merge upstream

Signed-off-by: Ashwin Naren <arihant2math@gmail.com>

.github/workflows/ai-review.yml

@@ -0,0 +1,23 @@
+name: PR Review
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: AI Code Review
+        uses: gitea-actions/ai-reviewer@v0.6
+        with:
+          access-token: ${{ secrets.ACCESS_TOKEN }}
+          full-context-model: "gpt-4o"
+          full-context-api-key: ${{ secrets.OPENAI_API_KEY }}
+          single-chunk-model: "claude-3-5-sonnet"
+          single-chunk-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+          exclude-files: "*.md,*.yaml"

.github/workflows/ci.yaml

@@ -229,6 +229,7 @@ jobs:
         uses: coolreader18/redoxer-action@v1
         with:
           command: check
+          args: --ignore-rust-version
 
   snippets_cpython:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
@@ -407,7 +408,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
-          target: wasm32-wasi
+          target: wasm32-wasip1
 
       - uses: Swatinem/rust-cache@v2
       - name: Setup Wasmer
@@ -415,8 +416,8 @@ jobs:
       - name: Install clang
         run: sudo apt-get update && sudo apt-get install clang -y
       - name: build rustpython
-        run: cargo build --release --target wasm32-wasi --features freeze-stdlib,stdlib --verbose
+        run: cargo build --release --target wasm32-wasip1 --features freeze-stdlib,stdlib --verbose
       - name: run snippets
-        run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py
+        run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py
       - name: run cpython unittest
-        run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py
+        run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py

.github/workflows/release.yml

@@ -0,0 +1,145 @@
+name: Release
+
+on:
+  schedule:
+    # 9 AM UTC on every Monday
+    - cron: "0 9 * * Mon"
+  workflow_dispatch:
+    inputs:
+      pre-release:
+        type: boolean
+        description: Mark "Pre-Release"
+        required: false
+        default: true
+
+permissions:
+  contents: write
+
+env:
+  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl
+
+jobs:
+  build:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: i686-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: aarch64-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: armv7-unknown-linux-gnueabi
+#          - runner: ubuntu-latest
+#            target: s390x-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: powerpc64le-unknown-linux-gnu
+          - runner: macos-latest
+            target: aarch64-apple-darwin
+#          - runner: macos-latest
+#            target: x86_64-apple-darwin
+          - runner: windows-latest
+            target: x86_64-pc-windows-msvc
+#          - runner: windows-latest
+#            target: i686-pc-windows-msvc
+#          - runner: windows-latest
+#            target: aarch64-pc-windows-msvc
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Set up Environment
+        shell: bash
+        run: rustup target add ${{ matrix.platform.target }}
+      - name: Set up Windows Environment
+        shell: bash
+        run: |
+          cargo install --target-dir=target -v cargo-vcpkg
+          cargo vcpkg -v build
+        if: runner.os == 'Windows'
+      - name: Set up MacOS Environment
+        run: brew install autoconf automake libtool
+        if: runner.os == 'macOS'
+
+      - name: Build RustPython
+        run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }}
+        if: runner.os == 'macOS'
+      - name: Build RustPython
+        run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }},jit
+        if: runner.os != 'macOS'
+
+      - name: Rename Binary
+        run: cp target/${{ matrix.platform.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}
+        if: runner.os != 'Windows'
+      - name: Rename Binary
+        run: cp target/${{ matrix.platform.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}.exe
+        if: runner.os == 'Windows'
+
+      - name: Upload Binary Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}
+          path: target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}*
+
+  build-wasm:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Set up Environment
+        shell: bash
+        run: rustup target add wasm32-wasip1
+
+      - name: Build RustPython
+        run: cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release
+
+      - name: Rename Binary
+        run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm
+
+      - name: Upload Binary Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rustpython-release-wasm32-wasip1
+          path: target/rustpython-release-wasm32-wasip1.wasm
+
+  release:
+    runs-on: ubuntu-latest
+    needs: [build, build-wasm]
+    steps:
+      - name: Download Binary Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: bin
+          pattern: rustpython-release-*
+          merge-multiple: true
+
+      - name: List Binaries
+        run: |
+          ls -lah bin/
+          file bin/*
+      - name: Create Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref_name }}
+          run: ${{ github.run_number }}
+        run: |
+          if [[ "${{ github.event.inputs.pre-release }}" == "false" ]]; then
+            RELEASE_TYPE_NAME=Release
+            PRERELEASE_ARG=
+          else
+            RELEASE_TYPE_NAME=Pre-Release
+            PRERELEASE_ARG=--prerelease
+          fi
+          
+          today=$(date '+%Y-%m-%d')
+          gh release create "$today-$tag-$run" \
+              --repo="$GITHUB_REPOSITORY" \
+              --title="RustPython $RELEASE_TYPE_NAME $today-$tag #$run" \
+              --target="$tag" \
+              --generate-notes \
+              $PRERELEASE_ARG \
+              bin/rustpython-release-*

Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "Inflector"
@@ -207,11 +207,10 @@ dependencies = [
 
 [[package]]
 name = "caseless"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f"
+checksum = "8b6fd507454086c8edfd769ca6ada439193cdb209c7681712ef6275cccbfe5d8"
 dependencies = [
- "regex",
  "unicode-normalization",
 ]
 
@@ -308,12 +307,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "convert_case"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
-
 [[package]]
 name = "core-foundation"
 version = "0.9.4"
@@ -564,15 +557,23 @@ dependencies = [
 
 [[package]]
 name = "derive_more"
-version = "0.99.18"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"
+checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05"
+dependencies = [
+ "derive_more-impl",
+]
+
+[[package]]
+name = "derive_more-impl"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
 dependencies = [
- "convert_case",
  "proc-macro2",
  "quote",
- "rustc_version",
  "syn 2.0.77",
+ "unicode-xid",
 ]
 
 [[package]]
@@ -1075,9 +1076,9 @@ dependencies = [
 
 [[package]]
 name = "libm"
-version = "0.2.8"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058"
+checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
 
 [[package]]
 name = "libredox"
@@ -1169,9 +1170,9 @@ dependencies = [
 
 [[package]]
 name = "malachite"
-version = "0.4.16"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5616515d632967cd329b6f6db96be9a03ea0b3a49cdbc45b0016803dad8a77b7"
+checksum = "4a6ecab92657eb234bfe98abd0b17920772c6b14ce69256950142e2eb36d000b"
 dependencies = [
  "malachite-base",
  "malachite-nz",
@@ -1180,9 +1181,9 @@ dependencies = [
 
 [[package]]
 name = "malachite-base"
-version = "0.4.16"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46059721011b0458b7bd6d9179be5d0b60294281c23320c207adceaecc54d13b"
+checksum = "06f6d078bb4dad5b76f6a85737e6c9113667d199612a73eef375c56e75d5d4d5"
 dependencies = [
  "hashbrown",
  "itertools 0.11.0",
@@ -1192,9 +1193,9 @@ dependencies = [
 
 [[package]]
 name = "malachite-bigint"
-version = "0.2.0"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17703a19c80bbdd0b7919f0f104f3b0597f7de4fc4e90a477c15366a5ba03faa"
+checksum = "63c7698e7abae9522edd41b54ae0395a9bd736ca5054b5fbe672316283a7b817"
 dependencies = [
  "derive_more",
  "malachite",
@@ -1205,9 +1206,9 @@ dependencies = [
 
 [[package]]
 name = "malachite-nz"
-version = "0.4.16"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1503b27e825cabd1c3d0ff1e95a39fb2ec9eab6fd3da6cfa41aec7091d273e78"
+checksum = "ad61a09cde72dcdb8b2baaf3e71430c5bd94fb6753da89f17301a459f3e466db"
 dependencies = [
  "itertools 0.11.0",
  "libm",
@@ -1216,9 +1217,9 @@ dependencies = [
 
 [[package]]
 name = "malachite-q"
-version = "0.4.16"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a475503a70a3679dbe3b9b230a23622516742528ba614a7b2490f180ea9cb514"
+checksum = "be4a9dfffb87667ae94e8320213d3f5419e3e37311bc6bf8f4d2ab9b44f3a535"
 dependencies = [
  "itertools 0.11.0",
  "malachite-base",
@@ -1566,13 +1567,13 @@ dependencies = [
 
 [[package]]
 name = "pmutil"
-version = "0.5.3"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3894e5d549cccbe44afecf72922f277f603cd4bb0219c8342631ef18fffbe004"
+checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -1824,24 +1825,23 @@ dependencies = [
 
 [[package]]
 name = "result-like"
-version = "0.4.6"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccc7ce6435c33898517a30e85578cd204cbb696875efb93dec19a2d31294f810"
+checksum = "abf7172fef6a7d056b5c26bf6c826570267562d51697f4982ff3ba4aec68a9df"
 dependencies = [
  "result-like-derive",
 ]
 
 [[package]]
 name = "result-like-derive"
-version = "0.4.6"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fabf0a2e54f711c68c50d49f648a1a8a37adcb57353f518ac4df374f0788f42"
+checksum = "a8d6574c02e894d66370cfc681e5d68fedbc9a548fb55b30a96b3f0ae22d0fe5"
 dependencies = [
  "pmutil",
  "proc-macro2",
  "quote",
- "syn 1.0.109",
- "syn-ext",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -1897,8 +1897,7 @@ dependencies = [
 [[package]]
 name = "rustpython-ast"
 version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5"
+source = "git+https://github.com/RustPython/Parser.git?rev=4588ea5c3e6327009640e7c9c89eb6fa9220358e#4588ea5c3e6327009640e7c9c89eb6fa9220358e"
 dependencies = [
  "is-macro",
  "malachite-bigint",
@@ -2010,8 +2009,7 @@ dependencies = [
 [[package]]
 name = "rustpython-format"
 version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0389039b132ad8e350552d771270ccd03186985696764bcee2239694e7839942"
+source = "git+https://github.com/RustPython/Parser.git?rev=4588ea5c3e6327009640e7c9c89eb6fa9220358e#4588ea5c3e6327009640e7c9c89eb6fa9220358e"
 dependencies = [
  "bitflags 2.6.0",
  "itertools 0.11.0",
@@ -2038,8 +2036,7 @@ dependencies = [
 [[package]]
 name = "rustpython-literal"
 version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8304be3cae00232a1721a911033e55877ca3810215f66798e964a2d8d22281d"
+source = "git+https://github.com/RustPython/Parser.git?rev=4588ea5c3e6327009640e7c9c89eb6fa9220358e#4588ea5c3e6327009640e7c9c89eb6fa9220358e"
 dependencies = [
  "hexf-parse",
  "is-macro",
@@ -2051,8 +2048,7 @@ dependencies = [
 [[package]]
 name = "rustpython-parser"
 version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb"
+source = "git+https://github.com/RustPython/Parser.git?rev=4588ea5c3e6327009640e7c9c89eb6fa9220358e#4588ea5c3e6327009640e7c9c89eb6fa9220358e"
 dependencies = [
  "anyhow",
  "is-macro",
@@ -2075,8 +2071,7 @@ dependencies = [
 [[package]]
 name = "rustpython-parser-core"
 version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad"
+source = "git+https://github.com/RustPython/Parser.git?rev=4588ea5c3e6327009640e7c9c89eb6fa9220358e#4588ea5c3e6327009640e7c9c89eb6fa9220358e"
 dependencies = [
  "is-macro",
  "memchr",
@@ -2086,8 +2081,7 @@ dependencies = [
 [[package]]
 name = "rustpython-parser-vendored"
 version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6"
+source = "git+https://github.com/RustPython/Parser.git?rev=4588ea5c3e6327009640e7c9c89eb6fa9220358e#4588ea5c3e6327009640e7c9c89eb6fa9220358e"
 dependencies = [
  "memchr",
  "once_cell",
@@ -2861,6 +2855,12 @@ version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
 
+[[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+
 [[package]]
 name = "unicode_names2"
 version = "1.3.0"
@@ -2948,9 +2948,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.93"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
+checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -2959,13 +2959,12 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.93"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
+checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79"
 dependencies = [
  "bumpalo",
  "log",
- "once_cell",
  "proc-macro2",
  "quote",
  "syn 2.0.77",
@@ -2986,9 +2985,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.93"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
+checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -2996,9 +2995,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.93"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
+checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3009,9 +3008,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.93"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
+checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
 
 [[package]]
 name = "web-sys"

Cargo.toml

@@ -104,7 +104,7 @@ members = [
 version = "0.4.0"
 authors = ["RustPython Team"]
 edition = "2021"
-rust-version = "1.80.0"
+rust-version = "1.83.0"
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
@@ -122,16 +122,16 @@ rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.
 rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
 rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
 
-rustpython-literal = { version = "0.4.0" }
-rustpython-parser-core = { version = "0.4.0" }
-rustpython-parser = { version = "0.4.0" }
-rustpython-ast = { version = "0.4.0" }
-rustpython-format= { version = "0.4.0" }
-# rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
+# rustpython-literal = { version = "0.4.0" }
+# rustpython-parser-core = { version = "0.4.0" }
+# rustpython-parser = { version = "0.4.0" }
+# rustpython-ast = { version = "0.4.0" }
+# rustpython-format= { version = "0.4.0" }
+rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "4588ea5c3e6327009640e7c9c89eb6fa9220358e" }
+rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "4588ea5c3e6327009640e7c9c89eb6fa9220358e" }
+rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "4588ea5c3e6327009640e7c9c89eb6fa9220358e" }
+rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "4588ea5c3e6327009640e7c9c89eb6fa9220358e" }
+rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "4588ea5c3e6327009640e7c9c89eb6fa9220358e" }
 # rustpython-literal = { path = "../RustPython-parser/literal" }
 # rustpython-parser-core = { path = "../RustPython-parser/core" }
 # rustpython-parser = { path = "../RustPython-parser/parser" }
@@ -157,9 +157,9 @@ junction = "1.0.0"
 libc = "0.2.153"
 log = "0.4.16"
 nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
-malachite-bigint = "0.2.0"
-malachite-q = "0.4.4"
-malachite-base = "0.4.4"
+malachite-bigint = "0.2.2"
+malachite-q = "<=0.4.18"
+malachite-base = "<=0.4.18"
 memchr = "2.7.2"
 num-complex = "0.4.0"
 num-integer = "0.1.44"

Lib/_dummy_os.py

@@ -5,22 +5,30 @@ A shim of the os module containing only simple path-related utilities
 try:
     from os import *
 except ImportError:
-    import abc
+    import abc, sys
 
     def __getattr__(name):
-        raise OSError("no os specific module found")
+        if name in {"_path_normpath", "__path__"}:
+            raise AttributeError(name)
+        if name.isupper():
+            return 0
+        def dummy(*args, **kwargs):
+            import io
+            return io.UnsupportedOperation(f"{name}: no os specific module found")
+        dummy.__name__ = f"dummy_{name}"
+        return dummy
 
-    def _shim():
-        import _dummy_os, sys
-        sys.modules['os'] = _dummy_os
-        sys.modules['os.path'] = _dummy_os.path
+    sys.modules['os'] = sys.modules['posix'] = sys.modules[__name__]
 
     import posixpath as path
-    import sys
     sys.modules['os.path'] = path
     del sys
 
     sep = path.sep
+    supports_dir_fd = set()
+    supports_effective_ids = set()
+    supports_fd = set()
+    supports_follow_symlinks = set()
 
 
     def fspath(path):

Lib/contextlib.py

@@ -145,14 +145,17 @@ class _GeneratorContextManager(
             except StopIteration:
                 return False
             else:
-                raise RuntimeError("generator didn't stop")
+                try:
+                    raise RuntimeError("generator didn't stop")
+                finally:
+                    self.gen.close()
         else:
             if value is None:
                 # Need to force instantiation so we can reliably
                 # tell if we get the same exception back
                 value = typ()
             try:
-                self.gen.throw(typ, value, traceback)
+                self.gen.throw(value)
             except StopIteration as exc:
                 # Suppress StopIteration *unless* it's the same exception that
                 # was passed to throw().  This prevents a StopIteration
@@ -187,7 +190,10 @@ class _GeneratorContextManager(
                     raise
                 exc.__traceback__ = traceback
                 return False
-            raise RuntimeError("generator didn't stop after throw()")
+            try:
+                raise RuntimeError("generator didn't stop after throw()")
+            finally:
+                self.gen.close()
 
 class _AsyncGeneratorContextManager(
     _GeneratorContextManagerBase,
@@ -212,14 +218,17 @@ class _AsyncGeneratorContextManager(
             except StopAsyncIteration:
                 return False
             else:
-                raise RuntimeError("generator didn't stop")
+                try:
+                    raise RuntimeError("generator didn't stop")
+                finally:
+                    await self.gen.aclose()
         else:
             if value is None:
                 # Need to force instantiation so we can reliably
                 # tell if we get the same exception back
                 value = typ()
             try:
-                await self.gen.athrow(typ, value, traceback)
+                await self.gen.athrow(value)
             except StopAsyncIteration as exc:
                 # Suppress StopIteration *unless* it's the same exception that
                 # was passed to throw().  This prevents a StopIteration
@@ -254,7 +263,10 @@ class _AsyncGeneratorContextManager(
                     raise
                 exc.__traceback__ = traceback
                 return False
-            raise RuntimeError("generator didn't stop after athrow()")
+            try:
+                raise RuntimeError("generator didn't stop after athrow()")
+            finally:
+                await self.gen.aclose()
 
 
 def contextmanager(func):
@@ -441,7 +453,16 @@ class suppress(AbstractContextManager):
         # exactly reproduce the limitations of the CPython interpreter.
         #
         # See http://bugs.python.org/issue12029 for more details
-        return exctype is not None and issubclass(exctype, self._exceptions)
+        if exctype is None:
+            return
+        if issubclass(exctype, self._exceptions):
+            return True
+        if issubclass(exctype, BaseExceptionGroup):
+            match, rest = excinst.split(self._exceptions)
+            if rest is None:
+                return True
+            raise rest
+        return False
 
 
 class _BaseExitStack:

Lib/ctypes/test/test_numbers.py

@@ -147,10 +147,10 @@ class NumberTestCase(unittest.TestCase):
 
             # alignment of the type...
             self.assertEqual((code, alignment(t)),
-                                 (code, align))
+                             (code, align))
             # and alignment of an instance
             self.assertEqual((code, alignment(t())),
-                                 (code, align))
+                             (code, align))
 
     def test_int_from_address(self):
         from array import array

Lib/io.py

@@ -55,10 +55,15 @@ import _io
 import abc
 
 from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
-                 open, open_code, FileIO, BytesIO, StringIO, BufferedReader,
+                 open, open_code, BytesIO, StringIO, BufferedReader,
                  BufferedWriter, BufferedRWPair, BufferedRandom,
                  IncrementalNewlineDecoder, text_encoding, TextIOWrapper)
 
+try:
+    from _io import FileIO
+except ImportError:
+    pass
+
 # Pretend this exception was created here.
 UnsupportedOperation.__module__ = "io"
 
@@ -82,7 +87,10 @@ class BufferedIOBase(_io._BufferedIOBase, IOBase):
 class TextIOBase(_io._TextIOBase, IOBase):
     __doc__ = _io._TextIOBase.__doc__
 
-RawIOBase.register(FileIO)
+try:
+    RawIOBase.register(FileIO)
+except NameError:
+    pass
 
 for klass in (BytesIO, BufferedReader, BufferedWriter, BufferedRandom,
               BufferedRWPair):

Lib/logging/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -18,13 +18,14 @@
 Logging package for Python. Based on PEP 282 and comments thereto in
 comp.lang.python.
 
-Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
 
 import sys, os, time, io, re, traceback, warnings, weakref, collections.abc
 
+from types import GenericAlias
 from string import Template
 from string import Formatter as StrFormatter
 
@@ -37,7 +38,8 @@ __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR',
            'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass',
            'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown',
            'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory',
-           'lastResort', 'raiseExceptions']
+           'lastResort', 'raiseExceptions', 'getLevelNamesMapping',
+           'getHandlerByName', 'getHandlerNames']
 
 import threading
 
@@ -63,20 +65,25 @@ _startTime = time.time()
 raiseExceptions = True
 
 #
-# If you don't want threading information in the log, set this to zero
+# If you don't want threading information in the log, set this to False
 #
 logThreads = True
 
 #
-# If you don't want multiprocessing information in the log, set this to zero
+# If you don't want multiprocessing information in the log, set this to False
 #
 logMultiprocessing = True
 
 #
-# If you don't want process information in the log, set this to zero
+# If you don't want process information in the log, set this to False
 #
 logProcesses = True
 
+#
+# If you don't want asyncio task information in the log, set this to False
+#
+logAsyncioTasks = True
+
 #---------------------------------------------------------------------------
 #   Level related stuff
 #---------------------------------------------------------------------------
@@ -116,6 +123,9 @@ _nameToLevel = {
     'NOTSET': NOTSET,
 }
 
+def getLevelNamesMapping():
+    return _nameToLevel.copy()
+
 def getLevelName(level):
     """
     Return the textual or numeric representation of logging level 'level'.
@@ -156,15 +166,15 @@ def addLevelName(level, levelName):
     finally:
         _releaseLock()
 
-if hasattr(sys, '_getframe'):
-    currentframe = lambda: sys._getframe(3)
+if hasattr(sys, "_getframe"):
+    currentframe = lambda: sys._getframe(1)
 else: #pragma: no cover
     def currentframe():
         """Return the frame object for the caller's stack frame."""
         try:
             raise Exception
-        except Exception:
-            return sys.exc_info()[2].tb_frame.f_back
+        except Exception as exc:
+            return exc.__traceback__.tb_frame.f_back
 
 #
 # _srcfile is used when walking the stack to check when we've got the first
@@ -181,13 +191,18 @@ else: #pragma: no cover
 _srcfile = os.path.normcase(addLevelName.__code__.co_filename)
 
 # _srcfile is only used in conjunction with sys._getframe().
-# To provide compatibility with older versions of Python, set _srcfile
-# to None if _getframe() is not available; this value will prevent
-# findCaller() from being called. You can also do this if you want to avoid
-# the overhead of fetching caller information, even when _getframe() is
-# available.
-#if not hasattr(sys, '_getframe'):
-#    _srcfile = None
+# Setting _srcfile to None will prevent findCaller() from being called. This
+# way, you can avoid the overhead of fetching caller information.
+
+# The following is based on warnings._is_internal_frame. It makes sure that
+# frames of the import mechanism are skipped when logging at module level and
+# using a stacklevel value greater than one.
+def _is_internal_frame(frame):
+    """Signal whether the frame is a CPython or logging module internal."""
+    filename = os.path.normcase(frame.f_code.co_filename)
+    return filename == _srcfile or (
+            "importlib" in filename and "_bootstrap" in filename
+    )
 
 
 def _checkLevel(level):
@@ -307,7 +322,7 @@ class LogRecord(object):
         # Thus, while not removing the isinstance check, it does now look
         # for collections.abc.Mapping rather than, as before, dict.
         if (args and len(args) == 1 and isinstance(args[0], collections.abc.Mapping)
-            and args[0]):
+                and args[0]):
             args = args[0]
         self.args = args
         self.levelname = getLevelName(level)
@@ -325,7 +340,7 @@ class LogRecord(object):
         self.lineno = lineno
         self.funcName = func
         self.created = ct
-        self.msecs = (ct - int(ct)) * 1000
+        self.msecs = int((ct - int(ct)) * 1000) + 0.0  # see gh-89047
         self.relativeCreated = (self.created - _startTime) * 1000
         if logThreads:
             self.thread = threading.get_ident()
@@ -352,9 +367,18 @@ class LogRecord(object):
         else:
             self.process = None
 
+        self.taskName = None
+        if logAsyncioTasks:
+            asyncio = sys.modules.get('asyncio')
+            if asyncio:
+                try:
+                    self.taskName = asyncio.current_task().get_name()
+                except Exception:
+                    pass
+
     def __repr__(self):
         return '<LogRecord: %s, %s, %s, %s, "%s">'%(self.name, self.levelno,
-            self.pathname, self.lineno, self.msg)
+                                                    self.pathname, self.lineno, self.msg)
 
     def getMessage(self):
         """
@@ -487,7 +511,7 @@ class StringTemplateStyle(PercentStyle):
 
     def usesTime(self):
         fmt = self._fmt
-        return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0
+        return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_search) >= 0
 
     def validate(self):
         pattern = Template.pattern
@@ -557,6 +581,7 @@ class Formatter(object):
                         (typically at application startup time)
     %(thread)d          Thread ID (if available)
     %(threadName)s      Thread name (if available)
+    %(taskName)s        Task name (if available)
     %(process)d         Process ID (if available)
     %(message)s         The result of record.getMessage(), computed just as
                         the record is emitted
@@ -583,7 +608,7 @@ class Formatter(object):
         """
         if style not in _STYLES:
             raise ValueError('Style must be one of: %s' % ','.join(
-                             _STYLES.keys()))
+                _STYLES.keys()))
         self._style = _STYLES[style][0](fmt, defaults=defaults)
         if validate:
             self._style.validate()
@@ -808,23 +833,36 @@ class Filterer(object):
         Determine if a record is loggable by consulting all the filters.
 
         The default is to allow the record to be logged; any filter can veto
-        this and the record is then dropped. Returns a zero value if a record
-        is to be dropped, else non-zero.
+        this by returning a false value.
+        If a filter attached to a handler returns a log record instance,
+        then that instance is used in place of the original log record in
+        any further processing of the event by that handler.
+        If a filter returns any other true value, the original log record
+        is used in any further processing of the event by that handler.
+
+        If none of the filters return false values, this method returns
+        a log record.
+        If any of the filters return a false value, this method returns
+        a false value.
 
         .. versionchanged:: 3.2
 
            Allow filters to be just callables.
+
+        .. versionchanged:: 3.12
+           Allow filters to return a LogRecord instead of
+           modifying it in place.
         """
-        rv = True
         for f in self.filters:
             if hasattr(f, 'filter'):
                 result = f.filter(record)
             else:
                 result = f(record) # assume callable - will raise if not
             if not result:
-                rv = False
-                break
-        return rv
+                return False
+            if isinstance(result, LogRecord):
+                record = result
+        return record
 
 #---------------------------------------------------------------------------
 #   Handler classes and functions
@@ -845,8 +883,9 @@ def _removeHandlerRef(wr):
     if acquire and release and handlers:
         acquire()
         try:
-            if wr in handlers:
-                handlers.remove(wr)
+            handlers.remove(wr)
+        except ValueError:
+            pass
         finally:
             release()
 
@@ -860,6 +899,23 @@ def _addHandlerRef(handler):
     finally:
         _releaseLock()
 
+
+def getHandlerByName(name):
+    """
+    Get a handler with the specified *name*, or None if there isn't one with
+    that name.
+    """
+    return _handlers.get(name)
+
+
+def getHandlerNames():
+    """
+    Return all known handler names as an immutable set.
+    """
+    result = set(_handlers.keys())
+    return frozenset(result)
+
+
 class Handler(Filterer):
     """
     Handler instances dispatch logging events to specific destinations.
@@ -958,10 +1014,14 @@ class Handler(Filterer):
 
         Emission depends on filters which may have been added to the handler.
         Wrap the actual emission of the record with acquisition/release of
-        the I/O thread lock. Returns whether the filter passed the record for
-        emission.
+        the I/O thread lock.
+
+        Returns an instance of the log record that was emitted
+        if it passed all filters, otherwise a false value is returned.
         """
         rv = self.filter(record)
+        if isinstance(rv, LogRecord):
+            record = rv
         if rv:
             self.acquire()
             try:
@@ -1032,7 +1092,7 @@ class Handler(Filterer):
                 else:
                     # couldn't find the right stack frame, for some reason
                     sys.stderr.write('Logged from file %s, line %s\n' % (
-                                     record.filename, record.lineno))
+                        record.filename, record.lineno))
                 # Issue 18671: output logging message and arguments
                 try:
                     sys.stderr.write('Message: %r\n'
@@ -1044,7 +1104,7 @@ class Handler(Filterer):
                     sys.stderr.write('Unable to print the message and arguments'
                                      ' - possible formatting error.\nUse the'
                                      ' traceback above to help find the error.\n'
-                                    )
+                                     )
             except OSError: #pragma: no cover
                 pass    # see issue 5971
             finally:
@@ -1136,6 +1196,8 @@ class StreamHandler(Handler):
             name += ' '
         return '<%s %s(%s)>' % (self.__class__.__name__, name, level)
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class FileHandler(StreamHandler):
     """
@@ -1459,7 +1521,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.debug("Houston, we have a %s", "thorny problem", exc_info=1)
+        logger.debug("Houston, we have a %s", "thorny problem", exc_info=True)
         """
         if self.isEnabledFor(DEBUG):
             self._log(DEBUG, msg, args, **kwargs)
@@ -1471,7 +1533,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.info("Houston, we have a %s", "interesting problem", exc_info=1)
+        logger.info("Houston, we have a %s", "notable problem", exc_info=True)
         """
         if self.isEnabledFor(INFO):
             self._log(INFO, msg, args, **kwargs)
@@ -1483,14 +1545,14 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1)
+        logger.warning("Houston, we have a %s", "bit of a problem", exc_info=True)
         """
         if self.isEnabledFor(WARNING):
             self._log(WARNING, msg, args, **kwargs)
 
     def warn(self, msg, *args, **kwargs):
         warnings.warn("The 'warn' method is deprecated, "
-            "use 'warning' instead", DeprecationWarning, 2)
+                      "use 'warning' instead", DeprecationWarning, 2)
         self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
@@ -1500,7 +1562,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.error("Houston, we have a %s", "major problem", exc_info=1)
+        logger.error("Houston, we have a %s", "major problem", exc_info=True)
         """
         if self.isEnabledFor(ERROR):
             self._log(ERROR, msg, args, **kwargs)
@@ -1518,7 +1580,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.critical("Houston, we have a %s", "major disaster", exc_info=1)
+        logger.critical("Houston, we have a %s", "major disaster", exc_info=True)
         """
         if self.isEnabledFor(CRITICAL):
             self._log(CRITICAL, msg, args, **kwargs)
@@ -1536,7 +1598,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.log(level, "We have a %s", "mysterious problem", exc_info=1)
+        logger.log(level, "We have a %s", "mysterious problem", exc_info=True)
         """
         if not isinstance(level, int):
             if raiseExceptions:
@@ -1554,33 +1616,31 @@ class Logger(Filterer):
         f = currentframe()
         #On some versions of IronPython, currentframe() returns None if
         #IronPython isn't run with -X:Frames.
-        if f is not None:
-            f = f.f_back
-        orig_f = f
-        while f and stacklevel > 1:
-            f = f.f_back
-            stacklevel -= 1
-        if not f:
-            f = orig_f
-        rv = "(unknown file)", 0, "(unknown function)", None
-        while hasattr(f, "f_code"):
-            co = f.f_code
-            filename = os.path.normcase(co.co_filename)
-            if filename == _srcfile:
-                f = f.f_back
-                continue
-            sinfo = None
-            if stack_info:
-                sio = io.StringIO()
-                sio.write('Stack (most recent call last):\n')
+        if f is None:
+            return "(unknown file)", 0, "(unknown function)", None
+        while stacklevel > 0:
+            next_f = f.f_back
+            if next_f is None:
+                ## We've got options here.
+                ## If we want to use the last (deepest) frame:
+                break
+                ## If we want to mimic the warnings module:
+                #return ("sys", 1, "(unknown function)", None)
+                ## If we want to be pedantic:
+                #raise ValueError("call stack is not deep enough")
+            f = next_f
+            if not _is_internal_frame(f):
+                stacklevel -= 1
+        co = f.f_code
+        sinfo = None
+        if stack_info:
+            with io.StringIO() as sio:
+                sio.write("Stack (most recent call last):\n")
                 traceback.print_stack(f, file=sio)
                 sinfo = sio.getvalue()
                 if sinfo[-1] == '\n':
                     sinfo = sinfo[:-1]
-                sio.close()
-            rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
-            break
-        return rv
+        return co.co_filename, f.f_lineno, co.co_name, sinfo
 
     def makeRecord(self, name, level, fn, lno, msg, args, exc_info,
                    func=None, extra=None, sinfo=None):
@@ -1589,7 +1649,7 @@ class Logger(Filterer):
         specialized LogRecords.
         """
         rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func,
-                             sinfo)
+                               sinfo)
         if extra is not None:
             for key in extra:
                 if (key in ["message", "asctime"]) or (key in rv.__dict__):
@@ -1630,8 +1690,14 @@ class Logger(Filterer):
         This method is used for unpickled records received from a socket, as
         well as those created locally. Logger-level filtering is applied.
         """
-        if (not self.disabled) and self.filter(record):
-            self.callHandlers(record)
+        if self.disabled:
+            return
+        maybe_record = self.filter(record)
+        if not maybe_record:
+            return
+        if isinstance(maybe_record, LogRecord):
+            record = maybe_record
+        self.callHandlers(record)
 
     def addHandler(self, hdlr):
         """
@@ -1737,7 +1803,7 @@ class Logger(Filterer):
                     is_enabled = self._cache[level] = False
                 else:
                     is_enabled = self._cache[level] = (
-                        level >= self.getEffectiveLevel()
+                            level >= self.getEffectiveLevel()
                     )
             finally:
                 _releaseLock()
@@ -1762,13 +1828,30 @@ class Logger(Filterer):
             suffix = '.'.join((self.name, suffix))
         return self.manager.getLogger(suffix)
 
+    def getChildren(self):
+
+        def _hierlevel(logger):
+            if logger is logger.manager.root:
+                return 0
+            return 1 + logger.name.count('.')
+
+        d = self.manager.loggerDict
+        _acquireLock()
+        try:
+            # exclude PlaceHolders - the last check is to ensure that lower-level
+            # descendants aren't returned - if there are placeholders, a logger's
+            # parent field might point to a grandparent or ancestor thereof.
+            return set(item for item in d.values()
+                       if isinstance(item, Logger) and item.parent is self and
+                       _hierlevel(item) == 1 + _hierlevel(item.parent))
+        finally:
+            _releaseLock()
+
     def __repr__(self):
         level = getLevelName(self.getEffectiveLevel())
         return '<%s %s (%s)>' % (self.__class__.__name__, self.name, level)
 
     def __reduce__(self):
-        # In general, only the root logger will not be accessible via its name.
-        # However, the root logger's class has its own __reduce__ method.
         if getLogger(self.name) is not self:
             import pickle
             raise pickle.PicklingError('logger cannot be pickled')
@@ -1848,7 +1931,7 @@ class LoggerAdapter(object):
 
     def warn(self, msg, *args, **kwargs):
         warnings.warn("The 'warn' method is deprecated, "
-            "use 'warning' instead", DeprecationWarning, 2)
+                      "use 'warning' instead", DeprecationWarning, 2)
         self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
@@ -1902,18 +1985,11 @@ class LoggerAdapter(object):
         """
         return self.logger.hasHandlers()
 
-    def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False):
+    def _log(self, level, msg, args, **kwargs):
         """
         Low-level log implementation, proxied to allow nested logger adapters.
         """
-        return self.logger._log(
-            level,
-            msg,
-            args,
-            exc_info=exc_info,
-            extra=extra,
-            stack_info=stack_info,
-        )
+        return self.logger._log(level, msg, args, **kwargs)
 
     @property
     def manager(self):
@@ -1932,6 +2008,8 @@ class LoggerAdapter(object):
         level = getLevelName(logger.getEffectiveLevel())
         return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level)
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 root = RootLogger(WARNING)
 Logger.root = root
 Logger.manager = Manager(Logger.root)
@@ -1971,7 +2049,7 @@ def basicConfig(**kwargs):
               that this argument is incompatible with 'filename' - if both
               are present, 'stream' is ignored.
     handlers  If specified, this should be an iterable of already created
-              handlers, which will be added to the root handler. Any handler
+              handlers, which will be added to the root logger. Any handler
               in the list which does not have a formatter assigned will be
               assigned the formatter created in this function.
     force     If this keyword  is specified as true, any existing handlers
@@ -2047,7 +2125,7 @@ def basicConfig(**kwargs):
             style = kwargs.pop("style", '%')
             if style not in _STYLES:
                 raise ValueError('Style must be one of: %s' % ','.join(
-                                 _STYLES.keys()))
+                    _STYLES.keys()))
             fs = kwargs.pop("format", _STYLES[style][1])
             fmt = Formatter(fs, dfs, style)
             for h in handlers:
@@ -2124,7 +2202,7 @@ def warning(msg, *args, **kwargs):
 
 def warn(msg, *args, **kwargs):
     warnings.warn("The 'warn' function is deprecated, "
-        "use 'warning' instead", DeprecationWarning, 2)
+                  "use 'warning' instead", DeprecationWarning, 2)
     warning(msg, *args, **kwargs)
 
 def info(msg, *args, **kwargs):
@@ -2179,7 +2257,11 @@ def shutdown(handlerList=_handlerList):
             if h:
                 try:
                     h.acquire()
-                    h.flush()
+                    # MemoryHandlers might not want to be flushed on close,
+                    # but circular imports prevent us scoping this to just
+                    # those handlers.  hence the default to True.
+                    if getattr(h, 'flushOnClose', True):
+                        h.flush()
                     h.close()
                 except (OSError, ValueError):
                     # Ignore errors which might be caused
@@ -2242,7 +2324,9 @@ def _showwarning(message, category, filename, lineno, file=None, line=None):
         logger = getLogger("py.warnings")
         if not logger.handlers:
             logger.addHandler(NullHandler())
-        logger.warning("%s", s)
+        # bpo-46557: Log str(s) as msg instead of logger.warning("%s", s)
+        # since some log aggregation tools group logs by the msg arg
+        logger.warning(str(s))
 
 def captureWarnings(capture):
     """

Lib/logging/config.py

@@ -1,4 +1,4 @@
-# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2023 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -19,18 +19,20 @@ Configuration functions for the logging package for Python. The core package
 is based on PEP 282 and comments thereto in comp.lang.python, and influenced
 by Apache's log4j system.
 
-Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
 
 import errno
+import functools
 import io
 import logging
 import logging.handlers
+import os
+import queue
 import re
 import struct
-import sys
 import threading
 import traceback
 
@@ -59,15 +61,24 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=Non
     """
     import configparser
 
+    if isinstance(fname, str):
+        if not os.path.exists(fname):
+            raise FileNotFoundError(f"{fname} doesn't exist")
+        elif not os.path.getsize(fname):
+            raise RuntimeError(f'{fname} is an empty file')
+
     if isinstance(fname, configparser.RawConfigParser):
         cp = fname
     else:
-        cp = configparser.ConfigParser(defaults)
-        if hasattr(fname, 'readline'):
-            cp.read_file(fname)
-        else:
-            encoding = io.text_encoding(encoding)
-            cp.read(fname, encoding=encoding)
+        try:
+            cp = configparser.ConfigParser(defaults)
+            if hasattr(fname, 'readline'):
+                cp.read_file(fname)
+            else:
+                encoding = io.text_encoding(encoding)
+                cp.read(fname, encoding=encoding)
+        except configparser.ParsingError as e:
+            raise RuntimeError(f'{fname} is invalid: {e}')
 
     formatters = _create_formatters(cp)
 
@@ -113,11 +124,18 @@ def _create_formatters(cp):
         fs = cp.get(sectname, "format", raw=True, fallback=None)
         dfs = cp.get(sectname, "datefmt", raw=True, fallback=None)
         stl = cp.get(sectname, "style", raw=True, fallback='%')
+        defaults = cp.get(sectname, "defaults", raw=True, fallback=None)
+
         c = logging.Formatter
         class_name = cp[sectname].get("class")
         if class_name:
             c = _resolve(class_name)
-        f = c(fs, dfs, stl)
+
+        if defaults is not None:
+            defaults = eval(defaults, vars(logging))
+            f = c(fs, dfs, stl, defaults=defaults)
+        else:
+            f = c(fs, dfs, stl)
         formatters[form] = f
     return formatters
 
@@ -296,7 +314,7 @@ class ConvertingMixin(object):
             if replace:
                 self[key] = result
             if type(result) in (ConvertingDict, ConvertingList,
-                               ConvertingTuple):
+                                ConvertingTuple):
                 result.parent = self
                 result.key = key
         return result
@@ -305,7 +323,7 @@ class ConvertingMixin(object):
         result = self.configurator.convert(value)
         if value is not result:
             if type(result) in (ConvertingDict, ConvertingList,
-                               ConvertingTuple):
+                                ConvertingTuple):
                 result.parent = self
         return result
 
@@ -392,11 +410,9 @@ class BaseConfigurator(object):
                     self.importer(used)
                     found = getattr(found, frag)
             return found
-        except ImportError:
-            e, tb = sys.exc_info()[1:]
+        except ImportError as e:
             v = ValueError('Cannot resolve %r: %s' % (s, e))
-            v.__cause__, v.__traceback__ = e, tb
-            raise v
+            raise v from e
 
     def ext_convert(self, value):
         """Default converter for the ext:// protocol."""
@@ -448,8 +464,8 @@ class BaseConfigurator(object):
         elif not isinstance(value, ConvertingList) and isinstance(value, list):
             value = ConvertingList(value)
             value.configurator = self
-        elif not isinstance(value, ConvertingTuple) and\
-                 isinstance(value, tuple) and not hasattr(value, '_fields'):
+        elif not isinstance(value, ConvertingTuple) and \
+                isinstance(value, tuple) and not hasattr(value, '_fields'):
             value = ConvertingTuple(value)
             value.configurator = self
         elif isinstance(value, str): # str for py3k
@@ -469,10 +485,10 @@ class BaseConfigurator(object):
         c = config.pop('()')
         if not callable(c):
             c = self.resolve(c)
-        props = config.pop('.', None)
         # Check for valid identifiers
-        kwargs = {k: config[k] for k in config if valid_ident(k)}
+        kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))}
         result = c(**kwargs)
+        props = config.pop('.', None)
         if props:
             for name, value in props.items():
                 setattr(result, name, value)
@@ -484,6 +500,33 @@ class BaseConfigurator(object):
             value = tuple(value)
         return value
 
+def _is_queue_like_object(obj):
+    """Check that *obj* implements the Queue API."""
+    if isinstance(obj, (queue.Queue, queue.SimpleQueue)):
+        return True
+    # defer importing multiprocessing as much as possible
+    from multiprocessing.queues import Queue as MPQueue
+    if isinstance(obj, MPQueue):
+        return True
+    # Depending on the multiprocessing start context, we cannot create
+    # a multiprocessing.managers.BaseManager instance 'mm' to get the
+    # runtime type of mm.Queue() or mm.JoinableQueue() (see gh-119819).
+    #
+    # Since we only need an object implementing the Queue API, we only
+    # do a protocol check, but we do not use typing.runtime_checkable()
+    # and typing.Protocol to reduce import time (see gh-121723).
+    #
+    # Ideally, we would have wanted to simply use strict type checking
+    # instead of a protocol-based type checking since the latter does
+    # not check the method signatures.
+    #
+    # Note that only 'put_nowait' and 'get' are required by the logging
+    # queue handler and queue listener (see gh-124653) and that other
+    # methods are either optional or unused.
+    minimal_queue_interface = ['put_nowait', 'get']
+    return all(callable(getattr(obj, method, None))
+               for method in minimal_queue_interface)
+
 class DictConfigurator(BaseConfigurator):
     """
     Configure logging using a dictionary-like object to describe the
@@ -542,7 +585,7 @@ class DictConfigurator(BaseConfigurator):
                 for name in formatters:
                     try:
                         formatters[name] = self.configure_formatter(
-                                                            formatters[name])
+                            formatters[name])
                     except Exception as e:
                         raise ValueError('Unable to configure '
                                          'formatter %r' % name) from e
@@ -566,7 +609,7 @@ class DictConfigurator(BaseConfigurator):
                         handler.name = name
                         handlers[name] = handler
                     except Exception as e:
-                        if 'target not configured yet' in str(e.__cause__):
+                        if ' not configured yet' in str(e.__cause__):
                             deferred.append(name)
                         else:
                             raise ValueError('Unable to configure handler '
@@ -669,18 +712,27 @@ class DictConfigurator(BaseConfigurator):
             dfmt = config.get('datefmt', None)
             style = config.get('style', '%')
             cname = config.get('class', None)
+            defaults = config.get('defaults', None)
 
             if not cname:
                 c = logging.Formatter
             else:
                 c = _resolve(cname)
 
+            kwargs  = {}
+
+            # Add defaults only if it exists.
+            # Prevents TypeError in custom formatter callables that do not
+            # accept it.
+            if defaults is not None:
+                kwargs['defaults'] = defaults
+
             # A TypeError would be raised if "validate" key is passed in with a formatter callable
             # that does not accept "validate" as a parameter
             if 'validate' in config:  # if user hasn't mentioned it, the default will be fine
-                result = c(fmt, dfmt, style, config['validate'])
+                result = c(fmt, dfmt, style, config['validate'], **kwargs)
             else:
-                result = c(fmt, dfmt, style)
+                result = c(fmt, dfmt, style, **kwargs)
 
         return result
 
@@ -697,10 +749,29 @@ class DictConfigurator(BaseConfigurator):
         """Add filters to a filterer from a list of names."""
         for f in filters:
             try:
-                filterer.addFilter(self.config['filters'][f])
+                if callable(f) or callable(getattr(f, 'filter', None)):
+                    filter_ = f
+                else:
+                    filter_ = self.config['filters'][f]
+                filterer.addFilter(filter_)
             except Exception as e:
                 raise ValueError('Unable to add filter %r' % f) from e
 
+    def _configure_queue_handler(self, klass, **kwargs):
+        if 'queue' in kwargs:
+            q = kwargs.pop('queue')
+        else:
+            q = queue.Queue()  # unbounded
+
+        rhl = kwargs.pop('respect_handler_level', False)
+        lklass = kwargs.pop('listener', logging.handlers.QueueListener)
+        handlers = kwargs.pop('handlers', [])
+
+        listener = lklass(q, *handlers, respect_handler_level=rhl)
+        handler = klass(q, **kwargs)
+        handler.listener = listener
+        return handler
+
     def configure_handler(self, config):
         """Configure a handler from a dictionary."""
         config_copy = dict(config)  # for restoring in case of error
@@ -720,28 +791,87 @@ class DictConfigurator(BaseConfigurator):
             factory = c
         else:
             cname = config.pop('class')
-            klass = self.resolve(cname)
-            #Special case for handler which refers to another handler
-            if issubclass(klass, logging.handlers.MemoryHandler) and\
-                'target' in config:
-                try:
-                    th = self.config['handlers'][config['target']]
-                    if not isinstance(th, logging.Handler):
-                        config.update(config_copy)  # restore for deferred cfg
-                        raise TypeError('target not configured yet')
-                    config['target'] = th
-                except Exception as e:
-                    raise ValueError('Unable to set target handler '
-                                     '%r' % config['target']) from e
-            elif issubclass(klass, logging.handlers.SMTPHandler) and\
-                'mailhost' in config:
+            if callable(cname):
+                klass = cname
+            else:
+                klass = self.resolve(cname)
+            if issubclass(klass, logging.handlers.MemoryHandler):
+                if 'flushLevel' in config:
+                    config['flushLevel'] = logging._checkLevel(config['flushLevel'])
+                if 'target' in config:
+                    # Special case for handler which refers to another handler
+                    try:
+                        tn = config['target']
+                        th = self.config['handlers'][tn]
+                        if not isinstance(th, logging.Handler):
+                            config.update(config_copy)  # restore for deferred cfg
+                            raise TypeError('target not configured yet')
+                        config['target'] = th
+                    except Exception as e:
+                        raise ValueError('Unable to set target handler %r' % tn) from e
+            elif issubclass(klass, logging.handlers.QueueHandler):
+                # Another special case for handler which refers to other handlers
+                # if 'handlers' not in config:
+                # raise ValueError('No handlers specified for a QueueHandler')
+                if 'queue' in config:
+                    qspec = config['queue']
+
+                    if isinstance(qspec, str):
+                        q = self.resolve(qspec)
+                        if not callable(q):
+                            raise TypeError('Invalid queue specifier %r' % qspec)
+                        config['queue'] = q()
+                    elif isinstance(qspec, dict):
+                        if '()' not in qspec:
+                            raise TypeError('Invalid queue specifier %r' % qspec)
+                        config['queue'] = self.configure_custom(dict(qspec))
+                    elif not _is_queue_like_object(qspec):
+                        raise TypeError('Invalid queue specifier %r' % qspec)
+
+                if 'listener' in config:
+                    lspec = config['listener']
+                    if isinstance(lspec, type):
+                        if not issubclass(lspec, logging.handlers.QueueListener):
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                    else:
+                        if isinstance(lspec, str):
+                            listener = self.resolve(lspec)
+                            if isinstance(listener, type) and \
+                                    not issubclass(listener, logging.handlers.QueueListener):
+                                raise TypeError('Invalid listener specifier %r' % lspec)
+                        elif isinstance(lspec, dict):
+                            if '()' not in lspec:
+                                raise TypeError('Invalid listener specifier %r' % lspec)
+                            listener = self.configure_custom(dict(lspec))
+                        else:
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                        if not callable(listener):
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                        config['listener'] = listener
+                if 'handlers' in config:
+                    hlist = []
+                    try:
+                        for hn in config['handlers']:
+                            h = self.config['handlers'][hn]
+                            if not isinstance(h, logging.Handler):
+                                config.update(config_copy)  # restore for deferred cfg
+                                raise TypeError('Required handler %r '
+                                                'is not configured yet' % hn)
+                            hlist.append(h)
+                    except Exception as e:
+                        raise ValueError('Unable to set required handler %r' % hn) from e
+                    config['handlers'] = hlist
+            elif issubclass(klass, logging.handlers.SMTPHandler) and \
+                    'mailhost' in config:
                 config['mailhost'] = self.as_tuple(config['mailhost'])
-            elif issubclass(klass, logging.handlers.SysLogHandler) and\
-                'address' in config:
+            elif issubclass(klass, logging.handlers.SysLogHandler) and \
+                    'address' in config:
                 config['address'] = self.as_tuple(config['address'])
-            factory = klass
-        props = config.pop('.', None)
-        kwargs = {k: config[k] for k in config if valid_ident(k)}
+            if issubclass(klass, logging.handlers.QueueHandler):
+                factory = functools.partial(self._configure_queue_handler, klass)
+            else:
+                factory = klass
+        kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))}
         try:
             result = factory(**kwargs)
         except TypeError as te:
@@ -759,6 +889,7 @@ class DictConfigurator(BaseConfigurator):
             result.setLevel(logging._checkLevel(level))
         if filters:
             self.add_filters(result, filters)
+        props = config.pop('.', None)
         if props:
             for name, value in props.items():
                 setattr(result, name, value)
@@ -794,6 +925,7 @@ class DictConfigurator(BaseConfigurator):
         """Configure a non-root logger from a dictionary."""
         logger = logging.getLogger(name)
         self.common_logger_config(logger, config, incremental)
+        logger.disabled = False
         propagate = config.get('propagate', None)
         if propagate is not None:
             logger.propagate = propagate

Lib/logging/handlers.py

@@ -187,15 +187,18 @@ class RotatingFileHandler(BaseRotatingHandler):
         Basically, see if the supplied record would cause the file to exceed
         the size limit we have.
         """
-        # See bpo-45401: Never rollover anything other than regular files
-        if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
-            return False
         if self.stream is None:                 # delay was set...
             self.stream = self._open()
         if self.maxBytes > 0:                   # are we rolling over?
+            pos = self.stream.tell()
+            if not pos:
+                # gh-116263: Never rollover an empty file
+                return False
             msg = "%s\n" % self.format(record)
-            self.stream.seek(0, 2)  #due to non-posix-compliant Windows feature
-            if self.stream.tell() + len(msg) >= self.maxBytes:
+            if pos + len(msg) >= self.maxBytes:
+                # See bpo-45401: Never rollover anything other than regular files
+                if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
+                    return False
                 return True
         return False
 
@@ -232,19 +235,19 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         if self.when == 'S':
             self.interval = 1 # one second
             self.suffix = "%Y-%m-%d_%H-%M-%S"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(?!\d)"
         elif self.when == 'M':
             self.interval = 60 # one minute
             self.suffix = "%Y-%m-%d_%H-%M"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(?!\d)"
         elif self.when == 'H':
             self.interval = 60 * 60 # one hour
             self.suffix = "%Y-%m-%d_%H"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}_\d{2}(?!\d)"
         elif self.when == 'D' or self.when == 'MIDNIGHT':
             self.interval = 60 * 60 * 24 # one day
             self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}(?!\d)"
         elif self.when.startswith('W'):
             self.interval = 60 * 60 * 24 * 7 # one week
             if len(self.when) != 2:
@@ -253,11 +256,17 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                 raise ValueError("Invalid day specified for weekly rollover: %s" % self.when)
             self.dayOfWeek = int(self.when[1])
             self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}(?!\d)"
         else:
             raise ValueError("Invalid rollover interval specified: %s" % self.when)
 
-        self.extMatch = re.compile(self.extMatch, re.ASCII)
+        # extMatch is a pattern for matching a datetime suffix in a file name.
+        # After custom naming, it is no longer guaranteed to be separated by
+        # periods from other parts of the filename.  The lookup statements
+        # (?<!\d) and (?!\d) ensure that the datetime suffix (which itself
+        # starts and ends with digits) is not preceded or followed by digits.
+        # This reduces the number of false matches and improves performance.
+        self.extMatch = re.compile(extMatch, re.ASCII)
         self.interval = self.interval * interval # multiply by units requested
         # The following line added because the filename passed in could be a
         # path object (see Issue #27493), but self.baseFilename will be a string
@@ -295,11 +304,11 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                 rotate_ts = _MIDNIGHT
             else:
                 rotate_ts = ((self.atTime.hour * 60 + self.atTime.minute)*60 +
-                    self.atTime.second)
+                             self.atTime.second)
 
             r = rotate_ts - ((currentHour * 60 + currentMinute) * 60 +
-                currentSecond)
-            if r < 0:
+                             currentSecond)
+            if r <= 0:
                 # Rotate time is before the current time (for example when
                 # self.rotateAt is 13:45 and it now 14:15), rotation is
                 # tomorrow.
@@ -328,17 +337,21 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                         daysToWait = self.dayOfWeek - day
                     else:
                         daysToWait = 6 - day + self.dayOfWeek + 1
-                    newRolloverAt = result + (daysToWait * (60 * 60 * 24))
-                    if not self.utc:
-                        dstNow = t[-1]
-                        dstAtRollover = time.localtime(newRolloverAt)[-1]
-                        if dstNow != dstAtRollover:
-                            if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
-                                addend = -3600
-                            else:           # DST bows out before next rollover, so we need to add an hour
-                                addend = 3600
-                            newRolloverAt += addend
-                    result = newRolloverAt
+                    result += daysToWait * _MIDNIGHT
+                result += self.interval - _MIDNIGHT * 7
+            else:
+                result += self.interval - _MIDNIGHT
+            if not self.utc:
+                dstNow = t[-1]
+                dstAtRollover = time.localtime(result)[-1]
+                if dstNow != dstAtRollover:
+                    if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
+                        addend = -3600
+                        if not time.localtime(result-3600)[-1]:
+                            addend = 0
+                    else:           # DST bows out before next rollover, so we need to add an hour
+                        addend = 3600
+                    result += addend
         return result
 
     def shouldRollover(self, record):
@@ -348,11 +361,15 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         record is not used, as we are just comparing times, but it is needed so
         the method signatures are the same
         """
-        # See bpo-45401: Never rollover anything other than regular files
-        if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
-            return False
         t = int(time.time())
         if t >= self.rolloverAt:
+            # See #89564: Never rollover anything other than regular files
+            if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
+                # The file is not a regular file, so do not rollover, but do
+                # set the next rollover time to avoid repeated checks.
+                self.rolloverAt = self.computeRollover(t)
+                return False
+
             return True
         return False
 
@@ -365,32 +382,28 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         dirName, baseName = os.path.split(self.baseFilename)
         fileNames = os.listdir(dirName)
         result = []
-        # See bpo-44753: Don't use the extension when computing the prefix.
-        n, e = os.path.splitext(baseName)
-        prefix = n + '.'
-        plen = len(prefix)
-        for fileName in fileNames:
-            if self.namer is None:
-                # Our files will always start with baseName
-                if not fileName.startswith(baseName):
-                    continue
-            else:
-                # Our files could be just about anything after custom naming, but
-                # likely candidates are of the form
-                # foo.log.DATETIME_SUFFIX or foo.DATETIME_SUFFIX.log
-                if (not fileName.startswith(baseName) and fileName.endswith(e) and
-                    len(fileName) > (plen + 1) and not fileName[plen+1].isdigit()):
-                    continue
-
-            if fileName[:plen] == prefix:
-                suffix = fileName[plen:]
-                # See bpo-45628: The date/time suffix could be anywhere in the
-                # filename
-                parts = suffix.split('.')
-                for part in parts:
-                    if self.extMatch.match(part):
+        if self.namer is None:
+            prefix = baseName + '.'
+            plen = len(prefix)
+            for fileName in fileNames:
+                if fileName[:plen] == prefix:
+                    suffix = fileName[plen:]
+                    if self.extMatch.fullmatch(suffix):
+                        result.append(os.path.join(dirName, fileName))
+        else:
+            for fileName in fileNames:
+                # Our files could be just about anything after custom naming,
+                # but they should contain the datetime suffix.
+                # Try to find the datetime suffix in the file name and verify
+                # that the file name can be generated by this handler.
+                m = self.extMatch.search(fileName)
+                while m:
+                    dfn = self.namer(self.baseFilename + "." + m[0])
+                    if os.path.basename(dfn) == fileName:
                         result.append(os.path.join(dirName, fileName))
                         break
+                    m = self.extMatch.search(fileName, m.start() + 1)
+
         if len(result) < self.backupCount:
             result = []
         else:
@@ -406,17 +419,14 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         then we have to get a list of matching filenames, sort them and remove
         the one with the oldest suffix.
         """
-        if self.stream:
-            self.stream.close()
-            self.stream = None
         # get the time that this sequence started at and make it a TimeTuple
         currentTime = int(time.time())
-        dstNow = time.localtime(currentTime)[-1]
         t = self.rolloverAt - self.interval
         if self.utc:
             timeTuple = time.gmtime(t)
         else:
             timeTuple = time.localtime(t)
+            dstNow = time.localtime(currentTime)[-1]
             dstThen = timeTuple[-1]
             if dstNow != dstThen:
                 if dstNow:
@@ -427,26 +437,19 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         dfn = self.rotation_filename(self.baseFilename + "." +
                                      time.strftime(self.suffix, timeTuple))
         if os.path.exists(dfn):
-            os.remove(dfn)
+            # Already rolled over.
+            return
+
+        if self.stream:
+            self.stream.close()
+            self.stream = None
         self.rotate(self.baseFilename, dfn)
         if self.backupCount > 0:
             for s in self.getFilesToDelete():
                 os.remove(s)
         if not self.delay:
             self.stream = self._open()
-        newRolloverAt = self.computeRollover(currentTime)
-        while newRolloverAt <= currentTime:
-            newRolloverAt = newRolloverAt + self.interval
-        #If DST changes and midnight or weekly rollover, adjust for this.
-        if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc:
-            dstAtRollover = time.localtime(newRolloverAt)[-1]
-            if dstNow != dstAtRollover:
-                if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
-                    addend = -3600
-                else:           # DST bows out before next rollover, so we need to add an hour
-                    addend = 3600
-                newRolloverAt += addend
-        self.rolloverAt = newRolloverAt
+        self.rolloverAt = self.computeRollover(currentTime)
 
 class WatchedFileHandler(logging.FileHandler):
     """
@@ -800,7 +803,7 @@ class SysLogHandler(logging.Handler):
         "panic":    LOG_EMERG,      #  DEPRECATED
         "warn":     LOG_WARNING,    #  DEPRECATED
         "warning":  LOG_WARNING,
-        }
+    }
 
     facility_names = {
         "auth":         LOG_AUTH,
@@ -827,12 +830,10 @@ class SysLogHandler(logging.Handler):
         "local5":       LOG_LOCAL5,
         "local6":       LOG_LOCAL6,
         "local7":       LOG_LOCAL7,
-        }
+    }
 
-    #The map below appears to be trivially lowercasing the key. However,
-    #there's more to it than meets the eye - in some locales, lowercasing
-    #gives unexpected results. See SF #1524081: in the Turkish locale,
-    #"INFO".lower() != "info"
+    # Originally added to work around GH-43683. Unnecessary since GH-50043 but kept
+    # for backwards compatibility.
     priority_map = {
         "DEBUG" : "debug",
         "INFO" : "info",
@@ -859,12 +860,49 @@ class SysLogHandler(logging.Handler):
         self.address = address
         self.facility = facility
         self.socktype = socktype
+        self.socket = None
+        self.createSocket()
+
+    def _connect_unixsocket(self, address):
+        use_socktype = self.socktype
+        if use_socktype is None:
+            use_socktype = socket.SOCK_DGRAM
+        self.socket = socket.socket(socket.AF_UNIX, use_socktype)
+        try:
+            self.socket.connect(address)
+            # it worked, so set self.socktype to the used type
+            self.socktype = use_socktype
+        except OSError:
+            self.socket.close()
+            if self.socktype is not None:
+                # user didn't specify falling back, so fail
+                raise
+            use_socktype = socket.SOCK_STREAM
+            self.socket = socket.socket(socket.AF_UNIX, use_socktype)
+            try:
+                self.socket.connect(address)
+                # it worked, so set self.socktype to the used type
+                self.socktype = use_socktype
+            except OSError:
+                self.socket.close()
+                raise
+
+    def createSocket(self):
+        """
+        Try to create a socket and, if it's not a datagram socket, connect it
+        to the other end. This method is called during handler initialization,
+        but it's not regarded as an error if the other end isn't listening yet
+        --- the method will be called again when emitting an event,
+        if there is no socket at that point.
+        """
+        address = self.address
+        socktype = self.socktype
 
         if isinstance(address, str):
             self.unixsocket = True
             # Syslog server may be unavailable during handler initialisation.
             # C's openlog() function also ignores connection errors.
-            # Moreover, we ignore these errors while logging, so it not worse
+            # Moreover, we ignore these errors while logging, so it's not worse
             # to ignore it also here.
             try:
                 self._connect_unixsocket(address)
@@ -895,30 +933,6 @@ class SysLogHandler(logging.Handler):
             self.socket = sock
             self.socktype = socktype
 
-    def _connect_unixsocket(self, address):
-        use_socktype = self.socktype
-        if use_socktype is None:
-            use_socktype = socket.SOCK_DGRAM
-        self.socket = socket.socket(socket.AF_UNIX, use_socktype)
-        try:
-            self.socket.connect(address)
-            # it worked, so set self.socktype to the used type
-            self.socktype = use_socktype
-        except OSError:
-            self.socket.close()
-            if self.socktype is not None:
-                # user didn't specify falling back, so fail
-                raise
-            use_socktype = socket.SOCK_STREAM
-            self.socket = socket.socket(socket.AF_UNIX, use_socktype)
-            try:
-                self.socket.connect(address)
-                # it worked, so set self.socktype to the used type
-                self.socktype = use_socktype
-            except OSError:
-                self.socket.close()
-                raise
-
     def encodePriority(self, facility, priority):
         """
         Encode the facility and priority. You can pass in strings or
@@ -938,7 +952,10 @@ class SysLogHandler(logging.Handler):
         """
         self.acquire()
         try:
-            self.socket.close()
+            sock = self.socket
+            if sock:
+                self.socket = None
+                sock.close()
             logging.Handler.close(self)
         finally:
             self.release()
@@ -978,6 +995,10 @@ class SysLogHandler(logging.Handler):
             # Message is a string. Convert to bytes as required by RFC 5424
             msg = msg.encode('utf-8')
             msg = prio + msg
+
+            if not self.socket:
+                self.createSocket()
+
             if self.unixsocket:
                 try:
                     self.socket.send(msg)
@@ -1094,7 +1115,16 @@ class NTEventLogHandler(logging.Handler):
                 dllname = os.path.join(dllname[0], r'win32service.pyd')
             self.dllname = dllname
             self.logtype = logtype
-            self._welu.AddSourceToRegistry(appname, dllname, logtype)
+            # Administrative privileges are required to add a source to the registry.
+            # This may not be available for a user that just wants to add to an
+            # existing source - handle this specific case.
+            try:
+                self._welu.AddSourceToRegistry(appname, dllname, logtype)
+            except Exception as e:
+                # This will probably be a pywintypes.error. Only raise if it's not
+                # an "access denied" error, else let it pass
+                if getattr(e, 'winerror', None) != 5:  # not access denied
+                    raise
             self.deftype = win32evtlog.EVENTLOG_ERROR_TYPE
             self.typemap = {
                 logging.DEBUG   : win32evtlog.EVENTLOG_INFORMATION_TYPE,
@@ -1102,10 +1132,10 @@ class NTEventLogHandler(logging.Handler):
                 logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE,
                 logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
                 logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
-         }
+            }
         except ImportError:
-            print("The Python Win32 extensions for NT (service, event "\
-                        "logging) appear not to be available.")
+            print("The Python Win32 extensions for NT (service, event " \
+                  "logging) appear not to be available.")
             self._welu = None
 
     def getMessageID(self, record):
@@ -1348,7 +1378,7 @@ class MemoryHandler(BufferingHandler):
         Check for buffer full or a record at the flushLevel or higher.
         """
         return (len(self.buffer) >= self.capacity) or \
-                (record.levelno >= self.flushLevel)
+            (record.levelno >= self.flushLevel)
 
     def setTarget(self, target):
         """
@@ -1366,7 +1396,7 @@ class MemoryHandler(BufferingHandler):
         records to the target, if there is one. Override if you want
         different behaviour.
 
-        The record buffer is also cleared by this operation.
+        The record buffer is only cleared if a target has been set.
         """
         self.acquire()
         try:
@@ -1411,6 +1441,7 @@ class QueueHandler(logging.Handler):
         """
         logging.Handler.__init__(self)
         self.queue = queue
+        self.listener = None  # will be set to listener if configured via dictConfig()
 
     def enqueue(self, record):
         """
@@ -1424,12 +1455,15 @@ class QueueHandler(logging.Handler):
 
     def prepare(self, record):
         """
-        Prepares a record for queuing. The object returned by this method is
+        Prepare a record for queuing. The object returned by this method is
         enqueued.
 
-        The base implementation formats the record to merge the message
-        and arguments, and removes unpickleable items from the record
-        in-place.
+        The base implementation formats the record to merge the message and
+        arguments, and removes unpickleable items from the record in-place.
+        Specifically, it overwrites the record's `msg` and
+        `message` attributes with the merged message (obtained by
+        calling the handler's `format` method), and sets the `args`,
+        `exc_info` and `exc_text` attributes to None.
 
         You might want to override this method if you want to convert
         the record to a dict or JSON string, or send a modified copy
@@ -1439,7 +1473,7 @@ class QueueHandler(logging.Handler):
         # (if there's exception data), and also returns the formatted
         # message. We can then use this to replace the original
         # msg + args, as these might be unpickleable. We also zap the
-        # exc_info and exc_text attributes, as they are no longer
+        # exc_info, exc_text and stack_info attributes, as they are no longer
         # needed and, if not None, will typically not be pickleable.
         msg = self.format(record)
         # bpo-35726: make copy of record to avoid affecting other handlers in the chain.
@@ -1449,6 +1483,7 @@ class QueueHandler(logging.Handler):
         record.args = None
         record.exc_info = None
         record.exc_text = None
+        record.stack_info = None
         return record
 
     def emit(self, record):

Lib/test/support/__init__.py

@@ -400,33 +400,37 @@ def skip_if_buildbot(reason=None):
         isbuildbot = False
     return unittest.skipIf(isbuildbot, reason)
 
-def check_sanitizer(*, address=False, memory=False, ub=False):
+def check_sanitizer(*, address=False, memory=False, ub=False, thread=False):
     """Returns True if Python is compiled with sanitizer support"""
-    if not (address or memory or ub):
-        raise ValueError('At least one of address, memory, or ub must be True')
+    if not (address or memory or ub or thread):
+        raise ValueError('At least one of address, memory, ub or thread must be True')
 
 
-    _cflags = sysconfig.get_config_var('CFLAGS') or ''
-    _config_args = sysconfig.get_config_var('CONFIG_ARGS') or ''
+    cflags = sysconfig.get_config_var('CFLAGS') or ''
+    config_args = sysconfig.get_config_var('CONFIG_ARGS') or ''
     memory_sanitizer = (
-        '-fsanitize=memory' in _cflags or
-        '--with-memory-sanitizer' in _config_args
+            '-fsanitize=memory' in cflags or
+            '--with-memory-sanitizer' in config_args
     )
     address_sanitizer = (
-        '-fsanitize=address' in _cflags or
-        '--with-address-sanitizer' in _config_args
+            '-fsanitize=address' in cflags or
+            '--with-address-sanitizer' in config_args
     )
     ub_sanitizer = (
-        '-fsanitize=undefined' in _cflags or
-        '--with-undefined-behavior-sanitizer' in _config_args
+            '-fsanitize=undefined' in cflags or
+            '--with-undefined-behavior-sanitizer' in config_args
+    )
+    thread_sanitizer = (
+            '-fsanitize=thread' in cflags or
+            '--with-thread-sanitizer' in config_args
     )
     return (
-        (memory and memory_sanitizer) or
-        (address and address_sanitizer) or
-        (ub and ub_sanitizer)
+            (memory and memory_sanitizer) or
+            (address and address_sanitizer) or
+            (ub and ub_sanitizer) or
+            (thread and thread_sanitizer)
     )
 
-
 def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False, thread=False):
     """Decorator raising SkipTest if running with a sanitizer active."""
     if not reason:
@@ -532,6 +536,10 @@ is_wasi = sys.platform == "wasi"
 
 has_fork_support = hasattr(os, "fork") and not is_emscripten and not is_wasi
 
+# From python 3.12.6
+is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x'
+skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x')
+
 def requires_fork():
     return unittest.skipUnless(has_fork_support, "requires working os.fork()")
 
@@ -2546,3 +2554,21 @@ C_RECURSION_LIMIT = 1500
 #Windows doesn't have os.uname() but it doesn't support s390x.
 skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
                                 'skipped on s390x')
+HAVE_ASAN_FORK_BUG = check_sanitizer(address=True)
+
+# From python 3.12.8
+class BrokenIter:
+    def __init__(self, init_raises=False, next_raises=False, iter_raises=False):
+        if init_raises:
+            1/0
+        self.next_raises = next_raises
+        self.iter_raises = iter_raises
+
+    def __next__(self):
+        if self.next_raises:
+            1/0
+
+    def __iter__(self):
+        if self.iter_raises:
+            1/0
+        return self

Lib/test/support/smtpd.py

@@ -0,0 +1,873 @@
+#! /usr/bin/env python3
+"""An RFC 5321 smtp proxy with optional RFC 1870 and RFC 6531 extensions.
+
+Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]]
+
+Options:
+
+    --nosetuid
+    -n
+        This program generally tries to setuid `nobody', unless this flag is
+        set.  The setuid call will fail if this program is not run as root (in
+        which case, use this flag).
+
+    --version
+    -V
+        Print the version number and exit.
+
+    --class classname
+    -c classname
+        Use `classname' as the concrete SMTP proxy class.  Uses `PureProxy' by
+        default.
+
+    --size limit
+    -s limit
+        Restrict the total size of the incoming message to "limit" number of
+        bytes via the RFC 1870 SIZE extension.  Defaults to 33554432 bytes.
+
+    --smtputf8
+    -u
+        Enable the SMTPUTF8 extension and behave as an RFC 6531 smtp proxy.
+
+    --debug
+    -d
+        Turn on debugging prints.
+
+    --help
+    -h
+        Print this message and exit.
+
+Version: %(__version__)s
+
+If localhost is not given then `localhost' is used, and if localport is not
+given then 8025 is used.  If remotehost is not given then `localhost' is used,
+and if remoteport is not given, then 25 is used.
+"""
+
+# Overview:
+#
+# This file implements the minimal SMTP protocol as defined in RFC 5321.  It
+# has a hierarchy of classes which implement the backend functionality for the
+# smtpd.  A number of classes are provided:
+#
+#   SMTPServer - the base class for the backend.  Raises NotImplementedError
+#   if you try to use it.
+#
+#   DebuggingServer - simply prints each message it receives on stdout.
+#
+#   PureProxy - Proxies all messages to a real smtpd which does final
+#   delivery.  One known problem with this class is that it doesn't handle
+#   SMTP errors from the backend server at all.  This should be fixed
+#   (contributions are welcome!).
+#
+#
+# Author: Barry Warsaw <barry@python.org>
+#
+# TODO:
+#
+# - support mailbox delivery
+# - alias files
+# - Handle more ESMTP extensions
+# - handle error codes from the backend smtpd
+
+import sys
+import os
+import errno
+import getopt
+import time
+import socket
+import collections
+from test.support import asyncore, asynchat
+from warnings import warn
+from email._header_value_parser import get_addr_spec, get_angle_addr
+
+__all__ = [
+    "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy",
+]
+
+program = sys.argv[0]
+__version__ = 'Python SMTP proxy version 0.3'
+
+
+class Devnull:
+    def write(self, msg): pass
+    def flush(self): pass
+
+
+DEBUGSTREAM = Devnull()
+NEWLINE = '\n'
+COMMASPACE = ', '
+DATA_SIZE_DEFAULT = 33554432
+
+
+def usage(code, msg=''):
+    print(__doc__ % globals(), file=sys.stderr)
+    if msg:
+        print(msg, file=sys.stderr)
+    sys.exit(code)
+
+
+class SMTPChannel(asynchat.async_chat):
+    COMMAND = 0
+    DATA = 1
+
+    command_size_limit = 512
+    command_size_limits = collections.defaultdict(lambda x=command_size_limit: x)
+
+    @property
+    def max_command_size_limit(self):
+        try:
+            return max(self.command_size_limits.values())
+        except ValueError:
+            return self.command_size_limit
+
+    def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT,
+                 map=None, enable_SMTPUTF8=False, decode_data=False):
+        asynchat.async_chat.__init__(self, conn, map=map)
+        self.smtp_server = server
+        self.conn = conn
+        self.addr = addr
+        self.data_size_limit = data_size_limit
+        self.enable_SMTPUTF8 = enable_SMTPUTF8
+        self._decode_data = decode_data
+        if enable_SMTPUTF8 and decode_data:
+            raise ValueError("decode_data and enable_SMTPUTF8 cannot"
+                             " be set to True at the same time")
+        if decode_data:
+            self._emptystring = ''
+            self._linesep = '\r\n'
+            self._dotsep = '.'
+            self._newline = NEWLINE
+        else:
+            self._emptystring = b''
+            self._linesep = b'\r\n'
+            self._dotsep = ord(b'.')
+            self._newline = b'\n'
+        self._set_rset_state()
+        self.seen_greeting = ''
+        self.extended_smtp = False
+        self.command_size_limits.clear()
+        self.fqdn = socket.getfqdn()
+        try:
+            self.peer = conn.getpeername()
+        except OSError as err:
+            # a race condition  may occur if the other end is closing
+            # before we can get the peername
+            self.close()
+            if err.errno != errno.ENOTCONN:
+                raise
+            return
+        print('Peer:', repr(self.peer), file=DEBUGSTREAM)
+        self.push('220 %s %s' % (self.fqdn, __version__))
+
+    def _set_post_data_state(self):
+        """Reset state variables to their post-DATA state."""
+        self.smtp_state = self.COMMAND
+        self.mailfrom = None
+        self.rcpttos = []
+        self.require_SMTPUTF8 = False
+        self.num_bytes = 0
+        self.set_terminator(b'\r\n')
+
+    def _set_rset_state(self):
+        """Reset all state variables except the greeting."""
+        self._set_post_data_state()
+        self.received_data = ''
+        self.received_lines = []
+
+
+    # properties for backwards-compatibility
+    @property
+    def __server(self):
+        warn("Access to __server attribute on SMTPChannel is deprecated, "
+             "use 'smtp_server' instead", DeprecationWarning, 2)
+        return self.smtp_server
+    @__server.setter
+    def __server(self, value):
+        warn("Setting __server attribute on SMTPChannel is deprecated, "
+             "set 'smtp_server' instead", DeprecationWarning, 2)
+        self.smtp_server = value
+
+    @property
+    def __line(self):
+        warn("Access to __line attribute on SMTPChannel is deprecated, "
+             "use 'received_lines' instead", DeprecationWarning, 2)
+        return self.received_lines
+    @__line.setter
+    def __line(self, value):
+        warn("Setting __line attribute on SMTPChannel is deprecated, "
+             "set 'received_lines' instead", DeprecationWarning, 2)
+        self.received_lines = value
+
+    @property
+    def __state(self):
+        warn("Access to __state attribute on SMTPChannel is deprecated, "
+             "use 'smtp_state' instead", DeprecationWarning, 2)
+        return self.smtp_state
+    @__state.setter
+    def __state(self, value):
+        warn("Setting __state attribute on SMTPChannel is deprecated, "
+             "set 'smtp_state' instead", DeprecationWarning, 2)
+        self.smtp_state = value
+
+    @property
+    def __greeting(self):
+        warn("Access to __greeting attribute on SMTPChannel is deprecated, "
+             "use 'seen_greeting' instead", DeprecationWarning, 2)
+        return self.seen_greeting
+    @__greeting.setter
+    def __greeting(self, value):
+        warn("Setting __greeting attribute on SMTPChannel is deprecated, "
+             "set 'seen_greeting' instead", DeprecationWarning, 2)
+        self.seen_greeting = value
+
+    @property
+    def __mailfrom(self):
+        warn("Access to __mailfrom attribute on SMTPChannel is deprecated, "
+             "use 'mailfrom' instead", DeprecationWarning, 2)
+        return self.mailfrom
+    @__mailfrom.setter
+    def __mailfrom(self, value):
+        warn("Setting __mailfrom attribute on SMTPChannel is deprecated, "
+             "set 'mailfrom' instead", DeprecationWarning, 2)
+        self.mailfrom = value
+
+    @property
+    def __rcpttos(self):
+        warn("Access to __rcpttos attribute on SMTPChannel is deprecated, "
+             "use 'rcpttos' instead", DeprecationWarning, 2)
+        return self.rcpttos
+    @__rcpttos.setter
+    def __rcpttos(self, value):
+        warn("Setting __rcpttos attribute on SMTPChannel is deprecated, "
+             "set 'rcpttos' instead", DeprecationWarning, 2)
+        self.rcpttos = value
+
+    @property
+    def __data(self):
+        warn("Access to __data attribute on SMTPChannel is deprecated, "
+             "use 'received_data' instead", DeprecationWarning, 2)
+        return self.received_data
+    @__data.setter
+    def __data(self, value):
+        warn("Setting __data attribute on SMTPChannel is deprecated, "
+             "set 'received_data' instead", DeprecationWarning, 2)
+        self.received_data = value
+
+    @property
+    def __fqdn(self):
+        warn("Access to __fqdn attribute on SMTPChannel is deprecated, "
+             "use 'fqdn' instead", DeprecationWarning, 2)
+        return self.fqdn
+    @__fqdn.setter
+    def __fqdn(self, value):
+        warn("Setting __fqdn attribute on SMTPChannel is deprecated, "
+             "set 'fqdn' instead", DeprecationWarning, 2)
+        self.fqdn = value
+
+    @property
+    def __peer(self):
+        warn("Access to __peer attribute on SMTPChannel is deprecated, "
+             "use 'peer' instead", DeprecationWarning, 2)
+        return self.peer
+    @__peer.setter
+    def __peer(self, value):
+        warn("Setting __peer attribute on SMTPChannel is deprecated, "
+             "set 'peer' instead", DeprecationWarning, 2)
+        self.peer = value
+
+    @property
+    def __conn(self):
+        warn("Access to __conn attribute on SMTPChannel is deprecated, "
+             "use 'conn' instead", DeprecationWarning, 2)
+        return self.conn
+    @__conn.setter
+    def __conn(self, value):
+        warn("Setting __conn attribute on SMTPChannel is deprecated, "
+             "set 'conn' instead", DeprecationWarning, 2)
+        self.conn = value
+
+    @property
+    def __addr(self):
+        warn("Access to __addr attribute on SMTPChannel is deprecated, "
+             "use 'addr' instead", DeprecationWarning, 2)
+        return self.addr
+    @__addr.setter
+    def __addr(self, value):
+        warn("Setting __addr attribute on SMTPChannel is deprecated, "
+             "set 'addr' instead", DeprecationWarning, 2)
+        self.addr = value
+
+    # Overrides base class for convenience.
+    def push(self, msg):
+        asynchat.async_chat.push(self, bytes(
+            msg + '\r\n', 'utf-8' if self.require_SMTPUTF8 else 'ascii'))
+
+    # Implementation of base class abstract method
+    def collect_incoming_data(self, data):
+        limit = None
+        if self.smtp_state == self.COMMAND:
+            limit = self.max_command_size_limit
+        elif self.smtp_state == self.DATA:
+            limit = self.data_size_limit
+        if limit and self.num_bytes > limit:
+            return
+        elif limit:
+            self.num_bytes += len(data)
+        if self._decode_data:
+            self.received_lines.append(str(data, 'utf-8'))
+        else:
+            self.received_lines.append(data)
+
+    # Implementation of base class abstract method
+    def found_terminator(self):
+        line = self._emptystring.join(self.received_lines)
+        print('Data:', repr(line), file=DEBUGSTREAM)
+        self.received_lines = []
+        if self.smtp_state == self.COMMAND:
+            sz, self.num_bytes = self.num_bytes, 0
+            if not line:
+                self.push('500 Error: bad syntax')
+                return
+            if not self._decode_data:
+                line = str(line, 'utf-8')
+            i = line.find(' ')
+            if i < 0:
+                command = line.upper()
+                arg = None
+            else:
+                command = line[:i].upper()
+                arg = line[i+1:].strip()
+            max_sz = (self.command_size_limits[command]
+                      if self.extended_smtp else self.command_size_limit)
+            if sz > max_sz:
+                self.push('500 Error: line too long')
+                return
+            method = getattr(self, 'smtp_' + command, None)
+            if not method:
+                self.push('500 Error: command "%s" not recognized' % command)
+                return
+            method(arg)
+            return
+        else:
+            if self.smtp_state != self.DATA:
+                self.push('451 Internal confusion')
+                self.num_bytes = 0
+                return
+            if self.data_size_limit and self.num_bytes > self.data_size_limit:
+                self.push('552 Error: Too much mail data')
+                self.num_bytes = 0
+                return
+            # Remove extraneous carriage returns and de-transparency according
+            # to RFC 5321, Section 4.5.2.
+            data = []
+            for text in line.split(self._linesep):
+                if text and text[0] == self._dotsep:
+                    data.append(text[1:])
+                else:
+                    data.append(text)
+            self.received_data = self._newline.join(data)
+            args = (self.peer, self.mailfrom, self.rcpttos, self.received_data)
+            kwargs = {}
+            if not self._decode_data:
+                kwargs = {
+                    'mail_options': self.mail_options,
+                    'rcpt_options': self.rcpt_options,
+                }
+            status = self.smtp_server.process_message(*args, **kwargs)
+            self._set_post_data_state()
+            if not status:
+                self.push('250 OK')
+            else:
+                self.push(status)
+
+    # SMTP and ESMTP commands
+    def smtp_HELO(self, arg):
+        if not arg:
+            self.push('501 Syntax: HELO hostname')
+            return
+        # See issue #21783 for a discussion of this behavior.
+        if self.seen_greeting:
+            self.push('503 Duplicate HELO/EHLO')
+            return
+        self._set_rset_state()
+        self.seen_greeting = arg
+        self.push('250 %s' % self.fqdn)
+
+    def smtp_EHLO(self, arg):
+        if not arg:
+            self.push('501 Syntax: EHLO hostname')
+            return
+        # See issue #21783 for a discussion of this behavior.
+        if self.seen_greeting:
+            self.push('503 Duplicate HELO/EHLO')
+            return
+        self._set_rset_state()
+        self.seen_greeting = arg
+        self.extended_smtp = True
+        self.push('250-%s' % self.fqdn)
+        if self.data_size_limit:
+            self.push('250-SIZE %s' % self.data_size_limit)
+            self.command_size_limits['MAIL'] += 26
+        if not self._decode_data:
+            self.push('250-8BITMIME')
+        if self.enable_SMTPUTF8:
+            self.push('250-SMTPUTF8')
+            self.command_size_limits['MAIL'] += 10
+        self.push('250 HELP')
+
+    def smtp_NOOP(self, arg):
+        if arg:
+            self.push('501 Syntax: NOOP')
+        else:
+            self.push('250 OK')
+
+    def smtp_QUIT(self, arg):
+        # args is ignored
+        self.push('221 Bye')
+        self.close_when_done()
+
+    def _strip_command_keyword(self, keyword, arg):
+        keylen = len(keyword)
+        if arg[:keylen].upper() == keyword:
+            return arg[keylen:].strip()
+        return ''
+
+    def _getaddr(self, arg):
+        if not arg:
+            return '', ''
+        if arg.lstrip().startswith('<'):
+            address, rest = get_angle_addr(arg)
+        else:
+            address, rest = get_addr_spec(arg)
+        if not address:
+            return address, rest
+        return address.addr_spec, rest
+
+    def _getparams(self, params):
+        # Return params as dictionary. Return None if not all parameters
+        # appear to be syntactically valid according to RFC 1869.
+        result = {}
+        for param in params:
+            param, eq, value = param.partition('=')
+            if not param.isalnum() or eq and not value:
+                return None
+            result[param] = value if eq else True
+        return result
+
+    def smtp_HELP(self, arg):
+        if arg:
+            extended = ' [SP <mail-parameters>]'
+            lc_arg = arg.upper()
+            if lc_arg == 'EHLO':
+                self.push('250 Syntax: EHLO hostname')
+            elif lc_arg == 'HELO':
+                self.push('250 Syntax: HELO hostname')
+            elif lc_arg == 'MAIL':
+                msg = '250 Syntax: MAIL FROM: <address>'
+                if self.extended_smtp:
+                    msg += extended
+                self.push(msg)
+            elif lc_arg == 'RCPT':
+                msg = '250 Syntax: RCPT TO: <address>'
+                if self.extended_smtp:
+                    msg += extended
+                self.push(msg)
+            elif lc_arg == 'DATA':
+                self.push('250 Syntax: DATA')
+            elif lc_arg == 'RSET':
+                self.push('250 Syntax: RSET')
+            elif lc_arg == 'NOOP':
+                self.push('250 Syntax: NOOP')
+            elif lc_arg == 'QUIT':
+                self.push('250 Syntax: QUIT')
+            elif lc_arg == 'VRFY':
+                self.push('250 Syntax: VRFY <address>')
+            else:
+                self.push('501 Supported commands: EHLO HELO MAIL RCPT '
+                          'DATA RSET NOOP QUIT VRFY')
+        else:
+            self.push('250 Supported commands: EHLO HELO MAIL RCPT DATA '
+                      'RSET NOOP QUIT VRFY')
+
+    def smtp_VRFY(self, arg):
+        if arg:
+            address, params = self._getaddr(arg)
+            if address:
+                self.push('252 Cannot VRFY user, but will accept message '
+                          'and attempt delivery')
+            else:
+                self.push('502 Could not VRFY %s' % arg)
+        else:
+            self.push('501 Syntax: VRFY <address>')
+
+    def smtp_MAIL(self, arg):
+        if not self.seen_greeting:
+            self.push('503 Error: send HELO first')
+            return
+        print('===> MAIL', arg, file=DEBUGSTREAM)
+        syntaxerr = '501 Syntax: MAIL FROM: <address>'
+        if self.extended_smtp:
+            syntaxerr += ' [SP <mail-parameters>]'
+        if arg is None:
+            self.push(syntaxerr)
+            return
+        arg = self._strip_command_keyword('FROM:', arg)
+        address, params = self._getaddr(arg)
+        if not address:
+            self.push(syntaxerr)
+            return
+        if not self.extended_smtp and params:
+            self.push(syntaxerr)
+            return
+        if self.mailfrom:
+            self.push('503 Error: nested MAIL command')
+            return
+        self.mail_options = params.upper().split()
+        params = self._getparams(self.mail_options)
+        if params is None:
+            self.push(syntaxerr)
+            return
+        if not self._decode_data:
+            body = params.pop('BODY', '7BIT')
+            if body not in ['7BIT', '8BITMIME']:
+                self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME')
+                return
+        if self.enable_SMTPUTF8:
+            smtputf8 = params.pop('SMTPUTF8', False)
+            if smtputf8 is True:
+                self.require_SMTPUTF8 = True
+            elif smtputf8 is not False:
+                self.push('501 Error: SMTPUTF8 takes no arguments')
+                return
+        size = params.pop('SIZE', None)
+        if size:
+            if not size.isdigit():
+                self.push(syntaxerr)
+                return
+            elif self.data_size_limit and int(size) > self.data_size_limit:
+                self.push('552 Error: message size exceeds fixed maximum message size')
+                return
+        if len(params.keys()) > 0:
+            self.push('555 MAIL FROM parameters not recognized or not implemented')
+            return
+        self.mailfrom = address
+        print('sender:', self.mailfrom, file=DEBUGSTREAM)
+        self.push('250 OK')
+
+    def smtp_RCPT(self, arg):
+        if not self.seen_greeting:
+            self.push('503 Error: send HELO first');
+            return
+        print('===> RCPT', arg, file=DEBUGSTREAM)
+        if not self.mailfrom:
+            self.push('503 Error: need MAIL command')
+            return
+        syntaxerr = '501 Syntax: RCPT TO: <address>'
+        if self.extended_smtp:
+            syntaxerr += ' [SP <mail-parameters>]'
+        if arg is None:
+            self.push(syntaxerr)
+            return
+        arg = self._strip_command_keyword('TO:', arg)
+        address, params = self._getaddr(arg)
+        if not address:
+            self.push(syntaxerr)
+            return
+        if not self.extended_smtp and params:
+            self.push(syntaxerr)
+            return
+        self.rcpt_options = params.upper().split()
+        params = self._getparams(self.rcpt_options)
+        if params is None:
+            self.push(syntaxerr)
+            return
+        # XXX currently there are no options we recognize.
+        if len(params.keys()) > 0:
+            self.push('555 RCPT TO parameters not recognized or not implemented')
+            return
+        self.rcpttos.append(address)
+        print('recips:', self.rcpttos, file=DEBUGSTREAM)
+        self.push('250 OK')
+
+    def smtp_RSET(self, arg):
+        if arg:
+            self.push('501 Syntax: RSET')
+            return
+        self._set_rset_state()
+        self.push('250 OK')
+
+    def smtp_DATA(self, arg):
+        if not self.seen_greeting:
+            self.push('503 Error: send HELO first');
+            return
+        if not self.rcpttos:
+            self.push('503 Error: need RCPT command')
+            return
+        if arg:
+            self.push('501 Syntax: DATA')
+            return
+        self.smtp_state = self.DATA
+        self.set_terminator(b'\r\n.\r\n')
+        self.push('354 End data with <CR><LF>.<CR><LF>')
+
+    # Commands that have not been implemented
+    def smtp_EXPN(self, arg):
+        self.push('502 EXPN not implemented')
+
+
+class SMTPServer(asyncore.dispatcher):
+    # SMTPChannel class to use for managing client connections
+    channel_class = SMTPChannel
+
+    def __init__(self, localaddr, remoteaddr,
+                 data_size_limit=DATA_SIZE_DEFAULT, map=None,
+                 enable_SMTPUTF8=False, decode_data=False):
+        self._localaddr = localaddr
+        self._remoteaddr = remoteaddr
+        self.data_size_limit = data_size_limit
+        self.enable_SMTPUTF8 = enable_SMTPUTF8
+        self._decode_data = decode_data
+        if enable_SMTPUTF8 and decode_data:
+            raise ValueError("decode_data and enable_SMTPUTF8 cannot"
+                             " be set to True at the same time")
+        asyncore.dispatcher.__init__(self, map=map)
+        try:
+            gai_results = socket.getaddrinfo(*localaddr,
+                                             type=socket.SOCK_STREAM)
+            self.create_socket(gai_results[0][0], gai_results[0][1])
+            # try to re-use a server port if possible
+            self.set_reuse_addr()
+            self.bind(localaddr)
+            self.listen(5)
+        except:
+            self.close()
+            raise
+        else:
+            print('%s started at %s\n\tLocal addr: %s\n\tRemote addr:%s' % (
+                self.__class__.__name__, time.ctime(time.time()),
+                localaddr, remoteaddr), file=DEBUGSTREAM)
+
+    def handle_accepted(self, conn, addr):
+        print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM)
+        channel = self.channel_class(self,
+                                     conn,
+                                     addr,
+                                     self.data_size_limit,
+                                     self._map,
+                                     self.enable_SMTPUTF8,
+                                     self._decode_data)
+
+    # API for "doing something useful with the message"
+    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
+        """Override this abstract method to handle messages from the client.
+
+        peer is a tuple containing (ipaddr, port) of the client that made the
+        socket connection to our smtp port.
+
+        mailfrom is the raw address the client claims the message is coming
+        from.
+
+        rcpttos is a list of raw addresses the client wishes to deliver the
+        message to.
+
+        data is a string containing the entire full text of the message,
+        headers (if supplied) and all.  It has been `de-transparencied'
+        according to RFC 821, Section 4.5.2.  In other words, a line
+        containing a `.' followed by other text has had the leading dot
+        removed.
+
+        kwargs is a dictionary containing additional information.  It is
+        empty if decode_data=True was given as init parameter, otherwise
+        it will contain the following keys:
+            'mail_options': list of parameters to the mail command.  All
+                            elements are uppercase strings.  Example:
+                            ['BODY=8BITMIME', 'SMTPUTF8'].
+            'rcpt_options': same, for the rcpt command.
+
+        This function should return None for a normal `250 Ok' response;
+        otherwise, it should return the desired response string in RFC 821
+        format.
+
+        """
+        raise NotImplementedError
+
+
+class DebuggingServer(SMTPServer):
+
+    def _print_message_content(self, peer, data):
+        inheaders = 1
+        lines = data.splitlines()
+        for line in lines:
+            # headers first
+            if inheaders and not line:
+                peerheader = 'X-Peer: ' + peer[0]
+                if not isinstance(data, str):
+                    # decoded_data=false; make header match other binary output
+                    peerheader = repr(peerheader.encode('utf-8'))
+                print(peerheader)
+                inheaders = 0
+            if not isinstance(data, str):
+                # Avoid spurious 'str on bytes instance' warning.
+                line = repr(line)
+            print(line)
+
+    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
+        print('---------- MESSAGE FOLLOWS ----------')
+        if kwargs:
+            if kwargs.get('mail_options'):
+                print('mail options: %s' % kwargs['mail_options'])
+            if kwargs.get('rcpt_options'):
+                print('rcpt options: %s\n' % kwargs['rcpt_options'])
+        self._print_message_content(peer, data)
+        print('------------ END MESSAGE ------------')
+
+
+class PureProxy(SMTPServer):
+    def __init__(self, *args, **kwargs):
+        if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']:
+            raise ValueError("PureProxy does not support SMTPUTF8.")
+        super(PureProxy, self).__init__(*args, **kwargs)
+
+    def process_message(self, peer, mailfrom, rcpttos, data):
+        lines = data.split('\n')
+        # Look for the last header
+        i = 0
+        for line in lines:
+            if not line:
+                break
+            i += 1
+        lines.insert(i, 'X-Peer: %s' % peer[0])
+        data = NEWLINE.join(lines)
+        refused = self._deliver(mailfrom, rcpttos, data)
+        # TBD: what to do with refused addresses?
+        print('we got some refusals:', refused, file=DEBUGSTREAM)
+
+    def _deliver(self, mailfrom, rcpttos, data):
+        import smtplib
+        refused = {}
+        try:
+            s = smtplib.SMTP()
+            s.connect(self._remoteaddr[0], self._remoteaddr[1])
+            try:
+                refused = s.sendmail(mailfrom, rcpttos, data)
+            finally:
+                s.quit()
+        except smtplib.SMTPRecipientsRefused as e:
+            print('got SMTPRecipientsRefused', file=DEBUGSTREAM)
+            refused = e.recipients
+        except (OSError, smtplib.SMTPException) as e:
+            print('got', e.__class__, file=DEBUGSTREAM)
+            # All recipients were refused.  If the exception had an associated
+            # error code, use it.  Otherwise,fake it with a non-triggering
+            # exception code.
+            errcode = getattr(e, 'smtp_code', -1)
+            errmsg = getattr(e, 'smtp_error', 'ignore')
+            for r in rcpttos:
+                refused[r] = (errcode, errmsg)
+        return refused
+
+
+class Options:
+    setuid = True
+    classname = 'PureProxy'
+    size_limit = None
+    enable_SMTPUTF8 = False
+
+
+def parseargs():
+    global DEBUGSTREAM
+    try:
+        opts, args = getopt.getopt(
+            sys.argv[1:], 'nVhc:s:du',
+            ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug',
+             'smtputf8'])
+    except getopt.error as e:
+        usage(1, e)
+
+    options = Options()
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+        elif opt in ('-V', '--version'):
+            print(__version__)
+            sys.exit(0)
+        elif opt in ('-n', '--nosetuid'):
+            options.setuid = False
+        elif opt in ('-c', '--class'):
+            options.classname = arg
+        elif opt in ('-d', '--debug'):
+            DEBUGSTREAM = sys.stderr
+        elif opt in ('-u', '--smtputf8'):
+            options.enable_SMTPUTF8 = True
+        elif opt in ('-s', '--size'):
+            try:
+                int_size = int(arg)
+                options.size_limit = int_size
+            except:
+                print('Invalid size: ' + arg, file=sys.stderr)
+                sys.exit(1)
+
+    # parse the rest of the arguments
+    if len(args) < 1:
+        localspec = 'localhost:8025'
+        remotespec = 'localhost:25'
+    elif len(args) < 2:
+        localspec = args[0]
+        remotespec = 'localhost:25'
+    elif len(args) < 3:
+        localspec = args[0]
+        remotespec = args[1]
+    else:
+        usage(1, 'Invalid arguments: %s' % COMMASPACE.join(args))
+
+    # split into host/port pairs
+    i = localspec.find(':')
+    if i < 0:
+        usage(1, 'Bad local spec: %s' % localspec)
+    options.localhost = localspec[:i]
+    try:
+        options.localport = int(localspec[i+1:])
+    except ValueError:
+        usage(1, 'Bad local port: %s' % localspec)
+    i = remotespec.find(':')
+    if i < 0:
+        usage(1, 'Bad remote spec: %s' % remotespec)
+    options.remotehost = remotespec[:i]
+    try:
+        options.remoteport = int(remotespec[i+1:])
+    except ValueError:
+        usage(1, 'Bad remote port: %s' % remotespec)
+    return options
+
+
+if __name__ == '__main__':
+    options = parseargs()
+    # Become nobody
+    classname = options.classname
+    if "." in classname:
+        lastdot = classname.rfind(".")
+        mod = __import__(classname[:lastdot], globals(), locals(), [""])
+        classname = classname[lastdot+1:]
+    else:
+        import __main__ as mod
+    class_ = getattr(mod, classname)
+    proxy = class_((options.localhost, options.localport),
+                   (options.remotehost, options.remoteport),
+                   options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8)
+    if options.setuid:
+        try:
+            import pwd
+        except ImportError:
+            print('Cannot import module "pwd"; try running with -n option.', file=sys.stderr)
+            sys.exit(1)
+        nobody = pwd.getpwnam('nobody')[2]
+        try:
+            os.setuid(nobody)
+        except PermissionError:
+            print('Cannot setuid "nobody"; try running with -n option.', file=sys.stderr)
+            sys.exit(1)
+    try:
+        asyncore.loop()
+    except KeyboardInterrupt:
+        pass

Lib/test/support/testcase.py

@@ -0,0 +1,65 @@
+from math import copysign, isnan
+
+
+class ExceptionIsLikeMixin:
+    def assertExceptionIsLike(self, exc, template):
+        """
+        Passes when the provided `exc` matches the structure of `template`.
+        Individual exceptions don't have to be the same objects or even pass
+        an equality test: they only need to be the same type and contain equal
+        `exc_obj.args`.
+        """
+        if exc is None and template is None:
+            return
+
+        if template is None:
+            self.fail(f"unexpected exception: {exc}")
+
+        if exc is None:
+            self.fail(f"expected an exception like {template!r}, got None")
+
+        if not isinstance(exc, ExceptionGroup):
+            self.assertEqual(exc.__class__, template.__class__)
+            self.assertEqual(exc.args[0], template.args[0])
+        else:
+            self.assertEqual(exc.message, template.message)
+            self.assertEqual(len(exc.exceptions), len(template.exceptions))
+            for e, t in zip(exc.exceptions, template.exceptions):
+                self.assertExceptionIsLike(e, t)
+
+
+class FloatsAreIdenticalMixin:
+    def assertFloatsAreIdentical(self, x, y):
+        """Fail unless floats x and y are identical, in the sense that:
+        (1) both x and y are nans, or
+        (2) both x and y are infinities, with the same sign, or
+        (3) both x and y are zeros, with the same sign, or
+        (4) x and y are both finite and nonzero, and x == y
+
+        """
+        msg = 'floats {!r} and {!r} are not identical'
+
+        if isnan(x) or isnan(y):
+            if isnan(x) and isnan(y):
+                return
+        elif x == y:
+            if x != 0.0:
+                return
+            # both zero; check that signs match
+            elif copysign(1.0, x) == copysign(1.0, y):
+                return
+            else:
+                msg += ': zeros have different signs'
+        self.fail(msg.format(x, y))
+
+
+class ComplexesAreIdenticalMixin(FloatsAreIdenticalMixin):
+    def assertComplexesAreIdentical(self, x, y):
+        """Fail unless complex numbers x and y have equal values and signs.
+
+        In particular, if x and y both have real (or imaginary) part
+        zero, but the zeros have different signs, this test will fail.
+
+        """
+        self.assertFloatsAreIdentical(x.real, y.real)
+        self.assertFloatsAreIdentical(x.imag, y.imag)

Lib/test/test_contextlib.py

@@ -10,6 +10,7 @@ import unittest
 from contextlib import *  # Tests __all__
 from test import support
 from test.support import os_helper
+from test.support.testcase import ExceptionIsLikeMixin
 import weakref
 
 
@@ -158,9 +159,45 @@ class ContextManagerTestCase(unittest.TestCase):
                 yield
         ctx = whoo()
         ctx.__enter__()
-        self.assertRaises(
-            RuntimeError, ctx.__exit__, TypeError, TypeError("foo"), None
-        )
+        with self.assertRaises(RuntimeError):
+            ctx.__exit__(TypeError, TypeError("foo"), None)
+        if support.check_impl_detail(cpython=True):
+            # The "gen" attribute is an implementation detail.
+            self.assertFalse(ctx.gen.gi_suspended)
+
+    def test_contextmanager_trap_no_yield(self):
+        @contextmanager
+        def whoo():
+            if False:
+                yield
+        ctx = whoo()
+        with self.assertRaises(RuntimeError):
+            ctx.__enter__()
+
+    def test_contextmanager_trap_second_yield(self):
+        @contextmanager
+        def whoo():
+            yield
+            yield
+        ctx = whoo()
+        ctx.__enter__()
+        with self.assertRaises(RuntimeError):
+            ctx.__exit__(None, None, None)
+        if support.check_impl_detail(cpython=True):
+            # The "gen" attribute is an implementation detail.
+            self.assertFalse(ctx.gen.gi_suspended)
+
+    def test_contextmanager_non_normalised(self):
+        @contextmanager
+        def whoo():
+            try:
+                yield
+            except RuntimeError:
+                raise SyntaxError
+        ctx = whoo()
+        ctx.__enter__()
+        with self.assertRaises(SyntaxError):
+            ctx.__exit__(RuntimeError, None, None)
 
     def test_contextmanager_except(self):
         state = []
@@ -241,6 +278,23 @@ def woohoo():
             self.assertEqual(ex.args[0], 'issue29692:Unchained')
             self.assertIsNone(ex.__cause__)
 
+    def test_contextmanager_wrap_runtimeerror(self):
+        @contextmanager
+        def woohoo():
+            try:
+                yield
+            except Exception as exc:
+                raise RuntimeError(f'caught {exc}') from exc
+        with self.assertRaises(RuntimeError):
+            with woohoo():
+                1 / 0
+        # If the context manager wrapped StopIteration in a RuntimeError,
+        # we also unwrap it, because we can't tell whether the wrapping was
+        # done by the generator machinery or by the generator itself.
+        with self.assertRaises(StopIteration):
+            with woohoo():
+                raise StopIteration
+
     def _create_contextmanager_attribs(self):
         def attribs(**kw):
             def decorate(func):
@@ -252,6 +306,7 @@ def woohoo():
         @attribs(foo='bar')
         def baz(spam):
             """Whee!"""
+            yield
         return baz
 
     def test_contextmanager_attribs(self):
@@ -308,8 +363,11 @@ def woohoo():
 
     def test_recursive(self):
         depth = 0
+        ncols = 0
         @contextmanager
         def woohoo():
+            nonlocal ncols
+            ncols += 1
             nonlocal depth
             before = depth
             depth += 1
@@ -323,6 +381,7 @@ def woohoo():
                 recursive()
 
         recursive()
+        self.assertEqual(ncols, 10)
         self.assertEqual(depth, 0)
 
 
@@ -374,12 +433,10 @@ class FileContextTestCase(unittest.TestCase):
     def testWithOpen(self):
         tfn = tempfile.mktemp()
         try:
-            f = None
             with open(tfn, "w", encoding="utf-8") as f:
                 self.assertFalse(f.closed)
                 f.write("Booh\n")
             self.assertTrue(f.closed)
-            f = None
             with self.assertRaises(ZeroDivisionError):
                 with open(tfn, "r", encoding="utf-8") as f:
                     self.assertFalse(f.closed)
@@ -1160,7 +1217,7 @@ class TestRedirectStderr(TestRedirectStream, unittest.TestCase):
     orig_stream = "stderr"
 
 
-class TestSuppress(unittest.TestCase):
+class TestSuppress(ExceptionIsLikeMixin, unittest.TestCase):
 
     @support.requires_docstrings
     def test_instance_docs(self):
@@ -1214,6 +1271,51 @@ class TestSuppress(unittest.TestCase):
             1/0
         self.assertTrue(outer_continued)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_exception_groups(self):
+        eg_ve = lambda: ExceptionGroup(
+            "EG with ValueErrors only",
+            [ValueError("ve1"), ValueError("ve2"), ValueError("ve3")],
+        )
+        eg_all = lambda: ExceptionGroup(
+            "EG with many types of exceptions",
+            [ValueError("ve1"), KeyError("ke1"), ValueError("ve2"), KeyError("ke2")],
+        )
+        with suppress(ValueError):
+            raise eg_ve()
+        with suppress(ValueError, KeyError):
+            raise eg_all()
+        with self.assertRaises(ExceptionGroup) as eg1:
+            with suppress(ValueError):
+                raise eg_all()
+        self.assertExceptionIsLike(
+            eg1.exception,
+            ExceptionGroup(
+                "EG with many types of exceptions",
+                [KeyError("ke1"), KeyError("ke2")],
+            ),
+        )
+
+        # Check handling of BaseExceptionGroup, using GeneratorExit so that
+        # we don't accidentally discard a ctrl-c with KeyboardInterrupt.
+        with suppress(GeneratorExit):
+            raise BaseExceptionGroup("message", [GeneratorExit()])
+        # If we raise a BaseException group, we can still suppress parts
+        with self.assertRaises(BaseExceptionGroup) as eg1:
+            with suppress(KeyError):
+                raise BaseExceptionGroup("message", [GeneratorExit("g"), KeyError("k")])
+        self.assertExceptionIsLike(
+            eg1.exception, BaseExceptionGroup("message", [GeneratorExit("g")]),
+        )
+        # If we suppress all the leaf BaseExceptions, we get a non-base ExceptionGroup
+        with self.assertRaises(ExceptionGroup) as eg1:
+            with suppress(GeneratorExit):
+                raise BaseExceptionGroup("message", [GeneratorExit("g"), KeyError("k")])
+        self.assertExceptionIsLike(
+            eg1.exception, ExceptionGroup("message", [KeyError("k")]),
+        )
+
 
 class TestChdir(unittest.TestCase):
     def make_relative_path(self, *parts):

Lib/test/test_float.py

@@ -8,7 +8,7 @@ import time
 import unittest
 
 from test import support
-from test.support import import_helper
+from test.support.testcase import FloatsAreIdenticalMixin
 from test.test_grammar import (VALID_UNDERSCORE_LITERALS,
                                INVALID_UNDERSCORE_LITERALS)
 from math import isinf, isnan, copysign, ldexp
@@ -19,7 +19,6 @@ try:
 except ImportError:
     _testcapi = None
 
-HAVE_IEEE_754 = float.__getformat__("double").startswith("IEEE")
 INF = float("inf")
 NAN = float("nan")
 
@@ -134,7 +133,7 @@ class GeneralFloatCases(unittest.TestCase):
             with self.assertRaises(ValueError, msg='float(%r)' % (s,)) as cm:
                 float(s)
             self.assertEqual(str(cm.exception),
-                'could not convert string to float: %r' % (s,))
+                             'could not convert string to float: %r' % (s,))
 
         check('\xbd')
         check('123\xbd')
@@ -156,7 +155,9 @@ class GeneralFloatCases(unittest.TestCase):
         # non-UTF-8 byte string
         check(b'123\xa0')
 
-    @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE')
+    # TODO: RUSTPYTHON
+    @unittest.skip("RustPython panics on this")
+    @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE', '')
     def test_float_with_comma(self):
         # set locale to something that doesn't use '.' for the decimal point
         # float must not accept the locale specific decimal point but
@@ -291,11 +292,11 @@ class GeneralFloatCases(unittest.TestCase):
 
     def test_floatasratio(self):
         for f, ratio in [
-                (0.875, (7, 8)),
-                (-0.875, (-7, 8)),
-                (0.0, (0, 1)),
-                (11.5, (23, 2)),
-            ]:
+            (0.875, (7, 8)),
+            (-0.875, (-7, 8)),
+            (0.0, (0, 1)),
+            (11.5, (23, 2)),
+        ]:
             self.assertEqual(f.as_integer_ratio(), ratio)
 
         for i in range(10000):
@@ -338,7 +339,7 @@ class GeneralFloatCases(unittest.TestCase):
             self.assertTrue((f,) == (f,), "(%r,) != (%r,)" % (f, f))
             self.assertTrue({f} == {f}, "{%r} != {%r}" % (f, f))
             self.assertTrue({f : None} == {f: None}, "{%r : None} != "
-                                                   "{%r : None}" % (f, f))
+                                                     "{%r : None}" % (f, f))
 
             # identical containers
             l, t, s, d = [f], (f,), {f}, {f: None}
@@ -668,8 +669,6 @@ class IEEEFormatTestCase(unittest.TestCase):
                           ('<f', LE_FLOAT_NAN)]:
             struct.unpack(fmt, data)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     @support.requires_IEEE_754
     @unittest.skipIf(_testcapi is None, 'needs _testcapi')
     def test_serialized_float_rounding(self):
@@ -742,8 +741,13 @@ class FormatTestCase(unittest.TestCase):
 
                 lhs, rhs = map(str.strip, line.split('->'))
                 fmt, arg = lhs.split()
-                self.assertEqual(fmt % float(arg), rhs)
-                self.assertEqual(fmt % -float(arg), '-' + rhs)
+                f = float(arg)
+                self.assertEqual(fmt % f, rhs)
+                self.assertEqual(fmt % -f, '-' + rhs)
+                if fmt != '%r':
+                    fmt2 = fmt[1:]
+                    self.assertEqual(format(f, fmt2), rhs)
+                    self.assertEqual(format(-f, fmt2), '-' + rhs)
 
     def test_issue5864(self):
         self.assertEqual(format(123.456, '.4'), '123.5')
@@ -772,7 +776,7 @@ class FormatTestCase(unittest.TestCase):
 class ReprTestCase(unittest.TestCase):
     def test_repr(self):
         with open(os.path.join(os.path.split(__file__)[0],
-                  'floating_points.txt'), encoding="utf-8") as floats_file:
+                               'floating_points.txt'), encoding="utf-8") as floats_file:
             for line in floats_file:
                 line = line.strip()
                 if not line or line.startswith('#'):
@@ -822,7 +826,7 @@ class ReprTestCase(unittest.TestCase):
             '2.86438000439698e+28',
             '8.89142905246179e+28',
             '3.08578087079232e+35',
-            ]
+        ]
 
         for s in test_strings:
             negs = '-'+s
@@ -833,7 +837,7 @@ class ReprTestCase(unittest.TestCase):
             self.assertEqual(repr(float(negs)), str(float(negs)))
 
 @support.requires_IEEE_754
-class RoundTestCase(unittest.TestCase):
+class RoundTestCase(unittest.TestCase, FloatsAreIdenticalMixin):
 
     def test_inf_nan(self):
         self.assertRaises(OverflowError, round, INF)
@@ -863,10 +867,10 @@ class RoundTestCase(unittest.TestCase):
 
     def test_small_n(self):
         for n in [-308, -309, -400, 1-2**31, -2**31, -2**31-1, -2**100]:
-            self.assertEqual(round(123.456, n), 0.0)
-            self.assertEqual(round(-123.456, n), -0.0)
-            self.assertEqual(round(1e300, n), 0.0)
-            self.assertEqual(round(1e-320, n), 0.0)
+            self.assertFloatsAreIdentical(round(123.456, n), 0.0)
+            self.assertFloatsAreIdentical(round(-123.456, n), -0.0)
+            self.assertFloatsAreIdentical(round(1e300, n), 0.0)
+            self.assertFloatsAreIdentical(round(1e-320, n), 0.0)
 
     def test_overflow(self):
         self.assertRaises(OverflowError, round, 1.6e308, -308)
@@ -879,7 +883,7 @@ class RoundTestCase(unittest.TestCase):
     def test_previous_round_bugs(self):
         # particular cases that have occurred in bug reports
         self.assertEqual(round(562949953421312.5, 1),
-                          562949953421312.5)
+                         562949953421312.5)
         self.assertEqual(round(56294995342131.5, 3),
                          56294995342131.5)
         # round-half-even
@@ -1053,32 +1057,22 @@ class InfNanTest(unittest.TestCase):
         self.assertEqual(copysign(1.0, float('inf')), 1.0)
         self.assertEqual(copysign(1.0, float('-inf')), -1.0)
 
-    @unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short',
-                         "applies only when using short float repr style")
     def test_nan_signs(self):
-        # When using the dtoa.c code, the sign of float('nan') should
-        # be predictable.
+        # The sign of float('nan') should be predictable.
         self.assertEqual(copysign(1.0, float('nan')), 1.0)
         self.assertEqual(copysign(1.0, float('-nan')), -1.0)
 
 
 fromHex = float.fromhex
 toHex = float.hex
-class HexFloatTestCase(unittest.TestCase):
+class HexFloatTestCase(FloatsAreIdenticalMixin, unittest.TestCase):
     MAX = fromHex('0x.fffffffffffff8p+1024')  # max normal
     MIN = fromHex('0x1p-1022')                # min normal
     TINY = fromHex('0x0.0000000000001p-1022') # min subnormal
     EPS = fromHex('0x0.0000000000001p0') # diff between 1.0 and next float up
 
     def identical(self, x, y):
-        # check that floats x and y are identical, or that both
-        # are NaNs
-        if isnan(x) or isnan(y):
-            if isnan(x) == isnan(y):
-                return
-        elif x == y and (x != 0.0 or copysign(1.0, x) == copysign(1.0, y)):
-            return
-        self.fail('%r not identical to %r' % (x, y))
+        self.assertFloatsAreIdentical(x, y)
 
     def test_ends(self):
         self.identical(self.MIN, ldexp(1.0, -1022))
@@ -1139,7 +1133,7 @@ class HexFloatTestCase(unittest.TestCase):
             '0x1.\uff10p0',
             '0x1p0 \n 0x2p0',
             '0x1p0\0 0x1p0',  # embedded null byte is not end of string
-            ]
+        ]
         for x in invalid_inputs:
             try:
                 result = fromHex(x)
@@ -1158,7 +1152,7 @@ class HexFloatTestCase(unittest.TestCase):
             ('1.0', 1.0),
             ('-0x.2', -0.125),
             ('-0.0', -0.0)
-            ]
+        ]
         whitespace = [
             '',
             ' ',
@@ -1168,7 +1162,7 @@ class HexFloatTestCase(unittest.TestCase):
             '\f',
             '\v',
             '\r'
-            ]
+        ]
         for inp, expected in value_pairs:
             for lead in whitespace:
                 for trail in whitespace:
@@ -1517,69 +1511,5 @@ class HexFloatTestCase(unittest.TestCase):
         self.assertEqual(getattr(f, 'foo', 'none'), 'bar')
 
 
-# Test PyFloat_Pack2(), PyFloat_Pack4() and PyFloat_Pack8()
-# Test PyFloat_Unpack2(), PyFloat_Unpack4() and PyFloat_Unpack8()
-BIG_ENDIAN = 0
-LITTLE_ENDIAN = 1
-EPSILON = {
-    2: 2.0 ** -11,  # binary16
-    4: 2.0 ** -24,  # binary32
-    8: 2.0 ** -53,  # binary64
-}
-
-@unittest.skipIf(_testcapi is None, 'needs _testcapi')
-class PackTests(unittest.TestCase):
-    def test_pack(self):
-        self.assertEqual(_testcapi.float_pack(2, 1.5, BIG_ENDIAN),
-                         b'>\x00')
-        self.assertEqual(_testcapi.float_pack(4, 1.5, BIG_ENDIAN),
-                         b'?\xc0\x00\x00')
-        self.assertEqual(_testcapi.float_pack(8, 1.5, BIG_ENDIAN),
-                         b'?\xf8\x00\x00\x00\x00\x00\x00')
-        self.assertEqual(_testcapi.float_pack(2, 1.5, LITTLE_ENDIAN),
-                         b'\x00>')
-        self.assertEqual(_testcapi.float_pack(4, 1.5, LITTLE_ENDIAN),
-                         b'\x00\x00\xc0?')
-        self.assertEqual(_testcapi.float_pack(8, 1.5, LITTLE_ENDIAN),
-                         b'\x00\x00\x00\x00\x00\x00\xf8?')
-
-    def test_unpack(self):
-        self.assertEqual(_testcapi.float_unpack(b'>\x00', BIG_ENDIAN),
-                         1.5)
-        self.assertEqual(_testcapi.float_unpack(b'?\xc0\x00\x00', BIG_ENDIAN),
-                         1.5)
-        self.assertEqual(_testcapi.float_unpack(b'?\xf8\x00\x00\x00\x00\x00\x00', BIG_ENDIAN),
-                         1.5)
-        self.assertEqual(_testcapi.float_unpack(b'\x00>', LITTLE_ENDIAN),
-                         1.5)
-        self.assertEqual(_testcapi.float_unpack(b'\x00\x00\xc0?', LITTLE_ENDIAN),
-                         1.5)
-        self.assertEqual(_testcapi.float_unpack(b'\x00\x00\x00\x00\x00\x00\xf8?', LITTLE_ENDIAN),
-                         1.5)
-
-    def test_roundtrip(self):
-        large = 2.0 ** 100
-        values = [1.0, 1.5, large, 1.0/7, math.pi]
-        if HAVE_IEEE_754:
-            values.extend((INF, NAN))
-        for value in values:
-            for size in (2, 4, 8,):
-                if size == 2 and value == large:
-                    # too large for 16-bit float
-                    continue
-                rel_tol = EPSILON[size]
-                for endian in (BIG_ENDIAN, LITTLE_ENDIAN):
-                    with self.subTest(value=value, size=size, endian=endian):
-                        data = _testcapi.float_pack(size, value, endian)
-                        value2 = _testcapi.float_unpack(data, endian)
-                        if isnan(value):
-                            self.assertTrue(isnan(value2), (value, value2))
-                        elif size < 8:
-                            self.assertTrue(math.isclose(value2, value, rel_tol=rel_tol),
-                                            (value, value2))
-                        else:
-                            self.assertEqual(value2, value)
-
-
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_list.py

@@ -22,6 +22,8 @@ class ListTest(list_tests.CommonTest):
 
         if sys.maxsize == 0x7fffffff:
             # This test can currently only work on 32-bit machines.
+            # XXX If/when PySequence_Length() returns a ssize_t, it should be
+            # XXX re-enabled.
             # Verify clearing of bug #556025.
             # This assumes that the max data size (sys.maxint) == max
             # address size this also assumes that the address size is at
@@ -97,7 +99,7 @@ class ListTest(list_tests.CommonTest):
         self.assertRaises((MemoryError, OverflowError), imul, lst, n)
 
     # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.skip("Crashes on windows debug build")
     def test_list_resize_overflow(self):
         # gh-97616: test new_allocated * sizeof(PyObject*) overflow
         # check in list_resize()
@@ -105,7 +107,7 @@ class ListTest(list_tests.CommonTest):
         del lst[1:]
         self.assertEqual(len(lst), 1)
 
-        size = ((2 ** (tuple.__itemsize__ * 8) - 1) // 2)
+        size = sys.maxsize
         with self.assertRaises((MemoryError, OverflowError)):
             lst * size
         with self.assertRaises((MemoryError, OverflowError)):
@@ -117,7 +119,7 @@ class ListTest(list_tests.CommonTest):
             l = [0] * n
             s = repr(l)
             self.assertEqual(s,
-                '[' + ', '.join(['0'] * n) + ']')
+                             '[' + ', '.join(['0'] * n) + ']')
         check(10)       # check our checking code
         check(1000000)
 
@@ -206,6 +208,7 @@ class ListTest(list_tests.CommonTest):
         with self.assertRaises(TypeError):
             (3,) + L([1,2])
 
+    # TODO: RUSTPYTHON
     @unittest.skip("TODO: RUSTPYTHON; hang")
     def test_equal_operator_modifying_operand(self):
         # test fix for seg fault reported in bpo-38588 part 2.
@@ -232,6 +235,33 @@ class ListTest(list_tests.CommonTest):
         list4 = [1]
         self.assertFalse(list3 == list4)
 
+    # TODO: RUSTPYTHON
+    @unittest.skip("TODO: RUSTPYTHON; hang")
+    def test_lt_operator_modifying_operand(self):
+        # See gh-120298
+        class evil:
+            def __lt__(self, other):
+                other.clear()
+                return NotImplemented
+
+        a = [[evil()]]
+        with self.assertRaises(TypeError):
+            a[0] < a
+
+    def test_list_index_modifing_operand(self):
+        # See gh-120384
+        class evil:
+            def __init__(self, lst):
+                self.lst = lst
+            def __iter__(self):
+                yield from self.lst
+                self.lst.clear()
+
+        lst = list(range(5))
+        operand = evil(lst)
+        with self.assertRaises(ValueError):
+            lst[::-1] = operand
+
     @cpython_only
     def test_preallocation(self):
         iterable = [0] * 10

Lib/test/test_listcomps.py

@@ -1,6 +1,11 @@
 import doctest
+import textwrap
+import traceback
+import types
 import unittest
 
+from test.support import BrokenIter
+
 
 doctests = """
 ########### Tests borrowed from or inspired by test_genexps.py ############
@@ -87,65 +92,661 @@ Make sure that None is a valid return value
     >>> [None for i in range(10)]
     [None, None, None, None, None, None, None, None, None, None]
 
-########### Tests for various scoping corner cases ############
-
-Return lambdas that use the iteration variable as a default argument
-
-    >>> items = [(lambda i=i: i) for i in range(5)]
-    >>> [x() for x in items]
-    [0, 1, 2, 3, 4]
-
-Same again, only this time as a closure variable
-
-    >>> items = [(lambda: i) for i in range(5)]
-    >>> [x() for x in items]
-    [4, 4, 4, 4, 4]
-
-Another way to test that the iteration variable is local to the list comp
-
-    >>> items = [(lambda: i) for i in range(5)]
-    >>> i = 20
-    >>> [x() for x in items]
-    [4, 4, 4, 4, 4]
-
-And confirm that a closure can jump over the list comp scope
-
-    >>> items = [(lambda: y) for i in range(5)]
-    >>> y = 2
-    >>> [x() for x in items]
-    [2, 2, 2, 2, 2]
-
-We also repeat each of the above scoping tests inside a function
-
-    >>> def test_func():
-    ...     items = [(lambda i=i: i) for i in range(5)]
-    ...     return [x() for x in items]
-    >>> test_func()
-    [0, 1, 2, 3, 4]
-
-    >>> def test_func():
-    ...     items = [(lambda: i) for i in range(5)]
-    ...     return [x() for x in items]
-    >>> test_func()
-    [4, 4, 4, 4, 4]
-
-    >>> def test_func():
-    ...     items = [(lambda: i) for i in range(5)]
-    ...     i = 20
-    ...     return [x() for x in items]
-    >>> test_func()
-    [4, 4, 4, 4, 4]
-
-    >>> def test_func():
-    ...     items = [(lambda: y) for i in range(5)]
-    ...     y = 2
-    ...     return [x() for x in items]
-    >>> test_func()
-    [2, 2, 2, 2, 2]
-
 """
 
 
+class ListComprehensionTest(unittest.TestCase):
+    def _check_in_scopes(self, code, outputs=None, ns=None, scopes=None, raises=(),
+                         exec_func=exec):
+        code = textwrap.dedent(code)
+        scopes = scopes or ["module", "class", "function"]
+        for scope in scopes:
+            with self.subTest(scope=scope):
+                if scope == "class":
+                    newcode = textwrap.dedent("""
+                        class _C:
+                            {code}
+                    """).format(code=textwrap.indent(code, "    "))
+                    def get_output(moddict, name):
+                        return getattr(moddict["_C"], name)
+                elif scope == "function":
+                    newcode = textwrap.dedent("""
+                        def _f():
+                            {code}
+                            return locals()
+                        _out = _f()
+                    """).format(code=textwrap.indent(code, "    "))
+                    def get_output(moddict, name):
+                        return moddict["_out"][name]
+                else:
+                    newcode = code
+                    def get_output(moddict, name):
+                        return moddict[name]
+                newns = ns.copy() if ns else {}
+                try:
+                    exec_func(newcode, newns)
+                except raises as e:
+                    # We care about e.g. NameError vs UnboundLocalError
+                    self.assertIs(type(e), raises)
+                else:
+                    for k, v in (outputs or {}).items():
+                        self.assertEqual(get_output(newns, k), v, k)
+
+    def test_lambdas_with_iteration_var_as_default(self):
+        code = """
+            items = [(lambda i=i: i) for i in range(5)]
+            y = [x() for x in items]
+        """
+        outputs = {"y": [0, 1, 2, 3, 4]}
+        self._check_in_scopes(code, outputs)
+
+    def test_lambdas_with_free_var(self):
+        code = """
+            items = [(lambda: i) for i in range(5)]
+            y = [x() for x in items]
+        """
+        outputs = {"y": [4, 4, 4, 4, 4]}
+        self._check_in_scopes(code, outputs)
+
+    def test_class_scope_free_var_with_class_cell(self):
+        class C:
+            def method(self):
+                super()
+                return __class__
+            items = [(lambda: i) for i in range(5)]
+            y = [x() for x in items]
+
+        self.assertEqual(C.y, [4, 4, 4, 4, 4])
+        self.assertIs(C().method(), C)
+
+    def test_references_super(self):
+        code = """
+            res = [super for x in [1]]
+        """
+        self._check_in_scopes(code, outputs={"res": [super]})
+
+    def test_references___class__(self):
+        code = """
+            res = [__class__ for x in [1]]
+        """
+        self._check_in_scopes(code, raises=NameError)
+
+    def test_references___class___defined(self):
+        code = """
+            __class__ = 2
+            res = [__class__ for x in [1]]
+        """
+        self._check_in_scopes(
+            code, outputs={"res": [2]}, scopes=["module", "function"])
+        self._check_in_scopes(code, raises=NameError, scopes=["class"])
+
+    def test_references___class___enclosing(self):
+        code = """
+            __class__ = 2
+            class C:
+                res = [__class__ for x in [1]]
+            res = C.res
+        """
+        self._check_in_scopes(code, raises=NameError)
+
+    def test_super_and_class_cell_in_sibling_comps(self):
+        code = """
+            [super for _ in [1]]
+            [__class__ for _ in [1]]
+        """
+        self._check_in_scopes(code, raises=NameError)
+
+    def test_inner_cell_shadows_outer(self):
+        code = """
+            items = [(lambda: i) for i in range(5)]
+            i = 20
+            y = [x() for x in items]
+        """
+        outputs = {"y": [4, 4, 4, 4, 4], "i": 20}
+        self._check_in_scopes(code, outputs)
+
+    def test_inner_cell_shadows_outer_no_store(self):
+        code = """
+            def f(x):
+                return [lambda: x for x in range(x)], x
+            fns, x = f(2)
+            y = [fn() for fn in fns]
+        """
+        outputs = {"y": [1, 1], "x": 2}
+        self._check_in_scopes(code, outputs)
+
+    def test_closure_can_jump_over_comp_scope(self):
+        code = """
+            items = [(lambda: y) for i in range(5)]
+            y = 2
+            z = [x() for x in items]
+        """
+        outputs = {"z": [2, 2, 2, 2, 2]}
+        self._check_in_scopes(code, outputs, scopes=["module", "function"])
+
+    def test_cell_inner_free_outer(self):
+        code = """
+            def f():
+                return [lambda: x for x in (x, [1])[1]]
+            x = ...
+            y = [fn() for fn in f()]
+        """
+        outputs = {"y": [1]}
+        self._check_in_scopes(code, outputs, scopes=["module", "function"])
+
+    def test_free_inner_cell_outer(self):
+        code = """
+            g = 2
+            def f():
+                return g
+            y = [g for x in [1]]
+        """
+        outputs = {"y": [2]}
+        self._check_in_scopes(code, outputs, scopes=["module", "function"])
+        self._check_in_scopes(code, scopes=["class"], raises=NameError)
+
+    def test_inner_cell_shadows_outer_redefined(self):
+        code = """
+            y = 10
+            items = [(lambda: y) for y in range(5)]
+            x = y
+            y = 20
+            out = [z() for z in items]
+        """
+        outputs = {"x": 10, "out": [4, 4, 4, 4, 4]}
+        self._check_in_scopes(code, outputs)
+
+    def test_shadows_outer_cell(self):
+        code = """
+            def inner():
+                return g
+            [g for g in range(5)]
+            x = inner()
+        """
+        outputs = {"x": -1}
+        self._check_in_scopes(code, outputs, ns={"g": -1})
+
+    def test_explicit_global(self):
+        code = """
+            global g
+            x = g
+            g = 2
+            items = [g for g in [1]]
+            y = g
+        """
+        outputs = {"x": 1, "y": 2, "items": [1]}
+        self._check_in_scopes(code, outputs, ns={"g": 1})
+
+    def test_explicit_global_2(self):
+        code = """
+            global g
+            x = g
+            g = 2
+            items = [g for x in [1]]
+            y = g
+        """
+        outputs = {"x": 1, "y": 2, "items": [2]}
+        self._check_in_scopes(code, outputs, ns={"g": 1})
+
+    def test_explicit_global_3(self):
+        code = """
+            global g
+            fns = [lambda: g for g in [2]]
+            items = [fn() for fn in fns]
+        """
+        outputs = {"items": [2]}
+        self._check_in_scopes(code, outputs, ns={"g": 1})
+
+    def test_assignment_expression(self):
+        code = """
+            x = -1
+            items = [(x:=y) for y in range(3)]
+        """
+        outputs = {"x": 2}
+        # assignment expression in comprehension is disallowed in class scope
+        self._check_in_scopes(code, outputs, scopes=["module", "function"])
+
+    def test_free_var_in_comp_child(self):
+        code = """
+            lst = range(3)
+            funcs = [lambda: x for x in lst]
+            inc = [x + 1 for x in lst]
+            [x for x in inc]
+            x = funcs[0]()
+        """
+        outputs = {"x": 2}
+        self._check_in_scopes(code, outputs)
+
+    def test_shadow_with_free_and_local(self):
+        code = """
+            lst = range(3)
+            x = -1
+            funcs = [lambda: x for x in lst]
+            items = [x + 1 for x in lst]
+        """
+        outputs = {"x": -1}
+        self._check_in_scopes(code, outputs)
+
+    def test_shadow_comp_iterable_name(self):
+        code = """
+            x = [1]
+            y = [x for x in x]
+        """
+        outputs = {"x": [1]}
+        self._check_in_scopes(code, outputs)
+
+    def test_nested_free(self):
+        code = """
+            x = 1
+            def g():
+                [x for x in range(3)]
+                return x
+            g()
+        """
+        outputs = {"x": 1}
+        self._check_in_scopes(code, outputs, scopes=["module", "function"])
+
+    def test_introspecting_frame_locals(self):
+        code = """
+            import sys
+            [i for i in range(2)]
+            i = 20
+            sys._getframe().f_locals
+        """
+        outputs = {"i": 20}
+        self._check_in_scopes(code, outputs)
+
+    def test_nested(self):
+        code = """
+            l = [2, 3]
+            y = [[x ** 2 for x in range(x)] for x in l]
+        """
+        outputs = {"y": [[0, 1], [0, 1, 4]]}
+        self._check_in_scopes(code, outputs)
+
+    def test_nested_2(self):
+        code = """
+            l = [1, 2, 3]
+            x = 3
+            y = [x for [x ** x for x in range(x)][x - 1] in l]
+        """
+        outputs = {"y": [3, 3, 3]}
+        self._check_in_scopes(code, outputs, scopes=["module", "function"])
+        self._check_in_scopes(code, scopes=["class"], raises=NameError)
+
+    def test_nested_3(self):
+        code = """
+            l = [(1, 2), (3, 4), (5, 6)]
+            y = [x for (x, [x ** x for x in range(x)][x - 1]) in l]
+        """
+        outputs = {"y": [1, 3, 5]}
+        self._check_in_scopes(code, outputs)
+
+    def test_nested_4(self):
+        code = """
+            items = [([lambda: x for x in range(2)], lambda: x) for x in range(3)]
+            out = [([fn() for fn in fns], fn()) for fns, fn in items]
+        """
+        outputs = {"out": [([1, 1], 2), ([1, 1], 2), ([1, 1], 2)]}
+        self._check_in_scopes(code, outputs)
+
+    def test_nameerror(self):
+        code = """
+            [x for x in [1]]
+            x
+        """
+
+        self._check_in_scopes(code, raises=NameError)
+
+    def test_dunder_name(self):
+        code = """
+            y = [__x for __x in [1]]
+        """
+        outputs = {"y": [1]}
+        self._check_in_scopes(code, outputs)
+
+    def test_unbound_local_after_comprehension(self):
+        def f():
+            if False:
+                x = 0
+            [x for x in [1]]
+            return x
+
+        with self.assertRaises(UnboundLocalError):
+            f()
+
+    def test_unbound_local_inside_comprehension(self):
+        def f():
+            l = [None]
+            return [1 for (l[0], l) in [[1, 2]]]
+
+        with self.assertRaises(UnboundLocalError):
+            f()
+
+    def test_global_outside_cellvar_inside_plus_freevar(self):
+        code = """
+            a = 1
+            def f():
+                func, = [(lambda: b) for b in [a]]
+                return b, func()
+            x = f()
+        """
+        self._check_in_scopes(
+            code, {"x": (2, 1)}, ns={"b": 2}, scopes=["function", "module"])
+        # inside a class, the `a = 1` assignment is not visible
+        self._check_in_scopes(code, raises=NameError, scopes=["class"])
+
+    def test_cell_in_nested_comprehension(self):
+        code = """
+            a = 1
+            def f():
+                (func, inner_b), = [[lambda: b for b in c] + [b] for c in [[a]]]
+                return b, inner_b, func()
+            x = f()
+        """
+        self._check_in_scopes(
+            code, {"x": (2, 2, 1)}, ns={"b": 2}, scopes=["function", "module"])
+        # inside a class, the `a = 1` assignment is not visible
+        self._check_in_scopes(code, raises=NameError, scopes=["class"])
+
+    def test_name_error_in_class_scope(self):
+        code = """
+            y = 1
+            [x + y for x in range(2)]
+        """
+        self._check_in_scopes(code, raises=NameError, scopes=["class"])
+
+    def test_global_in_class_scope(self):
+        code = """
+            y = 2
+            vals = [(x, y) for x in range(2)]
+        """
+        outputs = {"vals": [(0, 1), (1, 1)]}
+        self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["class"])
+
+    def test_in_class_scope_inside_function_1(self):
+        code = """
+            class C:
+                y = 2
+                vals = [(x, y) for x in range(2)]
+            vals = C.vals
+        """
+        outputs = {"vals": [(0, 1), (1, 1)]}
+        self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["function"])
+
+    def test_in_class_scope_inside_function_2(self):
+        code = """
+            y = 1
+            class C:
+                y = 2
+                vals = [(x, y) for x in range(2)]
+            vals = C.vals
+        """
+        outputs = {"vals": [(0, 1), (1, 1)]}
+        self._check_in_scopes(code, outputs, scopes=["function"])
+
+    def test_in_class_scope_with_global(self):
+        code = """
+            y = 1
+            class C:
+                global y
+                y = 2
+                # Ensure the listcomp uses the global, not the value in the
+                # class namespace
+                locals()['y'] = 3
+                vals = [(x, y) for x in range(2)]
+            vals = C.vals
+        """
+        outputs = {"vals": [(0, 2), (1, 2)]}
+        self._check_in_scopes(code, outputs, scopes=["module", "class"])
+        outputs = {"vals": [(0, 1), (1, 1)]}
+        self._check_in_scopes(code, outputs, scopes=["function"])
+
+    def test_in_class_scope_with_nonlocal(self):
+        code = """
+            y = 1
+            class C:
+                nonlocal y
+                y = 2
+                # Ensure the listcomp uses the global, not the value in the
+                # class namespace
+                locals()['y'] = 3
+                vals = [(x, y) for x in range(2)]
+            vals = C.vals
+        """
+        outputs = {"vals": [(0, 2), (1, 2)]}
+        self._check_in_scopes(code, outputs, scopes=["function"])
+
+    def test_nested_has_free_var(self):
+        code = """
+            items = [a for a in [1] if [a for _ in [0]]]
+        """
+        outputs = {"items": [1]}
+        self._check_in_scopes(code, outputs, scopes=["class"])
+
+    def test_nested_free_var_not_bound_in_outer_comp(self):
+        code = """
+            z = 1
+            items = [a for a in [1] if [x for x in [1] if z]]
+        """
+        self._check_in_scopes(code, {"items": [1]}, scopes=["module", "function"])
+        self._check_in_scopes(code, {"items": []}, ns={"z": 0}, scopes=["class"])
+
+    def test_nested_free_var_in_iter(self):
+        code = """
+            items = [_C for _C in [1] for [0, 1][[x for x in [1] if _C][0]] in [2]]
+        """
+        self._check_in_scopes(code, {"items": [1]})
+
+    def test_nested_free_var_in_expr(self):
+        code = """
+            items = [(_C, [x for x in [1] if _C]) for _C in [0, 1]]
+        """
+        self._check_in_scopes(code, {"items": [(0, []), (1, [1])]})
+
+    def test_nested_listcomp_in_lambda(self):
+        code = """
+            f = [(z, lambda y: [(x, y, z) for x in [3]]) for z in [1]]
+            (z, func), = f
+            out = func(2)
+        """
+        self._check_in_scopes(code, {"z": 1, "out": [(3, 2, 1)]})
+
+    def test_lambda_in_iter(self):
+        code = """
+            (func, c), = [(a, b) for b in [1] for a in [lambda : a]]
+            d = func()
+            assert d is func
+            # must use "a" in this scope
+            e = a if False else None
+        """
+        self._check_in_scopes(code, {"c": 1, "e": None})
+
+    def test_assign_to_comp_iter_var_in_outer_function(self):
+        code = """
+            a = [1 for a in [0]]
+        """
+        self._check_in_scopes(code, {"a": [1]}, scopes=["function"])
+
+    def test_no_leakage_to_locals(self):
+        code = """
+            def b():
+                [a for b in [1] for _ in []]
+                return b, locals()
+            r, s = b()
+            x = r is b
+            y = list(s.keys())
+        """
+        self._check_in_scopes(code, {"x": True, "y": []}, scopes=["module"])
+        self._check_in_scopes(code, {"x": True, "y": ["b"]}, scopes=["function"])
+        self._check_in_scopes(code, raises=NameError, scopes=["class"])
+
+    def test_iter_var_available_in_locals(self):
+        code = """
+            l = [1, 2]
+            y = 0
+            items = [locals()["x"] for x in l]
+            items2 = [vars()["x"] for x in l]
+            items3 = [("x" in dir()) for x in l]
+            items4 = [eval("x") for x in l]
+            # x is available, and does not overwrite y
+            [exec("y = x") for x in l]
+        """
+        self._check_in_scopes(
+            code,
+            {
+                "items": [1, 2],
+                "items2": [1, 2],
+                "items3": [True, True],
+                "items4": [1, 2],
+                "y": 0
+            }
+        )
+
+    def test_comp_in_try_except(self):
+        template = """
+            value = ["ab"]
+            result = snapshot = None
+            try:
+                result = [{func}(value) for value in value]
+            except:
+                snapshot = value
+                raise
+        """
+        # No exception.
+        code = template.format(func='len')
+        self._check_in_scopes(code, {"value": ["ab"], "result": [2], "snapshot": None})
+        # Handles exception.
+        code = template.format(func='int')
+        self._check_in_scopes(code, {"value": ["ab"], "result": None, "snapshot": ["ab"]},
+                              raises=ValueError)
+
+    def test_comp_in_try_finally(self):
+        template = """
+            value = ["ab"]
+            result = snapshot = None
+            try:
+                result = [{func}(value) for value in value]
+            finally:
+                snapshot = value
+        """
+        # No exception.
+        code = template.format(func='len')
+        self._check_in_scopes(code, {"value": ["ab"], "result": [2], "snapshot": ["ab"]})
+        # Handles exception.
+        code = template.format(func='int')
+        self._check_in_scopes(code, {"value": ["ab"], "result": None, "snapshot": ["ab"]},
+                              raises=ValueError)
+
+    def test_exception_in_post_comp_call(self):
+        code = """
+            value = [1, None]
+            try:
+                [v for v in value].sort()
+            except:
+                pass
+        """
+        self._check_in_scopes(code, {"value": [1, None]})
+
+    def test_frame_locals(self):
+        code = """
+            val = [sys._getframe().f_locals for a in [0]][0]["a"]
+        """
+        import sys
+        self._check_in_scopes(code, {"val": 0}, ns={"sys": sys})
+
+    def _recursive_replace(self, maybe_code):
+        if not isinstance(maybe_code, types.CodeType):
+            return maybe_code
+        return maybe_code.replace(co_consts=tuple(
+            self._recursive_replace(c) for c in maybe_code.co_consts
+        ))
+
+    def _replacing_exec(self, code_string, ns):
+        co = compile(code_string, "<string>", "exec")
+        co = self._recursive_replace(co)
+        exec(co, ns)
+
+    def test_code_replace(self):
+        code = """
+            x = 3
+            [x for x in (1, 2)]
+            dir()
+            y = [x]
+        """
+        self._check_in_scopes(code, {"y": [3], "x": 3})
+        self._check_in_scopes(code, {"y": [3], "x": 3}, exec_func=self._replacing_exec)
+
+    def test_code_replace_extended_arg(self):
+        num_names = 300
+        assignments = "; ".join(f"x{i} = {i}" for i in range(num_names))
+        name_list = ", ".join(f"x{i}" for i in range(num_names))
+        expected = {
+            "y": list(range(num_names)),
+            **{f"x{i}": i for i in range(num_names)}
+        }
+        code = f"""
+            {assignments}
+            [({name_list}) for {name_list} in (range(300),)]
+            dir()
+            y = [{name_list}]
+        """
+        self._check_in_scopes(code, expected)
+        self._check_in_scopes(code, expected, exec_func=self._replacing_exec)
+
+    def test_multiple_comprehension_name_reuse(self):
+        code = """
+            [x for x in [1]]
+            y = [x for _ in [1]]
+        """
+        self._check_in_scopes(code, {"y": [3]}, ns={"x": 3})
+
+        code = """
+            x = 2
+            [x for x in [1]]
+            y = [x for _ in [1]]
+        """
+        self._check_in_scopes(code, {"x": 2, "y": [3]}, ns={"x": 3}, scopes=["class"])
+        self._check_in_scopes(code, {"x": 2, "y": [2]}, ns={"x": 3}, scopes=["function", "module"])
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_exception_locations(self):
+        # The location of an exception raised from __init__ or
+        # __next__ should should be the iterator expression
+
+        def init_raises():
+            try:
+                [x for x in BrokenIter(init_raises=True)]
+            except Exception as e:
+                return e
+
+        def next_raises():
+            try:
+                [x for x in BrokenIter(next_raises=True)]
+            except Exception as e:
+                return e
+
+        def iter_raises():
+            try:
+                [x for x in BrokenIter(iter_raises=True)]
+            except Exception as e:
+                return e
+
+        for func, expected in [(init_raises, "BrokenIter(init_raises=True)"),
+                               (next_raises, "BrokenIter(next_raises=True)"),
+                               (iter_raises, "BrokenIter(iter_raises=True)"),
+                               ]:
+            with self.subTest(func):
+                exc = func()
+                f = traceback.extract_tb(exc.__traceback__)[0]
+                indent = 16
+                co = func.__code__
+                self.assertEqual(f.lineno, co.co_firstlineno + 2)
+                self.assertEqual(f.end_lineno, co.co_firstlineno + 2)
+                self.assertEqual(f.line[f.colno - indent : f.end_colno - indent],
+                                 expected)
+
 __test__ = {'doctests' : doctests}
 
 def load_tests(loader, tests, pattern):

Lib/test/test_smtpnet.py

@@ -0,0 +1,90 @@
+import unittest
+from test import support
+from test.support import import_helper
+from test.support import socket_helper
+import smtplib
+import socket
+
+ssl = import_helper.import_module("ssl")
+
+support.requires("network")
+
+def check_ssl_verifiy(host, port):
+    context = ssl.create_default_context()
+    with socket.create_connection((host, port)) as sock:
+        try:
+            sock = context.wrap_socket(sock, server_hostname=host)
+        except Exception:
+            return False
+        else:
+            sock.close()
+            return True
+
+
+class SmtpTest(unittest.TestCase):
+    testServer = 'smtp.gmail.com'
+    remotePort = 587
+
+    def test_connect_starttls(self):
+        support.get_attribute(smtplib, 'SMTP_SSL')
+        context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+        with socket_helper.transient_internet(self.testServer):
+            server = smtplib.SMTP(self.testServer, self.remotePort)
+            try:
+                server.starttls(context=context)
+            except smtplib.SMTPException as e:
+                if e.args[0] == 'STARTTLS extension not supported by server.':
+                    unittest.skip(e.args[0])
+                else:
+                    raise
+            server.ehlo()
+            server.quit()
+
+
+class SmtpSSLTest(unittest.TestCase):
+    testServer = 'smtp.gmail.com'
+    remotePort = 465
+
+    def test_connect(self):
+        support.get_attribute(smtplib, 'SMTP_SSL')
+        with socket_helper.transient_internet(self.testServer):
+            server = smtplib.SMTP_SSL(self.testServer, self.remotePort)
+            server.ehlo()
+            server.quit()
+
+    def test_connect_default_port(self):
+        support.get_attribute(smtplib, 'SMTP_SSL')
+        with socket_helper.transient_internet(self.testServer):
+            server = smtplib.SMTP_SSL(self.testServer)
+            server.ehlo()
+            server.quit()
+
+    @support.requires_resource('walltime')
+    def test_connect_using_sslcontext(self):
+        context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+        support.get_attribute(smtplib, 'SMTP_SSL')
+        with socket_helper.transient_internet(self.testServer):
+            server = smtplib.SMTP_SSL(self.testServer, self.remotePort, context=context)
+            server.ehlo()
+            server.quit()
+
+    def test_connect_using_sslcontext_verified(self):
+        with socket_helper.transient_internet(self.testServer):
+            can_verify = check_ssl_verifiy(self.testServer, self.remotePort)
+            if not can_verify:
+                self.skipTest("SSL certificate can't be verified")
+
+        support.get_attribute(smtplib, 'SMTP_SSL')
+        context = ssl.create_default_context()
+        with socket_helper.transient_internet(self.testServer):
+            server = smtplib.SMTP_SSL(self.testServer, self.remotePort, context=context)
+            server.ehlo()
+            server.quit()
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_unary.py

@@ -8,7 +8,6 @@ class UnaryOpTestCase(unittest.TestCase):
         self.assertTrue(-2 == 0 - 2)
         self.assertEqual(-0, 0)
         self.assertEqual(--2, 2)
-        self.assertTrue(-2 == 0 - 2)
         self.assertTrue(-2.0 == 0 - 2.0)
         self.assertTrue(-2j == 0 - 2j)
 
@@ -16,15 +15,13 @@ class UnaryOpTestCase(unittest.TestCase):
         self.assertEqual(+2, 2)
         self.assertEqual(+0, 0)
         self.assertEqual(++2, 2)
-        self.assertEqual(+2, 2)
         self.assertEqual(+2.0, 2.0)
         self.assertEqual(+2j, 2j)
 
     def test_invert(self):
-        self.assertTrue(-2 == 0 - 2)
-        self.assertEqual(-0, 0)
-        self.assertEqual(--2, 2)
-        self.assertTrue(-2 == 0 - 2)
+        self.assertTrue(~2 == -(2+1))
+        self.assertEqual(~0, -1)
+        self.assertEqual(~~2, 2)
 
     def test_no_overflow(self):
         nines = "9" * 32

Lib/test/test_zlib.py

@@ -7,20 +7,36 @@ import os
 import pickle
 import random
 import sys
-from test.support import bigmemtest, _1G, _4G
+from test.support import bigmemtest, _1G, _4G, is_s390x
 
 
 zlib = import_helper.import_module('zlib')
 
 requires_Compress_copy = unittest.skipUnless(
-        hasattr(zlib.compressobj(), "copy"),
-        'requires Compress.copy()')
+    hasattr(zlib.compressobj(), "copy"),
+    'requires Compress.copy()')
 requires_Decompress_copy = unittest.skipUnless(
-        hasattr(zlib.decompressobj(), "copy"),
-        'requires Decompress.copy()')
+    hasattr(zlib.decompressobj(), "copy"),
+    'requires Decompress.copy()')
 
-# bpo-46623: On s390x, when a hardware accelerator is used, using different
-# ways to compress data with zlib can produce different compressed data.
+
+# def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION):
+#     # Register "1.2.3" as "1.2.3.0"
+#     # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux"
+#     v = zlib_version.split('-', 1)[0].split('.')
+#     if len(v) < 4:
+#         v.append('0')
+#     elif not v[-1].isnumeric():
+#         v[-1] = '0'
+#     return tuple(map(int, v))
+#
+#
+# ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple()
+
+
+# bpo-46623: When a hardware accelerator is used (currently only on s390x),
+# using different ways to compress data with zlib can produce different
+# compressed data.
 # Simplified test_pair() code:
 #
 #   def func1(data):
@@ -43,10 +59,9 @@ requires_Decompress_copy = unittest.skipUnless(
 #
 #   zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data
 #
-# Make the assumption that s390x always has an accelerator to simplify the skip
-# condition. Windows doesn't have os.uname() but it doesn't support s390x.
-skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
-                                'skipped on s390x')
+# To simplify the skip condition, make the assumption that s390x always has an
+# accelerator, and nothing else has it.
+HW_ACCELERATED = is_s390x
 
 
 class VersionTestCase(unittest.TestCase):
@@ -141,7 +156,7 @@ class ExceptionTestCase(unittest.TestCase):
         self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, 0)
         # specifying total bits too large causes an error
         self.assertRaises(ValueError,
-                zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1)
+                          zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1)
 
     def test_baddecompressobj(self):
         # verify failure on building decompress object with bad params
@@ -214,12 +229,14 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
                                          bufsize=zlib.DEF_BUF_SIZE),
                          HAMLET_SCENE)
 
-    @skip_on_s390x
     def test_speech128(self):
         # compress more data
         data = HAMLET_SCENE * 128
         x = zlib.compress(data)
-        self.assertEqual(zlib.compress(bytearray(data)), x)
+        # With hardware acceleration, the compressed bytes
+        # might not be identical.
+        if not HW_ACCELERATED:
+            self.assertEqual(zlib.compress(bytearray(data)), x)
         for ob in x, bytearray(x):
             self.assertEqual(zlib.decompress(ob), data)
 
@@ -227,8 +244,8 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
         # A useful error message is given
         x = zlib.compress(HAMLET_SCENE)
         self.assertRaisesRegex(zlib.error,
-            "Error -5 while decompressing data: incomplete or truncated stream",
-            zlib.decompress, x[:-1])
+                               "Error -5 while decompressing data: incomplete or truncated stream",
+                               zlib.decompress, x[:-1])
 
     # Memory use of the following functions takes into account overallocation
 
@@ -268,7 +285,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
     # TODO: RUSTPYTHON
     @unittest.expectedFailure
     # Test compression object
-    @skip_on_s390x
     def test_pair(self):
         # straightforward compress/decompress objects
         datasrc = HAMLET_SCENE * 128
@@ -279,7 +295,10 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
             x1 = co.compress(data)
             x2 = co.flush()
             self.assertRaises(zlib.error, co.flush) # second flush should not work
-            self.assertEqual(x1 + x2, datazip)
+            # With hardware acceleration, the compressed bytes might not
+            # be identical.
+            if not HW_ACCELERATED:
+                self.assertEqual(x1 + x2, datazip)
         for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))):
             dco = zlib.decompressobj()
             y1 = dco.decompress(v1 + v2)
@@ -364,7 +383,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
             bufs.append(dco.decompress(combuf[i:i+dcx]))
             self.assertEqual(b'', dco.unconsumed_tail, ########
                              "(A) uct should be b'': not %d long" %
-                                       len(dco.unconsumed_tail))
+                             len(dco.unconsumed_tail))
             self.assertEqual(b'', dco.unused_data)
         if flush:
             bufs.append(dco.flush())
@@ -377,7 +396,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
                     break
         self.assertEqual(b'', dco.unconsumed_tail, ########
                          "(B) uct should be b'': not %d long" %
-                                       len(dco.unconsumed_tail))
+                         len(dco.unconsumed_tail))
         self.assertEqual(b'', dco.unused_data)
         self.assertEqual(data, b''.join(bufs))
         # Failure means: "decompressobj with init options failed"
@@ -406,7 +425,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
             #max_length = 1 + len(cb)//10
             chunk = dco.decompress(cb, dcx)
             self.assertFalse(len(chunk) > dcx,
-                    'chunk too big (%d>%d)' % (len(chunk), dcx))
+                             'chunk too big (%d>%d)' % (len(chunk), dcx))
             bufs.append(chunk)
             cb = dco.unconsumed_tail
         bufs.append(dco.flush())
@@ -431,7 +450,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
             max_length = 1 + len(cb)//10
             chunk = dco.decompress(cb, max_length)
             self.assertFalse(len(chunk) > max_length,
-                        'chunk too big (%d>%d)' % (len(chunk),max_length))
+                             'chunk too big (%d>%d)' % (len(chunk),max_length))
             bufs.append(chunk)
             cb = dco.unconsumed_tail
         if flush:
@@ -440,7 +459,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
             while chunk:
                 chunk = dco.decompress(b'', max_length)
                 self.assertFalse(len(chunk) > max_length,
-                            'chunk too big (%d>%d)' % (len(chunk),max_length))
+                                 'chunk too big (%d>%d)' % (len(chunk),max_length))
                 bufs.append(chunk)
         self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
 
@@ -487,9 +506,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
         sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH',
                     'Z_PARTIAL_FLUSH']
 
-        ver = tuple(int(v) for v in zlib.ZLIB_RUNTIME_VERSION.split('.'))
         # Z_BLOCK has a known failure prior to 1.2.5.3
-        if ver >= (1, 2, 5, 3):
+        if ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 5, 3):
             sync_opt.append('Z_BLOCK')
 
         sync_opt = [getattr(zlib, opt) for opt in sync_opt
@@ -498,20 +516,16 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
 
         for sync in sync_opt:
             for level in range(10):
-                try:
+                with self.subTest(sync=sync, level=level):
                     obj = zlib.compressobj( level )
                     a = obj.compress( data[:3000] )
                     b = obj.flush( sync )
                     c = obj.compress( data[3000:] )
                     d = obj.flush()
-                except:
-                    print("Error for flush mode={}, level={}"
-                          .format(sync, level))
-                    raise
-                self.assertEqual(zlib.decompress(b''.join([a,b,c,d])),
-                                 data, ("Decompress failed: flush "
-                                        "mode=%i, level=%i") % (sync, level))
-                del obj
+                    self.assertEqual(zlib.decompress(b''.join([a,b,c,d])),
+                                     data, ("Decompress failed: flush "
+                                            "mode=%i, level=%i") % (sync, level))
+                    del obj
 
     @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'),
                          'requires zlib.Z_SYNC_FLUSH')
@@ -526,18 +540,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
 
         # Try 17K of data
         # generate random data stream
-        try:
-            # In 2.3 and later, WichmannHill is the RNG of the bug report
-            gen = random.WichmannHill()
-        except AttributeError:
-            try:
-                # 2.2 called it Random
-                gen = random.Random()
-            except AttributeError:
-                # others might simply have a single RNG
-                gen = random
-        gen.seed(1)
-        data = gen.randbytes(17 * 1024)
+        data = random.randbytes(17 * 1024)
 
         # compress, sync-flush, and decompress
         first = co.compress(data)
@@ -642,7 +645,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
                         self.assertEqual(dco.unconsumed_tail, b'')
                     else:
                         data += dco.decompress(
-                                dco.unconsumed_tail + x[i : i + step], maxlen)
+                            dco.unconsumed_tail + x[i : i + step], maxlen)
                 data += dco.flush()
                 self.assertTrue(dco.eof)
                 self.assertEqual(data, source)
@@ -830,16 +833,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
     @unittest.expectedFailure
     def test_wbits(self):
         # wbits=0 only supported since zlib v1.2.3.5
-        # Register "1.2.3" as "1.2.3.0"
-        # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux"
-        v = zlib.ZLIB_RUNTIME_VERSION.split('-', 1)[0].split('.')
-        if len(v) < 4:
-            v.append('0')
-        elif not v[-1].isnumeric():
-            v[-1] = '0'
-
-        v = tuple(map(int, v))
-        supports_wbits_0 = v >= (1, 2, 3, 5)
+        supports_wbits_0 = ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 3, 5)
 
         co = zlib.compressobj(level=1, wbits=15)
         zlib15 = co.compress(HAMLET_SCENE) + co.flush()
@@ -965,6 +959,188 @@ LAERTES
        Farewell.
 """
 
+class ZlibDecompressorTest(unittest.TestCase):
+    # Test adopted from test_bz2.py
+    TEXT = HAMLET_SCENE
+    DATA = zlib.compress(HAMLET_SCENE)
+    BAD_DATA = b"Not a valid deflate block"
+    BIG_TEXT = DATA * ((128 * 1024 // len(DATA)) + 1)
+    BIG_DATA = zlib.compress(BIG_TEXT)
+
+    def test_Constructor(self):
+        self.assertRaises(TypeError, zlib._ZlibDecompressor, "ASDA")
+        self.assertRaises(TypeError, zlib._ZlibDecompressor, -15, "notbytes")
+        self.assertRaises(TypeError, zlib._ZlibDecompressor, -15, b"bytes", 5)
+
+    def testDecompress(self):
+        zlibd = zlib._ZlibDecompressor()
+        self.assertRaises(TypeError, zlibd.decompress)
+        text = zlibd.decompress(self.DATA)
+        self.assertEqual(text, self.TEXT)
+
+    def testDecompressChunks10(self):
+        zlibd = zlib._ZlibDecompressor()
+        text = b''
+        n = 0
+        while True:
+            str = self.DATA[n*10:(n+1)*10]
+            if not str:
+                break
+            text += zlibd.decompress(str)
+            n += 1
+        self.assertEqual(text, self.TEXT)
+
+    def testDecompressUnusedData(self):
+        zlibd = zlib._ZlibDecompressor()
+        unused_data = b"this is unused data"
+        text = zlibd.decompress(self.DATA+unused_data)
+        self.assertEqual(text, self.TEXT)
+        self.assertEqual(zlibd.unused_data, unused_data)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def testEOFError(self):
+        zlibd = zlib._ZlibDecompressor()
+        text = zlibd.decompress(self.DATA)
+        self.assertRaises(EOFError, zlibd.decompress, b"anything")
+        self.assertRaises(EOFError, zlibd.decompress, b"")
+
+    @support.skip_if_pgo_task
+    @bigmemtest(size=_4G + 100, memuse=3.3)
+    def testDecompress4G(self, size):
+        # "Test zlib._ZlibDecompressor.decompress() with >4GiB input"
+        blocksize = min(10 * 1024 * 1024, size)
+        block = random.randbytes(blocksize)
+        try:
+            data = block * ((size-1) // blocksize + 1)
+            compressed = zlib.compress(data)
+            zlibd = zlib._ZlibDecompressor()
+            decompressed = zlibd.decompress(compressed)
+            self.assertTrue(decompressed == data)
+        finally:
+            data = None
+            compressed = None
+            decompressed = None
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def testPickle(self):
+        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+            with self.assertRaises(TypeError):
+                pickle.dumps(zlib._ZlibDecompressor(), proto)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def testDecompressorChunksMaxsize(self):
+        zlibd = zlib._ZlibDecompressor()
+        max_length = 100
+        out = []
+
+        # Feed some input
+        len_ = len(self.BIG_DATA) - 64
+        out.append(zlibd.decompress(self.BIG_DATA[:len_],
+                                    max_length=max_length))
+        self.assertFalse(zlibd.needs_input)
+        self.assertEqual(len(out[-1]), max_length)
+
+        # Retrieve more data without providing more input
+        out.append(zlibd.decompress(b'', max_length=max_length))
+        self.assertFalse(zlibd.needs_input)
+        self.assertEqual(len(out[-1]), max_length)
+
+        # Retrieve more data while providing more input
+        out.append(zlibd.decompress(self.BIG_DATA[len_:],
+                                    max_length=max_length))
+        self.assertLessEqual(len(out[-1]), max_length)
+
+        # Retrieve remaining uncompressed data
+        while not zlibd.eof:
+            out.append(zlibd.decompress(b'', max_length=max_length))
+            self.assertLessEqual(len(out[-1]), max_length)
+
+        out = b"".join(out)
+        self.assertEqual(out, self.BIG_TEXT)
+        self.assertEqual(zlibd.unused_data, b"")
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_decompressor_inputbuf_1(self):
+        # Test reusing input buffer after moving existing
+        # contents to beginning
+        zlibd = zlib._ZlibDecompressor()
+        out = []
+
+        # Create input buffer and fill it
+        self.assertEqual(zlibd.decompress(self.DATA[:100],
+                                          max_length=0), b'')
+
+        # Retrieve some results, freeing capacity at beginning
+        # of input buffer
+        out.append(zlibd.decompress(b'', 2))
+
+        # Add more data that fits into input buffer after
+        # moving existing data to beginning
+        out.append(zlibd.decompress(self.DATA[100:105], 15))
+
+        # Decompress rest of data
+        out.append(zlibd.decompress(self.DATA[105:]))
+        self.assertEqual(b''.join(out), self.TEXT)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_decompressor_inputbuf_2(self):
+        # Test reusing input buffer by appending data at the
+        # end right away
+        zlibd = zlib._ZlibDecompressor()
+        out = []
+
+        # Create input buffer and empty it
+        self.assertEqual(zlibd.decompress(self.DATA[:200],
+                                          max_length=0), b'')
+        out.append(zlibd.decompress(b''))
+
+        # Fill buffer with new data
+        out.append(zlibd.decompress(self.DATA[200:280], 2))
+
+        # Append some more data, not enough to require resize
+        out.append(zlibd.decompress(self.DATA[280:300], 2))
+
+        # Decompress rest of data
+        out.append(zlibd.decompress(self.DATA[300:]))
+        self.assertEqual(b''.join(out), self.TEXT)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_decompressor_inputbuf_3(self):
+        # Test reusing input buffer after extending it
+
+        zlibd = zlib._ZlibDecompressor()
+        out = []
+
+        # Create almost full input buffer
+        out.append(zlibd.decompress(self.DATA[:200], 5))
+
+        # Add even more data to it, requiring resize
+        out.append(zlibd.decompress(self.DATA[200:300], 5))
+
+        # Decompress rest of data
+        out.append(zlibd.decompress(self.DATA[300:]))
+        self.assertEqual(b''.join(out), self.TEXT)
+
+    def test_failure(self):
+        zlibd = zlib._ZlibDecompressor()
+        self.assertRaises(Exception, zlibd.decompress, self.BAD_DATA * 30)
+        # Previously, a second call could crash due to internal inconsistency
+        self.assertRaises(Exception, zlibd.decompress, self.BAD_DATA * 30)
+
+    @support.refcount_test
+    def test_refleaks_in___init__(self):
+        gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount')
+        zlibd = zlib._ZlibDecompressor()
+        refs_before = gettotalrefcount()
+        for i in range(100):
+            zlibd.__init__()
+        self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10)
 
 class CustomInt:
     def __index__(self):

README.md

@@ -56,7 +56,7 @@ NOTE: For windows users, please set `RUSTPYTHONPATH` environment variable as `Li
 You can also install and run RustPython with the following:
 
 ```bash
-$ cargo install --git https://github.com/RustPython/RustPython
+$ cargo install --git https://github.com/RustPython/RustPython rustpython
 $ rustpython
 Welcome to the magnificent Rust Python interpreter
 >>>>>
@@ -91,13 +91,13 @@ You can compile RustPython to a standalone WebAssembly WASI module so it can run
 Build
 
 ```bash
-cargo build --target wasm32-wasi --no-default-features --features freeze-stdlib,stdlib --release
+cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release
 ```
 
 Run by wasmer
 
 ```bash
-wasmer run --dir `pwd` -- target/wasm32-wasi/release/rustpython.wasm `pwd`/extra_tests/snippets/stdlib_random.py
+wasmer run --dir `pwd` -- target/wasm32-wasip1/release/rustpython.wasm `pwd`/extra_tests/snippets/stdlib_random.py
 ```
 
 Run by wapm
@@ -114,10 +114,10 @@ $ wapm run rustpython
 You can build the WebAssembly WASI file with:
 
 ```bash
-cargo build --release --target wasm32-wasi --features="freeze-stdlib"
+cargo build --release --target wasm32-wasip1 --features="freeze-stdlib"
 ```
 
-> Note: we use the `freeze-stdlib` to include the standard library inside the binary. You also have to run once `rustup target add wasm32-wasi`.
+> Note: we use the `freeze-stdlib` to include the standard library inside the binary. You also have to run once `rustup target add wasm32-wasip1`.
 
 ### JIT (Just in time) compiler
 

common/src/boxvec.rs

@@ -1,7 +1,8 @@
 //! An unresizable vector backed by a `Box<[T]>`
 
+#![allow(clippy::needless_lifetimes)]
+
 use std::{
-    alloc,
     borrow::{Borrow, BorrowMut},
     cmp, fmt,
     mem::{self, MaybeUninit},
@@ -35,29 +36,11 @@ macro_rules! panic_oob {
     };
 }
 
-fn capacity_overflow() -> ! {
-    panic!("capacity overflow")
-}
-
 impl<T> BoxVec<T> {
     pub fn new(n: usize) -> BoxVec<T> {
-        unsafe {
-            let layout = match alloc::Layout::array::<T>(n) {
-                Ok(l) => l,
-                Err(_) => capacity_overflow(),
-            };
-            let ptr = if mem::size_of::<T>() == 0 {
-                ptr::NonNull::<MaybeUninit<T>>::dangling().as_ptr()
-            } else {
-                let ptr = alloc::alloc(layout);
-                if ptr.is_null() {
-                    alloc::handle_alloc_error(layout)
-                }
-                ptr as *mut MaybeUninit<T>
-            };
-            let ptr = ptr::slice_from_raw_parts_mut(ptr, n);
-            let xs = Box::from_raw(ptr);
-            BoxVec { xs, len: 0 }
+        BoxVec {
+            xs: Box::new_uninit_slice(n),
+            len: 0,
         }
     }
 

common/src/int.rs

@@ -60,9 +60,9 @@ pub fn bytes_to_int(lit: &[u8], mut base: u32) -> Option<BigInt> {
                     return Some(BigInt::zero());
                 }
             }
-            16 => lit.get(1).map_or(false, |&b| matches!(b, b'x' | b'X')),
-            2 => lit.get(1).map_or(false, |&b| matches!(b, b'b' | b'B')),
-            8 => lit.get(1).map_or(false, |&b| matches!(b, b'o' | b'O')),
+            16 => lit.get(1).is_some_and(|&b| matches!(b, b'x' | b'X')),
+            2 => lit.get(1).is_some_and(|&b| matches!(b, b'b' | b'B')),
+            8 => lit.get(1).is_some_and(|&b| matches!(b, b'o' | b'O')),
             _ => false,
         }
     } else {

common/src/lib.rs

@@ -1,6 +1,6 @@
 //! A crate to hold types and functions common to all rustpython components.
 
-#![cfg_attr(target_os = "redox", feature(byte_slice_trim_ascii))]
+#![cfg_attr(target_os = "redox", feature(byte_slice_trim_ascii, new_uninit))]
 
 #[macro_use]
 mod macros;

common/src/lock/cell_lock.rs

@@ -2,7 +2,7 @@ use lock_api::{
     GetThreadId, RawMutex, RawRwLock, RawRwLockDowngrade, RawRwLockRecursive, RawRwLockUpgrade,
     RawRwLockUpgradeDowngrade,
 };
-use std::{cell::Cell, num::NonZeroUsize};
+use std::{cell::Cell, num::NonZero};
 
 pub struct RawCellMutex {
     locked: Cell<bool>,
@@ -203,7 +203,7 @@ fn deadlock(lock_kind: &str, ty: &str) -> ! {
 pub struct SingleThreadId(());
 unsafe impl GetThreadId for SingleThreadId {
     const INIT: Self = SingleThreadId(());
-    fn nonzero_thread_id(&self) -> NonZeroUsize {
-        NonZeroUsize::new(1).unwrap()
+    fn nonzero_thread_id(&self) -> NonZero<usize> {
+        NonZero::new(1).unwrap()
     }
 }

common/src/lock/immutable_mutex.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::needless_lifetimes)]
+
 use lock_api::{MutexGuard, RawMutex};
 use std::{fmt, marker::PhantomData, ops::Deref};
 

common/src/lock/thread_mutex.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::needless_lifetimes)]
+
 use lock_api::{GetThreadId, GuardNoSend, RawMutex};
 use std::{
     cell::UnsafeCell,

common/src/str.rs

@@ -331,12 +331,13 @@ pub mod levenshtein {
 /// ```
 #[macro_export]
 macro_rules! ascii {
-    ($x:literal) => {{
-        const STR: &str = $x;
-        const _: () = if !STR.is_ascii() {
-            panic!("ascii!() argument is not an ascii string");
+    ($x:expr $(,)?) => {{
+        let s = const {
+            let s: &str = $x;
+            assert!(s.is_ascii(), "ascii!() argument is not an ascii string");
+            s
         };
-        unsafe { $crate::vendored::ascii::AsciiStr::from_ascii_unchecked(STR.as_bytes()) }
+        unsafe { $crate::vendored::ascii::AsciiStr::from_ascii_unchecked(s.as_bytes()) }
     }};
 }
 pub use ascii;

compiler/codegen/src/compile.rs

@@ -18,7 +18,10 @@ use num_complex::Complex64;
 use num_traits::ToPrimitive;
 use rustpython_ast::located::{self as located_ast, Located};
 use rustpython_compiler_core::{
-    bytecode::{self, Arg as OpArgMarker, CodeObject, ConstantData, Instruction, OpArg, OpArgType},
+    bytecode::{
+        self, Arg as OpArgMarker, CodeObject, ComparisonOperator, ConstantData, Instruction, OpArg,
+        OpArgType,
+    },
     Mode,
 };
 use rustpython_parser_core::source_code::{LineNumber, SourceLocation};
@@ -211,6 +214,12 @@ macro_rules! emit {
     };
 }
 
+struct PatternContext {
+    current_block: usize,
+    blocks: Vec<ir::BlockIdx>,
+    allow_irrefutable: bool,
+}
+
 impl Compiler {
     fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self {
         let module_code = ir::CodeInfo {
@@ -1755,14 +1764,152 @@ impl Compiler {
         Ok(())
     }
 
+    fn compile_pattern_value(
+        &mut self,
+        value: &located_ast::PatternMatchValue,
+        _pattern_context: &mut PatternContext,
+    ) -> CompileResult<()> {
+        self.compile_expression(&value.value)?;
+        emit!(
+            self,
+            Instruction::CompareOperation {
+                op: ComparisonOperator::Equal
+            }
+        );
+        Ok(())
+    }
+
+    fn compile_pattern_as(
+        &mut self,
+        as_pattern: &located_ast::PatternMatchAs,
+        pattern_context: &mut PatternContext,
+    ) -> CompileResult<()> {
+        if as_pattern.pattern.is_none() && !pattern_context.allow_irrefutable {
+            // TODO: better error message
+            if let Some(_name) = as_pattern.name.as_ref() {
+                return Err(
+                    self.error_loc(CodegenErrorType::InvalidMatchCase, as_pattern.location())
+                );
+            }
+            return Err(self.error_loc(CodegenErrorType::InvalidMatchCase, as_pattern.location()));
+        }
+        // Need to make a copy for (possibly) storing later:
+        emit!(self, Instruction::Duplicate);
+        if let Some(pattern) = &as_pattern.pattern {
+            self.compile_pattern_inner(pattern, pattern_context)?;
+        }
+        if let Some(name) = as_pattern.name.as_ref() {
+            self.store_name(name.as_str())?;
+        } else {
+            emit!(self, Instruction::Pop);
+        }
+        Ok(())
+    }
+
+    fn compile_pattern_inner(
+        &mut self,
+        pattern_type: &located_ast::Pattern,
+        pattern_context: &mut PatternContext,
+    ) -> CompileResult<()> {
+        match &pattern_type {
+            located_ast::Pattern::MatchValue(value) => {
+                self.compile_pattern_value(value, pattern_context)
+            }
+            located_ast::Pattern::MatchAs(as_pattern) => {
+                self.compile_pattern_as(as_pattern, pattern_context)
+            }
+            _ => {
+                eprintln!("not implemented pattern type: {pattern_type:?}");
+                Err(self.error(CodegenErrorType::NotImplementedYet))
+            }
+        }
+    }
+
+    fn compile_pattern(
+        &mut self,
+        pattern_type: &located_ast::Pattern,
+        pattern_context: &mut PatternContext,
+    ) -> CompileResult<()> {
+        self.compile_pattern_inner(pattern_type, pattern_context)?;
+        emit!(
+            self,
+            Instruction::JumpIfFalse {
+                target: pattern_context.blocks[pattern_context.current_block + 1]
+            }
+        );
+        Ok(())
+    }
+
+    fn compile_match_inner(
+        &mut self,
+        subject: &located_ast::Expr,
+        cases: &[located_ast::MatchCase],
+        pattern_context: &mut PatternContext,
+    ) -> CompileResult<()> {
+        self.compile_expression(subject)?;
+        pattern_context.blocks = std::iter::repeat_with(|| self.new_block())
+            .take(cases.len() + 1)
+            .collect::<Vec<_>>();
+        let end_block = *pattern_context.blocks.last().unwrap();
+
+        let _match_case_type = cases.last().expect("cases is not empty");
+        // TODO: get proper check for default case
+        // let has_default = match_case_type.pattern.is_match_as() && 1 < cases.len();
+        let has_default = false;
+        for i in 0..cases.len() - (has_default as usize) {
+            self.switch_to_block(pattern_context.blocks[i]);
+            pattern_context.current_block = i;
+            pattern_context.allow_irrefutable = cases[i].guard.is_some() || i == cases.len() - 1;
+            let m = &cases[i];
+            // Only copy the subject if we're *not* on the last case:
+            if i != cases.len() - has_default as usize - 1 {
+                emit!(self, Instruction::Duplicate);
+            }
+            self.compile_pattern(&m.pattern, pattern_context)?;
+            self.compile_statements(&m.body)?;
+            emit!(self, Instruction::Jump { target: end_block });
+        }
+        // TODO: below code is not called and does not work
+        if has_default {
+            // A trailing "case _" is common, and lets us save a bit of redundant
+            // pushing and popping in the loop above:
+            let m = &cases.last().unwrap();
+            self.switch_to_block(*pattern_context.blocks.last().unwrap());
+            if cases.len() == 1 {
+                // No matches. Done with the subject:
+                emit!(self, Instruction::Pop);
+            } else {
+                // Show line coverage for default case (it doesn't create bytecode)
+                // emit!(self, Instruction::Nop);
+            }
+            self.compile_statements(&m.body)?;
+        }
+
+        self.switch_to_block(end_block);
+
+        let code = self.current_code_info();
+        pattern_context
+            .blocks
+            .iter()
+            .zip(pattern_context.blocks.iter().skip(1))
+            .for_each(|(a, b)| {
+                code.blocks[a.0 as usize].next = *b;
+            });
+        Ok(())
+    }
+
     fn compile_match(
         &mut self,
         subject: &located_ast::Expr,
         cases: &[located_ast::MatchCase],
     ) -> CompileResult<()> {
-        eprintln!("match subject: {subject:?}");
-        eprintln!("match cases: {cases:?}");
-        Err(self.error(CodegenErrorType::NotImplementedYet))
+        let mut pattern_context = PatternContext {
+            current_block: usize::MAX,
+            blocks: Vec::new(),
+            allow_irrefutable: false,
+        };
+        self.compile_match_inner(subject, cases, &mut pattern_context)?;
+        Ok(())
     }
 
     fn compile_chained_comparison(
@@ -3036,16 +3183,17 @@ impl Compiler {
     fn switch_to_block(&mut self, block: ir::BlockIdx) {
         let code = self.current_code_info();
         let prev = code.current_block;
+        assert_ne!(prev, block, "recursive switching {prev:?} -> {block:?}");
         assert_eq!(
             code.blocks[block].next,
             ir::BlockIdx::NULL,
-            "switching to completed block"
+            "switching {prev:?} -> {block:?} to completed block"
         );
         let prev_block = &mut code.blocks[prev.0 as usize];
         assert_eq!(
             prev_block.next.0,
             u32::MAX,
-            "switching from block that's already got a next"
+            "switching {prev:?} -> {block:?} from block that's already got a next"
         );
         prev_block.next = block;
         code.current_block = block;
@@ -3102,19 +3250,18 @@ impl Compiler {
             Expr::Tuple(ExprTuple { elts, .. }) => elts.iter().any(Self::contains_await),
             Expr::Set(ExprSet { elts, .. }) => elts.iter().any(Self::contains_await),
             Expr::Dict(ExprDict { keys, values, .. }) => {
-                keys.iter()
-                    .any(|key| key.as_ref().map_or(false, Self::contains_await))
+                keys.iter().flatten().any(Self::contains_await)
                     || values.iter().any(Self::contains_await)
             }
             Expr::Slice(ExprSlice {
                 lower, upper, step, ..
             }) => {
-                lower.as_ref().map_or(false, |l| Self::contains_await(l))
-                    || upper.as_ref().map_or(false, |u| Self::contains_await(u))
-                    || step.as_ref().map_or(false, |s| Self::contains_await(s))
+                lower.as_deref().is_some_and(Self::contains_await)
+                    || upper.as_deref().is_some_and(Self::contains_await)
+                    || step.as_deref().is_some_and(Self::contains_await)
             }
             Expr::Yield(ExprYield { value, .. }) => {
-                value.as_ref().map_or(false, |v| Self::contains_await(v))
+                value.as_deref().is_some_and(Self::contains_await)
             }
             Expr::Await(ExprAwait { .. }) => true,
             Expr::YieldFrom(ExprYieldFrom { value, .. }) => Self::contains_await(value),
@@ -3128,9 +3275,7 @@ impl Compiler {
                 ..
             }) => {
                 Self::contains_await(value)
-                    || format_spec
-                        .as_ref()
-                        .map_or(false, |fs| Self::contains_await(fs))
+                    || format_spec.as_deref().is_some_and(Self::contains_await)
             }
             Expr::Name(located_ast::ExprName { .. }) => false,
             Expr::Lambda(located_ast::ExprLambda { body, .. }) => Self::contains_await(body),

compiler/codegen/src/error.rs

@@ -30,6 +30,8 @@ pub enum CodegenErrorType {
     TooManyStarUnpack,
     EmptyWithItems,
     EmptyWithBody,
+    DuplicateStore(String),
+    InvalidMatchCase,
     NotImplementedYet, // RustPython marker for unimplemented features
 }
 
@@ -75,6 +77,12 @@ impl fmt::Display for CodegenErrorType {
             EmptyWithBody => {
                 write!(f, "empty body on With")
             }
+            DuplicateStore(s) => {
+                write!(f, "duplicate store {s}")
+            }
+            InvalidMatchCase => {
+                write!(f, "invalid match case")
+            }
             NotImplementedYet => {
                 write!(f, "RustPython does not implement this feature yet")
             }

compiler/codegen/src/symboltable.rs

@@ -886,11 +886,13 @@ impl SymbolTableBuilder {
                 self.scan_statements(orelse)?;
                 self.scan_statements(finalbody)?;
             }
-            Stmt::Match(StmtMatch { subject, .. }) => {
-                return Err(SymbolTableError {
-                    error: "match expression is not implemented yet".to_owned(),
-                    location: Some(subject.location()),
-                });
+            Stmt::Match(StmtMatch { subject, cases, .. }) => {
+                self.scan_expression(subject, ExpressionContext::Load)?;
+                for case in cases {
+                    // TODO: below
+                    // self.scan_pattern(&case.pattern, ExpressionContext::Load)?;
+                    self.scan_statements(&case.body)?;
+                }
             }
             Stmt::Raise(StmtRaise { exc, cause, .. }) => {
                 if let Some(expression) = exc {

compiler/core/src/frozen.rs

@@ -102,8 +102,7 @@ impl FrozenLib<Vec<u8>> {
     /// Encode the given iterator of frozen modules into a compressed vector of bytes
     pub fn encode<'a, I, B: AsRef<[u8]>>(lib: I) -> FrozenLib<Vec<u8>>
     where
-        I: IntoIterator<Item = (&'a str, FrozenModule<B>)>,
-        I::IntoIter: ExactSizeIterator + Clone,
+        I: IntoIterator<Item = (&'a str, FrozenModule<B>), IntoIter: ExactSizeIterator + Clone>,
     {
         let iter = lib.into_iter();
         let mut bytes = Vec::new();

derive-impl/src/pyclass.rs

@@ -348,7 +348,7 @@ fn generate_class_def(
             && if let Ok(Meta::List(l)) = attr.parse_meta() {
                 l.nested
                     .into_iter()
-                    .any(|n| n.get_ident().map_or(false, |p| p == "PyStructSequence"))
+                    .any(|n| n.get_ident().is_some_and(|p| p == "PyStructSequence"))
             } else {
                 false
             }

derive-impl/src/util.rs

@@ -558,7 +558,7 @@ impl AttributeExt for Attribute {
             let has_name = list
                 .nested
                 .iter()
-                .any(|nested_meta| nested_meta.get_path().map_or(false, |p| p.is_ident(name)));
+                .any(|nested_meta| nested_meta.get_path().is_some_and(|p| p.is_ident(name)));
             if !has_name {
                 list.nested.push(new_item())
             }

extra_tests/snippets/operator_arithmetic.py

@@ -32,3 +32,23 @@ assert_raises(ValueError, lambda: 1 << -1)
 
 # Right shift raises value error on negative
 assert_raises(ValueError, lambda: 1 >> -1)
+
+# Bitwise or, and, xor raises value error on incompatible types
+assert_raises(TypeError, lambda: "abc" | True)
+assert_raises(TypeError, lambda: "abc" & True)
+assert_raises(TypeError, lambda: "abc" ^ True)
+assert_raises(TypeError, lambda: True | "abc")
+assert_raises(TypeError, lambda: True & "abc")
+assert_raises(TypeError, lambda: True ^ "abc")
+assert_raises(TypeError, lambda: "abc" | 1.5)
+assert_raises(TypeError, lambda: "abc" & 1.5)
+assert_raises(TypeError, lambda: "abc" ^ 1.5)
+assert_raises(TypeError, lambda: 1.5 | "abc")
+assert_raises(TypeError, lambda: 1.5 & "abc")
+assert_raises(TypeError, lambda: 1.5 ^ "abc")
+assert_raises(TypeError, lambda: True | 1.5)
+assert_raises(TypeError, lambda: True & 1.5)
+assert_raises(TypeError, lambda: True ^ 1.5)
+assert_raises(TypeError, lambda: 1.5 | True)
+assert_raises(TypeError, lambda: 1.5 & True)
+assert_raises(TypeError, lambda: 1.5 ^ True)

extra_tests/snippets/stdlib_os.py

@@ -502,9 +502,9 @@ with TestWithTempDir() as tmpdir:
             assert set(collected_files) == set(expected_files)
 
 # system()
-if "win" not in sys.platform:
-    assert os.system('ls') == 0
-    assert os.system('{') != 0
+if os.name in ('posix', 'nt'):
+    assert os.system('echo test') == 0
+    assert os.system('&') != 0
 
     for arg in [None, 1, 1.0, TabError]:
         assert_raises(TypeError, os.system, arg)

extra_tests/snippets/stdlib_subprocess.py

@@ -16,7 +16,7 @@ else:
         return ["cmd", "/C", f"echo {text}"]
     def sleep(secs):
         # TODO: make work in a non-unixy environment (something with timeout.exe?)
-        return ["sleep", str(secs)]
+        return ["powershell", "/C", "sleep", str(secs)]
 
 p = subprocess.Popen(echo("test"))
 

jit/src/instructions.rs

@@ -1,3 +1,5 @@
+use super::{JitCompileError, JitSig, JitType};
+use cranelift::codegen::ir::FuncRef;
 use cranelift::prelude::*;
 use num_traits::cast::ToPrimitive;
 use rustpython_compiler_core::bytecode::{
@@ -6,8 +8,6 @@ use rustpython_compiler_core::bytecode::{
 };
 use std::collections::HashMap;
 
-use super::{JitCompileError, JitSig, JitType};
-
 #[repr(u16)]
 enum CustomTrapCode {
     /// Raised when shifting by a negative number
@@ -27,6 +27,7 @@ enum JitValue {
     Bool(Value),
     None,
     Tuple(Vec<JitValue>),
+    FuncRef(FuncRef),
 }
 
 impl JitValue {
@@ -43,14 +44,14 @@ impl JitValue {
             JitValue::Int(_) => Some(JitType::Int),
             JitValue::Float(_) => Some(JitType::Float),
             JitValue::Bool(_) => Some(JitType::Bool),
-            JitValue::None | JitValue::Tuple(_) => None,
+            JitValue::None | JitValue::Tuple(_) | JitValue::FuncRef(_) => None,
         }
     }
 
     fn into_value(self) -> Option<Value> {
         match self {
             JitValue::Int(val) | JitValue::Float(val) | JitValue::Bool(val) => Some(val),
-            JitValue::None | JitValue::Tuple(_) => None,
+            JitValue::None | JitValue::Tuple(_) | JitValue::FuncRef(_) => None,
         }
     }
 }
@@ -68,6 +69,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
         builder: &'a mut FunctionBuilder<'b>,
         num_variables: usize,
         arg_types: &[JitType],
+        ret_type: Option<JitType>,
         entry_block: Block,
     ) -> FunctionCompiler<'a, 'b> {
         let mut compiler = FunctionCompiler {
@@ -77,7 +79,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
             label_to_block: HashMap::new(),
             sig: JitSig {
                 args: arg_types.to_vec(),
-                ret: None,
+                ret: ret_type,
             },
         };
         let params = compiler.builder.func.dfg.block_params(entry_block).to_vec();
@@ -132,7 +134,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
             }
             JitValue::Bool(val) => Ok(val),
             JitValue::None => Ok(self.builder.ins().iconst(types::I8, 0)),
-            JitValue::Tuple(_) => Err(JitCompileError::NotSupported),
+            JitValue::Tuple(_) | JitValue::FuncRef(_) => Err(JitCompileError::NotSupported),
         }
     }
 
@@ -146,6 +148,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
 
     pub fn compile<C: bytecode::Constant>(
         &mut self,
+        func_ref: FuncRef,
         bytecode: &CodeObject<C>,
     ) -> Result<(), JitCompileError> {
         // TODO: figure out if this is sufficient -- previously individual labels were associated
@@ -177,7 +180,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
                 continue;
             }
 
-            self.add_instruction(instruction, arg, &bytecode.constants)?;
+            self.add_instruction(func_ref, bytecode, instruction, arg)?;
         }
 
         Ok(())
@@ -229,9 +232,10 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
 
     pub fn add_instruction<C: bytecode::Constant>(
         &mut self,
+        func_ref: FuncRef,
+        bytecode: &CodeObject<C>,
         instruction: Instruction,
         arg: OpArg,
-        constants: &[C],
     ) -> Result<(), JitCompileError> {
         match instruction {
             Instruction::ExtendedArg => Ok(()),
@@ -282,7 +286,8 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
                 self.store_variable(idx.get(arg), val)
             }
             Instruction::LoadConst { idx } => {
-                let val = self.prepare_const(constants[idx.get(arg) as usize].borrow_constant())?;
+                let val = self
+                    .prepare_const(bytecode.constants[idx.get(arg) as usize].borrow_constant())?;
                 self.stack.push(val);
                 Ok(())
             }
@@ -311,7 +316,8 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
                 self.return_value(val)
             }
             Instruction::ReturnConst { idx } => {
-                let val = self.prepare_const(constants[idx.get(arg) as usize].borrow_constant())?;
+                let val = self
+                    .prepare_const(bytecode.constants[idx.get(arg) as usize].borrow_constant())?;
                 self.return_value(val)
             }
             Instruction::CompareOperation { op, .. } => {
@@ -508,6 +514,36 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
                 // TODO: block support
                 Ok(())
             }
+            Instruction::LoadGlobal(idx) => {
+                let name = &bytecode.names[idx.get(arg) as usize];
+
+                if name.as_ref() != bytecode.obj_name.as_ref() {
+                    Err(JitCompileError::NotSupported)
+                } else {
+                    self.stack.push(JitValue::FuncRef(func_ref));
+                    Ok(())
+                }
+            }
+            Instruction::CallFunctionPositional { nargs } => {
+                let nargs = nargs.get(arg);
+
+                let mut args = Vec::new();
+                for _ in 0..nargs {
+                    let arg = self.stack.pop().ok_or(JitCompileError::BadBytecode)?;
+                    args.push(arg.into_value().unwrap());
+                }
+
+                match self.stack.pop().ok_or(JitCompileError::BadBytecode)? {
+                    JitValue::FuncRef(reference) => {
+                        let call = self.builder.ins().call(reference, &args);
+                        let returns = self.builder.inst_results(call);
+                        self.stack.push(JitValue::Int(returns[0]));
+
+                        Ok(())
+                    }
+                    _ => Err(JitCompileError::BadBytecode),
+                }
+            }
             _ => Err(JitCompileError::NotSupported),
         }
     }

jit/src/lib.rs

@@ -49,6 +49,7 @@ impl Jit {
         &mut self,
         bytecode: &bytecode::CodeObject<C>,
         args: &[JitType],
+        ret: Option<JitType>,
     ) -> Result<(FuncId, JitSig), JitCompileError> {
         for arg in args {
             self.ctx
@@ -58,22 +59,13 @@ impl Jit {
                 .push(AbiParam::new(arg.to_cranelift()));
         }
 
-        let mut builder = FunctionBuilder::new(&mut self.ctx.func, &mut self.builder_context);
-        let entry_block = builder.create_block();
-        builder.append_block_params_for_function_params(entry_block);
-        builder.switch_to_block(entry_block);
-
-        let sig = {
-            let mut compiler =
-                FunctionCompiler::new(&mut builder, bytecode.varnames.len(), args, entry_block);
-
-            compiler.compile(bytecode)?;
-
-            compiler.sig
-        };
-
-        builder.seal_all_blocks();
-        builder.finalize();
+        if ret.is_some() {
+            self.ctx
+                .func
+                .signature
+                .returns
+                .push(AbiParam::new(ret.clone().unwrap().to_cranelift()));
+        }
 
         let id = self.module.declare_function(
             &format!("jit_{}", bytecode.obj_name.as_ref()),
@@ -81,6 +73,30 @@ impl Jit {
             &self.ctx.func.signature,
         )?;
 
+        let func_ref = self.module.declare_func_in_func(id, &mut self.ctx.func);
+
+        let mut builder = FunctionBuilder::new(&mut self.ctx.func, &mut self.builder_context);
+        let entry_block = builder.create_block();
+        builder.append_block_params_for_function_params(entry_block);
+        builder.switch_to_block(entry_block);
+
+        let sig = {
+            let mut compiler = FunctionCompiler::new(
+                &mut builder,
+                bytecode.varnames.len(),
+                args,
+                ret,
+                entry_block,
+            );
+
+            compiler.compile(func_ref, bytecode)?;
+
+            compiler.sig
+        };
+
+        builder.seal_all_blocks();
+        builder.finalize();
+
         self.module.define_function(id, &mut self.ctx)?;
 
         self.module.clear_context(&mut self.ctx);
@@ -92,10 +108,11 @@ impl Jit {
 pub fn compile<C: bytecode::Constant>(
     bytecode: &bytecode::CodeObject<C>,
     args: &[JitType],
+    ret: Option<JitType>,
 ) -> Result<CompiledCode, JitCompileError> {
     let mut jit = Jit::new();
 
-    let (id, sig) = jit.build_function(bytecode, args)?;
+    let (id, sig) = jit.build_function(bytecode, args, ret)?;
 
     jit.module.finalize_definitions();
 

jit/tests/common.rs

@@ -27,7 +27,17 @@ impl Function {
             arg_types.push(arg_type);
         }
 
-        rustpython_jit::compile(&self.code, &arg_types).expect("Compile failure")
+        let ret_type = match self.annotations.get("return") {
+            Some(StackValue::String(annotation)) => match annotation.as_str() {
+                "int" => Some(JitType::Int),
+                "float" => Some(JitType::Float),
+                "bool" => Some(JitType::Bool),
+                _ => panic!("Unrecognised jit type"),
+            },
+            _ => None,
+        };
+
+        rustpython_jit::compile(&self.code, &arg_types, ret_type).expect("Compile failure")
     }
 }
 

jit/tests/misc_tests.rs

@@ -113,3 +113,15 @@ fn test_unpack_tuple() {
     assert_eq!(unpack_tuple(0, 1), Ok(1));
     assert_eq!(unpack_tuple(1, 2), Ok(2));
 }
+
+#[test]
+fn test_recursive_fib() {
+    let fib = jit_function! { fib(n: i64) -> i64 => r##"
+        def fib(n: int) -> int:
+          if n == 0 or n == 1:
+            return 1
+          return fib(n-1) + fib(n-2)
+    "## };
+
+    assert_eq!(fib(10), Ok(89));
+}

scripts/release-wapm.sh

@@ -7,6 +7,6 @@ FEATURES_FOR_WAPM=(stdlib zlib)
 
 export BUILDTIME_RUSTPYTHONPATH="/lib/rustpython"
 
-cargo build --release --target wasm32-wasi --no-default-features --features="${FEATURES_FOR_WAPM[*]}"
+cargo build --release --target wasm32-wasip1 --no-default-features --features="${FEATURES_FOR_WAPM[*]}"
 
 wapm publish

src/lib.rs

@@ -58,6 +58,7 @@ use std::process::ExitCode;
 pub use interpreter::InterpreterConfig;
 pub use rustpython_vm as vm;
 pub use settings::{opts_with_clap, InstallPipMode, RunMode};
+pub use shell::run_shell;
 
 /// The main cli of the `rustpython` interpreter. This function will return `std::process::ExitCode`
 /// based on the return code of the python code ran through the cli.

src/shell/helper.rs

@@ -22,7 +22,7 @@ fn split_idents_on_dot(line: &str) -> Option<(usize, Vec<String>)> {
         match c {
             '.' => {
                 // check for a double dot
-                if i != 0 && words.last().map_or(false, |s| s.is_empty()) {
+                if i != 0 && words.last().is_some_and(|s| s.is_empty()) {
                     return None;
                 }
                 reverse_string(words.last_mut().unwrap());

stdlib/src/array.rs

@@ -1238,7 +1238,7 @@ mod array {
 
             let res = match array_a.cmp(&array_b) {
                 // fast path for same ArrayContentType type
-                Ok(partial_ord) => partial_ord.map_or(false, |ord| op.eval_ord(ord)),
+                Ok(partial_ord) => partial_ord.is_some_and(|ord| op.eval_ord(ord)),
                 Err(()) => {
                     let iter = Iterator::zip(array_a.iter(vm), array_b.iter(vm));
 

stdlib/src/contextvars.rs

@@ -129,18 +129,16 @@ mod _contextvars {
 
             super::CONTEXTS.with(|ctxs| {
                 let mut ctxs = ctxs.borrow_mut();
-                if !ctxs
-                    .last()
-                    .map_or(false, |ctx| ctx.get_id() == zelf.get_id())
-                {
+                // TODO: use Vec::pop_if once stabilized
+                if ctxs.last().is_some_and(|ctx| ctx.get_id() == zelf.get_id()) {
+                    let _ = ctxs.pop();
+                    Ok(())
+                } else {
                     let msg =
                         "cannot exit context: thread state references a different context object"
                             .to_owned();
-                    return Err(vm.new_runtime_error(msg));
+                    Err(vm.new_runtime_error(msg))
                 }
-
-                let _ = ctxs.pop();
-                Ok(())
             })?;
             zelf.inner.entered.set(false);
 

stdlib/src/grp.rs

@@ -83,7 +83,7 @@ mod grp {
     #[pyfunction]
     fn getgrall(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> {
         // setgrent, getgrent, etc are not thread safe. Could use fgetgrent_r, but this is easier
-        static GETGRALL: parking_lot::Mutex<()> = parking_lot::const_mutex(());
+        static GETGRALL: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
         let _guard = GETGRALL.lock();
         let mut list = Vec::new();
 

stdlib/src/lib.rs

@@ -1,6 +1,7 @@
 // to allow `mod foo {}` in foo.rs; clippy thinks this is a mistake/misunderstanding of
 // how `mod` works, but we want this sometimes for pymodule declarations
 #![allow(clippy::module_inception)]
+#![cfg_attr(target_os = "redox", feature(raw_ref_op))]
 
 #[macro_use]
 extern crate rustpython_derive;

stdlib/src/select.rs

@@ -36,7 +36,7 @@ mod platform {
     // based off winsock2.h: https://gist.github.com/piscisaureus/906386#file-winsock2-h-L128-L141
 
     pub unsafe fn FD_SET(fd: RawFd, set: *mut fd_set) {
-        let mut slot = std::ptr::addr_of_mut!((*set).fd_array).cast::<RawFd>();
+        let mut slot = (&raw mut (*set).fd_array).cast::<RawFd>();
         let fd_count = (*set).fd_count;
         for _ in 0..fd_count {
             if *slot == fd {

stdlib/src/socket.rs

@@ -2217,7 +2217,7 @@ mod _socket {
                 fn as_slice(&self) -> &[netioapi::MIB_IF_ROW2] {
                     unsafe {
                         let p = self.ptr.as_ptr();
-                        let ptr = ptr::addr_of!((*p).Table) as *const netioapi::MIB_IF_ROW2;
+                        let ptr = &raw const (*p).Table as *const netioapi::MIB_IF_ROW2;
                         std::slice::from_raw_parts(ptr, (*p).NumEntries as usize)
                     }
                 }

stdlib/src/sqlite.rs

@@ -76,7 +76,7 @@ mod _sqlite {
         ffi::{c_int, c_longlong, c_uint, c_void, CStr},
         fmt::Debug,
         ops::Deref,
-        ptr::{addr_of_mut, null, null_mut},
+        ptr::{null, null_mut},
         thread::ThreadId,
     };
 
@@ -1180,14 +1180,10 @@ mod _sqlite {
                 )
             };
 
-            // TODO: replace with Result.inspect_err when stable
-            if let Err(exc) = db.check(ret, vm) {
+            db.check(ret, vm).inspect_err(|_| {
                 // create_collation do not call destructor if error occur
                 let _ = unsafe { Box::from_raw(data) };
-                Err(exc)
-            } else {
-                Ok(())
-            }
+            })
         }
 
         #[pymethod]
@@ -2398,7 +2394,7 @@ mod _sqlite {
             let ret = unsafe {
                 sqlite3_open_v2(
                     path,
-                    addr_of_mut!(db),
+                    &raw mut db,
                     SQLITE_OPEN_READWRITE
                         | SQLITE_OPEN_CREATE
                         | if uri { SQLITE_OPEN_URI } else { 0 },

stdlib/src/unicodedata.rs

@@ -85,7 +85,7 @@ mod unicodedata {
         }
 
         fn check_age(&self, c: char) -> bool {
-            Age::of(c).map_or(false, |age| age.actual() <= self.unic_version)
+            Age::of(c).is_some_and(|age| age.actual() <= self.unic_version)
         }
 
         fn extract_char(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<Option<char>> {

stdlib/src/zlib.rs

@@ -47,6 +47,7 @@ mod zlib {
     use libz_sys::{
         Z_BLOCK, Z_DEFAULT_STRATEGY, Z_FILTERED, Z_FINISH, Z_FIXED, Z_HUFFMAN_ONLY, Z_RLE, Z_TREES,
     };
+    use rustpython_vm::types::Constructor;
 
     // copied from zlibmodule.c (commit 530f506ac91338)
     #[pyattr]
@@ -587,4 +588,141 @@ mod zlib {
             Ok(Self::new(int))
         }
     }
+
+    #[pyattr]
+    #[pyclass(name = "_ZlibDecompressor")]
+    #[derive(Debug, PyPayload)]
+    pub struct ZlibDecompressor {
+        decompress: PyMutex<Decompress>,
+        unused_data: PyMutex<PyBytesRef>,
+        unconsumed_tail: PyMutex<PyBytesRef>,
+    }
+
+    impl Constructor for ZlibDecompressor {
+        type Args = ();
+
+        fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult {
+            let decompress = Decompress::new(true);
+            let zlib_decompressor = Self {
+                decompress: PyMutex::new(decompress),
+                unused_data: PyMutex::new(PyBytes::from(vec![]).into_ref(&vm.ctx)),
+                unconsumed_tail: PyMutex::new(PyBytes::from(vec![]).into_ref(&vm.ctx)),
+            };
+            zlib_decompressor
+                .into_ref_with_type(vm, cls)
+                .map(Into::into)
+        }
+    }
+
+    #[pyclass(with(Constructor))]
+    impl ZlibDecompressor {
+        #[pygetset]
+        fn unused_data(&self) -> PyBytesRef {
+            self.unused_data.lock().clone()
+        }
+
+        #[pygetset]
+        fn unconsumed_tail(&self) -> PyBytesRef {
+            self.unconsumed_tail.lock().clone()
+        }
+
+        fn save_unused_input(
+            &self,
+            d: &Decompress,
+            data: &[u8],
+            stream_end: bool,
+            orig_in: u64,
+            vm: &VirtualMachine,
+        ) {
+            let leftover = &data[(d.total_in() - orig_in) as usize..];
+
+            if stream_end && !leftover.is_empty() {
+                let mut unused_data = self.unused_data.lock();
+                let unused: Vec<_> = unused_data
+                    .as_bytes()
+                    .iter()
+                    .chain(leftover)
+                    .copied()
+                    .collect();
+                *unused_data = vm.ctx.new_pyref(unused);
+            }
+        }
+
+        #[pymethod]
+        fn decompress(&self, args: PyBytesRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+            // let max_length = args.max_length.value;
+            // let max_length = (max_length != 0).then_some(max_length);
+            let max_length = None;
+            let data = args.as_bytes();
+
+            let mut d = self.decompress.lock();
+            let orig_in = d.total_in();
+
+            let (ret, stream_end) =
+                match _decompress(data, &mut d, DEF_BUF_SIZE, max_length, false, vm) {
+                    Ok((buf, true)) => {
+                        // Eof is true
+                        (Ok(buf), true)
+                    }
+                    Ok((buf, false)) => (Ok(buf), false),
+                    Err(err) => (Err(err), false),
+                };
+            self.save_unused_input(&d, data, stream_end, orig_in, vm);
+
+            let leftover = if stream_end {
+                b""
+            } else {
+                &data[(d.total_in() - orig_in) as usize..]
+            };
+
+            let mut unconsumed_tail = self.unconsumed_tail.lock();
+            if !leftover.is_empty() || !unconsumed_tail.is_empty() {
+                *unconsumed_tail = PyBytes::from(leftover.to_owned()).into_ref(&vm.ctx);
+            }
+
+            ret
+        }
+
+        #[pymethod]
+        fn flush(&self, length: OptionalArg<ArgSize>, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+            let length = match length {
+                OptionalArg::Present(l) => {
+                    let l: isize = l.into();
+                    if l <= 0 {
+                        return Err(
+                            vm.new_value_error("length must be greater than zero".to_owned())
+                        );
+                    } else {
+                        l as usize
+                    }
+                }
+                OptionalArg::Missing => DEF_BUF_SIZE,
+            };
+
+            let mut data = self.unconsumed_tail.lock();
+            let mut d = self.decompress.lock();
+
+            let orig_in = d.total_in();
+
+            let (ret, stream_end) = match _decompress(&data, &mut d, length, None, true, vm) {
+                Ok((buf, stream_end)) => (Ok(buf), stream_end),
+                Err(err) => (Err(err), false),
+            };
+            self.save_unused_input(&d, &data, stream_end, orig_in, vm);
+
+            *data = PyBytes::from(Vec::new()).into_ref(&vm.ctx);
+
+            // TODO: drop the inner decompressor, somehow
+            // if stream_end {
+            //
+            // }
+            ret
+        }
+
+        // TODO: Wait for getstate pyslot to be fixed
+        // #[pyslot]
+        // fn getstate(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyObject> {
+        //     Err(vm.new_type_error("cannot serialize '_ZlibDecompressor' object".to_owned()))
+        // }
+    }
 }

vm/Cargo.toml

@@ -73,12 +73,12 @@ thiserror = { workspace = true }
 thread_local = { workspace = true }
 memchr = { workspace = true }
 
-caseless = "0.2.1"
+caseless = "0.2.2"
 flamer = { version = "0.4", optional = true }
 half = "2"
 memoffset = "0.9.1"
 optional = "0.5.0"
-result-like = "0.4.6"
+result-like = "0.5.0"
 timsort = "0.1.2"
 
 ## unicode stuff

vm/src/anystr.rs

@@ -385,17 +385,14 @@ pub trait AnyStr {
             let (end_len, i_diff) = match *ch {
                 b'\n' => (keep, 1),
                 b'\r' => {
-                    let is_rn = enumerated.peek().map_or(false, |(_, ch)| **ch == b'\n');
+                    let is_rn = enumerated.next_if(|(_, ch)| **ch == b'\n').is_some();
                     if is_rn {
-                        let _ = enumerated.next();
                         (keep + keep, 2)
                     } else {
                         (keep, 1)
                     }
                 }
-                _ => {
-                    continue;
-                }
+                _ => continue,
             };
             let range = last_i..i + end_len;
             last_i = i + i_diff;

vm/src/builtins/asyncgenerator.rs

@@ -390,7 +390,7 @@ impl PyAsyncGenAThrow {
     }
 
     fn ignored_close(&self, res: &PyResult<PyIterReturn>) -> bool {
-        res.as_ref().map_or(false, |v| match v {
+        res.as_ref().is_ok_and(|v| match v {
             PyIterReturn::Return(obj) => obj.payload_is::<PyAsyncGenWrappedValue>(),
             PyIterReturn::StopIteration(_) => false,
         })

vm/src/builtins/bool.rs

@@ -127,8 +127,10 @@ impl PyBool {
             let lhs = get_value(&lhs);
             let rhs = get_value(&rhs);
             (lhs || rhs).to_pyobject(vm)
+        } else if let Some(lhs) = lhs.payload::<PyInt>() {
+            lhs.or(rhs, vm).to_pyobject(vm)
         } else {
-            get_py_int(&lhs).or(rhs, vm).to_pyobject(vm)
+            vm.ctx.not_implemented()
         }
     }
 
@@ -141,8 +143,10 @@ impl PyBool {
             let lhs = get_value(&lhs);
             let rhs = get_value(&rhs);
             (lhs && rhs).to_pyobject(vm)
+        } else if let Some(lhs) = lhs.payload::<PyInt>() {
+            lhs.and(rhs, vm).to_pyobject(vm)
         } else {
-            get_py_int(&lhs).and(rhs, vm).to_pyobject(vm)
+            vm.ctx.not_implemented()
         }
     }
 
@@ -155,8 +159,10 @@ impl PyBool {
             let lhs = get_value(&lhs);
             let rhs = get_value(&rhs);
             (lhs ^ rhs).to_pyobject(vm)
+        } else if let Some(lhs) = lhs.payload::<PyInt>() {
+            lhs.xor(rhs, vm).to_pyobject(vm)
         } else {
-            get_py_int(&lhs).xor(rhs, vm).to_pyobject(vm)
+            vm.ctx.not_implemented()
         }
     }
 }
@@ -207,7 +213,3 @@ pub(crate) fn init(context: &Context) {
 pub(crate) fn get_value(obj: &PyObject) -> bool {
     !obj.payload::<PyInt>().unwrap().as_bigint().is_zero()
 }
-
-fn get_py_int(obj: &PyObject) -> &PyInt {
-    obj.payload::<PyInt>().unwrap()
-}

vm/src/builtins/descriptor.rs

@@ -67,7 +67,7 @@ impl GetDescriptor for PyMethodDescriptor {
         let bound = match obj {
             Some(obj) => {
                 if descr.method.flags.contains(PyMethodFlags::METHOD) {
-                    if cls.map_or(false, |c| c.fast_isinstance(vm.ctx.types.type_type)) {
+                    if cls.is_some_and(|c| c.fast_isinstance(vm.ctx.types.type_type)) {
                         obj
                     } else {
                         return Err(vm.new_type_error(format!(

vm/src/builtins/dict.rs

@@ -728,11 +728,8 @@ impl ExactSizeIterator for DictIter<'_> {
 }
 
 #[pyclass]
-trait DictView: PyPayload + PyClassDef + Iterable + Representable
-where
-    Self::ReverseIter: PyPayload,
-{
-    type ReverseIter;
+trait DictView: PyPayload + PyClassDef + Iterable + Representable {
+    type ReverseIter: PyPayload;
 
     fn dict(&self) -> &PyDictRef;
     fn item(vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef) -> PyObjectRef;

vm/src/builtins/function.rs

@@ -506,7 +506,8 @@ impl PyFunction {
         zelf.jitted_code
             .get_or_try_init(|| {
                 let arg_types = jitfunc::get_jit_arg_types(&zelf, vm)?;
-                rustpython_jit::compile(&zelf.code.code, &arg_types)
+                let ret_type = jitfunc::jit_ret_type(&zelf, vm)?;
+                rustpython_jit::compile(&zelf.code.code, &arg_types, ret_type)
                     .map_err(|err| jitfunc::new_jit_error(err.to_string(), vm))
             })
             .map(drop)

vm/src/builtins/function/jitfunc.rs

@@ -52,7 +52,7 @@ fn get_jit_arg_type(dict: &PyDictRef, name: &str, vm: &VirtualMachine) -> PyResu
             Ok(JitType::Bool)
         } else {
             Err(new_jit_error(
-                "Jit requires argument to be either int or float".to_owned(),
+                "Jit requires argument to be either int, float or bool".to_owned(),
                 vm,
             ))
         }
@@ -106,6 +106,25 @@ pub fn get_jit_arg_types(func: &Py<PyFunction>, vm: &VirtualMachine) -> PyResult
     }
 }
 
+pub fn jit_ret_type(func: &Py<PyFunction>, vm: &VirtualMachine) -> PyResult<Option<JitType>> {
+    let func_obj: PyObjectRef = func.as_ref().to_owned();
+    let annotations = func_obj.get_attr("__annotations__", vm)?;
+    if vm.is_none(&annotations) {
+        Err(new_jit_error(
+            "Jitting function requires return type to have annotations".to_owned(),
+            vm,
+        ))
+    } else if let Ok(dict) = PyDictRef::try_from_object(vm, annotations) {
+        if dict.contains_key("return", vm) {
+            get_jit_arg_type(&dict, "return", vm).map_or(Ok(None), |t| Ok(Some(t)))
+        } else {
+            Ok(None)
+        }
+    } else {
+        Err(vm.new_type_error("Function annotations aren't a dict".to_owned()))
+    }
+}
+
 fn get_jit_value(vm: &VirtualMachine, obj: &PyObject) -> Result<AbiValue, ArgsError> {
     // This does exact type checks as subclasses of int/float can't be passed to jitted functions
     let cls = obj.class();

vm/src/builtins/mappingproxy.rs

@@ -126,7 +126,7 @@ impl PyMappingProxy {
         match &self.mapping {
             MappingProxyInner::Class(class) => Ok(key
                 .as_interned_str(vm)
-                .map_or(false, |key| class.attributes.read().contains_key(key))),
+                .is_some_and(|key| class.attributes.read().contains_key(key))),
             MappingProxyInner::Mapping(mapping) => mapping.to_sequence().contains(key, vm),
         }
     }

vm/src/builtins/str.rs

@@ -903,9 +903,8 @@ impl PyStr {
             let end_len = match ch {
                 '\n' => 1,
                 '\r' => {
-                    let is_rn = enumerated.peek().map_or(false, |(_, ch)| *ch == '\n');
+                    let is_rn = enumerated.next_if(|(_, ch)| *ch == '\n').is_some();
                     if is_rn {
-                        let _ = enumerated.next();
                         2
                     } else {
                         1
@@ -913,9 +912,7 @@ impl PyStr {
                 }
                 '\x0b' | '\x0c' | '\x1c' | '\x1d' | '\x1e' | '\u{0085}' | '\u{2028}'
                 | '\u{2029}' => ch.len_utf8(),
-                _ => {
-                    continue;
-                }
+                _ => continue,
             };
             let range = if args.keepends {
                 last_i..i + end_len
@@ -1160,7 +1157,7 @@ impl PyStr {
     #[pymethod]
     fn isidentifier(&self) -> bool {
         let mut chars = self.as_str().chars();
-        let is_identifier_start = chars.next().map_or(false, |c| c == '_' || is_xid_start(c));
+        let is_identifier_start = chars.next().is_some_and(|c| c == '_' || is_xid_start(c));
         // a string is not an identifier if it has whitespace or starts with a number
         is_identifier_start && chars.all(is_xid_continue)
     }

vm/src/cformat.rs

@@ -296,7 +296,7 @@ pub(crate) fn cformat_bytes(
         return if is_mapping
             || values_obj
                 .payload::<tuple::PyTuple>()
-                .map_or(false, |e| e.is_empty())
+                .is_some_and(|e| e.is_empty())
         {
             for (_, part) in format.iter_mut() {
                 match part {
@@ -397,7 +397,7 @@ pub(crate) fn cformat_string(
         return if is_mapping
             || values_obj
                 .payload::<tuple::PyTuple>()
-                .map_or(false, |e| e.is_empty())
+                .is_some_and(|e| e.is_empty())
         {
             for (_, part) in format.iter() {
                 match part {

vm/src/function/builtin.rs

@@ -55,19 +55,14 @@ const fn zst_ref_out_of_thin_air<T: 'static>(x: T) -> &'static T {
     // would never get called anyway if we consider this semantically a Box::leak(Box::new(x))-type
     // operation. if T isn't zero-sized, we don't have to worry about it because we'll fail to compile.
     std::mem::forget(x);
-    trait Zst: Sized + 'static {
-        const THIN_AIR: &'static Self = {
-            if std::mem::size_of::<Self>() == 0 {
-                // SAFETY: we just confirmed that Self is zero-sized, so we can
-                //         pull a value of it out of thin air.
-                unsafe { std::ptr::NonNull::<Self>::dangling().as_ref() }
-            } else {
-                panic!("can't use a non-zero-sized type here")
-            }
-        };
+    const {
+        if std::mem::size_of::<T>() != 0 {
+            panic!("can't use a non-zero-sized type here")
+        }
+        // SAFETY: we just confirmed that T is zero-sized, so we can
+        //         pull a value of it out of thin air.
+        unsafe { std::ptr::NonNull::<T>::dangling().as_ref() }
     }
-    impl<T: 'static> Zst for T {}
-    <T as Zst>::THIN_AIR
 }
 
 /// Get the [`STATIC_FUNC`](IntoPyNativeFn::STATIC_FUNC) of the passed function. The same

vm/src/intern.rs

@@ -228,12 +228,8 @@ mod sealed {
 }
 
 /// A sealed marker trait for `DictKey` types that always become an exact instance of `str`
-pub trait InternableString
-where
-    Self: sealed::SealedInternable + ToPyObject + AsRef<Self::Interned>,
-    Self::Interned: MaybeInternedString,
-{
-    type Interned: ?Sized;
+pub trait InternableString: sealed::SealedInternable + ToPyObject + AsRef<Self::Interned> {
+    type Interned: MaybeInternedString + ?Sized;
     fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact<PyStr>;
 }
 

vm/src/lib.rs

@@ -13,6 +13,7 @@
 #![allow(clippy::upper_case_acronyms)]
 #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
 #![doc(html_root_url = "https://docs.rs/rustpython-vm/")]
+#![cfg_attr(target_os = "redox", feature(raw_ref_op))]
 
 #[cfg(feature = "flame-it")]
 #[macro_use]

vm/src/object/core.rs

@@ -322,7 +322,7 @@ unsafe impl Link for WeakLink {
 
     #[inline(always)]
     unsafe fn pointers(target: NonNull<Self::Target>) -> NonNull<Pointers<Self::Target>> {
-        NonNull::new_unchecked(ptr::addr_of_mut!((*target.as_ptr()).0.payload.pointers))
+        NonNull::new_unchecked(&raw mut (*target.as_ptr()).0.payload.pointers)
     }
 }
 
@@ -364,8 +364,11 @@ impl PyWeak {
     fn drop_inner(&self) {
         let dealloc = {
             let mut guard = unsafe { self.parent.as_ref().lock() };
-            let offset = memoffset::offset_of!(PyInner<PyWeak>, payload);
-            let pyinner = (self as *const Self as usize - offset) as *const PyInner<Self>;
+            let offset = std::mem::offset_of!(PyInner<PyWeak>, payload);
+            let pyinner = (self as *const Self)
+                .cast::<u8>()
+                .wrapping_sub(offset)
+                .cast::<PyInner<Self>>();
             let node_ptr = unsafe { NonNull::new_unchecked(pyinner as *mut Py<Self>) };
             // the list doesn't have ownership over its PyRef<PyWeak>! we're being dropped
             // right now so that should be obvious!!
@@ -1044,7 +1047,7 @@ impl<T: PyObjectPayload> PyRef<T> {
     pub fn leak(pyref: Self) -> &'static Py<T> {
         let ptr = pyref.ptr;
         std::mem::forget(pyref);
-        unsafe { &*ptr.as_ptr() }
+        unsafe { ptr.as_ref() }
     }
 }
 

vm/src/object/traverse_object.rs

@@ -1,4 +1,4 @@
-use std::{fmt, marker::PhantomData};
+use std::fmt;
 
 use crate::{
     object::{
@@ -16,25 +16,18 @@ pub(in crate::object) struct PyObjVTable {
 }
 
 impl PyObjVTable {
-    pub fn of<T: PyObjectPayload>() -> &'static Self {
-        struct Helper<T: PyObjectPayload>(PhantomData<T>);
-        trait VtableHelper {
-            const VTABLE: PyObjVTable;
+    pub const fn of<T: PyObjectPayload>() -> &'static Self {
+        &PyObjVTable {
+            drop_dealloc: drop_dealloc_obj::<T>,
+            debug: debug_obj::<T>,
+            trace: const {
+                if T::IS_TRACE {
+                    Some(try_trace_obj::<T>)
+                } else {
+                    None
+                }
+            },
         }
-        impl<T: PyObjectPayload> VtableHelper for Helper<T> {
-            const VTABLE: PyObjVTable = PyObjVTable {
-                drop_dealloc: drop_dealloc_obj::<T>,
-                debug: debug_obj::<T>,
-                trace: {
-                    if T::IS_TRACE {
-                        Some(try_trace_obj::<T>)
-                    } else {
-                        None
-                    }
-                },
-            };
-        }
-        &Helper::<T>::VTABLE
     }
 }
 

vm/src/protocol/mapping.rs

@@ -95,7 +95,7 @@ impl PyMapping<'_> {
     // PyMapping::Check
     #[inline]
     pub fn check(obj: &PyObject) -> bool {
-        Self::find_methods(obj).map_or(false, |x| x.as_ref().check())
+        Self::find_methods(obj).is_some_and(|x| x.as_ref().check())
     }
 
     pub fn find_methods(obj: &PyObject) -> Option<PointerSlot<PyMappingMethods>> {

vm/src/stdlib/builtins.rs

@@ -429,8 +429,7 @@ mod builtins {
         let fd_matches = |obj, expected| {
             vm.call_method(obj, "fileno", ())
                 .and_then(|o| i64::try_from_object(vm, o))
-                .ok()
-                .map_or(false, |fd| fd == expected)
+                .is_ok_and(|fd| fd == expected)
         };
 
         // everything is normalish, we can just rely on rustyline to use stdin/stdout

vm/src/stdlib/itertools.rs

@@ -684,7 +684,7 @@ mod decl {
             self.grouper
                 .as_ref()
                 .and_then(|g| g.upgrade())
-                .map_or(false, |ref current_grouper| grouper.is(current_grouper))
+                .is_some_and(|current_grouper| grouper.is(&current_grouper))
         }
     }
 

vm/src/stdlib/nt.rs

@@ -62,7 +62,7 @@ pub(crate) mod module {
                 .as_path()
                 .parent()
                 .and_then(|dst_parent| dst_parent.join(&args.src).symlink_metadata().ok())
-                .map_or(false, |meta| meta.is_dir());
+                .is_some_and(|meta| meta.is_dir());
         let res = if dir {
             win_fs::symlink_dir(args.src.path, args.dst.path)
         } else {

vm/src/stdlib/os.rs

@@ -142,6 +142,7 @@ pub(super) mod _os {
         protocol::PyIterReturn,
         recursion::ReprGuard,
         types::{IterNext, Iterable, PyStructSequence, Representable, SelfIter},
+        utils::ToCString,
         vm::VirtualMachine,
         AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject,
     };
@@ -273,8 +274,8 @@ pub(super) mod _os {
     fn remove(path: OsPath, dir_fd: DirFd<0>, vm: &VirtualMachine) -> PyResult<()> {
         let [] = dir_fd.0;
         let is_junction = cfg!(windows)
-            && fs::metadata(&path).map_or(false, |meta| meta.file_type().is_dir())
-            && fs::symlink_metadata(&path).map_or(false, |meta| meta.file_type().is_symlink());
+            && fs::metadata(&path).is_ok_and(|meta| meta.file_type().is_dir())
+            && fs::symlink_metadata(&path).is_ok_and(|meta| meta.file_type().is_symlink());
         let res = if is_junction {
             fs::remove_dir(&path)
         } else {
@@ -1028,6 +1029,14 @@ pub(super) mod _os {
         })
     }
 
+    #[cfg(any(unix, windows))]
+    #[pyfunction]
+    fn system(command: PyStrRef, vm: &VirtualMachine) -> PyResult<i32> {
+        let cstr = command.to_cstring(vm)?;
+        let x = unsafe { libc::system(cstr.as_ptr()) };
+        Ok(x)
+    }
+
     #[derive(FromArgs)]
     struct UtimeArgs {
         path: OsPath,

vm/src/stdlib/posix.rs

@@ -901,13 +901,6 @@ pub mod module {
         nix::unistd::pipe2(oflags).map_err(|err| err.into_pyexception(vm))
     }
 
-    #[pyfunction]
-    fn system(command: PyStrRef, vm: &VirtualMachine) -> PyResult<i32> {
-        let cstr = command.to_cstring(vm)?;
-        let x = unsafe { libc::system(cstr.as_ptr()) };
-        Ok(x)
-    }
-
     fn _chmod(
         path: OsPath,
         dir_fd: DirFd<0>,

vm/src/stdlib/pwd.rs

@@ -92,7 +92,7 @@ mod pwd {
     #[pyfunction]
     fn getpwall(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> {
         // setpwent, getpwent, etc are not thread safe. Could use fgetpwent_r, but this is easier
-        static GETPWALL: parking_lot::Mutex<()> = parking_lot::const_mutex(());
+        static GETPWALL: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
         let _guard = GETPWALL.lock();
         let mut list = Vec::new();
 

vm/src/stdlib/signal.rs

@@ -11,10 +11,12 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
 
 #[pymodule]
 pub(crate) mod _signal {
+    #[cfg(any(unix, windows))]
     use crate::{
         convert::{IntoPyException, TryFromBorrowedObject},
-        signal, Py, PyObjectRef, PyResult, VirtualMachine,
+        Py,
     };
+    use crate::{signal, PyObjectRef, PyResult, VirtualMachine};
     use std::sync::atomic::{self, Ordering};
 
     #[cfg(any(unix, windows))]
@@ -110,28 +112,28 @@ pub(crate) mod _signal {
         module: &Py<crate::builtins::PyModule>,
         vm: &VirtualMachine,
     ) {
-        let sig_dfl = vm.new_pyobj(SIG_DFL as u8);
-        let sig_ign = vm.new_pyobj(SIG_IGN as u8);
-
-        for signum in 1..NSIG {
-            let handler = unsafe { libc::signal(signum as i32, SIG_IGN) };
-            if handler != SIG_ERR {
-                unsafe { libc::signal(signum as i32, handler) };
-            }
-            let py_handler = if handler == SIG_DFL {
-                Some(sig_dfl.clone())
-            } else if handler == SIG_IGN {
-                Some(sig_ign.clone())
-            } else {
-                None
-            };
-            vm.signal_handlers.as_deref().unwrap().borrow_mut()[signum] = py_handler;
-        }
-
-        let int_handler = module
-            .get_attr("default_int_handler", vm)
-            .expect("_signal does not have this attr?");
         if vm.state.settings.install_signal_handlers {
+            let sig_dfl = vm.new_pyobj(SIG_DFL as u8);
+            let sig_ign = vm.new_pyobj(SIG_IGN as u8);
+
+            for signum in 1..NSIG {
+                let handler = unsafe { libc::signal(signum as i32, SIG_IGN) };
+                if handler != SIG_ERR {
+                    unsafe { libc::signal(signum as i32, handler) };
+                }
+                let py_handler = if handler == SIG_DFL {
+                    Some(sig_dfl.clone())
+                } else if handler == SIG_IGN {
+                    Some(sig_ign.clone())
+                } else {
+                    None
+                };
+                vm.signal_handlers.as_deref().unwrap().borrow_mut()[signum] = py_handler;
+            }
+
+            let int_handler = module
+                .get_attr("default_int_handler", vm)
+                .expect("_signal does not have this attr?");
             signal(libc::SIGINT, int_handler, vm).expect("Failed to set sigint handler");
         }
     }

vm/src/types/slot.rs

@@ -470,7 +470,7 @@ impl PyType {
                     .attributes
                     .read()
                     .get(identifier!(ctx, __hash__))
-                    .map_or(false, |a| a.is(&ctx.none));
+                    .is_some_and(|a| a.is(&ctx.none));
                 let wrapper = if is_unhashable {
                     hash_not_implemented
                 } else {
@@ -945,7 +945,7 @@ pub trait GetDescriptor: PyPayload {
 
     #[inline]
     fn _cls_is(cls: &Option<PyObjectRef>, other: &impl Borrow<PyObject>) -> bool {
-        cls.as_ref().map_or(false, |cls| other.borrow().is(cls))
+        cls.as_ref().is_some_and(|cls| other.borrow().is(cls))
     }
 }
 

vm/src/vm/mod.rs

@@ -133,13 +133,10 @@ impl VirtualMachine {
         let import_func = ctx.none();
         let profile_func = RefCell::new(ctx.none());
         let trace_func = RefCell::new(ctx.none());
-        // hack to get around const array repeat expressions, rust issue #79270
-        const NONE: Option<PyObjectRef> = None;
-        // putting it in a const optimizes better, prevents linear initialization of the array
-        #[allow(clippy::declare_interior_mutable_const)]
-        const SIGNAL_HANDLERS: RefCell<[Option<PyObjectRef>; signal::NSIG]> =
-            RefCell::new([NONE; signal::NSIG]);
-        let signal_handlers = Some(Box::new(SIGNAL_HANDLERS));
+        let signal_handlers = Some(Box::new(
+            // putting it in a const optimizes better, prevents linear initialization of the array
+            const { RefCell::new([const { None }; signal::NSIG]) },
+        ));
 
         let module_inits = stdlib::get_module_inits();
 

vm/src/vm/vm_new.rs

@@ -365,7 +365,15 @@ impl VirtualMachine {
         let actual_class = obj.class();
         let actual_type = &*actual_class.name();
         let expected_type = &*class.name();
-        let msg = format!("Expected {msg} '{expected_type}' but '{actual_type}' found");
+        let msg = format!("Expected {msg} '{expected_type}' but '{actual_type}' found.");
+        #[cfg(debug_assertions)]
+        let msg = if class.get_id() == actual_class.get_id() {
+            let mut msg = msg;
+            msg += " Did you forget to add `#[pyclass(with(Constructor))]`?";
+            msg
+        } else {
+            msg
+        };
         self.new_exception_msg(error_type.to_owned(), msg)
     }
 
@@ -394,4 +402,9 @@ impl VirtualMachine {
             obj.as_object(),
         )
     }
+
+    pub fn new_eof_error(&self, msg: String) -> PyBaseExceptionRef {
+        let eof_error = self.ctx.exceptions.eof_error.to_owned();
+        self.new_exception_msg(eof_error, msg)
+    }
 }

vm/src/vm/vm_ops.rs

@@ -298,8 +298,8 @@ impl VirtualMachine {
         }
 
         if let Some(slot_c) = class_c.slots.as_number.left_ternary_op(op_slot) {
-            if slot_a.map_or(false, |slot_a| (slot_a as usize) != (slot_c as usize))
-                && slot_b.map_or(false, |slot_b| (slot_b as usize) != (slot_c as usize))
+            if slot_a.is_some_and(|slot_a| slot_a != slot_c)
+                && slot_b.is_some_and(|slot_b| slot_b != slot_c)
             {
                 let ret = slot_c(a, b, c, self)?;
                 if !ret.is(&self.ctx.not_implemented) {

vm/src/warn.rs

@@ -367,7 +367,7 @@ fn setup_context(
 
     // Stack level comparisons to Python code is off by one as there is no
     // warnings-related stack level to avoid.
-    if stack_level <= 0 || f.as_ref().map_or(false, |frame| frame.is_internal_frame()) {
+    if stack_level <= 0 || f.as_ref().is_some_and(|frame| frame.is_internal_frame()) {
         loop {
             stack_level -= 1;
             if stack_level <= 0 {

wapm.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/RustPython/RustPython"
 
 [[module]]
 name = "rustpython"
-source = "target/wasm32-wasi/release/rustpython.wasm"
+source = "target/wasm32-wasip1/release/rustpython.wasm"
 abi = "wasi"
 
 [[command]]

wasm/demo/package.json

@@ -4,24 +4,25 @@
     "description": "Bindings to the RustPython library for WebAssembly",
     "main": "index.js",
     "dependencies": {
-        "codemirror": "^5.42.0",
-        "local-echo": "^0.2.0",
-        "xterm": "^3.8.0"
+        "@codemirror/lang-python": "^6.1.6",
+        "@xterm/addon-fit": "^0.10.0",
+        "@xterm/xterm": "^5.3.0",
+        "codemirror": "^6.0.1",
+        "upgrade": "^1.1.0",
+        "xterm-readline": "^1.1.2"
     },
     "devDependencies": {
-        "@wasm-tool/wasm-pack-plugin": "^1.1.0",
-        "clean-webpack-plugin": "^3.0.0",
-        "css-loader": "^3.4.1",
-        "html-webpack-plugin": "^3.2.0",
-        "mini-css-extract-plugin": "^0.9.0",
-        "raw-loader": "^4.0.0",
-        "serve": "^11.0.2",
-        "webpack": "^4.16.3",
-        "webpack-cli": "^3.1.0",
-        "webpack-dev-server": "^3.1.5"
+        "@wasm-tool/wasm-pack-plugin": "^1.7.0",
+        "css-loader": "^7.1.2",
+        "html-webpack-plugin": "^5.6.3",
+        "mini-css-extract-plugin": "^2.9.2",
+        "serve": "^14.2.4",
+        "webpack": "^5.97.1",
+        "webpack-cli": "^6.0.1",
+        "webpack-dev-server": "^5.2.0"
     },
     "scripts": {
-        "dev": "webpack-dev-server -d",
+        "dev": "webpack serve",
         "build": "webpack",
         "dist": "webpack --mode production",
         "test": "webpack --mode production && cd ../tests && pytest"

wasm/demo/snippets/import_pypi.py

@@ -1,13 +1,12 @@
-import asyncweb
-import whlimport
-
-whlimport.setup()
-
 # make sys.modules['os'] a dumb version of the os module, which has posixpath
 # available as os.path as well as a few other utilities, but will raise an
 # OSError for anything that actually requires an OS
 import _dummy_os
-_dummy_os._shim()
+
+import asyncweb
+import whlimport
+
+whlimport.setup()
 
 @asyncweb.main
 async def main():

wasm/demo/src/index.ejs

@@ -14,7 +14,6 @@
             browser's devtools and play with <code>rp.pyEval('1 + 1')</code>
         </p>
         <div id="code-wrapper">
-            <textarea id="code"><%= defaultSnippet %></textarea>
             <select id="snippets">
                 <% for (const name of snippets) { %>
                     <option
@@ -77,7 +76,7 @@
         <a href="https://github.com/RustPython/RustPython">
             <img
                 style="position: absolute; top: 0; right: 0; border: 0;"
-                src="https://s3.amazonaws.com/github/ribbons/forkme_right_green_007200.png"
+                src="https://github.blog/wp-content/uploads/2008/12/forkme_right_green_007200.png"
                 alt="Fork me on GitHub"
             />
         </a>

wasm/demo/src/index.js

@@ -1,11 +1,13 @@
 import './style.css';
-import 'xterm/lib/xterm.css';
-import CodeMirror from 'codemirror';
-import 'codemirror/mode/python/python';
-import 'codemirror/addon/comment/comment';
-import 'codemirror/lib/codemirror.css';
-import { Terminal } from 'xterm';
-import LocalEchoController from 'local-echo';
+import '@xterm/xterm/css/xterm.css';
+import { EditorView, basicSetup } from 'codemirror';
+import { keymap } from '@codemirror/view';
+import { indentUnit } from '@codemirror/language';
+import { indentWithTab } from '@codemirror/commands';
+import { python } from '@codemirror/lang-python';
+import { Terminal } from '@xterm/xterm';
+import { FitAddon } from '@xterm/addon-fit';
+import { Readline } from 'xterm-readline';
 
 let rp;
 
@@ -22,23 +24,24 @@ import('rustpython')
         document.getElementById('error').textContent = e;
     });
 
-const editor = CodeMirror.fromTextArea(document.getElementById('code'), {
-    extraKeys: {
-        'Ctrl-Enter': runCodeFromTextarea,
-        'Cmd-Enter': runCodeFromTextarea,
-        'Shift-Tab': 'indentLess',
-        'Ctrl-/': 'toggleComment',
-        'Cmd-/': 'toggleComment',
-        Tab: (editor) => {
-            var spaces = Array(editor.getOption('indentUnit') + 1).join(' ');
-            editor.replaceSelection(spaces);
-        },
-    },
-    lineNumbers: true,
-    mode: 'text/x-python',
-    indentUnit: 4,
-    autofocus: true,
+const fixedHeightEditor = EditorView.theme({
+    '&': { height: '100%' },
+    '.cm-scroller': { overflow: 'auto' },
 });
+const editor = new EditorView({
+    parent: document.getElementById('code-wrapper'),
+    extensions: [
+        basicSetup,
+        python(),
+        keymap.of(
+            { key: 'Ctrl-Enter', mac: 'Cmd-Enter', run: runCodeFromTextarea },
+            indentWithTab,
+        ),
+        indentUnit.of('    '),
+        fixedHeightEditor,
+    ],
+});
+editor.focus();
 
 const consoleElement = document.getElementById('console');
 const errorElement = document.getElementById('error');
@@ -48,7 +51,7 @@ function runCodeFromTextarea() {
     consoleElement.value = '';
     errorElement.textContent = '';
 
-    const code = editor.getValue();
+    const code = editor.state.doc.toString();
     try {
         rp.pyExec(code, {
             stdout: (output) => {
@@ -78,18 +81,25 @@ function updateSnippet() {
     // the require here creates a webpack context; it's fine to use it
     // dynamically.
     // https://webpack.js.org/guides/dependency-management/
-    const { default: snippet } = require(
-        `raw-loader!../snippets/${selected}.py`,
-    );
+    const snippet = require(`../snippets/${selected}.py?raw`);
 
-    editor.setValue(snippet);
-    runCodeFromTextarea();
+    editor.dispatch({
+        changes: { from: 0, to: editor.state.doc.length, insert: snippet },
+    });
 }
+function updateSnippetAndRun() {
+    updateSnippet();
+    requestAnimationFrame(runCodeFromTextarea);
+}
+updateSnippet();
 
 const term = new Terminal();
+const readline = new Readline();
+const fitAddon = new FitAddon();
+term.loadAddon(readline);
+term.loadAddon(fitAddon);
 term.open(document.getElementById('terminal'));
-
-const localEcho = new LocalEchoController(term);
+fitAddon.fit();
 
 let terminalVM;
 
@@ -107,40 +117,33 @@ finally:
 }
 
 async function readPrompts() {
-    let continuing = false;
+    let continuing = '';
 
     while (true) {
-        const ps1 = getPrompt('ps1');
-        const ps2 = getPrompt('ps2');
-        let input;
+        let input = await readline.read(getPrompt(continuing ? 'ps2' : 'ps1'));
+        if (input.endsWith('\n')) input = input.slice(0, -1);
         if (continuing) {
-            const prom = localEcho.read(ps2, ps2);
-            localEcho._activePrompt.prompt = ps1;
-            localEcho._input = localEcho.history.entries.pop() + '\n';
-            localEcho._cursor = localEcho._input.length;
-            localEcho._active = true;
-            input = await prom;
-            if (!input.endsWith('\n')) continue;
-        } else {
-            input = await localEcho.read(ps1, ps2);
+            input = continuing += '\n' + input;
+            if (!continuing.endsWith('\n')) continue;
         }
         try {
+            console.log([input]);
             terminalVM.execSingle(input);
         } catch (err) {
             if (err.canContinue) {
-                continuing = true;
+                continuing = input;
                 continue;
             } else if (err instanceof WebAssembly.RuntimeError) {
                 err = window.__RUSTPYTHON_ERROR || err;
             }
-            localEcho.println(err);
+            readline.print('' + err);
         }
-        continuing = false;
+        continuing = '';
     }
 }
 
 function onReady() {
-    snippets.addEventListener('change', updateSnippet);
+    snippets.addEventListener('change', updateSnippetAndRun);
     document
         .getElementById('run-btn')
         .addEventListener('click', runCodeFromTextarea);
@@ -148,7 +151,7 @@ function onReady() {
     runCodeFromTextarea();
 
     terminalVM = rp.vmStore.init('term_vm');
-    terminalVM.setStdout((data) => localEcho.print(data));
+    terminalVM.setStdout((data) => readline.print(data));
     readPrompts().catch((err) => console.error(err));
 
     // so that the test knows that we're ready

wasm/demo/src/style.css

@@ -3,7 +3,7 @@ textarea {
     resize: vertical;
 }
 
-#code,
+#code-wrapper,
 #console {
     height: 30vh;
     width: calc(100% - 3px);

wasm/demo/webpack.config.js

@@ -1,7 +1,6 @@
 const HtmlWebpackPlugin = require('html-webpack-plugin');
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 const WasmPackPlugin = require('@wasm-tool/wasm-pack-plugin');
-const { CleanWebpackPlugin } = require('clean-webpack-plugin');
 
 const path = require('path');
 const fs = require('fs');
@@ -12,6 +11,7 @@ module.exports = (env = {}) => {
         output: {
             path: path.join(__dirname, 'dist'),
             filename: 'index.js',
+            clean: true,
         },
         mode: 'development',
         resolve: {
@@ -28,10 +28,17 @@ module.exports = (env = {}) => {
                     test: /\.css$/,
                     use: [MiniCssExtractPlugin.loader, 'css-loader'],
                 },
+                {
+                    resourceQuery: '?raw',
+                    type: 'asset/source',
+                },
+                {
+                    test: /\.wasm$/,
+                    type: 'webassembly/async',
+                },
             ],
         },
         plugins: [
-            new CleanWebpackPlugin(),
             new HtmlWebpackPlugin({
                 filename: 'index.html',
                 template: 'src/index.ejs',
@@ -51,6 +58,9 @@ module.exports = (env = {}) => {
                 filename: 'styles.css',
             }),
         ],
+        experiments: {
+            asyncWebAssembly: true,
+        },
     };
     if (!env.noWasmPack) {
         config.plugins.push(

wasm/lib/src/convert.rs

@@ -3,7 +3,7 @@
 use crate::js_module;
 use crate::vm_class::{stored_vm_from_wasm, WASMVirtualMachine};
 use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array};
-use rustpython_parser::ParseErrorType;
+use rustpython_parser::{lexer::LexicalErrorType, ParseErrorType};
 use rustpython_vm::{
     builtins::PyBaseExceptionRef,
     compiler::{CompileError, CompileErrorType},
@@ -34,7 +34,7 @@ extern "C" {
 
 pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyBaseExceptionRef) -> JsValue {
     let jserr = vm.try_class("_js", "JSError").ok();
-    let js_arg = if jserr.map_or(false, |jserr| py_err.fast_isinstance(&jserr)) {
+    let js_arg = if jserr.is_some_and(|jserr| py_err.fast_isinstance(&jserr)) {
         py_err.get_arg(0)
     } else {
         None
@@ -258,7 +258,15 @@ pub fn syntax_err(err: CompileError) -> SyntaxError {
         &"col".into(),
         &(err.location.unwrap().column.get()).into(),
     );
-    let can_continue = matches!(&err.error, CompileErrorType::Parse(ParseErrorType::Eof));
+    let can_continue = matches!(
+        &err.error,
+        CompileErrorType::Parse(
+            ParseErrorType::Eof
+                | ParseErrorType::Lexical(LexicalErrorType::Eof)
+                | ParseErrorType::Lexical(LexicalErrorType::IndentationError)
+                | ParseErrorType::UnrecognizedToken(rustpython_parser::Tok::Dedent, _)
+        )
+    );
     let _ = Reflect::set(&js_err, &"canContinue".into(), &can_continue.into());
     js_err
 }