Simplify unicode regex call sites

Agent-Logs-Url: https://github.com/RustPython/RustPython/sessions/b894057a-9bed-4f35-8400-a5731c63602d

Co-authored-by: youknowone <69878+youknowone@users.noreply.github.com>

.github/workflows/ci.yaml

@@ -8,6 +8,9 @@ on:
 
 name: CI
 
+permissions:
+  contents: read
+
 # Cancel previous workflows if they are the same workflow on same ref (branch/tags)
 # with the same event (push/pull_request) even they are in progress.
 # This setting will help reduce the number of duplicated workflows.
@@ -27,6 +30,8 @@ env:
   PYTHON_VERSION: "3.14.3"
   X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD
   X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include
+  CARGO_INCREMENTAL: 0
+  CARGO_TERM_COLOR: always
 
 jobs:
   rust_tests:
@@ -45,10 +50,9 @@ jobs:
         with:
           persist-credentials: false
 
-      - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9
+      - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
-          toolchain: stable
 
       - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
         with:
@@ -108,41 +112,39 @@ jobs:
 
   cargo_check:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    name: Ensure compilation on various targets
+    name: cargo check
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         include: 
         - os: ubuntu-latest
-          targets:
-            - aarch64-linux-android
-            - i686-unknown-linux-gnu
-            - i686-unknown-linux-musl
-            - wasm32-wasip2
-            - x86_64-unknown-freebsd
+          target: aarch64-linux-android
+        - os: ubuntu-latest
+          target: i686-unknown-linux-gnu
           dependencies:
             gcc-multilib: true
+        - os: ubuntu-latest
+          target: i686-unknown-linux-musl
+          dependencies:
             musl-tools: true
         - os: ubuntu-latest
-          targets:
-            - aarch64-unknown-linux-gnu
+          target: wasm32-wasip2
+        - os: ubuntu-latest
+          target: x86_64-unknown-freebsd
+        - os: ubuntu-latest
+          target: aarch64-unknown-linux-gnu
           dependencies:
-            gcc-aarch64-linux-gnu: true # conflict with `gcc-multilib`
+            gcc-aarch64-linux-gnu: true
         - os: macos-latest
-          targets:
-            - aarch64-apple-ios
-            - x86_64-apple-darwin
+          target: aarch64-apple-ios
+        - os: macos-latest
+          target: x86_64-apple-darwin
       fail-fast: false
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
-      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
-        with:
-          prefix-key: v0-rust-${{ join(matrix.targets, '-') }}
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-
       - name: Install dependencies
         uses: ./.github/actions/install-linux-deps
         # zizmor has an issue with dynamic `with`
@@ -152,13 +154,27 @@ jobs:
           musl-tools: ${{ matrix.dependencies.musl-tools || false }}
           gcc-aarch64-linux-gnu: ${{ matrix.dependencies.gcc-aarch64-linux-gnu || false }}
 
-      - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9
+      - name: Restore cache
+        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        if: ${{ github.ref != 'refs/heads/main' }} # Never restore on main
         with:
-          targets: ${{ join(matrix.targets, ',') }}
-          toolchain: stable
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          # key won't match, will rely on restore-keys
+          key: cargo-check-${{ runner.os }}-${{ matrix.target }}
+          restore-keys: |
+            cargo-check-${{ runner.os }}-${{ matrix.target }}-
+
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          target: ${{ matrix.target }}
 
       - name: Setup Android NDK
-        if: ${{ contains(matrix.targets, 'aarch64-linux-android') }}
+        if: ${{ matrix.target == 'aarch64-linux-android' }}
         id: setup-ndk
         uses: nttld/setup-ndk@v1
         with:
@@ -174,18 +190,24 @@ jobs:
       #     args: --ignore-rust-version
 
       - name: Check compilation
-        run: |
-          for target in ${{ join(matrix.targets, ' ') }}
-          do
-            echo "::group::${target}"
-            cargo check --target $target ${{ env.CARGO_ARGS_NO_SSL }}
-            echo "::endgroup::"
-          done
+        run: cargo check --target "${{ matrix.target }}" ${{ env.CARGO_ARGS_NO_SSL }}
         env:
           CC_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang
           AR_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar
           CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang
 
+      - name: Save cache
+        if: ${{ github.ref == 'refs/heads/main' }} # only save on main
+        uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: cargo-check-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }}-${{ github.sha }}
+
   snippets_cpython:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     env:
@@ -230,9 +252,7 @@ jobs:
         with:
           persist-credentials: false
 
-      - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9
-        with:
-          toolchain: stable
+      - uses: dtolnay/rust-toolchain@stable
 
       - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
         with:
@@ -345,9 +365,8 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9
+      - uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: stable
           components: rustfmt
 
       - uses: cargo-bins/cargo-binstall@113a77a4ce971c41332f2129c3d995df993cf746 # v1.17.8
@@ -372,7 +391,7 @@ jobs:
 
       - name: prek
         id: prek
-        uses: j178/prek-action@79f765515bd648eb4d6bb1b17277b7cb22cb6468 # v2.0.0
+        uses: j178/prek-action@53276d8b0d10f8b6672aa85b4588c6921d0370cc # v2.0.1
         with:
           cache: false
           show-verbose-logs: false
@@ -404,7 +423,7 @@ jobs:
         with:
           persist-credentials: false
 
-      - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9
+      - uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ env.NIGHTLY_CHANNEL }}
           components: miri
@@ -430,10 +449,9 @@ jobs:
         with:
           persist-credentials: false
 
-      - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9
+      - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
-          toolchain: stable
 
       - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
         with:
@@ -508,10 +526,9 @@ jobs:
         with:
           persist-credentials: false
 
-      - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9
+      - uses: dtolnay/rust-toolchain@stable
         with:
           target: wasm32-wasip1
-          toolchain: stable
 
       - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
         with:

.github/workflows/comment-commands.yml

@@ -18,4 +18,6 @@ jobs:
     steps:
       # Using REST API and not `gh issue edit`. https://github.com/cli/cli/issues/6235#issuecomment-1243487651
       - run: |
-          curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
+          curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ env.USER }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
+        env:
+          USER:  ${{ github.event.comment.user.login }}

.github/workflows/release.yml

@@ -12,9 +12,6 @@ on:
         required: false
         default: true
 
-permissions:
-  contents: write
-
 env:
   X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD
   X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include
@@ -115,8 +112,9 @@ jobs:
         with:
           package-manager-cache: false
 
-      - uses: mwilliamson/setup-wabt-action@v3
-        with: { wabt-version: "1.0.30" }
+      - uses: mwilliamson/setup-wabt-action@febe2a12b7ccb999a6e5d953a8362a3b7ffcf148 # v3.2.0
+        with:
+          wabt-version: "1.0.30"
 
       - name: build demo
         run: |
@@ -137,7 +135,7 @@ jobs:
 
       - name: Deploy demo to Github Pages
         if: ${{ github.repository == 'RustPython/RustPython' }}
-        uses: peaceiris/actions-gh-pages@v4
+        uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0
         with:
           deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
           publish_dir: ./wasm/demo/dist

.github/zizmor.yml

@@ -0,0 +1,14 @@
+rules:
+  unpinned-uses:
+    config:
+      policies:
+        # dtolnay/rust-toolchain is a trusted action that uses lightweight branch
+        # refs (@stable, @nightly, etc.) by design. Pinning to a hash would break
+        # the intended usage pattern.
+        # We can remove this once https://github.com/dtolnay/rust-toolchain/issues/180 is resolved
+        dtolnay/rust-toolchain: any
+  # dtolnay/rust-toolchain handles component installation, target addition, and
+  # override configuration beyond what a bare `rustup` invocation provides.
+  # See: https://github.com/zizmorcore/zizmor/issues/1817
+  superfluous-actions:
+    disable: true

Cargo.lock

@@ -79,9 +79,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
 
 [[package]]
 name = "anstream"
-version = "0.6.21"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -100,9 +100,9 @@ checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.7"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
 dependencies = [
  "utf8parse",
 ]
@@ -1143,9 +1143,9 @@ dependencies = [
 
 [[package]]
 name = "env_logger"
-version = "0.11.9"
+version = "0.11.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d"
+checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1503,6 +1503,90 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "icu_collections"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"
+dependencies = [
+ "displaydoc",
+ "potential_utf",
+ "yoke",
+ "zerofrom",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_locale_core"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"
+dependencies = [
+ "displaydoc",
+ "litemap",
+ "tinystr",
+ "writeable",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_normalizer"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"
+dependencies = [
+ "icu_collections",
+ "icu_normalizer_data",
+ "icu_properties",
+ "icu_provider",
+ "smallvec",
+ "utf16_iter",
+ "utf8_iter",
+ "write16",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_normalizer_data"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
+
+[[package]]
+name = "icu_properties"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
+dependencies = [
+ "icu_collections",
+ "icu_locale_core",
+ "icu_properties_data",
+ "icu_provider",
+ "zerotrie",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_properties_data"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
+
+[[package]]
+name = "icu_provider"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"
+dependencies = [
+ "displaydoc",
+ "icu_locale_core",
+ "writeable",
+ "yoke",
+ "zerofrom",
+ "zerotrie",
+ "zerovec",
+]
+
 [[package]]
 name = "indexmap"
 version = "2.13.0"
@@ -1585,9 +1669,9 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
 
 [[package]]
 name = "jiff"
-version = "0.2.18"
+version = "0.2.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50"
+checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359"
 dependencies = [
  "jiff-static",
  "log",
@@ -1598,9 +1682,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-static"
-version = "0.2.18"
+version = "0.2.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78"
+checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1824,6 +1908,12 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
 
+[[package]]
+name = "litemap"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
+
 [[package]]
 name = "lock_api"
 version = "0.4.14"
@@ -1944,12 +2034,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
 
-[[package]]
-name = "matches"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
-
 [[package]]
 name = "md-5"
 version = "0.10.6"
@@ -2479,6 +2563,15 @@ dependencies = [
  "portable-atomic",
 ]
 
+[[package]]
+name = "potential_utf"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77"
+dependencies = [
+ "zerovec",
+]
+
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -2877,72 +2970,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "ruff_python_ast"
-version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?rev=e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675#e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675"
-dependencies = [
- "aho-corasick",
- "bitflags 2.11.0",
- "compact_str",
- "get-size2",
- "is-macro",
- "memchr",
- "ruff_python_trivia",
- "ruff_source_file",
- "ruff_text_size",
- "rustc-hash",
- "thiserror 2.0.18",
-]
-
-[[package]]
-name = "ruff_python_parser"
-version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?rev=e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675#e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675"
-dependencies = [
- "bitflags 2.11.0",
- "bstr",
- "compact_str",
- "get-size2",
- "memchr",
- "ruff_python_ast",
- "ruff_python_trivia",
- "ruff_text_size",
- "rustc-hash",
- "static_assertions",
- "unicode-ident",
- "unicode-normalization",
- "unicode_names2 1.3.0",
-]
-
-[[package]]
-name = "ruff_python_trivia"
-version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?rev=e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675#e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675"
-dependencies = [
- "itertools 0.14.0",
- "ruff_source_file",
- "ruff_text_size",
- "unicode-ident",
-]
-
-[[package]]
-name = "ruff_source_file"
-version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?rev=e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675#e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675"
-dependencies = [
- "memchr",
- "ruff_text_size",
-]
-
-[[package]]
-name = "ruff_text_size"
-version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?rev=e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675#e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675"
-dependencies = [
- "get-size2",
-]
-
 [[package]]
 name = "rustc-hash"
 version = "2.1.1"
@@ -3068,9 +3095,9 @@ dependencies = [
  "libc",
  "log",
  "pyo3",
- "ruff_python_parser",
  "rustpython-compiler",
  "rustpython-pylib",
+ "rustpython-ruff_python_parser",
  "rustpython-stdlib",
  "rustpython-vm",
  "rustyline",
@@ -3091,14 +3118,14 @@ dependencies = [
  "memchr",
  "num-complex",
  "num-traits",
- "ruff_python_ast",
- "ruff_python_parser",
- "ruff_text_size",
  "rustpython-compiler-core",
  "rustpython-literal",
+ "rustpython-ruff_python_ast",
+ "rustpython-ruff_python_parser",
+ "rustpython-ruff_text_size",
+ "rustpython-unicode",
  "rustpython-wtf8",
  "thiserror 2.0.18",
- "unicode_names2 2.0.0",
 ]
 
 [[package]]
@@ -3121,9 +3148,9 @@ dependencies = [
  "parking_lot",
  "radium",
  "rustpython-literal",
+ "rustpython-unicode",
  "rustpython-wtf8",
  "siphasher",
- "unicode_names2 2.0.0",
  "widestring",
  "windows-sys 0.61.2",
 ]
@@ -3132,12 +3159,12 @@ dependencies = [
 name = "rustpython-compiler"
 version = "0.5.0"
 dependencies = [
- "ruff_python_ast",
- "ruff_python_parser",
- "ruff_source_file",
- "ruff_text_size",
  "rustpython-codegen",
  "rustpython-compiler-core",
+ "rustpython-ruff_python_ast",
+ "rustpython-ruff_python_parser",
+ "rustpython-ruff_source_file",
+ "rustpython-ruff_text_size",
  "thiserror 2.0.18",
 ]
 
@@ -3151,16 +3178,16 @@ dependencies = [
  "lz4_flex",
  "malachite-bigint",
  "num-complex",
- "ruff_source_file",
+ "rustpython-ruff_source_file",
  "rustpython-wtf8",
 ]
 
 [[package]]
 name = "rustpython-compiler-source"
-version = "0.5.0+deprecated"
+version = "0.4.1+deprecated"
 dependencies = [
- "ruff_source_file",
- "ruff_text_size",
+ "rustpython-ruff_source_file",
+ "rustpython-ruff_text_size",
 ]
 
 [[package]]
@@ -3219,8 +3246,8 @@ dependencies = [
  "lexical-parse-float",
  "num-traits",
  "rand 0.9.2",
+ "rustpython-unicode",
  "rustpython-wtf8",
- "unic-ucd-category",
 ]
 
 [[package]]
@@ -3232,6 +3259,77 @@ dependencies = [
  "rustpython-derive",
 ]
 
+[[package]]
+name = "rustpython-ruff_python_ast"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f021ff72cabf5e2cd6d8ec8813d376a8445a228dc610ab56c27bd9054cda70d4"
+dependencies = [
+ "aho-corasick",
+ "bitflags 2.11.0",
+ "compact_str",
+ "get-size2",
+ "is-macro",
+ "memchr",
+ "rustc-hash",
+ "rustpython-ruff_python_trivia",
+ "rustpython-ruff_source_file",
+ "rustpython-ruff_text_size",
+ "thiserror 2.0.18",
+]
+
+[[package]]
+name = "rustpython-ruff_python_parser"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01e6ee78bd9671fb5766664b2695fe1f2a92a961f4d9101646c570d8acdb1e0b"
+dependencies = [
+ "bitflags 2.11.0",
+ "bstr",
+ "compact_str",
+ "get-size2",
+ "memchr",
+ "rustc-hash",
+ "rustpython-ruff_python_ast",
+ "rustpython-ruff_python_trivia",
+ "rustpython-ruff_text_size",
+ "static_assertions",
+ "unicode-ident",
+ "unicode-normalization",
+ "unicode_names2 1.3.0",
+]
+
+[[package]]
+name = "rustpython-ruff_python_trivia"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79e7cfd1056f3a02ff0d2d0e4474286ca963260782f878b7b81c1dd87432e682"
+dependencies = [
+ "itertools 0.14.0",
+ "rustpython-ruff_source_file",
+ "rustpython-ruff_text_size",
+ "unicode-ident",
+]
+
+[[package]]
+name = "rustpython-ruff_source_file"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "948107aad62ddb12a11fc7bf68a49e52a0b0a3737d415a2505e54f5a9edac737"
+dependencies = [
+ "memchr",
+ "rustpython-ruff_text_size",
+]
+
+[[package]]
+name = "rustpython-ruff_text_size"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8291ee0f5a779e54ccd4e0151a0c426f8b49a123f99b5b6545db17ccdd4277aa"
+dependencies = [
+ "get-size2",
+]
+
 [[package]]
 name = "rustpython-sre_engine"
 version = "0.5.0"
@@ -3240,6 +3338,7 @@ dependencies = [
  "criterion",
  "num_enum",
  "optional",
+ "rustpython-unicode",
  "rustpython-wtf8",
 ]
 
@@ -3300,10 +3399,6 @@ dependencies = [
  "pkcs8",
  "pymath",
  "rand_core 0.9.5",
- "ruff_python_ast",
- "ruff_python_parser",
- "ruff_source_file",
- "ruff_text_size",
  "rustix",
  "rustls",
  "rustls-native-certs",
@@ -3311,6 +3406,11 @@ dependencies = [
  "rustls-platform-verifier",
  "rustpython-common",
  "rustpython-derive",
+ "rustpython-ruff_python_ast",
+ "rustpython-ruff_python_parser",
+ "rustpython-ruff_source_file",
+ "rustpython-ruff_text_size",
+ "rustpython-unicode",
  "rustpython-vm",
  "schannel",
  "sha-1",
@@ -3321,14 +3421,6 @@ dependencies = [
  "tcl-sys",
  "termios",
  "tk-sys",
- "ucd",
- "unic-char-property",
- "unic-normal",
- "unic-ucd-age",
- "unic-ucd-bidi",
- "unic-ucd-category",
- "unicode-bidi-mirroring",
- "unicode_names2 2.0.0",
  "uuid",
  "webpki-roots",
  "widestring",
@@ -3338,6 +3430,21 @@ dependencies = [
  "xml",
 ]
 
+[[package]]
+name = "rustpython-unicode"
+version = "0.5.0"
+dependencies = [
+ "caseless",
+ "icu_normalizer",
+ "icu_properties",
+ "itertools 0.14.0",
+ "rustpython-wtf8",
+ "ucd",
+ "unic-ucd-age",
+ "unicode-casing",
+ "unicode_names2 2.0.0",
+]
+
 [[package]]
 name = "rustpython-venvlauncher"
 version = "0.5.0"
@@ -3350,7 +3457,6 @@ dependencies = [
  "ascii",
  "bitflags 2.11.0",
  "bstr",
- "caseless",
  "cfg-if",
  "chrono",
  "constant_time_eq",
@@ -3384,9 +3490,6 @@ dependencies = [
  "paste",
  "psm",
  "result-like",
- "ruff_python_ast",
- "ruff_python_parser",
- "ruff_text_size",
  "rustix",
  "rustpython-codegen",
  "rustpython-common",
@@ -3395,7 +3498,11 @@ dependencies = [
  "rustpython-derive",
  "rustpython-jit",
  "rustpython-literal",
+ "rustpython-ruff_python_ast",
+ "rustpython-ruff_python_parser",
+ "rustpython-ruff_text_size",
  "rustpython-sre_engine",
+ "rustpython-unicode",
  "rustyline",
  "scoped-tls",
  "scopeguard",
@@ -3406,9 +3513,6 @@ dependencies = [
  "thiserror 2.0.18",
  "timsort",
  "uname",
- "unic-ucd-bidi",
- "unic-ucd-category",
- "unic-ucd-ident",
  "unicode-casing",
  "wasm-bindgen",
  "which",
@@ -3506,9 +3610,9 @@ dependencies = [
 
 [[package]]
 name = "schannel"
-version = "0.1.28"
+version = "0.1.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
 dependencies = [
  "windows-sys 0.61.2",
 ]
@@ -3959,6 +4063,16 @@ version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "639ce8ef6d2ba56be0383a94dd13b92138d58de44c62618303bb798fa92bdc00"
 
+[[package]]
+name = "tinystr"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869"
+dependencies = [
+ "displaydoc",
+ "zerovec",
+]
+
 [[package]]
 name = "tinytemplate"
 version = "1.2.1"
@@ -4101,15 +4215,6 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
 
-[[package]]
-name = "unic-normal"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f09d64d33589a94628bc2aeb037f35c2e25f3f049c7348b5aa5580b48e6bba62"
-dependencies = [
- "unic-ucd-normal",
-]
-
 [[package]]
 name = "unic-ucd-age"
 version = "0.9.0"
@@ -4121,61 +4226,6 @@ dependencies = [
  "unic-ucd-version",
 ]
 
-[[package]]
-name = "unic-ucd-bidi"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1d568b51222484e1f8209ce48caa6b430bf352962b877d592c29ab31fb53d8c"
-dependencies = [
- "unic-char-property",
- "unic-char-range",
- "unic-ucd-version",
-]
-
-[[package]]
-name = "unic-ucd-category"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b8d4591f5fcfe1bd4453baaf803c40e1b1e69ff8455c47620440b46efef91c0"
-dependencies = [
- "matches",
- "unic-char-property",
- "unic-char-range",
- "unic-ucd-version",
-]
-
-[[package]]
-name = "unic-ucd-hangul"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb1dc690e19010e1523edb9713224cba5ef55b54894fe33424439ec9a40c0054"
-dependencies = [
- "unic-ucd-version",
-]
-
-[[package]]
-name = "unic-ucd-ident"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987"
-dependencies = [
- "unic-char-property",
- "unic-char-range",
- "unic-ucd-version",
-]
-
-[[package]]
-name = "unic-ucd-normal"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86aed873b8202d22b13859dda5fe7c001d271412c31d411fd9b827e030569410"
-dependencies = [
- "unic-char-property",
- "unic-char-range",
- "unic-ucd-hangul",
- "unic-ucd-version",
-]
-
 [[package]]
 name = "unic-ucd-version"
 version = "0.9.0"
@@ -4185,12 +4235,6 @@ dependencies = [
  "unic-common",
 ]
 
-[[package]]
-name = "unicode-bidi-mirroring"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dfa6e8c60bb66d49db113e0125ee8711b7647b5579dc7f5f19c42357ed039fe"
-
 [[package]]
 name = "unicode-casing"
 version = "0.1.1"
@@ -4278,6 +4322,18 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
+[[package]]
+name = "utf16_iter"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
+
+[[package]]
+name = "utf8_iter"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+
 [[package]]
 name = "utf8parse"
 version = "0.2.2"
@@ -4809,6 +4865,18 @@ version = "0.51.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
 
+[[package]]
+name = "write16"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
+
+[[package]]
+name = "writeable"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
+
 [[package]]
 name = "x509-cert"
 version = "0.2.5"
@@ -4846,6 +4914,29 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a"
 
+[[package]]
+name = "yoke"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954"
+dependencies = [
+ "stable_deref_trait",
+ "yoke-derive",
+ "zerofrom",
+]
+
+[[package]]
+name = "yoke-derive"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "synstructure",
+]
+
 [[package]]
 name = "zerocopy"
 version = "0.8.34"
@@ -4866,6 +4957,27 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "zerofrom"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
+dependencies = [
+ "zerofrom-derive",
+]
+
+[[package]]
+name = "zerofrom-derive"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "synstructure",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.8.2"
@@ -4886,6 +4998,39 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "zerotrie"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851"
+dependencies = [
+ "displaydoc",
+ "yoke",
+ "zerofrom",
+]
+
+[[package]]
+name = "zerovec"
+version = "0.11.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002"
+dependencies = [
+ "yoke",
+ "zerofrom",
+ "zerovec-derive",
+]
+
+[[package]]
+name = "zerovec-derive"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "zlib-rs"
 version = "0.5.5"

Cargo.toml

@@ -153,15 +153,23 @@ rustpython-vm = { path = "crates/vm", default-features = false, version = "0.5.0
 rustpython-pylib = { path = "crates/pylib", version = "0.5.0" }
 rustpython-stdlib = { path = "crates/stdlib", default-features = false, version = "0.5.0" }
 rustpython-sre_engine = { path = "crates/sre_engine", version = "0.5.0" }
+rustpython-unicode = { path = "crates/unicode", default-features = false, version = "0.5.0" }
 rustpython-wtf8 = { path = "crates/wtf8", version = "0.5.0" }
 rustpython-doc = { path = "crates/doc", version = "0.5.0" }
 
-# Ruff tag 0.15.6 is based on commit e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675
+# Use RustPython-packaged Ruff crates from the published fork while keeping
+# existing crate names in the codebase.
+ruff_python_parser = { package = "rustpython-ruff_python_parser", version = "0.15.8" }
+ruff_python_ast = { package = "rustpython-ruff_python_ast", version = "0.15.8" }
+ruff_text_size = { package = "rustpython-ruff_text_size", version = "0.15.8" }
+ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.15.8" }
+# To update ruff crates, comment out the above lines and uncomment the following lines to pull directly from the Ruff repository at the specified commit hash.
+# Ruff tag 0.15.8 is based on commit c2a8815842f9dc5d24ec19385eae0f1a7188b0d9
 # at the time of this capture. We use the commit hash to ensure reproducible builds.
-ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" }
-ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" }
-ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" }
-ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" }
+# ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" }
+# ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" }
+# ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" }
+# ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" }
 
 phf = { version = "0.13.1", default-features = false, features = ["macros"]}
 ahash = "0.8.12"
@@ -207,7 +215,7 @@ rand_core = { version = "0.9", features = ["os_rng"] }
 rustix = { version = "1.1", features = ["event"] }
 rustyline = "17.0.1"
 serde = { package = "serde_core", version = "1.0.225", default-features = false, features = ["alloc"] }
-schannel = "0.1.28"
+schannel = "0.1.29"
 scoped-tls = "1"
 scopeguard = "1"
 static_assertions = "1.1"
@@ -215,15 +223,11 @@ strum = "0.28"
 strum_macros = "0.28"
 syn = "2"
 thiserror = "2.0"
+icu_properties = "2"
+icu_normalizer = "2"
 unicode-casing = "0.1.1"
-unic-char-property = "0.9.0"
-unic-normal = "0.9.0"
 unic-ucd-age = "0.9.0"
-unic-ucd-bidi = "0.9.0"
-unic-ucd-category = "0.9.0"
-unic-ucd-ident = "0.9.0"
 unicode_names2 = "2.0.0"
-unicode-bidi-mirroring = "0.4"
 widestring = "1.2.0"
 windows-sys = "0.61.2"
 wasm-bindgen = "0.2.106"

Lib/test/test_contextlib.py

@@ -99,8 +99,6 @@ class ContextManagerTestCase(unittest.TestCase):
                 raise ZeroDivisionError()
         self.assertEqual(state, [1, 42, 999])
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_contextmanager_traceback(self):
         @contextmanager
         def f():

Lib/test/test_contextlib_async.py

@@ -252,7 +252,6 @@ class AsyncContextManagerTestCase(unittest.IsolatedAsyncioTestCase):
             raise ZeroDivisionError(999)
         self.assertEqual(state, [1, 42, 999])
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON
     async def test_contextmanager_except_stopiter(self):
         @asynccontextmanager
         async def woohoo():

Lib/test/test_exceptions.py

@@ -2519,7 +2519,6 @@ class SyntaxErrorTests(unittest.TestCase):
         self.assertEqual(error, the_exception.text)
         self.assertEqual("bad bad", the_exception.msg)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_incorrect_constructor(self):
         args = ("bad.py", 1, 2)
         self.assertRaises(TypeError, SyntaxError, "bad bad", args)

Lib/test/test_file.py

@@ -126,7 +126,7 @@ class AutoFileTests:
         # it must also return None if an exception was given
         try:
             1/0
-        except:
+        except ZeroDivisionError:
             self.assertEqual(self.f.__exit__(*sys.exc_info()), None)
 
     def testReadWhenWriting(self):
@@ -216,6 +216,16 @@ class OtherFileTests:
         with self.assertWarnsRegex(RuntimeWarning, 'line buffering'):
             self._checkBufferSize(1)
 
+    def testDefaultBufferSize(self):
+        with self.open(TESTFN, 'wb') as f:
+            blksize = f.raw._blksize
+            f.write(b"\0" * 5_000_000)
+
+        with self.open(TESTFN, 'rb') as f:
+            data = f.read1()
+            expected_size = max(min(blksize, 8192 * 1024), io.DEFAULT_BUFFER_SIZE)
+            self.assertEqual(len(data), expected_size)
+
     def testTruncateOnWindows(self):
         # SF bug <https://bugs.python.org/issue801631>
         # "file.truncate fault on windows"
@@ -344,7 +354,6 @@ class OtherFileTests:
 class COtherFileTests(OtherFileTests, unittest.TestCase):
     open = io.open
 
-
 class PyOtherFileTests(OtherFileTests, unittest.TestCase):
     open = staticmethod(pyio.open)
 

Lib/test/test_largefile.py

@@ -2,13 +2,12 @@
 """
 
 import os
-import stat
 import sys
 import unittest
 import socket
 import shutil
 import threading
-from test.support import requires, bigmemtest
+from test.support import requires, bigmemtest, requires_resource
 from test.support import SHORT_TIMEOUT
 from test.support import socket_helper
 from test.support.os_helper import TESTFN, unlink
@@ -29,7 +28,7 @@ class LargeFileTest:
             mode = 'w+b'
 
         with self.open(TESTFN, mode) as f:
-            current_size = os.fstat(f.fileno())[stat.ST_SIZE]
+            current_size = os.fstat(f.fileno()).st_size
             if current_size == size+1:
                 return
 
@@ -40,13 +39,13 @@ class LargeFileTest:
             f.seek(size)
             f.write(b'a')
             f.flush()
-            self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1)
+            self.assertEqual(os.fstat(f.fileno()).st_size, size+1)
 
     @classmethod
     def tearDownClass(cls):
         with cls.open(TESTFN, 'wb'):
             pass
-        if not os.stat(TESTFN)[stat.ST_SIZE] == 0:
+        if not os.stat(TESTFN).st_size == 0:
             raise cls.failureException('File was not truncated by opening '
                                        'with mode "wb"')
         unlink(TESTFN2)
@@ -67,7 +66,7 @@ class TestFileMethods(LargeFileTest):
             self.assertEqual(f.tell(), size + 1)
 
     def test_osstat(self):
-        self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1)
+        self.assertEqual(os.stat(TESTFN).st_size, size+1)
 
     def test_seek_read(self):
         with self.open(TESTFN, 'rb') as f:
@@ -142,6 +141,9 @@ class TestFileMethods(LargeFileTest):
             f.truncate(1)
             self.assertEqual(f.tell(), 0)       # else pointer moved
             f.seek(0)
+            # Verify readall on a truncated file is well behaved. read()
+            # without a size can be unbounded, this should get just the byte
+            # that remains.
             self.assertEqual(len(f.read()), 1)  # else wasn't truncated
 
     def test_seekable(self):
@@ -152,6 +154,22 @@ class TestFileMethods(LargeFileTest):
                 f.seek(pos)
                 self.assertTrue(f.seekable())
 
+    @bigmemtest(size=size, memuse=2, dry_run=False)
+    def test_seek_readall(self, _size):
+        # Seek which doesn't change position should readall successfully.
+        with self.open(TESTFN, 'rb') as f:
+            self.assertEqual(f.seek(0, os.SEEK_CUR), 0)
+            self.assertEqual(len(f.read()), size + 1)
+
+        # Seek which changes (or might change) position should readall
+        # successfully.
+        with self.open(TESTFN, 'rb') as f:
+            self.assertEqual(f.seek(20, os.SEEK_SET), 20)
+            self.assertEqual(len(f.read()), size - 19)
+
+        with self.open(TESTFN, 'rb') as f:
+            self.assertEqual(f.seek(-3, os.SEEK_END), size - 2)
+            self.assertEqual(len(f.read()), 3)
 
 def skip_no_disk_space(path, required):
     def decorator(fun):
@@ -173,6 +191,7 @@ class TestCopyfile(LargeFileTest, unittest.TestCase):
     # Exact required disk space would be (size * 2), but let's give it a
     # bit more tolerance.
     @skip_no_disk_space(TESTFN, size * 2.5)
+    @requires_resource('cpu')
     def test_it(self):
         # Internally shutil.copyfile() can use "fast copy" methods like
         # os.sendfile().
@@ -222,6 +241,7 @@ class TestSocketSendfile(LargeFileTest, unittest.TestCase):
     # Exact required disk space would be (size * 2), but let's give it a
     # bit more tolerance.
     @skip_no_disk_space(TESTFN, size * 2.5)
+    @requires_resource('cpu')
     def test_it(self):
         port = socket_helper.find_unused_port()
         with socket.create_server(("", port)) as sock:

Lib/test/test_module/__init__.py

@@ -1,4 +1,5 @@
 # Test the module type
+import importlib.machinery
 import unittest
 import weakref
 from test.support import gc_collect
@@ -29,7 +30,7 @@ class ModuleTests(unittest.TestCase):
             self.fail("__name__ = %s" % repr(s))
         except AttributeError:
             pass
-        self.assertEqual(foo.__doc__, ModuleType.__doc__)
+        self.assertEqual(foo.__doc__, ModuleType.__doc__ or '')
 
     def test_uninitialized_missing_getattr(self):
         # Issue 8297
@@ -102,8 +103,7 @@ class ModuleTests(unittest.TestCase):
         gc_collect()
         self.assertEqual(f().__dict__["bar"], 4)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_clear_dict_in_ref_cycle(self):
         destroyed = []
         m = ModuleType("foo")
@@ -152,15 +152,13 @@ a = A(destroyed)"""
         if 'test.test_module.bad_getattr2' in sys.modules:
             del sys.modules['test.test_module.bad_getattr2']
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_module_dir(self):
         import test.test_module.good_getattr as gga
         self.assertEqual(dir(gga), ['a', 'b', 'c'])
         del sys.modules['test.test_module.good_getattr']
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_module_dir_errors(self):
         import test.test_module.bad_getattr as bga
         from test.test_module import bad_getattr2
@@ -270,11 +268,38 @@ a = A(destroyed)"""
         self.assertEqual(r[-len(ends_with):], ends_with,
                          '{!r} does not end with {!r}'.format(r, ends_with))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    def test_module_repr_with_namespace_package(self):
+        m = ModuleType('foo')
+        loader = importlib.machinery.NamespaceLoader('foo', ['bar'], 'baz')
+        spec = importlib.machinery.ModuleSpec('foo', loader)
+        m.__loader__ = loader
+        m.__spec__ = spec
+        self.assertEqual(repr(m), "<module 'foo' (namespace) from ['bar']>")
+
+    def test_module_repr_with_namespace_package_and_custom_loader(self):
+        m = ModuleType('foo')
+        loader = BareLoader()
+        spec = importlib.machinery.ModuleSpec('foo', loader)
+        m.__loader__ = loader
+        m.__spec__ = spec
+        expected_repr_pattern = r"<module 'foo' \(<.*\.BareLoader object at .+>\)>"
+        self.assertRegex(repr(m), expected_repr_pattern)
+        self.assertNotIn('from', repr(m))
+
+    def test_module_repr_with_fake_namespace_package(self):
+        m = ModuleType('foo')
+        loader = BareLoader()
+        loader._path = ['spam']
+        spec = importlib.machinery.ModuleSpec('foo', loader)
+        m.__loader__ = loader
+        m.__spec__ = spec
+        expected_repr_pattern = r"<module 'foo' \(<.*\.BareLoader object at .+>\)>"
+        self.assertRegex(repr(m), expected_repr_pattern)
+        self.assertNotIn('from', repr(m))
+
     def test_module_finalization_at_shutdown(self):
         # Module globals and builtins should still be available during shutdown
-        rc, out, err = assert_python_ok("-c", "from test import final_a")
+        rc, out, err = assert_python_ok("-c", "from test.test_module import final_a")
         self.assertFalse(err)
         lines = out.splitlines()
         self.assertEqual(set(lines), {

Lib/test/test_module/final_a.py

@@ -0,0 +1,19 @@
+"""
+Fodder for module finalization tests in test_module.
+"""
+
+import shutil
+import test.test_module.final_b
+
+x = 'a'
+
+class C:
+    def __del__(self):
+        # Inspect module globals and builtins
+        print("x =", x)
+        print("final_b.x =", test.test_module.final_b.x)
+        print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None))
+        print("len =", getattr(len, '__name__', None))
+
+c = C()
+_underscored = C()

Lib/test/test_module/final_b.py

@@ -0,0 +1,19 @@
+"""
+Fodder for module finalization tests in test_module.
+"""
+
+import shutil
+import test.test_module.final_a
+
+x = 'b'
+
+class C:
+    def __del__(self):
+        # Inspect module globals and builtins
+        print("x =", x)
+        print("final_a.x =", test.test_module.final_a.x)
+        print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None))
+        print("len =", getattr(len, '__name__', None))
+
+c = C()
+_underscored = C()

Lib/test/test_monitoring.py

@@ -1261,7 +1261,6 @@ class TestLineAndInstructionEvents(CheckEvents):
             ('instruction', 'func2', 46),
             ('line', 'get_events', 11)])
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; - instruction offsets differ from CPython
     def test_try_except(self):
 
         def func3():

Lib/test/test_optimizer.py

@@ -0,0 +1,90 @@
+import unittest
+import types
+from test.support import import_helper
+
+
+_testinternalcapi = import_helper.import_module("_testinternalcapi")
+
+
+class TestRareEventCounters(unittest.TestCase):
+    def setUp(self):
+        _testinternalcapi.reset_rare_event_counters()
+
+    def test_set_class(self):
+        class A:
+            pass
+        class B:
+            pass
+        a = A()
+
+        orig_counter = _testinternalcapi.get_rare_event_counters()["set_class"]
+        a.__class__ = B
+        self.assertEqual(
+            orig_counter + 1,
+            _testinternalcapi.get_rare_event_counters()["set_class"]
+        )
+
+    def test_set_bases(self):
+        class A:
+            pass
+        class B:
+            pass
+        class C(B):
+            pass
+
+        orig_counter = _testinternalcapi.get_rare_event_counters()["set_bases"]
+        C.__bases__ = (A,)
+        self.assertEqual(
+            orig_counter + 1,
+            _testinternalcapi.get_rare_event_counters()["set_bases"]
+        )
+
+    def test_set_eval_frame_func(self):
+        orig_counter = _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"]
+        _testinternalcapi.set_eval_frame_record([])
+        self.assertEqual(
+            orig_counter + 1,
+            _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"]
+        )
+        _testinternalcapi.set_eval_frame_default()
+
+    def test_builtin_dict(self):
+        orig_counter = _testinternalcapi.get_rare_event_counters()["builtin_dict"]
+        if isinstance(__builtins__, types.ModuleType):
+            builtins = __builtins__.__dict__
+        else:
+            builtins = __builtins__
+        builtins["FOO"] = 42
+        self.assertEqual(
+            orig_counter + 1,
+            _testinternalcapi.get_rare_event_counters()["builtin_dict"]
+        )
+        del builtins["FOO"]
+
+    def test_func_modification(self):
+        def func(x=0):
+            pass
+
+        for attribute in (
+            "__code__",
+            "__defaults__",
+            "__kwdefaults__"
+        ):
+            orig_counter = _testinternalcapi.get_rare_event_counters()["func_modification"]
+            setattr(func, attribute, getattr(func, attribute))
+            self.assertEqual(
+                orig_counter + 1,
+                _testinternalcapi.get_rare_event_counters()["func_modification"]
+            )
+
+
+class TestOptimizerSymbols(unittest.TestCase):
+
+    @unittest.skipUnless(hasattr(_testinternalcapi, "uop_symbols_test"),
+                "requires _testinternalcapi.uop_symbols_test")
+    def test_optimizer_symbols(self):
+        _testinternalcapi.uop_symbols_test()
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_peepholer.py

@@ -132,7 +132,6 @@ class TestTranforms(BytecodeTestCase):
         self.assertInBytecode(f, 'LOAD_CONST', None)
         self.check_lnotab(f)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; RETURN_VALUE
     def test_while_one(self):
         # Skip over:  LOAD_CONST trueconst  POP_JUMP_IF_FALSE xx
         def f():
@@ -530,7 +529,6 @@ class TestTranforms(BytecodeTestCase):
         self.assertEqual(len(returns), 1)
         self.check_lnotab(f)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; KeyError: 20
     def test_elim_jump_to_return(self):
         # JUMP_FORWARD to RETURN -->  RETURN
         def f(cond, true_value, false_value):
@@ -545,7 +543,6 @@ class TestTranforms(BytecodeTestCase):
         self.assertEqual(len(returns), 2)
         self.check_lnotab(f)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; absolute jump encoding
     def test_elim_jump_to_uncond_jump(self):
         # POP_JUMP_IF_FALSE to JUMP_FORWARD --> POP_JUMP_IF_FALSE to non-jump
         def f():

Lib/test/test_perfmaps.py

@@ -0,0 +1,23 @@
+import os
+import sys
+import unittest
+
+try:
+    from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry
+except ImportError:
+    raise unittest.SkipTest("requires _testinternalcapi")
+
+
+if sys.platform != 'linux':
+    raise unittest.SkipTest('Linux only')
+
+
+class TestPerfMapWriting(unittest.TestCase):
+    def test_write_perf_map_entry(self):
+        self.assertEqual(write_perf_map_entry(0x1234, 5678, "entry1"), 0)
+        self.assertEqual(write_perf_map_entry(0x2345, 6789, "entry2"), 0)
+        with open(f"/tmp/perf-{os.getpid()}.map") as f:
+            perf_file_contents = f.read()
+            self.assertIn("1234 162e entry1", perf_file_contents)
+            self.assertIn("2345 1a85 entry2", perf_file_contents)
+        perf_map_state_teardown()

Lib/test/test_scope.py

@@ -692,7 +692,6 @@ class ScopeTests(unittest.TestCase):
         self.assertEqual(c.dec(), 1)
         self.assertEqual(c.dec(), 0)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; figure out how to communicate that `y = 9` should be stored as a global rather than a STORE_NAME, even when the `global y` is in a nested subscope
     def testGlobalInParallelNestedFunctions(self):
         # A symbol table bug leaked the global statement from one
         # function to other nested functions in the same block.

Lib/test/test_str.py

@@ -854,6 +854,7 @@ class StrTest(string_tests.StringLikeTest,
         self.assertTrue('\U0001F46F'.isprintable())
         self.assertFalse('\U000E0020'.isprintable())
 
+    @unittest.expectedFailure  # TODO: RUSTPYTHON
     @support.requires_resource('cpu')
     def test_isprintable_invariant(self):
         for codepoint in range(sys.maxunicode + 1):

Lib/test/test_unicodedata.py

@@ -232,7 +232,6 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
         b = 'C\u0338' * 20  + '\xC7'
         self.assertEqual(self.db.normalize('NFC', a), b)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; ?  +
     def test_issue29456(self):
         # Fix #29456
         u1176_str_a = '\u1100\u1176\u11a8'
@@ -389,6 +388,7 @@ class NormalizationTest(unittest.TestCase):
         data = [int(x, 16) for x in data.split(" ")]
         return "".join([chr(x) for x in data])
 
+    @unittest.expectedFailure  # TODO: RUSTPYTHON; AssertionError: False is not true : 13055
     @requires_resource('network')
     @requires_resource('cpu')
     def test_normalization(self):

crates/codegen/Cargo.toml

@@ -14,6 +14,7 @@ std = ["thiserror/std", "itertools/use_std"]
 
 [dependencies]
 rustpython-compiler-core = { workspace = true }
+rustpython-unicode = { workspace = true, default-features = false }
 rustpython-literal = {workspace = true }
 rustpython-wtf8 = { workspace = true }
 ruff_python_ast = { workspace = true }
@@ -29,7 +30,6 @@ num-traits = { workspace = true }
 thiserror = { workspace = true }
 malachite-bigint = { workspace = true }
 memchr = { workspace = true }
-unicode_names2 = { workspace = true }
 
 [dev-dependencies]
 ruff_python_parser = { workspace = true }

crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap

@@ -1,6 +1,5 @@
 ---
 source: crates/codegen/src/compile.rs
-assertion_line: 9780
 expression: "compile_exec(\"\\\nasync def test():\n    for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n        with self.subTest(type=type(stop_exc)):\n            try:\n                async with egg():\n                    raise stop_exc\n            except Exception as ex:\n                self.assertIs(ex, stop_exc)\n            else:\n                self.fail(f'{stop_exc} was suppressed')\n\")"
 ---
   1           0 RESUME               (0)
@@ -24,7 +23,7 @@ expression: "compile_exec(\"\\\nasync def test():\n    for stop_exc in (StopIter
                  16 CACHE
                  17 CACHE
                  18 LOAD_CONST           ("ham")
-            >>   19 CALL                 (1)
+                 19 CALL                 (1)
                  20 CACHE
                  21 CACHE
                  22 CACHE
@@ -34,12 +33,12 @@ expression: "compile_exec(\"\\\nasync def test():\n    for stop_exc in (StopIter
                  26 CACHE
                  27 STORE_FAST           (0, stop_exc)
 
-  3         >>   28 LOAD_GLOBAL          (4, self)
+  3              28 LOAD_GLOBAL          (4, self)
                  29 CACHE
                  30 CACHE
                  31 CACHE
-                 32 CACHE
-            >>   33 LOAD_ATTR            (7, subTest, method=true)
+            >>   32 CACHE
+                 33 LOAD_ATTR            (7, subTest, method=true)
                  34 CACHE
                  35 CACHE
                  36 CACHE
@@ -52,9 +51,9 @@ expression: "compile_exec(\"\\\nasync def test():\n    for stop_exc in (StopIter
                  43 LOAD_GLOBAL          (9, NULL + type)
                  44 CACHE
                  45 CACHE
-            >>   46 CACHE
+                 46 CACHE
                  47 CACHE
-                 48 LOAD_FAST            (0, stop_exc)
+            >>   48 LOAD_FAST            (0, stop_exc)
                  49 CALL                 (1)
                  50 CACHE
                  51 CACHE
@@ -139,30 +138,30 @@ expression: "compile_exec(\"\\\nasync def test():\n    for stop_exc in (StopIter
                 125 POP_TOP
                 126 POP_TOP
                 127 POP_TOP
-                128 JUMP_FORWARD         (3)
+                128 JUMP_FORWARD         (48)
                 129 COPY                 (3)
                 130 POP_EXCEPT
                 131 RERAISE              (1)
-                132 JUMP_FORWARD         (46)
-                133 PUSH_EXC_INFO
+                132 PUSH_EXC_INFO
 
-  7             134 LOAD_GLOBAL          (12, Exception)
+  7             133 LOAD_GLOBAL          (12, Exception)
+                134 CACHE
                 135 CACHE
                 136 CACHE
                 137 CACHE
-                138 CACHE
-                139 CHECK_EXC_MATCH
-                140 POP_JUMP_IF_FALSE    (33)
-                141 CACHE
-                142 NOT_TAKEN
-                143 STORE_FAST           (1, ex)
+                138 CHECK_EXC_MATCH
+                139 POP_JUMP_IF_FALSE    (32)
+                140 CACHE
+                141 NOT_TAKEN
+                142 STORE_FAST           (1, ex)
 
-  8             144 LOAD_GLOBAL          (4, self)
+  8             143 LOAD_GLOBAL          (4, self)
+                144 CACHE
                 145 CACHE
                 146 CACHE
                 147 CACHE
-                148 CACHE
-                149 LOAD_ATTR            (15, assertIs, method=true)
+                148 LOAD_ATTR            (15, assertIs, method=true)
+                149 CACHE
                 150 CACHE
                 151 CACHE
                 152 CACHE
@@ -171,84 +170,85 @@ expression: "compile_exec(\"\\\nasync def test():\n    for stop_exc in (StopIter
                 155 CACHE
                 156 CACHE
                 157 CACHE
-                158 CACHE
-                159 LOAD_FAST_LOAD_FAST  (ex, stop_exc)
-                160 CALL                 (2)
+                158 LOAD_FAST_LOAD_FAST  (ex, stop_exc)
+                159 CALL                 (2)
+                160 CACHE
                 161 CACHE
                 162 CACHE
-                163 CACHE
-                164 POP_TOP
-                165 JUMP_FORWARD         (4)
-                166 LOAD_CONST           (None)
-                167 STORE_FAST           (1, ex)
-                168 DELETE_FAST          (1, ex)
-                169 RERAISE              (1)
-                170 POP_EXCEPT
-                171 LOAD_CONST           (None)
-                172 STORE_FAST           (1, ex)
-                173 DELETE_FAST          (1, ex)
-                174 JUMP_FORWARD         (28)
-                175 RERAISE              (0)
-                176 COPY                 (3)
-                177 POP_EXCEPT
-                178 RERAISE              (1)
+                163 POP_TOP
+                164 POP_EXCEPT
+                165 LOAD_CONST           (None)
+                166 STORE_FAST           (1, ex)
+                167 DELETE_FAST          (1, ex)
+                168 JUMP_FORWARD         (32)
+                169 LOAD_CONST           (None)
+                170 STORE_FAST           (1, ex)
+                171 DELETE_FAST          (1, ex)
+                172 RERAISE              (1)
+                173 RERAISE              (0)
+                174 COPY                 (3)
+                175 POP_EXCEPT
+                176 RERAISE              (1)
 
- 10             179 LOAD_GLOBAL          (4, self)
+ 10             177 LOAD_GLOBAL          (4, self)
+                178 CACHE
+                179 CACHE
                 180 CACHE
                 181 CACHE
-                182 CACHE
+                182 LOAD_ATTR            (17, fail, method=true)
                 183 CACHE
-                184 LOAD_ATTR            (17, fail, method=true)
+                184 CACHE
                 185 CACHE
                 186 CACHE
-                187 CACHE
+            >>  187 CACHE
                 188 CACHE
                 189 CACHE
                 190 CACHE
                 191 CACHE
-                192 CACHE
-                193 CACHE
-                194 LOAD_FAST_BORROW     (0, stop_exc)
-                195 FORMAT_SIMPLE
-                196 LOAD_CONST           (" was suppressed")
-                197 BUILD_STRING         (2)
-                198 CALL                 (1)
+                192 LOAD_FAST_BORROW     (0, stop_exc)
+                193 FORMAT_SIMPLE
+                194 LOAD_CONST           (" was suppressed")
+                195 BUILD_STRING         (2)
+                196 CALL                 (1)
+                197 CACHE
+                198 CACHE
                 199 CACHE
-                200 CACHE
-                201 CACHE
-                202 POP_TOP
-                203 NOP
+                200 POP_TOP
+                201 NOP
 
-  3             204 LOAD_CONST           (None)
-                205 LOAD_CONST           (None)
-                206 LOAD_CONST           (None)
-                207 CALL                 (3)
+  3             202 LOAD_CONST           (None)
+                203 LOAD_CONST           (None)
+            >>  204 LOAD_CONST           (None)
+                205 CALL                 (3)
+                206 CACHE
+                207 CACHE
                 208 CACHE
-            >>  209 CACHE
-                210 CACHE
-                211 POP_TOP
-                212 JUMP_FORWARD         (19)
-                213 PUSH_EXC_INFO
-                214 WITH_EXCEPT_START
-                215 TO_BOOL
+                209 POP_TOP
+                210 JUMP_BACKWARD        (187)
+                211 CACHE
+                212 PUSH_EXC_INFO
+                213 WITH_EXCEPT_START
+                214 TO_BOOL
+                215 CACHE
                 216 CACHE
                 217 CACHE
-                218 CACHE
-                219 POP_JUMP_IF_TRUE     (2)
-                220 CACHE
-                221 NOT_TAKEN
-                222 RERAISE              (2)
-                223 POP_TOP
-                224 POP_EXCEPT
+                218 POP_JUMP_IF_TRUE     (2)
+                219 CACHE
+                220 NOT_TAKEN
+                221 RERAISE              (2)
+                222 POP_TOP
+                223 POP_EXCEPT
+                224 POP_TOP
                 225 POP_TOP
                 226 POP_TOP
-                227 POP_TOP
-                228 JUMP_FORWARD         (3)
+                227 JUMP_BACKWARD        (204)
+                228 CACHE
                 229 COPY                 (3)
                 230 POP_EXCEPT
                 231 RERAISE              (1)
-                232 JUMP_BACKWARD        (209)
-                233 CACHE
+
+  2             232 CALL_INTRINSIC_1     (StopIterationError)
+                233 RERAISE              (1)
 
               2 MAKE_FUNCTION
               3 STORE_NAME           (0, test)

crates/codegen/src/string_parser.rs

@@ -113,7 +113,9 @@ impl StringParser {
         let name_and_ending = self.skip_bytes(close_idx + 1);
         let name = &name_and_ending[..name_and_ending.len() - 1];
 
-        unicode_names2::character(name).ok_or_else(|| unreachable!())
+        rustpython_unicode::data::lookup(name)
+            .and_then(char::from_u32)
+            .ok_or_else(|| unreachable!())
     }
 
     /// Parse an escaped character, returning the new character.

crates/codegen/src/symboltable.rs

@@ -2169,17 +2169,15 @@ impl SymbolTableBuilder {
         // Generator expressions need the is_generator flag
         self.tables.last_mut().unwrap().is_generator = is_generator;
 
-        // PEP 709: Mark non-generator comprehensions for inlining,
-        // but only inside function-like scopes (fastlocals).
-        // Module/class scope uses STORE_NAME which is incompatible
-        // with LOAD_FAST_AND_CLEAR / STORE_FAST save/restore.
-        // Async comprehensions cannot be inlined because they need
-        // their own coroutine scope.
-        // Note: tables.last() is the comprehension scope we just pushed,
-        // so we check the second-to-last for the parent scope.
+        // PEP 709: Mark non-generator comprehensions for inlining.
+        // Only in function-like scopes for now. Module/class scope inlining
+        // needs more work (Cell name resolution, __class__ handling).
+        // Also excluded: generator expressions, async comprehensions,
+        // and annotation scopes nested in classes (can_see_class_scope).
         let element_has_await = expr_contains_await(elt1) || elt2.is_some_and(expr_contains_await);
         if !is_generator && !has_async_gen && !element_has_await {
             let parent = self.tables.iter().rev().nth(1);
+            let parent_can_see_class = parent.is_some_and(|t| t.can_see_class_scope);
             let parent_is_func = parent.is_some_and(|t| {
                 matches!(
                     t.typ,
@@ -2189,8 +2187,7 @@ impl SymbolTableBuilder {
                         | CompilerScope::Comprehension
                 )
             });
-            let parent_can_see_class = parent.is_some_and(|t| t.can_see_class_scope);
-            if parent_is_func && !parent_can_see_class {
+            if !parent_can_see_class && parent_is_func {
                 self.tables.last_mut().unwrap().comp_inlined = true;
             }
         }

crates/common/Cargo.toml

@@ -16,6 +16,7 @@ wasm_js = ["getrandom/wasm_js"]
 
 [dependencies]
 rustpython-literal = { workspace = true }
+rustpython-unicode = { workspace = true, default-features = false }
 rustpython-wtf8 = { workspace = true }
 
 ascii = { workspace = true }
@@ -29,7 +30,6 @@ malachite-q = { workspace = true }
 malachite-base = { workspace = true }
 num-traits = { workspace = true }
 parking_lot = { workspace = true, optional = true }
-unicode_names2 = { workspace = true }
 radium = { workspace = true }
 
 lock_api = "0.4"

crates/common/src/encodings.rs

@@ -414,7 +414,7 @@ pub mod errors {
             let mut out = String::with_capacity(num_chars * 4);
             for c in err_str.code_points() {
                 let c_u32 = c.to_u32();
-                if let Some(c_name) = c.to_char().and_then(unicode_names2::name) {
+                if let Some(c_name) = rustpython_unicode::data::name(c_u32) {
                     write!(out, "\\N{{{c_name}}}").unwrap();
                 } else if c_u32 >= 0x10000 {
                     write!(out, "\\U{c_u32:08x}").unwrap();

crates/compiler-core/src/bytecode.rs

@@ -135,6 +135,72 @@ pub fn decode_exception_table(table: &[u8]) -> Vec<ExceptionTableEntry> {
     entries
 }
 
+/// Parse linetable to build a boolean mask indicating which code units
+/// have NO_LOCATION (line == -1). Returns a Vec<bool> of length `num_units`.
+pub fn build_no_location_mask(linetable: &[u8], num_units: usize) -> Vec<bool> {
+    let mut mask = Vec::new();
+    mask.resize(num_units, false);
+    let mut pos = 0;
+    let mut unit_idx = 0;
+
+    while pos < linetable.len() && unit_idx < num_units {
+        let header = linetable[pos];
+        pos += 1;
+        let code = (header >> 3) & 0xf;
+        let length = ((header & 7) + 1) as usize;
+
+        let is_no_location = code == PyCodeLocationInfoKind::None as u8;
+
+        // Skip payload bytes based on location kind
+        match code {
+            0..=9 => pos += 1,   // Short forms: 1 byte payload
+            10..=12 => pos += 2, // OneLine forms: 2 bytes payload
+            13 => {
+                // NoColumns: signed varint (line delta)
+                while pos < linetable.len() {
+                    let b = linetable[pos];
+                    pos += 1;
+                    if b & 0x40 == 0 {
+                        break;
+                    }
+                }
+            }
+            14 => {
+                // Long form: signed varint (line delta) + 3 unsigned varints
+                // line_delta
+                while pos < linetable.len() {
+                    let b = linetable[pos];
+                    pos += 1;
+                    if b & 0x40 == 0 {
+                        break;
+                    }
+                }
+                // end_line_delta, col+1, end_col+1
+                for _ in 0..3 {
+                    while pos < linetable.len() {
+                        let b = linetable[pos];
+                        pos += 1;
+                        if b & 0x40 == 0 {
+                            break;
+                        }
+                    }
+                }
+            }
+            15 => {} // None: no payload
+            _ => {}
+        }
+
+        for _ in 0..length {
+            if unit_idx < num_units {
+                mask[unit_idx] = is_no_location;
+                unit_idx += 1;
+            }
+        }
+    }
+
+    mask
+}
+
 /// CPython 3.11+ linetable location info codes
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 #[repr(u8)]
@@ -226,6 +292,8 @@ impl Constant for ConstantData {
             Self::Bytes { value } => Bytes { value },
             Self::Code { code } => Code { code },
             Self::Tuple { elements } => Tuple { elements },
+            Self::Slice { elements } => Slice { elements },
+            Self::Frozenset { elements } => Frozenset { elements },
             Self::None => None,
             Self::Ellipsis => Ellipsis,
         }
@@ -783,14 +851,37 @@ impl CodeUnits {
 /// ```
 #[derive(Debug, Clone)]
 pub enum ConstantData {
-    Tuple { elements: Vec<ConstantData> },
-    Integer { value: BigInt },
-    Float { value: f64 },
-    Complex { value: Complex64 },
-    Boolean { value: bool },
-    Str { value: Wtf8Buf },
-    Bytes { value: Vec<u8> },
-    Code { code: Box<CodeObject> },
+    Tuple {
+        elements: Vec<ConstantData>,
+    },
+    Integer {
+        value: BigInt,
+    },
+    Float {
+        value: f64,
+    },
+    Complex {
+        value: Complex64,
+    },
+    Boolean {
+        value: bool,
+    },
+    Str {
+        value: Wtf8Buf,
+    },
+    Bytes {
+        value: Vec<u8>,
+    },
+    Code {
+        code: Box<CodeObject>,
+    },
+    /// Constant slice(start, stop, step)
+    Slice {
+        elements: Box<[ConstantData; 3]>,
+    },
+    Frozenset {
+        elements: Vec<ConstantData>,
+    },
     None,
     Ellipsis,
 }
@@ -812,6 +903,8 @@ impl PartialEq for ConstantData {
             (Bytes { value: a }, Bytes { value: b }) => a == b,
             (Code { code: a }, Code { code: b }) => core::ptr::eq(a.as_ref(), b.as_ref()),
             (Tuple { elements: a }, Tuple { elements: b }) => a == b,
+            (Slice { elements: a }, Slice { elements: b }) => a == b,
+            (Frozenset { elements: a }, Frozenset { elements: b }) => a == b,
             (None, None) => true,
             (Ellipsis, Ellipsis) => true,
             _ => false,
@@ -838,6 +931,8 @@ impl hash::Hash for ConstantData {
             Bytes { value } => value.hash(state),
             Code { code } => core::ptr::hash(code.as_ref(), state),
             Tuple { elements } => elements.hash(state),
+            Slice { elements } => elements.hash(state),
+            Frozenset { elements } => elements.hash(state),
             None => {}
             Ellipsis => {}
         }
@@ -854,6 +949,8 @@ pub enum BorrowedConstant<'a, C: Constant> {
     Bytes { value: &'a [u8] },
     Code { code: &'a CodeObject<C> },
     Tuple { elements: &'a [C] },
+    Slice { elements: &'a [C; 3] },
+    Frozenset { elements: &'a [C] },
     None,
     Ellipsis,
 }
@@ -891,6 +988,28 @@ impl<C: Constant> BorrowedConstant<'_, C> {
                 }
                 write!(f, ")")
             }
+            BorrowedConstant::Slice { elements } => {
+                write!(f, "slice(")?;
+                elements[0].borrow_constant().fmt_display(f)?;
+                write!(f, ", ")?;
+                elements[1].borrow_constant().fmt_display(f)?;
+                write!(f, ", ")?;
+                elements[2].borrow_constant().fmt_display(f)?;
+                write!(f, ")")
+            }
+            BorrowedConstant::Frozenset { elements } => {
+                write!(f, "frozenset({{")?;
+                let mut first = true;
+                for c in *elements {
+                    if first {
+                        first = false
+                    } else {
+                        write!(f, ", ")?;
+                    }
+                    c.borrow_constant().fmt_display(f)?;
+                }
+                write!(f, "}})")
+            }
             BorrowedConstant::None => write!(f, "None"),
             BorrowedConstant::Ellipsis => write!(f, "..."),
         }
@@ -921,6 +1040,15 @@ impl<C: Constant> BorrowedConstant<'_, C> {
                     .map(|c| c.borrow_constant().to_owned())
                     .collect(),
             },
+            BorrowedConstant::Slice { elements } => Slice {
+                elements: Box::new(elements.each_ref().map(|c| c.borrow_constant().to_owned())),
+            },
+            BorrowedConstant::Frozenset { elements } => Frozenset {
+                elements: elements
+                    .iter()
+                    .map(|c| c.borrow_constant().to_owned())
+                    .collect(),
+            },
             BorrowedConstant::None => None,
             BorrowedConstant::Ellipsis => Ellipsis,
         }

crates/compiler-core/src/bytecode/oparg.rs

@@ -645,6 +645,10 @@ oparg_enum!(
         BuiltinAll = 3,
         /// Built-in `any` function
         BuiltinAny = 4,
+        /// Built-in `list` type
+        BuiltinList = 5,
+        /// Built-in `set` type
+        BuiltinSet = 6,
     }
 );
 
@@ -656,6 +660,8 @@ impl fmt::Display for CommonConstant {
             Self::BuiltinTuple => "tuple",
             Self::BuiltinAll => "all",
             Self::BuiltinAny => "any",
+            Self::BuiltinList => "list",
+            Self::BuiltinSet => "set",
         };
         write!(f, "{name}")
     }
@@ -918,8 +924,8 @@ impl VarNums {
 
 impl LoadAttr {
     #[must_use]
-    pub fn builder() -> LoadAttrBuilder {
-        LoadAttrBuilder::default()
+    pub const fn new(name_idx: u32, is_method: bool) -> Self {
+        Self::from_u32((name_idx << 1) | (is_method as u32))
     }
 
     #[must_use]
@@ -933,36 +939,10 @@ impl LoadAttr {
     }
 }
 
-#[derive(Clone, Copy, Default)]
-pub struct LoadAttrBuilder {
-    name_idx: u32,
-    is_method: bool,
-}
-
-impl LoadAttrBuilder {
-    #[must_use]
-    pub const fn build(self) -> LoadAttr {
-        let value = (self.name_idx << 1) | (self.is_method as u32);
-        LoadAttr::from_u32(value)
-    }
-
-    #[must_use]
-    pub const fn name_idx(mut self, value: u32) -> Self {
-        self.name_idx = value;
-        self
-    }
-
-    #[must_use]
-    pub const fn is_method(mut self, value: bool) -> Self {
-        self.is_method = value;
-        self
-    }
-}
-
 impl LoadSuperAttr {
     #[must_use]
-    pub fn builder() -> LoadSuperAttrBuilder {
-        LoadSuperAttrBuilder::default()
+    pub const fn new(name_idx: u32, is_load_method: bool, has_class: bool) -> Self {
+        Self::from_u32((name_idx << 2) | (is_load_method as u32) | ((has_class as u32) << 1))
     }
 
     #[must_use]
@@ -980,43 +960,3 @@ impl LoadSuperAttr {
         (self.0 & 2) == 2
     }
 }
-
-#[derive(Clone, Copy, Default)]
-pub struct LoadSuperAttrBuilder {
-    name_idx: u32,
-    is_load_method: bool,
-    has_class: bool,
-}
-
-impl LoadSuperAttrBuilder {
-    #[must_use]
-    pub const fn build(self) -> LoadSuperAttr {
-        let value =
-            (self.name_idx << 2) | ((self.has_class as u32) << 1) | (self.is_load_method as u32);
-        LoadSuperAttr::from_u32(value)
-    }
-
-    #[must_use]
-    pub const fn name_idx(mut self, value: u32) -> Self {
-        self.name_idx = value;
-        self
-    }
-
-    #[must_use]
-    pub const fn is_load_method(mut self, value: bool) -> Self {
-        self.is_load_method = value;
-        self
-    }
-
-    #[must_use]
-    pub const fn has_class(mut self, value: bool) -> Self {
-        self.has_class = value;
-        self
-    }
-}
-
-impl From<LoadSuperAttrBuilder> for LoadSuperAttr {
-    fn from(builder: LoadSuperAttrBuilder) -> Self {
-        builder.build()
-    }
-}

crates/compiler-core/src/marshal.rs

@@ -435,6 +435,20 @@ impl<Bag: ConstantBag> MarshalBag for Bag {
         self.make_tuple(elements)
     }
 
+    fn make_slice(
+        &self,
+        start: Self::Value,
+        stop: Self::Value,
+        step: Self::Value,
+    ) -> Result<Self::Value> {
+        let elements = [start, stop, step];
+        Ok(
+            self.make_constant::<Bag::Constant>(BorrowedConstant::Slice {
+                elements: &elements,
+            }),
+        )
+    }
+
     fn make_code(
         &self,
         code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
@@ -454,8 +468,13 @@ impl<Bag: ConstantBag> MarshalBag for Bag {
         Err(MarshalError::BadType)
     }
 
-    fn make_frozenset(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
-        Err(MarshalError::BadType)
+    fn make_frozenset(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
+        let elements: Vec<Self::Value> = it.collect();
+        Ok(
+            self.make_constant::<Bag::Constant>(BorrowedConstant::Frozenset {
+                elements: &elements,
+            }),
+        )
     }
 
     fn make_dict(
@@ -697,6 +716,10 @@ impl<'a, C: Constant> From<BorrowedConstant<'a, C>> for DumpableValue<'a, C> {
             BorrowedConstant::Bytes { value } => Self::Bytes(value),
             BorrowedConstant::Code { code } => Self::Code(code),
             BorrowedConstant::Tuple { elements } => Self::Tuple(elements),
+            BorrowedConstant::Slice { elements } => {
+                Self::Slice(&elements[0], &elements[1], &elements[2])
+            }
+            BorrowedConstant::Frozenset { elements } => Self::Frozenset(elements),
             BorrowedConstant::None => Self::None,
             BorrowedConstant::Ellipsis => Self::Ellipsis,
         }

crates/compiler-source/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "rustpython-compiler-source"
 description = "(DEPRECATED) RustPython Source and Index"
-version = "0.5.0+deprecated"
+version = "0.4.1+deprecated"
 authors.workspace = true
 edition.workspace = true
 rust-version.workspace = true

crates/doc/Cargo.toml

@@ -1,5 +1,6 @@
 [package]
 name = "rustpython-doc"
+description = "Python __doc__ database for RustPython"
 version.workspace = true
 authors.workspace = true
 edition.workspace = true

crates/jit/src/instructions.rs

@@ -736,6 +736,20 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
                 let val = self.stack.pop().ok_or(JitCompileError::BadBytecode)?;
                 self.store_variable(var_num.get(arg), val)
             }
+            Instruction::StoreFastLoadFast { var_nums } => {
+                let oparg = var_nums.get(arg);
+                let (store_idx, load_idx) = oparg.indexes();
+                let val = self.stack.pop().ok_or(JitCompileError::BadBytecode)?;
+                self.store_variable(store_idx, val)?;
+                let local = self.variables[load_idx]
+                    .as_ref()
+                    .ok_or(JitCompileError::BadBytecode)?;
+                self.stack.push(JitValue::from_type_and_value(
+                    local.ty.clone(),
+                    self.builder.use_var(local.var),
+                ));
+                Ok(())
+            }
             Instruction::StoreFastStoreFast { var_nums } => {
                 let oparg = var_nums.get(arg);
                 let (idx1, idx2) = oparg.indexes();

crates/jit/tests/common.rs

@@ -42,6 +42,7 @@ impl Function {
     }
 }
 
+#[allow(dead_code)]
 #[derive(Debug, Clone)]
 enum StackValue {
     String(String),
@@ -49,6 +50,8 @@ enum StackValue {
     Map(HashMap<Wtf8Buf, StackValue>),
     Code(Box<CodeObject>),
     Function(Function),
+    Slice(Box<[StackValue; 3]>),
+    Frozenset(Vec<StackValue>),
 }
 
 impl From<ConstantData> for StackValue {
@@ -59,6 +62,13 @@ impl From<ConstantData> for StackValue {
             }
             ConstantData::None => StackValue::None,
             ConstantData::Code { code } => StackValue::Code(code),
+            ConstantData::Slice { elements } => {
+                let [start, stop, step] = *elements;
+                StackValue::Slice(Box::new([start.into(), stop.into(), step.into()]))
+            }
+            ConstantData::Frozenset { elements } => {
+                StackValue::Frozenset(elements.into_iter().map(Into::into).collect())
+            }
             c => unimplemented!("constant {:?} isn't yet supported in py_function!", c),
         }
     }

crates/literal/Cargo.toml

@@ -9,13 +9,13 @@ license = { workspace = true }
 rust-version = { workspace = true }
 
 [dependencies]
+rustpython-unicode = { workspace = true, default-features = false }
 rustpython-wtf8 = { workspace = true }
 
 hexf-parse = "0.2.1"
 is-macro.workspace = true
 lexical-parse-float = { version = "1.0.6", features = ["format"] }
 num-traits = { workspace = true }
-unic-ucd-category = { workspace = true }
 
 [dev-dependencies]
 rand = { workspace = true }

crates/literal/src/char.rs

@@ -1,15 +0,0 @@
-use unic_ucd_category::GeneralCategory;
-
-/// According to python following categories aren't printable:
-/// * Cc (Other, Control)
-/// * Cf (Other, Format)
-/// * Cs (Other, Surrogate)
-/// * Co (Other, Private Use)
-/// * Cn (Other, Not Assigned)
-/// * Zl Separator, Line ('\u2028', LINE SEPARATOR)
-/// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR)
-/// * Zs (Separator, Space) other than ASCII space('\x20').
-pub fn is_printable(c: char) -> bool {
-    let cat = GeneralCategory::of(c);
-    !(cat.is_other() || cat.is_separator())
-}

crates/literal/src/escape.rs

@@ -204,7 +204,7 @@ impl UnicodeEscape<'_> {
             '\\' | '\t' | '\r' | '\n' => 2,
             ch if ch < ' ' || ch as u32 == 0x7f => 4, // \xHH
             ch if ch.is_ascii() => 1,
-            ch if crate::char::is_printable(ch) => {
+            ch if rustpython_unicode::classify::is_repr_printable(ch as u32) => {
                 // max = std::cmp::max(ch, max);
                 ch.len_utf8()
             }
@@ -238,7 +238,9 @@ impl UnicodeEscape<'_> {
             ch if ch.is_ascii() => {
                 write!(formatter, "\\x{:02x}", ch as u8)
             }
-            ch if crate::char::is_printable(ch) => formatter.write_char(ch),
+            ch if rustpython_unicode::classify::is_repr_printable(ch as u32) => {
+                formatter.write_char(ch)
+            }
             '\0'..='\u{ff}' => {
                 write!(formatter, "\\x{:02x}", ch as u32)
             }

crates/literal/src/lib.rs

@@ -2,7 +2,6 @@
 
 extern crate alloc;
 
-pub mod char;
 pub mod complex;
 pub mod escape;
 pub mod float;

crates/pylib/Cargo.toml

@@ -2,7 +2,7 @@
 name = "rustpython-pylib"
 description = "A subset of the Python standard library for use with RustPython"
 license-file = "Lib/PSF-LICENSE"
-include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"]
+include = ["Cargo.toml", "build.rs", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"]
 authors = ["CPython Developers"]
 version.workspace = true
 edition.workspace = true

crates/sre_engine/Cargo.toml

@@ -15,6 +15,7 @@ name = "benches"
 harness = false
 
 [dependencies]
+rustpython-unicode = { workspace = true, default-features = false }
 rustpython-wtf8 = { workspace = true }
 num_enum = { workspace = true }
 bitflags = { workspace = true }

crates/sre_engine/src/engine.rs

@@ -1,14 +1,10 @@
 // good luck to those that follow; here be dragons
 
-use crate::string::{
-    is_digit, is_linebreak, is_loc_word, is_space, is_uni_digit, is_uni_linebreak, is_uni_space,
-    is_uni_word, is_word, lower_ascii, lower_locate, lower_unicode, upper_locate, upper_unicode,
-};
-
 use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor};
 use alloc::{vec, vec::Vec};
 use core::{convert::TryFrom, ptr::null};
 use optional::Optioned;
+use rustpython_unicode::regex as unicode_regex;
 
 #[derive(Debug, Clone, Copy)]
 pub struct Request<'a, S> {
@@ -659,10 +655,10 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
                         }
                         SreOpcode::IN => general_op_in!(charset),
                         SreOpcode::IN_IGNORE => {
-                            general_op_in!(|set, c| charset(set, lower_ascii(c)))
+                            general_op_in!(|set, c| charset(set, unicode_regex::lower_ascii(c)))
                         }
                         SreOpcode::IN_UNI_IGNORE => {
-                            general_op_in!(|set, c| charset(set, lower_unicode(c)))
+                            general_op_in!(|set, c| charset(set, unicode_regex::lower_unicode(c)))
                         }
                         SreOpcode::IN_LOC_IGNORE => general_op_in!(charset_loc_ignore),
                         SreOpcode::MARK => {
@@ -803,25 +799,31 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
                         SreOpcode::LITERAL => general_op_literal!(|code, c| code == c),
                         SreOpcode::NOT_LITERAL => general_op_literal!(|code, c| code != c),
                         SreOpcode::LITERAL_IGNORE => {
-                            general_op_literal!(|code, c| code == lower_ascii(c))
+                            general_op_literal!(|code, c| code == unicode_regex::lower_ascii(c))
                         }
                         SreOpcode::NOT_LITERAL_IGNORE => {
-                            general_op_literal!(|code, c| code != lower_ascii(c))
+                            general_op_literal!(|code, c| code != unicode_regex::lower_ascii(c))
                         }
                         SreOpcode::LITERAL_UNI_IGNORE => {
-                            general_op_literal!(|code, c| code == lower_unicode(c))
+                            general_op_literal!(|code, c| code == unicode_regex::lower_unicode(c))
                         }
                         SreOpcode::NOT_LITERAL_UNI_IGNORE => {
-                            general_op_literal!(|code, c| code != lower_unicode(c))
+                            general_op_literal!(|code, c| code != unicode_regex::lower_unicode(c))
                         }
                         SreOpcode::LITERAL_LOC_IGNORE => general_op_literal!(char_loc_ignore),
                         SreOpcode::NOT_LITERAL_LOC_IGNORE => {
                             general_op_literal!(|code, c| !char_loc_ignore(code, c))
                         }
                         SreOpcode::GROUPREF => general_op_groupref!(|x| x),
-                        SreOpcode::GROUPREF_IGNORE => general_op_groupref!(lower_ascii),
-                        SreOpcode::GROUPREF_LOC_IGNORE => general_op_groupref!(lower_locate),
-                        SreOpcode::GROUPREF_UNI_IGNORE => general_op_groupref!(lower_unicode),
+                        SreOpcode::GROUPREF_IGNORE => {
+                            general_op_groupref!(unicode_regex::lower_ascii)
+                        }
+                        SreOpcode::GROUPREF_LOC_IGNORE => {
+                            general_op_groupref!(unicode_regex::lower_locale)
+                        }
+                        SreOpcode::GROUPREF_UNI_IGNORE => {
+                            general_op_groupref!(unicode_regex::lower_unicode)
+                        }
                         SreOpcode::GROUPREF_EXISTS => {
                             let (group_start, group_end) =
                                 state.marks.get(ctx.peek_code(req, 1) as usize);
@@ -1125,7 +1127,7 @@ impl MatchContext {
     }
 
     fn at_linebreak<S: StrDrive>(&self, req: &Request<'_, S>) -> bool {
-        !self.at_end(req) && is_linebreak(self.peek_char::<S>())
+        !self.at_end(req) && unicode_regex::is_linebreak(self.peek_char::<S>())
     }
 
     fn at_boundary<S: StrDrive, F: FnMut(u32) -> bool>(
@@ -1192,54 +1194,56 @@ impl MatchContext {
 fn at<S: StrDrive>(req: &Request<'_, S>, ctx: &MatchContext, at_code: SreAtCode) -> bool {
     match at_code {
         SreAtCode::BEGINNING | SreAtCode::BEGINNING_STRING => ctx.at_beginning(),
-        SreAtCode::BEGINNING_LINE => ctx.at_beginning() || is_linebreak(ctx.back_peek_char::<S>()),
-        SreAtCode::BOUNDARY => ctx.at_boundary(req, is_word),
-        SreAtCode::NON_BOUNDARY => ctx.at_non_boundary(req, is_word),
+        SreAtCode::BEGINNING_LINE => {
+            ctx.at_beginning() || unicode_regex::is_linebreak(ctx.back_peek_char::<S>())
+        }
+        SreAtCode::BOUNDARY => ctx.at_boundary(req, unicode_regex::is_word),
+        SreAtCode::NON_BOUNDARY => ctx.at_non_boundary(req, unicode_regex::is_word),
         SreAtCode::END => {
             (ctx.remaining_chars(req) == 1 && ctx.at_linebreak(req)) || ctx.at_end(req)
         }
         SreAtCode::END_LINE => ctx.at_linebreak(req) || ctx.at_end(req),
         SreAtCode::END_STRING => ctx.at_end(req),
-        SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, is_loc_word),
-        SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, is_loc_word),
-        SreAtCode::UNI_BOUNDARY => ctx.at_boundary(req, is_uni_word),
-        SreAtCode::UNI_NON_BOUNDARY => ctx.at_non_boundary(req, is_uni_word),
+        SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, unicode_regex::is_locale_word),
+        SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, unicode_regex::is_locale_word),
+        SreAtCode::UNI_BOUNDARY => ctx.at_boundary(req, unicode_regex::is_unicode_word),
+        SreAtCode::UNI_NON_BOUNDARY => ctx.at_non_boundary(req, unicode_regex::is_unicode_word),
     }
 }
 
 fn char_loc_ignore(code: u32, c: u32) -> bool {
-    code == c || code == lower_locate(c) || code == upper_locate(c)
+    code == c || code == unicode_regex::lower_locale(c) || code == unicode_regex::upper_locale(c)
 }
 
 fn charset_loc_ignore(set: &[u32], c: u32) -> bool {
-    let lo = lower_locate(c);
+    let lo = unicode_regex::lower_locale(c);
     if charset(set, c) {
         return true;
     }
-    let up = upper_locate(c);
+    let up = unicode_regex::upper_locale(c);
     up != lo && charset(set, up)
 }
 
 fn category(cat_code: SreCatCode, c: u32) -> bool {
     match cat_code {
-        SreCatCode::DIGIT => is_digit(c),
-        SreCatCode::NOT_DIGIT => !is_digit(c),
-        SreCatCode::SPACE => is_space(c),
-        SreCatCode::NOT_SPACE => !is_space(c),
-        SreCatCode::WORD => is_word(c),
-        SreCatCode::NOT_WORD => !is_word(c),
-        SreCatCode::LINEBREAK => is_linebreak(c),
-        SreCatCode::NOT_LINEBREAK => !is_linebreak(c),
-        SreCatCode::LOC_WORD => is_loc_word(c),
-        SreCatCode::LOC_NOT_WORD => !is_loc_word(c),
-        SreCatCode::UNI_DIGIT => is_uni_digit(c),
-        SreCatCode::UNI_NOT_DIGIT => !is_uni_digit(c),
-        SreCatCode::UNI_SPACE => is_uni_space(c),
-        SreCatCode::UNI_NOT_SPACE => !is_uni_space(c),
-        SreCatCode::UNI_WORD => is_uni_word(c),
-        SreCatCode::UNI_NOT_WORD => !is_uni_word(c),
-        SreCatCode::UNI_LINEBREAK => is_uni_linebreak(c),
-        SreCatCode::UNI_NOT_LINEBREAK => !is_uni_linebreak(c),
+        SreCatCode::DIGIT => unicode_regex::is_digit(c),
+        SreCatCode::NOT_DIGIT => !unicode_regex::is_digit(c),
+        SreCatCode::SPACE => unicode_regex::is_space(c),
+        SreCatCode::NOT_SPACE => !unicode_regex::is_space(c),
+        SreCatCode::WORD => unicode_regex::is_word(c),
+        SreCatCode::NOT_WORD => !unicode_regex::is_word(c),
+        SreCatCode::LINEBREAK => unicode_regex::is_linebreak(c),
+        SreCatCode::NOT_LINEBREAK => !unicode_regex::is_linebreak(c),
+        SreCatCode::LOC_WORD => unicode_regex::is_locale_word(c),
+        SreCatCode::LOC_NOT_WORD => !unicode_regex::is_locale_word(c),
+        SreCatCode::UNI_DIGIT => unicode_regex::is_unicode_digit(c),
+        SreCatCode::UNI_NOT_DIGIT => !unicode_regex::is_unicode_digit(c),
+        SreCatCode::UNI_SPACE => unicode_regex::is_unicode_space(c),
+        SreCatCode::UNI_NOT_SPACE => !unicode_regex::is_unicode_space(c),
+        SreCatCode::UNI_WORD => unicode_regex::is_unicode_word(c),
+        SreCatCode::UNI_NOT_WORD => !unicode_regex::is_unicode_word(c),
+        SreCatCode::UNI_LINEBREAK => unicode_regex::is_unicode_linebreak(c),
+        SreCatCode::UNI_NOT_LINEBREAK => !unicode_regex::is_unicode_linebreak(c),
     }
 }
 
@@ -1320,7 +1324,7 @@ fn charset(set: &[u32], ch: u32) -> bool {
                 if set[i + 1] <= ch && ch <= set[i + 2] {
                     return ok;
                 }
-                let ch = upper_unicode(ch);
+                let ch = unicode_regex::upper_unicode(ch);
                 if set[i + 1] <= ch && ch <= set[i + 2] {
                     return ok;
                 }
@@ -1368,10 +1372,14 @@ fn _count<S: StrDrive>(
             general_count_literal(req, ctx, end, |code, c| code != c);
         }
         SreOpcode::LITERAL_IGNORE => {
-            general_count_literal(req, ctx, end, |code, c| code == lower_ascii(c));
+            general_count_literal(req, ctx, end, |code, c| {
+                code == unicode_regex::lower_ascii(c)
+            });
         }
         SreOpcode::NOT_LITERAL_IGNORE => {
-            general_count_literal(req, ctx, end, |code, c| code != lower_ascii(c));
+            general_count_literal(req, ctx, end, |code, c| {
+                code != unicode_regex::lower_ascii(c)
+            });
         }
         SreOpcode::LITERAL_LOC_IGNORE => {
             general_count_literal(req, ctx, end, char_loc_ignore);
@@ -1380,10 +1388,14 @@ fn _count<S: StrDrive>(
             general_count_literal(req, ctx, end, |code, c| !char_loc_ignore(code, c));
         }
         SreOpcode::LITERAL_UNI_IGNORE => {
-            general_count_literal(req, ctx, end, |code, c| code == lower_unicode(c));
+            general_count_literal(req, ctx, end, |code, c| {
+                code == unicode_regex::lower_unicode(c)
+            });
         }
         SreOpcode::NOT_LITERAL_UNI_IGNORE => {
-            general_count_literal(req, ctx, end, |code, c| code != lower_unicode(c));
+            general_count_literal(req, ctx, end, |code, c| {
+                code != unicode_regex::lower_unicode(c)
+            });
         }
         _ => {
             /* General case */

crates/sre_engine/src/string.rs

@@ -331,136 +331,3 @@ const fn utf8_is_cont_byte(byte: u8) -> bool {
 
 /// Mask of the value bits of a continuation byte.
 const CONT_MASK: u8 = 0b0011_1111;
-
-const fn is_py_ascii_whitespace(b: u8) -> bool {
-    matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B')
-}
-
-#[inline]
-pub(crate) fn is_word(ch: u32) -> bool {
-    ch == '_' as u32
-        || u8::try_from(ch)
-            .map(|x| x.is_ascii_alphanumeric())
-            .unwrap_or(false)
-}
-#[inline]
-pub(crate) fn is_space(ch: u32) -> bool {
-    u8::try_from(ch)
-        .map(is_py_ascii_whitespace)
-        .unwrap_or(false)
-}
-#[inline]
-pub(crate) fn is_digit(ch: u32) -> bool {
-    u8::try_from(ch)
-        .map(|x| x.is_ascii_digit())
-        .unwrap_or(false)
-}
-#[inline]
-pub(crate) fn is_loc_alnum(ch: u32) -> bool {
-    // FIXME: Ignore the locales
-    u8::try_from(ch)
-        .map(|x| x.is_ascii_alphanumeric())
-        .unwrap_or(false)
-}
-#[inline]
-pub(crate) fn is_loc_word(ch: u32) -> bool {
-    ch == '_' as u32 || is_loc_alnum(ch)
-}
-#[inline]
-pub(crate) const fn is_linebreak(ch: u32) -> bool {
-    ch == '\n' as u32
-}
-#[inline]
-pub fn lower_ascii(ch: u32) -> u32 {
-    u8::try_from(ch)
-        .map(|x| x.to_ascii_lowercase() as u32)
-        .unwrap_or(ch)
-}
-#[inline]
-pub(crate) fn lower_locate(ch: u32) -> u32 {
-    // FIXME: Ignore the locales
-    lower_ascii(ch)
-}
-#[inline]
-pub(crate) fn upper_locate(ch: u32) -> u32 {
-    // FIXME: Ignore the locales
-    u8::try_from(ch)
-        .map(|x| x.to_ascii_uppercase() as u32)
-        .unwrap_or(ch)
-}
-#[inline]
-pub(crate) fn is_uni_digit(ch: u32) -> bool {
-    // TODO: check with cpython
-    char::try_from(ch)
-        .map(|x| x.is_ascii_digit())
-        .unwrap_or(false)
-}
-#[inline]
-pub(crate) fn is_uni_space(ch: u32) -> bool {
-    // TODO: check with cpython
-    is_space(ch)
-        || matches!(
-            ch,
-            0x0009
-                | 0x000A
-                | 0x000B
-                | 0x000C
-                | 0x000D
-                | 0x001C
-                | 0x001D
-                | 0x001E
-                | 0x001F
-                | 0x0020
-                | 0x0085
-                | 0x00A0
-                | 0x1680
-                | 0x2000
-                | 0x2001
-                | 0x2002
-                | 0x2003
-                | 0x2004
-                | 0x2005
-                | 0x2006
-                | 0x2007
-                | 0x2008
-                | 0x2009
-                | 0x200A
-                | 0x2028
-                | 0x2029
-                | 0x202F
-                | 0x205F
-                | 0x3000
-        )
-}
-#[inline]
-pub(crate) const fn is_uni_linebreak(ch: u32) -> bool {
-    matches!(
-        ch,
-        0x000A | 0x000B | 0x000C | 0x000D | 0x001C | 0x001D | 0x001E | 0x0085 | 0x2028 | 0x2029
-    )
-}
-#[inline]
-pub(crate) fn is_uni_alnum(ch: u32) -> bool {
-    // TODO: check with cpython
-    char::try_from(ch)
-        .map(|x| x.is_alphanumeric())
-        .unwrap_or(false)
-}
-#[inline]
-pub(crate) fn is_uni_word(ch: u32) -> bool {
-    ch == '_' as u32 || is_uni_alnum(ch)
-}
-#[inline]
-pub fn lower_unicode(ch: u32) -> u32 {
-    // TODO: check with cpython
-    char::try_from(ch)
-        .map(|x| x.to_lowercase().next().unwrap() as u32)
-        .unwrap_or(ch)
-}
-#[inline]
-pub fn upper_unicode(ch: u32) -> u32 {
-    // TODO: check with cpython
-    char::try_from(ch)
-        .map(|x| x.to_uppercase().next().unwrap() as u32)
-        .unwrap_or(ch)
-}

crates/stdlib/Cargo.toml

@@ -28,6 +28,7 @@ flame-it = ["flame"]
 [dependencies]
 # rustpython crates
 rustpython-derive = { workspace = true }
+rustpython-unicode = { workspace = true, features = ["casefold"] }
 rustpython-vm = { workspace = true, default-features = false, features = ["compiler"]}
 rustpython-common = { workspace = true }
 
@@ -76,16 +77,6 @@ pbkdf2 = { version = "0.12", features = ["hmac"] }
 constant_time_eq = { workspace = true }
 
 ## unicode stuff
-unicode_names2 = { workspace = true }
-# update version all at the same time
-unic-char-property = { workspace = true }
-unic-normal        = { workspace = true }
-unic-ucd-bidi      = { workspace = true }
-unic-ucd-category  = { workspace = true }
-unic-ucd-age       = { workspace = true }
-ucd = "0.1.1"
-unicode-bidi-mirroring = { workspace = true }
-
 # compression
 adler32 = "1.2.0"
 crc32fast = "1.3.2"

crates/stdlib/src/unicodedata.rs

@@ -6,55 +6,30 @@
 
 pub(crate) use unicodedata::module_def;
 
-use crate::vm::{
-    PyObject, PyResult, VirtualMachine, builtins::PyStr, convert::TryFromBorrowedObject,
-};
-
-enum NormalizeForm {
-    Nfc,
-    Nfkc,
-    Nfd,
-    Nfkd,
-}
-
-impl<'a> TryFromBorrowedObject<'a> for NormalizeForm {
-    fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> {
-        obj.try_value_with(
-            |form: &PyStr| match form.as_bytes() {
-                b"NFC" => Ok(Self::Nfc),
-                b"NFKC" => Ok(Self::Nfkc),
-                b"NFD" => Ok(Self::Nfd),
-                b"NFKD" => Ok(Self::Nfkd),
-                _ => Err(vm.new_value_error("invalid normalization form")),
-            },
-            vm,
-        )
-    }
-}
-
 #[pymodule]
 mod unicodedata {
-    use super::NormalizeForm::*;
     use crate::vm::{
         Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
         builtins::{PyModule, PyStrRef},
         function::OptionalArg,
     };
+
     use itertools::Itertools;
     use rustpython_common::wtf8::{CodePoint, Wtf8Buf};
-    use ucd::{Codepoint, DecompositionType, EastAsianWidth, Number, NumericType};
-    use unic_char_property::EnumeratedCharProperty;
-    use unic_normal::StrNormalForm;
-    use unic_ucd_age::{Age, UNICODE_VERSION, UnicodeVersion};
-    use unic_ucd_bidi::BidiClass;
-    use unic_ucd_category::GeneralCategory;
-    use unicode_bidi_mirroring::is_mirroring;
+    use rustpython_unicode::{NormalizeForm, UNICODE_VERSION, UnicodeVersion, data};
+
+    fn parse_normalize_form(form: PyStrRef, vm: &VirtualMachine) -> PyResult<NormalizeForm> {
+        form.to_str()
+            .ok_or_else(|| vm.new_value_error("invalid normalization form"))?
+            .parse()
+            .map_err(|()| vm.new_value_error("invalid normalization form"))
+    }
 
     pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
         __module_exec(vm, module);
 
         // Add UCD methods as module-level functions
-        let ucd: PyObjectRef = Ucd::new(UNICODE_VERSION).into_ref(&vm.ctx).into();
+        let ucd: PyObjectRef = PyUcd::new(data::Ucd::default()).into_ref(&vm.ctx).into();
 
         for attr in [
             "category",
@@ -80,56 +55,40 @@ mod unicodedata {
     #[pyattr]
     #[pyclass(name = "UCD")]
     #[derive(Debug, PyPayload)]
-    pub(super) struct Ucd {
-        unic_version: UnicodeVersion,
-    }
+    pub(super) struct PyUcd(data::Ucd);
 
-    impl Ucd {
-        pub const fn new(unic_version: UnicodeVersion) -> Self {
-            Self { unic_version }
+    impl PyUcd {
+        pub const fn new(ucd: data::Ucd) -> Self {
+            Self(ucd)
         }
 
-        fn check_age(&self, c: CodePoint) -> bool {
-            c.to_char()
-                .is_none_or(|c| Age::of(c).is_some_and(|age| age.actual() <= self.unic_version))
-        }
-
-        fn extract_char(
-            &self,
-            character: PyStrRef,
-            vm: &VirtualMachine,
-        ) -> PyResult<Option<CodePoint>> {
-            let c = character
+        fn extract_char(character: PyStrRef, vm: &VirtualMachine) -> PyResult<CodePoint> {
+            character
                 .as_wtf8()
                 .code_points()
                 .exactly_one()
-                .map_err(|_| vm.new_type_error("argument must be an unicode character, not str"))?;
-
-            Ok(self.check_age(c).then_some(c))
+                .map_err(|_| vm.new_type_error("argument must be a Unicode character, not str"))
         }
     }
 
     #[pyclass(flags(DISALLOW_INSTANTIATION))]
-    impl Ucd {
+    impl PyUcd {
         #[pymethod]
         fn category(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<String> {
             Ok(self
-                .extract_char(character, vm)?
-                .map_or(GeneralCategory::Unassigned, |c| {
-                    c.to_char()
-                        .map_or(GeneralCategory::Surrogate, GeneralCategory::of)
-                })
-                .abbr_name()
+                .0
+                .category(Self::extract_char(character, vm)?.to_u32())
                 .to_owned())
         }
 
         #[pymethod]
         fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<String> {
             if let Some(name_str) = name.to_str()
-                && let Some(character) = unicode_names2::character(name_str)
-                && self.check_age(character.into())
+                && let Some(character) = self.0.lookup(name_str)
             {
-                return Ok(character.to_string());
+                return Ok(char::from_u32(character)
+                    .expect("unicode_names2 only returns Unicode scalar values")
+                    .to_string());
             }
             Err(vm.new_key_error(
                 vm.ctx
@@ -145,13 +104,8 @@ mod unicodedata {
             default: OptionalArg<PyObjectRef>,
             vm: &VirtualMachine,
         ) -> PyResult {
-            let c = self.extract_char(character, vm)?;
-
-            if let Some(c) = c
-                && self.check_age(c)
-                && let Some(name) = c.to_char().and_then(unicode_names2::name)
-            {
-                return Ok(vm.ctx.new_str(name.to_string()).into());
+            if let Some(name) = self.0.name(Self::extract_char(character, vm)?.to_u32()) {
+                return Ok(vm.ctx.new_str(name).into());
             }
             default.ok_or_else(|| vm.new_value_error("no such name"))
         }
@@ -162,14 +116,9 @@ mod unicodedata {
             character: PyStrRef,
             vm: &VirtualMachine,
         ) -> PyResult<&'static str> {
-            let bidi = match self.extract_char(character, vm)? {
-                Some(c) => c
-                    .to_char()
-                    .map_or(BidiClass::LeftToRight, BidiClass::of)
-                    .abbr_name(),
-                None => "",
-            };
-            Ok(bidi)
+            Ok(self
+                .0
+                .bidirectional(Self::extract_char(character, vm)?.to_u32()))
         }
 
         /// NOTE: This function uses 9.0.0 database instead of 3.2.0
@@ -180,76 +129,51 @@ mod unicodedata {
             vm: &VirtualMachine,
         ) -> PyResult<&'static str> {
             Ok(self
-                .extract_char(character, vm)?
-                .and_then(|c| c.to_char())
-                .map_or(EastAsianWidth::Neutral, |c| c.east_asian_width())
-                .abbr_name())
+                .0
+                .east_asian_width(Self::extract_char(character, vm)?.to_u32()))
         }
 
         #[pymethod]
-        fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult<Wtf8Buf> {
-            let text = unistr.as_wtf8();
-            let normalized_text = match form {
-                Nfc => text.map_utf8(|s| s.nfc()).collect(),
-                Nfkc => text.map_utf8(|s| s.nfkc()).collect(),
-                Nfd => text.map_utf8(|s| s.nfd()).collect(),
-                Nfkd => text.map_utf8(|s| s.nfkd()).collect(),
-            };
-            Ok(normalized_text)
+        fn normalize(
+            &self,
+            form: PyStrRef,
+            unistr: PyStrRef,
+            vm: &VirtualMachine,
+        ) -> PyResult<Wtf8Buf> {
+            Ok(self
+                .0
+                .normalize(parse_normalize_form(form, vm)?, unistr.as_wtf8()))
         }
 
         #[pymethod]
-        fn is_normalized(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult<bool> {
-            let text = unistr.as_wtf8();
-            let normalized: Wtf8Buf = match form {
-                Nfc => text.map_utf8(|s| s.nfc()).collect(),
-                Nfkc => text.map_utf8(|s| s.nfkc()).collect(),
-                Nfd => text.map_utf8(|s| s.nfd()).collect(),
-                Nfkd => text.map_utf8(|s| s.nfkd()).collect(),
-            };
-            Ok(text == &*normalized)
+        fn is_normalized(
+            &self,
+            form: PyStrRef,
+            unistr: PyStrRef,
+            vm: &VirtualMachine,
+        ) -> PyResult<bool> {
+            Ok(self
+                .0
+                .is_normalized(parse_normalize_form(form, vm)?, unistr.as_wtf8()))
         }
 
         #[pymethod]
         fn mirrored(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<i32> {
-            match self.extract_char(character, vm)? {
-                Some(c) => {
-                    if let Some(ch) = c.to_char() {
-                        // Check if the character is mirrored in bidirectional text using Unicode standard
-                        Ok(if is_mirroring(ch) { 1 } else { 0 })
-                    } else {
-                        Ok(0)
-                    }
-                }
-                None => Ok(0),
-            }
+            Ok(self.0.mirrored(Self::extract_char(character, vm)?.to_u32()) as i32)
         }
 
         #[pymethod]
-        fn combining(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<i32> {
+        fn combining(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<u8> {
             Ok(self
-                .extract_char(character, vm)?
-                .and_then(|c| c.to_char())
-                .map_or(0, |ch| ch.canonical_combining_class() as i32))
+                .0
+                .combining(Self::extract_char(character, vm)?.to_u32()))
         }
 
         #[pymethod]
         fn decomposition(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<String> {
-            let ch = match self.extract_char(character, vm)?.and_then(|c| c.to_char()) {
-                Some(ch) => ch,
-                None => return Ok(String::new()),
-            };
-            let chars: Vec<char> = ch.decomposition_map().collect();
-            // If decomposition maps to just the character itself, there's no decomposition
-            if chars.len() == 1 && chars[0] == ch {
-                return Ok(String::new());
-            }
-            let hex_parts = chars.iter().map(|c| format!("{:04X}", *c as u32)).join(" ");
-            let tag = match ch.decomposition_type() {
-                Some(DecompositionType::Canonical) | None => return Ok(hex_parts),
-                Some(dt) => decomposition_type_tag(dt),
-            };
-            Ok(format!("<{tag}> {hex_parts}"))
+            Ok(self
+                .0
+                .decomposition(Self::extract_char(character, vm)?.to_u32()))
         }
 
         #[pymethod]
@@ -259,15 +183,8 @@ mod unicodedata {
             default: OptionalArg<PyObjectRef>,
             vm: &VirtualMachine,
         ) -> PyResult {
-            let ch = self.extract_char(character, vm)?.and_then(|c| c.to_char());
-            if let Some(ch) = ch
-                && matches!(
-                    ch.numeric_type(),
-                    Some(NumericType::Decimal) | Some(NumericType::Digit)
-                )
-                && let Some(Number::Integer(n)) = ch.numeric_value()
-            {
-                return Ok(vm.ctx.new_int(n).into());
+            if let Some(value) = self.0.digit(Self::extract_char(character, vm)?.to_u32()) {
+                return Ok(vm.ctx.new_int(value).into());
             }
             default.ok_or_else(|| vm.new_value_error("not a digit"))
         }
@@ -279,12 +196,8 @@ mod unicodedata {
             default: OptionalArg<PyObjectRef>,
             vm: &VirtualMachine,
         ) -> PyResult {
-            let ch = self.extract_char(character, vm)?.and_then(|c| c.to_char());
-            if let Some(ch) = ch
-                && ch.numeric_type() == Some(NumericType::Decimal)
-                && let Some(Number::Integer(n)) = ch.numeric_value()
-            {
-                return Ok(vm.ctx.new_int(n).into());
+            if let Some(value) = self.0.decimal(Self::extract_char(character, vm)?.to_u32()) {
+                return Ok(vm.ctx.new_int(value).into());
             }
             default.ok_or_else(|| vm.new_value_error("not a decimal"))
         }
@@ -296,75 +209,29 @@ mod unicodedata {
             default: OptionalArg<PyObjectRef>,
             vm: &VirtualMachine,
         ) -> PyResult {
-            let ch = self.extract_char(character, vm)?.and_then(|c| c.to_char());
-            if let Some(ch) = ch {
-                match ch.numeric_value() {
-                    Some(Number::Integer(n)) => {
-                        return Ok(vm.ctx.new_float(n as f64).into());
-                    }
-                    Some(Number::Rational(num, den)) => {
-                        return Ok(vm.ctx.new_float(num as f64 / den as f64).into());
-                    }
-                    None => {}
-                }
+            if let Some(value) = self.0.numeric(Self::extract_char(character, vm)?.to_u32()) {
+                let value = match value {
+                    data::NumericValue::Integer(n) => n as f64,
+                    data::NumericValue::Rational(num, den) => num as f64 / den as f64,
+                };
+                return Ok(vm.ctx.new_float(value).into());
             }
             default.ok_or_else(|| vm.new_value_error("not a numeric character"))
         }
 
         #[pygetset]
         fn unidata_version(&self) -> String {
-            self.unic_version.to_string()
-        }
-    }
-
-    fn decomposition_type_tag(dt: DecompositionType) -> &'static str {
-        match dt {
-            DecompositionType::Canonical => "canonical",
-            DecompositionType::Compat => "compat",
-            DecompositionType::Circle => "circle",
-            DecompositionType::Final => "final",
-            DecompositionType::Font => "font",
-            DecompositionType::Fraction => "fraction",
-            DecompositionType::Initial => "initial",
-            DecompositionType::Isolated => "isolated",
-            DecompositionType::Medial => "medial",
-            DecompositionType::Narrow => "narrow",
-            DecompositionType::Nobreak => "noBreak",
-            DecompositionType::Small => "small",
-            DecompositionType::Square => "square",
-            DecompositionType::Sub => "sub",
-            DecompositionType::Super => "super",
-            DecompositionType::Vertical => "vertical",
-            DecompositionType::Wide => "wide",
-        }
-    }
-
-    trait EastAsianWidthAbbrName {
-        fn abbr_name(&self) -> &'static str;
-    }
-
-    impl EastAsianWidthAbbrName for EastAsianWidth {
-        fn abbr_name(&self) -> &'static str {
-            match self {
-                Self::Narrow => "Na",
-                Self::Wide => "W",
-                Self::Neutral => "N",
-                Self::Ambiguous => "A",
-                Self::FullWidth => "F",
-                Self::HalfWidth => "H",
-            }
+            self.0.unicode_version().to_string()
         }
     }
 
     #[pyattr]
-    fn ucd_3_2_0(vm: &VirtualMachine) -> PyRef<Ucd> {
-        Ucd {
-            unic_version: UnicodeVersion {
-                major: 3,
-                minor: 2,
-                micro: 0,
-            },
-        }
+    fn ucd_3_2_0(vm: &VirtualMachine) -> PyRef<PyUcd> {
+        PyUcd::new(data::Ucd::new(UnicodeVersion {
+            major: 3,
+            minor: 2,
+            micro: 0,
+        }))
         .into_ref(&vm.ctx)
     }
 

crates/unicode/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "rustpython-unicode"
+description = "Shared Unicode semantics and data for RustPython and related Python tooling."
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+license.workspace = true
+
+[features]
+default = ["std", "casefold"]
+std = []
+casefold = ["std", "dep:caseless"]
+
+[dependencies]
+rustpython-wtf8 = { workspace = true }
+
+icu_normalizer = { workspace = true }
+icu_properties = { workspace = true }
+itertools = { workspace = true }
+unicode-casing = { workspace = true }
+unicode_names2 = { version = "2.0.0", default-features = false, features = ["no_std"] }
+unic-ucd-age = { workspace = true }
+ucd = "0.1.1"
+caseless = { version = "0.2.2", optional = true }
+
+[lints]
+workspace = true

crates/unicode/src/case.rs

@@ -0,0 +1,111 @@
+#[cfg(feature = "casefold")]
+use alloc::string::String;
+
+#[cfg(feature = "casefold")]
+use rustpython_wtf8::Wtf8Chunk;
+use rustpython_wtf8::{Wtf8, Wtf8Buf};
+use unicode_casing::CharExt;
+
+use crate::char_from_codepoint;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct CaseMapping {
+    len: u8,
+    codepoints: [u32; 3],
+}
+
+impl CaseMapping {
+    pub const fn identity(cp: u32) -> Self {
+        Self {
+            len: 1,
+            codepoints: [cp, 0, 0],
+        }
+    }
+
+    pub const fn first(self) -> Option<u32> {
+        if self.len == 0 {
+            None
+        } else {
+            Some(self.codepoints[0])
+        }
+    }
+
+    pub fn iter(self) -> impl Iterator<Item = u32> {
+        self.codepoints.into_iter().take(usize::from(self.len))
+    }
+}
+
+fn mapping_from_chars(chars: impl Iterator<Item = char>) -> CaseMapping {
+    let mut codepoints = [0; 3];
+    let mut len = 0;
+    for ch in chars.take(codepoints.len()) {
+        codepoints[len] = ch as u32;
+        len += 1;
+    }
+    CaseMapping {
+        len: len as u8,
+        codepoints,
+    }
+}
+
+#[cfg(feature = "casefold")]
+fn mapping_from_string(text: String) -> CaseMapping {
+    mapping_from_chars(text.chars())
+}
+
+pub fn to_lowercase(cp: u32) -> CaseMapping {
+    char_from_codepoint(cp).map_or_else(
+        || CaseMapping::identity(cp),
+        |ch| mapping_from_chars(ch.to_lowercase()),
+    )
+}
+
+pub fn to_uppercase(cp: u32) -> CaseMapping {
+    char_from_codepoint(cp).map_or_else(
+        || CaseMapping::identity(cp),
+        |ch| mapping_from_chars(ch.to_uppercase()),
+    )
+}
+
+pub fn to_titlecase(cp: u32) -> CaseMapping {
+    char_from_codepoint(cp).map_or_else(
+        || CaseMapping::identity(cp),
+        |ch| mapping_from_chars(ch.to_titlecase()),
+    )
+}
+
+pub fn to_lowercase_wtf8(text: &Wtf8) -> Wtf8Buf {
+    text.map_utf8(|s| s.chars().flat_map(char::to_lowercase))
+        .collect()
+}
+
+pub fn to_uppercase_wtf8(text: &Wtf8) -> Wtf8Buf {
+    text.map_utf8(|s| s.chars().flat_map(char::to_uppercase))
+        .collect()
+}
+
+#[cfg(feature = "casefold")]
+pub fn casefold(cp: u32) -> CaseMapping {
+    char_from_codepoint(cp).map_or_else(
+        || CaseMapping::identity(cp),
+        |ch| {
+            let mut buf = [0; 4];
+            mapping_from_string(caseless::default_case_fold_str(ch.encode_utf8(&mut buf)))
+        },
+    )
+}
+
+#[cfg(feature = "casefold")]
+pub fn casefold_str(text: &str) -> String {
+    caseless::default_case_fold_str(text)
+}
+
+#[cfg(feature = "casefold")]
+pub fn casefold_wtf8(text: &Wtf8) -> Wtf8Buf {
+    text.chunks()
+        .map(|chunk| match chunk {
+            Wtf8Chunk::Utf8(s) => Wtf8Buf::from_string(casefold_str(s)),
+            Wtf8Chunk::Surrogate(c) => Wtf8Buf::from(c),
+        })
+        .collect()
+}

crates/unicode/src/classify.rs

@@ -0,0 +1,67 @@
+use icu_properties::props::{BidiClass, EnumeratedProperty, GeneralCategory};
+use ucd::{Codepoint, NumericType};
+
+use crate::{char_from_codepoint, is_surrogate};
+
+pub fn general_category(cp: u32) -> GeneralCategory {
+    if is_surrogate(cp) {
+        GeneralCategory::Surrogate
+    } else {
+        char_from_codepoint(cp).map_or(GeneralCategory::Unassigned, GeneralCategory::for_char)
+    }
+}
+
+pub fn is_alpha(cp: u32) -> bool {
+    char_from_codepoint(cp).is_some_and(char::is_alphabetic)
+}
+
+pub fn is_alnum(cp: u32) -> bool {
+    char_from_codepoint(cp).is_some_and(char::is_alphanumeric)
+}
+
+pub fn is_decimal(cp: u32) -> bool {
+    matches!(general_category(cp), GeneralCategory::DecimalNumber)
+}
+
+pub fn is_digit(cp: u32) -> bool {
+    char_from_codepoint(cp).is_some_and(|ch| {
+        matches!(
+            ch.numeric_type(),
+            Some(NumericType::Decimal) | Some(NumericType::Digit)
+        )
+    })
+}
+
+pub fn is_numeric(cp: u32) -> bool {
+    char_from_codepoint(cp).is_some_and(|ch| ch.numeric_value().is_some())
+}
+
+pub fn is_space(cp: u32) -> bool {
+    char_from_codepoint(cp).is_some_and(|ch| {
+        matches!(general_category(cp), GeneralCategory::SpaceSeparator)
+            || matches!(
+                BidiClass::for_char(ch),
+                BidiClass::WhiteSpace | BidiClass::ParagraphSeparator | BidiClass::SegmentSeparator
+            )
+    })
+}
+
+/// Python's `str.isprintable()` semantics, which treat ASCII space as printable.
+pub fn is_printable(cp: u32) -> bool {
+    cp == '\u{0020}' as u32 || is_repr_printable(cp)
+}
+
+/// Repr/escape printable semantics, which exclude all Unicode space separators.
+pub fn is_repr_printable(cp: u32) -> bool {
+    !matches!(
+        general_category(cp),
+        GeneralCategory::SpaceSeparator
+            | GeneralCategory::LineSeparator
+            | GeneralCategory::ParagraphSeparator
+            | GeneralCategory::Control
+            | GeneralCategory::Format
+            | GeneralCategory::Surrogate
+            | GeneralCategory::PrivateUse
+            | GeneralCategory::Unassigned
+    )
+}

crates/unicode/src/data.rs

@@ -0,0 +1,230 @@
+use alloc::{format, string::String, vec::Vec};
+
+use icu_properties::{
+    CodePointSetData,
+    props::{
+        BidiClass, BidiMirrored, CanonicalCombiningClass, EastAsianWidth, EnumeratedProperty,
+        NamedEnumeratedProperty,
+    },
+};
+use itertools::Itertools;
+use ucd::{Codepoint, DecompositionType, Number, NumericType};
+use unic_ucd_age::{Age, UNICODE_VERSION, UnicodeVersion};
+
+use crate::{char_from_codepoint, classify, is_surrogate};
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum NumericValue {
+    Integer(i64),
+    Rational(i64, i64),
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Ucd {
+    unic_version: UnicodeVersion,
+}
+
+impl Default for Ucd {
+    fn default() -> Self {
+        Self::new(UNICODE_VERSION)
+    }
+}
+
+impl Ucd {
+    pub const fn new(unic_version: UnicodeVersion) -> Self {
+        Self { unic_version }
+    }
+
+    pub const fn unicode_version(&self) -> UnicodeVersion {
+        self.unic_version
+    }
+
+    pub fn category(&self, cp: u32) -> &'static str {
+        if self.contains(cp) {
+            category(cp)
+        } else {
+            "Cn"
+        }
+    }
+
+    pub fn lookup(&self, name: &str) -> Option<u32> {
+        let cp = lookup(name)?;
+        self.contains(cp).then_some(cp)
+    }
+
+    pub fn name(&self, cp: u32) -> Option<String> {
+        self.contains(cp).then(|| name(cp)).flatten()
+    }
+
+    pub fn bidirectional(&self, cp: u32) -> &'static str {
+        if self.contains(cp) {
+            bidirectional(cp)
+        } else {
+            ""
+        }
+    }
+
+    pub fn east_asian_width(&self, cp: u32) -> &'static str {
+        if self.contains(cp) {
+            east_asian_width(cp)
+        } else {
+            "N"
+        }
+    }
+
+    pub fn normalize(
+        &self,
+        form: crate::NormalizeForm,
+        text: &rustpython_wtf8::Wtf8,
+    ) -> rustpython_wtf8::Wtf8Buf {
+        crate::normalize::normalize(form, text)
+    }
+
+    pub fn is_normalized(&self, form: crate::NormalizeForm, text: &rustpython_wtf8::Wtf8) -> bool {
+        crate::normalize::is_normalized(form, text)
+    }
+
+    pub fn mirrored(&self, cp: u32) -> bool {
+        self.contains(cp) && mirrored(cp)
+    }
+
+    pub fn combining(&self, cp: u32) -> u8 {
+        if self.contains(cp) { combining(cp) } else { 0 }
+    }
+
+    pub fn decomposition(&self, cp: u32) -> String {
+        if self.contains(cp) {
+            decomposition(cp)
+        } else {
+            String::new()
+        }
+    }
+
+    pub fn digit(&self, cp: u32) -> Option<u32> {
+        self.contains(cp).then(|| digit(cp)).flatten()
+    }
+
+    pub fn decimal(&self, cp: u32) -> Option<u32> {
+        self.contains(cp).then(|| decimal(cp)).flatten()
+    }
+
+    pub fn numeric(&self, cp: u32) -> Option<NumericValue> {
+        self.contains(cp).then(|| numeric(cp)).flatten()
+    }
+
+    fn contains(&self, cp: u32) -> bool {
+        is_assigned_in_version(cp, self.unic_version)
+    }
+}
+
+pub fn is_assigned_in_version(cp: u32, version: UnicodeVersion) -> bool {
+    if is_surrogate(cp) {
+        true
+    } else {
+        char_from_codepoint(cp)
+            .is_some_and(|ch| Age::of(ch).is_some_and(|age| age.actual() <= version))
+    }
+}
+
+pub fn category(cp: u32) -> &'static str {
+    classify::general_category(cp).short_name()
+}
+
+pub fn lookup(name: &str) -> Option<u32> {
+    unicode_names2::character(name).map(u32::from)
+}
+
+pub fn name(cp: u32) -> Option<String> {
+    char_from_codepoint(cp)
+        .and_then(unicode_names2::name)
+        .map(|name| name.collect())
+}
+
+pub fn bidirectional(cp: u32) -> &'static str {
+    char_from_codepoint(cp)
+        .map_or(BidiClass::LeftToRight, BidiClass::for_char)
+        .short_name()
+}
+
+pub fn east_asian_width(cp: u32) -> &'static str {
+    char_from_codepoint(cp)
+        .map_or(EastAsianWidth::Neutral, EastAsianWidth::for_char)
+        .short_name()
+}
+
+pub fn mirrored(cp: u32) -> bool {
+    char_from_codepoint(cp).is_some_and(|ch| CodePointSetData::new::<BidiMirrored>().contains(ch))
+}
+
+pub fn combining(cp: u32) -> u8 {
+    char_from_codepoint(cp).map_or(0, |ch| {
+        CanonicalCombiningClass::for_char(ch).to_icu4c_value()
+    })
+}
+
+pub fn decomposition(cp: u32) -> String {
+    let ch = match char_from_codepoint(cp) {
+        Some(ch) => ch,
+        None => return String::new(),
+    };
+    let chars: Vec<char> = ch.decomposition_map().collect();
+    if chars.len() == 1 && chars[0] == ch {
+        return String::new();
+    }
+    let hex_parts = chars.iter().map(|c| format!("{:04X}", *c as u32)).join(" ");
+    match ch.decomposition_type() {
+        Some(DecompositionType::Canonical) | None => hex_parts,
+        Some(dt) => format!("<{}> {hex_parts}", decomposition_type_tag(dt)),
+    }
+}
+
+pub fn digit(cp: u32) -> Option<u32> {
+    let ch = char_from_codepoint(cp)?;
+    if matches!(
+        ch.numeric_type(),
+        Some(NumericType::Decimal) | Some(NumericType::Digit)
+    ) && let Some(Number::Integer(value)) = ch.numeric_value()
+    {
+        return u32::try_from(value).ok();
+    }
+    None
+}
+
+pub fn decimal(cp: u32) -> Option<u32> {
+    let ch = char_from_codepoint(cp)?;
+    if ch.numeric_type() == Some(NumericType::Decimal)
+        && let Some(Number::Integer(value)) = ch.numeric_value()
+    {
+        return u32::try_from(value).ok();
+    }
+    None
+}
+
+pub fn numeric(cp: u32) -> Option<NumericValue> {
+    match char_from_codepoint(cp)?.numeric_value()? {
+        Number::Integer(value) => Some(NumericValue::Integer(value)),
+        Number::Rational(num, den) => Some(NumericValue::Rational(num.into(), den.into())),
+    }
+}
+
+fn decomposition_type_tag(dt: DecompositionType) -> &'static str {
+    match dt {
+        DecompositionType::Canonical => "canonical",
+        DecompositionType::Compat => "compat",
+        DecompositionType::Circle => "circle",
+        DecompositionType::Final => "final",
+        DecompositionType::Font => "font",
+        DecompositionType::Fraction => "fraction",
+        DecompositionType::Initial => "initial",
+        DecompositionType::Isolated => "isolated",
+        DecompositionType::Medial => "medial",
+        DecompositionType::Narrow => "narrow",
+        DecompositionType::Nobreak => "noBreak",
+        DecompositionType::Small => "small",
+        DecompositionType::Square => "square",
+        DecompositionType::Sub => "sub",
+        DecompositionType::Super => "super",
+        DecompositionType::Vertical => "vertical",
+        DecompositionType::Wide => "wide",
+    }
+}

crates/unicode/src/identifier.rs

@@ -0,0 +1,27 @@
+use icu_properties::props::{BinaryProperty, XidContinue, XidStart};
+
+use crate::char_from_codepoint;
+
+pub fn is_xid_start(cp: u32) -> bool {
+    char_from_codepoint(cp).is_some_and(XidStart::for_char)
+}
+
+pub fn is_xid_continue(cp: u32) -> bool {
+    char_from_codepoint(cp).is_some_and(XidContinue::for_char)
+}
+
+pub fn is_python_identifier_start(cp: u32) -> bool {
+    cp == '_' as u32 || is_xid_start(cp)
+}
+
+pub fn is_python_identifier_continue(cp: u32) -> bool {
+    is_xid_continue(cp)
+}
+
+pub fn is_python_identifier(text: &str) -> bool {
+    let mut chars = text.chars();
+    let is_identifier_start = chars
+        .next()
+        .is_some_and(|ch| is_python_identifier_start(ch as u32));
+    is_identifier_start && chars.all(|ch| is_python_identifier_continue(ch as u32))
+}

crates/unicode/src/lib.rs

@@ -0,0 +1,77 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+
+extern crate alloc;
+
+pub mod case;
+pub mod classify;
+pub mod data;
+pub mod identifier;
+pub mod normalize;
+pub mod regex;
+
+pub use normalize::NormalizeForm;
+pub use unic_ucd_age::{UNICODE_VERSION, UnicodeVersion};
+
+use core::char;
+
+pub(crate) fn char_from_codepoint(cp: u32) -> Option<char> {
+    char::from_u32(cp)
+}
+
+pub(crate) const fn is_surrogate(cp: u32) -> bool {
+    matches!(cp, 0xD800..=0xDFFF)
+}
+
+#[cfg(test)]
+mod tests {
+    use alloc::vec::Vec;
+    use rustpython_wtf8::Wtf8Buf;
+
+    use crate::{NormalizeForm, case, classify, data, identifier, normalize, regex};
+
+    #[test]
+    fn printable_and_repr_printable_follow_python_rules() {
+        assert!(classify::is_printable(' ' as u32));
+        assert!(!classify::is_repr_printable(' ' as u32));
+        assert!(!classify::is_printable('\n' as u32));
+    }
+
+    #[test]
+    fn identifier_and_regex_predicates_share_unicode_tables() {
+        assert!(identifier::is_python_identifier_start('_' as u32));
+        assert!(identifier::is_python_identifier("유니코드"));
+        assert!(regex::is_unicode_word('가' as u32));
+        assert!(regex::is_unicode_digit('५' as u32));
+        assert!(regex::is_unicode_space('\u{3000}' as u32));
+    }
+
+    #[test]
+    fn case_and_normalization_helpers_support_full_mappings() {
+        let upper: Vec<_> = case::to_uppercase('ß' as u32).iter().collect();
+        assert_eq!(upper, vec!['S' as u32, 'S' as u32]);
+
+        let text = Wtf8Buf::from("e\u{301}");
+        assert_eq!(
+            normalize::normalize(NormalizeForm::Nfc, &text),
+            Wtf8Buf::from("é")
+        );
+        assert!(normalize::is_normalized(
+            NormalizeForm::Nfd,
+            &normalize::normalize(NormalizeForm::Nfd, &Wtf8Buf::from("é"))
+        ));
+    }
+
+    #[test]
+    fn unicode_data_queries_match_existing_unicodedata_behavior() {
+        assert_eq!(data::category('A' as u32), "Lu");
+        assert_eq!(data::category(0xD800), "Cs");
+        assert_eq!(data::lookup("SNOWMAN"), Some('☃' as u32));
+        assert_eq!(data::name('☃' as u32).as_deref(), Some("SNOWMAN"));
+        assert_eq!(data::decimal('५' as u32), Some(5));
+        assert_eq!(data::digit('²' as u32), Some(2));
+        assert_eq!(
+            data::numeric('⅓' as u32),
+            Some(data::NumericValue::Rational(1, 3))
+        );
+    }
+}

crates/unicode/src/normalize.rs

@@ -0,0 +1,55 @@
+use core::str::FromStr;
+use icu_normalizer::{ComposingNormalizerBorrowed, DecomposingNormalizerBorrowed};
+use rustpython_wtf8::{Wtf8, Wtf8Buf};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum NormalizeForm {
+    Nfc,
+    Nfkc,
+    Nfd,
+    Nfkd,
+}
+
+impl FromStr for NormalizeForm {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "NFC" => Ok(Self::Nfc),
+            "NFKC" => Ok(Self::Nfkc),
+            "NFD" => Ok(Self::Nfd),
+            "NFKD" => Ok(Self::Nfkd),
+            _ => Err(()),
+        }
+    }
+}
+
+pub fn normalize(form: NormalizeForm, text: &Wtf8) -> Wtf8Buf {
+    match form {
+        NormalizeForm::Nfc => {
+            let normalizer = ComposingNormalizerBorrowed::new_nfc();
+            text.map_utf8(|s| normalizer.normalize_iter(s.chars()))
+                .collect()
+        }
+        NormalizeForm::Nfkc => {
+            let normalizer = ComposingNormalizerBorrowed::new_nfkc();
+            text.map_utf8(|s| normalizer.normalize_iter(s.chars()))
+                .collect()
+        }
+        NormalizeForm::Nfd => {
+            let normalizer = DecomposingNormalizerBorrowed::new_nfd();
+            text.map_utf8(|s| normalizer.normalize_iter(s.chars()))
+                .collect()
+        }
+        NormalizeForm::Nfkd => {
+            let normalizer = DecomposingNormalizerBorrowed::new_nfkd();
+            text.map_utf8(|s| normalizer.normalize_iter(s.chars()))
+                .collect()
+        }
+    }
+}
+
+pub fn is_normalized(form: NormalizeForm, text: &Wtf8) -> bool {
+    let normalized = normalize(form, text);
+    text == &*normalized
+}

crates/unicode/src/regex.rs

@@ -0,0 +1,87 @@
+use crate::{case, classify};
+
+const UNDERSCORE: u32 = '_' as u32;
+
+const fn is_py_ascii_whitespace(byte: u8) -> bool {
+    matches!(byte, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B')
+}
+
+pub fn is_word(cp: u32) -> bool {
+    cp == UNDERSCORE
+        || u8::try_from(cp)
+            .map(|byte| byte.is_ascii_alphanumeric())
+            .unwrap_or(false)
+}
+
+pub fn is_space(cp: u32) -> bool {
+    u8::try_from(cp)
+        .map(is_py_ascii_whitespace)
+        .unwrap_or(false)
+}
+
+pub fn is_digit(cp: u32) -> bool {
+    u8::try_from(cp)
+        .map(|byte| byte.is_ascii_digit())
+        .unwrap_or(false)
+}
+
+pub fn is_locale_alnum(cp: u32) -> bool {
+    u8::try_from(cp)
+        .map(|byte| byte.is_ascii_alphanumeric())
+        .unwrap_or(false)
+}
+
+pub fn is_locale_word(cp: u32) -> bool {
+    cp == UNDERSCORE || is_locale_alnum(cp)
+}
+
+pub const fn is_linebreak(cp: u32) -> bool {
+    cp == '\n' as u32
+}
+
+pub fn lower_ascii(cp: u32) -> u32 {
+    u8::try_from(cp)
+        .map(|byte| byte.to_ascii_lowercase() as u32)
+        .unwrap_or(cp)
+}
+
+pub fn lower_locale(cp: u32) -> u32 {
+    lower_ascii(cp)
+}
+
+pub fn upper_locale(cp: u32) -> u32 {
+    u8::try_from(cp)
+        .map(|byte| byte.to_ascii_uppercase() as u32)
+        .unwrap_or(cp)
+}
+
+pub fn is_unicode_digit(cp: u32) -> bool {
+    classify::is_decimal(cp)
+}
+
+pub fn is_unicode_space(cp: u32) -> bool {
+    classify::is_space(cp)
+}
+
+pub const fn is_unicode_linebreak(cp: u32) -> bool {
+    matches!(
+        cp,
+        0x000A | 0x000B | 0x000C | 0x000D | 0x001C | 0x001D | 0x001E | 0x0085 | 0x2028 | 0x2029
+    )
+}
+
+pub fn is_unicode_alnum(cp: u32) -> bool {
+    classify::is_alnum(cp)
+}
+
+pub fn is_unicode_word(cp: u32) -> bool {
+    cp == UNDERSCORE || is_unicode_alnum(cp)
+}
+
+pub fn lower_unicode(cp: u32) -> u32 {
+    case::to_lowercase(cp).first().unwrap_or(cp)
+}
+
+pub fn upper_unicode(cp: u32) -> u32 {
+    case::to_uppercase(cp).first().unwrap_or(cp)
+}

crates/vm/Cargo.toml

@@ -41,6 +41,7 @@ ruff_text_size = { workspace = true, optional = true }
 rustpython-compiler-core = { workspace = true }
 rustpython-literal = { workspace = true }
 rustpython-sre_engine = { workspace = true }
+rustpython-unicode = { workspace = true, features = ["casefold"] }
 
 ascii = { workspace = true }
 ahash = { workspace = true }
@@ -74,7 +75,6 @@ strum_macros = { workspace = true }
 thiserror = { workspace = true }
 memchr = { workspace = true }
 
-caseless = "0.2.2"
 flamer = { version = "0.5", optional = true }
 half = "2"
 psm = "0.1"
@@ -86,10 +86,6 @@ timsort = "0.1.2"
 # TODO: use unic for this; needed for title case:
 # https://github.com/RustPython/RustPython/pull/832#discussion_r275428939
 unicode-casing = { workspace = true }
-# update version all at the same time
-unic-ucd-bidi = { workspace = true }
-unic-ucd-category = { workspace = true }
-unic-ucd-ident = { workspace = true }
 
 [target.'cfg(unix)'.dependencies]
 rustix = { workspace = true }

crates/vm/src/anystr.rs

@@ -396,6 +396,10 @@ pub trait AnyStr {
 
     fn py_zfill(&self, width: isize) -> Vec<u8> {
         let width = width.to_usize().unwrap_or(0);
+        let char_len = self.elements().count();
+        let width = self
+            .bytes_len()
+            .saturating_add(width.saturating_sub(char_len));
         rustpython_common::str::zfill(self.as_bytes(), width)
     }
 

crates/vm/src/builtins/code.rs

@@ -232,6 +232,23 @@ fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant<'_, Literal> {
         }
         super::singletons::PyNone => BorrowedConstant::None,
         super::slice::PyEllipsis => BorrowedConstant::Ellipsis,
+        ref s @ super::slice::PySlice => {
+            // Constant pool slices always store Some() for start/step (even for None).
+            // Box::leak the array so it outlives the borrow. Leak is acceptable since
+            // constant pool objects live for the program's lifetime.
+            let start = s.start.clone().unwrap();
+            let stop = s.stop.clone();
+            let step = s.step.clone().unwrap();
+            let arr = Box::leak(Box::new([Literal(start), Literal(stop), Literal(step)]));
+            BorrowedConstant::Slice { elements: arr }
+        }
+        ref fs @ super::set::PyFrozenSet => {
+            // Box::leak the elements so they outlive the borrow. Leak is acceptable since
+            // constant pool objects live for the program's lifetime.
+            let elems: Vec<Literal> = fs.elements().into_iter().map(Literal).collect();
+            let elements = Box::leak(elems.into_boxed_slice());
+            BorrowedConstant::Frozenset { elements }
+        }
         _ => panic!("unexpected payload for constant python value"),
     })
 }
@@ -283,6 +300,30 @@ impl ConstantBag for PyObjBag<'_> {
                     .collect();
                 ctx.new_tuple(elements).into()
             }
+            BorrowedConstant::Slice { elements } => {
+                let [start, stop, step] = elements;
+                let start_obj = self.make_constant(start.borrow_constant()).0;
+                let stop_obj = self.make_constant(stop.borrow_constant()).0;
+                let step_obj = self.make_constant(step.borrow_constant()).0;
+                // Store as PySlice with Some() for all fields (even None values)
+                // so borrow_obj_constant can reference them.
+                use crate::builtins::PySlice;
+                PySlice {
+                    start: Some(start_obj),
+                    stop: stop_obj,
+                    step: Some(step_obj),
+                }
+                .into_ref(ctx)
+                .into()
+            }
+            BorrowedConstant::Frozenset { elements: _ } => {
+                // Creating a frozenset requires VirtualMachine for element hashing.
+                // PyObjBag only has Context, so we cannot construct PyFrozenSet here.
+                // Frozenset constants from .pyc are handled by PyMarshalBag which has VM access.
+                unimplemented!(
+                    "frozenset constant in PyObjBag::make_constant requires VirtualMachine"
+                )
+            }
             BorrowedConstant::None => ctx.none(),
             BorrowedConstant::Ellipsis => ctx.ellipsis.clone().into(),
         };

crates/vm/src/builtins/str.rs

@@ -41,11 +41,9 @@ use rustpython_common::{
     hash,
     lock::PyMutex,
     str::DeduceStrKind,
-    wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Chunk, Wtf8Concat},
+    wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Concat},
 };
-use unic_ucd_bidi::BidiClass;
-use unic_ucd_category::GeneralCategory;
-use unic_ucd_ident::{is_xid_continue, is_xid_start};
+
 use unicode_casing::CharExt;
 
 impl<'a> TryFromBorrowedObject<'a> for String {
@@ -697,7 +695,7 @@ impl PyStr {
         match self.as_str_kind() {
             PyKindStr::Ascii(s) => s.to_ascii_lowercase().into(),
             PyKindStr::Utf8(s) => s.to_lowercase().into(),
-            PyKindStr::Wtf8(w) => w.to_lowercase().into(),
+            PyKindStr::Wtf8(w) => rustpython_unicode::case::to_lowercase_wtf8(w).into(),
         }
     }
 
@@ -705,16 +703,9 @@ impl PyStr {
     #[pymethod]
     fn casefold(&self) -> Self {
         match self.as_str_kind() {
-            PyKindStr::Ascii(s) => caseless::default_case_fold_str(s.as_str()).into(),
-            PyKindStr::Utf8(s) => caseless::default_case_fold_str(s).into(),
-            PyKindStr::Wtf8(w) => w
-                .chunks()
-                .map(|c| match c {
-                    Wtf8Chunk::Utf8(s) => Wtf8Buf::from_string(caseless::default_case_fold_str(s)),
-                    Wtf8Chunk::Surrogate(c) => Wtf8Buf::from(c),
-                })
-                .collect::<Wtf8Buf>()
-                .into(),
+            PyKindStr::Ascii(s) => rustpython_unicode::case::casefold_str(s.as_str()).into(),
+            PyKindStr::Utf8(s) => rustpython_unicode::case::casefold_str(s).into(),
+            PyKindStr::Wtf8(w) => rustpython_unicode::case::casefold_wtf8(w).into(),
         }
     }
 
@@ -723,7 +714,7 @@ impl PyStr {
         match self.as_str_kind() {
             PyKindStr::Ascii(s) => s.to_ascii_uppercase().into(),
             PyKindStr::Utf8(s) => s.to_uppercase().into(),
-            PyKindStr::Wtf8(w) => w.to_uppercase().into(),
+            PyKindStr::Wtf8(w) => rustpython_unicode::case::to_uppercase_wtf8(w).into(),
         }
     }
 
@@ -966,7 +957,7 @@ impl PyStr {
     #[pymethod]
     fn isdecimal(&self) -> bool {
         !self.data.is_empty()
-            && self.char_all(|c| GeneralCategory::of(c) == GeneralCategory::DecimalNumber)
+            && self.char_all(|c| rustpython_unicode::classify::is_decimal(c as u32))
     }
 
     fn __mod__(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Wtf8Buf> {
@@ -1086,17 +1077,12 @@ impl PyStr {
 
     #[pymethod]
     fn isprintable(&self) -> bool {
-        self.char_all(|c| c == '\u{0020}' || rustpython_literal::char::is_printable(c))
+        self.char_all(|c| rustpython_unicode::classify::is_printable(c as u32))
     }
 
     #[pymethod]
     fn isspace(&self) -> bool {
-        use unic_ucd_bidi::bidi_class::abbr_names::*;
-        !self.data.is_empty()
-            && self.char_all(|c| {
-                GeneralCategory::of(c) == GeneralCategory::SpaceSeparator
-                    || matches!(BidiClass::of(c), WS | B | S)
-            })
+        !self.data.is_empty() && self.char_all(|c| rustpython_unicode::classify::is_space(c as u32))
     }
 
     // Return true if all cased characters in the string are lowercase and there is at least one cased character, false otherwise.
@@ -1353,11 +1339,8 @@ impl PyStr {
 
     #[pymethod]
     pub fn isidentifier(&self) -> bool {
-        let Some(s) = self.to_str() else { return false };
-        let mut chars = s.chars();
-        let is_identifier_start = chars.next().is_some_and(|c| c == '_' || is_xid_start(c));
-        // a string is not an identifier if it has whitespace or starts with a number
-        is_identifier_start && chars.all(is_xid_continue)
+        self.to_str()
+            .is_some_and(rustpython_unicode::identifier::is_python_identifier)
     }
 
     // https://docs.python.org/3/library/stdtypes.html#str.translate

crates/vm/src/exceptions.rs

@@ -2397,6 +2397,22 @@ pub(super) mod types {
                     .downcast::<crate::builtins::PyTuple>()
             {
                 let location_tup_len = location_tuple.len();
+
+                match location_tup_len {
+                    4 | 6 => {}
+                    5 => {
+                        return Err(vm.new_type_error(
+                            "end_offset must be provided when end_lineno is provided".to_owned(),
+                        ));
+                    }
+                    _ => {
+                        return Err(vm.new_type_error(format!(
+                            "function takes exactly 4 or 6 arguments ({} given)",
+                            location_tup_len
+                        )));
+                    }
+                }
+
                 for (i, &attr) in [
                     "filename",
                     "lineno",

crates/vm/src/frame.rs

@@ -1755,12 +1755,6 @@ impl ExecutingFrame<'_> {
         exc_tb: PyObjectRef,
     ) -> PyResult<ExecutionResult> {
         self.monitoring_mask = vm.state.monitoring_events.load();
-        // Reset prev_line so that LINE monitoring events fire even if
-        // the exception handler is on the same line as the yield point.
-        // In CPython, _Py_call_instrumentation_line has a special case
-        // for RESUME: it fires LINE even when prev_line == current_line.
-        // Since gen_throw bypasses RESUME, we reset prev_line instead.
-        *self.prev_line = 0;
         if let Some(jen) = self.yield_from_target() {
             // Check if the exception is GeneratorExit (type or instance).
             // For GeneratorExit, close the sub-iterator instead of throwing.
@@ -1796,7 +1790,10 @@ impl ExecutingFrame<'_> {
                     self.push_value(vm.ctx.none());
                     vm.contextualize_exception(&err);
                     return match self.unwind_blocks(vm, UnwindReason::Raising { exception: err }) {
-                        Ok(None) => self.run(vm),
+                        Ok(None) => {
+                            *self.prev_line = 0;
+                            self.run(vm)
+                        }
                         Ok(Some(result)) => Ok(result),
                         Err(exception) => Err(exception),
                     };
@@ -1838,7 +1835,10 @@ impl ExecutingFrame<'_> {
                         self.push_value(vm.ctx.none());
                         vm.contextualize_exception(&err);
                         match self.unwind_blocks(vm, UnwindReason::Raising { exception: err }) {
-                            Ok(None) => self.run(vm),
+                            Ok(None) => {
+                                *self.prev_line = 0;
+                                self.run(vm)
+                            }
                             Ok(Some(result)) => Ok(result),
                             Err(exception) => Err(exception),
                         }
@@ -1906,7 +1906,13 @@ impl ExecutingFrame<'_> {
         self.push_value(vm.ctx.none());
 
         match self.unwind_blocks(vm, UnwindReason::Raising { exception }) {
-            Ok(None) => self.run(vm),
+            Ok(None) => {
+                // Reset prev_line so that the first instruction in the handler
+                // fires a LINE event. In CPython, gen_send_ex re-enters the
+                // eval loop which reinitializes its local prev_instr tracker.
+                *self.prev_line = 0;
+                self.run(vm)
+            }
             Ok(Some(result)) => Ok(result),
             Err(exception) => {
                 // Fire PY_UNWIND: exception escapes the generator frame.
@@ -2780,8 +2786,18 @@ impl ExecutingFrame<'_> {
                         vm.ctx.exceptions.not_implemented_error.to_owned().into()
                     }
                     CommonConstant::BuiltinTuple => vm.ctx.types.tuple_type.to_owned().into(),
-                    CommonConstant::BuiltinAll => vm.builtins.get_attr("all", vm)?,
-                    CommonConstant::BuiltinAny => vm.builtins.get_attr("any", vm)?,
+                    CommonConstant::BuiltinAll => vm
+                        .callable_cache
+                        .builtin_all
+                        .clone()
+                        .expect("builtin_all not initialized"),
+                    CommonConstant::BuiltinAny => vm
+                        .callable_cache
+                        .builtin_any
+                        .clone()
+                        .expect("builtin_any not initialized"),
+                    CommonConstant::BuiltinList => vm.ctx.types.list_type.to_owned().into(),
+                    CommonConstant::BuiltinSet => vm.ctx.types.set_type.to_owned().into(),
                 };
                 self.push_value(value);
                 Ok(None)
@@ -9440,20 +9456,25 @@ impl ExecutingFrame<'_> {
                 Ok(vm.ctx.new_tuple(list.borrow_vec().to_vec()).into())
             }
             bytecode::IntrinsicFunction1::StopIterationError => {
-                // Convert StopIteration to RuntimeError
-                // Used to ensure async generators don't raise StopIteration directly
-                // _PyGen_FetchStopIterationValue
-                // Use fast_isinstance to handle subclasses of StopIteration
+                // Convert StopIteration to RuntimeError (PEP 479)
+                // Returns the exception object; RERAISE will re-raise it
                 if arg.fast_isinstance(vm.ctx.exceptions.stop_iteration) {
-                    Err(vm.new_runtime_error("coroutine raised StopIteration"))
+                    let flags = &self.code.flags;
+                    let msg = if flags
+                        .contains(bytecode::CodeFlags::COROUTINE | bytecode::CodeFlags::GENERATOR)
+                    {
+                        "async generator raised StopIteration"
+                    } else if flags.contains(bytecode::CodeFlags::COROUTINE) {
+                        "coroutine raised StopIteration"
+                    } else {
+                        "generator raised StopIteration"
+                    };
+                    let err = vm.new_runtime_error(msg);
+                    err.set___cause__(arg.downcast().ok());
+                    Ok(err.into())
                 } else {
-                    // If not StopIteration, just re-raise the original exception
-                    Err(arg.downcast().unwrap_or_else(|obj| {
-                        vm.new_runtime_error(format!(
-                            "unexpected exception type: {:?}",
-                            obj.class()
-                        ))
-                    }))
+                    // Not StopIteration, pass through for RERAISE
+                    Ok(arg)
                 }
             }
             bytecode::IntrinsicFunction1::AsyncGenWrap => {

crates/vm/src/stdlib/_sre.rs

@@ -21,10 +21,8 @@ mod _sre {
     use crossbeam_utils::atomic::AtomicCell;
     use itertools::Itertools;
     use num_traits::ToPrimitive;
-    use rustpython_sre_engine::{
-        Request, SearchIter, SreFlag, State, StrDrive,
-        string::{lower_ascii, lower_unicode, upper_unicode},
-    };
+    use rustpython_sre_engine::{Request, SearchIter, SreFlag, State, StrDrive};
+    use rustpython_unicode::regex as unicode_regex;
 
     #[pyattr]
     pub use rustpython_sre_engine::{CODESIZE, MAXGROUPS, MAXREPEAT, SRE_MAGIC as MAGIC};
@@ -42,17 +40,17 @@ mod _sre {
     #[pyfunction]
     fn unicode_iscased(ch: i32) -> bool {
         let ch = ch as u32;
-        ch != lower_unicode(ch) || ch != upper_unicode(ch)
+        ch != unicode_regex::lower_unicode(ch) || ch != unicode_regex::upper_unicode(ch)
     }
 
     #[pyfunction]
     fn ascii_tolower(ch: i32) -> i32 {
-        lower_ascii(ch as u32) as i32
+        unicode_regex::lower_ascii(ch as u32) as i32
     }
 
     #[pyfunction]
     fn unicode_tolower(ch: i32) -> i32 {
-        lower_unicode(ch as u32) as i32
+        unicode_regex::lower_unicode(ch as u32) as i32
     }
 
     trait SreStr: StrDrive {

crates/vm/src/stdlib/_symtable.rs

@@ -174,6 +174,14 @@ mod _symtable {
                 .symtable
                 .sub_tables
                 .iter()
+                .flat_map(|t| {
+                    if t.comp_inlined {
+                        // Flatten: replace inlined comprehension tables with their children
+                        t.sub_tables.iter().collect::<Vec<_>>()
+                    } else {
+                        vec![t]
+                    }
+                })
                 .map(|t| to_py_symbol_table(t.clone()).into_pyobject(vm))
                 .collect();
             Ok(children)

crates/vm/src/stdlib/sys/monitoring.rs

@@ -368,6 +368,9 @@ pub fn instrument_code(code: &PyCode, events: u32) {
         // is_line_start[i] = true if position i should have INSTRUMENTED_LINE
         let mut is_line_start = vec![false; len];
 
+        // Build NO_LOCATION mask from linetable
+        let no_loc_mask = bytecode::build_no_location_mask(&code.code.linetable, len);
+
         // First pass: mark positions where the source line changes
         let mut prev_line: Option<u32> = None;
         for (i, unit) in code
@@ -395,6 +398,10 @@ pub fn instrument_code(code: &PyCode, events: u32) {
             ) {
                 continue;
             }
+            // Skip NO_LOCATION instructions
+            if no_loc_mask.get(i).copied().unwrap_or(false) {
+                continue;
+            }
             if let Some((loc, _)) = code.code.locations.get(i) {
                 let line = loc.line.get() as u32;
                 let is_new = prev_line != Some(line);
@@ -445,6 +452,7 @@ pub fn instrument_code(code: &PyCode, events: u32) {
             if let Some(target_idx) = target
                 && target_idx < len
                 && !is_line_start[target_idx]
+                && !no_loc_mask.get(target_idx).copied().unwrap_or(false)
             {
                 let target_op = code.code.instructions[target_idx].op;
                 let target_base = target_op.to_base().map_or(target_op, |b| b);
@@ -465,7 +473,10 @@ pub fn instrument_code(code: &PyCode, events: u32) {
         // Third pass: mark exception handler targets as line starts.
         for entry in bytecode::decode_exception_table(&code.code.exceptiontable) {
             let target_idx = entry.target as usize;
-            if target_idx < len && !is_line_start[target_idx] {
+            if target_idx < len
+                && !is_line_start[target_idx]
+                && !no_loc_mask.get(target_idx).copied().unwrap_or(false)
+            {
                 let target_op = code.code.instructions[target_idx].op;
                 let target_base = target_op.to_base().map_or(target_op, |b| b);
                 if !matches!(target_base, Instruction::PopIter)

crates/vm/src/version.rs

@@ -129,8 +129,8 @@ pub fn get_git_datetime() -> String {
 }
 
 // Must be aligned to Lib/importlib/_bootstrap_external.py
-// Bumped to 2997 for MAKE_CELL/COPY_FREE_VARS prolog and cell-local merging
-pub const PYC_MAGIC_NUMBER: u16 = 2997;
+// Bumped to 2994 for new CommonConstant discriminants (BuiltinList, BuiltinSet)
+pub const PYC_MAGIC_NUMBER: u16 = 2994;
 
 // CPython format: magic_number | ('\r' << 16) | ('\n' << 24)
 // This protects against text-mode file reads

crates/vm/src/vm/mod.rs

@@ -94,7 +94,7 @@ pub struct VirtualMachine {
     pub initialized: bool,
     recursion_depth: Cell<usize>,
     /// C stack soft limit for detecting stack overflow (like c_stack_soft_limit)
-    #[cfg_attr(miri, allow(dead_code))]
+    #[cfg_attr(any(miri, target_env = "musl"), allow(dead_code))]
     c_stack_soft_limit: Cell<usize>,
     /// Async generator firstiter hook (per-thread, set via sys.set_asyncgen_hooks)
     pub async_gen_firstiter: RefCell<Option<PyObjectRef>>,
@@ -576,6 +576,8 @@ pub(crate) struct CallableCache {
     pub len: Option<PyObjectRef>,
     pub isinstance: Option<PyObjectRef>,
     pub list_append: Option<PyObjectRef>,
+    pub builtin_all: Option<PyObjectRef>,
+    pub builtin_any: Option<PyObjectRef>,
 }
 
 pub struct PyGlobalState {
@@ -641,6 +643,8 @@ impl VirtualMachine {
             .get_attr(self.ctx.intern_str("append"))
             .ok_or_else(|| self.new_runtime_error("failed to cache list.append".to_owned()))?;
         self.callable_cache.list_append = Some(list_append);
+        self.callable_cache.builtin_all = Some(self.builtins.get_attr("all", self)?);
+        self.callable_cache.builtin_any = Some(self.builtins.get_attr("any", self)?);
         Ok(())
     }
 
@@ -1420,12 +1424,12 @@ impl VirtualMachine {
 
     /// Stack margin bytes (like _PyOS_STACK_MARGIN_BYTES).
     /// 2048 * sizeof(void*) = 16KB for 64-bit.
-    #[cfg_attr(miri, allow(dead_code))]
+    #[cfg_attr(any(miri, target_env = "musl"), allow(dead_code))]
     const STACK_MARGIN_BYTES: usize = 2048 * core::mem::size_of::<usize>();
 
     /// Get the stack boundaries using platform-specific APIs.
     /// Returns (base, top) where base is the lowest address and top is the highest.
-    #[cfg(all(not(miri), windows))]
+    #[cfg(all(not(miri), not(target_env = "musl"), windows))]
     fn get_stack_bounds() -> (usize, usize) {
         use windows_sys::Win32::System::Threading::{
             GetCurrentThreadStackLimits, SetThreadStackGuarantee,
@@ -1444,7 +1448,7 @@ impl VirtualMachine {
 
     /// Get stack boundaries on non-Windows platforms.
     /// Falls back to estimating based on current stack pointer.
-    #[cfg(all(not(miri), not(windows)))]
+    #[cfg(all(not(miri), not(target_env = "musl"), not(windows)))]
     fn get_stack_bounds() -> (usize, usize) {
         // Use pthread_attr_getstack on platforms that support it
         #[cfg(any(target_os = "linux", target_os = "android"))]
@@ -1495,15 +1499,16 @@ impl VirtualMachine {
 
     /// Calculate the C stack soft limit based on actual stack boundaries.
     /// soft_limit = base + 2 * margin (for downward-growing stacks)
-    #[cfg(not(miri))]
+    #[cfg(all(not(miri), not(target_env = "musl")))]
     fn calculate_c_stack_soft_limit() -> usize {
         let (base, _top) = Self::get_stack_bounds();
-        // Soft limit is 2 margins above the base
         base + Self::STACK_MARGIN_BYTES * 2
     }
 
-    /// Miri doesn't support inline assembly, so disable C stack checking.
-    #[cfg(miri)]
+    /// Musl currently reports stack bounds in a way that trips the VM's
+    /// native stack guard during frozen stdlib bootstrap, so keep the Python
+    /// recursion limit as the only guard there.
+    #[cfg(any(miri, target_env = "musl"))]
     fn calculate_c_stack_soft_limit() -> usize {
         0
     }
@@ -1511,19 +1516,18 @@ impl VirtualMachine {
     /// Check if we're near the C stack limit (like _Py_MakeRecCheck).
     /// Returns true only when stack pointer is in the "danger zone" between
     /// soft_limit and hard_limit (soft_limit - 2*margin).
-    #[cfg(not(miri))]
+    #[cfg(all(not(miri), not(target_env = "musl")))]
     #[inline(always)]
     fn check_c_stack_overflow(&self) -> bool {
         let current_sp = psm::stack_pointer() as usize;
         let soft_limit = self.c_stack_soft_limit.get();
-        // Stack grows downward: check if we're below soft limit but above hard limit
-        // This matches CPython's _Py_MakeRecCheck behavior
         current_sp < soft_limit
             && current_sp >= soft_limit.saturating_sub(Self::STACK_MARGIN_BYTES * 2)
     }
 
-    /// Miri doesn't support inline assembly, so always return false.
-    #[cfg(miri)]
+    /// Miri does not support the native stack probe, and musl currently trips
+    /// the probe during stdlib bootstrap.
+    #[cfg(any(miri, target_env = "musl"))]
     #[inline(always)]
     fn check_c_stack_overflow(&self) -> bool {
         false

extra_tests/snippets/builtin_str.py

@@ -61,6 +61,9 @@ assert a.endswith(("A", "lo"))
 assert not a.endswith("on")
 assert not a.endswith(("A", "ll"))
 assert a.zfill(8) == "000Hallo"
+assert "á".zfill(4) == "000á"
+assert "🙂".zfill(5) == "0000🙂"
+assert "+あ".zfill(5) == "+000あ"
 assert a.isalnum()
 assert not a.isdigit()
 assert not a.isdecimal()

extra_tests/snippets/stdlib_unicode_shared.py

@@ -0,0 +1,20 @@
+import re
+import unicodedata
+
+assert "유니코드".isidentifier()
+assert "५".isdecimal()
+assert "\u3000".isspace()
+assert " ".isprintable()
+assert not "\n".isprintable()
+
+assert unicodedata.category("\ud800") == "Cs"
+assert unicodedata.lookup("SNOWMAN") == "☃"
+assert unicodedata.name("☃") == "SNOWMAN"
+assert unicodedata.normalize("NFC", "e\u0301") == "é"
+assert unicodedata.digit("²") == 2
+assert unicodedata.decimal("५") == 5
+assert unicodedata.numeric("⅓") == 1 / 3
+
+assert re.fullmatch(r"\w+", "가나다")
+assert re.fullmatch(r"\d+", "५६७")
+assert re.fullmatch(r"\s+", "\u3000")

wasm/demo/package-lock.json

@@ -21,7 +21,7 @@
                 "css-loader": "^7.1.2",
                 "html-webpack-plugin": "^5.6.3",
                 "mini-css-extract-plugin": "^2.9.2",
-                "serve": "^14.2.5",
+                "serve": "^14.2.6",
                 "webpack": "^5.105.0",
                 "webpack-cli": "^6.0.1",
                 "webpack-dev-server": "^5.2.1"
@@ -839,9 +839,9 @@
             }
         },
         "node_modules/ajv": {
-            "version": "8.17.1",
-            "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz",
-            "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
+            "version": "8.18.0",
+            "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
+            "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
             "dev": true,
             "license": "MIT",
             "dependencies": {
@@ -1129,9 +1129,9 @@
             }
         },
         "node_modules/brace-expansion": {
-            "version": "1.1.11",
-            "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-            "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+            "version": "1.1.13",
+            "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz",
+            "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==",
             "dev": true,
             "license": "MIT",
             "dependencies": {
@@ -3266,9 +3266,9 @@
             }
         },
         "node_modules/lodash": {
-            "version": "4.17.23",
-            "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz",
-            "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==",
+            "version": "4.18.1",
+            "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz",
+            "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==",
             "dev": true,
             "license": "MIT"
         },
@@ -3448,9 +3448,9 @@
             "license": "ISC"
         },
         "node_modules/minimatch": {
-            "version": "3.1.2",
-            "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
-            "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+            "version": "3.1.5",
+            "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
+            "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
             "dev": true,
             "license": "ISC",
             "dependencies": {
@@ -4003,16 +4003,6 @@
                 "node": ">= 0.10"
             }
         },
-        "node_modules/punycode": {
-            "version": "2.3.1",
-            "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
-            "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
-            "dev": true,
-            "license": "MIT",
-            "engines": {
-                "node": ">=6"
-            }
-        },
         "node_modules/qs": {
             "version": "6.13.0",
             "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
@@ -4375,14 +4365,14 @@
             }
         },
         "node_modules/serve": {
-            "version": "14.2.5",
-            "resolved": "https://registry.npmjs.org/serve/-/serve-14.2.5.tgz",
-            "integrity": "sha512-Qn/qMkzCcMFVPb60E/hQy+iRLpiU8PamOfOSYoAHmmF+fFFmpPpqa6Oci2iWYpTdOUM3VF+TINud7CfbQnsZbA==",
+            "version": "14.2.6",
+            "resolved": "https://registry.npmjs.org/serve/-/serve-14.2.6.tgz",
+            "integrity": "sha512-QEjUSA+sD4Rotm1znR8s50YqA3kYpRGPmtd5GlFxbaL9n/FdUNbqMhxClqdditSk0LlZyA/dhud6XNRTOC9x2Q==",
             "dev": true,
             "license": "MIT",
             "dependencies": {
                 "@zeit/schemas": "2.36.0",
-                "ajv": "8.12.0",
+                "ajv": "8.18.0",
                 "arg": "5.0.2",
                 "boxen": "7.0.0",
                 "chalk": "5.0.1",
@@ -4390,7 +4380,7 @@
                 "clipboardy": "3.0.0",
                 "compression": "1.8.1",
                 "is-port-reachable": "4.0.0",
-                "serve-handler": "6.1.6",
+                "serve-handler": "6.1.7",
                 "update-check": "1.5.4"
             },
             "bin": {
@@ -4401,16 +4391,16 @@
             }
         },
         "node_modules/serve-handler": {
-            "version": "6.1.6",
-            "resolved": "https://registry.npmjs.org/serve-handler/-/serve-handler-6.1.6.tgz",
-            "integrity": "sha512-x5RL9Y2p5+Sh3D38Fh9i/iQ5ZK+e4xuXRd/pGbM4D13tgo/MGwbttUk8emytcr1YYzBYs+apnUngBDFYfpjPuQ==",
+            "version": "6.1.7",
+            "resolved": "https://registry.npmjs.org/serve-handler/-/serve-handler-6.1.7.tgz",
+            "integrity": "sha512-CinAq1xWb0vR3twAv9evEU8cNWkXCb9kd5ePAHUKJBkOsUpR1wt/CvGdeca7vqumL1U5cSaeVQ6zZMxiJ3yWsg==",
             "dev": true,
             "license": "MIT",
             "dependencies": {
                 "bytes": "3.0.0",
                 "content-disposition": "0.5.2",
                 "mime-types": "2.1.18",
-                "minimatch": "3.1.2",
+                "minimatch": "3.1.5",
                 "path-is-inside": "1.0.2",
                 "path-to-regexp": "3.3.0",
                 "range-parser": "1.2.0"
@@ -4524,23 +4514,6 @@
                 "node": ">= 0.8.0"
             }
         },
-        "node_modules/serve/node_modules/ajv": {
-            "version": "8.12.0",
-            "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz",
-            "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==",
-            "dev": true,
-            "license": "MIT",
-            "dependencies": {
-                "fast-deep-equal": "^3.1.1",
-                "json-schema-traverse": "^1.0.0",
-                "require-from-string": "^2.0.2",
-                "uri-js": "^4.2.2"
-            },
-            "funding": {
-                "type": "github",
-                "url": "https://github.com/sponsors/epoberezkin"
-            }
-        },
         "node_modules/serve/node_modules/chalk": {
             "version": "5.0.1",
             "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.1.tgz",
@@ -5203,16 +5176,6 @@
             "integrity": "sha512-NtkVvqVCqsJo5U3mYRum2Tw6uCltOxfIJ/AfTZeTmw6U39IB5X23xF+kRZ9aiPaORqeiQQ7Q209/ibhOvxzwHA==",
             "license": "MIT"
         },
-        "node_modules/uri-js": {
-            "version": "4.4.1",
-            "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
-            "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
-            "dev": true,
-            "license": "BSD-2-Clause",
-            "dependencies": {
-                "punycode": "^2.1.0"
-            }
-        },
         "node_modules/util-deprecate": {
             "version": "1.0.2",
             "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",

wasm/demo/package.json

@@ -16,7 +16,7 @@
         "css-loader": "^7.1.2",
         "html-webpack-plugin": "^5.6.3",
         "mini-css-extract-plugin": "^2.9.2",
-        "serve": "^14.2.5",
+        "serve": "^14.2.6",
         "webpack": "^5.105.0",
         "webpack-cli": "^6.0.1",
         "webpack-dev-server": "^5.2.1"