Update opcode from 3.13.7 (#6156)

* Update `opcode` from 3.13.7

* Base `_opcode`

* Add `test__opcode.py` from 3.13.7

* Impl `has_*` methods

* Add more methods

* Update `dis.py` from 3.13.7

* Update `support/bytecode_helper.py` from 3.13.7

* correct is_valid

* Patch failing tests

* Unpatch `support/__init__.py`

* clippy

* Make comments to doc

* impl `_varname_from_oparg` for code

* Unmark passing tests

* Revert changes to `dis`

* Mark failing tests

.cargo/config.toml

@@ -3,3 +3,6 @@ rustflags = "-C link-arg=/STACK:8000000"
 
 [target.'cfg(all(target_os = "windows", not(target_env = "msvc")))']
 rustflags = "-C link-args=-Wl,--stack,8000000"
+
+[target.wasm32-unknown-unknown]
+rustflags = ["--cfg=getrandom_backend=\"wasm_js\""]

.cspell.dict/cpython.txt

@@ -23,6 +23,7 @@ freevars
 fromlist
 heaptype
 HIGHRES
+Itertool
 IMMUTABLETYPE
 kwonlyarg
 kwonlyargs
@@ -58,4 +59,4 @@ weakreflist
 withitem
 withs
 xstat
-XXPRIME
+XXPRIME

.github/copilot-instructions.md

@@ -206,7 +206,7 @@ cargo run --features ssl
 
 ## Documentation
 
-- Check the [architecture document](architecture/architecture.md) for a high-level overview
-- Read the [development guide](DEVELOPMENT.md) for detailed setup instructions
+- Check the [architecture document](/architecture/architecture.md) for a high-level overview
+- Read the [development guide](/DEVELOPMENT.md) for detailed setup instructions
 - Generate documentation with `cargo doc --no-deps --all`
-- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/)
+- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/)

.github/dependabot.yml

@@ -1,13 +1,10 @@
-# Keep GitHub Actions up to date with GitHub's Dependabot...
-# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
-# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
 version: 2
 updates:
-  - package-ecosystem: github-actions
+  - package-ecosystem: cargo
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: github-actions
     directory: /
-    groups:
-      github-actions:
-        patterns:
-          - "*"  # Group all Actions updates into a single larger pull request
     schedule:
       interval: weekly

.github/workflows/ci.yaml

@@ -116,10 +116,13 @@ jobs:
     timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 30 }}
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
+        os:
+          - macos-latest
+          - ubuntu-latest
+          - windows-2025 # TODO: Switch to `windows-latest` on 2025/09/30
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
@@ -178,7 +181,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: i686-unknown-linux-gnu
@@ -242,13 +245,16 @@ jobs:
     timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 30 }}
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
+        os:
+          - macos-latest
+          - ubuntu-latest
+          - windows-2025 # TODO: Switch to `windows-latest` on 2025/09/30
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Set up the Windows environment
@@ -267,7 +273,7 @@ jobs:
       - name: build rustpython
         run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit
         if: runner.os != 'macOS'
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: run snippets
@@ -310,7 +316,7 @@ jobs:
     name: Check Rust code with rustfmt and clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: dtolnay/rust-toolchain@stable
         with:
             components: rustfmt, clippy
@@ -318,7 +324,7 @@ jobs:
         run: cargo fmt --check
       - name: run clippy on wasm
         run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: install ruff
@@ -351,7 +357,7 @@ jobs:
     env:
       NIGHTLY_CHANNEL: nightly
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: dtolnay/rust-toolchain@master
         with:
@@ -373,7 +379,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: Swatinem/rust-cache@v2
@@ -384,12 +390,12 @@ jobs:
           wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz
           mkdir geckodriver
           tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - run: python -m pip install -r requirements.txt
         working-directory: ./wasm/tests
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@v5
         with:
           cache: "npm"
           cache-dependency-path: "wasm/demo/package-lock.json"
@@ -434,7 +440,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: wasm32-wasip1

.github/workflows/cron-ci.yaml

@@ -18,6 +18,8 @@ jobs:
   codecov:
     name: Collect code coverage data
     runs-on: ubuntu-latest
+    # Disable this scheduled job when running on a fork.
+    if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -44,6 +46,8 @@ jobs:
   testdata:
     name: Collect regression test data
     runs-on: ubuntu-latest
+    # Disable this scheduled job when running on a fork.
+    if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -73,6 +77,8 @@ jobs:
   whatsleft:
     name: Collect what is left data
     runs-on: ubuntu-latest
+    # Disable this scheduled job when running on a fork.
+    if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -111,6 +117,8 @@ jobs:
   benchmark:
     name: Collect benchmark data
     runs-on: ubuntu-latest
+    # Disable this scheduled job when running on a fork.
+    if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable

.github/workflows/release.yml

@@ -21,6 +21,8 @@ env:
 jobs:
   build:
     runs-on: ${{ matrix.platform.runner }}
+    # Disable this scheduled job when running on a fork.
+    if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
     strategy:
       matrix:
         platform:
@@ -88,6 +90,8 @@ jobs:
 
   build-wasm:
     runs-on: ubuntu-latest
+    # Disable this scheduled job when running on a fork.
+    if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -108,7 +112,7 @@ jobs:
 
       - name: install wasm-pack
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@v5
       - uses: mwilliamson/setup-wabt-action@v3
         with: { wabt-version: "1.0.30" }
       - name: build demo
@@ -136,10 +140,12 @@ jobs:
 
   release:
     runs-on: ubuntu-latest
+    # Disable this scheduled job when running on a fork.
+    if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
     needs: [build, build-wasm]
     steps:
       - name: Download Binary Artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
         with:
           path: bin
           pattern: rustpython-*

.gitignore

@@ -21,3 +21,8 @@ flamescope.json
 
 extra_tests/snippets/resources
 extra_tests/not_impl.py
+
+Lib/site-packages/*
+!Lib/site-packages/README.txt
+Lib/test/data/*
+!Lib/test/data/README

Cargo.toml

@@ -51,7 +51,7 @@ rustyline = { workspace = true }
 
 [dev-dependencies]
 criterion = { workspace = true }
-pyo3 = { version = "0.24", features = ["auto-initialize"] }
+pyo3 = { version = "0.26", features = ["auto-initialize"] }
 
 [[bench]]
 name = "execution"
@@ -138,7 +138,7 @@ members = [
 version = "0.4.0"
 authors = ["RustPython Team"]
 edition = "2024"
-rust-version = "1.85.0"
+rust-version = "1.89.0"
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
@@ -163,28 +163,28 @@ ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0
 ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
 ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
 
-ahash = "0.8.11"
+ahash = "0.8.12"
 ascii = "1.1"
-bitflags = "2.4.2"
+bitflags = "2.9.4"
 bstr = "1"
 cfg-if = "1.0"
-chrono = "0.4.39"
+chrono = "0.4.42"
 constant_time_eq = "0.4"
-criterion = { version = "0.5", features = ["html_reports"] }
+criterion = { version = "0.7", features = ["html_reports"] }
 crossbeam-utils = "0.8.21"
 flame = "0.2.2"
 getrandom = { version = "0.3", features = ["std"] }
 glob = "0.3"
 hex = "0.4.3"
-indexmap = { version = "2.2.6", features = ["std"] }
+indexmap = { version = "2.11.3", features = ["std"] }
 insta = "1.42"
 itertools = "0.14.0"
 is-macro = "0.3.7"
-junction = "1.2.0"
+junction = "1.3.0"
 libc = "0.2.169"
 libffi = "4.1"
-log = "0.4.27"
-nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
+log = "0.4.28"
+nix = { version = "0.30", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
 malachite-bigint = "0.6"
 malachite-q = "0.6"
 malachite-base = "0.6"
@@ -204,25 +204,27 @@ radium = "1.1.1"
 rand = "0.9"
 rand_core = { version = "0.9", features = ["os_rng"] }
 rustix = { version = "1.0", features = ["event"] }
-rustyline = "15.0.0"
-serde = { version = "1.0.133", default-features = false }
-schannel = "0.1.27"
+rustyline = "17.0.1"
+serde = { version = "1.0.225", default-features = false }
+schannel = "0.1.28"
+scoped-tls = "1"
+scopeguard = "1"
 static_assertions = "1.1"
 strum = "0.27"
 strum_macros = "0.27"
 syn = "2"
 thiserror = "2.0"
-thread_local = "1.1.8"
-unicode-casing = "0.1.0"
+thread_local = "1.1.9"
+unicode-casing = "0.1.1"
 unic-char-property = "0.9.0"
 unic-normal = "0.9.0"
 unic-ucd-age = "0.9.0"
 unic-ucd-bidi = "0.9.0"
 unic-ucd-category = "0.9.0"
 unic-ucd-ident = "0.9.0"
-unicode_names2 = "1.3.0"
-unicode-bidi-mirroring = "0.2"
-widestring = "1.1.0"
+unicode_names2 = "2.0.0"
+unicode-bidi-mirroring = "0.4"
+widestring = "1.2.0"
 windows-sys = "0.59.0"
 wasm-bindgen = "0.2.100"
 

Lib/_opcode_metadata.py

@@ -0,0 +1,343 @@
+# This file is generated by Tools/cases_generator/py_metadata_generator.py
+# from:
+#   Python/bytecodes.c
+# Do not edit!
+_specializations = {
+    "RESUME": [
+        "RESUME_CHECK",
+    ],
+    "TO_BOOL": [
+        "TO_BOOL_ALWAYS_TRUE",
+        "TO_BOOL_BOOL",
+        "TO_BOOL_INT",
+        "TO_BOOL_LIST",
+        "TO_BOOL_NONE",
+        "TO_BOOL_STR",
+    ],
+    "BINARY_OP": [
+        "BINARY_OP_MULTIPLY_INT",
+        "BINARY_OP_ADD_INT",
+        "BINARY_OP_SUBTRACT_INT",
+        "BINARY_OP_MULTIPLY_FLOAT",
+        "BINARY_OP_ADD_FLOAT",
+        "BINARY_OP_SUBTRACT_FLOAT",
+        "BINARY_OP_ADD_UNICODE",
+        "BINARY_OP_INPLACE_ADD_UNICODE",
+    ],
+    "BINARY_SUBSCR": [
+        "BINARY_SUBSCR_DICT",
+        "BINARY_SUBSCR_GETITEM",
+        "BINARY_SUBSCR_LIST_INT",
+        "BINARY_SUBSCR_STR_INT",
+        "BINARY_SUBSCR_TUPLE_INT",
+    ],
+    "STORE_SUBSCR": [
+        "STORE_SUBSCR_DICT",
+        "STORE_SUBSCR_LIST_INT",
+    ],
+    "SEND": [
+        "SEND_GEN",
+    ],
+    "UNPACK_SEQUENCE": [
+        "UNPACK_SEQUENCE_TWO_TUPLE",
+        "UNPACK_SEQUENCE_TUPLE",
+        "UNPACK_SEQUENCE_LIST",
+    ],
+    "STORE_ATTR": [
+        "STORE_ATTR_INSTANCE_VALUE",
+        "STORE_ATTR_SLOT",
+        "STORE_ATTR_WITH_HINT",
+    ],
+    "LOAD_GLOBAL": [
+        "LOAD_GLOBAL_MODULE",
+        "LOAD_GLOBAL_BUILTIN",
+    ],
+    "LOAD_SUPER_ATTR": [
+        "LOAD_SUPER_ATTR_ATTR",
+        "LOAD_SUPER_ATTR_METHOD",
+    ],
+    "LOAD_ATTR": [
+        "LOAD_ATTR_INSTANCE_VALUE",
+        "LOAD_ATTR_MODULE",
+        "LOAD_ATTR_WITH_HINT",
+        "LOAD_ATTR_SLOT",
+        "LOAD_ATTR_CLASS",
+        "LOAD_ATTR_PROPERTY",
+        "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN",
+        "LOAD_ATTR_METHOD_WITH_VALUES",
+        "LOAD_ATTR_METHOD_NO_DICT",
+        "LOAD_ATTR_METHOD_LAZY_DICT",
+        "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES",
+        "LOAD_ATTR_NONDESCRIPTOR_NO_DICT",
+    ],
+    "COMPARE_OP": [
+        "COMPARE_OP_FLOAT",
+        "COMPARE_OP_INT",
+        "COMPARE_OP_STR",
+    ],
+    "CONTAINS_OP": [
+        "CONTAINS_OP_SET",
+        "CONTAINS_OP_DICT",
+    ],
+    "FOR_ITER": [
+        "FOR_ITER_LIST",
+        "FOR_ITER_TUPLE",
+        "FOR_ITER_RANGE",
+        "FOR_ITER_GEN",
+    ],
+    "CALL": [
+        "CALL_BOUND_METHOD_EXACT_ARGS",
+        "CALL_PY_EXACT_ARGS",
+        "CALL_TYPE_1",
+        "CALL_STR_1",
+        "CALL_TUPLE_1",
+        "CALL_BUILTIN_CLASS",
+        "CALL_BUILTIN_O",
+        "CALL_BUILTIN_FAST",
+        "CALL_BUILTIN_FAST_WITH_KEYWORDS",
+        "CALL_LEN",
+        "CALL_ISINSTANCE",
+        "CALL_LIST_APPEND",
+        "CALL_METHOD_DESCRIPTOR_O",
+        "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS",
+        "CALL_METHOD_DESCRIPTOR_NOARGS",
+        "CALL_METHOD_DESCRIPTOR_FAST",
+        "CALL_ALLOC_AND_ENTER_INIT",
+        "CALL_PY_GENERAL",
+        "CALL_BOUND_METHOD_GENERAL",
+        "CALL_NON_PY_GENERAL",
+    ],
+}
+
+_specialized_opmap = {
+    'BINARY_OP_ADD_FLOAT': 150,
+    'BINARY_OP_ADD_INT': 151,
+    'BINARY_OP_ADD_UNICODE': 152,
+    'BINARY_OP_INPLACE_ADD_UNICODE': 3,
+    'BINARY_OP_MULTIPLY_FLOAT': 153,
+    'BINARY_OP_MULTIPLY_INT': 154,
+    'BINARY_OP_SUBTRACT_FLOAT': 155,
+    'BINARY_OP_SUBTRACT_INT': 156,
+    'BINARY_SUBSCR_DICT': 157,
+    'BINARY_SUBSCR_GETITEM': 158,
+    'BINARY_SUBSCR_LIST_INT': 159,
+    'BINARY_SUBSCR_STR_INT': 160,
+    'BINARY_SUBSCR_TUPLE_INT': 161,
+    'CALL_ALLOC_AND_ENTER_INIT': 162,
+    'CALL_BOUND_METHOD_EXACT_ARGS': 163,
+    'CALL_BOUND_METHOD_GENERAL': 164,
+    'CALL_BUILTIN_CLASS': 165,
+    'CALL_BUILTIN_FAST': 166,
+    'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167,
+    'CALL_BUILTIN_O': 168,
+    'CALL_ISINSTANCE': 169,
+    'CALL_LEN': 170,
+    'CALL_LIST_APPEND': 171,
+    'CALL_METHOD_DESCRIPTOR_FAST': 172,
+    'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 173,
+    'CALL_METHOD_DESCRIPTOR_NOARGS': 174,
+    'CALL_METHOD_DESCRIPTOR_O': 175,
+    'CALL_NON_PY_GENERAL': 176,
+    'CALL_PY_EXACT_ARGS': 177,
+    'CALL_PY_GENERAL': 178,
+    'CALL_STR_1': 179,
+    'CALL_TUPLE_1': 180,
+    'CALL_TYPE_1': 181,
+    'COMPARE_OP_FLOAT': 182,
+    'COMPARE_OP_INT': 183,
+    'COMPARE_OP_STR': 184,
+    'CONTAINS_OP_DICT': 185,
+    'CONTAINS_OP_SET': 186,
+    'FOR_ITER_GEN': 187,
+    'FOR_ITER_LIST': 188,
+    'FOR_ITER_RANGE': 189,
+    'FOR_ITER_TUPLE': 190,
+    'LOAD_ATTR_CLASS': 191,
+    'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 192,
+    'LOAD_ATTR_INSTANCE_VALUE': 193,
+    'LOAD_ATTR_METHOD_LAZY_DICT': 194,
+    'LOAD_ATTR_METHOD_NO_DICT': 195,
+    'LOAD_ATTR_METHOD_WITH_VALUES': 196,
+    'LOAD_ATTR_MODULE': 197,
+    'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 198,
+    'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 199,
+    'LOAD_ATTR_PROPERTY': 200,
+    'LOAD_ATTR_SLOT': 201,
+    'LOAD_ATTR_WITH_HINT': 202,
+    'LOAD_GLOBAL_BUILTIN': 203,
+    'LOAD_GLOBAL_MODULE': 204,
+    'LOAD_SUPER_ATTR_ATTR': 205,
+    'LOAD_SUPER_ATTR_METHOD': 206,
+    'RESUME_CHECK': 207,
+    'SEND_GEN': 208,
+    'STORE_ATTR_INSTANCE_VALUE': 209,
+    'STORE_ATTR_SLOT': 210,
+    'STORE_ATTR_WITH_HINT': 211,
+    'STORE_SUBSCR_DICT': 212,
+    'STORE_SUBSCR_LIST_INT': 213,
+    'TO_BOOL_ALWAYS_TRUE': 214,
+    'TO_BOOL_BOOL': 215,
+    'TO_BOOL_INT': 216,
+    'TO_BOOL_LIST': 217,
+    'TO_BOOL_NONE': 218,
+    'TO_BOOL_STR': 219,
+    'UNPACK_SEQUENCE_LIST': 220,
+    'UNPACK_SEQUENCE_TUPLE': 221,
+    'UNPACK_SEQUENCE_TWO_TUPLE': 222,
+}
+
+opmap = {
+    'CACHE': 0,
+    'RESERVED': 17,
+    'RESUME': 149,
+    'INSTRUMENTED_LINE': 254,
+    'BEFORE_ASYNC_WITH': 1,
+    'BEFORE_WITH': 2,
+    'BINARY_SLICE': 4,
+    'BINARY_SUBSCR': 5,
+    'CHECK_EG_MATCH': 6,
+    'CHECK_EXC_MATCH': 7,
+    'CLEANUP_THROW': 8,
+    'DELETE_SUBSCR': 9,
+    'END_ASYNC_FOR': 10,
+    'END_FOR': 11,
+    'END_SEND': 12,
+    'EXIT_INIT_CHECK': 13,
+    'FORMAT_SIMPLE': 14,
+    'FORMAT_WITH_SPEC': 15,
+    'GET_AITER': 16,
+    'GET_ANEXT': 18,
+    'GET_ITER': 19,
+    'GET_LEN': 20,
+    'GET_YIELD_FROM_ITER': 21,
+    'INTERPRETER_EXIT': 22,
+    'LOAD_ASSERTION_ERROR': 23,
+    'LOAD_BUILD_CLASS': 24,
+    'LOAD_LOCALS': 25,
+    'MAKE_FUNCTION': 26,
+    'MATCH_KEYS': 27,
+    'MATCH_MAPPING': 28,
+    'MATCH_SEQUENCE': 29,
+    'NOP': 30,
+    'POP_EXCEPT': 31,
+    'POP_TOP': 32,
+    'PUSH_EXC_INFO': 33,
+    'PUSH_NULL': 34,
+    'RETURN_GENERATOR': 35,
+    'RETURN_VALUE': 36,
+    'SETUP_ANNOTATIONS': 37,
+    'STORE_SLICE': 38,
+    'STORE_SUBSCR': 39,
+    'TO_BOOL': 40,
+    'UNARY_INVERT': 41,
+    'UNARY_NEGATIVE': 42,
+    'UNARY_NOT': 43,
+    'WITH_EXCEPT_START': 44,
+    'BINARY_OP': 45,
+    'BUILD_CONST_KEY_MAP': 46,
+    'BUILD_LIST': 47,
+    'BUILD_MAP': 48,
+    'BUILD_SET': 49,
+    'BUILD_SLICE': 50,
+    'BUILD_STRING': 51,
+    'BUILD_TUPLE': 52,
+    'CALL': 53,
+    'CALL_FUNCTION_EX': 54,
+    'CALL_INTRINSIC_1': 55,
+    'CALL_INTRINSIC_2': 56,
+    'CALL_KW': 57,
+    'COMPARE_OP': 58,
+    'CONTAINS_OP': 59,
+    'CONVERT_VALUE': 60,
+    'COPY': 61,
+    'COPY_FREE_VARS': 62,
+    'DELETE_ATTR': 63,
+    'DELETE_DEREF': 64,
+    'DELETE_FAST': 65,
+    'DELETE_GLOBAL': 66,
+    'DELETE_NAME': 67,
+    'DICT_MERGE': 68,
+    'DICT_UPDATE': 69,
+    'ENTER_EXECUTOR': 70,
+    'EXTENDED_ARG': 71,
+    'FOR_ITER': 72,
+    'GET_AWAITABLE': 73,
+    'IMPORT_FROM': 74,
+    'IMPORT_NAME': 75,
+    'IS_OP': 76,
+    'JUMP_BACKWARD': 77,
+    'JUMP_BACKWARD_NO_INTERRUPT': 78,
+    'JUMP_FORWARD': 79,
+    'LIST_APPEND': 80,
+    'LIST_EXTEND': 81,
+    'LOAD_ATTR': 82,
+    'LOAD_CONST': 83,
+    'LOAD_DEREF': 84,
+    'LOAD_FAST': 85,
+    'LOAD_FAST_AND_CLEAR': 86,
+    'LOAD_FAST_CHECK': 87,
+    'LOAD_FAST_LOAD_FAST': 88,
+    'LOAD_FROM_DICT_OR_DEREF': 89,
+    'LOAD_FROM_DICT_OR_GLOBALS': 90,
+    'LOAD_GLOBAL': 91,
+    'LOAD_NAME': 92,
+    'LOAD_SUPER_ATTR': 93,
+    'MAKE_CELL': 94,
+    'MAP_ADD': 95,
+    'MATCH_CLASS': 96,
+    'POP_JUMP_IF_FALSE': 97,
+    'POP_JUMP_IF_NONE': 98,
+    'POP_JUMP_IF_NOT_NONE': 99,
+    'POP_JUMP_IF_TRUE': 100,
+    'RAISE_VARARGS': 101,
+    'RERAISE': 102,
+    'RETURN_CONST': 103,
+    'SEND': 104,
+    'SET_ADD': 105,
+    'SET_FUNCTION_ATTRIBUTE': 106,
+    'SET_UPDATE': 107,
+    'STORE_ATTR': 108,
+    'STORE_DEREF': 109,
+    'STORE_FAST': 110,
+    'STORE_FAST_LOAD_FAST': 111,
+    'STORE_FAST_STORE_FAST': 112,
+    'STORE_GLOBAL': 113,
+    'STORE_NAME': 114,
+    'SWAP': 115,
+    'UNPACK_EX': 116,
+    'UNPACK_SEQUENCE': 117,
+    'YIELD_VALUE': 118,
+    'INSTRUMENTED_RESUME': 236,
+    'INSTRUMENTED_END_FOR': 237,
+    'INSTRUMENTED_END_SEND': 238,
+    'INSTRUMENTED_RETURN_VALUE': 239,
+    'INSTRUMENTED_RETURN_CONST': 240,
+    'INSTRUMENTED_YIELD_VALUE': 241,
+    'INSTRUMENTED_LOAD_SUPER_ATTR': 242,
+    'INSTRUMENTED_FOR_ITER': 243,
+    'INSTRUMENTED_CALL': 244,
+    'INSTRUMENTED_CALL_KW': 245,
+    'INSTRUMENTED_CALL_FUNCTION_EX': 246,
+    'INSTRUMENTED_INSTRUCTION': 247,
+    'INSTRUMENTED_JUMP_FORWARD': 248,
+    'INSTRUMENTED_JUMP_BACKWARD': 249,
+    'INSTRUMENTED_POP_JUMP_IF_TRUE': 250,
+    'INSTRUMENTED_POP_JUMP_IF_FALSE': 251,
+    'INSTRUMENTED_POP_JUMP_IF_NONE': 252,
+    'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253,
+    'JUMP': 256,
+    'JUMP_NO_INTERRUPT': 257,
+    'LOAD_CLOSURE': 258,
+    'LOAD_METHOD': 259,
+    'LOAD_SUPER_METHOD': 260,
+    'LOAD_ZERO_SUPER_ATTR': 261,
+    'LOAD_ZERO_SUPER_METHOD': 262,
+    'POP_BLOCK': 263,
+    'SETUP_CLEANUP': 264,
+    'SETUP_FINALLY': 265,
+    'SETUP_WITH': 266,
+    'STORE_FAST_MAYBE_NULL': 267,
+}
+
+HAVE_ARGUMENT = 44
+MIN_INSTRUMENTED_OPCODE = 236

Lib/_strptime.py

@@ -10,10 +10,13 @@ FUNCTIONS:
     strptime -- Calculates the time struct represented by the passed-in string
 
 """
+import os
 import time
 import locale
 import calendar
+import re
 from re import compile as re_compile
+from re import sub as re_sub
 from re import IGNORECASE
 from re import escape as re_escape
 from datetime import (date as datetime_date,
@@ -27,6 +30,41 @@ def _getlang():
     # Figure out what the current language is set to.
     return locale.getlocale(locale.LC_TIME)
 
+def _findall(haystack, needle):
+    # Find all positions of needle in haystack.
+    if not needle:
+        return
+    i = 0
+    while True:
+        i = haystack.find(needle, i)
+        if i < 0:
+            break
+        yield i
+        i += len(needle)
+
+def _fixmonths(months):
+    yield from months
+    # The lower case of 'İ' ('\u0130') is 'i\u0307'.
+    # The re module only supports 1-to-1 character matching in
+    # case-insensitive mode.
+    for s in months:
+        if 'i\u0307' in s:
+            yield s.replace('i\u0307', '\u0130')
+
+lzh_TW_alt_digits = (
+    # 〇:一:二:三:四:五:六:七:八:九
+    '\u3007', '\u4e00', '\u4e8c', '\u4e09', '\u56db',
+    '\u4e94', '\u516d', '\u4e03', '\u516b', '\u4e5d',
+    # 十:十一:十二:十三:十四:十五:十六:十七:十八:十九
+    '\u5341', '\u5341\u4e00', '\u5341\u4e8c', '\u5341\u4e09', '\u5341\u56db',
+    '\u5341\u4e94', '\u5341\u516d', '\u5341\u4e03', '\u5341\u516b', '\u5341\u4e5d',
+    # 廿:廿一:廿二:廿三:廿四:廿五:廿六:廿七:廿八:廿九
+    '\u5eff', '\u5eff\u4e00', '\u5eff\u4e8c', '\u5eff\u4e09', '\u5eff\u56db',
+    '\u5eff\u4e94', '\u5eff\u516d', '\u5eff\u4e03', '\u5eff\u516b', '\u5eff\u4e5d',
+    # 卅:卅一
+    '\u5345', '\u5345\u4e00')
+
+
 class LocaleTime(object):
     """Stores and handles locale-specific information related to time.
 
@@ -70,6 +108,7 @@ class LocaleTime(object):
         self.__calc_weekday()
         self.__calc_month()
         self.__calc_am_pm()
+        self.__calc_alt_digits()
         self.__calc_timezone()
         self.__calc_date_time()
         if _getlang() != self.lang:
@@ -101,53 +140,184 @@ class LocaleTime(object):
         am_pm = []
         for hour in (1, 22):
             time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0))
-            am_pm.append(time.strftime("%p", time_tuple).lower())
+            # br_FR has AM/PM info (' ',' ').
+            am_pm.append(time.strftime("%p", time_tuple).lower().strip())
         self.am_pm = am_pm
 
+    def __calc_alt_digits(self):
+        # Set self.LC_alt_digits by using time.strftime().
+
+        # The magic data should contain all decimal digits.
+        time_tuple = time.struct_time((1998, 1, 27, 10, 43, 56, 1, 27, 0))
+        s = time.strftime("%x%X", time_tuple)
+        if s.isascii():
+            # Fast path -- all digits are ASCII.
+            self.LC_alt_digits = ()
+            return
+
+        digits = ''.join(sorted(set(re.findall(r'\d', s))))
+        if len(digits) == 10 and ord(digits[-1]) == ord(digits[0]) + 9:
+            # All 10 decimal digits from the same set.
+            if digits.isascii():
+                # All digits are ASCII.
+                self.LC_alt_digits = ()
+                return
+
+            self.LC_alt_digits = [a + b for a in digits for b in digits]
+            # Test whether the numbers contain leading zero.
+            time_tuple2 = time.struct_time((2000, 1, 1, 1, 1, 1, 5, 1, 0))
+            if self.LC_alt_digits[1] not in time.strftime("%x %X", time_tuple2):
+                self.LC_alt_digits[:10] = digits
+            return
+
+        # Either non-Gregorian calendar or non-decimal numbers.
+        if {'\u4e00', '\u4e03', '\u4e5d', '\u5341', '\u5eff'}.issubset(s):
+            # lzh_TW
+            self.LC_alt_digits = lzh_TW_alt_digits
+            return
+
+        self.LC_alt_digits = None
+
     def __calc_date_time(self):
-        # Set self.date_time, self.date, & self.time by using
-        # time.strftime().
+        # Set self.LC_date_time, self.LC_date, self.LC_time and
+        # self.LC_time_ampm by using time.strftime().
 
         # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of
         # overloaded numbers is minimized.  The order in which searches for
         # values within the format string is very important; it eliminates
         # possible ambiguity for what something represents.
         time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
-        date_time = [None, None, None]
-        date_time[0] = time.strftime("%c", time_tuple).lower()
-        date_time[1] = time.strftime("%x", time_tuple).lower()
-        date_time[2] = time.strftime("%X", time_tuple).lower()
-        replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'),
-                    (self.f_month[3], '%B'), (self.a_weekday[2], '%a'),
-                    (self.a_month[3], '%b'), (self.am_pm[1], '%p'),
-                    ('1999', '%Y'), ('99', '%y'), ('22', '%H'),
-                    ('44', '%M'), ('55', '%S'), ('76', '%j'),
-                    ('17', '%d'), ('03', '%m'), ('3', '%m'),
-                    # '3' needed for when no leading zero.
-                    ('2', '%w'), ('10', '%I')]
-        replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone
-                                                for tz in tz_values])
-        for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')):
-            current_format = date_time[offset]
-            for old, new in replacement_pairs:
+        time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0))
+        replacement_pairs = []
+
+        # Non-ASCII digits
+        if self.LC_alt_digits or self.LC_alt_digits is None:
+            for n, d in [(19, '%OC'), (99, '%Oy'), (22, '%OH'),
+                         (44, '%OM'), (55, '%OS'), (17, '%Od'),
+                         (3, '%Om'), (2, '%Ow'), (10, '%OI')]:
+                if self.LC_alt_digits is None:
+                    s = chr(0x660 + n // 10) + chr(0x660 + n % 10)
+                    replacement_pairs.append((s, d))
+                    if n < 10:
+                        replacement_pairs.append((s[1], d))
+                elif len(self.LC_alt_digits) > n:
+                    replacement_pairs.append((self.LC_alt_digits[n], d))
+                else:
+                    replacement_pairs.append((time.strftime(d, time_tuple), d))
+        replacement_pairs += [
+            ('1999', '%Y'), ('99', '%y'), ('22', '%H'),
+            ('44', '%M'), ('55', '%S'), ('76', '%j'),
+            ('17', '%d'), ('03', '%m'), ('3', '%m'),
+            # '3' needed for when no leading zero.
+            ('2', '%w'), ('10', '%I'),
+        ]
+
+        date_time = []
+        for directive in ('%c', '%x', '%X', '%r'):
+            current_format = time.strftime(directive, time_tuple).lower()
+            current_format = current_format.replace('%', '%%')
+            # The month and the day of the week formats are treated specially
+            # because of a possible ambiguity in some locales where the full
+            # and abbreviated names are equal or names of different types
+            # are equal. See doc of __find_month_format for more details.
+            lst, fmt = self.__find_weekday_format(directive)
+            if lst:
+                current_format = current_format.replace(lst[2], fmt, 1)
+            lst, fmt = self.__find_month_format(directive)
+            if lst:
+                current_format = current_format.replace(lst[3], fmt, 1)
+            if self.am_pm[1]:
                 # Must deal with possible lack of locale info
                 # manifesting itself as the empty string (e.g., Swedish's
                 # lack of AM/PM info) or a platform returning a tuple of empty
                 # strings (e.g., MacOS 9 having timezone as ('','')).
-                if old:
-                    current_format = current_format.replace(old, new)
+                current_format = current_format.replace(self.am_pm[1], '%p')
+            for tz_values in self.timezone:
+                for tz in tz_values:
+                    if tz:
+                        current_format = current_format.replace(tz, "%Z")
+            # Transform all non-ASCII digits to digits in range U+0660 to U+0669.
+            if not current_format.isascii() and self.LC_alt_digits is None:
+                current_format = re_sub(r'\d(?<![0-9])',
+                                        lambda m: chr(0x0660 + int(m[0])),
+                                        current_format)
+            for old, new in replacement_pairs:
+                current_format = current_format.replace(old, new)
             # If %W is used, then Sunday, 2005-01-03 will fall on week 0 since
             # 2005-01-03 occurs before the first Monday of the year.  Otherwise
             # %U is used.
-            time_tuple = time.struct_time((1999,1,3,1,1,1,6,3,0))
-            if '00' in time.strftime(directive, time_tuple):
+            if '00' in time.strftime(directive, time_tuple2):
                 U_W = '%W'
             else:
                 U_W = '%U'
-            date_time[offset] = current_format.replace('11', U_W)
+            current_format = current_format.replace('11', U_W)
+            date_time.append(current_format)
         self.LC_date_time = date_time[0]
         self.LC_date = date_time[1]
         self.LC_time = date_time[2]
+        self.LC_time_ampm = date_time[3]
+
+    def __find_month_format(self, directive):
+        """Find the month format appropriate for the current locale.
+
+        In some locales (for example French and Hebrew), the default month
+        used in __calc_date_time has the same name in full and abbreviated
+        form.  Also, the month name can by accident match other part of the
+        representation: the day of the week name (for example in Morisyen)
+        or the month number (for example in Japanese).  Thus, cycle months
+        of the year and find all positions that match the month name for
+        each month,  If no common positions are found, the representation
+        does not use the month name.
+        """
+        full_indices = abbr_indices = None
+        for m in range(1, 13):
+            time_tuple = time.struct_time((1999, m, 17, 22, 44, 55, 2, 76, 0))
+            datetime = time.strftime(directive, time_tuple).lower()
+            indices = set(_findall(datetime, self.f_month[m]))
+            if full_indices is None:
+                full_indices = indices
+            else:
+                full_indices &= indices
+            indices = set(_findall(datetime, self.a_month[m]))
+            if abbr_indices is None:
+                abbr_indices = set(indices)
+            else:
+                abbr_indices &= indices
+            if not full_indices and not abbr_indices:
+                return None, None
+        if full_indices:
+            return self.f_month, '%B'
+        if abbr_indices:
+            return self.a_month, '%b'
+        return None, None
+
+    def __find_weekday_format(self, directive):
+        """Find the day of the week format appropriate for the current locale.
+
+        Similar to __find_month_format().
+        """
+        full_indices = abbr_indices = None
+        for wd in range(7):
+            time_tuple = time.struct_time((1999, 3, 17, 22, 44, 55, wd, 76, 0))
+            datetime = time.strftime(directive, time_tuple).lower()
+            indices = set(_findall(datetime, self.f_weekday[wd]))
+            if full_indices is None:
+                full_indices = indices
+            else:
+                full_indices &= indices
+            if self.f_weekday[wd] != self.a_weekday[wd]:
+                indices = set(_findall(datetime, self.a_weekday[wd]))
+            if abbr_indices is None:
+                abbr_indices = set(indices)
+            else:
+                abbr_indices &= indices
+            if not full_indices and not abbr_indices:
+                return None, None
+        if full_indices:
+            return self.f_weekday, '%A'
+        if abbr_indices:
+            return self.a_weekday, '%a'
+        return None, None
 
     def __calc_timezone(self):
         # Set self.timezone by using time.tzname.
@@ -181,12 +351,14 @@ class TimeRE(dict):
         else:
             self.locale_time = LocaleTime()
         base = super()
-        base.__init__({
+        mapping = {
             # The " [1-9]" part of the regex is to make %c from ANSI C work
             'd': r"(?P<d>3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
             'f': r"(?P<f>[0-9]{1,6})",
-            'H': r"(?P<H>2[0-3]|[0-1]\d|\d)",
-            'I': r"(?P<I>1[0-2]|0[1-9]|[1-9])",
+            'H': r"(?P<H>2[0-3]|[0-1]\d|\d| \d)",
+            'k': r"(?P<H>2[0-3]|[0-1]\d|\d| \d)",
+            'I': r"(?P<I>1[0-2]|0[1-9]|[1-9]| [1-9])",
+            'l': r"(?P<I>1[0-2]|0[1-9]|[1-9]| [1-9])",
             'G': r"(?P<G>\d\d\d\d)",
             'j': r"(?P<j>36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])",
             'm': r"(?P<m>1[0-2]|0[1-9]|[1-9])",
@@ -198,25 +370,60 @@ class TimeRE(dict):
             'V': r"(?P<V>5[0-3]|0[1-9]|[1-4]\d|\d)",
             # W is set below by using 'U'
             'y': r"(?P<y>\d\d)",
-            #XXX: Does 'Y' need to worry about having less or more than
-            #     4 digits?
             'Y': r"(?P<Y>\d\d\d\d)",
             'z': r"(?P<z>[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))",
             'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
             'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
-            'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
-            'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'),
+            'B': self.__seqToRE(_fixmonths(self.locale_time.f_month[1:]), 'B'),
+            'b': self.__seqToRE(_fixmonths(self.locale_time.a_month[1:]), 'b'),
             'p': self.__seqToRE(self.locale_time.am_pm, 'p'),
             'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone
                                         for tz in tz_names),
                                 'Z'),
-            '%': '%'})
-        base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
-        base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
-        base.__setitem__('x', self.pattern(self.locale_time.LC_date))
-        base.__setitem__('X', self.pattern(self.locale_time.LC_time))
+            '%': '%'}
+        if self.locale_time.LC_alt_digits is None:
+            for d in 'dmyCHIMS':
+                mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d
+            mapping['Ow'] = r'(?P<w>\d)'
+        else:
+            mapping.update({
+                'Od': self.__seqToRE(self.locale_time.LC_alt_digits[1:32], 'd',
+                                     '3[0-1]|[1-2][0-9]|0[1-9]|[1-9]'),
+                'Om': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'm',
+                                     '1[0-2]|0[1-9]|[1-9]'),
+                'Ow': self.__seqToRE(self.locale_time.LC_alt_digits[:7], 'w',
+                                     '[0-6]'),
+                'Oy': self.__seqToRE(self.locale_time.LC_alt_digits, 'y',
+                                     '[0-9][0-9]'),
+                'OC': self.__seqToRE(self.locale_time.LC_alt_digits, 'C',
+                                     '[0-9][0-9]'),
+                'OH': self.__seqToRE(self.locale_time.LC_alt_digits[:24], 'H',
+                                     '2[0-3]|[0-1][0-9]|[0-9]'),
+                'OI': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'I',
+                                     '1[0-2]|0[1-9]|[1-9]'),
+                'OM': self.__seqToRE(self.locale_time.LC_alt_digits[:60], 'M',
+                                     '[0-5][0-9]|[0-9]'),
+                'OS': self.__seqToRE(self.locale_time.LC_alt_digits[:62], 'S',
+                                     '6[0-1]|[0-5][0-9]|[0-9]'),
+            })
+        mapping.update({
+            'e': mapping['d'],
+            'Oe': mapping['Od'],
+            'P': mapping['p'],
+            'Op': mapping['p'],
+            'W': mapping['U'].replace('U', 'W'),
+        })
+        mapping['W'] = mapping['U'].replace('U', 'W')
 
-    def __seqToRE(self, to_convert, directive):
+        base.__init__(mapping)
+        base.__setitem__('T', self.pattern('%H:%M:%S'))
+        base.__setitem__('R', self.pattern('%H:%M'))
+        base.__setitem__('r', self.pattern(self.locale_time.LC_time_ampm))
+        base.__setitem__('X', self.pattern(self.locale_time.LC_time))
+        base.__setitem__('x', self.pattern(self.locale_time.LC_date))
+        base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
+
+    def __seqToRE(self, to_convert, directive, altregex=None):
         """Convert a list to a regex string for matching a directive.
 
         Want possible matching values to be from longest to shortest.  This
@@ -232,8 +439,9 @@ class TimeRE(dict):
         else:
             return ''
         regex = '|'.join(re_escape(stuff) for stuff in to_convert)
-        regex = '(?P<%s>%s' % (directive, regex)
-        return '%s)' % regex
+        if altregex is not None:
+            regex += '|' + altregex
+        return '(?P<%s>%s)' % (directive, regex)
 
     def pattern(self, format):
         """Return regex pattern for the format string.
@@ -242,21 +450,36 @@ class TimeRE(dict):
         regex syntax are escaped.
 
         """
-        processed_format = ''
         # The sub() call escapes all characters that might be misconstrued
         # as regex syntax.  Cannot use re.escape since we have to deal with
         # format directives (%m, etc.).
-        regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
-        format = regex_chars.sub(r"\\\1", format)
-        whitespace_replacement = re_compile(r'\s+')
-        format = whitespace_replacement.sub(r'\\s+', format)
-        while '%' in format:
-            directive_index = format.index('%')+1
-            processed_format = "%s%s%s" % (processed_format,
-                                           format[:directive_index-1],
-                                           self[format[directive_index]])
-            format = format[directive_index+1:]
-        return "%s%s" % (processed_format, format)
+        format = re_sub(r"([\\.^$*+?\(\){}\[\]|])", r"\\\1", format)
+        format = re_sub(r'\s+', r'\\s+', format)
+        format = re_sub(r"'", "['\u02bc]", format)  # needed for br_FR
+        year_in_format = False
+        day_of_month_in_format = False
+        def repl(m):
+            format_char = m[1]
+            match format_char:
+                case 'Y' | 'y' | 'G':
+                    nonlocal year_in_format
+                    year_in_format = True
+                case 'd':
+                    nonlocal day_of_month_in_format
+                    day_of_month_in_format = True
+            return self[format_char]
+        format = re_sub(r'%[-_0^#]*[0-9]*([OE]?\\?.?)', repl, format)
+        if day_of_month_in_format and not year_in_format:
+            import warnings
+            warnings.warn("""\
+Parsing dates involving a day of month without a year specified is ambiguious
+and fails to parse leap day. The default behavior will change in Python 3.15
+to either always raise an exception or to use a different default year (TBD).
+To avoid trouble, add a specific year to the input & format.
+See https://github.com/python/cpython/issues/70647.""",
+                          DeprecationWarning,
+                          skip_file_prefixes=(os.path.dirname(__file__),))
+        return format
 
     def compile(self, format):
         """Return a compiled re object for the format string."""
@@ -319,14 +542,13 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
             # \\, in which case it was a stray % but with a space after it
             except KeyError as err:
                 bad_directive = err.args[0]
-                if bad_directive == "\\":
-                    bad_directive = "%"
                 del err
+                bad_directive = bad_directive.replace('\\s', '')
+                if not bad_directive:
+                    raise ValueError("stray %% in format '%s'" % format) from None
+                bad_directive = bad_directive.replace('\\', '', 1)
                 raise ValueError("'%s' is a bad directive in format '%s'" %
                                     (bad_directive, format)) from None
-            # IndexError only occurs when the format string is "%"
-            except IndexError:
-                raise ValueError("stray %% in format '%s'" % format) from None
             _regex_cache[format] = format_regex
     found = format_regex.match(data_string)
     if not found:
@@ -348,6 +570,15 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
     # values
     weekday = julian = None
     found_dict = found.groupdict()
+    if locale_time.LC_alt_digits:
+        def parse_int(s):
+            try:
+                return locale_time.LC_alt_digits.index(s)
+            except ValueError:
+                return int(s)
+    else:
+        parse_int = int
+
     for group_key in found_dict.keys():
         # Directives not explicitly handled below:
         #   c, x, X
@@ -355,30 +586,34 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
         #   U, W
         #      worthless without day of the week
         if group_key == 'y':
-            year = int(found_dict['y'])
-            # Open Group specification for strptime() states that a %y
-            #value in the range of [00, 68] is in the century 2000, while
-            #[69,99] is in the century 1900
-            if year <= 68:
-                year += 2000
+            year = parse_int(found_dict['y'])
+            if 'C' in found_dict:
+                century = parse_int(found_dict['C'])
+                year += century * 100
             else:
-                year += 1900
+                # Open Group specification for strptime() states that a %y
+                #value in the range of [00, 68] is in the century 2000, while
+                #[69,99] is in the century 1900
+                if year <= 68:
+                    year += 2000
+                else:
+                    year += 1900
         elif group_key == 'Y':
             year = int(found_dict['Y'])
         elif group_key == 'G':
             iso_year = int(found_dict['G'])
         elif group_key == 'm':
-            month = int(found_dict['m'])
+            month = parse_int(found_dict['m'])
         elif group_key == 'B':
             month = locale_time.f_month.index(found_dict['B'].lower())
         elif group_key == 'b':
             month = locale_time.a_month.index(found_dict['b'].lower())
         elif group_key == 'd':
-            day = int(found_dict['d'])
+            day = parse_int(found_dict['d'])
         elif group_key == 'H':
-            hour = int(found_dict['H'])
+            hour = parse_int(found_dict['H'])
         elif group_key == 'I':
-            hour = int(found_dict['I'])
+            hour = parse_int(found_dict['I'])
             ampm = found_dict.get('p', '').lower()
             # If there was no AM/PM indicator, we'll treat this like AM
             if ampm in ('', locale_time.am_pm[0]):
@@ -394,9 +629,9 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
                 if hour != 12:
                     hour += 12
         elif group_key == 'M':
-            minute = int(found_dict['M'])
+            minute = parse_int(found_dict['M'])
         elif group_key == 'S':
-            second = int(found_dict['S'])
+            second = parse_int(found_dict['S'])
         elif group_key == 'f':
             s = found_dict['f']
             # Pad to always return microseconds.

Lib/bz2.py

@@ -17,7 +17,7 @@ import _compression
 from _bz2 import BZ2Compressor, BZ2Decompressor
 
 
-_MODE_CLOSED   = 0
+# Value 0 no longer used
 _MODE_READ     = 1
 # Value 2 no longer used
 _MODE_WRITE    = 3
@@ -54,7 +54,7 @@ class BZ2File(_compression.BaseStream):
         """
         self._fp = None
         self._closefp = False
-        self._mode = _MODE_CLOSED
+        self._mode = None
 
         if not (1 <= compresslevel <= 9):
             raise ValueError("compresslevel must be between 1 and 9")
@@ -100,7 +100,7 @@ class BZ2File(_compression.BaseStream):
         May be called more than once without error. Once the file is
         closed, any other operation on it will raise a ValueError.
         """
-        if self._mode == _MODE_CLOSED:
+        if self.closed:
             return
         try:
             if self._mode == _MODE_READ:
@@ -115,13 +115,21 @@ class BZ2File(_compression.BaseStream):
             finally:
                 self._fp = None
                 self._closefp = False
-                self._mode = _MODE_CLOSED
                 self._buffer = None
 
     @property
     def closed(self):
         """True if this file is closed."""
-        return self._mode == _MODE_CLOSED
+        return self._fp is None
+
+    @property
+    def name(self):
+        self._check_not_closed()
+        return self._fp.name
+
+    @property
+    def mode(self):
+        return 'wb' if self._mode == _MODE_WRITE else 'rb'
 
     def fileno(self):
         """Return the file descriptor for the underlying file."""

Lib/codecs.py

@@ -111,6 +111,9 @@ class CodecInfo(tuple):
                 (self.__class__.__module__, self.__class__.__qualname__,
                  self.name, id(self))
 
+    def __getnewargs__(self):
+        return tuple(self)
+
 class Codec:
 
     """ Defines the interface for stateless encoders/decoders.
@@ -615,7 +618,7 @@ class StreamReader(Codec):
             method and are included in the list entries.
 
             sizehint, if given, is ignored since there is no efficient
-            way to finding the true end-of-line.
+            way of finding the true end-of-line.
 
         """
         data = self.read()
@@ -706,13 +709,13 @@ class StreamReaderWriter:
 
         return self.reader.read(size)
 
-    def readline(self, size=None):
+    def readline(self, size=None, keepends=True):
 
-        return self.reader.readline(size)
+        return self.reader.readline(size, keepends)
 
-    def readlines(self, sizehint=None):
+    def readlines(self, sizehint=None, keepends=True):
 
-        return self.reader.readlines(sizehint)
+        return self.reader.readlines(sizehint, keepends)
 
     def __next__(self):
 

Lib/configparser.py

@@ -18,8 +18,8 @@ ConfigParser -- responsible for parsing a list of
              delimiters=('=', ':'), comment_prefixes=('#', ';'),
              inline_comment_prefixes=None, strict=True,
              empty_lines_in_values=True, default_section='DEFAULT',
-             interpolation=<unset>, converters=<unset>):
-
+             interpolation=<unset>, converters=<unset>,
+             allow_unnamed_section=False):
         Create the parser. When `defaults` is given, it is initialized into the
         dictionary or intrinsic defaults. The keys must be strings, the values
         must be appropriate for %()s string interpolation.
@@ -68,6 +68,10 @@ ConfigParser -- responsible for parsing a list of
         converter gets its corresponding get*() method on the parser object and
         section proxies.
 
+        When `allow_unnamed_section` is True (default: False), options
+        without section are accepted: the section for these is
+        ``configparser.UNNAMED_SECTION``.
+
     sections()
         Return all the configuration section names, sans DEFAULT.
 
@@ -139,24 +143,28 @@ ConfigParser -- responsible for parsing a list of
         between keys and values are surrounded by spaces.
 """
 
-from collections.abc import MutableMapping
+# Do not import dataclasses; overhead is unacceptable (gh-117703)
+
+from collections.abc import Iterable, MutableMapping
 from collections import ChainMap as _ChainMap
+import contextlib
 import functools
 import io
 import itertools
 import os
 import re
 import sys
-import warnings
+import types
 
 __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError",
            "NoOptionError", "InterpolationError", "InterpolationDepthError",
            "InterpolationMissingOptionError", "InterpolationSyntaxError",
            "ParsingError", "MissingSectionHeaderError",
+           "MultilineContinuationError",
            "ConfigParser", "RawConfigParser",
            "Interpolation", "BasicInterpolation",  "ExtendedInterpolation",
-           "LegacyInterpolation", "SectionProxy", "ConverterMapping",
-           "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH")
+           "SectionProxy", "ConverterMapping",
+           "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION")
 
 _default_dict = dict
 DEFAULTSECT = "DEFAULT"
@@ -298,15 +306,33 @@ class InterpolationDepthError(InterpolationError):
 class ParsingError(Error):
     """Raised when a configuration file does not follow legal syntax."""
 
-    def __init__(self, source):
+    def __init__(self, source, *args):
         super().__init__(f'Source contains parsing errors: {source!r}')
         self.source = source
         self.errors = []
         self.args = (source, )
+        if args:
+            self.append(*args)
 
     def append(self, lineno, line):
         self.errors.append((lineno, line))
-        self.message += '\n\t[line %2d]: %s' % (lineno, line)
+        self.message += '\n\t[line %2d]: %s' % (lineno, repr(line))
+
+    def combine(self, others):
+        for other in others:
+            for error in other.errors:
+                self.append(*error)
+        return self
+
+    @staticmethod
+    def _raise_all(exceptions: Iterable['ParsingError']):
+        """
+        Combine any number of ParsingErrors into one and raise it.
+        """
+        exceptions = iter(exceptions)
+        with contextlib.suppress(StopIteration):
+            raise next(exceptions).combine(exceptions)
+
 
 
 class MissingSectionHeaderError(ParsingError):
@@ -323,6 +349,28 @@ class MissingSectionHeaderError(ParsingError):
         self.args = (filename, lineno, line)
 
 
+class MultilineContinuationError(ParsingError):
+    """Raised when a key without value is followed by continuation line"""
+    def __init__(self, filename, lineno, line):
+        Error.__init__(
+            self,
+            "Key without value continued with an indented line.\n"
+            "file: %r, line: %d\n%r"
+            %(filename, lineno, line))
+        self.source = filename
+        self.lineno = lineno
+        self.line = line
+        self.args = (filename, lineno, line)
+
+class _UnnamedSection:
+
+    def __repr__(self):
+        return "<UNNAMED_SECTION>"
+
+
+UNNAMED_SECTION = _UnnamedSection()
+
+
 # Used in parser getters to indicate the default behaviour when a specific
 # option is not found it to raise an exception. Created to enable `None` as
 # a valid fallback value.
@@ -478,6 +526,8 @@ class ExtendedInterpolation(Interpolation):
                 except (KeyError, NoSectionError, NoOptionError):
                     raise InterpolationMissingOptionError(
                         option, section, rawval, ":".join(path)) from None
+                if v is None:
+                    continue
                 if "$" in v:
                     self._interpolate_some(parser, opt, accum, v, sect,
                                            dict(parser.items(sect, raw=True)),
@@ -491,51 +541,50 @@ class ExtendedInterpolation(Interpolation):
                     "found: %r" % (rest,))
 
 
-class LegacyInterpolation(Interpolation):
-    """Deprecated interpolation used in old versions of ConfigParser.
-    Use BasicInterpolation or ExtendedInterpolation instead."""
+class _ReadState:
+    elements_added : set[str]
+    cursect : dict[str, str] | None = None
+    sectname : str | None = None
+    optname : str | None = None
+    lineno : int = 0
+    indent_level : int = 0
+    errors : list[ParsingError]
 
-    _KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
+    def __init__(self):
+        self.elements_added = set()
+        self.errors = list()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        warnings.warn(
-            "LegacyInterpolation has been deprecated since Python 3.2 "
-            "and will be removed from the configparser module in Python 3.13. "
-            "Use BasicInterpolation or ExtendedInterpolation instead.",
-            DeprecationWarning, stacklevel=2
+
+class _Line(str):
+
+    def __new__(cls, val, *args, **kwargs):
+        return super().__new__(cls, val)
+
+    def __init__(self, val, prefixes):
+        self.prefixes = prefixes
+
+    @functools.cached_property
+    def clean(self):
+        return self._strip_full() and self._strip_inline()
+
+    @property
+    def has_comments(self):
+        return self.strip() != self.clean
+
+    def _strip_inline(self):
+        """
+        Search for the earliest prefix at the beginning of the line or following a space.
+        """
+        matcher = re.compile(
+            '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline)
+            # match nothing if no prefixes
+            or '(?!)'
         )
+        match = matcher.search(self)
+        return self[:match.start() if match else None].strip()
 
-    def before_get(self, parser, section, option, value, vars):
-        rawval = value
-        depth = MAX_INTERPOLATION_DEPTH
-        while depth:                    # Loop through this until it's done
-            depth -= 1
-            if value and "%(" in value:
-                replace = functools.partial(self._interpolation_replace,
-                                            parser=parser)
-                value = self._KEYCRE.sub(replace, value)
-                try:
-                    value = value % vars
-                except KeyError as e:
-                    raise InterpolationMissingOptionError(
-                        option, section, rawval, e.args[0]) from None
-            else:
-                break
-        if value and "%(" in value:
-            raise InterpolationDepthError(option, section, rawval)
-        return value
-
-    def before_set(self, parser, section, option, value):
-        return value
-
-    @staticmethod
-    def _interpolation_replace(match, parser):
-        s = match.group(1)
-        if s is None:
-            return match.group()
-        else:
-            return "%%(%s)s" % parser.optionxform(s)
+    def _strip_full(self):
+        return '' if any(map(self.strip().startswith, self.prefixes.full)) else True
 
 
 class RawConfigParser(MutableMapping):
@@ -584,7 +633,8 @@ class RawConfigParser(MutableMapping):
                  comment_prefixes=('#', ';'), inline_comment_prefixes=None,
                  strict=True, empty_lines_in_values=True,
                  default_section=DEFAULTSECT,
-                 interpolation=_UNSET, converters=_UNSET):
+                 interpolation=_UNSET, converters=_UNSET,
+                 allow_unnamed_section=False,):
 
         self._dict = dict_type
         self._sections = self._dict()
@@ -603,8 +653,10 @@ class RawConfigParser(MutableMapping):
             else:
                 self._optcre = re.compile(self._OPT_TMPL.format(delim=d),
                                           re.VERBOSE)
-        self._comment_prefixes = tuple(comment_prefixes or ())
-        self._inline_comment_prefixes = tuple(inline_comment_prefixes or ())
+        self._prefixes = types.SimpleNamespace(
+            full=tuple(comment_prefixes or ()),
+            inline=tuple(inline_comment_prefixes or ()),
+        )
         self._strict = strict
         self._allow_no_value = allow_no_value
         self._empty_lines_in_values = empty_lines_in_values
@@ -623,6 +675,7 @@ class RawConfigParser(MutableMapping):
             self._converters.update(converters)
         if defaults:
             self._read_defaults(defaults)
+        self._allow_unnamed_section = allow_unnamed_section
 
     def defaults(self):
         return self._defaults
@@ -896,13 +949,19 @@ class RawConfigParser(MutableMapping):
         if self._defaults:
             self._write_section(fp, self.default_section,
                                     self._defaults.items(), d)
+        if UNNAMED_SECTION in self._sections:
+            self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True)
+
         for section in self._sections:
+            if section is UNNAMED_SECTION:
+                continue
             self._write_section(fp, section,
                                 self._sections[section].items(), d)
 
-    def _write_section(self, fp, section_name, section_items, delimiter):
-        """Write a single section to the specified `fp`."""
-        fp.write("[{}]\n".format(section_name))
+    def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False):
+        """Write a single section to the specified `fp'."""
+        if not unnamed:
+            fp.write("[{}]\n".format(section_name))
         for key, value in section_items:
             value = self._interpolation.before_write(self, section_name, key,
                                                      value)
@@ -988,110 +1047,113 @@ class RawConfigParser(MutableMapping):
         in an otherwise empty line or may be entered in lines holding values or
         section names. Please note that comments get stripped off when reading configuration files.
         """
-        elements_added = set()
-        cursect = None                        # None, or a dictionary
-        sectname = None
-        optname = None
-        lineno = 0
-        indent_level = 0
-        e = None                              # None, or an exception
-        for lineno, line in enumerate(fp, start=1):
-            comment_start = sys.maxsize
-            # strip inline comments
-            inline_prefixes = {p: -1 for p in self._inline_comment_prefixes}
-            while comment_start == sys.maxsize and inline_prefixes:
-                next_prefixes = {}
-                for prefix, index in inline_prefixes.items():
-                    index = line.find(prefix, index+1)
-                    if index == -1:
-                        continue
-                    next_prefixes[prefix] = index
-                    if index == 0 or (index > 0 and line[index-1].isspace()):
-                        comment_start = min(comment_start, index)
-                inline_prefixes = next_prefixes
-            # strip full line comments
-            for prefix in self._comment_prefixes:
-                if line.strip().startswith(prefix):
-                    comment_start = 0
-                    break
-            if comment_start == sys.maxsize:
-                comment_start = None
-            value = line[:comment_start].strip()
-            if not value:
+
+        try:
+            ParsingError._raise_all(self._read_inner(fp, fpname))
+        finally:
+            self._join_multiline_values()
+
+    def _read_inner(self, fp, fpname):
+        st = _ReadState()
+
+        Line = functools.partial(_Line, prefixes=self._prefixes)
+        for st.lineno, line in enumerate(map(Line, fp), start=1):
+            if not line.clean:
                 if self._empty_lines_in_values:
                     # add empty line to the value, but only if there was no
                     # comment on the line
-                    if (comment_start is None and
-                        cursect is not None and
-                        optname and
-                        cursect[optname] is not None):
-                        cursect[optname].append('') # newlines added at join
+                    if (not line.has_comments and
+                        st.cursect is not None and
+                        st.optname and
+                        st.cursect[st.optname] is not None):
+                        st.cursect[st.optname].append('') # newlines added at join
                 else:
                     # empty line marks end of value
-                    indent_level = sys.maxsize
+                    st.indent_level = sys.maxsize
                 continue
-            # continuation line?
+
             first_nonspace = self.NONSPACECRE.search(line)
-            cur_indent_level = first_nonspace.start() if first_nonspace else 0
-            if (cursect is not None and optname and
-                cur_indent_level > indent_level):
-                cursect[optname].append(value)
-            # a section header or option header?
-            else:
-                indent_level = cur_indent_level
-                # is it a section header?
-                mo = self.SECTCRE.match(value)
-                if mo:
-                    sectname = mo.group('header')
-                    if sectname in self._sections:
-                        if self._strict and sectname in elements_added:
-                            raise DuplicateSectionError(sectname, fpname,
-                                                        lineno)
-                        cursect = self._sections[sectname]
-                        elements_added.add(sectname)
-                    elif sectname == self.default_section:
-                        cursect = self._defaults
-                    else:
-                        cursect = self._dict()
-                        self._sections[sectname] = cursect
-                        self._proxies[sectname] = SectionProxy(self, sectname)
-                        elements_added.add(sectname)
-                    # So sections can't start with a continuation line
-                    optname = None
-                # no section header in the file?
-                elif cursect is None:
-                    raise MissingSectionHeaderError(fpname, lineno, line)
-                # an option line?
-                else:
-                    mo = self._optcre.match(value)
-                    if mo:
-                        optname, vi, optval = mo.group('option', 'vi', 'value')
-                        if not optname:
-                            e = self._handle_error(e, fpname, lineno, line)
-                        optname = self.optionxform(optname.rstrip())
-                        if (self._strict and
-                            (sectname, optname) in elements_added):
-                            raise DuplicateOptionError(sectname, optname,
-                                                       fpname, lineno)
-                        elements_added.add((sectname, optname))
-                        # This check is fine because the OPTCRE cannot
-                        # match if it would set optval to None
-                        if optval is not None:
-                            optval = optval.strip()
-                            cursect[optname] = [optval]
-                        else:
-                            # valueless option handling
-                            cursect[optname] = None
-                    else:
-                        # a non-fatal parsing error occurred. set up the
-                        # exception but keep going. the exception will be
-                        # raised at the end of the file and will contain a
-                        # list of all bogus lines
-                        e = self._handle_error(e, fpname, lineno, line)
-        self._join_multiline_values()
-        # if any parsing errors occurred, raise an exception
-        if e:
-            raise e
+            st.cur_indent_level = first_nonspace.start() if first_nonspace else 0
+
+            if self._handle_continuation_line(st, line, fpname):
+                continue
+
+            self._handle_rest(st, line, fpname)
+
+        return st.errors
+
+    def _handle_continuation_line(self, st, line, fpname):
+        # continuation line?
+        is_continue = (st.cursect is not None and st.optname and
+            st.cur_indent_level > st.indent_level)
+        if is_continue:
+            if st.cursect[st.optname] is None:
+                raise MultilineContinuationError(fpname, st.lineno, line)
+            st.cursect[st.optname].append(line.clean)
+        return is_continue
+
+    def _handle_rest(self, st, line, fpname):
+        # a section header or option header?
+        if self._allow_unnamed_section and st.cursect is None:
+            self._handle_header(st, UNNAMED_SECTION, fpname)
+
+        st.indent_level = st.cur_indent_level
+        # is it a section header?
+        mo = self.SECTCRE.match(line.clean)
+
+        if not mo and st.cursect is None:
+            raise MissingSectionHeaderError(fpname, st.lineno, line)
+
+        self._handle_header(st, mo.group('header'), fpname) if mo else self._handle_option(st, line, fpname)
+
+    def _handle_header(self, st, sectname, fpname):
+        st.sectname = sectname
+        if st.sectname in self._sections:
+            if self._strict and st.sectname in st.elements_added:
+                raise DuplicateSectionError(st.sectname, fpname,
+                                            st.lineno)
+            st.cursect = self._sections[st.sectname]
+            st.elements_added.add(st.sectname)
+        elif st.sectname == self.default_section:
+            st.cursect = self._defaults
+        else:
+            st.cursect = self._dict()
+            self._sections[st.sectname] = st.cursect
+            self._proxies[st.sectname] = SectionProxy(self, st.sectname)
+            st.elements_added.add(st.sectname)
+        # So sections can't start with a continuation line
+        st.optname = None
+
+    def _handle_option(self, st, line, fpname):
+        # an option line?
+        st.indent_level = st.cur_indent_level
+
+        mo = self._optcre.match(line.clean)
+        if not mo:
+            # a non-fatal parsing error occurred. set up the
+            # exception but keep going. the exception will be
+            # raised at the end of the file and will contain a
+            # list of all bogus lines
+            st.errors.append(ParsingError(fpname, st.lineno, line))
+            return
+
+        st.optname, vi, optval = mo.group('option', 'vi', 'value')
+        if not st.optname:
+            st.errors.append(ParsingError(fpname, st.lineno, line))
+        st.optname = self.optionxform(st.optname.rstrip())
+        if (self._strict and
+            (st.sectname, st.optname) in st.elements_added):
+            raise DuplicateOptionError(st.sectname, st.optname,
+                                    fpname, st.lineno)
+        st.elements_added.add((st.sectname, st.optname))
+        # This check is fine because the OPTCRE cannot
+        # match if it would set optval to None
+        if optval is not None:
+            optval = optval.strip()
+            st.cursect[st.optname] = [optval]
+        else:
+            # valueless option handling
+            st.cursect[st.optname] = None
 
     def _join_multiline_values(self):
         defaults = self.default_section, self._defaults
@@ -1111,12 +1173,6 @@ class RawConfigParser(MutableMapping):
         for key, value in defaults.items():
             self._defaults[self.optionxform(key)] = value
 
-    def _handle_error(self, exc, fpname, lineno, line):
-        if not exc:
-            exc = ParsingError(fpname)
-        exc.append(lineno, repr(line))
-        return exc
-
     def _unify_values(self, section, vars):
         """Create a sequence of lookups with 'vars' taking priority over
         the 'section' which takes priority over the DEFAULTSECT.

Lib/contextlib.py

@@ -20,6 +20,8 @@ class AbstractContextManager(abc.ABC):
 
     __class_getitem__ = classmethod(GenericAlias)
 
+    __slots__ = ()
+
     def __enter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -42,6 +44,8 @@ class AbstractAsyncContextManager(abc.ABC):
 
     __class_getitem__ = classmethod(GenericAlias)
 
+    __slots__ = ()
+
     async def __aenter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -565,11 +569,12 @@ class ExitStack(_BaseExitStack, AbstractContextManager):
         return self
 
     def __exit__(self, *exc_details):
-        received_exc = exc_details[0] is not None
+        exc = exc_details[1]
+        received_exc = exc is not None
 
         # We manipulate the exception state so it behaves as though
         # we were actually nesting multiple with statements
-        frame_exc = sys.exc_info()[1]
+        frame_exc = sys.exception()
         def _fix_exception_context(new_exc, old_exc):
             # Context may not be correct, so find the end of the chain
             while 1:
@@ -592,24 +597,28 @@ class ExitStack(_BaseExitStack, AbstractContextManager):
             is_sync, cb = self._exit_callbacks.pop()
             assert is_sync
             try:
+                if exc is None:
+                    exc_details = None, None, None
+                else:
+                    exc_details = type(exc), exc, exc.__traceback__
                 if cb(*exc_details):
                     suppressed_exc = True
                     pending_raise = False
-                    exc_details = (None, None, None)
-            except:
-                new_exc_details = sys.exc_info()
+                    exc = None
+            except BaseException as new_exc:
                 # simulate the stack of exceptions by setting the context
-                _fix_exception_context(new_exc_details[1], exc_details[1])
+                _fix_exception_context(new_exc, exc)
                 pending_raise = True
-                exc_details = new_exc_details
+                exc = new_exc
+
         if pending_raise:
             try:
-                # bare "raise exc_details[1]" replaces our carefully
+                # bare "raise exc" replaces our carefully
                 # set-up context
-                fixed_ctx = exc_details[1].__context__
-                raise exc_details[1]
+                fixed_ctx = exc.__context__
+                raise exc
             except BaseException:
-                exc_details[1].__context__ = fixed_ctx
+                exc.__context__ = fixed_ctx
                 raise
         return received_exc and suppressed_exc
 
@@ -705,11 +714,12 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         return self
 
     async def __aexit__(self, *exc_details):
-        received_exc = exc_details[0] is not None
+        exc = exc_details[1]
+        received_exc = exc is not None
 
         # We manipulate the exception state so it behaves as though
         # we were actually nesting multiple with statements
-        frame_exc = sys.exc_info()[1]
+        frame_exc = sys.exception()
         def _fix_exception_context(new_exc, old_exc):
             # Context may not be correct, so find the end of the chain
             while 1:
@@ -731,6 +741,10 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         while self._exit_callbacks:
             is_sync, cb = self._exit_callbacks.pop()
             try:
+                if exc is None:
+                    exc_details = None, None, None
+                else:
+                    exc_details = type(exc), exc, exc.__traceback__
                 if is_sync:
                     cb_suppress = cb(*exc_details)
                 else:
@@ -739,21 +753,21 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
                 if cb_suppress:
                     suppressed_exc = True
                     pending_raise = False
-                    exc_details = (None, None, None)
-            except:
-                new_exc_details = sys.exc_info()
+                    exc = None
+            except BaseException as new_exc:
                 # simulate the stack of exceptions by setting the context
-                _fix_exception_context(new_exc_details[1], exc_details[1])
+                _fix_exception_context(new_exc, exc)
                 pending_raise = True
-                exc_details = new_exc_details
+                exc = new_exc
+
         if pending_raise:
             try:
-                # bare "raise exc_details[1]" replaces our carefully
+                # bare "raise exc" replaces our carefully
                 # set-up context
-                fixed_ctx = exc_details[1].__context__
-                raise exc_details[1]
+                fixed_ctx = exc.__context__
+                raise exc
             except BaseException:
-                exc_details[1].__context__ = fixed_ctx
+                exc.__context__ = fixed_ctx
                 raise
         return received_exc and suppressed_exc
 

Lib/difflib.py

@@ -1200,6 +1200,25 @@ def context_diff(a, b, fromfile='', tofile='',
     strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'.
     The modification times are normally expressed in the ISO 8601 format.
     If not specified, the strings default to blanks.
+
+    Example:
+
+    >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True),
+    ...       'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')),
+    ...       end="")
+    *** Original
+    --- Current
+    ***************
+    *** 1,4 ****
+      one
+    ! two
+    ! three
+      four
+    --- 1,4 ----
+    + zero
+      one
+    ! tree
+      four
     """
 
     _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm)
@@ -1609,7 +1628,7 @@ _file_template = """
 </html>"""
 
 _styles = """
-        table.diff {font-family:Courier; border:medium;}
+        table.diff {font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; border:medium}
         .diff_header {background-color:#e0e0e0}
         td.diff_header {text-align:right}
         .diff_next {background-color:#c0c0c0}

Lib/encodings/aliases.py

@@ -209,6 +209,7 @@ aliases = {
     'ms932'              : 'cp932',
     'mskanji'            : 'cp932',
     'ms_kanji'           : 'cp932',
+    'windows_31j'        : 'cp932',
 
     # cp949 codec
     '949'                : 'cp949',

Lib/encodings/idna.py

@@ -11,7 +11,7 @@ ace_prefix = b"xn--"
 sace_prefix = "xn--"
 
 # This assumes query strings, so AllowUnassigned is true
-def nameprep(label):
+def nameprep(label):  # type: (str) -> str
     # Map
     newlabel = []
     for c in label:
@@ -25,7 +25,7 @@ def nameprep(label):
     label = unicodedata.normalize("NFKC", label)
 
     # Prohibit
-    for c in label:
+    for i, c in enumerate(label):
         if stringprep.in_table_c12(c) or \
            stringprep.in_table_c22(c) or \
            stringprep.in_table_c3(c) or \
@@ -35,7 +35,7 @@ def nameprep(label):
            stringprep.in_table_c7(c) or \
            stringprep.in_table_c8(c) or \
            stringprep.in_table_c9(c):
-            raise UnicodeError("Invalid character %r" % c)
+            raise UnicodeEncodeError("idna", label, i, i+1, f"Invalid character {c!r}")
 
     # Check bidi
     RandAL = [stringprep.in_table_d1(x) for x in label]
@@ -46,29 +46,38 @@ def nameprep(label):
         # This is table C.8, which was already checked
         # 2) If a string contains any RandALCat character, the string
         # MUST NOT contain any LCat character.
-        if any(stringprep.in_table_d2(x) for x in label):
-            raise UnicodeError("Violation of BIDI requirement 2")
+        for i, x in enumerate(label):
+            if stringprep.in_table_d2(x):
+                raise UnicodeEncodeError("idna", label, i, i+1,
+                                         "Violation of BIDI requirement 2")
         # 3) If a string contains any RandALCat character, a
         # RandALCat character MUST be the first character of the
         # string, and a RandALCat character MUST be the last
         # character of the string.
-        if not RandAL[0] or not RandAL[-1]:
-            raise UnicodeError("Violation of BIDI requirement 3")
+        if not RandAL[0]:
+            raise UnicodeEncodeError("idna", label, 0, 1,
+                                     "Violation of BIDI requirement 3")
+        if not RandAL[-1]:
+            raise UnicodeEncodeError("idna", label, len(label)-1, len(label),
+                                     "Violation of BIDI requirement 3")
 
     return label
 
-def ToASCII(label):
+def ToASCII(label):  # type: (str) -> bytes
     try:
         # Step 1: try ASCII
-        label = label.encode("ascii")
-    except UnicodeError:
+        label_ascii = label.encode("ascii")
+    except UnicodeEncodeError:
         pass
     else:
         # Skip to step 3: UseSTD3ASCIIRules is false, so
         # Skip to step 8.
-        if 0 < len(label) < 64:
-            return label
-        raise UnicodeError("label empty or too long")
+        if 0 < len(label_ascii) < 64:
+            return label_ascii
+        if len(label) == 0:
+            raise UnicodeEncodeError("idna", label, 0, 1, "label empty")
+        else:
+            raise UnicodeEncodeError("idna", label, 0, len(label), "label too long")
 
     # Step 2: nameprep
     label = nameprep(label)
@@ -76,29 +85,34 @@ def ToASCII(label):
     # Step 3: UseSTD3ASCIIRules is false
     # Step 4: try ASCII
     try:
-        label = label.encode("ascii")
-    except UnicodeError:
+        label_ascii = label.encode("ascii")
+    except UnicodeEncodeError:
         pass
     else:
         # Skip to step 8.
         if 0 < len(label) < 64:
-            return label
-        raise UnicodeError("label empty or too long")
+            return label_ascii
+        if len(label) == 0:
+            raise UnicodeEncodeError("idna", label, 0, 1, "label empty")
+        else:
+            raise UnicodeEncodeError("idna", label, 0, len(label), "label too long")
 
     # Step 5: Check ACE prefix
-    if label.startswith(sace_prefix):
-        raise UnicodeError("Label starts with ACE prefix")
+    if label.lower().startswith(sace_prefix):
+        raise UnicodeEncodeError(
+            "idna", label, 0, len(sace_prefix), "Label starts with ACE prefix")
 
     # Step 6: Encode with PUNYCODE
-    label = label.encode("punycode")
+    label_ascii = label.encode("punycode")
 
     # Step 7: Prepend ACE prefix
-    label = ace_prefix + label
+    label_ascii = ace_prefix + label_ascii
 
     # Step 8: Check size
-    if 0 < len(label) < 64:
-        return label
-    raise UnicodeError("label empty or too long")
+    # do not check for empty as we prepend ace_prefix.
+    if len(label_ascii) < 64:
+        return label_ascii
+    raise UnicodeEncodeError("idna", label, 0, len(label), "label too long")
 
 def ToUnicode(label):
     if len(label) > 1024:
@@ -110,7 +124,9 @@ def ToUnicode(label):
         # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still
         # preventing us from wasting time decoding a big thing that'll just
         # hit the actual <= 63 length limit in Step 6.
-        raise UnicodeError("label way too long")
+        if isinstance(label, str):
+            label = label.encode("utf-8", errors="backslashreplace")
+        raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long")
     # Step 1: Check for ASCII
     if isinstance(label, bytes):
         pure_ascii = True
@@ -118,25 +134,32 @@ def ToUnicode(label):
         try:
             label = label.encode("ascii")
             pure_ascii = True
-        except UnicodeError:
+        except UnicodeEncodeError:
             pure_ascii = False
     if not pure_ascii:
+        assert isinstance(label, str)
         # Step 2: Perform nameprep
         label = nameprep(label)
         # It doesn't say this, but apparently, it should be ASCII now
         try:
             label = label.encode("ascii")
-        except UnicodeError:
-            raise UnicodeError("Invalid character in IDN label")
+        except UnicodeEncodeError as exc:
+            raise UnicodeEncodeError("idna", label, exc.start, exc.end,
+                                     "Invalid character in IDN label")
     # Step 3: Check for ACE prefix
-    if not label.startswith(ace_prefix):
+    assert isinstance(label, bytes)
+    if not label.lower().startswith(ace_prefix):
         return str(label, "ascii")
 
     # Step 4: Remove ACE prefix
     label1 = label[len(ace_prefix):]
 
     # Step 5: Decode using PUNYCODE
-    result = label1.decode("punycode")
+    try:
+        result = label1.decode("punycode")
+    except UnicodeDecodeError as exc:
+        offset = len(ace_prefix)
+        raise UnicodeDecodeError("idna", label, offset+exc.start, offset+exc.end, exc.reason)
 
     # Step 6: Apply ToASCII
     label2 = ToASCII(result)
@@ -144,7 +167,8 @@ def ToUnicode(label):
     # Step 7: Compare the result of step 6 with the one of step 3
     # label2 will already be in lower case.
     if str(label, "ascii").lower() != str(label2, "ascii"):
-        raise UnicodeError("IDNA does not round-trip", label, label2)
+        raise UnicodeDecodeError("idna", label, 0, len(label),
+                                 f"IDNA does not round-trip, '{label!r}' != '{label2!r}'")
 
     # Step 8: return the result of step 5
     return result
@@ -156,7 +180,7 @@ class Codec(codecs.Codec):
 
         if errors != 'strict':
             # IDNA is quite clear that implementations must be strict
-            raise UnicodeError("unsupported error handling "+errors)
+            raise UnicodeError(f"Unsupported error handling: {errors}")
 
         if not input:
             return b'', 0
@@ -168,11 +192,16 @@ class Codec(codecs.Codec):
         else:
             # ASCII name: fast path
             labels = result.split(b'.')
-            for label in labels[:-1]:
-                if not (0 < len(label) < 64):
-                    raise UnicodeError("label empty or too long")
-            if len(labels[-1]) >= 64:
-                raise UnicodeError("label too long")
+            for i, label in enumerate(labels[:-1]):
+                if len(label) == 0:
+                    offset = sum(len(l) for l in labels[:i]) + i
+                    raise UnicodeEncodeError("idna", input, offset, offset+1,
+                                             "label empty")
+            for i, label in enumerate(labels):
+                if len(label) >= 64:
+                    offset = sum(len(l) for l in labels[:i]) + i
+                    raise UnicodeEncodeError("idna", input, offset, offset+len(label),
+                                             "label too long")
             return result, len(input)
 
         result = bytearray()
@@ -182,17 +211,27 @@ class Codec(codecs.Codec):
             del labels[-1]
         else:
             trailing_dot = b''
-        for label in labels:
+        for i, label in enumerate(labels):
             if result:
                 # Join with U+002E
                 result.extend(b'.')
-            result.extend(ToASCII(label))
+            try:
+                result.extend(ToASCII(label))
+            except (UnicodeEncodeError, UnicodeDecodeError) as exc:
+                offset = sum(len(l) for l in labels[:i]) + i
+                raise UnicodeEncodeError(
+                    "idna",
+                    input,
+                    offset + exc.start,
+                    offset + exc.end,
+                    exc.reason,
+                )
         return bytes(result+trailing_dot), len(input)
 
     def decode(self, input, errors='strict'):
 
         if errors != 'strict':
-            raise UnicodeError("Unsupported error handling "+errors)
+            raise UnicodeError(f"Unsupported error handling: {errors}")
 
         if not input:
             return "", 0
@@ -202,7 +241,7 @@ class Codec(codecs.Codec):
             # XXX obviously wrong, see #3232
             input = bytes(input)
 
-        if ace_prefix not in input:
+        if ace_prefix not in input.lower():
             # Fast path
             try:
                 return input.decode('ascii'), len(input)
@@ -218,8 +257,15 @@ class Codec(codecs.Codec):
             trailing_dot = ''
 
         result = []
-        for label in labels:
-            result.append(ToUnicode(label))
+        for i, label in enumerate(labels):
+            try:
+                u_label = ToUnicode(label)
+            except (UnicodeEncodeError, UnicodeDecodeError) as exc:
+                offset = sum(len(x) for x in labels[:i]) + len(labels[:i])
+                raise UnicodeDecodeError(
+                    "idna", input, offset+exc.start, offset+exc.end, exc.reason)
+            else:
+                result.append(u_label)
 
         return ".".join(result)+trailing_dot, len(input)
 
@@ -227,7 +273,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
     def _buffer_encode(self, input, errors, final):
         if errors != 'strict':
             # IDNA is quite clear that implementations must be strict
-            raise UnicodeError("unsupported error handling "+errors)
+            raise UnicodeError(f"Unsupported error handling: {errors}")
 
         if not input:
             return (b'', 0)
@@ -251,7 +297,16 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
                 # Join with U+002E
                 result.extend(b'.')
                 size += 1
-            result.extend(ToASCII(label))
+            try:
+                result.extend(ToASCII(label))
+            except (UnicodeEncodeError, UnicodeDecodeError) as exc:
+                raise UnicodeEncodeError(
+                    "idna",
+                    input,
+                    size + exc.start,
+                    size + exc.end,
+                    exc.reason,
+                )
             size += len(label)
 
         result += trailing_dot
@@ -261,7 +316,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
 class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
     def _buffer_decode(self, input, errors, final):
         if errors != 'strict':
-            raise UnicodeError("Unsupported error handling "+errors)
+            raise UnicodeError(f"Unsupported error handling: {errors}")
 
         if not input:
             return ("", 0)
@@ -271,7 +326,11 @@ class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
             labels = dots.split(input)
         else:
             # Must be ASCII string
-            input = str(input, "ascii")
+            try:
+                input = str(input, "ascii")
+            except (UnicodeEncodeError, UnicodeDecodeError) as exc:
+                raise UnicodeDecodeError("idna", input,
+                                         exc.start, exc.end, exc.reason)
             labels = input.split(".")
 
         trailing_dot = ''
@@ -288,7 +347,18 @@ class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
         result = []
         size = 0
         for label in labels:
-            result.append(ToUnicode(label))
+            try:
+                u_label = ToUnicode(label)
+            except (UnicodeEncodeError, UnicodeDecodeError) as exc:
+                raise UnicodeDecodeError(
+                    "idna",
+                    input.encode("ascii", errors="backslashreplace"),
+                    size + exc.start,
+                    size + exc.end,
+                    exc.reason,
+                )
+            else:
+                result.append(u_label)
             if size:
                 size += 1
             size += len(label)

Lib/encodings/palmos.py

@@ -201,7 +201,7 @@ decoding_table = (
     '\u02dc'   #  0x98 -> SMALL TILDE
     '\u2122'   #  0x99 -> TRADE MARK SIGN
     '\u0161'   #  0x9A -> LATIN SMALL LETTER S WITH CARON
-    '\x9b'     #  0x9B -> <control>
+    '\u203a'   #  0x9B -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
     '\u0153'   #  0x9C -> LATIN SMALL LIGATURE OE
     '\x9d'     #  0x9D -> <control>
     '\x9e'     #  0x9E -> <control>

Lib/encodings/punycode.py

@@ -1,4 +1,4 @@
-""" Codec for the Punicode encoding, as specified in RFC 3492
+""" Codec for the Punycode encoding, as specified in RFC 3492
 
 Written by Martin v. Löwis.
 """
@@ -131,10 +131,11 @@ def decode_generalized_number(extended, extpos, bias, errors):
     j = 0
     while 1:
         try:
-            char = ord(extended[extpos])
+            char = extended[extpos]
         except IndexError:
             if errors == "strict":
-                raise UnicodeError("incomplete punicode string")
+                raise UnicodeDecodeError("punycode", extended, extpos, extpos+1,
+                                         "incomplete punycode string")
             return extpos + 1, None
         extpos += 1
         if 0x41 <= char <= 0x5A: # A-Z
@@ -142,8 +143,8 @@ def decode_generalized_number(extended, extpos, bias, errors):
         elif 0x30 <= char <= 0x39:
             digit = char - 22 # 0x30-26
         elif errors == "strict":
-            raise UnicodeError("Invalid extended code point '%s'"
-                               % extended[extpos-1])
+            raise UnicodeDecodeError("punycode", extended, extpos-1, extpos,
+                                     f"Invalid extended code point '{extended[extpos-1]}'")
         else:
             return extpos, None
         t = T(j, bias)
@@ -155,11 +156,14 @@ def decode_generalized_number(extended, extpos, bias, errors):
 
 
 def insertion_sort(base, extended, errors):
-    """3.2 Insertion unsort coding"""
+    """3.2 Insertion sort coding"""
+    # This function raises UnicodeDecodeError with position in the extended.
+    # Caller should add the offset.
     char = 0x80
     pos = -1
     bias = 72
     extpos = 0
+
     while extpos < len(extended):
         newpos, delta = decode_generalized_number(extended, extpos,
                                                   bias, errors)
@@ -171,7 +175,9 @@ def insertion_sort(base, extended, errors):
         char += pos // (len(base) + 1)
         if char > 0x10FFFF:
             if errors == "strict":
-                raise UnicodeError("Invalid character U+%x" % char)
+                raise UnicodeDecodeError(
+                    "punycode", extended, pos-1, pos,
+                    f"Invalid character U+{char:x}")
             char = ord('?')
         pos = pos % (len(base) + 1)
         base = base[:pos] + chr(char) + base[pos:]
@@ -187,11 +193,21 @@ def punycode_decode(text, errors):
     pos = text.rfind(b"-")
     if pos == -1:
         base = ""
-        extended = str(text, "ascii").upper()
+        extended = text.upper()
     else:
-        base = str(text[:pos], "ascii", errors)
-        extended = str(text[pos+1:], "ascii").upper()
-    return insertion_sort(base, extended, errors)
+        try:
+            base = str(text[:pos], "ascii", errors)
+        except UnicodeDecodeError as exc:
+            raise UnicodeDecodeError("ascii", text, exc.start, exc.end,
+                                     exc.reason) from None
+        extended = text[pos+1:].upper()
+    try:
+        return insertion_sort(base, extended, errors)
+    except UnicodeDecodeError as exc:
+        offset = pos + 1
+        raise UnicodeDecodeError("punycode", text,
+                                 offset+exc.start, offset+exc.end,
+                                 exc.reason) from None
 
 ### Codec APIs
 
@@ -203,7 +219,7 @@ class Codec(codecs.Codec):
 
     def decode(self, input, errors='strict'):
         if errors not in ('strict', 'replace', 'ignore'):
-            raise UnicodeError("Unsupported error handling "+errors)
+            raise UnicodeError(f"Unsupported error handling: {errors}")
         res = punycode_decode(input, errors)
         return res, len(input)
 
@@ -214,7 +230,7 @@ class IncrementalEncoder(codecs.IncrementalEncoder):
 class IncrementalDecoder(codecs.IncrementalDecoder):
     def decode(self, input, final=False):
         if self.errors not in ('strict', 'replace', 'ignore'):
-            raise UnicodeError("Unsupported error handling "+self.errors)
+            raise UnicodeError(f"Unsupported error handling: {self.errors}")
         return punycode_decode(input, self.errors)
 
 class StreamWriter(Codec,codecs.StreamWriter):

Lib/encodings/undefined.py

@@ -1,6 +1,6 @@
 """ Python 'undefined' Codec
 
-    This codec will always raise a ValueError exception when being
+    This codec will always raise a UnicodeError exception when being
     used. It is intended for use by the site.py file to switch off
     automatic string to Unicode coercion.
 

Lib/encodings/utf_16.py

@@ -64,7 +64,7 @@ class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
             elif byteorder == 1:
                 self.decoder = codecs.utf_16_be_decode
             elif consumed >= 2:
-                raise UnicodeError("UTF-16 stream does not start with BOM")
+                raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM")
             return (output, consumed)
         return self.decoder(input, self.errors, final)
 
@@ -138,7 +138,7 @@ class StreamReader(codecs.StreamReader):
         elif byteorder == 1:
             self.decode = codecs.utf_16_be_decode
         elif consumed>=2:
-            raise UnicodeError("UTF-16 stream does not start with BOM")
+            raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM")
         return (object, consumed)
 
 ### encodings module API

Lib/encodings/utf_32.py

@@ -59,7 +59,7 @@ class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
             elif byteorder == 1:
                 self.decoder = codecs.utf_32_be_decode
             elif consumed >= 4:
-                raise UnicodeError("UTF-32 stream does not start with BOM")
+                raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM")
             return (output, consumed)
         return self.decoder(input, self.errors, final)
 
@@ -132,8 +132,8 @@ class StreamReader(codecs.StreamReader):
             self.decode = codecs.utf_32_le_decode
         elif byteorder == 1:
             self.decode = codecs.utf_32_be_decode
-        elif consumed>=4:
-            raise UnicodeError("UTF-32 stream does not start with BOM")
+        elif consumed >= 4:
+            raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM")
         return (object, consumed)
 
 ### encodings module API

Lib/fnmatch.py

@@ -16,12 +16,6 @@ import functools
 
 __all__ = ["filter", "fnmatch", "fnmatchcase", "translate"]
 
-# Build a thread-safe incrementing counter to help create unique regexp group
-# names across calls.
-from itertools import count
-_nextgroupnum = count().__next__
-del count
-
 def fnmatch(name, pat):
     """Test whether FILENAME matches PATTERN.
 
@@ -41,7 +35,7 @@ def fnmatch(name, pat):
     pat = os.path.normcase(pat)
     return fnmatchcase(name, pat)
 
-@functools.lru_cache(maxsize=256, typed=True)
+@functools.lru_cache(maxsize=32768, typed=True)
 def _compile_pattern(pat):
     if isinstance(pat, bytes):
         pat_str = str(pat, 'ISO-8859-1')
@@ -84,6 +78,11 @@ def translate(pat):
     """
 
     STAR = object()
+    parts = _translate(pat, STAR, '.')
+    return _join_translated_parts(parts, STAR)
+
+
+def _translate(pat, STAR, QUESTION_MARK):
     res = []
     add = res.append
     i, n = 0, len(pat)
@@ -95,7 +94,7 @@ def translate(pat):
             if (not res) or res[-1] is not STAR:
                 add(STAR)
         elif c == '?':
-            add('.')
+            add(QUESTION_MARK)
         elif c == '[':
             j = i
             if j < n and pat[j] == '!':
@@ -152,9 +151,11 @@ def translate(pat):
         else:
             add(re.escape(c))
     assert i == n
+    return res
 
+
+def _join_translated_parts(inp, STAR):
     # Deal with STARs.
-    inp = res
     res = []
     add = res.append
     i, n = 0, len(inp)
@@ -165,17 +166,10 @@ def translate(pat):
     # Now deal with STAR fixed STAR fixed ...
     # For an interior `STAR fixed` pairing, we want to do a minimal
     # .*? match followed by `fixed`, with no possibility of backtracking.
-    # We can't spell that directly, but can trick it into working by matching
-    #    .*?fixed
-    # in a lookahead assertion, save the matched part in a group, then
-    # consume that group via a backreference. If the overall match fails,
-    # the lookahead assertion won't try alternatives. So the translation is:
-    #     (?=(?P<name>.*?fixed))(?P=name)
-    # Group names are created as needed: g0, g1, g2, ...
-    # The numbers are obtained from _nextgroupnum() to ensure they're unique
-    # across calls and across threads. This is because people rely on the
-    # undocumented ability to join multiple translate() results together via
-    # "|" to build large regexps matching "one of many" shell patterns.
+    # Atomic groups ("(?>...)") allow us to spell that directly.
+    # Note: people rely on the undocumented ability to join multiple
+    # translate() results together via "|" to build large regexps matching
+    # "one of many" shell patterns.
     while i < n:
         assert inp[i] is STAR
         i += 1
@@ -192,8 +186,7 @@ def translate(pat):
             add(".*")
             add(fixed)
         else:
-            groupnum = _nextgroupnum()
-            add(f"(?=(?P<g{groupnum}>.*?{fixed}))(?P=g{groupnum})")
+            add(f"(?>.*?{fixed})")
     assert i == n
     res = "".join(res)
     return fr'(?s:{res})\Z'

Lib/genericpath.py

@@ -7,8 +7,8 @@ import os
 import stat
 
 __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime',
-           'getsize', 'isdir', 'isfile', 'islink', 'samefile', 'sameopenfile',
-           'samestat']
+           'getsize', 'isdevdrive', 'isdir', 'isfile', 'isjunction', 'islink',
+           'lexists', 'samefile', 'sameopenfile', 'samestat', 'ALLOW_MISSING']
 
 
 # Does a path exist?
@@ -22,6 +22,15 @@ def exists(path):
     return True
 
 
+# Being true for dangling symbolic links is also useful.
+def lexists(path):
+    """Test whether a path exists.  Returns True for broken symbolic links"""
+    try:
+        os.lstat(path)
+    except (OSError, ValueError):
+        return False
+    return True
+
 # This follows symbolic links, so both islink() and isdir() can be true
 # for the same path on systems that support symlinks
 def isfile(path):
@@ -57,6 +66,21 @@ def islink(path):
     return stat.S_ISLNK(st.st_mode)
 
 
+# Is a path a junction?
+def isjunction(path):
+    """Test whether a path is a junction
+    Junctions are not supported on the current platform"""
+    os.fspath(path)
+    return False
+
+
+def isdevdrive(path):
+    """Determines whether the specified path is on a Windows Dev Drive.
+    Dev Drives are not supported on the current platform"""
+    os.fspath(path)
+    return False
+
+
 def getsize(filename):
     """Return the size of a file, reported by os.stat()."""
     return os.stat(filename).st_size
@@ -165,3 +189,12 @@ def _check_arg_types(funcname, *args):
                             f'os.PathLike object, not {s.__class__.__name__!r}') from None
     if hasstr and hasbytes:
         raise TypeError("Can't mix strings and bytes in path components") from None
+
+# A singleton with a true boolean value.
+@object.__new__
+class ALLOW_MISSING:
+    """Special value for use in realpath()."""
+    def __repr__(self):
+        return 'os.path.ALLOW_MISSING'
+    def __reduce__(self):
+        return self.__class__.__name__

Lib/gettext.py

@@ -46,6 +46,7 @@ internationalized, to the local language and cultural habits.
 #   find this format documented anywhere.
 
 
+import operator
 import os
 import re
 import sys
@@ -166,14 +167,28 @@ def _parse(tokens, priority=-1):
 
 def _as_int(n):
     try:
-        i = round(n)
+        round(n)
     except TypeError:
         raise TypeError('Plural value must be an integer, got %s' %
                         (n.__class__.__name__,)) from None
+    return _as_int2(n)
+
+def _as_int2(n):
+    try:
+        return operator.index(n)
+    except TypeError:
+        pass
+
     import warnings
+    frame = sys._getframe(1)
+    stacklevel = 2
+    while frame.f_back is not None and frame.f_globals.get('__name__') == __name__:
+        stacklevel += 1
+        frame = frame.f_back
     warnings.warn('Plural value must be an integer, got %s' %
                   (n.__class__.__name__,),
-                  DeprecationWarning, 4)
+                  DeprecationWarning,
+                  stacklevel)
     return n
 
 
@@ -200,7 +215,7 @@ def c2py(plural):
             elif c == ')':
                 depth -= 1
 
-        ns = {'_as_int': _as_int}
+        ns = {'_as_int': _as_int, '__name__': __name__}
         exec('''if True:
             def func(n):
                 if not isinstance(n, int):
@@ -280,6 +295,7 @@ class NullTranslations:
     def ngettext(self, msgid1, msgid2, n):
         if self._fallback:
             return self._fallback.ngettext(msgid1, msgid2, n)
+        n = _as_int2(n)
         if n == 1:
             return msgid1
         else:
@@ -293,6 +309,7 @@ class NullTranslations:
     def npgettext(self, context, msgid1, msgid2, n):
         if self._fallback:
             return self._fallback.npgettext(context, msgid1, msgid2, n)
+        n = _as_int2(n)
         if n == 1:
             return msgid1
         else:
@@ -579,6 +596,7 @@ def dngettext(domain, msgid1, msgid2, n):
     try:
         t = translation(domain, _localedirs.get(domain, None))
     except OSError:
+        n = _as_int2(n)
         if n == 1:
             return msgid1
         else:
@@ -598,6 +616,7 @@ def dnpgettext(domain, context, msgid1, msgid2, n):
     try:
         t = translation(domain, _localedirs.get(domain, None))
     except OSError:
+        n = _as_int2(n)
         if n == 1:
             return msgid1
         else:

Lib/glob.py

@@ -4,11 +4,14 @@ import contextlib
 import os
 import re
 import fnmatch
+import functools
 import itertools
+import operator
 import stat
 import sys
 
-__all__ = ["glob", "iglob", "escape"]
+
+__all__ = ["glob", "iglob", "escape", "translate"]
 
 def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
         include_hidden=False):
@@ -104,8 +107,8 @@ def _iglob(pathname, root_dir, dir_fd, recursive, dironly,
 
 def _glob1(dirname, pattern, dir_fd, dironly, include_hidden=False):
     names = _listdir(dirname, dir_fd, dironly)
-    if include_hidden or not _ishidden(pattern):
-        names = (x for x in names if include_hidden or not _ishidden(x))
+    if not (include_hidden or _ishidden(pattern)):
+        names = (x for x in names if not _ishidden(x))
     return fnmatch.filter(names, pattern)
 
 def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False):
@@ -119,12 +122,19 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False):
             return [basename]
     return []
 
-# Following functions are not public but can be used by third-party code.
+_deprecated_function_message = (
+    "{name} is deprecated and will be removed in Python {remove}. Use "
+    "glob.glob and pass a directory to its root_dir argument instead."
+)
 
 def glob0(dirname, pattern):
+    import warnings
+    warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15))
     return _glob0(dirname, pattern, None, False)
 
 def glob1(dirname, pattern):
+    import warnings
+    warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15))
     return _glob1(dirname, pattern, None, False)
 
 # This helper function recursively yields relative pathnames inside a literal
@@ -249,4 +259,287 @@ def escape(pathname):
     return drive + pathname
 
 
+_special_parts = ('', '.', '..')
 _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0)
+_no_recurse_symlinks = object()
+
+
+def translate(pat, *, recursive=False, include_hidden=False, seps=None):
+    """Translate a pathname with shell wildcards to a regular expression.
+
+    If `recursive` is true, the pattern segment '**' will match any number of
+    path segments.
+
+    If `include_hidden` is true, wildcards can match path segments beginning
+    with a dot ('.').
+
+    If a sequence of separator characters is given to `seps`, they will be
+    used to split the pattern into segments and match path separators. If not
+    given, os.path.sep and os.path.altsep (where available) are used.
+    """
+    if not seps:
+        if os.path.altsep:
+            seps = (os.path.sep, os.path.altsep)
+        else:
+            seps = os.path.sep
+    escaped_seps = ''.join(map(re.escape, seps))
+    any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps
+    not_sep = f'[^{escaped_seps}]'
+    if include_hidden:
+        one_last_segment = f'{not_sep}+'
+        one_segment = f'{one_last_segment}{any_sep}'
+        any_segments = f'(?:.+{any_sep})?'
+        any_last_segments = '.*'
+    else:
+        one_last_segment = f'[^{escaped_seps}.]{not_sep}*'
+        one_segment = f'{one_last_segment}{any_sep}'
+        any_segments = f'(?:{one_segment})*'
+        any_last_segments = f'{any_segments}(?:{one_last_segment})?'
+
+    results = []
+    parts = re.split(any_sep, pat)
+    last_part_idx = len(parts) - 1
+    for idx, part in enumerate(parts):
+        if part == '*':
+            results.append(one_segment if idx < last_part_idx else one_last_segment)
+        elif recursive and part == '**':
+            if idx < last_part_idx:
+                if parts[idx + 1] != '**':
+                    results.append(any_segments)
+            else:
+                results.append(any_last_segments)
+        else:
+            if part:
+                if not include_hidden and part[0] in '*?':
+                    results.append(r'(?!\.)')
+                results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep))
+            if idx < last_part_idx:
+                results.append(any_sep)
+    res = ''.join(results)
+    return fr'(?s:{res})\Z'
+
+
+@functools.lru_cache(maxsize=512)
+def _compile_pattern(pat, sep, case_sensitive, recursive=True):
+    """Compile given glob pattern to a re.Pattern object (observing case
+    sensitivity)."""
+    flags = re.NOFLAG if case_sensitive else re.IGNORECASE
+    regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep)
+    return re.compile(regex, flags=flags).match
+
+
+class _Globber:
+    """Class providing shell-style pattern matching and globbing.
+    """
+
+    def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False):
+        self.sep = sep
+        self.case_sensitive = case_sensitive
+        self.case_pedantic = case_pedantic
+        self.recursive = recursive
+
+    # Low-level methods
+
+    lstat = operator.methodcaller('lstat')
+    add_slash = operator.methodcaller('joinpath', '')
+
+    @staticmethod
+    def scandir(path):
+        """Emulates os.scandir(), which returns an object that can be used as
+        a context manager. This method is called by walk() and glob().
+        """
+        return contextlib.nullcontext(path.iterdir())
+
+    @staticmethod
+    def concat_path(path, text):
+        """Appends text to the given path.
+        """
+        return path.with_segments(path._raw_path + text)
+
+    @staticmethod
+    def parse_entry(entry):
+        """Returns the path of an entry yielded from scandir().
+        """
+        return entry
+
+    # High-level methods
+
+    def compile(self, pat):
+        return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive)
+
+    def selector(self, parts):
+        """Returns a function that selects from a given path, walking and
+        filtering according to the glob-style pattern parts in *parts*.
+        """
+        if not parts:
+            return self.select_exists
+        part = parts.pop()
+        if self.recursive and part == '**':
+            selector = self.recursive_selector
+        elif part in _special_parts:
+            selector = self.special_selector
+        elif not self.case_pedantic and magic_check.search(part) is None:
+            selector = self.literal_selector
+        else:
+            selector = self.wildcard_selector
+        return selector(part, parts)
+
+    def special_selector(self, part, parts):
+        """Returns a function that selects special children of the given path.
+        """
+        select_next = self.selector(parts)
+
+        def select_special(path, exists=False):
+            path = self.concat_path(self.add_slash(path), part)
+            return select_next(path, exists)
+        return select_special
+
+    def literal_selector(self, part, parts):
+        """Returns a function that selects a literal descendant of a path.
+        """
+
+        # Optimization: consume and join any subsequent literal parts here,
+        # rather than leaving them for the next selector. This reduces the
+        # number of string concatenation operations and calls to add_slash().
+        while parts and magic_check.search(parts[-1]) is None:
+            part += self.sep + parts.pop()
+
+        select_next = self.selector(parts)
+
+        def select_literal(path, exists=False):
+            path = self.concat_path(self.add_slash(path), part)
+            return select_next(path, exists=False)
+        return select_literal
+
+    def wildcard_selector(self, part, parts):
+        """Returns a function that selects direct children of a given path,
+        filtering by pattern.
+        """
+
+        match = None if part == '*' else self.compile(part)
+        dir_only = bool(parts)
+        if dir_only:
+            select_next = self.selector(parts)
+
+        def select_wildcard(path, exists=False):
+            try:
+                # We must close the scandir() object before proceeding to
+                # avoid exhausting file descriptors when globbing deep trees.
+                with self.scandir(path) as scandir_it:
+                    entries = list(scandir_it)
+            except OSError:
+                pass
+            else:
+                for entry in entries:
+                    if match is None or match(entry.name):
+                        if dir_only:
+                            try:
+                                if not entry.is_dir():
+                                    continue
+                            except OSError:
+                                continue
+                        entry_path = self.parse_entry(entry)
+                        if dir_only:
+                            yield from select_next(entry_path, exists=True)
+                        else:
+                            yield entry_path
+        return select_wildcard
+
+    def recursive_selector(self, part, parts):
+        """Returns a function that selects a given path and all its children,
+        recursively, filtering by pattern.
+        """
+        # Optimization: consume following '**' parts, which have no effect.
+        while parts and parts[-1] == '**':
+            parts.pop()
+
+        # Optimization: consume and join any following non-special parts here,
+        # rather than leaving them for the next selector. They're used to
+        # build a regular expression, which we use to filter the results of
+        # the recursive walk. As a result, non-special pattern segments
+        # following a '**' wildcard don't require additional filesystem access
+        # to expand.
+        follow_symlinks = self.recursive is not _no_recurse_symlinks
+        if follow_symlinks:
+            while parts and parts[-1] not in _special_parts:
+                part += self.sep + parts.pop()
+
+        match = None if part == '**' else self.compile(part)
+        dir_only = bool(parts)
+        select_next = self.selector(parts)
+
+        def select_recursive(path, exists=False):
+            path = self.add_slash(path)
+            match_pos = len(str(path))
+            if match is None or match(str(path), match_pos):
+                yield from select_next(path, exists)
+            stack = [path]
+            while stack:
+                yield from select_recursive_step(stack, match_pos)
+
+        def select_recursive_step(stack, match_pos):
+            path = stack.pop()
+            try:
+                # We must close the scandir() object before proceeding to
+                # avoid exhausting file descriptors when globbing deep trees.
+                with self.scandir(path) as scandir_it:
+                    entries = list(scandir_it)
+            except OSError:
+                pass
+            else:
+                for entry in entries:
+                    is_dir = False
+                    try:
+                        if entry.is_dir(follow_symlinks=follow_symlinks):
+                            is_dir = True
+                    except OSError:
+                        pass
+
+                    if is_dir or not dir_only:
+                        entry_path = self.parse_entry(entry)
+                        if match is None or match(str(entry_path), match_pos):
+                            if dir_only:
+                                yield from select_next(entry_path, exists=True)
+                            else:
+                                # Optimization: directly yield the path if this is
+                                # last pattern part.
+                                yield entry_path
+                        if is_dir:
+                            stack.append(entry_path)
+
+        return select_recursive
+
+    def select_exists(self, path, exists=False):
+        """Yields the given path, if it exists.
+        """
+        if exists:
+            # Optimization: this path is already known to exist, e.g. because
+            # it was returned from os.scandir(), so we skip calling lstat().
+            yield path
+        else:
+            try:
+                self.lstat(path)
+                yield path
+            except OSError:
+                pass
+
+
+class _StringGlobber(_Globber):
+    lstat = staticmethod(os.lstat)
+    scandir = staticmethod(os.scandir)
+    parse_entry = operator.attrgetter('path')
+    concat_path = operator.add
+
+    if os.name == 'nt':
+        @staticmethod
+        def add_slash(pathname):
+            tail = os.path.splitroot(pathname)[2]
+            if not tail or tail[-1] in '\\/':
+                return pathname
+            return f'{pathname}\\'
+    else:
+        @staticmethod
+        def add_slash(pathname):
+            if not pathname or pathname[-1] == '/':
+                return pathname
+            return f'{pathname}/'

Lib/html/parser.py

@@ -27,18 +27,48 @@ charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
 attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?')
 
 starttagopen = re.compile('<[a-zA-Z]')
+endtagopen = re.compile('</[a-zA-Z]')
 piclose = re.compile('>')
-commentclose = re.compile(r'--\s*>')
+commentclose = re.compile(r'--!?>')
+commentabruptclose = re.compile(r'-?>')
 # Note:
-#  1) if you change tagfind/attrfind remember to update locatestarttagend too;
-#  2) if you change tagfind/attrfind and/or locatestarttagend the parser will
+#  1) if you change tagfind/attrfind remember to update locatetagend too;
+#  2) if you change tagfind/attrfind and/or locatetagend the parser will
 #     explode, so don't do it.
-# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state
-# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state
-tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*')
-attrfind_tolerant = re.compile(
-    r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*'
-    r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*')
+# see the HTML5 specs section "13.2.5.6 Tag open state",
+# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state".
+# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
+# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
+# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
+tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*')
+attrfind_tolerant = re.compile(r"""
+  (
+    (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]*  # attribute name
+   )
+  ([\t\n\r\f ]*=[\t\n\r\f ]*        # value indicator
+    ('[^']*'                        # LITA-enclosed value
+    |"[^"]*"                        # LIT-enclosed value
+    |(?!['"])[^>\t\n\r\f ]*         # bare value
+    )
+   )?
+  (?:[\t\n\r\f ]|/(?!>))*           # possibly followed by a space
+""", re.VERBOSE)
+locatetagend = re.compile(r"""
+  [a-zA-Z][^\t\n\r\f />]*           # tag name
+  [\t\n\r\f /]*                     # optional whitespace before attribute name
+  (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]*  # attribute name
+    (?:[\t\n\r\f ]*=[\t\n\r\f ]*    # value indicator
+      (?:'[^']*'                    # LITA-enclosed value
+        |"[^"]*"                    # LIT-enclosed value
+        |(?!['"])[^>\t\n\r\f ]*     # bare value
+       )
+     )?
+    [\t\n\r\f /]*                   # possibly followed by a space
+   )*
+   >?
+""", re.VERBOSE)
+# The following variables are not used, but are temporarily left for
+# backward compatibility.
 locatestarttagend_tolerant = re.compile(r"""
   <[a-zA-Z][^\t\n\r\f />\x00]*       # tag name
   (?:[\s/]*                          # optional whitespace before attribute name
@@ -55,8 +85,6 @@ locatestarttagend_tolerant = re.compile(r"""
   \s*                                # trailing whitespace
 """, re.VERBOSE)
 endendtag = re.compile('>')
-# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between
-# </ and the tag name, so maybe this should be fixed
 endtagfind = re.compile(r'</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
 
 # Character reference processing logic specific to attribute values
@@ -100,6 +128,7 @@ class HTMLParser(_markupbase.ParserBase):
     """
 
     CDATA_CONTENT_ELEMENTS = ("script", "style")
+    RCDATA_CONTENT_ELEMENTS = ("textarea", "title")
 
     def __init__(self, *, convert_charrefs=True):
         """Initialize and reset this instance.
@@ -117,6 +146,7 @@ class HTMLParser(_markupbase.ParserBase):
         self.lasttag = '???'
         self.interesting = interesting_normal
         self.cdata_elem = None
+        self._escapable = True
         super().reset()
 
     def feed(self, data):
@@ -138,13 +168,20 @@ class HTMLParser(_markupbase.ParserBase):
         """Return full source of start tag: '<...>'."""
         return self.__starttag_text
 
-    def set_cdata_mode(self, elem):
+    def set_cdata_mode(self, elem, *, escapable=False):
         self.cdata_elem = elem.lower()
-        self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
+        self._escapable = escapable
+        if escapable and not self.convert_charrefs:
+            self.interesting = re.compile(r'&|</%s(?=[\t\n\r\f />])' % self.cdata_elem,
+                                          re.IGNORECASE|re.ASCII)
+        else:
+            self.interesting = re.compile(r'</%s(?=[\t\n\r\f />])' % self.cdata_elem,
+                                          re.IGNORECASE|re.ASCII)
 
     def clear_cdata_mode(self):
         self.interesting = interesting_normal
         self.cdata_elem = None
+        self._escapable = True
 
     # Internal -- handle data as far as reasonable.  May leave state
     # and data to be processed by a subsequent call.  If 'end' is
@@ -165,7 +202,7 @@ class HTMLParser(_markupbase.ParserBase):
                     # & near the end and see if it's followed by a space or ;.
                     amppos = rawdata.rfind('&', max(i, n-34))
                     if (amppos >= 0 and
-                        not re.compile(r'[\s;]').search(rawdata, amppos)):
+                        not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)):
                         break  # wait till we get all the text
                     j = n
             else:
@@ -177,7 +214,7 @@ class HTMLParser(_markupbase.ParserBase):
                         break
                     j = n
             if i < j:
-                if self.convert_charrefs and not self.cdata_elem:
+                if self.convert_charrefs and self._escapable:
                     self.handle_data(unescape(rawdata[i:j]))
                 else:
                     self.handle_data(rawdata[i:j])
@@ -195,7 +232,7 @@ class HTMLParser(_markupbase.ParserBase):
                     k = self.parse_pi(i)
                 elif startswith("<!", i):
                     k = self.parse_html_declaration(i)
-                elif (i + 1) < n:
+                elif (i + 1) < n or end:
                     self.handle_data("<")
                     k = i + 1
                 else:
@@ -203,17 +240,35 @@ class HTMLParser(_markupbase.ParserBase):
                 if k < 0:
                     if not end:
                         break
-                    k = rawdata.find('>', i + 1)
-                    if k < 0:
-                        k = rawdata.find('<', i + 1)
-                        if k < 0:
-                            k = i + 1
+                    if starttagopen.match(rawdata, i):  # < + letter
+                        pass
+                    elif startswith("</", i):
+                        if i + 2 == n:
+                            self.handle_data("</")
+                        elif endtagopen.match(rawdata, i):  # </ + letter
+                            pass
+                        else:
+                            # bogus comment
+                            self.handle_comment(rawdata[i+2:])
+                    elif startswith("<!--", i):
+                        j = n
+                        for suffix in ("--!", "--", "-"):
+                            if rawdata.endswith(suffix, i+4):
+                                j -= len(suffix)
+                                break
+                        self.handle_comment(rawdata[i+4:j])
+                    elif startswith("<![CDATA[", i):
+                        self.unknown_decl(rawdata[i+3:])
+                    elif rawdata[i:i+9].lower() == '<!doctype':
+                        self.handle_decl(rawdata[i+2:])
+                    elif startswith("<!", i):
+                        # bogus comment
+                        self.handle_comment(rawdata[i+2:])
+                    elif startswith("<?", i):
+                        self.handle_pi(rawdata[i+2:])
                     else:
-                        k += 1
-                    if self.convert_charrefs and not self.cdata_elem:
-                        self.handle_data(unescape(rawdata[i:k]))
-                    else:
-                        self.handle_data(rawdata[i:k])
+                        raise AssertionError("we should not get here!")
+                    k = n
                 i = self.updatepos(i, k)
             elif startswith("&#", i):
                 match = charref.match(rawdata, i)
@@ -261,7 +316,7 @@ class HTMLParser(_markupbase.ParserBase):
                 assert 0, "interesting.search() lied"
         # end while
         if end and i < n:
-            if self.convert_charrefs and not self.cdata_elem:
+            if self.convert_charrefs and self._escapable:
                 self.handle_data(unescape(rawdata[i:n]))
             else:
                 self.handle_data(rawdata[i:n])
@@ -290,8 +345,23 @@ class HTMLParser(_markupbase.ParserBase):
         else:
             return self.parse_bogus_comment(i)
 
+    # Internal -- parse comment, return length or -1 if not terminated
+    # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state
+    def parse_comment(self, i, report=True):
+        rawdata = self.rawdata
+        assert rawdata.startswith('<!--', i), 'unexpected call to parse_comment()'
+        match = commentclose.search(rawdata, i+4)
+        if not match:
+            match = commentabruptclose.match(rawdata, i+4)
+            if not match:
+                return -1
+        if report:
+            j = match.start()
+            self.handle_comment(rawdata[i+4: j])
+        return match.end()
+
     # Internal -- parse bogus comment, return length or -1 if not terminated
-    # see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state
+    # see https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state
     def parse_bogus_comment(self, i, report=1):
         rawdata = self.rawdata
         assert rawdata[i:i+2] in ('<!', '</'), ('unexpected call to '
@@ -317,6 +387,8 @@ class HTMLParser(_markupbase.ParserBase):
 
     # Internal -- handle starttag, return end or -1 if not terminated
     def parse_starttag(self, i):
+        # See the HTML5 specs section "13.2.5.8 Tag name state"
+        # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
         self.__starttag_text = None
         endpos = self.check_for_whole_start_tag(i)
         if endpos < 0:
@@ -356,82 +428,50 @@ class HTMLParser(_markupbase.ParserBase):
             self.handle_starttag(tag, attrs)
             if tag in self.CDATA_CONTENT_ELEMENTS:
                 self.set_cdata_mode(tag)
+            elif tag in self.RCDATA_CONTENT_ELEMENTS:
+                self.set_cdata_mode(tag, escapable=True)
         return endpos
 
     # Internal -- check to see if we have a complete starttag; return end
     # or -1 if incomplete.
     def check_for_whole_start_tag(self, i):
         rawdata = self.rawdata
-        m = locatestarttagend_tolerant.match(rawdata, i)
-        if m:
-            j = m.end()
-            next = rawdata[j:j+1]
-            if next == ">":
-                return j + 1
-            if next == "/":
-                if rawdata.startswith("/>", j):
-                    return j + 2
-                if rawdata.startswith("/", j):
-                    # buffer boundary
-                    return -1
-                # else bogus input
-                if j > i:
-                    return j
-                else:
-                    return i + 1
-            if next == "":
-                # end of input
-                return -1
-            if next in ("abcdefghijklmnopqrstuvwxyz=/"
-                        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
-                # end of input in or before attribute value, or we have the
-                # '/' from a '/>' ending
-                return -1
-            if j > i:
-                return j
-            else:
-                return i + 1
-        raise AssertionError("we should not get here!")
+        match = locatetagend.match(rawdata, i+1)
+        assert match
+        j = match.end()
+        if rawdata[j-1] != ">":
+            return -1
+        return j
 
     # Internal -- parse endtag, return end or -1 if incomplete
     def parse_endtag(self, i):
+        # See the HTML5 specs section "13.2.5.7 End tag open state"
+        # https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
         rawdata = self.rawdata
         assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag"
-        match = endendtag.search(rawdata, i+1) # >
-        if not match:
+        if rawdata.find('>', i+2) < 0:  # fast check
             return -1
-        gtpos = match.end()
-        match = endtagfind.match(rawdata, i) # </ + tag + >
-        if not match:
-            if self.cdata_elem is not None:
-                self.handle_data(rawdata[i:gtpos])
-                return gtpos
-            # find the name: w3.org/TR/html5/tokenization.html#tag-name-state
-            namematch = tagfind_tolerant.match(rawdata, i+2)
-            if not namematch:
-                # w3.org/TR/html5/tokenization.html#end-tag-open-state
-                if rawdata[i:i+3] == '</>':
-                    return i+3
-                else:
-                    return self.parse_bogus_comment(i)
-            tagname = namematch.group(1).lower()
-            # consume and ignore other stuff between the name and the >
-            # Note: this is not 100% correct, since we might have things like
-            # </tag attr=">">, but looking for > after the name should cover
-            # most of the cases and is much simpler
-            gtpos = rawdata.find('>', namematch.end())
-            self.handle_endtag(tagname)
-            return gtpos+1
+        if not endtagopen.match(rawdata, i):  # </ + letter
+            if rawdata[i+2:i+3] == '>':  # </> is ignored
+                # "missing-end-tag-name" parser error
+                return i+3
+            else:
+                return self.parse_bogus_comment(i)
 
-        elem = match.group(1).lower() # script or style
-        if self.cdata_elem is not None:
-            if elem != self.cdata_elem:
-                self.handle_data(rawdata[i:gtpos])
-                return gtpos
+        match = locatetagend.match(rawdata, i+2)
+        assert match
+        j = match.end()
+        if rawdata[j-1] != ">":
+            return -1
 
-        self.handle_endtag(elem)
+        # find the name: "13.2.5.8 Tag name state"
+        # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
+        match = tagfind_tolerant.match(rawdata, i+2)
+        assert match
+        tag = match.group(1).lower()
+        self.handle_endtag(tag)
         self.clear_cdata_mode()
-        return gtpos
+        return j
 
     # Overridable -- finish processing of start+end tag: <tag.../>
     def handle_startendtag(self, tag, attrs):

Lib/importlib/metadata/__init__.py

@@ -56,12 +56,6 @@ class PackageNotFoundError(ModuleNotFoundError):
         (name,) = self.args
         return name
 
-    # TODO: RUSTPYTHON; the entire setter is added to avoid errors
-    @name.setter
-    def name(self, value):
-        import sys
-        sys.stderr.write("set value to PackageNotFoundError ignored\n")
-
 
 class Sectioned:
     """

Lib/io.py

@@ -46,23 +46,17 @@ __all__ = ["BlockingIOError", "open", "open_code", "IOBase", "RawIOBase",
            "BufferedReader", "BufferedWriter", "BufferedRWPair",
            "BufferedRandom", "TextIOBase", "TextIOWrapper",
            "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END",
-           "DEFAULT_BUFFER_SIZE", "text_encoding",
-           "IncrementalNewlineDecoder"
-           ]
+           "DEFAULT_BUFFER_SIZE", "text_encoding", "IncrementalNewlineDecoder"]
 
 
 import _io
 import abc
 
 from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
-                 open, open_code, BytesIO, StringIO, BufferedReader,
+                 open, open_code, FileIO, BytesIO, StringIO, BufferedReader,
                  BufferedWriter, BufferedRWPair, BufferedRandom,
                  IncrementalNewlineDecoder, text_encoding, TextIOWrapper)
 
-try:
-    from _io import FileIO
-except ImportError:
-    pass
 
 # Pretend this exception was created here.
 UnsupportedOperation.__module__ = "io"
@@ -87,10 +81,7 @@ class BufferedIOBase(_io._BufferedIOBase, IOBase):
 class TextIOBase(_io._TextIOBase, IOBase):
     __doc__ = _io._TextIOBase.__doc__
 
-try:
-    RawIOBase.register(FileIO)
-except NameError:
-    pass
+RawIOBase.register(FileIO)
 
 for klass in (BytesIO, BufferedReader, BufferedWriter, BufferedRandom,
               BufferedRWPair):

Lib/locale.py

@@ -25,8 +25,8 @@ import functools
 # Yuck:  LC_MESSAGES is non-standard:  can't tell whether it exists before
 # trying the import.  So __all__ is also fiddled at the end of the file.
 __all__ = ["getlocale", "getdefaultlocale", "getpreferredencoding", "Error",
-           "setlocale", "resetlocale", "localeconv", "strcoll", "strxfrm",
-           "str", "atof", "atoi", "format", "format_string", "currency",
+           "setlocale", "localeconv", "strcoll", "strxfrm",
+           "str", "atof", "atoi", "format_string", "currency",
            "normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY",
            "LC_NUMERIC", "LC_ALL", "CHAR_MAX", "getencoding"]
 
@@ -247,21 +247,6 @@ def format_string(f, val, grouping=False, monetary=False):
 
     return new_f % val
 
-def format(percent, value, grouping=False, monetary=False, *additional):
-    """Deprecated, use format_string instead."""
-    import warnings
-    warnings.warn(
-        "This method will be removed in a future version of Python. "
-        "Use 'locale.format_string()' instead.",
-        DeprecationWarning, stacklevel=2
-    )
-
-    match = _percent_re.match(percent)
-    if not match or len(match.group())!= len(percent):
-        raise ValueError(("format() must be given exactly one %%char "
-                         "format specifier, %s not valid") % repr(percent))
-    return _format(percent, value, grouping, monetary, *additional)
-
 def currency(val, symbol=True, grouping=False, international=False):
     """Formats val according to the currency settings
     in the current locale."""
@@ -556,11 +541,15 @@ def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')):
     """
 
     import warnings
-    warnings.warn(
-        "Use setlocale(), getencoding() and getlocale() instead",
-        DeprecationWarning, stacklevel=2
-    )
+    warnings._deprecated(
+        "locale.getdefaultlocale",
+        "{name!r} is deprecated and slated for removal in Python {remove}. "
+        "Use setlocale(), getencoding() and getlocale() instead.",
+        remove=(3, 15))
+    return _getdefaultlocale(envvars)
 
+
+def _getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')):
     try:
         # check if it's supported by the _locale module
         import _locale
@@ -625,40 +614,15 @@ def setlocale(category, locale=None):
         locale = normalize(_build_localename(locale))
     return _setlocale(category, locale)
 
-def resetlocale(category=LC_ALL):
-
-    """ Sets the locale for category to the default setting.
-
-        The default setting is determined by calling
-        getdefaultlocale(). category defaults to LC_ALL.
-
-    """
-    import warnings
-    warnings.warn(
-        'Use locale.setlocale(locale.LC_ALL, "") instead',
-        DeprecationWarning, stacklevel=2
-    )
-
-    with warnings.catch_warnings():
-        warnings.simplefilter('ignore', category=DeprecationWarning)
-        loc = getdefaultlocale()
-
-    _setlocale(category, _build_localename(loc))
-
 
 try:
     from _locale import getencoding
 except ImportError:
+    # When _locale.getencoding() is missing, locale.getencoding() uses the
+    # Python filesystem encoding.
     def getencoding():
-        if hasattr(sys, 'getandroidapilevel'):
-            # On Android langinfo.h and CODESET are missing, and UTF-8 is
-            # always used in mbstowcs() and wcstombs().
-            return 'utf-8'
-        encoding = getdefaultlocale()[1]
-        if encoding is None:
-            # LANG not set, default to UTF-8
-            encoding = 'utf-8'
-        return encoding
+        return sys.getfilesystemencoding()
+
 
 try:
     CODESET
@@ -896,6 +860,28 @@ del k, v
 #    updated 'ca_es@valencia' -> 'ca_ES.ISO8859-15@valencia' to 'ca_ES.UTF-8@valencia'
 #    updated 'kk_kz' -> 'kk_KZ.RK1048' to 'kk_KZ.ptcp154'
 #    updated 'russian' -> 'ru_RU.ISO8859-5' to 'ru_RU.KOI8-R'
+#
+# SS 2025-02-04:
+# Updated alias mapping with glibc 2.41 supported locales and the latest
+# X lib alias mapping.
+#
+# These are the differences compared to the old mapping (Python 3.13.1
+# and older):
+#
+#    updated 'c.utf8' -> 'C.UTF-8' to 'en_US.UTF-8'
+#    updated 'de_it' -> 'de_IT.ISO8859-1' to 'de_IT.UTF-8'
+#    removed 'de_li.utf8'
+#    updated 'en_il' -> 'en_IL.UTF-8' to 'en_IL.ISO8859-1'
+#    removed 'english.iso88591'
+#    updated 'es_cu' -> 'es_CU.UTF-8' to 'es_CU.ISO8859-1'
+#    updated 'russian' -> 'ru_RU.KOI8-R' to 'ru_RU.ISO8859-5'
+#    updated 'sr@latn' -> 'sr_CS.UTF-8@latin' to 'sr_RS.UTF-8@latin'
+#    removed 'univ'
+#    removed 'universal'
+#
+# SS 2025-06-10:
+# Remove 'c.utf8' -> 'en_US.UTF-8' because 'en_US.UTF-8' does not exist
+# on all platforms.
 
 locale_alias = {
     'a3':                                   'az_AZ.KOI8-C',
@@ -975,7 +961,6 @@ locale_alias = {
     'c.ascii':                              'C',
     'c.en':                                 'C',
     'c.iso88591':                           'en_US.ISO8859-1',
-    'c.utf8':                               'en_US.UTF-8',
     'c_c':                                  'C',
     'c_c.c':                                'C',
     'ca':                                   'ca_ES.ISO8859-1',
@@ -992,6 +977,7 @@ locale_alias = {
     'chr_us':                               'chr_US.UTF-8',
     'ckb_iq':                               'ckb_IQ.UTF-8',
     'cmn_tw':                               'cmn_TW.UTF-8',
+    'crh_ru':                               'crh_RU.UTF-8',
     'crh_ua':                               'crh_UA.UTF-8',
     'croatian':                             'hr_HR.ISO8859-2',
     'cs':                                   'cs_CZ.ISO8859-2',
@@ -1013,11 +999,12 @@ locale_alias = {
     'de_be':                                'de_BE.ISO8859-1',
     'de_ch':                                'de_CH.ISO8859-1',
     'de_de':                                'de_DE.ISO8859-1',
-    'de_it':                                'de_IT.ISO8859-1',
-    'de_li.utf8':                           'de_LI.UTF-8',
+    'de_it':                                'de_IT.UTF-8',
+    'de_li':                                'de_LI.ISO8859-1',
     'de_lu':                                'de_LU.ISO8859-1',
     'deutsch':                              'de_DE.ISO8859-1',
     'doi_in':                               'doi_IN.UTF-8',
+    'dsb_de':                               'dsb_DE.UTF-8',
     'dutch':                                'nl_NL.ISO8859-1',
     'dutch.iso88591':                       'nl_BE.ISO8859-1',
     'dv_mv':                                'dv_MV.UTF-8',
@@ -1040,7 +1027,7 @@ locale_alias = {
     'en_gb':                                'en_GB.ISO8859-1',
     'en_hk':                                'en_HK.ISO8859-1',
     'en_ie':                                'en_IE.ISO8859-1',
-    'en_il':                                'en_IL.UTF-8',
+    'en_il':                                'en_IL.ISO8859-1',
     'en_in':                                'en_IN.ISO8859-1',
     'en_ng':                                'en_NG.UTF-8',
     'en_nz':                                'en_NZ.ISO8859-1',
@@ -1056,7 +1043,6 @@ locale_alias = {
     'en_zw.utf8':                           'en_ZS.UTF-8',
     'eng_gb':                               'en_GB.ISO8859-1',
     'english':                              'en_EN.ISO8859-1',
-    'english.iso88591':                     'en_US.ISO8859-1',
     'english_uk':                           'en_GB.ISO8859-1',
     'english_united-states':                'en_US.ISO8859-1',
     'english_united-states.437':            'C',
@@ -1072,7 +1058,7 @@ locale_alias = {
     'es_cl':                                'es_CL.ISO8859-1',
     'es_co':                                'es_CO.ISO8859-1',
     'es_cr':                                'es_CR.ISO8859-1',
-    'es_cu':                                'es_CU.UTF-8',
+    'es_cu':                                'es_CU.ISO8859-1',
     'es_do':                                'es_DO.ISO8859-1',
     'es_ec':                                'es_EC.ISO8859-1',
     'es_es':                                'es_ES.ISO8859-1',
@@ -1122,6 +1108,7 @@ locale_alias = {
     'ga_ie':                                'ga_IE.ISO8859-1',
     'galego':                               'gl_ES.ISO8859-1',
     'galician':                             'gl_ES.ISO8859-1',
+    'gbm_in':                               'gbm_IN.UTF-8',
     'gd':                                   'gd_GB.ISO8859-1',
     'gd_gb':                                'gd_GB.ISO8859-1',
     'ger_de':                               'de_DE.ISO8859-1',
@@ -1162,6 +1149,7 @@ locale_alias = {
     'icelandic':                            'is_IS.ISO8859-1',
     'id':                                   'id_ID.ISO8859-1',
     'id_id':                                'id_ID.ISO8859-1',
+    'ie':                                   'ie.UTF-8',
     'ig_ng':                                'ig_NG.UTF-8',
     'ik_ca':                                'ik_CA.UTF-8',
     'in':                                   'id_ID.ISO8859-1',
@@ -1216,6 +1204,7 @@ locale_alias = {
     'ks_in':                                'ks_IN.UTF-8',
     'ks_in@devanagari.utf8':                'ks_IN.UTF-8@devanagari',
     'ku_tr':                                'ku_TR.ISO8859-9',
+    'kv_ru':                                'kv_RU.UTF-8',
     'kw':                                   'kw_GB.ISO8859-1',
     'kw_gb':                                'kw_GB.ISO8859-1',
     'ky':                                   'ky_KG.UTF-8',
@@ -1234,6 +1223,7 @@ locale_alias = {
     'lo_la.mulelao1':                       'lo_LA.MULELAO-1',
     'lt':                                   'lt_LT.ISO8859-13',
     'lt_lt':                                'lt_LT.ISO8859-13',
+    'ltg_lv.utf8':                          'ltg_LV.UTF-8',
     'lv':                                   'lv_LV.ISO8859-13',
     'lv_lv':                                'lv_LV.ISO8859-13',
     'lzh_tw':                               'lzh_TW.UTF-8',
@@ -1241,6 +1231,7 @@ locale_alias = {
     'mai':                                  'mai_IN.UTF-8',
     'mai_in':                               'mai_IN.UTF-8',
     'mai_np':                               'mai_NP.UTF-8',
+    'mdf_ru':                               'mdf_RU.UTF-8',
     'mfe_mu':                               'mfe_MU.UTF-8',
     'mg_mg':                                'mg_MG.ISO8859-15',
     'mhr_ru':                               'mhr_RU.UTF-8',
@@ -1254,6 +1245,7 @@ locale_alias = {
     'ml_in':                                'ml_IN.UTF-8',
     'mn_mn':                                'mn_MN.UTF-8',
     'mni_in':                               'mni_IN.UTF-8',
+    'mnw_mm':                               'mnw_MM.UTF-8',
     'mr':                                   'mr_IN.UTF-8',
     'mr_in':                                'mr_IN.UTF-8',
     'ms':                                   'ms_MY.ISO8859-1',
@@ -1322,6 +1314,7 @@ locale_alias = {
     'pt_pt':                                'pt_PT.ISO8859-1',
     'quz_pe':                               'quz_PE.UTF-8',
     'raj_in':                               'raj_IN.UTF-8',
+    'rif_ma':                               'rif_MA.UTF-8',
     'ro':                                   'ro_RO.ISO8859-2',
     'ro_ro':                                'ro_RO.ISO8859-2',
     'romanian':                             'ro_RO.ISO8859-2',
@@ -1329,12 +1322,14 @@ locale_alias = {
     'ru_ru':                                'ru_RU.UTF-8',
     'ru_ua':                                'ru_UA.KOI8-U',
     'rumanian':                             'ro_RO.ISO8859-2',
-    'russian':                              'ru_RU.KOI8-R',
+    'russian':                              'ru_RU.ISO8859-5',
     'rw':                                   'rw_RW.ISO8859-1',
     'rw_rw':                                'rw_RW.ISO8859-1',
     'sa_in':                                'sa_IN.UTF-8',
+    'sah_ru':                               'sah_RU.UTF-8',
     'sat_in':                               'sat_IN.UTF-8',
     'sc_it':                                'sc_IT.UTF-8',
+    'scn_it':                               'scn_IT.UTF-8',
     'sd':                                   'sd_IN.UTF-8',
     'sd_in':                                'sd_IN.UTF-8',
     'sd_in@devanagari.utf8':                'sd_IN.UTF-8@devanagari',
@@ -1376,7 +1371,7 @@ locale_alias = {
     'sq_mk':                                'sq_MK.UTF-8',
     'sr':                                   'sr_RS.UTF-8',
     'sr@cyrillic':                          'sr_RS.UTF-8',
-    'sr@latn':                              'sr_CS.UTF-8@latin',
+    'sr@latn':                              'sr_RS.UTF-8@latin',
     'sr_cs':                                'sr_CS.UTF-8',
     'sr_cs.iso88592@latn':                  'sr_CS.ISO8859-2',
     'sr_cs@latn':                           'sr_CS.UTF-8@latin',
@@ -1395,14 +1390,17 @@ locale_alias = {
     'sr_yu@cyrillic':                       'sr_RS.UTF-8',
     'ss':                                   'ss_ZA.ISO8859-1',
     'ss_za':                                'ss_ZA.ISO8859-1',
+    'ssy_er':                               'ssy_ER.UTF-8',
     'st':                                   'st_ZA.ISO8859-1',
     'st_za':                                'st_ZA.ISO8859-1',
+    'su_id':                                'su_ID.UTF-8',
     'sv':                                   'sv_SE.ISO8859-1',
     'sv_fi':                                'sv_FI.ISO8859-1',
     'sv_se':                                'sv_SE.ISO8859-1',
     'sw_ke':                                'sw_KE.UTF-8',
     'sw_tz':                                'sw_TZ.UTF-8',
     'swedish':                              'sv_SE.ISO8859-1',
+    'syr':                                  'syr.UTF-8',
     'szl_pl':                               'szl_PL.UTF-8',
     'ta':                                   'ta_IN.TSCII-0',
     'ta_in':                                'ta_IN.TSCII-0',
@@ -1429,6 +1427,7 @@ locale_alias = {
     'tn':                                   'tn_ZA.ISO8859-15',
     'tn_za':                                'tn_ZA.ISO8859-15',
     'to_to':                                'to_TO.UTF-8',
+    'tok':                                  'tok.UTF-8',
     'tpi_pg':                               'tpi_PG.UTF-8',
     'tr':                                   'tr_TR.ISO8859-9',
     'tr_cy':                                'tr_CY.ISO8859-9',
@@ -1443,8 +1442,7 @@ locale_alias = {
     'ug_cn':                                'ug_CN.UTF-8',
     'uk':                                   'uk_UA.KOI8-U',
     'uk_ua':                                'uk_UA.KOI8-U',
-    'univ':                                 'en_US.utf',
-    'universal':                            'en_US.utf',
+    'univ.utf8':                            'en_US.UTF-8',
     'universal.utf8@ucs4':                  'en_US.UTF-8',
     'unm_us':                               'unm_US.UTF-8',
     'ur':                                   'ur_PK.CP1256',
@@ -1473,6 +1471,7 @@ locale_alias = {
     'yo_ng':                                'yo_NG.UTF-8',
     'yue_hk':                               'yue_HK.UTF-8',
     'yuw_pg':                               'yuw_PG.UTF-8',
+    'zgh_ma':                               'zgh_MA.UTF-8',
     'zh':                                   'zh_CN.eucCN',
     'zh_cn':                                'zh_CN.gb2312',
     'zh_cn.big5':                           'zh_TW.big5',
@@ -1496,7 +1495,8 @@ locale_alias = {
 # to include every locale up to Windows Vista.
 #
 # NOTE: this mapping is incomplete.  If your language is missing, please
-# submit a bug report to the Python bug tracker at http://bugs.python.org/
+# submit a bug report as detailed in the Python devguide at:
+#    https://devguide.python.org/triage/issue-tracker/
 # Make sure you include the missing language identifier and the suggested
 # locale code.
 #
@@ -1742,17 +1742,6 @@ def _print_locale():
         print('   Encoding: ', enc or '(undefined)')
         print()
 
-    print()
-    print('Locale settings after calling resetlocale():')
-    print('-'*72)
-    resetlocale()
-    for name,category in categories.items():
-        print(name, '...')
-        lang, enc = getlocale(category)
-        print('   Language: ', lang or '(undefined)')
-        print('   Encoding: ', enc or '(undefined)')
-        print()
-
     try:
         setlocale(LC_ALL, "")
     except:

Lib/lzma.py

@@ -128,7 +128,7 @@ class LZMAFile(_compression.BaseStream):
 
         if self._mode == _MODE_READ:
             raw = _compression.DecompressReader(self._fp, LZMADecompressor,
-                                                trailing_error=LZMAError, format=format, filters=filters)
+                trailing_error=LZMAError, format=format, filters=filters)
             self._buffer = io.BufferedReader(raw)
 
     def close(self):

Lib/netrc.py

@@ -2,11 +2,24 @@
 
 # Module and documentation by Eric S. Raymond, 21 Dec 1998
 
-import os, shlex, stat
+import os, stat
 
 __all__ = ["netrc", "NetrcParseError"]
 
 
+def _can_security_check():
+    # On WASI, getuid() is indicated as a stub but it may also be missing.
+    return os.name == 'posix' and hasattr(os, 'getuid')
+
+
+def _getpwuid(uid):
+    try:
+        import pwd
+        return pwd.getpwuid(uid)[0]
+    except (ImportError, LookupError):
+        return f'uid {uid}'
+
+
 class NetrcParseError(Exception):
     """Exception raised on syntax errors in the .netrc file."""
     def __init__(self, msg, filename=None, lineno=None):
@@ -142,18 +155,12 @@ class netrc:
             self._security_check(fp, default_netrc, self.hosts[entryname][0])
 
     def _security_check(self, fp, default_netrc, login):
-        if os.name == 'posix' and default_netrc and login != "anonymous":
+        if _can_security_check() and default_netrc and login != "anonymous":
             prop = os.fstat(fp.fileno())
-            if prop.st_uid != os.getuid():
-                import pwd
-                try:
-                    fowner = pwd.getpwuid(prop.st_uid)[0]
-                except KeyError:
-                    fowner = 'uid %s' % prop.st_uid
-                try:
-                    user = pwd.getpwuid(os.getuid())[0]
-                except KeyError:
-                    user = 'uid %s' % os.getuid()
+            current_user_id = os.getuid()
+            if prop.st_uid != current_user_id:
+                fowner = _getpwuid(prop.st_uid)
+                user = _getpwuid(current_user_id)
                 raise NetrcParseError(
                     (f"~/.netrc file owner ({fowner}, {user}) does not match"
                      " current user"))

Lib/ntpath.py

@@ -19,18 +19,17 @@ devnull = 'nul'
 
 import os
 import sys
-import stat
 import genericpath
 from genericpath import *
 
-
 __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext",
            "basename","dirname","commonprefix","getsize","getmtime",
            "getatime","getctime", "islink","exists","lexists","isdir","isfile",
-           "ismount", "expanduser","expandvars","normpath","abspath",
-           "curdir","pardir","sep","pathsep","defpath","altsep",
+           "ismount","isreserved","expanduser","expandvars","normpath",
+           "abspath","curdir","pardir","sep","pathsep","defpath","altsep",
            "extsep","devnull","realpath","supports_unicode_filenames","relpath",
-           "samefile", "sameopenfile", "samestat", "commonpath", "isjunction"]
+           "samefile", "sameopenfile", "samestat", "commonpath", "isjunction",
+           "isdevdrive", "ALLOW_MISSING"]
 
 def _get_bothseps(path):
     if isinstance(path, bytes):
@@ -78,12 +77,6 @@ except ImportError:
         return s.replace('/', '\\').lower()
 
 
-# Return whether a path is absolute.
-# Trivial in Posix, harder on Windows.
-# For Windows it is absolute if it starts with a slash or backslash (current
-# volume), or if a pathname after the volume-letter-and-colon or UNC-resource
-# starts with a slash or backslash.
-
 def isabs(s):
     """Test whether a path is absolute"""
     s = os.fspath(s)
@@ -91,16 +84,15 @@ def isabs(s):
         sep = b'\\'
         altsep = b'/'
         colon_sep = b':\\'
+        double_sep = b'\\\\'
     else:
         sep = '\\'
         altsep = '/'
         colon_sep = ':\\'
+        double_sep = '\\\\'
     s = s[:3].replace(altsep, sep)
     # Absolute: UNC, device, and paths with a drive and root.
-    # LEGACY BUG: isabs("/x") should be false since the path has no drive.
-    if s.startswith(sep) or s.startswith(colon_sep, 1):
-        return True
-    return False
+    return s.startswith(colon_sep, 1) or s.startswith(double_sep)
 
 
 # Join two (or more) paths.
@@ -109,16 +101,14 @@ def join(path, *paths):
     if isinstance(path, bytes):
         sep = b'\\'
         seps = b'\\/'
-        colon = b':'
+        colon_seps = b':\\/'
     else:
         sep = '\\'
         seps = '\\/'
-        colon = ':'
+        colon_seps = ':\\/'
     try:
-        if not paths:
-            path[:0] + sep  #23780: Ensure compatible data type even if p is null.
         result_drive, result_root, result_path = splitroot(path)
-        for p in map(os.fspath, paths):
+        for p in paths:
             p_drive, p_root, p_path = splitroot(p)
             if p_root:
                 # Second path is absolute
@@ -142,7 +132,7 @@ def join(path, *paths):
             result_path = result_path + p_path
         ## add separator between UNC and non-absolute path
         if (result_path and not result_root and
-            result_drive and result_drive[-1:] not in colon + seps):
+            result_drive and result_drive[-1] not in colon_seps):
             return result_drive + sep + result_path
         return result_drive + result_root + result_path
     except (TypeError, AttributeError, BytesWarning):
@@ -176,56 +166,52 @@ def splitdrive(p):
     return drive, root + tail
 
 
-def splitroot(p):
-    """Split a pathname into drive, root and tail. The drive is defined
-    exactly as in splitdrive(). On Windows, the root may be a single path
-    separator or an empty string. The tail contains anything after the root.
-    For example:
+try:
+    from nt import _path_splitroot_ex as splitroot
+except ImportError:
+    def splitroot(p):
+        """Split a pathname into drive, root and tail.
 
-        splitroot('//server/share/') == ('//server/share', '/', '')
-        splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney')
-        splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham')
-        splitroot('Windows/notepad') == ('', '', 'Windows/notepad')
-    """
-    p = os.fspath(p)
-    if isinstance(p, bytes):
-        sep = b'\\'
-        altsep = b'/'
-        colon = b':'
-        unc_prefix = b'\\\\?\\UNC\\'
-        empty = b''
-    else:
-        sep = '\\'
-        altsep = '/'
-        colon = ':'
-        unc_prefix = '\\\\?\\UNC\\'
-        empty = ''
-    normp = p.replace(altsep, sep)
-    if normp[:1] == sep:
-        if normp[1:2] == sep:
-            # UNC drives, e.g. \\server\share or \\?\UNC\server\share
-            # Device drives, e.g. \\.\device or \\?\device
-            start = 8 if normp[:8].upper() == unc_prefix else 2
-            index = normp.find(sep, start)
-            if index == -1:
-                return p, empty, empty
-            index2 = normp.find(sep, index + 1)
-            if index2 == -1:
-                return p, empty, empty
-            return p[:index2], p[index2:index2 + 1], p[index2 + 1:]
+        The tail contains anything after the root."""
+        p = os.fspath(p)
+        if isinstance(p, bytes):
+            sep = b'\\'
+            altsep = b'/'
+            colon = b':'
+            unc_prefix = b'\\\\?\\UNC\\'
+            empty = b''
         else:
-            # Relative path with root, e.g. \Windows
-            return empty, p[:1], p[1:]
-    elif normp[1:2] == colon:
-        if normp[2:3] == sep:
-            # Absolute drive-letter path, e.g. X:\Windows
-            return p[:2], p[2:3], p[3:]
+            sep = '\\'
+            altsep = '/'
+            colon = ':'
+            unc_prefix = '\\\\?\\UNC\\'
+            empty = ''
+        normp = p.replace(altsep, sep)
+        if normp[:1] == sep:
+            if normp[1:2] == sep:
+                # UNC drives, e.g. \\server\share or \\?\UNC\server\share
+                # Device drives, e.g. \\.\device or \\?\device
+                start = 8 if normp[:8].upper() == unc_prefix else 2
+                index = normp.find(sep, start)
+                if index == -1:
+                    return p, empty, empty
+                index2 = normp.find(sep, index + 1)
+                if index2 == -1:
+                    return p, empty, empty
+                return p[:index2], p[index2:index2 + 1], p[index2 + 1:]
+            else:
+                # Relative path with root, e.g. \Windows
+                return empty, p[:1], p[1:]
+        elif normp[1:2] == colon:
+            if normp[2:3] == sep:
+                # Absolute drive-letter path, e.g. X:\Windows
+                return p[:2], p[2:3], p[3:]
+            else:
+                # Relative path with drive, e.g. X:Windows
+                return p[:2], empty, p[2:]
         else:
-            # Relative path with drive, e.g. X:Windows
-            return p[:2], empty, p[2:]
-    else:
-        # Relative path, e.g. Windows
-        return empty, empty, p
+            # Relative path, e.g. Windows
+            return empty, empty, p
 
 
 # Split a path in head (everything up to the last '/') and tail (the
@@ -277,33 +263,6 @@ def dirname(p):
     return split(p)[0]
 
 
-# Is a path a junction?
-
-if hasattr(os.stat_result, 'st_reparse_tag'):
-    def isjunction(path):
-        """Test whether a path is a junction"""
-        try:
-            st = os.lstat(path)
-        except (OSError, ValueError, AttributeError):
-            return False
-        return bool(st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)
-else:
-    def isjunction(path):
-        """Test whether a path is a junction"""
-        os.fspath(path)
-        return False
-
-
-# Being true for dangling symbolic links is also useful.
-
-def lexists(path):
-    """Test whether a path exists.  Returns True for broken symbolic links"""
-    try:
-        st = os.lstat(path)
-    except (OSError, ValueError):
-        return False
-    return True
-
 # Is a path a mount point?
 # Any drive letter root (eg c:\)
 # Any share UNC (eg \\server\share)
@@ -338,6 +297,40 @@ def ismount(path):
         return False
 
 
+_reserved_chars = frozenset(
+    {chr(i) for i in range(32)} |
+    {'"', '*', ':', '<', '>', '?', '|', '/', '\\'}
+)
+
+_reserved_names = frozenset(
+    {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} |
+    {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} |
+    {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'}
+)
+
+def isreserved(path):
+    """Return true if the pathname is reserved by the system."""
+    # Refer to "Naming Files, Paths, and Namespaces":
+    # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    path = os.fsdecode(splitroot(path)[2]).replace(altsep, sep)
+    return any(_isreservedname(name) for name in reversed(path.split(sep)))
+
+def _isreservedname(name):
+    """Return true if the filename is reserved by the system."""
+    # Trailing dots and spaces are reserved.
+    if name[-1:] in ('.', ' '):
+        return name not in ('.', '..')
+    # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved.
+    # ASCII control characters (0-31) are reserved.
+    # Colon is reserved for file streams (e.g. "name:stream[:type]").
+    if _reserved_chars.intersection(name):
+        return True
+    # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules
+    # are complex and vary across Windows versions. On the side of
+    # caution, return True for names that may not be reserved.
+    return name.partition('.')[0].rstrip(' ').upper() in _reserved_names
+
+
 # Expand paths beginning with '~' or '~user'.
 # '~' means $HOME; '~user' means that user's home directory.
 # If the path doesn't begin with '~', or if the user or $HOME is unknown,
@@ -353,24 +346,23 @@ def expanduser(path):
     If user or $HOME is unknown, do nothing."""
     path = os.fspath(path)
     if isinstance(path, bytes):
+        seps = b'\\/'
         tilde = b'~'
     else:
+        seps = '\\/'
         tilde = '~'
     if not path.startswith(tilde):
         return path
     i, n = 1, len(path)
-    while i < n and path[i] not in _get_bothseps(path):
+    while i < n and path[i] not in seps:
         i += 1
 
     if 'USERPROFILE' in os.environ:
         userhome = os.environ['USERPROFILE']
-    elif not 'HOMEPATH' in os.environ:
+    elif 'HOMEPATH' not in os.environ:
         return path
     else:
-        try:
-            drive = os.environ['HOMEDRIVE']
-        except KeyError:
-            drive = ''
+        drive = os.environ.get('HOMEDRIVE', '')
         userhome = join(drive, os.environ['HOMEPATH'])
 
     if i != 1: #~user
@@ -521,7 +513,7 @@ def expandvars(path):
 # Previously, this function also truncated pathnames to 8+3 format,
 # but as this module is called "ntpath", that's obviously wrong!
 try:
-    from nt import _path_normpath
+    from nt import _path_normpath as normpath
 
 except ImportError:
     def normpath(path):
@@ -560,37 +552,22 @@ except ImportError:
             comps.append(curdir)
         return prefix + sep.join(comps)
 
-else:
-    def normpath(path):
-        """Normalize path, eliminating double slashes, etc."""
-        path = os.fspath(path)
-        if isinstance(path, bytes):
-            return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
-        return _path_normpath(path) or "."
-
-
-def _abspath_fallback(path):
-    """Return the absolute version of a path as a fallback function in case
-    `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for
-    more.
-
-    """
-
-    path = os.fspath(path)
-    if not isabs(path):
-        if isinstance(path, bytes):
-            cwd = os.getcwdb()
-        else:
-            cwd = os.getcwd()
-        path = join(cwd, path)
-    return normpath(path)
 
 # Return an absolute path.
 try:
     from nt import _getfullpathname
 
 except ImportError: # not running on Windows - mock up something sensible
-    abspath = _abspath_fallback
+    def abspath(path):
+        """Return the absolute version of a path."""
+        path = os.fspath(path)
+        if not isabs(path):
+            if isinstance(path, bytes):
+                cwd = os.getcwdb()
+            else:
+                cwd = os.getcwd()
+            path = join(cwd, path)
+        return normpath(path)
 
 else:  # use native Windows method on Windows
     def abspath(path):
@@ -598,15 +575,36 @@ else:  # use native Windows method on Windows
         try:
             return _getfullpathname(normpath(path))
         except (OSError, ValueError):
-            return _abspath_fallback(path)
+            # See gh-75230, handle outside for cleaner traceback
+            pass
+        path = os.fspath(path)
+        if not isabs(path):
+            if isinstance(path, bytes):
+                sep = b'\\'
+                getcwd = os.getcwdb
+            else:
+                sep = '\\'
+                getcwd = os.getcwd
+            drive, root, path = splitroot(path)
+            # Either drive or root can be nonempty, but not both.
+            if drive or root:
+                try:
+                    path = join(_getfullpathname(drive + root), path)
+                except (OSError, ValueError):
+                    # Drive "\0:" cannot exist; use the root directory.
+                    path = drive + sep + path
+            else:
+                path = join(getcwd(), path)
+        return normpath(path)
 
 try:
-    from nt import _getfinalpathname, readlink as _nt_readlink
+    from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink
 except ImportError:
     # realpath is a no-op on systems without _getfinalpathname support.
-    realpath = abspath
+    def realpath(path, *, strict=False):
+        return abspath(path)
 else:
-    def _readlink_deep(path):
+    def _readlink_deep(path, ignored_error=OSError):
         # These error codes indicate that we should stop reading links and
         # return the path we currently have.
         # 1: ERROR_INVALID_FUNCTION
@@ -639,7 +637,7 @@ else:
                         path = old_path
                         break
                     path = normpath(join(dirname(old_path), path))
-            except OSError as ex:
+            except ignored_error as ex:
                 if ex.winerror in allowed_winerror:
                     break
                 raise
@@ -648,7 +646,7 @@ else:
                 break
         return path
 
-    def _getfinalpathname_nonstrict(path):
+    def _getfinalpathname_nonstrict(path, ignored_error=OSError):
         # These error codes indicate that we should stop resolving the path
         # and return the value we currently have.
         # 1: ERROR_INVALID_FUNCTION
@@ -664,9 +662,10 @@ else:
         # 87: ERROR_INVALID_PARAMETER
         # 123: ERROR_INVALID_NAME
         # 161: ERROR_BAD_PATHNAME
+        # 1005: ERROR_UNRECOGNIZED_VOLUME
         # 1920: ERROR_CANT_ACCESS_FILE
         # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink)
-        allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921
+        allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1005, 1920, 1921
 
         # Non-strict algorithm is to find as much of the target directory
         # as we can and join the rest.
@@ -675,23 +674,29 @@ else:
             try:
                 path = _getfinalpathname(path)
                 return join(path, tail) if tail else path
-            except OSError as ex:
+            except ignored_error as ex:
                 if ex.winerror not in allowed_winerror:
                     raise
                 try:
                     # The OS could not resolve this path fully, so we attempt
                     # to follow the link ourselves. If we succeed, join the tail
                     # and return.
-                    new_path = _readlink_deep(path)
+                    new_path = _readlink_deep(path,
+                                              ignored_error=ignored_error)
                     if new_path != path:
                         return join(new_path, tail) if tail else new_path
-                except OSError:
+                except ignored_error:
                     # If we fail to readlink(), let's keep traversing
                     pass
-                path, name = split(path)
-                # TODO (bpo-38186): Request the real file name from the directory
-                # entry using FindFirstFileW. For now, we will return the path
-                # as best we have it
+                # If we get these errors, try to get the real name of the file without accessing it.
+                if ex.winerror in (1, 5, 32, 50, 87, 1920, 1921):
+                    try:
+                        name = _findfirstfile(path)
+                        path, _ = split(path)
+                    except ignored_error:
+                        path, name = split(path)
+                else:
+                    path, name = split(path)
                 if path and not name:
                     return path + tail
                 tail = join(name, tail) if tail else name
@@ -705,7 +710,8 @@ else:
             new_unc_prefix = b'\\\\'
             cwd = os.getcwdb()
             # bpo-38081: Special case for realpath(b'nul')
-            if normcase(path) == normcase(os.fsencode(devnull)):
+            devnull = b'nul'
+            if normcase(path) == devnull:
                 return b'\\\\.\\NUL'
         else:
             prefix = '\\\\?\\'
@@ -713,9 +719,19 @@ else:
             new_unc_prefix = '\\\\'
             cwd = os.getcwd()
             # bpo-38081: Special case for realpath('nul')
-            if normcase(path) == normcase(devnull):
+            devnull = 'nul'
+            if normcase(path) == devnull:
                 return '\\\\.\\NUL'
         had_prefix = path.startswith(prefix)
+
+        if strict is ALLOW_MISSING:
+            ignored_error = FileNotFoundError
+            strict = True
+        elif strict:
+            ignored_error = ()
+        else:
+            ignored_error = OSError
+
         if not had_prefix and not isabs(path):
             path = join(cwd, path)
         try:
@@ -723,17 +739,16 @@ else:
             initial_winerror = 0
         except ValueError as ex:
             # gh-106242: Raised for embedded null characters
-            # In strict mode, we convert into an OSError.
+            # In strict modes, we convert into an OSError.
             # Non-strict mode returns the path as-is, since we've already
             # made it absolute.
             if strict:
                 raise OSError(str(ex)) from None
             path = normpath(path)
-        except OSError as ex:
-            if strict:
-                raise
+        except ignored_error as ex:
             initial_winerror = ex.winerror
-            path = _getfinalpathname_nonstrict(path)
+            path = _getfinalpathname_nonstrict(path,
+                                               ignored_error=ignored_error)
         # The path returned by _getfinalpathname will always start with \\?\ -
         # strip off that prefix unless it was already provided on the original
         # path.
@@ -766,6 +781,9 @@ supports_unicode_filenames = True
 def relpath(path, start=None):
     """Return a relative version of a path"""
     path = os.fspath(path)
+    if not path:
+        raise ValueError("no path specified")
+
     if isinstance(path, bytes):
         sep = b'\\'
         curdir = b'.'
@@ -777,22 +795,20 @@ def relpath(path, start=None):
 
     if start is None:
         start = curdir
+    else:
+        start = os.fspath(start)
 
-    if not path:
-        raise ValueError("no path specified")
-
-    start = os.fspath(start)
     try:
-        start_abs = abspath(normpath(start))
-        path_abs = abspath(normpath(path))
+        start_abs = abspath(start)
+        path_abs = abspath(path)
         start_drive, _, start_rest = splitroot(start_abs)
         path_drive, _, path_rest = splitroot(path_abs)
         if normcase(start_drive) != normcase(path_drive):
             raise ValueError("path is on mount %r, start on mount %r" % (
                 path_drive, start_drive))
 
-        start_list = [x for x in start_rest.split(sep) if x]
-        path_list = [x for x in path_rest.split(sep) if x]
+        start_list = start_rest.split(sep) if start_rest else []
+        path_list = path_rest.split(sep) if path_rest else []
         # Work out how much of the filepath is shared by start and path.
         i = 0
         for e1, e2 in zip(start_list, path_list):
@@ -803,29 +819,28 @@ def relpath(path, start=None):
         rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
         if not rel_list:
             return curdir
-        return join(*rel_list)
+        return sep.join(rel_list)
     except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning):
         genericpath._check_arg_types('relpath', path, start)
         raise
 
 
-# Return the longest common sub-path of the sequence of paths given as input.
+# Return the longest common sub-path of the iterable of paths given as input.
 # The function is case-insensitive and 'separator-insensitive', i.e. if the
 # only difference between two paths is the use of '\' versus '/' as separator,
 # they are deemed to be equal.
 #
 # However, the returned path will have the standard '\' separator (even if the
 # given paths had the alternative '/' separator) and will have the case of the
-# first path given in the sequence. Additionally, any trailing separator is
+# first path given in the iterable. Additionally, any trailing separator is
 # stripped from the returned path.
 
 def commonpath(paths):
-    """Given a sequence of path names, returns the longest common sub-path."""
-
-    if not paths:
-        raise ValueError('commonpath() arg is an empty sequence')
-
+    """Given an iterable of path names, returns the longest common sub-path."""
     paths = tuple(map(os.fspath, paths))
+    if not paths:
+        raise ValueError('commonpath() arg is an empty iterable')
+
     if isinstance(paths[0], bytes):
         sep = b'\\'
         altsep = b'/'
@@ -839,9 +854,6 @@ def commonpath(paths):
         drivesplits = [splitroot(p.replace(altsep, sep).lower()) for p in paths]
         split_paths = [p.split(sep) for d, r, p in drivesplits]
 
-        if len({r for d, r, p in drivesplits}) != 1:
-            raise ValueError("Can't mix absolute and relative paths")
-
         # Check that all drive letters or UNC paths match. The check is made only
         # now otherwise type errors for mixing strings and bytes would not be
         # caught.
@@ -849,6 +861,12 @@ def commonpath(paths):
             raise ValueError("Paths don't have the same drive")
 
         drive, root, path = splitroot(paths[0].replace(altsep, sep))
+        if len({r for d, r, p in drivesplits}) != 1:
+            if drive:
+                raise ValueError("Can't mix absolute and relative paths")
+            else:
+                raise ValueError("Can't mix rooted and not-rooted paths")
+
         common = path.split(sep)
         common = [c for c in common if c and c != curdir]
 
@@ -869,13 +887,15 @@ def commonpath(paths):
 
 
 try:
-    # The isdir(), isfile(), islink() and exists() implementations in
-    # genericpath use os.stat(). This is overkill on Windows. Use simpler
+    # The isdir(), isfile(), islink(), exists() and lexists() implementations
+    # in genericpath use os.stat(). This is overkill on Windows. Use simpler
     # builtin functions if they are available.
     from nt import _path_isdir as isdir
     from nt import _path_isfile as isfile
     from nt import _path_islink as islink
+    from nt import _path_isjunction as isjunction
     from nt import _path_exists as exists
+    from nt import _path_lexists as lexists
 except ImportError:
     # Use genericpath.* as imported above
     pass
@@ -883,15 +903,12 @@ except ImportError:
 
 try:
     from nt import _path_isdevdrive
-except ImportError:
-    def isdevdrive(path):
-        """Determines whether the specified path is on a Windows Dev Drive."""
-        # Never a Dev Drive
-        return False
-else:
     def isdevdrive(path):
         """Determines whether the specified path is on a Windows Dev Drive."""
         try:
             return _path_isdevdrive(abspath(path))
         except OSError:
             return False
+except ImportError:
+    # Use genericpath.isdevdrive as imported above
+    pass

Lib/numbers.py

@@ -290,18 +290,27 @@ Real.register(float)
 
 
 class Rational(Real):
-    """.numerator and .denominator should be in lowest terms."""
+    """To Real, Rational adds numerator and denominator properties.
+
+    The numerator and denominator values should be in lowest terms,
+    with a positive denominator.
+    """
 
     __slots__ = ()
 
     @property
     @abstractmethod
     def numerator(self):
+        """The numerator of a rational number in lowest terms."""
         raise NotImplementedError
 
     @property
     @abstractmethod
     def denominator(self):
+        """The denominator of a rational number in lowest terms.
+
+        This denominator should be positive.
+        """
         raise NotImplementedError
 
     # Concrete implementation of Real's conversion to float.

Lib/opcode.py

@@ -4,404 +4,47 @@ opcode module - potentially shared between dis and other modules which
 operate on bytecodes (e.g. peephole optimizers).
 """
 
-__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs",
-           "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap",
-           "HAVE_ARGUMENT", "EXTENDED_ARG"]
 
-# It's a chicken-and-egg I'm afraid:
-# We're imported before _opcode's made.
-# With exception unheeded
-# (stack_effect is not needed)
-# Both our chickens and eggs are allayed.
-#     --Larry Hastings, 2013/11/23
+__all__ = ["cmp_op", "stack_effect", "hascompare", "opname", "opmap",
+           "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname",
+           "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"]
 
-try:
-    from _opcode import stack_effect
-    __all__.append('stack_effect')
-except ImportError:
-    pass
+import _opcode
+from _opcode import stack_effect
 
-cmp_op = ('<', '<=', '==', '!=', '>', '>=')
+from _opcode_metadata import (_specializations, _specialized_opmap, opmap,
+                              HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE)
+EXTENDED_ARG = opmap['EXTENDED_ARG']
 
-hasarg = []
-hasconst = []
-hasname = []
-hasjrel = []
-hasjabs = []
-haslocal = []
-hascompare = []
-hasfree = []
-hasexc = []
-
-def is_pseudo(op):
-    return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE
-
-oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs,
-           haslocal, hascompare, hasfree, hasexc]
-
-opmap = {}
-
-## pseudo opcodes (used in the compiler) mapped to the values
-## they can become in the actual code.
-_pseudo_ops = {}
-
-def def_op(name, op):
-    opmap[name] = op
-
-def name_op(name, op):
-    def_op(name, op)
-    hasname.append(op)
-
-def jrel_op(name, op):
-    def_op(name, op)
-    hasjrel.append(op)
-
-def jabs_op(name, op):
-    def_op(name, op)
-    hasjabs.append(op)
-
-def pseudo_op(name, op, real_ops):
-    def_op(name, op)
-    _pseudo_ops[name] = real_ops
-    # add the pseudo opcode to the lists its targets are in
-    for oplist in oplists:
-        res = [opmap[rop] in oplist for rop in real_ops]
-        if any(res):
-            assert all(res)
-            oplist.append(op)
-
-
-# Instruction opcodes for compiled code
-# Blank lines correspond to available opcodes
-
-def_op('CACHE', 0)
-def_op('POP_TOP', 1)
-def_op('PUSH_NULL', 2)
-
-def_op('NOP', 9)
-def_op('UNARY_POSITIVE', 10)
-def_op('UNARY_NEGATIVE', 11)
-def_op('UNARY_NOT', 12)
-
-def_op('UNARY_INVERT', 15)
-
-def_op('BINARY_SUBSCR', 25)
-def_op('BINARY_SLICE', 26)
-def_op('STORE_SLICE', 27)
-
-def_op('GET_LEN', 30)
-def_op('MATCH_MAPPING', 31)
-def_op('MATCH_SEQUENCE', 32)
-def_op('MATCH_KEYS', 33)
-
-def_op('PUSH_EXC_INFO', 35)
-def_op('CHECK_EXC_MATCH', 36)
-def_op('CHECK_EG_MATCH', 37)
-
-def_op('WITH_EXCEPT_START', 49)
-def_op('GET_AITER', 50)
-def_op('GET_ANEXT', 51)
-def_op('BEFORE_ASYNC_WITH', 52)
-def_op('BEFORE_WITH', 53)
-def_op('END_ASYNC_FOR', 54)
-def_op('CLEANUP_THROW', 55)
-
-def_op('STORE_SUBSCR', 60)
-def_op('DELETE_SUBSCR', 61)
-
-# TODO: RUSTPYTHON
-# Delete below def_op after updating coroutines.py
-def_op('YIELD_FROM', 72)
-
-def_op('GET_ITER', 68)
-def_op('GET_YIELD_FROM_ITER', 69)
-def_op('PRINT_EXPR', 70)
-def_op('LOAD_BUILD_CLASS', 71)
-
-def_op('LOAD_ASSERTION_ERROR', 74)
-def_op('RETURN_GENERATOR', 75)
-
-def_op('LIST_TO_TUPLE', 82)
-def_op('RETURN_VALUE', 83)
-def_op('IMPORT_STAR', 84)
-def_op('SETUP_ANNOTATIONS', 85)
-
-def_op('ASYNC_GEN_WRAP', 87)
-def_op('PREP_RERAISE_STAR', 88)
-def_op('POP_EXCEPT', 89)
-
-HAVE_ARGUMENT = 90             # real opcodes from here have an argument:
-
-name_op('STORE_NAME', 90)       # Index in name list
-name_op('DELETE_NAME', 91)      # ""
-def_op('UNPACK_SEQUENCE', 92)   # Number of tuple items
-jrel_op('FOR_ITER', 93)
-def_op('UNPACK_EX', 94)
-name_op('STORE_ATTR', 95)       # Index in name list
-name_op('DELETE_ATTR', 96)      # ""
-name_op('STORE_GLOBAL', 97)     # ""
-name_op('DELETE_GLOBAL', 98)    # ""
-def_op('SWAP', 99)
-def_op('LOAD_CONST', 100)       # Index in const list
-hasconst.append(100)
-name_op('LOAD_NAME', 101)       # Index in name list
-def_op('BUILD_TUPLE', 102)      # Number of tuple items
-def_op('BUILD_LIST', 103)       # Number of list items
-def_op('BUILD_SET', 104)        # Number of set items
-def_op('BUILD_MAP', 105)        # Number of dict entries
-name_op('LOAD_ATTR', 106)       # Index in name list
-def_op('COMPARE_OP', 107)       # Comparison operator
-hascompare.append(107)
-name_op('IMPORT_NAME', 108)     # Index in name list
-name_op('IMPORT_FROM', 109)     # Index in name list
-jrel_op('JUMP_FORWARD', 110)    # Number of words to skip
-jrel_op('JUMP_IF_FALSE_OR_POP', 111) # Number of words to skip
-jrel_op('JUMP_IF_TRUE_OR_POP', 112)  # ""
-jrel_op('POP_JUMP_IF_FALSE', 114)
-jrel_op('POP_JUMP_IF_TRUE', 115)
-name_op('LOAD_GLOBAL', 116)     # Index in name list
-def_op('IS_OP', 117)
-def_op('CONTAINS_OP', 118)
-def_op('RERAISE', 119)
-def_op('COPY', 120)
-def_op('BINARY_OP', 122)
-jrel_op('SEND', 123) # Number of bytes to skip
-def_op('LOAD_FAST', 124)        # Local variable number, no null check
-haslocal.append(124)
-def_op('STORE_FAST', 125)       # Local variable number
-haslocal.append(125)
-def_op('DELETE_FAST', 126)      # Local variable number
-haslocal.append(126)
-def_op('LOAD_FAST_CHECK', 127)  # Local variable number
-haslocal.append(127)
-jrel_op('POP_JUMP_IF_NOT_NONE', 128)
-jrel_op('POP_JUMP_IF_NONE', 129)
-def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
-def_op('GET_AWAITABLE', 131)
-def_op('MAKE_FUNCTION', 132)    # Flags
-def_op('BUILD_SLICE', 133)      # Number of items
-jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards)
-def_op('MAKE_CELL', 135)
-hasfree.append(135)
-def_op('LOAD_CLOSURE', 136)
-hasfree.append(136)
-def_op('LOAD_DEREF', 137)
-hasfree.append(137)
-def_op('STORE_DEREF', 138)
-hasfree.append(138)
-def_op('DELETE_DEREF', 139)
-hasfree.append(139)
-jrel_op('JUMP_BACKWARD', 140)    # Number of words to skip (backwards)
-
-def_op('CALL_FUNCTION_EX', 142)  # Flags
-
-def_op('EXTENDED_ARG', 144)
-EXTENDED_ARG = 144
-def_op('LIST_APPEND', 145)
-def_op('SET_ADD', 146)
-def_op('MAP_ADD', 147)
-def_op('LOAD_CLASSDEREF', 148)
-hasfree.append(148)
-def_op('COPY_FREE_VARS', 149)
-def_op('YIELD_VALUE', 150)
-def_op('RESUME', 151)   # This must be kept in sync with deepfreeze.py
-def_op('MATCH_CLASS', 152)
-
-def_op('FORMAT_VALUE', 155)
-def_op('BUILD_CONST_KEY_MAP', 156)
-def_op('BUILD_STRING', 157)
-
-def_op('LIST_EXTEND', 162)
-def_op('SET_UPDATE', 163)
-def_op('DICT_MERGE', 164)
-def_op('DICT_UPDATE', 165)
-
-def_op('CALL', 171)
-def_op('KW_NAMES', 172)
-hasconst.append(172)
-
-
-hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT])
-
-MIN_PSEUDO_OPCODE = 256
-
-pseudo_op('SETUP_FINALLY', 256, ['NOP'])
-hasexc.append(256)
-pseudo_op('SETUP_CLEANUP', 257, ['NOP'])
-hasexc.append(257)
-pseudo_op('SETUP_WITH', 258, ['NOP'])
-hasexc.append(258)
-pseudo_op('POP_BLOCK', 259, ['NOP'])
-
-pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD'])
-pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT'])
-
-pseudo_op('LOAD_METHOD', 262, ['LOAD_ATTR'])
-
-MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1
-
-del def_op, name_op, jrel_op, jabs_op, pseudo_op
-
-opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)]
+opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)]
 for op, i in opmap.items():
     opname[i] = op
 
+cmp_op = ('<', '<=', '==', '!=', '>', '>=')
 
-_nb_ops = [
-    ("NB_ADD", "+"),
-    ("NB_AND", "&"),
-    ("NB_FLOOR_DIVIDE", "//"),
-    ("NB_LSHIFT", "<<"),
-    ("NB_MATRIX_MULTIPLY", "@"),
-    ("NB_MULTIPLY", "*"),
-    ("NB_REMAINDER", "%"),
-    ("NB_OR", "|"),
-    ("NB_POWER", "**"),
-    ("NB_RSHIFT", ">>"),
-    ("NB_SUBTRACT", "-"),
-    ("NB_TRUE_DIVIDE", "/"),
-    ("NB_XOR", "^"),
-    ("NB_INPLACE_ADD", "+="),
-    ("NB_INPLACE_AND", "&="),
-    ("NB_INPLACE_FLOOR_DIVIDE", "//="),
-    ("NB_INPLACE_LSHIFT", "<<="),
-    ("NB_INPLACE_MATRIX_MULTIPLY", "@="),
-    ("NB_INPLACE_MULTIPLY", "*="),
-    ("NB_INPLACE_REMAINDER", "%="),
-    ("NB_INPLACE_OR", "|="),
-    ("NB_INPLACE_POWER", "**="),
-    ("NB_INPLACE_RSHIFT", ">>="),
-    ("NB_INPLACE_SUBTRACT", "-="),
-    ("NB_INPLACE_TRUE_DIVIDE", "/="),
-    ("NB_INPLACE_XOR", "^="),
-]
+# These lists are documented as part of the dis module's API
+hasarg = [op for op in opmap.values() if _opcode.has_arg(op)]
+hasconst = [op for op in opmap.values() if _opcode.has_const(op)]
+hasname = [op for op in opmap.values() if _opcode.has_name(op)]
+hasjump = [op for op in opmap.values() if _opcode.has_jump(op)]
+hasjrel = hasjump  # for backward compatibility
+hasjabs = []
+hasfree = [op for op in opmap.values() if _opcode.has_free(op)]
+haslocal = [op for op in opmap.values() if _opcode.has_local(op)]
+hasexc = [op for op in opmap.values() if _opcode.has_exc(op)]
 
-_specializations = {
-    "BINARY_OP": [
-        "BINARY_OP_ADAPTIVE",
-        "BINARY_OP_ADD_FLOAT",
-        "BINARY_OP_ADD_INT",
-        "BINARY_OP_ADD_UNICODE",
-        "BINARY_OP_INPLACE_ADD_UNICODE",
-        "BINARY_OP_MULTIPLY_FLOAT",
-        "BINARY_OP_MULTIPLY_INT",
-        "BINARY_OP_SUBTRACT_FLOAT",
-        "BINARY_OP_SUBTRACT_INT",
-    ],
-    "BINARY_SUBSCR": [
-        "BINARY_SUBSCR_ADAPTIVE",
-        "BINARY_SUBSCR_DICT",
-        "BINARY_SUBSCR_GETITEM",
-        "BINARY_SUBSCR_LIST_INT",
-        "BINARY_SUBSCR_TUPLE_INT",
-    ],
-    "CALL": [
-        "CALL_ADAPTIVE",
-        "CALL_PY_EXACT_ARGS",
-        "CALL_PY_WITH_DEFAULTS",
-        "CALL_BOUND_METHOD_EXACT_ARGS",
-        "CALL_BUILTIN_CLASS",
-        "CALL_BUILTIN_FAST_WITH_KEYWORDS",
-        "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS",
-        "CALL_NO_KW_BUILTIN_FAST",
-        "CALL_NO_KW_BUILTIN_O",
-        "CALL_NO_KW_ISINSTANCE",
-        "CALL_NO_KW_LEN",
-        "CALL_NO_KW_LIST_APPEND",
-        "CALL_NO_KW_METHOD_DESCRIPTOR_FAST",
-        "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS",
-        "CALL_NO_KW_METHOD_DESCRIPTOR_O",
-        "CALL_NO_KW_STR_1",
-        "CALL_NO_KW_TUPLE_1",
-        "CALL_NO_KW_TYPE_1",
-    ],
-    "COMPARE_OP": [
-        "COMPARE_OP_ADAPTIVE",
-        "COMPARE_OP_FLOAT_JUMP",
-        "COMPARE_OP_INT_JUMP",
-        "COMPARE_OP_STR_JUMP",
-    ],
-    "EXTENDED_ARG": [
-        "EXTENDED_ARG_QUICK",
-    ],
-    "FOR_ITER": [
-        "FOR_ITER_ADAPTIVE",
-        "FOR_ITER_LIST",
-        "FOR_ITER_RANGE",
-    ],
-    "JUMP_BACKWARD": [
-        "JUMP_BACKWARD_QUICK",
-    ],
-    "LOAD_ATTR": [
-        "LOAD_ATTR_ADAPTIVE",
-        # These potentially push [NULL, bound method] onto the stack.
-        "LOAD_ATTR_CLASS",
-        "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN",
-        "LOAD_ATTR_INSTANCE_VALUE",
-        "LOAD_ATTR_MODULE",
-        "LOAD_ATTR_PROPERTY",
-        "LOAD_ATTR_SLOT",
-        "LOAD_ATTR_WITH_HINT",
-        # These will always push [unbound method, self] onto the stack.
-        "LOAD_ATTR_METHOD_LAZY_DICT",
-        "LOAD_ATTR_METHOD_NO_DICT",
-        "LOAD_ATTR_METHOD_WITH_DICT",
-        "LOAD_ATTR_METHOD_WITH_VALUES",
-    ],
-    "LOAD_CONST": [
-        "LOAD_CONST__LOAD_FAST",
-    ],
-    "LOAD_FAST": [
-        "LOAD_FAST__LOAD_CONST",
-        "LOAD_FAST__LOAD_FAST",
-    ],
-    "LOAD_GLOBAL": [
-        "LOAD_GLOBAL_ADAPTIVE",
-        "LOAD_GLOBAL_BUILTIN",
-        "LOAD_GLOBAL_MODULE",
-    ],
-    "RESUME": [
-        "RESUME_QUICK",
-    ],
-    "STORE_ATTR": [
-        "STORE_ATTR_ADAPTIVE",
-        "STORE_ATTR_INSTANCE_VALUE",
-        "STORE_ATTR_SLOT",
-        "STORE_ATTR_WITH_HINT",
-    ],
-    "STORE_FAST": [
-        "STORE_FAST__LOAD_FAST",
-        "STORE_FAST__STORE_FAST",
-    ],
-    "STORE_SUBSCR": [
-        "STORE_SUBSCR_ADAPTIVE",
-        "STORE_SUBSCR_DICT",
-        "STORE_SUBSCR_LIST_INT",
-    ],
-    "UNPACK_SEQUENCE": [
-        "UNPACK_SEQUENCE_ADAPTIVE",
-        "UNPACK_SEQUENCE_LIST",
-        "UNPACK_SEQUENCE_TUPLE",
-        "UNPACK_SEQUENCE_TWO_TUPLE",
-    ],
-}
-_specialized_instructions = [
-    opcode for family in _specializations.values() for opcode in family
-]
-_specialization_stats = [
-    "success",
-    "failure",
-    "hit",
-    "deferred",
-    "miss",
-    "deopt",
-]
+
+_intrinsic_1_descs = _opcode.get_intrinsic1_descs()
+_intrinsic_2_descs = _opcode.get_intrinsic2_descs()
+_nb_ops = _opcode.get_nb_ops()
+
+hascompare = [opmap["COMPARE_OP"]]
 
 _cache_format = {
     "LOAD_GLOBAL": {
         "counter": 1,
         "index": 1,
-        "module_keys_version": 2,
+        "module_keys_version": 1,
         "builtin_keys_version": 1,
     },
     "BINARY_OP": {
@@ -412,16 +55,19 @@ _cache_format = {
     },
     "COMPARE_OP": {
         "counter": 1,
-        "mask": 1,
+    },
+    "CONTAINS_OP": {
+        "counter": 1,
     },
     "BINARY_SUBSCR": {
         "counter": 1,
-        "type_version": 2,
-        "func_version": 1,
     },
     "FOR_ITER": {
         "counter": 1,
     },
+    "LOAD_SUPER_ATTR": {
+        "counter": 1,
+    },
     "LOAD_ATTR": {
         "counter": 1,
         "version": 2,
@@ -436,13 +82,34 @@ _cache_format = {
     "CALL": {
         "counter": 1,
         "func_version": 2,
-        "min_args": 1,
     },
     "STORE_SUBSCR": {
         "counter": 1,
     },
+    "SEND": {
+        "counter": 1,
+    },
+    "JUMP_BACKWARD": {
+        "counter": 1,
+    },
+    "TO_BOOL": {
+        "counter": 1,
+        "version": 2,
+    },
+    "POP_JUMP_IF_TRUE": {
+        "counter": 1,
+    },
+    "POP_JUMP_IF_FALSE": {
+        "counter": 1,
+    },
+    "POP_JUMP_IF_NONE": {
+        "counter": 1,
+    },
+    "POP_JUMP_IF_NOT_NONE": {
+        "counter": 1,
+    },
 }
 
-_inline_cache_entries = [
-    sum(_cache_format.get(opname[opcode], {}).values()) for opcode in range(256)
-]
+_inline_cache_entries = {
+    name : sum(value.values()) for (name, value) in _cache_format.items()
+}

Lib/os.py

@@ -110,6 +110,7 @@ if _exists("_have_functions"):
     _add("HAVE_FCHMODAT",   "chmod")
     _add("HAVE_FCHOWNAT",   "chown")
     _add("HAVE_FSTATAT",    "stat")
+    _add("HAVE_LSTAT",      "lstat")
     _add("HAVE_FUTIMESAT",  "utime")
     _add("HAVE_LINKAT",     "link")
     _add("HAVE_MKDIRAT",    "mkdir")
@@ -131,6 +132,7 @@ if _exists("_have_functions"):
     _set = set()
     _add("HAVE_FCHDIR",     "chdir")
     _add("HAVE_FCHMOD",     "chmod")
+    _add("MS_WINDOWS",      "chmod")
     _add("HAVE_FCHOWN",     "chown")
     _add("HAVE_FDOPENDIR",  "listdir")
     _add("HAVE_FDOPENDIR",  "scandir")
@@ -171,6 +173,7 @@ if _exists("_have_functions"):
     _add("HAVE_FSTATAT",    "stat")
     _add("HAVE_LCHFLAGS",   "chflags")
     _add("HAVE_LCHMOD",     "chmod")
+    _add("MS_WINDOWS",      "chmod")
     if _exists("lchown"): # mac os x10.3
         _add("HAVE_LCHOWN", "chown")
     _add("HAVE_LINKAT",     "link")
@@ -279,6 +282,10 @@ def renames(old, new):
 
 __all__.extend(["makedirs", "removedirs", "renames"])
 
+# Private sentinel that makes walk() classify all symlinks and junctions as
+# regular files.
+_walk_symlinks_as_files = object()
+
 def walk(top, topdown=True, onerror=None, followlinks=False):
     """Directory tree generator.
 
@@ -331,12 +338,12 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
 
     import os
     from os.path import join, getsize
-    for root, dirs, files in os.walk('python/Lib/email'):
+    for root, dirs, files in os.walk('python/Lib/xml'):
         print(root, "consumes ")
         print(sum(getsize(join(root, name)) for name in files), end=" ")
         print("bytes in", len(files), "non-directory files")
-        if 'CVS' in dirs:
-            dirs.remove('CVS')  # don't visit CVS directories
+        if '__pycache__' in dirs:
+            dirs.remove('__pycache__')  # don't visit __pycache__ directories
 
     """
     sys.audit("os.walk", top, topdown, onerror, followlinks)
@@ -380,7 +387,10 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
                     break
 
                 try:
-                    is_dir = entry.is_dir()
+                    if followlinks is _walk_symlinks_as_files:
+                        is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction()
+                    else:
+                        is_dir = entry.is_dir()
                 except OSError:
                     # If is_dir() raises an OSError, consider the entry not to
                     # be a directory, same behaviour as os.path.isdir().
@@ -459,34 +469,69 @@ if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd:
         Example:
 
         import os
-        for root, dirs, files, rootfd in os.fwalk('python/Lib/email'):
+        for root, dirs, files, rootfd in os.fwalk('python/Lib/xml'):
             print(root, "consumes", end="")
             print(sum(os.stat(name, dir_fd=rootfd).st_size for name in files),
                   end="")
             print("bytes in", len(files), "non-directory files")
-            if 'CVS' in dirs:
-                dirs.remove('CVS')  # don't visit CVS directories
+            if '__pycache__' in dirs:
+                dirs.remove('__pycache__')  # don't visit __pycache__ directories
         """
         sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd)
         top = fspath(top)
-        # Note: To guard against symlink races, we use the standard
-        # lstat()/open()/fstat() trick.
-        if not follow_symlinks:
-            orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd)
-        topfd = open(top, O_RDONLY | O_NONBLOCK, dir_fd=dir_fd)
+        stack = [(_fwalk_walk, (True, dir_fd, top, top, None))]
+        isbytes = isinstance(top, bytes)
         try:
-            if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and
-                                    path.samestat(orig_st, stat(topfd)))):
-                yield from _fwalk(topfd, top, isinstance(top, bytes),
-                                  topdown, onerror, follow_symlinks)
+            while stack:
+                yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks)
         finally:
-            close(topfd)
+            # Close any file descriptors still on the stack.
+            while stack:
+                action, value = stack.pop()
+                if action == _fwalk_close:
+                    close(value)
 
-    def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks):
+    # Each item in the _fwalk() stack is a pair (action, args).
+    _fwalk_walk = 0  # args: (isroot, dirfd, toppath, topname, entry)
+    _fwalk_yield = 1  # args: (toppath, dirnames, filenames, topfd)
+    _fwalk_close = 2  # args: dirfd
+
+    def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks):
         # Note: This uses O(depth of the directory tree) file descriptors: if
         # necessary, it can be adapted to only require O(1) FDs, see issue
         # #13734.
 
+        action, value = stack.pop()
+        if action == _fwalk_close:
+            close(value)
+            return
+        elif action == _fwalk_yield:
+            yield value
+            return
+        assert action == _fwalk_walk
+        isroot, dirfd, toppath, topname, entry = value
+        try:
+            if not follow_symlinks:
+                # Note: To guard against symlink races, we use the standard
+                # lstat()/open()/fstat() trick.
+                if entry is None:
+                    orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd)
+                else:
+                    orig_st = entry.stat(follow_symlinks=False)
+            topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd)
+        except OSError as err:
+            if isroot:
+                raise
+            if onerror is not None:
+                onerror(err)
+            return
+        stack.append((_fwalk_close, topfd))
+        if not follow_symlinks:
+            if isroot and not st.S_ISDIR(orig_st.st_mode):
+                return
+            if not path.samestat(orig_st, stat(topfd)):
+                return
+
         scandir_it = scandir(topfd)
         dirs = []
         nondirs = []
@@ -512,31 +557,18 @@ if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd:
 
         if topdown:
             yield toppath, dirs, nondirs, topfd
+        else:
+            stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd)))
 
-        for name in dirs if entries is None else zip(dirs, entries):
-            try:
-                if not follow_symlinks:
-                    if topdown:
-                        orig_st = stat(name, dir_fd=topfd, follow_symlinks=False)
-                    else:
-                        assert entries is not None
-                        name, entry = name
-                        orig_st = entry.stat(follow_symlinks=False)
-                dirfd = open(name, O_RDONLY | O_NONBLOCK, dir_fd=topfd)
-            except OSError as err:
-                if onerror is not None:
-                    onerror(err)
-                continue
-            try:
-                if follow_symlinks or path.samestat(orig_st, stat(dirfd)):
-                    dirpath = path.join(toppath, name)
-                    yield from _fwalk(dirfd, dirpath, isbytes,
-                                      topdown, onerror, follow_symlinks)
-            finally:
-                close(dirfd)
-
-        if not topdown:
-            yield toppath, dirs, nondirs, topfd
+        toppath = path.join(toppath, toppath[:0])  # Add trailing slash.
+        if entries is None:
+            stack.extend(
+                (_fwalk_walk, (False, topfd, toppath + name, name, None))
+                for name in dirs[::-1])
+        else:
+            stack.extend(
+                (_fwalk_walk, (False, topfd, toppath + name, name, entry))
+                for name, entry in zip(dirs[::-1], entries[::-1]))
 
     __all__.append("fwalk")
 
@@ -1061,6 +1093,12 @@ def _fspath(path):
         else:
             raise TypeError("expected str, bytes or os.PathLike object, "
                             "not " + path_type.__name__)
+    except TypeError:
+        if path_type.__fspath__ is None:
+            raise TypeError("expected str, bytes or os.PathLike object, "
+                            "not " + path_type.__name__) from None
+        else:
+            raise
     if isinstance(path_repr, (str, bytes)):
         return path_repr
     else:
@@ -1079,6 +1117,8 @@ class PathLike(abc.ABC):
 
     """Abstract base class for implementing the file system path protocol."""
 
+    __slots__ = ()
+
     @abc.abstractmethod
     def __fspath__(self):
         """Return the file system path representation of the object."""
@@ -1128,3 +1168,17 @@ if name == 'nt':
             cookie,
             nt._remove_dll_directory
         )
+
+
+if _exists('sched_getaffinity') and sys._get_cpu_count_config() < 0:
+    def process_cpu_count():
+        """
+        Get the number of CPUs of the current process.
+
+        Return the number of logical CPUs usable by the calling thread of the
+        current process. Return None if indeterminable.
+        """
+        return len(sched_getaffinity(0))
+else:
+    # Just an alias to cpu_count() (same docstring)
+    process_cpu_count = cpu_count

Lib/pickle.py

@@ -314,16 +314,17 @@ class _Unframer:
 # Tools used for pickling.
 
 def _getattribute(obj, name):
+    top = obj
     for subpath in name.split('.'):
         if subpath == '<locals>':
             raise AttributeError("Can't get local attribute {!r} on {!r}"
-                                 .format(name, obj))
+                                 .format(name, top))
         try:
             parent = obj
             obj = getattr(obj, subpath)
         except AttributeError:
             raise AttributeError("Can't get attribute {!r} on {!r}"
-                                 .format(name, obj)) from None
+                                 .format(name, top)) from None
     return obj, parent
 
 def whichmodule(obj, name):
@@ -396,6 +397,8 @@ def decode_long(data):
     return int.from_bytes(data, byteorder='little', signed=True)
 
 
+_NoValue = object()
+
 # Pickling machinery
 
 class _Pickler:
@@ -530,10 +533,11 @@ class _Pickler:
         self.framer.commit_frame()
 
         # Check for persistent id (defined by a subclass)
-        pid = self.persistent_id(obj)
-        if pid is not None and save_persistent_id:
-            self.save_pers(pid)
-            return
+        if save_persistent_id:
+            pid = self.persistent_id(obj)
+            if pid is not None:
+                self.save_pers(pid)
+                return
 
         # Check the memo
         x = self.memo.get(id(obj))
@@ -542,8 +546,8 @@ class _Pickler:
             return
 
         rv = NotImplemented
-        reduce = getattr(self, "reducer_override", None)
-        if reduce is not None:
+        reduce = getattr(self, "reducer_override", _NoValue)
+        if reduce is not _NoValue:
             rv = reduce(obj)
 
         if rv is NotImplemented:
@@ -556,8 +560,8 @@ class _Pickler:
 
             # Check private dispatch table if any, or else
             # copyreg.dispatch_table
-            reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
-            if reduce is not None:
+            reduce = getattr(self, 'dispatch_table', dispatch_table).get(t, _NoValue)
+            if reduce is not _NoValue:
                 rv = reduce(obj)
             else:
                 # Check for a class with a custom metaclass; treat as regular
@@ -567,12 +571,12 @@ class _Pickler:
                     return
 
                 # Check for a __reduce_ex__ method, fall back to __reduce__
-                reduce = getattr(obj, "__reduce_ex__", None)
-                if reduce is not None:
+                reduce = getattr(obj, "__reduce_ex__", _NoValue)
+                if reduce is not _NoValue:
                     rv = reduce(self.proto)
                 else:
-                    reduce = getattr(obj, "__reduce__", None)
-                    if reduce is not None:
+                    reduce = getattr(obj, "__reduce__", _NoValue)
+                    if reduce is not _NoValue:
                         rv = reduce()
                     else:
                         raise PicklingError("Can't pickle %r object: %r" %
@@ -780,14 +784,10 @@ class _Pickler:
             self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
     dispatch[float] = save_float
 
-    def save_bytes(self, obj):
-        if self.proto < 3:
-            if not obj: # bytes object is empty
-                self.save_reduce(bytes, (), obj=obj)
-            else:
-                self.save_reduce(codecs.encode,
-                                 (str(obj, 'latin1'), 'latin1'), obj=obj)
-            return
+    def _save_bytes_no_memo(self, obj):
+        # helper for writing bytes objects for protocol >= 3
+        # without memoizing them
+        assert self.proto >= 3
         n = len(obj)
         if n <= 0xff:
             self.write(SHORT_BINBYTES + pack("<B", n) + obj)
@@ -797,9 +797,29 @@ class _Pickler:
             self._write_large_bytes(BINBYTES + pack("<I", n), obj)
         else:
             self.write(BINBYTES + pack("<I", n) + obj)
+
+    def save_bytes(self, obj):
+        if self.proto < 3:
+            if not obj: # bytes object is empty
+                self.save_reduce(bytes, (), obj=obj)
+            else:
+                self.save_reduce(codecs.encode,
+                                 (str(obj, 'latin1'), 'latin1'), obj=obj)
+            return
+        self._save_bytes_no_memo(obj)
         self.memoize(obj)
     dispatch[bytes] = save_bytes
 
+    def _save_bytearray_no_memo(self, obj):
+        # helper for writing bytearray objects for protocol >= 5
+        # without memoizing them
+        assert self.proto >= 5
+        n = len(obj)
+        if n >= self.framer._FRAME_SIZE_TARGET:
+            self._write_large_bytes(BYTEARRAY8 + pack("<Q", n), obj)
+        else:
+            self.write(BYTEARRAY8 + pack("<Q", n) + obj)
+
     def save_bytearray(self, obj):
         if self.proto < 5:
             if not obj:  # bytearray is empty
@@ -807,18 +827,14 @@ class _Pickler:
             else:
                 self.save_reduce(bytearray, (bytes(obj),), obj=obj)
             return
-        n = len(obj)
-        if n >= self.framer._FRAME_SIZE_TARGET:
-            self._write_large_bytes(BYTEARRAY8 + pack("<Q", n), obj)
-        else:
-            self.write(BYTEARRAY8 + pack("<Q", n) + obj)
+        self._save_bytearray_no_memo(obj)
         self.memoize(obj)
     dispatch[bytearray] = save_bytearray
 
     if _HAVE_PICKLE_BUFFER:
         def save_picklebuffer(self, obj):
             if self.proto < 5:
-                raise PicklingError("PickleBuffer can only pickled with "
+                raise PicklingError("PickleBuffer can only be pickled with "
                                     "protocol >= 5")
             with obj.raw() as m:
                 if not m.contiguous:
@@ -830,10 +846,18 @@ class _Pickler:
                 if in_band:
                     # Write data in-band
                     # XXX The C implementation avoids a copy here
+                    buf = m.tobytes()
+                    in_memo = id(buf) in self.memo
                     if m.readonly:
-                        self.save_bytes(m.tobytes())
+                        if in_memo:
+                            self._save_bytes_no_memo(buf)
+                        else:
+                            self.save_bytes(buf)
                     else:
-                        self.save_bytearray(m.tobytes())
+                        if in_memo:
+                            self._save_bytearray_no_memo(buf)
+                        else:
+                            self.save_bytearray(buf)
                 else:
                     # Write data out-of-band
                     self.write(NEXT_BUFFER)
@@ -1070,11 +1094,16 @@ class _Pickler:
                     (obj, module_name, name))
 
         if self.proto >= 2:
-            code = _extension_registry.get((module_name, name))
-            if code:
-                assert code > 0
+            code = _extension_registry.get((module_name, name), _NoValue)
+            if code is not _NoValue:
                 if code <= 0xff:
-                    write(EXT1 + pack("<B", code))
+                    data = pack("<B", code)
+                    if data == b'\0':
+                        # Should never happen in normal circumstances,
+                        # since the type and the value of the code are
+                        # checked in copyreg.add_extension().
+                        raise RuntimeError("extension code 0 is out of range")
+                    write(EXT1 + data)
                 elif code <= 0xffff:
                     write(EXT2 + pack("<H", code))
                 else:
@@ -1088,11 +1117,35 @@ class _Pickler:
             self.save(module_name)
             self.save(name)
             write(STACK_GLOBAL)
-        elif parent is not module:
-            self.save_reduce(getattr, (parent, lastname))
-        elif self.proto >= 3:
-            write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
-                  bytes(name, "utf-8") + b'\n')
+        elif '.' in name:
+            # In protocol < 4, objects with multi-part __qualname__
+            # are represented as
+            # getattr(getattr(..., attrname1), attrname2).
+            dotted_path = name.split('.')
+            name = dotted_path.pop(0)
+            save = self.save
+            for attrname in dotted_path:
+                save(getattr)
+                if self.proto < 2:
+                    write(MARK)
+            self._save_toplevel_by_name(module_name, name)
+            for attrname in dotted_path:
+                save(attrname)
+                if self.proto < 2:
+                    write(TUPLE)
+                else:
+                    write(TUPLE2)
+                write(REDUCE)
+        else:
+            self._save_toplevel_by_name(module_name, name)
+
+        self.memoize(obj)
+
+    def _save_toplevel_by_name(self, module_name, name):
+        if self.proto >= 3:
+            # Non-ASCII identifiers are supported only with protocols >= 3.
+            self.write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
+                       bytes(name, "utf-8") + b'\n')
         else:
             if self.fix_imports:
                 r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
@@ -1102,14 +1155,12 @@ class _Pickler:
                 elif module_name in r_import_mapping:
                     module_name = r_import_mapping[module_name]
             try:
-                write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
-                      bytes(name, "ascii") + b'\n')
+                self.write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
+                           bytes(name, "ascii") + b'\n')
             except UnicodeEncodeError:
                 raise PicklingError(
                     "can't pickle global identifier '%s.%s' using "
-                    "pickle protocol %i" % (module, name, self.proto)) from None
-
-        self.memoize(obj)
+                    "pickle protocol %i" % (module_name, name, self.proto)) from None
 
     def save_type(self, obj):
         if obj is type(None):
@@ -1546,9 +1597,8 @@ class _Unpickler:
     dispatch[EXT4[0]] = load_ext4
 
     def get_extension(self, code):
-        nil = []
-        obj = _extension_cache.get(code, nil)
-        if obj is not nil:
+        obj = _extension_cache.get(code, _NoValue)
+        if obj is not _NoValue:
             self.append(obj)
             return
         key = _inverted_registry.get(code)
@@ -1705,8 +1755,8 @@ class _Unpickler:
         stack = self.stack
         state = stack.pop()
         inst = stack[-1]
-        setstate = getattr(inst, "__setstate__", None)
-        if setstate is not None:
+        setstate = getattr(inst, "__setstate__", _NoValue)
+        if setstate is not _NoValue:
             setstate(state)
             return
         slotstate = None

Lib/pickletools.py

@@ -312,7 +312,7 @@ uint8 = ArgumentDescriptor(
             doc="Eight-byte unsigned integer, little-endian.")
 
 
-def read_stringnl(f, decode=True, stripquotes=True):
+def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'):
     r"""
     >>> import io
     >>> read_stringnl(io.BytesIO(b"'abcd'\nefg\n"))
@@ -356,7 +356,7 @@ def read_stringnl(f, decode=True, stripquotes=True):
             raise ValueError("no string quotes around %r" % data)
 
     if decode:
-        data = codecs.escape_decode(data)[0].decode("ascii")
+        data = codecs.escape_decode(data)[0].decode(encoding)
     return data
 
 stringnl = ArgumentDescriptor(
@@ -370,7 +370,7 @@ stringnl = ArgumentDescriptor(
                    """)
 
 def read_stringnl_noescape(f):
-    return read_stringnl(f, stripquotes=False)
+    return read_stringnl(f, stripquotes=False, encoding='utf-8')
 
 stringnl_noescape = ArgumentDescriptor(
                         name='stringnl_noescape',
@@ -2513,7 +2513,10 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0):
             # make a mild effort to align arguments
             line += ' ' * (10 - len(opcode.name))
             if arg is not None:
-                line += ' ' + repr(arg)
+                if opcode.name in ("STRING", "BINSTRING", "SHORT_BINSTRING"):
+                    line += ' ' + ascii(arg)
+                else:
+                    line += ' ' + repr(arg)
             if markmsg:
                 line += ' ' + markmsg
         if annotate:

Lib/posixpath.py

@@ -22,6 +22,7 @@ defpath = '/bin:/usr/bin'
 altsep = None
 devnull = '/dev/null'
 
+import errno
 import os
 import sys
 import stat
@@ -35,7 +36,7 @@ __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext"
            "samefile","sameopenfile","samestat",
            "curdir","pardir","sep","pathsep","defpath","altsep","extsep",
            "devnull","realpath","supports_unicode_filenames","relpath",
-           "commonpath", "isjunction"]
+           "commonpath", "isjunction","isdevdrive","ALLOW_MISSING"]
 
 
 def _get_sep(path):
@@ -77,12 +78,11 @@ def join(a, *p):
     sep = _get_sep(a)
     path = a
     try:
-        if not p:
-            path[:0] + sep  #23780: Ensure compatible data type even if p is null.
-        for b in map(os.fspath, p):
-            if b.startswith(sep):
+        for b in p:
+            b = os.fspath(b)
+            if b.startswith(sep) or not path:
                 path = b
-            elif not path or path.endswith(sep):
+            elif path.endswith(sep):
                 path += b
             else:
                 path += sep + b
@@ -135,33 +135,30 @@ def splitdrive(p):
     return p[:0], p
 
 
-def splitroot(p):
-    """Split a pathname into drive, root and tail. On Posix, drive is always
-    empty; the root may be empty, a single slash, or two slashes. The tail
-    contains anything after the root. For example:
+try:
+    from posix import _path_splitroot_ex as splitroot
+except ImportError:
+    def splitroot(p):
+        """Split a pathname into drive, root and tail.
 
-        splitroot('foo/bar') == ('', '', 'foo/bar')
-        splitroot('/foo/bar') == ('', '/', 'foo/bar')
-        splitroot('//foo/bar') == ('', '//', 'foo/bar')
-        splitroot('///foo/bar') == ('', '/', '//foo/bar')
-    """
-    p = os.fspath(p)
-    if isinstance(p, bytes):
-        sep = b'/'
-        empty = b''
-    else:
-        sep = '/'
-        empty = ''
-    if p[:1] != sep:
-        # Relative path, e.g.: 'foo'
-        return empty, empty, p
-    elif p[1:2] != sep or p[2:3] == sep:
-        # Absolute path, e.g.: '/foo', '///foo', '////foo', etc.
-        return empty, sep, p[1:]
-    else:
-        # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see
-        # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13
-        return empty, p[:2], p[2:]
+        The tail contains anything after the root."""
+        p = os.fspath(p)
+        if isinstance(p, bytes):
+            sep = b'/'
+            empty = b''
+        else:
+            sep = '/'
+            empty = ''
+        if p[:1] != sep:
+            # Relative path, e.g.: 'foo'
+            return empty, empty, p
+        elif p[1:2] != sep or p[2:3] == sep:
+            # Absolute path, e.g.: '/foo', '///foo', '////foo', etc.
+            return empty, sep, p[1:]
+        else:
+            # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see
+            # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13
+            return empty, p[:2], p[2:]
 
 
 # Return the tail (basename) part of a path, same as split(path)[1].
@@ -187,26 +184,6 @@ def dirname(p):
     return head
 
 
-# Is a path a junction?
-
-def isjunction(path):
-    """Test whether a path is a junction
-    Junctions are not a part of posix semantics"""
-    os.fspath(path)
-    return False
-
-
-# Being true for dangling symbolic links is also useful.
-
-def lexists(path):
-    """Test whether a path exists.  Returns True for broken symbolic links"""
-    try:
-        os.lstat(path)
-    except (OSError, ValueError):
-        return False
-    return True
-
-
 # Is a path a mount point?
 # (Does this work for all UNIXes?  Is it even guaranteed to work by Posix?)
 
@@ -227,21 +204,17 @@ def ismount(path):
         parent = join(path, b'..')
     else:
         parent = join(path, '..')
-    parent = realpath(parent)
     try:
         s2 = os.lstat(parent)
-    except (OSError, ValueError):
-        return False
+    except OSError:
+        parent = realpath(parent)
+        try:
+            s2 = os.lstat(parent)
+        except OSError:
+            return False
 
-    dev1 = s1.st_dev
-    dev2 = s2.st_dev
-    if dev1 != dev2:
-        return True     # path/.. on a different device as path
-    ino1 = s1.st_ino
-    ino2 = s2.st_ino
-    if ino1 == ino2:
-        return True     # path/.. is the same i-node as path
-    return False
+    # path/.. on a different device as path or the same i-node as path
+    return s1.st_dev != s2.st_dev or s1.st_ino == s2.st_ino
 
 
 # Expand paths beginning with '~' or '~user'.
@@ -290,7 +263,7 @@ def expanduser(path):
             return path
         name = path[1:i]
         if isinstance(name, bytes):
-            name = str(name, 'ASCII')
+            name = os.fsdecode(name)
         try:
             pwent = pwd.getpwnam(name)
         except KeyError:
@@ -303,11 +276,8 @@ def expanduser(path):
         return path
     if isinstance(path, bytes):
         userhome = os.fsencode(userhome)
-        root = b'/'
-    else:
-        root = '/'
-    userhome = userhome.rstrip(root)
-    return (userhome + path[i:]) or root
+    userhome = userhome.rstrip(sep)
+    return (userhome + path[i:]) or sep
 
 
 # Expand paths containing shell variable substitutions.
@@ -371,7 +341,7 @@ def expandvars(path):
 # if it contains symbolic links!
 
 try:
-    from posix import _path_normpath
+    from posix import _path_normpath as normpath
 
 except ImportError:
     def normpath(path):
@@ -379,21 +349,19 @@ except ImportError:
         path = os.fspath(path)
         if isinstance(path, bytes):
             sep = b'/'
-            empty = b''
             dot = b'.'
             dotdot = b'..'
         else:
             sep = '/'
-            empty = ''
             dot = '.'
             dotdot = '..'
-        if path == empty:
+        if not path:
             return dot
         _, initial_slashes, path = splitroot(path)
         comps = path.split(sep)
         new_comps = []
         for comp in comps:
-            if comp in (empty, dot):
+            if not comp or comp == dot:
                 continue
             if (comp != dotdot or (not initial_slashes and not new_comps) or
                  (new_comps and new_comps[-1] == dotdot)):
@@ -404,24 +372,16 @@ except ImportError:
         path = initial_slashes + sep.join(comps)
         return path or dot
 
-else:
-    def normpath(path):
-        """Normalize path, eliminating double slashes, etc."""
-        path = os.fspath(path)
-        if isinstance(path, bytes):
-            return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
-        return _path_normpath(path) or "."
-
 
 def abspath(path):
     """Return an absolute path."""
     path = os.fspath(path)
-    if not isabs(path):
-        if isinstance(path, bytes):
-            cwd = os.getcwdb()
-        else:
-            cwd = os.getcwd()
-        path = join(cwd, path)
+    if isinstance(path, bytes):
+        if not path.startswith(b'/'):
+            path = join(os.getcwdb(), path)
+    else:
+        if not path.startswith('/'):
+            path = join(os.getcwd(), path)
     return normpath(path)
 
 
@@ -432,72 +392,109 @@ def realpath(filename, *, strict=False):
     """Return the canonical path of the specified filename, eliminating any
 symbolic links encountered in the path."""
     filename = os.fspath(filename)
-    path, ok = _joinrealpath(filename[:0], filename, strict, {})
-    return abspath(path)
-
-# Join two paths, normalizing and eliminating any symbolic links
-# encountered in the second path.
-def _joinrealpath(path, rest, strict, seen):
-    if isinstance(path, bytes):
+    if isinstance(filename, bytes):
         sep = b'/'
         curdir = b'.'
         pardir = b'..'
+        getcwd = os.getcwdb
     else:
         sep = '/'
         curdir = '.'
         pardir = '..'
+        getcwd = os.getcwd
+    if strict is ALLOW_MISSING:
+        ignored_error = FileNotFoundError
+        strict = True
+    elif strict:
+        ignored_error = ()
+    else:
+        ignored_error = OSError
 
-    if isabs(rest):
-        rest = rest[1:]
-        path = sep
+    maxlinks = None
 
-    while rest:
-        name, _, rest = rest.partition(sep)
+    # The stack of unresolved path parts. When popped, a special value of None
+    # indicates that a symlink target has been resolved, and that the original
+    # symlink path can be retrieved by popping again. The [::-1] slice is a
+    # very fast way of spelling list(reversed(...)).
+    rest = filename.split(sep)[::-1]
+
+    # Number of unprocessed parts in 'rest'. This can differ from len(rest)
+    # later, because 'rest' might contain markers for unresolved symlinks.
+    part_count = len(rest)
+
+    # The resolved path, which is absolute throughout this function.
+    # Note: getcwd() returns a normalized and symlink-free path.
+    path = sep if filename.startswith(sep) else getcwd()
+
+    # Mapping from symlink paths to *fully resolved* symlink targets. If a
+    # symlink is encountered but not yet resolved, the value is None. This is
+    # used both to detect symlink loops and to speed up repeated traversals of
+    # the same links.
+    seen = {}
+
+    while part_count:
+        name = rest.pop()
+        if name is None:
+            # resolved symlink target
+            seen[rest.pop()] = path
+            continue
+        part_count -= 1
         if not name or name == curdir:
             # current dir
             continue
         if name == pardir:
             # parent dir
-            if path:
-                path, name = split(path)
-                if name == pardir:
-                    path = join(path, pardir, pardir)
-            else:
-                path = pardir
+            path = path[:path.rindex(sep)] or sep
             continue
-        newpath = join(path, name)
-        try:
-            st = os.lstat(newpath)
-        except OSError:
-            if strict:
-                raise
-            is_link = False
+        if path == sep:
+            newpath = path + name
         else:
-            is_link = stat.S_ISLNK(st.st_mode)
-        if not is_link:
-            path = newpath
-            continue
-        # Resolve the symbolic link
-        if newpath in seen:
-            # Already seen this path
-            path = seen[newpath]
-            if path is not None:
-                # use cached value
+            newpath = path + sep + name
+        try:
+            st_mode = os.lstat(newpath).st_mode
+            if not stat.S_ISLNK(st_mode):
+                if strict and part_count and not stat.S_ISDIR(st_mode):
+                    raise OSError(errno.ENOTDIR, os.strerror(errno.ENOTDIR),
+                                  newpath)
+                path = newpath
                 continue
-            # The symlink is not resolved, so we must have a symlink loop.
-            if strict:
-                # Raise OSError(errno.ELOOP)
-                os.stat(newpath)
-            else:
-                # Return already resolved part + rest of the path unchanged.
-                return join(newpath, rest), False
-        seen[newpath] = None # not resolved symlink
-        path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen)
-        if not ok:
-            return join(path, rest), False
-        seen[newpath] = path # resolved symlink
+            if newpath in seen:
+                # Already seen this path
+                path = seen[newpath]
+                if path is not None:
+                    # use cached value
+                    continue
+                # The symlink is not resolved, so we must have a symlink loop.
+                if strict:
+                    # Raise OSError(errno.ELOOP)
+                    os.stat(newpath)
+                path = newpath
+                continue
+            target = os.readlink(newpath)
+        except ignored_error:
+            pass
+        else:
+            # Resolve the symbolic link
+            if target.startswith(sep):
+                # Symlink target is absolute; reset resolved path.
+                path = sep
+            if maxlinks is None:
+                # Mark this symlink as seen but not fully resolved.
+                seen[newpath] = None
+                # Push the symlink path onto the stack, and signal its specialness
+                # by also pushing None. When these entries are popped, we'll
+                # record the fully-resolved symlink target in the 'seen' mapping.
+                rest.append(newpath)
+                rest.append(None)
+            # Push the unresolved symlink target parts onto the stack.
+            target_parts = target.split(sep)[::-1]
+            rest.extend(target_parts)
+            part_count += len(target_parts)
+            continue
+        # An error occurred and was ignored.
+        path = newpath
 
-    return path, True
+    return path
 
 
 supports_unicode_filenames = (sys.platform == 'darwin')
@@ -505,10 +502,10 @@ supports_unicode_filenames = (sys.platform == 'darwin')
 def relpath(path, start=None):
     """Return a relative version of a path"""
 
+    path = os.fspath(path)
     if not path:
         raise ValueError("no path specified")
 
-    path = os.fspath(path)
     if isinstance(path, bytes):
         curdir = b'.'
         sep = b'/'
@@ -524,15 +521,17 @@ def relpath(path, start=None):
         start = os.fspath(start)
 
     try:
-        start_list = [x for x in abspath(start).split(sep) if x]
-        path_list = [x for x in abspath(path).split(sep) if x]
+        start_tail = abspath(start).lstrip(sep)
+        path_tail = abspath(path).lstrip(sep)
+        start_list = start_tail.split(sep) if start_tail else []
+        path_list = path_tail.split(sep) if path_tail else []
         # Work out how much of the filepath is shared by start and path.
         i = len(commonprefix([start_list, path_list]))
 
         rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
         if not rel_list:
             return curdir
-        return join(*rel_list)
+        return sep.join(rel_list)
     except (TypeError, AttributeError, BytesWarning, DeprecationWarning):
         genericpath._check_arg_types('relpath', path, start)
         raise
@@ -546,10 +545,11 @@ def relpath(path, start=None):
 def commonpath(paths):
     """Given a sequence of path names, returns the longest common sub-path."""
 
+    paths = tuple(map(os.fspath, paths))
+
     if not paths:
         raise ValueError('commonpath() arg is an empty sequence')
 
-    paths = tuple(map(os.fspath, paths))
     if isinstance(paths[0], bytes):
         sep = b'/'
         curdir = b'.'
@@ -561,7 +561,7 @@ def commonpath(paths):
         split_paths = [path.split(sep) for path in paths]
 
         try:
-            isabs, = set(p[:1] == sep for p in paths)
+            isabs, = {p.startswith(sep) for p in paths}
         except ValueError:
             raise ValueError("Can't mix absolute and relative paths") from None
 

Lib/runpy.py

@@ -14,18 +14,20 @@ import sys
 import importlib.machinery # importlib first so we can test #15386 via -m
 import importlib.util
 import io
-import types
 import os
 
 __all__ = [
     "run_module", "run_path",
 ]
 
+# avoid 'import types' just for ModuleType
+ModuleType = type(sys)
+
 class _TempModule(object):
     """Temporarily replace a module in sys.modules with an empty namespace"""
     def __init__(self, mod_name):
         self.mod_name = mod_name
-        self.module = types.ModuleType(mod_name)
+        self.module = ModuleType(mod_name)
         self._saved_module = []
 
     def __enter__(self):
@@ -245,17 +247,17 @@ def _get_main_module_details(error=ImportError):
         sys.modules[main_name] = saved_main
 
 
-def _get_code_from_file(run_name, fname):
+def _get_code_from_file(fname):
     # Check for a compiled file first
     from pkgutil import read_code
-    decoded_path = os.path.abspath(os.fsdecode(fname))
-    with io.open_code(decoded_path) as f:
+    code_path = os.path.abspath(fname)
+    with io.open_code(code_path) as f:
         code = read_code(f)
     if code is None:
         # That didn't work, so try it as normal source code
-        with io.open_code(decoded_path) as f:
+        with io.open_code(code_path) as f:
             code = compile(f.read(), fname, 'exec')
-    return code, fname
+    return code
 
 def run_path(path_name, init_globals=None, run_name=None):
     """Execute code located at the specified filesystem location.
@@ -277,17 +279,13 @@ def run_path(path_name, init_globals=None, run_name=None):
     pkg_name = run_name.rpartition(".")[0]
     from pkgutil import get_importer
     importer = get_importer(path_name)
-    # Trying to avoid importing imp so as to not consume the deprecation warning.
-    is_NullImporter = False
-    if type(importer).__module__ == 'imp':
-        if type(importer).__name__ == 'NullImporter':
-            is_NullImporter = True
-    if isinstance(importer, type(None)) or is_NullImporter:
+    path_name = os.fsdecode(path_name)
+    if isinstance(importer, type(None)):
         # Not a valid sys.path entry, so run the code directly
         # execfile() doesn't help as we want to allow compiled files
-        code, fname = _get_code_from_file(run_name, path_name)
+        code = _get_code_from_file(path_name)
         return _run_module_code(code, init_globals, run_name,
-                                pkg_name=pkg_name, script_name=fname)
+                                pkg_name=pkg_name, script_name=path_name)
     else:
         # Finder is defined for path, so add it to
         # the start of sys.path

Lib/site.py

@@ -75,6 +75,7 @@ import builtins
 import _sitebuiltins
 import io
 import stat
+import errno
 
 # Prefixes for site-packages; add additional prefixes like /usr/local here
 PREFIXES = [sys.prefix, sys.exec_prefix]
@@ -179,35 +180,46 @@ def addpackage(sitedir, name, known_paths):
         return
     _trace(f"Processing .pth file: {fullname!r}")
     try:
-        # locale encoding is not ideal especially on Windows. But we have used
-        # it for a long time. setuptools uses the locale encoding too.
-        f = io.TextIOWrapper(io.open_code(fullname), encoding="locale")
+        with io.open_code(fullname) as f:
+            pth_content = f.read()
     except OSError:
         return
-    with f:
-        for n, line in enumerate(f):
-            if line.startswith("#"):
+
+    try:
+        # Accept BOM markers in .pth files as we do in source files
+        # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM)
+        pth_content = pth_content.decode("utf-8-sig")
+    except UnicodeDecodeError:
+        # Fallback to locale encoding for backward compatibility.
+        # We will deprecate this fallback in the future.
+        import locale
+        pth_content = pth_content.decode(locale.getencoding())
+        _trace(f"Cannot read {fullname!r} as UTF-8. "
+               f"Using fallback encoding {locale.getencoding()!r}")
+
+    for n, line in enumerate(pth_content.splitlines(), 1):
+        if line.startswith("#"):
+            continue
+        if line.strip() == "":
+            continue
+        try:
+            if line.startswith(("import ", "import\t")):
+                exec(line)
                 continue
-            if line.strip() == "":
-                continue
-            try:
-                if line.startswith(("import ", "import\t")):
-                    exec(line)
-                    continue
-                line = line.rstrip()
-                dir, dircase = makepath(sitedir, line)
-                if not dircase in known_paths and os.path.exists(dir):
-                    sys.path.append(dir)
-                    known_paths.add(dircase)
-            except Exception as exc:
-                print("Error processing line {:d} of {}:\n".format(n+1, fullname),
-                      file=sys.stderr)
-                import traceback
-                for record in traceback.format_exception(exc):
-                    for line in record.splitlines():
-                        print('  '+line, file=sys.stderr)
-                print("\nRemainder of file ignored", file=sys.stderr)
-                break
+            line = line.rstrip()
+            dir, dircase = makepath(sitedir, line)
+            if dircase not in known_paths and os.path.exists(dir):
+                sys.path.append(dir)
+                known_paths.add(dircase)
+        except Exception as exc:
+            print(f"Error processing line {n:d} of {fullname}:\n",
+                  file=sys.stderr)
+            import traceback
+            for record in traceback.format_exception(exc):
+                for line in record.splitlines():
+                    print('  '+line, file=sys.stderr)
+            print("\nRemainder of file ignored", file=sys.stderr)
+            break
     if reset:
         known_paths = None
     return known_paths
@@ -270,14 +282,18 @@ def check_enableusersite():
 #
 # See https://bugs.python.org/issue29585
 
+# Copy of sysconfig._get_implementation()
+def _get_implementation():
+    return 'RustPython' # XXX: RustPython; for site-packages
+
 # Copy of sysconfig._getuserbase()
 def _getuserbase():
     env_base = os.environ.get("PYTHONUSERBASE", None)
     if env_base:
         return env_base
 
-    # Emscripten, VxWorks, and WASI have no home directories
-    if sys.platform in {"emscripten", "vxworks", "wasi"}:
+    # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories
+    if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}:
         return None
 
     def joinuser(*args):
@@ -285,8 +301,7 @@ def _getuserbase():
 
     if os.name == "nt":
         base = os.environ.get("APPDATA") or "~"
-        # XXX: RUSTPYTHON; please keep this change for site-packages
-        return joinuser(base, "RustPython")
+        return joinuser(base, _get_implementation())
 
     if sys.platform == "darwin" and sys._framework:
         return joinuser("~", "Library", sys._framework,
@@ -298,15 +313,22 @@ def _getuserbase():
 # Same to sysconfig.get_path('purelib', os.name+'_user')
 def _get_path(userbase):
     version = sys.version_info
+    if hasattr(sys, 'abiflags') and 't' in sys.abiflags:
+        abi_thread = 't'
+    else:
+        abi_thread = ''
 
+    implementation = _get_implementation()
+    implementation_lower = implementation.lower()
     if os.name == 'nt':
         ver_nodot = sys.winver.replace('.', '')
-        return f'{userbase}\\RustPython{ver_nodot}\\site-packages'
+        return f'{userbase}\\{implementation}{ver_nodot}\\site-packages'
 
     if sys.platform == 'darwin' and sys._framework:
-        return f'{userbase}/lib/rustpython/site-packages'
+        return f'{userbase}/lib/{implementation_lower}/site-packages'
 
-    return f'{userbase}/lib/rustpython{version[0]}.{version[1]}/site-packages'
+    # XXX: RUSTPYTHON
+    return f'{userbase}/lib/rustpython{version[0]}.{version[1]}{abi_thread}/site-packages'
 
 
 def getuserbase():
@@ -372,6 +394,12 @@ def getsitepackages(prefixes=None):
             continue
         seen.add(prefix)
 
+        implementation = _get_implementation().lower()
+        ver = sys.version_info
+        if hasattr(sys, 'abiflags') and 't' in sys.abiflags:
+            abi_thread = 't'
+        else:
+            abi_thread = ''
         if os.sep == '/':
             libdirs = [sys.platlibdir]
             if sys.platlibdir != "lib":
@@ -379,8 +407,7 @@ def getsitepackages(prefixes=None):
 
             for libdir in libdirs:
                 path = os.path.join(prefix, libdir,
-                                    # XXX: RUSTPYTHON; please keep this change for site-packages
-                                    "rustpython%d.%d" % sys.version_info[:2],
+                                    f"{implementation}{ver[0]}.{ver[1]}{abi_thread}",
                                     "site-packages")
                 sitepackages.append(path)
         else:
@@ -417,8 +444,9 @@ def setcopyright():
     """Set 'copyright' and 'credits' in builtins"""
     builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright)
     builtins.credits = _sitebuiltins._Printer("credits", """\
-    Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
-    for supporting Python development.  See www.python.org for more information.""")
+    Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software
+    Foundation, and a cast of thousands for supporting Python
+    development.  See www.python.org for more information.""")
     files, dirs = [], []
     # Not all modules are required to have a __file__ attribute.  See
     # PEP 420 for more details.
@@ -437,27 +465,76 @@ def setcopyright():
 def sethelper():
     builtins.help = _sitebuiltins._Helper()
 
+
+def gethistoryfile():
+    """Check if the PYTHON_HISTORY environment variable is set and define
+    it as the .python_history file.  If PYTHON_HISTORY is not set, use the
+    default .python_history file.
+    """
+    if not sys.flags.ignore_environment:
+        history = os.environ.get("PYTHON_HISTORY")
+        if history:
+            return history
+    return os.path.join(os.path.expanduser('~'),
+        '.python_history')
+
+
 def enablerlcompleter():
     """Enable default readline configuration on interactive prompts, by
     registering a sys.__interactivehook__.
+    """
+    sys.__interactivehook__ = register_readline
+
+
+def register_readline():
+    """Configure readline completion on interactive prompts.
 
     If the readline module can be imported, the hook will set the Tab key
     as completion key and register ~/.python_history as history file.
     This can be overridden in the sitecustomize or usercustomize module,
     or in a PYTHONSTARTUP file.
     """
-    def register_readline():
-        import atexit
+    if not sys.flags.ignore_environment:
+        PYTHON_BASIC_REPL = os.getenv("PYTHON_BASIC_REPL")
+    else:
+        PYTHON_BASIC_REPL = False
+
+    import atexit
+
+    try:
         try:
             import readline
-            import rlcompleter
         except ImportError:
-            return
+            readline = None
+        else:
+            import rlcompleter  # noqa: F401
+    except ImportError:
+        return
 
+    try:
+        if PYTHON_BASIC_REPL:
+            CAN_USE_PYREPL = False
+        else:
+            original_path = sys.path
+            sys.path = [p for p in original_path if p != '']
+            try:
+                import _pyrepl.readline
+                if os.name == "nt":
+                    import _pyrepl.windows_console
+                    console_errors = (_pyrepl.windows_console._error,)
+                else:
+                    import _pyrepl.unix_console
+                    console_errors = _pyrepl.unix_console._error
+                from _pyrepl.main import CAN_USE_PYREPL
+            finally:
+                sys.path = original_path
+    except ImportError:
+        return
+
+    if readline is not None:
         # Reading the initialization (config) file may not be enough to set a
         # completion key, so we set one first and then read the file.
-        readline_doc = getattr(readline, '__doc__', '')
-        if readline_doc is not None and 'libedit' in readline_doc:
+        if readline.backend == 'editline':
             readline.parse_and_bind('bind ^I rl_complete')
         else:
             readline.parse_and_bind('tab: complete')
@@ -471,30 +548,44 @@ def enablerlcompleter():
             # want to ignore the exception.
             pass
 
-        if readline.get_current_history_length() == 0:
-            # If no history was loaded, default to .python_history.
-            # The guard is necessary to avoid doubling history size at
-            # each interpreter exit when readline was already configured
-            # through a PYTHONSTARTUP hook, see:
-            # http://bugs.python.org/issue5845#msg198636
-            history = os.path.join(os.path.expanduser('~'),
-                                   '.python_history')
+    if readline is None or readline.get_current_history_length() == 0:
+        # If no history was loaded, default to .python_history,
+        # or PYTHON_HISTORY.
+        # The guard is necessary to avoid doubling history size at
+        # each interpreter exit when readline was already configured
+        # through a PYTHONSTARTUP hook, see:
+        # http://bugs.python.org/issue5845#msg198636
+        history = gethistoryfile()
+
+        if CAN_USE_PYREPL:
+            readline_module = _pyrepl.readline
+            exceptions = (OSError, *console_errors)
+        else:
+            if readline is None:
+                return
+            readline_module = readline
+            exceptions = OSError
+
+        try:
+            readline_module.read_history_file(history)
+        except exceptions:
+            pass
+
+        def write_history():
             try:
-                readline.read_history_file(history)
-            except OSError:
+                readline_module.write_history_file(history)
+            except (FileNotFoundError, PermissionError):
+                # home directory does not exist or is not writable
+                # https://bugs.python.org/issue19891
                 pass
+            except OSError:
+                if errno.EROFS:
+                    pass  # gh-128066: read-only file system
+                else:
+                    raise
 
-            def write_history():
-                try:
-                    readline.write_history_file(history)
-                except OSError:
-                    # bpo-19891, bpo-41193: Home directory does not exist
-                    # or is not writable, or the filesystem is read-only.
-                    pass
+        atexit.register(write_history)
 
-            atexit.register(write_history)
-
-    sys.__interactivehook__ = register_readline
 
 def venv(known_paths):
     global PREFIXES, ENABLE_USER_SITE
@@ -679,17 +770,5 @@ def _script():
         print(textwrap.dedent(help % (sys.argv[0], os.pathsep)))
         sys.exit(10)
 
-def gethistoryfile():
-    """Check if the PYTHON_HISTORY environment variable is set and define
-    it as the .python_history file.  If PYTHON_HISTORY is not set, use the
-    default .python_history file.
-    """
-    if not sys.flags.ignore_environment:
-        history = os.environ.get("PYTHON_HISTORY")
-        if history:
-            return history
-    return os.path.join(os.path.expanduser('~'),
-        '.python_history')
-
 if __name__ == '__main__':
     _script()

Lib/symtable.py

@@ -0,0 +1,414 @@
+"""Interface to the compiler's internal symbol tables"""
+
+import _symtable
+from _symtable import (USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM,
+     DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE,
+     LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL)
+
+import weakref
+from enum import StrEnum
+
+__all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"]
+
+def symtable(code, filename, compile_type):
+    """ Return the toplevel *SymbolTable* for the source code.
+
+    *filename* is the name of the file with the code
+    and *compile_type* is the *compile()* mode argument.
+    """
+    top = _symtable.symtable(code, filename, compile_type)
+    return _newSymbolTable(top, filename)
+
+class SymbolTableFactory:
+    def __init__(self):
+        self.__memo = weakref.WeakValueDictionary()
+
+    def new(self, table, filename):
+        if table.type == _symtable.TYPE_FUNCTION:
+            return Function(table, filename)
+        if table.type == _symtable.TYPE_CLASS:
+            return Class(table, filename)
+        return SymbolTable(table, filename)
+
+    def __call__(self, table, filename):
+        key = table, filename
+        obj = self.__memo.get(key, None)
+        if obj is None:
+            obj = self.__memo[key] = self.new(table, filename)
+        return obj
+
+_newSymbolTable = SymbolTableFactory()
+
+
+class SymbolTableType(StrEnum):
+    MODULE = "module"
+    FUNCTION = "function"
+    CLASS = "class"
+    ANNOTATION = "annotation"
+    TYPE_ALIAS = "type alias"
+    TYPE_PARAMETERS = "type parameters"
+    TYPE_VARIABLE = "type variable"
+
+
+class SymbolTable:
+
+    def __init__(self, raw_table, filename):
+        self._table = raw_table
+        self._filename = filename
+        self._symbols = {}
+
+    def __repr__(self):
+        if self.__class__ == SymbolTable:
+            kind = ""
+        else:
+            kind = "%s " % self.__class__.__name__
+
+        if self._table.name == "top":
+            return "<{0}SymbolTable for module {1}>".format(kind, self._filename)
+        else:
+            return "<{0}SymbolTable for {1} in {2}>".format(kind,
+                                                            self._table.name,
+                                                            self._filename)
+
+    def get_type(self):
+        """Return the type of the symbol table.
+
+        The value returned is one of the values in
+        the ``SymbolTableType`` enumeration.
+        """
+        if self._table.type == _symtable.TYPE_MODULE:
+            return SymbolTableType.MODULE
+        if self._table.type == _symtable.TYPE_FUNCTION:
+            return SymbolTableType.FUNCTION
+        if self._table.type == _symtable.TYPE_CLASS:
+            return SymbolTableType.CLASS
+        if self._table.type == _symtable.TYPE_ANNOTATION:
+            return SymbolTableType.ANNOTATION
+        if self._table.type == _symtable.TYPE_TYPE_ALIAS:
+            return SymbolTableType.TYPE_ALIAS
+        if self._table.type == _symtable.TYPE_TYPE_PARAMETERS:
+            return SymbolTableType.TYPE_PARAMETERS
+        if self._table.type == _symtable.TYPE_TYPE_VARIABLE:
+            return SymbolTableType.TYPE_VARIABLE
+        assert False, f"unexpected type: {self._table.type}"
+
+    def get_id(self):
+        """Return an identifier for the table.
+        """
+        return self._table.id
+
+    def get_name(self):
+        """Return the table's name.
+
+        This corresponds to the name of the class, function
+        or 'top' if the table is for a class, function or
+        global respectively.
+        """
+        return self._table.name
+
+    def get_lineno(self):
+        """Return the number of the first line in the
+        block for the table.
+        """
+        return self._table.lineno
+
+    def is_optimized(self):
+        """Return *True* if the locals in the table
+        are optimizable.
+        """
+        return bool(self._table.type == _symtable.TYPE_FUNCTION)
+
+    def is_nested(self):
+        """Return *True* if the block is a nested class
+        or function."""
+        return bool(self._table.nested)
+
+    def has_children(self):
+        """Return *True* if the block has nested namespaces.
+        """
+        return bool(self._table.children)
+
+    def get_identifiers(self):
+        """Return a view object containing the names of symbols in the table.
+        """
+        return self._table.symbols.keys()
+
+    def lookup(self, name):
+        """Lookup a *name* in the table.
+
+        Returns a *Symbol* instance.
+        """
+        sym = self._symbols.get(name)
+        if sym is None:
+            flags = self._table.symbols[name]
+            namespaces = self.__check_children(name)
+            module_scope = (self._table.name == "top")
+            sym = self._symbols[name] = Symbol(name, flags, namespaces,
+                                               module_scope=module_scope)
+        return sym
+
+    def get_symbols(self):
+        """Return a list of *Symbol* instances for
+        names in the table.
+        """
+        return [self.lookup(ident) for ident in self.get_identifiers()]
+
+    def __check_children(self, name):
+        return [_newSymbolTable(st, self._filename)
+                for st in self._table.children
+                if st.name == name]
+
+    def get_children(self):
+        """Return a list of the nested symbol tables.
+        """
+        return [_newSymbolTable(st, self._filename)
+                for st in self._table.children]
+
+
+class Function(SymbolTable):
+
+    # Default values for instance variables
+    __params = None
+    __locals = None
+    __frees = None
+    __globals = None
+    __nonlocals = None
+
+    def __idents_matching(self, test_func):
+        return tuple(ident for ident in self.get_identifiers()
+                     if test_func(self._table.symbols[ident]))
+
+    def get_parameters(self):
+        """Return a tuple of parameters to the function.
+        """
+        if self.__params is None:
+            self.__params = self.__idents_matching(lambda x:x & DEF_PARAM)
+        return self.__params
+
+    def get_locals(self):
+        """Return a tuple of locals in the function.
+        """
+        if self.__locals is None:
+            locs = (LOCAL, CELL)
+            test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs
+            self.__locals = self.__idents_matching(test)
+        return self.__locals
+
+    def get_globals(self):
+        """Return a tuple of globals in the function.
+        """
+        if self.__globals is None:
+            glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
+            test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob
+            self.__globals = self.__idents_matching(test)
+        return self.__globals
+
+    def get_nonlocals(self):
+        """Return a tuple of nonlocals in the function.
+        """
+        if self.__nonlocals is None:
+            self.__nonlocals = self.__idents_matching(lambda x:x & DEF_NONLOCAL)
+        return self.__nonlocals
+
+    def get_frees(self):
+        """Return a tuple of free variables in the function.
+        """
+        if self.__frees is None:
+            is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE
+            self.__frees = self.__idents_matching(is_free)
+        return self.__frees
+
+
+class Class(SymbolTable):
+
+    __methods = None
+
+    def get_methods(self):
+        """Return a tuple of methods declared in the class.
+        """
+        if self.__methods is None:
+            d = {}
+
+            def is_local_symbol(ident):
+                flags = self._table.symbols.get(ident, 0)
+                return ((flags >> SCOPE_OFF) & SCOPE_MASK) == LOCAL
+
+            for st in self._table.children:
+                # pick the function-like symbols that are local identifiers
+                if is_local_symbol(st.name):
+                    match st.type:
+                        case _symtable.TYPE_FUNCTION:
+                            # generators are of type TYPE_FUNCTION with a ".0"
+                            # parameter as a first parameter (which makes them
+                            # distinguishable from a function named 'genexpr')
+                            if st.name == 'genexpr' and '.0' in st.varnames:
+                                continue
+                            d[st.name] = 1
+                        case _symtable.TYPE_TYPE_PARAMETERS:
+                            # Get the function-def block in the annotation
+                            # scope 'st' with the same identifier, if any.
+                            scope_name = st.name
+                            for c in st.children:
+                                if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION:
+                                    # A generic generator of type TYPE_FUNCTION
+                                    # cannot be a direct child of 'st' (but it
+                                    # can be a descendant), e.g.:
+                                    #
+                                    # class A:
+                                    #   type genexpr[genexpr] = (x for x in [])
+                                    assert scope_name != 'genexpr' or '.0' not in c.varnames
+                                    d[scope_name] = 1
+                                    break
+            self.__methods = tuple(d)
+        return self.__methods
+
+
+class Symbol:
+
+    def __init__(self, name, flags, namespaces=None, *, module_scope=False):
+        self.__name = name
+        self.__flags = flags
+        self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope()
+        self.__namespaces = namespaces or ()
+        self.__module_scope = module_scope
+
+    def __repr__(self):
+        flags_str = '|'.join(self._flags_str())
+        return f'<symbol {self.__name!r}: {self._scope_str()}, {flags_str}>'
+
+    def _scope_str(self):
+        return _scopes_value_to_name.get(self.__scope) or str(self.__scope)
+
+    def _flags_str(self):
+        for flagname, flagvalue in _flags:
+            if self.__flags & flagvalue == flagvalue:
+                yield flagname
+
+    def get_name(self):
+        """Return a name of a symbol.
+        """
+        return self.__name
+
+    def is_referenced(self):
+        """Return *True* if the symbol is used in
+        its block.
+        """
+        return bool(self.__flags & _symtable.USE)
+
+    def is_parameter(self):
+        """Return *True* if the symbol is a parameter.
+        """
+        return bool(self.__flags & DEF_PARAM)
+
+    def is_global(self):
+        """Return *True* if the symbol is global.
+        """
+        return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
+                    or (self.__module_scope and self.__flags & DEF_BOUND))
+
+    def is_nonlocal(self):
+        """Return *True* if the symbol is nonlocal."""
+        return bool(self.__flags & DEF_NONLOCAL)
+
+    def is_declared_global(self):
+        """Return *True* if the symbol is declared global
+        with a global statement."""
+        return bool(self.__scope == GLOBAL_EXPLICIT)
+
+    def is_local(self):
+        """Return *True* if the symbol is local.
+        """
+        return bool(self.__scope in (LOCAL, CELL)
+                    or (self.__module_scope and self.__flags & DEF_BOUND))
+
+    def is_annotated(self):
+        """Return *True* if the symbol is annotated.
+        """
+        return bool(self.__flags & DEF_ANNOT)
+
+    def is_free(self):
+        """Return *True* if a referenced symbol is
+        not assigned to.
+        """
+        return bool(self.__scope == FREE)
+
+    def is_imported(self):
+        """Return *True* if the symbol is created from
+        an import statement.
+        """
+        return bool(self.__flags & DEF_IMPORT)
+
+    def is_assigned(self):
+        """Return *True* if a symbol is assigned to."""
+        return bool(self.__flags & DEF_LOCAL)
+
+    def is_namespace(self):
+        """Returns *True* if name binding introduces new namespace.
+
+        If the name is used as the target of a function or class
+        statement, this will be true.
+
+        Note that a single name can be bound to multiple objects.  If
+        is_namespace() is true, the name may also be bound to other
+        objects, like an int or list, that does not introduce a new
+        namespace.
+        """
+        return bool(self.__namespaces)
+
+    def get_namespaces(self):
+        """Return a list of namespaces bound to this name"""
+        return self.__namespaces
+
+    def get_namespace(self):
+        """Return the single namespace bound to this name.
+
+        Raises ValueError if the name is bound to multiple namespaces
+        or no namespace.
+        """
+        if len(self.__namespaces) == 0:
+            raise ValueError("name is not bound to any namespaces")
+        elif len(self.__namespaces) > 1:
+            raise ValueError("name is bound to multiple namespaces")
+        else:
+            return self.__namespaces[0]
+
+
+_flags = [('USE', USE)]
+_flags.extend(kv for kv in globals().items() if kv[0].startswith('DEF_'))
+_scopes_names = ('FREE', 'LOCAL', 'GLOBAL_IMPLICIT', 'GLOBAL_EXPLICIT', 'CELL')
+_scopes_value_to_name = {globals()[n]: n for n in _scopes_names}
+
+
+def main(args):
+    import sys
+    def print_symbols(table, level=0):
+        indent = '    ' * level
+        nested = "nested " if table.is_nested() else ""
+        if table.get_type() == 'module':
+            what = f'from file {table._filename!r}'
+        else:
+            what = f'{table.get_name()!r}'
+        print(f'{indent}symbol table for {nested}{table.get_type()} {what}:')
+        for ident in table.get_identifiers():
+            symbol = table.lookup(ident)
+            flags = ', '.join(symbol._flags_str()).lower()
+            print(f'    {indent}{symbol._scope_str().lower()} symbol {symbol.get_name()!r}: {flags}')
+        print()
+
+        for table2 in table.get_children():
+            print_symbols(table2, level + 1)
+
+    for filename in args or ['-']:
+        if filename == '-':
+            src = sys.stdin.read()
+            filename = '<stdin>'
+        else:
+            with open(filename, 'rb') as f:
+                src = f.read()
+        mod = symtable(src, filename, 'exec')
+        print_symbols(mod)
+
+
+if __name__ == "__main__":
+    import sys
+    main(sys.argv[1:])

Lib/sysconfig/__init__.py

@@ -1,10 +1,9 @@
-# XXX: RUSTPYTHON; Trick to make sysconfig work as RustPython
-exec(r'''
 """Access to Python's configuration information."""
 
 import os
 import sys
-from os.path import pardir, realpath
+import threading
+from os.path import realpath
 
 __all__ = [
     'get_config_h_filename',
@@ -22,29 +21,30 @@ __all__ = [
 
 # Keys for get_config_var() that are never converted to Python integers.
 _ALWAYS_STR = {
+    'IPHONEOS_DEPLOYMENT_TARGET',
     'MACOSX_DEPLOYMENT_TARGET',
 }
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}',
-        'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}',
-        'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages',
+        'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
+        'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
+        'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
+        'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
         'include':
-            '{installed_base}/include/python{py_version_short}{abiflags}',
+            '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
         'platinclude':
-            '{installed_platbase}/include/python{py_version_short}{abiflags}',
+            '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'posix_home': {
-        'stdlib': '{installed_base}/lib/python',
-        'platstdlib': '{base}/lib/python',
-        'purelib': '{base}/lib/python',
-        'platlib': '{base}/lib/python',
-        'include': '{installed_base}/include/python',
-        'platinclude': '{installed_base}/include/python',
+        'stdlib': '{installed_base}/lib/{implementation_lower}',
+        'platstdlib': '{base}/lib/{implementation_lower}',
+        'purelib': '{base}/lib/{implementation_lower}',
+        'platlib': '{base}/lib/{implementation_lower}',
+        'include': '{installed_base}/include/{implementation_lower}',
+        'platinclude': '{installed_base}/include/{implementation_lower}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
@@ -58,6 +58,7 @@ _INSTALL_SCHEMES = {
         'scripts': '{base}/Scripts',
         'data': '{base}',
         },
+
     # Downstream distributors can overwrite the default install scheme.
     # This is done to support downstream modifications where distributors change
     # the installation layout (eg. different site-packages directory).
@@ -76,14 +77,14 @@ _INSTALL_SCHEMES = {
     # Downstream distributors who patch posix_prefix/nt scheme are encouraged to
     # leave the following schemes unchanged
     'posix_venv': {
-        'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}',
-        'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}',
-        'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages',
+        'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
+        'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
+        'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
+        'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
         'include':
-            '{installed_base}/include/python{py_version_short}{abiflags}',
+            '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
         'platinclude':
-            '{installed_platbase}/include/python{py_version_short}{abiflags}',
+            '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
@@ -105,6 +106,8 @@ if os.name == 'nt':
 else:
     _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv']
 
+def _get_implementation():
+    return 'RustPython' # XXX: For site-packages
 
 # NOTE: site.py has copy of this function.
 # Sync it when modify this function.
@@ -113,8 +116,8 @@ def _getuserbase():
     if env_base:
         return env_base
 
-    # Emscripten, VxWorks, and WASI have no home directories
-    if sys.platform in {"emscripten", "vxworks", "wasi"}:
+    # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories
+    if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}:
         return None
 
     def joinuser(*args):
@@ -122,7 +125,7 @@ def _getuserbase():
 
     if os.name == "nt":
         base = os.environ.get("APPDATA") or "~"
-        return joinuser(base, "Python")
+        return joinuser(base,  _get_implementation())
 
     if sys.platform == "darwin" and sys._framework:
         return joinuser("~", "Library", sys._framework,
@@ -136,29 +139,29 @@ if _HAS_USER_BASE:
     _INSTALL_SCHEMES |= {
         # NOTE: When modifying "purelib" scheme, update site._get_path() too.
         'nt_user': {
-            'stdlib': '{userbase}/Python{py_version_nodot_plat}',
-            'platstdlib': '{userbase}/Python{py_version_nodot_plat}',
-            'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages',
-            'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages',
-            'include': '{userbase}/Python{py_version_nodot_plat}/Include',
-            'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts',
+            'stdlib': '{userbase}/{implementation}{py_version_nodot_plat}',
+            'platstdlib': '{userbase}/{implementation}{py_version_nodot_plat}',
+            'purelib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages',
+            'platlib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages',
+            'include': '{userbase}/{implementation}{py_version_nodot_plat}/Include',
+            'scripts': '{userbase}/{implementation}{py_version_nodot_plat}/Scripts',
             'data': '{userbase}',
             },
         'posix_user': {
-            'stdlib': '{userbase}/{platlibdir}/python{py_version_short}',
-            'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}',
-            'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-            'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-            'include': '{userbase}/include/python{py_version_short}',
+            'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
+            'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}',
+            'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
+            'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages',
+            'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}',
             'scripts': '{userbase}/bin',
             'data': '{userbase}',
             },
         'osx_framework_user': {
-            'stdlib': '{userbase}/lib/python',
-            'platstdlib': '{userbase}/lib/python',
-            'purelib': '{userbase}/lib/python/site-packages',
-            'platlib': '{userbase}/lib/python/site-packages',
-            'include': '{userbase}/include/python{py_version_short}',
+            'stdlib': '{userbase}/lib/{implementation_lower}',
+            'platstdlib': '{userbase}/lib/{implementation_lower}',
+            'purelib': '{userbase}/lib/{implementation_lower}/site-packages',
+            'platlib': '{userbase}/lib/{implementation_lower}/site-packages',
+            'include': '{userbase}/include/{implementation_lower}{py_version_short}',
             'scripts': '{userbase}/bin',
             'data': '{userbase}',
             },
@@ -170,19 +173,15 @@ _SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include',
 _PY_VERSION = sys.version.split()[0]
 _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}'
 _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}'
-_PREFIX = os.path.normpath(sys.prefix)
 _BASE_PREFIX = os.path.normpath(sys.base_prefix)
-_EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
 _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix)
+# Mutex guarding initialization of _CONFIG_VARS.
+_CONFIG_VARS_LOCK = threading.RLock()
 _CONFIG_VARS = None
+# True iff _CONFIG_VARS has been fully initialized.
+_CONFIG_VARS_INITIALIZED = False
 _USER_BASE = None
 
-# Regexes needed for parsing Makefile (and similar syntaxes,
-# like old-style Setup files).
-_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)"
-_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)"
-_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}"
-
 
 def _safe_realpath(path):
     try:
@@ -221,8 +220,15 @@ if "_PYTHON_PROJECT_BASE" in os.environ:
 def is_python_build(check_home=None):
     if check_home is not None:
         import warnings
-        warnings.warn("check_home argument is deprecated and ignored.",
-                      DeprecationWarning, stacklevel=2)
+        warnings.warn(
+            (
+                'The check_home argument of sysconfig.is_python_build is '
+                'deprecated and its value is ignored. '
+                'It will be removed in Python 3.15.'
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
     for fn in ("Setup", "Setup.local"):
         if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)):
             return True
@@ -291,6 +297,7 @@ def _get_preferred_schemes():
             'home': 'posix_home',
             'user': 'osx_framework_user',
         }
+
     return {
         'prefix': 'posix_prefix',
         'home': 'posix_home',
@@ -314,134 +321,6 @@ def get_default_scheme():
     return get_preferred_scheme('prefix')
 
 
-def _parse_makefile(filename, vars=None, keep_unresolved=True):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    import re
-
-    if vars is None:
-        vars = {}
-    done = {}
-    notdone = {}
-
-    with open(filename, encoding=sys.getfilesystemencoding(),
-              errors="surrogateescape") as f:
-        lines = f.readlines()
-
-    for line in lines:
-        if line.startswith('#') or line.strip() == '':
-            continue
-        m = re.match(_variable_rx, line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    if n in _ALWAYS_STR:
-                        raise ValueError
-
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    variables = list(notdone.keys())
-
-    # Variables with a 'PY_' prefix in the makefile. These need to
-    # be made available without that prefix through sysconfig.
-    # Special care is needed to ensure that variable expansion works, even
-    # if the expansion uses the name without a prefix.
-    renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS')
-
-    while len(variables) > 0:
-        for name in tuple(variables):
-            value = notdone[name]
-            m1 = re.search(_findvar1_rx, value)
-            m2 = re.search(_findvar2_rx, value)
-            if m1 and m2:
-                m = m1 if m1.start() < m2.start() else m2
-            else:
-                m = m1 if m1 else m2
-            if m is not None:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-
-                elif n in renamed_variables:
-                    if (name.startswith('PY_') and
-                        name[3:] in renamed_variables):
-                        item = ""
-
-                    elif 'PY_' + n in notdone:
-                        found = False
-
-                    else:
-                        item = str(done['PY_' + n])
-
-                else:
-                    done[n] = item = ""
-
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try:
-                            if name in _ALWAYS_STR:
-                                raise ValueError
-                            value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        variables.remove(name)
-
-                        if name.startswith('PY_') \
-                        and name[3:] in renamed_variables:
-
-                            name = name[3:]
-                            if name not in done:
-                                done[name] = value
-
-            else:
-                # Adds unresolved variables to the done dict.
-                # This is disabled when called from distutils.sysconfig
-                if keep_unresolved:
-                    done[name] = value
-                # bogus variable reference (e.g. "prefix=$/opt/python");
-                # just drop it since we can't deal
-                variables.remove(name)
-
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    vars.update(done)
-    return vars
-
-
 def get_makefile_filename():
     """Return the path of the Makefile."""
     if _PYTHON_BUILD:
@@ -462,91 +341,44 @@ def _get_sysconfigdata_name():
         f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}',
     )
 
-
-def _generate_posix_vars():
-    """Generate the Python module containing build-time variables."""
-    import pprint
-    vars = {}
-    # load the installed Makefile:
-    makefile = get_makefile_filename()
-    try:
-        _parse_makefile(makefile, vars)
-    except OSError as e:
-        msg = f"invalid Python installation: unable to open {makefile}"
-        if hasattr(e, "strerror"):
-            msg = f"{msg} ({e.strerror})"
-        raise OSError(msg)
-    # load the installed pyconfig.h:
-    config_h = get_config_h_filename()
-    try:
-        with open(config_h, encoding="utf-8") as f:
-            parse_config_h(f, vars)
-    except OSError as e:
-        msg = f"invalid Python installation: unable to open {config_h}"
-        if hasattr(e, "strerror"):
-            msg = f"{msg} ({e.strerror})"
-        raise OSError(msg)
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if _PYTHON_BUILD:
-        vars['BLDSHARED'] = vars['LDSHARED']
-
-    # There's a chicken-and-egg situation on OS X with regards to the
-    # _sysconfigdata module after the changes introduced by #15298:
-    # get_config_vars() is called by get_platform() as part of the
-    # `make pybuilddir.txt` target -- which is a precursor to the
-    # _sysconfigdata.py module being constructed.  Unfortunately,
-    # get_config_vars() eventually calls _init_posix(), which attempts
-    # to import _sysconfigdata, which we won't have built yet.  In order
-    # for _init_posix() to work, if we're on Darwin, just mock up the
-    # _sysconfigdata module manually and populate it with the build vars.
-    # This is more than sufficient for ensuring the subsequent call to
-    # get_platform() succeeds.
-    name = _get_sysconfigdata_name()
-    if 'darwin' in sys.platform:
-        import types
-        module = types.ModuleType(name)
-        module.build_time_vars = vars
-        sys.modules[name] = module
-
-    pybuilddir = f'build/lib.{get_platform()}-{_PY_VERSION_SHORT}'
-    if hasattr(sys, "gettotalrefcount"):
-        pybuilddir += '-pydebug'
-    os.makedirs(pybuilddir, exist_ok=True)
-    destfile = os.path.join(pybuilddir, name + '.py')
-
-    with open(destfile, 'w', encoding='utf8') as f:
-        f.write('# system configuration generated and used by'
-                ' the sysconfig module\n')
-        f.write('build_time_vars = ')
-        pprint.pprint(vars, stream=f)
-
-    # Create file used for sys.path fixup -- see Modules/getpath.c
-    with open('pybuilddir.txt', 'w', encoding='utf8') as f:
-        f.write(pybuilddir)
-
 def _init_posix(vars):
     """Initialize the module as appropriate for POSIX systems."""
     # _sysconfigdata is generated at build time, see _generate_posix_vars()
     name = _get_sysconfigdata_name()
-    _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0)
+
+    # For cross builds, the path to the target's sysconfigdata must be specified
+    # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in
+    # sys.path can cause crashes when loaded by the host interpreter.
+    # Rely on truthiness as a valueless env variable is still an empty string.
+    # See OS X note in _generate_posix_vars re _sysconfigdata.
+    if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')):
+        from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES
+        from importlib.util import module_from_spec
+        spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name)
+        _temp = module_from_spec(spec)
+        spec.loader.exec_module(_temp)
+    else:
+        _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0)
     build_time_vars = _temp.build_time_vars
     vars.update(build_time_vars)
 
 def _init_non_posix(vars):
     """Initialize the module as appropriate for NT"""
     # set basic install directories
-    import _imp
+    import _winapi
+    import _sysconfig
     vars['LIBDEST'] = get_path('stdlib')
     vars['BINLIBDEST'] = get_path('platstdlib')
     vars['INCLUDEPY'] = get_path('include')
-    try:
-        # GH-99201: _imp.extension_suffixes may be empty when
-        # HAVE_DYNAMIC_LOADING is not set. In this case, don't set EXT_SUFFIX.
-        vars['EXT_SUFFIX'] = _imp.extension_suffixes()[0]
-    except IndexError:
-        pass
+
+    # Add EXT_SUFFIX, SOABI, and Py_GIL_DISABLED
+    vars.update(_sysconfig.config_vars())
+
+    vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs'))
+    if hasattr(sys, 'dllhandle'):
+        dllhandle = _winapi.GetModuleFileName(sys.dllhandle)
+        vars['LIBRARY'] = os.path.basename(_safe_realpath(dllhandle))
+        vars['LDLIBRARY'] = vars['LIBRARY']
     vars['EXE'] = '.exe'
     vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT
     vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable))
@@ -595,7 +427,7 @@ def get_config_h_filename():
     """Return the path of pyconfig.h."""
     if _PYTHON_BUILD:
         if os.name == "nt":
-            inc_dir = os.path.join(_PROJECT_BASE, "PC")
+            inc_dir = os.path.dirname(sys._base_executable)
         else:
             inc_dir = _PROJECT_BASE
     else:
@@ -633,6 +465,78 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True):
     return get_paths(scheme, vars, expand)[name]
 
 
+def _init_config_vars():
+    global _CONFIG_VARS
+    _CONFIG_VARS = {}
+    # Normalized versions of prefix and exec_prefix are handy to have;
+    # in fact, these are the standard versions used most places in the
+    # Distutils.
+    _PREFIX = os.path.normpath(sys.prefix)
+    _EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+    _CONFIG_VARS['prefix'] = _PREFIX  # FIXME: This gets overwriten by _init_posix.
+    _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX  # FIXME: This gets overwriten by _init_posix.
+    _CONFIG_VARS['py_version'] = _PY_VERSION
+    _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT
+    _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT
+    _CONFIG_VARS['installed_base'] = _BASE_PREFIX
+    _CONFIG_VARS['base'] = _PREFIX
+    _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX
+    _CONFIG_VARS['platbase'] = _EXEC_PREFIX
+    _CONFIG_VARS['projectbase'] = _PROJECT_BASE
+    _CONFIG_VARS['platlibdir'] = sys.platlibdir
+    _CONFIG_VARS['implementation'] = _get_implementation()
+    _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
+    try:
+        _CONFIG_VARS['abiflags'] = sys.abiflags
+    except AttributeError:
+        # sys.abiflags may not be defined on all platforms.
+        _CONFIG_VARS['abiflags'] = ''
+    try:
+        _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '')
+    except AttributeError:
+        _CONFIG_VARS['py_version_nodot_plat'] = ''
+
+    if os.name == 'nt':
+        _init_non_posix(_CONFIG_VARS)
+        _CONFIG_VARS['VPATH'] = sys._vpath
+    if os.name == 'posix':
+        _init_posix(_CONFIG_VARS)
+    if _HAS_USER_BASE:
+        # Setting 'userbase' is done below the call to the
+        # init function to enable using 'get_config_var' in
+        # the init-function.
+        _CONFIG_VARS['userbase'] = _getuserbase()
+
+    # e.g., 't' for free-threaded or '' for default build
+    _CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else ''
+
+    # Always convert srcdir to an absolute path
+    srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE)
+    if os.name == 'posix':
+        if _PYTHON_BUILD:
+            # If srcdir is a relative path (typically '.' or '..')
+            # then it should be interpreted relative to the directory
+            # containing Makefile.
+            base = os.path.dirname(get_makefile_filename())
+            srcdir = os.path.join(base, srcdir)
+        else:
+            # srcdir is not meaningful since the installation is
+            # spread about the filesystem.  We choose the
+            # directory containing the Makefile since we know it
+            # exists.
+            srcdir = os.path.dirname(get_makefile_filename())
+    _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir)
+
+    # OS X platforms require special customization to handle
+    # multi-architecture, multi-os-version installers
+    if sys.platform == 'darwin':
+        import _osx_support
+        _osx_support.customize_config_vars(_CONFIG_VARS)
+
+    global _CONFIG_VARS_INITIALIZED
+    _CONFIG_VARS_INITIALIZED = True
+
+
 def get_config_vars(*args):
     """With no arguments, return a dictionary of all configuration
     variables relevant for the current platform.
@@ -643,66 +547,26 @@ def get_config_vars(*args):
     With arguments, return a list of values that result from looking up
     each argument in the configuration variable dictionary.
     """
-    global _CONFIG_VARS
-    if _CONFIG_VARS is None:
-        _CONFIG_VARS = {}
-        # Normalized versions of prefix and exec_prefix are handy to have;
-        # in fact, these are the standard versions used most places in the
-        # Distutils.
-        _CONFIG_VARS['prefix'] = _PREFIX
-        _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX
-        _CONFIG_VARS['py_version'] = _PY_VERSION
-        _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT
-        _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT
-        _CONFIG_VARS['installed_base'] = _BASE_PREFIX
-        _CONFIG_VARS['base'] = _PREFIX
-        _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX
-        _CONFIG_VARS['platbase'] = _EXEC_PREFIX
-        _CONFIG_VARS['projectbase'] = _PROJECT_BASE
-        _CONFIG_VARS['platlibdir'] = sys.platlibdir
-        try:
-            _CONFIG_VARS['abiflags'] = sys.abiflags
-        except AttributeError:
-            # sys.abiflags may not be defined on all platforms.
-            _CONFIG_VARS['abiflags'] = ''
-        try:
-            _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '')
-        except AttributeError:
-            _CONFIG_VARS['py_version_nodot_plat'] = ''
+    global _CONFIG_VARS_INITIALIZED
 
-        if os.name == 'nt':
-            _init_non_posix(_CONFIG_VARS)
-            _CONFIG_VARS['VPATH'] = sys._vpath
-        if os.name == 'posix':
-            _init_posix(_CONFIG_VARS)
-        if _HAS_USER_BASE:
-            # Setting 'userbase' is done below the call to the
-            # init function to enable using 'get_config_var' in
-            # the init-function.
-            _CONFIG_VARS['userbase'] = _getuserbase()
-
-        # Always convert srcdir to an absolute path
-        srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE)
-        if os.name == 'posix':
-            if _PYTHON_BUILD:
-                # If srcdir is a relative path (typically '.' or '..')
-                # then it should be interpreted relative to the directory
-                # containing Makefile.
-                base = os.path.dirname(get_makefile_filename())
-                srcdir = os.path.join(base, srcdir)
-            else:
-                # srcdir is not meaningful since the installation is
-                # spread about the filesystem.  We choose the
-                # directory containing the Makefile since we know it
-                # exists.
-                srcdir = os.path.dirname(get_makefile_filename())
-        _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir)
-
-        # OS X platforms require special customization to handle
-        # multi-architecture, multi-os-version installers
-        if sys.platform == 'darwin':
-            import _osx_support
-            _osx_support.customize_config_vars(_CONFIG_VARS)
+    # Avoid claiming the lock once initialization is complete.
+    if not _CONFIG_VARS_INITIALIZED:
+        with _CONFIG_VARS_LOCK:
+            # Test again with the lock held to avoid races. Note that
+            # we test _CONFIG_VARS here, not _CONFIG_VARS_INITIALIZED,
+            # to ensure that recursive calls to get_config_vars()
+            # don't re-enter init_config_vars().
+            if _CONFIG_VARS is None:
+                _init_config_vars()
+    else:
+        # If the site module initialization happened after _CONFIG_VARS was
+        # initialized, a virtual environment might have been activated, resulting in
+        # variables like sys.prefix changing their value, so we need to re-init the
+        # config vars (see GH-126789).
+        if _CONFIG_VARS['base'] != os.path.normpath(sys.prefix):
+            with _CONFIG_VARS_LOCK:
+                _CONFIG_VARS_INITIALIZED = False
+                _init_config_vars()
 
     if args:
         vals = []
@@ -737,7 +601,8 @@ def get_platform():
        solaris-2.6-sun4u
 
     Windows will return one of:
-       win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
+       win-amd64 (64-bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
+       win-arm64 (64-bit Windows on ARM64 (aka AArch64)
        win32 (all others - specifically, sys.platform is returned)
 
     For other non-POSIX platforms, currently just returns 'sys.platform'.
@@ -770,10 +635,22 @@ def get_platform():
     machine = machine.replace('/', '-')
 
     if osname[:5] == "linux":
-        # At least on Linux/Intel, 'machine' is the processor --
-        # i386, etc.
-        # XXX what about Alpha, SPARC, etc?
-        return  f"{osname}-{machine}"
+        if sys.platform == "android":
+            osname = "android"
+            release = get_config_var("ANDROID_API_LEVEL")
+
+            # Wheel tags use the ABI names from Android's own tools.
+            machine = {
+                "x86_64": "x86_64",
+                "i686": "x86",
+                "aarch64": "arm64_v8a",
+                "armv7l": "armeabi_v7a",
+            }[machine]
+        else:
+            # At least on Linux/Intel, 'machine' is the processor --
+            # i386, etc.
+            # XXX what about Alpha, SPARC, etc?
+            return  f"{osname}-{machine}"
     elif osname[:5] == "sunos":
         if release[0] >= "5":           # SunOS 5 == Solaris 2
             osname = "solaris"
@@ -795,10 +672,15 @@ def get_platform():
         if m:
             release = m.group()
     elif osname[:6] == "darwin":
-        import _osx_support
-        osname, release, machine = _osx_support.get_platform_osx(
-                                            get_config_vars(),
-                                            osname, release, machine)
+        if sys.platform == "ios":
+            release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0")
+            osname = sys.platform
+            machine = sys.implementation._multiarch
+        else:
+            import _osx_support
+            osname, release, machine = _osx_support.get_platform_osx(
+                                                get_config_vars(),
+                                                osname, release, machine)
 
     return f"{osname}-{release}-{machine}"
 
@@ -807,6 +689,10 @@ def get_python_version():
     return _PY_VERSION_SHORT
 
 
+def _get_python_version_abi():
+    return _PY_VERSION_SHORT + get_config_var("abi_thread")
+
+
 def expand_makefile_vars(s, vars):
     """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
     'string' according to 'vars' (a dictionary mapping variable names to
@@ -817,6 +703,9 @@ def expand_makefile_vars(s, vars):
     """
     import re
 
+    _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)"
+    _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}"
+
     # This algorithm does multiple expansion, so if vars['foo'] contains
     # "${bar}", it will expand ${foo} to ${bar}, and then expand
     # ${bar}... and so forth.  This is fine as long as 'vars' comes from
@@ -831,28 +720,3 @@ def expand_makefile_vars(s, vars):
         else:
             break
     return s
-
-
-def _print_dict(title, data):
-    for index, (key, value) in enumerate(sorted(data.items())):
-        if index == 0:
-            print(f'{title}: ')
-        print(f'\t{key} = "{value}"')
-
-
-def _main():
-    """Display all information sysconfig detains."""
-    if '--generate-posix-vars' in sys.argv:
-        _generate_posix_vars()
-        return
-    print(f'Platform: "{get_platform()}"')
-    print(f'Python version: "{get_python_version()}"')
-    print(f'Current installation scheme: "{get_default_scheme()}"')
-    print()
-    _print_dict('Paths', get_paths())
-    print()
-    _print_dict('Variables', get_config_vars())
-
-if __name__ == '__main__':
-    _main()
-'''.replace("Python", "RustPython").replace("/python", "/rustpython"))

Lib/sysconfig/__main__.py

@@ -0,0 +1,248 @@
+import os
+import sys
+from sysconfig import (
+    _ALWAYS_STR,
+    _PYTHON_BUILD,
+    _get_sysconfigdata_name,
+    get_config_h_filename,
+    get_config_vars,
+    get_default_scheme,
+    get_makefile_filename,
+    get_paths,
+    get_platform,
+    get_python_version,
+    parse_config_h,
+)
+
+
+# Regexes needed for parsing Makefile (and similar syntaxes,
+# like old-style Setup files).
+_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)"
+_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)"
+_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}"
+
+
+def _parse_makefile(filename, vars=None, keep_unresolved=True):
+    """Parse a Makefile-style file.
+
+    A dictionary containing name/value pairs is returned.  If an
+    optional dictionary is passed in as the second argument, it is
+    used instead of a new dictionary.
+    """
+    import re
+
+    if vars is None:
+        vars = {}
+    done = {}
+    notdone = {}
+
+    with open(filename, encoding=sys.getfilesystemencoding(),
+              errors="surrogateescape") as f:
+        lines = f.readlines()
+
+    for line in lines:
+        if line.startswith('#') or line.strip() == '':
+            continue
+        m = re.match(_variable_rx, line)
+        if m:
+            n, v = m.group(1, 2)
+            v = v.strip()
+            # `$$' is a literal `$' in make
+            tmpv = v.replace('$$', '')
+
+            if "$" in tmpv:
+                notdone[n] = v
+            else:
+                try:
+                    if n in _ALWAYS_STR:
+                        raise ValueError
+
+                    v = int(v)
+                except ValueError:
+                    # insert literal `$'
+                    done[n] = v.replace('$$', '$')
+                else:
+                    done[n] = v
+
+    # do variable interpolation here
+    variables = list(notdone.keys())
+
+    # Variables with a 'PY_' prefix in the makefile. These need to
+    # be made available without that prefix through sysconfig.
+    # Special care is needed to ensure that variable expansion works, even
+    # if the expansion uses the name without a prefix.
+    renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS')
+
+    while len(variables) > 0:
+        for name in tuple(variables):
+            value = notdone[name]
+            m1 = re.search(_findvar1_rx, value)
+            m2 = re.search(_findvar2_rx, value)
+            if m1 and m2:
+                m = m1 if m1.start() < m2.start() else m2
+            else:
+                m = m1 if m1 else m2
+            if m is not None:
+                n = m.group(1)
+                found = True
+                if n in done:
+                    item = str(done[n])
+                elif n in notdone:
+                    # get it on a subsequent round
+                    found = False
+                elif n in os.environ:
+                    # do it like make: fall back to environment
+                    item = os.environ[n]
+
+                elif n in renamed_variables:
+                    if (name.startswith('PY_') and
+                        name[3:] in renamed_variables):
+                        item = ""
+
+                    elif 'PY_' + n in notdone:
+                        found = False
+
+                    else:
+                        item = str(done['PY_' + n])
+
+                else:
+                    done[n] = item = ""
+
+                if found:
+                    after = value[m.end():]
+                    value = value[:m.start()] + item + after
+                    if "$" in after:
+                        notdone[name] = value
+                    else:
+                        try:
+                            if name in _ALWAYS_STR:
+                                raise ValueError
+                            value = int(value)
+                        except ValueError:
+                            done[name] = value.strip()
+                        else:
+                            done[name] = value
+                        variables.remove(name)
+
+                        if name.startswith('PY_') \
+                        and name[3:] in renamed_variables:
+
+                            name = name[3:]
+                            if name not in done:
+                                done[name] = value
+
+            else:
+                # Adds unresolved variables to the done dict.
+                # This is disabled when called from distutils.sysconfig
+                if keep_unresolved:
+                    done[name] = value
+                # bogus variable reference (e.g. "prefix=$/opt/python");
+                # just drop it since we can't deal
+                variables.remove(name)
+
+    # strip spurious spaces
+    for k, v in done.items():
+        if isinstance(v, str):
+            done[k] = v.strip()
+
+    # save the results in the global dictionary
+    vars.update(done)
+    return vars
+
+
+def _print_config_dict(d, stream):
+    print ("{", file=stream)
+    for k, v in sorted(d.items()):
+        print(f"    {k!r}: {v!r},", file=stream)
+    print ("}", file=stream)
+
+
+def _generate_posix_vars():
+    """Generate the Python module containing build-time variables."""
+    vars = {}
+    # load the installed Makefile:
+    makefile = get_makefile_filename()
+    try:
+        _parse_makefile(makefile, vars)
+    except OSError as e:
+        msg = f"invalid Python installation: unable to open {makefile}"
+        if hasattr(e, "strerror"):
+            msg = f"{msg} ({e.strerror})"
+        raise OSError(msg)
+    # load the installed pyconfig.h:
+    config_h = get_config_h_filename()
+    try:
+        with open(config_h, encoding="utf-8") as f:
+            parse_config_h(f, vars)
+    except OSError as e:
+        msg = f"invalid Python installation: unable to open {config_h}"
+        if hasattr(e, "strerror"):
+            msg = f"{msg} ({e.strerror})"
+        raise OSError(msg)
+    # On AIX, there are wrong paths to the linker scripts in the Makefile
+    # -- these paths are relative to the Python source, but when installed
+    # the scripts are in another directory.
+    if _PYTHON_BUILD:
+        vars['BLDSHARED'] = vars['LDSHARED']
+
+    # There's a chicken-and-egg situation on OS X with regards to the
+    # _sysconfigdata module after the changes introduced by #15298:
+    # get_config_vars() is called by get_platform() as part of the
+    # `make pybuilddir.txt` target -- which is a precursor to the
+    # _sysconfigdata.py module being constructed.  Unfortunately,
+    # get_config_vars() eventually calls _init_posix(), which attempts
+    # to import _sysconfigdata, which we won't have built yet.  In order
+    # for _init_posix() to work, if we're on Darwin, just mock up the
+    # _sysconfigdata module manually and populate it with the build vars.
+    # This is more than sufficient for ensuring the subsequent call to
+    # get_platform() succeeds.
+    name = _get_sysconfigdata_name()
+    if 'darwin' in sys.platform:
+        import types
+        module = types.ModuleType(name)
+        module.build_time_vars = vars
+        sys.modules[name] = module
+
+    pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}'
+    if hasattr(sys, "gettotalrefcount"):
+        pybuilddir += '-pydebug'
+    os.makedirs(pybuilddir, exist_ok=True)
+    destfile = os.path.join(pybuilddir, name + '.py')
+
+    with open(destfile, 'w', encoding='utf8') as f:
+        f.write('# system configuration generated and used by'
+                ' the sysconfig module\n')
+        f.write('build_time_vars = ')
+        _print_config_dict(vars, stream=f)
+
+    # Create file used for sys.path fixup -- see Modules/getpath.c
+    with open('pybuilddir.txt', 'w', encoding='utf8') as f:
+        f.write(pybuilddir)
+
+
+def _print_dict(title, data):
+    for index, (key, value) in enumerate(sorted(data.items())):
+        if index == 0:
+            print(f'{title}: ')
+        print(f'\t{key} = "{value}"')
+
+
+def _main():
+    """Display all information sysconfig detains."""
+    if '--generate-posix-vars' in sys.argv:
+        _generate_posix_vars()
+        return
+    print(f'Platform: "{get_platform()}"')
+    print(f'Python version: "{get_python_version()}"')
+    print(f'Current installation scheme: "{get_default_scheme()}"')
+    print()
+    _print_dict('Paths', get_paths())
+    print()
+    _print_dict('Variables', get_config_vars())
+
+
+if __name__ == '__main__':
+    try:
+        _main()
+    except BrokenPipeError:
+        pass

Lib/test/archivetestdata/README.md

@@ -0,0 +1,36 @@
+# Test data for `test_zipfile`, `test_tarfile` (and even some others)
+
+## `test_zipfile`
+
+The test executables in this directory are created manually from `header.sh` and
+the `testdata_module_inside_zip.py` file.  You must have Info-ZIP's zip utility
+installed (`apt install zip` on Debian).
+
+### Purpose of `exe_with_zip` and `exe_with_z64`
+
+These are used to test executable files with an appended zipfile, in a scenario
+where the executable is _not_ a Python interpreter itself so our automatic
+zipimport machinery (that'd look for `__main__.py`) is not being used.
+
+### Updating the test executables
+
+If you update header.sh or the testdata_module_inside_zip.py file, rerun the
+commands below.  These are expected to be rarely changed, if ever.
+
+#### Standard old format (2.0) zip file
+
+```
+zip -0 zip2.zip testdata_module_inside_zip.py
+cat header.sh zip2.zip >exe_with_zip
+rm zip2.zip
+```
+
+#### Modern format (4.5) zip64 file
+
+Redirecting from stdin forces Info-ZIP's zip tool to create a zip64.
+
+```
+zip -0 <testdata_module_inside_zip.py >zip64.zip
+cat header.sh zip64.zip >exe_with_z64
+rm zip64.zip
+```

Lib/test/archivetestdata/header.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+INTERPRETER_UNDER_TEST="$1"
+if [[ ! -x "${INTERPRETER_UNDER_TEST}" ]]; then
+    echo "Interpreter must be the command line argument."
+    exit 4
+fi
+EXECUTABLE="$0" exec "${INTERPRETER_UNDER_TEST}" -E - <<END_OF_PYTHON
+import os
+import zipfile
+
+namespace = {}
+
+filename = os.environ['EXECUTABLE']
+print(f'Opening {filename} as a zipfile.')
+with zipfile.ZipFile(filename, mode='r') as exe_zip:
+  for file_info in exe_zip.infolist():
+    data = exe_zip.read(file_info)
+    exec(data, namespace, namespace)
+    break  # Only use the first file in the archive.
+
+print('Favorite number in executable:', namespace["FAVORITE_NUMBER"])
+
+### Archive contents will be appended after this file. ###
+END_OF_PYTHON

Lib/test/archivetestdata/testdata_module_inside_zip.py

@@ -0,0 +1,2 @@
+# Test data file to be stored within a zip file.
+FAVORITE_NUMBER = 5

Lib/test/data/README

@@ -0,0 +1,2 @@
+This empty directory serves as destination for temporary files
+created by some tests, in particular, the test_codecmaps_* tests.

Lib/test/dtracedata/assert_usable.d

@@ -0,0 +1,5 @@
+BEGIN
+{
+    printf("probe: success\n");
+    exit(0);
+}

Lib/test/dtracedata/assert_usable.stp

@@ -0,0 +1,5 @@
+probe begin
+{
+  println("probe: success")
+  exit ()
+}

Lib/test/dtracedata/call_stack.d

@@ -0,0 +1,31 @@
+self int indent;
+
+python$target:::function-entry
+/copyinstr(arg1) == "start"/
+{
+    self->trace = 1;
+}
+
+python$target:::function-entry
+/self->trace/
+{
+    printf("%d\t%*s:", timestamp, 15, probename);
+    printf("%*s", self->indent, "");
+    printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2);
+    self->indent++;
+}
+
+python$target:::function-return
+/self->trace/
+{
+    self->indent--;
+    printf("%d\t%*s:", timestamp, 15, probename);
+    printf("%*s", self->indent, "");
+    printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2);
+}
+
+python$target:::function-return
+/copyinstr(arg1) == "start"/
+{
+    self->trace = 0;
+}

Lib/test/dtracedata/call_stack.d.expected

@@ -0,0 +1,18 @@
+ function-entry:call_stack.py:start:23
+ function-entry: call_stack.py:function_1:1
+ function-entry:  call_stack.py:function_3:9
+function-return:  call_stack.py:function_3:10
+function-return: call_stack.py:function_1:2
+ function-entry: call_stack.py:function_2:5
+ function-entry:  call_stack.py:function_1:1
+ function-entry:   call_stack.py:function_3:9
+function-return:   call_stack.py:function_3:10
+function-return:  call_stack.py:function_1:2
+function-return: call_stack.py:function_2:6
+ function-entry: call_stack.py:function_3:9
+function-return: call_stack.py:function_3:10
+ function-entry: call_stack.py:function_4:13
+function-return: call_stack.py:function_4:14
+ function-entry: call_stack.py:function_5:18
+function-return: call_stack.py:function_5:21
+function-return:call_stack.py:start:28

Lib/test/dtracedata/call_stack.py

@@ -0,0 +1,30 @@
+def function_1():
+    function_3(1, 2)
+
+# Check stacktrace
+def function_2():
+    function_1()
+
+# CALL_FUNCTION_VAR
+def function_3(dummy, dummy2):
+    pass
+
+# CALL_FUNCTION_KW
+def function_4(**dummy):
+    return 1
+    return 2  # unreachable
+
+# CALL_FUNCTION_VAR_KW
+def function_5(dummy, dummy2, **dummy3):
+    if False:
+        return 7
+    return 8
+
+def start():
+    function_1()
+    function_2()
+    function_3(1, 2)
+    function_4(test=42)
+    function_5(*(1, 2), **{"test": 42})
+
+start()

Lib/test/dtracedata/call_stack.stp

@@ -0,0 +1,41 @@
+global tracing
+
+function basename:string(path:string)
+{
+    last_token = token = tokenize(path, "/");
+    while (token != "") {
+        last_token = token;
+        token = tokenize("", "/");
+    }
+    return last_token;
+}
+
+probe process.mark("function__entry")
+{
+    funcname = user_string($arg2);
+
+    if (funcname == "start") {
+        tracing = 1;
+    }
+}
+
+probe process.mark("function__entry"), process.mark("function__return")
+{
+    filename = user_string($arg1);
+    funcname = user_string($arg2);
+    lineno = $arg3;
+
+    if (tracing) {
+        printf("%d\t%s:%s:%s:%d\n", gettimeofday_us(), $$name,
+               basename(filename), funcname, lineno);
+    }
+}
+
+probe process.mark("function__return")
+{
+    funcname = user_string($arg2);
+
+    if (funcname == "start") {
+        tracing = 0;
+    }
+}

Lib/test/dtracedata/call_stack.stp.expected

@@ -0,0 +1,14 @@
+function__entry:call_stack.py:start:23
+function__entry:call_stack.py:function_1:1
+function__return:call_stack.py:function_1:2
+function__entry:call_stack.py:function_2:5
+function__entry:call_stack.py:function_1:1
+function__return:call_stack.py:function_1:2
+function__return:call_stack.py:function_2:6
+function__entry:call_stack.py:function_3:9
+function__return:call_stack.py:function_3:10
+function__entry:call_stack.py:function_4:13
+function__return:call_stack.py:function_4:14
+function__entry:call_stack.py:function_5:18
+function__return:call_stack.py:function_5:21
+function__return:call_stack.py:start:28

Lib/test/dtracedata/gc.d

@@ -0,0 +1,18 @@
+python$target:::function-entry
+/copyinstr(arg1) == "start"/
+{
+    self->trace = 1;
+}
+
+python$target:::gc-start,
+python$target:::gc-done
+/self->trace/
+{
+    printf("%d\t%s:%ld\n", timestamp, probename, arg0);
+}
+
+python$target:::function-return
+/copyinstr(arg1) == "start"/
+{
+    self->trace = 0;
+}

Lib/test/dtracedata/gc.d.expected

@@ -0,0 +1,8 @@
+gc-start:0
+gc-done:0
+gc-start:1
+gc-done:0
+gc-start:2
+gc-done:0
+gc-start:2
+gc-done:1

Lib/test/dtracedata/gc.py

@@ -0,0 +1,13 @@
+import gc
+
+def start():
+    gc.collect(0)
+    gc.collect(1)
+    gc.collect(2)
+    l = []
+    l.append(l)
+    del l
+    gc.collect(2)
+
+gc.collect()
+start()

Lib/test/dtracedata/gc.stp

@@ -0,0 +1,26 @@
+global tracing
+
+probe process.mark("function__entry")
+{
+    funcname = user_string($arg2);
+
+    if (funcname == "start") {
+        tracing = 1;
+    }
+}
+
+probe process.mark("gc__start"), process.mark("gc__done")
+{
+    if (tracing) {
+        printf("%d\t%s:%ld\n", gettimeofday_us(), $$name, $arg1);
+    }
+}
+
+probe process.mark("function__return")
+{
+    funcname = user_string($arg2);
+
+    if (funcname == "start") {
+        tracing = 0;
+    }
+}

Lib/test/dtracedata/gc.stp.expected

@@ -0,0 +1,8 @@
+gc__start:0
+gc__done:0
+gc__start:1
+gc__done:0
+gc__start:2
+gc__done:0
+gc__start:2
+gc__done:1

Lib/test/dtracedata/instance.py

@@ -0,0 +1,24 @@
+import gc
+
+class old_style_class():
+    pass
+class new_style_class(object):
+    pass
+
+a = old_style_class()
+del a
+gc.collect()
+b = new_style_class()
+del b
+gc.collect()
+
+a = old_style_class()
+del old_style_class
+gc.collect()
+b = new_style_class()
+del new_style_class
+gc.collect()
+del a
+gc.collect()
+del b
+gc.collect()

Lib/test/dtracedata/line.d

@@ -0,0 +1,7 @@
+python$target:::line
+/(copyinstr(arg1)=="test_line")/
+{
+    printf("%d\t%s:%s:%s:%d\n", timestamp,
+        probename, basename(copyinstr(arg0)),
+        copyinstr(arg1), arg2);
+}

Lib/test/dtracedata/line.d.expected

@@ -0,0 +1,20 @@
+line:line.py:test_line:2
+line:line.py:test_line:3
+line:line.py:test_line:4
+line:line.py:test_line:5
+line:line.py:test_line:6
+line:line.py:test_line:7
+line:line.py:test_line:8
+line:line.py:test_line:9
+line:line.py:test_line:10
+line:line.py:test_line:11
+line:line.py:test_line:4
+line:line.py:test_line:5
+line:line.py:test_line:6
+line:line.py:test_line:7
+line:line.py:test_line:8
+line:line.py:test_line:10
+line:line.py:test_line:11
+line:line.py:test_line:4
+line:line.py:test_line:12
+line:line.py:test_line:13

Lib/test/dtracedata/line.py

@@ -0,0 +1,17 @@
+def test_line():
+    a = 1
+    print('# Preamble', a)
+    for i in range(2):
+        a = i
+        b = i+2
+        c = i+3
+        if c < 4:
+            a = c
+        d = a + b +c
+        print('#', a, b, c, d)
+    a = 1
+    print('# Epilogue', a)
+
+
+if __name__ == '__main__':
+    test_line()

Lib/test/mathdata/formatfloat_testcases.txt

@@ -0,0 +1,355 @@
+-- 'f' code formatting, with explicit precision (>= 0).  Output always
+-- has the given number of places after the point;  zeros are added if
+-- necessary to make this true.
+
+-- zeros
+%.0f 0 -> 0
+%.1f 0 -> 0.0
+%.2f 0 -> 0.00
+%.3f 0 -> 0.000
+%.50f 0 -> 0.00000000000000000000000000000000000000000000000000
+
+-- precision 0;  result should never include a .
+%.0f 1.5 -> 2
+%.0f 2.5 -> 2
+%.0f 3.5 -> 4
+%.0f 0.0 -> 0
+%.0f 0.1 -> 0
+%.0f 0.001 -> 0
+%.0f 10.0 -> 10
+%.0f 10.1 -> 10
+%.0f 10.01 -> 10
+%.0f 123.456 -> 123
+%.0f 1234.56 -> 1235
+%.0f 1e49 -> 9999999999999999464902769475481793196872414789632
+%.0f 9.9999999999999987e+49 -> 99999999999999986860582406952576489172979654066176
+%.0f 1e50 -> 100000000000000007629769841091887003294964970946560
+
+-- precision 1
+%.1f 0.0001 -> 0.0
+%.1f 0.001 -> 0.0
+%.1f 0.01 -> 0.0
+%.1f 0.04 -> 0.0
+%.1f 0.06 -> 0.1
+%.1f 0.25 -> 0.2
+%.1f 0.75 -> 0.8
+%.1f 1.4 -> 1.4
+%.1f 1.5 -> 1.5
+%.1f 10.0 -> 10.0
+%.1f 1000.03 -> 1000.0
+%.1f 1234.5678 -> 1234.6
+%.1f 1234.7499 -> 1234.7
+%.1f 1234.75 -> 1234.8
+
+-- precision 2
+%.2f 0.0001 -> 0.00
+%.2f 0.001 -> 0.00
+%.2f 0.004999 -> 0.00
+%.2f 0.005001 -> 0.01
+%.2f 0.01 -> 0.01
+%.2f 0.125 -> 0.12
+%.2f 0.375 -> 0.38
+%.2f 1234500 -> 1234500.00
+%.2f 1234560 -> 1234560.00
+%.2f 1234567 -> 1234567.00
+%.2f 1234567.8 -> 1234567.80
+%.2f 1234567.89 -> 1234567.89
+%.2f 1234567.891 -> 1234567.89
+%.2f 1234567.8912 -> 1234567.89
+
+-- alternate form always includes a decimal point.  This only
+-- makes a difference when the precision is 0.
+%#.0f 0 -> 0.
+%#.1f 0 -> 0.0
+%#.0f 1.5 -> 2.
+%#.0f 2.5 -> 2.
+%#.0f 10.1 -> 10.
+%#.0f 1234.56 -> 1235.
+%#.1f 1.4 -> 1.4
+%#.2f 0.375 -> 0.38
+
+-- if precision is omitted it defaults to 6
+%f 0 -> 0.000000
+%f 1230000 -> 1230000.000000
+%f 1234567 -> 1234567.000000
+%f 123.4567 -> 123.456700
+%f 1.23456789 -> 1.234568
+%f 0.00012 -> 0.000120
+%f 0.000123 -> 0.000123
+%f 0.00012345 -> 0.000123
+%f 0.000001 -> 0.000001
+%f 0.0000005001 -> 0.000001
+%f 0.0000004999 -> 0.000000
+
+-- 'e' code formatting with explicit precision (>= 0). Output should
+-- always have exactly the number of places after the point that were
+-- requested.
+
+-- zeros
+%.0e 0 -> 0e+00
+%.1e 0 -> 0.0e+00
+%.2e 0 -> 0.00e+00
+%.10e 0 -> 0.0000000000e+00
+%.50e 0 -> 0.00000000000000000000000000000000000000000000000000e+00
+
+-- precision 0.  no decimal point in the output
+%.0e 0.01 -> 1e-02
+%.0e 0.1 -> 1e-01
+%.0e 1 -> 1e+00
+%.0e 10 -> 1e+01
+%.0e 100 -> 1e+02
+%.0e 0.012 -> 1e-02
+%.0e 0.12 -> 1e-01
+%.0e 1.2 -> 1e+00
+%.0e 12 -> 1e+01
+%.0e 120 -> 1e+02
+%.0e 123.456 -> 1e+02
+%.0e 0.000123456 -> 1e-04
+%.0e 123456000 -> 1e+08
+%.0e 0.5 -> 5e-01
+%.0e 1.4 -> 1e+00
+%.0e 1.5 -> 2e+00
+%.0e 1.6 -> 2e+00
+%.0e 2.4999999 -> 2e+00
+%.0e 2.5 -> 2e+00
+%.0e 2.5000001 -> 3e+00
+%.0e 3.499999999999 -> 3e+00
+%.0e 3.5 -> 4e+00
+%.0e 4.5 -> 4e+00
+%.0e 5.5 -> 6e+00
+%.0e 6.5 -> 6e+00
+%.0e 7.5 -> 8e+00
+%.0e 8.5 -> 8e+00
+%.0e 9.4999 -> 9e+00
+%.0e 9.5 -> 1e+01
+%.0e 10.5 -> 1e+01
+%.0e 14.999 -> 1e+01
+%.0e 15 -> 2e+01
+
+-- precision 1
+%.1e 0.0001 -> 1.0e-04
+%.1e 0.001 -> 1.0e-03
+%.1e 0.01 -> 1.0e-02
+%.1e 0.1 -> 1.0e-01
+%.1e 1 -> 1.0e+00
+%.1e 10 -> 1.0e+01
+%.1e 100 -> 1.0e+02
+%.1e 120 -> 1.2e+02
+%.1e 123 -> 1.2e+02
+%.1e 123.4 -> 1.2e+02
+
+-- precision 2
+%.2e 0.00013 -> 1.30e-04
+%.2e 0.000135 -> 1.35e-04
+%.2e 0.0001357 -> 1.36e-04
+%.2e 0.0001 -> 1.00e-04
+%.2e 0.001 -> 1.00e-03
+%.2e 0.01 -> 1.00e-02
+%.2e 0.1 -> 1.00e-01
+%.2e 1 -> 1.00e+00
+%.2e 10 -> 1.00e+01
+%.2e 100 -> 1.00e+02
+%.2e 1000 -> 1.00e+03
+%.2e 1500 -> 1.50e+03
+%.2e 1590 -> 1.59e+03
+%.2e 1598 -> 1.60e+03
+%.2e 1598.7 -> 1.60e+03
+%.2e 1598.76 -> 1.60e+03
+%.2e 9999 -> 1.00e+04
+
+-- omitted precision defaults to 6
+%e 0 -> 0.000000e+00
+%e 165 -> 1.650000e+02
+%e 1234567 -> 1.234567e+06
+%e 12345678 -> 1.234568e+07
+%e 1.1 -> 1.100000e+00
+
+-- alternate form always contains a decimal point.  This only makes
+-- a difference when precision is 0.
+
+%#.0e 0.01 -> 1.e-02
+%#.0e 0.1 -> 1.e-01
+%#.0e 1 -> 1.e+00
+%#.0e 10 -> 1.e+01
+%#.0e 100 -> 1.e+02
+%#.0e 0.012 -> 1.e-02
+%#.0e 0.12 -> 1.e-01
+%#.0e 1.2 -> 1.e+00
+%#.0e 12 -> 1.e+01
+%#.0e 120 -> 1.e+02
+%#.0e 123.456 -> 1.e+02
+%#.0e 0.000123456 -> 1.e-04
+%#.0e 123456000 -> 1.e+08
+%#.0e 0.5 -> 5.e-01
+%#.0e 1.4 -> 1.e+00
+%#.0e 1.5 -> 2.e+00
+%#.0e 1.6 -> 2.e+00
+%#.0e 2.4999999 -> 2.e+00
+%#.0e 2.5 -> 2.e+00
+%#.0e 2.5000001 -> 3.e+00
+%#.0e 3.499999999999 -> 3.e+00
+%#.0e 3.5 -> 4.e+00
+%#.0e 4.5 -> 4.e+00
+%#.0e 5.5 -> 6.e+00
+%#.0e 6.5 -> 6.e+00
+%#.0e 7.5 -> 8.e+00
+%#.0e 8.5 -> 8.e+00
+%#.0e 9.4999 -> 9.e+00
+%#.0e 9.5 -> 1.e+01
+%#.0e 10.5 -> 1.e+01
+%#.0e 14.999 -> 1.e+01
+%#.0e 15 -> 2.e+01
+%#.1e 123.4 -> 1.2e+02
+%#.2e 0.0001357 -> 1.36e-04
+
+-- 'g' code formatting.
+
+-- zeros
+%.0g 0 -> 0
+%.1g 0 -> 0
+%.2g 0 -> 0
+%.3g 0 -> 0
+%.4g 0 -> 0
+%.10g 0 -> 0
+%.50g 0 -> 0
+%.100g 0 -> 0
+
+-- precision 0 doesn't make a lot of sense for the 'g' code (what does
+-- it mean to have no significant digits?); in practice, it's interpreted
+-- as identical to precision 1
+%.0g 1000 -> 1e+03
+%.0g 100 -> 1e+02
+%.0g 10 -> 1e+01
+%.0g 1 -> 1
+%.0g 0.1 -> 0.1
+%.0g 0.01 -> 0.01
+%.0g 1e-3 -> 0.001
+%.0g 1e-4 -> 0.0001
+%.0g 1e-5 -> 1e-05
+%.0g 1e-6 -> 1e-06
+%.0g 12 -> 1e+01
+%.0g 120 -> 1e+02
+%.0g 1.2 -> 1
+%.0g 0.12 -> 0.1
+%.0g 0.012 -> 0.01
+%.0g 0.0012 -> 0.001
+%.0g 0.00012 -> 0.0001
+%.0g 0.000012 -> 1e-05
+%.0g 0.0000012 -> 1e-06
+
+-- precision 1 identical to precision 0
+%.1g 1000 -> 1e+03
+%.1g 100 -> 1e+02
+%.1g 10 -> 1e+01
+%.1g 1 -> 1
+%.1g 0.1 -> 0.1
+%.1g 0.01 -> 0.01
+%.1g 1e-3 -> 0.001
+%.1g 1e-4 -> 0.0001
+%.1g 1e-5 -> 1e-05
+%.1g 1e-6 -> 1e-06
+%.1g 12 -> 1e+01
+%.1g 120 -> 1e+02
+%.1g 1.2 -> 1
+%.1g 0.12 -> 0.1
+%.1g 0.012 -> 0.01
+%.1g 0.0012 -> 0.001
+%.1g 0.00012 -> 0.0001
+%.1g 0.000012 -> 1e-05
+%.1g 0.0000012 -> 1e-06
+
+-- precision 2
+%.2g 1000 -> 1e+03
+%.2g 100 -> 1e+02
+%.2g 10 -> 10
+%.2g 1 -> 1
+%.2g 0.1 -> 0.1
+%.2g 0.01 -> 0.01
+%.2g 0.001 -> 0.001
+%.2g 1e-4 -> 0.0001
+%.2g 1e-5 -> 1e-05
+%.2g 1e-6 -> 1e-06
+%.2g 1234 -> 1.2e+03
+%.2g 123 -> 1.2e+02
+%.2g 12.3 -> 12
+%.2g 1.23 -> 1.2
+%.2g 0.123 -> 0.12
+%.2g 0.0123 -> 0.012
+%.2g 0.00123 -> 0.0012
+%.2g 0.000123 -> 0.00012
+%.2g 0.0000123 -> 1.2e-05
+
+-- bad cases from http://bugs.python.org/issue9980
+%.12g 38210.0 -> 38210
+%.12g 37210.0 -> 37210
+%.12g 36210.0 -> 36210
+
+-- alternate g formatting:  always include decimal point and
+-- exactly <precision> significant digits.
+%#.0g 0 -> 0.
+%#.1g 0 -> 0.
+%#.2g 0 -> 0.0
+%#.3g 0 -> 0.00
+%#.4g 0 -> 0.000
+
+%#.0g 0.2 -> 0.2
+%#.1g 0.2 -> 0.2
+%#.2g 0.2 -> 0.20
+%#.3g 0.2 -> 0.200
+%#.4g 0.2 -> 0.2000
+%#.10g 0.2 -> 0.2000000000
+
+%#.0g 2 -> 2.
+%#.1g 2 -> 2.
+%#.2g 2 -> 2.0
+%#.3g 2 -> 2.00
+%#.4g 2 -> 2.000
+
+%#.0g 20 -> 2.e+01
+%#.1g 20 -> 2.e+01
+%#.2g 20 -> 20.
+%#.3g 20 -> 20.0
+%#.4g 20 -> 20.00
+
+%#.0g 234.56 -> 2.e+02
+%#.1g 234.56 -> 2.e+02
+%#.2g 234.56 -> 2.3e+02
+%#.3g 234.56 -> 235.
+%#.4g 234.56 -> 234.6
+%#.5g 234.56 -> 234.56
+%#.6g 234.56 -> 234.560
+
+-- repr formatting.  Result always includes decimal point and at
+-- least one digit after the point, or an exponent.
+%r 0 -> 0.0
+%r 1 -> 1.0
+
+%r 0.01 -> 0.01
+%r 0.02 -> 0.02
+%r 0.03 -> 0.03
+%r 0.04 -> 0.04
+%r 0.05 -> 0.05
+
+-- values >= 1e16 get an exponent
+%r 10 -> 10.0
+%r 100 -> 100.0
+%r 1e15 -> 1000000000000000.0
+%r 9.999e15 -> 9999000000000000.0
+%r 9999999999999998 -> 9999999999999998.0
+%r 9999999999999999 -> 1e+16
+%r 1e16 -> 1e+16
+%r 1e17 -> 1e+17
+
+-- as do values < 1e-4
+%r 1e-3 -> 0.001
+%r 1.001e-4 -> 0.0001001
+%r 1.0000000000000001e-4 -> 0.0001
+%r 1.000000000000001e-4 -> 0.0001000000000000001
+%r 1.00000000001e-4 -> 0.000100000000001
+%r 1.0000000001e-4 -> 0.00010000000001
+%r 1e-4 -> 0.0001
+%r 0.99999999999999999e-4 -> 0.0001
+%r 0.9999999999999999e-4 -> 9.999999999999999e-05
+%r 0.999999999999e-4 -> 9.99999999999e-05
+%r 0.999e-4 -> 9.99e-05
+%r 1e-5 -> 1e-05

Lib/test/site-packages/README.txt

@@ -0,0 +1,2 @@
+This directory exists so that 3rd party packages can be installed
+here.  Read the source for site.py for more details.

Lib/test/support/__init__.py

@@ -7,7 +7,7 @@ import contextlib
 import dataclasses
 import functools
 import logging
-# import _opcode # TODO: RUSTPYTHON
+import _opcode
 import os
 import re
 import stat
@@ -514,7 +514,12 @@ def has_no_debug_ranges():
     return not bool(config['code_debug_ranges'])
 
 def requires_debug_ranges(reason='requires co_positions / debug_ranges'):
-    return unittest.skipIf(has_no_debug_ranges(), reason)
+    try:
+        skip = has_no_debug_ranges()
+    except unittest.SkipTest as e:
+        skip = True
+        reason = e.args[0] if e.args else reason
+    return unittest.skipIf(skip, reason)
 
 @contextlib.contextmanager
 def suppress_immortalization(suppress=True):

Lib/test/support/bytecode_helper.py

@@ -3,10 +3,26 @@
 import unittest
 import dis
 import io
-from _testinternalcapi import compiler_codegen, optimize_cfg, assemble_code_object
+import opcode
+try:
+    import _testinternalcapi
+except ImportError:
+    _testinternalcapi = None
 
 _UNSPECIFIED = object()
 
+def instructions_with_positions(instrs, co_positions):
+    # Return (instr, positions) pairs from the instrs list and co_positions
+    # iterator. The latter contains items for cache lines and the former
+    # doesn't, so those need to be skipped.
+
+    co_positions = co_positions or iter(())
+    for instr in instrs:
+        yield instr, next(co_positions, ())
+        for _, size, _ in (instr.cache_info or ()):
+            for i in range(size):
+                next(co_positions, ())
+
 class BytecodeTestCase(unittest.TestCase):
     """Custom assertion methods for inspecting bytecode."""
 
@@ -53,16 +69,14 @@ class CompilationStepTestCase(unittest.TestCase):
     class Label:
         pass
 
-    def assertInstructionsMatch(self, actual_, expected_):
-        # get two lists where each entry is a label or
-        # an instruction tuple. Normalize the labels to the
-        # instruction count of the target, and compare the lists.
+    def assertInstructionsMatch(self, actual_seq, expected):
+        # get an InstructionSequence and an expected list, where each
+        # entry is a label or an instruction tuple. Construct an expcted
+        # instruction sequence and compare with the one given.
 
-        self.assertIsInstance(actual_, list)
-        self.assertIsInstance(expected_, list)
-
-        actual = self.normalize_insts(actual_)
-        expected = self.normalize_insts(expected_)
+        self.assertIsInstance(expected, list)
+        actual = actual_seq.get_instructions()
+        expected = self.seq_from_insts(expected).get_instructions()
         self.assertEqual(len(actual), len(expected))
 
         # compare instructions
@@ -72,10 +86,8 @@ class CompilationStepTestCase(unittest.TestCase):
                 continue
             self.assertIsInstance(exp, tuple)
             self.assertIsInstance(act, tuple)
-            # crop comparison to the provided expected values
-            if len(act) > len(exp):
-                act = act[:len(exp)]
-            self.assertEqual(exp, act)
+            idx = max([p[0] for p in enumerate(exp) if p[1] != -1])
+            self.assertEqual(exp[:idx], act[:idx])
 
     def resolveAndRemoveLabels(self, insts):
         idx = 0
@@ -90,54 +102,57 @@ class CompilationStepTestCase(unittest.TestCase):
 
         return res
 
-    def normalize_insts(self, insts):
-        """ Map labels to instruction index.
-            Map opcodes to opnames.
-        """
-        insts = self.resolveAndRemoveLabels(insts)
-        res = []
+    def seq_from_insts(self, insts):
+        labels = {item for item in insts if isinstance(item, self.Label)}
+        for i, lbl in enumerate(labels):
+            lbl.value = i
+
+        seq = _testinternalcapi.new_instruction_sequence()
         for item in insts:
-            assert isinstance(item, tuple)
-            opcode, oparg, *loc = item
-            opcode = dis.opmap.get(opcode, opcode)
-            if isinstance(oparg, self.Label):
-                arg = oparg.value
+            if isinstance(item, self.Label):
+                seq.use_label(item.value)
             else:
-                arg = oparg if opcode in self.HAS_ARG else None
-            opcode = dis.opname[opcode]
-            res.append((opcode, arg, *loc))
-        return res
+                op = item[0]
+                if isinstance(op, str):
+                    op = opcode.opmap[op]
+                arg, *loc = item[1:]
+                if isinstance(arg, self.Label):
+                    arg = arg.value
+                loc = loc + [-1] * (4 - len(loc))
+                seq.addop(op, arg or 0, *loc)
+        return seq
 
-    def complete_insts_info(self, insts):
-        # fill in omitted fields in location, and oparg 0 for ops with no arg.
-        res = []
-        for item in insts:
-            assert isinstance(item, tuple)
-            inst = list(item)
-            opcode = dis.opmap[inst[0]]
-            oparg = inst[1]
-            loc = inst[2:] + [-1] * (6 - len(inst))
-            res.append((opcode, oparg, *loc))
-        return res
+    def check_instructions(self, insts):
+        for inst in insts:
+            if isinstance(inst, self.Label):
+                continue
+            op, arg, *loc = inst
+            if isinstance(op, str):
+                op = opcode.opmap[op]
+            self.assertEqual(op in opcode.hasarg,
+                             arg is not None,
+                             f"{opcode.opname[op]=} {arg=}")
+            self.assertTrue(all(isinstance(l, int) for l in loc))
 
 
+@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi")
 class CodegenTestCase(CompilationStepTestCase):
 
     def generate_code(self, ast):
-        insts, _ = compiler_codegen(ast, "my_file.py", 0)
+        insts, _ = _testinternalcapi.compiler_codegen(ast, "my_file.py", 0)
         return insts
 
 
+@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi")
 class CfgOptimizationTestCase(CompilationStepTestCase):
 
-    def get_optimized(self, insts, consts, nlocals=0):
-        insts = self.normalize_insts(insts)
-        insts = self.complete_insts_info(insts)
-        insts = optimize_cfg(insts, consts, nlocals)
+    def get_optimized(self, seq, consts, nlocals=0):
+        insts = _testinternalcapi.optimize_cfg(seq, consts, nlocals)
         return insts, consts
 
+@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi")
 class AssemblerTestCase(CompilationStepTestCase):
 
     def get_code_object(self, filename, insts, metadata):
-        co = assemble_code_object(filename, insts, metadata)
+        co = _testinternalcapi.assemble_code_object(filename, insts, metadata)
         return co

Lib/test/support/hypothesis_helper.py

@@ -5,6 +5,14 @@ try:
 except ImportError:
     from . import _hypothesis_stubs as hypothesis
 else:
+    # Regrtest changes to use a tempdir as the working directory, so we have
+    # to tell Hypothesis to use the original in order to persist the database.
+    from test.support import has_socket_support
+    from test.support.os_helper import SAVEDCWD
+    from hypothesis.configuration import set_hypothesis_home_dir
+
+    set_hypothesis_home_dir(os.path.join(SAVEDCWD, ".hypothesis"))
+
     # When using the real Hypothesis, we'll configure it to ignore occasional
     # slow tests (avoiding flakiness from random VM slowness in CI).
     hypothesis.settings.register_profile(
@@ -21,7 +29,14 @@ else:
     # of failing examples, and also use a pull-through cache to automatically
     # replay any failing examples discovered in CI.  For details on how this
     # works, see https://hypothesis.readthedocs.io/en/latest/database.html
-    if "CI" not in os.environ:
+    # We only do that if a GITHUB_TOKEN env var is provided, see:
+    # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens
+    # And Python is built with socket support:
+    if (
+        has_socket_support
+        and "CI" not in os.environ
+        and "GITHUB_TOKEN" in os.environ
+    ):
         from hypothesis.database import (
             GitHubArtifactDatabase,
             MultiplexedDatabase,

Lib/test/support/interpreters/__init__.py

@@ -0,0 +1,258 @@
+"""Subinterpreters High Level Module."""
+
+import threading
+import weakref
+import _interpreters
+
+# aliases:
+from _interpreters import (
+    InterpreterError, InterpreterNotFoundError, NotShareableError,
+    is_shareable,
+)
+
+
+__all__ = [
+    'get_current', 'get_main', 'create', 'list_all', 'is_shareable',
+    'Interpreter',
+    'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed',
+    'NotShareableError',
+    'create_queue', 'Queue', 'QueueEmpty', 'QueueFull',
+]
+
+
+_queuemod = None
+
+def __getattr__(name):
+    if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'):
+        global create_queue, Queue, QueueEmpty, QueueFull
+        ns = globals()
+        from .queues import (
+            create as create_queue,
+            Queue, QueueEmpty, QueueFull,
+        )
+        return ns[name]
+    else:
+        raise AttributeError(name)
+
+
+_EXEC_FAILURE_STR = """
+{superstr}
+
+Uncaught in the interpreter:
+
+{formatted}
+""".strip()
+
+class ExecutionFailed(InterpreterError):
+    """An unhandled exception happened during execution.
+
+    This is raised from Interpreter.exec() and Interpreter.call().
+    """
+
+    def __init__(self, excinfo):
+        msg = excinfo.formatted
+        if not msg:
+            if excinfo.type and excinfo.msg:
+                msg = f'{excinfo.type.__name__}: {excinfo.msg}'
+            else:
+                msg = excinfo.type.__name__ or excinfo.msg
+        super().__init__(msg)
+        self.excinfo = excinfo
+
+    def __str__(self):
+        try:
+            formatted = self.excinfo.errdisplay
+        except Exception:
+            return super().__str__()
+        else:
+            return _EXEC_FAILURE_STR.format(
+                superstr=super().__str__(),
+                formatted=formatted,
+            )
+
+
+def create():
+    """Return a new (idle) Python interpreter."""
+    id = _interpreters.create(reqrefs=True)
+    return Interpreter(id, _ownsref=True)
+
+
+def list_all():
+    """Return all existing interpreters."""
+    return [Interpreter(id, _whence=whence)
+            for id, whence in _interpreters.list_all(require_ready=True)]
+
+
+def get_current():
+    """Return the currently running interpreter."""
+    id, whence = _interpreters.get_current()
+    return Interpreter(id, _whence=whence)
+
+
+def get_main():
+    """Return the main interpreter."""
+    id, whence = _interpreters.get_main()
+    assert whence == _interpreters.WHENCE_RUNTIME, repr(whence)
+    return Interpreter(id, _whence=whence)
+
+
+_known = weakref.WeakValueDictionary()
+
+class Interpreter:
+    """A single Python interpreter.
+
+    Attributes:
+
+    "id" - the unique process-global ID number for the interpreter
+    "whence" - indicates where the interpreter was created
+
+    If the interpreter wasn't created by this module
+    then any method that modifies the interpreter will fail,
+    i.e. .close(), .prepare_main(), .exec(), and .call()
+    """
+
+    _WHENCE_TO_STR = {
+       _interpreters.WHENCE_UNKNOWN: 'unknown',
+       _interpreters.WHENCE_RUNTIME: 'runtime init',
+       _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API',
+       _interpreters.WHENCE_CAPI: 'C-API',
+       _interpreters.WHENCE_XI: 'cross-interpreter C-API',
+       _interpreters.WHENCE_STDLIB: '_interpreters module',
+    }
+
+    def __new__(cls, id, /, _whence=None, _ownsref=None):
+        # There is only one instance for any given ID.
+        if not isinstance(id, int):
+            raise TypeError(f'id must be an int, got {id!r}')
+        id = int(id)
+        if _whence is None:
+            if _ownsref:
+                _whence = _interpreters.WHENCE_STDLIB
+            else:
+                _whence = _interpreters.whence(id)
+        assert _whence in cls._WHENCE_TO_STR, repr(_whence)
+        if _ownsref is None:
+            _ownsref = (_whence == _interpreters.WHENCE_STDLIB)
+        try:
+            self = _known[id]
+            assert hasattr(self, '_ownsref')
+        except KeyError:
+            self = super().__new__(cls)
+            _known[id] = self
+            self._id = id
+            self._whence = _whence
+            self._ownsref = _ownsref
+            if _ownsref:
+                # This may raise InterpreterNotFoundError:
+                _interpreters.incref(id)
+        return self
+
+    def __repr__(self):
+        return f'{type(self).__name__}({self.id})'
+
+    def __hash__(self):
+        return hash(self._id)
+
+    def __del__(self):
+        self._decref()
+
+    # for pickling:
+    def __getnewargs__(self):
+        return (self._id,)
+
+    # for pickling:
+    def __getstate__(self):
+        return None
+
+    def _decref(self):
+        if not self._ownsref:
+            return
+        self._ownsref = False
+        try:
+            _interpreters.decref(self._id)
+        except InterpreterNotFoundError:
+            pass
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def whence(self):
+        return self._WHENCE_TO_STR[self._whence]
+
+    def is_running(self):
+        """Return whether or not the identified interpreter is running."""
+        return _interpreters.is_running(self._id)
+
+    # Everything past here is available only to interpreters created by
+    # interpreters.create().
+
+    def close(self):
+        """Finalize and destroy the interpreter.
+
+        Attempting to destroy the current interpreter results
+        in an InterpreterError.
+        """
+        return _interpreters.destroy(self._id, restrict=True)
+
+    def prepare_main(self, ns=None, /, **kwargs):
+        """Bind the given values into the interpreter's __main__.
+
+        The values must be shareable.
+        """
+        ns = dict(ns, **kwargs) if ns is not None else kwargs
+        _interpreters.set___main___attrs(self._id, ns, restrict=True)
+
+    def exec(self, code, /):
+        """Run the given source code in the interpreter.
+
+        This is essentially the same as calling the builtin "exec"
+        with this interpreter, using the __dict__ of its __main__
+        module as both globals and locals.
+
+        There is no return value.
+
+        If the code raises an unhandled exception then an ExecutionFailed
+        exception is raised, which summarizes the unhandled exception.
+        The actual exception is discarded because objects cannot be
+        shared between interpreters.
+
+        This blocks the current Python thread until done.  During
+        that time, the previous interpreter is allowed to run
+        in other threads.
+        """
+        excinfo = _interpreters.exec(self._id, code, restrict=True)
+        if excinfo is not None:
+            raise ExecutionFailed(excinfo)
+
+    def call(self, callable, /):
+        """Call the object in the interpreter with given args/kwargs.
+
+        Only functions that take no arguments and have no closure
+        are supported.
+
+        The return value is discarded.
+
+        If the callable raises an exception then the error display
+        (including full traceback) is send back between the interpreters
+        and an ExecutionFailed exception is raised, much like what
+        happens with Interpreter.exec().
+        """
+        # XXX Support args and kwargs.
+        # XXX Support arbitrary callables.
+        # XXX Support returning the return value (e.g. via pickle).
+        excinfo = _interpreters.call(self._id, callable, restrict=True)
+        if excinfo is not None:
+            raise ExecutionFailed(excinfo)
+
+    def call_in_thread(self, callable, /):
+        """Return a new thread that calls the object in the interpreter.
+
+        The return value and any raised exception are discarded.
+        """
+        def task():
+            self.call(callable)
+        t = threading.Thread(target=task)
+        t.start()
+        return t

Lib/test/support/interpreters/_crossinterp.py

@@ -0,0 +1,102 @@
+"""Common code between queues and channels."""
+
+
+class ItemInterpreterDestroyed(Exception):
+    """Raised when trying to get an item whose interpreter was destroyed."""
+
+
+class classonly:
+    """A non-data descriptor that makes a value only visible on the class.
+
+    This is like the "classmethod" builtin, but does not show up on
+    instances of the class.  It may be used as a decorator.
+    """
+
+    def __init__(self, value):
+        self.value = value
+        self.getter = classmethod(value).__get__
+        self.name = None
+
+    def __set_name__(self, cls, name):
+        if self.name is not None:
+            raise TypeError('already used')
+        self.name = name
+
+    def __get__(self, obj, cls):
+        if obj is not None:
+            raise AttributeError(self.name)
+        # called on the class
+        return self.getter(None, cls)
+
+
+class UnboundItem:
+    """Represents a cross-interpreter item no longer bound to an interpreter.
+
+    An item is unbound when the interpreter that added it to the
+    cross-interpreter container is destroyed.
+    """
+
+    __slots__ = ()
+
+    @classonly
+    def singleton(cls, kind, module, name='UNBOUND'):
+        doc = cls.__doc__.replace('cross-interpreter container', kind)
+        doc = doc.replace('cross-interpreter', kind)
+        subclass = type(
+            f'Unbound{kind.capitalize()}Item',
+            (cls,),
+            dict(
+                _MODULE=module,
+                _NAME=name,
+                __doc__=doc,
+            ),
+        )
+        return object.__new__(subclass)
+
+    _MODULE = __name__
+    _NAME = 'UNBOUND'
+
+    def __new__(cls):
+        raise Exception(f'use {cls._MODULE}.{cls._NAME}')
+
+    def __repr__(self):
+        return f'{self._MODULE}.{self._NAME}'
+#        return f'interpreters.queues.UNBOUND'
+
+
+UNBOUND = object.__new__(UnboundItem)
+UNBOUND_ERROR = object()
+UNBOUND_REMOVE = object()
+
+_UNBOUND_CONSTANT_TO_FLAG = {
+    UNBOUND_REMOVE: 1,
+    UNBOUND_ERROR: 2,
+    UNBOUND: 3,
+}
+_UNBOUND_FLAG_TO_CONSTANT = {v: k
+                             for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()}
+
+
+def serialize_unbound(unbound):
+    op = unbound
+    try:
+        flag = _UNBOUND_CONSTANT_TO_FLAG[op]
+    except KeyError:
+        raise NotImplementedError(f'unsupported unbound replacement op {op!r}')
+    return flag,
+
+
+def resolve_unbound(flag, exctype_destroyed):
+    try:
+        op = _UNBOUND_FLAG_TO_CONSTANT[flag]
+    except KeyError:
+        raise NotImplementedError(f'unsupported unbound replacement op {flag!r}')
+    if op is UNBOUND_REMOVE:
+        # "remove" not possible here
+        raise NotImplementedError
+    elif op is UNBOUND_ERROR:
+        raise exctype_destroyed("item's original interpreter destroyed")
+    elif op is UNBOUND:
+        return UNBOUND
+    else:
+        raise NotImplementedError(repr(op))

Lib/test/support/interpreters/channels.py

@@ -0,0 +1,257 @@
+"""Cross-interpreter Channels High Level Module."""
+
+import time
+import _interpchannels as _channels
+from . import _crossinterp
+
+# aliases:
+from _interpchannels import (
+    ChannelError, ChannelNotFoundError, ChannelClosedError,
+    ChannelEmptyError, ChannelNotEmptyError,
+)
+from ._crossinterp import (
+    UNBOUND_ERROR, UNBOUND_REMOVE,
+)
+
+
+__all__ = [
+    'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE',
+    'create', 'list_all',
+    'SendChannel', 'RecvChannel',
+    'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError',
+    'ItemInterpreterDestroyed',
+]
+
+
+class ItemInterpreterDestroyed(ChannelError,
+                               _crossinterp.ItemInterpreterDestroyed):
+    """Raised from get() and get_nowait()."""
+
+
+UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__)
+
+
+def _serialize_unbound(unbound):
+    if unbound is UNBOUND:
+        unbound = _crossinterp.UNBOUND
+    return _crossinterp.serialize_unbound(unbound)
+
+
+def _resolve_unbound(flag):
+    resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed)
+    if resolved is _crossinterp.UNBOUND:
+        resolved = UNBOUND
+    return resolved
+
+
+def create(*, unbounditems=UNBOUND):
+    """Return (recv, send) for a new cross-interpreter channel.
+
+    The channel may be used to pass data safely between interpreters.
+
+    "unbounditems" sets the default for the send end of the channel.
+    See SendChannel.send() for supported values.  The default value
+    is UNBOUND, which replaces the unbound item when received.
+    """
+    unbound = _serialize_unbound(unbounditems)
+    unboundop, = unbound
+    cid = _channels.create(unboundop)
+    recv, send = RecvChannel(cid), SendChannel(cid, _unbound=unbound)
+    return recv, send
+
+
+def list_all():
+    """Return a list of (recv, send) for all open channels."""
+    return [(RecvChannel(cid), SendChannel(cid, _unbound=unbound))
+            for cid, unbound in _channels.list_all()]
+
+
+class _ChannelEnd:
+    """The base class for RecvChannel and SendChannel."""
+
+    _end = None
+
+    def __new__(cls, cid):
+        self = super().__new__(cls)
+        if self._end == 'send':
+            cid = _channels._channel_id(cid, send=True, force=True)
+        elif self._end == 'recv':
+            cid = _channels._channel_id(cid, recv=True, force=True)
+        else:
+            raise NotImplementedError(self._end)
+        self._id = cid
+        return self
+
+    def __repr__(self):
+        return f'{type(self).__name__}(id={int(self._id)})'
+
+    def __hash__(self):
+        return hash(self._id)
+
+    def __eq__(self, other):
+        if isinstance(self, RecvChannel):
+            if not isinstance(other, RecvChannel):
+                return NotImplemented
+        elif not isinstance(other, SendChannel):
+            return NotImplemented
+        return other._id == self._id
+
+    # for pickling:
+    def __getnewargs__(self):
+        return (int(self._id),)
+
+    # for pickling:
+    def __getstate__(self):
+        return None
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def _info(self):
+        return _channels.get_info(self._id)
+
+    @property
+    def is_closed(self):
+        return self._info.closed
+
+
+_NOT_SET = object()
+
+
+class RecvChannel(_ChannelEnd):
+    """The receiving end of a cross-interpreter channel."""
+
+    _end = 'recv'
+
+    def recv(self, timeout=None, *,
+             _sentinel=object(),
+             _delay=10 / 1000,  # 10 milliseconds
+             ):
+        """Return the next object from the channel.
+
+        This blocks until an object has been sent, if none have been
+        sent already.
+        """
+        if timeout is not None:
+            timeout = int(timeout)
+            if timeout < 0:
+                raise ValueError(f'timeout value must be non-negative')
+            end = time.time() + timeout
+        obj, unboundop = _channels.recv(self._id, _sentinel)
+        while obj is _sentinel:
+            time.sleep(_delay)
+            if timeout is not None and time.time() >= end:
+                raise TimeoutError
+            obj, unboundop = _channels.recv(self._id, _sentinel)
+        if unboundop is not None:
+            assert obj is None, repr(obj)
+            return _resolve_unbound(unboundop)
+        return obj
+
+    def recv_nowait(self, default=_NOT_SET):
+        """Return the next object from the channel.
+
+        If none have been sent then return the default if one
+        is provided or fail with ChannelEmptyError.  Otherwise this
+        is the same as recv().
+        """
+        if default is _NOT_SET:
+            obj, unboundop = _channels.recv(self._id)
+        else:
+            obj, unboundop = _channels.recv(self._id, default)
+        if unboundop is not None:
+            assert obj is None, repr(obj)
+            return _resolve_unbound(unboundop)
+        return obj
+
+    def close(self):
+        _channels.close(self._id, recv=True)
+
+
+class SendChannel(_ChannelEnd):
+    """The sending end of a cross-interpreter channel."""
+
+    _end = 'send'
+
+    def __new__(cls, cid, *, _unbound=None):
+        if _unbound is None:
+            try:
+                op = _channels.get_channel_defaults(cid)
+                _unbound = (op,)
+            except ChannelNotFoundError:
+                _unbound = _serialize_unbound(UNBOUND)
+        self = super().__new__(cls, cid)
+        self._unbound = _unbound
+        return self
+
+    @property
+    def is_closed(self):
+        info = self._info
+        return info.closed or info.closing
+
+    def send(self, obj, timeout=None, *,
+             unbound=None,
+             ):
+        """Send the object (i.e. its data) to the channel's receiving end.
+
+        This blocks until the object is received.
+        """
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True)
+
+    def send_nowait(self, obj, *,
+                    unbound=None,
+                    ):
+        """Send the object to the channel's receiving end.
+
+        If the object is immediately received then return True
+        (else False).  Otherwise this is the same as send().
+        """
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        # XXX Note that at the moment channel_send() only ever returns
+        # None.  This should be fixed when channel_send_wait() is added.
+        # See bpo-32604 and gh-19829.
+        return _channels.send(self._id, obj, unboundop, blocking=False)
+
+    def send_buffer(self, obj, timeout=None, *,
+                    unbound=None,
+                    ):
+        """Send the object's buffer to the channel's receiving end.
+
+        This blocks until the object is received.
+        """
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        _channels.send_buffer(self._id, obj, unboundop,
+                              timeout=timeout, blocking=True)
+
+    def send_buffer_nowait(self, obj, *,
+                           unbound=None,
+                           ):
+        """Send the object's buffer to the channel's receiving end.
+
+        If the object is immediately received then return True
+        (else False).  Otherwise this is the same as send().
+        """
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        return _channels.send_buffer(self._id, obj, unboundop, blocking=False)
+
+    def close(self):
+        _channels.close(self._id, send=True)
+
+
+# XXX This is causing leaks (gh-110318):
+_channels._register_end_types(SendChannel, RecvChannel)

Lib/test/support/interpreters/queues.py

@@ -0,0 +1,313 @@
+"""Cross-interpreter Queues High Level Module."""
+
+import pickle
+import queue
+import time
+import weakref
+import _interpqueues as _queues
+from . import _crossinterp
+
+# aliases:
+from _interpqueues import (
+    QueueError, QueueNotFoundError,
+)
+from ._crossinterp import (
+    UNBOUND_ERROR, UNBOUND_REMOVE,
+)
+
+__all__ = [
+    'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE',
+    'create', 'list_all',
+    'Queue',
+    'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull',
+    'ItemInterpreterDestroyed',
+]
+
+
+class QueueEmpty(QueueError, queue.Empty):
+    """Raised from get_nowait() when the queue is empty.
+
+    It is also raised from get() if it times out.
+    """
+
+
+class QueueFull(QueueError, queue.Full):
+    """Raised from put_nowait() when the queue is full.
+
+    It is also raised from put() if it times out.
+    """
+
+
+class ItemInterpreterDestroyed(QueueError,
+                               _crossinterp.ItemInterpreterDestroyed):
+    """Raised from get() and get_nowait()."""
+
+
+_SHARED_ONLY = 0
+_PICKLED = 1
+
+
+UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__)
+
+
+def _serialize_unbound(unbound):
+    if unbound is UNBOUND:
+        unbound = _crossinterp.UNBOUND
+    return _crossinterp.serialize_unbound(unbound)
+
+
+def _resolve_unbound(flag):
+    resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed)
+    if resolved is _crossinterp.UNBOUND:
+        resolved = UNBOUND
+    return resolved
+
+
+def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND):
+    """Return a new cross-interpreter queue.
+
+    The queue may be used to pass data safely between interpreters.
+
+    "syncobj" sets the default for Queue.put()
+    and Queue.put_nowait().
+
+    "unbounditems" likewise sets the default.  See Queue.put() for
+    supported values.  The default value is UNBOUND, which replaces
+    the unbound item.
+    """
+    fmt = _SHARED_ONLY if syncobj else _PICKLED
+    unbound = _serialize_unbound(unbounditems)
+    unboundop, = unbound
+    qid = _queues.create(maxsize, fmt, unboundop)
+    return Queue(qid, _fmt=fmt, _unbound=unbound)
+
+
+def list_all():
+    """Return a list of all open queues."""
+    return [Queue(qid, _fmt=fmt, _unbound=(unboundop,))
+            for qid, fmt, unboundop in _queues.list_all()]
+
+
+_known_queues = weakref.WeakValueDictionary()
+
+class Queue:
+    """A cross-interpreter queue."""
+
+    def __new__(cls, id, /, *, _fmt=None, _unbound=None):
+        # There is only one instance for any given ID.
+        if isinstance(id, int):
+            id = int(id)
+        else:
+            raise TypeError(f'id must be an int, got {id!r}')
+        if _fmt is None:
+            if _unbound is None:
+                _fmt, op = _queues.get_queue_defaults(id)
+                _unbound = (op,)
+            else:
+                _fmt, _ = _queues.get_queue_defaults(id)
+        elif _unbound is None:
+            _, op = _queues.get_queue_defaults(id)
+            _unbound = (op,)
+        try:
+            self = _known_queues[id]
+        except KeyError:
+            self = super().__new__(cls)
+            self._id = id
+            self._fmt = _fmt
+            self._unbound = _unbound
+            _known_queues[id] = self
+            _queues.bind(id)
+        return self
+
+    def __del__(self):
+        try:
+            _queues.release(self._id)
+        except QueueNotFoundError:
+            pass
+        try:
+            del _known_queues[self._id]
+        except KeyError:
+            pass
+
+    def __repr__(self):
+        return f'{type(self).__name__}({self.id})'
+
+    def __hash__(self):
+        return hash(self._id)
+
+    # for pickling:
+    def __getnewargs__(self):
+        return (self._id,)
+
+    # for pickling:
+    def __getstate__(self):
+        return None
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def maxsize(self):
+        try:
+            return self._maxsize
+        except AttributeError:
+            self._maxsize = _queues.get_maxsize(self._id)
+            return self._maxsize
+
+    def empty(self):
+        return self.qsize() == 0
+
+    def full(self):
+        return _queues.is_full(self._id)
+
+    def qsize(self):
+        return _queues.get_count(self._id)
+
+    def put(self, obj, timeout=None, *,
+            syncobj=None,
+            unbound=None,
+            _delay=10 / 1000,  # 10 milliseconds
+            ):
+        """Add the object to the queue.
+
+        This blocks while the queue is full.
+
+        If "syncobj" is None (the default) then it uses the
+        queue's default, set with create_queue().
+
+        If "syncobj" is false then all objects are supported,
+        at the expense of worse performance.
+
+        If "syncobj" is true then the object must be "shareable".
+        Examples of "shareable" objects include the builtin singletons,
+        str, and memoryview.  One benefit is that such objects are
+        passed through the queue efficiently.
+
+        The key difference, though, is conceptual: the corresponding
+        object returned from Queue.get() will be strictly equivalent
+        to the given obj.  In other words, the two objects will be
+        effectively indistinguishable from each other, even if the
+        object is mutable.  The received object may actually be the
+        same object, or a copy (immutable values only), or a proxy.
+        Regardless, the received object should be treated as though
+        the original has been shared directly, whether or not it
+        actually is.  That's a slightly different and stronger promise
+        than just (initial) equality, which is all "syncobj=False"
+        can promise.
+
+        "unbound" controls the behavior of Queue.get() for the given
+        object if the current interpreter (calling put()) is later
+        destroyed.
+
+        If "unbound" is None (the default) then it uses the
+        queue's default, set with create_queue(),
+        which is usually UNBOUND.
+
+        If "unbound" is UNBOUND_ERROR then get() will raise an
+        ItemInterpreterDestroyed exception if the original interpreter
+        has been destroyed.  This does not otherwise affect the queue;
+        the next call to put() will work like normal, returning the next
+        item in the queue.
+
+        If "unbound" is UNBOUND_REMOVE then the item will be removed
+        from the queue as soon as the original interpreter is destroyed.
+        Be aware that this will introduce an imbalance between put()
+        and get() calls.
+
+        If "unbound" is UNBOUND then it is returned by get() in place
+        of the unbound item.
+        """
+        if syncobj is None:
+            fmt = self._fmt
+        else:
+            fmt = _SHARED_ONLY if syncobj else _PICKLED
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        if timeout is not None:
+            timeout = int(timeout)
+            if timeout < 0:
+                raise ValueError(f'timeout value must be non-negative')
+            end = time.time() + timeout
+        if fmt is _PICKLED:
+            obj = pickle.dumps(obj)
+        while True:
+            try:
+                _queues.put(self._id, obj, fmt, unboundop)
+            except QueueFull as exc:
+                if timeout is not None and time.time() >= end:
+                    raise  # re-raise
+                time.sleep(_delay)
+            else:
+                break
+
+    def put_nowait(self, obj, *, syncobj=None, unbound=None):
+        if syncobj is None:
+            fmt = self._fmt
+        else:
+            fmt = _SHARED_ONLY if syncobj else _PICKLED
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        if fmt is _PICKLED:
+            obj = pickle.dumps(obj)
+        _queues.put(self._id, obj, fmt, unboundop)
+
+    def get(self, timeout=None, *,
+            _delay=10 / 1000,  # 10 milliseconds
+            ):
+        """Return the next object from the queue.
+
+        This blocks while the queue is empty.
+
+        If the next item's original interpreter has been destroyed
+        then the "next object" is determined by the value of the
+        "unbound" argument to put().
+        """
+        if timeout is not None:
+            timeout = int(timeout)
+            if timeout < 0:
+                raise ValueError(f'timeout value must be non-negative')
+            end = time.time() + timeout
+        while True:
+            try:
+                obj, fmt, unboundop = _queues.get(self._id)
+            except QueueEmpty as exc:
+                if timeout is not None and time.time() >= end:
+                    raise  # re-raise
+                time.sleep(_delay)
+            else:
+                break
+        if unboundop is not None:
+            assert obj is None, repr(obj)
+            return _resolve_unbound(unboundop)
+        if fmt == _PICKLED:
+            obj = pickle.loads(obj)
+        else:
+            assert fmt == _SHARED_ONLY
+        return obj
+
+    def get_nowait(self):
+        """Return the next object from the channel.
+
+        If the queue is empty then raise QueueEmpty.  Otherwise this
+        is the same as get().
+        """
+        try:
+            obj, fmt, unboundop = _queues.get(self._id)
+        except QueueEmpty as exc:
+            raise  # re-raise
+        if unboundop is not None:
+            assert obj is None, repr(obj)
+            return _resolve_unbound(unboundop)
+        if fmt == _PICKLED:
+            obj = pickle.loads(obj)
+        else:
+            assert fmt == _SHARED_ONLY
+        return obj
+
+
+_queues._register_heap_types(Queue, QueueEmpty, QueueFull)

Lib/test/test__opcode.py

@@ -0,0 +1,143 @@
+import dis
+from test.support.import_helper import import_module
+import unittest
+import opcode
+
+_opcode = import_module("_opcode")
+from _opcode import stack_effect
+
+
+class OpListTests(unittest.TestCase):
+    def check_bool_function_result(self, func, ops, expected):
+        for op in ops:
+            if isinstance(op, str):
+                op = dis.opmap[op]
+            with self.subTest(opcode=op, func=func):
+                self.assertIsInstance(func(op), bool)
+                self.assertEqual(func(op), expected)
+
+    def test_invalid_opcodes(self):
+        invalid = [-100, -1, 255, 512, 513, 1000]
+        self.check_bool_function_result(_opcode.is_valid, invalid, False)
+        self.check_bool_function_result(_opcode.has_arg, invalid, False)
+        self.check_bool_function_result(_opcode.has_const, invalid, False)
+        self.check_bool_function_result(_opcode.has_name, invalid, False)
+        self.check_bool_function_result(_opcode.has_jump, invalid, False)
+        self.check_bool_function_result(_opcode.has_free, invalid, False)
+        self.check_bool_function_result(_opcode.has_local, invalid, False)
+        self.check_bool_function_result(_opcode.has_exc, invalid, False)
+
+    @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'dis' has no attribute 'opmap'
+    def test_is_valid(self):
+        names = [
+            'CACHE',
+            'POP_TOP',
+            'IMPORT_NAME',
+            'JUMP',
+            'INSTRUMENTED_RETURN_VALUE',
+        ]
+        opcodes = [dis.opmap[opname] for opname in names]
+        self.check_bool_function_result(_opcode.is_valid, opcodes, True)
+
+    @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'dis' has no attribute 'hasarg'
+    def test_oplists(self):
+        def check_function(self, func, expected):
+            for op in [-10, 520]:
+                with self.subTest(opcode=op, func=func):
+                    res = func(op)
+                    self.assertIsInstance(res, bool)
+                    self.assertEqual(res, op in expected)
+
+        check_function(self, _opcode.has_arg, dis.hasarg)
+        check_function(self, _opcode.has_const, dis.hasconst)
+        check_function(self, _opcode.has_name, dis.hasname)
+        check_function(self, _opcode.has_jump, dis.hasjump)
+        check_function(self, _opcode.has_free, dis.hasfree)
+        check_function(self, _opcode.has_local, dis.haslocal)
+        check_function(self, _opcode.has_exc, dis.hasexc)
+
+
+class StackEffectTests(unittest.TestCase):
+    @unittest.expectedFailure # TODO: RUSTPYTHON
+    def test_stack_effect(self):
+        self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1)
+        self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1)
+        self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 1), -1)
+        self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 3), -2)
+        self.assertRaises(ValueError, stack_effect, 30000)
+        # All defined opcodes
+        has_arg = dis.hasarg
+        for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()):
+            if code >= opcode.MIN_INSTRUMENTED_OPCODE:
+                continue
+            with self.subTest(opname=name):
+                stack_effect(code)
+                stack_effect(code, 0)
+        # All not defined opcodes
+        for code in set(range(256)) - set(dis.opmap.values()):
+            with self.subTest(opcode=code):
+                self.assertRaises(ValueError, stack_effect, code)
+                self.assertRaises(ValueError, stack_effect, code, 0)
+
+    @unittest.expectedFailure # TODO: RUSTPYTHON
+    def test_stack_effect_jump(self):
+        FOR_ITER = dis.opmap['FOR_ITER']
+        self.assertEqual(stack_effect(FOR_ITER, 0), 1)
+        self.assertEqual(stack_effect(FOR_ITER, 0, jump=True), 1)
+        self.assertEqual(stack_effect(FOR_ITER, 0, jump=False), 1)
+        JUMP_FORWARD = dis.opmap['JUMP_FORWARD']
+        self.assertEqual(stack_effect(JUMP_FORWARD, 0), 0)
+        self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=True), 0)
+        self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=False), 0)
+        # All defined opcodes
+        has_arg = dis.hasarg
+        has_exc = dis.hasexc
+        has_jump = dis.hasjabs + dis.hasjrel
+        for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()):
+            if code >= opcode.MIN_INSTRUMENTED_OPCODE:
+                continue
+            with self.subTest(opname=name):
+                if code not in has_arg:
+                    common = stack_effect(code)
+                    jump = stack_effect(code, jump=True)
+                    nojump = stack_effect(code, jump=False)
+                else:
+                    common = stack_effect(code, 0)
+                    jump = stack_effect(code, 0, jump=True)
+                    nojump = stack_effect(code, 0, jump=False)
+                if code in has_jump or code in has_exc:
+                    self.assertEqual(common, max(jump, nojump))
+                else:
+                    self.assertEqual(jump, common)
+                    self.assertEqual(nojump, common)
+
+
+class SpecializationStatsTests(unittest.TestCase):
+    def test_specialization_stats(self):
+        stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"]
+        specialized_opcodes = [
+            op.lower()
+            for op in opcode._specializations
+            if opcode._inline_cache_entries.get(op, 0)
+        ]
+        self.assertIn('load_attr', specialized_opcodes)
+        self.assertIn('binary_subscr', specialized_opcodes)
+
+        stats = _opcode.get_specialization_stats()
+        if stats is not None:
+            self.assertIsInstance(stats, dict)
+            self.assertCountEqual(stats.keys(), specialized_opcodes)
+            self.assertCountEqual(
+                stats['load_attr'].keys(),
+                stat_names + ['failure_kinds'])
+            for sn in stat_names:
+                self.assertIsInstance(stats['load_attr'][sn], int)
+            self.assertIsInstance(
+                stats['load_attr']['failure_kinds'],
+                tuple)
+            for v in stats['load_attr']['failure_kinds']:
+                self.assertIsInstance(v, int)
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_ast/test_ast.py

@@ -72,8 +72,7 @@ class AST_Tests(unittest.TestCase):
             # "ast.AST constructor takes 0 positional arguments"
             ast.AST(2)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_AST_fields_NULL_check(self):
         # See: https://github.com/python/cpython/issues/126105
         old_value = ast.AST._fields
@@ -91,8 +90,7 @@ class AST_Tests(unittest.TestCase):
         with self.assertRaisesRegex(AttributeError, msg):
             ast.AST()
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_AST_garbage_collection(self):
         class X:
             pass
@@ -105,8 +103,7 @@ class AST_Tests(unittest.TestCase):
         support.gc_collect()
         self.assertIsNone(ref())
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_snippets(self):
         for input, output, kind in (
             (exec_tests, exec_results, "exec"),
@@ -121,8 +118,7 @@ class AST_Tests(unittest.TestCase):
                 with self.subTest(action="compiling", input=i, kind=kind):
                     compile(ast_tree, "?", kind)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_ast_validation(self):
         # compile() is the only function that calls PyAST_Validate
         snippets_to_validate = exec_tests + single_tests + eval_tests
@@ -130,8 +126,7 @@ class AST_Tests(unittest.TestCase):
             tree = ast.parse(snippet)
             compile(tree, "<string>", "exec")
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_optimization_levels__debug__(self):
         cases = [(-1, "__debug__"), (0, "__debug__"), (1, False), (2, False)]
         for optval, expected in cases:
@@ -147,8 +142,7 @@ class AST_Tests(unittest.TestCase):
                         self.assertIsInstance(res.body[0].value, ast.Name)
                         self.assertEqual(res.body[0].value.id, expected)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_optimization_levels_const_folding(self):
         folded = ("Expr", (1, 0, 1, 5), ("Constant", (1, 0, 1, 5), 3, None))
         not_folded = (
@@ -172,8 +166,7 @@ class AST_Tests(unittest.TestCase):
                     res = to_tuple(tree.body[0])
                     self.assertEqual(res, expected)
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_invalid_position_information(self):
         invalid_linenos = [(10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1)]
 
@@ -198,8 +191,7 @@ class AST_Tests(unittest.TestCase):
                 with self.assertRaises(ValueError):
                     compile(tree, "<string>", "exec")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_compilation_of_ast_nodes_with_default_end_position_values(self):
         tree = ast.Module(
             body=[
@@ -220,8 +212,7 @@ class AST_Tests(unittest.TestCase):
         # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode.
         compile(tree, "<string>", "exec")
 
-    # TODO: RUSTPYTHON; TypeError: required field "end_lineno" missing from alias
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: required field "end_lineno" missing from alias
     def test_negative_locations_for_compile(self):
         # See https://github.com/python/cpython/issues/130775
         alias = ast.alias(name='traceback', lineno=0, col_offset=0)
@@ -328,8 +319,7 @@ class AST_Tests(unittest.TestCase):
                 if isinstance(x, ast.AST):
                     self.assertIs(type(x._fields), tuple)
 
-    # TODO: RUSTPYTHON; type object 'Module' has no attribute '__annotations__'
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; type object 'Module' has no attribute '__annotations__'
     def test_field_attr_existence(self):
         for name, item in ast.__dict__.items():
             # These emit DeprecationWarnings
@@ -356,8 +346,7 @@ class AST_Tests(unittest.TestCase):
                 kwargs[name] = self._construct_ast_class(typ)
         return cls(**kwargs)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_arguments(self):
         x = ast.arguments()
         self.assertEqual(
@@ -406,8 +395,7 @@ class AST_Tests(unittest.TestCase):
         x._fields = 666
         self.assertEqual(x._fields, 666)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_classattrs_deprecated(self):
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", "", DeprecationWarning)
@@ -499,8 +487,7 @@ class AST_Tests(unittest.TestCase):
             ],
         )
 
-    # TODO: RUSTPYTHON; DeprecationWarning not triggered
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered
     def test_classattrs(self):
         with self.assertWarns(DeprecationWarning):
             x = ast.Constant()
@@ -706,8 +693,7 @@ class AST_Tests(unittest.TestCase):
         with assertNumDeprecated():
             self.assertNotIsInstance(Constant(S("42")), Num)
 
-    # TODO: RUSTPYTHON; will be removed in Python 3.14
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; will be removed in Python 3.14
     def test_constant_subclasses_deprecated(self):
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", "", DeprecationWarning)
@@ -774,8 +760,7 @@ class AST_Tests(unittest.TestCase):
         x = ast.Module(body, [])
         self.assertEqual(x.body, body)
 
-    # TODO: RUSTPYTHON; DeprecationWarning not triggered
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered
     def test_nodeclasses(self):
         # Zero arguments constructor explicitly allowed (but deprecated)
         with self.assertWarns(DeprecationWarning):
@@ -827,8 +812,7 @@ class AST_Tests(unittest.TestCase):
         x = ast.Sub()
         self.assertEqual(x._fields, ())
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_invalid_sum(self):
         pos = dict(lineno=2, col_offset=3)
         m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], [])
@@ -836,8 +820,7 @@ class AST_Tests(unittest.TestCase):
             compile(m, "<test>", "exec")
         self.assertIn("but got <ast.expr", str(cm.exception))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_invalid_identifier(self):
         m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))], [])
         ast.fix_missing_locations(m)
@@ -852,8 +835,7 @@ class AST_Tests(unittest.TestCase):
             with self.assertRaisesRegex(TypeError, "invalid type in Constant: type"):
                 compile(e, "<test>", "eval")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_empty_yield_from(self):
         # Issue 16546: yield from value is not optional.
         empty_yield_from = ast.parse("def f():\n yield from g()")
@@ -910,8 +892,7 @@ class AST_Tests(unittest.TestCase):
         attr_b = tree.body[0].decorator_list[0].value
         self.assertEqual(attr_b.end_col_offset, 4)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_ast_asdl_signature(self):
         self.assertEqual(
             ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)"
@@ -926,8 +907,7 @@ class AST_Tests(unittest.TestCase):
         expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}"
         self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions)
 
-    # TODO: RUSTPYTHON; SyntaxError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised
     def test_positional_only_feature_version(self):
         ast.parse("def foo(x, /): ...", feature_version=(3, 8))
         ast.parse("def bar(x=1, /): ...", feature_version=(3, 8))
@@ -943,8 +923,7 @@ class AST_Tests(unittest.TestCase):
         with self.assertRaises(SyntaxError):
             ast.parse("lambda x=1, /: ...", feature_version=(3, 7))
 
-    # TODO: RUSTPYTHON; SyntaxError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised
     def test_assignment_expression_feature_version(self):
         ast.parse("(x := 0)", feature_version=(3, 8))
         with self.assertRaises(SyntaxError):
@@ -954,8 +933,7 @@ class AST_Tests(unittest.TestCase):
         # regression test for gh-115881
         ast.parse("with (x() if y else z()): ...", feature_version=(3, 8))
 
-    # TODO: RUSTPYTHON; SyntaxError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised
     def test_exception_groups_feature_version(self):
         code = dedent("""
         try: ...
@@ -965,8 +943,7 @@ class AST_Tests(unittest.TestCase):
         with self.assertRaises(SyntaxError):
             ast.parse(code, feature_version=(3, 10))
 
-    # TODO: RUSTPYTHON; SyntaxError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised
     def test_type_params_feature_version(self):
         samples = [
             "type X = int",
@@ -979,8 +956,7 @@ class AST_Tests(unittest.TestCase):
                 with self.assertRaises(SyntaxError):
                     ast.parse(sample, feature_version=(3, 11))
 
-    # TODO: RUSTPYTHON; SyntaxError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised
     def test_type_params_default_feature_version(self):
         samples = [
             "type X[*Ts=int] = int",
@@ -999,8 +975,7 @@ class AST_Tests(unittest.TestCase):
         with self.assertRaises(ValueError):
             ast.parse("pass", feature_version=(4, 0))
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_constant_as_name(self):
         for constant in "True", "False", "None":
             expr = ast.Expression(ast.Name(constant, ast.Load()))
@@ -1010,8 +985,7 @@ class AST_Tests(unittest.TestCase):
             ):
                 compile(expr, "<test>", "eval")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_constant_as_unicode_name(self):
         constants = [
             ("True", b"Tru\xe1\xb5\x89"),
@@ -1023,8 +997,7 @@ class AST_Tests(unittest.TestCase):
                 f"identifier field can't represent '{constant[0]}' constant"):
                 ast.parse(constant[1], mode="eval")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_precedence_enum(self):
         class _Precedence(enum.IntEnum):
             """Precedence table that originated from python grammar."""
@@ -1101,8 +1074,7 @@ class AST_Tests(unittest.TestCase):
             with self.assertRaisesRegex(ValueError, f"^{e}$"):
                 compile(tree, "<test>", "exec")
 
-    # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None
     def test_none_checks(self) -> None:
         tests = [
             (ast.alias, "name", "import spam as SPAM"),
@@ -1120,8 +1092,7 @@ class AST_Tests(unittest.TestCase):
 class CopyTests(unittest.TestCase):
     """Test copying and pickling AST nodes."""
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_pickling(self):
         import pickle
 
@@ -1202,8 +1173,7 @@ class CopyTests(unittest.TestCase):
 class ASTHelpers_Test(unittest.TestCase):
     maxDiff = None
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_parse(self):
         a = ast.parse("foo(1 + 1)")
         b = compile("foo(1 + 1)", "<unknown>", "exec", ast.PyCF_ONLY_AST)
@@ -1217,8 +1187,7 @@ class ASTHelpers_Test(unittest.TestCase):
                 ast.literal_eval(r"'\U'")
             self.assertIsNotNone(e.exception.__context__)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_dump(self):
         node = ast.parse('spam(eggs, "and cheese")')
         self.assertEqual(
@@ -1242,8 +1211,7 @@ class ASTHelpers_Test(unittest.TestCase):
             "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])",
         )
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_dump_indent(self):
         node = ast.parse('spam(eggs, "and cheese")')
         self.assertEqual(
@@ -1308,8 +1276,7 @@ Module(
          end_col_offset=24)])""",
         )
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_dump_incomplete(self):
         node = ast.Raise(lineno=3, col_offset=4)
         self.assertEqual(ast.dump(node), "Raise()")
@@ -1377,8 +1344,7 @@ Module(
             "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])",
         )
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_dump_show_empty(self):
         def check_node(node, empty, full, **kwargs):
             with self.subTest(show_empty=False):
@@ -1469,8 +1435,7 @@ Module(
             full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])",
         )
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_copy_location(self):
         src = ast.parse("1 + 1", mode="eval")
         src.body.right = ast.copy_location(ast.Constant(2), src.body.right)
@@ -1491,8 +1456,7 @@ Module(
         self.assertEqual(new.lineno, 1)
         self.assertEqual(new.col_offset, 1)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_fix_missing_locations(self):
         src = ast.parse('write("spam")')
         src.body.append(
@@ -1514,8 +1478,7 @@ Module(
             "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])",
         )
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_increment_lineno(self):
         src = ast.parse("1 + 1", mode="eval")
         self.assertEqual(ast.increment_lineno(src, n=3), src)
@@ -1542,8 +1505,7 @@ Module(
         self.assertEqual(ast.increment_lineno(src).lineno, 2)
         self.assertIsNone(ast.increment_lineno(src).end_lineno)
 
-    # TODO: RUSTPYTHON; IndexError: index out of range
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range
     def test_increment_lineno_on_module(self):
         src = ast.parse(
             dedent("""\
@@ -1565,8 +1527,7 @@ Module(
         self.assertEqual(d.pop("func").id, "foo")
         self.assertEqual(d, {"keywords": [], "args": []})
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_iter_child_nodes(self):
         node = ast.parse("spam(23, 42, eggs='leek')", mode="eval")
         self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4)
@@ -1681,8 +1642,7 @@ Module(
         self.assertRaises(ValueError, ast.literal_eval, "+True")
         self.assertRaises(ValueError, ast.literal_eval, "2+3")
 
-    # TODO: RUSTPYTHON; SyntaxError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised
     def test_literal_eval_str_int_limit(self):
         with support.adjust_int_max_str_digits(4000):
             ast.literal_eval("3" * 4000)  # no error
@@ -1722,8 +1682,7 @@ Module(
         )
         self.assertRaises(ValueError, ast.literal_eval, malformed)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_literal_eval_trailing_ws(self):
         self.assertEqual(ast.literal_eval("    -1"), -1)
         self.assertEqual(ast.literal_eval("\t\t-1"), -1)
@@ -1741,8 +1700,7 @@ Module(
         with self.assertRaisesRegex(ValueError, msg):
             ast.literal_eval(node)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_literal_eval_syntax_errors(self):
         with self.assertRaisesRegex(SyntaxError, "unexpected indent"):
             ast.literal_eval(r"""
@@ -1750,8 +1708,7 @@ Module(
                 (\
             \ """)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_bad_integer(self):
         # issue13436: Bad error message with invalid numeric values
         body = [
@@ -1768,8 +1725,7 @@ Module(
             compile(mod, "test", "exec")
         self.assertIn("invalid integer value: None", str(cm.exception))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_level_as_none(self):
         body = [
             ast.ImportFrom(
@@ -1786,8 +1742,7 @@ Module(
         exec(code, ns)
         self.assertIn("sleep", ns)
 
-    # TODO: RUSTPYTHON
-    @unittest.skip("TODO: RUSTPYTHON; crash")
+    @unittest.skip('TODO: RUSTPYTHON; crash')
     def test_recursion_direct(self):
         e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))
         e.operand = e
@@ -1795,8 +1750,7 @@ Module(
             with support.infinite_recursion():
                 compile(ast.Expression(e), "<test>", "eval")
 
-    # TODO: RUSTPYTHON
-    @unittest.skip("TODO: RUSTPYTHON; crash")
+    @unittest.skip('TODO: RUSTPYTHON; crash')
     def test_recursion_indirect(self):
         e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))
         f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))
@@ -1826,8 +1780,7 @@ class ASTValidatorTests(unittest.TestCase):
         mod = ast.Module([stmt], [])
         self.mod(mod, msg)
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_module(self):
         m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))])
         self.mod(m, "must have Load context", "single")
@@ -1884,8 +1837,7 @@ class ASTValidatorTests(unittest.TestCase):
             "must have Load context",
         )
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_funcdef(self):
         a = ast.arguments([], [], None, [], [], None, [])
         f = ast.FunctionDef("x", a, [], [], None, None, [])
@@ -1906,8 +1858,7 @@ class ASTValidatorTests(unittest.TestCase):
 
         self._check_arguments(fac, self.stmt)
 
-    # TODO: RUSTPYTHON; called `Result::unwrap()` on an `Err` value: StackUnderflow
-    '''
+    @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: class pattern defines no positional sub-patterns (__match_args__ missing)
     def test_funcdef_pattern_matching(self):
         # gh-104799: New fields on FunctionDef should be added at the end
         def matcher(node):
@@ -1932,10 +1883,8 @@ class ASTValidatorTests(unittest.TestCase):
         funcdef = source.body[0]
         self.assertIsInstance(funcdef, ast.FunctionDef)
         self.assertTrue(matcher(funcdef))
-    '''
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_classdef(self):
         def cls(
             bases=None, keywords=None, body=None, decorator_list=None, type_params=None
@@ -1965,15 +1914,13 @@ class ASTValidatorTests(unittest.TestCase):
             cls(decorator_list=[ast.Name("x", ast.Store())]), "must have Load context"
         )
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_delete(self):
         self.stmt(ast.Delete([]), "empty targets on Delete")
         self.stmt(ast.Delete([None]), "None disallowed")
         self.stmt(ast.Delete([ast.Name("x", ast.Load())]), "must have Del context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_assign(self):
         self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign")
         self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed")
@@ -1986,8 +1933,7 @@ class ASTValidatorTests(unittest.TestCase):
             "must have Load context",
         )
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_augassign(self):
         aug = ast.AugAssign(
             ast.Name("x", ast.Load()), ast.Add(), ast.Name("y", ast.Load())
@@ -1998,8 +1944,7 @@ class ASTValidatorTests(unittest.TestCase):
         )
         self.stmt(aug, "must have Load context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_for(self):
         x = ast.Name("x", ast.Store())
         y = ast.Name("y", ast.Load())
@@ -2015,8 +1960,7 @@ class ASTValidatorTests(unittest.TestCase):
         self.stmt(ast.For(x, y, [e], []), "must have Load context")
         self.stmt(ast.For(x, y, [p], [e]), "must have Load context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_while(self):
         self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While")
         self.stmt(
@@ -2030,8 +1974,7 @@ class ASTValidatorTests(unittest.TestCase):
             "must have Load context",
         )
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_if(self):
         self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If")
         i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], [])
@@ -2043,8 +1986,7 @@ class ASTValidatorTests(unittest.TestCase):
         )
         self.stmt(i, "must have Load context")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_with(self):
         p = ast.Pass()
         self.stmt(ast.With([], [p]), "empty items on With")
@@ -2055,8 +1997,7 @@ class ASTValidatorTests(unittest.TestCase):
         i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load()))
         self.stmt(ast.With([i], [p]), "must have Store context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_raise(self):
         r = ast.Raise(None, ast.Constant(3))
         self.stmt(r, "Raise with cause but no exception")
@@ -2065,8 +2006,7 @@ class ASTValidatorTests(unittest.TestCase):
         r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store()))
         self.stmt(r, "must have Load context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_try(self):
         p = ast.Pass()
         t = ast.Try([], [], [], [p])
@@ -2087,8 +2027,7 @@ class ASTValidatorTests(unittest.TestCase):
         t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))])
         self.stmt(t, "must have Load context")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_try_star(self):
         p = ast.Pass()
         t = ast.TryStar([], [], [], [p])
@@ -2109,8 +2048,7 @@ class ASTValidatorTests(unittest.TestCase):
         t = ast.TryStar([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))])
         self.stmt(t, "must have Load context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_assert(self):
         self.stmt(
             ast.Assert(ast.Name("x", ast.Store()), None), "must have Load context"
@@ -2118,36 +2056,30 @@ class ASTValidatorTests(unittest.TestCase):
         assrt = ast.Assert(ast.Name("x", ast.Load()), ast.Name("y", ast.Store()))
         self.stmt(assrt, "must have Load context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_import(self):
         self.stmt(ast.Import([]), "empty names on Import")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_importfrom(self):
         imp = ast.ImportFrom(None, [ast.alias("x", None)], -42)
         self.stmt(imp, "Negative ImportFrom level")
         self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_global(self):
         self.stmt(ast.Global([]), "empty names on Global")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_nonlocal(self):
         self.stmt(ast.Nonlocal([]), "empty names on Nonlocal")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_expr(self):
         e = ast.Expr(ast.Name("x", ast.Store()))
         self.stmt(e, "must have Load context")
 
-    # TODO: RUSTPYTHON
-    @unittest.skip("TODO: RUSTPYTHON; called `Option::unwrap()` on a `None` value")
+    @unittest.skip('TODO: RUSTPYTHON; called `Option::unwrap()` on a `None` value')
     def test_boolop(self):
         b = ast.BoolOp(ast.And(), [])
         self.expr(b, "less than 2 values")
@@ -2158,14 +2090,12 @@ class ASTValidatorTests(unittest.TestCase):
         b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())])
         self.expr(b, "must have Load context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_unaryop(self):
         u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store()))
         self.expr(u, "must have Load context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_lambda(self):
         a = ast.arguments([], [], None, [], [], None, [])
         self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), "must have Load context")
@@ -2175,24 +2105,21 @@ class ASTValidatorTests(unittest.TestCase):
 
         self._check_arguments(fac, self.expr)
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_ifexp(self):
         l = ast.Name("x", ast.Load())
         s = ast.Name("y", ast.Store())
         for args in (s, l, l), (l, s, l), (l, l, s):
             self.expr(ast.IfExp(*args), "must have Load context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_dict(self):
         d = ast.Dict([], [ast.Name("x", ast.Load())])
         self.expr(d, "same number of keys as values")
         d = ast.Dict([ast.Name("x", ast.Load())], [None])
         self.expr(d, "None disallowed")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_set(self):
         self.expr(ast.Set([None]), "None disallowed")
         s = ast.Set([ast.Name("x", ast.Store())])
@@ -2226,23 +2153,19 @@ class ASTValidatorTests(unittest.TestCase):
 
         self._check_comprehension(wrap)
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_listcomp(self):
         self._simple_comp(ast.ListComp)
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_setcomp(self):
         self._simple_comp(ast.SetComp)
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_generatorexp(self):
         self._simple_comp(ast.GeneratorExp)
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_dictcomp(self):
         g = ast.comprehension(
             ast.Name("y", ast.Store()), ast.Name("p", ast.Load()), [], 0
@@ -2259,13 +2182,11 @@ class ASTValidatorTests(unittest.TestCase):
 
         self._check_comprehension(factory)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_yield(self):
         self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load")
         self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load")
 
-    # TODO: RUSTPYTHON
     @unittest.skip("TODO: RUSTPYTHON; thread 'main' panicked")
     def test_compare(self):
         left = ast.Name("x", ast.Load())
@@ -2278,8 +2199,7 @@ class ASTValidatorTests(unittest.TestCase):
         comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")])
         self.expr(comp)
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_call(self):
         func = ast.Name("x", ast.Load())
         args = [ast.Name("y", ast.Load())]
@@ -2325,14 +2245,12 @@ class ASTValidatorTests(unittest.TestCase):
             ],
         )
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_attribute(self):
         attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load())
         self.expr(attr, "must have Load context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_subscript(self):
         sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), ast.Load())
         self.expr(sub, "must have Load context")
@@ -2348,8 +2266,7 @@ class ASTValidatorTests(unittest.TestCase):
         sl = ast.Tuple([s], ast.Load())
         self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context")
 
-    # TODO: RUSTPYTHON; ValueError not raised
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised
     def test_starred(self):
         left = ast.List(
             [ast.Starred(ast.Name("x", ast.Load()), ast.Store())], ast.Store()
@@ -2363,13 +2280,11 @@ class ASTValidatorTests(unittest.TestCase):
             fac([ast.Name("x", ast.Store())], ast.Load()), "must have Load context"
         )
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_list(self):
         self._sequence(ast.List)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_tuple(self):
         self._sequence(ast.Tuple)
 
@@ -2389,8 +2304,7 @@ class ASTValidatorTests(unittest.TestCase):
             ],
         )
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     @support.requires_resource("cpu")
     def test_stdlib_validates(self):
         stdlib = os.path.dirname(ast.__file__)
@@ -2486,7 +2400,6 @@ class ASTValidatorTests(unittest.TestCase):
         ast.MatchMapping([], [], rest="_"),
     ]
 
-    # TODO: RUSTPYTHON
     @unittest.skip("TODO: RUSTPYTHON; thread 'main' panicked")
     def test_match_validation_pattern(self):
         name_x = ast.Name("x", ast.Load())
@@ -2524,16 +2437,14 @@ class ConstantTests(unittest.TestCase):
             self.compile_constant([1, 2, 3])
         self.assertEqual(str(cm.exception), "got an invalid type in Constant: list")
 
-    # TODO: RUSTPYTHON; b'' is not b''
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; b'' is not b''
     def test_singletons(self):
         for const in (None, False, True, Ellipsis, b"", frozenset()):
             with self.subTest(const=const):
                 value = self.compile_constant(const)
                 self.assertIs(value, const)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_values(self):
         nested_tuple = (1,)
         nested_frozenset = frozenset({1})
@@ -2556,8 +2467,7 @@ class ConstantTests(unittest.TestCase):
                 result = self.compile_constant(value)
                 self.assertEqual(result, value)
 
-    # TODO: RUSTPYTHON; SyntaxError: cannot assign to literal
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: cannot assign to literal
     def test_assign_to_constant(self):
         tree = ast.parse("x = 1")
 
@@ -3079,8 +2989,7 @@ class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase
 
         self.assertASTEqual(result_ast, expected_ast)
 
-    # TODO: RUSTPYTHON; <class 'object'> is not <class 'NoneType'>
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; <class 'object'> is not <class 'NoneType'>
     def test_node_remove_single(self):
         code = "def func(arg) -> SomeType: ..."
         expected = "def func(arg): ..."
@@ -3118,8 +3027,7 @@ class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase
 
         self.assertASTTransformation(YieldRemover, code, expected)
 
-    # TODO: RUSTPYTHON; <class 'object'> is not <class 'NoneType'>
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; <class 'object'> is not <class 'NoneType'>
     def test_node_return_list(self):
         code = """
         class DSL(Base, kw1=True): ...
@@ -3160,8 +3068,7 @@ class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase
 
         self.assertASTTransformation(PrintToLog, code, expected)
 
-    # TODO: RUSTPYTHON; <class 'object'> is not <class 'NoneType'>
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; <class 'object'> is not <class 'NoneType'>
     def test_node_replace(self):
         code = """
         def func(arg):
@@ -3193,8 +3100,7 @@ class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase
 class ASTConstructorTests(unittest.TestCase):
     """Test the autogenerated constructors for AST nodes."""
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_FunctionDef(self):
         args = ast.arguments()
         self.assertEqual(args.args, [])
@@ -3210,8 +3116,7 @@ class ASTConstructorTests(unittest.TestCase):
         self.assertEqual(node.name, "foo")
         self.assertEqual(node.decorator_list, [])
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_expr_context(self):
         name = ast.Name("x")
         self.assertEqual(name.id, "x")
@@ -3260,8 +3165,7 @@ class ASTConstructorTests(unittest.TestCase):
         obj = FieldsAndTypes(a=1)
         self.assertEqual(obj.a, 1)
 
-    # TODO: RUSTPYTHON; DeprecationWarning not triggered
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered
     def test_custom_attributes(self):
         class MyAttrs(ast.AST):
             _attributes = ("a", "b")
@@ -3276,8 +3180,7 @@ class ASTConstructorTests(unittest.TestCase):
         ):
             obj = MyAttrs(c=3)
 
-    # TODO: RUSTPYTHON; DeprecationWarning not triggered
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered
     def test_fields_and_types_no_default(self):
         class FieldsAndTypesNoDefault(ast.AST):
             _fields = ("a",)
@@ -3293,8 +3196,7 @@ class ASTConstructorTests(unittest.TestCase):
         obj = FieldsAndTypesNoDefault(a=1)
         self.assertEqual(obj.a, 1)
 
-    # TODO: RUSTPYTHON; DeprecationWarning not triggered
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered
     def test_incomplete_field_types(self):
         class MoreFieldsThanTypes(ast.AST):
             _fields = ("a", "b")
@@ -3314,8 +3216,7 @@ class ASTConstructorTests(unittest.TestCase):
         self.assertEqual(obj.a, 1)
         self.assertEqual(obj.b, 2)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_complete_field_types(self):
         class _AllFieldTypes(ast.AST):
             _fields = ("a", "b")
@@ -3416,8 +3317,7 @@ class ModuleStateTests(unittest.TestCase):
 class ASTMainTests(unittest.TestCase):
     # Tests `ast.main()` function.
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_cli_file_input(self):
         code = "print(1, 2, 3)"
         expected = ast.dump(ast.parse(code), indent=3)
@@ -3435,7 +3335,7 @@ class ASTMainTests(unittest.TestCase):
 def compare(left, right):
     return ast.dump(left) == ast.dump(right)
 
-class ASTOptimiziationTests(unittest.TestCase):
+class ASTOptimizationTests(unittest.TestCase):
     binop = {
         "+": ast.Add(),
         "-": ast.Sub(),
@@ -3492,8 +3392,7 @@ class ASTOptimiziationTests(unittest.TestCase):
     def create_binop(self, operand, left=ast.Constant(1), right=ast.Constant(1)):
             return ast.BinOp(left=left, op=self.binop[operand], right=right)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_binop(self):
         code = "1 %s 1"
         operators = self.binop.keys()
@@ -3517,8 +3416,7 @@ class ASTOptimiziationTests(unittest.TestCase):
 
         self.assert_ast(code, non_optimized_target, optimized_target)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_unaryop(self):
         code = "%s1"
         operators = self.unaryop.keys()
@@ -3538,8 +3436,7 @@ class ASTOptimiziationTests(unittest.TestCase):
             ):
                 self.assert_ast(result_code, non_optimized_target, optimized_target)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_not(self):
         code = "not (1 %s (1,))"
         operators = {
@@ -3572,8 +3469,7 @@ class ASTOptimiziationTests(unittest.TestCase):
             ):
                 self.assert_ast(result_code, non_optimized_target, optimized_target)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_format(self):
         code = "'%s' % (a,)"
 
@@ -3594,8 +3490,7 @@ class ASTOptimiziationTests(unittest.TestCase):
         self.assert_ast(code, non_optimized_target, optimized_target)
 
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_tuple(self):
         code = "(1,)"
 
@@ -3604,8 +3499,7 @@ class ASTOptimiziationTests(unittest.TestCase):
 
         self.assert_ast(code, non_optimized_target, optimized_target)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_comparator(self):
         code = "1 %s %s1%s"
         operators = [("in", ast.In()), ("not in", ast.NotIn())]
@@ -3625,8 +3519,7 @@ class ASTOptimiziationTests(unittest.TestCase):
                 ))
                 self.assert_ast(code % (op, left, right), non_optimized_target, optimized_target)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_iter(self):
         code = "for _ in %s1%s: pass"
         braces = [
@@ -3648,8 +3541,7 @@ class ASTOptimiziationTests(unittest.TestCase):
 
             self.assert_ast(code % (left, right), non_optimized_target, optimized_target)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_subscript(self):
         code = "(1,)[0]"
 
@@ -3660,8 +3552,7 @@ class ASTOptimiziationTests(unittest.TestCase):
 
         self.assert_ast(code, non_optimized_target, optimized_target)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_type_param_in_function_def(self):
         code = "def foo[%s = 1 + 1](): pass"
 
@@ -3692,8 +3583,7 @@ class ASTOptimiziationTests(unittest.TestCase):
             )
             self.assert_ast(result_code, non_optimized_target, optimized_target)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_type_param_in_class_def(self):
         code = "class foo[%s = 1 + 1]: pass"
 
@@ -3722,8 +3612,7 @@ class ASTOptimiziationTests(unittest.TestCase):
             )
             self.assert_ast(result_code, non_optimized_target, optimized_target)
 
-    # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags
     def test_folding_type_param_in_type_alias(self):
         code = "type foo[%s = 1 + 1] = 1"
 

Lib/test/test_binascii.py

@@ -4,7 +4,8 @@ import unittest
 import binascii
 import array
 import re
-from test.support import bigmemtest, _1G, _4G, warnings_helper
+from test.support import bigmemtest, _1G, _4G
+from test.support.hypothesis_helper import hypothesis
 
 
 # Note: "*_hex" functions are aliases for "(un)hexlify"
@@ -27,6 +28,14 @@ class BinASCIITest(unittest.TestCase):
     def setUp(self):
         self.data = self.type2test(self.rawdata)
 
+    def assertConversion(self, original, converted, restored, **kwargs):
+        self.assertIsInstance(original, bytes)
+        self.assertIsInstance(converted, bytes)
+        self.assertIsInstance(restored, bytes)
+        if converted:
+            self.assertLess(max(converted), 128)
+        self.assertEqual(original, restored, msg=f'{self.type2test=} {kwargs=}')
+
     def test_exceptions(self):
         # Check module exceptions
         self.assertTrue(issubclass(binascii.Error, Exception))
@@ -52,9 +61,7 @@ class BinASCIITest(unittest.TestCase):
                 self.fail("{}/{} conversion raises {!r}".format(fb, fa, err))
             self.assertEqual(res, raw, "{}/{} conversion: "
                              "{!r} != {!r}".format(fb, fa, res, raw))
-            self.assertIsInstance(res, bytes)
-            self.assertIsInstance(a, bytes)
-            self.assertLess(max(a), 128)
+            self.assertConversion(raw, a, res)
         self.assertIsInstance(binascii.crc_hqx(raw, 0), int)
         self.assertIsInstance(binascii.crc32(raw), int)
 
@@ -110,6 +117,7 @@ class BinASCIITest(unittest.TestCase):
         # empty strings. TBD: shouldn't it raise an exception instead ?
         self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'')
 
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_base64_strict_mode(self):
         # Test base64 with strict mode on
         def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes):
@@ -132,13 +140,21 @@ class BinASCIITest(unittest.TestCase):
         def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes):
             _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result)
 
+        def assertExcessPadding(data, non_strict_mode_expected_result: bytes):
+            _assertRegexTemplate(r'(?i)Excess padding', data, non_strict_mode_expected_result)
+
         # Test excess data exceptions
         assertExcessData(b'ab==a', b'i')
         assertExcessData(b'ab===', b'i')
+        assertExcessData(b'ab====', b'i')
         assertExcessData(b'ab==:', b'i')
         assertExcessData(b'abc=a', b'i\xb7')
         assertExcessData(b'abc=:', b'i\xb7')
         assertExcessData(b'ab==\n', b'i')
+        assertExcessData(b'abc==', b'i\xb7')
+        assertExcessData(b'abc===', b'i\xb7')
+        assertExcessData(b'abc====', b'i\xb7')
+        assertExcessData(b'abc=====', b'i\xb7')
 
         # Test non-base64 data exceptions
         assertNonBase64Data(b'\nab==', b'i')
@@ -150,8 +166,16 @@ class BinASCIITest(unittest.TestCase):
         assertLeadingPadding(b'=', b'')
         assertLeadingPadding(b'==', b'')
         assertLeadingPadding(b'===', b'')
+        assertLeadingPadding(b'====', b'')
+        assertLeadingPadding(b'=====', b'')
         assertDiscontinuousPadding(b'ab=c=', b'i\xb7')
         assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b')
+        assertExcessPadding(b'abcd=', b'i\xb7\x1d')
+        assertExcessPadding(b'abcd==', b'i\xb7\x1d')
+        assertExcessPadding(b'abcd===', b'i\xb7\x1d')
+        assertExcessPadding(b'abcd====', b'i\xb7\x1d')
+        assertExcessPadding(b'abcd=====', b'i\xb7\x1d')
+
 
     def test_base64errors(self):
         # Test base64 with invalid padding
@@ -221,6 +245,15 @@ class BinASCIITest(unittest.TestCase):
         with self.assertRaises(TypeError):
             binascii.b2a_uu(b"", True)
 
+    @hypothesis.given(
+        binary=hypothesis.strategies.binary(max_size=45),
+        backtick=hypothesis.strategies.booleans(),
+    )
+    def test_b2a_roundtrip(self, binary, backtick):
+        converted = binascii.b2a_uu(self.type2test(binary), backtick=backtick)
+        restored = binascii.a2b_uu(self.type2test(converted))
+        self.assertConversion(binary, converted, restored, backtick=backtick)
+
     def test_crc_hqx(self):
         crc = binascii.crc_hqx(self.type2test(b"Test the CRC-32 of"), 0)
         crc = binascii.crc_hqx(self.type2test(b" this string."), crc)
@@ -258,6 +291,12 @@ class BinASCIITest(unittest.TestCase):
         self.assertEqual(binascii.hexlify(self.type2test(s)), t)
         self.assertEqual(binascii.unhexlify(self.type2test(t)), u)
 
+    @hypothesis.given(binary=hypothesis.strategies.binary())
+    def test_hex_roundtrip(self, binary):
+        converted = binascii.hexlify(self.type2test(binary))
+        restored = binascii.unhexlify(self.type2test(converted))
+        self.assertConversion(binary, converted, restored)
+
     def test_hex_separator(self):
         """Test that hexlify and b2a_hex are binary versions of bytes.hex."""
         # Logic of separators is tested in test_bytes.py.  This checks that
@@ -372,6 +411,21 @@ class BinASCIITest(unittest.TestCase):
         self.assertEqual(b2a_qp(type2test(b'a.\n')), b'a.\n')
         self.assertEqual(b2a_qp(type2test(b'.a')[:-1]), b'=2E')
 
+    @hypothesis.given(
+        binary=hypothesis.strategies.binary(),
+        quotetabs=hypothesis.strategies.booleans(),
+        istext=hypothesis.strategies.booleans(),
+        header=hypothesis.strategies.booleans(),
+    )
+    def test_b2a_qp_a2b_qp_round_trip(self, binary, quotetabs, istext, header):
+        converted = binascii.b2a_qp(
+            self.type2test(binary),
+            quotetabs=quotetabs, istext=istext, header=header,
+        )
+        restored = binascii.a2b_qp(self.type2test(converted), header=header)
+        self.assertConversion(binary, converted, restored,
+                              quotetabs=quotetabs, istext=istext, header=header)
+
     def test_empty_string(self):
         # A test for SF bug #1022953.  Make sure SystemError is not raised.
         empty = self.type2test(b'')
@@ -427,6 +481,22 @@ class BinASCIITest(unittest.TestCase):
         self.assertEqual(binascii.b2a_base64(b, newline=False),
                          b'aGVsbG8=')
 
+    @hypothesis.given(
+        binary=hypothesis.strategies.binary(),
+        newline=hypothesis.strategies.booleans(),
+    )
+    def test_base64_roundtrip(self, binary, newline):
+        converted = binascii.b2a_base64(self.type2test(binary), newline=newline)
+        restored = binascii.a2b_base64(self.type2test(converted))
+        self.assertConversion(binary, converted, restored, newline=newline)
+
+    @unittest.expectedFailure # TODO: RUSTPYTHON
+    def test_c_contiguity(self):
+        m = memoryview(bytearray(b'noncontig'))
+        noncontig_writable = m[::-2]
+        with self.assertRaises(BufferError):
+            binascii.b2a_hex(noncontig_writable)
+
 
 class ArrayBinASCIITest(BinASCIITest):
     def type2test(self, s):

Lib/test/test_bz2.py

@@ -3,19 +3,19 @@ from test.support import bigmemtest, _4G
 
 import array
 import unittest
+import io
 from io import BytesIO, DEFAULT_BUFFER_SIZE
 import os
 import pickle
 import glob
 import tempfile
-import pathlib
 import random
 import shutil
 import subprocess
 import threading
 from test.support import import_helper
 from test.support import threading_helper
-from test.support.os_helper import unlink
+from test.support.os_helper import unlink, FakePath
 import _compression
 import sys
 
@@ -476,7 +476,6 @@ class BZ2FileTest(BaseTest):
         self.assertEqual(xlines, [b'Test'])
 
     def testContextProtocol(self):
-        f = None
         with BZ2File(self.filename, "wb") as f:
             f.write(b"xxx")
         f = BZ2File(self.filename, "rb")
@@ -537,26 +536,210 @@ class BZ2FileTest(BaseTest):
         with BZ2File(self.filename) as bz2f:
             self.assertEqual(bz2f.read(), data1 + data2)
 
+    def testOpenFilename(self):
+        with BZ2File(self.filename, "wb") as f:
+            f.write(b'content')
+            self.assertEqual(f.name, self.filename)
+            self.assertIsInstance(f.fileno(), int)
+            self.assertEqual(f.mode, 'wb')
+            self.assertIs(f.readable(), False)
+            self.assertIs(f.writable(), True)
+            self.assertIs(f.seekable(), False)
+            self.assertIs(f.closed, False)
+        self.assertIs(f.closed, True)
+        with self.assertRaises(ValueError):
+            f.name
+        self.assertRaises(ValueError, f.fileno)
+        self.assertEqual(f.mode, 'wb')
+        self.assertRaises(ValueError, f.readable)
+        self.assertRaises(ValueError, f.writable)
+        self.assertRaises(ValueError, f.seekable)
+
+        with BZ2File(self.filename, "ab") as f:
+            f.write(b'appendix')
+            self.assertEqual(f.name, self.filename)
+            self.assertIsInstance(f.fileno(), int)
+            self.assertEqual(f.mode, 'wb')
+            self.assertIs(f.readable(), False)
+            self.assertIs(f.writable(), True)
+            self.assertIs(f.seekable(), False)
+            self.assertIs(f.closed, False)
+        self.assertIs(f.closed, True)
+        with self.assertRaises(ValueError):
+            f.name
+        self.assertRaises(ValueError, f.fileno)
+        self.assertEqual(f.mode, 'wb')
+        self.assertRaises(ValueError, f.readable)
+        self.assertRaises(ValueError, f.writable)
+        self.assertRaises(ValueError, f.seekable)
+
+        with BZ2File(self.filename, 'rb') as f:
+            self.assertEqual(f.read(), b'contentappendix')
+            self.assertEqual(f.name, self.filename)
+            self.assertIsInstance(f.fileno(), int)
+            self.assertEqual(f.mode, 'rb')
+            self.assertIs(f.readable(), True)
+            self.assertIs(f.writable(), False)
+            self.assertIs(f.seekable(), True)
+            self.assertIs(f.closed, False)
+        self.assertIs(f.closed, True)
+        with self.assertRaises(ValueError):
+            f.name
+        self.assertRaises(ValueError, f.fileno)
+        self.assertEqual(f.mode, 'rb')
+        self.assertRaises(ValueError, f.readable)
+        self.assertRaises(ValueError, f.writable)
+        self.assertRaises(ValueError, f.seekable)
+
+    def testOpenFileWithName(self):
+        with open(self.filename, 'wb') as raw:
+            with BZ2File(raw, 'wb') as f:
+                f.write(b'content')
+                self.assertEqual(f.name, raw.name)
+                self.assertEqual(f.fileno(), raw.fileno())
+                self.assertEqual(f.mode, 'wb')
+                self.assertIs(f.readable(), False)
+                self.assertIs(f.writable(), True)
+                self.assertIs(f.seekable(), False)
+                self.assertIs(f.closed, False)
+            self.assertIs(f.closed, True)
+            with self.assertRaises(ValueError):
+                f.name
+            self.assertRaises(ValueError, f.fileno)
+            self.assertEqual(f.mode, 'wb')
+            self.assertRaises(ValueError, f.readable)
+            self.assertRaises(ValueError, f.writable)
+            self.assertRaises(ValueError, f.seekable)
+
+        with open(self.filename, 'ab') as raw:
+            with BZ2File(raw, 'ab') as f:
+                f.write(b'appendix')
+                self.assertEqual(f.name, raw.name)
+                self.assertEqual(f.fileno(), raw.fileno())
+                self.assertEqual(f.mode, 'wb')
+                self.assertIs(f.readable(), False)
+                self.assertIs(f.writable(), True)
+                self.assertIs(f.seekable(), False)
+                self.assertIs(f.closed, False)
+            self.assertIs(f.closed, True)
+            with self.assertRaises(ValueError):
+                f.name
+            self.assertRaises(ValueError, f.fileno)
+            self.assertEqual(f.mode, 'wb')
+            self.assertRaises(ValueError, f.readable)
+            self.assertRaises(ValueError, f.writable)
+            self.assertRaises(ValueError, f.seekable)
+
+        with open(self.filename, 'rb') as raw:
+            with BZ2File(raw, 'rb') as f:
+                self.assertEqual(f.read(), b'contentappendix')
+                self.assertEqual(f.name, raw.name)
+                self.assertEqual(f.fileno(), raw.fileno())
+                self.assertEqual(f.mode, 'rb')
+                self.assertIs(f.readable(), True)
+                self.assertIs(f.writable(), False)
+                self.assertIs(f.seekable(), True)
+                self.assertIs(f.closed, False)
+            self.assertIs(f.closed, True)
+            with self.assertRaises(ValueError):
+                f.name
+            self.assertRaises(ValueError, f.fileno)
+            self.assertEqual(f.mode, 'rb')
+            self.assertRaises(ValueError, f.readable)
+            self.assertRaises(ValueError, f.writable)
+            self.assertRaises(ValueError, f.seekable)
+
+    def testOpenFileWithoutName(self):
+        bio = BytesIO()
+        with BZ2File(bio, 'wb') as f:
+            f.write(b'content')
+            with self.assertRaises(AttributeError):
+                f.name
+            self.assertRaises(io.UnsupportedOperation, f.fileno)
+            self.assertEqual(f.mode, 'wb')
+        with self.assertRaises(ValueError):
+            f.name
+        self.assertRaises(ValueError, f.fileno)
+
+        with BZ2File(bio, 'ab') as f:
+            f.write(b'appendix')
+            with self.assertRaises(AttributeError):
+                f.name
+            self.assertRaises(io.UnsupportedOperation, f.fileno)
+            self.assertEqual(f.mode, 'wb')
+        with self.assertRaises(ValueError):
+            f.name
+        self.assertRaises(ValueError, f.fileno)
+
+        bio.seek(0)
+        with BZ2File(bio, 'rb') as f:
+            self.assertEqual(f.read(), b'contentappendix')
+            with self.assertRaises(AttributeError):
+                f.name
+            self.assertRaises(io.UnsupportedOperation, f.fileno)
+            self.assertEqual(f.mode, 'rb')
+        with self.assertRaises(ValueError):
+            f.name
+        self.assertRaises(ValueError, f.fileno)
+
+    def testOpenFileWithIntName(self):
+        fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
+        with open(fd, 'wb') as raw:
+            with BZ2File(raw, 'wb') as f:
+                f.write(b'content')
+                self.assertEqual(f.name, raw.name)
+                self.assertEqual(f.fileno(), raw.fileno())
+                self.assertEqual(f.mode, 'wb')
+            with self.assertRaises(ValueError):
+                f.name
+            self.assertRaises(ValueError, f.fileno)
+
+        fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND)
+        with open(fd, 'ab') as raw:
+            with BZ2File(raw, 'ab') as f:
+                f.write(b'appendix')
+                self.assertEqual(f.name, raw.name)
+                self.assertEqual(f.fileno(), raw.fileno())
+                self.assertEqual(f.mode, 'wb')
+            with self.assertRaises(ValueError):
+                f.name
+            self.assertRaises(ValueError, f.fileno)
+
+        fd = os.open(self.filename, os.O_RDONLY)
+        with open(fd, 'rb') as raw:
+            with BZ2File(raw, 'rb') as f:
+                self.assertEqual(f.read(), b'contentappendix')
+                self.assertEqual(f.name, raw.name)
+                self.assertEqual(f.fileno(), raw.fileno())
+                self.assertEqual(f.mode, 'rb')
+            with self.assertRaises(ValueError):
+                f.name
+            self.assertRaises(ValueError, f.fileno)
+
     def testOpenBytesFilename(self):
         str_filename = self.filename
-        try:
-            bytes_filename = str_filename.encode("ascii")
-        except UnicodeEncodeError:
-            self.skipTest("Temporary file name needs to be ASCII")
+        bytes_filename = os.fsencode(str_filename)
         with BZ2File(bytes_filename, "wb") as f:
             f.write(self.DATA)
+            self.assertEqual(f.name, bytes_filename)
         with BZ2File(bytes_filename, "rb") as f:
             self.assertEqual(f.read(), self.DATA)
+            self.assertEqual(f.name, bytes_filename)
         # Sanity check that we are actually operating on the right file.
         with BZ2File(str_filename, "rb") as f:
             self.assertEqual(f.read(), self.DATA)
+            self.assertEqual(f.name, str_filename)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def testOpenPathLikeFilename(self):
-        filename = pathlib.Path(self.filename)
+        filename = FakePath(self.filename)
         with BZ2File(filename, "wb") as f:
             f.write(self.DATA)
+            self.assertEqual(f.name, self.filename)
         with BZ2File(filename, "rb") as f:
             self.assertEqual(f.read(), self.DATA)
+            self.assertEqual(f.name, self.filename)
 
     def testDecompressLimited(self):
         """Decompressed data buffering should be limited"""
@@ -577,6 +760,9 @@ class BZ2FileTest(BaseTest):
             with BZ2File(bio) as bz2f:
                 self.assertRaises(TypeError, bz2f.read, float())
                 self.assertEqual(bz2f.read(), self.TEXT)
+                with self.assertRaises(AttributeError):
+                    bz2.name
+                self.assertEqual(bz2f.mode, 'rb')
             self.assertFalse(bio.closed)
 
     def testPeekBytesIO(self):
@@ -592,6 +778,9 @@ class BZ2FileTest(BaseTest):
             with BZ2File(bio, "w") as bz2f:
                 self.assertRaises(TypeError, bz2f.write)
                 bz2f.write(self.TEXT)
+                with self.assertRaises(AttributeError):
+                    bz2.name
+                self.assertEqual(bz2f.mode, 'wb')
             self.assertEqual(ext_decompress(bio.getvalue()), self.TEXT)
             self.assertFalse(bio.closed)
 

Lib/test/test_cmath.py

@@ -1,4 +1,5 @@
 from test.support import requires_IEEE_754, cpython_only, import_helper
+from test.support.testcase import ComplexesAreIdenticalMixin
 from test.test_math import parse_testfile, test_file
 import test.test_math as test_math
 import unittest
@@ -49,7 +50,7 @@ complex_nans = [complex(x, y) for x, y in [
         (INF, NAN)
         ]]
 
-class CMathTests(unittest.TestCase):
+class CMathTests(ComplexesAreIdenticalMixin, unittest.TestCase):
     # list of all functions in cmath
     test_functions = [getattr(cmath, fname) for fname in [
             'acos', 'acosh', 'asin', 'asinh', 'atan', 'atanh',
@@ -65,39 +66,6 @@ class CMathTests(unittest.TestCase):
     def tearDown(self):
         self.test_values.close()
 
-    def assertFloatIdentical(self, x, y):
-        """Fail unless floats x and y are identical, in the sense that:
-        (1) both x and y are nans, or
-        (2) both x and y are infinities, with the same sign, or
-        (3) both x and y are zeros, with the same sign, or
-        (4) x and y are both finite and nonzero, and x == y
-
-        """
-        msg = 'floats {!r} and {!r} are not identical'
-
-        if math.isnan(x) or math.isnan(y):
-            if math.isnan(x) and math.isnan(y):
-                return
-        elif x == y:
-            if x != 0.0:
-                return
-            # both zero; check that signs match
-            elif math.copysign(1.0, x) == math.copysign(1.0, y):
-                return
-            else:
-                msg += ': zeros have different signs'
-        self.fail(msg.format(x, y))
-
-    def assertComplexIdentical(self, x, y):
-        """Fail unless complex numbers x and y have equal values and signs.
-
-        In particular, if x and y both have real (or imaginary) part
-        zero, but the zeros have different signs, this test will fail.
-
-        """
-        self.assertFloatIdentical(x.real, y.real)
-        self.assertFloatIdentical(x.imag, y.imag)
-
     def rAssertAlmostEqual(self, a, b, rel_err = 2e-15, abs_err = 5e-323,
                            msg=None):
         """Fail if the two floating-point numbers are not almost equal.
@@ -299,12 +267,6 @@ class CMathTests(unittest.TestCase):
             for v in values:
                 z = complex_fn(v)
                 self.rAssertAlmostEqual(float_fn(v), z.real)
-                # TODO: RUSTPYTHON
-                # This line currently fails for acos and asin.
-                # cmath.asin/acos(0.2) should produce a real number,
-                # but imaginary part is 1.1102230246251565e-16 for both.
-                if fn in {"asin", "acos"}:
-                    continue
                 self.assertEqual(0., z.imag)
 
         # test two-argument version of log with various bases
@@ -314,8 +276,7 @@ class CMathTests(unittest.TestCase):
                 self.rAssertAlmostEqual(math.log(v, base), z.real)
                 self.assertEqual(0., z.imag)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     @requires_IEEE_754
     def test_specific_values(self):
         # Some tests need to be skipped on ancient OS X versions.
@@ -563,25 +524,23 @@ class CMathTests(unittest.TestCase):
     @requires_IEEE_754
     def testTanhSign(self):
         for z in complex_zeros:
-            self.assertComplexIdentical(cmath.tanh(z), z)
+            self.assertComplexesAreIdentical(cmath.tanh(z), z)
 
     # The algorithm used for atan and atanh makes use of the system
     # log1p function; If that system function doesn't respect the sign
     # of zero, then atan and atanh will also have difficulties with
     # the sign of complex zeros.
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     @requires_IEEE_754
     def testAtanSign(self):
         for z in complex_zeros:
-            self.assertComplexIdentical(cmath.atan(z), z)
+            self.assertComplexesAreIdentical(cmath.atan(z), z)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     @requires_IEEE_754
     def testAtanhSign(self):
         for z in complex_zeros:
-            self.assertComplexIdentical(cmath.atanh(z), z)
+            self.assertComplexesAreIdentical(cmath.atanh(z), z)
 
 
 class IsCloseTests(test_math.IsCloseTests):
@@ -624,8 +583,7 @@ class IsCloseTests(test_math.IsCloseTests):
         self.assertIsClose(0.001-0.001j, 0.001+0.001j, abs_tol=2e-03)
         self.assertIsNotClose(0.001-0.001j, 0.001+0.001j, abs_tol=1e-03)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_complex_special(self):
         self.assertIsNotClose(INF, INF*1j)
         self.assertIsNotClose(INF*1j, INF)

Lib/test/test_code.py

@@ -347,8 +347,6 @@ class CodeTest(unittest.TestCase):
         newcode = code.replace(co_name="func")  # Should not raise SystemError
         self.assertEqual(code, newcode)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_empty_linetable(self):
         def func():
             pass
@@ -468,8 +466,6 @@ class CodeTest(unittest.TestCase):
 
     # co_positions behavior when info is missing.
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     # @requires_debug_ranges()
     def test_co_positions_empty_linetable(self):
         def func():
@@ -480,8 +476,6 @@ class CodeTest(unittest.TestCase):
             self.assertIsNone(line)
             self.assertEqual(end_line, new_code.co_firstlineno + 1)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_code_equality(self):
         def f():
             try:
@@ -522,8 +516,6 @@ class CodeTest(unittest.TestCase):
         self.assertNotEqual(c, swapped)
         self.assertNotEqual(hash(c), hash(swapped))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_code_hash_uses_bytecode(self):
         c = (lambda x, y: x + y).__code__
         d = (lambda x, y: x * y).__code__
@@ -735,8 +727,6 @@ class CodeLocationTest(unittest.TestCase):
             self.assertEqual(l1, l2)
         self.assertEqual(len(pos1), len(pos2))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_positions(self):
         self.check_positions(parse_location_table)
         self.check_positions(misshappen)
@@ -751,8 +741,6 @@ class CodeLocationTest(unittest.TestCase):
             self.assertEqual(l1, l2)
         self.assertEqual(len(lines1), len(lines2))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_lines(self):
         self.check_lines(parse_location_table)
         self.check_lines(misshappen)

Lib/test/test_collections.py

@@ -262,8 +262,7 @@ class TestChainMap(unittest.TestCase):
         d = c.new_child(b=20, c=30)
         self.assertEqual(d.maps, [{'b': 20, 'c': 30}, {'a': 1, 'b': 2}])
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_union_operators(self):
         cm1 = ChainMap(dict(a=1, b=2), dict(c=3, d=4))
         cm2 = ChainMap(dict(a=10, e=5), dict(b=20, d=4))
@@ -470,8 +469,7 @@ class TestNamedTuple(unittest.TestCase):
         NT = namedtuple('NT', ['x', 'y'], module=collections)
         self.assertEqual(NT.__module__, collections)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_instance(self):
         Point = namedtuple('Point', 'x y')
         p = Point(11, 22)
@@ -740,7 +738,7 @@ class ABCTestCase(unittest.TestCase):
             stubs = methodstubs.copy()
             del stubs[name]
             C = type('C', (abc,), stubs)
-            self.assertRaises(TypeError, C, name)
+            self.assertRaises(TypeError, C)
 
     def validate_isinstance(self, abc, name):
         stub = lambda s, *args: 0
@@ -790,8 +788,7 @@ def _test_gen():
 
 class TestOneTrickPonyABCs(ABCTestCase):
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_Awaitable(self):
         def gen():
             yield
@@ -844,8 +841,7 @@ class TestOneTrickPonyABCs(ABCTestCase):
         CoroLike = None
         support.gc_collect() # Kill CoroLike to clean-up ABCMeta cache
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_Coroutine(self):
         def gen():
             yield
@@ -971,7 +967,7 @@ class TestOneTrickPonyABCs(ABCTestCase):
             async def __anext__(self):
                 raise StopAsyncIteration
         self.assertNotIsInstance(AnextOnly(), AsyncIterator)
-        self.validate_abstract_methods(AsyncIterator, '__anext__', '__aiter__')
+        self.validate_abstract_methods(AsyncIterator, '__anext__')
 
     def test_Iterable(self):
         # Check some non-iterables
@@ -1168,7 +1164,7 @@ class TestOneTrickPonyABCs(ABCTestCase):
         for x in samples:
             self.assertIsInstance(x, Iterator)
             self.assertTrue(issubclass(type(x), Iterator), repr(type(x)))
-        self.validate_abstract_methods(Iterator, '__next__', '__iter__')
+        self.validate_abstract_methods(Iterator, '__next__')
 
         # Issue 10565
         class NextOnly:
@@ -1852,8 +1848,7 @@ class TestCollectionABCs(ABCTestCase):
         for sample in [dict]:
             self.assertIsInstance(sample(), Mapping)
             self.assertTrue(issubclass(sample, Mapping))
-        self.validate_abstract_methods(Mapping, '__contains__', '__iter__', '__len__',
-            '__getitem__')
+        self.validate_abstract_methods(Mapping, '__iter__', '__len__', '__getitem__')
         class MyMapping(Mapping):
             def __len__(self):
                 return 0
@@ -1868,7 +1863,7 @@ class TestCollectionABCs(ABCTestCase):
         for sample in [dict]:
             self.assertIsInstance(sample(), MutableMapping)
             self.assertTrue(issubclass(sample, MutableMapping))
-        self.validate_abstract_methods(MutableMapping, '__contains__', '__iter__', '__len__',
+        self.validate_abstract_methods(MutableMapping, '__iter__', '__len__',
             '__getitem__', '__setitem__', '__delitem__')
 
     def test_MutableMapping_subclass(self):
@@ -1907,8 +1902,7 @@ class TestCollectionABCs(ABCTestCase):
         self.assertIsInstance(memoryview(b""), Sequence)
         self.assertTrue(issubclass(memoryview, Sequence))
         self.assertTrue(issubclass(str, Sequence))
-        self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__',
-            '__getitem__')
+        self.validate_abstract_methods(Sequence, '__len__', '__getitem__')
 
     def test_Sequence_mixins(self):
         class SequenceSubclass(Sequence):
@@ -1967,10 +1961,7 @@ class TestCollectionABCs(ABCTestCase):
             # No metaclass conflict
             class Z(ByteString, Awaitable): pass
 
-    # TODO: RUSTPYTHON
-    # Need to implement __buffer__ and __release_buffer__
-    # https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types)
     def test_Buffer(self):
         for sample in [bytes, bytearray, memoryview]:
             self.assertIsInstance(sample(b"x"), Buffer)
@@ -1989,8 +1980,8 @@ class TestCollectionABCs(ABCTestCase):
             self.assertTrue(issubclass(sample, MutableSequence))
         self.assertTrue(issubclass(array.array, MutableSequence))
         self.assertFalse(issubclass(str, MutableSequence))
-        self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__',
-            '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert')
+        self.validate_abstract_methods(MutableSequence, '__len__', '__getitem__',
+                                       '__setitem__', '__delitem__', 'insert')
 
     def test_MutableSequence_mixins(self):
         # Test the mixins of MutableSequence by creating a minimal concrete
@@ -2042,8 +2033,7 @@ class TestCollectionABCs(ABCTestCase):
         self.assertEqual(len(mss), len(mss2))
         self.assertEqual(list(mss), list(mss2))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_illegal_patma_flags(self):
         with self.assertRaises(TypeError):
             class Both(Collection):

Lib/test/test_compile.py

@@ -138,7 +138,6 @@ def f(x):
     def test_argument_order(self):
         self.assertRaises(SyntaxError, exec, 'def f(a=1, b): pass')
 
-    @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: ParseFloatError { kind: Invalid }'")
     def test_float_literals(self):
         # testing bad float literals
         self.assertRaises(SyntaxError, eval, "2e")
@@ -201,6 +200,8 @@ if 1:
         self.assertEqual(eval("0o777"), 511)
         self.assertEqual(eval("-0o0000010"), -8)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_int_literals_too_long(self):
         n = 3000
         source = f"a = 1\nb = 2\nc = {'3'*n}\nd = 4"
@@ -274,6 +275,8 @@ if 1:
             self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single')
             self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec')
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_import(self):
         succeed = [
             'import sys',
@@ -821,6 +824,8 @@ if 1:
             self.assertEqual(None, opcodes[1].argval)
             self.assertEqual('RETURN_VALUE', opcodes[2].opname)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_consts_in_conditionals(self):
         def and_true(x):
             return True and x
@@ -844,6 +849,8 @@ if 1:
                 self.assertIn('LOAD_', opcodes[-2].opname)
                 self.assertEqual('RETURN_VALUE', opcodes[-1].opname)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_imported_load_method(self):
         sources = [
             """\
@@ -886,6 +893,8 @@ if 1:
         line1 = call.__code__.co_firstlineno + 1
         assert line1 not in [line for (_, _, line) in call.__code__.co_lines()]
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_lineno_after_implicit_return(self):
         TRUE = True
         # Don't use constant True or False, as compiler will remove test
@@ -920,6 +929,8 @@ if 1:
                 func(save_caller_frame)
                 self.assertEqual(frame.f_lineno-frame.f_code.co_firstlineno, lastline)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_lineno_after_no_code(self):
         def no_code1():
             "doc string"
@@ -944,6 +955,8 @@ if 1:
                 last_line = line
         return res
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_lineno_attribute(self):
         def load_attr():
             return (
@@ -988,6 +1001,8 @@ if 1:
                 code_lines = self.get_code_lines(func.__code__)
                 self.assertEqual(lines, code_lines)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_line_number_genexp(self):
 
         def return_genexp():
@@ -1002,6 +1017,8 @@ if 1:
         code_lines = self.get_code_lines(genexp_code)
         self.assertEqual(genexp_lines, code_lines)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_line_number_implicit_return_after_async_for(self):
 
         async def test(aseq):
@@ -1022,6 +1039,8 @@ if 1:
         the_dict = "{" + ",".join(f"{x}:{x}" for x in range(dict_size)) + "}"
         self.assertEqual(len(eval(the_dict)), dict_size)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_redundant_jump_in_if_else_break(self):
         # Check if bytecode containing jumps that simply point to the next line
         # is generated around if-else-break style structures. See bpo-42615.
@@ -1051,6 +1070,8 @@ if 1:
             elif instr.opname in HANDLED_JUMPS:
                 self.assertNotEqual(instr.arg, (line + 1)*INSTR_SIZE)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_no_wraparound_jump(self):
         # See https://bugs.python.org/issue46724
 
@@ -1061,6 +1082,8 @@ if 1:
         for instr in dis.Bytecode(while_not_chained):
             self.assertNotEqual(instr.opname, "EXTENDED_ARG")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_compare_positions(self):
         for opname, op in [
             ("COMPARE_OP", "<"),
@@ -1361,64 +1384,66 @@ class TestExpressionStackSize(unittest.TestCase):
         max_size = math.ceil(math.log(len(code.co_code)))
         self.assertLessEqual(code.co_stacksize, max_size)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_and(self):
         self.check_stack_size("x and " * self.N + "x")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_or(self):
         self.check_stack_size("x or " * self.N + "x")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_and_or(self):
         self.check_stack_size("x and x or " * self.N + "x")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_chained_comparison(self):
         self.check_stack_size("x < " * self.N + "x")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_if_else(self):
         self.check_stack_size("x if x else " * self.N + "x")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_binop(self):
         self.check_stack_size("x + " * self.N + "x")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_list(self):
         self.check_stack_size("[" + "x, " * self.N + "x]")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_tuple(self):
         self.check_stack_size("(" + "x, " * self.N + "x)")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_set(self):
         self.check_stack_size("{" + "x, " * self.N + "x}")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_dict(self):
         self.check_stack_size("{" + "x:x, " * self.N + "x:x}")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_func_args(self):
         self.check_stack_size("f(" + "x, " * self.N + ")")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_func_kwargs(self):
         kwargs = (f'a{i}=x' for i in range(self.N))
         self.check_stack_size("f(" +  ", ".join(kwargs) + ")")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_meth_args(self):
         self.check_stack_size("o.m(" + "x, " * self.N + ")")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_meth_kwargs(self):
         kwargs = (f'a{i}=x' for i in range(self.N))
         self.check_stack_size("o.m(" +  ", ".join(kwargs) + ")")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_func_and(self):
         code = "def f(x):\n"
         code += "   x and x\n" * self.N
@@ -1513,6 +1538,8 @@ class TestStackSizeStability(unittest.TestCase):
             """
         self.check_stack_size(snippet)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_try_except_star_qualified(self):
         snippet = """
             try:
@@ -1524,6 +1551,8 @@ class TestStackSizeStability(unittest.TestCase):
             """
         self.check_stack_size(snippet)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_try_except_star_as(self):
         snippet = """
             try:
@@ -1535,6 +1564,8 @@ class TestStackSizeStability(unittest.TestCase):
             """
         self.check_stack_size(snippet)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_try_except_star_finally(self):
         snippet = """
                 try:

Lib/test/test_configparser.py

@@ -2,10 +2,8 @@ import collections
 import configparser
 import io
 import os
-import pathlib
 import textwrap
 import unittest
-import warnings
 
 from test import support
 from test.support import os_helper
@@ -545,7 +543,7 @@ boolean {0[0]} NO
                                 "[Foo]\n  wrong-indent\n")
             self.assertEqual(e.args, ('<???>',))
             # read_file on a real file
-            tricky = support.findfile("cfgparser.3")
+            tricky = support.findfile("cfgparser.3", subdir="configdata")
             if self.delimiters[0] == '=':
                 error = configparser.ParsingError
                 expected = (tricky,)
@@ -648,6 +646,21 @@ boolean {0[0]} NO
                                      "'opt' in section 'Bar' already exists")
             self.assertEqual(e.args, ("Bar", "opt", "<dict>", None))
 
+    def test_get_after_duplicate_option_error(self):
+        cf = self.newconfig()
+        ini = textwrap.dedent("""\
+            [Foo]
+            x{equals}1
+            y{equals}2
+            y{equals}3
+        """.format(equals=self.delimiters[0]))
+        if self.strict:
+            with self.assertRaises(configparser.DuplicateOptionError):
+                cf.read_string(ini)
+        else:
+            cf.read_string(ini)
+        self.assertEqual(cf.get('Foo', 'x'), '1')
+
     def test_write(self):
         config_string = (
             "[Long Line]\n"
@@ -719,7 +732,7 @@ boolean {0[0]} NO
     def test_read_returns_file_list(self):
         if self.delimiters[0] != '=':
             self.skipTest('incompatible format')
-        file1 = support.findfile("cfgparser.1")
+        file1 = support.findfile("cfgparser.1", subdir="configdata")
         # check when we pass a mix of readable and non-readable files:
         cf = self.newconfig()
         parsed_files = cf.read([file1, "nonexistent-file"], encoding="utf-8")
@@ -732,12 +745,12 @@ boolean {0[0]} NO
         self.assertEqual(cf.get("Foo Bar", "foo"), "newbar")
         # check when we pass only a Path object:
         cf = self.newconfig()
-        parsed_files = cf.read(pathlib.Path(file1), encoding="utf-8")
+        parsed_files = cf.read(os_helper.FakePath(file1), encoding="utf-8")
         self.assertEqual(parsed_files, [file1])
         self.assertEqual(cf.get("Foo Bar", "foo"), "newbar")
         # check when we passed both a filename and a Path object:
         cf = self.newconfig()
-        parsed_files = cf.read([pathlib.Path(file1), file1], encoding="utf-8")
+        parsed_files = cf.read([os_helper.FakePath(file1), file1], encoding="utf-8")
         self.assertEqual(parsed_files, [file1, file1])
         self.assertEqual(cf.get("Foo Bar", "foo"), "newbar")
         # check when we pass only missing files:
@@ -753,7 +766,7 @@ boolean {0[0]} NO
     def test_read_returns_file_list_with_bytestring_path(self):
         if self.delimiters[0] != '=':
             self.skipTest('incompatible format')
-        file1_bytestring = support.findfile("cfgparser.1").encode()
+        file1_bytestring = support.findfile("cfgparser.1", subdir="configdata").encode()
         # check when passing an existing bytestring path
         cf = self.newconfig()
         parsed_files = cf.read(file1_bytestring, encoding="utf-8")
@@ -909,9 +922,6 @@ class ConfigParserTestCase(BasicTestCase, unittest.TestCase):
         if self.interpolation == configparser._UNSET:
             self.assertEqual(e.args, ("bar11", "Foo",
                 "something %(with11)s lots of interpolation (11 steps)"))
-        elif isinstance(self.interpolation, configparser.LegacyInterpolation):
-            self.assertEqual(e.args, ("bar11", "Foo",
-                "something %(with11)s lots of interpolation (11 steps)"))
 
     def test_interpolation_missing_value(self):
         cf = self.get_interpolation_config()
@@ -923,9 +933,6 @@ class ConfigParserTestCase(BasicTestCase, unittest.TestCase):
         if self.interpolation == configparser._UNSET:
             self.assertEqual(e.args, ('name', 'Interpolation Error',
                                     '%(reference)s', 'reference'))
-        elif isinstance(self.interpolation, configparser.LegacyInterpolation):
-            self.assertEqual(e.args, ('name', 'Interpolation Error',
-                                    '%(reference)s', 'reference'))
 
     def test_items(self):
         self.check_items_config([('default', '<default>'),
@@ -944,9 +951,6 @@ class ConfigParserTestCase(BasicTestCase, unittest.TestCase):
         self.assertEqual(cf.get("section", "ok"), "xxx/%s")
         if self.interpolation == configparser._UNSET:
             self.assertEqual(cf.get("section", "not_ok"), "xxx/xxx/%s")
-        elif isinstance(self.interpolation, configparser.LegacyInterpolation):
-            with self.assertRaises(TypeError):
-                cf.get("section", "not_ok")
 
     def test_set_malformatted_interpolation(self):
         cf = self.fromstring("[sect]\n"
@@ -1027,31 +1031,6 @@ class ConfigParserTestCaseNoInterpolation(BasicTestCase, unittest.TestCase):
         cf.read_string(self.ini)
         self.assertMatchesIni(cf)
 
-
-class ConfigParserTestCaseLegacyInterpolation(ConfigParserTestCase):
-    config_class = configparser.ConfigParser
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", DeprecationWarning)
-        interpolation = configparser.LegacyInterpolation()
-
-    def test_set_malformatted_interpolation(self):
-        cf = self.fromstring("[sect]\n"
-                             "option1{eq}foo\n".format(eq=self.delimiters[0]))
-
-        self.assertEqual(cf.get('sect', "option1"), "foo")
-
-        cf.set("sect", "option1", "%foo")
-        self.assertEqual(cf.get('sect', "option1"), "%foo")
-        cf.set("sect", "option1", "foo%")
-        self.assertEqual(cf.get('sect', "option1"), "foo%")
-        cf.set("sect", "option1", "f%oo")
-        self.assertEqual(cf.get('sect', "option1"), "f%oo")
-
-        # bug #5741: double percents are *not* malformed
-        cf.set("sect", "option2", "foo%%bar")
-        self.assertEqual(cf.get("sect", "option2"), "foo%%bar")
-
-
 class ConfigParserTestCaseInvalidInterpolationType(unittest.TestCase):
     def test_error_on_wrong_type_for_interpolation(self):
         for value in [configparser.ExtendedInterpolation,  42,  "a string"]:
@@ -1163,7 +1142,7 @@ class RawConfigParserTestSambaConf(CfgParserTestCaseClass, unittest.TestCase):
     empty_lines_in_values = False
 
     def test_reading(self):
-        smbconf = support.findfile("cfgparser.2")
+        smbconf = support.findfile("cfgparser.2", subdir="configdata")
         # check when we pass a mix of readable and non-readable files:
         cf = self.newconfig()
         parsed_files = cf.read([smbconf, "nonexistent-file"], encoding='utf-8')
@@ -1351,6 +1330,47 @@ class ConfigParserTestCaseNoValue(ConfigParserTestCase):
     allow_no_value = True
 
 
+class NoValueAndExtendedInterpolation(CfgParserTestCaseClass):
+    interpolation = configparser.ExtendedInterpolation()
+    allow_no_value = True
+
+    def test_interpolation_with_allow_no_value(self):
+        config = textwrap.dedent("""
+            [dummy]
+            a
+            b = ${a}
+        """)
+        cf = self.fromstring(config)
+
+        self.assertIs(cf["dummy"]["a"], None)
+        self.assertEqual(cf["dummy"]["b"], "")
+
+    def test_explicit_none(self):
+        config = textwrap.dedent("""
+            [dummy]
+            a = None
+            b = ${a}
+        """)
+        cf = self.fromstring(config)
+
+        self.assertEqual(cf["dummy"]["a"], "None")
+        self.assertEqual(cf["dummy"]["b"], "None")
+
+
+class ConfigParserNoValueAndExtendedInterpolationTest(
+    NoValueAndExtendedInterpolation,
+    unittest.TestCase,
+):
+    config_class = configparser.ConfigParser
+
+
+class RawConfigParserNoValueAndExtendedInterpolationTest(
+    NoValueAndExtendedInterpolation,
+    unittest.TestCase,
+):
+    config_class = configparser.RawConfigParser
+
+
 class ConfigParserTestCaseTrickyFile(CfgParserTestCaseClass, unittest.TestCase):
     config_class = configparser.ConfigParser
     delimiters = {'='}
@@ -1358,7 +1378,7 @@ class ConfigParserTestCaseTrickyFile(CfgParserTestCaseClass, unittest.TestCase):
     allow_no_value = True
 
     def test_cfgparser_dot_3(self):
-        tricky = support.findfile("cfgparser.3")
+        tricky = support.findfile("cfgparser.3", subdir="configdata")
         cf = self.newconfig()
         self.assertEqual(len(cf.read(tricky, encoding='utf-8')), 1)
         self.assertEqual(cf.sections(), ['strange',
@@ -1390,7 +1410,7 @@ class ConfigParserTestCaseTrickyFile(CfgParserTestCaseClass, unittest.TestCase):
         self.assertEqual(cf.get('more interpolation', 'lets'), 'go shopping')
 
     def test_unicode_failure(self):
-        tricky = support.findfile("cfgparser.3")
+        tricky = support.findfile("cfgparser.3", subdir="configdata")
         cf = self.newconfig()
         with self.assertRaises(UnicodeDecodeError):
             cf.read(tricky, encoding='ascii')
@@ -1491,7 +1511,7 @@ class CopyTestCase(BasicTestCase, unittest.TestCase):
 
 class FakeFile:
     def __init__(self):
-        file_path = support.findfile("cfgparser.1")
+        file_path = support.findfile("cfgparser.1", subdir="configdata")
         with open(file_path, encoding="utf-8") as f:
             self.lines = f.readlines()
             self.lines.reverse()
@@ -1512,7 +1532,7 @@ def readline_generator(f):
 
 class ReadFileTestCase(unittest.TestCase):
     def test_file(self):
-        file_paths = [support.findfile("cfgparser.1")]
+        file_paths = [support.findfile("cfgparser.1", subdir="configdata")]
         try:
             file_paths.append(file_paths[0].encode('utf8'))
         except UnicodeEncodeError:
@@ -1592,6 +1612,30 @@ class ReadFileTestCase(unittest.TestCase):
             "'[badbad'"
         )
 
+    def test_keys_without_value_with_extra_whitespace(self):
+        lines = [
+            '[SECT]\n',
+            'KEY1\n',
+            ' KEY2 = VAL2\n', # note the Space before the key!
+        ]
+        parser = configparser.ConfigParser(
+            comment_prefixes="",
+            allow_no_value=True,
+            strict=False,
+            delimiters=('=',),
+            interpolation=None,
+        )
+        with self.assertRaises(configparser.MultilineContinuationError) as dse:
+            parser.read_file(lines)
+        self.assertEqual(
+            str(dse.exception),
+            "Key without value continued with an indented line.\n"
+            "file: '<???>', line: 3\n"
+            "' KEY2 = VAL2\\n'"
+        )
+
+
+
 
 class CoverageOneHundredTestCase(unittest.TestCase):
     """Covers edge cases in the codebase."""
@@ -1638,14 +1682,6 @@ class CoverageOneHundredTestCase(unittest.TestCase):
         self.assertEqual(str(cm.exception), "bad interpolation variable "
                                             "reference '%(()'")
 
-    def test_legacyinterpolation_deprecation(self):
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter("always", DeprecationWarning)
-            configparser.LegacyInterpolation()
-        self.assertGreaterEqual(len(w), 1)
-        for warning in w:
-            self.assertIs(warning.category, DeprecationWarning)
-
     def test_sectionproxy_repr(self):
         parser = configparser.ConfigParser()
         parser.read_string("""
@@ -2121,6 +2157,63 @@ class BlatantOverrideConvertersTestCase(unittest.TestCase):
             self.assertEqual(cfg['two'].getlen('one'), 5)
 
 
+class SectionlessTestCase(unittest.TestCase):
+
+    def fromstring(self, string):
+        cfg = configparser.ConfigParser(allow_unnamed_section=True)
+        cfg.read_string(string)
+        return cfg
+
+    def test_no_first_section(self):
+        cfg1 = self.fromstring("""
+        a = 1
+        b = 2
+        [sect1]
+        c = 3
+        """)
+
+        self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg1.sections()))
+        self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a'])
+        self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b'])
+        self.assertEqual('3', cfg1['sect1']['c'])
+
+        output = io.StringIO()
+        cfg1.write(output)
+        cfg2 = self.fromstring(output.getvalue())
+
+        #self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg2.sections()))
+        self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a'])
+        self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b'])
+        self.assertEqual('3', cfg2['sect1']['c'])
+
+    def test_no_section(self):
+        cfg1 = self.fromstring("""
+        a = 1
+        b = 2
+        """)
+
+        self.assertEqual([configparser.UNNAMED_SECTION], cfg1.sections())
+        self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a'])
+        self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b'])
+
+        output = io.StringIO()
+        cfg1.write(output)
+        cfg2 = self.fromstring(output.getvalue())
+
+        self.assertEqual([configparser.UNNAMED_SECTION], cfg2.sections())
+        self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a'])
+        self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b'])
+
+    def test_multiple_configs(self):
+        cfg = configparser.ConfigParser(allow_unnamed_section=True)
+        cfg.read_string('a = 1')
+        cfg.read_string('b = 2')
+
+        self.assertEqual([configparser.UNNAMED_SECTION], cfg.sections())
+        self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a'])
+        self.assertEqual('2', cfg[configparser.UNNAMED_SECTION]['b'])
+
+
 class MiscTestCase(unittest.TestCase):
     def test__all__(self):
         support.check__all__(self, configparser, not_exported={"Error"})