Merge pull request #2360 from RustPython/master

Merge to release

.cargo/config.toml

@@ -1,5 +0,0 @@
-[target.'cfg(target_env = "msvc")']
-rustflags = "-C link-arg=/STACK:8000000"
-
-[target.'cfg(all(target_os = "windows", not(target_env = "msvc")))']
-rustflags = "-C link-args=-Wl,--stack,8000000"

.cspell.json

@@ -1,145 +0,0 @@
-// See: https://github.com/streetsidesoftware/cspell/tree/master/packages/cspell
-{
-  "version": "0.2",
-  // language - current active spelling language
-  "language": "en",
-  // dictionaries - list of the names of the dictionaries to use
-  "dictionaries": [
-    "en_US",
-    "softwareTerms",
-    "c",
-    "cpp",
-    "python",
-    "python-custom",
-    "rust",
-    "unix",
-    "posix",
-    "winapi"
-  ],
-  // dictionaryDefinitions - this list defines any custom dictionaries to use
-  "dictionaryDefinitions": [],
-  "ignorePaths": [
-    "**/__pycache__/**",
-    "Lib/**"
-  ],
-  // words - list of words to be always considered correct
-  "words": [
-    // Rust
-    "ahash",
-    "bitflags",
-    "bindgen",
-    "cstring",
-    "chrono",
-    "peekable",
-    "lalrpop",
-    "memmap",
-    "Manually",
-    "rustc",
-    "unistd",
-    "unic",
-    // Python
-    "cformat",
-    "cpython",
-    "fspath",
-    "kwarg",
-    "kwargs",
-    "vararg",
-    "varargs",
-    "metaclass",
-    "metaclasses",
-    "fstring",
-    "fstrings",
-    "docstring",
-    "docstrings",
-    "fileencoding",
-    "linearization",
-    "linearize",
-    "PYTHONDEBUG",
-    "PYTHONINSPECT",
-    "PYTHONPATH",
-    "PYTHONHOME",
-    "PYTHONPATH",
-    "PYTHONVERBOSE",
-    "PYTHONOPTIMIZE",
-    "PYTHONWARNINGS",
-    "basicsize",
-    "itemsize",
-    "getattro",
-    "setattro",
-    "iternext",
-    "maxsplit",
-    "fdel",
-    "subclasscheck",
-    "qualname",
-    "eventmask",
-    "instanceof",
-    "abstractmethods",
-    "aiter",
-    "anext",
-    "rdiv",
-    "idiv",
-    "ndim",
-    "varnames",
-    "getweakrefs",
-    "getweakrefcount",
-    "stacklevel",
-    "MemoryView",
-    "warningregistry",
-    "defaultaction",
-    "unraisablehook",
-    "descr",
-    "xopts",
-    "warnopts",
-    "weakproxy",
-    "mappingproxy",
-    // RustPython
-    "RustPython",
-    "pyarg",
-    "pyargs",
-    "pygetset",
-    "pyobj",
-    "pystr",
-    "pyc",
-    "pyref",
-    "pyslot",
-    "PyFunction",
-    "PyMethod",
-    "PyClassMethod",
-    "PyStaticMethod",
-    "PyProperty",
-    "PyClass",
-    "pyimpl",
-    "pyarg",
-    "PyModule",
-    "PyAttr",
-    "PyResult",
-    "PyObject",
-    "PyException",
-    "GetSet",
-    "zelf",
-    "wasi",
-    "Bytecode",
-    "richcompare",
-    "makeunicodedata",
-    "unhashable",
-    "unraisable",
-    "typealiases",
-    "Unconstructible",
-    "posonlyargs",
-    "kwonlyargs",
-    "uninit",
-    "miri"
-  ],
-  // flagWords - list of words to be always considered incorrect
-  "flagWords": [
-  ],
-  "ignoreRegExpList": [
-  ],
-  // languageSettings - allow for per programming language configuration settings.
-  "languageSettings": [
-    {
-      "languageId": "python",
-      "locale": "en"
-    }
-  ]
-}

.devcontainer/devcontainer.json

@@ -1,6 +0,0 @@
-{
-  "image": "mcr.microsoft.com/devcontainers/universal:2",
-  "features": {
-    "ghcr.io/devcontainers/features/rust:1": {}
-  }
-}

.flake8

@@ -1,3 +0,0 @@
-[flake8]
-# black's line length
-max-line-length = 88

.gitattributes

@@ -1,8 +1 @@
-Lib/**      linguist-vendored
-Cargo.lock  linguist-generated -merge
-*.snap      linguist-generated -merge
-ast/src/ast_gen.rs          linguist-generated -merge
-vm/src/stdlib/ast/gen.rs    linguist-generated -merge
-compiler/parser/python.lalrpop       text eol=LF
-Lib/*.py    text working-tree-encoding=UTF-8 eol=LF
-**/*.rs     text working-tree-encoding=UTF-8 eol=LF
+Lib/* linguist-vendored

.github/workflows/ci.yaml

@@ -1,231 +1,61 @@
 on:
   push:
-    branches: [main, release]
+    branches: [master, release]
   pull_request:
-    types: [labeled, unlabeled, opened, synchronize, reopened]
 
 name: CI
 
-# Cancel previous workflows if they are the same workflow on same ref (branch/tags)
-# with the same event (push/pull_request) even they are in progress.
-# This setting will help reduce the number of duplicated workflows.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
-  cancel-in-progress: true
-
 env:
-  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
-  NON_WASM_PACKAGES: >-
+  CARGO_ARGS: --features "ssl jit"
+  NON_WASM_PACKAGES: >
+    -p rustpython-bytecode
     -p rustpython-common
-    -p rustpython-compiler-core
     -p rustpython-compiler
-    -p rustpython-codegen
     -p rustpython-parser
     -p rustpython-vm
-    -p rustpython-stdlib
     -p rustpython-jit
     -p rustpython-derive
     -p rustpython
-  PLATFORM_INDEPENDENT_TESTS: >-
-    test_argparse
-    test_array
-    test_asyncgen
-    test_binop
-    test_bisect
-    test_bool
-    test_bytes
-    test_call
-    test_class
-    test_cmath
-    test_collections
-    test_complex
-    test_contains
-    test_copy
-    test_dataclasses
-    test_decimal
-    test_decorators
-    test_defaultdict
-    test_deque
-    test_dict
-    test_dictcomps
-    test_dictviews
-    test_dis
-    test_enumerate
-    test_exception_variations
-    test_exceptions
-    test_float
-    test_format
-    test_fractions
-    test_genericalias
-    test_genericclass
-    test_grammar
-    test_range
-    test_index
-    test_int
-    test_int_literal
-    test_isinstance
-    test_iter
-    test_iterlen
-    test_itertools
-    test_json
-    test_keyword
-    test_keywordonlyarg
-    test_list
-    test_long
-    test_longexp
-    test_math
-    test_operator
-    test_ordered_dict
-    test_pow
-    test_raise
-    test_richcmp
-    test_scope
-    test_set
-    test_slice
-    test_sort
-    test_string
-    test_string_literals
-    test_strtod
-    test_structseq
-    test_subclassinit
-    test_super
-    test_syntax
-    test_tuple
-    test_types
-    test_unary
-    test_unicode
-    test_unpack
-    test_weakref
-    test_yield_from
 
 jobs:
   rust_tests:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    env:
-      RUST_BACKTRACE: full
     name: Run rust tests
-    needs: lalrpop
     runs-on:  ${{ matrix.os }}
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          components: clippy
+      - uses: actions/checkout@master
+      - uses: actions-rs/toolchain@v1
       - name: Set up the Windows environment
-        shell: bash
         run: |
-          choco install llvm openssl
-          echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >>$GITHUB_ENV
+          choco install llvm
+          powershell.exe scripts/symlinks-to-hardlinks.ps1
         if: runner.os == 'Windows'
       - name: Set up the Mac environment
         run: brew install autoconf automake libtool
         if: runner.os == 'macOS'
-      - uses: Swatinem/rust-cache@v1
-
-      - name: run clippy
-        run: cargo clippy ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }} -- -Dwarnings
-
-      - name: run rust tests
-        run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }}
-      - name: check compilation without threading
-        run: cargo check ${{ env.CARGO_ARGS }}
-
-      - name: prepare AppleSilicon build
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          target: aarch64-apple-darwin
-        if: runner.os == 'macOS'
-      - name: Check compilation for Apple Silicon
-        run: cargo check --target aarch64-apple-darwin
-        if: runner.os == 'macOS'
-      - name: prepare iOS build
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          target: aarch64-apple-ios
-        if: runner.os == 'macOS'
-      - name: Check compilation for iOS
-        run: cargo check --target aarch64-apple-ios
-        if: runner.os == 'macOS'
-
-  exotic_targets:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    name: Ensure compilation on various targets
-    needs: lalrpop
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
+      - name: Cache cargo dependencies
         uses: actions/cache@v2
         with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-
-      - uses: dtolnay/rust-toolchain@stable
+          path: |
+              ~/.cargo/registry
+              ~/.cargo/git
+              target
+          key: ${{ runner.os }}-debug_opt3-${{ hashFiles('**/Cargo.lock') }}
+      - name: run rust tests
+        uses: actions-rs/cargo@v1
         with:
-          target: i686-unknown-linux-gnu
-
-      - name: Install gcc-multilib and musl-tools
-        run: sudo apt-get update && sudo apt-get install gcc-multilib musl-tools
-      - name: Check compilation for x86 32bit
-        run: cargo check --target i686-unknown-linux-gnu
-
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          target: aarch64-linux-android
-
-      - name: Check compilation for android
-        run: cargo check --target aarch64-linux-android
-
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          target: i686-unknown-linux-musl
-
-      - name: Check compilation for musl
-        run: cargo check --target i686-unknown-linux-musl
-
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          target: x86_64-unknown-freebsd
-
-      - name: Check compilation for freebsd
-        run: cargo check --target x86_64-unknown-freebsd
-
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          target: wasm32-unknown-unknown
-
-      - name: Check compilation for wasm32
-        run: cargo check --target wasm32-unknown-unknown --no-default-features
-
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          target: x86_64-unknown-freebsd
-
-      - name: Check compilation for freeBSD
-        run: cargo check --target x86_64-unknown-freebsd
-
-      - name: Prepare repository for redox compilation
-        run: bash scripts/redox/uncomment-cargo.sh
-      - name: Check compilation for Redox
-        if: false # FIXME: redoxer toolchain is from ~july 2021, edition2021 isn't stabilized
-        uses: coolreader18/redoxer-action@v1
+          command: test
+          args: --verbose ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }}
+      - name: check compilation without threading
+        uses: actions-rs/cargo@v1
         with:
           command: check
+          args: ${{ env.CARGO_ARGS }} --no-default-features
 
   snippets_cpython:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    needs: lalrpop
-    env:
-      RUST_BACKTRACE: full
     name: Run snippets and cpython tests
     runs-on:  ${{ matrix.os }}
     strategy:
@@ -233,176 +63,152 @@ jobs:
         os: [macos-latest, ubuntu-latest, windows-latest]
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-
-      - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/checkout@master
+      - uses: actions-rs/toolchain@v1
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.10"
+          python-version: 3.8
       - name: Set up the Windows environment
-        shell: bash
         run: |
-          choco install llvm openssl
-          echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >>$GITHUB_ENV
+          choco install llvm
+          powershell.exe scripts/symlinks-to-hardlinks.ps1
         if: runner.os == 'Windows'
       - name: Set up the Mac environment
         run: brew install autoconf automake libtool
         if: runner.os == 'macOS'
-      - uses: Swatinem/rust-cache@v1
-      - name: build rustpython
-        run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }}
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.10"
-      - name: run snippets
-        run: python -m pip install -r requirements.txt && pytest -v
-        working-directory: ./extra_tests
-      - if: runner.os == 'Linux'
-        name: run cpython platform-independent tests
-        run:
-          target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }}
-      - if: runner.os != 'Windows'
-        name: run cpython platform-dependent tests
-        run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }}
-      - if: runner.os == 'Windows'
-        name: run cpython platform-dependent tests (windows partial - fixme)
-        run:
-          target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }}
-            test_glob
-            test_importlib
-            test_io
-            test_iter
-            test_os
-            test_pathlib
-            test_posixpath
-            test_shutil
-            test_venv
-      - if: runner.os != 'Windows'
-        name: check that --install-pip succeeds
-        run: |
-          mkdir site-packages
-          target/release/rustpython --install-pip ensurepip --user
-
-  lalrpop:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    name: Generate parser with lalrpop
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest]
-    runs-on:  ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
+      - name: Cache cargo dependencies
         uses: actions/cache@v2
+        # cache gets corrupted for some reason on mac
+        if: runner.os != 'macOS'
         with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - name: Check if cached generated parser exists
-        id: generated_parser
-        uses: andstor/file-existence-action@v1
+          path: |
+              ~/.cargo/registry
+              ~/.cargo/git
+              target
+          key: ${{ runner.os }}-release-${{ hashFiles('**/Cargo.lock') }}
+      - name: build rustpython
+        uses: actions-rs/cargo@v1
         with:
-          files: "compiler/parser/python.rs"
-      - if: runner.os == 'Windows'
-        name: Force python.lalrpop to be lf  # actions@checkout ignore .gitattributes
+          command: build
+          args: --release --verbose ${{ env.CARGO_ARGS }}
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Install pipenv
         run: |
-          set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
-      - name: Install lalrpop
-        if: steps.generated_parser.outputs.files_exists == 'false'
-        uses: baptiste0928/cargo-install@v1
-        with:
-          crate: lalrpop
-          version: "0.19.8"
-      - name: Run lalrpop
-        if: steps.generated_parser.outputs.files_exists == 'false'
-        run: lalrpop compiler/parser/python.lalrpop
+          python -V
+          python -m pip install --upgrade pip
+          python -m pip install pipenv
+      - run: pipenv install --python 3.8
+        working-directory: ./extra_tests
+      - name: run snippets
+        run: pipenv run pytest -v
+        working-directory: ./extra_tests
+      - name: run cpython tests
+        run: target/release/rustpython -m test -v
+        env:
+          RUSTPYTHONPATH: ${{ github.workspace }}/Lib
+        if: runner.os == 'Linux'
+      - name: run cpython tests (macOS lightweight)
+        run:
+          target/release/rustpython -m test -v -x
+            test_argparse test_json test_bytes test_bytearray test_long test_unicode test_array
+            test_asyncgen test_list test_complex test_json test_set test_dis test_calendar
+        env:
+          RUSTPYTHONPATH: ${{ github.workspace }}/Lib
+        if: runner.os == 'macOS'
+      - name: run cpython tests (windows partial - fixme)
+        run:
+          target/release/rustpython -m test -v -x
+            test_argparse test_json test_bytes test_long test_pwd test_bool test_cgi test_complex
+            test_exception_hierarchy test_glob test_iter test_list test_os test_pathlib
+            test_py_compile test_set test_shutil test_sys test_unicode test_unittest test_venv
+            test_zipimport test_importlib test_io
+        env:
+          RUSTPYTHONPATH: ${{ github.workspace }}/Lib
+        if: runner.os == 'Windows'
 
   lint:
     name: Check Rust code with rustfmt and clippy
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
+      - uses: actions/checkout@master
+      - uses: actions-rs/toolchain@v1
         with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-            components: rustfmt, clippy
+            profile: minimal
+            toolchain: stable
+            components: rustfmt
+            override: true
       - name: run rustfmt
-        run: cargo fmt --all -- --check
-      - name: run clippy on wasm
-        run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
-      - uses: actions/setup-python@v2
+        uses: actions-rs/cargo@v1
         with:
-          python-version: "3.10"
+          command: fmt
+          args: --all -- --check
+      - name: run clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }} -- -Dwarnings
+      - name: run clippy on wasm
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
       - name: install flake8
         run: python -m pip install flake8
       - name: run lint
         run: flake8 . --count --exclude=./.*,./Lib,./vm/Lib,./benches/  --select=E9,F63,F7,F82 --show-source --statistics
-      - name: install prettier
-        run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH
-      - name: check wasm code with prettier
-        # prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506
-        run: cd wasm && git ls-files -z | xargs -0 prettier --check -u
-      - name: Check update_asdl.sh consistency
-        run: bash scripts/update_asdl.sh && git diff --exit-code
-      - name: Check whats_left is not broken
-        run: python -I whats_left.py
-
   miri:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Run tests under miri
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - uses: dtolnay/rust-toolchain@master
+      - uses: actions/checkout@master
+      - uses: actions-rs/toolchain@v1
         with:
+            profile: minimal
             toolchain: nightly
             components: miri
-      - uses: Swatinem/rust-cache@v1
+            override: true
       - name: Run tests under miri
         # miri-ignore-leaks because the type-object circular reference means that there will always be
         # a memory leak, at least until we have proper cyclic gc
         run: MIRIFLAGS='-Zmiri-ignore-leaks' cargo +nightly miri test -p rustpython-vm -- miri_test
 
   wasm:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Check the WASM package and demo
-    needs: lalrpop
+    needs: rust_tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
+      - uses: actions/checkout@master
+      - name: Cache cargo dependencies
         uses: actions/cache@v2
         with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v1
+          path: |
+              ~/.cargo/registry
+              ~/.cargo/git
+              target
+          key: ${{ runner.os }}-wasm_opt3-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+               ${{ runner.os }}-debug_opt3-${{ hashFiles('**/Cargo.lock') }}
       - name: install wasm-pack
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: install geckodriver
         run: |
-          wget https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz
+          wget https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckodriver-v0.24.0-linux32.tar.gz
           mkdir geckodriver
-          tar -xzf geckodriver-v0.30.0-linux64.tar.gz -C geckodriver
-      - uses: actions/setup-python@v2
+          tar -xzf geckodriver-v0.24.0-linux32.tar.gz -C geckodriver
+      - uses: actions/setup-python@v1
         with:
-          python-version: "3.10"
-      - run: python -m pip install -r requirements.txt
+          python-version: 3.8
+      - name: Install pipenv
+        run: |
+          python -V
+          python -m pip install --upgrade pip
+          python -m pip install pipenv
+      - run: pipenv install
         working-directory: ./wasm/tests
       - uses: actions/setup-node@v1
       - name: run test
@@ -427,27 +233,3 @@ jobs:
           EXTERNAL_REPOSITORY: RustPython/demo
           PUBLISH_BRANCH: master
 
-  wasm-wasi:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    name: Run snippets and cpython tests on wasm-wasi
-    needs: lalrpop
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          target: wasm32-wasi
-      - uses: Swatinem/rust-cache@v1
-      - name: Setup Wasmer
-        uses: wasmerio/setup-wasmer@v1
-      - name: Install clang
-        run: sudo apt-get update && sudo apt-get install clang -y
-      - name: build rustpython
-        run: cargo build --release --target wasm32-wasi --features freeze-stdlib,stdlib --verbose
-      - name: run snippets
-        run: wasmer run --dir . target/wasm32-wasi/release/rustpython.wasm -- extra_tests/snippets/stdlib_random.py

.github/workflows/cron-ci.yaml

@@ -1,79 +1,72 @@
 on:
   schedule:
     - cron: '0 0 * * 6'
-  workflow_dispatch:
-
-name: Periodic checks/tasks
-
-env:
-  CARGO_ARGS: --features ssl,jit
 
 jobs:
+  redox:
+    name: Check compilation on Redox
+    runs-on: ubuntu-latest
+    container:
+      image: redoxos/redoxer:latest
+    steps:
+      - uses: actions/checkout@master
+      - name: prepare repository for redoxer compilation
+        run: bash scripts/redox/uncomment-cargo.sh
+      - name: compile for redox
+        run: redoxer build --verbose
+
   codecov:
     name: Collect code coverage data
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
+      - uses: actions/checkout@master
+      - uses: actions-rs/toolchain@v1
         with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - uses: dtolnay/rust-toolchain@stable
+          toolchain: nightly
+          override: true
+      - uses: actions-rs/cargo@v1
         with:
-          components: llvm-tools-preview
-      - run: sudo apt-get update && sudo apt-get -y install lcov
-      - run: cargo build --release --verbose ${{ env.CARGO_ARGS }}
+          command: build
+          args: --verbose
         env:
-          RUSTC_WRAPPER: './scripts/codecoverage-rustc-wrapper.sh'
-      - uses: actions/setup-python@v2
+          CARGO_INCREMENTAL: '0'
+          RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests' # -Cpanic=abort
+      - uses: actions/setup-python@v1
         with:
-          python-version: "3.10"
-      - run: python -m pip install pytest
+          python-version: 3.8
+      - name: Install pipenv
+        run: |
+          python -V
+          python -m pip install --upgrade pip
+          python -m pip install pipenv
+      - run: pipenv install
         working-directory: ./extra_tests
       - name: run snippets
-        run: LLVM_PROFILE_FILE="$PWD/snippet-%p.profraw" pytest -v
+        run: pipenv run pytest -v
         working-directory: ./extra_tests
-        continue-on-error: true
+        env:
+          RUSTPYTHON_DEBUG: 'true'
       - name: run cpython tests
-        run: |
-          alltests=($(target/release/rustpython -c 'from test.libregrtest.runtest import findtests; print(*findtests())'))
-          i=0
-          # chunk into chunks of 10 tests each. idk at this point
-          while subtests=("${alltests[@]:$i:10}"); [[ ${#subtests[@]} -ne 0 ]]; do
-            LLVM_PROFILE_FILE="$PWD/regrtest-%p.profraw" target/release/rustpython -m test -v "${subtests[@]}" || true
-            ((i+=10))
-          done
-        continue-on-error: true
-      - name: prepare code coverage data
-        run: |
-          rusttool() {
-            local tool=$1; shift; "$(rustc --print target-libdir)/../bin/llvm-$tool" "$@"
-          }
-          rusttool profdata merge extra_tests/snippet-*.profraw regrtest-*.profraw --output codecov.profdata
-          rusttool cov export --instr-profile codecov.profdata target/release/rustpython --format lcov > codecov_tmp.lcov
-          lcov -e codecov_tmp.lcov "$PWD"/'*' -o codecov_tmp2.lcov
-          lcov -r codecov_tmp2.lcov "$PWD"/target/'*' -o codecov.lcov  # remove LALRPOP-generated parser
+        run: cargo run -- -m test -v
+        env:
+          RUSTPYTHONPATH: ${{ github.workspace }}/Lib
+      - uses: actions-rs/grcov@v0.1
+        id: coverage
       - name: upload to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v1
         with:
-          file: ./codecov.lcov
+          file: ${{ steps.coverage.outputs.report }}
 
   testdata:
     name: Collect regression test data
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/checkout@master
       - name: build rustpython
-        run: cargo build --release --verbose
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --release --verbose
       - name: collect tests data
         run: cargo run --release extra_tests/jsontests.py
         env:
@@ -91,127 +84,5 @@ jobs:
           cd website
           cp ../extra_tests/cpython_tests_results.json ./_data/regrtests_results.json
           git add ./_data/regrtests_results.json
-          if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results" --author="$GITHUB_ACTOR"; then
-            git push
-          fi
-
-  whatsleft:
-    name: Collect what is left data
-    needs: lalrpop
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - uses: dtolnay/rust-toolchain@stable
-      - name: build rustpython
-        run: cargo build --release --verbose
-      - name: Collect what is left data
-        run: |
-          chmod +x ./whats_left.py
-          ./whats_left.py > whats_left.temp
-        env:
-          RUSTPYTHONPATH: ${{ github.workspace }}/Lib
-      - name: Upload data to the website
-        env:
-          SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
-          GITHUB_ACTOR: ${{ github.actor }}
-        run: |
-          echo "$SSHKEY" >~/github_key
-          chmod 600 ~/github_key
-          export GIT_SSH_COMMAND="ssh -i ~/github_key"
-
-          git clone git@github.com:RustPython/rustpython.github.io.git website
-          cd website
-          [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp
-          cp ../whats_left.temp ./_data/whats_left.temp
-          git add -A
-          if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then
-            git push
-          fi
-
-  benchmark:
-    name: Collect benchmark data
-    needs: lalrpop
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - uses: dtolnay/rust-toolchain@stable
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - run: cargo install cargo-criterion
-      - name: build benchmarks
-        run: cargo build --release --benches
-      - name: collect execution benchmark data
-        run: cargo criterion --bench execution
-      - name: collect microbenchmarks data
-        run: cargo criterion --bench microbenchmarks
-      - name: restructure generated files
-        run: |
-          cd ./target/criterion/reports
-          find -type d -name cpython | xargs rm -rf
-          find -type d -name rustpython | xargs rm -rf
-          find -mindepth 2 -maxdepth 2 -name violin.svg | xargs rm -rf
-          find -type f -not -name violin.svg | xargs rm -rf
-          for file in $(find -type f -name violin.svg); do mv $file $(echo $file | sed -E "s_\./([^/]+)/([^/]+)/violin\.svg_./\1/\2.svg_"); done
-          find -mindepth 2 -maxdepth 2 -type d | xargs rm -rf
-          cd ..
-          mv reports/* .
-          rmdir reports
-      - name: upload benchmark data to the website
-        env:
-          SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
-        run: |
-          echo "$SSHKEY" >~/github_key
-          chmod 600 ~/github_key
-          export GIT_SSH_COMMAND="ssh -i ~/github_key"
-
-          git clone git@github.com:RustPython/rustpython.github.io.git website
-          cd website
-          rm -rf ./assets/criterion
-          cp -r ../target/criterion ./assets/criterion
-          git add ./assets/criterion
-          if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then
-            git push
-          fi
-
-  lalrpop:
-    name: Generate parser with lalrpop
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest]
-    runs-on:  ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - name: Check if cached generated parser exists
-        id: generated_parser
-        uses: andstor/file-existence-action@v1
-        with:
-          files: "compiler/parser/python.rs"
-      - if: runner.os == 'Windows'
-        name: Force python.lalrpop to be lf  # actions@checkout ignore .gitattributes
-        run: |
-          set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
-      - name: Install lalrpop
-        if: steps.generated_parser.outputs.files_exists == 'false'
-        uses: baptiste0928/cargo-install@v1
-        with:
-          crate: lalrpop
-          version: "0.19.8"
-      - name: Run lalrpop
-        if: steps.generated_parser.outputs.files_exists == 'false'
-        run: lalrpop compiler/parser/python.lalrpop
+          git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results" --author="$GITHUB_ACTOR"
+          git push

.gitignore

@@ -9,6 +9,7 @@ __pycache__
 .vscode
 wasm-pack.log
 .idea/
+extra_tests/snippets/resources
 
 flame-graph.html
 flame.txt
@@ -16,8 +17,3 @@ flamescope.json
 /wapm.lock
 /wapm_packages
 /.cargo/config
-
-extra_tests/snippets/resources
-extra_tests/not_impl.py
-
-compiler/parser/python.rs

.mailmap

@@ -1,8 +0,0 @@
-#
-# This list is used by git-shortlog to aggregate contributions.  It is
-# necessary when either the author's full name is not always written
-# the same way, and/or the same author contributes from different
-# email addresses.
-#
-
-Noa <coolreader18@gmail.com> <33094578+coolreader18@users.noreply.github.com>

.vscode/launch.json

@@ -1,289 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug executable 'rustpython'",
-            "cargo": {
-                "args": [
-                    "build",
-                    "--package=rustpython"
-                ],
-            },
-            "preLaunchTask": "Build RustPython Debug",
-            "program": "target/debug/rustpython",
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython"
-                ],
-                "filter": {
-                    "name": "rustpython",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug benchmark 'execution'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--bench=execution",
-                    "--package=rustpython"
-                ],
-                "filter": {
-                    "name": "execution",
-                    "kind": "bench"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug benchmark 'microbenchmarks'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--bench=microbenchmarks",
-                    "--package=rustpython"
-                ],
-                "filter": {
-                    "name": "microbenchmarks",
-                    "kind": "bench"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython-pylib'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython-pylib"
-                ],
-                "filter": {
-                    "name": "rustpython-pylib",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython-bytecode'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython-bytecode"
-                ],
-                "filter": {
-                    "name": "rustpython-bytecode",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython-compiler'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython-compiler"
-                ],
-                "filter": {
-                    "name": "rustpython-compiler",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython-compiler-core'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython-compiler-core"
-                ],
-                "filter": {
-                    "name": "rustpython-compiler-core",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython-ast'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython-ast"
-                ],
-                "filter": {
-                    "name": "rustpython-ast",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython-parser'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython-parser"
-                ],
-                "filter": {
-                    "name": "rustpython-parser",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython-vm'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython-vm"
-                ],
-                "filter": {
-                    "name": "rustpython-vm",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython-common'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython-common"
-                ],
-                "filter": {
-                    "name": "rustpython-common",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython-jit'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython-jit"
-                ],
-                "filter": {
-                    "name": "rustpython-jit",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug integration test 'integration'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--test=integration",
-                    "--package=rustpython-jit"
-                ],
-                "filter": {
-                    "name": "integration",
-                    "kind": "test"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        },
-        {
-            "type": "lldb",
-            "request": "launch",
-            "name": "Debug unit tests in library 'rustpython_wasm'",
-            "cargo": {
-                "args": [
-                    "test",
-                    "--no-run",
-                    "--lib",
-                    "--package=rustpython_wasm"
-                ],
-                "filter": {
-                    "name": "rustpython_wasm",
-                    "kind": "lib"
-                }
-            },
-            "args": [],
-            "cwd": "${workspaceFolder}"
-        }
-    ]
-}

.vscode/tasks.json

@@ -1,20 +0,0 @@
-{
-    "version": "2.0.0",
-    "tasks": [
-        {
-            "label": "Build RustPython Debug",
-            "type": "shell",
-            "command": "cargo",
-            "args": [
-                "build",
-            ],
-            "problemMatcher": [
-                "$rustc",
-            ],
-            "group": {
-                "kind": "build",
-                "isDefault": true,
-            },
-        }
-    ],
-}

Cargo.toml

@@ -1,62 +1,50 @@
-# REDOX START
-# cargo-features = ["edition2021"]
-# REDOX END
 [package]
 name = "rustpython"
-version = "0.2.0"
+version = "0.1.2"
 authors = ["RustPython Team"]
-edition = "2021"
+edition = "2018"
 description = "A python interpreter written in rust."
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 
 [workspace]
-resolver = "2"
 members = [
-    "compiler", "compiler/ast", "compiler/core", "compiler/codegen", "compiler/parser",
-    ".", "common", "derive", "jit", "vm", "pylib", "stdlib", "wasm/lib", "derive-impl",
+    ".", "ast", "bytecode", "common", "compiler", "compiler/porcelain",
+    "derive", "jit", "parser", "vm", "vm/pylib-crate", "wasm/lib",
 ]
 
 [features]
-default = ["threading", "stdlib", "zlib", "importlib", "encodings", "rustpython-parser/lalrpop"]
-importlib = ["rustpython-vm/importlib"]
-encodings = ["rustpython-vm/encodings"]
-stdlib = ["rustpython-stdlib", "rustpython-pylib"]
+default = ["threading", "pylib"]
 flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
-freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
+freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
-threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
-zlib = ["stdlib", "rustpython-stdlib/zlib"]
-bz2 = ["stdlib", "rustpython-stdlib/bz2"]
-ssl = ["rustpython-stdlib/ssl"]
-ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
+threading = ["rustpython-vm/threading"]
+
+ssl = ["rustpython-vm/ssl"]
 
 [dependencies]
-rustpython-compiler = { path = "compiler", version = "0.2.0" }
-rustpython-parser = { path = "compiler/parser", version = "0.2.0" }
-rustpython-pylib = { path = "pylib", optional = true, default-features = false }
-rustpython-stdlib = { path = "stdlib", optional = true, default-features = false }
-rustpython-vm = { path = "vm", version = "0.2.0", default-features = false, features = ["compiler"] }
+log = "0.4"
+env_logger = "0.7"
+clap = "2.33"
+rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" }
+rustpython-parser = { path = "parser", version = "0.1.1" }
+rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
+pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1.0", default-features = false, optional = true }
+dirs = { package = "dirs-next", version = "1.0" }
+num-traits = "0.2.8"
+cfg-if = "0.1"
+libc = "0.2"
 
-cfg-if = "1.0.0"
-clap = "2.34"
-dirs = { package = "dirs-next", version = "2.0.0" }
-env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
-flame = { version = "0.2.2", optional = true }
-flamescope = { version = "0.1.2", optional = true }
-libc = "0.2.133"
-log = "0.4.16"
-num-traits = "0.2.14"
-atty = "0.2.14"
+flame = { version = "0.2", optional = true }
+flamescope = { version = "0.1", optional = true }
 
-[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-rustyline = "10.0.0"
+[target.'cfg(not(target_os = "wasi"))'.dependencies]
+rustyline = "6.0"
 
 [dev-dependencies]
-cpython = "0.7.0"
-criterion = "0.3.5"
-python3-sys = "0.7.0"
+cpython = "0.5.0"
+criterion = "0.3"
 
 [[bench]]
 name = "execution"
@@ -73,19 +61,16 @@ path = "src/main.rs"
 [profile.dev.package."*"]
 opt-level = 3
 
-[profile.test]
-opt-level = 3
-# https://github.com/rust-lang/rust/issues/92869
-# lto = "thin"
-
 [profile.bench]
-lto = "thin"
+lto = true
 codegen-units = 1
 opt-level = 3
 
-[profile.release]
-lto = "thin"
-
 [patch.crates-io]
 # REDOX START, Uncommment when you want to compile/check with redoxer
+# # following patches are just waiting on a new version to be released to crates.io
+# nix = { git = "https://github.com/nix-rust/nix" }
+# crossbeam-utils = { git = "https://github.com/crossbeam-rs/crossbeam" }
+# socket2 = { git = "https://github.com/alexcrichton/socket2-rs" }
 # REDOX END
+

DEVELOPMENT.md

@@ -19,13 +19,13 @@ The contents of the Development Guide include:
 
 RustPython requires the following:
 
-- Rust latest stable version (e.g 1.51.0 as of Apr 2 2021)
+- Rust latest stable version (e.g 1.38.0 at Oct 1st 2019)
     - To check Rust version: `rustc --version` 
     - If you have `rustup` on your system, enter to update to the latest
       stable version: `rustup update stable`
     - If you do not have Rust installed, use [rustup](https://rustup.rs/) to
       do so.
-- CPython version 3.10 or higher
+- CPython version 3.7.4 or higher
     - CPython can be installed by your operating system's package manager,
       from the [Python website](https://www.python.org/downloads/), or
       using a third-party distribution, such as 
@@ -41,12 +41,11 @@ RustPython requires the following:
 
 The Rust code style used is the default
 [rustfmt](https://github.com/rust-lang/rustfmt) codestyle. Please format your
-code accordingly, or run `cargo fmt` to autoformat it. We also use
-[clippy](https://github.com/rust-lang/rust-clippy) to lint Rust code, which
-you can check yourself with `cargo clippy`.
+code accordingly. We also use [clippy](https://github.com/rust-lang/rust-clippy)
+to detect rust code issues. 
 
-Custom Python code (i.e. code not copied from CPython's standard library) should
-follow the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style. We also use
+Python code should follow the 
+[PEP 8](https://www.python.org/dev/peps/pep-0008/) style. We also use
 [flake8](http://flake8.pycqa.org/en/latest/) to check Python code style.
 
 ## Testing
@@ -62,34 +61,7 @@ $ pytest -v
 Rust unit tests can be run with `cargo`:
 
 ```shell
-$ cargo test --workspace --exclude rustpython_wasm
-```
-
-Python unit tests can be run by compiling RustPython and running the test module:
-
-```shell
-$ cargo run --release -- -m test
-```
-
-There are a few test options that are especially useful:
-
-- `-j <n>` enables parallel testing (which is a lot faster), where `<n>` is the
-number of threads to be used, ideally the same as number of cores on your CPU.
-If you don't know, `-j 4` or `-j 8` are good options.
-- `-v` enables verbose mode, adding additional information about the tests being
-run.
-- `<test_name>` specifies a single test to run instead of running all tests.
-
-For example, to run all tests in parallel:
-
-```shell
-$ cargo run --release -- -m test -j 4
-```
-
-To run only `test_cmath` (located at `Lib/test/test_cmath`) verbosely:
-
-```shell
-$ cargo run --release -- -m test test_cmath -v
+$ cargo test --all
 ```
 
 ## Profiling
@@ -115,29 +87,35 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i
 Understanding a new codebase takes time. Here's a brief view of the
 repository's structure:
 
+- `bytecode/src`: python bytecode representation in rust structures
 - `compiler/src`: python compilation to bytecode
-  - `core/src`: python bytecode representation in rust structures
-  - `parser/src`: python lexing, parsing and ast
 - `derive/src`: Rust language extensions and macros specific to rustpython
+- `parser/src`: python lexing, parsing and ast
 - `Lib`: Carefully selected / copied files from CPython sourcecode. This is
    the python side of the standard library.
   - `test`: CPython test suite
 - `vm/src`: python virtual machine
-  - `builtins`: Builtin functions and types
+  - `builtins.rs`: Builtin functions
+  - `compile.rs`: the python compiler from ast to bytecode
+  - `obj`: python builtin types
   - `stdlib`: Standard library parts implemented in rust.
 - `src`: using the other subcrates to bring rustpython to life.
+- `docs`: documentation (work in progress)
+- `py_code_object`: CPython bytecode to rustpython bytecode converter (work in
+  progress)
 - `wasm`: Binary crate and resources for WebAssembly build
-- `extra_tests`: extra integration test snippets as a supplement to `Lib/test`
+- `extra_tests`: extra integration test snippets as supplement of `Lib/test`
 
 ## Understanding Internals
 
 The RustPython workspace includes the `rustpython` top-level crate. The `Cargo.toml`
 file in the root of the repo provide configuration of the crate and the
-implementation is found in the `src` directory (specifically, `src/lib.rs`).
+implementation is found in the `src` directory (specifically, 
+`src/main.rs`).
 
 The top-level `rustpython` binary depends on several lower-level crates including:
 
-- `rustpython-parser` (implementation in `compiler/parser/src`)
+- `rustpython-parser` (implementation in `parser/src`)
 - `rustpython-compiler` (implementation in `compiler/src`)
 - `rustpython-vm` (implementation in `vm/src`)
 
@@ -155,26 +133,25 @@ enable a line of code to go through a series of steps:
 This crate contains the lexer and parser to convert a line of code to
 an Abstract Syntax Tree (AST):
 
-- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens
-- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses
+- Lexer: `parser/lexer.rs` converts Python source code into tokens
+- Parser: `parser/parser.rs` takes the tokens generated by the lexer and parses
   the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax
   tree are Rust structs and enums.
-  - The Parser relies on `LALRPOP`, a Rust parser generator framework. The
-    LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`.
+  - The Parser relies on `LALRPOP`, a Rust parser generator framework.
   - More information on parsers and a tutorial can be found in the 
-    [LALRPOP book](https://lalrpop.github.io/lalrpop/).
-- AST: `compiler/ast/` implements in Rust the Python types and expressions
+    [LALRPOP book](https://lalrpop.github.io/lalrpop/README.html).
+- AST: `parser/ast.rs` implements in Rust the Python types and expressions
   represented by the AST nodes.
 
 ### rustpython-compiler
 
 The `rustpython-compiler` crate's purpose is to transform the AST (Abstract Syntax
 Tree) to bytecode. The implementation of the compiler is found in the
-`compiler/src` directory. The compiler implements Python's symbol table,
-ast->bytecode compiler, and bytecode optimizer in Rust.
+`compiler/src` directory. The compiler implements Python's peephole optimizer
+implementation, Symbol table, and streams in Rust.
 
-Implementation of bytecode structure in Rust is found in the `compiler/core/src`
-directory. `compiler/core/src/bytecode.rs` contains the representation of
+Implementation of bytecode structure in Rust is found in the `bytecode/src`
+directory. The `bytecode/src/bytecode.rs` contains the representation of
 instructions and operations in Rust. Further information about Python's
 bytecode instructions can be found in the
 [Python documentation](https://docs.python.org/3/library/dis.html#bytecodes).
@@ -186,20 +163,10 @@ executes Python's instructions. The `vm/src` directory contains code to
 implement the read and evaluation loop that fetches and dispatches
 instructions. This directory also contains the implementation of the
 Python Standard Library modules in Rust (`vm/src/stdlib`). In Python
-everything can be represented as an object. The `vm/src/builtins` directory holds
-the Rust code used to represent different Python objects and their methods. The
-core implementation of what a Python object is can be found in
-`vm/src/object/core.rs`.
-
-### Code generation
-
-There are some code generations involved in building RustPython:
-
-- some part of the AST code is generated from `vm/src/stdlib/ast/gen.rs` to `compiler/ast/src/ast_gen.rs`.
-- the `__doc__` attributes are generated by the 
-  [__doc__](https://github.com/RustPython/__doc__) project which is then included as the `rustpython-doc` crate.
+everything can be represented as an Object. `vm/src/obj` directory holds
+the Rust code used to represent a Python Object and its methods.
 
 ## Questions
 
 Have you tried these steps and have a question, please chat with us on
-[Discord](https://discord.gg/vru8NypEhv).
+[gitter](https://gitter.im/rustpython/Lobby).

Lib/__future__.py

@@ -42,7 +42,7 @@ CompilerFlag is the (bitfield) flag that should be passed in the fourth
 argument to the builtin function compile() to enable the feature in
 dynamically compiled code.  This flag is stored in the .compiler_flag
 attribute on _Future instances.  These values must match the appropriate
-#defines of CO_xxx flags in Include/cpython/compile.h.
+#defines of CO_xxx flags in Include/compile.h.
 
 No feature line is ever to be deleted from this file.
 """
@@ -57,29 +57,25 @@ all_feature_names = [
     "unicode_literals",
     "barry_as_FLUFL",
     "generator_stop",
-    "annotations",
 ]
 
 __all__ = ["all_feature_names"] + all_feature_names
 
-# The CO_xxx symbols are defined here under the same names defined in
-# code.h and used by compile.h, so that an editor search will find them here.
-# However, they're not exported in __all__, because they don't really belong to
+# The CO_xxx symbols are defined here under the same names used by
+# compile.h, so that an editor search will find them here.  However,
+# they're not exported in __all__, because they don't really belong to
 # this module.
-CO_NESTED = 0x0010                      # nested_scopes
-CO_GENERATOR_ALLOWED = 0                # generators (obsolete, was 0x1000)
-CO_FUTURE_DIVISION = 0x20000            # division
-CO_FUTURE_ABSOLUTE_IMPORT = 0x40000     # perform absolute imports by default
-CO_FUTURE_WITH_STATEMENT = 0x80000      # with statement
-CO_FUTURE_PRINT_FUNCTION = 0x100000     # print function
-CO_FUTURE_UNICODE_LITERALS = 0x200000   # unicode string literals
-CO_FUTURE_BARRY_AS_BDFL = 0x400000
-CO_FUTURE_GENERATOR_STOP = 0x800000     # StopIteration becomes RuntimeError in generators
-CO_FUTURE_ANNOTATIONS = 0x1000000       # annotations become strings at runtime
-
+CO_NESTED            = 0x0010   # nested_scopes
+CO_GENERATOR_ALLOWED = 0        # generators (obsolete, was 0x1000)
+CO_FUTURE_DIVISION   = 0x2000   # division
+CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
+CO_FUTURE_WITH_STATEMENT  = 0x8000   # with statement
+CO_FUTURE_PRINT_FUNCTION  = 0x10000   # print function
+CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
+CO_FUTURE_BARRY_AS_BDFL = 0x40000
+CO_FUTURE_GENERATOR_STOP  = 0x80000 # StopIteration becomes RuntimeError in generators
 
 class _Feature:
-
     def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
         self.optional = optionalRelease
         self.mandatory = mandatoryRelease
@@ -90,6 +86,7 @@ class _Feature:
 
         This is a 5-tuple, of the same form as sys.version_info.
         """
+
         return self.optional
 
     def getMandatoryRelease(self):
@@ -98,6 +95,7 @@ class _Feature:
         This is a 5-tuple, of the same form as sys.version_info, or, if
         the feature was dropped, is None.
         """
+
         return self.mandatory
 
     def __repr__(self):
@@ -105,7 +103,6 @@ class _Feature:
                                   self.mandatory,
                                   self.compiler_flag))
 
-
 nested_scopes = _Feature((2, 1, 0, "beta",  1),
                          (2, 2, 0, "alpha", 0),
                          CO_NESTED)
@@ -135,13 +132,9 @@ unicode_literals = _Feature((2, 6, 0, "alpha", 2),
                             CO_FUTURE_UNICODE_LITERALS)
 
 barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2),
-                          (4, 0, 0, "alpha", 0),
-                          CO_FUTURE_BARRY_AS_BDFL)
+                         (3, 9, 0, "alpha", 0),
+                         CO_FUTURE_BARRY_AS_BDFL)
 
 generator_stop = _Feature((3, 5, 0, "beta", 1),
-                          (3, 7, 0, "alpha", 0),
-                          CO_FUTURE_GENERATOR_STOP)
-
-annotations = _Feature((3, 7, 0, "beta", 1),
-                       (3, 11, 0, "alpha", 0),
-                       CO_FUTURE_ANNOTATIONS)
+                         (3, 7, 0, "alpha", 0),
+                         CO_FUTURE_GENERATOR_STOP)

Lib/_collections_abc.py

@@ -9,12 +9,6 @@ Unit tests are in test_collections.
 from abc import ABCMeta, abstractmethod
 import sys
 
-GenericAlias = type(list[int])
-EllipsisType = type(...)
-def _f(): pass
-FunctionType = type(_f)
-del _f
-
 __all__ = ["Awaitable", "Coroutine",
            "AsyncIterable", "AsyncIterator", "AsyncGenerator",
            "Hashable", "Iterable", "Iterator", "Generator", "Reversible",
@@ -73,7 +67,7 @@ async_generator = type(_ag)
 del _ag
 
 
-### ONE-TRICK PONIES ###
+# ## ONE-TRICK PONIES ###
 
 def _check_methods(C, *methods):
     mro = C.__mro__
@@ -116,8 +110,6 @@ class Awaitable(metaclass=ABCMeta):
             return _check_methods(C, "__await__")
         return NotImplemented
 
-    __class_getitem__ = classmethod(GenericAlias)
-
 
 class Coroutine(Awaitable):
 
@@ -177,8 +169,6 @@ class AsyncIterable(metaclass=ABCMeta):
             return _check_methods(C, "__aiter__")
         return NotImplemented
 
-    __class_getitem__ = classmethod(GenericAlias)
-
 
 class AsyncIterator(AsyncIterable):
 
@@ -265,8 +255,6 @@ class Iterable(metaclass=ABCMeta):
             return _check_methods(C, "__iter__")
         return NotImplemented
 
-    __class_getitem__ = classmethod(GenericAlias)
-
 
 class Iterator(Iterable):
 
@@ -286,10 +274,9 @@ class Iterator(Iterable):
             return _check_methods(C, '__iter__', '__next__')
         return NotImplemented
 
-
 Iterator.register(bytes_iterator)
 Iterator.register(bytearray_iterator)
-#Iterator.register(callable_iterator)
+# Iterator.register(callable_iterator)
 Iterator.register(dict_keyiterator)
 Iterator.register(dict_valueiterator)
 Iterator.register(dict_itemiterator)
@@ -366,10 +353,8 @@ class Generator(Iterator):
                                   'send', 'throw', 'close')
         return NotImplemented
 
-
 Generator.register(generator)
 
-
 class Sized(metaclass=ABCMeta):
 
     __slots__ = ()
@@ -399,9 +384,6 @@ class Container(metaclass=ABCMeta):
             return _check_methods(C, "__contains__")
         return NotImplemented
 
-    __class_getitem__ = classmethod(GenericAlias)
-
-
 class Collection(Sized, Iterable, Container):
 
     __slots__ = ()
@@ -412,141 +394,6 @@ class Collection(Sized, Iterable, Container):
             return _check_methods(C,  "__len__", "__iter__", "__contains__")
         return NotImplemented
 
-
-class _CallableGenericAlias(GenericAlias):
-    """ Represent `Callable[argtypes, resulttype]`.
-
-    This sets ``__args__`` to a tuple containing the flattened ``argtypes``
-    followed by ``resulttype``.
-
-    Example: ``Callable[[int, str], float]`` sets ``__args__`` to
-    ``(int, str, float)``.
-    """
-
-    __slots__ = ()
-
-    def __new__(cls, origin, args):
-        if not (isinstance(args, tuple) and len(args) == 2):
-            raise TypeError(
-                "Callable must be used as Callable[[arg, ...], result].")
-        t_args, t_result = args
-        if isinstance(t_args, list):
-            args = (*t_args, t_result)
-        elif not _is_param_expr(t_args):
-            raise TypeError(f"Expected a list of types, an ellipsis, "
-                            f"ParamSpec, or Concatenate. Got {t_args}")
-        return super().__new__(cls, origin, args)
-
-    @property
-    def __parameters__(self):
-        params = []
-        for arg in self.__args__:
-            # Looks like a genericalias
-            if hasattr(arg, "__parameters__") and isinstance(arg.__parameters__, tuple):
-                params.extend(arg.__parameters__)
-            else:
-                if _is_typevarlike(arg):
-                    params.append(arg)
-        return tuple(dict.fromkeys(params))
-
-    def __repr__(self):
-        if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]):
-            return super().__repr__()
-        return (f'collections.abc.Callable'
-                f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], '
-                f'{_type_repr(self.__args__[-1])}]')
-
-    def __reduce__(self):
-        args = self.__args__
-        if not (len(args) == 2 and _is_param_expr(args[0])):
-            args = list(args[:-1]), args[-1]
-        return _CallableGenericAlias, (Callable, args)
-
-    def __getitem__(self, item):
-        # Called during TypeVar substitution, returns the custom subclass
-        # rather than the default types.GenericAlias object.  Most of the
-        # code is copied from typing's _GenericAlias and the builtin
-        # types.GenericAlias.
-
-        # A special case in PEP 612 where if X = Callable[P, int],
-        # then X[int, str] == X[[int, str]].
-        param_len = len(self.__parameters__)
-        if param_len == 0:
-            raise TypeError(f'{self} is not a generic class')
-        if not isinstance(item, tuple):
-            item = (item,)
-        if (param_len == 1 and _is_param_expr(self.__parameters__[0])
-                and item and not _is_param_expr(item[0])):
-            item = (list(item),)
-        item_len = len(item)
-        if item_len != param_len:
-            raise TypeError(f'Too {"many" if item_len > param_len else "few"}'
-                            f' arguments for {self};'
-                            f' actual {item_len}, expected {param_len}')
-        subst = dict(zip(self.__parameters__, item))
-        new_args = []
-        for arg in self.__args__:
-            if _is_typevarlike(arg):
-                if _is_param_expr(arg):
-                    arg = subst[arg]
-                    if not _is_param_expr(arg):
-                        raise TypeError(f"Expected a list of types, an ellipsis, "
-                                        f"ParamSpec, or Concatenate. Got {arg}")
-                else:
-                    arg = subst[arg]
-            # Looks like a GenericAlias
-            elif hasattr(arg, '__parameters__') and isinstance(arg.__parameters__, tuple):
-                subparams = arg.__parameters__
-                if subparams:
-                    subargs = tuple(subst[x] for x in subparams)
-                    arg = arg[subargs]
-            new_args.append(arg)
-
-        # args[0] occurs due to things like Z[[int, str, bool]] from PEP 612
-        if not isinstance(new_args[0], list):
-            t_result = new_args[-1]
-            t_args = new_args[:-1]
-            new_args = (t_args, t_result)
-        return _CallableGenericAlias(Callable, tuple(new_args))
-
-
-def _is_typevarlike(arg):
-    obj = type(arg)
-    # looks like a TypeVar/ParamSpec
-    return (obj.__module__ == 'typing'
-            and obj.__name__ in {'ParamSpec', 'TypeVar'})
-
-def _is_param_expr(obj):
-    """Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or
-    ``_ConcatenateGenericAlias`` from typing.py
-    """
-    if obj is Ellipsis:
-        return True
-    if isinstance(obj, list):
-        return True
-    obj = type(obj)
-    names = ('ParamSpec', '_ConcatenateGenericAlias')
-    return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names)
-
-def _type_repr(obj):
-    """Return the repr() of an object, special-casing types (internal helper).
-
-    Copied from :mod:`typing` since collections.abc
-    shouldn't depend on that module.
-    """
-    if isinstance(obj, GenericAlias):
-        return repr(obj)
-    if isinstance(obj, type):
-        if obj.__module__ == 'builtins':
-            return obj.__qualname__
-        return f'{obj.__module__}.{obj.__qualname__}'
-    if obj is Ellipsis:
-        return '...'
-    if isinstance(obj, FunctionType):
-        return obj.__name__
-    return repr(obj)
-
-
 class Callable(metaclass=ABCMeta):
 
     __slots__ = ()
@@ -561,13 +408,12 @@ class Callable(metaclass=ABCMeta):
             return _check_methods(C, "__call__")
         return NotImplemented
 
-    __class_getitem__ = classmethod(_CallableGenericAlias)
-
 
 ### SETS ###
 
 
 class Set(Collection):
+
     """A set is a finite, iterable container.
 
     This class provides concrete generic implementations of all
@@ -695,7 +541,6 @@ class Set(Collection):
             hx = hash(x)
             h ^= (hx ^ (hx << 16) ^ 89869747)  * 3644798167
             h &= MASK
-        h ^= (h >> 11) ^ (h >> 25)
         h = h * 69069 + 907133923
         h &= MASK
         if h > MAX:
@@ -704,7 +549,6 @@ class Set(Collection):
             h = 590923713
         return h
 
-
 Set.register(frozenset)
 
 
@@ -787,25 +631,24 @@ class MutableSet(Set):
                 self.discard(value)
         return self
 
-
 MutableSet.register(set)
 
 
 ### MAPPINGS ###
 
+
 class Mapping(Collection):
+
+    __slots__ = ()
+
     """A Mapping is a generic container for associating key/value
     pairs.
 
     This class provides concrete generic implementations of all
     methods except for __getitem__, __iter__, and __len__.
+
     """
 
-    __slots__ = ()
-
-    # Tell ABCMeta.__new__ that this class should have TPFLAGS_MAPPING set.
-    __abc_tpflags__ = 1 << 6 # Py_TPFLAGS_MAPPING
-
     @abstractmethod
     def __getitem__(self, key):
         raise KeyError
@@ -860,8 +703,6 @@ class MappingView(Sized):
     def __repr__(self):
         return '{0.__class__.__name__}({0._mapping!r})'.format(self)
 
-    __class_getitem__ = classmethod(GenericAlias)
-
 
 class KeysView(MappingView, Set):
 
@@ -877,7 +718,6 @@ class KeysView(MappingView, Set):
     def __iter__(self):
         yield from self._mapping
 
-
 KeysView.register(dict_keys)
 
 
@@ -902,7 +742,6 @@ class ItemsView(MappingView, Set):
         for key in self._mapping:
             yield (key, self._mapping[key])
 
-
 ItemsView.register(dict_items)
 
 
@@ -921,20 +760,21 @@ class ValuesView(MappingView, Collection):
         for key in self._mapping:
             yield self._mapping[key]
 
-
 ValuesView.register(dict_values)
 
 
 class MutableMapping(Mapping):
+
+    __slots__ = ()
+
     """A MutableMapping is a generic container for associating
     key/value pairs.
 
     This class provides concrete generic implementations of all
     methods except for __getitem__, __setitem__, __delitem__,
     __iter__, and __len__.
-    """
 
-    __slots__ = ()
+    """
 
     @abstractmethod
     def __setitem__(self, key, value):
@@ -980,21 +820,34 @@ class MutableMapping(Mapping):
         except KeyError:
             pass
 
-    def update(self, other=(), /, **kwds):
+    def update(*args, **kwds):
         ''' D.update([E, ]**F) -> None.  Update D from mapping/iterable E and F.
             If E present and has a .keys() method, does:     for k in E: D[k] = E[k]
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''
-        if isinstance(other, Mapping):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, "keys"):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
+        if not args:
+            raise TypeError("descriptor 'update' of 'MutableMapping' object "
+                            "needs an argument")
+        self, *args = args
+        if len(args) > 1:
+            raise TypeError('update expected at most 1 arguments, got %d' %
+                            len(args))
+        if args:
+            other = args[0]
+            try:
+                mapping_inst = isinstance(other, Mapping)
+            except TypeError:
+                mapping_inst = False
+            if mapping_inst:
+                for key in other:
+                    self[key] = other[key]
+            elif hasattr(other, "keys"):
+                for key in other.keys():
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
         for key, value in kwds.items():
             self[key] = value
 
@@ -1006,13 +859,14 @@ class MutableMapping(Mapping):
             self[key] = default
         return default
 
-
 MutableMapping.register(dict)
 
 
 ### SEQUENCES ###
 
+
 class Sequence(Reversible, Collection):
+
     """All the operations on a read-only sequence.
 
     Concrete subclasses must override __new__ or __init__,
@@ -1021,9 +875,6 @@ class Sequence(Reversible, Collection):
 
     __slots__ = ()
 
-    # Tell ABCMeta.__new__ that this class should have TPFLAGS_SEQUENCE set.
-    __abc_tpflags__ = 1 << 5 # Py_TPFLAGS_SEQUENCE
-
     @abstractmethod
     def __getitem__(self, index):
         raise IndexError
@@ -1082,6 +933,7 @@ Sequence.register(memoryview)
 
 
 class ByteString(Sequence):
+
     """This unifies bytes and bytearray.
 
     XXX Should add all their methods.
@@ -1094,13 +946,15 @@ ByteString.register(bytearray)
 
 
 class MutableSequence(Sequence):
+
+    __slots__ = ()
+
     """All the operations on a read-write sequence.
 
     Concrete subclasses must provide __new__ or __init__,
     __getitem__, __setitem__, __delitem__, __len__, and insert().
-    """
 
-    __slots__ = ()
+    """
 
     @abstractmethod
     def __setitem__(self, index, value):
@@ -1158,6 +1012,5 @@ class MutableSequence(Sequence):
         self.extend(values)
         return self
 
-
 MutableSequence.register(list)
 MutableSequence.register(bytearray)  # Multiply inheriting, see ByteString

Lib/_compat_pickle.py

@@ -148,14 +148,6 @@ except NameError:
 else:
     PYTHON2_EXCEPTIONS += ("WindowsError",)
 
-# NOTE: RUSTPYTHON exceptions
-try:
-    JitError
-except NameError:
-    pass
-else:
-    PYTHON2_EXCEPTIONS += ("JitError",)
-
 for excname in PYTHON2_EXCEPTIONS:
     NAME_MAPPING[("exceptions", excname)] = ("builtins", excname)
 

Lib/_compression.py

@@ -1,152 +0,0 @@
-"""Internal classes used by the gzip, lzma and bz2 modules"""
-
-import io
-
-
-BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE  # Compressed data read chunk size
-
-
-class BaseStream(io.BufferedIOBase):
-    """Mode-checking helper functions."""
-
-    def _check_not_closed(self):
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-
-    def _check_can_read(self):
-        if not self.readable():
-            raise io.UnsupportedOperation("File not open for reading")
-
-    def _check_can_write(self):
-        if not self.writable():
-            raise io.UnsupportedOperation("File not open for writing")
-
-    def _check_can_seek(self):
-        if not self.readable():
-            raise io.UnsupportedOperation("Seeking is only supported "
-                                          "on files open for reading")
-        if not self.seekable():
-            raise io.UnsupportedOperation("The underlying file object "
-                                          "does not support seeking")
-
-
-class DecompressReader(io.RawIOBase):
-    """Adapts the decompressor API to a RawIOBase reader API"""
-
-    def readable(self):
-        return True
-
-    def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
-        self._fp = fp
-        self._eof = False
-        self._pos = 0  # Current offset in decompressed stream
-
-        # Set to size of decompressed stream once it is known, for SEEK_END
-        self._size = -1
-
-        # Save the decompressor factory and arguments.
-        # If the file contains multiple compressed streams, each
-        # stream will need a separate decompressor object. A new decompressor
-        # object is also needed when implementing a backwards seek().
-        self._decomp_factory = decomp_factory
-        self._decomp_args = decomp_args
-        self._decompressor = self._decomp_factory(**self._decomp_args)
-
-        # Exception class to catch from decompressor signifying invalid
-        # trailing data to ignore
-        self._trailing_error = trailing_error
-
-    def close(self):
-        self._decompressor = None
-        return super().close()
-
-    def seekable(self):
-        return self._fp.seekable()
-
-    def readinto(self, b):
-        with memoryview(b) as view, view.cast("B") as byte_view:
-            data = self.read(len(byte_view))
-            byte_view[:len(data)] = data
-        return len(data)
-
-    def read(self, size=-1):
-        if size < 0:
-            return self.readall()
-
-        if not size or self._eof:
-            return b""
-        data = None  # Default if EOF is encountered
-        # Depending on the input data, our call to the decompressor may not
-        # return any data. In this case, try again after reading another block.
-        while True:
-            if self._decompressor.eof:
-                rawblock = (self._decompressor.unused_data or
-                            self._fp.read(BUFFER_SIZE))
-                if not rawblock:
-                    break
-                # Continue to next stream.
-                self._decompressor = self._decomp_factory(
-                    **self._decomp_args)
-                try:
-                    data = self._decompressor.decompress(rawblock, size)
-                except self._trailing_error:
-                    # Trailing data isn't a valid compressed stream; ignore it.
-                    break
-            else:
-                if self._decompressor.needs_input:
-                    rawblock = self._fp.read(BUFFER_SIZE)
-                    if not rawblock:
-                        raise EOFError("Compressed file ended before the "
-                                       "end-of-stream marker was reached")
-                else:
-                    rawblock = b""
-                data = self._decompressor.decompress(rawblock, size)
-            if data:
-                break
-        if not data:
-            self._eof = True
-            self._size = self._pos
-            return b""
-        self._pos += len(data)
-        return data
-
-    # Rewind the file to the beginning of the data stream.
-    def _rewind(self):
-        self._fp.seek(0)
-        self._eof = False
-        self._pos = 0
-        self._decompressor = self._decomp_factory(**self._decomp_args)
-
-    def seek(self, offset, whence=io.SEEK_SET):
-        # Recalculate offset as an absolute file position.
-        if whence == io.SEEK_SET:
-            pass
-        elif whence == io.SEEK_CUR:
-            offset = self._pos + offset
-        elif whence == io.SEEK_END:
-            # Seeking relative to EOF - we need to know the file's size.
-            if self._size < 0:
-                while self.read(io.DEFAULT_BUFFER_SIZE):
-                    pass
-            offset = self._size + offset
-        else:
-            raise ValueError("Invalid value for whence: {}".format(whence))
-
-        # Make it so that offset is the number of bytes to skip forward.
-        if offset < self._pos:
-            self._rewind()
-        else:
-            offset -= self._pos
-
-        # Read and discard data until we reach the desired position.
-        while offset > 0:
-            data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset))
-            if not data:
-                break
-            offset -= len(data)
-
-        return self._pos
-
-    def tell(self):
-        """Return the current file position."""
-        return self._pos

Lib/_dummy_os.py

@@ -1,66 +0,0 @@
-"""
-A shim of the os module containing only simple path-related utilities
-"""
-
-try:
-    from os import *
-except ImportError:
-    import abc
-
-    def __getattr__(name):
-        raise OSError("no os specific module found")
-
-    def _shim():
-        import _dummy_os, sys
-        sys.modules['os'] = _dummy_os
-        sys.modules['os.path'] = _dummy_os.path
-
-    import posixpath as path
-    import sys
-    sys.modules['os.path'] = path
-    del sys
-
-    sep = path.sep
-
-
-    def fspath(path):
-        """Return the path representation of a path-like object.
-
-        If str or bytes is passed in, it is returned unchanged. Otherwise the
-        os.PathLike interface is used to get the path representation. If the
-        path representation is not str or bytes, TypeError is raised. If the
-        provided path is not str, bytes, or os.PathLike, TypeError is raised.
-        """
-        if isinstance(path, (str, bytes)):
-            return path
-
-        # Work from the object's type to match method resolution of other magic
-        # methods.
-        path_type = type(path)
-        try:
-            path_repr = path_type.__fspath__(path)
-        except AttributeError:
-            if hasattr(path_type, '__fspath__'):
-                raise
-            else:
-                raise TypeError("expected str, bytes or os.PathLike object, "
-                                "not " + path_type.__name__)
-        if isinstance(path_repr, (str, bytes)):
-            return path_repr
-        else:
-            raise TypeError("expected {}.__fspath__() to return str or bytes, "
-                            "not {}".format(path_type.__name__,
-                                            type(path_repr).__name__))
-
-    class PathLike(abc.ABC):
-
-        """Abstract base class for implementing the file system path protocol."""
-
-        @abc.abstractmethod
-        def __fspath__(self):
-            """Return the file system path representation of the object."""
-            raise NotImplementedError
-
-        @classmethod
-        def __subclasshook__(cls, subclass):
-            return hasattr(subclass, '__fspath__')

Lib/_dummy_thread.py

@@ -14,8 +14,7 @@ Suggested usage is::
 # Exports only things specified by thread documentation;
 # skipping obsolete synonyms allocate(), start_new(), exit_thread().
 __all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
-           'interrupt_main', 'LockType', 'RLock',
-           '_count']
+           'interrupt_main', 'LockType', 'RLock']
 
 # A dummy value
 TIMEOUT_MAX = 2**31
@@ -86,10 +85,6 @@ def _set_sentinel():
     """Dummy implementation of _thread._set_sentinel()."""
     return LockType()
 
-def _count():
-    """Dummy implementation of _thread._count()."""
-    return 0
-
 class LockType(object):
     """Class implementing dummy implementation of _thread.LockType.
 

Lib/_osx_support.py

@@ -52,7 +52,7 @@ def _find_executable(executable, path=None):
         return executable
 
 
-def _read_output(commandstring, capture_stderr=False):
+def _read_output(commandstring):
     """Output from successful command execution or None"""
     # Similar to os.popen(commandstring, "r").read(),
     # but without actually using os.popen because that
@@ -67,10 +67,7 @@ def _read_output(commandstring, capture_stderr=False):
             os.getpid(),), "w+b")
 
     with contextlib.closing(fp) as fp:
-        if capture_stderr:
-            cmd = "%s >'%s' 2>&1" % (commandstring, fp.name)
-        else:
-            cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
+        cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
         return fp.read().decode('utf-8').strip() if not os.system(cmd) else None
 
 
@@ -96,7 +93,7 @@ def _get_system_version():
     if _SYSTEM_VERSION is None:
         _SYSTEM_VERSION = ''
         try:
-            f = open('/System/Library/CoreServices/SystemVersion.plist', encoding="utf-8")
+            f = open('/System/Library/CoreServices/SystemVersion.plist')
         except OSError:
             # We're on a plain darwin box, fall back to the default
             # behaviour.
@@ -113,26 +110,6 @@ def _get_system_version():
 
     return _SYSTEM_VERSION
 
-_SYSTEM_VERSION_TUPLE = None
-def _get_system_version_tuple():
-    """
-    Return the macOS system version as a tuple
-
-    The return value is safe to use to compare
-    two version numbers.
-    """
-    global _SYSTEM_VERSION_TUPLE
-    if _SYSTEM_VERSION_TUPLE is None:
-        osx_version = _get_system_version()
-        if osx_version:
-            try:
-                _SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.'))
-            except ValueError:
-                _SYSTEM_VERSION_TUPLE = ()
-
-    return _SYSTEM_VERSION_TUPLE
-
-
 def _remove_original_values(_config_vars):
     """Remove original unmodified values for testing"""
     # This is needed for higher-level cross-platform tests of get_platform.
@@ -148,33 +125,6 @@ def _save_modified_value(_config_vars, cv, newvalue):
         _config_vars[_INITPRE + cv] = oldvalue
     _config_vars[cv] = newvalue
 
-
-_cache_default_sysroot = None
-def _default_sysroot(cc):
-    """ Returns the root of the default SDK for this system, or '/' """
-    global _cache_default_sysroot
-
-    if _cache_default_sysroot is not None:
-        return _cache_default_sysroot
-
-    contents = _read_output('%s -c -E -v - </dev/null' % (cc,), True)
-    in_incdirs = False
-    for line in contents.splitlines():
-        if line.startswith("#include <...>"):
-            in_incdirs = True
-        elif line.startswith("End of search list"):
-            in_incdirs = False
-        elif in_incdirs:
-            line = line.strip()
-            if line == '/usr/include':
-                _cache_default_sysroot = '/'
-            elif line.endswith(".sdk/usr/include"):
-                _cache_default_sysroot = line[:-12]
-    if _cache_default_sysroot is None:
-        _cache_default_sysroot = '/'
-
-    return _cache_default_sysroot
-
 def _supports_universal_builds():
     """Returns True if universal builds are supported on this system"""
     # As an approximation, we assume that if we are running on 10.4 or above,
@@ -182,18 +132,14 @@ def _supports_universal_builds():
     # builds, in particular -isysroot and -arch arguments to the compiler. This
     # is in support of allowing 10.4 universal builds to run on 10.3.x systems.
 
-    osx_version = _get_system_version_tuple()
+    osx_version = _get_system_version()
+    if osx_version:
+        try:
+            osx_version = tuple(int(i) for i in osx_version.split('.'))
+        except ValueError:
+            osx_version = ''
     return bool(osx_version >= (10, 4)) if osx_version else False
 
-def _supports_arm64_builds():
-    """Returns True if arm64 builds are supported on this system"""
-    # There are two sets of systems supporting macOS/arm64 builds:
-    # 1. macOS 11 and later, unconditionally
-    # 2. macOS 10.15 with Xcode 12.2 or later
-    # For now the second category is ignored.
-    osx_version = _get_system_version_tuple()
-    return osx_version >= (11, 0) if osx_version else False
-
 
 def _find_appropriate_compiler(_config_vars):
     """Find appropriate C compiler for extension module builds"""
@@ -265,7 +211,7 @@ def _remove_universal_flags(_config_vars):
         if cv in _config_vars and cv not in os.environ:
             flags = _config_vars[cv]
             flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII)
-            flags = re.sub(r'-isysroot\s*\S+', ' ', flags)
+            flags = re.sub('-isysroot [^ \t]*', ' ', flags)
             _save_modified_value(_config_vars, cv, flags)
 
     return _config_vars
@@ -341,7 +287,7 @@ def _check_for_unavailable_sdk(_config_vars):
     # to /usr and /System/Library by either a standalone CLT
     # package or the CLT component within Xcode.
     cflags = _config_vars.get('CFLAGS', '')
-    m = re.search(r'-isysroot\s*(\S+)', cflags)
+    m = re.search(r'-isysroot\s+(\S+)', cflags)
     if m is not None:
         sdk = m.group(1)
         if not os.path.exists(sdk):
@@ -349,7 +295,7 @@ def _check_for_unavailable_sdk(_config_vars):
                 # Do not alter a config var explicitly overridden by env var
                 if cv in _config_vars and cv not in os.environ:
                     flags = _config_vars[cv]
-                    flags = re.sub(r'-isysroot\s*\S+(?:\s|$)', ' ', flags)
+                    flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags)
                     _save_modified_value(_config_vars, cv, flags)
 
     return _config_vars
@@ -374,7 +320,7 @@ def compiler_fixup(compiler_so, cc_args):
         stripArch = stripSysroot = True
     else:
         stripArch = '-arch' in cc_args
-        stripSysroot = any(arg for arg in cc_args if arg.startswith('-isysroot'))
+        stripSysroot = '-isysroot' in cc_args
 
     if stripArch or 'ARCHFLAGS' in os.environ:
         while True:
@@ -385,12 +331,6 @@ def compiler_fixup(compiler_so, cc_args):
             except ValueError:
                 break
 
-    elif not _supports_arm64_builds():
-        # Look for "-arch arm64" and drop that
-        for idx in reversed(range(len(compiler_so))):
-            if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64":
-                del compiler_so[idx:idx+2]
-
     if 'ARCHFLAGS' in os.environ and not stripArch:
         # User specified different -arch flags in the environ,
         # see also distutils.sysconfig
@@ -398,39 +338,29 @@ def compiler_fixup(compiler_so, cc_args):
 
     if stripSysroot:
         while True:
-            indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
-            if not indices:
-                break
-            index = indices[0]
-            if compiler_so[index] == '-isysroot':
+            try:
+                index = compiler_so.index('-isysroot')
                 # Strip this argument and the next one:
                 del compiler_so[index:index+2]
-            else:
-                # It's '-isysroot/some/path' in one arg
-                del compiler_so[index:index+1]
+            except ValueError:
+                break
 
     # Check if the SDK that is used during compilation actually exists,
     # the universal build requires the usage of a universal SDK and not all
     # users have that installed by default.
     sysroot = None
-    argvar = cc_args
-    indices = [i for i,x in enumerate(cc_args) if x.startswith('-isysroot')]
-    if not indices:
-        argvar = compiler_so
-        indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
-
-    for idx in indices:
-        if argvar[idx] == '-isysroot':
-            sysroot = argvar[idx+1]
-            break
-        else:
-            sysroot = argvar[idx][len('-isysroot'):]
-            break
+    if '-isysroot' in cc_args:
+        idx = cc_args.index('-isysroot')
+        sysroot = cc_args[idx+1]
+    elif '-isysroot' in compiler_so:
+        idx = compiler_so.index('-isysroot')
+        sysroot = compiler_so[idx+1]
 
     if sysroot and not os.path.isdir(sysroot):
-        sys.stderr.write(f"Compiling with an SDK that doesn't seem to exist: {sysroot}\n")
-        sys.stderr.write("Please check your Xcode installation\n")
-        sys.stderr.flush()
+        from distutils import log
+        log.warn("Compiling with an SDK that doesn't seem to exist: %s",
+                sysroot)
+        log.warn("Please check your Xcode installation")
 
     return compiler_so
 
@@ -481,7 +411,7 @@ def customize_compiler(_config_vars):
 
     This customization is performed when the first
     extension module build is requested
-    in distutils.sysconfig.customize_compiler.
+    in distutils.sysconfig.customize_compiler).
     """
 
     # Find a compiler to use for extension module builds
@@ -524,10 +454,10 @@ def get_platform_osx(_config_vars, osname, release, machine):
             try:
                 macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
             except ValueError:
-                macrelease = (10, 3)
+                macrelease = (10, 0)
         else:
             # assume no universal support
-            macrelease = (10, 3)
+            macrelease = (10, 0)
 
         if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
             # The universal build will build fat binaries, but not on
@@ -540,8 +470,6 @@ def get_platform_osx(_config_vars, osname, release, machine):
 
             if len(archs) == 1:
                 machine = archs[0]
-            elif archs == ('arm64', 'x86_64'):
-                machine = 'universal2'
             elif archs == ('i386', 'ppc'):
                 machine = 'fat'
             elif archs == ('i386', 'x86_64'):

Lib/_py_abc.py

@@ -32,7 +32,7 @@ class ABCMeta(type):
     #       external code.
     _abc_invalidation_counter = 0
 
-    def __new__(mcls, name, bases, namespace, /, **kwargs):
+    def __new__(mcls, name, bases, namespace, **kwargs):
         cls = super().__new__(mcls, name, bases, namespace, **kwargs)
         # Compute set of abstract method names
         abstracts = {name
@@ -43,7 +43,7 @@ class ABCMeta(type):
                 value = getattr(cls, name, None)
                 if getattr(value, "__isabstractmethod__", False):
                     abstracts.add(name)
-        cls.__abstractmethods__ = frozenset(abstracts)
+        cls.__abstractmethods__ = set(abstracts)
         # Set up inheritance registry
         cls._abc_registry = WeakSet()
         cls._abc_cache = WeakSet()

Lib/_pyio.py

@@ -36,40 +36,8 @@ BlockingIOError = BlockingIOError
 # Does io.IOBase finalizer log the exception if the close() method fails?
 # The exception is ignored silently by default in release build.
 _IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode)
-# Does open() check its 'errors' argument?
-_CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE
 
 
-def text_encoding(encoding, stacklevel=2):
-    """
-    A helper function to choose the text encoding.
-
-    When encoding is not None, just return it.
-    Otherwise, return the default text encoding (i.e. "locale").
-
-    This function emits an EncodingWarning if *encoding* is None and
-    sys.flags.warn_default_encoding is true.
-
-    This can be used in APIs with an encoding=None parameter
-    that pass it to TextIOWrapper or open.
-    However, please consider using encoding="utf-8" for new APIs.
-    """
-    if encoding is None:
-        encoding = "locale"
-        if sys.flags.warn_default_encoding:
-            import warnings
-            warnings.warn("'encoding' argument not specified.",
-                          EncodingWarning, stacklevel + 1)
-    return encoding
-
-
-# Wrapper for builtins.open
-#
-# Trick so that open() won't become a bound method when stored
-# as a class variable (as dbm.dumb does).
-#
-# See init_set_builtins_open() in Python/pylifecycle.c.
-@staticmethod
 def open(file, mode="r", buffering=-1, encoding=None, errors=None,
          newline=None, closefd=True, opener=None):
 
@@ -278,7 +246,6 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
         result = buffer
         if binary:
             return result
-        encoding = text_encoding(encoding)
         text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
         result = text
         text.mode = mode
@@ -311,20 +278,27 @@ except AttributeError:
     open_code = _open_code_with_warning
 
 
-def __getattr__(name):
-    if name == "OpenWrapper":
-        # bpo-43680: Until Python 3.9, _pyio.open was not a static method and
-        # builtins.open was set to OpenWrapper to not become a bound method
-        # when set to a class variable. _io.open is a built-in function whereas
-        # _pyio.open is a Python function. In Python 3.10, _pyio.open() is now
-        # a static method, and builtins.open() is now io.open().
-        import warnings
-        warnings.warn('OpenWrapper is deprecated, use open instead',
-                      DeprecationWarning, stacklevel=2)
-        global OpenWrapper
-        OpenWrapper = open
-        return OpenWrapper
-    raise AttributeError(name)
+class DocDescriptor:
+    """Helper for builtins.open.__doc__
+    """
+    def __get__(self, obj, typ=None):
+        return (
+            "open(file, mode='r', buffering=-1, encoding=None, "
+                 "errors=None, newline=None, closefd=True)\n\n" +
+            open.__doc__)
+
+class OpenWrapper:
+    """Wrapper for builtins.open
+
+    Trick so that open won't become a bound method when stored
+    as a class variable (as dbm.dumb does).
+
+    See initstdio() in Python/pylifecycle.c.
+    """
+    __doc__ = DocDescriptor()
+
+    def __new__(cls, *args, **kwargs):
+        return open(*args, **kwargs)
 
 
 # In normal operation, both `UnsupportedOperation`s should be bound to the
@@ -828,9 +802,6 @@ class _BufferedIOMixin(BufferedIOBase):
         return pos
 
     def truncate(self, pos=None):
-        self._checkClosed()
-        self._checkWritable()
-
         # Flush the stream.  We're mixing buffered I/O with lower-level I/O,
         # and a flush may be necessary to synch both views of the current
         # file state.
@@ -1600,7 +1571,7 @@ class FileIO(RawIOBase):
                     raise IsADirectoryError(errno.EISDIR,
                                             os.strerror(errno.EISDIR), file)
             except AttributeError:
-                # Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR
+                # Ignore the AttribueError if stat.S_ISDIR or errno.EISDIR
                 # don't exist.
                 pass
             self._blksize = getattr(fdfstat, 'st_blksize', 0)
@@ -2028,22 +1999,19 @@ class TextIOWrapper(TextIOBase):
     def __init__(self, buffer, encoding=None, errors=None, newline=None,
                  line_buffering=False, write_through=False):
         self._check_newline(newline)
-        encoding = text_encoding(encoding)
-
-        if encoding == "locale":
+        if encoding is None:
             try:
-                encoding = os.device_encoding(buffer.fileno()) or "locale"
+                encoding = os.device_encoding(buffer.fileno())
             except (AttributeError, UnsupportedOperation):
                 pass
-
-        if encoding == "locale":
-            try:
-                import locale
-            except ImportError:
-                # Importing locale may fail if Python is being built
-                encoding = "utf-8"
-            else:
-                encoding = locale.getpreferredencoding(False)
+            if encoding is None:
+                try:
+                    import locale
+                except ImportError:
+                    # Importing locale may fail if Python is being built
+                    encoding = "ascii"
+                else:
+                    encoding = locale.getpreferredencoding(False)
 
         if not isinstance(encoding, str):
             raise ValueError("invalid encoding: %r" % encoding)
@@ -2058,8 +2026,6 @@ class TextIOWrapper(TextIOBase):
         else:
             if not isinstance(errors, str):
                 raise ValueError("invalid errors: %r" % errors)
-            if _CHECK_ERRORS:
-                codecs.lookup_error(errors)
 
         self._buffer = buffer
         self._decoded_chars = ''  # buffer for text returned from decoder
@@ -2329,7 +2295,7 @@ class TextIOWrapper(TextIOBase):
         return not eof
 
     def _pack_cookie(self, position, dec_flags=0,
-                           bytes_to_feed=0, need_eof=False, chars_to_skip=0):
+                           bytes_to_feed=0, need_eof=0, chars_to_skip=0):
         # The meaning of a tell() cookie is: seek to position, set the
         # decoder flags to dec_flags, read bytes_to_feed bytes, feed them
         # into the decoder with need_eof as the EOF flag, then skip
@@ -2343,7 +2309,7 @@ class TextIOWrapper(TextIOBase):
         rest, dec_flags = divmod(rest, 1<<64)
         rest, bytes_to_feed = divmod(rest, 1<<64)
         need_eof, chars_to_skip = divmod(rest, 1<<64)
-        return position, dec_flags, bytes_to_feed, bool(need_eof), chars_to_skip
+        return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip
 
     def tell(self):
         if not self._seekable:
@@ -2417,7 +2383,7 @@ class TextIOWrapper(TextIOBase):
             # (a point where the decoder has nothing buffered, so seek()
             # can safely start from there and advance to this location).
             bytes_fed = 0
-            need_eof = False
+            need_eof = 0
             # Chars decoded since `start_pos`
             chars_decoded = 0
             for i in range(skip_bytes, len(next_input)):
@@ -2434,7 +2400,7 @@ class TextIOWrapper(TextIOBase):
             else:
                 # We didn't get enough decoded data; signal EOF to get more.
                 chars_decoded += len(decoder.decode(b'', final=True))
-                need_eof = True
+                need_eof = 1
                 if chars_decoded < chars_to_skip:
                     raise OSError("can't reconstruct logical file position")
 

Lib/_weakrefset.py

@@ -3,7 +3,6 @@
 # by abc.py to load everything else at startup.
 
 from _weakref import ref
-from types import GenericAlias
 
 __all__ = ['WeakSet']
 
@@ -51,14 +50,10 @@ class WeakSet:
             self.update(data)
 
     def _commit_removals(self):
-        pop = self._pending_removals.pop
+        l = self._pending_removals
         discard = self.data.discard
-        while True:
-            try:
-                item = pop()
-            except IndexError:
-                return
-            discard(item)
+        while l:
+            discard(l.pop())
 
     def __iter__(self):
         with _IterationGuard(self):
@@ -202,5 +197,3 @@ class WeakSet:
 
     def __repr__(self):
         return repr(self.data)
-
-    __class_getitem__ = classmethod(GenericAlias)

Lib/abc.py

@@ -11,8 +11,7 @@ def abstractmethod(funcobj):
     class that has a metaclass derived from ABCMeta cannot be
     instantiated unless all of its abstract methods are overridden.
     The abstract methods can be called using any of the normal
-    'super' call mechanisms.  abstractmethod() may be used to declare
-    abstract methods for properties and descriptors.
+    'super' call mechanisms.
 
     Usage:
 
@@ -28,14 +27,17 @@ def abstractmethod(funcobj):
 class abstractclassmethod(classmethod):
     """A decorator indicating abstract classmethods.
 
-    Deprecated, use 'classmethod' with 'abstractmethod' instead:
+    Similar to abstractmethod.
 
-        class C(ABC):
-            @classmethod
-            @abstractmethod
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractclassmethod
             def my_abstract_classmethod(cls, ...):
                 ...
 
+    'abstractclassmethod' is deprecated. Use 'classmethod' with
+    'abstractmethod' instead.
     """
 
     __isabstractmethod__ = True
@@ -48,14 +50,17 @@ class abstractclassmethod(classmethod):
 class abstractstaticmethod(staticmethod):
     """A decorator indicating abstract staticmethods.
 
-    Deprecated, use 'staticmethod' with 'abstractmethod' instead:
+    Similar to abstractmethod.
 
-        class C(ABC):
-            @staticmethod
-            @abstractmethod
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractstaticmethod
             def my_abstract_staticmethod(...):
                 ...
 
+    'abstractstaticmethod' is deprecated. Use 'staticmethod' with
+    'abstractmethod' instead.
     """
 
     __isabstractmethod__ = True
@@ -68,14 +73,29 @@ class abstractstaticmethod(staticmethod):
 class abstractproperty(property):
     """A decorator indicating abstract properties.
 
-    Deprecated, use 'property' with 'abstractmethod' instead:
+    Requires that the metaclass is ABCMeta or derived from it.  A
+    class that has a metaclass derived from ABCMeta cannot be
+    instantiated unless all of its abstract properties are overridden.
+    The abstract properties can be called using any of the normal
+    'super' call mechanisms.
 
-        class C(ABC):
-            @property
-            @abstractmethod
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractproperty
             def my_abstract_property(self):
                 ...
 
+    This defines a read-only property; you can also define a read-write
+    abstract property using the 'long' form of property declaration:
+
+        class C(metaclass=ABCMeta):
+            def getx(self): ...
+            def setx(self, value): ...
+            x = abstractproperty(getx, setx)
+
+    'abstractproperty' is deprecated. Use 'property' with 'abstractmethod'
+    instead.
     """
 
     __isabstractmethod__ = True
@@ -85,10 +105,6 @@ try:
     from _abc import (get_cache_token, _abc_init, _abc_register,
                       _abc_instancecheck, _abc_subclasscheck, _get_dump,
                       _reset_registry, _reset_caches)
-# TODO: RUSTPYTHON missing _abc module implementation.
-except ModuleNotFoundError:
-    from _py_abc import ABCMeta, get_cache_token
-    ABCMeta.__module__ = 'abc'
 except ImportError:
     from _py_abc import ABCMeta, get_cache_token
     ABCMeta.__module__ = 'abc'
@@ -147,44 +163,6 @@ else:
             _reset_caches(cls)
 
 
-def update_abstractmethods(cls):
-    """Recalculate the set of abstract methods of an abstract class.
-
-    If a class has had one of its abstract methods implemented after the
-    class was created, the method will not be considered implemented until
-    this function is called. Alternatively, if a new abstract method has been
-    added to the class, it will only be considered an abstract method of the
-    class after this function is called.
-
-    This function should be called before any use is made of the class,
-    usually in class decorators that add methods to the subject class.
-
-    Returns cls, to allow usage as a class decorator.
-
-    If cls is not an instance of ABCMeta, does nothing.
-    """
-    if not hasattr(cls, '__abstractmethods__'):
-        # We check for __abstractmethods__ here because cls might by a C
-        # implementation or a python implementation (especially during
-        # testing), and we want to handle both cases.
-        return cls
-
-    abstracts = set()
-    # Check the existing abstract methods of the parents, keep only the ones
-    # that are not implemented.
-    for scls in cls.__bases__:
-        for name in getattr(scls, '__abstractmethods__', ()):
-            value = getattr(cls, name, None)
-            if getattr(value, "__isabstractmethod__", False):
-                abstracts.add(name)
-    # Also add any other newly added abstract methods.
-    for name, value in cls.__dict__.items():
-        if getattr(value, "__isabstractmethod__", False):
-            abstracts.add(name)
-    cls.__abstractmethods__ = frozenset(abstracts)
-    return cls
-
-
 class ABC(metaclass=ABCMeta):
     """Helper class that provides a standard way to create an ABC using
     inheritance.

Lib/argparse.py

@@ -1,5 +1,4 @@
 # Author: Steven J. Bethard <steven.bethard@gmail.com>.
-# New maintainer as of 29 August 2019:  Raymond Hettinger <raymond.hettinger@gmail.com>
 
 """Command-line parsing library
 
@@ -67,7 +66,6 @@ __all__ = [
     'ArgumentParser',
     'ArgumentError',
     'ArgumentTypeError',
-    'BooleanOptionalAction',
     'FileType',
     'HelpFormatter',
     'ArgumentDefaultsHelpFormatter',
@@ -129,7 +127,7 @@ class _AttributeHolder(object):
         return '%s(%s)' % (type_name, ', '.join(arg_strings))
 
     def _get_kwargs(self):
-        return list(self.__dict__.items())
+        return sorted(self.__dict__.items())
 
     def _get_args(self):
         return []
@@ -166,12 +164,15 @@ class HelpFormatter(object):
 
         # default setting for width
         if width is None:
-            import shutil
-            width = shutil.get_terminal_size().columns
+            try:
+                width = int(_os.environ['COLUMNS'])
+            except (KeyError, ValueError):
+                width = 80
             width -= 2
 
         self._prog = prog
         self._indent_increment = indent_increment
+        self._max_help_position = max_help_position
         self._max_help_position = min(max_help_position,
                                       max(width - 20, indent_increment * 2))
         self._width = width
@@ -264,7 +265,7 @@ class HelpFormatter(object):
                 invocations.append(get_invocation(subaction))
 
             # update the maximum item length
-            invocation_length = max(map(len, invocations))
+            invocation_length = max([len(s) for s in invocations])
             action_length = invocation_length + self._current_indent
             self._action_max_length = max(self._action_max_length,
                                           action_length)
@@ -392,9 +393,6 @@ class HelpFormatter(object):
         group_actions = set()
         inserts = {}
         for group in groups:
-            if not group._group_actions:
-                raise ValueError(f'empty group {group}')
-
             try:
                 start = actions.index(group._group_actions[0])
             except ValueError:
@@ -409,19 +407,13 @@ class HelpFormatter(object):
                             inserts[start] += ' ['
                         else:
                             inserts[start] = '['
-                        if end in inserts:
-                            inserts[end] += ']'
-                        else:
-                            inserts[end] = ']'
+                        inserts[end] = ']'
                     else:
                         if start in inserts:
                             inserts[start] += ' ('
                         else:
                             inserts[start] = '('
-                        if end in inserts:
-                            inserts[end] += ')'
-                        else:
-                            inserts[end] = ')'
+                        inserts[end] = ')'
                     for i in range(start + 1, end):
                         inserts[i] = '|'
 
@@ -458,7 +450,7 @@ class HelpFormatter(object):
                 # if the Optional doesn't take a value, format is:
                 #    -s or --long
                 if action.nargs == 0:
-                    part = action.format_usage()
+                    part = '%s' % option_string
 
                 # if the Optional takes a value, format is:
                 #    -s ARGS or --long ARGS
@@ -529,13 +521,12 @@ class HelpFormatter(object):
         parts = [action_header]
 
         # if there was help for the action, add lines of help text
-        if action.help and action.help.strip():
+        if action.help:
             help_text = self._expand_help(action)
-            if help_text:
-                help_lines = self._split_lines(help_text, help_width)
-                parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
-                for line in help_lines[1:]:
-                    parts.append('%*s%s\n' % (help_position, '', line))
+            help_lines = self._split_lines(help_text, help_width)
+            parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
+            for line in help_lines[1:]:
+                parts.append('%*s%s\n' % (help_position, '', line))
 
         # or add a newline if the description doesn't end with one
         elif not action_header.endswith('\n'):
@@ -595,11 +586,7 @@ class HelpFormatter(object):
         elif action.nargs == OPTIONAL:
             result = '[%s]' % get_metavar(1)
         elif action.nargs == ZERO_OR_MORE:
-            metavar = get_metavar(1)
-            if len(metavar) == 2:
-                result = '[%s [%s ...]]' % metavar
-            else:
-                result = '[%s ...]' % metavar
+            result = '[%s [%s ...]]' % get_metavar(2)
         elif action.nargs == ONE_OR_MORE:
             result = '%s [%s ...]' % get_metavar(2)
         elif action.nargs == REMAINDER:
@@ -609,10 +596,7 @@ class HelpFormatter(object):
         elif action.nargs == SUPPRESS:
             result = ''
         else:
-            try:
-                formats = ['%s' for _ in range(action.nargs)]
-            except TypeError:
-                raise ValueError("invalid nargs value") from None
+            formats = ['%s' for _ in range(action.nargs)]
             result = ' '.join(formats) % get_metavar(action.nargs)
         return result
 
@@ -726,13 +710,11 @@ def _get_action_name(argument):
     if argument is None:
         return None
     elif argument.option_strings:
-        return '/'.join(argument.option_strings)
+        return  '/'.join(argument.option_strings)
     elif argument.metavar not in (None, SUPPRESS):
         return argument.metavar
     elif argument.dest not in (None, SUPPRESS):
         return argument.dest
-    elif argument.choices:
-        return '{' + ','.join(argument.choices) + '}'
     else:
         return None
 
@@ -848,58 +830,14 @@ class Action(_AttributeHolder):
             'default',
             'type',
             'choices',
-            'required',
             'help',
             'metavar',
         ]
         return [(name, getattr(self, name)) for name in names]
 
-    def format_usage(self):
-        return self.option_strings[0]
-
     def __call__(self, parser, namespace, values, option_string=None):
         raise NotImplementedError(_('.__call__() not defined'))
 
-class BooleanOptionalAction(Action):
-    def __init__(self,
-                 option_strings,
-                 dest,
-                 default=None,
-                 type=None,
-                 choices=None,
-                 required=False,
-                 help=None,
-                 metavar=None):
-
-        _option_strings = []
-        for option_string in option_strings:
-            _option_strings.append(option_string)
-
-            if option_string.startswith('--'):
-                option_string = '--no-' + option_string[2:]
-                _option_strings.append(option_string)
-
-        if help is not None and default is not None and default is not SUPPRESS:
-            help += " (default: %(default)s)"
-
-        super().__init__(
-            option_strings=_option_strings,
-            dest=dest,
-            nargs=0,
-            default=default,
-            type=type,
-            choices=choices,
-            required=required,
-            help=help,
-            metavar=metavar)
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        if option_string in self.option_strings:
-            setattr(namespace, self.dest, not option_string.startswith('--no-'))
-
-    def format_usage(self):
-        return ' | '.join(self.option_strings)
-
 
 class _StoreAction(Action):
 
@@ -915,7 +853,7 @@ class _StoreAction(Action):
                  help=None,
                  metavar=None):
         if nargs == 0:
-            raise ValueError('nargs for store actions must be != 0; if you '
+            raise ValueError('nargs for store actions must be > 0; if you '
                              'have nothing to store, actions such as store '
                              'true or store const may be more appropriate')
         if const is not None and nargs != OPTIONAL:
@@ -1007,7 +945,7 @@ class _AppendAction(Action):
                  help=None,
                  metavar=None):
         if nargs == 0:
-            raise ValueError('nargs for append actions must be != 0; if arg '
+            raise ValueError('nargs for append actions must be > 0; if arg '
                              'strings are not supplying the value to append, '
                              'the append const action may be more appropriate')
         if const is not None and nargs != OPTIONAL:
@@ -1219,12 +1157,6 @@ class _SubParsersAction(Action):
             vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
             getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
 
-class _ExtendAction(_AppendAction):
-    def __call__(self, parser, namespace, values, option_string=None):
-        items = getattr(namespace, self.dest, None)
-        items = _copy_items(items)
-        items.extend(values)
-        setattr(namespace, self.dest, items)
 
 # ==============
 # Type classes
@@ -1257,9 +1189,9 @@ class FileType(object):
         # the special argument "-" means sys.std{in,out}
         if string == '-':
             if 'r' in self._mode:
-                return _sys.stdin.buffer if 'b' in self._mode else _sys.stdin
-            elif any(c in self._mode for c in 'wax'):
-                return _sys.stdout.buffer if 'b' in self._mode else _sys.stdout
+                return _sys.stdin
+            elif 'w' in self._mode:
+                return _sys.stdout
             else:
                 msg = _('argument "-" with mode %r') % self._mode
                 raise ValueError(msg)
@@ -1269,9 +1201,8 @@ class FileType(object):
             return open(string, self._mode, self._bufsize, self._encoding,
                         self._errors)
         except OSError as e:
-            args = {'filename': string, 'error': e}
-            message = _("can't open '%(filename)s': %(error)s")
-            raise ArgumentTypeError(message % args)
+            message = _("can't open '%s': %s")
+            raise ArgumentTypeError(message % (string, e))
 
     def __repr__(self):
         args = self._mode, self._bufsize
@@ -1334,7 +1265,6 @@ class _ActionsContainer(object):
         self.register('action', 'help', _HelpAction)
         self.register('action', 'version', _VersionAction)
         self.register('action', 'parsers', _SubParsersAction)
-        self.register('action', 'extend', _ExtendAction)
 
         # raise an exception if the conflict handler is invalid
         self._get_handler()
@@ -1427,10 +1357,6 @@ class _ActionsContainer(object):
         if not callable(type_func):
             raise ValueError('%r is not callable' % (type_func,))
 
-        if type_func is FileType:
-            raise ValueError('%r is a FileType class object, instance of it'
-                             ' must be passed' % (type_func,))
-
         # raise an error if the metavar does not match the type
         if hasattr(self, "_get_formatter"):
             try:
@@ -1545,8 +1471,10 @@ class _ActionsContainer(object):
 
             # strings starting with two prefix characters are long options
             option_strings.append(option_string)
-            if len(option_string) > 1 and option_string[1] in self.prefix_chars:
-                long_option_strings.append(option_string)
+            if option_string[0] in self.prefix_chars:
+                if len(option_string) > 1:
+                    if option_string[1] in self.prefix_chars:
+                        long_option_strings.append(option_string)
 
         # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
         dest = kwargs.pop('dest', None)
@@ -1673,8 +1601,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
     """Object for parsing command line strings into Python objects.
 
     Keyword Arguments:
-        - prog -- The name of the program (default:
-            ``os.path.basename(sys.argv[0])``)
+        - prog -- The name of the program (default: sys.argv[0])
         - usage -- A usage message (default: auto-generated from arguments)
         - description -- A description of what the program does
         - epilog -- Text following the argument descriptions
@@ -1687,8 +1614,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         - conflict_handler -- String indicating how to handle conflicts
         - add_help -- Add a -h/-help option
         - allow_abbrev -- Allow long options to be abbreviated unambiguously
-        - exit_on_error -- Determines whether or not ArgumentParser exits with
-            error info when an error occurs
     """
 
     def __init__(self,
@@ -1703,14 +1628,19 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                  argument_default=None,
                  conflict_handler='error',
                  add_help=True,
-                 allow_abbrev=True,
-                 exit_on_error=True):
-
-        superinit = super(ArgumentParser, self).__init__
-        superinit(description=description,
-                  prefix_chars=prefix_chars,
-                  argument_default=argument_default,
-                  conflict_handler=conflict_handler)
+                 allow_abbrev=True):
+        _ActionsContainer.__init__(self,
+                                description=description,
+                                prefix_chars=prefix_chars,
+                                argument_default=argument_default,
+                                conflict_handler=conflict_handler)
+        # FIXME: get multiple inheritance method resolution right so we can use
+        # what's below instead of the modified version above
+        # superinit = super(ArgumentParser, self).__init__
+        # superinit(description=description,
+        #           prefix_chars=prefix_chars,
+        #           argument_default=argument_default,
+        #           conflict_handler=conflict_handler)
 
         # default setting for prog
         if prog is None:
@@ -1723,11 +1653,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         self.fromfile_prefix_chars = fromfile_prefix_chars
         self.add_help = add_help
         self.allow_abbrev = allow_abbrev
-        self.exit_on_error = exit_on_error
 
         add_group = self.add_argument_group
         self._positionals = add_group(_('positional arguments'))
-        self._optionals = add_group(_('options'))
+        self._optionals = add_group(_('optional arguments'))
         self._subparsers = None
 
         # register types
@@ -1854,19 +1783,15 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                 setattr(namespace, dest, self._defaults[dest])
 
         # parse the arguments and exit if there are any errors
-        if self.exit_on_error:
-            try:
-                namespace, args = self._parse_known_args(args, namespace)
-            except ArgumentError:
-                err = _sys.exc_info()[1]
-                self.error(str(err))
-        else:
+        try:
             namespace, args = self._parse_known_args(args, namespace)
-
-        if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
-            args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
-            delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
-        return namespace, args
+            if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
+                args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
+                delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
+            return namespace, args
+        except ArgumentError:
+            err = _sys.exc_info()[1]
+            self.error(str(err))
 
     def _parse_known_args(self, arg_strings, namespace):
         # replace arg strings that are file references
@@ -2155,11 +2080,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                 OPTIONAL: _('expected at most one argument'),
                 ONE_OR_MORE: _('expected at least one argument'),
             }
-            msg = nargs_errors.get(action.nargs)
-            if msg is None:
-                msg = ngettext('expected %s argument',
+            default = ngettext('expected %s argument',
                                'expected %s arguments',
                                action.nargs) % action.nargs
+            msg = nargs_errors.get(action.nargs, default)
             raise ArgumentError(action, msg)
 
         # return the number of arguments matched
@@ -2206,23 +2130,24 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                 action = self._option_string_actions[option_string]
                 return action, option_string, explicit_arg
 
-        # search through all possible prefixes of the option string
-        # and all actions in the parser for possible interpretations
-        option_tuples = self._get_option_tuples(arg_string)
+        if self.allow_abbrev:
+            # search through all possible prefixes of the option string
+            # and all actions in the parser for possible interpretations
+            option_tuples = self._get_option_tuples(arg_string)
 
-        # if multiple actions match, the option string was ambiguous
-        if len(option_tuples) > 1:
-            options = ', '.join([option_string
-                for action, option_string, explicit_arg in option_tuples])
-            args = {'option': arg_string, 'matches': options}
-            msg = _('ambiguous option: %(option)s could match %(matches)s')
-            self.error(msg % args)
+            # if multiple actions match, the option string was ambiguous
+            if len(option_tuples) > 1:
+                options = ', '.join([option_string
+                    for action, option_string, explicit_arg in option_tuples])
+                args = {'option': arg_string, 'matches': options}
+                msg = _('ambiguous option: %(option)s could match %(matches)s')
+                self.error(msg % args)
 
-        # if exactly one action matched, this segmentation is good,
-        # so return the parsed action
-        elif len(option_tuples) == 1:
-            option_tuple, = option_tuples
-            return option_tuple
+            # if exactly one action matched, this segmentation is good,
+            # so return the parsed action
+            elif len(option_tuples) == 1:
+                option_tuple, = option_tuples
+                return option_tuple
 
         # if it was not found as an option, but it looks like a negative
         # number, it was meant to be positional
@@ -2246,17 +2171,16 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         # split at the '='
         chars = self.prefix_chars
         if option_string[0] in chars and option_string[1] in chars:
-            if self.allow_abbrev:
-                if '=' in option_string:
-                    option_prefix, explicit_arg = option_string.split('=', 1)
-                else:
-                    option_prefix = option_string
-                    explicit_arg = None
-                for option_string in self._option_string_actions:
-                    if option_string.startswith(option_prefix):
-                        action = self._option_string_actions[option_string]
-                        tup = action, option_string, explicit_arg
-                        result.append(tup)
+            if '=' in option_string:
+                option_prefix, explicit_arg = option_string.split('=', 1)
+            else:
+                option_prefix = option_string
+                explicit_arg = None
+            for option_string in self._option_string_actions:
+                if option_string.startswith(option_prefix):
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, explicit_arg
+                    result.append(tup)
 
         # single character options can be concatenated with their arguments
         # but multiple character options always have to have their argument

Lib/asyncio/futures.py

@@ -184,9 +184,6 @@ class Future:
                 context['source_traceback'] = self._source_traceback
             self._loop.call_exception_handler(context)
 
-    def __class_getitem__(cls, type):
-        return cls
-
     def cancel(self):
         """Cancel the future and schedule callbacks.
 

Lib/asyncio/queues.py

@@ -81,9 +81,6 @@ class Queue:
     def __str__(self):
         return '<{} {}>'.format(type(self).__name__, self._format())
 
-    def __class_getitem__(cls, type):
-        return cls
-
     def _format(self):
         result = 'maxsize={!r}'.format(self._maxsize)
         if getattr(self, '_queue', None):

Lib/bdb.py

@@ -34,8 +34,6 @@ class Bdb:
         self.fncache = {}
         self.frame_returning = None
 
-        self._load_breaks()
-
     def canonic(self, filename):
         """Return canonical form of filename.
 
@@ -119,7 +117,7 @@ class Bdb:
         """Invoke user function and return trace function for call event.
 
         If the debugger stops on this function call, invoke
-        self.user_call(). Raise BdbQuit if self.quitting is set.
+        self.user_call(). Raise BbdQuit if self.quitting is set.
         Return self.trace_dispatch to continue tracing in this scope.
         """
         # XXX 'arg' is no longer used
@@ -367,12 +365,6 @@ class Bdb:
     # Call self.get_*break*() to see the breakpoints or better
     # for bp in Breakpoint.bpbynumber: if bp: bp.bpprint().
 
-    def _add_to_breaks(self, filename, lineno):
-        """Add breakpoint to breaks, if not already there."""
-        bp_linenos = self.breaks.setdefault(filename, [])
-        if lineno not in bp_linenos:
-            bp_linenos.append(lineno)
-
     def set_break(self, filename, lineno, temporary=False, cond=None,
                   funcname=None):
         """Set a new breakpoint for filename:lineno.
@@ -385,21 +377,12 @@ class Bdb:
         line = linecache.getline(filename, lineno)
         if not line:
             return 'Line %s:%d does not exist' % (filename, lineno)
-        self._add_to_breaks(filename, lineno)
+        list = self.breaks.setdefault(filename, [])
+        if lineno not in list:
+            list.append(lineno)
         bp = Breakpoint(filename, lineno, temporary, cond, funcname)
         return None
 
-    def _load_breaks(self):
-        """Apply all breakpoints (set in other instances) to this one.
-
-        Populates this instance's breaks list from the Breakpoint class's
-        list, which can have breakpoints set by another Bdb instance. This
-        is necessary for interactive sessions to keep the breakpoints
-        active across multiple calls to run().
-        """
-        for (filename, lineno) in Breakpoint.bplist.keys():
-            self._add_to_breaks(filename, lineno)
-
     def _prune_breaks(self, filename, lineno):
         """Prune breakpoints for filename:lineno.
 
@@ -628,11 +611,26 @@ class Bdb:
 
     # This method is more useful to debug a single function call.
 
-    def runcall(self, func, /, *args, **kwds):
+    def runcall(*args, **kwds):
         """Debug a single function call.
 
         Return the result of the function call.
         """
+        if len(args) >= 2:
+            self, func, *args = args
+        elif not args:
+            raise TypeError("descriptor 'runcall' of 'Bdb' object "
+                            "needs an argument")
+        elif 'func' in kwds:
+            func = kwds.pop('func')
+            self, *args = args
+            import warnings
+            warnings.warn("Passing 'func' as keyword argument is deprecated",
+                          DeprecationWarning, stacklevel=2)
+        else:
+            raise TypeError('runcall expected at least 1 positional argument, '
+                            'got %d' % (len(args)-1))
+
         self.reset()
         sys.settrace(self.trace_dispatch)
         res = None
@@ -644,6 +642,7 @@ class Bdb:
             self.quitting = True
             sys.settrace(None)
         return res
+    runcall.__text_signature__ = '($self, func, /, *args, **kwds)'
 
 
 def set_trace():
@@ -698,12 +697,6 @@ class Breakpoint:
         else:
             self.bplist[file, line] = [self]
 
-    @staticmethod
-    def clearBreakpoints():
-        Breakpoint.next = 1
-        Breakpoint.bplist = {}
-        Breakpoint.bpbynumber = [None]
-
     def deleteMe(self):
         """Delete the breakpoint from the list associated to a file:line.
 

Lib/binhex.py

@@ -1,502 +0,0 @@
-"""Macintosh binhex compression/decompression.
-
-easy interface:
-binhex(inputfilename, outputfilename)
-hexbin(inputfilename, outputfilename)
-"""
-
-#
-# Jack Jansen, CWI, August 1995.
-#
-# The module is supposed to be as compatible as possible. Especially the
-# easy interface should work "as expected" on any platform.
-# XXXX Note: currently, textfiles appear in mac-form on all platforms.
-# We seem to lack a simple character-translate in python.
-# (we should probably use ISO-Latin-1 on all but the mac platform).
-# XXXX The simple routines are too simple: they expect to hold the complete
-# files in-core. Should be fixed.
-# XXXX It would be nice to handle AppleDouble format on unix
-# (for servers serving macs).
-# XXXX I don't understand what happens when you get 0x90 times the same byte on
-# input. The resulting code (xx 90 90) would appear to be interpreted as an
-# escaped *value* of 0x90. All coders I've seen appear to ignore this nicety...
-#
-import binascii
-import contextlib
-import io
-import os
-import struct
-import warnings
-
-warnings.warn('the binhex module is deprecated', DeprecationWarning,
-              stacklevel=2)
-
-
-__all__ = ["binhex","hexbin","Error"]
-
-class Error(Exception):
-    pass
-
-# States (what have we written)
-_DID_HEADER = 0
-_DID_DATA = 1
-
-# Various constants
-REASONABLY_LARGE = 32768  # Minimal amount we pass the rle-coder
-LINELEN = 64
-RUNCHAR = b"\x90"
-
-#
-# This code is no longer byte-order dependent
-
-
-class FInfo:
-    def __init__(self):
-        self.Type = '????'
-        self.Creator = '????'
-        self.Flags = 0
-
-def getfileinfo(name):
-    finfo = FInfo()
-    with io.open(name, 'rb') as fp:
-        # Quick check for textfile
-        data = fp.read(512)
-        if 0 not in data:
-            finfo.Type = 'TEXT'
-        fp.seek(0, 2)
-        dsize = fp.tell()
-    dir, file = os.path.split(name)
-    file = file.replace(':', '-', 1)
-    return file, finfo, dsize, 0
-
-class openrsrc:
-    def __init__(self, *args):
-        pass
-
-    def read(self, *args):
-        return b''
-
-    def write(self, *args):
-        pass
-
-    def close(self):
-        pass
-
-
-# DeprecationWarning is already emitted on "import binhex". There is no need
-# to repeat the warning at each call to deprecated binascii functions.
-@contextlib.contextmanager
-def _ignore_deprecation_warning():
-    with warnings.catch_warnings():
-        warnings.filterwarnings('ignore', '', DeprecationWarning)
-        yield
-
-
-class _Hqxcoderengine:
-    """Write data to the coder in 3-byte chunks"""
-
-    def __init__(self, ofp):
-        self.ofp = ofp
-        self.data = b''
-        self.hqxdata = b''
-        self.linelen = LINELEN - 1
-
-    def write(self, data):
-        self.data = self.data + data
-        datalen = len(self.data)
-        todo = (datalen // 3) * 3
-        data = self.data[:todo]
-        self.data = self.data[todo:]
-        if not data:
-            return
-        with _ignore_deprecation_warning():
-            self.hqxdata = self.hqxdata + binascii.b2a_hqx(data)
-        self._flush(0)
-
-    def _flush(self, force):
-        first = 0
-        while first <= len(self.hqxdata) - self.linelen:
-            last = first + self.linelen
-            self.ofp.write(self.hqxdata[first:last] + b'\r')
-            self.linelen = LINELEN
-            first = last
-        self.hqxdata = self.hqxdata[first:]
-        if force:
-            self.ofp.write(self.hqxdata + b':\r')
-
-    def close(self):
-        if self.data:
-            with _ignore_deprecation_warning():
-                self.hqxdata = self.hqxdata + binascii.b2a_hqx(self.data)
-        self._flush(1)
-        self.ofp.close()
-        del self.ofp
-
-class _Rlecoderengine:
-    """Write data to the RLE-coder in suitably large chunks"""
-
-    def __init__(self, ofp):
-        self.ofp = ofp
-        self.data = b''
-
-    def write(self, data):
-        self.data = self.data + data
-        if len(self.data) < REASONABLY_LARGE:
-            return
-        with _ignore_deprecation_warning():
-            rledata = binascii.rlecode_hqx(self.data)
-        self.ofp.write(rledata)
-        self.data = b''
-
-    def close(self):
-        if self.data:
-            with _ignore_deprecation_warning():
-                rledata = binascii.rlecode_hqx(self.data)
-            self.ofp.write(rledata)
-        self.ofp.close()
-        del self.ofp
-
-class BinHex:
-    def __init__(self, name_finfo_dlen_rlen, ofp):
-        name, finfo, dlen, rlen = name_finfo_dlen_rlen
-        close_on_error = False
-        if isinstance(ofp, str):
-            ofname = ofp
-            ofp = io.open(ofname, 'wb')
-            close_on_error = True
-        try:
-            ofp.write(b'(This file must be converted with BinHex 4.0)\r\r:')
-            hqxer = _Hqxcoderengine(ofp)
-            self.ofp = _Rlecoderengine(hqxer)
-            self.crc = 0
-            if finfo is None:
-                finfo = FInfo()
-            self.dlen = dlen
-            self.rlen = rlen
-            self._writeinfo(name, finfo)
-            self.state = _DID_HEADER
-        except:
-            if close_on_error:
-                ofp.close()
-            raise
-
-    def _writeinfo(self, name, finfo):
-        nl = len(name)
-        if nl > 63:
-            raise Error('Filename too long')
-        d = bytes([nl]) + name.encode("latin-1") + b'\0'
-        tp, cr = finfo.Type, finfo.Creator
-        if isinstance(tp, str):
-            tp = tp.encode("latin-1")
-        if isinstance(cr, str):
-            cr = cr.encode("latin-1")
-        d2 = tp + cr
-
-        # Force all structs to be packed with big-endian
-        d3 = struct.pack('>h', finfo.Flags)
-        d4 = struct.pack('>ii', self.dlen, self.rlen)
-        info = d + d2 + d3 + d4
-        self._write(info)
-        self._writecrc()
-
-    def _write(self, data):
-        self.crc = binascii.crc_hqx(data, self.crc)
-        self.ofp.write(data)
-
-    def _writecrc(self):
-        # XXXX Should this be here??
-        # self.crc = binascii.crc_hqx('\0\0', self.crc)
-        if self.crc < 0:
-            fmt = '>h'
-        else:
-            fmt = '>H'
-        self.ofp.write(struct.pack(fmt, self.crc))
-        self.crc = 0
-
-    def write(self, data):
-        if self.state != _DID_HEADER:
-            raise Error('Writing data at the wrong time')
-        self.dlen = self.dlen - len(data)
-        self._write(data)
-
-    def close_data(self):
-        if self.dlen != 0:
-            raise Error('Incorrect data size, diff=%r' % (self.rlen,))
-        self._writecrc()
-        self.state = _DID_DATA
-
-    def write_rsrc(self, data):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error('Writing resource data at the wrong time')
-        self.rlen = self.rlen - len(data)
-        self._write(data)
-
-    def close(self):
-        if self.state is None:
-            return
-        try:
-            if self.state < _DID_DATA:
-                self.close_data()
-            if self.state != _DID_DATA:
-                raise Error('Close at the wrong time')
-            if self.rlen != 0:
-                raise Error("Incorrect resource-datasize, diff=%r" % (self.rlen,))
-            self._writecrc()
-        finally:
-            self.state = None
-            ofp = self.ofp
-            del self.ofp
-            ofp.close()
-
-def binhex(inp, out):
-    """binhex(infilename, outfilename): create binhex-encoded copy of a file"""
-    finfo = getfileinfo(inp)
-    ofp = BinHex(finfo, out)
-
-    with io.open(inp, 'rb') as ifp:
-        # XXXX Do textfile translation on non-mac systems
-        while True:
-            d = ifp.read(128000)
-            if not d: break
-            ofp.write(d)
-        ofp.close_data()
-
-    ifp = openrsrc(inp, 'rb')
-    while True:
-        d = ifp.read(128000)
-        if not d: break
-        ofp.write_rsrc(d)
-    ofp.close()
-    ifp.close()
-
-class _Hqxdecoderengine:
-    """Read data via the decoder in 4-byte chunks"""
-
-    def __init__(self, ifp):
-        self.ifp = ifp
-        self.eof = 0
-
-    def read(self, totalwtd):
-        """Read at least wtd bytes (or until EOF)"""
-        decdata = b''
-        wtd = totalwtd
-        #
-        # The loop here is convoluted, since we don't really now how
-        # much to decode: there may be newlines in the incoming data.
-        while wtd > 0:
-            if self.eof: return decdata
-            wtd = ((wtd + 2) // 3) * 4
-            data = self.ifp.read(wtd)
-            #
-            # Next problem: there may not be a complete number of
-            # bytes in what we pass to a2b. Solve by yet another
-            # loop.
-            #
-            while True:
-                try:
-                    with _ignore_deprecation_warning():
-                        decdatacur, self.eof = binascii.a2b_hqx(data)
-                    break
-                except binascii.Incomplete:
-                    pass
-                newdata = self.ifp.read(1)
-                if not newdata:
-                    raise Error('Premature EOF on binhex file')
-                data = data + newdata
-            decdata = decdata + decdatacur
-            wtd = totalwtd - len(decdata)
-            if not decdata and not self.eof:
-                raise Error('Premature EOF on binhex file')
-        return decdata
-
-    def close(self):
-        self.ifp.close()
-
-class _Rledecoderengine:
-    """Read data via the RLE-coder"""
-
-    def __init__(self, ifp):
-        self.ifp = ifp
-        self.pre_buffer = b''
-        self.post_buffer = b''
-        self.eof = 0
-
-    def read(self, wtd):
-        if wtd > len(self.post_buffer):
-            self._fill(wtd - len(self.post_buffer))
-        rv = self.post_buffer[:wtd]
-        self.post_buffer = self.post_buffer[wtd:]
-        return rv
-
-    def _fill(self, wtd):
-        self.pre_buffer = self.pre_buffer + self.ifp.read(wtd + 4)
-        if self.ifp.eof:
-            with _ignore_deprecation_warning():
-                self.post_buffer = self.post_buffer + \
-                    binascii.rledecode_hqx(self.pre_buffer)
-            self.pre_buffer = b''
-            return
-
-        #
-        # Obfuscated code ahead. We have to take care that we don't
-        # end up with an orphaned RUNCHAR later on. So, we keep a couple
-        # of bytes in the buffer, depending on what the end of
-        # the buffer looks like:
-        # '\220\0\220' - Keep 3 bytes: repeated \220 (escaped as \220\0)
-        # '?\220' - Keep 2 bytes: repeated something-else
-        # '\220\0' - Escaped \220: Keep 2 bytes.
-        # '?\220?' - Complete repeat sequence: decode all
-        # otherwise: keep 1 byte.
-        #
-        mark = len(self.pre_buffer)
-        if self.pre_buffer[-3:] == RUNCHAR + b'\0' + RUNCHAR:
-            mark = mark - 3
-        elif self.pre_buffer[-1:] == RUNCHAR:
-            mark = mark - 2
-        elif self.pre_buffer[-2:] == RUNCHAR + b'\0':
-            mark = mark - 2
-        elif self.pre_buffer[-2:-1] == RUNCHAR:
-            pass # Decode all
-        else:
-            mark = mark - 1
-
-        with _ignore_deprecation_warning():
-            self.post_buffer = self.post_buffer + \
-                binascii.rledecode_hqx(self.pre_buffer[:mark])
-        self.pre_buffer = self.pre_buffer[mark:]
-
-    def close(self):
-        self.ifp.close()
-
-class HexBin:
-    def __init__(self, ifp):
-        if isinstance(ifp, str):
-            ifp = io.open(ifp, 'rb')
-        #
-        # Find initial colon.
-        #
-        while True:
-            ch = ifp.read(1)
-            if not ch:
-                raise Error("No binhex data found")
-            # Cater for \r\n terminated lines (which show up as \n\r, hence
-            # all lines start with \r)
-            if ch == b'\r':
-                continue
-            if ch == b':':
-                break
-
-        hqxifp = _Hqxdecoderengine(ifp)
-        self.ifp = _Rledecoderengine(hqxifp)
-        self.crc = 0
-        self._readheader()
-
-    def _read(self, len):
-        data = self.ifp.read(len)
-        self.crc = binascii.crc_hqx(data, self.crc)
-        return data
-
-    def _checkcrc(self):
-        filecrc = struct.unpack('>h', self.ifp.read(2))[0] & 0xffff
-        #self.crc = binascii.crc_hqx('\0\0', self.crc)
-        # XXXX Is this needed??
-        self.crc = self.crc & 0xffff
-        if filecrc != self.crc:
-            raise Error('CRC error, computed %x, read %x'
-                        % (self.crc, filecrc))
-        self.crc = 0
-
-    def _readheader(self):
-        len = self._read(1)
-        fname = self._read(ord(len))
-        rest = self._read(1 + 4 + 4 + 2 + 4 + 4)
-        self._checkcrc()
-
-        type = rest[1:5]
-        creator = rest[5:9]
-        flags = struct.unpack('>h', rest[9:11])[0]
-        self.dlen = struct.unpack('>l', rest[11:15])[0]
-        self.rlen = struct.unpack('>l', rest[15:19])[0]
-
-        self.FName = fname
-        self.FInfo = FInfo()
-        self.FInfo.Creator = creator
-        self.FInfo.Type = type
-        self.FInfo.Flags = flags
-
-        self.state = _DID_HEADER
-
-    def read(self, *n):
-        if self.state != _DID_HEADER:
-            raise Error('Read data at wrong time')
-        if n:
-            n = n[0]
-            n = min(n, self.dlen)
-        else:
-            n = self.dlen
-        rv = b''
-        while len(rv) < n:
-            rv = rv + self._read(n-len(rv))
-        self.dlen = self.dlen - n
-        return rv
-
-    def close_data(self):
-        if self.state != _DID_HEADER:
-            raise Error('close_data at wrong time')
-        if self.dlen:
-            dummy = self._read(self.dlen)
-        self._checkcrc()
-        self.state = _DID_DATA
-
-    def read_rsrc(self, *n):
-        if self.state == _DID_HEADER:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error('Read resource data at wrong time')
-        if n:
-            n = n[0]
-            n = min(n, self.rlen)
-        else:
-            n = self.rlen
-        self.rlen = self.rlen - n
-        return self._read(n)
-
-    def close(self):
-        if self.state is None:
-            return
-        try:
-            if self.rlen:
-                dummy = self.read_rsrc(self.rlen)
-            self._checkcrc()
-        finally:
-            self.state = None
-            self.ifp.close()
-
-def hexbin(inp, out):
-    """hexbin(infilename, outfilename) - Decode binhexed file"""
-    ifp = HexBin(inp)
-    finfo = ifp.FInfo
-    if not out:
-        out = ifp.FName
-
-    with io.open(out, 'wb') as ofp:
-        # XXXX Do translation on non-mac systems
-        while True:
-            d = ifp.read(128000)
-            if not d: break
-            ofp.write(d)
-    ifp.close_data()
-
-    d = ifp.read_rsrc(128000)
-    if d:
-        ofp = openrsrc(out, 'wb')
-        ofp.write(d)
-        while True:
-            d = ifp.read_rsrc(128000)
-            if not d: break
-            ofp.write(d)
-        ofp.close()
-
-    ifp.close()

Lib/bisect.py

@@ -1,7 +1,6 @@
 """Bisection algorithms."""
 
-
-def insort_right(a, x, lo=0, hi=None, *, key=None):
+def insort_right(a, x, lo=0, hi=None):
     """Insert item x in list a, and keep it sorted assuming a is sorted.
 
     If x is already in a, insert it to the right of the rightmost x.
@@ -9,18 +8,24 @@ def insort_right(a, x, lo=0, hi=None, *, key=None):
     Optional args lo (default 0) and hi (default len(a)) bound the
     slice of a to be searched.
     """
-    if key is None:
-        lo = bisect_right(a, x, lo, hi)
-    else:
-        lo = bisect_right(a, key(x), lo, hi, key=key)
+
+    if lo < 0:
+        raise ValueError('lo must be non-negative')
+    if hi is None:
+        hi = len(a)
+    while lo < hi:
+        mid = (lo+hi)//2
+        if x < a[mid]: hi = mid
+        else: lo = mid+1
     a.insert(lo, x)
 
+insort = insort_right   # backward compatibility
 
-def bisect_right(a, x, lo=0, hi=None, *, key=None):
+def bisect_right(a, x, lo=0, hi=None):
     """Return the index where to insert item x in list a, assuming a is sorted.
 
     The return value i is such that all e in a[:i] have e <= x, and all e in
-    a[i:] have e > x.  So if x already appears in the list, a.insert(i, x) will
+    a[i:] have e > x.  So if x already appears in the list, a.insert(x) will
     insert just after the rightmost x already there.
 
     Optional args lo (default 0) and hi (default len(a)) bound the
@@ -31,26 +36,15 @@ def bisect_right(a, x, lo=0, hi=None, *, key=None):
         raise ValueError('lo must be non-negative')
     if hi is None:
         hi = len(a)
-    # Note, the comparison uses "<" to match the
-    # __lt__() logic in list.sort() and in heapq.
-    if key is None:
-        while lo < hi:
-            mid = (lo + hi) // 2
-            if x < a[mid]:
-                hi = mid
-            else:
-                lo = mid + 1
-    else:
-        while lo < hi:
-            mid = (lo + hi) // 2
-            if x < key(a[mid]):
-                hi = mid
-            else:
-                lo = mid + 1
+    while lo < hi:
+        mid = (lo+hi)//2
+        if x < a[mid]: hi = mid
+        else: lo = mid+1
     return lo
 
+bisect = bisect_right   # backward compatibility
 
-def insort_left(a, x, lo=0, hi=None, *, key=None):
+def insort_left(a, x, lo=0, hi=None):
     """Insert item x in list a, and keep it sorted assuming a is sorted.
 
     If x is already in a, insert it to the left of the leftmost x.
@@ -59,17 +53,22 @@ def insort_left(a, x, lo=0, hi=None, *, key=None):
     slice of a to be searched.
     """
 
-    if key is None:
-        lo = bisect_left(a, x, lo, hi)
-    else:
-        lo = bisect_left(a, key(x), lo, hi, key=key)
+    if lo < 0:
+        raise ValueError('lo must be non-negative')
+    if hi is None:
+        hi = len(a)
+    while lo < hi:
+        mid = (lo+hi)//2
+        if a[mid] < x: lo = mid+1
+        else: hi = mid
     a.insert(lo, x)
 
-def bisect_left(a, x, lo=0, hi=None, *, key=None):
+
+def bisect_left(a, x, lo=0, hi=None):
     """Return the index where to insert item x in list a, assuming a is sorted.
 
     The return value i is such that all e in a[:i] have e < x, and all e in
-    a[i:] have e >= x.  So if x already appears in the list, a.insert(i, x) will
+    a[i:] have e >= x.  So if x already appears in the list, a.insert(x) will
     insert just before the leftmost x already there.
 
     Optional args lo (default 0) and hi (default len(a)) bound the
@@ -80,31 +79,14 @@ def bisect_left(a, x, lo=0, hi=None, *, key=None):
         raise ValueError('lo must be non-negative')
     if hi is None:
         hi = len(a)
-    # Note, the comparison uses "<" to match the
-    # __lt__() logic in list.sort() and in heapq.
-    if key is None:
-        while lo < hi:
-            mid = (lo + hi) // 2
-            if a[mid] < x:
-                lo = mid + 1
-            else:
-                hi = mid
-    else:
-        while lo < hi:
-            mid = (lo + hi) // 2
-            if key(a[mid]) < x:
-                lo = mid + 1
-            else:
-                hi = mid
+    while lo < hi:
+        mid = (lo+hi)//2
+        if a[mid] < x: lo = mid+1
+        else: hi = mid
     return lo
 
-
 # Overwrite above definitions with a fast C implementation
 try:
     from _bisect import *
 except ImportError:
     pass
-
-# Create aliases
-bisect = bisect_right
-insort = insort_right

Lib/bz2.py

@@ -1,344 +0,0 @@
-"""Interface to the libbzip2 compression library.
-
-This module provides a file interface, classes for incremental
-(de)compression, and functions for one-shot (de)compression.
-"""
-
-__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor",
-           "open", "compress", "decompress"]
-
-__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
-
-from builtins import open as _builtin_open
-import io
-import os
-import _compression
-
-from _bz2 import BZ2Compressor, BZ2Decompressor
-
-
-_MODE_CLOSED   = 0
-_MODE_READ     = 1
-# Value 2 no longer used
-_MODE_WRITE    = 3
-
-
-class BZ2File(_compression.BaseStream):
-
-    """A file object providing transparent bzip2 (de)compression.
-
-    A BZ2File can act as a wrapper for an existing file object, or refer
-    directly to a named file on disk.
-
-    Note that BZ2File provides a *binary* file interface - data read is
-    returned as bytes, and data to be written should be given as bytes.
-    """
-
-    def __init__(self, filename, mode="r", *, compresslevel=9):
-        """Open a bzip2-compressed file.
-
-        If filename is a str, bytes, or PathLike object, it gives the
-        name of the file to be opened. Otherwise, it should be a file
-        object, which will be used to read or write the compressed data.
-
-        mode can be 'r' for reading (default), 'w' for (over)writing,
-        'x' for creating exclusively, or 'a' for appending. These can
-        equivalently be given as 'rb', 'wb', 'xb', and 'ab'.
-
-        If mode is 'w', 'x' or 'a', compresslevel can be a number between 1
-        and 9 specifying the level of compression: 1 produces the least
-        compression, and 9 (default) produces the most compression.
-
-        If mode is 'r', the input file may be the concatenation of
-        multiple compressed streams.
-        """
-        self._fp = None
-        self._closefp = False
-        self._mode = _MODE_CLOSED
-
-        if not (1 <= compresslevel <= 9):
-            raise ValueError("compresslevel must be between 1 and 9")
-
-        if mode in ("", "r", "rb"):
-            mode = "rb"
-            mode_code = _MODE_READ
-        elif mode in ("w", "wb"):
-            mode = "wb"
-            mode_code = _MODE_WRITE
-            self._compressor = BZ2Compressor(compresslevel)
-        elif mode in ("x", "xb"):
-            mode = "xb"
-            mode_code = _MODE_WRITE
-            self._compressor = BZ2Compressor(compresslevel)
-        elif mode in ("a", "ab"):
-            mode = "ab"
-            mode_code = _MODE_WRITE
-            self._compressor = BZ2Compressor(compresslevel)
-        else:
-            raise ValueError("Invalid mode: %r" % (mode,))
-
-        if isinstance(filename, (str, bytes, os.PathLike)):
-            self._fp = _builtin_open(filename, mode)
-            self._closefp = True
-            self._mode = mode_code
-        elif hasattr(filename, "read") or hasattr(filename, "write"):
-            self._fp = filename
-            self._mode = mode_code
-        else:
-            raise TypeError("filename must be a str, bytes, file or PathLike object")
-
-        if self._mode == _MODE_READ:
-            raw = _compression.DecompressReader(self._fp,
-                BZ2Decompressor, trailing_error=OSError)
-            self._buffer = io.BufferedReader(raw)
-        else:
-            self._pos = 0
-
-    def close(self):
-        """Flush and close the file.
-
-        May be called more than once without error. Once the file is
-        closed, any other operation on it will raise a ValueError.
-        """
-        if self._mode == _MODE_CLOSED:
-            return
-        try:
-            if self._mode == _MODE_READ:
-                self._buffer.close()
-            elif self._mode == _MODE_WRITE:
-                self._fp.write(self._compressor.flush())
-                self._compressor = None
-        finally:
-            try:
-                if self._closefp:
-                    self._fp.close()
-            finally:
-                self._fp = None
-                self._closefp = False
-                self._mode = _MODE_CLOSED
-                self._buffer = None
-
-    @property
-    def closed(self):
-        """True if this file is closed."""
-        return self._mode == _MODE_CLOSED
-
-    def fileno(self):
-        """Return the file descriptor for the underlying file."""
-        self._check_not_closed()
-        return self._fp.fileno()
-
-    def seekable(self):
-        """Return whether the file supports seeking."""
-        return self.readable() and self._buffer.seekable()
-
-    def readable(self):
-        """Return whether the file was opened for reading."""
-        self._check_not_closed()
-        return self._mode == _MODE_READ
-
-    def writable(self):
-        """Return whether the file was opened for writing."""
-        self._check_not_closed()
-        return self._mode == _MODE_WRITE
-
-    def peek(self, n=0):
-        """Return buffered data without advancing the file position.
-
-        Always returns at least one byte of data, unless at EOF.
-        The exact number of bytes returned is unspecified.
-        """
-        self._check_can_read()
-        # Relies on the undocumented fact that BufferedReader.peek()
-        # always returns at least one byte (except at EOF), independent
-        # of the value of n
-        return self._buffer.peek(n)
-
-    def read(self, size=-1):
-        """Read up to size uncompressed bytes from the file.
-
-        If size is negative or omitted, read until EOF is reached.
-        Returns b'' if the file is already at EOF.
-        """
-        self._check_can_read()
-        return self._buffer.read(size)
-
-    def read1(self, size=-1):
-        """Read up to size uncompressed bytes, while trying to avoid
-        making multiple reads from the underlying stream. Reads up to a
-        buffer's worth of data if size is negative.
-
-        Returns b'' if the file is at EOF.
-        """
-        self._check_can_read()
-        if size < 0:
-            size = io.DEFAULT_BUFFER_SIZE
-        return self._buffer.read1(size)
-
-    def readinto(self, b):
-        """Read bytes into b.
-
-        Returns the number of bytes read (0 for EOF).
-        """
-        self._check_can_read()
-        return self._buffer.readinto(b)
-
-    def readline(self, size=-1):
-        """Read a line of uncompressed bytes from the file.
-
-        The terminating newline (if present) is retained. If size is
-        non-negative, no more than size bytes will be read (in which
-        case the line may be incomplete). Returns b'' if already at EOF.
-        """
-        if not isinstance(size, int):
-            if not hasattr(size, "__index__"):
-                raise TypeError("Integer argument expected")
-            size = size.__index__()
-        self._check_can_read()
-        return self._buffer.readline(size)
-
-    def readlines(self, size=-1):
-        """Read a list of lines of uncompressed bytes from the file.
-
-        size can be specified to control the number of lines read: no
-        further lines will be read once the total size of the lines read
-        so far equals or exceeds size.
-        """
-        if not isinstance(size, int):
-            if not hasattr(size, "__index__"):
-                raise TypeError("Integer argument expected")
-            size = size.__index__()
-        self._check_can_read()
-        return self._buffer.readlines(size)
-
-    def write(self, data):
-        """Write a byte string to the file.
-
-        Returns the number of uncompressed bytes written, which is
-        always the length of data in bytes. Note that due to buffering,
-        the file on disk may not reflect the data written until close()
-        is called.
-        """
-        self._check_can_write()
-        if isinstance(data, (bytes, bytearray)):
-            length = len(data)
-        else:
-            # accept any data that supports the buffer protocol
-            data = memoryview(data)
-            length = data.nbytes
-
-        compressed = self._compressor.compress(data)
-        self._fp.write(compressed)
-        self._pos += length
-        return length
-
-    def writelines(self, seq):
-        """Write a sequence of byte strings to the file.
-
-        Returns the number of uncompressed bytes written.
-        seq can be any iterable yielding byte strings.
-
-        Line separators are not added between the written byte strings.
-        """
-        return _compression.BaseStream.writelines(self, seq)
-
-    def seek(self, offset, whence=io.SEEK_SET):
-        """Change the file position.
-
-        The new position is specified by offset, relative to the
-        position indicated by whence. Values for whence are:
-
-            0: start of stream (default); offset must not be negative
-            1: current stream position
-            2: end of stream; offset must not be positive
-
-        Returns the new file position.
-
-        Note that seeking is emulated, so depending on the parameters,
-        this operation may be extremely slow.
-        """
-        self._check_can_seek()
-        return self._buffer.seek(offset, whence)
-
-    def tell(self):
-        """Return the current file position."""
-        self._check_not_closed()
-        if self._mode == _MODE_READ:
-            return self._buffer.tell()
-        return self._pos
-
-
-def open(filename, mode="rb", compresslevel=9,
-         encoding=None, errors=None, newline=None):
-    """Open a bzip2-compressed file in binary or text mode.
-
-    The filename argument can be an actual filename (a str, bytes, or
-    PathLike object), or an existing file object to read from or write
-    to.
-
-    The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or
-    "ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode.
-    The default mode is "rb", and the default compresslevel is 9.
-
-    For binary mode, this function is equivalent to the BZ2File
-    constructor: BZ2File(filename, mode, compresslevel). In this case,
-    the encoding, errors and newline arguments must not be provided.
-
-    For text mode, a BZ2File object is created, and wrapped in an
-    io.TextIOWrapper instance with the specified encoding, error
-    handling behavior, and line ending(s).
-
-    """
-    if "t" in mode:
-        if "b" in mode:
-            raise ValueError("Invalid mode: %r" % (mode,))
-    else:
-        if encoding is not None:
-            raise ValueError("Argument 'encoding' not supported in binary mode")
-        if errors is not None:
-            raise ValueError("Argument 'errors' not supported in binary mode")
-        if newline is not None:
-            raise ValueError("Argument 'newline' not supported in binary mode")
-
-    bz_mode = mode.replace("t", "")
-    binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel)
-
-    if "t" in mode:
-        encoding = io.text_encoding(encoding)
-        return io.TextIOWrapper(binary_file, encoding, errors, newline)
-    else:
-        return binary_file
-
-
-def compress(data, compresslevel=9):
-    """Compress a block of data.
-
-    compresslevel, if given, must be a number between 1 and 9.
-
-    For incremental compression, use a BZ2Compressor object instead.
-    """
-    comp = BZ2Compressor(compresslevel)
-    return comp.compress(data) + comp.flush()
-
-
-def decompress(data):
-    """Decompress a block of data.
-
-    For incremental decompression, use a BZ2Decompressor object instead.
-    """
-    results = []
-    while data:
-        decomp = BZ2Decompressor()
-        try:
-            res = decomp.decompress(data)
-        except OSError:
-            if results:
-                break  # Leftover data is not a valid bzip2 stream; ignore it.
-            else:
-                raise  # Error on the first iteration; bail out.
-        results.append(res)
-        if not decomp.eof:
-            raise ValueError("Compressed data ended before the "
-                             "end-of-stream marker was reached")
-        data = decomp.unused_data
-    return b"".join(results)

Lib/cgitb.py

@@ -1,321 +0,0 @@
-"""More comprehensive traceback formatting for Python scripts.
-
-To enable this module, do:
-
-    import cgitb; cgitb.enable()
-
-at the top of your script.  The optional arguments to enable() are:
-
-    display     - if true, tracebacks are displayed in the web browser
-    logdir      - if set, tracebacks are written to files in this directory
-    context     - number of lines of source code to show for each stack frame
-    format      - 'text' or 'html' controls the output format
-
-By default, tracebacks are displayed but not saved, the context is 5 lines
-and the output format is 'html' (for backwards compatibility with the
-original use of this module)
-
-Alternatively, if you have caught an exception and want cgitb to display it
-for you, call cgitb.handler().  The optional argument to handler() is a
-3-item tuple (etype, evalue, etb) just like the value of sys.exc_info().
-The default handler displays output as HTML.
-
-"""
-import inspect
-import keyword
-import linecache
-import os
-import pydoc
-import sys
-import tempfile
-import time
-import tokenize
-import traceback
-
-def reset():
-    """Return a string that resets the CGI and browser to a known state."""
-    return '''<!--: spam
-Content-Type: text/html
-
-<body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> -->
-<body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> --> -->
-</font> </font> </font> </script> </object> </blockquote> </pre>
-</table> </table> </table> </table> </table> </font> </font> </font>'''
-
-__UNDEF__ = []                          # a special sentinel object
-def small(text):
-    if text:
-        return '<small>' + text + '</small>'
-    else:
-        return ''
-
-def strong(text):
-    if text:
-        return '<strong>' + text + '</strong>'
-    else:
-        return ''
-
-def grey(text):
-    if text:
-        return '<font color="#909090">' + text + '</font>'
-    else:
-        return ''
-
-def lookup(name, frame, locals):
-    """Find the value for a given name in the given environment."""
-    if name in locals:
-        return 'local', locals[name]
-    if name in frame.f_globals:
-        return 'global', frame.f_globals[name]
-    if '__builtins__' in frame.f_globals:
-        builtins = frame.f_globals['__builtins__']
-        if type(builtins) is type({}):
-            if name in builtins:
-                return 'builtin', builtins[name]
-        else:
-            if hasattr(builtins, name):
-                return 'builtin', getattr(builtins, name)
-    return None, __UNDEF__
-
-def scanvars(reader, frame, locals):
-    """Scan one logical line of Python and look up values of variables used."""
-    vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__
-    for ttype, token, start, end, line in tokenize.generate_tokens(reader):
-        if ttype == tokenize.NEWLINE: break
-        if ttype == tokenize.NAME and token not in keyword.kwlist:
-            if lasttoken == '.':
-                if parent is not __UNDEF__:
-                    value = getattr(parent, token, __UNDEF__)
-                    vars.append((prefix + token, prefix, value))
-            else:
-                where, value = lookup(token, frame, locals)
-                vars.append((token, where, value))
-        elif token == '.':
-            prefix += lasttoken + '.'
-            parent = value
-        else:
-            parent, prefix = None, ''
-        lasttoken = token
-    return vars
-
-def html(einfo, context=5):
-    """Return a nice HTML document describing a given traceback."""
-    etype, evalue, etb = einfo
-    if isinstance(etype, type):
-        etype = etype.__name__
-    pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
-    date = time.ctime(time.time())
-    head = '<body bgcolor="#f0f0f8">' + pydoc.html.heading(
-        '<big><big>%s</big></big>' %
-        strong(pydoc.html.escape(str(etype))),
-        '#ffffff', '#6622aa', pyver + '<br>' + date) + '''
-<p>A problem occurred in a Python script.  Here is the sequence of
-function calls leading up to the error, in the order they occurred.</p>'''
-
-    indent = '<tt>' + small('&nbsp;' * 5) + '&nbsp;</tt>'
-    frames = []
-    records = inspect.getinnerframes(etb, context)
-    for frame, file, lnum, func, lines, index in records:
-        if file:
-            file = os.path.abspath(file)
-            link = '<a href="file://%s">%s</a>' % (file, pydoc.html.escape(file))
-        else:
-            file = link = '?'
-        args, varargs, varkw, locals = inspect.getargvalues(frame)
-        call = ''
-        if func != '?':
-            call = 'in ' + strong(pydoc.html.escape(func))
-            if func != "<module>":
-                call += inspect.formatargvalues(args, varargs, varkw, locals,
-                    formatvalue=lambda value: '=' + pydoc.html.repr(value))
-
-        highlight = {}
-        def reader(lnum=[lnum]):
-            highlight[lnum[0]] = 1
-            try: return linecache.getline(file, lnum[0])
-            finally: lnum[0] += 1
-        vars = scanvars(reader, frame, locals)
-
-        rows = ['<tr><td bgcolor="#d8bbff">%s%s %s</td></tr>' %
-                ('<big>&nbsp;</big>', link, call)]
-        if index is not None:
-            i = lnum - index
-            for line in lines:
-                num = small('&nbsp;' * (5-len(str(i))) + str(i)) + '&nbsp;'
-                if i in highlight:
-                    line = '<tt>=&gt;%s%s</tt>' % (num, pydoc.html.preformat(line))
-                    rows.append('<tr><td bgcolor="#ffccee">%s</td></tr>' % line)
-                else:
-                    line = '<tt>&nbsp;&nbsp;%s%s</tt>' % (num, pydoc.html.preformat(line))
-                    rows.append('<tr><td>%s</td></tr>' % grey(line))
-                i += 1
-
-        done, dump = {}, []
-        for name, where, value in vars:
-            if name in done: continue
-            done[name] = 1
-            if value is not __UNDEF__:
-                if where in ('global', 'builtin'):
-                    name = ('<em>%s</em> ' % where) + strong(name)
-                elif where == 'local':
-                    name = strong(name)
-                else:
-                    name = where + strong(name.split('.')[-1])
-                dump.append('%s&nbsp;= %s' % (name, pydoc.html.repr(value)))
-            else:
-                dump.append(name + ' <em>undefined</em>')
-
-        rows.append('<tr><td>%s</td></tr>' % small(grey(', '.join(dump))))
-        frames.append('''
-<table width="100%%" cellspacing=0 cellpadding=0 border=0>
-%s</table>''' % '\n'.join(rows))
-
-    exception = ['<p>%s: %s' % (strong(pydoc.html.escape(str(etype))),
-                                pydoc.html.escape(str(evalue)))]
-    for name in dir(evalue):
-        if name[:1] == '_': continue
-        value = pydoc.html.repr(getattr(evalue, name))
-        exception.append('\n<br>%s%s&nbsp;=\n%s' % (indent, name, value))
-
-    return head + ''.join(frames) + ''.join(exception) + '''
-
-
-<!-- The above is a description of an error in a Python program, formatted
-     for a Web browser because the 'cgitb' module was enabled.  In case you
-     are not reading this in a Web browser, here is the original traceback:
-
-%s
--->
-''' % pydoc.html.escape(
-          ''.join(traceback.format_exception(etype, evalue, etb)))
-
-def text(einfo, context=5):
-    """Return a plain text document describing a given traceback."""
-    etype, evalue, etb = einfo
-    if isinstance(etype, type):
-        etype = etype.__name__
-    pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
-    date = time.ctime(time.time())
-    head = "%s\n%s\n%s\n" % (str(etype), pyver, date) + '''
-A problem occurred in a Python script.  Here is the sequence of
-function calls leading up to the error, in the order they occurred.
-'''
-
-    frames = []
-    records = inspect.getinnerframes(etb, context)
-    for frame, file, lnum, func, lines, index in records:
-        file = file and os.path.abspath(file) or '?'
-        args, varargs, varkw, locals = inspect.getargvalues(frame)
-        call = ''
-        if func != '?':
-            call = 'in ' + func
-            if func != "<module>":
-                call += inspect.formatargvalues(args, varargs, varkw, locals,
-                    formatvalue=lambda value: '=' + pydoc.text.repr(value))
-
-        highlight = {}
-        def reader(lnum=[lnum]):
-            highlight[lnum[0]] = 1
-            try: return linecache.getline(file, lnum[0])
-            finally: lnum[0] += 1
-        vars = scanvars(reader, frame, locals)
-
-        rows = [' %s %s' % (file, call)]
-        if index is not None:
-            i = lnum - index
-            for line in lines:
-                num = '%5d ' % i
-                rows.append(num+line.rstrip())
-                i += 1
-
-        done, dump = {}, []
-        for name, where, value in vars:
-            if name in done: continue
-            done[name] = 1
-            if value is not __UNDEF__:
-                if where == 'global': name = 'global ' + name
-                elif where != 'local': name = where + name.split('.')[-1]
-                dump.append('%s = %s' % (name, pydoc.text.repr(value)))
-            else:
-                dump.append(name + ' undefined')
-
-        rows.append('\n'.join(dump))
-        frames.append('\n%s\n' % '\n'.join(rows))
-
-    exception = ['%s: %s' % (str(etype), str(evalue))]
-    for name in dir(evalue):
-        value = pydoc.text.repr(getattr(evalue, name))
-        exception.append('\n%s%s = %s' % (" "*4, name, value))
-
-    return head + ''.join(frames) + ''.join(exception) + '''
-
-The above is a description of an error in a Python program.  Here is
-the original traceback:
-
-%s
-''' % ''.join(traceback.format_exception(etype, evalue, etb))
-
-class Hook:
-    """A hook to replace sys.excepthook that shows tracebacks in HTML."""
-
-    def __init__(self, display=1, logdir=None, context=5, file=None,
-                 format="html"):
-        self.display = display          # send tracebacks to browser if true
-        self.logdir = logdir            # log tracebacks to files if not None
-        self.context = context          # number of source code lines per frame
-        self.file = file or sys.stdout  # place to send the output
-        self.format = format
-
-    def __call__(self, etype, evalue, etb):
-        self.handle((etype, evalue, etb))
-
-    def handle(self, info=None):
-        info = info or sys.exc_info()
-        if self.format == "html":
-            self.file.write(reset())
-
-        formatter = (self.format=="html") and html or text
-        plain = False
-        try:
-            doc = formatter(info, self.context)
-        except:                         # just in case something goes wrong
-            doc = ''.join(traceback.format_exception(*info))
-            plain = True
-
-        if self.display:
-            if plain:
-                doc = pydoc.html.escape(doc)
-                self.file.write('<pre>' + doc + '</pre>\n')
-            else:
-                self.file.write(doc + '\n')
-        else:
-            self.file.write('<p>A problem occurred in a Python script.\n')
-
-        if self.logdir is not None:
-            suffix = ['.txt', '.html'][self.format=="html"]
-            (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir)
-
-            try:
-                with os.fdopen(fd, 'w') as file:
-                    file.write(doc)
-                msg = '%s contains the description of this error.' % path
-            except:
-                msg = 'Tried to save traceback to %s, but failed.' % path
-
-            if self.format == 'html':
-                self.file.write('<p>%s</p>\n' % msg)
-            else:
-                self.file.write(msg + '\n')
-        try:
-            self.file.flush()
-        except: pass
-
-handler = Hook().handle
-def enable(display=1, logdir=None, context=5, format="html"):
-    """Install an exception handler that formats tracebacks as HTML.
-
-    The optional argument 'display' can be set to 0 to suppress sending the
-    traceback to the browser, and 'logdir' can be set to a directory to cause
-    tracebacks to be written to files there."""
-    sys.excepthook = Hook(display=display, logdir=logdir,
-                          context=context, format=format)

Lib/codecs.py

@@ -83,7 +83,7 @@ BOM64_BE = BOM_UTF32_BE
 class CodecInfo(tuple):
     """Codec details when looking up the codec registry"""
 
-    # Private API to allow Python 3.4 to denylist the known non-Unicode
+    # Private API to allow Python 3.4 to blacklist the known non-Unicode
     # codecs in the standard library. A more general mechanism to
     # reliably distinguish test encodings from other codecs will hopefully
     # be defined for Python 3.5
@@ -386,7 +386,7 @@ class StreamWriter(Codec):
 
     def reset(self):
 
-        """ Resets the codec buffers used for keeping internal state.
+        """ Flushes and resets the codec buffers used for keeping state.
 
             Calling this method should ensure that the data on the
             output is put into a clean state, that allows appending
@@ -620,7 +620,7 @@ class StreamReader(Codec):
 
     def reset(self):
 
-        """ Resets the codec buffers used for keeping internal state.
+        """ Resets the codec buffers used for keeping state.
 
             Note that no stream repositioning should take place.
             This method is primarily intended to be able to recover
@@ -838,7 +838,7 @@ class StreamRecoder:
 
     def writelines(self, list):
 
-        data = b''.join(list)
+        data = ''.join(list)
         data, bytesdecoded = self.decode(data, self.errors)
         return self.writer.write(data)
 
@@ -847,12 +847,6 @@ class StreamRecoder:
         self.reader.reset()
         self.writer.reset()
 
-    def seek(self, offset, whence=0):
-        # Seeks must be propagated to both the readers and writers
-        # as they might need to reset their internal buffers.
-        self.reader.seek(offset, whence)
-        self.writer.seek(offset, whence)
-
     def __getattr__(self, name,
                     getattr=getattr):
 
@@ -868,7 +862,7 @@ class StreamRecoder:
 
 ### Shortcuts
 
-def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
+def open(filename, mode='r', encoding=None, errors='strict', buffering=1):
 
     """ Open an encoded file using the given mode and return
         a wrapped version providing transparent encoding/decoding.
@@ -889,8 +883,7 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
         encoding error occurs.
 
         buffering has the same meaning as for the builtin open() API.
-        It defaults to -1 which means that the default buffer size will
-        be used.
+        It defaults to line buffered.
 
         The returned wrapped file object provides an extra attribute
         .encoding which allows querying the used encoding. This
@@ -905,16 +898,11 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
     file = builtins.open(filename, mode, buffering)
     if encoding is None:
         return file
-
-    try:
-        info = lookup(encoding)
-        srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
-        # Add attributes to simplify introspection
-        srw.encoding = encoding
-        return srw
-    except:
-        file.close()
-        raise
+    info = lookup(encoding)
+    srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
+    # Add attributes to simplify introspection
+    srw.encoding = encoding
+    return srw
 
 def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'):
 

Lib/codeop.py

@@ -57,7 +57,6 @@ Compile():
 """
 
 import __future__
-import warnings
 
 _features = [getattr(__future__, fname)
              for fname in __future__.all_feature_names]
@@ -66,7 +65,6 @@ __all__ = ["compile_command", "Compile", "CommandCompiler"]
 
 PyCF_DONT_IMPLY_DEDENT = 0x200          # Matches pythonrun.h
 
-
 def _maybe_compile(compiler, source, filename, symbol):
     # Check for source consisting of only blank lines and comments
     for line in source.split("\n"):
@@ -82,37 +80,27 @@ def _maybe_compile(compiler, source, filename, symbol):
 
     try:
         code = compiler(source, filename, symbol)
-    except SyntaxError:
+    except SyntaxError as err:
         pass
 
-    # Catch syntax warnings after the first compile
-    # to emit warnings (SyntaxWarning, DeprecationWarning) at most once.
-    with warnings.catch_warnings():
-        warnings.simplefilter("error")
-
-        try:
-            code1 = compiler(source + "\n", filename, symbol)
-        except SyntaxError as e:
-            err1 = e
-
-        try:
-            code2 = compiler(source + "\n\n", filename, symbol)
-        except SyntaxError as e:
-            err2 = e
+    try:
+        code1 = compiler(source + "\n", filename, symbol)
+    except SyntaxError as e:
+        err1 = e
 
     try:
-        if code:
-            return code
-        if not code1 and repr(err1) == repr(err2):
-            raise err1
-    finally:
-        err1 = err2 = None
+        code2 = compiler(source + "\n\n", filename, symbol)
+    except SyntaxError as e:
+        err2 = e
 
+    if code:
+        return code
+    if not code1 and repr(err1) == repr(err2):
+        raise err1
 
 def _compile(source, filename, symbol):
     return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT)
 
-
 def compile_command(source, filename="<input>", symbol="single"):
     r"""Compile a command and determine whether it is incomplete.
 
@@ -121,8 +109,7 @@ def compile_command(source, filename="<input>", symbol="single"):
     source -- the source string; may contain \n characters
     filename -- optional filename from which source was read; default
                 "<input>"
-    symbol -- optional grammar start symbol; "single" (default), "exec"
-              or "eval"
+    symbol -- optional grammar start symbol; "single" (default) or "eval"
 
     Return value / exceptions raised:
 
@@ -134,24 +121,21 @@ def compile_command(source, filename="<input>", symbol="single"):
     """
     return _maybe_compile(_compile, source, filename, symbol)
 
-
 class Compile:
     """Instances of this class behave much like the built-in compile
     function, but if one is used to compile text containing a future
     statement, it "remembers" and compiles all subsequent program texts
     with the statement in force."""
-
     def __init__(self):
         self.flags = PyCF_DONT_IMPLY_DEDENT
 
     def __call__(self, source, filename, symbol):
-        codeob = compile(source, filename, symbol, self.flags, True)
+        codeob = compile(source, filename, symbol, self.flags, 1)
         for feature in _features:
             if codeob.co_flags & feature.compiler_flag:
                 self.flags |= feature.compiler_flag
         return codeob
 
-
 class CommandCompiler:
     """Instances of this class have __call__ methods identical in
     signature to compile_command; the difference is that if the

Lib/collections/_defaultdict.py

@@ -1,11 +1,7 @@
-from reprlib import recursive_repr as _recursive_repr
-
 class defaultdict(dict):
     def __init__(self, *args, **kwargs):
         if len(args) >= 1:
             default_factory = args[0]
-            if default_factory is not None and not callable(default_factory):
-                raise TypeError("first argument must be callable or None")
             args = args[1:]
         else:
             default_factory = None
@@ -13,46 +9,11 @@ class defaultdict(dict):
         self.default_factory = default_factory
 
     def __missing__(self, key):
-        if self.default_factory is not None:
-            val = self.default_factory()
+        if self.default_factory:
+            return self.default_factory()
         else:
             raise KeyError(key)
-        self[key] = val
-        return val
-
-    @_recursive_repr()
-    def __repr_factory(factory):
-        return repr(factory)
 
     def __repr__(self):
-        return f"{type(self).__name__}({defaultdict.__repr_factory(self.default_factory)}, {dict.__repr__(self)})"
+        return f"defaultdict({self.default_factory}, {dict.__repr__(self)})"
 
-    def copy(self):
-        return type(self)(self.default_factory, self)
-
-    __copy__ = copy
-
-    def __reduce__(self):
-        if self.default_factory is not None:
-            args = self.default_factory,
-        else:
-            args = ()
-        return type(self), args, None, None, iter(self.items())
-
-    def __or__(self, other):
-        if not isinstance(other, dict):
-            return NotImplemented
-
-        new = defaultdict(self.default_factory, self)
-        new.update(other)
-        return new
-
-    def __ror__(self, other):
-        if not isinstance(other, dict):
-            return NotImplemented
-
-        new = defaultdict(self.default_factory, other)
-        new.update(self)
-        return new
-
-defaultdict.__module__ = 'collections'

Lib/collections/abc.py

@@ -1,3 +1,2 @@
 from _collections_abc import *
 from _collections_abc import __all__
-from _collections_abc import _CallableGenericAlias

Lib/concurrent/futures/__init__.py

@@ -10,44 +10,9 @@ from concurrent.futures._base import (FIRST_COMPLETED,
                                       ALL_COMPLETED,
                                       CancelledError,
                                       TimeoutError,
-                                      InvalidStateError,
-                                      BrokenExecutor,
                                       Future,
                                       Executor,
                                       wait,
                                       as_completed)
-
-__all__ = (
-    'FIRST_COMPLETED',
-    'FIRST_EXCEPTION',
-    'ALL_COMPLETED',
-    'CancelledError',
-    'TimeoutError',
-    'BrokenExecutor',
-    'Future',
-    'Executor',
-    'wait',
-    'as_completed',
-    'ProcessPoolExecutor',
-    'ThreadPoolExecutor',
-)
-
-
-def __dir__():
-    return __all__ + ('__author__', '__doc__')
-
-
-def __getattr__(name):
-    global ProcessPoolExecutor, ThreadPoolExecutor
-
-    if name == 'ProcessPoolExecutor':
-        from .process import ProcessPoolExecutor as pe
-        ProcessPoolExecutor = pe
-        return pe
-
-    if name == 'ThreadPoolExecutor':
-        from .thread import ThreadPoolExecutor as te
-        ThreadPoolExecutor = te
-        return te
-
-    raise AttributeError(f"module {__name__} has no attribute {name}")
+from concurrent.futures.process import ProcessPoolExecutor
+from concurrent.futures.thread import ThreadPoolExecutor

Lib/concurrent/futures/_base.py

@@ -7,7 +7,6 @@ import collections
 import logging
 import threading
 import time
-import types
 
 FIRST_COMPLETED = 'FIRST_COMPLETED'
 FIRST_EXCEPTION = 'FIRST_EXCEPTION'
@@ -54,10 +53,6 @@ class TimeoutError(Error):
     """The operation exceeded the given deadline."""
     pass
 
-class InvalidStateError(Error):
-    """The operation is not allowed in this state."""
-    pass
-
 class _Waiter(object):
     """Provides the event that wait() and as_completed() block on."""
     def __init__(self):
@@ -175,29 +170,6 @@ def _create_and_install_waiters(fs, return_when):
 
     return waiter
 
-
-def _yield_finished_futures(fs, waiter, ref_collect):
-    """
-    Iterate on the list *fs*, yielding finished futures one by one in
-    reverse order.
-    Before yielding a future, *waiter* is removed from its waiters
-    and the future is removed from each set in the collection of sets
-    *ref_collect*.
-
-    The aim of this function is to avoid keeping stale references after
-    the future is yielded and before the iterator resumes.
-    """
-    while fs:
-        f = fs[-1]
-        for futures_set in ref_collect:
-            futures_set.remove(f)
-        with f._condition:
-            f._waiters.remove(waiter)
-        del f
-        # Careful not to keep a reference to the popped value
-        yield fs.pop()
-
-
 def as_completed(fs, timeout=None):
     """An iterator over the given futures that yields each as it completes.
 
@@ -217,30 +189,28 @@ def as_completed(fs, timeout=None):
             before the given timeout.
     """
     if timeout is not None:
-        end_time = timeout + time.monotonic()
+        end_time = timeout + time.time()
 
     fs = set(fs)
-    total_futures = len(fs)
     with _AcquireFutures(fs):
         finished = set(
                 f for f in fs
                 if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
         pending = fs - finished
         waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
-    finished = list(finished)
+
     try:
-        yield from _yield_finished_futures(finished, waiter,
-                                           ref_collect=(fs,))
+        yield from finished
 
         while pending:
             if timeout is None:
                 wait_timeout = None
             else:
-                wait_timeout = end_time - time.monotonic()
+                wait_timeout = end_time - time.time()
                 if wait_timeout < 0:
                     raise TimeoutError(
                             '%d (of %d) futures unfinished' % (
-                            len(pending), total_futures))
+                            len(pending), len(fs)))
 
             waiter.event.wait(wait_timeout)
 
@@ -249,13 +219,11 @@ def as_completed(fs, timeout=None):
                 waiter.finished_futures = []
                 waiter.event.clear()
 
-            # reverse to keep finishing order
-            finished.reverse()
-            yield from _yield_finished_futures(finished, waiter,
-                                               ref_collect=(fs, pending))
+            for future in finished:
+                yield future
+                pending.remove(future)
 
     finally:
-        # Remove waiter from unfinished futures
         for f in fs:
             with f._condition:
                 f._waiters.remove(waiter)
@@ -284,14 +252,13 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
         A named 2-tuple of sets. The first set, named 'done', contains the
         futures that completed (is finished or cancelled) before the wait
         completed. The second set, named 'not_done', contains uncompleted
-        futures. Duplicate futures given to *fs* are removed and will be 
-        returned only once.
+        futures.
     """
-    fs = set(fs)
     with _AcquireFutures(fs):
-        done = {f for f in fs
-                   if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]}
-        not_done = fs - done
+        done = set(f for f in fs
+                   if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
+        not_done = set(fs) - done
+
         if (return_when == FIRST_COMPLETED) and done:
             return DoneAndNotDoneFutures(done, not_done)
         elif (return_when == FIRST_EXCEPTION) and done:
@@ -310,7 +277,7 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
             f._waiters.remove(waiter)
 
     done.update(waiter.finished_futures)
-    return DoneAndNotDoneFutures(done, fs - done)
+    return DoneAndNotDoneFutures(done, set(fs) - done)
 
 class Future(object):
     """Represents the result of an asynchronous computation."""
@@ -381,17 +348,13 @@ class Future(object):
             return self._state == RUNNING
 
     def done(self):
-        """Return True if the future was cancelled or finished executing."""
+        """Return True of the future was cancelled or finished executing."""
         with self._condition:
             return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]
 
     def __get_result(self):
         if self._exception:
-            try:
-                raise self._exception
-            finally:
-                # Break a reference cycle with the exception in self._exception
-                self = None
+            raise self._exception
         else:
             return self._result
 
@@ -410,10 +373,7 @@ class Future(object):
             if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]:
                 self._done_callbacks.append(fn)
                 return
-        try:
-            fn(self)
-        except Exception:
-            LOGGER.exception('exception calling callback for %r', self)
+        fn(self)
 
     def result(self, timeout=None):
         """Return the result of the call that the future represents.
@@ -431,24 +391,20 @@ class Future(object):
                 timeout.
             Exception: If the call raised then that exception will be raised.
         """
-        try:
-            with self._condition:
-                if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
-                    raise CancelledError()
-                elif self._state == FINISHED:
-                    return self.__get_result()
+        with self._condition:
+            if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
+                raise CancelledError()
+            elif self._state == FINISHED:
+                return self.__get_result()
 
-                self._condition.wait(timeout)
+            self._condition.wait(timeout)
 
-                if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
-                    raise CancelledError()
-                elif self._state == FINISHED:
-                    return self.__get_result()
-                else:
-                    raise TimeoutError()
-        finally:
-            # Break a reference cycle with the exception in self._exception
-            self = None
+            if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
+                raise CancelledError()
+            elif self._state == FINISHED:
+                return self.__get_result()
+            else:
+                raise TimeoutError()
 
     def exception(self, timeout=None):
         """Return the exception raised by the call that the future represents.
@@ -530,8 +486,6 @@ class Future(object):
         Should only be used by Executor implementations and unit tests.
         """
         with self._condition:
-            if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
-                raise InvalidStateError('{}: {!r}'.format(self._state, self))
             self._result = result
             self._state = FINISHED
             for waiter in self._waiters:
@@ -545,8 +499,6 @@ class Future(object):
         Should only be used by Executor implementations and unit tests.
         """
         with self._condition:
-            if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
-                raise InvalidStateError('{}: {!r}'.format(self._state, self))
             self._exception = exception
             self._state = FINISHED
             for waiter in self._waiters:
@@ -554,12 +506,10 @@ class Future(object):
             self._condition.notify_all()
         self._invoke_callbacks()
 
-    __class_getitem__ = classmethod(types.GenericAlias)
-
 class Executor(object):
     """This is an abstract base class for concrete asynchronous executors."""
 
-    def submit(self, fn, /, *args, **kwargs):
+    def submit(self, fn, *args, **kwargs):
         """Submits a callable to be executed with the given arguments.
 
         Schedules the callable to be executed as fn(*args, **kwargs) and returns
@@ -593,7 +543,7 @@ class Executor(object):
             Exception: If fn(*args) raises for any values.
         """
         if timeout is not None:
-            end_time = timeout + time.monotonic()
+            end_time = timeout + time.time()
 
         fs = [self.submit(fn, *args) for args in zip(*iterables)]
 
@@ -601,20 +551,17 @@ class Executor(object):
         # before the first iterator value is required.
         def result_iterator():
             try:
-                # reverse to keep finishing order
-                fs.reverse()
-                while fs:
-                    # Careful not to keep a reference to the popped future
+                for future in fs:
                     if timeout is None:
-                        yield fs.pop().result()
+                        yield future.result()
                     else:
-                        yield fs.pop().result(end_time - time.monotonic())
+                        yield future.result(end_time - time.time())
             finally:
                 for future in fs:
                     future.cancel()
         return result_iterator()
 
-    def shutdown(self, wait=True, *, cancel_futures=False):
+    def shutdown(self, wait=True):
         """Clean-up the resources associated with the Executor.
 
         It is safe to call this method several times. Otherwise, no other
@@ -624,9 +571,6 @@ class Executor(object):
             wait: If True then shutdown will not return until all running
                 futures have finished executing and the resources used by the
                 executor have been reclaimed.
-            cancel_futures: If True then shutdown will cancel all pending
-                futures. Futures that are completed or running will not be
-                cancelled.
         """
         pass
 
@@ -636,9 +580,3 @@ class Executor(object):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.shutdown(wait=True)
         return False
-
-
-class BrokenExecutor(RuntimeError):
-    """
-    Raised when a executor has become non-functional after a severe failure.
-    """

Lib/concurrent/futures/process.py

@@ -3,15 +3,15 @@
 
 """Implements ProcessPoolExecutor.
 
-The following diagram and text describe the data-flow through the system:
+The follow diagram and text describe the data-flow through the system:
 
 |======================= In-process =====================|== Out-of-process ==|
 
 +----------+     +----------+       +--------+     +-----------+    +---------+
-|          |  => | Work Ids |       |        |     | Call Q    |    | Process |
-|          |     +----------+       |        |     +-----------+    |  Pool   |
-|          |     | ...      |       |        |     | ...       |    +---------+
-|          |     | 6        |    => |        |  => | 5, call() | => |         |
+|          |  => | Work Ids |    => |        |  => | Call Q    | => |         |
+|          |     +----------+       |        |     +-----------+    |         |
+|          |     | ...      |       |        |     | ...       |    |         |
+|          |     | 6        |       |        |     | 5, call() |    |         |
 |          |     | 7        |       |        |     | ...       |    |         |
 | Process  |     | ...      |       | Local  |     +-----------+    | Process |
 |  Pool    |     +----------+       | Worker |                      |  #1..n  |
@@ -45,74 +45,52 @@ Process #1..n:
 
 __author__ = 'Brian Quinlan (brian@sweetapp.com)'
 
+import atexit
 import os
 from concurrent.futures import _base
 import queue
-import multiprocessing as mp
-import multiprocessing.connection
-from multiprocessing.queues import Queue
+from queue import Full
+import multiprocessing
+from multiprocessing import SimpleQueue
+from multiprocessing.connection import wait
 import threading
 import weakref
 from functools import partial
 import itertools
-import sys
 import traceback
 
+# Workers are created as daemon threads and processes. This is done to allow the
+# interpreter to exit when there are still idle processes in a
+# ProcessPoolExecutor's process pool (i.e. shutdown() was not called). However,
+# allowing workers to die with the interpreter has two undesirable properties:
+#   - The workers would still be running during interpreter shutdown,
+#     meaning that they would fail in unpredictable ways.
+#   - The workers could be killed while evaluating a work item, which could
+#     be bad if the callable being evaluated has external side-effects e.g.
+#     writing to a file.
+#
+# To work around this problem, an exit handler is installed which tells the
+# workers to exit when their work queues are empty and then waits until the
+# threads/processes finish.
 
-_threads_wakeups = weakref.WeakKeyDictionary()
-_global_shutdown = False
-
-
-class _ThreadWakeup:
-    def __init__(self):
-        self._closed = False
-        self._reader, self._writer = mp.Pipe(duplex=False)
-
-    def close(self):
-        if not self._closed:
-            self._closed = True
-            self._writer.close()
-            self._reader.close()
-
-    def wakeup(self):
-        if not self._closed:
-            self._writer.send_bytes(b"")
-
-    def clear(self):
-        if not self._closed:
-            while self._reader.poll():
-                self._reader.recv_bytes()
-
+_threads_queues = weakref.WeakKeyDictionary()
+_shutdown = False
 
 def _python_exit():
-    global _global_shutdown
-    _global_shutdown = True
-    items = list(_threads_wakeups.items())
-    for _, thread_wakeup in items:
-        # call not protected by ProcessPoolExecutor._shutdown_lock
-        thread_wakeup.wakeup()
-    for t, _ in items:
+    global _shutdown
+    _shutdown = True
+    items = list(_threads_queues.items())
+    for t, q in items:
+        q.put(None)
+    for t, q in items:
         t.join()
 
-# Register for `_python_exit()` to be called just before joining all
-# non-daemon threads. This is used instead of `atexit.register()` for
-# compatibility with subinterpreters, which no longer support daemon threads.
-# See bpo-39812 for context.
-threading._register_atexit(_python_exit)
-
 # Controls how many more calls than processes will be queued in the call queue.
 # A smaller number will mean that processes spend more time idle waiting for
 # work while a larger number will make Future.cancel() succeed less frequently
 # (Futures in the call queue cannot be cancelled).
 EXTRA_QUEUED_CALLS = 1
 
-
-# On Windows, WaitForMultipleObjects is used to wait for processes to finish.
-# It can wait on, at most, 63 objects. There is an overhead of two objects:
-# - the result queue reader
-# - the thread wakeup reader
-_MAX_WINDOWS_WORKERS = 63 - 2
-
 # Hack to embed stringification of remote traceback in local traceback
 
 class _RemoteTraceback(Exception):
@@ -126,9 +104,6 @@ class _ExceptionWithTraceback:
         tb = traceback.format_exception(type(exc), exc, tb)
         tb = ''.join(tb)
         self.exc = exc
-        # Traceback object needs to be garbage-collected as its frames
-        # contain references to all the objects in the exception scope
-        self.exc.__traceback__ = None
         self.tb = '\n"""\n%s"""' % tb
     def __reduce__(self):
         return _rebuild_exc, (self.exc, self.tb)
@@ -157,32 +132,6 @@ class _CallItem(object):
         self.args = args
         self.kwargs = kwargs
 
-
-class _SafeQueue(Queue):
-    """Safe Queue set exception to the future object linked to a job"""
-    def __init__(self, max_size=0, *, ctx, pending_work_items, shutdown_lock,
-                 thread_wakeup):
-        self.pending_work_items = pending_work_items
-        self.shutdown_lock = shutdown_lock
-        self.thread_wakeup = thread_wakeup
-        super().__init__(max_size, ctx=ctx)
-
-    def _on_queue_feeder_error(self, e, obj):
-        if isinstance(obj, _CallItem):
-            tb = traceback.format_exception(type(e), e, e.__traceback__)
-            e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb)))
-            work_item = self.pending_work_items.pop(obj.work_id, None)
-            with self.shutdown_lock:
-                self.thread_wakeup.wakeup()
-            # work_item can be None if another process terminated. In this
-            # case, the executor_manager_thread fails all work_items
-            # with BrokenProcessPool
-            if work_item is not None:
-                work_item.future.set_exception(e)
-        else:
-            super()._on_queue_feeder_error(e, obj)
-
-
 def _get_chunks(*iterables, chunksize):
     """ Iterates over zip()ed iterables in chunks. """
     it = zip(*iterables)
@@ -192,7 +141,6 @@ def _get_chunks(*iterables, chunksize):
             return
         yield chunk
 
-
 def _process_chunk(fn, chunk):
     """ Processes a chunk of an iterable passed to map.
 
@@ -204,38 +152,19 @@ def _process_chunk(fn, chunk):
     """
     return [fn(*args) for args in chunk]
 
-
-def _sendback_result(result_queue, work_id, result=None, exception=None):
-    """Safely send back the given result or exception"""
-    try:
-        result_queue.put(_ResultItem(work_id, result=result,
-                                     exception=exception))
-    except BaseException as e:
-        exc = _ExceptionWithTraceback(e, e.__traceback__)
-        result_queue.put(_ResultItem(work_id, exception=exc))
-
-
-def _process_worker(call_queue, result_queue, initializer, initargs):
+def _process_worker(call_queue, result_queue):
     """Evaluates calls from call_queue and places the results in result_queue.
 
     This worker is run in a separate process.
 
     Args:
-        call_queue: A ctx.Queue of _CallItems that will be read and
+        call_queue: A multiprocessing.Queue of _CallItems that will be read and
             evaluated by the worker.
-        result_queue: A ctx.Queue of _ResultItems that will written
+        result_queue: A multiprocessing.Queue of _ResultItems that will written
             to by the worker.
-        initializer: A callable initializer, or None
-        initargs: A tuple of args for the initializer
+        shutdown: A multiprocessing.Event that will be set as a signal to the
+            worker that it should exit when call_queue is empty.
     """
-    if initializer is not None:
-        try:
-            initializer(*initargs)
-        except BaseException:
-            _base.LOGGER.critical('Exception in initializer:', exc_info=True)
-            # The parent will notice that the process stopped and
-            # mark the pool broken
-            return
     while True:
         call_item = call_queue.get(block=True)
         if call_item is None:
@@ -246,303 +175,171 @@ def _process_worker(call_queue, result_queue, initializer, initargs):
             r = call_item.fn(*call_item.args, **call_item.kwargs)
         except BaseException as e:
             exc = _ExceptionWithTraceback(e, e.__traceback__)
-            _sendback_result(result_queue, call_item.work_id, exception=exc)
+            result_queue.put(_ResultItem(call_item.work_id, exception=exc))
         else:
-            _sendback_result(result_queue, call_item.work_id, result=r)
-            del r
+            result_queue.put(_ResultItem(call_item.work_id,
+                                         result=r))
 
-        # Liberate the resource as soon as possible, to avoid holding onto
-        # open files or shared memory that is not needed anymore
-        del call_item
+def _add_call_item_to_queue(pending_work_items,
+                            work_ids,
+                            call_queue):
+    """Fills call_queue with _WorkItems from pending_work_items.
 
-
-class _ExecutorManagerThread(threading.Thread):
-    """Manages the communication between this process and the worker processes.
-
-    The manager is run in a local thread.
+    This function never blocks.
 
     Args:
-        executor: A reference to the ProcessPoolExecutor that owns
-            this thread. A weakref will be own by the manager as well as
-            references to internal objects used to introspect the state of
-            the executor.
+        pending_work_items: A dict mapping work ids to _WorkItems e.g.
+            {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
+        work_ids: A queue.Queue of work ids e.g. Queue([5, 6, ...]). Work ids
+            are consumed and the corresponding _WorkItems from
+            pending_work_items are transformed into _CallItems and put in
+            call_queue.
+        call_queue: A multiprocessing.Queue that will be filled with _CallItems
+            derived from _WorkItems.
     """
+    while True:
+        if call_queue.full():
+            return
+        try:
+            work_id = work_ids.get(block=False)
+        except queue.Empty:
+            return
+        else:
+            work_item = pending_work_items[work_id]
 
-    def __init__(self, executor):
-        # Store references to necessary internals of the executor.
-
-        # A _ThreadWakeup to allow waking up the queue_manager_thread from the
-        # main Thread and avoid deadlocks caused by permanently locked queues.
-        self.thread_wakeup = executor._executor_manager_thread_wakeup
-        self.shutdown_lock = executor._shutdown_lock
-
-        # A weakref.ref to the ProcessPoolExecutor that owns this thread. Used
-        # to determine if the ProcessPoolExecutor has been garbage collected
-        # and that the manager can exit.
-        # When the executor gets garbage collected, the weakref callback
-        # will wake up the queue management thread so that it can terminate
-        # if there is no pending work item.
-        def weakref_cb(_,
-                       thread_wakeup=self.thread_wakeup,
-                       shutdown_lock=self.shutdown_lock):
-            mp.util.debug('Executor collected: triggering callback for'
-                          ' QueueManager wakeup')
-            with shutdown_lock:
-                thread_wakeup.wakeup()
-
-        self.executor_reference = weakref.ref(executor, weakref_cb)
-
-        # A list of the ctx.Process instances used as workers.
-        self.processes = executor._processes
-
-        # A ctx.Queue that will be filled with _CallItems derived from
-        # _WorkItems for processing by the process workers.
-        self.call_queue = executor._call_queue
-
-        # A ctx.SimpleQueue of _ResultItems generated by the process workers.
-        self.result_queue = executor._result_queue
-
-        # A queue.Queue of work ids e.g. Queue([5, 6, ...]).
-        self.work_ids_queue = executor._work_ids
-
-        # A dict mapping work ids to _WorkItems e.g.
-        #     {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
-        self.pending_work_items = executor._pending_work_items
-
-        super().__init__()
-
-    def run(self):
-        # Main loop for the executor manager thread.
-
-        while True:
-            self.add_call_item_to_queue()
-
-            result_item, is_broken, cause = self.wait_result_broken_or_wakeup()
-
-            if is_broken:
-                self.terminate_broken(cause)
-                return
-            if result_item is not None:
-                self.process_result_item(result_item)
-                # Delete reference to result_item to avoid keeping references
-                # while waiting on new results.
-                del result_item
-
-                # attempt to increment idle process count
-                executor = self.executor_reference()
-                if executor is not None:
-                    executor._idle_worker_semaphore.release()
-                del executor
-
-            if self.is_shutting_down():
-                self.flag_executor_shutting_down()
-
-                # Since no new work items can be added, it is safe to shutdown
-                # this thread if there are no pending work items.
-                if not self.pending_work_items:
-                    self.join_executor_internals()
-                    return
-
-    def add_call_item_to_queue(self):
-        # Fills call_queue with _WorkItems from pending_work_items.
-        # This function never blocks.
-        while True:
-            if self.call_queue.full():
-                return
-            try:
-                work_id = self.work_ids_queue.get(block=False)
-            except queue.Empty:
-                return
+            if work_item.future.set_running_or_notify_cancel():
+                call_queue.put(_CallItem(work_id,
+                                         work_item.fn,
+                                         work_item.args,
+                                         work_item.kwargs),
+                               block=True)
             else:
-                work_item = self.pending_work_items[work_id]
+                del pending_work_items[work_id]
+                continue
 
-                if work_item.future.set_running_or_notify_cancel():
-                    self.call_queue.put(_CallItem(work_id,
-                                                  work_item.fn,
-                                                  work_item.args,
-                                                  work_item.kwargs),
-                                        block=True)
-                else:
-                    del self.pending_work_items[work_id]
-                    continue
+def _queue_management_worker(executor_reference,
+                             processes,
+                             pending_work_items,
+                             work_ids_queue,
+                             call_queue,
+                             result_queue):
+    """Manages the communication between this process and the worker processes.
 
-    def wait_result_broken_or_wakeup(self):
-        # Wait for a result to be ready in the result_queue while checking
-        # that all worker processes are still running, or for a wake up
-        # signal send. The wake up signals come either from new tasks being
-        # submitted, from the executor being shutdown/gc-ed, or from the
-        # shutdown of the python interpreter.
-        result_reader = self.result_queue._reader
-        assert not self.thread_wakeup._closed
-        wakeup_reader = self.thread_wakeup._reader
-        readers = [result_reader, wakeup_reader]
-        worker_sentinels = [p.sentinel for p in list(self.processes.values())]
-        ready = mp.connection.wait(readers + worker_sentinels)
+    This function is run in a local thread.
 
-        cause = None
-        is_broken = True
-        result_item = None
-        if result_reader in ready:
-            try:
-                result_item = result_reader.recv()
-                is_broken = False
-            except BaseException as e:
-                cause = traceback.format_exception(type(e), e, e.__traceback__)
+    Args:
+        executor_reference: A weakref.ref to the ProcessPoolExecutor that owns
+            this thread. Used to determine if the ProcessPoolExecutor has been
+            garbage collected and that this function can exit.
+        process: A list of the multiprocessing.Process instances used as
+            workers.
+        pending_work_items: A dict mapping work ids to _WorkItems e.g.
+            {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
+        work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]).
+        call_queue: A multiprocessing.Queue that will be filled with _CallItems
+            derived from _WorkItems for processing by the process workers.
+        result_queue: A multiprocessing.Queue of _ResultItems generated by the
+            process workers.
+    """
+    executor = None
 
-        elif wakeup_reader in ready:
-            is_broken = False
+    def shutting_down():
+        return _shutdown or executor is None or executor._shutdown_thread
 
-        with self.shutdown_lock:
-            self.thread_wakeup.clear()
+    def shutdown_worker():
+        # This is an upper bound
+        nb_children_alive = sum(p.is_alive() for p in processes.values())
+        for i in range(0, nb_children_alive):
+            call_queue.put_nowait(None)
+        # Release the queue's resources as soon as possible.
+        call_queue.close()
+        # If .join() is not called on the created processes then
+        # some multiprocessing.Queue methods may deadlock on Mac OS X.
+        for p in processes.values():
+            p.join()
 
-        return result_item, is_broken, cause
+    reader = result_queue._reader
 
-    def process_result_item(self, result_item):
-        # Process the received a result_item. This can be either the PID of a
-        # worker that exited gracefully or a _ResultItem
+    while True:
+        _add_call_item_to_queue(pending_work_items,
+                                work_ids_queue,
+                                call_queue)
 
+        sentinels = [p.sentinel for p in processes.values()]
+        assert sentinels
+        ready = wait([reader] + sentinels)
+        if reader in ready:
+            result_item = reader.recv()
+        else:
+            # Mark the process pool broken so that submits fail right now.
+            executor = executor_reference()
+            if executor is not None:
+                executor._broken = True
+                executor._shutdown_thread = True
+                executor = None
+            # All futures in flight must be marked failed
+            for work_id, work_item in pending_work_items.items():
+                work_item.future.set_exception(
+                    BrokenProcessPool(
+                        "A process in the process pool was "
+                        "terminated abruptly while the future was "
+                        "running or pending."
+                    ))
+                # Delete references to object. See issue16284
+                del work_item
+            pending_work_items.clear()
+            # Terminate remaining workers forcibly: the queues or their
+            # locks may be in a dirty state and block forever.
+            for p in processes.values():
+                p.terminate()
+            shutdown_worker()
+            return
         if isinstance(result_item, int):
             # Clean shutdown of a worker using its PID
             # (avoids marking the executor broken)
-            assert self.is_shutting_down()
-            p = self.processes.pop(result_item)
+            assert shutting_down()
+            p = processes.pop(result_item)
             p.join()
-            if not self.processes:
-                self.join_executor_internals()
+            if not processes:
+                shutdown_worker()
                 return
-        else:
-            # Received a _ResultItem so mark the future as completed.
-            work_item = self.pending_work_items.pop(result_item.work_id, None)
+        elif result_item is not None:
+            work_item = pending_work_items.pop(result_item.work_id, None)
             # work_item can be None if another process terminated (see above)
             if work_item is not None:
                 if result_item.exception:
                     work_item.future.set_exception(result_item.exception)
                 else:
                     work_item.future.set_result(result_item.result)
-
-    def is_shutting_down(self):
-        # Check whether we should start shutting down the executor.
-        executor = self.executor_reference()
+                # Delete references to object. See issue16284
+                del work_item
+        # Check whether we should start shutting down.
+        executor = executor_reference()
         # No more work items can be added if:
         #   - The interpreter is shutting down OR
         #   - The executor that owns this worker has been collected OR
         #   - The executor that owns this worker has been shutdown.
-        return (_global_shutdown or executor is None
-                or executor._shutdown_thread)
-
-    def terminate_broken(self, cause):
-        # Terminate the executor because it is in a broken state. The cause
-        # argument can be used to display more information on the error that
-        # lead the executor into becoming broken.
-
-        # Mark the process pool broken so that submits fail right now.
-        executor = self.executor_reference()
-        if executor is not None:
-            executor._broken = ('A child process terminated '
-                                'abruptly, the process pool is not '
-                                'usable anymore')
-            executor._shutdown_thread = True
-            executor = None
-
-        # All pending tasks are to be marked failed with the following
-        # BrokenProcessPool error
-        bpe = BrokenProcessPool("A process in the process pool was "
-                                "terminated abruptly while the future was "
-                                "running or pending.")
-        if cause is not None:
-            bpe.__cause__ = _RemoteTraceback(
-                f"\n'''\n{''.join(cause)}'''")
-
-        # Mark pending tasks as failed.
-        for work_id, work_item in self.pending_work_items.items():
-            work_item.future.set_exception(bpe)
-            # Delete references to object. See issue16284
-            del work_item
-        self.pending_work_items.clear()
-
-        # Terminate remaining workers forcibly: the queues or their
-        # locks may be in a dirty state and block forever.
-        for p in self.processes.values():
-            p.terminate()
-
-        # clean up resources
-        self.join_executor_internals()
-
-    def flag_executor_shutting_down(self):
-        # Flag the executor as shutting down and cancel remaining tasks if
-        # requested as early as possible if it is not gc-ed yet.
-        executor = self.executor_reference()
-        if executor is not None:
-            executor._shutdown_thread = True
-            # Cancel pending work items if requested.
-            if executor._cancel_pending_futures:
-                # Cancel all pending futures and update pending_work_items
-                # to only have futures that are currently running.
-                new_pending_work_items = {}
-                for work_id, work_item in self.pending_work_items.items():
-                    if not work_item.future.cancel():
-                        new_pending_work_items[work_id] = work_item
-                self.pending_work_items = new_pending_work_items
-                # Drain work_ids_queue since we no longer need to
-                # add items to the call queue.
-                while True:
-                    try:
-                        self.work_ids_queue.get_nowait()
-                    except queue.Empty:
-                        break
-                # Make sure we do this only once to not waste time looping
-                # on running processes over and over.
-                executor._cancel_pending_futures = False
-
-    def shutdown_workers(self):
-        n_children_to_stop = self.get_n_children_alive()
-        n_sentinels_sent = 0
-        # Send the right number of sentinels, to make sure all children are
-        # properly terminated.
-        while (n_sentinels_sent < n_children_to_stop
-                and self.get_n_children_alive() > 0):
-            for i in range(n_children_to_stop - n_sentinels_sent):
-                try:
-                    self.call_queue.put_nowait(None)
-                    n_sentinels_sent += 1
-                except queue.Full:
-                    break
-
-    def join_executor_internals(self):
-        self.shutdown_workers()
-        # Release the queue's resources as soon as possible.
-        self.call_queue.close()
-        self.call_queue.join_thread()
-        with self.shutdown_lock:
-            self.thread_wakeup.close()
-        # If .join() is not called on the created processes then
-        # some ctx.Queue methods may deadlock on Mac OS X.
-        for p in self.processes.values():
-            p.join()
-
-    def get_n_children_alive(self):
-        # This is an upper bound on the number of children alive.
-        return sum(p.is_alive() for p in self.processes.values())
-
+        if shutting_down():
+            try:
+                # Since no new work items can be added, it is safe to shutdown
+                # this thread if there are no pending work items.
+                if not pending_work_items:
+                    shutdown_worker()
+                    return
+            except Full:
+                # This is not a problem: we will eventually be woken up (in
+                # result_queue.get()) and be able to send a sentinel again.
+                pass
+        executor = None
 
 _system_limits_checked = False
 _system_limited = None
-
-
 def _check_system_limits():
     global _system_limits_checked, _system_limited
     if _system_limits_checked:
         if _system_limited:
             raise NotImplementedError(_system_limited)
     _system_limits_checked = True
-    try:
-        import multiprocessing.synchronize
-    except ImportError:
-        _system_limited = (
-            "This Python build lacks multiprocessing.synchronize, usually due "
-            "to named semaphores being unavailable on this platform."
-        )
-        raise NotImplementedError(_system_limited)
     try:
         nsems_max = os.sysconf("SC_SEM_NSEMS_MAX")
     except (AttributeError, ValueError):
@@ -556,24 +353,11 @@ def _check_system_limits():
         # minimum number of semaphores available
         # according to POSIX
         return
-    _system_limited = ("system provides too few semaphores (%d"
-                       " available, 256 necessary)" % nsems_max)
+    _system_limited = "system provides too few semaphores (%d available, 256 necessary)" % nsems_max
     raise NotImplementedError(_system_limited)
 
 
-def _chain_from_iterable_of_lists(iterable):
-    """
-    Specialized implementation of itertools.chain.from_iterable.
-    Each item in *iterable* should be a list.  This function is
-    careful not to keep references to yielded objects.
-    """
-    for element in iterable:
-        element.reverse()
-        while element:
-            yield element.pop()
-
-
-class BrokenProcessPool(_base.BrokenExecutor):
+class BrokenProcessPool(RuntimeError):
     """
     Raised when a process in a ProcessPoolExecutor terminated abruptly
     while a future was in the running state.
@@ -581,143 +365,82 @@ class BrokenProcessPool(_base.BrokenExecutor):
 
 
 class ProcessPoolExecutor(_base.Executor):
-    def __init__(self, max_workers=None, mp_context=None,
-                 initializer=None, initargs=()):
+    def __init__(self, max_workers=None):
         """Initializes a new ProcessPoolExecutor instance.
 
         Args:
             max_workers: The maximum number of processes that can be used to
                 execute the given calls. If None or not given then as many
                 worker processes will be created as the machine has processors.
-            mp_context: A multiprocessing context to launch the workers. This
-                object should provide SimpleQueue, Queue and Process.
-            initializer: A callable used to initialize worker processes.
-            initargs: A tuple of arguments to pass to the initializer.
         """
         _check_system_limits()
 
         if max_workers is None:
             self._max_workers = os.cpu_count() or 1
-            if sys.platform == 'win32':
-                self._max_workers = min(_MAX_WINDOWS_WORKERS,
-                                        self._max_workers)
         else:
             if max_workers <= 0:
                 raise ValueError("max_workers must be greater than 0")
-            elif (sys.platform == 'win32' and
-                max_workers > _MAX_WINDOWS_WORKERS):
-                raise ValueError(
-                    f"max_workers must be <= {_MAX_WINDOWS_WORKERS}")
 
             self._max_workers = max_workers
 
-        if mp_context is None:
-            mp_context = mp.get_context()
-        self._mp_context = mp_context
-
-        # https://github.com/python/cpython/issues/90622
-        self._safe_to_dynamically_spawn_children = (
-                self._mp_context.get_start_method(allow_none=False) != "fork")
-
-        if initializer is not None and not callable(initializer):
-            raise TypeError("initializer must be a callable")
-        self._initializer = initializer
-        self._initargs = initargs
-
-        # Management thread
-        self._executor_manager_thread = None
-
+        # Make the call queue slightly larger than the number of processes to
+        # prevent the worker processes from idling. But don't make it too big
+        # because futures in the call queue cannot be cancelled.
+        self._call_queue = multiprocessing.Queue(self._max_workers +
+                                                 EXTRA_QUEUED_CALLS)
+        # Killed worker processes can produce spurious "broken pipe"
+        # tracebacks in the queue's own worker thread. But we detect killed
+        # processes anyway, so silence the tracebacks.
+        self._call_queue._ignore_epipe = True
+        self._result_queue = SimpleQueue()
+        self._work_ids = queue.Queue()
+        self._queue_management_thread = None
         # Map of pids to processes
         self._processes = {}
 
         # Shutdown is a two-step process.
         self._shutdown_thread = False
         self._shutdown_lock = threading.Lock()
-        self._idle_worker_semaphore = threading.Semaphore(0)
         self._broken = False
         self._queue_count = 0
         self._pending_work_items = {}
-        self._cancel_pending_futures = False
 
-        # _ThreadWakeup is a communication channel used to interrupt the wait
-        # of the main loop of executor_manager_thread from another thread (e.g.
-        # when calling executor.submit or executor.shutdown). We do not use the
-        # _result_queue to send wakeup signals to the executor_manager_thread
-        # as it could result in a deadlock if a worker process dies with the
-        # _result_queue write lock still acquired.
-        #
-        # _shutdown_lock must be locked to access _ThreadWakeup.
-        self._executor_manager_thread_wakeup = _ThreadWakeup()
-
-        # Create communication channels for the executor
-        # Make the call queue slightly larger than the number of processes to
-        # prevent the worker processes from idling. But don't make it too big
-        # because futures in the call queue cannot be cancelled.
-        queue_size = self._max_workers + EXTRA_QUEUED_CALLS
-        self._call_queue = _SafeQueue(
-            max_size=queue_size, ctx=self._mp_context,
-            pending_work_items=self._pending_work_items,
-            shutdown_lock=self._shutdown_lock,
-            thread_wakeup=self._executor_manager_thread_wakeup)
-        # Killed worker processes can produce spurious "broken pipe"
-        # tracebacks in the queue's own worker thread. But we detect killed
-        # processes anyway, so silence the tracebacks.
-        self._call_queue._ignore_epipe = True
-        self._result_queue = mp_context.SimpleQueue()
-        self._work_ids = queue.Queue()
-
-    def _start_executor_manager_thread(self):
-        if self._executor_manager_thread is None:
+    def _start_queue_management_thread(self):
+        # When the executor gets lost, the weakref callback will wake up
+        # the queue management thread.
+        def weakref_cb(_, q=self._result_queue):
+            q.put(None)
+        if self._queue_management_thread is None:
             # Start the processes so that their sentinels are known.
-            if not self._safe_to_dynamically_spawn_children:  # ie, using fork.
-                self._launch_processes()
-            self._executor_manager_thread = _ExecutorManagerThread(self)
-            self._executor_manager_thread.start()
-            _threads_wakeups[self._executor_manager_thread] = \
-                self._executor_manager_thread_wakeup
+            self._adjust_process_count()
+            self._queue_management_thread = threading.Thread(
+                    target=_queue_management_worker,
+                    args=(weakref.ref(self, weakref_cb),
+                          self._processes,
+                          self._pending_work_items,
+                          self._work_ids,
+                          self._call_queue,
+                          self._result_queue))
+            self._queue_management_thread.daemon = True
+            self._queue_management_thread.start()
+            _threads_queues[self._queue_management_thread] = self._result_queue
 
     def _adjust_process_count(self):
-        # if there's an idle process, we don't need to spawn a new one.
-        if self._idle_worker_semaphore.acquire(blocking=False):
-            return
-
-        process_count = len(self._processes)
-        if process_count < self._max_workers:
-            # Assertion disabled as this codepath is also used to replace a
-            # worker that unexpectedly dies, even when using the 'fork' start
-            # method. That means there is still a potential deadlock bug. If a
-            # 'fork' mp_context worker dies, we'll be forking a new one when
-            # we know a thread is running (self._executor_manager_thread).
-            #assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622'
-            self._spawn_process()
-
-    def _launch_processes(self):
-        # https://github.com/python/cpython/issues/90622
-        assert not self._executor_manager_thread, (
-                'Processes cannot be fork()ed after the thread has started, '
-                'deadlock in the child processes could result.')
         for _ in range(len(self._processes), self._max_workers):
-            self._spawn_process()
+            p = multiprocessing.Process(
+                    target=_process_worker,
+                    args=(self._call_queue,
+                          self._result_queue))
+            p.start()
+            self._processes[p.pid] = p
 
-    def _spawn_process(self):
-        p = self._mp_context.Process(
-            target=_process_worker,
-            args=(self._call_queue,
-                  self._result_queue,
-                  self._initializer,
-                  self._initargs))
-        p.start()
-        self._processes[p.pid] = p
-
-    def submit(self, fn, /, *args, **kwargs):
+    def submit(self, fn, *args, **kwargs):
         with self._shutdown_lock:
             if self._broken:
-                raise BrokenProcessPool(self._broken)
+                raise BrokenProcessPool('A child process terminated '
+                    'abruptly, the process pool is not usable anymore')
             if self._shutdown_thread:
                 raise RuntimeError('cannot schedule new futures after shutdown')
-            if _global_shutdown:
-                raise RuntimeError('cannot schedule new futures after '
-                                   'interpreter shutdown')
 
             f = _base.Future()
             w = _WorkItem(f, fn, args, kwargs)
@@ -726,11 +449,9 @@ class ProcessPoolExecutor(_base.Executor):
             self._work_ids.put(self._queue_count)
             self._queue_count += 1
             # Wake up queue management thread
-            self._executor_manager_thread_wakeup.wakeup()
+            self._result_queue.put(None)
 
-            if self._safe_to_dynamically_spawn_children:
-                self._adjust_process_count()
-            self._start_executor_manager_thread()
+            self._start_queue_management_thread()
             return f
     submit.__doc__ = _base.Executor.submit.__doc__
 
@@ -761,26 +482,22 @@ class ProcessPoolExecutor(_base.Executor):
         results = super().map(partial(_process_chunk, fn),
                               _get_chunks(*iterables, chunksize=chunksize),
                               timeout=timeout)
-        return _chain_from_iterable_of_lists(results)
+        return itertools.chain.from_iterable(results)
 
-    def shutdown(self, wait=True, *, cancel_futures=False):
+    def shutdown(self, wait=True):
         with self._shutdown_lock:
-            self._cancel_pending_futures = cancel_futures
             self._shutdown_thread = True
-            if self._executor_manager_thread_wakeup is not None:
-                # Wake up queue management thread
-                self._executor_manager_thread_wakeup.wakeup()
-
-        if self._executor_manager_thread is not None and wait:
-            self._executor_manager_thread.join()
+        if self._queue_management_thread:
+            # Wake up queue management thread
+            self._result_queue.put(None)
+            if wait:
+                self._queue_management_thread.join()
         # To reduce the risk of opening too many files, remove references to
         # objects that use file descriptors.
-        self._executor_manager_thread = None
+        self._queue_management_thread = None
         self._call_queue = None
-        if self._result_queue is not None and wait:
-            self._result_queue.close()
         self._result_queue = None
         self._processes = None
-        self._executor_manager_thread_wakeup = None
-
     shutdown.__doc__ = _base.Executor.shutdown.__doc__
+
+atexit.register(_python_exit)

Lib/concurrent/futures/thread.py

@@ -5,43 +5,40 @@
 
 __author__ = 'Brian Quinlan (brian@sweetapp.com)'
 
+import atexit
 from concurrent.futures import _base
-import itertools
 import queue
 import threading
-import types
 import weakref
 import os
 
+# Workers are created as daemon threads. This is done to allow the interpreter
+# to exit when there are still idle threads in a ThreadPoolExecutor's thread
+# pool (i.e. shutdown() was not called). However, allowing workers to die with
+# the interpreter has two undesirable properties:
+#   - The workers would still be running during interpreter shutdown,
+#     meaning that they would fail in unpredictable ways.
+#   - The workers could be killed while evaluating a work item, which could
+#     be bad if the callable being evaluated has external side-effects e.g.
+#     writing to a file.
+#
+# To work around this problem, an exit handler is installed which tells the
+# workers to exit when their work queues are empty and then waits until the
+# threads finish.
 
 _threads_queues = weakref.WeakKeyDictionary()
 _shutdown = False
-# Lock that ensures that new workers are not created while the interpreter is
-# shutting down. Must be held while mutating _threads_queues and _shutdown.
-_global_shutdown_lock = threading.Lock()
 
 def _python_exit():
     global _shutdown
-    with _global_shutdown_lock:
-        _shutdown = True
+    _shutdown = True
     items = list(_threads_queues.items())
     for t, q in items:
         q.put(None)
     for t, q in items:
         t.join()
 
-# Register for `_python_exit()` to be called just before joining all
-# non-daemon threads. This is used instead of `atexit.register()` for
-# compatibility with subinterpreters, which no longer support daemon threads.
-# See bpo-39812 for context.
-threading._register_atexit(_python_exit)
-
-# At fork, reinitialize the `_global_shutdown_lock` lock in the child process
-if hasattr(os, 'register_at_fork'):
-    os.register_at_fork(before=_global_shutdown_lock.acquire,
-                        after_in_child=_global_shutdown_lock._at_fork_reinit,
-                        after_in_parent=_global_shutdown_lock.release)
-
+atexit.register(_python_exit)
 
 class _WorkItem(object):
     def __init__(self, future, fn, args, kwargs):
@@ -56,26 +53,12 @@ class _WorkItem(object):
 
         try:
             result = self.fn(*self.args, **self.kwargs)
-        except BaseException as exc:
-            self.future.set_exception(exc)
-            # Break a reference cycle with the exception 'exc'
-            self = None
+        except BaseException as e:
+            self.future.set_exception(e)
         else:
             self.future.set_result(result)
 
-    __class_getitem__ = classmethod(types.GenericAlias)
-
-
-def _worker(executor_reference, work_queue, initializer, initargs):
-    if initializer is not None:
-        try:
-            initializer(*initargs)
-        except BaseException:
-            _base.LOGGER.critical('Exception in initializer:', exc_info=True)
-            executor = executor_reference()
-            if executor is not None:
-                executor._initializer_failed()
-            return
+def _worker(executor_reference, work_queue):
     try:
         while True:
             work_item = work_queue.get(block=True)
@@ -83,24 +66,13 @@ def _worker(executor_reference, work_queue, initializer, initargs):
                 work_item.run()
                 # Delete references to object. See issue16284
                 del work_item
-
-                # attempt to increment idle count
-                executor = executor_reference()
-                if executor is not None:
-                    executor._idle_semaphore.release()
-                del executor
                 continue
-
             executor = executor_reference()
             # Exit if:
             #   - The interpreter is shutting down OR
             #   - The executor that owns the worker has been collected OR
             #   - The executor that owns the worker has been shutdown.
             if _shutdown or executor is None or executor._shutdown:
-                # Flag the executor as shutting down as early as possible if it
-                # is not gc-ed yet.
-                if executor is not None:
-                    executor._shutdown = True
                 # Notice other workers
                 work_queue.put(None)
                 return
@@ -108,66 +80,33 @@ def _worker(executor_reference, work_queue, initializer, initargs):
     except BaseException:
         _base.LOGGER.critical('Exception in worker', exc_info=True)
 
-
-class BrokenThreadPool(_base.BrokenExecutor):
-    """
-    Raised when a worker thread in a ThreadPoolExecutor failed initializing.
-    """
-
-
 class ThreadPoolExecutor(_base.Executor):
-
-    # Used to assign unique thread names when thread_name_prefix is not supplied.
-    _counter = itertools.count().__next__
-
-    def __init__(self, max_workers=None, thread_name_prefix='',
-                 initializer=None, initargs=()):
+    def __init__(self, max_workers=None, thread_name_prefix=''):
         """Initializes a new ThreadPoolExecutor instance.
 
         Args:
             max_workers: The maximum number of threads that can be used to
                 execute the given calls.
             thread_name_prefix: An optional name prefix to give our threads.
-            initializer: A callable used to initialize worker threads.
-            initargs: A tuple of arguments to pass to the initializer.
         """
         if max_workers is None:
-            # ThreadPoolExecutor is often used to:
-            # * CPU bound task which releases GIL
-            # * I/O bound task (which releases GIL, of course)
-            #
-            # We use cpu_count + 4 for both types of tasks.
-            # But we limit it to 32 to avoid consuming surprisingly large resource
-            # on many core machine.
-            max_workers = min(32, (os.cpu_count() or 1) + 4)
+            # Use this number because ThreadPoolExecutor is often
+            # used to overlap I/O instead of CPU work.
+            max_workers = (os.cpu_count() or 1) * 5
         if max_workers <= 0:
             raise ValueError("max_workers must be greater than 0")
 
-        if initializer is not None and not callable(initializer):
-            raise TypeError("initializer must be a callable")
-
         self._max_workers = max_workers
-        self._work_queue = queue.SimpleQueue()
-        self._idle_semaphore = threading.Semaphore(0)
+        self._work_queue = queue.Queue()
         self._threads = set()
-        self._broken = False
         self._shutdown = False
         self._shutdown_lock = threading.Lock()
-        self._thread_name_prefix = (thread_name_prefix or
-                                    ("ThreadPoolExecutor-%d" % self._counter()))
-        self._initializer = initializer
-        self._initargs = initargs
-
-    def submit(self, fn, /, *args, **kwargs):
-        with self._shutdown_lock, _global_shutdown_lock:
-            if self._broken:
-                raise BrokenThreadPool(self._broken)
+        self._thread_name_prefix = thread_name_prefix
 
+    def submit(self, fn, *args, **kwargs):
+        with self._shutdown_lock:
             if self._shutdown:
                 raise RuntimeError('cannot schedule new futures after shutdown')
-            if _shutdown:
-                raise RuntimeError('cannot schedule new futures after '
-                                   'interpreter shutdown')
 
             f = _base.Future()
             w = _WorkItem(f, fn, args, kwargs)
@@ -178,57 +117,27 @@ class ThreadPoolExecutor(_base.Executor):
     submit.__doc__ = _base.Executor.submit.__doc__
 
     def _adjust_thread_count(self):
-        # if idle threads are available, don't spin new threads
-        if self._idle_semaphore.acquire(timeout=0):
-            return
-
         # When the executor gets lost, the weakref callback will wake up
         # the worker threads.
         def weakref_cb(_, q=self._work_queue):
             q.put(None)
-
+        # TODO(bquinlan): Should avoid creating new threads if there are more
+        # idle threads than items in the work queue.
         num_threads = len(self._threads)
         if num_threads < self._max_workers:
             thread_name = '%s_%d' % (self._thread_name_prefix or self,
                                      num_threads)
             t = threading.Thread(name=thread_name, target=_worker,
                                  args=(weakref.ref(self, weakref_cb),
-                                       self._work_queue,
-                                       self._initializer,
-                                       self._initargs))
+                                       self._work_queue))
+            t.daemon = True
             t.start()
             self._threads.add(t)
             _threads_queues[t] = self._work_queue
 
-    def _initializer_failed(self):
-        with self._shutdown_lock:
-            self._broken = ('A thread initializer failed, the thread pool '
-                            'is not usable anymore')
-            # Drain work queue and mark pending futures failed
-            while True:
-                try:
-                    work_item = self._work_queue.get_nowait()
-                except queue.Empty:
-                    break
-                if work_item is not None:
-                    work_item.future.set_exception(BrokenThreadPool(self._broken))
-
-    def shutdown(self, wait=True, *, cancel_futures=False):
+    def shutdown(self, wait=True):
         with self._shutdown_lock:
             self._shutdown = True
-            if cancel_futures:
-                # Drain all work items from the queue, and then cancel their
-                # associated futures.
-                while True:
-                    try:
-                        work_item = self._work_queue.get_nowait()
-                    except queue.Empty:
-                        break
-                    if work_item is not None:
-                        work_item.future.cancel()
-
-            # Send a wake-up to prevent threads calling
-            # _work_queue.get(block=True) from permanently blocking.
             self._work_queue.put(None)
         if wait:
             for t in self._threads:

Lib/contextvars.py

@@ -1,4 +0,0 @@
-from _contextvars import Context, ContextVar, Token, copy_context
-
-
-__all__ = ('Context', 'ContextVar', 'Token', 'copy_context')

Lib/copy.py

@@ -75,20 +75,24 @@ def copy(x):
     if copier:
         return copier(x)
 
-    if issubclass(cls, type):
+    try:
+        issc = issubclass(cls, type)
+    except TypeError: # cls is not a class
+        issc = False
+    if issc:
         # treat it as a regular class:
         return _copy_immutable(x)
 
     copier = getattr(cls, "__copy__", None)
-    if copier is not None:
+    if copier:
         return copier(x)
 
     reductor = dispatch_table.get(cls)
-    if reductor is not None:
+    if reductor:
         rv = reductor(x)
     else:
         reductor = getattr(x, "__reduce_ex__", None)
-        if reductor is not None:
+        if reductor:
             rv = reductor(4)
         else:
             reductor = getattr(x, "__reduce__", None)
@@ -107,7 +111,7 @@ _copy_dispatch = d = {}
 def _copy_immutable(x):
     return x
 for t in (type(None), int, float, bool, complex, str, tuple,
-          bytes, frozenset, type, range, slice, property,
+          bytes, frozenset, type, range, slice,
           types.BuiltinFunctionType, type(Ellipsis), type(NotImplemented),
           types.FunctionType, weakref.ref):
     d[t] = _copy_immutable
@@ -142,14 +146,18 @@ def deepcopy(x, memo=None, _nil=[]):
     cls = type(x)
 
     copier = _deepcopy_dispatch.get(cls)
-    if copier is not None:
+    if copier:
         y = copier(x, memo)
     else:
-        if issubclass(cls, type):
+        try:
+            issc = issubclass(cls, type)
+        except TypeError: # cls is not a class (old Boost; see SF #502085)
+            issc = 0
+        if issc:
             y = _deepcopy_atomic(x, memo)
         else:
             copier = getattr(x, "__deepcopy__", None)
-            if copier is not None:
+            if copier:
                 y = copier(memo)
             else:
                 reductor = dispatch_table.get(cls)
@@ -157,7 +165,7 @@ def deepcopy(x, memo=None, _nil=[]):
                     rv = reductor(x)
                 else:
                     reductor = getattr(x, "__reduce_ex__", None)
-                    if reductor is not None:
+                    if reductor:
                         rv = reductor(4)
                     else:
                         reductor = getattr(x, "__reduce__", None)
@@ -190,12 +198,14 @@ d[bool] = _deepcopy_atomic
 d[complex] = _deepcopy_atomic
 d[bytes] = _deepcopy_atomic
 d[str] = _deepcopy_atomic
-d[types.CodeType] = _deepcopy_atomic
+try:
+    d[types.CodeType] = _deepcopy_atomic
+except AttributeError:
+    pass
 d[type] = _deepcopy_atomic
 d[types.BuiltinFunctionType] = _deepcopy_atomic
 d[types.FunctionType] = _deepcopy_atomic
 d[weakref.ref] = _deepcopy_atomic
-d[property] = _deepcopy_atomic
 
 def _deepcopy_list(x, memo, deepcopy=deepcopy):
     y = []

Lib/copyreg.py

@@ -53,8 +53,7 @@ _HEAPTYPE = 1<<9
 
 def _reduce_ex(self, proto):
     assert proto < 2
-    cls = self.__class__
-    for base in cls.__mro__:
+    for base in self.__class__.__mro__:
         if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE:
             break
     else:
@@ -62,18 +61,16 @@ def _reduce_ex(self, proto):
     if base is object:
         state = None
     else:
-        if base is cls:
-            raise TypeError(f"cannot pickle {cls.__name__!r} object")
+        if base is self.__class__:
+            raise TypeError("can't pickle %s objects" % base.__name__)
         state = base(self)
-    args = (cls, base, state)
+    args = (self.__class__, base, state)
     try:
         getstate = self.__getstate__
     except AttributeError:
         if getattr(self, "__slots__", None):
-            raise TypeError(f"cannot pickle {cls.__name__!r} object: "
-                            f"a class that defines __slots__ without "
-                            f"defining __getstate__ cannot be pickled "
-                            f"with protocol {proto}") from None
+            raise TypeError("a class that defines __slots__ without "
+                            "defining __getstate__ cannot be pickled")
         try:
             dict = self.__dict__
         except AttributeError:

Lib/csv.py

@@ -4,7 +4,7 @@ csv.py - read/write/investigate CSV files
 """
 
 import re
-from _csv import Error, writer, reader, \
+from _csv import Error, reader, \
                  QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \
                  __doc__
 

Lib/datetime.py

@@ -4,14 +4,9 @@ See http://www.iana.org/time-zones/repository/tz-link.html for
 time zone and DST data sources.
 """
 
-__all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo",
-           "MINYEAR", "MAXYEAR", "UTC")
-
-
 import time as _time
 import math as _math
 import sys
-from operator import index as _index
 
 def _cmp(x, y):
     return 0 if x == y else 1 if x > y else -1
@@ -381,10 +376,27 @@ def _check_utc_offset(name, offset):
                          "-timedelta(hours=24) and timedelta(hours=24)" %
                          (name, offset))
 
+def _check_int_field(value):
+    if isinstance(value, int):
+        return value
+    if not isinstance(value, float):
+        try:
+            value = value.__int__()
+        except AttributeError:
+            pass
+        else:
+            if isinstance(value, int):
+                return value
+            raise TypeError('__int__ returned non-int (type %s)' %
+                            type(value).__name__)
+        raise TypeError('an integer is required (got type %s)' %
+                        type(value).__name__)
+    raise TypeError('integer argument expected, got float')
+
 def _check_date_fields(year, month, day):
-    year = _index(year)
-    month = _index(month)
-    day = _index(day)
+    year = _check_int_field(year)
+    month = _check_int_field(month)
+    day = _check_int_field(day)
     if not MINYEAR <= year <= MAXYEAR:
         raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year)
     if not 1 <= month <= 12:
@@ -395,10 +407,10 @@ def _check_date_fields(year, month, day):
     return year, month, day
 
 def _check_time_fields(hour, minute, second, microsecond, fold):
-    hour = _index(hour)
-    minute = _index(minute)
-    second = _index(second)
-    microsecond = _index(microsecond)
+    hour = _check_int_field(hour)
+    minute = _check_int_field(minute)
+    second = _check_int_field(second)
+    microsecond = _check_int_field(microsecond)
     if not 0 <= hour <= 23:
         raise ValueError('hour must be in 0..23', hour)
     if not 0 <= minute <= 59:
@@ -706,31 +718,31 @@ class timedelta:
         if isinstance(other, timedelta):
             return self._cmp(other) == 0
         else:
-            return NotImplemented
+            return False
 
     def __le__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) <= 0
         else:
-            return NotImplemented
+            _cmperror(self, other)
 
     def __lt__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) < 0
         else:
-            return NotImplemented
+            _cmperror(self, other)
 
     def __ge__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) >= 0
         else:
-            return NotImplemented
+            _cmperror(self, other)
 
     def __gt__(self, other):
         if isinstance(other, timedelta):
             return self._cmp(other) > 0
         else:
-            return NotImplemented
+            _cmperror(self, other)
 
     def _cmp(self, other):
         assert isinstance(other, timedelta)
@@ -857,40 +869,6 @@ class date:
         except Exception:
             raise ValueError(f'Invalid isoformat string: {date_string!r}')
 
-    @classmethod
-    def fromisocalendar(cls, year, week, day):
-        """Construct a date from the ISO year, week number and weekday.
-
-        This is the inverse of the date.isocalendar() function"""
-        # Year is bounded this way because 9999-12-31 is (9999, 52, 5)
-        if not MINYEAR <= year <= MAXYEAR:
-            raise ValueError(f"Year is out of range: {year}")
-
-        if not 0 < week < 53:
-            out_of_range = True
-
-            if week == 53:
-                # ISO years have 53 weeks in them on years starting with a
-                # Thursday and leap years starting on a Wednesday
-                first_weekday = _ymd2ord(year, 1, 1) % 7
-                if (first_weekday == 4 or (first_weekday == 3 and
-                                           _is_leap(year))):
-                    out_of_range = False
-
-            if out_of_range:
-                raise ValueError(f"Invalid week: {week}")
-
-        if not 0 < day < 8:
-            raise ValueError(f"Invalid weekday: {day} (range is [1, 7])")
-
-        # Now compute the offset from (Y, 1, 1) in days:
-        day_offset = (week - 1) * 7 + (day - 1)
-
-        # Calculate the ordinal day for monday, week 1
-        day_1 = _isoweek1monday(year)
-        ord_day = day_1 + day_offset
-
-        return cls(*_ord2ymd(ord_day))
 
     # Conversions to string
 
@@ -1036,7 +1014,7 @@ class date:
         if isinstance(other, timedelta):
             o = self.toordinal() + other.days
             if 0 < o <= _MAXORDINAL:
-                return type(self).fromordinal(o)
+                return date.fromordinal(o)
             raise OverflowError("result out of range")
         return NotImplemented
 
@@ -1064,7 +1042,7 @@ class date:
         return self.toordinal() % 7 or 7
 
     def isocalendar(self):
-        """Return a named tuple containing ISO year, week number, and weekday.
+        """Return a 3-tuple containing ISO year, week number, and weekday.
 
         The first ISO week of the year is the (Mon-Sun) week
         containing the year's first Thursday; everything else derives
@@ -1089,7 +1067,7 @@ class date:
             if today >= _isoweek1monday(year+1):
                 year += 1
                 week = 0
-        return _IsoCalendarDate(year, week+1, day+1)
+        return year, week+1, day+1
 
     # Pickle support.
 
@@ -1179,36 +1157,6 @@ class tzinfo:
         else:
             return (self.__class__, args, state)
 
-
-class IsoCalendarDate(tuple):
-
-    def __new__(cls, year, week, weekday, /):
-        return super().__new__(cls, (year, week, weekday))
-
-    @property
-    def year(self):
-        return self[0]
-
-    @property
-    def week(self):
-        return self[1]
-
-    @property
-    def weekday(self):
-        return self[2]
-
-    def __reduce__(self):
-        # This code is intended to pickle the object without making the
-        # class public. See https://bugs.python.org/msg352381
-        return (tuple, (tuple(self),))
-
-    def __repr__(self):
-        return (f'{self.__class__.__name__}'
-                f'(year={self[0]}, week={self[1]}, weekday={self[2]})')
-
-
-_IsoCalendarDate = IsoCalendarDate
-del IsoCalendarDate
 _tzinfo_class = tzinfo
 
 class time:
@@ -1313,31 +1261,31 @@ class time:
         if isinstance(other, time):
             return self._cmp(other, allow_mixed=True) == 0
         else:
-            return NotImplemented
+            return False
 
     def __le__(self, other):
         if isinstance(other, time):
             return self._cmp(other) <= 0
         else:
-            return NotImplemented
+            _cmperror(self, other)
 
     def __lt__(self, other):
         if isinstance(other, time):
             return self._cmp(other) < 0
         else:
-            return NotImplemented
+            _cmperror(self, other)
 
     def __ge__(self, other):
         if isinstance(other, time):
             return self._cmp(other) >= 0
         else:
-            return NotImplemented
+            _cmperror(self, other)
 
     def __gt__(self, other):
         if isinstance(other, time):
             return self._cmp(other) > 0
         else:
-            return NotImplemented
+            _cmperror(self, other)
 
     def _cmp(self, other, allow_mixed=False):
         assert isinstance(other, time)
@@ -1421,8 +1369,7 @@ class time:
         part is omitted if self.microsecond == 0.
 
         The optional argument timespec specifies the number of additional
-        terms of the time to include. Valid options are 'auto', 'hours',
-        'minutes', 'seconds', 'milliseconds' and 'microseconds'.
+        terms of the time to include.
         """
         s = _format_time(self._hour, self._minute, self._second,
                           self._microsecond, timespec)
@@ -1548,7 +1495,7 @@ class time:
         self._tzinfo = tzinfo
 
     def __reduce_ex__(self, protocol):
-        return (self.__class__, self._getstate(protocol))
+        return (time, self._getstate(protocol))
 
     def __reduce__(self):
         return self.__reduce_ex__(2)
@@ -1559,7 +1506,6 @@ time.min = time(0, 0, 0)
 time.max = time(23, 59, 59, 999999)
 time.resolution = timedelta(microseconds=1)
 
-
 class datetime(date):
     """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])
 
@@ -1652,7 +1598,7 @@ class datetime(date):
         y, m, d, hh, mm, ss, weekday, jday, dst = converter(t)
         ss = min(ss, 59)    # clamp out leap seconds if the platform has them
         result = cls(y, m, d, hh, mm, ss, us, tz)
-        if tz is None and not utc:
+        if tz is None:
             # As of version 2015f max fold in IANA database is
             # 23 hours at 1969-09-30 13:00:00 in Kwajalein.
             # Let's probe 24 hours in the past to detect a transition:
@@ -1673,7 +1619,7 @@ class datetime(date):
                 probe2 = cls(y, m, d, hh, mm, ss, us, tz)
                 if probe2 == result:
                     result._fold = 1
-        elif tz is not None:
+        else:
             result = tz.fromutc(result)
         return result
 
@@ -1852,10 +1798,17 @@ class datetime(date):
             ts = (self - _EPOCH) // timedelta(seconds=1)
         localtm = _time.localtime(ts)
         local = datetime(*localtm[:6])
-        # Extract TZ data
-        gmtoff = localtm.tm_gmtoff
-        zone = localtm.tm_zone
-        return timezone(timedelta(seconds=gmtoff), zone)
+        try:
+            # Extract TZ data if available
+            gmtoff = localtm.tm_gmtoff
+            zone = localtm.tm_zone
+        except AttributeError:
+            delta = local - datetime(*_time.gmtime(ts)[:6])
+            zone = _time.strftime('%Z', localtm)
+            tz = timezone(delta, zone)
+        else:
+            tz = timezone(timedelta(seconds=gmtoff), zone)
+        return tz
 
     def astimezone(self, tz=None):
         if tz is None:
@@ -1907,8 +1860,7 @@ class datetime(date):
         time, default 'T'.
 
         The optional argument timespec specifies the number of additional
-        terms of the time to include. Valid options are 'auto', 'hours',
-        'minutes', 'seconds', 'milliseconds' and 'microseconds'.
+        terms of the time to include.
         """
         s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) +
              _format_time(self._hour, self._minute, self._second,
@@ -2079,10 +2031,10 @@ class datetime(date):
         hour, rem = divmod(delta.seconds, 3600)
         minute, second = divmod(rem, 60)
         if 0 < delta.days <= _MAXORDINAL:
-            return type(self).combine(date.fromordinal(delta.days),
-                                      time(hour, minute, second,
-                                           delta.microseconds,
-                                           tzinfo=self._tzinfo))
+            return datetime.combine(date.fromordinal(delta.days),
+                                    time(hour, minute, second,
+                                         delta.microseconds,
+                                         tzinfo=self._tzinfo))
         raise OverflowError("result out of range")
 
     __radd__ = __add__
@@ -2181,7 +2133,6 @@ def _isoweek1monday(year):
         week1monday += 7
     return week1monday
 
-
 class timezone(tzinfo):
     __slots__ = '_offset', '_name'
 
@@ -2216,9 +2167,9 @@ class timezone(tzinfo):
         return (self._offset, self._name)
 
     def __eq__(self, other):
-        if isinstance(other, timezone):
-            return self._offset == other._offset
-        return NotImplemented
+        if type(other) != timezone:
+            return False
+        return self._offset == other._offset
 
     def __hash__(self):
         return hash(self._offset)
@@ -2275,7 +2226,7 @@ class timezone(tzinfo):
         raise TypeError("fromutc() argument must be a datetime instance"
                         " or None")
 
-    _maxoffset = timedelta(hours=24, microseconds=-1)
+    _maxoffset = timedelta(hours=23, minutes=59)
     _minoffset = -_maxoffset
 
     @staticmethod
@@ -2298,12 +2249,9 @@ class timezone(tzinfo):
             return f'UTC{sign}{hours:02d}:{minutes:02d}:{seconds:02d}'
         return f'UTC{sign}{hours:02d}:{minutes:02d}'
 
-UTC = timezone.utc = timezone._create(timedelta(0))
-# bpo-37642: These attributes are rounded to the nearest minute for backwards
-# compatibility, even though the constructor will accept a wider range of
-# values. This may change in the future.
-timezone.min = timezone._create(-timedelta(hours=23, minutes=59))
-timezone.max = timezone._create(timedelta(hours=23, minutes=59))
+timezone.utc = timezone._create(timedelta(0))
+timezone.min = timezone._create(timezone._minoffset)
+timezone.max = timezone._create(timezone._maxoffset)
 _EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
 
 # Some time zone algebra.  For a datetime x, let
@@ -2327,7 +2275,7 @@ _EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
 #    This is again a requirement for a sane tzinfo class.
 #
 # 4. (x+k).s = x.s
-#    This follows from #2, and that datetime.timetz+timedelta preserves tzinfo.
+#    This follows from #2, and that datimetimetz+timedelta preserves tzinfo.
 #
 # 5. (x+k).n = x.n + k
 #    Again follows from how arithmetic is defined.
@@ -2510,13 +2458,13 @@ else:
     # Clean up unused names
     del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH, _DI100Y, _DI400Y,
          _DI4Y, _EPOCH, _MAXORDINAL, _MONTHNAMES, _build_struct_time,
-         _check_date_fields, _check_time_fields,
+         _check_date_fields, _check_int_field, _check_time_fields,
          _check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror,
          _date_class, _days_before_month, _days_before_year, _days_in_month,
-         _format_time, _format_offset, _index, _is_leap, _isoweek1monday, _math,
+         _format_time, _format_offset, _is_leap, _isoweek1monday, _math,
          _ord2ymd, _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord,
          _divide_and_round, _parse_isoformat_date, _parse_isoformat_time,
-         _parse_hh_mm_ss_ff, _IsoCalendarDate)
+         _parse_hh_mm_ss_ff)
     # XXX Since import * above excludes names that start with _,
     # docstring does not get overwritten. In the future, it may be
     # appropriate to maintain a single module level docstring and

Lib/dbm/__init__.py

@@ -1,189 +0,0 @@
-"""Generic interface to all dbm clones.
-
-Use
-
-        import dbm
-        d = dbm.open(file, 'w', 0o666)
-
-The returned object is a dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the
-type of database being opened (determined by the whichdb function) in the case
-of an existing dbm. If the dbm does not exist and the create or new flag ('c'
-or 'n') was specified, the dbm type will be determined by the availability of
-the modules (tested in the above order).
-
-It has the following interface (key and data are strings):
-
-        d[key] = data   # store data at key (may override data at
-                        # existing key)
-        data = d[key]   # retrieve data at key (raise KeyError if no
-                        # such key)
-        del d[key]      # delete data stored at key (raises KeyError
-                        # if no such key)
-        flag = key in d # true if the key exists
-        list = d.keys() # return a list of all existing keys (slow!)
-
-Future versions may change the order in which implementations are
-tested for existence, and add interfaces to other dbm-like
-implementations.
-"""
-
-__all__ = ['open', 'whichdb', 'error']
-
-import io
-import os
-import struct
-import sys
-
-
-class error(Exception):
-    pass
-
-_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.dumb']
-_defaultmod = None
-_modules = {}
-
-error = (error, OSError)
-
-try:
-    from dbm import ndbm
-except ImportError:
-    ndbm = None
-
-
-def open(file, flag='r', mode=0o666):
-    """Open or create database at path given by *file*.
-
-    Optional argument *flag* can be 'r' (default) for read-only access, 'w'
-    for read-write access of an existing database, 'c' for read-write access
-    to a new or existing database, and 'n' for read-write access to a new
-    database.
-
-    Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it
-    only if it doesn't exist; and 'n' always creates a new database.
-    """
-    global _defaultmod
-    if _defaultmod is None:
-        for name in _names:
-            try:
-                mod = __import__(name, fromlist=['open'])
-            except ImportError:
-                continue
-            if not _defaultmod:
-                _defaultmod = mod
-            _modules[name] = mod
-        if not _defaultmod:
-            raise ImportError("no dbm clone found; tried %s" % _names)
-
-    # guess the type of an existing database, if not creating a new one
-    result = whichdb(file) if 'n' not in flag else None
-    if result is None:
-        # db doesn't exist or 'n' flag was specified to create a new db
-        if 'c' in flag or 'n' in flag:
-            # file doesn't exist and the new flag was used so use default type
-            mod = _defaultmod
-        else:
-            raise error[0]("db file doesn't exist; "
-                           "use 'c' or 'n' flag to create a new db")
-    elif result == "":
-        # db type cannot be determined
-        raise error[0]("db type could not be determined")
-    elif result not in _modules:
-        raise error[0]("db type is {0}, but the module is not "
-                       "available".format(result))
-    else:
-        mod = _modules[result]
-    return mod.open(file, flag, mode)
-
-
-def whichdb(filename):
-    """Guess which db package to use to open a db file.
-
-    Return values:
-
-    - None if the database file can't be read;
-    - empty string if the file can be read but can't be recognized
-    - the name of the dbm submodule (e.g. "ndbm" or "gnu") if recognized.
-
-    Importing the given module may still fail, and opening the
-    database using that module may still fail.
-    """
-
-    # Check for ndbm first -- this has a .pag and a .dir file
-    try:
-        f = io.open(filename + ".pag", "rb")
-        f.close()
-        f = io.open(filename + ".dir", "rb")
-        f.close()
-        return "dbm.ndbm"
-    except OSError:
-        # some dbm emulations based on Berkeley DB generate a .db file
-        # some do not, but they should be caught by the bsd checks
-        try:
-            f = io.open(filename + ".db", "rb")
-            f.close()
-            # guarantee we can actually open the file using dbm
-            # kind of overkill, but since we are dealing with emulations
-            # it seems like a prudent step
-            if ndbm is not None:
-                d = ndbm.open(filename)
-                d.close()
-                return "dbm.ndbm"
-        except OSError:
-            pass
-
-    # Check for dumbdbm next -- this has a .dir and a .dat file
-    try:
-        # First check for presence of files
-        os.stat(filename + ".dat")
-        size = os.stat(filename + ".dir").st_size
-        # dumbdbm files with no keys are empty
-        if size == 0:
-            return "dbm.dumb"
-        f = io.open(filename + ".dir", "rb")
-        try:
-            if f.read(1) in (b"'", b'"'):
-                return "dbm.dumb"
-        finally:
-            f.close()
-    except OSError:
-        pass
-
-    # See if the file exists, return None if not
-    try:
-        f = io.open(filename, "rb")
-    except OSError:
-        return None
-
-    with f:
-        # Read the start of the file -- the magic number
-        s16 = f.read(16)
-    s = s16[0:4]
-
-    # Return "" if not at least 4 bytes
-    if len(s) != 4:
-        return ""
-
-    # Convert to 4-byte int in native byte order -- return "" if impossible
-    try:
-        (magic,) = struct.unpack("=l", s)
-    except struct.error:
-        return ""
-
-    # Check for GNU dbm
-    if magic in (0x13579ace, 0x13579acd, 0x13579acf):
-        return "dbm.gnu"
-
-    # Later versions of Berkeley db hash file have a 12-byte pad in
-    # front of the file type
-    try:
-        (magic,) = struct.unpack("=l", s16[-4:])
-    except struct.error:
-        return ""
-
-    # Unknown
-    return ""
-
-
-if __name__ == "__main__":
-    for filename in sys.argv[1:]:
-        print(whichdb(filename) or "UNKNOWN", filename)

Lib/dbm/dumb.py

@@ -1,316 +0,0 @@
-"""A dumb and slow but simple dbm clone.
-
-For database spam, spam.dir contains the index (a text file),
-spam.bak *may* contain a backup of the index (also a text file),
-while spam.dat contains the data (a binary file).
-
-XXX TO DO:
-
-- seems to contain a bug when updating...
-
-- reclaim free space (currently, space once occupied by deleted or expanded
-items is never reused)
-
-- support concurrent access (currently, if two processes take turns making
-updates, they can mess up the index)
-
-- support efficient access to large databases (currently, the whole index
-is read when the database is opened, and some updates rewrite the whole index)
-
-- support opening for read-only (flag = 'm')
-
-"""
-
-import ast as _ast
-import io as _io
-import os as _os
-import collections.abc
-
-__all__ = ["error", "open"]
-
-_BLOCKSIZE = 512
-
-error = OSError
-
-class _Database(collections.abc.MutableMapping):
-
-    # The on-disk directory and data files can remain in mutually
-    # inconsistent states for an arbitrarily long time (see comments
-    # at the end of __setitem__).  This is only repaired when _commit()
-    # gets called.  One place _commit() gets called is from __del__(),
-    # and if that occurs at program shutdown time, module globals may
-    # already have gotten rebound to None.  Since it's crucial that
-    # _commit() finish successfully, we can't ignore shutdown races
-    # here, and _commit() must not reference any globals.
-    _os = _os       # for _commit()
-    _io = _io       # for _commit()
-
-    def __init__(self, filebasename, mode, flag='c'):
-        self._mode = mode
-        self._readonly = (flag == 'r')
-
-        # The directory file is a text file.  Each line looks like
-        #    "%r, (%d, %d)\n" % (key, pos, siz)
-        # where key is the string key, pos is the offset into the dat
-        # file of the associated value's first byte, and siz is the number
-        # of bytes in the associated value.
-        self._dirfile = filebasename + '.dir'
-
-        # The data file is a binary file pointed into by the directory
-        # file, and holds the values associated with keys.  Each value
-        # begins at a _BLOCKSIZE-aligned byte offset, and is a raw
-        # binary 8-bit string value.
-        self._datfile = filebasename + '.dat'
-        self._bakfile = filebasename + '.bak'
-
-        # The index is an in-memory dict, mirroring the directory file.
-        self._index = None  # maps keys to (pos, siz) pairs
-
-        # Handle the creation
-        self._create(flag)
-        self._update(flag)
-
-    def _create(self, flag):
-        if flag == 'n':
-            for filename in (self._datfile, self._bakfile, self._dirfile):
-                try:
-                    _os.remove(filename)
-                except OSError:
-                    pass
-        # Mod by Jack: create data file if needed
-        try:
-            f = _io.open(self._datfile, 'r', encoding="Latin-1")
-        except OSError:
-            if flag not in ('c', 'n'):
-                raise
-            with _io.open(self._datfile, 'w', encoding="Latin-1") as f:
-                self._chmod(self._datfile)
-        else:
-            f.close()
-
-    # Read directory file into the in-memory index dict.
-    def _update(self, flag):
-        self._modified = False
-        self._index = {}
-        try:
-            f = _io.open(self._dirfile, 'r', encoding="Latin-1")
-        except OSError:
-            if flag not in ('c', 'n'):
-                raise
-            self._modified = True
-        else:
-            with f:
-                for line in f:
-                    line = line.rstrip()
-                    key, pos_and_siz_pair = _ast.literal_eval(line)
-                    key = key.encode('Latin-1')
-                    self._index[key] = pos_and_siz_pair
-
-    # Write the index dict to the directory file.  The original directory
-    # file (if any) is renamed with a .bak extension first.  If a .bak
-    # file currently exists, it's deleted.
-    def _commit(self):
-        # CAUTION:  It's vital that _commit() succeed, and _commit() can
-        # be called from __del__().  Therefore we must never reference a
-        # global in this routine.
-        if self._index is None or not self._modified:
-            return  # nothing to do
-
-        try:
-            self._os.unlink(self._bakfile)
-        except OSError:
-            pass
-
-        try:
-            self._os.rename(self._dirfile, self._bakfile)
-        except OSError:
-            pass
-
-        with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f:
-            self._chmod(self._dirfile)
-            for key, pos_and_siz_pair in self._index.items():
-                # Use Latin-1 since it has no qualms with any value in any
-                # position; UTF-8, though, does care sometimes.
-                entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair)
-                f.write(entry)
-
-    sync = _commit
-
-    def _verify_open(self):
-        if self._index is None:
-            raise error('DBM object has already been closed')
-
-    def __getitem__(self, key):
-        if isinstance(key, str):
-            key = key.encode('utf-8')
-        self._verify_open()
-        pos, siz = self._index[key]     # may raise KeyError
-        with _io.open(self._datfile, 'rb') as f:
-            f.seek(pos)
-            dat = f.read(siz)
-        return dat
-
-    # Append val to the data file, starting at a _BLOCKSIZE-aligned
-    # offset.  The data file is first padded with NUL bytes (if needed)
-    # to get to an aligned offset.  Return pair
-    #     (starting offset of val, len(val))
-    def _addval(self, val):
-        with _io.open(self._datfile, 'rb+') as f:
-            f.seek(0, 2)
-            pos = int(f.tell())
-            npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE
-            f.write(b'\0'*(npos-pos))
-            pos = npos
-            f.write(val)
-        return (pos, len(val))
-
-    # Write val to the data file, starting at offset pos.  The caller
-    # is responsible for ensuring that there's enough room starting at
-    # pos to hold val, without overwriting some other value.  Return
-    # pair (pos, len(val)).
-    def _setval(self, pos, val):
-        with _io.open(self._datfile, 'rb+') as f:
-            f.seek(pos)
-            f.write(val)
-        return (pos, len(val))
-
-    # key is a new key whose associated value starts in the data file
-    # at offset pos and with length siz.  Add an index record to
-    # the in-memory index dict, and append one to the directory file.
-    def _addkey(self, key, pos_and_siz_pair):
-        self._index[key] = pos_and_siz_pair
-        with _io.open(self._dirfile, 'a', encoding="Latin-1") as f:
-            self._chmod(self._dirfile)
-            f.write("%r, %r\n" % (key.decode("Latin-1"), pos_and_siz_pair))
-
-    def __setitem__(self, key, val):
-        if self._readonly:
-            raise error('The database is opened for reading only')
-        if isinstance(key, str):
-            key = key.encode('utf-8')
-        elif not isinstance(key, (bytes, bytearray)):
-            raise TypeError("keys must be bytes or strings")
-        if isinstance(val, str):
-            val = val.encode('utf-8')
-        elif not isinstance(val, (bytes, bytearray)):
-            raise TypeError("values must be bytes or strings")
-        self._verify_open()
-        self._modified = True
-        if key not in self._index:
-            self._addkey(key, self._addval(val))
-        else:
-            # See whether the new value is small enough to fit in the
-            # (padded) space currently occupied by the old value.
-            pos, siz = self._index[key]
-            oldblocks = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE
-            newblocks = (len(val) + _BLOCKSIZE - 1) // _BLOCKSIZE
-            if newblocks <= oldblocks:
-                self._index[key] = self._setval(pos, val)
-            else:
-                # The new value doesn't fit in the (padded) space used
-                # by the old value.  The blocks used by the old value are
-                # forever lost.
-                self._index[key] = self._addval(val)
-
-            # Note that _index may be out of synch with the directory
-            # file now:  _setval() and _addval() don't update the directory
-            # file.  This also means that the on-disk directory and data
-            # files are in a mutually inconsistent state, and they'll
-            # remain that way until _commit() is called.  Note that this
-            # is a disaster (for the database) if the program crashes
-            # (so that _commit() never gets called).
-
-    def __delitem__(self, key):
-        if self._readonly:
-            raise error('The database is opened for reading only')
-        if isinstance(key, str):
-            key = key.encode('utf-8')
-        self._verify_open()
-        self._modified = True
-        # The blocks used by the associated value are lost.
-        del self._index[key]
-        # XXX It's unclear why we do a _commit() here (the code always
-        # XXX has, so I'm not changing it).  __setitem__ doesn't try to
-        # XXX keep the directory file in synch.  Why should we?  Or
-        # XXX why shouldn't __setitem__?
-        self._commit()
-
-    def keys(self):
-        try:
-            return list(self._index)
-        except TypeError:
-            raise error('DBM object has already been closed') from None
-
-    def items(self):
-        self._verify_open()
-        return [(key, self[key]) for key in self._index.keys()]
-
-    def __contains__(self, key):
-        if isinstance(key, str):
-            key = key.encode('utf-8')
-        try:
-            return key in self._index
-        except TypeError:
-            if self._index is None:
-                raise error('DBM object has already been closed') from None
-            else:
-                raise
-
-    def iterkeys(self):
-        try:
-            return iter(self._index)
-        except TypeError:
-            raise error('DBM object has already been closed') from None
-    __iter__ = iterkeys
-
-    def __len__(self):
-        try:
-            return len(self._index)
-        except TypeError:
-            raise error('DBM object has already been closed') from None
-
-    def close(self):
-        try:
-            self._commit()
-        finally:
-            self._index = self._datfile = self._dirfile = self._bakfile = None
-
-    __del__ = close
-
-    def _chmod(self, file):
-        self._os.chmod(file, self._mode)
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        self.close()
-
-
-def open(file, flag='c', mode=0o666):
-    """Open the database file, filename, and return corresponding object.
-
-    The flag argument, used to control how the database is opened in the
-    other DBM implementations, supports only the semantics of 'c' and 'n'
-    values.  Other values will default to the semantics of 'c' value:
-    the database will always opened for update and will be created if it
-    does not exist.
-
-    The optional mode argument is the UNIX mode of the file, used only when
-    the database has to be created.  It defaults to octal code 0o666 (and
-    will be modified by the prevailing umask).
-
-    """
-
-    # Modify mode depending on the umask
-    try:
-        um = _os.umask(0)
-        _os.umask(um)
-    except AttributeError:
-        pass
-    else:
-        # Turn off any bits that are set in the umask
-        mode = mode & (~um)
-    if flag not in ('r', 'w', 'c', 'n'):
-        raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n'")
-    return _Database(file, mode, flag=flag)

Lib/difflib.py

@@ -32,7 +32,6 @@ __all__ = ['get_close_matches', 'ndiff', 'restore', 'SequenceMatcher',
 
 from heapq import nlargest as _nlargest
 from collections import namedtuple as _namedtuple
-from types import GenericAlias
 
 Match = _namedtuple('Match', 'a b size')
 
@@ -686,8 +685,6 @@ class SequenceMatcher:
         # shorter sequence
         return _calculate_ratio(min(la, lb), la + lb)
 
-    __class_getitem__ = classmethod(GenericAlias)
-
 def get_close_matches(word, possibilities, n=3, cutoff=0.6):
     """Use SequenceMatcher to return list of the best "good enough" matches.
 
@@ -736,15 +733,20 @@ def get_close_matches(word, possibilities, n=3, cutoff=0.6):
     # Strip scores for the best n matches
     return [x for score, x in result]
 
+def _count_leading(line, ch):
+    """
+    Return number of `ch` characters at the start of `line`.
 
-def _keep_original_ws(s, tag_s):
-    """Replace whitespace with the original whitespace characters in `s`"""
-    return ''.join(
-        c if tag_c == " " and c.isspace() else tag_c
-        for c, tag_c in zip(s, tag_s)
-    )
+    Example:
 
+    >>> _count_leading('   abc', ' ')
+    3
+    """
 
+    i, n = 0, len(line)
+    while i < n and line[i] == ch:
+        i += 1
+    return i
 
 class Differ:
     r"""
@@ -1031,7 +1033,7 @@ class Differ:
 
     def _qformat(self, aline, bline, atags, btags):
         r"""
-        Format "?" output and deal with tabs.
+        Format "?" output and deal with leading tabs.
 
         Example:
 
@@ -1045,16 +1047,22 @@ class Differ:
         '+ \tabcdefGhijkl\n'
         '? \t ^ ^  ^\n'
         """
-        atags = _keep_original_ws(aline, atags).rstrip()
-        btags = _keep_original_ws(bline, btags).rstrip()
+
+        # Can hurt, but will probably help most of the time.
+        common = min(_count_leading(aline, "\t"),
+                     _count_leading(bline, "\t"))
+        common = min(common, _count_leading(atags[:common], " "))
+        common = min(common, _count_leading(btags[:common], " "))
+        atags = atags[common:].rstrip()
+        btags = btags[common:].rstrip()
 
         yield "- " + aline
         if atags:
-            yield f"? {atags}\n"
+            yield "? %s%s\n" % ("\t" * common, atags)
 
         yield "+ " + bline
         if btags:
-            yield f"? {btags}\n"
+            yield "? %s%s\n" % ("\t" * common, btags)
 
 # With respect to junk, an earlier version of ndiff simply refused to
 # *start* a match with a junk element.  The result was cases like this:
@@ -1077,7 +1085,7 @@ import re
 
 def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match):
     r"""
-    Return True for ignorable line: iff `line` is blank or contains a single '#'.
+    Return 1 for ignorable line: iff `line` is blank or contains a single '#'.
 
     Examples:
 
@@ -1093,7 +1101,7 @@ def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match):
 
 def IS_CHARACTER_JUNK(ch, ws=" \t"):
     r"""
-    Return True for ignorable character: iff `ch` is a space or tab.
+    Return 1 for ignorable character: iff `ch` is a space or tab.
 
     Examples:
 

Lib/dis.py

@@ -1,18 +0,0 @@
-from _dis import *
-
-
-# Disassembling a file by following cpython Lib/dis.py
-def _test():
-    """Simple test program to disassemble a file."""
-    import argparse
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('infile', type=argparse.FileType('rb'), nargs='?', default='-')
-    args = parser.parse_args()
-    with args.infile as infile:
-        source = infile.read()
-    code = compile(source, args.infile.name, "exec")
-    dis(code)
-
-if __name__ == "__main__":
-    _test()

Lib/distutils/command/install.py

@@ -78,11 +78,6 @@ if HAS_USER_SITE:
         'data'   : '$userbase',
         }
 
-# XXX RUSTPYTHON: replace python with rustpython in all these paths
-for group in INSTALL_SCHEMES.values():
-    for key in group.keys():
-        group[key] = group[key].replace("Python", "RustPython").replace("python", "rustpython")
-
 # The keys to an installation scheme; if any new types of files are to be
 # installed, be sure to add an entry to every installation scheme above,
 # and to SCHEME_KEYS here.

Lib/distutils/sysconfig.py

@@ -13,6 +13,7 @@ import _imp
 import os
 import re
 import sys
+import fnmatch
 
 from .errors import DistutilsPlatformError
 
@@ -23,47 +24,36 @@ BASE_PREFIX = os.path.normpath(sys.base_prefix)
 BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix)
 
 # Path to the base directory of the project. On Windows the binary may
-# live in project/PCbuild/win32 or project/PCbuild/amd64.
+# live in project/PCBuild/win32 or project/PCBuild/amd64.
 # set for cross builds
 if "_PYTHON_PROJECT_BASE" in os.environ:
     project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"])
 else:
-    if sys.executable:
-        project_base = os.path.dirname(os.path.abspath(sys.executable))
-    else:
-        # sys.executable can be empty if argv[0] has been changed and Python is
-        # unable to retrieve the real program name
-        project_base = os.getcwd()
-
+    project_base = os.path.dirname(os.path.abspath(sys.executable))
+if (os.name == 'nt' and
+    project_base.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))):
+    project_base = os.path.dirname(os.path.dirname(project_base))
 
 # python_build: (Boolean) if true, we're either building Python or
 # building an extension with an un-installed Python, so we use
 # different (hard-wired) directories.
+# Setup.local is available for Makefile builds including VPATH builds,
+# Setup.dist is available on Windows
 def _is_python_source_dir(d):
-    for fn in ("Setup", "Setup.local"):
+    for fn in ("Setup.dist", "Setup.local"):
         if os.path.isfile(os.path.join(d, "Modules", fn)):
             return True
     return False
-
 _sys_home = getattr(sys, '_home', None)
-
-if os.name == 'nt':
-    def _fix_pcbuild(d):
-        if d and os.path.normcase(d).startswith(
-                os.path.normcase(os.path.join(PREFIX, "PCbuild"))):
-            return PREFIX
-        return d
-    project_base = _fix_pcbuild(project_base)
-    _sys_home = _fix_pcbuild(_sys_home)
-
+if (_sys_home and os.name == 'nt' and
+    _sys_home.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))):
+    _sys_home = os.path.dirname(os.path.dirname(_sys_home))
 def _python_build():
     if _sys_home:
         return _is_python_source_dir(_sys_home)
     return _is_python_source_dir(project_base)
-
 python_build = _python_build()
 
-
 # Calculate the build qualifier flags if they are defined.  Adding the flags
 # to the include and lib directories only makes sense for an installation, not
 # an in-source build.
@@ -110,13 +100,11 @@ def get_python_inc(plat_specific=0, prefix=None):
                 incdir = os.path.join(get_config_var('srcdir'), 'Include')
                 return os.path.normpath(incdir)
         python_dir = 'python' + get_python_version() + build_flags
+        if not python_build and plat_specific:
+            import sysconfig
+            return sysconfig.get_path('platinclude')
         return os.path.join(prefix, "include", python_dir)
     elif os.name == "nt":
-        if python_build:
-            # Include both the include and PC dir to ensure we can find
-            # pyconfig.h
-            return (os.path.join(prefix, "include") + os.path.pathsep +
-                    os.path.join(prefix, "PC"))
         return os.path.join(prefix, "include")
     else:
         raise DistutilsPlatformError(
@@ -138,6 +126,7 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
     If 'prefix' is supplied, use it instead of sys.base_prefix or
     sys.base_exec_prefix -- i.e., ignore 'plat_specific'.
     """
+    is_default_prefix = not prefix or os.path.normpath(prefix) in ('/usr', '/usr/local')
     if prefix is None:
         if standard_lib:
             prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
@@ -145,18 +134,16 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
             prefix = plat_specific and EXEC_PREFIX or PREFIX
 
     if os.name == "posix":
-        if plat_specific or standard_lib:
-            # Platform-specific modules (any module from a non-pure-Python
-            # module distribution) or standard Python library modules.
-            libdir = sys.platlibdir
-        else:
-            # Pure Python
-            libdir = "lib"
-        libpython = os.path.join(prefix, libdir,
-                                 # XXX RUSTPYTHON: changed from python->rustpython
-                                 "rustpython" + get_python_version())
+        libpython = os.path.join(prefix,
+                                 "lib", "python" + get_python_version())
         if standard_lib:
             return libpython
+        elif (is_default_prefix and
+              'PYTHONUSERBASE' not in os.environ and
+              'VIRTUAL_ENV' not in os.environ and
+              'real_prefix' not in sys.__dict__ and
+              sys.prefix == sys.base_prefix):
+            return os.path.join(prefix, "lib", "python3", "dist-packages")
         else:
             return os.path.join(libpython, "site-packages")
     elif os.name == "nt":
@@ -194,9 +181,11 @@ def customize_compiler(compiler):
                 _osx_support.customize_compiler(_config_vars)
                 _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
 
-        (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \
-            get_config_vars('CC', 'CXX', 'CFLAGS',
-                            'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS')
+        (cc, cxx, opt, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags,
+         configure_cppflags, configure_cflags, configure_ldflags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS',
+                            'CONFIGURE_CPPFLAGS', 'CONFIGURE_CFLAGS', 'CONFIGURE_LDFLAGS')
 
         if 'CC' in os.environ:
             newcc = os.environ['CC']
@@ -209,6 +198,10 @@ def customize_compiler(compiler):
             cc = newcc
         if 'CXX' in os.environ:
             cxx = os.environ['CXX']
+        if fnmatch.filter([cc, cxx], '*-4.[0-8]'):
+            configure_cflags = configure_cflags.replace('-fstack-protector-strong', '-fstack-protector')
+            ldshared = ldshared.replace('-fstack-protector-strong', '-fstack-protector')
+            cflags = cflags.replace('-fstack-protector-strong', '-fstack-protector')
         if 'LDSHARED' in os.environ:
             ldshared = os.environ['LDSHARED']
         if 'CPP' in os.environ:
@@ -217,13 +210,22 @@ def customize_compiler(compiler):
             cpp = cc + " -E"           # not always
         if 'LDFLAGS' in os.environ:
             ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        elif configure_ldflags:
+            ldshared = ldshared + ' ' + configure_ldflags
         if 'CFLAGS' in os.environ:
-            cflags = cflags + ' ' + os.environ['CFLAGS']
+            cflags = opt + ' ' + os.environ['CFLAGS']
             ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        elif configure_cflags:
+            cflags = opt + ' ' + configure_cflags
+            ldshared = ldshared + ' ' + configure_cflags
         if 'CPPFLAGS' in os.environ:
             cpp = cpp + ' ' + os.environ['CPPFLAGS']
             cflags = cflags + ' ' + os.environ['CPPFLAGS']
             ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        elif configure_cppflags:
+            cpp = cpp + ' ' + configure_cppflags
+            cflags = cflags + ' ' + configure_cppflags
+            ldshared = ldshared + ' ' + configure_cppflags
         if 'AR' in os.environ:
             ar = os.environ['AR']
         if 'ARFLAGS' in os.environ:

Lib/doctest.py

@@ -102,7 +102,7 @@ import re
 import sys
 import traceback
 import unittest
-from io import StringIO  # XXX: RUSTPYTHON; , IncrementalNewlineDecoder
+from io import StringIO
 from collections import namedtuple
 
 TestResults = namedtuple('TestResults', 'failed attempted')
@@ -211,28 +211,17 @@ def _normalize_module(module, depth=2):
     else:
         raise TypeError("Expected a module, string, or None")
 
-def _newline_convert(data):
-    # The IO module provides a handy decoder for universal newline conversion
-    return IncrementalNewlineDecoder(None, True).decode(data, True)
-
 def _load_testfile(filename, package, module_relative, encoding):
     if module_relative:
         package = _normalize_module(package, 3)
         filename = _module_relative_path(package, filename)
-        if (loader := getattr(package, '__loader__', None)) is None:
-            try:
-                loader = package.__spec__.loader
-            except AttributeError:
-                pass
-        if hasattr(loader, 'get_data'):
-            file_contents = loader.get_data(filename)
-            file_contents = file_contents.decode(encoding)
-            # get_data() opens files as 'rb', so one must do the equivalent
-            # conversion as universal newlines would do.
-
-            # TODO: RUSTPYTHON; use _newline_convert once io.IncrementalNewlineDecoder is implemented
-            return file_contents.replace(os.linesep, '\n'), filename
-            # return _newline_convert(file_contents), filename
+        if getattr(package, '__loader__', None) is not None:
+            if hasattr(package.__loader__, 'get_data'):
+                file_contents = package.__loader__.get_data(filename)
+                file_contents = file_contents.decode(encoding)
+                # get_data() opens files as 'rb', so one must do the equivalent
+                # conversion as universal newlines would do.
+                return file_contents.replace(os.linesep, '\n'), filename
     with open(filename, encoding=encoding) as f:
         return f.read(), filename
 
@@ -976,17 +965,6 @@ class DocTestFinder:
         else:
             raise ValueError("object must be a class or function")
 
-    def _is_routine(self, obj):
-        """
-        Safely unwrap objects and determine if they are functions.
-        """
-        maybe_routine = obj
-        try:
-            maybe_routine = inspect.unwrap(maybe_routine)
-        except ValueError:
-            pass
-        return inspect.isroutine(maybe_routine)
-
     def _find(self, tests, obj, name, module, source_lines, globs, seen):
         """
         Find tests for the given object and any contained objects, and
@@ -1009,9 +987,9 @@ class DocTestFinder:
         if inspect.ismodule(obj) and self._recurse:
             for valname, val in obj.__dict__.items():
                 valname = '%s.%s' % (name, valname)
-
                 # Recurse to functions & classes.
-                if ((self._is_routine(val) or inspect.isclass(val)) and
+                if ((inspect.isroutine(inspect.unwrap(val))
+                     or inspect.isclass(val)) and
                     self._from_module(module, val)):
                     self._find(tests, val, valname, module, source_lines,
                                globs, seen)
@@ -1037,8 +1015,10 @@ class DocTestFinder:
         if inspect.isclass(obj) and self._recurse:
             for valname, val in obj.__dict__.items():
                 # Special handling for staticmethod/classmethod.
-                if isinstance(val, (staticmethod, classmethod)):
-                    val = val.__func__
+                if isinstance(val, staticmethod):
+                    val = getattr(obj, valname)
+                if isinstance(val, classmethod):
+                    val = getattr(obj, valname).__func__
 
                 # Recurse to methods, properties, and nested classes.
                 if ((inspect.isroutine(val) or inspect.isclass(val) or
@@ -1088,21 +1068,19 @@ class DocTestFinder:
 
     def _find_lineno(self, obj, source_lines):
         """
-        Return a line number of the given object's docstring.
-
-        Returns `None` if the given object does not have a docstring.
+        Return a line number of the given object's docstring.  Note:
+        this method assumes that the object has a docstring.
         """
         lineno = None
-        docstring = getattr(obj, '__doc__', None)
 
         # Find the line number for modules.
-        if inspect.ismodule(obj) and docstring is not None:
+        if inspect.ismodule(obj):
             lineno = 0
 
         # Find the line number for classes.
         # Note: this could be fooled if a class is defined multiple
         # times in a single file.
-        if inspect.isclass(obj) and docstring is not None:
+        if inspect.isclass(obj):
             if source_lines is None:
                 return None
             pat = re.compile(r'^\s*class\s*%s\b' %
@@ -1114,9 +1092,7 @@ class DocTestFinder:
 
         # Find the line number for functions & methods.
         if inspect.ismethod(obj): obj = obj.__func__
-        if inspect.isfunction(obj) and getattr(obj, '__doc__', None):
-            # We don't use `docstring` var here, because `obj` can be changed.
-            obj = obj.__code__
+        if inspect.isfunction(obj): obj = obj.__code__
         if inspect.istraceback(obj): obj = obj.tb_frame
         if inspect.isframe(obj): obj = obj.f_code
         if inspect.iscode(obj):
@@ -1351,7 +1327,7 @@ class DocTestRunner:
             try:
                 # Don't blink!  This is where the user's code gets run.
                 exec(compile(example.source, filename, "single",
-                             compileflags, True), test.globs)
+                             compileflags, 1), test.globs)
                 self.debugger.set_continue() # ==== Example Finished ====
                 exception = None
             except KeyboardInterrupt:
@@ -2178,7 +2154,6 @@ class DocTestCase(unittest.TestCase):
         unittest.TestCase.__init__(self)
         self._dt_optionflags = optionflags
         self._dt_checker = checker
-        self._dt_globs = test.globs.copy()
         self._dt_test = test
         self._dt_setUp = setUp
         self._dt_tearDown = tearDown
@@ -2195,9 +2170,7 @@ class DocTestCase(unittest.TestCase):
         if self._dt_tearDown is not None:
             self._dt_tearDown(test)
 
-        # restore the original globs
         test.globs.clear()
-        test.globs.update(self._dt_globs)
 
     def runTest(self):
         test = self._dt_test

Lib/dummy_threading.py

@@ -1,78 +0,0 @@
-"""Faux ``threading`` version using ``dummy_thread`` instead of ``thread``.
-
-The module ``_dummy_threading`` is added to ``sys.modules`` in order
-to not have ``threading`` considered imported.  Had ``threading`` been
-directly imported it would have made all subsequent imports succeed
-regardless of whether ``_thread`` was available which is not desired.
-
-"""
-from sys import modules as sys_modules
-
-import _dummy_thread
-
-# Declaring now so as to not have to nest ``try``s to get proper clean-up.
-holding_thread = False
-holding_threading = False
-holding__threading_local = False
-
-try:
-    # Could have checked if ``_thread`` was not in sys.modules and gone
-    # a different route, but decided to mirror technique used with
-    # ``threading`` below.
-    if '_thread' in sys_modules:
-        held_thread = sys_modules['_thread']
-        holding_thread = True
-    # Must have some module named ``_thread`` that implements its API
-    # in order to initially import ``threading``.
-    sys_modules['_thread'] = sys_modules['_dummy_thread']
-
-    if 'threading' in sys_modules:
-        # If ``threading`` is already imported, might as well prevent
-        # trying to import it more than needed by saving it if it is
-        # already imported before deleting it.
-        held_threading = sys_modules['threading']
-        holding_threading = True
-        del sys_modules['threading']
-
-    if '_threading_local' in sys_modules:
-        # If ``_threading_local`` is already imported, might as well prevent
-        # trying to import it more than needed by saving it if it is
-        # already imported before deleting it.
-        held__threading_local = sys_modules['_threading_local']
-        holding__threading_local = True
-        del sys_modules['_threading_local']
-
-    import threading
-    # Need a copy of the code kept somewhere...
-    sys_modules['_dummy_threading'] = sys_modules['threading']
-    del sys_modules['threading']
-    sys_modules['_dummy__threading_local'] = sys_modules['_threading_local']
-    del sys_modules['_threading_local']
-    from _dummy_threading import *
-    from _dummy_threading import __all__
-
-finally:
-    # Put back ``threading`` if we overwrote earlier
-
-    if holding_threading:
-        sys_modules['threading'] = held_threading
-        del held_threading
-    del holding_threading
-
-    # Put back ``_threading_local`` if we overwrote earlier
-
-    if holding__threading_local:
-        sys_modules['_threading_local'] = held__threading_local
-        del held__threading_local
-    del holding__threading_local
-
-    # Put back ``thread`` if we overwrote, else del the entry we made
-    if holding_thread:
-        sys_modules['_thread'] = held_thread
-        del held_thread
-    else:
-        del sys_modules['_thread']
-    del holding_thread
-
-    del _dummy_thread
-    del sys_modules

Lib/encodings/__init__.py

@@ -12,7 +12,7 @@
     * getregentry() -> codecs.CodecInfo object
     The getregentry() API must return a CodecInfo object with encoder, decoder,
     incrementalencoder, incrementaldecoder, streamwriter and streamreader
-    attributes which adhere to the Python Codec Interface Standard.
+    atttributes which adhere to the Python Codec Interface Standard.
 
     In addition, a module may optionally also define the following
     APIs which are then used by the package's codec search function:
@@ -49,7 +49,8 @@ def normalize_encoding(encoding):
         collapsed and replaced with a single underscore, e.g. '  -;#'
         becomes '_'. Leading and trailing underscores are removed.
 
-        Note that encoding names should be ASCII only.
+        Note that encoding names should be ASCII only; if they do use
+        non-ASCII characters, these must be Latin-1 compatible.
 
     """
     if isinstance(encoding, bytes):
@@ -57,12 +58,11 @@ def normalize_encoding(encoding):
 
     chars = []
     punct = False
-    for c in encoding:
+    for c in encoding.lower():
         if c.isalnum() or c == '.':
             if punct and chars:
                 chars.append('_')
-            if c.isascii():
-                chars.append(c)
+            chars.append(c)
             punct = False
         else:
             punct = True

Lib/encodings/aliases.py

@@ -266,8 +266,6 @@ aliases = {
     'roman8'             : 'hp_roman8',
     'r8'                 : 'hp_roman8',
     'csHPRoman8'         : 'hp_roman8',
-    'cp1051'             : 'hp_roman8',
-    'ibm1051'            : 'hp_roman8',
 
     # hz codec
     'hzgb'               : 'hz',
@@ -450,7 +448,6 @@ aliases = {
 
     # mac_latin2 codec
     'maccentraleurope'   : 'mac_latin2',
-    'mac_centeuro'       : 'mac_latin2',
     'maclatin2'          : 'mac_latin2',
 
     # mac_roman codec
@@ -494,6 +491,9 @@ aliases = {
     'sjisx0213'          : 'shift_jisx0213',
     's_jisx0213'         : 'shift_jisx0213',
 
+    # tactis codec
+    'tis260'             : 'tactis',
+
     # tis_620 codec
     'tis620'             : 'tis_620',
     'tis_620_0'          : 'tis_620',
@@ -534,7 +534,6 @@ aliases = {
     'utf8'               : 'utf_8',
     'utf8_ucs2'          : 'utf_8',
     'utf8_ucs4'          : 'utf_8',
-    'cp65001'            : 'utf_8',
 
     # uu_codec codec
     'uu'                 : 'uu_codec',

Lib/encodings/punycode.py

@@ -143,7 +143,7 @@ def decode_generalized_number(extended, extpos, bias, errors):
             digit = char - 22 # 0x30-26
         elif errors == "strict":
             raise UnicodeError("Invalid extended code point '%s'"
-                               % extended[extpos-1])
+                               % extended[extpos])
         else:
             return extpos, None
         t = T(j, bias)

Lib/encodings/raw_unicode_escape.py

@@ -21,16 +21,15 @@ class IncrementalEncoder(codecs.IncrementalEncoder):
     def encode(self, input, final=False):
         return codecs.raw_unicode_escape_encode(input, self.errors)[0]
 
-class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
-    def _buffer_decode(self, input, errors, final):
-        return codecs.raw_unicode_escape_decode(input, errors, final)
+class IncrementalDecoder(codecs.IncrementalDecoder):
+    def decode(self, input, final=False):
+        return codecs.raw_unicode_escape_decode(input, self.errors)[0]
 
 class StreamWriter(Codec,codecs.StreamWriter):
     pass
 
 class StreamReader(Codec,codecs.StreamReader):
-    def decode(self, input, errors='strict'):
-        return codecs.raw_unicode_escape_decode(input, errors, False)
+    pass
 
 ### encodings module API
 

Lib/encodings/unicode_escape.py

@@ -21,16 +21,15 @@ class IncrementalEncoder(codecs.IncrementalEncoder):
     def encode(self, input, final=False):
         return codecs.unicode_escape_encode(input, self.errors)[0]
 
-class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
-    def _buffer_decode(self, input, errors, final):
-        return codecs.unicode_escape_decode(input, errors, final)
+class IncrementalDecoder(codecs.IncrementalDecoder):
+    def decode(self, input, final=False):
+        return codecs.unicode_escape_decode(input, self.errors)[0]
 
 class StreamWriter(Codec,codecs.StreamWriter):
     pass
 
 class StreamReader(Codec,codecs.StreamReader):
-    def decode(self, input, errors='strict'):
-        return codecs.unicode_escape_decode(input, errors, False)
+    pass
 
 ### encodings module API
 

Lib/encodings/uu_codec.py

@@ -20,10 +20,6 @@ def uu_encode(input, errors='strict', filename='<data>', mode=0o666):
     read = infile.read
     write = outfile.write
 
-    # Remove newline chars from filename
-    filename = filename.replace('\n','\\n')
-    filename = filename.replace('\r','\\r')
-
     # Encode
     write(('begin %o %s\n' % (mode & 0o777, filename)).encode('ascii'))
     chunk = read(45)

Lib/ensurepip/__init__.py

@@ -1,284 +0,0 @@
-import collections
-import os
-import os.path
-import subprocess
-import sys
-import sysconfig
-import tempfile
-from importlib import resources
-
-
-
-__all__ = ["version", "bootstrap"]
-_PACKAGE_NAMES = ('setuptools', 'pip')
-_SETUPTOOLS_VERSION = "58.1.0"
-_PIP_VERSION = "22.0.4"
-_PROJECTS = [
-    ("setuptools", _SETUPTOOLS_VERSION, "py3"),
-    ("pip", _PIP_VERSION, "py3"),
-]
-
-# Packages bundled in ensurepip._bundled have wheel_name set.
-# Packages from WHEEL_PKG_DIR have wheel_path set.
-_Package = collections.namedtuple('Package',
-                                  ('version', 'wheel_name', 'wheel_path'))
-
-# Directory of system wheel packages. Some Linux distribution packaging
-# policies recommend against bundling dependencies. For example, Fedora
-# installs wheel packages in the /usr/share/python-wheels/ directory and don't
-# install the ensurepip._bundled package.
-_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR')
-
-
-def _find_packages(path):
-    packages = {}
-    try:
-        filenames = os.listdir(path)
-    except OSError:
-        # Ignore: path doesn't exist or permission error
-        filenames = ()
-    # Make the code deterministic if a directory contains multiple wheel files
-    # of the same package, but don't attempt to implement correct version
-    # comparison since this case should not happen.
-    filenames = sorted(filenames)
-    for filename in filenames:
-        # filename is like 'pip-21.2.4-py3-none-any.whl'
-        if not filename.endswith(".whl"):
-            continue
-        for name in _PACKAGE_NAMES:
-            prefix = name + '-'
-            if filename.startswith(prefix):
-                break
-        else:
-            continue
-
-        # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl'
-        version = filename.removeprefix(prefix).partition('-')[0]
-        wheel_path = os.path.join(path, filename)
-        packages[name] = _Package(version, None, wheel_path)
-    return packages
-
-
-def _get_packages():
-    global _PACKAGES, _WHEEL_PKG_DIR
-    if _PACKAGES is not None:
-        return _PACKAGES
-
-    packages = {}
-    for name, version, py_tag in _PROJECTS:
-        wheel_name = f"{name}-{version}-{py_tag}-none-any.whl"
-        packages[name] = _Package(version, wheel_name, None)
-    if _WHEEL_PKG_DIR:
-        dir_packages = _find_packages(_WHEEL_PKG_DIR)
-        # only used the wheel package directory if all packages are found there
-        if all(name in dir_packages for name in _PACKAGE_NAMES):
-            packages = dir_packages
-    _PACKAGES = packages
-    return packages
-_PACKAGES = None
-
-
-def _run_pip(args, additional_paths=None):
-    # Run the bootstraping in a subprocess to avoid leaking any state that happens
-    # after pip has executed. Particulary, this avoids the case when pip holds onto
-    # the files in *additional_paths*, preventing us to remove them at the end of the
-    # invocation.
-    code = f"""
-import runpy
-import sys
-sys.path = {additional_paths or []} + sys.path
-sys.argv[1:] = {args}
-runpy.run_module("pip", run_name="__main__", alter_sys=True)
-"""
-    return subprocess.run([sys.executable, '-W', 'ignore::DeprecationWarning',
-                           "-c", code], check=True).returncode
-
-
-def version():
-    """
-    Returns a string specifying the bundled version of pip.
-    """
-    return _get_packages()['pip'].version
-
-
-def _disable_pip_configuration_settings():
-    # We deliberately ignore all pip environment variables
-    # when invoking pip
-    # See http://bugs.python.org/issue19734 for details
-    keys_to_remove = [k for k in os.environ if k.startswith("PIP_")]
-    for k in keys_to_remove:
-        del os.environ[k]
-    # We also ignore the settings in the default pip configuration file
-    # See http://bugs.python.org/issue20053 for details
-    os.environ['PIP_CONFIG_FILE'] = os.devnull
-
-
-def bootstrap(*, root=None, upgrade=False, user=False,
-              altinstall=False, default_pip=False,
-              verbosity=0):
-    """
-    Bootstrap pip into the current Python installation (or the given root
-    directory).
-
-    Note that calling this function will alter both sys.path and os.environ.
-    """
-    # Discard the return value
-    _bootstrap(root=root, upgrade=upgrade, user=user,
-               altinstall=altinstall, default_pip=default_pip,
-               verbosity=verbosity)
-
-
-def _bootstrap(*, root=None, upgrade=False, user=False,
-              altinstall=False, default_pip=False,
-              verbosity=0):
-    """
-    Bootstrap pip into the current Python installation (or the given root
-    directory). Returns pip command status code.
-
-    Note that calling this function will alter both sys.path and os.environ.
-    """
-    if altinstall and default_pip:
-        raise ValueError("Cannot use altinstall and default_pip together")
-
-    sys.audit("ensurepip.bootstrap", root)
-
-    _disable_pip_configuration_settings()
-
-    # By default, installing pip and setuptools installs all of the
-    # following scripts (X.Y == running Python version):
-    #
-    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
-    #
-    # pip 1.5+ allows ensurepip to request that some of those be left out
-    if altinstall:
-        # omit pip, pipX and easy_install
-        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
-    elif not default_pip:
-        # omit pip and easy_install
-        os.environ["ENSUREPIP_OPTIONS"] = "install"
-
-    with tempfile.TemporaryDirectory() as tmpdir:
-        # Put our bundled wheels into a temporary directory and construct the
-        # additional paths that need added to sys.path
-        additional_paths = []
-        for name, package in _get_packages().items():
-            if package.wheel_name:
-                # Use bundled wheel package
-                from ensurepip import _bundled
-                wheel_name = package.wheel_name
-                whl = resources.read_binary(_bundled, wheel_name)
-            else:
-                # Use the wheel package directory
-                with open(package.wheel_path, "rb") as fp:
-                    whl = fp.read()
-                wheel_name = os.path.basename(package.wheel_path)
-
-            filename = os.path.join(tmpdir, wheel_name)
-            with open(filename, "wb") as fp:
-                fp.write(whl)
-
-            additional_paths.append(filename)
-
-        # Construct the arguments to be passed to the pip command
-        args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir]
-        if root:
-            args += ["--root", root]
-        if upgrade:
-            args += ["--upgrade"]
-        if user:
-            args += ["--user"]
-        if verbosity:
-            args += ["-" + "v" * verbosity]
-
-        return _run_pip([*args, *_PACKAGE_NAMES], additional_paths)
-
-def _uninstall_helper(*, verbosity=0):
-    """Helper to support a clean default uninstall process on Windows
-
-    Note that calling this function may alter os.environ.
-    """
-    # Nothing to do if pip was never installed, or has been removed
-    try:
-        import pip
-    except ImportError:
-        return
-
-    # If the installed pip version doesn't match the available one,
-    # leave it alone
-    available_version = version()
-    if pip.__version__ != available_version:
-        print(f"ensurepip will only uninstall a matching version "
-              f"({pip.__version__!r} installed, "
-              f"{available_version!r} available)",
-              file=sys.stderr)
-        return
-
-    _disable_pip_configuration_settings()
-
-    # Construct the arguments to be passed to the pip command
-    args = ["uninstall", "-y", "--disable-pip-version-check"]
-    if verbosity:
-        args += ["-" + "v" * verbosity]
-
-    return _run_pip([*args, *reversed(_PACKAGE_NAMES)])
-
-
-def _main(argv=None):
-    import argparse
-    parser = argparse.ArgumentParser(prog="python -m ensurepip")
-    parser.add_argument(
-        "--version",
-        action="version",
-        version="pip {}".format(version()),
-        help="Show the version of pip that is bundled with this Python.",
-    )
-    parser.add_argument(
-        "-v", "--verbose",
-        action="count",
-        default=0,
-        dest="verbosity",
-        help=("Give more output. Option is additive, and can be used up to 3 "
-              "times."),
-    )
-    parser.add_argument(
-        "-U", "--upgrade",
-        action="store_true",
-        default=False,
-        help="Upgrade pip and dependencies, even if already installed.",
-    )
-    parser.add_argument(
-        "--user",
-        action="store_true",
-        default=False,
-        help="Install using the user scheme.",
-    )
-    parser.add_argument(
-        "--root",
-        default=None,
-        help="Install everything relative to this alternate root directory.",
-    )
-    parser.add_argument(
-        "--altinstall",
-        action="store_true",
-        default=False,
-        help=("Make an alternate install, installing only the X.Y versioned "
-              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)."),
-    )
-    parser.add_argument(
-        "--default-pip",
-        action="store_true",
-        default=False,
-        help=("Make a default pip install, installing the unqualified pip "
-              "and easy_install in addition to the versioned scripts."),
-    )
-
-    args = parser.parse_args(argv)
-
-    return _bootstrap(
-        root=args.root,
-        upgrade=args.upgrade,
-        user=args.user,
-        verbosity=args.verbosity,
-        altinstall=args.altinstall,
-        default_pip=args.default_pip,
-    )

Lib/ensurepip/__main__.py

@@ -1,5 +0,0 @@
-import ensurepip
-import sys
-
-if __name__ == "__main__":
-    sys.exit(ensurepip._main())

Lib/ensurepip/_uninstall.py

@@ -1,31 +0,0 @@
-"""Basic pip uninstallation support, helper for the Windows uninstaller"""
-
-import argparse
-import ensurepip
-import sys
-
-
-def _main(argv=None):
-    parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall")
-    parser.add_argument(
-        "--version",
-        action="version",
-        version="pip {}".format(ensurepip.version()),
-        help="Show the version of pip this will attempt to uninstall.",
-    )
-    parser.add_argument(
-        "-v", "--verbose",
-        action="count",
-        default=0,
-        dest="verbosity",
-        help=("Give more output. Option is additive, and can be used up to 3 "
-              "times."),
-    )
-
-    args = parser.parse_args(argv)
-
-    return ensurepip._uninstall_helper(verbosity=args.verbosity)
-
-
-if __name__ == "__main__":
-    sys.exit(_main())

Lib/enum.py

@@ -10,41 +10,31 @@ __all__ = [
 
 
 def _is_descriptor(obj):
-    """
-    Returns True if obj is a descriptor, False otherwise.
-    """
+    """Returns True if obj is a descriptor, False otherwise."""
     return (
             hasattr(obj, '__get__') or
             hasattr(obj, '__set__') or
-            hasattr(obj, '__delete__')
-            )
+            hasattr(obj, '__delete__'))
+
 
 def _is_dunder(name):
-    """
-    Returns True if a __dunder__ name, False otherwise.
-    """
-    return (
-            len(name) > 4 and
+    """Returns True if a __dunder__ name, False otherwise."""
+    return (len(name) > 4 and
             name[:2] == name[-2:] == '__' and
             name[2] != '_' and
-            name[-3] != '_'
-            )
+            name[-3] != '_')
+
 
 def _is_sunder(name):
-    """
-    Returns True if a _sunder_ name, False otherwise.
-    """
-    return (
-            len(name) > 2 and
+    """Returns True if a _sunder_ name, False otherwise."""
+    return (len(name) > 2 and
             name[0] == name[-1] == '_' and
             name[1:2] != '_' and
-            name[-2:-1] != '_'
-            )
+            name[-2:-1] != '_')
+
 
 def _make_class_unpicklable(cls):
-    """
-    Make the given class un-picklable.
-    """
+    """Make the given class un-picklable."""
     def _break_on_call_reduce(self, proto):
         raise TypeError('%r cannot be pickled' % self)
     cls.__reduce_ex__ = _break_on_call_reduce
@@ -59,27 +49,26 @@ class auto:
 
 
 class _EnumDict(dict):
-    """
-    Track enum member order and ensure member names are not reused.
+    """Track enum member order and ensure member names are not reused.
 
     EnumMeta will use the names found in self._member_names as the
     enumeration member names.
+
     """
     def __init__(self):
         super().__init__()
         self._member_names = []
         self._last_values = []
         self._ignore = []
-        self._auto_called = False
 
     def __setitem__(self, key, value):
-        """
-        Changes anything not dundered or not a descriptor.
+        """Changes anything not dundered or not a descriptor.
 
         If an enum member name is used twice, an error is raised; duplicate
         values are not checked for.
 
         Single underscore (sunder) names are reserved.
+
         """
         if _is_sunder(key):
             if key not in (
@@ -88,9 +77,6 @@ class _EnumDict(dict):
                     ):
                 raise ValueError('_names_ are reserved for future Enum use')
             if key == '_generate_next_value_':
-                # check if members already defined as auto()
-                if self._auto_called:
-                    raise TypeError("_generate_next_value_ must be defined before members")
                 setattr(self, '_generate_next_value', value)
             elif key == '_ignore_':
                 if isinstance(value, str):
@@ -100,10 +86,7 @@ class _EnumDict(dict):
                 self._ignore = value
                 already = set(value) & set(self._member_names)
                 if already:
-                    raise ValueError(
-                            '_ignore_ cannot specify already set names: %r'
-                            % (already, )
-                            )
+                    raise ValueError('_ignore_ cannot specify already set names: %r' % (already, ))
         elif _is_dunder(key):
             if key == '__order__':
                 key = '_order_'
@@ -118,13 +101,7 @@ class _EnumDict(dict):
                 raise TypeError('%r already defined as: %r' % (key, self[key]))
             if isinstance(value, auto):
                 if value.value == _auto_null:
-                    value.value = self._generate_next_value(
-                            key,
-                            1,
-                            len(self._member_names),
-                            self._last_values[:],
-                            )
-                    self._auto_called = True
+                    value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:])
                 value = value.value
             self._member_names.append(key)
             self._last_values.append(value)
@@ -136,22 +113,17 @@ class _EnumDict(dict):
 # This is also why there are checks in EnumMeta like `if Enum is not None`
 Enum = None
 
+
 class EnumMeta(type):
-    """
-    Metaclass for Enum
-    """
+    """Metaclass for Enum"""
     @classmethod
     def __prepare__(metacls, cls, bases):
-        # check that previous enum members do not exist
-        metacls._check_for_existing_members(cls, bases)
         # create the namespace dict
         enum_dict = _EnumDict()
         # inherit previous flags and _generate_next_value_ function
-        member_type, first_enum = metacls._get_mixins_(cls, bases)
+        member_type, first_enum = metacls._get_mixins_(bases)
         if first_enum is not None:
-            enum_dict['_generate_next_value_'] = getattr(
-                    first_enum, '_generate_next_value_', None,
-                    )
+            enum_dict['_generate_next_value_'] = getattr(first_enum, '_generate_next_value_', None)
         return enum_dict
 
     def __new__(metacls, cls, bases, classdict):
@@ -165,10 +137,9 @@ class EnumMeta(type):
         ignore = classdict['_ignore_']
         for key in ignore:
             classdict.pop(key, None)
-        member_type, first_enum = metacls._get_mixins_(cls, bases)
-        __new__, save_new, use_args = metacls._find_new_(
-                classdict, member_type, first_enum,
-                )
+        member_type, first_enum = metacls._get_mixins_(bases)
+        __new__, save_new, use_args = metacls._find_new_(classdict, member_type,
+                                                        first_enum)
 
         # save enum items into separate mapping so they don't get baked into
         # the new class
@@ -197,11 +168,9 @@ class EnumMeta(type):
 
         # save DynamicClassAttribute attributes from super classes so we know
         # if we can take the shortcut of storing members in the class dict
-        dynamic_attributes = {
-                k for c in enum_class.mro()
-                for k, v in c.__dict__.items()
-                if isinstance(v, DynamicClassAttribute)
-                }
+        dynamic_attributes = {k for c in enum_class.mro()
+                              for k, v in c.__dict__.items()
+                              if isinstance(v, DynamicClassAttribute)}
 
         # Reverse value->name map for hashable values.
         enum_class._value2member_map_ = {}
@@ -275,11 +244,7 @@ class EnumMeta(type):
 
         # double check that repr and friends are not the mixin's or various
         # things break (such as pickle)
-        # however, if the method is defined in the Enum itself, don't replace
-        # it
         for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'):
-            if name in classdict:
-                continue
             class_method = getattr(enum_class, name)
             obj_method = getattr(member_type, name, None)
             enum_method = getattr(first_enum, name, None)
@@ -311,8 +276,7 @@ class EnumMeta(type):
         return True
 
     def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1):
-        """
-        Either returns an existing member, or creates a new enum class.
+        """Either returns an existing member, or creates a new enum class.
 
         This method is used both when an enum class is given a value to match
         to an enumeration member (i.e. Color(3)) and for the functional API
@@ -334,18 +298,12 @@ class EnumMeta(type):
         not correct, unpickling will fail in some circumstances.
 
         `type`, if set, will be mixed in as the first base class.
+
         """
         if names is None:  # simple value lookup
             return cls.__new__(cls, value)
         # otherwise, functional API: we're creating a new Enum type
-        return cls._create_(
-                value,
-                names,
-                module=module,
-                qualname=qualname,
-                type=type,
-                start=start,
-                )
+        return cls._create_(value, names, module=module, qualname=qualname, type=type, start=start)
 
     def __contains__(cls, member):
         if not isinstance(member, Enum):
@@ -358,23 +316,22 @@ class EnumMeta(type):
         # nicer error message when someone tries to delete an attribute
         # (see issue19025).
         if attr in cls._member_map_:
-            raise AttributeError("%s: cannot delete Enum member." % cls.__name__)
+            raise AttributeError(
+                    "%s: cannot delete Enum member." % cls.__name__)
         super().__delattr__(attr)
 
     def __dir__(self):
-        return (
-                ['__class__', '__doc__', '__members__', '__module__']
-                + self._member_names_
-                )
+        return (['__class__', '__doc__', '__members__', '__module__'] +
+                self._member_names_)
 
     def __getattr__(cls, name):
-        """
-        Return the enum member matching `name`
+        """Return the enum member matching `name`
 
         We use __getattr__ instead of descriptors or inserting into the enum
         class' __dict__ in order to support `name` and `value` being both
         properties for enum members (which live in the class' __dict__) and
         enum members themselves.
+
         """
         if _is_dunder(name):
             raise AttributeError(name)
@@ -387,9 +344,6 @@ class EnumMeta(type):
         return cls._member_map_[name]
 
     def __iter__(cls):
-        """
-        Returns members in definition order.
-        """
         return (cls._member_map_[name] for name in cls._member_names_)
 
     def __len__(cls):
@@ -397,11 +351,11 @@ class EnumMeta(type):
 
     @property
     def __members__(cls):
-        """
-        Returns a mapping of member name->value.
+        """Returns a mapping of member name->value.
 
         This mapping lists all enum members, including aliases. Note that this
         is a read-only view of the internal mapping.
+
         """
         return MappingProxyType(cls._member_map_)
 
@@ -409,18 +363,15 @@ class EnumMeta(type):
         return "<enum %r>" % cls.__name__
 
     def __reversed__(cls):
-        """
-        Returns members in reverse definition order.
-        """
         return (cls._member_map_[name] for name in reversed(cls._member_names_))
 
     def __setattr__(cls, name, value):
-        """
-        Block attempts to reassign Enum members.
+        """Block attempts to reassign Enum members.
 
         A simple assignment to the class namespace only changes one of the
         several possible ways to get an Enum member from the Enum class,
         resulting in an inconsistent Enumeration.
+
         """
         member_map = cls.__dict__.get('_member_map_', {})
         if name in member_map:
@@ -428,8 +379,7 @@ class EnumMeta(type):
         super().__setattr__(name, value)
 
     def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1):
-        """
-        Convenience method to create a new Enum class.
+        """Convenience method to create a new Enum class.
 
         `names` can be:
 
@@ -438,10 +388,11 @@ class EnumMeta(type):
         * An iterable of member names.  Values are incremented by 1 from `start`.
         * An iterable of (member name, value) pairs.
         * A mapping of member name -> value pairs.
+
         """
         metacls = cls.__class__
         bases = (cls, ) if type is None else (type, cls)
-        _, first_enum = cls._get_mixins_(cls, bases)
+        _, first_enum = cls._get_mixins_(bases)
         classdict = metacls.__prepare__(class_name, bases)
 
         # special processing needed for names?
@@ -520,50 +471,25 @@ class EnumMeta(type):
         return cls._convert_(*args, **kwargs)
 
     @staticmethod
-    def _check_for_existing_members(class_name, bases):
-        for chain in bases:
-            for base in chain.__mro__:
-                if issubclass(base, Enum) and base._member_names_:
-                    raise TypeError(
-                            "%s: cannot extend enumeration %r"
-                            % (class_name, base.__name__)
-                            )
-
-    @staticmethod
-    def _get_mixins_(class_name, bases):
-        """
-        Returns the type for creating enum members, and the first inherited
+    def _get_mixins_(bases):
+        """Returns the type for creating enum members, and the first inherited
         enum class.
 
         bases: the tuple of bases that was given to __new__
+
         """
         if not bases:
             return object, Enum
 
         def _find_data_type(bases):
-            data_types = []
             for chain in bases:
-                candidate = None
                 for base in chain.__mro__:
                     if base is object:
                         continue
-                    elif issubclass(base, Enum):
-                        if base._member_type_ is not object:
-                            data_types.append(base._member_type_)
-                            break
                     elif '__new__' in base.__dict__:
                         if issubclass(base, Enum):
                             continue
-                        data_types.append(candidate or base)
-                        break
-                    else:
-                        candidate = base
-            if len(data_types) > 1:
-                raise TypeError('%r: too many data types: %r' % (class_name, data_types))
-            elif data_types:
-                return data_types[0]
-            else:
-                return None
+                        return base
 
         # ensure final parent class is an Enum derivative, find any concrete
         # data type, and check that Enum has no members
@@ -578,12 +504,12 @@ class EnumMeta(type):
 
     @staticmethod
     def _find_new_(classdict, member_type, first_enum):
-        """
-        Returns the __new__ to be used for creating the enum members.
+        """Returns the __new__ to be used for creating the enum members.
 
         classdict: the class dictionary given to __new__
         member_type: the data type whose __new__ will be used by default
         first_enum: enumeration to check for an overriding __new__
+
         """
         # now find the correct __new__, checking to see of one was defined
         # by the user; also check earlier enum classes in case a __new__ was
@@ -623,10 +549,10 @@ class EnumMeta(type):
 
 
 class Enum(metaclass=EnumMeta):
-    """
-    Generic enumeration.
+    """Generic enumeration.
 
     Derive from this class to define new enumerations.
+
     """
     def __new__(cls, value):
         # all enum instances are actually created during class construction
@@ -669,14 +595,6 @@ class Enum(metaclass=EnumMeta):
             raise exc
 
     def _generate_next_value_(name, start, count, last_values):
-        """
-        Generate the next value when not given.
-
-        name: the name of the member
-        start: the initial start value or None
-        count: the number of existing members
-        last_value: the last value assigned or None
-        """
         for last_value in reversed(last_values):
             try:
                 return last_value + 1
@@ -687,7 +605,7 @@ class Enum(metaclass=EnumMeta):
 
     @classmethod
     def _missing_(cls, value):
-        return None
+        raise ValueError("%r is not a valid %s" % (value, cls.__name__))
 
     def __repr__(self):
         return "<%s.%s: %r>" % (
@@ -697,28 +615,21 @@ class Enum(metaclass=EnumMeta):
         return "%s.%s" % (self.__class__.__name__, self._name_)
 
     def __dir__(self):
-        """
-        Returns all members and all public methods
-        """
         added_behavior = [
                 m
                 for cls in self.__class__.mro()
                 for m in cls.__dict__
                 if m[0] != '_' and m not in self._member_map_
-                ] + [m for m in self.__dict__ if m[0] != '_']
+                ]
         return (['__class__', '__doc__', '__module__'] + added_behavior)
 
     def __format__(self, format_spec):
-        """
-        Returns format using actual value type unless __str__ has been overridden.
-        """
         # mixed-in Enums should use the mixed-in type's __format__, otherwise
         # we can get strange results with the Enum name showing up instead of
         # the value
 
-        # pure Enum branch, or branch with __str__ explicitly overridden
-        str_overridden = type(self).__str__ not in (Enum.__str__, Flag.__str__)
-        if self._member_type_ is object or str_overridden:
+        # pure Enum branch
+        if self._member_type_ is object:
             cls = str
             val = str(self)
         # mix-in branch
@@ -759,9 +670,7 @@ def _reduce_ex_by_name(self, proto):
     return self.name
 
 class Flag(Enum):
-    """
-    Support for flags
-    """
+    """Support for flags"""
 
     def _generate_next_value_(name, start, count, last_values):
         """
@@ -784,9 +693,6 @@ class Flag(Enum):
 
     @classmethod
     def _missing_(cls, value):
-        """
-        Returns member (possibly creating it) if one can be found for value.
-        """
         original_value = value
         if value < 0:
             value = ~value
@@ -816,9 +722,6 @@ class Flag(Enum):
         return pseudo_member
 
     def __contains__(self, other):
-        """
-        Returns True if self has at least the same flags set as other.
-        """
         if not isinstance(other, self.__class__):
             raise TypeError(
                 "unsupported operand type(s) for 'in': '%s' and '%s'" % (
@@ -877,15 +780,10 @@ class Flag(Enum):
 
 
 class IntFlag(int, Flag):
-    """
-    Support for integer-based Flags
-    """
+    """Support for integer-based Flags"""
 
     @classmethod
     def _missing_(cls, value):
-        """
-        Returns member (possibly creating it) if one can be found for value.
-        """
         if not isinstance(value, int):
             raise ValueError("%r is not a valid %s" % (value, cls.__name__))
         new_member = cls._create_pseudo_member_(value)
@@ -893,9 +791,6 @@ class IntFlag(int, Flag):
 
     @classmethod
     def _create_pseudo_member_(cls, value):
-        """
-        Create a composite member iff value contains only members.
-        """
         pseudo_member = cls._value2member_map_.get(value, None)
         if pseudo_member is None:
             need_to_create = [value]
@@ -950,15 +845,11 @@ class IntFlag(int, Flag):
 
 
 def _high_bit(value):
-    """
-    returns index of highest bit, or -1 if value is zero or negative
-    """
+    """returns index of highest bit, or -1 if value is zero or negative"""
     return value.bit_length() - 1
 
 def unique(enumeration):
-    """
-    Class decorator for enumerations ensuring unique member values.
-    """
+    """Class decorator for enumerations ensuring unique member values."""
     duplicates = []
     for name, member in enumeration.__members__.items():
         if name != member.name:
@@ -971,9 +862,7 @@ def unique(enumeration):
     return enumeration
 
 def _decompose(flag, value):
-    """
-    Extract all members from the value.
-    """
+    """Extract all members from the value."""
     # _decompose is only called if the value is not named
     not_covered = value
     negative = value < 0

Lib/filecmp.py

@@ -1,316 +0,0 @@
-"""Utilities for comparing files and directories.
-
-Classes:
-    dircmp
-
-Functions:
-    cmp(f1, f2, shallow=True) -> int
-    cmpfiles(a, b, common) -> ([], [], [])
-    clear_cache()
-
-"""
-
-try:
-    import os
-except ImportError:
-    import _dummy_os as os
-import stat
-from itertools import filterfalse
-from types import GenericAlias
-
-__all__ = ['clear_cache', 'cmp', 'dircmp', 'cmpfiles', 'DEFAULT_IGNORES']
-
-_cache = {}
-BUFSIZE = 8*1024
-
-DEFAULT_IGNORES = [
-    'RCS', 'CVS', 'tags', '.git', '.hg', '.bzr', '_darcs', '__pycache__']
-
-def clear_cache():
-    """Clear the filecmp cache."""
-    _cache.clear()
-
-def cmp(f1, f2, shallow=True):
-    """Compare two files.
-
-    Arguments:
-
-    f1 -- First file name
-
-    f2 -- Second file name
-
-    shallow -- treat files as identical if their stat signatures (type, size,
-               mtime) are identical. Otherwise, files are considered different
-               if their sizes or contents differ.  [default: True]
-
-    Return value:
-
-    True if the files are the same, False otherwise.
-
-    This function uses a cache for past comparisons and the results,
-    with cache entries invalidated if their stat information
-    changes.  The cache may be cleared by calling clear_cache().
-
-    """
-
-    s1 = _sig(os.stat(f1))
-    s2 = _sig(os.stat(f2))
-    if s1[0] != stat.S_IFREG or s2[0] != stat.S_IFREG:
-        return False
-    if shallow and s1 == s2:
-        return True
-    if s1[1] != s2[1]:
-        return False
-
-    outcome = _cache.get((f1, f2, s1, s2))
-    if outcome is None:
-        outcome = _do_cmp(f1, f2)
-        if len(_cache) > 100:      # limit the maximum size of the cache
-            clear_cache()
-        _cache[f1, f2, s1, s2] = outcome
-    return outcome
-
-def _sig(st):
-    return (stat.S_IFMT(st.st_mode),
-            st.st_size,
-            st.st_mtime)
-
-def _do_cmp(f1, f2):
-    bufsize = BUFSIZE
-    with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2:
-        while True:
-            b1 = fp1.read(bufsize)
-            b2 = fp2.read(bufsize)
-            if b1 != b2:
-                return False
-            if not b1:
-                return True
-
-# Directory comparison class.
-#
-class dircmp:
-    """A class that manages the comparison of 2 directories.
-
-    dircmp(a, b, ignore=None, hide=None)
-      A and B are directories.
-      IGNORE is a list of names to ignore,
-        defaults to DEFAULT_IGNORES.
-      HIDE is a list of names to hide,
-        defaults to [os.curdir, os.pardir].
-
-    High level usage:
-      x = dircmp(dir1, dir2)
-      x.report() -> prints a report on the differences between dir1 and dir2
-       or
-      x.report_partial_closure() -> prints report on differences between dir1
-            and dir2, and reports on common immediate subdirectories.
-      x.report_full_closure() -> like report_partial_closure,
-            but fully recursive.
-
-    Attributes:
-     left_list, right_list: The files in dir1 and dir2,
-        filtered by hide and ignore.
-     common: a list of names in both dir1 and dir2.
-     left_only, right_only: names only in dir1, dir2.
-     common_dirs: subdirectories in both dir1 and dir2.
-     common_files: files in both dir1 and dir2.
-     common_funny: names in both dir1 and dir2 where the type differs between
-        dir1 and dir2, or the name is not stat-able.
-     same_files: list of identical files.
-     diff_files: list of filenames which differ.
-     funny_files: list of files which could not be compared.
-     subdirs: a dictionary of dircmp instances (or MyDirCmp instances if this
-       object is of type MyDirCmp, a subclass of dircmp), keyed by names
-       in common_dirs.
-     """
-
-    def __init__(self, a, b, ignore=None, hide=None): # Initialize
-        self.left = a
-        self.right = b
-        if hide is None:
-            self.hide = [os.curdir, os.pardir] # Names never to be shown
-        else:
-            self.hide = hide
-        if ignore is None:
-            self.ignore = DEFAULT_IGNORES
-        else:
-            self.ignore = ignore
-
-    def phase0(self): # Compare everything except common subdirectories
-        self.left_list = _filter(os.listdir(self.left),
-                                 self.hide+self.ignore)
-        self.right_list = _filter(os.listdir(self.right),
-                                  self.hide+self.ignore)
-        self.left_list.sort()
-        self.right_list.sort()
-
-    def phase1(self): # Compute common names
-        a = dict(zip(map(os.path.normcase, self.left_list), self.left_list))
-        b = dict(zip(map(os.path.normcase, self.right_list), self.right_list))
-        self.common = list(map(a.__getitem__, filter(b.__contains__, a)))
-        self.left_only = list(map(a.__getitem__, filterfalse(b.__contains__, a)))
-        self.right_only = list(map(b.__getitem__, filterfalse(a.__contains__, b)))
-
-    def phase2(self): # Distinguish files, directories, funnies
-        self.common_dirs = []
-        self.common_files = []
-        self.common_funny = []
-
-        for x in self.common:
-            a_path = os.path.join(self.left, x)
-            b_path = os.path.join(self.right, x)
-
-            ok = 1
-            try:
-                a_stat = os.stat(a_path)
-            except OSError:
-                # print('Can\'t stat', a_path, ':', why.args[1])
-                ok = 0
-            try:
-                b_stat = os.stat(b_path)
-            except OSError:
-                # print('Can\'t stat', b_path, ':', why.args[1])
-                ok = 0
-
-            if ok:
-                a_type = stat.S_IFMT(a_stat.st_mode)
-                b_type = stat.S_IFMT(b_stat.st_mode)
-                if a_type != b_type:
-                    self.common_funny.append(x)
-                elif stat.S_ISDIR(a_type):
-                    self.common_dirs.append(x)
-                elif stat.S_ISREG(a_type):
-                    self.common_files.append(x)
-                else:
-                    self.common_funny.append(x)
-            else:
-                self.common_funny.append(x)
-
-    def phase3(self): # Find out differences between common files
-        xx = cmpfiles(self.left, self.right, self.common_files)
-        self.same_files, self.diff_files, self.funny_files = xx
-
-    def phase4(self): # Find out differences between common subdirectories
-        # A new dircmp (or MyDirCmp if dircmp was subclassed) object is created
-        # for each common subdirectory,
-        # these are stored in a dictionary indexed by filename.
-        # The hide and ignore properties are inherited from the parent
-        self.subdirs = {}
-        for x in self.common_dirs:
-            a_x = os.path.join(self.left, x)
-            b_x = os.path.join(self.right, x)
-            self.subdirs[x]  = self.__class__(a_x, b_x, self.ignore, self.hide)
-
-    def phase4_closure(self): # Recursively call phase4() on subdirectories
-        self.phase4()
-        for sd in self.subdirs.values():
-            sd.phase4_closure()
-
-    def report(self): # Print a report on the differences between a and b
-        # Output format is purposely lousy
-        print('diff', self.left, self.right)
-        if self.left_only:
-            self.left_only.sort()
-            print('Only in', self.left, ':', self.left_only)
-        if self.right_only:
-            self.right_only.sort()
-            print('Only in', self.right, ':', self.right_only)
-        if self.same_files:
-            self.same_files.sort()
-            print('Identical files :', self.same_files)
-        if self.diff_files:
-            self.diff_files.sort()
-            print('Differing files :', self.diff_files)
-        if self.funny_files:
-            self.funny_files.sort()
-            print('Trouble with common files :', self.funny_files)
-        if self.common_dirs:
-            self.common_dirs.sort()
-            print('Common subdirectories :', self.common_dirs)
-        if self.common_funny:
-            self.common_funny.sort()
-            print('Common funny cases :', self.common_funny)
-
-    def report_partial_closure(self): # Print reports on self and on subdirs
-        self.report()
-        for sd in self.subdirs.values():
-            print()
-            sd.report()
-
-    def report_full_closure(self): # Report on self and subdirs recursively
-        self.report()
-        for sd in self.subdirs.values():
-            print()
-            sd.report_full_closure()
-
-    methodmap = dict(subdirs=phase4,
-                     same_files=phase3, diff_files=phase3, funny_files=phase3,
-                     common_dirs = phase2, common_files=phase2, common_funny=phase2,
-                     common=phase1, left_only=phase1, right_only=phase1,
-                     left_list=phase0, right_list=phase0)
-
-    def __getattr__(self, attr):
-        if attr not in self.methodmap:
-            raise AttributeError(attr)
-        self.methodmap[attr](self)
-        return getattr(self, attr)
-
-    __class_getitem__ = classmethod(GenericAlias)
-
-
-def cmpfiles(a, b, common, shallow=True):
-    """Compare common files in two directories.
-
-    a, b -- directory names
-    common -- list of file names found in both directories
-    shallow -- if true, do comparison based solely on stat() information
-
-    Returns a tuple of three lists:
-      files that compare equal
-      files that are different
-      filenames that aren't regular files.
-
-    """
-    res = ([], [], [])
-    for x in common:
-        ax = os.path.join(a, x)
-        bx = os.path.join(b, x)
-        res[_cmp(ax, bx, shallow)].append(x)
-    return res
-
-
-# Compare two files.
-# Return:
-#       0 for equal
-#       1 for different
-#       2 for funny cases (can't stat, etc.)
-#
-def _cmp(a, b, sh, abs=abs, cmp=cmp):
-    try:
-        return not abs(cmp(a, b, sh))
-    except OSError:
-        return 2
-
-
-# Return a copy with items that occur in skip removed.
-#
-def _filter(flist, skip):
-    return list(filterfalse(skip.__contains__, flist))
-
-
-# Demonstration and testing.
-#
-def demo():
-    import sys
-    import getopt
-    options, args = getopt.getopt(sys.argv[1:], 'r')
-    if len(args) != 2:
-        raise getopt.GetoptError('need exactly two args', None)
-    dd = dircmp(args[0], args[1])
-    if ('-r', '') in options:
-        dd.report_full_closure()
-    else:
-        dd.report()
-
-if __name__ == '__main__':
-    demo()

Lib/fileinput.py

@@ -1,462 +0,0 @@
-"""Helper class to quickly write a loop over all standard input files.
-
-Typical use is:
-
-    import fileinput
-    for line in fileinput.input(encoding="utf-8"):
-        process(line)
-
-This iterates over the lines of all files listed in sys.argv[1:],
-defaulting to sys.stdin if the list is empty.  If a filename is '-' it
-is also replaced by sys.stdin and the optional arguments mode and
-openhook are ignored.  To specify an alternative list of filenames,
-pass it as the argument to input().  A single file name is also allowed.
-
-Functions filename(), lineno() return the filename and cumulative line
-number of the line that has just been read; filelineno() returns its
-line number in the current file; isfirstline() returns true iff the
-line just read is the first line of its file; isstdin() returns true
-iff the line was read from sys.stdin.  Function nextfile() closes the
-current file so that the next iteration will read the first line from
-the next file (if any); lines not read from the file will not count
-towards the cumulative line count; the filename is not changed until
-after the first line of the next file has been read.  Function close()
-closes the sequence.
-
-Before any lines have been read, filename() returns None and both line
-numbers are zero; nextfile() has no effect.  After all lines have been
-read, filename() and the line number functions return the values
-pertaining to the last line read; nextfile() has no effect.
-
-All files are opened in text mode by default, you can override this by
-setting the mode parameter to input() or FileInput.__init__().
-If an I/O error occurs during opening or reading a file, the OSError
-exception is raised.
-
-If sys.stdin is used more than once, the second and further use will
-return no lines, except perhaps for interactive use, or if it has been
-explicitly reset (e.g. using sys.stdin.seek(0)).
-
-Empty files are opened and immediately closed; the only time their
-presence in the list of filenames is noticeable at all is when the
-last file opened is empty.
-
-It is possible that the last line of a file doesn't end in a newline
-character; otherwise lines are returned including the trailing
-newline.
-
-Class FileInput is the implementation; its methods filename(),
-lineno(), fileline(), isfirstline(), isstdin(), nextfile() and close()
-correspond to the functions in the module.  In addition it has a
-readline() method which returns the next input line, and a
-__getitem__() method which implements the sequence behavior.  The
-sequence must be accessed in strictly sequential order; sequence
-access and readline() cannot be mixed.
-
-Optional in-place filtering: if the keyword argument inplace=1 is
-passed to input() or to the FileInput constructor, the file is moved
-to a backup file and standard output is directed to the input file.
-This makes it possible to write a filter that rewrites its input file
-in place.  If the keyword argument backup=".<some extension>" is also
-given, it specifies the extension for the backup file, and the backup
-file remains around; by default, the extension is ".bak" and it is
-deleted when the output file is closed.  In-place filtering is
-disabled when standard input is read.  XXX The current implementation
-does not work for MS-DOS 8+3 filesystems.
-"""
-
-import io
-import sys, os
-from types import GenericAlias
-
-__all__ = ["input", "close", "nextfile", "filename", "lineno", "filelineno",
-           "fileno", "isfirstline", "isstdin", "FileInput", "hook_compressed",
-           "hook_encoded"]
-
-_state = None
-
-def input(files=None, inplace=False, backup="", *, mode="r", openhook=None,
-          encoding=None, errors=None):
-    """Return an instance of the FileInput class, which can be iterated.
-
-    The parameters are passed to the constructor of the FileInput class.
-    The returned instance, in addition to being an iterator,
-    keeps global state for the functions of this module,.
-    """
-    global _state
-    if _state and _state._file:
-        raise RuntimeError("input() already active")
-    _state = FileInput(files, inplace, backup, mode=mode, openhook=openhook,
-                       encoding=encoding, errors=errors)
-    return _state
-
-def close():
-    """Close the sequence."""
-    global _state
-    state = _state
-    _state = None
-    if state:
-        state.close()
-
-def nextfile():
-    """
-    Close the current file so that the next iteration will read the first
-    line from the next file (if any); lines not read from the file will
-    not count towards the cumulative line count. The filename is not
-    changed until after the first line of the next file has been read.
-    Before the first line has been read, this function has no effect;
-    it cannot be used to skip the first file. After the last line of the
-    last file has been read, this function has no effect.
-    """
-    if not _state:
-        raise RuntimeError("no active input()")
-    return _state.nextfile()
-
-def filename():
-    """
-    Return the name of the file currently being read.
-    Before the first line has been read, returns None.
-    """
-    if not _state:
-        raise RuntimeError("no active input()")
-    return _state.filename()
-
-def lineno():
-    """
-    Return the cumulative line number of the line that has just been read.
-    Before the first line has been read, returns 0. After the last line
-    of the last file has been read, returns the line number of that line.
-    """
-    if not _state:
-        raise RuntimeError("no active input()")
-    return _state.lineno()
-
-def filelineno():
-    """
-    Return the line number in the current file. Before the first line
-    has been read, returns 0. After the last line of the last file has
-    been read, returns the line number of that line within the file.
-    """
-    if not _state:
-        raise RuntimeError("no active input()")
-    return _state.filelineno()
-
-def fileno():
-    """
-    Return the file number of the current file. When no file is currently
-    opened, returns -1.
-    """
-    if not _state:
-        raise RuntimeError("no active input()")
-    return _state.fileno()
-
-def isfirstline():
-    """
-    Returns true the line just read is the first line of its file,
-    otherwise returns false.
-    """
-    if not _state:
-        raise RuntimeError("no active input()")
-    return _state.isfirstline()
-
-def isstdin():
-    """
-    Returns true if the last line was read from sys.stdin,
-    otherwise returns false.
-    """
-    if not _state:
-        raise RuntimeError("no active input()")
-    return _state.isstdin()
-
-class FileInput:
-    """FileInput([files[, inplace[, backup]]], *, mode=None, openhook=None)
-
-    Class FileInput is the implementation of the module; its methods
-    filename(), lineno(), fileline(), isfirstline(), isstdin(), fileno(),
-    nextfile() and close() correspond to the functions of the same name
-    in the module.
-    In addition it has a readline() method which returns the next
-    input line, and a __getitem__() method which implements the
-    sequence behavior. The sequence must be accessed in strictly
-    sequential order; random access and readline() cannot be mixed.
-    """
-
-    def __init__(self, files=None, inplace=False, backup="", *,
-                 mode="r", openhook=None, encoding=None, errors=None):
-        if isinstance(files, str):
-            files = (files,)
-        elif isinstance(files, os.PathLike):
-            files = (os.fspath(files), )
-        else:
-            if files is None:
-                files = sys.argv[1:]
-            if not files:
-                files = ('-',)
-            else:
-                files = tuple(files)
-        self._files = files
-        self._inplace = inplace
-        self._backup = backup
-        self._savestdout = None
-        self._output = None
-        self._filename = None
-        self._startlineno = 0
-        self._filelineno = 0
-        self._file = None
-        self._isstdin = False
-        self._backupfilename = None
-        self._encoding = encoding
-        self._errors = errors
-
-        # We can not use io.text_encoding() here because old openhook doesn't
-        # take encoding parameter.
-        if (sys.flags.warn_default_encoding and
-                "b" not in mode and encoding is None and openhook is None):
-            import warnings
-            warnings.warn("'encoding' argument not specified.",
-                          EncodingWarning, 2)
-
-        # restrict mode argument to reading modes
-        if mode not in ('r', 'rU', 'U', 'rb'):
-            raise ValueError("FileInput opening mode must be one of "
-                             "'r', 'rU', 'U' and 'rb'")
-        if 'U' in mode:
-            import warnings
-            warnings.warn("'U' mode is deprecated",
-                          DeprecationWarning, 2)
-        self._mode = mode
-        self._write_mode = mode.replace('r', 'w') if 'U' not in mode else 'w'
-        if openhook:
-            if inplace:
-                raise ValueError("FileInput cannot use an opening hook in inplace mode")
-            if not callable(openhook):
-                raise ValueError("FileInput openhook must be callable")
-        self._openhook = openhook
-
-    def __del__(self):
-        self.close()
-
-    def close(self):
-        try:
-            self.nextfile()
-        finally:
-            self._files = ()
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, type, value, traceback):
-        self.close()
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        while True:
-            line = self._readline()
-            if line:
-                self._filelineno += 1
-                return line
-            if not self._file:
-                raise StopIteration
-            self.nextfile()
-            # repeat with next file
-
-    def __getitem__(self, i):
-        import warnings
-        warnings.warn(
-            "Support for indexing FileInput objects is deprecated. "
-            "Use iterator protocol instead.",
-            DeprecationWarning,
-            stacklevel=2
-        )
-        if i != self.lineno():
-            raise RuntimeError("accessing lines out of order")
-        try:
-            return self.__next__()
-        except StopIteration:
-            raise IndexError("end of input reached")
-
-    def nextfile(self):
-        savestdout = self._savestdout
-        self._savestdout = None
-        if savestdout:
-            sys.stdout = savestdout
-
-        output = self._output
-        self._output = None
-        try:
-            if output:
-                output.close()
-        finally:
-            file = self._file
-            self._file = None
-            try:
-                del self._readline  # restore FileInput._readline
-            except AttributeError:
-                pass
-            try:
-                if file and not self._isstdin:
-                    file.close()
-            finally:
-                backupfilename = self._backupfilename
-                self._backupfilename = None
-                if backupfilename and not self._backup:
-                    try: os.unlink(backupfilename)
-                    except OSError: pass
-
-                self._isstdin = False
-
-    def readline(self):
-        while True:
-            line = self._readline()
-            if line:
-                self._filelineno += 1
-                return line
-            if not self._file:
-                return line
-            self.nextfile()
-            # repeat with next file
-
-    def _readline(self):
-        if not self._files:
-            if 'b' in self._mode:
-                return b''
-            else:
-                return ''
-        self._filename = self._files[0]
-        self._files = self._files[1:]
-        self._startlineno = self.lineno()
-        self._filelineno = 0
-        self._file = None
-        self._isstdin = False
-        self._backupfilename = 0
-
-        # EncodingWarning is emitted in __init__() already
-        if "b" not in self._mode:
-            encoding = self._encoding or "locale"
-        else:
-            encoding = None
-
-        if self._filename == '-':
-            self._filename = '<stdin>'
-            if 'b' in self._mode:
-                self._file = getattr(sys.stdin, 'buffer', sys.stdin)
-            else:
-                self._file = sys.stdin
-            self._isstdin = True
-        else:
-            if self._inplace:
-                self._backupfilename = (
-                    os.fspath(self._filename) + (self._backup or ".bak"))
-                try:
-                    os.unlink(self._backupfilename)
-                except OSError:
-                    pass
-                # The next few lines may raise OSError
-                os.rename(self._filename, self._backupfilename)
-                self._file = open(self._backupfilename, self._mode,
-                                  encoding=encoding, errors=self._errors)
-                try:
-                    perm = os.fstat(self._file.fileno()).st_mode
-                except OSError:
-                    self._output = open(self._filename, self._write_mode,
-                                        encoding=encoding, errors=self._errors)
-                else:
-                    mode = os.O_CREAT | os.O_WRONLY | os.O_TRUNC
-                    if hasattr(os, 'O_BINARY'):
-                        mode |= os.O_BINARY
-
-                    fd = os.open(self._filename, mode, perm)
-                    self._output = os.fdopen(fd, self._write_mode,
-                                             encoding=encoding, errors=self._errors)
-                    try:
-                        os.chmod(self._filename, perm)
-                    except OSError:
-                        pass
-                self._savestdout = sys.stdout
-                sys.stdout = self._output
-            else:
-                # This may raise OSError
-                if self._openhook:
-                    # Custom hooks made previous to Python 3.10 didn't have
-                    # encoding argument
-                    if self._encoding is None:
-                        self._file = self._openhook(self._filename, self._mode)
-                    else:
-                        self._file = self._openhook(
-                            self._filename, self._mode, encoding=self._encoding, errors=self._errors)
-                else:
-                    self._file = open(self._filename, self._mode, encoding=encoding, errors=self._errors)
-        self._readline = self._file.readline  # hide FileInput._readline
-        return self._readline()
-
-    def filename(self):
-        return self._filename
-
-    def lineno(self):
-        return self._startlineno + self._filelineno
-
-    def filelineno(self):
-        return self._filelineno
-
-    def fileno(self):
-        if self._file:
-            try:
-                return self._file.fileno()
-            except ValueError:
-                return -1
-        else:
-            return -1
-
-    def isfirstline(self):
-        return self._filelineno == 1
-
-    def isstdin(self):
-        return self._isstdin
-
-    __class_getitem__ = classmethod(GenericAlias)
-
-
-def hook_compressed(filename, mode, *, encoding=None, errors=None):
-    if encoding is None:  # EncodingWarning is emitted in FileInput() already.
-        encoding = "locale"
-    ext = os.path.splitext(filename)[1]
-    if ext == '.gz':
-        import gzip
-        stream = gzip.open(filename, mode)
-    elif ext == '.bz2':
-        import bz2
-        stream = bz2.BZ2File(filename, mode)
-    else:
-        return open(filename, mode, encoding=encoding, errors=errors)
-
-    # gzip and bz2 are binary mode by default.
-    if "b" not in mode:
-        stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors)
-    return stream
-
-
-def hook_encoded(encoding, errors=None):
-    def openhook(filename, mode):
-        return open(filename, mode, encoding=encoding, errors=errors)
-    return openhook
-
-
-def _test():
-    import getopt
-    inplace = False
-    backup = False
-    opts, args = getopt.getopt(sys.argv[1:], "ib:")
-    for o, a in opts:
-        if o == '-i': inplace = True
-        if o == '-b': backup = a
-    for line in input(args, inplace=inplace, backup=backup):
-        if line[-1:] == '\n': line = line[:-1]
-        if line[-1:] == '\r': line = line[:-1]
-        print("%d: %s[%d]%s %s" % (lineno(), filename(), filelineno(),
-                                   isfirstline() and "*" or "", line))
-    print("%d: %s[%d]" % (lineno(), filename(), filelineno()))
-
-if __name__ == '__main__':
-    _test()

Lib/fnmatch.py

@@ -16,12 +16,6 @@ import functools
 
 __all__ = ["filter", "fnmatch", "fnmatchcase", "translate"]
 
-# Build a thread-safe incrementing counter to help create unique regexp group
-# names across calls.
-from itertools import count
-_nextgroupnum = count().__next__
-del count
-
 def fnmatch(name, pat):
     """Test whether FILENAME matches PATTERN.
 
@@ -52,7 +46,7 @@ def _compile_pattern(pat):
     return re.compile(res).match
 
 def filter(names, pat):
-    """Construct a list from those elements of the iterable NAMES that match PAT."""
+    """Return the subset of the list NAMES that match PAT."""
     result = []
     pat = os.path.normcase(pat)
     match = _compile_pattern(pat)
@@ -83,19 +77,15 @@ def translate(pat):
     There is no way to quote meta-characters.
     """
 
-    STAR = object()
-    res = []
-    add = res.append
     i, n = 0, len(pat)
+    res = ''
     while i < n:
         c = pat[i]
         i = i+1
         if c == '*':
-            # compress consecutive `*` into one
-            if (not res) or res[-1] is not STAR:
-                add(STAR)
+            res = res + '.*'
         elif c == '?':
-            add('.')
+            res = res + '.'
         elif c == '[':
             j = i
             if j < n and pat[j] == '!':
@@ -105,10 +95,10 @@ def translate(pat):
             while j < n and pat[j] != ']':
                 j = j+1
             if j >= n:
-                add('\\[')
+                res = res + '\\['
             else:
                 stuff = pat[i:j]
-                if '-' not in stuff:
+                if '--' not in stuff:
                     stuff = stuff.replace('\\', r'\\')
                 else:
                     chunks = []
@@ -120,16 +110,7 @@ def translate(pat):
                         chunks.append(pat[i:k])
                         i = k+1
                         k = k+3
-                    chunk = pat[i:j]
-                    if chunk:
-                        chunks.append(chunk)
-                    else:
-                        chunks[-1] += '-'
-                    # Remove empty ranges -- invalid in RE.
-                    for k in range(len(chunks)-1, 0, -1):
-                        if chunks[k-1][-1] > chunks[k][0]:
-                            chunks[k-1] = chunks[k-1][:-1] + chunks[k][1:]
-                            del chunks[k]
+                    chunks.append(pat[i:j])
                     # Escape backslashes and hyphens for set difference (--).
                     # Hyphens that create ranges shouldn't be escaped.
                     stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-')
@@ -137,63 +118,11 @@ def translate(pat):
                 # Escape set operations (&&, ~~ and ||).
                 stuff = re.sub(r'([&~|])', r'\\\1', stuff)
                 i = j+1
-                if not stuff:
-                    # Empty range: never match.
-                    add('(?!)')
-                elif stuff == '!':
-                    # Negated empty range: match any character.
-                    add('.')
-                else:
-                    if stuff[0] == '!':
-                        stuff = '^' + stuff[1:]
-                    elif stuff[0] in ('^', '['):
-                        stuff = '\\' + stuff
-                    add(f'[{stuff}]')
+                if stuff[0] == '!':
+                    stuff = '^' + stuff[1:]
+                elif stuff[0] in ('^', '['):
+                    stuff = '\\' + stuff
+                res = '%s[%s]' % (res, stuff)
         else:
-            add(re.escape(c))
-    assert i == n
-
-    # Deal with STARs.
-    inp = res
-    res = []
-    add = res.append
-    i, n = 0, len(inp)
-    # Fixed pieces at the start?
-    while i < n and inp[i] is not STAR:
-        add(inp[i])
-        i += 1
-    # Now deal with STAR fixed STAR fixed ...
-    # For an interior `STAR fixed` pairing, we want to do a minimal
-    # .*? match followed by `fixed`, with no possibility of backtracking.
-    # We can't spell that directly, but can trick it into working by matching
-    #    .*?fixed
-    # in a lookahead assertion, save the matched part in a group, then
-    # consume that group via a backreference. If the overall match fails,
-    # the lookahead assertion won't try alternatives. So the translation is:
-    #     (?=(?P<name>.*?fixed))(?P=name)
-    # Group names are created as needed: g0, g1, g2, ...
-    # The numbers are obtained from _nextgroupnum() to ensure they're unique
-    # across calls and across threads. This is because people rely on the
-    # undocumented ability to join multiple translate() results together via
-    # "|" to build large regexps matching "one of many" shell patterns.
-    while i < n:
-        assert inp[i] is STAR
-        i += 1
-        if i == n:
-            add(".*")
-            break
-        assert inp[i] is not STAR
-        fixed = []
-        while i < n and inp[i] is not STAR:
-            fixed.append(inp[i])
-            i += 1
-        fixed = "".join(fixed)
-        if i == n:
-            add(".*")
-            add(fixed)
-        else:
-            groupnum = _nextgroupnum()
-            add(f"(?=(?P<g{groupnum}>.*?{fixed}))(?P=g{groupnum})")
-    assert i == n
-    res = "".join(res)
-    return fr'(?s:{res})\Z'
+            res = res + re.escape(c)
+    return r'(?s:%s)\Z' % res

Lib/functools.py

@@ -18,8 +18,10 @@ from abc import get_cache_token
 from collections import namedtuple
 # import types, weakref  # Deferred to single_dispatch()
 from reprlib import recursive_repr
-from _thread import RLock
-from types import GenericAlias
+try:
+    from _thread import RLock
+except ModuleNotFoundError:
+    from _dummy_thread import RLock
 
 
 ################################################################################
@@ -425,8 +427,6 @@ class partialmethod(object):
     def __isabstractmethod__(self):
         return getattr(self.func, "__isabstractmethod__", False)
 
-    __class_getitem__ = classmethod(GenericAlias)
-
 # Helper functions
 
 def _unwrap_partial(func):
@@ -977,5 +977,3 @@ class cached_property:
                         )
                         raise TypeError(msg) from None
         return val
-
-    __class_getitem__ = classmethod(GenericAlias)

Lib/gc.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python  # [built-in module gc]
+"""
+This module provides access to the garbage collector for reference cycles.
+
+enable() -- Enable automatic garbage collection.
+disable() -- Disable automatic garbage collection.
+isenabled() -- Returns true if automatic collection is enabled.
+collect() -- Do a full collection right now.
+get_count() -- Return the current collection counts.
+get_stats() -- Return list of dictionaries containing per-generation stats.
+set_debug() -- Set debugging flags.
+get_debug() -- Get debugging flags.
+set_threshold() -- Set the collection thresholds.
+get_threshold() -- Return the current the collection thresholds.
+get_objects() -- Return a list of all objects tracked by the collector.
+is_tracked() -- Returns true if a given object is tracked.
+get_referrers() -- Return the list of objects that refer to an object.
+get_referents() -- Return the list of objects that an object refers to.
+"""
+
+## DATA ##
+
+DEBUG_COLLECTABLE = 2
+# None
+DEBUG_LEAK = 38
+# None
+DEBUG_SAVEALL = 32
+# None
+DEBUG_STATS = 1
+# None
+DEBUG_UNCOLLECTABLE = 4
+# None
+callbacks = []
+# None
+garbage = []
+# None
+
+## FUNCTIONS ##
+
+
+def collect(*args, **kwargs):  # unknown args #
+    """
+    collect([generation]) -> n
+
+    With no arguments, run a full collection.  The optional argument
+    may be an integer specifying which generation to collect.  A ValueError
+    is raised if the generation number is invalid.
+
+    The number of unreachable objects is returned.
+    """
+    return 0
+
+
+def disable(*args, **kwargs):  # unknown args #
+    """
+    disable() -> None
+
+    Disable automatic garbage collection.
+    """
+    raise NotImplementedError()
+
+
+def enable(*args, **kwargs):  # unknown args #
+    """
+    enable() -> None
+
+    Enable automatic garbage collection.
+    """
+    raise NotImplementedError()
+
+
+def get_count(*args, **kwargs):  # unknown args #
+    """
+    get_count() -> (count0, count1, count2)
+
+    Return the current collection counts
+    """
+    raise NotImplementedError()
+
+
+def get_debug(*args, **kwargs):  # unknown args #
+    """
+    get_debug() -> flags
+
+    Get the garbage collection debugging flags.
+    """
+    raise NotImplementedError()
+
+
+def get_objects(*args, **kwargs):  # unknown args #
+    """
+    get_objects() -> [...]
+
+    Return a list of objects tracked by the collector (excluding the list
+    returned).
+    """
+    raise NotImplementedError()
+
+
+def get_referents(*args, **kwargs):  # unknown args #
+    """
+    get_referents(*objs) -> list
+    Return the list of objects that are directly referred to by objs.
+    """
+    raise NotImplementedError()
+
+
+def get_referrers(*args, **kwargs):  # unknown args #
+    """
+    get_referrers(*objs) -> list
+    Return the list of objects that directly refer to any of objs.
+    """
+    raise NotImplementedError()
+
+
+def get_stats(*args, **kwargs):  # unknown args #
+    """
+    get_stats() -> [...]
+
+    Return a list of dictionaries containing per-generation statistics.
+    """
+    raise NotImplementedError()
+
+
+def get_threshold(*args, **kwargs):  # unknown args #
+    """
+    get_threshold() -> (threshold0, threshold1, threshold2)
+
+    Return the current collection thresholds
+    """
+    raise NotImplementedError()
+
+
+def is_tracked(*args, **kwargs):  # unknown args #
+    """
+    is_tracked(obj) -> bool
+
+    Returns true if the object is tracked by the garbage collector.
+    Simple atomic objects will return false.
+    """
+    raise NotImplementedError()
+
+
+def isenabled(*args, **kwargs):  # unknown args #
+    """
+    isenabled() -> status
+
+    Returns true if automatic garbage collection is enabled.
+    """
+    raise NotImplementedError()
+
+
+def set_debug(*args, **kwargs):  # unknown args #
+    """
+    set_debug(flags) -> None
+
+    Set the garbage collection debugging flags. Debugging information is
+    written to sys.stderr.
+
+    flags is an integer and can have the following bits turned on:
+
+      DEBUG_STATS - Print statistics during collection.
+      DEBUG_COLLECTABLE - Print collectable objects found.
+      DEBUG_UNCOLLECTABLE - Print unreachable but uncollectable objects found.
+      DEBUG_SAVEALL - Save objects to gc.garbage rather than freeing them.
+      DEBUG_LEAK - Debug leaking programs (everything but STATS).
+    """
+    raise NotImplementedError()
+
+
+def set_threshold(*args, **kwargs):  # unknown args #
+    """
+    set_threshold(threshold0, [threshold1, threshold2]) -> None
+
+    Sets the collection thresholds.  Setting threshold0 to zero disables
+    collection.
+    """
+    raise NotImplementedError()

Lib/genericpath.py

@@ -3,10 +3,7 @@ Path operations common to more than one OS
 Do not use directly.  The OS specific modules import the appropriate
 functions from this module themselves.
 """
-try:
-    import os
-except ImportError:
-    import _dummy_os as os
+import os
 import stat
 
 __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime',
@@ -95,11 +92,7 @@ def samestat(s1, s2):
 
 # Are two filenames really pointing to the same file?
 def samefile(f1, f2):
-    """Test whether two pathnames reference the same actual file or directory
-
-    This is determined by the device number and i-node number and
-    raises an exception if an os.stat() call on either pathname fails.
-    """
+    """Test whether two pathnames reference the same actual file"""
     s1 = os.stat(f1)
     s2 = os.stat(f2)
     return samestat(s1, s2)
@@ -152,7 +145,7 @@ def _check_arg_types(funcname, *args):
         elif isinstance(s, bytes):
             hasbytes = True
         else:
-            raise TypeError(f'{funcname}() argument must be str, bytes, or '
-                            f'os.PathLike object, not {s.__class__.__name__!r}') from None
+            raise TypeError('%s() argument must be str or bytes, not %r' %
+                            (funcname, s.__class__.__name__)) from None
     if hasstr and hasbytes:
         raise TypeError("Can't mix strings and bytes in path components") from None

Lib/getpass.py

@@ -7,6 +7,7 @@ GetPassWarning - This UserWarning is issued when getpass() cannot prevent
                  echoing of the password contents while reading.
 
 On Windows, the msvcrt module will be used.
+On the Mac EasyDialogs.AskPassword is used, if available.
 
 """
 
@@ -52,7 +53,7 @@ def unix_getpass(prompt='Password: ', stream=None):
             stack.enter_context(input)
             if not stream:
                 stream = input
-        except OSError:
+        except OSError as e:
             # If that fails, see if stdin can be controlled.
             stack.close()
             try:
@@ -95,7 +96,7 @@ def unix_getpass(prompt='Password: ', stream=None):
 
 
 def win_getpass(prompt='Password: ', stream=None):
-    """Prompt for password with echo off, using Windows getwch()."""
+    """Prompt for password with echo off, using Windows getch()."""
     if sys.stdin is not sys.__stdin__:
         return fallback_getpass(prompt, stream)
 

Lib/glob.py

@@ -1,16 +1,12 @@
 """Filename globbing utility."""
 
-import contextlib
 import os
 import re
 import fnmatch
-import itertools
-import stat
-import sys
 
 __all__ = ["glob", "iglob", "escape"]
 
-def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False):
+def glob(pathname, *, recursive=False):
     """Return a list of paths matching a pathname pattern.
 
     The pattern may contain simple shell-style wildcards a la
@@ -21,9 +17,9 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False):
     If recursive is true, the pattern '**' will match any files and
     zero or more directories and subdirectories.
     """
-    return list(iglob(pathname, root_dir=root_dir, dir_fd=dir_fd, recursive=recursive))
+    return list(iglob(pathname, recursive=recursive))
 
-def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False):
+def iglob(pathname, *, recursive=False):
     """Return an iterator which yields the paths matching a pathname pattern.
 
     The pattern may contain simple shell-style wildcards a la
@@ -34,45 +30,35 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False):
     If recursive is true, the pattern '**' will match any files and
     zero or more directories and subdirectories.
     """
-    sys.audit("glob.glob", pathname, recursive)
-    sys.audit("glob.glob/2", pathname, recursive, root_dir, dir_fd)
-    if root_dir is not None:
-        root_dir = os.fspath(root_dir)
-    else:
-        root_dir = pathname[:0]
-    it = _iglob(pathname, root_dir, dir_fd, recursive, False)
-    if not pathname or recursive and _isrecursive(pathname[:2]):
-        try:
-            s = next(it)  # skip empty string
-            if s:
-                it = itertools.chain((s,), it)
-        except StopIteration:
-            pass
+    it = _iglob(pathname, recursive, False)
+    if recursive and _isrecursive(pathname):
+        s = next(it)  # skip empty string
+        assert not s
     return it
 
-def _iglob(pathname, root_dir, dir_fd, recursive, dironly):
+def _iglob(pathname, recursive, dironly):
     dirname, basename = os.path.split(pathname)
     if not has_magic(pathname):
         assert not dironly
         if basename:
-            if _lexists(_join(root_dir, pathname), dir_fd):
+            if os.path.lexists(pathname):
                 yield pathname
         else:
             # Patterns ending with a slash should match only directories
-            if _isdir(_join(root_dir, dirname), dir_fd):
+            if os.path.isdir(dirname):
                 yield pathname
         return
     if not dirname:
         if recursive and _isrecursive(basename):
-            yield from _glob2(root_dir, basename, dir_fd, dironly)
+            yield from _glob2(dirname, basename, dironly)
         else:
-            yield from _glob1(root_dir, basename, dir_fd, dironly)
+            yield from _glob1(dirname, basename, dironly)
         return
     # `os.path.split()` returns the argument itself as a dirname if it is a
     # drive or UNC path.  Prevent an infinite recursion if a drive or UNC path
     # contains magic characters (i.e. r'\\?\C:').
     if dirname != pathname and has_magic(dirname):
-        dirs = _iglob(dirname, root_dir, dir_fd, recursive, True)
+        dirs = _iglob(dirname, recursive, True)
     else:
         dirs = [dirname]
     if has_magic(basename):
@@ -83,125 +69,76 @@ def _iglob(pathname, root_dir, dir_fd, recursive, dironly):
     else:
         glob_in_dir = _glob0
     for dirname in dirs:
-        for name in glob_in_dir(_join(root_dir, dirname), basename, dir_fd, dironly):
+        for name in glob_in_dir(dirname, basename, dironly):
             yield os.path.join(dirname, name)
 
 # These 2 helper functions non-recursively glob inside a literal directory.
 # They return a list of basenames.  _glob1 accepts a pattern while _glob0
 # takes a literal basename (so it only has to check for its existence).
 
-def _glob1(dirname, pattern, dir_fd, dironly):
-    names = _listdir(dirname, dir_fd, dironly)
+def _glob1(dirname, pattern, dironly):
+    names = list(_iterdir(dirname, dironly))
     if not _ishidden(pattern):
         names = (x for x in names if not _ishidden(x))
     return fnmatch.filter(names, pattern)
 
-def _glob0(dirname, basename, dir_fd, dironly):
-    if basename:
-        if _lexists(_join(dirname, basename), dir_fd):
-            return [basename]
-    else:
+def _glob0(dirname, basename, dironly):
+    if not basename:
         # `os.path.split()` returns an empty basename for paths ending with a
         # directory separator.  'q*x/' should match only directories.
-        if _isdir(dirname, dir_fd):
+        if os.path.isdir(dirname):
+            return [basename]
+    else:
+        if os.path.lexists(os.path.join(dirname, basename)):
             return [basename]
     return []
 
 # Following functions are not public but can be used by third-party code.
 
 def glob0(dirname, pattern):
-    return _glob0(dirname, pattern, None, False)
+    return _glob0(dirname, pattern, False)
 
 def glob1(dirname, pattern):
-    return _glob1(dirname, pattern, None, False)
+    return _glob1(dirname, pattern, False)
 
 # This helper function recursively yields relative pathnames inside a literal
 # directory.
 
-def _glob2(dirname, pattern, dir_fd, dironly):
+def _glob2(dirname, pattern, dironly):
     assert _isrecursive(pattern)
     yield pattern[:0]
-    yield from _rlistdir(dirname, dir_fd, dironly)
+    yield from _rlistdir(dirname, dironly)
 
 # If dironly is false, yields all file names inside a directory.
 # If dironly is true, yields only directory names.
-def _iterdir(dirname, dir_fd, dironly):
-    try:
-        fd = None
-        fsencode = None
-        if dir_fd is not None:
-            if dirname:
-                fd = arg = os.open(dirname, _dir_open_flags, dir_fd=dir_fd)
-            else:
-                arg = dir_fd
-            if isinstance(dirname, bytes):
-                fsencode = os.fsencode
-        elif dirname:
-            arg = dirname
-        elif isinstance(dirname, bytes):
-            arg = bytes(os.curdir, 'ASCII')
+def _iterdir(dirname, dironly):
+    if not dirname:
+        if isinstance(dirname, bytes):
+            dirname = bytes(os.curdir, 'ASCII')
         else:
-            arg = os.curdir
-        try:
-            with os.scandir(arg) as it:
-                for entry in it:
-                    try:
-                        if not dironly or entry.is_dir():
-                            if fsencode is not None:
-                                yield fsencode(entry.name)
-                            else:
-                                yield entry.name
-                    except OSError:
-                        pass
-        finally:
-            if fd is not None:
-                os.close(fd)
+            dirname = os.curdir
+    try:
+        with os.scandir(dirname) as it:
+            for entry in it:
+                try:
+                    if not dironly or entry.is_dir():
+                        yield entry.name
+                except OSError:
+                    pass
     except OSError:
         return
 
-def _listdir(dirname, dir_fd, dironly):
-    with contextlib.closing(_iterdir(dirname, dir_fd, dironly)) as it:
-        return list(it)
-
 # Recursively yields relative pathnames inside a literal directory.
-def _rlistdir(dirname, dir_fd, dironly):
-    names = _listdir(dirname, dir_fd, dironly)
+def _rlistdir(dirname, dironly):
+    names = list(_iterdir(dirname, dironly))
     for x in names:
         if not _ishidden(x):
             yield x
-            path = _join(dirname, x) if dirname else x
-            for y in _rlistdir(path, dir_fd, dironly):
-                yield _join(x, y)
+            path = os.path.join(dirname, x) if dirname else x
+            for y in _rlistdir(path, dironly):
+                yield os.path.join(x, y)
 
 
-def _lexists(pathname, dir_fd):
-    # Same as os.path.lexists(), but with dir_fd
-    if dir_fd is None:
-        return os.path.lexists(pathname)
-    try:
-        os.lstat(pathname, dir_fd=dir_fd)
-    except (OSError, ValueError):
-        return False
-    else:
-        return True
-
-def _isdir(pathname, dir_fd):
-    # Same as os.path.isdir(), but with dir_fd
-    if dir_fd is None:
-        return os.path.isdir(pathname)
-    try:
-        st = os.stat(pathname, dir_fd=dir_fd)
-    except (OSError, ValueError):
-        return False
-    else:
-        return stat.S_ISDIR(st.st_mode)
-
-def _join(dirname, basename):
-    # It is common if dirname or basename is empty
-    if not dirname or not basename:
-        return dirname or basename
-    return os.path.join(dirname, basename)
-
 magic_check = re.compile('([*?[])')
 magic_check_bytes = re.compile(b'([*?[])')
 
@@ -232,6 +169,3 @@ def escape(pathname):
     else:
         pathname = magic_check.sub(r'[\1]', pathname)
     return drive + pathname
-
-
-_dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0)

Lib/gzip.py

@@ -1,609 +0,0 @@
-"""Functions that read and write gzipped files.
-
-The user of the file doesn't have to worry about the compression,
-but random access is not allowed."""
-
-# based on Andrew Kuchling's minigzip.py distributed with the zlib module
-
-import struct, sys, time, os
-import zlib
-import builtins
-import io
-import _compression
-
-__all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"]
-
-FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
-
-READ, WRITE = 1, 2
-
-_COMPRESS_LEVEL_FAST = 1
-_COMPRESS_LEVEL_TRADEOFF = 6
-_COMPRESS_LEVEL_BEST = 9
-
-
-def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST,
-         encoding=None, errors=None, newline=None):
-    """Open a gzip-compressed file in binary or text mode.
-
-    The filename argument can be an actual filename (a str or bytes object), or
-    an existing file object to read from or write to.
-
-    The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
-    binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
-    "rb", and the default compresslevel is 9.
-
-    For binary mode, this function is equivalent to the GzipFile constructor:
-    GzipFile(filename, mode, compresslevel). In this case, the encoding, errors
-    and newline arguments must not be provided.
-
-    For text mode, a GzipFile object is created, and wrapped in an
-    io.TextIOWrapper instance with the specified encoding, error handling
-    behavior, and line ending(s).
-
-    """
-    if "t" in mode:
-        if "b" in mode:
-            raise ValueError("Invalid mode: %r" % (mode,))
-    else:
-        if encoding is not None:
-            raise ValueError("Argument 'encoding' not supported in binary mode")
-        if errors is not None:
-            raise ValueError("Argument 'errors' not supported in binary mode")
-        if newline is not None:
-            raise ValueError("Argument 'newline' not supported in binary mode")
-
-    gz_mode = mode.replace("t", "")
-    if isinstance(filename, (str, bytes, os.PathLike)):
-        binary_file = GzipFile(filename, gz_mode, compresslevel)
-    elif hasattr(filename, "read") or hasattr(filename, "write"):
-        binary_file = GzipFile(None, gz_mode, compresslevel, filename)
-    else:
-        raise TypeError("filename must be a str or bytes object, or a file")
-
-    if "t" in mode:
-        encoding = io.text_encoding(encoding)
-        return io.TextIOWrapper(binary_file, encoding, errors, newline)
-    else:
-        return binary_file
-
-def write32u(output, value):
-    # The L format writes the bit pattern correctly whether signed
-    # or unsigned.
-    output.write(struct.pack("<L", value))
-
-class _PaddedFile:
-    """Minimal read-only file object that prepends a string to the contents
-    of an actual file. Shouldn't be used outside of gzip.py, as it lacks
-    essential functionality."""
-
-    def __init__(self, f, prepend=b''):
-        self._buffer = prepend
-        self._length = len(prepend)
-        self.file = f
-        self._read = 0
-
-    def read(self, size):
-        if self._read is None:
-            return self.file.read(size)
-        if self._read + size <= self._length:
-            read = self._read
-            self._read += size
-            return self._buffer[read:self._read]
-        else:
-            read = self._read
-            self._read = None
-            return self._buffer[read:] + \
-                   self.file.read(size-self._length+read)
-
-    def prepend(self, prepend=b''):
-        if self._read is None:
-            self._buffer = prepend
-        else:  # Assume data was read since the last prepend() call
-            self._read -= len(prepend)
-            return
-        self._length = len(self._buffer)
-        self._read = 0
-
-    def seek(self, off):
-        self._read = None
-        self._buffer = None
-        return self.file.seek(off)
-
-    def seekable(self):
-        return True  # Allows fast-forwarding even in unseekable streams
-
-
-class BadGzipFile(OSError):
-    """Exception raised in some cases for invalid gzip files."""
-
-
-class GzipFile(_compression.BaseStream):
-    """The GzipFile class simulates most of the methods of a file object with
-    the exception of the truncate() method.
-
-    This class only supports opening files in binary mode. If you need to open a
-    compressed file in text mode, use the gzip.open() function.
-
-    """
-
-    # Overridden with internal file object to be closed, if only a filename
-    # is passed in
-    myfileobj = None
-
-    def __init__(self, filename=None, mode=None,
-                 compresslevel=_COMPRESS_LEVEL_BEST, fileobj=None, mtime=None):
-        """Constructor for the GzipFile class.
-
-        At least one of fileobj and filename must be given a
-        non-trivial value.
-
-        The new class instance is based on fileobj, which can be a regular
-        file, an io.BytesIO object, or any other object which simulates a file.
-        It defaults to None, in which case filename is opened to provide
-        a file object.
-
-        When fileobj is not None, the filename argument is only used to be
-        included in the gzip file header, which may include the original
-        filename of the uncompressed file.  It defaults to the filename of
-        fileobj, if discernible; otherwise, it defaults to the empty string,
-        and in this case the original filename is not included in the header.
-
-        The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', 'wb', 'x', or
-        'xb' depending on whether the file will be read or written.  The default
-        is the mode of fileobj if discernible; otherwise, the default is 'rb'.
-        A mode of 'r' is equivalent to one of 'rb', and similarly for 'w' and
-        'wb', 'a' and 'ab', and 'x' and 'xb'.
-
-        The compresslevel argument is an integer from 0 to 9 controlling the
-        level of compression; 1 is fastest and produces the least compression,
-        and 9 is slowest and produces the most compression. 0 is no compression
-        at all. The default is 9.
-
-        The mtime argument is an optional numeric timestamp to be written
-        to the last modification time field in the stream when compressing.
-        If omitted or None, the current time is used.
-
-        """
-
-        if mode and ('t' in mode or 'U' in mode):
-            raise ValueError("Invalid mode: {!r}".format(mode))
-        if mode and 'b' not in mode:
-            mode += 'b'
-        if fileobj is None:
-            fileobj = self.myfileobj = builtins.open(filename, mode or 'rb')
-        if filename is None:
-            filename = getattr(fileobj, 'name', '')
-            if not isinstance(filename, (str, bytes)):
-                filename = ''
-        else:
-            filename = os.fspath(filename)
-        origmode = mode
-        if mode is None:
-            mode = getattr(fileobj, 'mode', 'rb')
-
-        if mode.startswith('r'):
-            self.mode = READ
-            raw = _GzipReader(fileobj)
-            self._buffer = io.BufferedReader(raw)
-            self.name = filename
-
-        elif mode.startswith(('w', 'a', 'x')):
-            if origmode is None:
-                import warnings
-                warnings.warn(
-                    "GzipFile was opened for writing, but this will "
-                    "change in future Python releases.  "
-                    "Specify the mode argument for opening it for writing.",
-                    FutureWarning, 2)
-            self.mode = WRITE
-            self._init_write(filename)
-            self.compress = zlib.compressobj(compresslevel,
-                                             zlib.DEFLATED,
-                                             -zlib.MAX_WBITS,
-                                             zlib.DEF_MEM_LEVEL,
-                                             0)
-            self._write_mtime = mtime
-        else:
-            raise ValueError("Invalid mode: {!r}".format(mode))
-
-        self.fileobj = fileobj
-
-        if self.mode == WRITE:
-            self._write_gzip_header(compresslevel)
-
-    @property
-    def filename(self):
-        import warnings
-        warnings.warn("use the name attribute", DeprecationWarning, 2)
-        if self.mode == WRITE and self.name[-3:] != ".gz":
-            return self.name + ".gz"
-        return self.name
-
-    @property
-    def mtime(self):
-        """Last modification time read from stream, or None"""
-        return self._buffer.raw._last_mtime
-
-    def __repr__(self):
-        s = repr(self.fileobj)
-        return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>'
-
-    def _init_write(self, filename):
-        self.name = filename
-        self.crc = zlib.crc32(b"")
-        self.size = 0
-        self.writebuf = []
-        self.bufsize = 0
-        self.offset = 0  # Current file offset for seek(), tell(), etc
-
-    def _write_gzip_header(self, compresslevel):
-        self.fileobj.write(b'\037\213')             # magic header
-        self.fileobj.write(b'\010')                 # compression method
-        try:
-            # RFC 1952 requires the FNAME field to be Latin-1. Do not
-            # include filenames that cannot be represented that way.
-            fname = os.path.basename(self.name)
-            if not isinstance(fname, bytes):
-                fname = fname.encode('latin-1')
-            if fname.endswith(b'.gz'):
-                fname = fname[:-3]
-        except UnicodeEncodeError:
-            fname = b''
-        flags = 0
-        if fname:
-            flags = FNAME
-        self.fileobj.write(chr(flags).encode('latin-1'))
-        mtime = self._write_mtime
-        if mtime is None:
-            mtime = time.time()
-        write32u(self.fileobj, int(mtime))
-        if compresslevel == _COMPRESS_LEVEL_BEST:
-            xfl = b'\002'
-        elif compresslevel == _COMPRESS_LEVEL_FAST:
-            xfl = b'\004'
-        else:
-            xfl = b'\000'
-        self.fileobj.write(xfl)
-        self.fileobj.write(b'\377')
-        if fname:
-            self.fileobj.write(fname + b'\000')
-
-    def write(self,data):
-        self._check_not_closed()
-        if self.mode != WRITE:
-            import errno
-            raise OSError(errno.EBADF, "write() on read-only GzipFile object")
-
-        if self.fileobj is None:
-            raise ValueError("write() on closed GzipFile object")
-
-        if isinstance(data, (bytes, bytearray)):
-            length = len(data)
-        else:
-            # accept any data that supports the buffer protocol
-            data = memoryview(data)
-            length = data.nbytes
-
-        if length > 0:
-            self.fileobj.write(self.compress.compress(data))
-            self.size += length
-            self.crc = zlib.crc32(data, self.crc)
-            self.offset += length
-
-        return length
-
-    def read(self, size=-1):
-        self._check_not_closed()
-        if self.mode != READ:
-            import errno
-            raise OSError(errno.EBADF, "read() on write-only GzipFile object")
-        return self._buffer.read(size)
-
-    def read1(self, size=-1):
-        """Implements BufferedIOBase.read1()
-
-        Reads up to a buffer's worth of data if size is negative."""
-        self._check_not_closed()
-        if self.mode != READ:
-            import errno
-            raise OSError(errno.EBADF, "read1() on write-only GzipFile object")
-
-        if size < 0:
-            size = io.DEFAULT_BUFFER_SIZE
-        return self._buffer.read1(size)
-
-    def peek(self, n):
-        self._check_not_closed()
-        if self.mode != READ:
-            import errno
-            raise OSError(errno.EBADF, "peek() on write-only GzipFile object")
-        return self._buffer.peek(n)
-
-    @property
-    def closed(self):
-        return self.fileobj is None
-
-    def close(self):
-        fileobj = self.fileobj
-        if fileobj is None:
-            return
-        self.fileobj = None
-        try:
-            if self.mode == WRITE:
-                fileobj.write(self.compress.flush())
-                write32u(fileobj, self.crc)
-                # self.size may exceed 2 GiB, or even 4 GiB
-                write32u(fileobj, self.size & 0xffffffff)
-            elif self.mode == READ:
-                self._buffer.close()
-        finally:
-            myfileobj = self.myfileobj
-            if myfileobj:
-                self.myfileobj = None
-                myfileobj.close()
-
-    def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
-        self._check_not_closed()
-        if self.mode == WRITE:
-            # Ensure the compressor's buffer is flushed
-            self.fileobj.write(self.compress.flush(zlib_mode))
-            self.fileobj.flush()
-
-    def fileno(self):
-        """Invoke the underlying file object's fileno() method.
-
-        This will raise AttributeError if the underlying file object
-        doesn't support fileno().
-        """
-        return self.fileobj.fileno()
-
-    def rewind(self):
-        '''Return the uncompressed stream file position indicator to the
-        beginning of the file'''
-        if self.mode != READ:
-            raise OSError("Can't rewind in write mode")
-        self._buffer.seek(0)
-
-    def readable(self):
-        return self.mode == READ
-
-    def writable(self):
-        return self.mode == WRITE
-
-    def seekable(self):
-        return True
-
-    def seek(self, offset, whence=io.SEEK_SET):
-        if self.mode == WRITE:
-            if whence != io.SEEK_SET:
-                if whence == io.SEEK_CUR:
-                    offset = self.offset + offset
-                else:
-                    raise ValueError('Seek from end not supported')
-            if offset < self.offset:
-                raise OSError('Negative seek in write mode')
-            count = offset - self.offset
-            chunk = b'\0' * 1024
-            for i in range(count // 1024):
-                self.write(chunk)
-            self.write(b'\0' * (count % 1024))
-        elif self.mode == READ:
-            self._check_not_closed()
-            return self._buffer.seek(offset, whence)
-
-        return self.offset
-
-    def readline(self, size=-1):
-        self._check_not_closed()
-        return self._buffer.readline(size)
-
-
-class _GzipReader(_compression.DecompressReader):
-    def __init__(self, fp):
-        super().__init__(_PaddedFile(fp), zlib.decompressobj,
-                         wbits=-zlib.MAX_WBITS)
-        # Set flag indicating start of a new member
-        self._new_member = True
-        self._last_mtime = None
-
-    def _init_read(self):
-        self._crc = zlib.crc32(b"")
-        self._stream_size = 0  # Decompressed size of unconcatenated stream
-
-    def _read_exact(self, n):
-        '''Read exactly *n* bytes from `self._fp`
-
-        This method is required because self._fp may be unbuffered,
-        i.e. return short reads.
-        '''
-
-        data = self._fp.read(n)
-        while len(data) < n:
-            b = self._fp.read(n - len(data))
-            if not b:
-                raise EOFError("Compressed file ended before the "
-                               "end-of-stream marker was reached")
-            data += b
-        return data
-
-    def _read_gzip_header(self):
-        magic = self._fp.read(2)
-        if magic == b'':
-            return False
-
-        if magic != b'\037\213':
-            raise BadGzipFile('Not a gzipped file (%r)' % magic)
-
-        (method, flag,
-         self._last_mtime) = struct.unpack("<BBIxx", self._read_exact(8))
-        if method != 8:
-            raise BadGzipFile('Unknown compression method')
-
-        if flag & FEXTRA:
-            # Read & discard the extra field, if present
-            extra_len, = struct.unpack("<H", self._read_exact(2))
-            self._read_exact(extra_len)
-        if flag & FNAME:
-            # Read and discard a null-terminated string containing the filename
-            while True:
-                s = self._fp.read(1)
-                if not s or s==b'\000':
-                    break
-        if flag & FCOMMENT:
-            # Read and discard a null-terminated string containing a comment
-            while True:
-                s = self._fp.read(1)
-                if not s or s==b'\000':
-                    break
-        if flag & FHCRC:
-            self._read_exact(2)     # Read & discard the 16-bit header CRC
-        return True
-
-    def read(self, size=-1):
-        if size < 0:
-            return self.readall()
-        # size=0 is special because decompress(max_length=0) is not supported
-        if not size:
-            return b""
-
-        # For certain input data, a single
-        # call to decompress() may not return
-        # any data. In this case, retry until we get some data or reach EOF.
-        while True:
-            if self._decompressor.eof:
-                # Ending case: we've come to the end of a member in the file,
-                # so finish up this member, and read a new gzip header.
-                # Check the CRC and file size, and set the flag so we read
-                # a new member
-                self._read_eof()
-                self._new_member = True
-                self._decompressor = self._decomp_factory(
-                    **self._decomp_args)
-
-            if self._new_member:
-                # If the _new_member flag is set, we have to
-                # jump to the next member, if there is one.
-                self._init_read()
-                if not self._read_gzip_header():
-                    self._size = self._pos
-                    return b""
-                self._new_member = False
-
-            # Read a chunk of data from the file
-            buf = self._fp.read(io.DEFAULT_BUFFER_SIZE)
-
-            uncompress = self._decompressor.decompress(buf, size)
-            if self._decompressor.unconsumed_tail != b"":
-                self._fp.prepend(self._decompressor.unconsumed_tail)
-            elif self._decompressor.unused_data != b"":
-                # Prepend the already read bytes to the fileobj so they can
-                # be seen by _read_eof() and _read_gzip_header()
-                self._fp.prepend(self._decompressor.unused_data)
-
-            if uncompress != b"":
-                break
-            if buf == b"":
-                raise EOFError("Compressed file ended before the "
-                               "end-of-stream marker was reached")
-
-        self._add_read_data( uncompress )
-        self._pos += len(uncompress)
-        return uncompress
-
-    def _add_read_data(self, data):
-        self._crc = zlib.crc32(data, self._crc)
-        self._stream_size = self._stream_size + len(data)
-
-    def _read_eof(self):
-        # We've read to the end of the file
-        # We check that the computed CRC and size of the
-        # uncompressed data matches the stored values.  Note that the size
-        # stored is the true file size mod 2**32.
-        crc32, isize = struct.unpack("<II", self._read_exact(8))
-        if crc32 != self._crc:
-            raise BadGzipFile("CRC check failed %s != %s" % (hex(crc32),
-                                                             hex(self._crc)))
-        elif isize != (self._stream_size & 0xffffffff):
-            raise BadGzipFile("Incorrect length of data produced")
-
-        # Gzip files can be padded with zeroes and still have archives.
-        # Consume all zero bytes and set the file position to the first
-        # non-zero byte. See http://www.gzip.org/#faq8
-        c = b"\x00"
-        while c == b"\x00":
-            c = self._fp.read(1)
-        if c:
-            self._fp.prepend(c)
-
-    def _rewind(self):
-        super()._rewind()
-        self._new_member = True
-
-def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
-    """Compress data in one shot and return the compressed string.
-    Optional argument is the compression level, in range of 0-9.
-    """
-    buf = io.BytesIO()
-    with GzipFile(fileobj=buf, mode='wb', compresslevel=compresslevel, mtime=mtime) as f:
-        f.write(data)
-    return buf.getvalue()
-
-def decompress(data):
-    """Decompress a gzip compressed string in one shot.
-    Return the decompressed string.
-    """
-    with GzipFile(fileobj=io.BytesIO(data)) as f:
-        return f.read()
-
-
-def main():
-    from argparse import ArgumentParser
-    parser = ArgumentParser(description=
-        "A simple command line interface for the gzip module: act like gzip, "
-        "but do not delete the input file.")
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument('--fast', action='store_true', help='compress faster')
-    group.add_argument('--best', action='store_true', help='compress better')
-    group.add_argument("-d", "--decompress", action="store_true",
-                        help="act like gunzip instead of gzip")
-
-    parser.add_argument("args", nargs="*", default=["-"], metavar='file')
-    args = parser.parse_args()
-
-    compresslevel = _COMPRESS_LEVEL_TRADEOFF
-    if args.fast:
-        compresslevel = _COMPRESS_LEVEL_FAST
-    elif args.best:
-        compresslevel = _COMPRESS_LEVEL_BEST
-
-    for arg in args.args:
-        if args.decompress:
-            if arg == "-":
-                f = GzipFile(filename="", mode="rb", fileobj=sys.stdin.buffer)
-                g = sys.stdout.buffer
-            else:
-                if arg[-3:] != ".gz":
-                    sys.exit(f"filename doesn't end in .gz: {arg!r}")
-                f = open(arg, "rb")
-                g = builtins.open(arg[:-3], "wb")
-        else:
-            if arg == "-":
-                f = sys.stdin.buffer
-                g = GzipFile(filename="", mode="wb", fileobj=sys.stdout.buffer,
-                             compresslevel=compresslevel)
-            else:
-                f = builtins.open(arg, "rb")
-                g = open(arg + ".gz", "wb")
-        while True:
-            chunk = f.read(io.DEFAULT_BUFFER_SIZE)
-            if not chunk:
-                break
-            g.write(chunk)
-        if g is not sys.stdout.buffer:
-            g.close()
-        if f is not sys.stdin.buffer:
-            f.close()
-
-if __name__ == '__main__':
-    main()

Lib/heapq.py

@@ -597,5 +597,5 @@ except ImportError:
 
 if __name__ == "__main__":
 
-    import doctest # pragma: no cover
-    print(doctest.testmod()) # pragma: no cover
+    import doctest
+    print(doctest.testmod())

Lib/html/entities.py

@@ -4,7 +4,6 @@ __all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs']
 
 
 # maps the HTML entity name to the Unicode code point
-# from https://html.spec.whatwg.org/multipage/named-characters.html
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1

Lib/html/parser.py

@@ -9,6 +9,7 @@
 
 
 import re
+import warnings
 import _markupbase
 
 from html import unescape
@@ -46,7 +47,7 @@ locatestarttagend_tolerant = re.compile(r"""
           |"[^"]*"                   # LIT-enclosed value
           |(?!['"])[^>\s]*           # bare value
          )
-        \s*                          # possibly followed by a space
+         (?:\s*,)*                   # possibly followed by a comma
        )?(?:\s|/(?!>))*
      )*
    )?
@@ -405,7 +406,7 @@ class HTMLParser(_markupbase.ParserBase):
             tagname = namematch.group(1).lower()
             # consume and ignore other stuff between the name and the >
             # Note: this is not 100% correct, since we might have things like
-            # </tag attr=">">, but looking for > after the name should cover
+            # </tag attr=">">, but looking for > after tha name should cover
             # most of the cases and is much simpler
             gtpos = rawdata.find('>', namematch.end())
             self.handle_endtag(tagname)
@@ -417,7 +418,7 @@ class HTMLParser(_markupbase.ParserBase):
                 self.handle_data(rawdata[i:gtpos])
                 return gtpos
 
-        self.handle_endtag(elem)
+        self.handle_endtag(elem.lower())
         self.clear_cdata_mode()
         return gtpos
 
@@ -460,3 +461,10 @@ class HTMLParser(_markupbase.ParserBase):
 
     def unknown_decl(self, data):
         pass
+
+    # Internal -- helper to remove special character quoting
+    def unescape(self, s):
+        warnings.warn('The unescape method is deprecated and will be removed '
+                      'in 3.5, use html.unescape() instead.',
+                      DeprecationWarning, stacklevel=2)
+        return unescape(s)

Lib/http/__init__.py

@@ -2,7 +2,6 @@ from enum import IntEnum
 
 __all__ = ['HTTPStatus']
 
-
 class HTTPStatus(IntEnum):
     """HTTP status codes and reason phrases
 
@@ -16,11 +15,6 @@ class HTTPStatus(IntEnum):
         * RFC 7238: Permanent Redirect
         * RFC 2295: Transparent Content Negotiation in HTTP
         * RFC 2774: An HTTP Extension Framework
-        * RFC 7725: An HTTP Status Code to Report Legal Obstacles
-        * RFC 7540: Hypertext Transfer Protocol Version 2 (HTTP/2)
-        * RFC 2324: Hyper Text Coffee Pot Control Protocol (HTCPCP/1.0)
-        * RFC 8297: An HTTP Status Code for Indicating Hints
-        * RFC 8470: Using Early Data in HTTP
     """
     def __new__(cls, value, phrase, description=''):
         obj = int.__new__(cls, value)
@@ -35,7 +29,6 @@ class HTTPStatus(IntEnum):
     SWITCHING_PROTOCOLS = (101, 'Switching Protocols',
             'Switching to new protocol; obey Upgrade header')
     PROCESSING = 102, 'Processing'
-    EARLY_HINTS = 103, 'Early Hints'
 
     # success
     OK = 200, 'OK', 'Request fulfilled, document follows'
@@ -65,7 +58,7 @@ class HTTPStatus(IntEnum):
     TEMPORARY_REDIRECT = (307, 'Temporary Redirect',
         'Object moved temporarily -- see URI list')
     PERMANENT_REDIRECT = (308, 'Permanent Redirect',
-        'Object moved permanently -- see URI list')
+        'Object moved temporarily -- see URI list')
 
     # client error
     BAD_REQUEST = (400, 'Bad Request',
@@ -105,14 +98,9 @@ class HTTPStatus(IntEnum):
         'Cannot satisfy request range')
     EXPECTATION_FAILED = (417, 'Expectation Failed',
         'Expect condition could not be satisfied')
-    IM_A_TEAPOT = (418, 'I\'m a Teapot',
-        'Server refuses to brew coffee because it is a teapot.')
-    MISDIRECTED_REQUEST = (421, 'Misdirected Request',
-        'Server is not able to produce a response')
     UNPROCESSABLE_ENTITY = 422, 'Unprocessable Entity'
     LOCKED = 423, 'Locked'
     FAILED_DEPENDENCY = 424, 'Failed Dependency'
-    TOO_EARLY = 425, 'Too Early'
     UPGRADE_REQUIRED = 426, 'Upgrade Required'
     PRECONDITION_REQUIRED = (428, 'Precondition Required',
         'The origin server requires the request to be conditional')
@@ -123,10 +111,6 @@ class HTTPStatus(IntEnum):
         'Request Header Fields Too Large',
         'The server is unwilling to process the request because its header '
         'fields are too large')
-    UNAVAILABLE_FOR_LEGAL_REASONS = (451,
-        'Unavailable For Legal Reasons',
-        'The server is denying access to the '
-        'resource as a consequence of a legal demand')
 
     # server errors
     INTERNAL_SERVER_ERROR = (500, 'Internal Server Error',

Lib/http/client.py

@@ -70,13 +70,12 @@ Req-sent-unread-response       _CS_REQ_SENT       <response_class>
 
 import email.parser
 import email.message
-import errno
 import http
 import io
+import os
 import re
 import socket
-import sys
-import collections.abc
+import collections
 from urllib.parse import urlsplit
 
 # HTTPMessage, parse_headers(), and the HTTP status code constants are
@@ -107,6 +106,9 @@ globals().update(http.HTTPStatus.__members__)
 # Mapping status codes to official W3C names
 responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()}
 
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
 # maximal line length when calling readline().
 _MAXLINE = 65536
 _MAXHEADERS = 100
@@ -139,20 +141,6 @@ _MAXHEADERS = 100
 _is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch
 _is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search
 
-# These characters are not allowed within HTTP URL paths.
-#  See https://tools.ietf.org/html/rfc3986#section-3.3 and the
-#  https://tools.ietf.org/html/rfc3986#appendix-A pchar definition.
-# Prevents CVE-2019-9740.  Includes control characters such as \r\n.
-# We don't restrict chars above \x7f as putrequest() limits us to ASCII.
-_contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')
-# Arguably only these _should_ allowed:
-#  _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
-# We are more lenient for assumed real world compatibility purposes.
-
-# These characters are not allowed within HTTP method names
-# to prevent http header injection.
-_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]')
-
 # We always set the Content-Length header for these methods because some
 # servers will otherwise respond with a 411
 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
@@ -203,11 +191,15 @@ class HTTPMessage(email.message.Message):
                 lst.append(line)
         return lst
 
-def _read_headers(fp):
-    """Reads potential header lines into a list from a file pointer.
+def parse_headers(fp, _class=HTTPMessage):
+    """Parses only RFC2822 headers from a file pointer.
+
+    email Parser wants to see strings rather than bytes.
+    But a TextIOWrapper around self.rfile would buffer too many bytes
+    from the stream, bytes which we later need to read as bytes.
+    So we read the correct bytes here, as bytes, for email Parser
+    to parse.
 
-    Length of line is limited by _MAXLINE, and number of
-    headers is limited by _MAXHEADERS.
     """
     headers = []
     while True:
@@ -219,19 +211,6 @@ def _read_headers(fp):
             raise HTTPException("got more than %d headers" % _MAXHEADERS)
         if line in (b'\r\n', b'\n', b''):
             break
-    return headers
-
-def parse_headers(fp, _class=HTTPMessage):
-    """Parses only RFC2822 headers from a file pointer.
-
-    email Parser wants to see strings rather than bytes.
-    But a TextIOWrapper around self.rfile would buffer too many bytes
-    from the stream, bytes which we later need to read as bytes.
-    So we read the correct bytes here, as bytes, for email Parser
-    to parse.
-
-    """
-    headers = _read_headers(fp)
     hstring = b''.join(headers).decode('iso-8859-1')
     return email.parser.Parser(_class=_class).parsestr(hstring)
 
@@ -319,10 +298,15 @@ class HTTPResponse(io.BufferedIOBase):
             if status != CONTINUE:
                 break
             # skip the header from the 100 response
-            skipped_headers = _read_headers(self.fp)
-            if self.debuglevel > 0:
-                print("headers:", skipped_headers)
-            del skipped_headers
+            while True:
+                skip = self.fp.readline(_MAXLINE + 1)
+                if len(skip) > _MAXLINE:
+                    raise LineTooLong("header line")
+                skip = skip.strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print("header:", skip)
 
         self.code = self.status = status
         self.reason = reason.strip()
@@ -337,8 +321,8 @@ class HTTPResponse(io.BufferedIOBase):
         self.headers = self.msg = parse_headers(self.fp)
 
         if self.debuglevel > 0:
-            for hdr, val in self.headers.items():
-                print("header:", hdr + ":", val)
+            for hdr in self.headers:
+                print("header:", hdr, end=" ")
 
         # are we using the chunked-style of transfer encoding?
         tr_enc = self.headers.get("transfer-encoding")
@@ -355,6 +339,9 @@ class HTTPResponse(io.BufferedIOBase):
         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
         self.length = None
         length = self.headers.get("content-length")
+
+         # are we using the chunked-style of transfer encoding?
+        tr_enc = self.headers.get("transfer-encoding")
         if length and not self.chunked:
             try:
                 self.length = int(length)
@@ -385,6 +372,7 @@ class HTTPResponse(io.BufferedIOBase):
         if self.version == 11:
             # An HTTP/1.1 proxy is assumed to stay open unless
             # explicitly closed.
+            conn = self.headers.get("connection")
             if conn and "close" in conn.lower():
                 return True
             return False
@@ -455,25 +443,18 @@ class HTTPResponse(io.BufferedIOBase):
             self._close_conn()
             return b""
 
-        if self.chunked:
-            return self._read_chunked(amt)
-
         if amt is not None:
-            if self.length is not None and amt > self.length:
-                # clip the read to the "end of response"
-                amt = self.length
-            s = self.fp.read(amt)
-            if not s and amt:
-                # Ideally, we would raise IncompleteRead if the content-length
-                # wasn't satisfied, but it might break compatibility.
-                self._close_conn()
-            elif self.length is not None:
-                self.length -= len(s)
-                if not self.length:
-                    self._close_conn()
-            return s
+            # Amount is given, implement using readinto
+            b = bytearray(amt)
+            n = self.readinto(b)
+            return memoryview(b)[:n].tobytes()
         else:
             # Amount is not given (unbounded read) so we must check self.length
+            # and self.chunked
+
+            if self.chunked:
+                return self._readall_chunked()
+
             if self.length is None:
                 s = self.fp.read()
             else:
@@ -559,7 +540,7 @@ class HTTPResponse(io.BufferedIOBase):
         chunk_left = self.chunk_left
         if not chunk_left: # Can be 0 or None
             if chunk_left is not None:
-                # We are at the end of chunk, discard chunk end
+                # We are at the end of chunk. dicard chunk end
                 self._safe_read(2)  # toss the CRLF at the end of the chunk
             try:
                 chunk_left = self._read_next_chunk_size()
@@ -574,7 +555,7 @@ class HTTPResponse(io.BufferedIOBase):
             self.chunk_left = chunk_left
         return chunk_left
 
-    def _read_chunked(self, amt=None):
+    def _readall_chunked(self):
         assert self.chunked != _UNKNOWN
         value = []
         try:
@@ -582,15 +563,7 @@ class HTTPResponse(io.BufferedIOBase):
                 chunk_left = self._get_chunk_left()
                 if chunk_left is None:
                     break
-
-                if amt is not None and amt <= chunk_left:
-                    value.append(self._safe_read(amt))
-                    self.chunk_left = chunk_left - amt
-                    break
-
                 value.append(self._safe_read(chunk_left))
-                if amt is not None:
-                    amt -= chunk_left
                 self.chunk_left = 0
             return b''.join(value)
         except IncompleteRead:
@@ -621,24 +594,43 @@ class HTTPResponse(io.BufferedIOBase):
             raise IncompleteRead(bytes(b[0:total_bytes]))
 
     def _safe_read(self, amt):
-        """Read the number of bytes requested.
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
 
         This function should be used when <amt> bytes "should" be present for
         reading. If the bytes are truly not available (due to EOF), then the
         IncompleteRead exception can be used to detect the problem.
         """
-        data = self.fp.read(amt)
-        if len(data) < amt:
-            raise IncompleteRead(data, amt-len(data))
-        return data
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(b''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return b"".join(s)
 
     def _safe_readinto(self, b):
         """Same as _safe_read, but for reading into a buffer."""
-        amt = len(b)
-        n = self.fp.readinto(b)
-        if n < amt:
-            raise IncompleteRead(bytes(b[:n]), amt-n)
-        return n
+        total_bytes = 0
+        mvb = memoryview(b)
+        while total_bytes < len(b):
+            if MAXAMOUNT < len(mvb):
+                temp_mvb = mvb[0:MAXAMOUNT]
+                n = self.fp.readinto(temp_mvb)
+            else:
+                n = self.fp.readinto(mvb)
+            if not n:
+                raise IncompleteRead(bytes(mvb[0:total_bytes]), len(b))
+            mvb = mvb[n:]
+            total_bytes += n
+        return total_bytes
 
     def read1(self, n=-1):
         """Read with at most one underlying system call.  If at least one
@@ -650,7 +642,14 @@ class HTTPResponse(io.BufferedIOBase):
             return self._read1_chunked(n)
         if self.length is not None and (n < 0 or n > self.length):
             n = self.length
-        result = self.fp.read1(n)
+        try:
+            result = self.fp.read1(n)
+        except ValueError:
+            if n >= 0:
+                raise
+            # some implementations, like BufferedReader, don't support -1
+            # Read an arbitrarily selected largeish chunk.
+            result = self.fp.read1(16*1024)
         if not result and n:
             self._close_conn()
         elif self.length is not None:
@@ -835,10 +834,9 @@ class HTTPConnection:
         return None
 
     def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-                 source_address=None, blocksize=8192):
+                 source_address=None):
         self.timeout = timeout
         self.source_address = source_address
-        self.blocksize = blocksize
         self.sock = None
         self._buffer = []
         self.__response = None
@@ -850,8 +848,6 @@ class HTTPConnection:
 
         (self.host, self.port) = self._get_hostport(host, port)
 
-        self._validate_host(self.host)
-
         # This is stored as an instance variable to allow unit
         # tests to replace it with a suitable mockup
         self._create_connection = socket.create_connection
@@ -864,7 +860,7 @@ class HTTPConnection:
         the endpoint passed to `set_tunnel`. This done by sending an HTTP
         CONNECT request to the proxy server when the connection is established.
 
-        This method must be called before the HTTP connection has been
+        This method must be called before the HTML connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers to send
@@ -904,24 +900,23 @@ class HTTPConnection:
         self.debuglevel = level
 
     def _tunnel(self):
-        connect = b"CONNECT %s:%d HTTP/1.0\r\n" % (
-            self._tunnel_host.encode("ascii"), self._tunnel_port)
-        headers = [connect]
+        connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host,
+            self._tunnel_port)
+        connect_bytes = connect_str.encode("ascii")
+        self.send(connect_bytes)
         for header, value in self._tunnel_headers.items():
-            headers.append(f"{header}: {value}\r\n".encode("latin-1"))
-        headers.append(b"\r\n")
-        # Making a single send() call instead of one per line encourages
-        # the host OS to use a more optimal packet size instead of
-        # potentially emitting a series of small packets.
-        self.send(b"".join(headers))
-        del headers
+            header_str = "%s: %s\r\n" % (header, value)
+            header_bytes = header_str.encode("latin-1")
+            self.send(header_bytes)
+        self.send(b'\r\n')
 
         response = self.response_class(self.sock, method=self._method)
         (version, code, message) = response._read_status()
 
         if code != http.HTTPStatus.OK:
             self.close()
-            raise OSError(f"Tunnel connection failed: {code} {message.strip()}")
+            raise OSError("Tunnel connection failed: %d %s" % (code,
+                                                               message.strip()))
         while True:
             line = response.fp.readline(_MAXLINE + 1)
             if len(line) > _MAXLINE:
@@ -937,15 +932,9 @@ class HTTPConnection:
 
     def connect(self):
         """Connect to the host and port specified in __init__."""
-        sys.audit("http.client.connect", self, self.host, self.port)
         self.sock = self._create_connection(
             (self.host,self.port), self.timeout, self.source_address)
-        # Might fail in OSs that don't implement TCP_NODELAY
-        try:
-             self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-        except OSError as e:
-            if e.errno != errno.ENOPROTOOPT:
-                raise
+        self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
 
         if self._tunnel_host:
             self._tunnel()
@@ -978,6 +967,7 @@ class HTTPConnection:
 
         if self.debuglevel > 0:
             print("send:", repr(data))
+        blocksize = 8192
         if hasattr(data, "read") :
             if self.debuglevel > 0:
                 print("sendIng a read()able")
@@ -985,19 +975,17 @@ class HTTPConnection:
             if encode and self.debuglevel > 0:
                 print("encoding file using iso-8859-1")
             while 1:
-                datablock = data.read(self.blocksize)
+                datablock = data.read(blocksize)
                 if not datablock:
                     break
                 if encode:
                     datablock = datablock.encode("iso-8859-1")
-                sys.audit("http.client.send", self, datablock)
                 self.sock.sendall(datablock)
             return
-        sys.audit("http.client.send", self, data)
         try:
             self.sock.sendall(data)
         except TypeError:
-            if isinstance(data, collections.abc.Iterable):
+            if isinstance(data, collections.Iterable):
                 for d in data:
                     self.sock.sendall(d)
             else:
@@ -1012,13 +1000,14 @@ class HTTPConnection:
         self._buffer.append(s)
 
     def _read_readable(self, readable):
+        blocksize = 8192
         if self.debuglevel > 0:
             print("sendIng a read()able")
         encode = self._is_textIO(readable)
         if encode and self.debuglevel > 0:
             print("encoding file using iso-8859-1")
         while True:
-            datablock = readable.read(self.blocksize)
+            datablock = readable.read(blocksize)
             if not datablock:
                 break
             if encode:
@@ -1118,17 +1107,14 @@ class HTTPConnection:
         else:
             raise CannotSendRequest(self.__state)
 
-        self._validate_method(method)
-
-        # Save the method for use later in the response phase
+        # Save the method we use, we need it later in the response phase
         self._method = method
-
-        url = url or '/'
-        self._validate_path(url)
-
+        if not url:
+            url = '/'
         request = '%s %s %s' % (method, url, self._http_vsn_str)
 
-        self._output(self._encode_request(request))
+        # Non-ASCII characters should have been eliminated earlier
+        self._output(request.encode('ascii'))
 
         if self._http_vsn == 11:
             # Issue some standard headers for better HTTP/1.1 compliance
@@ -1206,35 +1192,6 @@ class HTTPConnection:
             # For HTTP/1.0, the server will assume "not chunked"
             pass
 
-    def _encode_request(self, request):
-        # ASCII also helps prevent CVE-2019-9740.
-        return request.encode('ascii')
-
-    def _validate_method(self, method):
-        """Validate a method name for putrequest."""
-        # prevent http header injection
-        match = _contains_disallowed_method_pchar_re.search(method)
-        if match:
-            raise ValueError(
-                    f"method can't contain control characters. {method!r} "
-                    f"(found at least {match.group()!r})")
-
-    def _validate_path(self, url):
-        """Validate a url for putrequest."""
-        # Prevent CVE-2019-9740.
-        match = _contains_disallowed_url_pchar_re.search(url)
-        if match:
-            raise InvalidURL(f"URL can't contain control characters. {url!r} "
-                             f"(found at least {match.group()!r})")
-
-    def _validate_host(self, host):
-        """Validate a host so it doesn't contain control characters."""
-        # Prevent CVE-2019-18348.
-        match = _contains_disallowed_url_pchar_re.search(host)
-        if match:
-            raise InvalidURL(f"URL can't contain control characters. {host!r} "
-                             f"(found at least {match.group()!r})")
-
     def putheader(self, header, *values):
         """Send a request header line to the server.
 
@@ -1405,10 +1362,9 @@ else:
         def __init__(self, host, port=None, key_file=None, cert_file=None,
                      timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
                      source_address=None, *, context=None,
-                     check_hostname=None, blocksize=8192):
+                     check_hostname=None):
             super(HTTPSConnection, self).__init__(host, port, timeout,
-                                                  source_address,
-                                                  blocksize=blocksize)
+                                                  source_address)
             if (key_file is not None or cert_file is not None or
                         check_hostname is not None):
                 import warnings
@@ -1419,12 +1375,6 @@ else:
             self.cert_file = cert_file
             if context is None:
                 context = ssl._create_default_https_context()
-                # send ALPN extension to indicate HTTP/1.1 protocol
-                if self._http_vsn == 11:
-                    context.set_alpn_protocols(['http/1.1'])
-                # enable PHA for TLS 1.3 connections if available
-                if context.post_handshake_auth is not None:
-                    context.post_handshake_auth = True
             will_verify = context.verify_mode != ssl.CERT_NONE
             if check_hostname is None:
                 check_hostname = context.check_hostname
@@ -1433,13 +1383,8 @@ else:
                                  "either CERT_OPTIONAL or CERT_REQUIRED")
             if key_file or cert_file:
                 context.load_cert_chain(cert_file, key_file)
-                # cert and key file means the user wants to authenticate.
-                # enable TLS 1.3 PHA implicitly even for custom contexts.
-                if context.post_handshake_auth is not None:
-                    context.post_handshake_auth = True
             self._context = context
-            if check_hostname is not None:
-                self._context.check_hostname = check_hostname
+            self._check_hostname = check_hostname
 
         def connect(self):
             "Connect to a host on a given (SSL) port."
@@ -1453,6 +1398,13 @@ else:
 
             self.sock = self._context.wrap_socket(self.sock,
                                                   server_hostname=server_hostname)
+            if not self._context.check_hostname and self._check_hostname:
+                try:
+                    ssl.match_hostname(self.sock.getpeercert(), server_hostname)
+                except Exception:
+                    self.sock.shutdown(socket.SHUT_RDWR)
+                    self.sock.close()
+                    raise
 
     __all__.append("HTTPSConnection")
 
@@ -1490,7 +1442,8 @@ class IncompleteRead(HTTPException):
             e = ''
         return '%s(%i bytes read%s)' % (self.__class__.__name__,
                                         len(self.partial), e)
-    __str__ = object.__str__
+    def __str__(self):
+        return repr(self)
 
 class ImproperConnectionState(HTTPException):
     pass

Lib/http/cookiejar.py

@@ -28,7 +28,6 @@ http://wwwsearch.sf.net/):
 __all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy',
            'FileCookieJar', 'LWPCookieJar', 'LoadError', 'MozillaCookieJar']
 
-import os
 import copy
 import datetime
 import re
@@ -53,18 +52,10 @@ def _debug(*args):
         logger = logging.getLogger("http.cookiejar")
     return logger.debug(*args)
 
-HTTPONLY_ATTR = "HTTPOnly"
-HTTPONLY_PREFIX = "#HttpOnly_"
+
 DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT)
-NETSCAPE_MAGIC_RGX = re.compile("#( Netscape)? HTTP Cookie File")
 MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "
                          "instance initialised with one)")
-NETSCAPE_HEADER_TEXT =  """\
-# Netscape HTTP Cookie File
-# http://curl.haxx.se/rfc/cookie_spec.html
-# This is a generated file!  Do not edit.
-
-"""
 
 def _warn_unhandled_exception():
     # There are a few catch-all except: statements in this module, for
@@ -225,14 +216,10 @@ LOOSE_HTTP_DATE_RE = re.compile(
        (?::(\d\d))?    # optional seconds
     )?                 # optional clock
        \s*
-    (?:
-       ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+) # timezone
+    ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone
        \s*
-    )?
-    (?:
-       \(\w+\)         # ASCII representation of timezone in parens.
-       \s*
-    )?$""", re.X | re.ASCII)
+    (?:\(\w+\))?       # ASCII representation of timezone in parens.
+       \s*$""", re.X | re.ASCII)
 def http2time(text):
     """Returns time in seconds since epoch of time represented by a string.
 
@@ -302,11 +289,9 @@ ISO_DATE_RE = re.compile(
       (?::?(\d\d(?:\.\d*)?))?  # optional seconds (and fractional)
    )?                    # optional clock
       \s*
-   (?:
-      ([-+]?\d\d?:?(:?\d\d)?
-       |Z|z)             # timezone  (Z is "zero meridian", i.e. GMT)
-      \s*
-   )?$""", re.X | re. ASCII)
+   ([-+]?\d\d?:?(:?\d\d)?
+    |Z|z)?               # timezone  (Z is "zero meridian", i.e. GMT)
+      \s*$""", re.X | re. ASCII)
 def iso2time(text):
     """
     As for http2time, but parses the ISO 8601 formats:
@@ -896,7 +881,6 @@ class DefaultCookiePolicy(CookiePolicy):
                  strict_ns_domain=DomainLiberal,
                  strict_ns_set_initial_dollar=False,
                  strict_ns_set_path=False,
-                 secure_protocols=("https", "wss")
                  ):
         """Constructor arguments should be passed as keyword arguments only."""
         self.netscape = netscape
@@ -909,7 +893,6 @@ class DefaultCookiePolicy(CookiePolicy):
         self.strict_ns_domain = strict_ns_domain
         self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar
         self.strict_ns_set_path = strict_ns_set_path
-        self.secure_protocols = secure_protocols
 
         if blocked_domains is not None:
             self._blocked_domains = tuple(blocked_domains)
@@ -1010,7 +993,7 @@ class DefaultCookiePolicy(CookiePolicy):
             req_path = request_path(request)
             if ((cookie.version > 0 or
                  (cookie.version == 0 and self.strict_ns_set_path)) and
-                not self.path_return_ok(cookie.path, request)):
+                not req_path.startswith(cookie.path)):
                 _debug("   path attribute %s is not a prefix of request "
                        "path %s", cookie.path, req_path)
                 return False
@@ -1136,7 +1119,7 @@ class DefaultCookiePolicy(CookiePolicy):
         return True
 
     def return_ok_secure(self, cookie, request):
-        if cookie.secure and request.type not in self.secure_protocols:
+        if cookie.secure and request.type != "https":
             _debug("   secure cookie with non-secure request")
             return False
         return True
@@ -1165,11 +1148,6 @@ class DefaultCookiePolicy(CookiePolicy):
         req_host, erhn = eff_request_host(request)
         domain = cookie.domain
 
-        if domain and not domain.startswith("."):
-            dotdomain = "." + domain
-        else:
-            dotdomain = domain
-
         # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't
         if (cookie.version == 0 and
             (self.strict_ns_domain & self.DomainStrictNonDomain) and
@@ -1182,7 +1160,7 @@ class DefaultCookiePolicy(CookiePolicy):
             _debug("   effective request-host name %s does not domain-match "
                    "RFC 2965 cookie domain %s", erhn, domain)
             return False
-        if cookie.version == 0 and not ("."+erhn).endswith(dotdomain):
+        if cookie.version == 0 and not ("."+erhn).endswith(domain):
             _debug("   request-host %s does not match Netscape cookie domain "
                    "%s", req_host, domain)
             return False
@@ -1196,11 +1174,7 @@ class DefaultCookiePolicy(CookiePolicy):
             req_host = "."+req_host
         if not erhn.startswith("."):
             erhn = "."+erhn
-        if domain and not domain.startswith("."):
-            dotdomain = "." + domain
-        else:
-            dotdomain = domain
-        if not (req_host.endswith(dotdomain) or erhn.endswith(dotdomain)):
+        if not (req_host.endswith(domain) or erhn.endswith(domain)):
             #_debug("   request domain %s does not match cookie domain %s",
             #       req_host, domain)
             return False
@@ -1217,15 +1191,11 @@ class DefaultCookiePolicy(CookiePolicy):
     def path_return_ok(self, path, request):
         _debug("- checking cookie path=%s", path)
         req_path = request_path(request)
-        pathlen = len(path)
-        if req_path == path:
-            return True
-        elif (req_path.startswith(path) and
-              (path.endswith("/") or req_path[pathlen:pathlen+1] == "/")):
-            return True
+        if not req_path.startswith(path):
+            _debug("  %s does not path-match %s", req_path, path)
+            return False
+        return True
 
-        _debug("  %s does not path-match %s", req_path, path)
-        return False
 
 def vals_sorted_by_key(adict):
     keys = sorted(adict.keys())
@@ -1610,7 +1580,6 @@ class CookieJar:
         headers = response.info()
         rfc2965_hdrs = headers.get_all("Set-Cookie2", [])
         ns_hdrs = headers.get_all("Set-Cookie", [])
-        self._policy._now = self._now = int(time.time())
 
         rfc2965 = self._policy.rfc2965
         netscape = self._policy.netscape
@@ -1690,6 +1659,8 @@ class CookieJar:
         _debug("extract_cookies: %s", response.info())
         self._cookies_lock.acquire()
         try:
+            self._policy._now = self._now = int(time.time())
+
             for cookie in self.make_cookies(response, request):
                 if self._policy.set_ok(cookie, request):
                     _debug(" setting cookie: %s", cookie)
@@ -1792,7 +1763,10 @@ class FileCookieJar(CookieJar):
         """
         CookieJar.__init__(self, policy)
         if filename is not None:
-            filename = os.fspath(filename)
+            try:
+                filename+""
+            except:
+                raise ValueError("filename must be string-like")
         self.filename = filename
         self.delayload = bool(delayload)
 
@@ -2015,11 +1989,19 @@ class MozillaCookieJar(FileCookieJar):
     header by default (Mozilla can cope with that).
 
     """
+    magic_re = re.compile("#( Netscape)? HTTP Cookie File")
+    header = """\
+# Netscape HTTP Cookie File
+# http://curl.haxx.se/rfc/cookie_spec.html
+# This is a generated file!  Do not edit.
+
+"""
 
     def _really_load(self, f, filename, ignore_discard, ignore_expires):
         now = time.time()
 
-        if not NETSCAPE_MAGIC_RGX.match(f.readline()):
+        magic = f.readline()
+        if not self.magic_re.search(magic):
             raise LoadError(
                 "%r does not look like a Netscape format cookies file" %
                 filename)
@@ -2027,17 +2009,8 @@ class MozillaCookieJar(FileCookieJar):
         try:
             while 1:
                 line = f.readline()
-                rest = {}
-
                 if line == "": break
 
-                # httponly is a cookie flag as defined in rfc6265
-                # when encoded in a netscape cookie file,
-                # the line is prepended with "#HttpOnly_"
-                if line.startswith(HTTPONLY_PREFIX):
-                    rest[HTTPONLY_ATTR] = ""
-                    line = line[len(HTTPONLY_PREFIX):]
-
                 # last field may be absent, so keep any trailing tab
                 if line.endswith("\n"): line = line[:-1]
 
@@ -2075,7 +2048,7 @@ class MozillaCookieJar(FileCookieJar):
                            discard,
                            None,
                            None,
-                           rest)
+                           {})
                 if not ignore_discard and c.discard:
                     continue
                 if not ignore_expires and c.is_expired(now):
@@ -2095,17 +2068,16 @@ class MozillaCookieJar(FileCookieJar):
             else: raise ValueError(MISSING_FILENAME_TEXT)
 
         with open(filename, "w") as f:
-            f.write(NETSCAPE_HEADER_TEXT)
+            f.write(self.header)
             now = time.time()
             for cookie in self:
-                domain = cookie.domain
                 if not ignore_discard and cookie.discard:
                     continue
                 if not ignore_expires and cookie.is_expired(now):
                     continue
                 if cookie.secure: secure = "TRUE"
                 else: secure = "FALSE"
-                if domain.startswith("."): initial_dot = "TRUE"
+                if cookie.domain.startswith("."): initial_dot = "TRUE"
                 else: initial_dot = "FALSE"
                 if cookie.expires is not None:
                     expires = str(cookie.expires)
@@ -2120,9 +2092,7 @@ class MozillaCookieJar(FileCookieJar):
                 else:
                     name = cookie.name
                     value = cookie.value
-                if cookie.has_nonstandard_attr(HTTPONLY_ATTR):
-                    domain = HTTPONLY_PREFIX + domain
                 f.write(
-                    "\t".join([domain, initial_dot, cookie.path,
+                    "\t".join([cookie.domain, initial_dot, cookie.path,
                                secure, expires, name, value])+
                     "\n")

Lib/http/cookies.py

@@ -131,7 +131,6 @@ Finis.
 #
 import re
 import string
-import types
 
 __all__ = ["CookieError", "BaseCookie", "SimpleCookie"]
 
@@ -139,6 +138,12 @@ _nulljoin = ''.join
 _semispacejoin = '; '.join
 _spacejoin = ' '.join
 
+def _warn_deprecated_setter(setter):
+    import warnings
+    msg = ('The .%s setter is deprecated. The attribute will be read-only in '
+           'future releases. Please use the set() method instead.' % setter)
+    warnings.warn(msg, DeprecationWarning, stacklevel=3)
+
 #
 # Define an exception visible to External modules
 #
@@ -257,7 +262,8 @@ class Morsel(dict):
     In a cookie, each such pair may have several attributes, so this class is
     used to keep the attributes associated with the appropriate key,value pair.
     This class also includes a coded_value attribute, which is used to hold
-    the network representation of the value.
+    the network representation of the value.  This is most useful when Python
+    objects are pickled for network transit.
     """
     # RFC 2109 lists these attributes as reserved:
     #   path       comment         domain
@@ -281,7 +287,6 @@ class Morsel(dict):
         "secure"   : "Secure",
         "httponly" : "HttpOnly",
         "version"  : "Version",
-        "samesite" : "SameSite",
     }
 
     _flags = {'secure', 'httponly'}
@@ -298,14 +303,29 @@ class Morsel(dict):
     def key(self):
         return self._key
 
+    @key.setter
+    def key(self, key):
+        _warn_deprecated_setter('key')
+        self._key = key
+
     @property
     def value(self):
         return self._value
 
+    @value.setter
+    def value(self, value):
+        _warn_deprecated_setter('value')
+        self._value = value
+
     @property
     def coded_value(self):
         return self._coded_value
 
+    @coded_value.setter
+    def coded_value(self, coded_value):
+        _warn_deprecated_setter('coded_value')
+        self._coded_value = coded_value
+
     def __setitem__(self, K, V):
         K = K.lower()
         if not K in self._reserved:
@@ -346,7 +366,14 @@ class Morsel(dict):
     def isReservedKey(self, K):
         return K.lower() in self._reserved
 
-    def set(self, key, val, coded_val):
+    def set(self, key, val, coded_val, LegalChars=_LegalChars):
+        if LegalChars != _LegalChars:
+            import warnings
+            warnings.warn(
+                'LegalChars parameter is deprecated, ignored and will '
+                'be removed in future versions.', DeprecationWarning,
+                stacklevel=2)
+
         if key.lower() in self._reserved:
             raise CookieError('Attempt to set a reserved key %r' % (key,))
         if not _is_legal_key(key):
@@ -409,8 +436,6 @@ class Morsel(dict):
                 append("%s=%s" % (self._reserved[key], _getdate(value)))
             elif key == "max-age" and isinstance(value, int):
                 append("%s=%d" % (self._reserved[key], value))
-            elif key == "comment" and isinstance(value, str):
-                append("%s=%s" % (self._reserved[key], _quote(value)))
             elif key in self._flags:
                 if value:
                     append(str(self._reserved[key]))
@@ -420,8 +445,6 @@ class Morsel(dict):
         # Return the result
         return _semispacejoin(result)
 
-    __class_getitem__ = classmethod(types.GenericAlias)
-
 
 #
 # Pattern for finding cookie

Lib/http/server.py

@@ -83,12 +83,10 @@ XXX To do:
 __version__ = "0.6"
 
 __all__ = [
-    "HTTPServer", "ThreadingHTTPServer", "BaseHTTPRequestHandler",
+    "HTTPServer", "BaseHTTPRequestHandler",
     "SimpleHTTPRequestHandler", "CGIHTTPRequestHandler",
 ]
 
-import copy
-import datetime
 import email.utils
 import html
 import http.client
@@ -103,6 +101,8 @@ import socketserver
 import sys
 import time
 import urllib.parse
+import copy
+import argparse
 
 from http import HTTPStatus
 
@@ -139,10 +139,6 @@ class HTTPServer(socketserver.TCPServer):
         self.server_port = port
 
 
-class ThreadingHTTPServer(socketserver.ThreadingMixIn, HTTPServer):
-    daemon_threads = True
-
-
 class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
 
     """HTTP request handler base class.
@@ -271,8 +267,8 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
         are in self.command, self.path, self.request_version and
         self.headers.
 
-        Return True for success, False for failure; on failure, any relevant
-        error response has already been sent back.
+        Return True for success, False for failure; on failure, an
+        error is sent back.
 
         """
         self.command = None  # set in case of error on the first line
@@ -282,13 +278,10 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
         requestline = requestline.rstrip('\r\n')
         self.requestline = requestline
         words = requestline.split()
-        if len(words) == 0:
-            return False
-
-        if len(words) >= 3:  # Enough to determine protocol version
-            version = words[-1]
+        if len(words) == 3:
+            command, path, version = words
             try:
-                if not version.startswith('HTTP/'):
+                if version[:5] != 'HTTP/':
                     raise ValueError
                 base_version_number = version.split('/', 1)[1]
                 version_number = base_version_number.split(".")
@@ -313,22 +306,22 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
                     HTTPStatus.HTTP_VERSION_NOT_SUPPORTED,
                     "Invalid HTTP version (%s)" % base_version_number)
                 return False
-            self.request_version = version
-
-        if not 2 <= len(words) <= 3:
-            self.send_error(
-                HTTPStatus.BAD_REQUEST,
-                "Bad request syntax (%r)" % requestline)
-            return False
-        command, path = words[:2]
-        if len(words) == 2:
+        elif len(words) == 2:
+            command, path = words
             self.close_connection = True
             if command != 'GET':
                 self.send_error(
                     HTTPStatus.BAD_REQUEST,
                     "Bad HTTP/0.9 request type (%r)" % command)
                 return False
-        self.command, self.path = command, path
+        elif not words:
+            return False
+        else:
+            self.send_error(
+                HTTPStatus.BAD_REQUEST,
+                "Bad request syntax (%r)" % requestline)
+            return False
+        self.command, self.path, self.request_version = command, path, version
 
         # Examine the headers and look for a Connection directive.
         try:
@@ -412,7 +405,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
             method = getattr(self, mname)
             method()
             self.wfile.flush() #actually send the response if not already done.
-        except TimeoutError as e:
+        except socket.timeout as e:
             #a read or a write timed out.  Discard this connection
             self.log_error("Request timed out: %r", e)
             self.close_connection = True
@@ -473,7 +466,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
             })
             body = content.encode('UTF-8', 'replace')
             self.send_header("Content-Type", self.error_content_type)
-            self.send_header('Content-Length', str(len(body)))
+            self.send_header('Content-Length', int(len(body)))
         self.end_headers()
 
         if self.command != 'HEAD' and body:
@@ -637,18 +630,6 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
     """
 
     server_version = "SimpleHTTP/" + __version__
-    extensions_map = _encodings_map_default = {
-        '.gz': 'application/gzip',
-        '.Z': 'application/octet-stream',
-        '.bz2': 'application/x-bzip2',
-        '.xz': 'application/x-xz',
-    }
-
-    def __init__(self, *args, directory=None, **kwargs):
-        if directory is None:
-            directory = os.getcwd()
-        self.directory = os.fspath(directory)
-        super().__init__(*args, **kwargs)
 
     def do_GET(self):
         """Serve a GET request."""
@@ -687,7 +668,6 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
                              parts[3], parts[4])
                 new_url = urllib.parse.urlunsplit(new_parts)
                 self.send_header("Location", new_url)
-                self.send_header("Content-Length", "0")
                 self.end_headers()
                 return None
             for index in "index.html", "index.htm":
@@ -698,55 +678,17 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
             else:
                 return self.list_directory(path)
         ctype = self.guess_type(path)
-        # check for trailing "/" which should return 404. See Issue17324
-        # The test for this was added in test_httpserver.py
-        # However, some OS platforms accept a trailingSlash as a filename
-        # See discussion on python-dev and Issue34711 regarding
-        # parseing and rejection of filenames with a trailing slash
-        if path.endswith("/"):
-            self.send_error(HTTPStatus.NOT_FOUND, "File not found")
-            return None
         try:
             f = open(path, 'rb')
         except OSError:
             self.send_error(HTTPStatus.NOT_FOUND, "File not found")
             return None
-
         try:
-            fs = os.fstat(f.fileno())
-            # Use browser cache if possible
-            if ("If-Modified-Since" in self.headers
-                    and "If-None-Match" not in self.headers):
-                # compare If-Modified-Since and time of last file modification
-                try:
-                    ims = email.utils.parsedate_to_datetime(
-                        self.headers["If-Modified-Since"])
-                except (TypeError, IndexError, OverflowError, ValueError):
-                    # ignore ill-formed values
-                    pass
-                else:
-                    if ims.tzinfo is None:
-                        # obsolete format with no timezone, cf.
-                        # https://tools.ietf.org/html/rfc7231#section-7.1.1.1
-                        ims = ims.replace(tzinfo=datetime.timezone.utc)
-                    if ims.tzinfo is datetime.timezone.utc:
-                        # compare to UTC datetime of last modification
-                        last_modif = datetime.datetime.fromtimestamp(
-                            fs.st_mtime, datetime.timezone.utc)
-                        # remove microseconds, like in If-Modified-Since
-                        last_modif = last_modif.replace(microsecond=0)
-
-                        if last_modif <= ims:
-                            self.send_response(HTTPStatus.NOT_MODIFIED)
-                            self.end_headers()
-                            f.close()
-                            return None
-
             self.send_response(HTTPStatus.OK)
             self.send_header("Content-type", ctype)
+            fs = os.fstat(f.fileno())
             self.send_header("Content-Length", str(fs[6]))
-            self.send_header("Last-Modified",
-                self.date_time_string(fs.st_mtime))
+            self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
             self.end_headers()
             return f
         except:
@@ -831,7 +773,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         path = posixpath.normpath(path)
         words = path.split('/')
         words = filter(None, words)
-        path = self.directory
+        path = os.getcwd()
         for word in words:
             if os.path.dirname(word) or word in (os.curdir, os.pardir):
                 # Ignore components that are not a simple file/directory name
@@ -871,16 +813,25 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         slow) to look inside the data to make a better guess.
 
         """
+
         base, ext = posixpath.splitext(path)
         if ext in self.extensions_map:
             return self.extensions_map[ext]
         ext = ext.lower()
         if ext in self.extensions_map:
             return self.extensions_map[ext]
-        guess, _ = mimetypes.guess_type(path)
-        if guess:
-            return guess
-        return 'application/octet-stream'
+        else:
+            return self.extensions_map['']
+
+    if not mimetypes.inited:
+        mimetypes.init() # try to read system mime.types
+    extensions_map = mimetypes.types_map.copy()
+    extensions_map.update({
+        '': 'application/octet-stream', # Default
+        '.py': 'text/plain',
+        '.c': 'text/plain',
+        '.h': 'text/plain',
+        })
 
 
 # Utilities for CGIHTTPRequestHandler
@@ -1011,10 +962,8 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
         """
         collapsed_path = _url_collapse_path(self.path)
         dir_sep = collapsed_path.find('/', 1)
-        while dir_sep > 0 and not collapsed_path[:dir_sep] in self.cgi_directories:
-            dir_sep = collapsed_path.find('/', dir_sep+1)
-        if dir_sep > 0:
-            head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
+        head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
+        if head in self.cgi_directories:
             self.cgi_info = head, tail
             return True
         return False
@@ -1091,7 +1040,8 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
         env['PATH_INFO'] = uqrest
         env['PATH_TRANSLATED'] = self.translate_path(uqrest)
         env['SCRIPT_NAME'] = scriptname
-        env['QUERY_STRING'] = query
+        if query:
+            env['QUERY_STRING'] = query
         env['REMOTE_ADDR'] = self.client_address[0]
         authorization = self.headers.get("authorization")
         if authorization:
@@ -1121,7 +1071,12 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
         referer = self.headers.get('referer')
         if referer:
             env['HTTP_REFERER'] = referer
-        accept = self.headers.get_all('accept', ())
+        accept = []
+        for line in self.headers.getallmatchingheaders('accept'):
+            if line[:1] in "\t\n\r ":
+                accept.append(line.strip())
+            else:
+                accept = accept + line[7:].split(',')
         env['HTTP_ACCEPT'] = ','.join(accept)
         ua = self.headers.get('user-agent')
         if ua:
@@ -1157,9 +1112,8 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
                 while select.select([self.rfile], [], [], 0)[0]:
                     if not self.rfile.read(1):
                         break
-                exitcode = os.waitstatus_to_exitcode(sts)
-                if exitcode:
-                    self.log_error(f"CGI script exit code {exitcode}")
+                if sts:
+                    self.log_error("CGI script exit status %#x", sts)
                 return
             # Child
             try:
@@ -1218,33 +1172,20 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
                 self.log_message("CGI script exited OK")
 
 
-def _get_best_family(*address):
-    infos = socket.getaddrinfo(
-        *address,
-        type=socket.SOCK_STREAM,
-        flags=socket.AI_PASSIVE,
-    )
-    family, type, proto, canonname, sockaddr = next(iter(infos))
-    return family, sockaddr
-
-
 def test(HandlerClass=BaseHTTPRequestHandler,
-         ServerClass=ThreadingHTTPServer,
-         protocol="HTTP/1.0", port=8000, bind=None):
+         ServerClass=HTTPServer, protocol="HTTP/1.0", port=8000, bind=""):
     """Test the HTTP request handler class.
 
     This runs an HTTP server on port 8000 (or the port argument).
 
     """
-    ServerClass.address_family, addr = _get_best_family(bind, port)
+    server_address = (bind, port)
+
     HandlerClass.protocol_version = protocol
-    with ServerClass(addr, HandlerClass) as httpd:
-        host, port = httpd.socket.getsockname()[:2]
-        url_host = f'[{host}]' if ':' in host else host
-        print(
-            f"Serving HTTP on {host} port {port} "
-            f"(http://{url_host}:{port}/) ..."
-        )
+    with ServerClass(server_address, HandlerClass) as httpd:
+        sa = httpd.socket.getsockname()
+        serve_message = "Serving HTTP on {host} port {port} (http://{host}:{port}/) ..."
+        print(serve_message.format(host=sa[0], port=sa[1]))
         try:
             httpd.serve_forever()
         except KeyboardInterrupt:
@@ -1252,44 +1193,19 @@ def test(HandlerClass=BaseHTTPRequestHandler,
             sys.exit(0)
 
 if __name__ == '__main__':
-    import argparse
-    import contextlib
-
     parser = argparse.ArgumentParser()
     parser.add_argument('--cgi', action='store_true',
-                        help='run as CGI server')
-    parser.add_argument('--bind', '-b', metavar='ADDRESS',
-                        help='specify alternate bind address '
-                             '(default: all interfaces)')
-    parser.add_argument('--directory', '-d', default=os.getcwd(),
-                        help='specify alternate directory '
-                             '(default: current directory)')
-    parser.add_argument('port', action='store', default=8000, type=int,
+                       help='Run as CGI Server')
+    parser.add_argument('--bind', '-b', default='', metavar='ADDRESS',
+                        help='Specify alternate bind address '
+                             '[default: all interfaces]')
+    parser.add_argument('port', action='store',
+                        default=8000, type=int,
                         nargs='?',
-                        help='specify alternate port (default: 8000)')
+                        help='Specify alternate port [default: 8000]')
     args = parser.parse_args()
     if args.cgi:
         handler_class = CGIHTTPRequestHandler
     else:
         handler_class = SimpleHTTPRequestHandler
-
-    # ensure dual-stack is not disabled; ref #38907
-    class DualStackServer(ThreadingHTTPServer):
-
-        def server_bind(self):
-            # suppress exception when protocol is IPv4
-            with contextlib.suppress(Exception):
-                self.socket.setsockopt(
-                    socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
-            return super().server_bind()
-
-        def finish_request(self, request, client_address):
-            self.RequestHandlerClass(request, client_address, self,
-                                     directory=args.directory)
-
-    test(
-        HandlerClass=handler_class,
-        ServerClass=DualStackServer,
-        port=args.port,
-        bind=args.bind,
-    )
+    test(HandlerClass=handler_class, port=args.port, bind=args.bind)

Lib/imghdr.py

@@ -1,6 +1,6 @@
 """Recognize image file formats based on their first few bytes."""
 
-from os import PathLike
+#from os import PathLike
 
 __all__ = ["what"]
 
@@ -14,7 +14,8 @@ def what(file, h=None):
     f = None
     try:
         if h is None:
-            if isinstance(file, (str, PathLike)):
+            # if isinstance(file, (str, PathLike))
+            if isinstance(file, str): # FIXME(corona10): RustPython doesn't support PathLike yet.
                 f = FileIO(file, 'rb')
                 h = f.read(32)
             else:

Lib/imp.py

@@ -28,8 +28,7 @@ import tokenize
 import types
 import warnings
 
-warnings.warn("the imp module is deprecated in favour of importlib and slated "
-              "for removal in Python 3.12; "
+warnings.warn("the imp module is deprecated in favour of importlib; "
               "see the module's documentation for alternative uses",
               DeprecationWarning, stacklevel=2)
 

Lib/importlib/__init__.py

@@ -34,7 +34,7 @@ try:
     import _frozen_importlib_external as _bootstrap_external
 except ImportError:
     from . import _bootstrap_external
-    _bootstrap_external._set_bootstrap_module(_bootstrap)
+    _bootstrap_external._setup(_bootstrap)
     _bootstrap._bootstrap_external = _bootstrap_external
 else:
     _bootstrap_external.__name__ = 'importlib._bootstrap_external'
@@ -54,6 +54,7 @@ _unpack_uint32 = _bootstrap_external._unpack_uint32
 # Fully bootstrapped at this point, import whatever you like, circular
 # dependencies and startup overhead minimisation permitting :)
 
+import types
 import warnings
 
 
@@ -78,8 +79,8 @@ def find_loader(name, path=None):
     This function is deprecated in favor of importlib.util.find_spec().
 
     """
-    warnings.warn('Deprecated since Python 3.4 and slated for removal in '
-                  'Python 3.12; use importlib.util.find_spec() instead',
+    warnings.warn('Deprecated since Python 3.4. '
+                  'Use importlib.util.find_spec() instead.',
                   DeprecationWarning, stacklevel=2)
     try:
         loader = sys.modules[name].__loader__
@@ -135,13 +136,12 @@ def reload(module):
     The module must have been successfully imported before.
 
     """
+    if not module or not isinstance(module, types.ModuleType):
+        raise TypeError("reload() argument must be a module")
     try:
         name = module.__spec__.name
     except AttributeError:
-        try:
-            name = module.__name__
-        except AttributeError:
-            raise TypeError("reload() argument must be a module")
+        name = module.__name__
 
     if sys.modules.get(name) is not module:
         msg = "module {} not in sys.modules"

Lib/importlib/_abc.py

@@ -1,54 +0,0 @@
-"""Subset of importlib.abc used to reduce importlib.util imports."""
-from . import _bootstrap
-import abc
-import warnings
-
-
-class Loader(metaclass=abc.ABCMeta):
-
-    """Abstract base class for import loaders."""
-
-    def create_module(self, spec):
-        """Return a module to initialize and into which to load.
-
-        This method should raise ImportError if anything prevents it
-        from creating a new module.  It may return None to indicate
-        that the spec should create the new module.
-        """
-        # By default, defer to default semantics for the new module.
-        return None
-
-    # We don't define exec_module() here since that would break
-    # hasattr checks we do to support backward compatibility.
-
-    def load_module(self, fullname):
-        """Return the loaded module.
-
-        The module must be added to sys.modules and have import-related
-        attributes set properly.  The fullname is a str.
-
-        ImportError is raised on failure.
-
-        This method is deprecated in favor of loader.exec_module(). If
-        exec_module() exists then it is used to provide a backwards-compatible
-        functionality for this method.
-
-        """
-        if not hasattr(self, 'exec_module'):
-            raise ImportError
-        # Warning implemented in _load_module_shim().
-        return _bootstrap._load_module_shim(self, fullname)
-
-    def module_repr(self, module):
-        """Return a module's repr.
-
-        Used by the module type when the method does not raise
-        NotImplementedError.
-
-        This method is deprecated.
-
-        """
-        warnings.warn("importlib.abc.Loader.module_repr() is deprecated and "
-                      "slated for removal in Python 3.12", DeprecationWarning)
-        # The exception will cause ModuleType.__repr__ to ignore this method.
-        raise NotImplementedError

Lib/importlib/_adapters.py

@@ -1,83 +0,0 @@
-from contextlib import suppress
-
-from . import abc
-
-
-class SpecLoaderAdapter:
-    """
-    Adapt a package spec to adapt the underlying loader.
-    """
-
-    def __init__(self, spec, adapter=lambda spec: spec.loader):
-        self.spec = spec
-        self.loader = adapter(spec)
-
-    def __getattr__(self, name):
-        return getattr(self.spec, name)
-
-
-class TraversableResourcesLoader:
-    """
-    Adapt a loader to provide TraversableResources.
-    """
-
-    def __init__(self, spec):
-        self.spec = spec
-
-    def get_resource_reader(self, name):
-        return DegenerateFiles(self.spec)._native()
-
-
-class DegenerateFiles:
-    """
-    Adapter for an existing or non-existant resource reader
-    to provide a degenerate .files().
-    """
-
-    class Path(abc.Traversable):
-        def iterdir(self):
-            return iter(())
-
-        def is_dir(self):
-            return False
-
-        is_file = exists = is_dir  # type: ignore
-
-        def joinpath(self, other):
-            return DegenerateFiles.Path()
-
-        @property
-        def name(self):
-            return ''
-
-        def open(self, mode='rb', *args, **kwargs):
-            raise ValueError()
-
-    def __init__(self, spec):
-        self.spec = spec
-
-    @property
-    def _reader(self):
-        with suppress(AttributeError):
-            return self.spec.loader.get_resource_reader(self.spec.name)
-
-    def _native(self):
-        """
-        Return the native reader if it supports files().
-        """
-        reader = self._reader
-        return reader if hasattr(reader, 'files') else self
-
-    def __getattr__(self, attr):
-        return getattr(self._reader, attr)
-
-    def files(self):
-        return DegenerateFiles.Path()
-
-
-def wrap_spec(package):
-    """
-    Construct a package spec with traversable compatibility
-    on the spec/loader/reader.
-    """
-    return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)

Lib/importlib/_bootstrap.py

@@ -20,23 +20,10 @@ work. One should use importlib as the public-facing version of this module.
 # reference any injected objects! This includes not only global code but also
 # anything specified at the class level.
 
-def _object_name(obj):
-    try:
-        return obj.__qualname__
-    except AttributeError:
-        return type(obj).__qualname__
-
 # Bootstrap-related code ######################################################
 
-# Modules injected manually by _setup()
-_thread = None
-_warnings = None
-_weakref = None
-
-# Import done by _install_external_importers()
 _bootstrap_external = None
 
-
 def _wrap(new, old):
     """Simple substitute for functools.update_wrapper."""
     for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
@@ -80,7 +67,6 @@ class _ModuleLock:
         # Deadlock avoidance for concurrent circular imports.
         me = _thread.get_ident()
         tid = self.owner
-        seen = set()
         while True:
             lock = _blocking_on.get(tid)
             if lock is None:
@@ -88,14 +74,6 @@ class _ModuleLock:
             tid = lock.owner
             if tid == me:
                 return True
-            if tid in seen:
-                # bpo 38091: the chain of tid's we encounter here
-                # eventually leads to a fixpoint or a cycle, but
-                # does not reach 'me'.  This means we would not
-                # actually deadlock.  This can happen if other
-                # threads are at the beginning of acquire() below.
-                return False
-            seen.add(tid)
 
     def acquire(self):
         """
@@ -243,8 +221,7 @@ def _call_with_frames_removed(f, *args, **kwds):
 
 def _verbose_message(message, *args, verbosity=1):
     """Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
-    # XXX RUSTPYTHON: hasattr check because we might be bootstrapping and we wouldn't have stderr yet
-    if sys.flags.verbose >= verbosity and hasattr(sys, "stderr"):
+    if sys.flags.verbose >= verbosity:
         if not message.startswith(('#', 'import ')):
             message = '# ' + message
         print(message.format(*args), file=sys.stderr)
@@ -276,12 +253,9 @@ def _requires_frozen(fxn):
 def _load_module_shim(self, fullname):
     """Load the specified module into sys.modules and return it.
 
-    This method is deprecated.  Use loader.exec_module() instead.
+    This method is deprecated.  Use loader.exec_module instead.
 
     """
-    msg = ("the load_module() method is deprecated and slated for removal in "
-          "Python 3.12; use exec_module() instead")
-    _warnings.warn(msg, DeprecationWarning)
     spec = spec_from_loader(fullname, self)
     if fullname in sys.modules:
         module = sys.modules[fullname]
@@ -293,16 +267,26 @@ def _load_module_shim(self, fullname):
 # Module specifications #######################################################
 
 def _module_repr(module):
-    """The implementation of ModuleType.__repr__()."""
+    # The implementation of ModuleType.__repr__().
     loader = getattr(module, '__loader__', None)
-    if spec := getattr(module, "__spec__", None):
-        return _module_repr_from_spec(spec)
-    elif hasattr(loader, 'module_repr'):
+    if hasattr(loader, 'module_repr'):
+        # As soon as BuiltinImporter, FrozenImporter, and NamespaceLoader
+        # drop their implementations for module_repr. we can add a
+        # deprecation warning here.
         try:
             return loader.module_repr(module)
         except Exception:
             pass
-    # Fall through to a catch-all which always succeeds.
+    try:
+        spec = module.__spec__
+    except AttributeError:
+        pass
+    else:
+        if spec is not None:
+            return _module_repr_from_spec(spec)
+
+    # We could use module.__class__.__name__ instead of 'module' in the
+    # various repr permutations.
     try:
         name = module.__name__
     except AttributeError:
@@ -387,7 +371,7 @@ class ModuleSpec:
                     self.cached == other.cached and
                     self.has_location == other.has_location)
         except AttributeError:
-            return NotImplemented
+            return False
 
     @property
     def cached(self):
@@ -612,9 +596,9 @@ def _exec(spec, module):
             else:
                 _init_module_attrs(spec, module, override=True)
                 if not hasattr(spec.loader, 'exec_module'):
-                    msg = (f"{_object_name(spec.loader)}.exec_module() not found; "
-                           "falling back to load_module()")
-                    _warnings.warn(msg, ImportWarning)
+                    # (issue19713) Once BuiltinImporter and ExtensionFileLoader
+                    # have exec_module() implemented, we can add a deprecation
+                    # warning here.
                     spec.loader.load_module(name)
                 else:
                     spec.loader.exec_module(module)
@@ -627,8 +611,9 @@ def _exec(spec, module):
 
 
 def _load_backward_compatible(spec):
-    # It is assumed that all callers have been warned about using load_module()
-    # appropriately before calling this function.
+    # (issue19713) Once BuiltinImporter and ExtensionFileLoader
+    # have exec_module() implemented, we can add a deprecation
+    # warning here.
     try:
         spec.loader.load_module(spec.name)
     except:
@@ -667,9 +652,6 @@ def _load_unlocked(spec):
     if spec.loader is not None:
         # Not a namespace package.
         if not hasattr(spec.loader, 'exec_module'):
-            msg = (f"{_object_name(spec.loader)}.exec_module() not found; "
-                    "falling back to load_module()")
-            _warnings.warn(msg, ImportWarning)
             return _load_backward_compatible(spec)
 
     module = module_from_spec(spec)
@@ -731,8 +713,6 @@ class BuiltinImporter:
 
     """
 
-    _ORIGIN = "built-in"
-
     @staticmethod
     def module_repr(module):
         """Return repr for the module.
@@ -740,16 +720,14 @@ class BuiltinImporter:
         The method is deprecated.  The import machinery does the job itself.
 
         """
-        _warnings.warn("BuiltinImporter.module_repr() is deprecated and "
-                       "slated for removal in Python 3.12", DeprecationWarning)
-        return f'<module {module.__name__!r} ({BuiltinImporter._ORIGIN})>'
+        return '<module {!r} (built-in)>'.format(module.__name__)
 
     @classmethod
     def find_spec(cls, fullname, path=None, target=None):
         if path is not None:
             return None
         if _imp.is_builtin(fullname):
-            return spec_from_loader(fullname, cls, origin=cls._ORIGIN)
+            return spec_from_loader(fullname, cls, origin='built-in')
         else:
             return None
 
@@ -762,22 +740,19 @@ class BuiltinImporter:
         This method is deprecated.  Use find_spec() instead.
 
         """
-        _warnings.warn("BuiltinImporter.find_module() is deprecated and "
-                       "slated for removal in Python 3.12; use find_spec() instead",
-                       DeprecationWarning)
         spec = cls.find_spec(fullname, path)
         return spec.loader if spec is not None else None
 
-    @staticmethod
-    def create_module(spec):
+    @classmethod
+    def create_module(self, spec):
         """Create a built-in module"""
         if spec.name not in sys.builtin_module_names:
             raise ImportError('{!r} is not a built-in module'.format(spec.name),
                               name=spec.name)
         return _call_with_frames_removed(_imp.create_builtin, spec)
 
-    @staticmethod
-    def exec_module(module):
+    @classmethod
+    def exec_module(self, module):
         """Exec a built-in module"""
         _call_with_frames_removed(_imp.exec_builtin, module)
 
@@ -820,8 +795,6 @@ class FrozenImporter:
         The method is deprecated.  The import machinery does the job itself.
 
         """
-        _warnings.warn("FrozenImporter.module_repr() is deprecated and "
-                       "slated for removal in Python 3.12", DeprecationWarning)
         return '<module {!r} ({})>'.format(m.__name__, FrozenImporter._ORIGIN)
 
     @classmethod
@@ -838,13 +811,10 @@ class FrozenImporter:
         This method is deprecated.  Use find_spec() instead.
 
         """
-        _warnings.warn("FrozenImporter.find_module() is deprecated and "
-                       "slated for removal in Python 3.12; use find_spec() instead",
-                       DeprecationWarning)
         return cls if _imp.is_frozen(fullname) else None
 
-    @staticmethod
-    def create_module(spec):
+    @classmethod
+    def create_module(cls, spec):
         """Use default semantics for module creation."""
 
     @staticmethod
@@ -863,7 +833,6 @@ class FrozenImporter:
         This method is deprecated.  Use exec_module() instead.
 
         """
-        # Warning about deprecation implemented in _load_module_shim().
         return _load_module_shim(cls, fullname)
 
     @classmethod
@@ -904,15 +873,14 @@ def _resolve_name(name, package, level):
     """Resolve a relative module name to an absolute one."""
     bits = package.rsplit('.', level - 1)
     if len(bits) < level:
-        raise ImportError('attempted relative import beyond top-level package')
+        raise ValueError('attempted relative import beyond top-level package')
     base = bits[0]
     return '{}.{}'.format(base, name) if name else base
 
 
 def _find_spec_legacy(finder, name, path):
-    msg = (f"{_object_name(finder)}.find_spec() not found; "
-                           "falling back to find_module()")
-    _warnings.warn(msg, ImportWarning)
+    # This would be a good place for a DeprecationWarning if
+    # we ended up going that route.
     loader = finder.find_module(name, path)
     if loader is None:
         return None
@@ -1008,12 +976,7 @@ def _find_and_load_unlocked(name, import_):
     if parent:
         # Set the module as an attribute on its parent.
         parent_module = sys.modules[parent]
-        child = name.rpartition('.')[2]
-        try:
-            setattr(parent_module, child, module)
-        except AttributeError:
-            msg = f"Cannot set an attribute on {parent!r} for child module {child!r}"
-            _warnings.warn(msg, ImportWarning)
+        setattr(parent_module, name.rpartition('.')[2], module)
     return module
 
 
@@ -1186,12 +1149,21 @@ def _setup(sys_module, _imp_module):
 
     # Directly load built-in modules needed during bootstrap.
     self_module = sys.modules[__name__]
-    for builtin_name in ('_thread', '_warnings', '_weakref'):
+    for builtin_name in ('_warnings', '_weakref'):
         if builtin_name not in sys.modules:
             builtin_module = _builtin_from_name(builtin_name)
         else:
             builtin_module = sys.modules[builtin_name]
         setattr(self_module, builtin_name, builtin_module)
+    # _thread was part of the above loop, but other parts of the code allow for it
+    # to be None, so we handle it separately here
+    builtin_name = '_thread'
+    if builtin_name in sys.modules:
+        builtin_module = sys.modules[builtin_name]
+    else:
+        builtin_spec = BuiltinImporter.find_spec(builtin_name)
+        builtin_module = builtin_spec and _load_unlocked(builtin_spec)
+    setattr(self_module, builtin_name, builtin_module)
 
 
 def _install(sys_module, _imp_module):

Lib/importlib/_bootstrap_external.py

@@ -19,37 +19,6 @@ work. One should use importlib as the public-facing version of this module.
 # reference any injected objects! This includes not only global code but also
 # anything specified at the class level.
 
-# Module injected manually by _set_bootstrap_module()
-_bootstrap = None
-
-# Import builtin modules
-import _imp
-import _io
-import sys
-import _warnings
-import marshal
-
-
-_MS_WINDOWS = (sys.platform == 'win32')
-if _MS_WINDOWS:
-    import nt as _os
-    import winreg
-else:
-    import posix as _os
-
-
-if _MS_WINDOWS:
-    path_separators = ['\\', '/']
-else:
-    path_separators = ['/']
-# Assumption made in _path_join()
-assert all(len(sep) == 1 for sep in path_separators)
-path_sep = path_separators[0]
-path_sep_tuple = tuple(path_separators)
-path_separators = ''.join(path_separators)
-_pathseps_with_colon = {f':{s}' for s in path_separators}
-
-
 # Bootstrap-related code ######################################################
 _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win',
 _CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin'
@@ -65,16 +34,14 @@ def _make_relax_case():
             key = b'PYTHONCASEOK'
 
         def _relax_case():
-            """True if filenames must be checked case-insensitively and ignore environment flags are not set."""
-            return not sys.flags.ignore_environment and key in _os.environ
+            """True if filenames must be checked case-insensitively."""
+            return key in _os.environ
     else:
         def _relax_case():
             """True if filenames must be checked case-insensitively."""
             return False
     return _relax_case
 
-_relax_case = _make_relax_case()
-
 
 def _pack_uint32(x):
     """Convert a 32-bit integer to little-endian."""
@@ -92,49 +59,22 @@ def _unpack_uint16(data):
     return int.from_bytes(data, 'little')
 
 
-if _MS_WINDOWS:
-    def _path_join(*path_parts):
-        """Replacement for os.path.join()."""
-        if not path_parts:
-            return ""
-        if len(path_parts) == 1:
-            return path_parts[0]
-        root = ""
-        path = []
-        for new_root, tail in map(_os._path_splitroot, path_parts):
-            if new_root.startswith(path_sep_tuple) or new_root.endswith(path_sep_tuple):
-                root = new_root.rstrip(path_separators) or root
-                path = [path_sep + tail]
-            elif new_root.endswith(':'):
-                if root.casefold() != new_root.casefold():
-                    # Drive relative paths have to be resolved by the OS, so we reset the
-                    # tail but do not add a path_sep prefix.
-                    root = new_root
-                    path = [tail]
-                else:
-                    path.append(tail)
-            else:
-                root = new_root or root
-                path.append(tail)
-        path = [p.rstrip(path_separators) for p in path if p]
-        if len(path) == 1 and not path[0]:
-            # Avoid losing the root's trailing separator when joining with nothing
-            return root + path_sep
-        return root + path_sep.join(path)
-
-else:
-    def _path_join(*path_parts):
-        """Replacement for os.path.join()."""
-        return path_sep.join([part.rstrip(path_separators)
-                              for part in path_parts if part])
+def _path_join(*path_parts):
+    """Replacement for os.path.join()."""
+    return path_sep.join([part.rstrip(path_separators)
+                          for part in path_parts if part])
 
 
 def _path_split(path):
     """Replacement for os.path.split()."""
-    i = max(path.rfind(p) for p in path_separators)
-    if i < 0:
-        return '', path
-    return path[:i], path[i + 1:]
+    if len(path_separators) == 1:
+        front, _, tail = path.rpartition(path_sep)
+        return front, tail
+    for x in reversed(path):
+        if x in path_separators:
+            front, tail = path.rsplit(x, maxsplit=1)
+            return front, tail
+    return '', path
 
 
 def _path_stat(path):
@@ -168,18 +108,13 @@ def _path_isdir(path):
     return _path_is_mode_type(path, 0o040000)
 
 
-if _MS_WINDOWS:
-    def _path_isabs(path):
-        """Replacement for os.path.isabs."""
-        if not path:
-            return False
-        root = _os._path_splitroot(path)[0].replace('/', '\\')
-        return len(root) > 1 and (root.startswith('\\\\') or root.endswith('\\'))
+def _path_isabs(path):
+    """Replacement for os.path.isabs.
 
-else:
-    def _path_isabs(path):
-        """Replacement for os.path.isabs."""
-        return path.startswith(path_separators)
+    Considers a Windows drive-relative path (no drive, but starts with slash) to
+    still be "absolute".
+    """
+    return path.startswith(path_separators) or path[1:3] in _pathseps_with_colon
 
 
 def _write_atomic(path, data, mode=0o666):
@@ -336,23 +271,6 @@ _code_type = type(_write_atomic.__code__)
 #     Python 3.8b2  3412 (Swap the position of positional args and positional
 #                         only args in ast.arguments #37593)
 #     Python 3.8b4  3413 (Fix "break" and "continue" in "finally" #37830)
-#     Python 3.9a0  3420 (add LOAD_ASSERTION_ERROR #34880)
-#     Python 3.9a0  3421 (simplified bytecode for with blocks #32949)
-#     Python 3.9a0  3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387)
-#     Python 3.9a2  3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156)
-#     Python 3.9a2  3424 (simplify bytecodes for *value unpacking)
-#     Python 3.9a2  3425 (simplify bytecodes for **value unpacking)
-#     Python 3.10a1 3430 (Make 'annotations' future by default)
-#     Python 3.10a1 3431 (New line number table format -- PEP 626)
-#     Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202)
-#     Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0)
-#     Python 3.10a6 3434 (PEP 634: Structural Pattern Matching)
-#     Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets).
-#     Python 3.10b1 3436 (Add GEN_START bytecode #43683)
-#     Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!)
-#     Python 3.10b1 3438 Safer line number table handling.
-#     Python 3.10b1 3439 (Add ROT_N)
-
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually
@@ -361,17 +279,13 @@ _code_type = type(_write_atomic.__code__)
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3439).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3410).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'
 _OPT = 'opt-'
 
-SOURCE_SUFFIXES = ['.py']
-if _MS_WINDOWS:
-    SOURCE_SUFFIXES.append('.pyw')
-
-EXTENSION_SUFFIXES = _imp.extension_suffixes()
+SOURCE_SUFFIXES = ['.py']  # _setup() adds .pyw as needed.
 
 BYTECODE_SUFFIXES = ['.pyc']
 # Deprecated.
@@ -546,18 +460,15 @@ def _check_name(method):
             raise ImportError('loader for %s cannot handle %s' %
                                 (self.name, name), name=name)
         return method(self, name, *args, **kwargs)
-
-    # FIXME: @_check_name is used to define class methods before the
-    # _bootstrap module is set by _set_bootstrap_module().
-    if _bootstrap is not None:
+    try:
         _wrap = _bootstrap._wrap
-    else:
+    except NameError:
+        # XXX yuck
         def _wrap(new, old):
             for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
                 if hasattr(old, replace):
                     setattr(new, replace, getattr(old, replace))
             new.__dict__.update(old.__dict__)
-
     _wrap(_check_name_wrapper, method)
     return _check_name_wrapper
 
@@ -569,9 +480,6 @@ def _find_module_shim(self, fullname):
     This method is deprecated in favor of finder.find_spec().
 
     """
-    _warnings.warn("find_module() is deprecated and "
-                   "slated for removal in Python 3.12; use find_spec() instead",
-                   DeprecationWarning)
     # Call find_loader(). If it returns a string (indicating this
     # is a namespace package portion), generate a warning and
     # return None.
@@ -743,11 +651,6 @@ def spec_from_file_location(name, location=None, *, loader=None,
                 pass
     else:
         location = _os.fspath(location)
-        if not _path_isabs(location):
-            try:
-                location = _path_join(_os.getcwd(), location)
-            except OSError:
-                pass
 
     # If the location is on the filesystem, but doesn't actually exist,
     # we could return None here, indicating that the location is not
@@ -801,14 +704,14 @@ class WindowsRegistryFinder:
     REGISTRY_KEY_DEBUG = (
         'Software\\Python\\PythonCore\\{sys_version}'
         '\\Modules\\{fullname}\\Debug')
-    DEBUG_BUILD = (_MS_WINDOWS and '_d.pyd' in EXTENSION_SUFFIXES)
+    DEBUG_BUILD = False  # Changed in _setup()
 
-    @staticmethod
-    def _open_registry(key):
+    @classmethod
+    def _open_registry(cls, key):
         try:
-            return winreg.OpenKey(winreg.HKEY_CURRENT_USER, key)
+            return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key)
         except OSError:
-            return winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key)
+            return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key)
 
     @classmethod
     def _search_registry(cls, fullname):
@@ -820,7 +723,7 @@ class WindowsRegistryFinder:
                                   sys_version='%d.%d' % sys.version_info[:2])
         try:
             with cls._open_registry(key) as hkey:
-                filepath = winreg.QueryValue(hkey, '')
+                filepath = _winreg.QueryValue(hkey, '')
         except OSError:
             return None
         return filepath
@@ -845,12 +748,9 @@ class WindowsRegistryFinder:
     def find_module(cls, fullname, path=None):
         """Find module named in the registry.
 
-        This method is deprecated.  Use find_spec() instead.
+        This method is deprecated.  Use exec_module() instead.
 
         """
-        _warnings.warn("WindowsRegistryFinder.find_module() is deprecated and "
-                       "slated for removal in Python 3.12; use find_spec() instead",
-                       DeprecationWarning)
         spec = cls.find_spec(fullname, path)
         if spec is not None:
             return spec.loader
@@ -883,8 +783,7 @@ class _LoaderBasics:
         _bootstrap._call_with_frames_removed(exec, code, module.__dict__)
 
     def load_module(self, fullname):
-        """This method is deprecated."""
-        # Warning implemented in _load_module_shim().
+        """This module is deprecated."""
         return _bootstrap._load_module_shim(self, fullname)
 
 
@@ -1059,7 +958,7 @@ class FileLoader:
         """
         # The only reason for this method is for the name check.
         # Issue #14857: Avoid the zero-argument form of super so the implementation
-        # of that form can be updated without breaking the frozen module.
+        # of that form can be updated without breaking the frozen module
         return super(FileLoader, self).load_module(fullname)
 
     @_check_name
@@ -1076,10 +975,32 @@ class FileLoader:
             with _io.FileIO(path, 'r') as file:
                 return file.read()
 
+    # ResourceReader ABC API.
+
     @_check_name
     def get_resource_reader(self, module):
-        from importlib.readers import FileReader
-        return FileReader(self)
+        if self.is_package(module):
+            return self
+        return None
+
+    def open_resource(self, resource):
+        path = _path_join(_path_split(self.path)[0], resource)
+        return _io.FileIO(path, 'r')
+
+    def resource_path(self, resource):
+        if not self.is_resource(resource):
+            raise FileNotFoundError
+        path = _path_join(_path_split(self.path)[0], resource)
+        return path
+
+    def is_resource(self, name):
+        if path_sep in name:
+            return False
+        path = _path_join(_path_split(self.path)[0], name)
+        return _path_isfile(path)
+
+    def contents(self):
+        return iter(_os.listdir(_path_split(self.path)[0]))
 
 
 class SourceFileLoader(FileLoader, SourceLoader):
@@ -1152,6 +1073,10 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics):
         return None
 
 
+# Filled in by _setup().
+EXTENSION_SUFFIXES = []
+
+
 class ExtensionFileLoader(FileLoader, _LoaderBasics):
 
     """Loader for extension modules.
@@ -1212,15 +1137,10 @@ class _NamespacePath:
     using path_finder.  For top-level modules, the parent module's path
     is sys.path."""
 
-    # When invalidate_caches() is called, this epoch is incremented
-    # https://bugs.python.org/issue45703
-    _epoch = 0
-
     def __init__(self, name, path, path_finder):
         self._name = name
         self._path = path
         self._last_parent_path = tuple(self._get_parent_path())
-        self._last_epoch = self._epoch
         self._path_finder = path_finder
 
     def _find_parent_path_names(self):
@@ -1240,7 +1160,7 @@ class _NamespacePath:
     def _recalculate(self):
         # If the parent's path has changed, recalculate _path
         parent_path = tuple(self._get_parent_path()) # Make a copy
-        if parent_path != self._last_parent_path or self._epoch != self._last_epoch:
+        if parent_path != self._last_parent_path:
             spec = self._path_finder(self._name, parent_path)
             # Note that no changes are made if a loader is returned, but we
             #  do remember the new parent path
@@ -1248,7 +1168,6 @@ class _NamespacePath:
                 if spec.submodule_search_locations:
                     self._path = spec.submodule_search_locations
             self._last_parent_path = parent_path     # Save the copy
-            self._last_epoch = self._epoch
         return self._path
 
     def __iter__(self):
@@ -1278,15 +1197,13 @@ class _NamespaceLoader:
     def __init__(self, name, path, path_finder):
         self._path = _NamespacePath(name, path, path_finder)
 
-    @staticmethod
-    def module_repr(module):
+    @classmethod
+    def module_repr(cls, module):
         """Return repr for the module.
 
         The method is deprecated.  The import machinery does the job itself.
 
         """
-        _warnings.warn("_NamespaceLoader.module_repr() is deprecated and "
-                       "slated for removal in Python 3.12", DeprecationWarning)
         return '<module {!r} (namespace)>'.format(module.__name__)
 
     def is_package(self, fullname):
@@ -1313,13 +1230,8 @@ class _NamespaceLoader:
         # The import system never calls this method.
         _bootstrap._verbose_message('namespace module loaded with path {!r}',
                                     self._path)
-        # Warning implemented in _load_module_shim().
         return _bootstrap._load_module_shim(self, fullname)
 
-    def get_resource_reader(self, module):
-        from importlib.readers import NamespaceReader
-        return NamespaceReader(self._path)
-
 
 # Finders #####################################################################
 
@@ -1327,8 +1239,8 @@ class PathFinder:
 
     """Meta path finder for sys.path and package __path__ attributes."""
 
-    @staticmethod
-    def invalidate_caches():
+    @classmethod
+    def invalidate_caches(cls):
         """Call the invalidate_caches() method on all path entry finders
         stored in sys.path_importer_caches (where implemented)."""
         for name, finder in list(sys.path_importer_cache.items()):
@@ -1336,12 +1248,9 @@ class PathFinder:
                 del sys.path_importer_cache[name]
             elif hasattr(finder, 'invalidate_caches'):
                 finder.invalidate_caches()
-        # Also invalidate the caches of _NamespacePaths
-        # https://bugs.python.org/issue45703
-        _NamespacePath._epoch += 1
 
-    @staticmethod
-    def _path_hooks(path):
+    @classmethod
+    def _path_hooks(cls, path):
         """Search sys.path_hooks for a finder for 'path'."""
         if sys.path_hooks is not None and not sys.path_hooks:
             _warnings.warn('sys.path_hooks is empty', ImportWarning)
@@ -1380,14 +1289,8 @@ class PathFinder:
         # This would be a good place for a DeprecationWarning if
         # we ended up going that route.
         if hasattr(finder, 'find_loader'):
-            msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; "
-                    "falling back to find_loader()")
-            _warnings.warn(msg, ImportWarning)
             loader, portions = finder.find_loader(fullname)
         else:
-            msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; "
-                    "falling back to find_module()")
-            _warnings.warn(msg, ImportWarning)
             loader = finder.find_module(fullname)
             portions = []
         if loader is not None:
@@ -1460,16 +1363,13 @@ class PathFinder:
         This method is deprecated.  Use find_spec() instead.
 
         """
-        _warnings.warn("PathFinder.find_module() is deprecated and "
-                       "slated for removal in Python 3.12; use find_spec() instead",
-                       DeprecationWarning)
         spec = cls.find_spec(fullname, path)
         if spec is None:
             return None
         return spec.loader
 
-    @staticmethod
-    def find_distributions(*args, **kwargs):
+    @classmethod
+    def find_distributions(cls, *args, **kwargs):
         """
         Find distributions.
 
@@ -1501,8 +1401,6 @@ class FileFinder:
         self._loaders = loaders
         # Base (directory) path
         self.path = path or '.'
-        if not _path_isabs(self.path):
-            self.path = _path_join(_os.getcwd(), self.path)
         self._path_mtime = -1
         self._path_cache = set()
         self._relaxed_path_cache = set()
@@ -1520,9 +1418,6 @@ class FileFinder:
         This method is deprecated.  Use find_spec() instead.
 
         """
-        _warnings.warn("FileFinder.find_loader() is deprecated and "
-                       "slated for removal in Python 3.12; use find_spec() instead",
-                       DeprecationWarning)
         spec = self.find_spec(fullname)
         if spec is None:
             return None, []
@@ -1568,10 +1463,7 @@ class FileFinder:
                 is_namespace = _path_isdir(base_path)
         # Check for a file w/ a proper suffix exists.
         for suffix, loader_class in self._loaders:
-            try:
-                full_path = _path_join(self.path, tail_module + suffix)
-            except ValueError:
-                return None
+            full_path = _path_join(self.path, tail_module + suffix)
             _bootstrap._verbose_message('trying {}', full_path, verbosity=2)
             if cache_module + suffix in cache:
                 if _path_isfile(full_path):
@@ -1673,14 +1565,77 @@ def _get_supported_file_loaders():
     return [extensions, source, bytecode]
 
 
-def _set_bootstrap_module(_bootstrap_module):
-    global _bootstrap
+def _setup(_bootstrap_module):
+    """Setup the path-based importers for importlib by importing needed
+    built-in modules and injecting them into the global namespace.
+
+    Other components are extracted from the core bootstrap module.
+
+    """
+    global sys, _imp, _bootstrap
     _bootstrap = _bootstrap_module
+    sys = _bootstrap.sys
+    _imp = _bootstrap._imp
+
+    # Directly load built-in modules needed during bootstrap.
+    self_module = sys.modules[__name__]
+    for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
+        if builtin_name not in sys.modules:
+            builtin_module = _bootstrap._builtin_from_name(builtin_name)
+        else:
+            builtin_module = sys.modules[builtin_name]
+        setattr(self_module, builtin_name, builtin_module)
+
+    # Directly load the os module (needed during bootstrap).
+    os_details = ('posix', ['/']), ('nt', ['\\', '/'])
+    for builtin_os, path_separators in os_details:
+        # Assumption made in _path_join()
+        assert all(len(sep) == 1 for sep in path_separators)
+        path_sep = path_separators[0]
+        if builtin_os in sys.modules:
+            os_module = sys.modules[builtin_os]
+            break
+        else:
+            try:
+                os_module = _bootstrap._builtin_from_name(builtin_os)
+                break
+            except ImportError:
+                continue
+    else:
+        raise ImportError('importlib requires posix or nt')
+    setattr(self_module, '_os', os_module)
+    setattr(self_module, 'path_sep', path_sep)
+    setattr(self_module, 'path_separators', ''.join(path_separators))
+    setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators})
+
+    # Directly load the _thread module (needed during bootstrap).
+    try:
+        thread_module = _bootstrap._builtin_from_name('_thread')
+    except ImportError:
+        thread_module = None
+    setattr(self_module, '_thread', thread_module)
+
+    # Directly load the _weakref module (needed during bootstrap).
+    weakref_module = _bootstrap._builtin_from_name('_weakref')
+    setattr(self_module, '_weakref', weakref_module)
+
+    # Directly load the winreg module (needed during bootstrap).
+    if builtin_os == 'nt':
+        winreg_module = _bootstrap._builtin_from_name('winreg')
+        setattr(self_module, '_winreg', winreg_module)
+
+    # Constants
+    setattr(self_module, '_relax_case', _make_relax_case())
+    EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
+    if builtin_os == 'nt':
+        SOURCE_SUFFIXES.append('.pyw')
+        if '_d.pyd' in EXTENSION_SUFFIXES:
+            WindowsRegistryFinder.DEBUG_BUILD = True
 
 
 def _install(_bootstrap_module):
     """Install the path-based import components."""
-    _set_bootstrap_module(_bootstrap_module)
+    _setup(_bootstrap_module)
     supported_loaders = _get_supported_file_loaders()
     sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
     sys.meta_path.append(PathFinder)

Lib/importlib/_common.py

@@ -1,118 +0,0 @@
-import os
-import pathlib
-import tempfile
-import functools
-import contextlib
-import types
-import importlib
-
-from typing import Union, Any, Optional
-from .abc import ResourceReader, Traversable
-
-from ._adapters import wrap_spec
-
-Package = Union[types.ModuleType, str]
-
-
-def files(package):
-    # type: (Package) -> Traversable
-    """
-    Get a Traversable resource from a package
-    """
-    return from_package(get_package(package))
-
-
-def normalize_path(path):
-    # type: (Any) -> str
-    """Normalize a path by ensuring it is a string.
-
-    If the resulting string contains path separators, an exception is raised.
-    """
-    str_path = str(path)
-    parent, file_name = os.path.split(str_path)
-    if parent:
-        raise ValueError(f'{path!r} must be only a file name')
-    return file_name
-
-
-def get_resource_reader(package):
-    # type: (types.ModuleType) -> Optional[ResourceReader]
-    """
-    Return the package's loader if it's a ResourceReader.
-    """
-    # We can't use
-    # a issubclass() check here because apparently abc.'s __subclasscheck__()
-    # hook wants to create a weak reference to the object, but
-    # zipimport.zipimporter does not support weak references, resulting in a
-    # TypeError.  That seems terrible.
-    spec = package.__spec__
-    reader = getattr(spec.loader, 'get_resource_reader', None)  # type: ignore
-    if reader is None:
-        return None
-    return reader(spec.name)  # type: ignore
-
-
-def resolve(cand):
-    # type: (Package) -> types.ModuleType
-    return cand if isinstance(cand, types.ModuleType) else importlib.import_module(cand)
-
-
-def get_package(package):
-    # type: (Package) -> types.ModuleType
-    """Take a package name or module object and return the module.
-
-    Raise an exception if the resolved module is not a package.
-    """
-    resolved = resolve(package)
-    if wrap_spec(resolved).submodule_search_locations is None:
-        raise TypeError(f'{package!r} is not a package')
-    return resolved
-
-
-def from_package(package):
-    """
-    Return a Traversable object for the given package.
-
-    """
-    spec = wrap_spec(package)
-    reader = spec.loader.get_resource_reader(spec.name)
-    return reader.files()
-
-
-@contextlib.contextmanager
-def _tempfile(reader, suffix='',
-              # gh-93353: Keep a reference to call os.remove() in late Python
-              # finalization.
-              *, _os_remove=os.remove):
-    # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try'
-    # blocks due to the need to close the temporary file to work on Windows
-    # properly.
-    fd, raw_path = tempfile.mkstemp(suffix=suffix)
-    try:
-        os.write(fd, reader())
-        os.close(fd)
-        del reader
-        yield pathlib.Path(raw_path)
-    finally:
-        try:
-            _os_remove(raw_path)
-        except FileNotFoundError:
-            pass
-
-
-@functools.singledispatch
-def as_file(path):
-    """
-    Given a Traversable object, return that object as a
-    path on the local file system in a context manager.
-    """
-    return _tempfile(path.read_bytes, suffix=path.name)
-
-
-@as_file.register(pathlib.Path)
-@contextlib.contextmanager
-def _(path):
-    """
-    Degenerate behavior for pathlib.Path objects.
-    """
-    yield path

Lib/importlib/abc.py

@@ -1,4 +1,5 @@
 """Abstract base classes related to import."""
+from . import _bootstrap
 from . import _bootstrap_external
 from . import machinery
 try:
@@ -9,13 +10,10 @@ except ImportError as exc:
     _frozen_importlib = None
 try:
     import _frozen_importlib_external
-except ImportError:
+except ImportError as exc:
     _frozen_importlib_external = _bootstrap_external
-from ._abc import Loader
 import abc
 import warnings
-from typing import BinaryIO, Iterable, Text
-from typing import Protocol, runtime_checkable
 
 
 def _register(abstract_cls, *classes):
@@ -41,27 +39,15 @@ class Finder(metaclass=abc.ABCMeta):
     Deprecated since Python 3.3
     """
 
-    def __init__(self):
-        warnings.warn("the Finder ABC is deprecated and "
-                       "slated for removal in Python 3.12; use MetaPathFinder "
-                       "or PathEntryFinder instead",
-                       DeprecationWarning)
-
     @abc.abstractmethod
     def find_module(self, fullname, path=None):
         """An abstract method that should find a module.
         The fullname is a str and the optional path is a str or None.
         Returns a Loader object or None.
         """
-        warnings.warn("importlib.abc.Finder along with its find_module() "
-                      "method are deprecated and "
-                       "slated for removal in Python 3.12; use "
-                       "MetaPathFinder.find_spec() or "
-                       "PathEntryFinder.find_spec() instead",
-                       DeprecationWarning)
 
 
-class MetaPathFinder(metaclass=abc.ABCMeta):
+class MetaPathFinder(Finder):
 
     """Abstract base class for import finders on sys.meta_path."""
 
@@ -80,8 +66,8 @@ class MetaPathFinder(metaclass=abc.ABCMeta):
 
         """
         warnings.warn("MetaPathFinder.find_module() is deprecated since Python "
-                      "3.4 in favor of MetaPathFinder.find_spec() and is "
-                      "slated for removal in Python 3.12",
+                      "3.4 in favor of MetaPathFinder.find_spec() "
+                      "(available since 3.4)",
                       DeprecationWarning,
                       stacklevel=2)
         if not hasattr(self, 'find_spec'):
@@ -98,7 +84,7 @@ _register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter,
           machinery.PathFinder, machinery.WindowsRegistryFinder)
 
 
-class PathEntryFinder(metaclass=abc.ABCMeta):
+class PathEntryFinder(Finder):
 
     """Abstract base class for path entry finders used by PathFinder."""
 
@@ -147,6 +133,53 @@ class PathEntryFinder(metaclass=abc.ABCMeta):
 _register(PathEntryFinder, machinery.FileFinder)
 
 
+class Loader(metaclass=abc.ABCMeta):
+
+    """Abstract base class for import loaders."""
+
+    def create_module(self, spec):
+        """Return a module to initialize and into which to load.
+
+        This method should raise ImportError if anything prevents it
+        from creating a new module.  It may return None to indicate
+        that the spec should create the new module.
+        """
+        # By default, defer to default semantics for the new module.
+        return None
+
+    # We don't define exec_module() here since that would break
+    # hasattr checks we do to support backward compatibility.
+
+    def load_module(self, fullname):
+        """Return the loaded module.
+
+        The module must be added to sys.modules and have import-related
+        attributes set properly.  The fullname is a str.
+
+        ImportError is raised on failure.
+
+        This method is deprecated in favor of loader.exec_module(). If
+        exec_module() exists then it is used to provide a backwards-compatible
+        functionality for this method.
+
+        """
+        if not hasattr(self, 'exec_module'):
+            raise ImportError
+        return _bootstrap._load_module_shim(self, fullname)
+
+    def module_repr(self, module):
+        """Return a module's repr.
+
+        Used by the module type when the method does not raise
+        NotImplementedError.
+
+        This method is deprecated.
+
+        """
+        # The exception will cause ModuleType.__repr__ to ignore this method.
+        raise NotImplementedError
+
+
 class ResourceLoader(Loader):
 
     """Abstract base class for loaders which can return data from their
@@ -310,133 +343,46 @@ _register(SourceLoader, machinery.SourceFileLoader)
 
 
 class ResourceReader(metaclass=abc.ABCMeta):
-    """Abstract base class for loaders to provide resource reading support."""
+
+    """Abstract base class to provide resource-reading support.
+
+    Loaders that support resource reading are expected to implement
+    the ``get_resource_reader(fullname)`` method and have it either return None
+    or an object compatible with this ABC.
+    """
 
     @abc.abstractmethod
-    def open_resource(self, resource: Text) -> BinaryIO:
+    def open_resource(self, resource):
         """Return an opened, file-like object for binary reading.
 
-        The 'resource' argument is expected to represent only a file name.
+        The 'resource' argument is expected to represent only a file name
+        and thus not contain any subdirectory components.
+
         If the resource cannot be found, FileNotFoundError is raised.
         """
-        # This deliberately raises FileNotFoundError instead of
-        # NotImplementedError so that if this method is accidentally called,
-        # it'll still do the right thing.
         raise FileNotFoundError
 
     @abc.abstractmethod
-    def resource_path(self, resource: Text) -> Text:
+    def resource_path(self, resource):
         """Return the file system path to the specified resource.
 
-        The 'resource' argument is expected to represent only a file name.
+        The 'resource' argument is expected to represent only a file name
+        and thus not contain any subdirectory components.
+
         If the resource does not exist on the file system, raise
         FileNotFoundError.
         """
-        # This deliberately raises FileNotFoundError instead of
-        # NotImplementedError so that if this method is accidentally called,
-        # it'll still do the right thing.
         raise FileNotFoundError
 
     @abc.abstractmethod
-    def is_resource(self, path: Text) -> bool:
-        """Return True if the named 'path' is a resource.
-
-        Files are resources, directories are not.
-        """
+    def is_resource(self, name):
+        """Return True if the named 'name' is consider a resource."""
         raise FileNotFoundError
 
     @abc.abstractmethod
-    def contents(self) -> Iterable[str]:
-        """Return an iterable of entries in `package`."""
-        raise FileNotFoundError
-
-
-@runtime_checkable
-class Traversable(Protocol):
-    """
-    An object with a subset of pathlib.Path methods suitable for
-    traversing directories and opening files.
-    """
-
-    @abc.abstractmethod
-    def iterdir(self):
-        """
-        Yield Traversable objects in self
-        """
-
-    def read_bytes(self):
-        """
-        Read contents of self as bytes
-        """
-        with self.open('rb') as strm:
-            return strm.read()
-
-    def read_text(self, encoding=None):
-        """
-        Read contents of self as text
-        """
-        with self.open(encoding=encoding) as strm:
-            return strm.read()
-
-    @abc.abstractmethod
-    def is_dir(self) -> bool:
-        """
-        Return True if self is a dir
-        """
-
-    @abc.abstractmethod
-    def is_file(self) -> bool:
-        """
-        Return True if self is a file
-        """
-
-    @abc.abstractmethod
-    def joinpath(self, child):
-        """
-        Return Traversable child in self
-        """
-
-    def __truediv__(self, child):
-        """
-        Return Traversable child in self
-        """
-        return self.joinpath(child)
-
-    @abc.abstractmethod
-    def open(self, mode='r', *args, **kwargs):
-        """
-        mode may be 'r' or 'rb' to open as text or binary. Return a handle
-        suitable for reading (same as pathlib.Path.open).
-
-        When opening as text, accepts encoding parameters such as those
-        accepted by io.TextIOWrapper.
-        """
-
-    @abc.abstractproperty
-    def name(self) -> str:
-        """
-        The base name of this object without any parent references.
-        """
-
-
-class TraversableResources(ResourceReader):
-    """
-    The required interface for providing traversable
-    resources.
-    """
-
-    @abc.abstractmethod
-    def files(self):
-        """Return a Traversable object for the loaded package."""
-
-    def open_resource(self, resource):
-        return self.files().joinpath(resource).open('rb')
-
-    def resource_path(self, resource):
-        raise FileNotFoundError(resource)
-
-    def is_resource(self, path):
-        return self.files().joinpath(path).is_file()
-
     def contents(self):
-        return (item.name for item in self.files().iterdir())
+        """Return an iterable of strings over the contents of the package."""
+        return []
+
+
+_register(ResourceReader, machinery.SourceFileLoader)

Lib/importlib/machinery.py

@@ -1,5 +1,7 @@
 """The machinery of importlib: finders, loaders, hooks, etc."""
 
+import _imp
+
 from ._bootstrap import ModuleSpec
 from ._bootstrap import BuiltinImporter
 from ._bootstrap import FrozenImporter

Lib/importlib/metadata.py

@@ -0,0 +1,566 @@
+import io
+import os
+import re
+import abc
+import csv
+import sys
+import email
+import pathlib
+import zipfile
+import operator
+import functools
+import itertools
+import posixpath
+import collections
+
+from configparser import ConfigParser
+from contextlib import suppress
+from importlib import import_module
+from importlib.abc import MetaPathFinder
+from itertools import starmap
+
+
+__all__ = [
+    'Distribution',
+    'DistributionFinder',
+    'PackageNotFoundError',
+    'distribution',
+    'distributions',
+    'entry_points',
+    'files',
+    'metadata',
+    'requires',
+    'version',
+    ]
+
+
+class PackageNotFoundError(ModuleNotFoundError):
+    """The package was not found."""
+
+
+class EntryPoint(
+        collections.namedtuple('EntryPointBase', 'name value group')):
+    """An entry point as defined by Python packaging conventions.
+
+    See `the packaging docs on entry points
+    <https://packaging.python.org/specifications/entry-points/>`_
+    for more information.
+    """
+
+    pattern = re.compile(
+        r'(?P<module>[\w.]+)\s*'
+        r'(:\s*(?P<attr>[\w.]+))?\s*'
+        r'(?P<extras>\[.*\])?\s*$'
+        )
+    """
+    A regular expression describing the syntax for an entry point,
+    which might look like:
+
+        - module
+        - package.module
+        - package.module:attribute
+        - package.module:object.attribute
+        - package.module:attr [extra1, extra2]
+
+    Other combinations are possible as well.
+
+    The expression is lenient about whitespace around the ':',
+    following the attr, and following any extras.
+    """
+
+    def load(self):
+        """Load the entry point from its definition. If only a module
+        is indicated by the value, return that module. Otherwise,
+        return the named object.
+        """
+        match = self.pattern.match(self.value)
+        module = import_module(match.group('module'))
+        attrs = filter(None, (match.group('attr') or '').split('.'))
+        return functools.reduce(getattr, attrs, module)
+
+    @property
+    def extras(self):
+        match = self.pattern.match(self.value)
+        return list(re.finditer(r'\w+', match.group('extras') or ''))
+
+    @classmethod
+    def _from_config(cls, config):
+        return [
+            cls(name, value, group)
+            for group in config.sections()
+            for name, value in config.items(group)
+            ]
+
+    @classmethod
+    def _from_text(cls, text):
+        config = ConfigParser(delimiters='=')
+        # case sensitive: https://stackoverflow.com/q/1611799/812183
+        config.optionxform = str
+        try:
+            config.read_string(text)
+        except AttributeError:  # pragma: nocover
+            # Python 2 has no read_string
+            config.readfp(io.StringIO(text))
+        return EntryPoint._from_config(config)
+
+    def __iter__(self):
+        """
+        Supply iter so one may construct dicts of EntryPoints easily.
+        """
+        return iter((self.name, self))
+
+    def __reduce__(self):
+        return (
+            self.__class__,
+            (self.name, self.value, self.group),
+            )
+
+
+class PackagePath(pathlib.PurePosixPath):
+    """A reference to a path in a package"""
+
+    def read_text(self, encoding='utf-8'):
+        with self.locate().open(encoding=encoding) as stream:
+            return stream.read()
+
+    def read_binary(self):
+        with self.locate().open('rb') as stream:
+            return stream.read()
+
+    def locate(self):
+        """Return a path-like object for this path"""
+        return self.dist.locate_file(self)
+
+
+class FileHash:
+    def __init__(self, spec):
+        self.mode, _, self.value = spec.partition('=')
+
+    def __repr__(self):
+        return '<FileHash mode: {} value: {}>'.format(self.mode, self.value)
+
+
+class Distribution:
+    """A Python distribution package."""
+
+    @abc.abstractmethod
+    def read_text(self, filename):
+        """Attempt to load metadata file given by the name.
+
+        :param filename: The name of the file in the distribution info.
+        :return: The text if found, otherwise None.
+        """
+
+    @abc.abstractmethod
+    def locate_file(self, path):
+        """
+        Given a path to a file in this distribution, return a path
+        to it.
+        """
+
+    @classmethod
+    def from_name(cls, name):
+        """Return the Distribution for the given package name.
+
+        :param name: The name of the distribution package to search for.
+        :return: The Distribution instance (or subclass thereof) for the named
+            package, if found.
+        :raises PackageNotFoundError: When the named package's distribution
+            metadata cannot be found.
+        """
+        for resolver in cls._discover_resolvers():
+            dists = resolver(DistributionFinder.Context(name=name))
+            dist = next(dists, None)
+            if dist is not None:
+                return dist
+        else:
+            raise PackageNotFoundError(name)
+
+    @classmethod
+    def discover(cls, **kwargs):
+        """Return an iterable of Distribution objects for all packages.
+
+        Pass a ``context`` or pass keyword arguments for constructing
+        a context.
+
+        :context: A ``DistributionFinder.Context`` object.
+        :return: Iterable of Distribution objects for all packages.
+        """
+        context = kwargs.pop('context', None)
+        if context and kwargs:
+            raise ValueError("cannot accept context and kwargs")
+        context = context or DistributionFinder.Context(**kwargs)
+        return itertools.chain.from_iterable(
+            resolver(context)
+            for resolver in cls._discover_resolvers()
+            )
+
+    @staticmethod
+    def at(path):
+        """Return a Distribution for the indicated metadata path
+
+        :param path: a string or path-like object
+        :return: a concrete Distribution instance for the path
+        """
+        return PathDistribution(pathlib.Path(path))
+
+    @staticmethod
+    def _discover_resolvers():
+        """Search the meta_path for resolvers."""
+        declared = (
+            getattr(finder, 'find_distributions', None)
+            for finder in sys.meta_path
+            )
+        return filter(None, declared)
+
+    @property
+    def metadata(self):
+        """Return the parsed metadata for this Distribution.
+
+        The returned object will have keys that name the various bits of
+        metadata.  See PEP 566 for details.
+        """
+        text = (
+            self.read_text('METADATA')
+            or self.read_text('PKG-INFO')
+            # This last clause is here to support old egg-info files.  Its
+            # effect is to just end up using the PathDistribution's self._path
+            # (which points to the egg-info file) attribute unchanged.
+            or self.read_text('')
+            )
+        return email.message_from_string(text)
+
+    @property
+    def version(self):
+        """Return the 'Version' metadata for the distribution package."""
+        return self.metadata['Version']
+
+    @property
+    def entry_points(self):
+        return EntryPoint._from_text(self.read_text('entry_points.txt'))
+
+    @property
+    def files(self):
+        """Files in this distribution.
+
+        :return: List of PackagePath for this distribution or None
+
+        Result is `None` if the metadata file that enumerates files
+        (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is
+        missing.
+        Result may be empty if the metadata exists but is empty.
+        """
+        file_lines = self._read_files_distinfo() or self._read_files_egginfo()
+
+        def make_file(name, hash=None, size_str=None):
+            result = PackagePath(name)
+            result.hash = FileHash(hash) if hash else None
+            result.size = int(size_str) if size_str else None
+            result.dist = self
+            return result
+
+        return file_lines and list(starmap(make_file, csv.reader(file_lines)))
+
+    def _read_files_distinfo(self):
+        """
+        Read the lines of RECORD
+        """
+        text = self.read_text('RECORD')
+        return text and text.splitlines()
+
+    def _read_files_egginfo(self):
+        """
+        SOURCES.txt might contain literal commas, so wrap each line
+        in quotes.
+        """
+        text = self.read_text('SOURCES.txt')
+        return text and map('"{}"'.format, text.splitlines())
+
+    @property
+    def requires(self):
+        """Generated requirements specified for this Distribution"""
+        reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs()
+        return reqs and list(reqs)
+
+    def _read_dist_info_reqs(self):
+        return self.metadata.get_all('Requires-Dist')
+
+    def _read_egg_info_reqs(self):
+        source = self.read_text('requires.txt')
+        return source and self._deps_from_requires_text(source)
+
+    @classmethod
+    def _deps_from_requires_text(cls, source):
+        section_pairs = cls._read_sections(source.splitlines())
+        sections = {
+            section: list(map(operator.itemgetter('line'), results))
+            for section, results in
+            itertools.groupby(section_pairs, operator.itemgetter('section'))
+            }
+        return cls._convert_egg_info_reqs_to_simple_reqs(sections)
+
+    @staticmethod
+    def _read_sections(lines):
+        section = None
+        for line in filter(None, lines):
+            section_match = re.match(r'\[(.*)\]$', line)
+            if section_match:
+                section = section_match.group(1)
+                continue
+            yield locals()
+
+    @staticmethod
+    def _convert_egg_info_reqs_to_simple_reqs(sections):
+        """
+        Historically, setuptools would solicit and store 'extra'
+        requirements, including those with environment markers,
+        in separate sections. More modern tools expect each
+        dependency to be defined separately, with any relevant
+        extras and environment markers attached directly to that
+        requirement. This method converts the former to the
+        latter. See _test_deps_from_requires_text for an example.
+        """
+        def make_condition(name):
+            return name and 'extra == "{name}"'.format(name=name)
+
+        def parse_condition(section):
+            section = section or ''
+            extra, sep, markers = section.partition(':')
+            if extra and markers:
+                markers = '({markers})'.format(markers=markers)
+            conditions = list(filter(None, [markers, make_condition(extra)]))
+            return '; ' + ' and '.join(conditions) if conditions else ''
+
+        for section, deps in sections.items():
+            for dep in deps:
+                yield dep + parse_condition(section)
+
+
+class DistributionFinder(MetaPathFinder):
+    """
+    A MetaPathFinder capable of discovering installed distributions.
+    """
+
+    class Context:
+        """
+        Keyword arguments presented by the caller to
+        ``distributions()`` or ``Distribution.discover()``
+        to narrow the scope of a search for distributions
+        in all DistributionFinders.
+
+        Each DistributionFinder may expect any parameters
+        and should attempt to honor the canonical
+        parameters defined below when appropriate.
+        """
+
+        name = None
+        """
+        Specific name for which a distribution finder should match.
+        A name of ``None`` matches all distributions.
+        """
+
+        def __init__(self, **kwargs):
+            vars(self).update(kwargs)
+
+        @property
+        def path(self):
+            """
+            The path that a distribution finder should search.
+
+            Typically refers to Python package paths and defaults
+            to ``sys.path``.
+            """
+            return vars(self).get('path', sys.path)
+
+    @abc.abstractmethod
+    def find_distributions(self, context=Context()):
+        """
+        Find distributions.
+
+        Return an iterable of all Distribution instances capable of
+        loading the metadata for packages matching the ``context``,
+        a DistributionFinder.Context instance.
+        """
+
+
+class FastPath:
+    """
+    Micro-optimized class for searching a path for
+    children.
+    """
+
+    def __init__(self, root):
+        self.root = root
+        self.base = os.path.basename(root).lower()
+
+    def joinpath(self, child):
+        return pathlib.Path(self.root, child)
+
+    def children(self):
+        with suppress(Exception):
+            return os.listdir(self.root or '')
+        with suppress(Exception):
+            return self.zip_children()
+        return []
+
+    def zip_children(self):
+        zip_path = zipfile.Path(self.root)
+        names = zip_path.root.namelist()
+        self.joinpath = zip_path.joinpath
+
+        return (
+            posixpath.split(child)[0]
+            for child in names
+            )
+
+    def is_egg(self, search):
+        base = self.base
+        return (
+            base == search.versionless_egg_name
+            or base.startswith(search.prefix)
+            and base.endswith('.egg'))
+
+    def search(self, name):
+        for child in self.children():
+            n_low = child.lower()
+            if (n_low in name.exact_matches
+                    or n_low.startswith(name.prefix)
+                    and n_low.endswith(name.suffixes)
+                    # legacy case:
+                    or self.is_egg(name) and n_low == 'egg-info'):
+                yield self.joinpath(child)
+
+
+class Prepared:
+    """
+    A prepared search for metadata on a possibly-named package.
+    """
+    normalized = ''
+    prefix = ''
+    suffixes = '.dist-info', '.egg-info'
+    exact_matches = [''][:0]
+    versionless_egg_name = ''
+
+    def __init__(self, name):
+        self.name = name
+        if name is None:
+            return
+        self.normalized = name.lower().replace('-', '_')
+        self.prefix = self.normalized + '-'
+        self.exact_matches = [
+            self.normalized + suffix for suffix in self.suffixes]
+        self.versionless_egg_name = self.normalized + '.egg'
+
+
+class MetadataPathFinder(DistributionFinder):
+    @classmethod
+    def find_distributions(cls, context=DistributionFinder.Context()):
+        """
+        Find distributions.
+
+        Return an iterable of all Distribution instances capable of
+        loading the metadata for packages matching ``context.name``
+        (or all names if ``None`` indicated) along the paths in the list
+        of directories ``context.path``.
+        """
+        found = cls._search_paths(context.name, context.path)
+        return map(PathDistribution, found)
+
+    @classmethod
+    def _search_paths(cls, name, paths):
+        """Find metadata directories in paths heuristically."""
+        return itertools.chain.from_iterable(
+            path.search(Prepared(name))
+            for path in map(FastPath, paths)
+            )
+
+
+
+class PathDistribution(Distribution):
+    def __init__(self, path):
+        """Construct a distribution from a path to the metadata directory.
+
+        :param path: A pathlib.Path or similar object supporting
+                     .joinpath(), __div__, .parent, and .read_text().
+        """
+        self._path = path
+
+    def read_text(self, filename):
+        with suppress(FileNotFoundError, IsADirectoryError, KeyError,
+                      NotADirectoryError, PermissionError):
+            return self._path.joinpath(filename).read_text(encoding='utf-8')
+    read_text.__doc__ = Distribution.read_text.__doc__
+
+    def locate_file(self, path):
+        return self._path.parent / path
+
+
+def distribution(distribution_name):
+    """Get the ``Distribution`` instance for the named package.
+
+    :param distribution_name: The name of the distribution package as a string.
+    :return: A ``Distribution`` instance (or subclass thereof).
+    """
+    return Distribution.from_name(distribution_name)
+
+
+def distributions(**kwargs):
+    """Get all ``Distribution`` instances in the current environment.
+
+    :return: An iterable of ``Distribution`` instances.
+    """
+    return Distribution.discover(**kwargs)
+
+
+def metadata(distribution_name):
+    """Get the metadata for the named package.
+
+    :param distribution_name: The name of the distribution package to query.
+    :return: An email.Message containing the parsed metadata.
+    """
+    return Distribution.from_name(distribution_name).metadata
+
+
+def version(distribution_name):
+    """Get the version string for the named package.
+
+    :param distribution_name: The name of the distribution package to query.
+    :return: The version string for the package as defined in the package's
+        "Version" metadata key.
+    """
+    return distribution(distribution_name).version
+
+
+def entry_points():
+    """Return EntryPoint objects for all installed packages.
+
+    :return: EntryPoint objects for all installed packages.
+    """
+    eps = itertools.chain.from_iterable(
+        dist.entry_points for dist in distributions())
+    by_group = operator.attrgetter('group')
+    ordered = sorted(eps, key=by_group)
+    grouped = itertools.groupby(ordered, by_group)
+    return {
+        group: tuple(eps)
+        for group, eps in grouped
+        }
+
+
+def files(distribution_name):
+    """Return a list of files for the named package.
+
+    :param distribution_name: The name of the distribution package to query.
+    :return: List of files composing the distribution.
+    """
+    return distribution(distribution_name).files
+
+
+def requires(distribution_name):
+    """
+    Return a list of requirements for the named package.
+
+    :return: An iterator of requirements, suitable for
+    packaging.requirement.Requirement.
+    """
+    return distribution(distribution_name).requires