Update dependencies

Merge pull request #5788 from coolreader18/fix-prec
Fix panic with high precision
2025-05-19 14:41:24 +09:00 · 2025-05-16 20:28:25 -05:00 · 2025-05-16 15:20:10 -05:00 · 2025-05-16 14:17:25 -05:00 · 2025-05-16 18:52:14 +09:00 · 2025-05-16 18:25:08 +09:00
633 changed files with 65877 additions and 28544 deletions
--- a/.cspell.dict/cpython.txt
+++ b/.cspell.dict/cpython.txt
@@ -0,0 +1,59 @@
 argtypes
 asdl
 asname
 augassign
 badsyntax
 basetype
 boolop
 bxor
 cached_tsver
 cellarg
 cellvar
 cellvars
 cmpop
 denom
 dictoffset
 elts
 excepthandler
 fileutils
 finalbody
 formatfloat
 freevar
 freevars
 fromlist
 heaptype
 HIGHRES
 IMMUTABLETYPE
 kwonlyarg
 kwonlyargs
 lasti
 linearise
 maxdepth
 mult
 nkwargs
 noraise
 numer
 orelse
 pathconfig
 patma
 posonlyarg
 posonlyargs
 prec
 preinitialized
 PYTHREAD_NAME
 SA_ONSTACK
 stackdepth
 stringlib
 structseq
 tok_oldval
 unaryop
 unparse
 unparser
 VARKEYWORDS
 varkwarg
 wbits
 weakreflist
 withitem
 withs
 xstat
 XXPRIME
--- a/.cspell.dict/python-more.txt
+++ b/.cspell.dict/python-more.txt
@@ -0,0 +1,257 @@
 abiflags
 abstractmethods
 aenter
 aexit
 aiter
 anext
 appendleft
 argcount
 arrayiterator
 arraytype
 asend
 asyncgen
 athrow
 backslashreplace
 baserepl
 basicsize
 bdfl
 bigcharset
 bignum
 breakpointhook
 cformat
 chunksize
 classcell
 closefd
 closesocket
 codepoint
 codepoints
 codesize
 contextvar
 cpython
 cratio
 dealloc
 debugbuild
 decompressor
 defaultaction
 descr
 dictcomp
 dictitems
 dictkeys
 dictview
 digestmod
 dllhandle
 docstring
 docstrings
 dunder
 endianness
 endpos
 eventmask
 excepthook
 exceptiongroup
 exitfuncs
 extendleft
 fastlocals
 fdel
 fedcba
 fget
 fileencoding
 fillchar
 fillvalue
 finallyhandler
 firstiter
 firstlineno
 fnctl
 frombytes
 fromhex
 fromunicode
 fset
 fspath
 fstring
 fstrings
 ftruncate
 genexpr
 getattro
 getcodesize
 getdefaultencoding
 getfilesystemencodeerrors
 getfilesystemencoding
 getformat
 getframe
 getnewargs
 getpip
 getrandom
 getrecursionlimit
 getrefcount
 getsizeof
 getweakrefcount
 getweakrefs
 getwindowsversion
 gmtoff
 groupdict
 groupindex
 hamt
 hostnames
 idfunc
 idiv
 idxs
 impls
 indexgroup
 infj
 instancecheck
 instanceof
 irepeat
 isabstractmethod
 isbytes
 iscased
 isfinal
 istext
 itemiterator
 itemsize
 iternext
 keepends
 keyfunc
 keyiterator
 kwarg
 kwargs
 kwdefaults
 kwonlyargcount
 lastgroup
 lastindex
 linearization
 linearize
 listcomp
 longrange
 lvalue
 mappingproxy
 maskpri
 maxdigits
 MAXGROUPS
 MAXREPEAT
 maxsplit
 maxunicode
 memoryview
 memoryviewiterator
 metaclass
 metaclasses
 metatype
 mformat
 mro
 mros
 multiarch
 namereplace
 nanj
 nbytes
 ncallbacks
 ndigits
 ndim
 nldecoder
 nlocals
 NOARGS
 nonbytes
 Nonprintable
 origname
 ospath
 pendingcr
 phello
 platlibdir
 popleft
 posixsubprocess
 posonly
 posonlyargcount
 prepending
 profilefunc
 pycache
 pycodecs
 pycs
 pyexpat
 PYTHONBREAKPOINT
 PYTHONDEBUG
 PYTHONHASHSEED
 PYTHONHOME
 PYTHONINSPECT
 PYTHONOPTIMIZE
 PYTHONPATH
 PYTHONPATH
 PYTHONSAFEPATH
 PYTHONVERBOSE
 PYTHONWARNDEFAULTENCODING
 PYTHONWARNINGS
 pytraverse
 PYVENV
 qualname
 quotetabs
 radd
 rdiv
 rdivmod
 readall
 readbuffer
 reconstructor
 refcnt
 releaselevel
 reverseitemiterator
 reverseiterator
 reversekeyiterator
 reversevalueiterator
 rfloordiv
 rlshift
 rmod
 rpow
 rrshift
 rsub
 rtruediv
 rvalue
 scproxy
 seennl
 setattro
 setcomp
 setrecursionlimit
 showwarnmsg
 signum
 slotnames
 STACKLESS
 stacklevel
 stacksize
 startpos
 subclassable
 subclasscheck
 subclasshook
 suboffset
 suboffsets
 SUBPATTERN
 sumprod
 surrogateescape
 surrogatepass
 sysconf
 sysconfigdata
 sysvars
 teedata
 thisclass
 titlecased
 tkapp
 tobytes
 tolist
 toreadonly
 TPFLAGS
 tracefunc
 unimportable
 unionable
 unraisablehook
 unsliceable
 urandom
 valueiterator
 vararg
 varargs
 varnames
 warningregistry
 warnmsg
 warnoptions
 warnopts
 weaklist
 weakproxy
 weakrefs
 winver
 withdata
 xmlcharrefreplace
 xoptions
 xopts
 yieldfrom
--- a/.cspell.dict/rust-more.txt
+++ b/.cspell.dict/rust-more.txt
@@ -0,0 +1,82 @@
 ahash
 arrayvec
 bidi
 biguint
 bindgen
 bitflags
 bitor
 bstr
 byteorder
 byteset
 caseless
 chrono
 consts
 cranelift
 cstring
 datelike
 deserializer
 fdiv
 flamescope
 flate2
 fract
 getres
 hasher
 hexf
 hexversion
 idents
 illumos
 indexmap
 insta
 keccak
 lalrpop
 lexopt
 libc
 libloading
 libz
 longlong
 Manually
 maplit
 memmap
 memmem
 metas
 modpow
 msvc
 muldiv
 nanos
 nonoverlapping
 objclass
 peekable
 powc
 powf
 powi
 prepended
 punct
 replacen
 rmatch
 rposition
 rsplitn
 rustc
 rustfmt
 rustyline
 seedable
 seekfrom
 siphash
 siphasher
 splitn
 subsec
 thiserror
 timelike
 timsort
 trai
 ulonglong
 unic
 unistd
 unraw
 unsync
 wasip1
 wasip2
 wasmbind
 wasmtime
 widestring
 winapi
 winsock
--- a/.cspell.json
+++ b/.cspell.json
@@ -1,210 +1,81 @@
 // See: https://github.com/streetsidesoftware/cspell/tree/master/packages/cspell
 {
  "version": "0.2",
  "import": [
    "@cspell/dict-en_us/cspell-ext.json",
    // "@cspell/dict-cpp/cspell-ext.json",
    "@cspell/dict-python/cspell-ext.json",
    "@cspell/dict-rust/cspell-ext.json",
    "@cspell/dict-win32/cspell-ext.json",
    "@cspell/dict-shell/cspell-ext.json",
  ],
  // language - current active spelling language
  "language": "en",
  // dictionaries - list of the names of the dictionaries to use
  "dictionaries": [
    "cpython",  // Sometimes keeping same terms with cpython is easy
    "python-more",  // Python API terms not listed in python
    "rust-more",  // Rust API terms not listed in rust
    "en_US",
    "softwareTerms",
    "c",
    "cpp",
    "python",
    "python-custom",
    "rust",
-    "unix",
+    "shell",
-    "posix",
+    "win32"
    "winapi"
  ],
  // dictionaryDefinitions - this list defines any custom dictionaries to use
-  "dictionaryDefinitions": [],
+  "dictionaryDefinitions": [
    {
      "name": "cpython",
      "path": "./.cspell.dict/cpython.txt"
    },
    {
      "name": "python-more",
      "path": "./.cspell.dict/python-more.txt"
    },
    {
      "name": "rust-more",
      "path": "./.cspell.dict/rust-more.txt"
    }
  ],
  "ignorePaths": [
    "**/__pycache__/**",
    "Lib/**"
  ],
  // words - list of words to be always considered correct
  "words": [
-    // Rust
+    "RUSTPYTHONPATH",
-    "ahash",
+    // RustPython terms
-    "bidi",
+    "aiterable",
-    "biguint",
+    "alnum",
    "bindgen",
    "bitflags",
    "bstr",
    "byteorder",
    "chrono",
    "consts",
    "cstring",
    "flate2",
    "fract",
    "hasher",
    "idents",
    "indexmap",
    "insta",
    "keccak",
    "lalrpop",
    "libc",
    "libz",
    "longlong",
    "Manually",
    "maplit",
    "memmap",
    "metas",
    "modpow",
    "nanos",
    "objclass",
    "peekable",
    "powc",
    "powf",
    "prepended",
    "punct",
    "replacen",
    "rsplitn",
    "rustc",
    "rustfmt",
    "seekfrom",
    "splitn",
    "subsec",
    "timsort",
    "trai",
    "ulonglong",
    "unic",
    "unistd",
    "winapi",
    "winsock",
    // Python
    "abstractmethods",
    "aiter",
    "anext",
    "arrayiterator",
    "arraytype",
    "asend",
    "athrow",
    "basicsize",
    "cformat",
    "classcell",
    "closesocket",
    "codepoint",
    "codepoints",
    "cpython",
    "decompressor",
    "defaultaction",
    "descr",
    "dictcomp",
    "dictitems",
    "dictkeys",
    "dictview",
    "docstring",
    "docstrings",
    "dunder",
    "eventmask",
    "fdel",
    "fget",
    "fileencoding",
    "fillchar",
    "finallyhandler",
    "frombytes",
    "fromhex",
    "fromunicode",
    "fset",
    "fspath",
    "fstring",
    "fstrings",
    "genexpr",
    "getattro",
    "getformat",
    "getnewargs",
    "getweakrefcount",
    "getweakrefs",
    "hostnames",
    "idiv",
    "impls",
    "infj",
    "instancecheck",
    "instanceof",
    "isabstractmethod",
    "itemiterator",
    "itemsize",
    "iternext",
    "keyiterator",
    "kwarg",
    "kwargs",
    "linearization",
    "linearize",
    "listcomp",
    "mappingproxy",
    "maxsplit",
    "memoryview",
    "memoryviewiterator",
    "metaclass",
    "metaclasses",
    "metatype",
    "mro",
    "mros",
    "nanj",
    "ndigits",
    "ndim",
    "nonbytes",
    "origname",
    "posixsubprocess",
    "pyexpat",
    "PYTHONDEBUG",
    "PYTHONHOME",
    "PYTHONINSPECT",
    "PYTHONOPTIMIZE",
    "PYTHONPATH",
    "PYTHONPATH",
    "PYTHONVERBOSE",
    "PYTHONWARNINGS",
    "qualname",
    "radd",
    "rdiv",
    "rdivmod",
    "reconstructor",
    "reversevalueiterator",
    "rfloordiv",
    "rlshift",
    "rmod",
    "rpow",
    "rrshift",
    "rsub",
    "rtruediv",
    "scproxy",
    "setattro",
    "setcomp",
    "showwarnmsg",
    "warnmsg",
    "stacklevel",
    "subclasscheck",
    "subclasshook",
    "unionable",
    "unraisablehook",
    "valueiterator",
    "vararg",
    "varargs",
    "varnames",
    "warningregistry",
    "warnopts",
    "weakproxy",
    "xopts",
    // RustPython
    "baseclass",
    "boxvec",
    "Bytecode",
    "cfgs",
    "codegen",
    "coro",
    "dedentations",
    "dedents",
    "deduped",
    "downcasted",
    "dumpable",
    "emscripten",
    "excs",
    "finalizer",
    "GetSet",
    "groupref",
    "internable",
    "lossily",
    "makeunicodedata",
    "miri",
    "notrace",
    "openat",
    "pyarg",
    "pyarg",
    "pyargs",
    "pyast",
    "PyAttr",
    "pyc",
    "PyClass",
@@ -213,6 +84,7 @@
    "PyFunction",
    "pygetset",
    "pyimpl",
    "pylib",
    "pymember",
    "PyMethod",
    "PyModule",
@@ -225,6 +97,7 @@
    "PyResult",
    "pyslot",
    "PyStaticMethod",
    "pystone",
    "pystr",
    "pystruct",
    "pystructseq",
@@ -232,57 +105,31 @@
    "reducelib",
    "richcompare",
    "RustPython",
    "significand",
    "struc",
    "summands", // plural of summand
    "sysmodule",
    "tracebacks",
    "typealiases",
-    "Unconstructible",
+    "unconstructible",
    "unhashable",
    "uninit",
    "unraisable",
    "unresizable",
    "wasi",
    "zelf",
-    // cpython
+    // unix
-    "argtypes",
+    "CLOEXEC",
-    "asdl",
+    "codeset",
-    "asname",
+    "endgrent",
-    "augassign",
+    "gethrvtime",
-    "badsyntax",
+    "getrusage",
-    "basetype",
+    "nanosleep",
-    "boolop",
+    "sigaction",
-    "bxor",
+    "WRLCK",
-    "cellarg",
+    // win32
-    "cellvar",
+    "birthtime",
-    "cellvars",
+    "IFEXEC",
    "cmpop",
    "dictoffset",
    "elts",
    "excepthandler",
    "finalbody",
    "freevar",
    "freevars",
    "fromlist",
    "heaptype",
    "IMMUTABLETYPE",
    "kwonlyarg",
    "kwonlyargs",
    "linearise",
    "maxdepth",
    "mult",
    "nkwargs",
    "orelse",
    "patma",
    "posonlyarg",
    "posonlyargs",
    "prec",
    "stackdepth",
    "unaryop",
    "unparse",
    "unparser",
    "VARKEYWORDS",
    "varkwarg",
    "wbits",
    "withitem",
    "withs"
  ],
  // flagWords - list of words to be always considered incorrect
  "flagWords": [
--- a/.devcontainer/DOCKERFILE
+++ b/.devcontainer/DOCKERFILE
@@ -0,0 +1,6 @@
 FROM mcr.microsoft.com/vscode/devcontainers/rust:1-bullseye
 # Install clang
 RUN apt-get update \
  && apt-get install -y clang \
  && rm -rf /var/lib/apt/lists/*
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,4 +1,25 @@
 {
-  "image": "mcr.microsoft.com/devcontainers/base:jammy",
+  "name": "Rust",
-  "onCreateCommand": "curl https://sh.rustup.rs -sSf | sh -s -- -y"
+  "build": {
-}
+    "dockerfile": "Dockerfile"
  },
  "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
  "customizations": {
    "vscode": {
      "settings": {
        "lldb.executable": "/usr/bin/lldb",
        // VS Code don't watch files under ./target
        "files.watcherExclude": {
          "**/target/**": true
        },
        "extensions": [
          "rust-lang.rust-analyzer",
          "tamasfe.even-better-toml",
          "vadimcn.vscode-lldb",
          "mutantdino.resourcemonitor"
        ]
      }
    }
  },
  "remoteUser": "vscode"
 }
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,5 +1,5 @@
 Lib/**      linguist-vendored
-Cargo.lock  linguist-generated -merge
+Cargo.lock  linguist-generated
 *.snap      linguist-generated -merge
 vm/src/stdlib/ast/gen.rs    linguist-generated -merge
 Lib/*.py    text working-tree-encoding=UTF-8 eol=LF
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -0,0 +1,186 @@
 # GitHub Copilot Instructions for RustPython
 This document provides guidelines for working with GitHub Copilot when contributing to the RustPython project.
 ## Project Overview
 RustPython is a Python 3 interpreter written in Rust, implementing Python 3.13.0+ compatibility. The project aims to provide:
 - A complete Python-3 environment entirely in Rust (not CPython bindings)
 - A clean implementation without compatibility hacks
 - Cross-platform support, including WebAssembly compilation
 - The ability to embed Python scripting in Rust applications
 ## Repository Structure
 - `src/` - Top-level code for the RustPython binary
 - `vm/` - The Python virtual machine implementation
  - `builtins/` - Python built-in types and functions
  - `stdlib/` - Essential standard library modules implemented in Rust, required to run the Python core
 - `compiler/` - Python compiler components
  - `parser/` - Parser for converting Python source to AST
  - `core/` - Bytecode representation in Rust structures
  - `codegen/` - AST to bytecode compiler
 - `Lib/` - CPython's standard library in Python (copied from CPython)
 - `derive/` - Rust macros for RustPython
 - `common/` - Common utilities
 - `extra_tests/` - Integration tests and snippets
 - `stdlib/` - Non-essential Python standard library modules implemented in Rust (useful but not required for core functionality)
 - `wasm/` - WebAssembly support
 - `jit/` - Experimental JIT compiler implementation
 - `pylib/` - Python standard library packaging (do not modify this directory directly - its contents are generated automatically)
 ## Important Development Notes
 ### Running Python Code
 When testing Python code, always use RustPython instead of the standard `python` command:
 ```bash
 # Use this instead of python script.py
 cargo run -- script.py
 # For interactive REPL
 cargo run
 # With specific features
 cargo run --features ssl
 # Release mode (recommended for better performance)
 cargo run --release -- script.py
 ```
 ### Comparing with CPython
 When you need to compare behavior with CPython or run test suites:
 ```bash
 # Use python command to explicitly run CPython
 python my_test_script.py
 # Run RustPython
 cargo run -- my_test_script.py
 ```
 ### Working with the Lib Directory
 The `Lib/` directory contains Python standard library files copied from the CPython repository. Important notes:
 - These files should be edited very conservatively
 - Modifications should be minimal and only to work around RustPython limitations
 - Tests in `Lib/test` often use one of the following markers:
  - Add a `# TODO: RUSTPYTHON` comment when modifications are made
  - `unittest.skip("TODO: RustPython <reason>")`
  - `unittest.expectedFailure` with `# TODO: RUSTPYTHON <reason>` comment
 ### Testing
 ```bash
 # Run Rust unit tests
 cargo test --workspace --exclude rustpython_wasm
 # Run Python snippets tests
 cd extra_tests
 pytest -v
 # Run the Python test module
 cargo run --release -- -m test
 ```
 ### Determining What to Implement
 Run `./whats_left.py` to get a list of unimplemented methods, which is helpful when looking for contribution opportunities.
 ## Coding Guidelines
 ### Rust Code
 - Follow the default rustfmt code style (`cargo fmt` to format)
 - Use clippy to lint code (`cargo clippy`)
 - Follow Rust best practices for error handling and memory management
 - Use the macro system (`pyclass`, `pymodule`, `pyfunction`, etc.) when implementing Python functionality in Rust
 ### Python Code
 - Follow PEP 8 style for custom Python code
 - Use ruff for linting Python code
 - Minimize modifications to CPython standard library files
 ## Integration Between Rust and Python
 The project provides several mechanisms for integration:
 - `pymodule` macro for creating Python modules in Rust
 - `pyclass` macro for implementing Python classes in Rust
 - `pyfunction` macro for exposing Rust functions to Python
 - `PyObjectRef` and other types for working with Python objects in Rust
 ## Common Patterns
 ### Implementing a Python Module in Rust
 ```rust
 #[pymodule]
 mod mymodule {
    use rustpython_vm::prelude::*;
    #[pyfunction]
    fn my_function(value: i32) -> i32 {
        value * 2
    }
    #[pyattr]
    #[pyclass(name = "MyClass")]
    #[derive(Debug, PyPayload)]
    struct MyClass {
        value: usize,
    }
    #[pyclass]
    impl MyClass {
        #[pymethod]
        fn get_value(&self) -> usize {
            self.value
        }
    }
 }
 ```
 ### Adding a Python Module to the Interpreter
 ```rust
 vm.add_native_module(
    "my_module_name".to_owned(),
    Box::new(my_module::make_module),
 );
 ```
 ## Building for Different Targets
 ### WebAssembly
 ```bash
 # Build for WASM
 cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release
 ```
 ### JIT Support
 ```bash
 # Enable JIT support
 cargo run --features jit
 ```
 ### SSL Support
 ```bash
 # Enable SSL support
 cargo run --features ssl
 ```
 ## Documentation
 - Check the [architecture document](architecture/architecture.md) for a high-level overview
 - Read the [development guide](DEVELOPMENT.md) for detailed setup instructions
 - Generate documentation with `cargo doc --no-deps --all`
 - Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/)
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -16,7 +16,7 @@ concurrency:
  cancel-in-progress: true
 env:
-  CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl
  # Skip additional tests on Windows. They are checked on Linux and MacOS.
  # test_glob: many failing tests
  # test_io: many failing tests
@@ -132,6 +132,7 @@ jobs:
      - name: Set up the Windows environment
        shell: bash
        run: |
          git config --system core.longpaths true
          cargo install --target-dir=target -v cargo-vcpkg
          cargo vcpkg -v build
        if: runner.os == 'Windows'
@@ -140,7 +141,7 @@ jobs:
        if: runner.os == 'macOS'
      - name: run clippy
-        run: cargo clippy ${{ env.CARGO_ARGS }} --workspace  --exclude rustpython_wasm -- -Dwarnings
+        run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets --exclude rustpython_wasm -- -Dwarnings
      - name: run rust tests
        run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }}
@@ -255,6 +256,7 @@ jobs:
      - name: Set up the Windows environment
        shell: bash
        run: |
          git config --system core.longpaths true
          cargo install cargo-vcpkg
          cargo vcpkg build
        if: runner.os == 'Windows'
@@ -322,14 +324,26 @@ jobs:
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: install ruff
-        run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778
+        run: python -m pip install ruff==0.11.8
-      - name: run python lint
+      - name: Ensure docs generate no warnings
-        run: ruff extra_tests wasm examples  --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source
+        run: cargo doc
      - name: run ruff check
        run: ruff check --diff
      - name: run ruff format
        run: ruff format --check
      - name: install prettier
        run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH
      - name: check wasm code with prettier
        # prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506
        run: cd wasm && git ls-files -z | xargs -0 prettier --check -u
      # Keep cspell check as the last step. This is optional test.
      - name: install extra dictionaries
        run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell
      - name: spell checker
        uses: streetsidesoftware/cspell-action@v7
        with:
          files: '**/*.rs'
          incremental_files_only: true
  miri:
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
--- a/.github/workflows/cron-ci.yaml
+++ b/.github/workflows/cron-ci.yaml
@@ -84,7 +84,7 @@ jobs:
      - name: Collect what is left data
        run: |
          chmod +x ./whats_left.py
-          ./whats_left.py > whats_left.temp
+          ./whats_left.py --features "ssl,sqlite" > whats_left.temp
        env:
          RUSTPYTHONPATH: ${{ github.workspace }}/Lib
      - name: Upload data to the website
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -50,6 +50,7 @@ jobs:
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
      - uses: cargo-bins/cargo-binstall@main
      - name: Set up Environment
        shell: bash
@@ -57,6 +58,7 @@ jobs:
      - name: Set up Windows Environment
        shell: bash
        run: |
          git config --global core.longpaths true
          cargo install --target-dir=target -v cargo-vcpkg
          cargo vcpkg -v build
        if: runner.os == 'Windows'
@@ -140,7 +142,7 @@ jobs:
        uses: actions/download-artifact@v4
        with:
          path: bin
-          pattern: rustpython-release-*
+          pattern: rustpython-*
          merge-multiple: true
      - name: List Binaries
--- a/.gitignore
+++ b/.gitignore
@@ -2,11 +2,11 @@
 /*/target
 **/*.rs.bk
 **/*.bytecode
-__pycache__
+__pycache__/
 **/*.pytest_cache
 .*sw*
 .repl_history.txt
-.vscode
+.vscode/
 wasm-pack.log
 .idea/
 .envrc
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,33 +10,34 @@ repository.workspace = true
 license.workspace = true
 [features]
-default = ["threading", "stdlib", "importlib"]
+default = ["threading", "stdlib", "stdio", "importlib"]
 importlib = ["rustpython-vm/importlib"]
 encodings = ["rustpython-vm/encodings"]
 stdio = ["rustpython-vm/stdio"]
 stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"]
 flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
 freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
 threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
 bz2 = ["stdlib", "rustpython-stdlib/bz2"]
 sqlite = ["rustpython-stdlib/sqlite"]
 ssl = ["rustpython-stdlib/ssl"]
 ssl-vendor = ["ssl", "rustpython-stdlib/ssl-vendor"]
 tkinter = ["rustpython-stdlib/tkinter"]
 [dependencies]
 rustpython-compiler = { workspace = true }
 rustpython-pylib = { workspace = true, optional = true }
 rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
 rustpython-vm = { workspace = true, features = ["compiler"] }
-rustpython-parser = { workspace = true }
+ruff_python_parser = { workspace = true }
 cfg-if = { workspace = true }
 log = { workspace = true }
 flame = { workspace = true, optional = true }
-clap = "2.34"
+lexopt = "0.3"
-dirs = { package = "dirs-next", version = "2.0.0" }
+dirs = { package = "dirs-next", version = "2.0" }
-env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
+env_logger = "0.11"
 flamescope = { version = "0.1.2", optional = true }
 [target.'cfg(windows)'.dependencies]
@@ -47,7 +48,7 @@ rustyline = { workspace = true }
 [dev-dependencies]
 criterion = { workspace = true }
-pyo3 = { version = "0.22", features = ["auto-initialize"] }
+pyo3 = { version = "0.24", features = ["auto-initialize"] }
 [[bench]]
 name = "execution"
@@ -78,6 +79,7 @@ opt-level = 3
 lto = "thin"
 [patch.crates-io]
 radium = { version = "1.1.0", git = "https://github.com/youknowone/ferrilab", branch = "fix-nightly" }
 # REDOX START, Uncomment when you want to compile/check with redoxer
 # REDOX END
@@ -91,11 +93,30 @@ rev = "2024.02.14"
 [package.metadata.vcpkg.target]
 x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] }
 [package.metadata.packager]
 product-name = "RustPython"
 identifier = "com.rustpython.rustpython"
 description = "An open source Python 3 interpreter written in Rust"
 homepage = "https://rustpython.github.io/"
 license_file = "LICENSE"
 authors = ["RustPython Team"]
 publisher = "RustPython Team"
 resources = ["LICENSE", "README.md", "Lib"]
 icons = ["32x32.png"]
 [package.metadata.packager.nsis]
 installer_mode = "both"
 template = "installer-config/installer.nsi"
 [package.metadata.packager.wix]
 template = "installer-config/installer.wxs"
 [workspace]
 resolver = "2"
 members = [
-    "compiler", "compiler/core", "compiler/codegen",
+    "compiler", "compiler/core", "compiler/codegen", "compiler/literal", "compiler/source",
-    ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl",
+    ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", "wtf8",
    "wasm/lib",
 ]
@@ -108,6 +129,7 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 [workspace.dependencies]
 rustpython-compiler-source = { path = "compiler/source" }
 rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
 rustpython-compiler = { path = "compiler", version = "0.4.0" }
 rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }
@@ -115,27 +137,18 @@ rustpython-common = { path = "common", version = "0.4.0" }
 rustpython-derive = { path = "derive", version = "0.4.0" }
 rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" }
 rustpython-jit = { path = "jit", version = "0.4.0" }
 rustpython-literal = { path = "compiler/literal", version = "0.4.0" }
 rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" }
 rustpython-pylib = { path = "pylib", version = "0.4.0" }
 rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" }
 rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
 rustpython-wtf8 = { path = "wtf8", version = "0.4.0" }
 rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
-# rustpython-literal = { version = "0.4.0" }
+ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
-# rustpython-parser-core = { version = "0.4.0" }
+ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
-# rustpython-parser = { version = "0.4.0" }
+ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
-# rustpython-ast = { version = "0.4.0" }
+ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
 # rustpython-format= { version = "0.4.0" }
 rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
 rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
 rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
 rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
 rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
 # rustpython-literal = { path = "../RustPython-parser/literal" }
 # rustpython-parser-core = { path = "../RustPython-parser/core" }
 # rustpython-parser = { path = "../RustPython-parser/parser" }
 # rustpython-ast = { path = "../RustPython-parser/ast" }
 # rustpython-format = { path = "../RustPython-parser/format" }
 ahash = "0.8.11"
 ascii = "1.1"
@@ -143,33 +156,41 @@ bitflags = "2.4.2"
 bstr = "1"
 cfg-if = "1.0"
 chrono = "0.4.39"
-criterion = { version = "0.3.5", features = ["html_reports"] }
+constant_time_eq = "0.4"
 criterion = { version = "0.5", features = ["html_reports"] }
 crossbeam-utils = "0.8.21"
 flame = "0.2.2"
-getrandom = "0.3"
+getrandom = { version = "0.3", features = ["std"] }
 glob = "0.3"
 hex = "0.4.3"
 indexmap = { version = "2.2.6", features = ["std"] }
-insta = "1.38.0"
+insta = "1.42"
 itertools = "0.14.0"
 is-macro = "0.3.7"
 junction = "1.2.0"
 libc = "0.2.169"
-log = "0.4.25"
+libffi = "4.0"
 log = "0.4.27"
 nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
-malachite-bigint = "0.2.3"
+malachite-bigint = "0.6"
-malachite-q = "0.4.22"
+malachite-q = "0.6"
-malachite-base = "0.4.22"
+malachite-base = "0.6"
 memchr = "2.7.4"
 num-complex = "0.4.6"
 num-integer = "0.1.46"
 num-traits = "0.2"
 num_enum = { version = "0.7", default-features = false }
 optional = "0.5"
 once_cell = "1.20.3"
 parking_lot = "0.12.3"
 paste = "1.0.15"
 proc-macro2 = "1.0.93"
 pymath = "0.0.2"
 quote = "1.0.38"
 radium = "1.1"
 rand = "0.9"
-rustix = { version = "0.38", features = ["event"] }
+rand_core = { version = "0.9", features = ["os_rng"] }
 rustix = { version = "1.0", features = ["event"] }
 rustyline = "15.0.0"
 serde = { version = "1.0.133", default-features = false }
 schannel = "0.1.27"
@@ -179,6 +200,13 @@ strum_macros = "0.27"
 syn = "2"
 thiserror = "2.0"
 thread_local = "1.1.8"
 unicode-casing = "0.1.0"
 unic-char-property = "0.9.0"
 unic-normal = "0.9.0"
 unic-ucd-age = "0.9.0"
 unic-ucd-bidi = "0.9.0"
 unic-ucd-category = "0.9.0"
 unic-ucd-ident = "0.9.0"
 unicode_names2 = "1.3.0"
 widestring = "1.1.0"
 windows-sys = "0.59.0"
--- a/Lib/_aix_support.py
+++ b/Lib/_aix_support.py
@@ -0,0 +1,108 @@
 """Shared AIX support functions."""
 import sys
 import sysconfig
 # Taken from _osx_support _read_output function
 def _read_cmd_output(commandstring, capture_stderr=False):
    """Output from successful command execution or None"""
    # Similar to os.popen(commandstring, "r").read(),
    # but without actually using os.popen because that
    # function is not usable during python bootstrap.
    import os
    import contextlib
    fp = open("/tmp/_aix_support.%s"%(
        os.getpid(),), "w+b")
    with contextlib.closing(fp) as fp:
        if capture_stderr:
            cmd = "%s >'%s' 2>&1" % (commandstring, fp.name)
        else:
            cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
        return fp.read() if not os.system(cmd) else None
 def _aix_tag(vrtl, bd):
    # type: (List[int], int) -> str
    # Infer the ABI bitwidth from maxsize (assuming 64 bit as the default)
    _sz = 32 if sys.maxsize == (2**31-1) else 64
    _bd = bd if bd != 0 else 9988
    # vrtl[version, release, technology_level]
    return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], _bd, _sz)
 # extract version, release and technology level from a VRMF string
 def _aix_vrtl(vrmf):
    # type: (str) -> List[int]
    v, r, tl = vrmf.split(".")[:3]
    return [int(v[-1]), int(r), int(tl)]
 def _aix_bos_rte():
    # type: () -> Tuple[str, int]
    """
    Return a Tuple[str, int] e.g., ['7.1.4.34', 1806]
    The fileset bos.rte represents the current AIX run-time level. It's VRMF and
    builddate reflect the current ABI levels of the runtime environment.
    If no builddate is found give a value that will satisfy pep425 related queries
    """
    # All AIX systems to have lslpp installed in this location
    # subprocess may not be available during python bootstrap
    try:
        import subprocess
        out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.rte"])
    except ImportError:
        out = _read_cmd_output("/usr/bin/lslpp -Lqc bos.rte")
    out = out.decode("utf-8")
    out = out.strip().split(":")  # type: ignore
    _bd = int(out[-1]) if out[-1] != '' else 9988
    return (str(out[2]), _bd)
 def aix_platform():
    # type: () -> str
    """
    AIX filesets are identified by four decimal values: V.R.M.F.
    V (version) and R (release) can be retrieved using ``uname``
    Since 2007, starting with AIX 5.3 TL7, the M value has been
    included with the fileset bos.rte and represents the Technology
    Level (TL) of AIX. The F (Fix) value also increases, but is not
    relevant for comparing releases and binary compatibility.
    For binary compatibility the so-called builddate is needed.
    Again, the builddate of an AIX release is associated with bos.rte.
    AIX ABI compatibility is described  as guaranteed at: https://www.ibm.com/\
    support/knowledgecenter/en/ssw_aix_72/install/binary_compatability.html
    For pep425 purposes the AIX platform tag becomes:
    "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(v, r, tl, builddate, bitsize)
    e.g., "aix-6107-1415-32" for AIX 6.1 TL7 bd 1415, 32-bit
    and, "aix-6107-1415-64" for AIX 6.1 TL7 bd 1415, 64-bit
    """
    vrmf, bd = _aix_bos_rte()
    return _aix_tag(_aix_vrtl(vrmf), bd)
 # extract vrtl from the BUILD_GNU_TYPE as an int
 def _aix_bgt():
    # type: () -> List[int]
    gnu_type = sysconfig.get_config_var("BUILD_GNU_TYPE")
    if not gnu_type:
        raise ValueError("BUILD_GNU_TYPE is not defined")
    return _aix_vrtl(vrmf=gnu_type)
 def aix_buildtag():
    # type: () -> str
    """
    Return the platform_tag of the system Python was built on.
    """
    # AIX_BUILDDATE is defined by configure with:
    # lslpp -Lcq bos.rte | awk -F:  '{ print $NF }'
    build_date = sysconfig.get_config_var("AIX_BUILDDATE")
    try:
        build_date = int(build_date)
    except (ValueError, TypeError):
        raise ValueError(f"AIX_BUILDDATE is not defined or invalid: "
                         f"{build_date!r}")
    return _aix_tag(_aix_bgt(), build_date)
--- a/Lib/_py_abc.py
+++ b/Lib/_py_abc.py
@@ -33,6 +33,8 @@ class ABCMeta(type):
    _abc_invalidation_counter = 0
    def __new__(mcls, name, bases, namespace, /, **kwargs):
        # TODO: RUSTPYTHON remove this line (prevents duplicate bases)
        bases = tuple(dict.fromkeys(bases))
        cls = super().__new__(mcls, name, bases, namespace, **kwargs)
        # Compute set of abstract method names
        abstracts = {name
@@ -98,8 +100,8 @@ class ABCMeta(type):
        subtype = type(instance)
        if subtype is subclass:
            if (cls._abc_negative_cache_version ==
-                ABCMeta._abc_invalidation_counter and
+                    ABCMeta._abc_invalidation_counter and
-                subclass in cls._abc_negative_cache):
+                    subclass in cls._abc_negative_cache):
                return False
            # Fall back to the subclass check.
            return cls.__subclasscheck__(subclass)
--- a/Lib/_pycodecs.py
+++ b/Lib/_pycodecs.py
@@ -1086,11 +1086,13 @@ def charmapencode_output(c, mapping):
    rep = mapping[c]
    if isinstance(rep, int) or isinstance(rep, int):
        if rep < 256:
-            return rep
+            return [rep]
        else:
            raise TypeError("character mapping must be in range(256)")
    elif isinstance(rep, str):
-        return ord(rep)
+        return [ord(rep)]
    elif isinstance(rep, bytes):
        return rep
    elif rep == None:
        raise KeyError("character maps to <undefined>")
    else:
@@ -1113,12 +1115,13 @@ def PyUnicode_EncodeCharmap(p, size, mapping='latin-1', errors='strict'):
        #/* try to encode it */
        try:
            x = charmapencode_output(ord(p[inpos]), mapping)
-            res += [x]
+            res += x
        except KeyError:
            x = unicode_call_errorhandler(errors, "charmap",
            "character maps to <undefined>", p, inpos, inpos+1, False)
            try:
-                res += [charmapencode_output(ord(y), mapping) for y in x[0]]
+                for y in x[0]:
                    res += charmapencode_output(ord(y), mapping)
            except KeyError:
                raise UnicodeEncodeError("charmap", p, inpos, inpos+1,
                                        "character maps to <undefined>")
--- a/Lib/_pylong.py
+++ b/Lib/_pylong.py
@@ -0,0 +1,363 @@
 """Python implementations of some algorithms for use by longobject.c.
 The goal is to provide asymptotically faster algorithms that can be
 used for operations on integers with many digits.  In those cases, the
 performance overhead of the Python implementation is not significant
 since the asymptotic behavior is what dominates runtime. Functions
 provided by this module should be considered private and not part of any
 public API.
 Note: for ease of maintainability, please prefer clear code and avoid
 "micro-optimizations".  This module will only be imported and used for
 integers with a huge number of digits.  Saving a few microseconds with
 tricky or non-obvious code is not worth it.  For people looking for
 maximum performance, they should use something like gmpy2."""
 import re
 import decimal
 try:
    import _decimal
 except ImportError:
    _decimal = None
 # A number of functions have this form, where `w` is a desired number of
 # digits in base `base`:
 #
 #    def inner(...w...):
 #        if w <= LIMIT:
 #            return something
 #        lo = w >> 1
 #        hi = w - lo
 #        something involving base**lo, inner(...lo...), j, and inner(...hi...)
 #    figure out largest w needed
 #    result = inner(w)
 #
 # They all had some on-the-fly scheme to cache `base**lo` results for reuse.
 # Power is costly.
 #
 # This routine aims to compute all amd only the needed powers in advance, as
 # efficiently as reasonably possible. This isn't trivial, and all the
 # on-the-fly methods did needless work in many cases. The driving code above
 # changes to:
 #
 #    figure out largest w needed
 #    mycache = compute_powers(w, base, LIMIT)
 #    result = inner(w)
 #
 # and `mycache[lo]` replaces `base**lo` in the inner function.
 #
 # While this does give minor speedups (a few percent at best), the primary
 # intent is to simplify the functions using this, by eliminating the need for
 # them to craft their own ad-hoc caching schemes.
 def compute_powers(w, base, more_than, show=False):
    seen = set()
    need = set()
    ws = {w}
    while ws:
        w = ws.pop() # any element is fine to use next
        if w in seen or w <= more_than:
            continue
        seen.add(w)
        lo = w >> 1
        # only _need_ lo here; some other path may, or may not, need hi
        need.add(lo)
        ws.add(lo)
        if w & 1:
            ws.add(lo + 1)
    d = {}
    if not need:
        return d
    it = iter(sorted(need))
    first = next(it)
    if show:
        print("pow at", first)
    d[first] = base ** first
    for this in it:
        if this - 1 in d:
            if show:
                print("* base at", this)
            d[this] = d[this - 1] * base # cheap
        else:
            lo = this >> 1
            hi = this - lo
            assert lo in d
            if show:
                print("square at", this)
            # Multiplying a bigint by itself (same object!) is about twice
            # as fast in CPython.
            sq = d[lo] * d[lo]
            if hi != lo:
                assert hi == lo + 1
                if show:
                    print("    and * base")
                sq *= base
            d[this] = sq
    return d
 _unbounded_dec_context = decimal.getcontext().copy()
 _unbounded_dec_context.prec = decimal.MAX_PREC
 _unbounded_dec_context.Emax = decimal.MAX_EMAX
 _unbounded_dec_context.Emin = decimal.MIN_EMIN
 _unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check
 def int_to_decimal(n):
    """Asymptotically fast conversion of an 'int' to Decimal."""
    # Function due to Tim Peters.  See GH issue #90716 for details.
    # https://github.com/python/cpython/issues/90716
    #
    # The implementation in longobject.c of base conversion algorithms
    # between power-of-2 and non-power-of-2 bases are quadratic time.
    # This function implements a divide-and-conquer algorithm that is
    # faster for large numbers.  Builds an equal decimal.Decimal in a
    # "clever" recursive way.  If we want a string representation, we
    # apply str to _that_.
    from decimal import Decimal as D
    BITLIM = 200
    # Don't bother caching the "lo" mask in this; the time to compute it is
    # tiny compared to the multiply.
    def inner(n, w):
        if w <= BITLIM:
            return D(n)
        w2 = w >> 1
        hi = n >> w2
        lo = n & ((1 << w2) - 1)
        return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2]
    with decimal.localcontext(_unbounded_dec_context):
        nbits = n.bit_length()
        w2pow = compute_powers(nbits, D(2), BITLIM)
        if n < 0:
            negate = True
            n = -n
        else:
            negate = False
        result = inner(n, nbits)
        if negate:
            result = -result
    return result
 def int_to_decimal_string(n):
    """Asymptotically fast conversion of an 'int' to a decimal string."""
    w = n.bit_length()
    if w > 450_000 and _decimal is not None:
        # It is only usable with the C decimal implementation.
        # _pydecimal.py calls str() on very large integers, which in its
        # turn calls int_to_decimal_string(), causing very deep recursion.
        return str(int_to_decimal(n))
    # Fallback algorithm for the case when the C decimal module isn't
    # available.  This algorithm is asymptotically worse than the algorithm
    # using the decimal module, but better than the quadratic time
    # implementation in longobject.c.
    DIGLIM = 1000
    def inner(n, w):
        if w <= DIGLIM:
            return str(n)
        w2 = w >> 1
        hi, lo = divmod(n, pow10[w2])
        return inner(hi, w - w2) + inner(lo, w2).zfill(w2)
    # The estimation of the number of decimal digits.
    # There is no harm in small error.  If we guess too large, there may
    # be leading 0's that need to be stripped.  If we guess too small, we
    # may need to call str() recursively for the remaining highest digits,
    # which can still potentially be a large integer. This is manifested
    # only if the number has way more than 10**15 digits, that exceeds
    # the 52-bit physical address limit in both Intel64 and AMD64.
    w = int(w * 0.3010299956639812 + 1)  # log10(2)
    pow10 = compute_powers(w, 5, DIGLIM)
    for k, v in pow10.items():
        pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k
    if n < 0:
        n = -n
        sign = '-'
    else:
        sign = ''
    s = inner(n, w)
    if s[0] == '0' and n:
        # If our guess of w is too large, there may be leading 0's that
        # need to be stripped.
        s = s.lstrip('0')
    return sign + s
 def _str_to_int_inner(s):
    """Asymptotically fast conversion of a 'str' to an 'int'."""
    # Function due to Bjorn Martinsson.  See GH issue #90716 for details.
    # https://github.com/python/cpython/issues/90716
    #
    # The implementation in longobject.c of base conversion algorithms
    # between power-of-2 and non-power-of-2 bases are quadratic time.
    # This function implements a divide-and-conquer algorithm making use
    # of Python's built in big int multiplication. Since Python uses the
    # Karatsuba algorithm for multiplication, the time complexity
    # of this function is O(len(s)**1.58).
    DIGLIM = 2048
    def inner(a, b):
        if b - a <= DIGLIM:
            return int(s[a:b])
        mid = (a + b + 1) >> 1
        return (inner(mid, b)
                + ((inner(a, mid) * w5pow[b - mid])
                    << (b - mid)))
    w5pow = compute_powers(len(s), 5, DIGLIM)
    return inner(0, len(s))
 def int_from_string(s):
    """Asymptotically fast version of PyLong_FromString(), conversion
    of a string of decimal digits into an 'int'."""
    # PyLong_FromString() has already removed leading +/-, checked for invalid
    # use of underscore characters, checked that string consists of only digits
    # and underscores, and stripped leading whitespace.  The input can still
    # contain underscores and have trailing whitespace.
    s = s.rstrip().replace('_', '')
    return _str_to_int_inner(s)
 def str_to_int(s):
    """Asymptotically fast version of decimal string to 'int' conversion."""
    # FIXME: this doesn't support the full syntax that int() supports.
    m = re.match(r'\s*([+-]?)([0-9_]+)\s*', s)
    if not m:
        raise ValueError('invalid literal for int() with base 10')
    v = int_from_string(m.group(2))
    if m.group(1) == '-':
        v = -v
    return v
 # Fast integer division, based on code from Mark Dickinson, fast_div.py
 # GH-47701. Additional refinements and optimizations by Bjorn Martinsson.  The
 # algorithm is due to Burnikel and Ziegler, in their paper "Fast Recursive
 # Division".
 _DIV_LIMIT = 4000
 def _div2n1n(a, b, n):
    """Divide a 2n-bit nonnegative integer a by an n-bit positive integer
    b, using a recursive divide-and-conquer algorithm.
    Inputs:
      n is a positive integer
      b is a positive integer with exactly n bits
      a is a nonnegative integer such that a < 2**n * b
    Output:
      (q, r) such that a = b*q+r and 0 <= r < b.
    """
    if a.bit_length() - n <= _DIV_LIMIT:
        return divmod(a, b)
    pad = n & 1
    if pad:
        a <<= 1
        b <<= 1
        n += 1
    half_n = n >> 1
    mask = (1 << half_n) - 1
    b1, b2 = b >> half_n, b & mask
    q1, r = _div3n2n(a >> n, (a >> half_n) & mask, b, b1, b2, half_n)
    q2, r = _div3n2n(r, a & mask, b, b1, b2, half_n)
    if pad:
        r >>= 1
    return q1 << half_n | q2, r
 def _div3n2n(a12, a3, b, b1, b2, n):
    """Helper function for _div2n1n; not intended to be called directly."""
    if a12 >> n == b1:
        q, r = (1 << n) - 1, a12 - (b1 << n) + b1
    else:
        q, r = _div2n1n(a12, b1, n)
    r = (r << n | a3) - q * b2
    while r < 0:
        q -= 1
        r += b
    return q, r
 def _int2digits(a, n):
    """Decompose non-negative int a into base 2**n
    Input:
      a is a non-negative integer
    Output:
      List of the digits of a in base 2**n in little-endian order,
      meaning the most significant digit is last. The most
      significant digit is guaranteed to be non-zero.
      If a is 0 then the output is an empty list.
    """
    a_digits = [0] * ((a.bit_length() + n - 1) // n)
    def inner(x, L, R):
        if L + 1 == R:
            a_digits[L] = x
            return
        mid = (L + R) >> 1
        shift = (mid - L) * n
        upper = x >> shift
        lower = x ^ (upper << shift)
        inner(lower, L, mid)
        inner(upper, mid, R)
    if a:
        inner(a, 0, len(a_digits))
    return a_digits
 def _digits2int(digits, n):
    """Combine base-2**n digits into an int. This function is the
    inverse of `_int2digits`. For more details, see _int2digits.
    """
    def inner(L, R):
        if L + 1 == R:
            return digits[L]
        mid = (L + R) >> 1
        shift = (mid - L) * n
        return (inner(mid, R) << shift) + inner(L, mid)
    return inner(0, len(digits)) if digits else 0
 def _divmod_pos(a, b):
    """Divide a non-negative integer a by a positive integer b, giving
    quotient and remainder."""
    # Use grade-school algorithm in base 2**n, n = nbits(b)
    n = b.bit_length()
    a_digits = _int2digits(a, n)
    r = 0
    q_digits = []
    for a_digit in reversed(a_digits):
        q_digit, r = _div2n1n((r << n) + a_digit, b, n)
        q_digits.append(q_digit)
    q_digits.reverse()
    q = _digits2int(q_digits, n)
    return q, r
 def int_divmod(a, b):
    """Asymptotically fast replacement for divmod, for 'int'.
    Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b).
    """
    if b == 0:
        raise ZeroDivisionError
    elif b < 0:
        q, r = int_divmod(-a, -b)
        return q, -r
    elif a < 0:
        q, r = int_divmod(~a, b)
        return ~q, b + ~r
    else:
        return _divmod_pos(a, b)
--- a/Lib/_pyrepl/init.py
+++ b/Lib/_pyrepl/init.py
@@ -0,0 +1,19 @@
 #   Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com>
 #                       Armin Rigo
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
--- a/Lib/_pyrepl/main.py
+++ b/Lib/_pyrepl/main.py
@@ -0,0 +1,6 @@
 # Important: don't add things to this module, as they will end up in the REPL's
 # default globals.  Use _pyrepl.main instead.
 if __name__ == "__main__":
    from .main import interactive_console as __pyrepl_interactive_console
    __pyrepl_interactive_console()
--- a/Lib/_pyrepl/_minimal_curses.py
+++ b/Lib/_pyrepl/_minimal_curses.py
@@ -0,0 +1,68 @@
 """Minimal '_curses' module, the low-level interface for curses module
 which is not meant to be used directly.
 Based on ctypes.  It's too incomplete to be really called '_curses', so
 to use it, you have to import it and stick it in sys.modules['_curses']
 manually.
 Note that there is also a built-in module _minimal_curses which will
 hide this one if compiled in.
 """
 import ctypes
 import ctypes.util
 class error(Exception):
    pass
 def _find_clib() -> str:
    trylibs = ["ncursesw", "ncurses", "curses"]
    for lib in trylibs:
        path = ctypes.util.find_library(lib)
        if path:
            return path
    raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses")
 _clibpath = _find_clib()
 clib = ctypes.cdll.LoadLibrary(_clibpath)
 clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
 clib.setupterm.restype = ctypes.c_int
 clib.tigetstr.argtypes = [ctypes.c_char_p]
 clib.tigetstr.restype = ctypes.c_ssize_t
 clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int]  # type: ignore[operator]
 clib.tparm.restype = ctypes.c_char_p
 OK = 0
 ERR = -1
 # ____________________________________________________________
 def setupterm(termstr, fd):
    err = ctypes.c_int(0)
    result = clib.setupterm(termstr, fd, ctypes.byref(err))
    if result == ERR:
        raise error("setupterm() failed (err=%d)" % err.value)
 def tigetstr(cap):
    if not isinstance(cap, bytes):
        cap = cap.encode("ascii")
    result = clib.tigetstr(cap)
    if result == ERR:
        return None
    return ctypes.cast(result, ctypes.c_char_p).value
 def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0):
    result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9)
    if result is None:
        raise error("tparm() returned NULL")
    return result
--- a/Lib/_pyrepl/_threading_handler.py
+++ b/Lib/_pyrepl/_threading_handler.py
@@ -0,0 +1,74 @@
 from __future__ import annotations
 from dataclasses import dataclass, field
 import traceback
 TYPE_CHECKING = False
 if TYPE_CHECKING:
    from threading import Thread
    from types import TracebackType
    from typing import Protocol
    class ExceptHookArgs(Protocol):
        @property
        def exc_type(self) -> type[BaseException]: ...
        @property
        def exc_value(self) -> BaseException | None: ...
        @property
        def exc_traceback(self) -> TracebackType | None: ...
        @property
        def thread(self) -> Thread | None: ...
    class ShowExceptions(Protocol):
        def __call__(self) -> int: ...
        def add(self, s: str) -> None: ...
    from .reader import Reader
 def install_threading_hook(reader: Reader) -> None:
    import threading
    @dataclass
    class ExceptHookHandler:
        lock: threading.Lock = field(default_factory=threading.Lock)
        messages: list[str] = field(default_factory=list)
        def show(self) -> int:
            count = 0
            with self.lock:
                if not self.messages:
                    return 0
                reader.restore()
                for tb in self.messages:
                    count += 1
                    if tb:
                        print(tb)
                self.messages.clear()
                reader.scheduled_commands.append("ctrl-c")
                reader.prepare()
            return count
        def add(self, s: str) -> None:
            with self.lock:
                self.messages.append(s)
        def exception(self, args: ExceptHookArgs) -> None:
            lines = traceback.format_exception(
                args.exc_type,
                args.exc_value,
                args.exc_traceback,
                colorize=reader.can_colorize,
            )  # type: ignore[call-overload]
            pre = f"\nException in {args.thread.name}:\n" if args.thread else "\n"
            tb = pre + "".join(lines)
            self.add(tb)
        def __call__(self) -> int:
            return self.show()
    handler = ExceptHookHandler()
    reader.threading_hook = handler
    threading.excepthook = handler.exception
--- a/Lib/_pyrepl/commands.py
+++ b/Lib/_pyrepl/commands.py
@@ -0,0 +1,489 @@
 #   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
 #                       Antonio Cuni
 #                       Armin Rigo
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 from __future__ import annotations
 import os
 # Categories of actions:
 #  killing
 #  yanking
 #  motion
 #  editing
 #  history
 #  finishing
 # [completion]
 # types
 if False:
    from .historical_reader import HistoricalReader
 class Command:
    finish: bool = False
    kills_digit_arg: bool = True
    def __init__(
        self, reader: HistoricalReader, event_name: str, event: list[str]
    ) -> None:
        # Reader should really be "any reader" but there's too much usage of
        # HistoricalReader methods and fields in the code below for us to
        # refactor at the moment.
        self.reader = reader
        self.event = event
        self.event_name = event_name
    def do(self) -> None:
        pass
 class KillCommand(Command):
    def kill_range(self, start: int, end: int) -> None:
        if start == end:
            return
        r = self.reader
        b = r.buffer
        text = b[start:end]
        del b[start:end]
        if is_kill(r.last_command):
            if start < r.pos:
                r.kill_ring[-1] = text + r.kill_ring[-1]
            else:
                r.kill_ring[-1] = r.kill_ring[-1] + text
        else:
            r.kill_ring.append(text)
        r.pos = start
        r.dirty = True
 class YankCommand(Command):
    pass
 class MotionCommand(Command):
    pass
 class EditCommand(Command):
    pass
 class FinishCommand(Command):
    finish = True
    pass
 def is_kill(command: type[Command] | None) -> bool:
    return command is not None and issubclass(command, KillCommand)
 def is_yank(command: type[Command] | None) -> bool:
    return command is not None and issubclass(command, YankCommand)
 # etc
 class digit_arg(Command):
    kills_digit_arg = False
    def do(self) -> None:
        r = self.reader
        c = self.event[-1]
        if c == "-":
            if r.arg is not None:
                r.arg = -r.arg
            else:
                r.arg = -1
        else:
            d = int(c)
            if r.arg is None:
                r.arg = d
            else:
                if r.arg < 0:
                    r.arg = 10 * r.arg - d
                else:
                    r.arg = 10 * r.arg + d
        r.dirty = True
 class clear_screen(Command):
    def do(self) -> None:
        r = self.reader
        r.console.clear()
        r.dirty = True
 class refresh(Command):
    def do(self) -> None:
        self.reader.dirty = True
 class repaint(Command):
    def do(self) -> None:
        self.reader.dirty = True
        self.reader.console.repaint()
 class kill_line(KillCommand):
    def do(self) -> None:
        r = self.reader
        b = r.buffer
        eol = r.eol()
        for c in b[r.pos : eol]:
            if not c.isspace():
                self.kill_range(r.pos, eol)
                return
        else:
            self.kill_range(r.pos, eol + 1)
 class unix_line_discard(KillCommand):
    def do(self) -> None:
        r = self.reader
        self.kill_range(r.bol(), r.pos)
 class unix_word_rubout(KillCommand):
    def do(self) -> None:
        r = self.reader
        for i in range(r.get_arg()):
            self.kill_range(r.bow(), r.pos)
 class kill_word(KillCommand):
    def do(self) -> None:
        r = self.reader
        for i in range(r.get_arg()):
            self.kill_range(r.pos, r.eow())
 class backward_kill_word(KillCommand):
    def do(self) -> None:
        r = self.reader
        for i in range(r.get_arg()):
            self.kill_range(r.bow(), r.pos)
 class yank(YankCommand):
    def do(self) -> None:
        r = self.reader
        if not r.kill_ring:
            r.error("nothing to yank")
            return
        r.insert(r.kill_ring[-1])
 class yank_pop(YankCommand):
    def do(self) -> None:
        r = self.reader
        b = r.buffer
        if not r.kill_ring:
            r.error("nothing to yank")
            return
        if not is_yank(r.last_command):
            r.error("previous command was not a yank")
            return
        repl = len(r.kill_ring[-1])
        r.kill_ring.insert(0, r.kill_ring.pop())
        t = r.kill_ring[-1]
        b[r.pos - repl : r.pos] = t
        r.pos = r.pos - repl + len(t)
        r.dirty = True
 class interrupt(FinishCommand):
    def do(self) -> None:
        import signal
        self.reader.console.finish()
        self.reader.finish()
        os.kill(os.getpid(), signal.SIGINT)
 class ctrl_c(Command):
    def do(self) -> None:
        self.reader.console.finish()
        self.reader.finish()
        raise KeyboardInterrupt
 class suspend(Command):
    def do(self) -> None:
        import signal
        r = self.reader
        p = r.pos
        r.console.finish()
        os.kill(os.getpid(), signal.SIGSTOP)
        ## this should probably be done
        ## in a handler for SIGCONT?
        r.console.prepare()
        r.pos = p
        # r.posxy = 0, 0  # XXX this is invalid
        r.dirty = True
        r.console.screen = []
 class up(MotionCommand):
    def do(self) -> None:
        r = self.reader
        for _ in range(r.get_arg()):
            x, y = r.pos2xy()
            new_y = y - 1
            if r.bol() == 0:
                if r.historyi > 0:
                    r.select_item(r.historyi - 1)
                    return
                r.pos = 0
                r.error("start of buffer")
                return
            if (
                x
                > (
                    new_x := r.max_column(new_y)
                )  # we're past the end of the previous line
                or x == r.max_column(y)
                and any(
                    not i.isspace() for i in r.buffer[r.bol() :]
                )  # move between eols
            ):
                x = new_x
            r.setpos_from_xy(x, new_y)
 class down(MotionCommand):
    def do(self) -> None:
        r = self.reader
        b = r.buffer
        for _ in range(r.get_arg()):
            x, y = r.pos2xy()
            new_y = y + 1
            if r.eol() == len(b):
                if r.historyi < len(r.history):
                    r.select_item(r.historyi + 1)
                    r.pos = r.eol(0)
                    return
                r.pos = len(b)
                r.error("end of buffer")
                return
            if (
                x
                > (
                    new_x := r.max_column(new_y)
                )  # we're past the end of the previous line
                or x == r.max_column(y)
                and any(
                    not i.isspace() for i in r.buffer[r.bol() :]
                )  # move between eols
            ):
                x = new_x
            r.setpos_from_xy(x, new_y)
 class left(MotionCommand):
    def do(self) -> None:
        r = self.reader
        for _ in range(r.get_arg()):
            p = r.pos - 1
            if p >= 0:
                r.pos = p
            else:
                self.reader.error("start of buffer")
 class right(MotionCommand):
    def do(self) -> None:
        r = self.reader
        b = r.buffer
        for _ in range(r.get_arg()):
            p = r.pos + 1
            if p <= len(b):
                r.pos = p
            else:
                self.reader.error("end of buffer")
 class beginning_of_line(MotionCommand):
    def do(self) -> None:
        self.reader.pos = self.reader.bol()
 class end_of_line(MotionCommand):
    def do(self) -> None:
        self.reader.pos = self.reader.eol()
 class home(MotionCommand):
    def do(self) -> None:
        self.reader.pos = 0
 class end(MotionCommand):
    def do(self) -> None:
        self.reader.pos = len(self.reader.buffer)
 class forward_word(MotionCommand):
    def do(self) -> None:
        r = self.reader
        for i in range(r.get_arg()):
            r.pos = r.eow()
 class backward_word(MotionCommand):
    def do(self) -> None:
        r = self.reader
        for i in range(r.get_arg()):
            r.pos = r.bow()
 class self_insert(EditCommand):
    def do(self) -> None:
        r = self.reader
        text = self.event * r.get_arg()
        r.insert(text)
 class insert_nl(EditCommand):
    def do(self) -> None:
        r = self.reader
        r.insert("\n" * r.get_arg())
 class transpose_characters(EditCommand):
    def do(self) -> None:
        r = self.reader
        b = r.buffer
        s = r.pos - 1
        if s < 0:
            r.error("cannot transpose at start of buffer")
        else:
            if s == len(b):
                s -= 1
            t = min(s + r.get_arg(), len(b) - 1)
            c = b[s]
            del b[s]
            b.insert(t, c)
            r.pos = t
            r.dirty = True
 class backspace(EditCommand):
    def do(self) -> None:
        r = self.reader
        b = r.buffer
        for i in range(r.get_arg()):
            if r.pos > 0:
                r.pos -= 1
                del b[r.pos]
                r.dirty = True
            else:
                self.reader.error("can't backspace at start")
 class delete(EditCommand):
    def do(self) -> None:
        r = self.reader
        b = r.buffer
        if (
            r.pos == 0
            and len(b) == 0  # this is something of a hack
            and self.event[-1] == "\004"
        ):
            r.update_screen()
            r.console.finish()
            raise EOFError
        for i in range(r.get_arg()):
            if r.pos != len(b):
                del b[r.pos]
                r.dirty = True
            else:
                self.reader.error("end of buffer")
 class accept(FinishCommand):
    def do(self) -> None:
        pass
 class help(Command):
    def do(self) -> None:
        import _sitebuiltins
        with self.reader.suspend():
            self.reader.msg = _sitebuiltins._Helper()()  # type: ignore[assignment, call-arg]
 class invalid_key(Command):
    def do(self) -> None:
        pending = self.reader.console.getpending()
        s = "".join(self.event) + pending.data
        self.reader.error("`%r' not bound" % s)
 class invalid_command(Command):
    def do(self) -> None:
        s = self.event_name
        self.reader.error("command `%s' not known" % s)
 class show_history(Command):
    def do(self) -> None:
        from .pager import get_pager
        from site import gethistoryfile  # type: ignore[attr-defined]
        history = os.linesep.join(self.reader.history[:])
        self.reader.console.restore()
        pager = get_pager()
        pager(history, gethistoryfile())
        self.reader.console.prepare()
        # We need to copy over the state so that it's consistent between
        # console and reader, and console does not overwrite/append stuff
        self.reader.console.screen = self.reader.screen.copy()
        self.reader.console.posxy = self.reader.cxy
 class paste_mode(Command):
    def do(self) -> None:
        self.reader.paste_mode = not self.reader.paste_mode
        self.reader.dirty = True
 class enable_bracketed_paste(Command):
    def do(self) -> None:
        self.reader.paste_mode = True
        self.reader.in_bracketed_paste = True
 class disable_bracketed_paste(Command):
    def do(self) -> None:
        self.reader.paste_mode = False
        self.reader.in_bracketed_paste = False
        self.reader.dirty = True
--- a/Lib/_pyrepl/completing_reader.py
+++ b/Lib/_pyrepl/completing_reader.py
@@ -0,0 +1,295 @@
 #   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
 #                       Antonio Cuni
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 from __future__ import annotations
 from dataclasses import dataclass, field
 import re
 from . import commands, console, reader
 from .reader import Reader
 # types
 Command = commands.Command
 if False:
    from .types import KeySpec, CommandName
 def prefix(wordlist: list[str], j: int = 0) -> str:
    d = {}
    i = j
    try:
        while 1:
            for word in wordlist:
                d[word[i]] = 1
            if len(d) > 1:
                return wordlist[0][j:i]
            i += 1
            d = {}
    except IndexError:
        return wordlist[0][j:i]
    return ""
 STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]")
 def stripcolor(s: str) -> str:
    return STRIPCOLOR_REGEX.sub('', s)
 def real_len(s: str) -> int:
    return len(stripcolor(s))
 def left_align(s: str, maxlen: int) -> str:
    stripped = stripcolor(s)
    if len(stripped) > maxlen:
        # too bad, we remove the color
        return stripped[:maxlen]
    padding = maxlen - len(stripped)
    return s + ' '*padding
 def build_menu(
        cons: console.Console,
        wordlist: list[str],
        start: int,
        use_brackets: bool,
        sort_in_column: bool,
 ) -> tuple[list[str], int]:
    if use_brackets:
        item = "[ %s ]"
        padding = 4
    else:
        item = "%s  "
        padding = 2
    maxlen = min(max(map(real_len, wordlist)), cons.width - padding)
    cols = int(cons.width / (maxlen + padding))
    rows = int((len(wordlist) - 1)/cols + 1)
    if sort_in_column:
        # sort_in_column=False (default)     sort_in_column=True
        #          A B C                       A D G
        #          D E F                       B E
        #          G                           C F
        #
        # "fill" the table with empty words, so we always have the same amout
        # of rows for each column
        missing = cols*rows - len(wordlist)
        wordlist = wordlist + ['']*missing
        indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))]
        wordlist = [wordlist[i] for i in indexes]
    menu = []
    i = start
    for r in range(rows):
        row = []
        for col in range(cols):
            row.append(item % left_align(wordlist[i], maxlen))
            i += 1
            if i >= len(wordlist):
                break
        menu.append(''.join(row))
        if i >= len(wordlist):
            i = 0
            break
        if r + 5 > cons.height:
            menu.append("   %d more... " % (len(wordlist) - i))
            break
    return menu, i
 # this gets somewhat user interface-y, and as a result the logic gets
 # very convoluted.
 #
 #  To summarise the summary of the summary:- people are a problem.
 #                  -- The Hitch-Hikers Guide to the Galaxy, Episode 12
 #### Desired behaviour of the completions commands.
 # the considerations are:
 # (1) how many completions are possible
 # (2) whether the last command was a completion
 # (3) if we can assume that the completer is going to return the same set of
 #     completions: this is controlled by the ``assume_immutable_completions``
 #     variable on the reader, which is True by default to match the historical
 #     behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match
 #     more closely readline's semantics (this is needed e.g. by
 #     fancycompleter)
 #
 # if there's no possible completion, beep at the user and point this out.
 # this is easy.
 #
 # if there's only one possible completion, stick it in.  if the last thing
 # user did was a completion, point out that he isn't getting anywhere, but
 # only if the ``assume_immutable_completions`` is True.
 #
 # now it gets complicated.
 #
 # for the first press of a completion key:
 #  if there's a common prefix, stick it in.
 #  irrespective of whether anything got stuck in, if the word is now
 #  complete, show the "complete but not unique" message
 #  if there's no common prefix and if the word is not now complete,
 #  beep.
 #        common prefix ->    yes          no
 #        word complete \/
 #            yes           "cbnu"      "cbnu"
 #            no              -          beep
 # for the second bang on the completion key
 #  there will necessarily be no common prefix
 #  show a menu of the choices.
 # for subsequent bangs, rotate the menu around (if there are sufficient
 # choices).
 class complete(commands.Command):
    def do(self) -> None:
        r: CompletingReader
        r = self.reader  # type: ignore[assignment]
        last_is_completer = r.last_command_is(self.__class__)
        immutable_completions = r.assume_immutable_completions
        completions_unchangable = last_is_completer and immutable_completions
        stem = r.get_stem()
        if not completions_unchangable:
            r.cmpltn_menu_choices = r.get_completions(stem)
        completions = r.cmpltn_menu_choices
        if not completions:
            r.error("no matches")
        elif len(completions) == 1:
            if completions_unchangable and len(completions[0]) == len(stem):
                r.msg = "[ sole completion ]"
                r.dirty = True
            r.insert(completions[0][len(stem):])
        else:
            p = prefix(completions, len(stem))
            if p:
                r.insert(p)
            if last_is_completer:
                r.cmpltn_menu_visible = True
                r.cmpltn_message_visible = False
                r.cmpltn_menu, r.cmpltn_menu_end = build_menu(
                    r.console, completions, r.cmpltn_menu_end,
                    r.use_brackets, r.sort_in_column)
                r.dirty = True
            elif not r.cmpltn_menu_visible:
                r.cmpltn_message_visible = True
                if stem + p in completions:
                    r.msg = "[ complete but not unique ]"
                    r.dirty = True
                else:
                    r.msg = "[ not unique ]"
                    r.dirty = True
 class self_insert(commands.self_insert):
    def do(self) -> None:
        r: CompletingReader
        r = self.reader  # type: ignore[assignment]
        commands.self_insert.do(self)
        if r.cmpltn_menu_visible:
            stem = r.get_stem()
            if len(stem) < 1:
                r.cmpltn_reset()
            else:
                completions = [w for w in r.cmpltn_menu_choices
                               if w.startswith(stem)]
                if completions:
                    r.cmpltn_menu, r.cmpltn_menu_end = build_menu(
                        r.console, completions, 0,
                        r.use_brackets, r.sort_in_column)
                else:
                    r.cmpltn_reset()
@dataclass
 class CompletingReader(Reader):
    """Adds completion support"""
    ### Class variables
    # see the comment for the complete command
    assume_immutable_completions = True
    use_brackets = True  # display completions inside []
    sort_in_column = False
    ### Instance variables
    cmpltn_menu: list[str] = field(init=False)
    cmpltn_menu_visible: bool = field(init=False)
    cmpltn_message_visible: bool = field(init=False)
    cmpltn_menu_end: int = field(init=False)
    cmpltn_menu_choices: list[str] = field(init=False)
    def __post_init__(self) -> None:
        super().__post_init__()
        self.cmpltn_reset()
        for c in (complete, self_insert):
            self.commands[c.__name__] = c
            self.commands[c.__name__.replace('_', '-')] = c
    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
        return super().collect_keymap() + (
            (r'\t', 'complete'),)
    def after_command(self, cmd: Command) -> None:
        super().after_command(cmd)
        if not isinstance(cmd, (complete, self_insert)):
            self.cmpltn_reset()
    def calc_screen(self) -> list[str]:
        screen = super().calc_screen()
        if self.cmpltn_menu_visible:
            # We display the completions menu below the current prompt
            ly = self.lxy[1] + 1
            screen[ly:ly] = self.cmpltn_menu
            # If we're not in the middle of multiline edit, don't append to screeninfo
            # since that screws up the position calculation in pos2xy function.
            # This is a hack to prevent the cursor jumping
            # into the completions menu when pressing left or down arrow.
            if self.pos != len(self.buffer):
                self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu)
        return screen
    def finish(self) -> None:
        super().finish()
        self.cmpltn_reset()
    def cmpltn_reset(self) -> None:
        self.cmpltn_menu = []
        self.cmpltn_menu_visible = False
        self.cmpltn_message_visible = False
        self.cmpltn_menu_end = 0
        self.cmpltn_menu_choices = []
    def get_stem(self) -> str:
        st = self.syntax_table
        SW = reader.SYNTAX_WORD
        b = self.buffer
        p = self.pos - 1
        while p >= 0 and st.get(b[p], SW) == SW:
            p -= 1
        return ''.join(b[p+1:self.pos])
    def get_completions(self, stem: str) -> list[str]:
        return []
--- a/Lib/_pyrepl/console.py
+++ b/Lib/_pyrepl/console.py
@@ -0,0 +1,213 @@
 #   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 from __future__ import annotations
 import _colorize  # type: ignore[import-not-found]
 from abc import ABC, abstractmethod
 import ast
 import code
 from dataclasses import dataclass, field
 import os.path
 import sys
 TYPE_CHECKING = False
 if TYPE_CHECKING:
    from typing import IO
    from typing import Callable
@dataclass
 class Event:
    evt: str
    data: str
    raw: bytes = b""
@dataclass
 class Console(ABC):
    posxy: tuple[int, int]
    screen: list[str] = field(default_factory=list)
    height: int = 25
    width: int = 80
    def __init__(
        self,
        f_in: IO[bytes] | int = 0,
        f_out: IO[bytes] | int = 1,
        term: str = "",
        encoding: str = "",
    ):
        self.encoding = encoding or sys.getdefaultencoding()
        if isinstance(f_in, int):
            self.input_fd = f_in
        else:
            self.input_fd = f_in.fileno()
        if isinstance(f_out, int):
            self.output_fd = f_out
        else:
            self.output_fd = f_out.fileno()
    @abstractmethod
    def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ...
    @abstractmethod
    def prepare(self) -> None: ...
    @abstractmethod
    def restore(self) -> None: ...
    @abstractmethod
    def move_cursor(self, x: int, y: int) -> None: ...
    @abstractmethod
    def set_cursor_vis(self, visible: bool) -> None: ...
    @abstractmethod
    def getheightwidth(self) -> tuple[int, int]:
        """Return (height, width) where height and width are the height
        and width of the terminal window in characters."""
        ...
    @abstractmethod
    def get_event(self, block: bool = True) -> Event | None:
        """Return an Event instance.  Returns None if |block| is false
        and there is no event pending, otherwise waits for the
        completion of an event."""
        ...
    @abstractmethod
    def push_char(self, char: int | bytes) -> None:
        """
        Push a character to the console event queue.
        """
        ...
    @abstractmethod
    def beep(self) -> None: ...
    @abstractmethod
    def clear(self) -> None:
        """Wipe the screen"""
        ...
    @abstractmethod
    def finish(self) -> None:
        """Move the cursor to the end of the display and otherwise get
        ready for end.  XXX could be merged with restore?  Hmm."""
        ...
    @abstractmethod
    def flushoutput(self) -> None:
        """Flush all output to the screen (assuming there's some
        buffering going on somewhere)."""
        ...
    @abstractmethod
    def forgetinput(self) -> None:
        """Forget all pending, but not yet processed input."""
        ...
    @abstractmethod
    def getpending(self) -> Event:
        """Return the characters that have been typed but not yet
        processed."""
        ...
    @abstractmethod
    def wait(self, timeout: float | None) -> bool:
        """Wait for an event. The return value is True if an event is
        available, False if the timeout has been reached. If timeout is
        None, wait forever. The timeout is in milliseconds."""
        ...
    @property
    def input_hook(self) -> Callable[[], int] | None:
        """Returns the current input hook."""
        ...
    @abstractmethod
    def repaint(self) -> None: ...
 class InteractiveColoredConsole(code.InteractiveConsole):
    def __init__(
        self,
        locals: dict[str, object] | None = None,
        filename: str = "<console>",
        *,
        local_exit: bool = False,
    ) -> None:
        super().__init__(locals=locals, filename=filename, local_exit=local_exit)  # type: ignore[call-arg]
        self.can_colorize = _colorize.can_colorize()
    def showsyntaxerror(self, filename=None, **kwargs):
        super().showsyntaxerror(filename=filename, **kwargs)
    def _excepthook(self, typ, value, tb):
        import traceback
        lines = traceback.format_exception(
                typ, value, tb,
                colorize=self.can_colorize,
                limit=traceback.BUILTIN_EXCEPTION_LIMIT)
        self.write(''.join(lines))
    def runsource(self, source, filename="<input>", symbol="single"):
        try:
            tree = self.compile.compiler(
                source,
                filename,
                "exec",
                ast.PyCF_ONLY_AST,
                incomplete_input=False,
            )
        except (SyntaxError, OverflowError, ValueError):
            self.showsyntaxerror(filename, source=source)
            return False
        if tree.body:
            *_, last_stmt = tree.body
        for stmt in tree.body:
            wrapper = ast.Interactive if stmt is last_stmt else ast.Module
            the_symbol = symbol if stmt is last_stmt else "exec"
            item = wrapper([stmt])
            try:
                code = self.compile.compiler(item, filename, the_symbol)
            except SyntaxError as e:
                if e.args[0] == "'await' outside function":
                    python = os.path.basename(sys.executable)
                    e.add_note(
                        f"Try the asyncio REPL ({python} -m asyncio) to use"
                        f" top-level 'await' and run background asyncio tasks."
                    )
                self.showsyntaxerror(filename, source=source)
                return False
            except (OverflowError, ValueError):
                self.showsyntaxerror(filename, source=source)
                return False
            if code is None:
                return True
            self.runcode(code)
        return False
--- a/Lib/_pyrepl/curses.py
+++ b/Lib/_pyrepl/curses.py
@@ -0,0 +1,33 @@
 #   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
 #                       Armin Rigo
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 try:
    import _curses
 except ImportError:
    try:
        import curses as _curses  # type: ignore[no-redef]
    except ImportError:
        from . import _minimal_curses as _curses  # type: ignore[no-redef]
 setupterm = _curses.setupterm
 tigetstr = _curses.tigetstr
 tparm = _curses.tparm
 error = _curses.error
--- a/Lib/_pyrepl/fancy_termios.py
+++ b/Lib/_pyrepl/fancy_termios.py
@@ -0,0 +1,76 @@
 #   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 import termios
 class TermState:
    def __init__(self, tuples):
        (
            self.iflag,
            self.oflag,
            self.cflag,
            self.lflag,
            self.ispeed,
            self.ospeed,
            self.cc,
        ) = tuples
    def as_list(self):
        return [
            self.iflag,
            self.oflag,
            self.cflag,
            self.lflag,
            self.ispeed,
            self.ospeed,
            # Always return a copy of the control characters list to ensure
            # there are not any additional references to self.cc
            self.cc[:],
        ]
    def copy(self):
        return self.__class__(self.as_list())
 def tcgetattr(fd):
    return TermState(termios.tcgetattr(fd))
 def tcsetattr(fd, when, attrs):
    termios.tcsetattr(fd, when, attrs.as_list())
 class Term(TermState):
    TS__init__ = TermState.__init__
    def __init__(self, fd=0):
        self.TS__init__(termios.tcgetattr(fd))
        self.fd = fd
        self.stack = []
    def save(self):
        self.stack.append(self.as_list())
    def set(self, when=termios.TCSANOW):
        termios.tcsetattr(self.fd, when, self.as_list())
    def restore(self):
        self.TS__init__(self.stack.pop())
        self.set()
--- a/Lib/_pyrepl/historical_reader.py
+++ b/Lib/_pyrepl/historical_reader.py
@@ -0,0 +1,419 @@
 #   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 from __future__ import annotations
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from . import commands, input
 from .reader import Reader
 if False:
    from .types import SimpleContextManager, KeySpec, CommandName
 isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple(
    [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"]
    + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"]
    + [
        ("\\%03o" % c, "isearch-add-character")
        for c in range(256)
        if chr(c).isalpha() and chr(c) != "\\"
    ]
    + [
        ("\\\\", "self-insert"),
        (r"\C-r", "isearch-backwards"),
        (r"\C-s", "isearch-forwards"),
        (r"\C-c", "isearch-cancel"),
        (r"\C-g", "isearch-cancel"),
        (r"\<backspace>", "isearch-backspace"),
    ]
 )
 ISEARCH_DIRECTION_NONE = ""
 ISEARCH_DIRECTION_BACKWARDS = "r"
 ISEARCH_DIRECTION_FORWARDS = "f"
 class next_history(commands.Command):
    def do(self) -> None:
        r = self.reader
        if r.historyi == len(r.history):
            r.error("end of history list")
            return
        r.select_item(r.historyi + 1)
 class previous_history(commands.Command):
    def do(self) -> None:
        r = self.reader
        if r.historyi == 0:
            r.error("start of history list")
            return
        r.select_item(r.historyi - 1)
 class history_search_backward(commands.Command):
    def do(self) -> None:
        r = self.reader
        r.search_next(forwards=False)
 class history_search_forward(commands.Command):
    def do(self) -> None:
        r = self.reader
        r.search_next(forwards=True)
 class restore_history(commands.Command):
    def do(self) -> None:
        r = self.reader
        if r.historyi != len(r.history):
            if r.get_unicode() != r.history[r.historyi]:
                r.buffer = list(r.history[r.historyi])
                r.pos = len(r.buffer)
                r.dirty = True
 class first_history(commands.Command):
    def do(self) -> None:
        self.reader.select_item(0)
 class last_history(commands.Command):
    def do(self) -> None:
        self.reader.select_item(len(self.reader.history))
 class operate_and_get_next(commands.FinishCommand):
    def do(self) -> None:
        self.reader.next_history = self.reader.historyi + 1
 class yank_arg(commands.Command):
    def do(self) -> None:
        r = self.reader
        if r.last_command is self.__class__:
            r.yank_arg_i += 1
        else:
            r.yank_arg_i = 0
        if r.historyi < r.yank_arg_i:
            r.error("beginning of history list")
            return
        a = r.get_arg(-1)
        # XXX how to split?
        words = r.get_item(r.historyi - r.yank_arg_i - 1).split()
        if a < -len(words) or a >= len(words):
            r.error("no such arg")
            return
        w = words[a]
        b = r.buffer
        if r.yank_arg_i > 0:
            o = len(r.yank_arg_yanked)
        else:
            o = 0
        b[r.pos - o : r.pos] = list(w)
        r.yank_arg_yanked = w
        r.pos += len(w) - o
        r.dirty = True
 class forward_history_isearch(commands.Command):
    def do(self) -> None:
        r = self.reader
        r.isearch_direction = ISEARCH_DIRECTION_FORWARDS
        r.isearch_start = r.historyi, r.pos
        r.isearch_term = ""
        r.dirty = True
        r.push_input_trans(r.isearch_trans)
 class reverse_history_isearch(commands.Command):
    def do(self) -> None:
        r = self.reader
        r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS
        r.dirty = True
        r.isearch_term = ""
        r.push_input_trans(r.isearch_trans)
        r.isearch_start = r.historyi, r.pos
 class isearch_cancel(commands.Command):
    def do(self) -> None:
        r = self.reader
        r.isearch_direction = ISEARCH_DIRECTION_NONE
        r.pop_input_trans()
        r.select_item(r.isearch_start[0])
        r.pos = r.isearch_start[1]
        r.dirty = True
 class isearch_add_character(commands.Command):
    def do(self) -> None:
        r = self.reader
        b = r.buffer
        r.isearch_term += self.event[-1]
        r.dirty = True
        p = r.pos + len(r.isearch_term) - 1
        if b[p : p + 1] != [r.isearch_term[-1]]:
            r.isearch_next()
 class isearch_backspace(commands.Command):
    def do(self) -> None:
        r = self.reader
        if len(r.isearch_term) > 0:
            r.isearch_term = r.isearch_term[:-1]
            r.dirty = True
        else:
            r.error("nothing to rubout")
 class isearch_forwards(commands.Command):
    def do(self) -> None:
        r = self.reader
        r.isearch_direction = ISEARCH_DIRECTION_FORWARDS
        r.isearch_next()
 class isearch_backwards(commands.Command):
    def do(self) -> None:
        r = self.reader
        r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS
        r.isearch_next()
 class isearch_end(commands.Command):
    def do(self) -> None:
        r = self.reader
        r.isearch_direction = ISEARCH_DIRECTION_NONE
        r.console.forgetinput()
        r.pop_input_trans()
        r.dirty = True
@dataclass
 class HistoricalReader(Reader):
    """Adds history support (with incremental history searching) to the
    Reader class.
    """
    history: list[str] = field(default_factory=list)
    historyi: int = 0
    next_history: int | None = None
    transient_history: dict[int, str] = field(default_factory=dict)
    isearch_term: str = ""
    isearch_direction: str = ISEARCH_DIRECTION_NONE
    isearch_start: tuple[int, int] = field(init=False)
    isearch_trans: input.KeymapTranslator = field(init=False)
    yank_arg_i: int = 0
    yank_arg_yanked: str = ""
    def __post_init__(self) -> None:
        super().__post_init__()
        for c in [
            next_history,
            previous_history,
            restore_history,
            first_history,
            last_history,
            yank_arg,
            forward_history_isearch,
            reverse_history_isearch,
            isearch_end,
            isearch_add_character,
            isearch_cancel,
            isearch_add_character,
            isearch_backspace,
            isearch_forwards,
            isearch_backwards,
            operate_and_get_next,
            history_search_backward,
            history_search_forward,
        ]:
            self.commands[c.__name__] = c
            self.commands[c.__name__.replace("_", "-")] = c
        self.isearch_start = self.historyi, self.pos
        self.isearch_trans = input.KeymapTranslator(
            isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character
        )
    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
        return super().collect_keymap() + (
            (r"\C-n", "next-history"),
            (r"\C-p", "previous-history"),
            (r"\C-o", "operate-and-get-next"),
            (r"\C-r", "reverse-history-isearch"),
            (r"\C-s", "forward-history-isearch"),
            (r"\M-r", "restore-history"),
            (r"\M-.", "yank-arg"),
            (r"\<page down>", "history-search-forward"),
            (r"\x1b[6~", "history-search-forward"),
            (r"\<page up>", "history-search-backward"),
            (r"\x1b[5~", "history-search-backward"),
        )
    def select_item(self, i: int) -> None:
        self.transient_history[self.historyi] = self.get_unicode()
        buf = self.transient_history.get(i)
        if buf is None:
            buf = self.history[i].rstrip()
        self.buffer = list(buf)
        self.historyi = i
        self.pos = len(self.buffer)
        self.dirty = True
        self.last_refresh_cache.invalidated = True
    def get_item(self, i: int) -> str:
        if i != len(self.history):
            return self.transient_history.get(i, self.history[i])
        else:
            return self.transient_history.get(i, self.get_unicode())
    @contextmanager
    def suspend(self) -> SimpleContextManager:
        with super().suspend(), self.suspend_history():
            yield
    @contextmanager
    def suspend_history(self) -> SimpleContextManager:
        try:
            old_history = self.history[:]
            del self.history[:]
            yield
        finally:
            self.history[:] = old_history
    def prepare(self) -> None:
        super().prepare()
        try:
            self.transient_history = {}
            if self.next_history is not None and self.next_history < len(self.history):
                self.historyi = self.next_history
                self.buffer[:] = list(self.history[self.next_history])
                self.pos = len(self.buffer)
                self.transient_history[len(self.history)] = ""
            else:
                self.historyi = len(self.history)
            self.next_history = None
        except:
            self.restore()
            raise
    def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
        if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE:
            d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS]
            return "(%s-search `%s') " % (d, self.isearch_term)
        else:
            return super().get_prompt(lineno, cursor_on_line)
    def search_next(self, *, forwards: bool) -> None:
        """Search history for the current line contents up to the cursor.
        Selects the first item found. If nothing is under the cursor, any next
        item in history is selected.
        """
        pos = self.pos
        s = self.get_unicode()
        history_index = self.historyi
        # In multiline contexts, we're only interested in the current line.
        nl_index = s.rfind('\n', 0, pos)
        prefix = s[nl_index + 1:pos]
        pos = len(prefix)
        match_prefix = len(prefix)
        len_item = 0
        if history_index < len(self.history):
            len_item = len(self.get_item(history_index))
        if len_item and pos == len_item:
            match_prefix = False
        elif not pos:
            match_prefix = False
        while 1:
            if forwards:
                out_of_bounds = history_index >= len(self.history) - 1
            else:
                out_of_bounds = history_index == 0
            if out_of_bounds:
                if forwards and not match_prefix:
                    self.pos = 0
                    self.buffer = []
                    self.dirty = True
                else:
                    self.error("not found")
                return
            history_index += 1 if forwards else -1
            s = self.get_item(history_index)
            if not match_prefix:
                self.select_item(history_index)
                return
            len_acc = 0
            for i, line in enumerate(s.splitlines(keepends=True)):
                if line.startswith(prefix):
                    self.select_item(history_index)
                    self.pos = pos + len_acc
                    return
                len_acc += len(line)
    def isearch_next(self) -> None:
        st = self.isearch_term
        p = self.pos
        i = self.historyi
        s = self.get_unicode()
        forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS
        while 1:
            if forwards:
                p = s.find(st, p + 1)
            else:
                p = s.rfind(st, 0, p + len(st) - 1)
            if p != -1:
                self.select_item(i)
                self.pos = p
                return
            elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0):
                self.error("not found")
                return
            else:
                if forwards:
                    i += 1
                    s = self.get_item(i)
                    p = -1
                else:
                    i -= 1
                    s = self.get_item(i)
                    p = len(s)
    def finish(self) -> None:
        super().finish()
        ret = self.get_unicode()
        for i, t in self.transient_history.items():
            if i < len(self.history) and i != self.historyi:
                self.history[i] = t
        if ret and should_auto_add_history:
            self.history.append(ret)
 should_auto_add_history = True
--- a/Lib/_pyrepl/input.py
+++ b/Lib/_pyrepl/input.py
@@ -0,0 +1,114 @@
 #   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 # (naming modules after builtin functions is not such a hot idea...)
 # an KeyTrans instance translates Event objects into Command objects
 # hmm, at what level do we want [C-i] and [tab] to be equivalent?
 # [meta-a] and [esc a]?  obviously, these are going to be equivalent
 # for the UnixConsole, but should they be for PygameConsole?
 # it would in any situation seem to be a bad idea to bind, say, [tab]
 # and [C-i] to *different* things... but should binding one bind the
 # other?
 # executive, temporary decision: [tab] and [C-i] are distinct, but
 # [meta-key] is identified with [esc key].  We demand that any console
 # class does quite a lot towards emulating a unix terminal.
 from __future__ import annotations
 from abc import ABC, abstractmethod
 import unicodedata
 from collections import deque
 # types
 if False:
    from .types import EventTuple
 class InputTranslator(ABC):
    @abstractmethod
    def push(self, evt: EventTuple) -> None:
        pass
    @abstractmethod
    def get(self) -> EventTuple | None:
        return None
    @abstractmethod
    def empty(self) -> bool:
        return True
 class KeymapTranslator(InputTranslator):
    def __init__(self, keymap, verbose=False, invalid_cls=None, character_cls=None):
        self.verbose = verbose
        from .keymap import compile_keymap, parse_keys
        self.keymap = keymap
        self.invalid_cls = invalid_cls
        self.character_cls = character_cls
        d = {}
        for keyspec, command in keymap:
            keyseq = tuple(parse_keys(keyspec))
            d[keyseq] = command
        if self.verbose:
            print(d)
        self.k = self.ck = compile_keymap(d, ())
        self.results = deque()
        self.stack = []
    def push(self, evt):
        if self.verbose:
            print("pushed", evt.data, end="")
        key = evt.data
        d = self.k.get(key)
        if isinstance(d, dict):
            if self.verbose:
                print("transition")
            self.stack.append(key)
            self.k = d
        else:
            if d is None:
                if self.verbose:
                    print("invalid")
                if self.stack or len(key) > 1 or unicodedata.category(key) == "C":
                    self.results.append((self.invalid_cls, self.stack + [key]))
                else:
                    # small optimization:
                    self.k[key] = self.character_cls
                    self.results.append((self.character_cls, [key]))
            else:
                if self.verbose:
                    print("matched", d)
                self.results.append((d, self.stack + [key]))
            self.stack = []
            self.k = self.ck
    def get(self):
        if self.results:
            return self.results.popleft()
        else:
            return None
    def empty(self) -> bool:
        return not self.results
--- a/Lib/_pyrepl/keymap.py
+++ b/Lib/_pyrepl/keymap.py
@@ -0,0 +1,213 @@
 #   Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com>
 #                       Armin Rigo
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 """
 Keymap contains functions for parsing keyspecs and turning keyspecs into
 appropriate sequences.
 A keyspec is a string representing a sequence of key presses that can
 be bound to a command. All characters other than the backslash represent
 themselves. In the traditional manner, a backslash introduces an escape
 sequence.
 pyrepl uses its own keyspec format that is meant to be a strict superset of
 readline's KEYSEQ format. This means that if a spec is found that readline
 accepts that this doesn't, it should be logged as a bug. Note that this means
 we're using the `\\C-o' style of readline's keyspec, not the `Control-o' sort.
 The extension to readline is that the sequence \\<KEY> denotes the
 sequence of characters produced by hitting KEY.
 Examples:
 `a'      - what you get when you hit the `a' key
 `\\EOA'  - Escape - O - A (up, on my terminal)
 `\\<UP>' - the up arrow key
 `\\<up>' - ditto (keynames are case-insensitive)
 `\\C-o', `\\c-o'  - control-o
 `\\M-.'  - meta-period
 `\\E.'   - ditto (that's how meta works for pyrepl)
 `\\<tab>', `\\<TAB>', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I'
   - all of these are the tab character.
 """
 _escapes = {
    "\\": "\\",
    "'": "'",
    '"': '"',
    "a": "\a",
    "b": "\b",
    "e": "\033",
    "f": "\f",
    "n": "\n",
    "r": "\r",
    "t": "\t",
    "v": "\v",
 }
 _keynames = {
    "backspace": "backspace",
    "delete": "delete",
    "down": "down",
    "end": "end",
    "enter": "\r",
    "escape": "\033",
    "f1": "f1",
    "f2": "f2",
    "f3": "f3",
    "f4": "f4",
    "f5": "f5",
    "f6": "f6",
    "f7": "f7",
    "f8": "f8",
    "f9": "f9",
    "f10": "f10",
    "f11": "f11",
    "f12": "f12",
    "f13": "f13",
    "f14": "f14",
    "f15": "f15",
    "f16": "f16",
    "f17": "f17",
    "f18": "f18",
    "f19": "f19",
    "f20": "f20",
    "home": "home",
    "insert": "insert",
    "left": "left",
    "page down": "page down",
    "page up": "page up",
    "return": "\r",
    "right": "right",
    "space": " ",
    "tab": "\t",
    "up": "up",
 }
 class KeySpecError(Exception):
    pass
 def parse_keys(keys: str) -> list[str]:
    """Parse keys in keyspec format to a sequence of keys."""
    s = 0
    r: list[str] = []
    while s < len(keys):
        k, s = _parse_single_key_sequence(keys, s)
        r.extend(k)
    return r
 def _parse_single_key_sequence(key: str, s: int) -> tuple[list[str], int]:
    ctrl = 0
    meta = 0
    ret = ""
    while not ret and s < len(key):
        if key[s] == "\\":
            c = key[s + 1].lower()
            if c in _escapes:
                ret = _escapes[c]
                s += 2
            elif c == "c":
                if key[s + 2] != "-":
                    raise KeySpecError(
                        "\\C must be followed by `-' (char %d of %s)"
                        % (s + 2, repr(key))
                    )
                if ctrl:
                    raise KeySpecError(
                        "doubled \\C- (char %d of %s)" % (s + 1, repr(key))
                    )
                ctrl = 1
                s += 3
            elif c == "m":
                if key[s + 2] != "-":
                    raise KeySpecError(
                        "\\M must be followed by `-' (char %d of %s)"
                        % (s + 2, repr(key))
                    )
                if meta:
                    raise KeySpecError(
                        "doubled \\M- (char %d of %s)" % (s + 1, repr(key))
                    )
                meta = 1
                s += 3
            elif c.isdigit():
                n = key[s + 1 : s + 4]
                ret = chr(int(n, 8))
                s += 4
            elif c == "x":
                n = key[s + 2 : s + 4]
                ret = chr(int(n, 16))
                s += 4
            elif c == "<":
                t = key.find(">", s)
                if t == -1:
                    raise KeySpecError(
                        "unterminated \\< starting at char %d of %s"
                        % (s + 1, repr(key))
                    )
                ret = key[s + 2 : t].lower()
                if ret not in _keynames:
                    raise KeySpecError(
                        "unrecognised keyname `%s' at char %d of %s"
                        % (ret, s + 2, repr(key))
                    )
                ret = _keynames[ret]
                s = t + 1
            else:
                raise KeySpecError(
                    "unknown backslash escape %s at char %d of %s"
                    % (repr(c), s + 2, repr(key))
                )
        else:
            ret = key[s]
            s += 1
    if ctrl:
        if len(ret) == 1:
            ret = chr(ord(ret) & 0x1F)  # curses.ascii.ctrl()
        elif ret in {"left", "right"}:
            ret = f"ctrl {ret}"
        else:
            raise KeySpecError("\\C- followed by invalid key")
    result = [ret], s
    if meta:
        result[0].insert(0, "\033")
    return result
 def compile_keymap(keymap, empty=b""):
    r = {}
    for key, value in keymap.items():
        if isinstance(key, bytes):
            first = key[:1]
        else:
            first = key[0]
        r.setdefault(first, {})[key[1:]] = value
    for key, value in r.items():
        if empty in value:
            if len(value) != 1:
                raise KeySpecError("key definitions for %s clash" % (value.values(),))
            else:
                r[key] = value[empty]
        else:
            r[key] = compile_keymap(value, empty)
    return r
--- a/Lib/_pyrepl/main.py
+++ b/Lib/_pyrepl/main.py
@@ -0,0 +1,59 @@
 import errno
 import os
 import sys
 CAN_USE_PYREPL: bool
 FAIL_REASON: str
 try:
    if sys.platform == "win32" and sys.getwindowsversion().build < 10586:
        raise RuntimeError("Windows 10 TH2 or later required")
    if not os.isatty(sys.stdin.fileno()):
        raise OSError(errno.ENOTTY, "tty required", "stdin")
    from .simple_interact import check
    if err := check():
        raise RuntimeError(err)
 except Exception as e:
    CAN_USE_PYREPL = False
    FAIL_REASON = f"warning: can't use pyrepl: {e}"
 else:
    CAN_USE_PYREPL = True
    FAIL_REASON = ""
 def interactive_console(mainmodule=None, quiet=False, pythonstartup=False):
    if not CAN_USE_PYREPL:
        if not os.getenv('PYTHON_BASIC_REPL') and FAIL_REASON:
            from .trace import trace
            trace(FAIL_REASON)
            print(FAIL_REASON, file=sys.stderr)
        return sys._baserepl()
    if mainmodule:
        namespace = mainmodule.__dict__
    else:
        import __main__
        namespace = __main__.__dict__
        namespace.pop("__pyrepl_interactive_console", None)
    # sys._baserepl() above does this internally, we do it here
    startup_path = os.getenv("PYTHONSTARTUP")
    if pythonstartup and startup_path:
        sys.audit("cpython.run_startup", startup_path)
        import tokenize
        with tokenize.open(startup_path) as f:
            startup_code = compile(f.read(), startup_path, "exec")
            exec(startup_code, namespace)
    # set sys.{ps1,ps2} just before invoking the interactive interpreter. This
    # mimics what CPython does in pythonrun.c
    if not hasattr(sys, "ps1"):
        sys.ps1 = ">>> "
    if not hasattr(sys, "ps2"):
        sys.ps2 = "... "
    from .console import InteractiveColoredConsole
    from .simple_interact import run_multiline_interactive_console
    console = InteractiveColoredConsole(namespace, filename="<stdin>")
    run_multiline_interactive_console(console)
--- a/Lib/_pyrepl/mypy.ini
+++ b/Lib/_pyrepl/mypy.ini
@@ -0,0 +1,24 @@
 # Config file for running mypy on _pyrepl.
 # Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini`
 # on the command-line from the repo root
 [mypy]
 files = Lib/_pyrepl
 explicit_package_bases = True
 python_version = 3.12
 platform = linux
 pretty = True
 # Enable most stricter settings
 enable_error_code = ignore-without-code,redundant-expr
 strict = True
 # Various stricter settings that we can't yet enable
 # Try to enable these in the following order:
 disallow_untyped_calls = False
 disallow_untyped_defs = False
 check_untyped_defs = False
 # Various internal modules that typeshed deliberately doesn't have stubs for:
 [mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*]
 ignore_missing_imports = True
--- a/Lib/_pyrepl/pager.py
+++ b/Lib/_pyrepl/pager.py
@@ -0,0 +1,175 @@
 from __future__ import annotations
 import io
 import os
 import re
 import sys
 # types
 if False:
    from typing import Protocol
    class Pager(Protocol):
        def __call__(self, text: str, title: str = "") -> None:
            ...
 def get_pager() -> Pager:
    """Decide what method to use for paging through text."""
    if not hasattr(sys.stdin, "isatty"):
        return plain_pager
    if not hasattr(sys.stdout, "isatty"):
        return plain_pager
    if not sys.stdin.isatty() or not sys.stdout.isatty():
        return plain_pager
    if sys.platform == "emscripten":
        return plain_pager
    use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER')
    if use_pager:
        if sys.platform == 'win32': # pipes completely broken in Windows
            return lambda text, title='': tempfile_pager(plain(text), use_pager)
        elif os.environ.get('TERM') in ('dumb', 'emacs'):
            return lambda text, title='': pipe_pager(plain(text), use_pager, title)
        else:
            return lambda text, title='': pipe_pager(text, use_pager, title)
    if os.environ.get('TERM') in ('dumb', 'emacs'):
        return plain_pager
    if sys.platform == 'win32':
        return lambda text, title='': tempfile_pager(plain(text), 'more <')
    if hasattr(os, 'system') and os.system('(pager) 2>/dev/null') == 0:
        return lambda text, title='': pipe_pager(text, 'pager', title)
    if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0:
        return lambda text, title='': pipe_pager(text, 'less', title)
    import tempfile
    (fd, filename) = tempfile.mkstemp()
    os.close(fd)
    try:
        if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0:
            return lambda text, title='': pipe_pager(text, 'more', title)
        else:
            return tty_pager
    finally:
        os.unlink(filename)
 def escape_stdout(text: str) -> str:
    # Escape non-encodable characters to avoid encoding errors later
    encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8'
    return text.encode(encoding, 'backslashreplace').decode(encoding)
 def escape_less(s: str) -> str:
    return re.sub(r'([?:.%\\])', r'\\\1', s)
 def plain(text: str) -> str:
    """Remove boldface formatting from text."""
    return re.sub('.\b', '', text)
 def tty_pager(text: str, title: str = '') -> None:
    """Page through text on a text terminal."""
    lines = plain(escape_stdout(text)).split('\n')
    has_tty = False
    try:
        import tty
        import termios
        fd = sys.stdin.fileno()
        old = termios.tcgetattr(fd)
        tty.setcbreak(fd)
        has_tty = True
        def getchar() -> str:
            return sys.stdin.read(1)
    except (ImportError, AttributeError, io.UnsupportedOperation):
        def getchar() -> str:
            return sys.stdin.readline()[:-1][:1]
    try:
        try:
            h = int(os.environ.get('LINES', 0))
        except ValueError:
            h = 0
        if h <= 1:
            h = 25
        r = inc = h - 1
        sys.stdout.write('\n'.join(lines[:inc]) + '\n')
        while lines[r:]:
            sys.stdout.write('-- more --')
            sys.stdout.flush()
            c = getchar()
            if c in ('q', 'Q'):
                sys.stdout.write('\r          \r')
                break
            elif c in ('\r', '\n'):
                sys.stdout.write('\r          \r' + lines[r] + '\n')
                r = r + 1
                continue
            if c in ('b', 'B', '\x1b'):
                r = r - inc - inc
                if r < 0: r = 0
            sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n')
            r = r + inc
    finally:
        if has_tty:
            termios.tcsetattr(fd, termios.TCSAFLUSH, old)
 def plain_pager(text: str, title: str = '') -> None:
    """Simply print unformatted text.  This is the ultimate fallback."""
    sys.stdout.write(plain(escape_stdout(text)))
 def pipe_pager(text: str, cmd: str, title: str = '') -> None:
    """Page through text by feeding it to another program."""
    import subprocess
    env = os.environ.copy()
    if title:
        title += ' '
    esc_title = escape_less(title)
    prompt_string = (
        f' {esc_title}' +
        '?ltline %lt?L/%L.'
        ':byte %bB?s/%s.'
        '.'
        '?e (END):?pB %pB\\%..'
        ' (press h for help or q to quit)')
    env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string)
    proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
                            errors='backslashreplace', env=env)
    assert proc.stdin is not None
    try:
        with proc.stdin as pipe:
            try:
                pipe.write(text)
            except KeyboardInterrupt:
                # We've hereby abandoned whatever text hasn't been written,
                # but the pager is still in control of the terminal.
                pass
    except OSError:
        pass # Ignore broken pipes caused by quitting the pager program.
    while True:
        try:
            proc.wait()
            break
        except KeyboardInterrupt:
            # Ignore ctl-c like the pager itself does.  Otherwise the pager is
            # left running and the terminal is in raw mode and unusable.
            pass
 def tempfile_pager(text: str, cmd: str, title: str = '') -> None:
    """Page through text by invoking a program on a temporary file."""
    import tempfile
    with tempfile.TemporaryDirectory() as tempdir:
        filename = os.path.join(tempdir, 'pydoc.out')
        with open(filename, 'w', errors='backslashreplace',
                  encoding=os.device_encoding(0) if
                  sys.platform == 'win32' else None
                  ) as file:
            file.write(text)
        os.system(cmd + ' "' + filename + '"')
--- a/Lib/_pyrepl/reader.py
+++ b/Lib/_pyrepl/reader.py
@@ -0,0 +1,816 @@
 #   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
 #                       Antonio Cuni
 #                       Armin Rigo
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 from __future__ import annotations
 import sys
 from contextlib import contextmanager
 from dataclasses import dataclass, field, fields
 import unicodedata
 from _colorize import can_colorize, ANSIColors  # type: ignore[import-not-found]
 from . import commands, console, input
 from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width
 from .trace import trace
 # types
 Command = commands.Command
 from .types import Callback, SimpleContextManager, KeySpec, CommandName
 def disp_str(buffer: str) -> tuple[str, list[int]]:
    """disp_str(buffer:string) -> (string, [int])
    Return the string that should be the printed representation of
    |buffer| and a list detailing where the characters of |buffer|
    get used up.  E.g.:
    >>> disp_str(chr(3))
    ('^C', [1, 0])
    """
    b: list[int] = []
    s: list[str] = []
    for c in buffer:
        if c == '\x1a':
            s.append(c)
            b.append(2)
        elif ord(c) < 128:
            s.append(c)
            b.append(1)
        elif unicodedata.category(c).startswith("C"):
            c = r"\u%04x" % ord(c)
            s.append(c)
            b.extend([0] * (len(c) - 1))
        else:
            s.append(c)
            b.append(str_width(c))
    return "".join(s), b
 # syntax classes:
 SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3)
 def make_default_syntax_table() -> dict[str, int]:
    # XXX perhaps should use some unicodedata here?
    st: dict[str, int] = {}
    for c in map(chr, range(256)):
        st[c] = SYNTAX_SYMBOL
    for c in [a for a in map(chr, range(256)) if a.isalnum()]:
        st[c] = SYNTAX_WORD
    st["\n"] = st[" "] = SYNTAX_WHITESPACE
    return st
 def make_default_commands() -> dict[CommandName, type[Command]]:
    result: dict[CommandName, type[Command]] = {}
    for v in vars(commands).values():
        if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower():
            result[v.__name__] = v
            result[v.__name__.replace("_", "-")] = v
    return result
 default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple(
    [
        (r"\C-a", "beginning-of-line"),
        (r"\C-b", "left"),
        (r"\C-c", "interrupt"),
        (r"\C-d", "delete"),
        (r"\C-e", "end-of-line"),
        (r"\C-f", "right"),
        (r"\C-g", "cancel"),
        (r"\C-h", "backspace"),
        (r"\C-j", "accept"),
        (r"\<return>", "accept"),
        (r"\C-k", "kill-line"),
        (r"\C-l", "clear-screen"),
        (r"\C-m", "accept"),
        (r"\C-t", "transpose-characters"),
        (r"\C-u", "unix-line-discard"),
        (r"\C-w", "unix-word-rubout"),
        (r"\C-x\C-u", "upcase-region"),
        (r"\C-y", "yank"),
        *(() if sys.platform == "win32" else ((r"\C-z", "suspend"), )),
        (r"\M-b", "backward-word"),
        (r"\M-c", "capitalize-word"),
        (r"\M-d", "kill-word"),
        (r"\M-f", "forward-word"),
        (r"\M-l", "downcase-word"),
        (r"\M-t", "transpose-words"),
        (r"\M-u", "upcase-word"),
        (r"\M-y", "yank-pop"),
        (r"\M--", "digit-arg"),
        (r"\M-0", "digit-arg"),
        (r"\M-1", "digit-arg"),
        (r"\M-2", "digit-arg"),
        (r"\M-3", "digit-arg"),
        (r"\M-4", "digit-arg"),
        (r"\M-5", "digit-arg"),
        (r"\M-6", "digit-arg"),
        (r"\M-7", "digit-arg"),
        (r"\M-8", "digit-arg"),
        (r"\M-9", "digit-arg"),
        (r"\M-\n", "accept"),
        ("\\\\", "self-insert"),
        (r"\x1b[200~", "enable_bracketed_paste"),
        (r"\x1b[201~", "disable_bracketed_paste"),
        (r"\x03", "ctrl-c"),
    ]
    + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"]
    + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()]
    + [
        (r"\<up>", "up"),
        (r"\<down>", "down"),
        (r"\<left>", "left"),
        (r"\C-\<left>", "backward-word"),
        (r"\<right>", "right"),
        (r"\C-\<right>", "forward-word"),
        (r"\<delete>", "delete"),
        (r"\x1b[3~", "delete"),
        (r"\<backspace>", "backspace"),
        (r"\M-\<backspace>", "backward-kill-word"),
        (r"\<end>", "end-of-line"),  # was 'end'
        (r"\<home>", "beginning-of-line"),  # was 'home'
        (r"\<f1>", "help"),
        (r"\<f2>", "show-history"),
        (r"\<f3>", "paste-mode"),
        (r"\EOF", "end"),  # the entries in the terminfo database for xterms
        (r"\EOH", "home"),  # seem to be wrong.  this is a less than ideal
        # workaround
    ]
 )
@dataclass(slots=True)
 class Reader:
    """The Reader class implements the bare bones of a command reader,
    handling such details as editing and cursor motion.  What it does
    not support are such things as completion or history support -
    these are implemented elsewhere.
    Instance variables of note include:
      * buffer:
        A *list* (*not* a string at the moment :-) containing all the
        characters that have been entered.
      * console:
        Hopefully encapsulates the OS dependent stuff.
      * pos:
        A 0-based index into `buffer' for where the insertion point
        is.
      * screeninfo:
        Ahem.  This list contains some info needed to move the
        insertion point around reasonably efficiently.
      * cxy, lxy:
        the position of the insertion point in screen ...
      * syntax_table:
        Dictionary mapping characters to `syntax class'; read the
        emacs docs to see what this means :-)
      * commands:
        Dictionary mapping command names to command classes.
      * arg:
        The emacs-style prefix argument.  It will be None if no such
        argument has been provided.
      * dirty:
        True if we need to refresh the display.
      * kill_ring:
        The emacs-style kill-ring; manipulated with yank & yank-pop
      * ps1, ps2, ps3, ps4:
        prompts.  ps1 is the prompt for a one-line input; for a
        multiline input it looks like:
            ps2> first line of input goes here
            ps3> second and further
            ps3> lines get ps3
            ...
            ps4> and the last one gets ps4
        As with the usual top-level, you can set these to instances if
        you like; str() will be called on them (once) at the beginning
        of each command.  Don't put really long or newline containing
        strings here, please!
        This is just the default policy; you can change it freely by
        overriding get_prompt() (and indeed some standard subclasses
        do).
      * finished:
        handle1 will set this to a true value if a command signals
        that we're done.
    """
    console: console.Console
    ## state
    buffer: list[str] = field(default_factory=list)
    pos: int = 0
    ps1: str = "->> "
    ps2: str = "/>> "
    ps3: str = "|.. "
    ps4: str = R"\__ "
    kill_ring: list[list[str]] = field(default_factory=list)
    msg: str = ""
    arg: int | None = None
    dirty: bool = False
    finished: bool = False
    paste_mode: bool = False
    in_bracketed_paste: bool = False
    commands: dict[str, type[Command]] = field(default_factory=make_default_commands)
    last_command: type[Command] | None = None
    syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table)
    keymap: tuple[tuple[str, str], ...] = ()
    input_trans: input.KeymapTranslator = field(init=False)
    input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list)
    screen: list[str] = field(default_factory=list)
    screeninfo: list[tuple[int, list[int]]] = field(init=False)
    cxy: tuple[int, int] = field(init=False)
    lxy: tuple[int, int] = field(init=False)
    scheduled_commands: list[str] = field(default_factory=list)
    can_colorize: bool = False
    threading_hook: Callback | None = None
    ## cached metadata to speed up screen refreshes
    @dataclass
    class RefreshCache:
        in_bracketed_paste: bool = False
        screen: list[str] = field(default_factory=list)
        screeninfo: list[tuple[int, list[int]]] = field(init=False)
        line_end_offsets: list[int] = field(default_factory=list)
        pos: int = field(init=False)
        cxy: tuple[int, int] = field(init=False)
        dimensions: tuple[int, int] = field(init=False)
        invalidated: bool = False
        def update_cache(self,
                         reader: Reader,
                         screen: list[str],
                         screeninfo: list[tuple[int, list[int]]],
            ) -> None:
            self.in_bracketed_paste = reader.in_bracketed_paste
            self.screen = screen.copy()
            self.screeninfo = screeninfo.copy()
            self.pos = reader.pos
            self.cxy = reader.cxy
            self.dimensions = reader.console.width, reader.console.height
            self.invalidated = False
        def valid(self, reader: Reader) -> bool:
            if self.invalidated:
                return False
            dimensions = reader.console.width, reader.console.height
            dimensions_changed = dimensions != self.dimensions
            paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste
            return not (dimensions_changed or paste_changed)
        def get_cached_location(self, reader: Reader) -> tuple[int, int]:
            if self.invalidated:
                raise ValueError("Cache is invalidated")
            offset = 0
            earliest_common_pos = min(reader.pos, self.pos)
            num_common_lines = len(self.line_end_offsets)
            while num_common_lines > 0:
                offset = self.line_end_offsets[num_common_lines - 1]
                if earliest_common_pos > offset:
                    break
                num_common_lines -= 1
            else:
                offset = 0
            return offset, num_common_lines
    last_refresh_cache: RefreshCache = field(default_factory=RefreshCache)
    def __post_init__(self) -> None:
        # Enable the use of `insert` without a `prepare` call - necessary to
        # facilitate the tab completion hack implemented for
        # <https://bugs.python.org/issue25660>.
        self.keymap = self.collect_keymap()
        self.input_trans = input.KeymapTranslator(
            self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
        )
        self.screeninfo = [(0, [])]
        self.cxy = self.pos2xy()
        self.lxy = (self.pos, 0)
        self.can_colorize = can_colorize()
        self.last_refresh_cache.screeninfo = self.screeninfo
        self.last_refresh_cache.pos = self.pos
        self.last_refresh_cache.cxy = self.cxy
        self.last_refresh_cache.dimensions = (0, 0)
    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
        return default_keymap
    def calc_screen(self) -> list[str]:
        """Translate changes in self.buffer into changes in self.console.screen."""
        # Since the last call to calc_screen:
        # screen and screeninfo may differ due to a completion menu being shown
        # pos and cxy may differ due to edits, cursor movements, or completion menus
        # Lines that are above both the old and new cursor position can't have changed,
        # unless the terminal has been resized (which might cause reflowing) or we've
        # entered or left paste mode (which changes prompts, causing reflowing).
        num_common_lines = 0
        offset = 0
        if self.last_refresh_cache.valid(self):
            offset, num_common_lines = self.last_refresh_cache.get_cached_location(self)
        screen = self.last_refresh_cache.screen
        del screen[num_common_lines:]
        screeninfo = self.last_refresh_cache.screeninfo
        del screeninfo[num_common_lines:]
        last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets
        del last_refresh_line_end_offsets[num_common_lines:]
        pos = self.pos
        pos -= offset
        prompt_from_cache = (offset and self.buffer[offset - 1] != "\n")
        lines = "".join(self.buffer[offset:]).split("\n")
        cursor_found = False
        lines_beyond_cursor = 0
        for ln, line in enumerate(lines, num_common_lines):
            ll = len(line)
            if 0 <= pos <= ll:
                self.lxy = pos, ln
                cursor_found = True
            elif cursor_found:
                lines_beyond_cursor += 1
                if lines_beyond_cursor > self.console.height:
                    # No need to keep formatting lines.
                    # The console can't show them.
                    break
            if prompt_from_cache:
                # Only the first line's prompt can come from the cache
                prompt_from_cache = False
                prompt = ""
            else:
                prompt = self.get_prompt(ln, ll >= pos >= 0)
            while "\n" in prompt:
                pre_prompt, _, prompt = prompt.partition("\n")
                last_refresh_line_end_offsets.append(offset)
                screen.append(pre_prompt)
                screeninfo.append((0, []))
            pos -= ll + 1
            prompt, lp = self.process_prompt(prompt)
            l, l2 = disp_str(line)
            wrapcount = (wlen(l) + lp) // self.console.width
            if wrapcount == 0:
                offset += ll + 1  # Takes all of the line plus the newline
                last_refresh_line_end_offsets.append(offset)
                screen.append(prompt + l)
                screeninfo.append((lp, l2))
            else:
                i = 0
                while l:
                    prelen = lp if i == 0 else 0
                    index_to_wrap_before = 0
                    column = 0
                    for character_width in l2:
                        if column + character_width >= self.console.width - prelen:
                            break
                        index_to_wrap_before += 1
                        column += character_width
                    pre = prompt if i == 0 else ""
                    if len(l) > index_to_wrap_before:
                        offset += index_to_wrap_before
                        post = "\\"
                        after = [1]
                    else:
                        offset += index_to_wrap_before + 1  # Takes the newline
                        post = ""
                        after = []
                    last_refresh_line_end_offsets.append(offset)
                    screen.append(pre + l[:index_to_wrap_before] + post)
                    screeninfo.append((prelen, l2[:index_to_wrap_before] + after))
                    l = l[index_to_wrap_before:]
                    l2 = l2[index_to_wrap_before:]
                    i += 1
        self.screeninfo = screeninfo
        self.cxy = self.pos2xy()
        if self.msg:
            for mline in self.msg.split("\n"):
                screen.append(mline)
                screeninfo.append((0, []))
        self.last_refresh_cache.update_cache(self, screen, screeninfo)
        return screen
    @staticmethod
    def process_prompt(prompt: str) -> tuple[str, int]:
        """Process the prompt.
        This means calculate the length of the prompt. The character \x01
        and \x02 are used to bracket ANSI control sequences and need to be
        excluded from the length calculation.  So also a copy of the prompt
        is returned with these control characters removed."""
        # The logic below also ignores the length of common escape
        # sequences if they were not explicitly within \x01...\x02.
        # They are CSI (or ANSI) sequences  ( ESC [ ... LETTER )
        # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars,
        # which breaks the logic below so we redefine it here.
        def wlen(s: str) -> int:
            return sum(str_width(i) for i in s)
        out_prompt = ""
        l = wlen(prompt)
        pos = 0
        while True:
            s = prompt.find("\x01", pos)
            if s == -1:
                break
            e = prompt.find("\x02", s)
            if e == -1:
                break
            # Found start and end brackets, subtract from string length
            l = l - (e - s + 1)
            keep = prompt[pos:s]
            l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep)))
            out_prompt += keep + prompt[s + 1 : e]
            pos = e + 1
        keep = prompt[pos:]
        l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep)))
        out_prompt += keep
        return out_prompt, l
    def bow(self, p: int | None = None) -> int:
        """Return the 0-based index of the word break preceding p most
        immediately.
        p defaults to self.pos; word boundaries are determined using
        self.syntax_table."""
        if p is None:
            p = self.pos
        st = self.syntax_table
        b = self.buffer
        p -= 1
        while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD:
            p -= 1
        while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
            p -= 1
        return p + 1
    def eow(self, p: int | None = None) -> int:
        """Return the 0-based index of the word break following p most
        immediately.
        p defaults to self.pos; word boundaries are determined using
        self.syntax_table."""
        if p is None:
            p = self.pos
        st = self.syntax_table
        b = self.buffer
        while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD:
            p += 1
        while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
            p += 1
        return p
    def bol(self, p: int | None = None) -> int:
        """Return the 0-based index of the line break preceding p most
        immediately.
        p defaults to self.pos."""
        if p is None:
            p = self.pos
        b = self.buffer
        p -= 1
        while p >= 0 and b[p] != "\n":
            p -= 1
        return p + 1
    def eol(self, p: int | None = None) -> int:
        """Return the 0-based index of the line break following p most
        immediately.
        p defaults to self.pos."""
        if p is None:
            p = self.pos
        b = self.buffer
        while p < len(b) and b[p] != "\n":
            p += 1
        return p
    def max_column(self, y: int) -> int:
        """Return the last x-offset for line y"""
        return self.screeninfo[y][0] + sum(self.screeninfo[y][1])
    def max_row(self) -> int:
        return len(self.screeninfo) - 1
    def get_arg(self, default: int = 1) -> int:
        """Return any prefix argument that the user has supplied,
        returning `default' if there is None.  Defaults to 1.
        """
        if self.arg is None:
            return default
        return self.arg
    def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
        """Return what should be in the left-hand margin for line
        `lineno'."""
        if self.arg is not None and cursor_on_line:
            prompt = f"(arg: {self.arg}) "
        elif self.paste_mode and not self.in_bracketed_paste:
            prompt = "(paste) "
        elif "\n" in self.buffer:
            if lineno == 0:
                prompt = self.ps2
            elif self.ps4 and lineno == self.buffer.count("\n"):
                prompt = self.ps4
            else:
                prompt = self.ps3
        else:
            prompt = self.ps1
        if self.can_colorize:
            prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}"
        return prompt
    def push_input_trans(self, itrans: input.KeymapTranslator) -> None:
        self.input_trans_stack.append(self.input_trans)
        self.input_trans = itrans
    def pop_input_trans(self) -> None:
        self.input_trans = self.input_trans_stack.pop()
    def setpos_from_xy(self, x: int, y: int) -> None:
        """Set pos according to coordinates x, y"""
        pos = 0
        i = 0
        while i < y:
            prompt_len, character_widths = self.screeninfo[i]
            offset = len(character_widths) - character_widths.count(0)
            in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width
            if in_wrapped_line:
                pos += offset - 1  # -1 cause backslash is not in buffer
            else:
                pos += offset + 1  # +1 cause newline is in buffer
            i += 1
        j = 0
        cur_x = self.screeninfo[i][0]
        while cur_x < x:
            if self.screeninfo[i][1][j] == 0:
                continue
            cur_x += self.screeninfo[i][1][j]
            j += 1
            pos += 1
        self.pos = pos
    def pos2xy(self) -> tuple[int, int]:
        """Return the x, y coordinates of position 'pos'."""
        # this *is* incomprehensible, yes.
        p, y = 0, 0
        l2: list[int] = []
        pos = self.pos
        assert 0 <= pos <= len(self.buffer)
        if pos == len(self.buffer) and len(self.screeninfo) > 0:
            y = len(self.screeninfo) - 1
            p, l2 = self.screeninfo[y]
            return p + sum(l2) + l2.count(0), y
        for p, l2 in self.screeninfo:
            l = len(l2) - l2.count(0)
            in_wrapped_line = p + sum(l2) >= self.console.width
            offset = l - 1 if in_wrapped_line else l  # need to remove backslash
            if offset >= pos:
                break
            if p + sum(l2) >= self.console.width:
                pos -= l - 1  # -1 cause backslash is not in buffer
            else:
                pos -= l + 1  # +1 cause newline is in buffer
            y += 1
        return p + sum(l2[:pos]), y
    def insert(self, text: str | list[str]) -> None:
        """Insert 'text' at the insertion point."""
        self.buffer[self.pos : self.pos] = list(text)
        self.pos += len(text)
        self.dirty = True
    def update_cursor(self) -> None:
        """Move the cursor to reflect changes in self.pos"""
        self.cxy = self.pos2xy()
        self.console.move_cursor(*self.cxy)
    def after_command(self, cmd: Command) -> None:
        """This function is called to allow post command cleanup."""
        if getattr(cmd, "kills_digit_arg", True):
            if self.arg is not None:
                self.dirty = True
            self.arg = None
    def prepare(self) -> None:
        """Get ready to run.  Call restore when finished.  You must not
        write to the console in between the calls to prepare and
        restore."""
        try:
            self.console.prepare()
            self.arg = None
            self.finished = False
            del self.buffer[:]
            self.pos = 0
            self.dirty = True
            self.last_command = None
            self.calc_screen()
        except BaseException:
            self.restore()
            raise
        while self.scheduled_commands:
            cmd = self.scheduled_commands.pop()
            self.do_cmd((cmd, []))
    def last_command_is(self, cls: type) -> bool:
        if not self.last_command:
            return False
        return issubclass(cls, self.last_command)
    def restore(self) -> None:
        """Clean up after a run."""
        self.console.restore()
    @contextmanager
    def suspend(self) -> SimpleContextManager:
        """A context manager to delegate to another reader."""
        prev_state = {f.name: getattr(self, f.name) for f in fields(self)}
        try:
            self.restore()
            yield
        finally:
            for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"):
                setattr(self, arg, prev_state[arg])
            self.prepare()
    def finish(self) -> None:
        """Called when a command signals that we're finished."""
        pass
    def error(self, msg: str = "none") -> None:
        self.msg = "! " + msg + " "
        self.dirty = True
        self.console.beep()
    def update_screen(self) -> None:
        if self.dirty:
            self.refresh()
    def refresh(self) -> None:
        """Recalculate and refresh the screen."""
        if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n":
            return
        # this call sets up self.cxy, so call it first.
        self.screen = self.calc_screen()
        self.console.refresh(self.screen, self.cxy)
        self.dirty = False
    def do_cmd(self, cmd: tuple[str, list[str]]) -> None:
        """`cmd` is a tuple of "event_name" and "event", which in the current
        implementation is always just the "buffer" which happens to be a list
        of single-character strings."""
        trace("received command {cmd}", cmd=cmd)
        if isinstance(cmd[0], str):
            command_type = self.commands.get(cmd[0], commands.invalid_command)
        elif isinstance(cmd[0], type):
            command_type = cmd[0]
        else:
            return  # nothing to do
        command = command_type(self, *cmd)  # type: ignore[arg-type]
        command.do()
        self.after_command(command)
        if self.dirty:
            self.refresh()
        else:
            self.update_cursor()
        if not isinstance(cmd, commands.digit_arg):
            self.last_command = command_type
        self.finished = bool(command.finish)
        if self.finished:
            self.console.finish()
            self.finish()
    def run_hooks(self) -> None:
        threading_hook = self.threading_hook
        if threading_hook is None and 'threading' in sys.modules:
            from ._threading_handler import install_threading_hook
            install_threading_hook(self)
        if threading_hook is not None:
            try:
                threading_hook()
            except Exception:
                pass
        input_hook = self.console.input_hook
        if input_hook:
            try:
                input_hook()
            except Exception:
                pass
    def handle1(self, block: bool = True) -> bool:
        """Handle a single event.  Wait as long as it takes if block
        is true (the default), otherwise return False if no event is
        pending."""
        if self.msg:
            self.msg = ""
            self.dirty = True
        while True:
            # We use the same timeout as in readline.c: 100ms
            self.run_hooks()
            self.console.wait(100)
            event = self.console.get_event(block=False)
            if not event:
                if block:
                    continue
                return False
            translate = True
            if event.evt == "key":
                self.input_trans.push(event)
            elif event.evt == "scroll":
                self.refresh()
            elif event.evt == "resize":
                self.refresh()
            else:
                translate = False
            if translate:
                cmd = self.input_trans.get()
            else:
                cmd = [event.evt, event.data]
            if cmd is None:
                if block:
                    continue
                return False
            self.do_cmd(cmd)
            return True
    def push_char(self, char: int | bytes) -> None:
        self.console.push_char(char)
        self.handle1(block=False)
    def readline(self, startup_hook: Callback | None = None) -> str:
        """Read a line.  The implementation of this method also shows
        how to drive Reader if you want more control over the event
        loop."""
        self.prepare()
        try:
            if startup_hook is not None:
                startup_hook()
            self.refresh()
            while not self.finished:
                self.handle1()
            return self.get_unicode()
        finally:
            self.restore()
    def bind(self, spec: KeySpec, command: CommandName) -> None:
        self.keymap = self.keymap + ((spec, command),)
        self.input_trans = input.KeymapTranslator(
            self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
        )
    def get_unicode(self) -> str:
        """Return the current buffer as a unicode string."""
        return "".join(self.buffer)
--- a/Lib/_pyrepl/readline.py
+++ b/Lib/_pyrepl/readline.py
@@ -0,0 +1,598 @@
 #   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
 #                       Alex Gaynor
 #                       Antonio Cuni
 #                       Armin Rigo
 #                       Holger Krekel
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 """A compatibility wrapper reimplementing the 'readline' standard module
 on top of pyrepl.  Not all functionalities are supported.  Contains
 extensions for multiline input.
 """
 from __future__ import annotations
 import warnings
 from dataclasses import dataclass, field
 import os
 from site import gethistoryfile   # type: ignore[attr-defined]
 import sys
 from rlcompleter import Completer as RLCompleter
 from . import commands, historical_reader
 from .completing_reader import CompletingReader
 from .console import Console as ConsoleType
 Console: type[ConsoleType]
 _error: tuple[type[Exception], ...] | type[Exception]
 try:
    from .unix_console import UnixConsole as Console, _error
 except ImportError:
    from .windows_console import WindowsConsole as Console, _error
 ENCODING = sys.getdefaultencoding() or "latin1"
 # types
 Command = commands.Command
 from collections.abc import Callable, Collection
 from .types import Callback, Completer, KeySpec, CommandName
 TYPE_CHECKING = False
 if TYPE_CHECKING:
    from typing import Any, Mapping
 MoreLinesCallable = Callable[[str], bool]
 __all__ = [
    "add_history",
    "clear_history",
    "get_begidx",
    "get_completer",
    "get_completer_delims",
    "get_current_history_length",
    "get_endidx",
    "get_history_item",
    "get_history_length",
    "get_line_buffer",
    "insert_text",
    "parse_and_bind",
    "read_history_file",
    # "read_init_file",
    # "redisplay",
    "remove_history_item",
    "replace_history_item",
    "set_auto_history",
    "set_completer",
    "set_completer_delims",
    "set_history_length",
    # "set_pre_input_hook",
    "set_startup_hook",
    "write_history_file",
    # ---- multiline extensions ----
    "multiline_input",
 ]
 # ____________________________________________________________
@dataclass
 class ReadlineConfig:
    readline_completer: Completer | None = None
    completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?")
@dataclass(kw_only=True)
 class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader):
    # Class fields
    assume_immutable_completions = False
    use_brackets = False
    sort_in_column = True
    # Instance fields
    config: ReadlineConfig
    more_lines: MoreLinesCallable | None = None
    last_used_indentation: str | None = None
    def __post_init__(self) -> None:
        super().__post_init__()
        self.commands["maybe_accept"] = maybe_accept
        self.commands["maybe-accept"] = maybe_accept
        self.commands["backspace_dedent"] = backspace_dedent
        self.commands["backspace-dedent"] = backspace_dedent
    def error(self, msg: str = "none") -> None:
        pass  # don't show error messages by default
    def get_stem(self) -> str:
        b = self.buffer
        p = self.pos - 1
        completer_delims = self.config.completer_delims
        while p >= 0 and b[p] not in completer_delims:
            p -= 1
        return "".join(b[p + 1 : self.pos])
    def get_completions(self, stem: str) -> list[str]:
        if len(stem) == 0 and self.more_lines is not None:
            b = self.buffer
            p = self.pos
            while p > 0 and b[p - 1] != "\n":
                p -= 1
            num_spaces = 4 - ((self.pos - p) % 4)
            return [" " * num_spaces]
        result = []
        function = self.config.readline_completer
        if function is not None:
            try:
                stem = str(stem)  # rlcompleter.py seems to not like unicode
            except UnicodeEncodeError:
                pass  # but feed unicode anyway if we have no choice
            state = 0
            while True:
                try:
                    next = function(stem, state)
                except Exception:
                    break
                if not isinstance(next, str):
                    break
                result.append(next)
                state += 1
            # emulate the behavior of the standard readline that sorts
            # the completions before displaying them.
            result.sort()
        return result
    def get_trimmed_history(self, maxlength: int) -> list[str]:
        if maxlength >= 0:
            cut = len(self.history) - maxlength
            if cut < 0:
                cut = 0
        else:
            cut = 0
        return self.history[cut:]
    def update_last_used_indentation(self) -> None:
        indentation = _get_first_indentation(self.buffer)
        if indentation is not None:
            self.last_used_indentation = indentation
    # --- simplified support for reading multiline Python statements ---
    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
        return super().collect_keymap() + (
            (r"\n", "maybe-accept"),
            (r"\<backspace>", "backspace-dedent"),
        )
    def after_command(self, cmd: Command) -> None:
        super().after_command(cmd)
        if self.more_lines is None:
            # Force single-line input if we are in raw_input() mode.
            # Although there is no direct way to add a \n in this mode,
            # multiline buffers can still show up using various
            # commands, e.g. navigating the history.
            try:
                index = self.buffer.index("\n")
            except ValueError:
                pass
            else:
                self.buffer = self.buffer[:index]
                if self.pos > len(self.buffer):
                    self.pos = len(self.buffer)
 def set_auto_history(_should_auto_add_history: bool) -> None:
    """Enable or disable automatic history"""
    historical_reader.should_auto_add_history = bool(_should_auto_add_history)
 def _get_this_line_indent(buffer: list[str], pos: int) -> int:
    indent = 0
    while pos > 0 and buffer[pos - 1] in " \t":
        indent += 1
        pos -= 1
    if pos > 0 and buffer[pos - 1] == "\n":
        return indent
    return 0
 def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]:
    prevlinestart = pos
    while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n":
        prevlinestart -= 1
    prevlinetext = prevlinestart
    while prevlinetext < pos and buffer[prevlinetext] in " \t":
        prevlinetext += 1
    if prevlinetext == pos:
        indent = None
    else:
        indent = prevlinetext - prevlinestart
    return prevlinestart, indent
 def _get_first_indentation(buffer: list[str]) -> str | None:
    indented_line_start = None
    for i in range(len(buffer)):
        if (i < len(buffer) - 1
            and buffer[i] == "\n"
            and buffer[i + 1] in " \t"
        ):
            indented_line_start = i + 1
        elif indented_line_start is not None and buffer[i] not in " \t\n":
            return ''.join(buffer[indented_line_start : i])
    return None
 def _should_auto_indent(buffer: list[str], pos: int) -> bool:
    # check if last character before "pos" is a colon, ignoring
    # whitespaces and comments.
    last_char = None
    while pos > 0:
        pos -= 1
        if last_char is None:
            if buffer[pos] not in " \t\n#":  # ignore whitespaces and comments
                last_char = buffer[pos]
        else:
            # even if we found a non-whitespace character before
            # original pos, we keep going back until newline is reached
            # to make sure we ignore comments
            if buffer[pos] == "\n":
                break
            if buffer[pos] == "#":
                last_char = None
    return last_char == ":"
 class maybe_accept(commands.Command):
    def do(self) -> None:
        r: ReadlineAlikeReader
        r = self.reader  # type: ignore[assignment]
        r.dirty = True  # this is needed to hide the completion menu, if visible
        if self.reader.in_bracketed_paste:
            r.insert("\n")
            return
        # if there are already several lines and the cursor
        # is not on the last one, always insert a new \n.
        text = r.get_unicode()
        if "\n" in r.buffer[r.pos :] or (
            r.more_lines is not None and r.more_lines(text)
        ):
            def _newline_before_pos():
                before_idx = r.pos - 1
                while before_idx > 0 and text[before_idx].isspace():
                    before_idx -= 1
                return text[before_idx : r.pos].count("\n") > 0
            # if there's already a new line before the cursor then
            # even if the cursor is followed by whitespace, we assume
            # the user is trying to terminate the block
            if _newline_before_pos() and text[r.pos:].isspace():
                self.finish = True
                return
            # auto-indent the next line like the previous line
            prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos)
            r.insert("\n")
            if not self.reader.paste_mode:
                if indent:
                    for i in range(prevlinestart, prevlinestart + indent):
                        r.insert(r.buffer[i])
                r.update_last_used_indentation()
                if _should_auto_indent(r.buffer, r.pos):
                    if r.last_used_indentation is not None:
                        indentation = r.last_used_indentation
                    else:
                        # default
                        indentation = " " * 4
                    r.insert(indentation)
        elif not self.reader.paste_mode:
            self.finish = True
        else:
            r.insert("\n")
 class backspace_dedent(commands.Command):
    def do(self) -> None:
        r = self.reader
        b = r.buffer
        if r.pos > 0:
            repeat = 1
            if b[r.pos - 1] != "\n":
                indent = _get_this_line_indent(b, r.pos)
                if indent > 0:
                    ls = r.pos - indent
                    while ls > 0:
                        ls, pi = _get_previous_line_indent(b, ls - 1)
                        if pi is not None and pi < indent:
                            repeat = indent - pi
                            break
            r.pos -= repeat
            del b[r.pos : r.pos + repeat]
            r.dirty = True
        else:
            self.reader.error("can't backspace at start")
 # ____________________________________________________________
@dataclass(slots=True)
 class _ReadlineWrapper:
    f_in: int = -1
    f_out: int = -1
    reader: ReadlineAlikeReader | None = field(default=None, repr=False)
    saved_history_length: int = -1
    startup_hook: Callback | None = None
    config: ReadlineConfig = field(default_factory=ReadlineConfig, repr=False)
    def __post_init__(self) -> None:
        if self.f_in == -1:
            self.f_in = os.dup(0)
        if self.f_out == -1:
            self.f_out = os.dup(1)
    def get_reader(self) -> ReadlineAlikeReader:
        if self.reader is None:
            console = Console(self.f_in, self.f_out, encoding=ENCODING)
            self.reader = ReadlineAlikeReader(console=console, config=self.config)
        return self.reader
    def input(self, prompt: object = "") -> str:
        try:
            reader = self.get_reader()
        except _error:
            assert raw_input is not None
            return raw_input(prompt)
        prompt_str = str(prompt)
        reader.ps1 = prompt_str
        sys.audit("builtins.input", prompt_str)
        result = reader.readline(startup_hook=self.startup_hook)
        sys.audit("builtins.input/result", result)
        return result
    def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str:
        """Read an input on possibly multiple lines, asking for more
        lines as long as 'more_lines(unicodetext)' returns an object whose
        boolean value is true.
        """
        reader = self.get_reader()
        saved = reader.more_lines
        try:
            reader.more_lines = more_lines
            reader.ps1 = ps1
            reader.ps2 = ps1
            reader.ps3 = ps2
            reader.ps4 = ""
            with warnings.catch_warnings(action="ignore"):
                return reader.readline()
        finally:
            reader.more_lines = saved
            reader.paste_mode = False
    def parse_and_bind(self, string: str) -> None:
        pass  # XXX we don't support parsing GNU-readline-style init files
    def set_completer(self, function: Completer | None = None) -> None:
        self.config.readline_completer = function
    def get_completer(self) -> Completer | None:
        return self.config.readline_completer
    def set_completer_delims(self, delimiters: Collection[str]) -> None:
        self.config.completer_delims = frozenset(delimiters)
    def get_completer_delims(self) -> str:
        return "".join(sorted(self.config.completer_delims))
    def _histline(self, line: str) -> str:
        line = line.rstrip("\n")
        return line
    def get_history_length(self) -> int:
        return self.saved_history_length
    def set_history_length(self, length: int) -> None:
        self.saved_history_length = length
    def get_current_history_length(self) -> int:
        return len(self.get_reader().history)
    def read_history_file(self, filename: str = gethistoryfile()) -> None:
        # multiline extension (really a hack) for the end of lines that
        # are actually continuations inside a single multiline_input()
        # history item: we use \r\n instead of just \n.  If the history
        # file is passed to GNU readline, the extra \r are just ignored.
        history = self.get_reader().history
        with open(os.path.expanduser(filename), 'rb') as f:
            is_editline = f.readline().startswith(b"_HiStOrY_V2_")
            if is_editline:
                encoding = "unicode-escape"
            else:
                f.seek(0)
                encoding = "utf-8"
            lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')]
            buffer = []
            for line in lines:
                if line.endswith("\r"):
                    buffer.append(line+'\n')
                else:
                    line = self._histline(line)
                    if buffer:
                        line = self._histline("".join(buffer).replace("\r", "") + line)
                        del buffer[:]
                    if line:
                        history.append(line)
    def write_history_file(self, filename: str = gethistoryfile()) -> None:
        maxlength = self.saved_history_length
        history = self.get_reader().get_trimmed_history(maxlength)
        f = open(os.path.expanduser(filename), "w",
                 encoding="utf-8", newline="\n")
        with f:
            for entry in history:
                entry = entry.replace("\n", "\r\n")  # multiline history support
                f.write(entry + "\n")
    def clear_history(self) -> None:
        del self.get_reader().history[:]
    def get_history_item(self, index: int) -> str | None:
        history = self.get_reader().history
        if 1 <= index <= len(history):
            return history[index - 1]
        else:
            return None  # like readline.c
    def remove_history_item(self, index: int) -> None:
        history = self.get_reader().history
        if 0 <= index < len(history):
            del history[index]
        else:
            raise ValueError("No history item at position %d" % index)
            # like readline.c
    def replace_history_item(self, index: int, line: str) -> None:
        history = self.get_reader().history
        if 0 <= index < len(history):
            history[index] = self._histline(line)
        else:
            raise ValueError("No history item at position %d" % index)
            # like readline.c
    def add_history(self, line: str) -> None:
        self.get_reader().history.append(self._histline(line))
    def set_startup_hook(self, function: Callback | None = None) -> None:
        self.startup_hook = function
    def get_line_buffer(self) -> str:
        return self.get_reader().get_unicode()
    def _get_idxs(self) -> tuple[int, int]:
        start = cursor = self.get_reader().pos
        buf = self.get_line_buffer()
        for i in range(cursor - 1, -1, -1):
            if buf[i] in self.get_completer_delims():
                break
            start = i
        return start, cursor
    def get_begidx(self) -> int:
        return self._get_idxs()[0]
    def get_endidx(self) -> int:
        return self._get_idxs()[1]
    def insert_text(self, text: str) -> None:
        self.get_reader().insert(text)
 _wrapper = _ReadlineWrapper()
 # ____________________________________________________________
 # Public API
 parse_and_bind = _wrapper.parse_and_bind
 set_completer = _wrapper.set_completer
 get_completer = _wrapper.get_completer
 set_completer_delims = _wrapper.set_completer_delims
 get_completer_delims = _wrapper.get_completer_delims
 get_history_length = _wrapper.get_history_length
 set_history_length = _wrapper.set_history_length
 get_current_history_length = _wrapper.get_current_history_length
 read_history_file = _wrapper.read_history_file
 write_history_file = _wrapper.write_history_file
 clear_history = _wrapper.clear_history
 get_history_item = _wrapper.get_history_item
 remove_history_item = _wrapper.remove_history_item
 replace_history_item = _wrapper.replace_history_item
 add_history = _wrapper.add_history
 set_startup_hook = _wrapper.set_startup_hook
 get_line_buffer = _wrapper.get_line_buffer
 get_begidx = _wrapper.get_begidx
 get_endidx = _wrapper.get_endidx
 insert_text = _wrapper.insert_text
 # Extension
 multiline_input = _wrapper.multiline_input
 # Internal hook
 _get_reader = _wrapper.get_reader
 # ____________________________________________________________
 # Stubs
 def _make_stub(_name: str, _ret: object) -> None:
    def stub(*args: object, **kwds: object) -> None:
        import warnings
        warnings.warn("readline.%s() not implemented" % _name, stacklevel=2)
    stub.__name__ = _name
    globals()[_name] = stub
 for _name, _ret in [
    ("read_init_file", None),
    ("redisplay", None),
    ("set_pre_input_hook", None),
 ]:
    assert _name not in globals(), _name
    _make_stub(_name, _ret)
 # ____________________________________________________________
 def _setup(namespace: Mapping[str, Any]) -> None:
    global raw_input
    if raw_input is not None:
        return  # don't run _setup twice
    try:
        f_in = sys.stdin.fileno()
        f_out = sys.stdout.fileno()
    except (AttributeError, ValueError):
        return
    if not os.isatty(f_in) or not os.isatty(f_out):
        return
    _wrapper.f_in = f_in
    _wrapper.f_out = f_out
    # set up namespace in rlcompleter, which requires it to be a bona fide dict
    if not isinstance(namespace, dict):
        namespace = dict(namespace)
    _wrapper.config.readline_completer = RLCompleter(namespace).complete
    # this is not really what readline.c does.  Better than nothing I guess
    import builtins
    raw_input = builtins.input
    builtins.input = _wrapper.input
 raw_input: Callable[[object], str] | None = None
--- a/Lib/_pyrepl/simple_interact.py
+++ b/Lib/_pyrepl/simple_interact.py
@@ -0,0 +1,167 @@
 #   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
 #                       Armin Rigo
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 """This is an alternative to python_reader which tries to emulate
 the CPython prompt as closely as possible, with the exception of
 allowing multiline input and multiline history entries.
 """
 from __future__ import annotations
 import _sitebuiltins
 import linecache
 import functools
 import os
 import sys
 import code
 from .readline import _get_reader, multiline_input
 TYPE_CHECKING = False
 if TYPE_CHECKING:
    from typing import Any
 _error: tuple[type[Exception], ...] | type[Exception]
 try:
    from .unix_console import _error
 except ModuleNotFoundError:
    from .windows_console import _error
 def check() -> str:
    """Returns the error message if there is a problem initializing the state."""
    try:
        _get_reader()
    except _error as e:
        if term := os.environ.get("TERM", ""):
            term = f"; TERM={term}"
        return str(str(e) or repr(e) or "unknown error") + term
    return ""
 def _strip_final_indent(text: str) -> str:
    # kill spaces and tabs at the end, but only if they follow '\n'.
    # meant to remove the auto-indentation only (although it would of
    # course also remove explicitly-added indentation).
    short = text.rstrip(" \t")
    n = len(short)
    if n > 0 and text[n - 1] == "\n":
        return short
    return text
 def _clear_screen():
    reader = _get_reader()
    reader.scheduled_commands.append("clear_screen")
 REPL_COMMANDS = {
    "exit": _sitebuiltins.Quitter('exit', ''),
    "quit": _sitebuiltins.Quitter('quit' ,''),
    "copyright": _sitebuiltins._Printer('copyright', sys.copyright),
    "help": _sitebuiltins._Helper(),
    "clear": _clear_screen,
    "\x1a": _sitebuiltins.Quitter('\x1a', ''),
 }
 def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool:
    # ooh, look at the hack:
    src = _strip_final_indent(unicodetext)
    try:
        code = console.compile(src, "<stdin>", "single")
    except (OverflowError, SyntaxError, ValueError):
        lines = src.splitlines(keepends=True)
        if len(lines) == 1:
            return False
        last_line = lines[-1]
        was_indented = last_line.startswith((" ", "\t"))
        not_empty = last_line.strip() != ""
        incomplete = not last_line.endswith("\n")
        return (was_indented or not_empty) and incomplete
    else:
        return code is None
 def run_multiline_interactive_console(
    console: code.InteractiveConsole,
    *,
    future_flags: int = 0,
 ) -> None:
    from .readline import _setup
    _setup(console.locals)
    if future_flags:
        console.compile.compiler.flags |= future_flags
    more_lines = functools.partial(_more_lines, console)
    input_n = 0
    def maybe_run_command(statement: str) -> bool:
        statement = statement.strip()
        if statement in console.locals or statement not in REPL_COMMANDS:
            return False
        reader = _get_reader()
        reader.history.pop()  # skip internal commands in history
        command = REPL_COMMANDS[statement]
        if callable(command):
            # Make sure that history does not change because of commands
            with reader.suspend_history():
                command()
            return True
        return False
    while 1:
        try:
            try:
                sys.stdout.flush()
            except Exception:
                pass
            ps1 = getattr(sys, "ps1", ">>> ")
            ps2 = getattr(sys, "ps2", "... ")
            try:
                statement = multiline_input(more_lines, ps1, ps2)
            except EOFError:
                break
            if maybe_run_command(statement):
                continue
            input_name = f"<python-input-{input_n}>"
            linecache._register_code(input_name, statement, "<stdin>")  # type: ignore[attr-defined]
            more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single")  # type: ignore[call-arg]
            assert not more
            input_n += 1
        except KeyboardInterrupt:
            r = _get_reader()
            if r.input_trans is r.isearch_trans:
                r.do_cmd(("isearch-end", [""]))
            r.pos = len(r.get_unicode())
            r.dirty = True
            r.refresh()
            r.in_bracketed_paste = False
            console.write("\nKeyboardInterrupt\n")
            console.resetbuffer()
        except MemoryError:
            console.write("\nMemoryError\n")
            console.resetbuffer()
--- a/Lib/_pyrepl/trace.py
+++ b/Lib/_pyrepl/trace.py
@@ -0,0 +1,21 @@
 from __future__ import annotations
 import os
 # types
 if False:
    from typing import IO
 trace_file: IO[str] | None = None
 if trace_filename := os.environ.get("PYREPL_TRACE"):
    trace_file = open(trace_filename, "a")
 def trace(line: str, *k: object, **kw: object) -> None:
    if trace_file is None:
        return
    if k or kw:
        line = line.format(*k, **kw)
    trace_file.write(line + "\n")
    trace_file.flush()
--- a/Lib/_pyrepl/types.py
+++ b/Lib/_pyrepl/types.py
@@ -0,0 +1,8 @@
 from collections.abc import Callable, Iterator
 Callback = Callable[[], object]
 SimpleContextManager = Iterator[None]
 KeySpec = str  # like r"\C-c"
 CommandName = str  # like "interrupt"
 EventTuple = tuple[CommandName, str]
 Completer = Callable[[str, int], str | None]
--- a/Lib/_pyrepl/unix_console.py
+++ b/Lib/_pyrepl/unix_console.py
@@ -0,0 +1,810 @@
 #   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
 #                       Antonio Cuni
 #                       Armin Rigo
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 from __future__ import annotations
 import errno
 import os
 import re
 import select
 import signal
 import struct
 import termios
 import time
 import platform
 from fcntl import ioctl
 from . import curses
 from .console import Console, Event
 from .fancy_termios import tcgetattr, tcsetattr
 from .trace import trace
 from .unix_eventqueue import EventQueue
 from .utils import wlen
 TYPE_CHECKING = False
 # types
 if TYPE_CHECKING:
    from typing import IO, Literal, overload
 else:
    overload = lambda func: None
 class InvalidTerminal(RuntimeError):
    pass
 _error = (termios.error, curses.error, InvalidTerminal)
 SIGWINCH_EVENT = "repaint"
 FIONREAD = getattr(termios, "FIONREAD", None)
 TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None)
 # ------------ start of baudrate definitions ------------
 # Add (possibly) missing baudrates (check termios man page) to termios
 def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None:
    baudrate_name = "B%d" % rate
    if hasattr(termios, baudrate_name):
        dictionary[getattr(termios, baudrate_name)] = rate
 # Check the termios man page (Line speed) to know where these
 # values come from.
 potential_baudrates = [
    0,
    110,
    115200,
    1200,
    134,
    150,
    1800,
    19200,
    200,
    230400,
    2400,
    300,
    38400,
    460800,
    4800,
    50,
    57600,
    600,
    75,
    9600,
 ]
 ratedict: dict[int, int] = {}
 for rate in potential_baudrates:
    add_baudrate_if_supported(ratedict, rate)
 # Clean up variables to avoid unintended usage
 del rate, add_baudrate_if_supported
 # ------------ end of baudrate definitions ------------
 delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>")
 try:
    poll: type[select.poll] = select.poll
 except AttributeError:
    # this is exactly the minumum necessary to support what we
    # do with poll objects
    class MinimalPoll:
        def __init__(self):
            pass
        def register(self, fd, flag):
            self.fd = fd
        # note: The 'timeout' argument is received as *milliseconds*
        def poll(self, timeout: float | None = None) -> list[int]:
            if timeout is None:
                r, w, e = select.select([self.fd], [], [])
            else:
                r, w, e = select.select([self.fd], [], [], timeout/1000)
            return r
    poll = MinimalPoll  # type: ignore[assignment]
 class UnixConsole(Console):
    def __init__(
        self,
        f_in: IO[bytes] | int = 0,
        f_out: IO[bytes] | int = 1,
        term: str = "",
        encoding: str = "",
    ):
        """
        Initialize the UnixConsole.
        Parameters:
        - f_in (int or file-like object): Input file descriptor or object.
        - f_out (int or file-like object): Output file descriptor or object.
        - term (str): Terminal name.
        - encoding (str): Encoding to use for I/O operations.
        """
        super().__init__(f_in, f_out, term, encoding)
        self.pollob = poll()
        self.pollob.register(self.input_fd, select.POLLIN)
        self.input_buffer = b""
        self.input_buffer_pos = 0
        curses.setupterm(term or None, self.output_fd)
        self.term = term
        @overload
        def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ...
        @overload
        def _my_getstr(cap: str, optional: bool) -> bytes | None: ...
        def _my_getstr(cap: str, optional: bool = False) -> bytes | None:
            r = curses.tigetstr(cap)
            if not optional and r is None:
                raise InvalidTerminal(
                    f"terminal doesn't have the required {cap} capability"
                )
            return r
        self._bel = _my_getstr("bel")
        self._civis = _my_getstr("civis", optional=True)
        self._clear = _my_getstr("clear")
        self._cnorm = _my_getstr("cnorm", optional=True)
        self._cub = _my_getstr("cub", optional=True)
        self._cub1 = _my_getstr("cub1", optional=True)
        self._cud = _my_getstr("cud", optional=True)
        self._cud1 = _my_getstr("cud1", optional=True)
        self._cuf = _my_getstr("cuf", optional=True)
        self._cuf1 = _my_getstr("cuf1", optional=True)
        self._cup = _my_getstr("cup")
        self._cuu = _my_getstr("cuu", optional=True)
        self._cuu1 = _my_getstr("cuu1", optional=True)
        self._dch1 = _my_getstr("dch1", optional=True)
        self._dch = _my_getstr("dch", optional=True)
        self._el = _my_getstr("el")
        self._hpa = _my_getstr("hpa", optional=True)
        self._ich = _my_getstr("ich", optional=True)
        self._ich1 = _my_getstr("ich1", optional=True)
        self._ind = _my_getstr("ind", optional=True)
        self._pad = _my_getstr("pad", optional=True)
        self._ri = _my_getstr("ri", optional=True)
        self._rmkx = _my_getstr("rmkx", optional=True)
        self._smkx = _my_getstr("smkx", optional=True)
        self.__setup_movement()
        self.event_queue = EventQueue(self.input_fd, self.encoding)
        self.cursor_visible = 1
    def more_in_buffer(self) -> bool:
        return bool(
            self.input_buffer
            and self.input_buffer_pos < len(self.input_buffer)
        )
    def __read(self, n: int) -> bytes:
        if not self.more_in_buffer():
            self.input_buffer = os.read(self.input_fd, 10000)
        ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n]
        self.input_buffer_pos += len(ret)
        if self.input_buffer_pos >= len(self.input_buffer):
            self.input_buffer = b""
            self.input_buffer_pos = 0
        return ret
    def change_encoding(self, encoding: str) -> None:
        """
        Change the encoding used for I/O operations.
        Parameters:
        - encoding (str): New encoding to use.
        """
        self.encoding = encoding
    def refresh(self, screen, c_xy):
        """
        Refresh the console screen.
        Parameters:
        - screen (list): List of strings representing the screen contents.
        - c_xy (tuple): Cursor position (x, y) on the screen.
        """
        cx, cy = c_xy
        if not self.__gone_tall:
            while len(self.screen) < min(len(screen), self.height):
                self.__hide_cursor()
                self.__move(0, len(self.screen) - 1)
                self.__write("\n")
                self.posxy = 0, len(self.screen)
                self.screen.append("")
        else:
            while len(self.screen) < len(screen):
                self.screen.append("")
        if len(screen) > self.height:
            self.__gone_tall = 1
            self.__move = self.__move_tall
        px, py = self.posxy
        old_offset = offset = self.__offset
        height = self.height
        # we make sure the cursor is on the screen, and that we're
        # using all of the screen if we can
        if cy < offset:
            offset = cy
        elif cy >= offset + height:
            offset = cy - height + 1
        elif offset > 0 and len(screen) < offset + height:
            offset = max(len(screen) - height, 0)
            screen.append("")
        oldscr = self.screen[old_offset : old_offset + height]
        newscr = screen[offset : offset + height]
        # use hardware scrolling if we have it.
        if old_offset > offset and self._ri:
            self.__hide_cursor()
            self.__write_code(self._cup, 0, 0)
            self.posxy = 0, old_offset
            for i in range(old_offset - offset):
                self.__write_code(self._ri)
                oldscr.pop(-1)
                oldscr.insert(0, "")
        elif old_offset < offset and self._ind:
            self.__hide_cursor()
            self.__write_code(self._cup, self.height - 1, 0)
            self.posxy = 0, old_offset + self.height - 1
            for i in range(offset - old_offset):
                self.__write_code(self._ind)
                oldscr.pop(0)
                oldscr.append("")
        self.__offset = offset
        for (
            y,
            oldline,
            newline,
        ) in zip(range(offset, offset + height), oldscr, newscr):
            if oldline != newline:
                self.__write_changed_line(y, oldline, newline, px)
        y = len(newscr)
        while y < len(oldscr):
            self.__hide_cursor()
            self.__move(0, y)
            self.posxy = 0, y
            self.__write_code(self._el)
            y += 1
        self.__show_cursor()
        self.screen = screen.copy()
        self.move_cursor(cx, cy)
        self.flushoutput()
    def move_cursor(self, x, y):
        """
        Move the cursor to the specified position on the screen.
        Parameters:
        - x (int): X coordinate.
        - y (int): Y coordinate.
        """
        if y < self.__offset or y >= self.__offset + self.height:
            self.event_queue.insert(Event("scroll", None))
        else:
            self.__move(x, y)
            self.posxy = x, y
            self.flushoutput()
    def prepare(self):
        """
        Prepare the console for input/output operations.
        """
        self.__svtermstate = tcgetattr(self.input_fd)
        raw = self.__svtermstate.copy()
        raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON)
        raw.oflag &= ~(termios.OPOST)
        raw.cflag &= ~(termios.CSIZE | termios.PARENB)
        raw.cflag |= termios.CS8
        raw.iflag |= termios.BRKINT
        raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN)
        raw.lflag |= termios.ISIG
        raw.cc[termios.VMIN] = 1
        raw.cc[termios.VTIME] = 0
        tcsetattr(self.input_fd, termios.TCSADRAIN, raw)
        # In macOS terminal we need to deactivate line wrap via ANSI escape code
        if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal":
            os.write(self.output_fd, b"\033[?7l")
        self.screen = []
        self.height, self.width = self.getheightwidth()
        self.__buffer = []
        self.posxy = 0, 0
        self.__gone_tall = 0
        self.__move = self.__move_short
        self.__offset = 0
        self.__maybe_write_code(self._smkx)
        try:
            self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch)
        except ValueError:
            pass
        self.__enable_bracketed_paste()
    def restore(self):
        """
        Restore the console to the default state
        """
        self.__disable_bracketed_paste()
        self.__maybe_write_code(self._rmkx)
        self.flushoutput()
        tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate)
        if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal":
            os.write(self.output_fd, b"\033[?7h")
        if hasattr(self, "old_sigwinch"):
            signal.signal(signal.SIGWINCH, self.old_sigwinch)
            del self.old_sigwinch
    def push_char(self, char: int | bytes) -> None:
        """
        Push a character to the console event queue.
        """
        trace("push char {char!r}", char=char)
        self.event_queue.push(char)
    def get_event(self, block: bool = True) -> Event | None:
        """
        Get an event from the console event queue.
        Parameters:
        - block (bool): Whether to block until an event is available.
        Returns:
        - Event: Event object from the event queue.
        """
        if not block and not self.wait(timeout=0):
            return None
        while self.event_queue.empty():
            while True:
                try:
                    self.push_char(self.__read(1))
                except OSError as err:
                    if err.errno == errno.EINTR:
                        if not self.event_queue.empty():
                            return self.event_queue.get()
                        else:
                            continue
                    else:
                        raise
                else:
                    break
        return self.event_queue.get()
    def wait(self, timeout: float | None = None) -> bool:
        """
        Wait for events on the console.
        """
        return (
            not self.event_queue.empty()
            or self.more_in_buffer()
            or bool(self.pollob.poll(timeout))
        )
    def set_cursor_vis(self, visible):
        """
        Set the visibility of the cursor.
        Parameters:
        - visible (bool): Visibility flag.
        """
        if visible:
            self.__show_cursor()
        else:
            self.__hide_cursor()
    if TIOCGWINSZ:
        def getheightwidth(self):
            """
            Get the height and width of the console.
            Returns:
            - tuple: Height and width of the console.
            """
            try:
                return int(os.environ["LINES"]), int(os.environ["COLUMNS"])
            except (KeyError, TypeError, ValueError):
                try:
                    size = ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8)
                except OSError:
                    return 25, 80
                height, width = struct.unpack("hhhh", size)[0:2]
                if not height:
                    return 25, 80
                return height, width
    else:
        def getheightwidth(self):
            """
            Get the height and width of the console.
            Returns:
            - tuple: Height and width of the console.
            """
            try:
                return int(os.environ["LINES"]), int(os.environ["COLUMNS"])
            except (KeyError, TypeError, ValueError):
                return 25, 80
    def forgetinput(self):
        """
        Discard any pending input on the console.
        """
        termios.tcflush(self.input_fd, termios.TCIFLUSH)
    def flushoutput(self):
        """
        Flush the output buffer.
        """
        for text, iscode in self.__buffer:
            if iscode:
                self.__tputs(text)
            else:
                os.write(self.output_fd, text.encode(self.encoding, "replace"))
        del self.__buffer[:]
    def finish(self):
        """
        Finish console operations and flush the output buffer.
        """
        y = len(self.screen) - 1
        while y >= 0 and not self.screen[y]:
            y -= 1
        self.__move(0, min(y, self.height + self.__offset - 1))
        self.__write("\n\r")
        self.flushoutput()
    def beep(self):
        """
        Emit a beep sound.
        """
        self.__maybe_write_code(self._bel)
        self.flushoutput()
    if FIONREAD:
        def getpending(self):
            """
            Get pending events from the console event queue.
            Returns:
            - Event: Pending event from the event queue.
            """
            e = Event("key", "", b"")
            while not self.event_queue.empty():
                e2 = self.event_queue.get()
                e.data += e2.data
                e.raw += e.raw
            amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0]
            raw = self.__read(amount)
            data = str(raw, self.encoding, "replace")
            e.data += data
            e.raw += raw
            return e
    else:
        def getpending(self):
            """
            Get pending events from the console event queue.
            Returns:
            - Event: Pending event from the event queue.
            """
            e = Event("key", "", b"")
            while not self.event_queue.empty():
                e2 = self.event_queue.get()
                e.data += e2.data
                e.raw += e.raw
            amount = 10000
            raw = self.__read(amount)
            data = str(raw, self.encoding, "replace")
            e.data += data
            e.raw += raw
            return e
    def clear(self):
        """
        Clear the console screen.
        """
        self.__write_code(self._clear)
        self.__gone_tall = 1
        self.__move = self.__move_tall
        self.posxy = 0, 0
        self.screen = []
    @property
    def input_hook(self):
        try:
            import posix
        except ImportError:
            return None
        if posix._is_inputhook_installed():
            return posix._inputhook
    def __enable_bracketed_paste(self) -> None:
        os.write(self.output_fd, b"\x1b[?2004h")
    def __disable_bracketed_paste(self) -> None:
        os.write(self.output_fd, b"\x1b[?2004l")
    def __setup_movement(self):
        """
        Set up the movement functions based on the terminal capabilities.
        """
        if 0 and self._hpa:  # hpa don't work in windows telnet :-(
            self.__move_x = self.__move_x_hpa
        elif self._cub and self._cuf:
            self.__move_x = self.__move_x_cub_cuf
        elif self._cub1 and self._cuf1:
            self.__move_x = self.__move_x_cub1_cuf1
        else:
            raise RuntimeError("insufficient terminal (horizontal)")
        if self._cuu and self._cud:
            self.__move_y = self.__move_y_cuu_cud
        elif self._cuu1 and self._cud1:
            self.__move_y = self.__move_y_cuu1_cud1
        else:
            raise RuntimeError("insufficient terminal (vertical)")
        if self._dch1:
            self.dch1 = self._dch1
        elif self._dch:
            self.dch1 = curses.tparm(self._dch, 1)
        else:
            self.dch1 = None
        if self._ich1:
            self.ich1 = self._ich1
        elif self._ich:
            self.ich1 = curses.tparm(self._ich, 1)
        else:
            self.ich1 = None
        self.__move = self.__move_short
    def __write_changed_line(self, y, oldline, newline, px_coord):
        # this is frustrating; there's no reason to test (say)
        # self.dch1 inside the loop -- but alternative ways of
        # structuring this function are equally painful (I'm trying to
        # avoid writing code generators these days...)
        minlen = min(wlen(oldline), wlen(newline))
        x_pos = 0
        x_coord = 0
        px_pos = 0
        j = 0
        for c in oldline:
            if j >= px_coord:
                break
            j += wlen(c)
            px_pos += 1
        # reuse the oldline as much as possible, but stop as soon as we
        # encounter an ESCAPE, because it might be the start of an escape
        # sequene
        while (
            x_coord < minlen
            and oldline[x_pos] == newline[x_pos]
            and newline[x_pos] != "\x1b"
        ):
            x_coord += wlen(newline[x_pos])
            x_pos += 1
        # if we need to insert a single character right after the first detected change
        if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1:
            if (
                y == self.posxy[1]
                and x_coord > self.posxy[0]
                and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1]
            ):
                x_pos = px_pos
                x_coord = px_coord
            character_width = wlen(newline[x_pos])
            self.__move(x_coord, y)
            self.__write_code(self.ich1)
            self.__write(newline[x_pos])
            self.posxy = x_coord + character_width, y
        # if it's a single character change in the middle of the line
        elif (
            x_coord < minlen
            and oldline[x_pos + 1 :] == newline[x_pos + 1 :]
            and wlen(oldline[x_pos]) == wlen(newline[x_pos])
        ):
            character_width = wlen(newline[x_pos])
            self.__move(x_coord, y)
            self.__write(newline[x_pos])
            self.posxy = x_coord + character_width, y
        # if this is the last character to fit in the line and we edit in the middle of the line
        elif (
            self.dch1
            and self.ich1
            and wlen(newline) == self.width
            and x_coord < wlen(newline) - 2
            and newline[x_pos + 1 : -1] == oldline[x_pos:-2]
        ):
            self.__hide_cursor()
            self.__move(self.width - 2, y)
            self.posxy = self.width - 2, y
            self.__write_code(self.dch1)
            character_width = wlen(newline[x_pos])
            self.__move(x_coord, y)
            self.__write_code(self.ich1)
            self.__write(newline[x_pos])
            self.posxy = character_width + 1, y
        else:
            self.__hide_cursor()
            self.__move(x_coord, y)
            if wlen(oldline) > wlen(newline):
                self.__write_code(self._el)
            self.__write(newline[x_pos:])
            self.posxy = wlen(newline), y
        if "\x1b" in newline:
            # ANSI escape characters are present, so we can't assume
            # anything about the position of the cursor.  Moving the cursor
            # to the left margin should work to get to a known position.
            self.move_cursor(0, y)
    def __write(self, text):
        self.__buffer.append((text, 0))
    def __write_code(self, fmt, *args):
        self.__buffer.append((curses.tparm(fmt, *args), 1))
    def __maybe_write_code(self, fmt, *args):
        if fmt:
            self.__write_code(fmt, *args)
    def __move_y_cuu1_cud1(self, y):
        assert self._cud1 is not None
        assert self._cuu1 is not None
        dy = y - self.posxy[1]
        if dy > 0:
            self.__write_code(dy * self._cud1)
        elif dy < 0:
            self.__write_code((-dy) * self._cuu1)
    def __move_y_cuu_cud(self, y):
        dy = y - self.posxy[1]
        if dy > 0:
            self.__write_code(self._cud, dy)
        elif dy < 0:
            self.__write_code(self._cuu, -dy)
    def __move_x_hpa(self, x: int) -> None:
        if x != self.posxy[0]:
            self.__write_code(self._hpa, x)
    def __move_x_cub1_cuf1(self, x: int) -> None:
        assert self._cuf1 is not None
        assert self._cub1 is not None
        dx = x - self.posxy[0]
        if dx > 0:
            self.__write_code(self._cuf1 * dx)
        elif dx < 0:
            self.__write_code(self._cub1 * (-dx))
    def __move_x_cub_cuf(self, x: int) -> None:
        dx = x - self.posxy[0]
        if dx > 0:
            self.__write_code(self._cuf, dx)
        elif dx < 0:
            self.__write_code(self._cub, -dx)
    def __move_short(self, x, y):
        self.__move_x(x)
        self.__move_y(y)
    def __move_tall(self, x, y):
        assert 0 <= y - self.__offset < self.height, y - self.__offset
        self.__write_code(self._cup, y - self.__offset, x)
    def __sigwinch(self, signum, frame):
        self.height, self.width = self.getheightwidth()
        self.event_queue.insert(Event("resize", None))
    def __hide_cursor(self):
        if self.cursor_visible:
            self.__maybe_write_code(self._civis)
            self.cursor_visible = 0
    def __show_cursor(self):
        if not self.cursor_visible:
            self.__maybe_write_code(self._cnorm)
            self.cursor_visible = 1
    def repaint(self):
        if not self.__gone_tall:
            self.posxy = 0, self.posxy[1]
            self.__write("\r")
            ns = len(self.screen) * ["\000" * self.width]
            self.screen = ns
        else:
            self.posxy = 0, self.__offset
            self.__move(0, self.__offset)
            ns = self.height * ["\000" * self.width]
            self.screen = ns
    def __tputs(self, fmt, prog=delayprog):
        """A Python implementation of the curses tputs function; the
        curses one can't really be wrapped in a sane manner.
        I have the strong suspicion that this is complexity that
        will never do anyone any good."""
        # using .get() means that things will blow up
        # only if the bps is actually needed (which I'm
        # betting is pretty unlkely)
        bps = ratedict.get(self.__svtermstate.ospeed)
        while 1:
            m = prog.search(fmt)
            if not m:
                os.write(self.output_fd, fmt)
                break
            x, y = m.span()
            os.write(self.output_fd, fmt[:x])
            fmt = fmt[y:]
            delay = int(m.group(1))
            if b"*" in m.group(2):
                delay *= self.height
            if self._pad and bps is not None:
                nchars = (bps * delay) / 1000
                os.write(self.output_fd, self._pad * nchars)
            else:
                time.sleep(float(delay) / 1000.0)
--- a/Lib/_pyrepl/unix_eventqueue.py
+++ b/Lib/_pyrepl/unix_eventqueue.py
@@ -0,0 +1,152 @@
 #   Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com>
 #                       Armin Rigo
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 from collections import deque
 from . import keymap
 from .console import Event
 from . import curses
 from .trace import trace
 from termios import tcgetattr, VERASE
 import os
 # Mapping of human-readable key names to their terminal-specific codes
 TERMINAL_KEYNAMES = {
    "delete": "kdch1",
    "down": "kcud1",
    "end": "kend",
    "enter": "kent",
    "home": "khome",
    "insert": "kich1",
    "left": "kcub1",
    "page down": "knp",
    "page up": "kpp",
    "right": "kcuf1",
    "up": "kcuu1",
 }
 # Function keys F1-F20 mapping
 TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21))
 # Known CTRL-arrow keycodes
 CTRL_ARROW_KEYCODES= {
    # for xterm, gnome-terminal, xfce terminal, etc.
    b'\033[1;5D': 'ctrl left',
    b'\033[1;5C': 'ctrl right',
    # for rxvt
    b'\033Od': 'ctrl left',
    b'\033Oc': 'ctrl right',
 }
 def get_terminal_keycodes() -> dict[bytes, str]:
    """
    Generates a dictionary mapping terminal keycodes to human-readable names.
    """
    keycodes = {}
    for key, terminal_code in TERMINAL_KEYNAMES.items():
        keycode = curses.tigetstr(terminal_code)
        trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals())
        if keycode:
            keycodes[keycode] = key
    keycodes.update(CTRL_ARROW_KEYCODES)
    return keycodes
 class EventQueue:
    def __init__(self, fd: int, encoding: str) -> None:
        self.keycodes = get_terminal_keycodes()
        if os.isatty(fd):
            backspace = tcgetattr(fd)[6][VERASE]
            self.keycodes[backspace] = "backspace"
        self.compiled_keymap = keymap.compile_keymap(self.keycodes)
        self.keymap = self.compiled_keymap
        trace("keymap {k!r}", k=self.keymap)
        self.encoding = encoding
        self.events: deque[Event] = deque()
        self.buf = bytearray()
    def get(self) -> Event | None:
        """
        Retrieves the next event from the queue.
        """
        if self.events:
            return self.events.popleft()
        else:
            return None
    def empty(self) -> bool:
        """
        Checks if the queue is empty.
        """
        return not self.events
    def flush_buf(self) -> bytearray:
        """
        Flushes the buffer and returns its contents.
        """
        old = self.buf
        self.buf = bytearray()
        return old
    def insert(self, event: Event) -> None:
        """
        Inserts an event into the queue.
        """
        trace('added event {event}', event=event)
        self.events.append(event)
    def push(self, char: int | bytes) -> None:
        """
        Processes a character by updating the buffer and handling special key mappings.
        """
        ord_char = char if isinstance(char, int) else ord(char)
        char = bytes(bytearray((ord_char,)))
        self.buf.append(ord_char)
        if char in self.keymap:
            if self.keymap is self.compiled_keymap:
                #sanity check, buffer is empty when a special key comes
                assert len(self.buf) == 1
            k = self.keymap[char]
            trace('found map {k!r}', k=k)
            if isinstance(k, dict):
                self.keymap = k
            else:
                self.insert(Event('key', k, self.flush_buf()))
                self.keymap = self.compiled_keymap
        elif self.buf and self.buf[0] == 27:  # escape
            # escape sequence not recognized by our keymap: propagate it
            # outside so that i can be recognized as an M-... key (see also
            # the docstring in keymap.py
            trace('unrecognized escape sequence, propagating...')
            self.keymap = self.compiled_keymap
            self.insert(Event('key', '\033', bytearray(b'\033')))
            for _c in self.flush_buf()[1:]:
                self.push(_c)
        else:
            try:
                decoded = bytes(self.buf).decode(self.encoding)
            except UnicodeError:
                return
            else:
                self.insert(Event('key', decoded, self.flush_buf()))
            self.keymap = self.compiled_keymap
--- a/Lib/_pyrepl/utils.py
+++ b/Lib/_pyrepl/utils.py
@@ -0,0 +1,25 @@
 import re
 import unicodedata
 import functools
 ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]")
@functools.cache
 def str_width(c: str) -> int:
    if ord(c) < 128:
        return 1
    w = unicodedata.east_asian_width(c)
    if w in ('N', 'Na', 'H', 'A'):
        return 1
    return 2
 def wlen(s: str) -> int:
    if len(s) == 1 and s != '\x1a':
        return str_width(s)
    length = sum(str_width(i) for i in s)
    # remove lengths of any escape sequences
    sequence = ANSI_ESCAPE_SEQUENCE.findall(s)
    ctrl_z_cnt = s.count('\x1a')
    return length - sum(len(i) for i in sequence) + ctrl_z_cnt
--- a/Lib/_pyrepl/windows_console.py
+++ b/Lib/_pyrepl/windows_console.py
@@ -0,0 +1,618 @@
 #   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
 #
 #                        All Rights Reserved
 #
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose is hereby granted without fee,
 # provided that the above copyright notice appear in all copies and
 # that both that copyright notice and this permission notice appear in
 # supporting documentation.
 #
 # THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
 # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 # AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
 # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 from __future__ import annotations
 import io
 import os
 import sys
 import time
 import msvcrt
 from collections import deque
 import ctypes
 from ctypes.wintypes import (
    _COORD,
    WORD,
    SMALL_RECT,
    BOOL,
    HANDLE,
    CHAR,
    DWORD,
    WCHAR,
    SHORT,
 )
 from ctypes import Structure, POINTER, Union
 from .console import Event, Console
 from .trace import trace
 from .utils import wlen
 try:
    from ctypes import GetLastError, WinDLL, windll, WinError  # type: ignore[attr-defined]
 except:
    # Keep MyPy happy off Windows
    from ctypes import CDLL as WinDLL, cdll as windll
    def GetLastError() -> int:
        return 42
    class WinError(OSError):  # type: ignore[no-redef]
        def __init__(self, err: int | None, descr: str | None = None) -> None:
            self.err = err
            self.descr = descr
 TYPE_CHECKING = False
 if TYPE_CHECKING:
    from typing import IO
 # Virtual-Key Codes: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
 VK_MAP: dict[int, str] = {
    0x23: "end",  # VK_END
    0x24: "home",  # VK_HOME
    0x25: "left",  # VK_LEFT
    0x26: "up",  # VK_UP
    0x27: "right",  # VK_RIGHT
    0x28: "down",  # VK_DOWN
    0x2E: "delete",  # VK_DELETE
    0x70: "f1",  # VK_F1
    0x71: "f2",  # VK_F2
    0x72: "f3",  # VK_F3
    0x73: "f4",  # VK_F4
    0x74: "f5",  # VK_F5
    0x75: "f6",  # VK_F6
    0x76: "f7",  # VK_F7
    0x77: "f8",  # VK_F8
    0x78: "f9",  # VK_F9
    0x79: "f10",  # VK_F10
    0x7A: "f11",  # VK_F11
    0x7B: "f12",  # VK_F12
    0x7C: "f13",  # VK_F13
    0x7D: "f14",  # VK_F14
    0x7E: "f15",  # VK_F15
    0x7F: "f16",  # VK_F16
    0x80: "f17",  # VK_F17
    0x81: "f18",  # VK_F18
    0x82: "f19",  # VK_F19
    0x83: "f20",  # VK_F20
 }
 # Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
 ERASE_IN_LINE = "\x1b[K"
 MOVE_LEFT = "\x1b[{}D"
 MOVE_RIGHT = "\x1b[{}C"
 MOVE_UP = "\x1b[{}A"
 MOVE_DOWN = "\x1b[{}B"
 CLEAR = "\x1b[H\x1b[J"
 class _error(Exception):
    pass
 class WindowsConsole(Console):
    def __init__(
        self,
        f_in: IO[bytes] | int = 0,
        f_out: IO[bytes] | int = 1,
        term: str = "",
        encoding: str = "",
    ):
        super().__init__(f_in, f_out, term, encoding)
        SetConsoleMode(
            OutHandle,
            ENABLE_WRAP_AT_EOL_OUTPUT
            | ENABLE_PROCESSED_OUTPUT
            | ENABLE_VIRTUAL_TERMINAL_PROCESSING,
        )
        self.screen: list[str] = []
        self.width = 80
        self.height = 25
        self.__offset = 0
        self.event_queue: deque[Event] = deque()
        try:
            self.out = io._WindowsConsoleIO(self.output_fd, "w")  # type: ignore[attr-defined]
        except ValueError:
            # Console I/O is redirected, fallback...
            self.out = None
    def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None:
        """
        Refresh the console screen.
        Parameters:
        - screen (list): List of strings representing the screen contents.
        - c_xy (tuple): Cursor position (x, y) on the screen.
        """
        cx, cy = c_xy
        while len(self.screen) < min(len(screen), self.height):
            self._hide_cursor()
            self._move_relative(0, len(self.screen) - 1)
            self.__write("\n")
            self.posxy = 0, len(self.screen)
            self.screen.append("")
        px, py = self.posxy
        old_offset = offset = self.__offset
        height = self.height
        # we make sure the cursor is on the screen, and that we're
        # using all of the screen if we can
        if cy < offset:
            offset = cy
        elif cy >= offset + height:
            offset = cy - height + 1
            scroll_lines = offset - old_offset
            # Scrolling the buffer as the current input is greater than the visible
            # portion of the window.  We need to scroll the visible portion and the
            # entire history
            self._scroll(scroll_lines, self._getscrollbacksize())
            self.posxy = self.posxy[0], self.posxy[1] + scroll_lines
            self.__offset += scroll_lines
            for i in range(scroll_lines):
                self.screen.append("")
        elif offset > 0 and len(screen) < offset + height:
            offset = max(len(screen) - height, 0)
            screen.append("")
        oldscr = self.screen[old_offset : old_offset + height]
        newscr = screen[offset : offset + height]
        self.__offset = offset
        self._hide_cursor()
        for (
            y,
            oldline,
            newline,
        ) in zip(range(offset, offset + height), oldscr, newscr):
            if oldline != newline:
                self.__write_changed_line(y, oldline, newline, px)
        y = len(newscr)
        while y < len(oldscr):
            self._move_relative(0, y)
            self.posxy = 0, y
            self._erase_to_end()
            y += 1
        self._show_cursor()
        self.screen = screen
        self.move_cursor(cx, cy)
    @property
    def input_hook(self):
        try:
            import nt
        except ImportError:
            return None
        if nt._is_inputhook_installed():
            return nt._inputhook
    def __write_changed_line(
        self, y: int, oldline: str, newline: str, px_coord: int
    ) -> None:
        # this is frustrating; there's no reason to test (say)
        # self.dch1 inside the loop -- but alternative ways of
        # structuring this function are equally painful (I'm trying to
        # avoid writing code generators these days...)
        minlen = min(wlen(oldline), wlen(newline))
        x_pos = 0
        x_coord = 0
        px_pos = 0
        j = 0
        for c in oldline:
            if j >= px_coord:
                break
            j += wlen(c)
            px_pos += 1
        # reuse the oldline as much as possible, but stop as soon as we
        # encounter an ESCAPE, because it might be the start of an escape
        # sequene
        while (
            x_coord < minlen
            and oldline[x_pos] == newline[x_pos]
            and newline[x_pos] != "\x1b"
        ):
            x_coord += wlen(newline[x_pos])
            x_pos += 1
        self._hide_cursor()
        self._move_relative(x_coord, y)
        if wlen(oldline) > wlen(newline):
            self._erase_to_end()
        self.__write(newline[x_pos:])
        if wlen(newline) == self.width:
            # If we wrapped we want to start at the next line
            self._move_relative(0, y + 1)
            self.posxy = 0, y + 1
        else:
            self.posxy = wlen(newline), y
            if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline:
                # ANSI escape characters are present, so we can't assume
                # anything about the position of the cursor.  Moving the cursor
                # to the left margin should work to get to a known position.
                self.move_cursor(0, y)
    def _scroll(
        self, top: int, bottom: int, left: int | None = None, right: int | None = None
    ) -> None:
        scroll_rect = SMALL_RECT()
        scroll_rect.Top = SHORT(top)
        scroll_rect.Bottom = SHORT(bottom)
        scroll_rect.Left = SHORT(0 if left is None else left)
        scroll_rect.Right = SHORT(
            self.getheightwidth()[1] - 1 if right is None else right
        )
        destination_origin = _COORD()
        fill_info = CHAR_INFO()
        fill_info.UnicodeChar = " "
        if not ScrollConsoleScreenBuffer(
            OutHandle, scroll_rect, None, destination_origin, fill_info
        ):
            raise WinError(GetLastError())
    def _hide_cursor(self):
        self.__write("\x1b[?25l")
    def _show_cursor(self):
        self.__write("\x1b[?25h")
    def _enable_blinking(self):
        self.__write("\x1b[?12h")
    def _disable_blinking(self):
        self.__write("\x1b[?12l")
    def __write(self, text: str) -> None:
        if "\x1a" in text:
            text = ''.join(["^Z" if x == '\x1a' else x for x in text])
        if self.out is not None:
            self.out.write(text.encode(self.encoding, "replace"))
            self.out.flush()
        else:
            os.write(self.output_fd, text.encode(self.encoding, "replace"))
    @property
    def screen_xy(self) -> tuple[int, int]:
        info = CONSOLE_SCREEN_BUFFER_INFO()
        if not GetConsoleScreenBufferInfo(OutHandle, info):
            raise WinError(GetLastError())
        return info.dwCursorPosition.X, info.dwCursorPosition.Y
    def _erase_to_end(self) -> None:
        self.__write(ERASE_IN_LINE)
    def prepare(self) -> None:
        trace("prepare")
        self.screen = []
        self.height, self.width = self.getheightwidth()
        self.posxy = 0, 0
        self.__gone_tall = 0
        self.__offset = 0
    def restore(self) -> None:
        pass
    def _move_relative(self, x: int, y: int) -> None:
        """Moves relative to the current posxy"""
        dx = x - self.posxy[0]
        dy = y - self.posxy[1]
        if dx < 0:
            self.__write(MOVE_LEFT.format(-dx))
        elif dx > 0:
            self.__write(MOVE_RIGHT.format(dx))
        if dy < 0:
            self.__write(MOVE_UP.format(-dy))
        elif dy > 0:
            self.__write(MOVE_DOWN.format(dy))
    def move_cursor(self, x: int, y: int) -> None:
        if x < 0 or y < 0:
            raise ValueError(f"Bad cursor position {x}, {y}")
        if y < self.__offset or y >= self.__offset + self.height:
            self.event_queue.insert(0, Event("scroll", ""))
        else:
            self._move_relative(x, y)
            self.posxy = x, y
    def set_cursor_vis(self, visible: bool) -> None:
        if visible:
            self._show_cursor()
        else:
            self._hide_cursor()
    def getheightwidth(self) -> tuple[int, int]:
        """Return (height, width) where height and width are the height
        and width of the terminal window in characters."""
        info = CONSOLE_SCREEN_BUFFER_INFO()
        if not GetConsoleScreenBufferInfo(OutHandle, info):
            raise WinError(GetLastError())
        return (
            info.srWindow.Bottom - info.srWindow.Top + 1,
            info.srWindow.Right - info.srWindow.Left + 1,
        )
    def _getscrollbacksize(self) -> int:
        info = CONSOLE_SCREEN_BUFFER_INFO()
        if not GetConsoleScreenBufferInfo(OutHandle, info):
            raise WinError(GetLastError())
        return info.srWindow.Bottom  # type: ignore[no-any-return]
    def _read_input(self, block: bool = True) -> INPUT_RECORD | None:
        if not block:
            events = DWORD()
            if not GetNumberOfConsoleInputEvents(InHandle, events):
                raise WinError(GetLastError())
            if not events.value:
                return None
        rec = INPUT_RECORD()
        read = DWORD()
        if not ReadConsoleInput(InHandle, rec, 1, read):
            raise WinError(GetLastError())
        return rec
    def get_event(self, block: bool = True) -> Event | None:
        """Return an Event instance.  Returns None if |block| is false
        and there is no event pending, otherwise waits for the
        completion of an event."""
        if self.event_queue:
            return self.event_queue.pop()
        while True:
            rec = self._read_input(block)
            if rec is None:
                return None
            if rec.EventType == WINDOW_BUFFER_SIZE_EVENT:
                return Event("resize", "")
            if rec.EventType != KEY_EVENT or not rec.Event.KeyEvent.bKeyDown:
                # Only process keys and keydown events
                if block:
                    continue
                return None
            key = rec.Event.KeyEvent.uChar.UnicodeChar
            if rec.Event.KeyEvent.uChar.UnicodeChar == "\r":
                # Make enter make unix-like
                return Event(evt="key", data="\n", raw=b"\n")
            elif rec.Event.KeyEvent.wVirtualKeyCode == 8:
                # Turn backspace directly into the command
                return Event(
                    evt="key",
                    data="backspace",
                    raw=rec.Event.KeyEvent.uChar.UnicodeChar,
                )
            elif rec.Event.KeyEvent.uChar.UnicodeChar == "\x00":
                # Handle special keys like arrow keys and translate them into the appropriate command
                code = VK_MAP.get(rec.Event.KeyEvent.wVirtualKeyCode)
                if code:
                    return Event(
                        evt="key", data=code, raw=rec.Event.KeyEvent.uChar.UnicodeChar
                    )
                if block:
                    continue
                return None
            return Event(evt="key", data=key, raw=rec.Event.KeyEvent.uChar.UnicodeChar)
    def push_char(self, char: int | bytes) -> None:
        """
        Push a character to the console event queue.
        """
        raise NotImplementedError("push_char not supported on Windows")
    def beep(self) -> None:
        self.__write("\x07")
    def clear(self) -> None:
        """Wipe the screen"""
        self.__write(CLEAR)
        self.posxy = 0, 0
        self.screen = [""]
    def finish(self) -> None:
        """Move the cursor to the end of the display and otherwise get
        ready for end.  XXX could be merged with restore?  Hmm."""
        y = len(self.screen) - 1
        while y >= 0 and not self.screen[y]:
            y -= 1
        self._move_relative(0, min(y, self.height + self.__offset - 1))
        self.__write("\r\n")
    def flushoutput(self) -> None:
        """Flush all output to the screen (assuming there's some
        buffering going on somewhere).
        All output on Windows is unbuffered so this is a nop"""
        pass
    def forgetinput(self) -> None:
        """Forget all pending, but not yet processed input."""
        if not FlushConsoleInputBuffer(InHandle):
            raise WinError(GetLastError())
    def getpending(self) -> Event:
        """Return the characters that have been typed but not yet
        processed."""
        return Event("key", "", b"")
    def wait(self, timeout: float | None) -> bool:
        """Wait for an event."""
        # Poor man's Windows select loop
        start_time = time.time()
        while True:
            if msvcrt.kbhit(): # type: ignore[attr-defined]
                return True
            if timeout and time.time() - start_time > timeout / 1000:
                return False
            time.sleep(0.01)
    def repaint(self) -> None:
        raise NotImplementedError("No repaint support")
 # Windows interop
 class CONSOLE_SCREEN_BUFFER_INFO(Structure):
    _fields_ = [
        ("dwSize", _COORD),
        ("dwCursorPosition", _COORD),
        ("wAttributes", WORD),
        ("srWindow", SMALL_RECT),
        ("dwMaximumWindowSize", _COORD),
    ]
 class CONSOLE_CURSOR_INFO(Structure):
    _fields_ = [
        ("dwSize", DWORD),
        ("bVisible", BOOL),
    ]
 class CHAR_INFO(Structure):
    _fields_ = [
        ("UnicodeChar", WCHAR),
        ("Attributes", WORD),
    ]
 class Char(Union):
    _fields_ = [
        ("UnicodeChar", WCHAR),
        ("Char", CHAR),
    ]
 class KeyEvent(ctypes.Structure):
    _fields_ = [
        ("bKeyDown", BOOL),
        ("wRepeatCount", WORD),
        ("wVirtualKeyCode", WORD),
        ("wVirtualScanCode", WORD),
        ("uChar", Char),
        ("dwControlKeyState", DWORD),
    ]
 class WindowsBufferSizeEvent(ctypes.Structure):
    _fields_ = [("dwSize", _COORD)]
 class ConsoleEvent(ctypes.Union):
    _fields_ = [
        ("KeyEvent", KeyEvent),
        ("WindowsBufferSizeEvent", WindowsBufferSizeEvent),
    ]
 class INPUT_RECORD(Structure):
    _fields_ = [("EventType", WORD), ("Event", ConsoleEvent)]
 KEY_EVENT = 0x01
 FOCUS_EVENT = 0x10
 MENU_EVENT = 0x08
 MOUSE_EVENT = 0x02
 WINDOW_BUFFER_SIZE_EVENT = 0x04
 ENABLE_PROCESSED_OUTPUT = 0x01
 ENABLE_WRAP_AT_EOL_OUTPUT = 0x02
 ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04
 STD_INPUT_HANDLE = -10
 STD_OUTPUT_HANDLE = -11
 if sys.platform == "win32":
    _KERNEL32 = WinDLL("kernel32", use_last_error=True)
    GetStdHandle = windll.kernel32.GetStdHandle
    GetStdHandle.argtypes = [DWORD]
    GetStdHandle.restype = HANDLE
    GetConsoleScreenBufferInfo = _KERNEL32.GetConsoleScreenBufferInfo
    GetConsoleScreenBufferInfo.argtypes = [
        HANDLE,
        ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO),
    ]
    GetConsoleScreenBufferInfo.restype = BOOL
    ScrollConsoleScreenBuffer = _KERNEL32.ScrollConsoleScreenBufferW
    ScrollConsoleScreenBuffer.argtypes = [
        HANDLE,
        POINTER(SMALL_RECT),
        POINTER(SMALL_RECT),
        _COORD,
        POINTER(CHAR_INFO),
    ]
    ScrollConsoleScreenBuffer.restype = BOOL
    SetConsoleMode = _KERNEL32.SetConsoleMode
    SetConsoleMode.argtypes = [HANDLE, DWORD]
    SetConsoleMode.restype = BOOL
    ReadConsoleInput = _KERNEL32.ReadConsoleInputW
    ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)]
    ReadConsoleInput.restype = BOOL
    GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents
    GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)]
    GetNumberOfConsoleInputEvents.restype = BOOL
    FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer
    FlushConsoleInputBuffer.argtypes = [HANDLE]
    FlushConsoleInputBuffer.restype = BOOL
    OutHandle = GetStdHandle(STD_OUTPUT_HANDLE)
    InHandle = GetStdHandle(STD_INPUT_HANDLE)
 else:
    def _win_only(*args, **kwargs):
        raise NotImplementedError("Windows only")
    GetStdHandle = _win_only
    GetConsoleScreenBufferInfo = _win_only
    ScrollConsoleScreenBuffer = _win_only
    SetConsoleMode = _win_only
    ReadConsoleInput = _win_only
    GetNumberOfConsoleInputEvents = _win_only
    FlushConsoleInputBuffer = _win_only
    OutHandle = 0
    InHandle = 0
--- a/Lib/aifc.py
+++ b/Lib/aifc.py
@@ -1,984 +0,0 @@
 """Stuff to parse AIFF-C and AIFF files.
 Unless explicitly stated otherwise, the description below is true
 both for AIFF-C files and AIFF files.
 An AIFF-C file has the following structure.
  +-----------------+
  | FORM            |
  +-----------------+
  | <size>          |
  +----+------------+
  |    | AIFC       |
  |    +------------+
  |    | <chunks>   |
  |    |    .       |
  |    |    .       |
  |    |    .       |
  +----+------------+
 An AIFF file has the string "AIFF" instead of "AIFC".
 A chunk consists of an identifier (4 bytes) followed by a size (4 bytes,
 big endian order), followed by the data.  The size field does not include
 the size of the 8 byte header.
 The following chunk types are recognized.
  FVER
      <version number of AIFF-C defining document> (AIFF-C only).
  MARK
      <# of markers> (2 bytes)
      list of markers:
          <marker ID> (2 bytes, must be > 0)
          <position> (4 bytes)
          <marker name> ("pstring")
  COMM
      <# of channels> (2 bytes)
      <# of sound frames> (4 bytes)
      <size of the samples> (2 bytes)
      <sampling frequency> (10 bytes, IEEE 80-bit extended
          floating point)
      in AIFF-C files only:
      <compression type> (4 bytes)
      <human-readable version of compression type> ("pstring")
  SSND
      <offset> (4 bytes, not used by this program)
      <blocksize> (4 bytes, not used by this program)
      <sound data>
 A pstring consists of 1 byte length, a string of characters, and 0 or 1
 byte pad to make the total length even.
 Usage.
 Reading AIFF files:
  f = aifc.open(file, 'r')
 where file is either the name of a file or an open file pointer.
 The open file pointer must have methods read(), seek(), and close().
 In some types of audio files, if the setpos() method is not used,
 the seek() method is not necessary.
 This returns an instance of a class with the following public methods:
  getnchannels()  -- returns number of audio channels (1 for
             mono, 2 for stereo)
  getsampwidth()  -- returns sample width in bytes
  getframerate()  -- returns sampling frequency
  getnframes()    -- returns number of audio frames
  getcomptype()   -- returns compression type ('NONE' for AIFF files)
  getcompname()   -- returns human-readable version of
             compression type ('not compressed' for AIFF files)
  getparams() -- returns a namedtuple consisting of all of the
             above in the above order
  getmarkers()    -- get the list of marks in the audio file or None
             if there are no marks
  getmark(id) -- get mark with the specified id (raises an error
             if the mark does not exist)
  readframes(n)   -- returns at most n frames of audio
  rewind()    -- rewind to the beginning of the audio stream
  setpos(pos) -- seek to the specified position
  tell()      -- return the current position
  close()     -- close the instance (make it unusable)
 The position returned by tell(), the position given to setpos() and
 the position of marks are all compatible and have nothing to do with
 the actual position in the file.
 The close() method is called automatically when the class instance
 is destroyed.
 Writing AIFF files:
  f = aifc.open(file, 'w')
 where file is either the name of a file or an open file pointer.
 The open file pointer must have methods write(), tell(), seek(), and
 close().
 This returns an instance of a class with the following public methods:
  aiff()      -- create an AIFF file (AIFF-C default)
  aifc()      -- create an AIFF-C file
  setnchannels(n) -- set the number of channels
  setsampwidth(n) -- set the sample width
  setframerate(n) -- set the frame rate
  setnframes(n)   -- set the number of frames
  setcomptype(type, name)
          -- set the compression type and the
             human-readable compression type
  setparams(tuple)
          -- set all parameters at once
  setmark(id, pos, name)
          -- add specified mark to the list of marks
  tell()      -- return current position in output file (useful
             in combination with setmark())
  writeframesraw(data)
          -- write audio frames without pathing up the
             file header
  writeframes(data)
          -- write audio frames and patch up the file header
  close()     -- patch up the file header and close the
             output file
 You should set the parameters before the first writeframesraw or
 writeframes.  The total number of frames does not need to be set,
 but when it is set to the correct value, the header does not have to
 be patched up.
 It is best to first set all parameters, perhaps possibly the
 compression type, and then write audio frames using writeframesraw.
 When all frames have been written, either call writeframes(b'') or
 close() to patch up the sizes in the header.
 Marks can be added anytime.  If there are any marks, you must call
 close() after all frames have been written.
 The close() method is called automatically when the class instance
 is destroyed.
 When a file is opened with the extension '.aiff', an AIFF file is
 written, otherwise an AIFF-C file is written.  This default can be
 changed by calling aiff() or aifc() before the first writeframes or
 writeframesraw.
 """
 import struct
 import builtins
 import warnings
 __all__ = ["Error", "open"]
 warnings._deprecated(__name__, remove=(3, 13))
 class Error(Exception):
    pass
 _AIFC_version = 0xA2805140     # Version 1 of AIFF-C
 def _read_long(file):
    try:
        return struct.unpack('>l', file.read(4))[0]
    except struct.error:
        raise EOFError from None
 def _read_ulong(file):
    try:
        return struct.unpack('>L', file.read(4))[0]
    except struct.error:
        raise EOFError from None
 def _read_short(file):
    try:
        return struct.unpack('>h', file.read(2))[0]
    except struct.error:
        raise EOFError from None
 def _read_ushort(file):
    try:
        return struct.unpack('>H', file.read(2))[0]
    except struct.error:
        raise EOFError from None
 def _read_string(file):
    length = ord(file.read(1))
    if length == 0:
        data = b''
    else:
        data = file.read(length)
    if length & 1 == 0:
        dummy = file.read(1)
    return data
 _HUGE_VAL = 1.79769313486231e+308 # See <limits.h>
 def _read_float(f): # 10 bytes
    expon = _read_short(f) # 2 bytes
    sign = 1
    if expon < 0:
        sign = -1
        expon = expon + 0x8000
    himant = _read_ulong(f) # 4 bytes
    lomant = _read_ulong(f) # 4 bytes
    if expon == himant == lomant == 0:
        f = 0.0
    elif expon == 0x7FFF:
        f = _HUGE_VAL
    else:
        expon = expon - 16383
        f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63)
    return sign * f
 def _write_short(f, x):
    f.write(struct.pack('>h', x))
 def _write_ushort(f, x):
    f.write(struct.pack('>H', x))
 def _write_long(f, x):
    f.write(struct.pack('>l', x))
 def _write_ulong(f, x):
    f.write(struct.pack('>L', x))
 def _write_string(f, s):
    if len(s) > 255:
        raise ValueError("string exceeds maximum pstring length")
    f.write(struct.pack('B', len(s)))
    f.write(s)
    if len(s) & 1 == 0:
        f.write(b'\x00')
 def _write_float(f, x):
    import math
    if x < 0:
        sign = 0x8000
        x = x * -1
    else:
        sign = 0
    if x == 0:
        expon = 0
        himant = 0
        lomant = 0
    else:
        fmant, expon = math.frexp(x)
        if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN
            expon = sign|0x7FFF
            himant = 0
            lomant = 0
        else:                   # Finite
            expon = expon + 16382
            if expon < 0:           # denormalized
                fmant = math.ldexp(fmant, expon)
                expon = 0
            expon = expon | sign
            fmant = math.ldexp(fmant, 32)
            fsmant = math.floor(fmant)
            himant = int(fsmant)
            fmant = math.ldexp(fmant - fsmant, 32)
            fsmant = math.floor(fmant)
            lomant = int(fsmant)
    _write_ushort(f, expon)
    _write_ulong(f, himant)
    _write_ulong(f, lomant)
 with warnings.catch_warnings():
    warnings.simplefilter("ignore", DeprecationWarning)
    from chunk import Chunk
 from collections import namedtuple
 _aifc_params = namedtuple('_aifc_params',
                          'nchannels sampwidth framerate nframes comptype compname')
 _aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)'
 _aifc_params.sampwidth.__doc__ = 'Sample width in bytes'
 _aifc_params.framerate.__doc__ = 'Sampling frequency'
 _aifc_params.nframes.__doc__ = 'Number of audio frames'
 _aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)'
 _aifc_params.compname.__doc__ = ("""\
 A human-readable version of the compression type
 ('not compressed' for AIFF files)""")
 class Aifc_read:
    # Variables used in this class:
    #
    # These variables are available to the user though appropriate
    # methods of this class:
    # _file -- the open file with methods read(), close(), and seek()
    #       set through the __init__() method
    # _nchannels -- the number of audio channels
    #       available through the getnchannels() method
    # _nframes -- the number of audio frames
    #       available through the getnframes() method
    # _sampwidth -- the number of bytes per audio sample
    #       available through the getsampwidth() method
    # _framerate -- the sampling frequency
    #       available through the getframerate() method
    # _comptype -- the AIFF-C compression type ('NONE' if AIFF)
    #       available through the getcomptype() method
    # _compname -- the human-readable AIFF-C compression type
    #       available through the getcomptype() method
    # _markers -- the marks in the audio file
    #       available through the getmarkers() and getmark()
    #       methods
    # _soundpos -- the position in the audio stream
    #       available through the tell() method, set through the
    #       setpos() method
    #
    # These variables are used internally only:
    # _version -- the AIFF-C version number
    # _decomp -- the decompressor from builtin module cl
    # _comm_chunk_read -- 1 iff the COMM chunk has been read
    # _aifc -- 1 iff reading an AIFF-C file
    # _ssnd_seek_needed -- 1 iff positioned correctly in audio
    #       file for readframes()
    # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
    # _framesize -- size of one frame in the file
    _file = None  # Set here since __del__ checks it
    def initfp(self, file):
        self._version = 0
        self._convert = None
        self._markers = []
        self._soundpos = 0
        self._file = file
        chunk = Chunk(file)
        if chunk.getname() != b'FORM':
            raise Error('file does not start with FORM id')
        formdata = chunk.read(4)
        if formdata == b'AIFF':
            self._aifc = 0
        elif formdata == b'AIFC':
            self._aifc = 1
        else:
            raise Error('not an AIFF or AIFF-C file')
        self._comm_chunk_read = 0
        self._ssnd_chunk = None
        while 1:
            self._ssnd_seek_needed = 1
            try:
                chunk = Chunk(self._file)
            except EOFError:
                break
            chunkname = chunk.getname()
            if chunkname == b'COMM':
                self._read_comm_chunk(chunk)
                self._comm_chunk_read = 1
            elif chunkname == b'SSND':
                self._ssnd_chunk = chunk
                dummy = chunk.read(8)
                self._ssnd_seek_needed = 0
            elif chunkname == b'FVER':
                self._version = _read_ulong(chunk)
            elif chunkname == b'MARK':
                self._readmark(chunk)
            chunk.skip()
        if not self._comm_chunk_read or not self._ssnd_chunk:
            raise Error('COMM chunk and/or SSND chunk missing')
    def __init__(self, f):
        if isinstance(f, str):
            file_object = builtins.open(f, 'rb')
            try:
                self.initfp(file_object)
            except:
                file_object.close()
                raise
        else:
            # assume it is an open file object already
            self.initfp(f)
    def __enter__(self):
        return self
    def __exit__(self, *args):
        self.close()
    #
    # User visible methods.
    #
    def getfp(self):
        return self._file
    def rewind(self):
        self._ssnd_seek_needed = 1
        self._soundpos = 0
    def close(self):
        file = self._file
        if file is not None:
            self._file = None
            file.close()
    def tell(self):
        return self._soundpos
    def getnchannels(self):
        return self._nchannels
    def getnframes(self):
        return self._nframes
    def getsampwidth(self):
        return self._sampwidth
    def getframerate(self):
        return self._framerate
    def getcomptype(self):
        return self._comptype
    def getcompname(self):
        return self._compname
 ##  def getversion(self):
 ##      return self._version
    def getparams(self):
        return _aifc_params(self.getnchannels(), self.getsampwidth(),
                            self.getframerate(), self.getnframes(),
                            self.getcomptype(), self.getcompname())
    def getmarkers(self):
        if len(self._markers) == 0:
            return None
        return self._markers
    def getmark(self, id):
        for marker in self._markers:
            if id == marker[0]:
                return marker
        raise Error('marker {0!r} does not exist'.format(id))
    def setpos(self, pos):
        if pos < 0 or pos > self._nframes:
            raise Error('position not in range')
        self._soundpos = pos
        self._ssnd_seek_needed = 1
    def readframes(self, nframes):
        if self._ssnd_seek_needed:
            self._ssnd_chunk.seek(0)
            dummy = self._ssnd_chunk.read(8)
            pos = self._soundpos * self._framesize
            if pos:
                self._ssnd_chunk.seek(pos + 8)
            self._ssnd_seek_needed = 0
        if nframes == 0:
            return b''
        data = self._ssnd_chunk.read(nframes * self._framesize)
        if self._convert and data:
            data = self._convert(data)
        self._soundpos = self._soundpos + len(data) // (self._nchannels
                                                        * self._sampwidth)
        return data
    #
    # Internal methods.
    #
    def _alaw2lin(self, data):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=DeprecationWarning)
            import audioop
        return audioop.alaw2lin(data, 2)
    def _ulaw2lin(self, data):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=DeprecationWarning)
            import audioop
        return audioop.ulaw2lin(data, 2)
    def _adpcm2lin(self, data):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=DeprecationWarning)
            import audioop
        if not hasattr(self, '_adpcmstate'):
            # first time
            self._adpcmstate = None
        data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
        return data
    def _sowt2lin(self, data):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=DeprecationWarning)
            import audioop
        return audioop.byteswap(data, 2)
    def _read_comm_chunk(self, chunk):
        self._nchannels = _read_short(chunk)
        self._nframes = _read_long(chunk)
        self._sampwidth = (_read_short(chunk) + 7) // 8
        self._framerate = int(_read_float(chunk))
        if self._sampwidth <= 0:
            raise Error('bad sample width')
        if self._nchannels <= 0:
            raise Error('bad # of channels')
        self._framesize = self._nchannels * self._sampwidth
        if self._aifc:
            #DEBUG: SGI's soundeditor produces a bad size :-(
            kludge = 0
            if chunk.chunksize == 18:
                kludge = 1
                warnings.warn('Warning: bad COMM chunk size')
                chunk.chunksize = 23
            #DEBUG end
            self._comptype = chunk.read(4)
            #DEBUG start
            if kludge:
                length = ord(chunk.file.read(1))
                if length & 1 == 0:
                    length = length + 1
                chunk.chunksize = chunk.chunksize + length
                chunk.file.seek(-1, 1)
            #DEBUG end
            self._compname = _read_string(chunk)
            if self._comptype != b'NONE':
                if self._comptype == b'G722':
                    self._convert = self._adpcm2lin
                elif self._comptype in (b'ulaw', b'ULAW'):
                    self._convert = self._ulaw2lin
                elif self._comptype in (b'alaw', b'ALAW'):
                    self._convert = self._alaw2lin
                elif self._comptype in (b'sowt', b'SOWT'):
                    self._convert = self._sowt2lin
                else:
                    raise Error('unsupported compression type')
                self._sampwidth = 2
        else:
            self._comptype = b'NONE'
            self._compname = b'not compressed'
    def _readmark(self, chunk):
        nmarkers = _read_short(chunk)
        # Some files appear to contain invalid counts.
        # Cope with this by testing for EOF.
        try:
            for i in range(nmarkers):
                id = _read_short(chunk)
                pos = _read_long(chunk)
                name = _read_string(chunk)
                if pos or name:
                    # some files appear to have
                    # dummy markers consisting of
                    # a position 0 and name ''
                    self._markers.append((id, pos, name))
        except EOFError:
            w = ('Warning: MARK chunk contains only %s marker%s instead of %s' %
                 (len(self._markers), '' if len(self._markers) == 1 else 's',
                  nmarkers))
            warnings.warn(w)
 class Aifc_write:
    # Variables used in this class:
    #
    # These variables are user settable through appropriate methods
    # of this class:
    # _file -- the open file with methods write(), close(), tell(), seek()
    #       set through the __init__() method
    # _comptype -- the AIFF-C compression type ('NONE' in AIFF)
    #       set through the setcomptype() or setparams() method
    # _compname -- the human-readable AIFF-C compression type
    #       set through the setcomptype() or setparams() method
    # _nchannels -- the number of audio channels
    #       set through the setnchannels() or setparams() method
    # _sampwidth -- the number of bytes per audio sample
    #       set through the setsampwidth() or setparams() method
    # _framerate -- the sampling frequency
    #       set through the setframerate() or setparams() method
    # _nframes -- the number of audio frames written to the header
    #       set through the setnframes() or setparams() method
    # _aifc -- whether we're writing an AIFF-C file or an AIFF file
    #       set through the aifc() method, reset through the
    #       aiff() method
    #
    # These variables are used internally only:
    # _version -- the AIFF-C version number
    # _comp -- the compressor from builtin module cl
    # _nframeswritten -- the number of audio frames actually written
    # _datalength -- the size of the audio samples written to the header
    # _datawritten -- the size of the audio samples actually written
    _file = None  # Set here since __del__ checks it
    def __init__(self, f):
        if isinstance(f, str):
            file_object = builtins.open(f, 'wb')
            try:
                self.initfp(file_object)
            except:
                file_object.close()
                raise
            # treat .aiff file extensions as non-compressed audio
            if f.endswith('.aiff'):
                self._aifc = 0
        else:
            # assume it is an open file object already
            self.initfp(f)
    def initfp(self, file):
        self._file = file
        self._version = _AIFC_version
        self._comptype = b'NONE'
        self._compname = b'not compressed'
        self._convert = None
        self._nchannels = 0
        self._sampwidth = 0
        self._framerate = 0
        self._nframes = 0
        self._nframeswritten = 0
        self._datawritten = 0
        self._datalength = 0
        self._markers = []
        self._marklength = 0
        self._aifc = 1      # AIFF-C is default
    def __del__(self):
        self.close()
    def __enter__(self):
        return self
    def __exit__(self, *args):
        self.close()
    #
    # User visible methods.
    #
    def aiff(self):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        self._aifc = 0
    def aifc(self):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        self._aifc = 1
    def setnchannels(self, nchannels):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        if nchannels < 1:
            raise Error('bad # of channels')
        self._nchannels = nchannels
    def getnchannels(self):
        if not self._nchannels:
            raise Error('number of channels not set')
        return self._nchannels
    def setsampwidth(self, sampwidth):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        if sampwidth < 1 or sampwidth > 4:
            raise Error('bad sample width')
        self._sampwidth = sampwidth
    def getsampwidth(self):
        if not self._sampwidth:
            raise Error('sample width not set')
        return self._sampwidth
    def setframerate(self, framerate):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        if framerate <= 0:
            raise Error('bad frame rate')
        self._framerate = framerate
    def getframerate(self):
        if not self._framerate:
            raise Error('frame rate not set')
        return self._framerate
    def setnframes(self, nframes):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        self._nframes = nframes
    def getnframes(self):
        return self._nframeswritten
    def setcomptype(self, comptype, compname):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        if comptype not in (b'NONE', b'ulaw', b'ULAW',
                            b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
            raise Error('unsupported compression type')
        self._comptype = comptype
        self._compname = compname
    def getcomptype(self):
        return self._comptype
    def getcompname(self):
        return self._compname
 ##  def setversion(self, version):
 ##      if self._nframeswritten:
 ##          raise Error, 'cannot change parameters after starting to write'
 ##      self._version = version
    def setparams(self, params):
        nchannels, sampwidth, framerate, nframes, comptype, compname = params
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        if comptype not in (b'NONE', b'ulaw', b'ULAW',
                            b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
            raise Error('unsupported compression type')
        self.setnchannels(nchannels)
        self.setsampwidth(sampwidth)
        self.setframerate(framerate)
        self.setnframes(nframes)
        self.setcomptype(comptype, compname)
    def getparams(self):
        if not self._nchannels or not self._sampwidth or not self._framerate:
            raise Error('not all parameters set')
        return _aifc_params(self._nchannels, self._sampwidth, self._framerate,
                            self._nframes, self._comptype, self._compname)
    def setmark(self, id, pos, name):
        if id <= 0:
            raise Error('marker ID must be > 0')
        if pos < 0:
            raise Error('marker position must be >= 0')
        if not isinstance(name, bytes):
            raise Error('marker name must be bytes')
        for i in range(len(self._markers)):
            if id == self._markers[i][0]:
                self._markers[i] = id, pos, name
                return
        self._markers.append((id, pos, name))
    def getmark(self, id):
        for marker in self._markers:
            if id == marker[0]:
                return marker
        raise Error('marker {0!r} does not exist'.format(id))
    def getmarkers(self):
        if len(self._markers) == 0:
            return None
        return self._markers
    def tell(self):
        return self._nframeswritten
    def writeframesraw(self, data):
        if not isinstance(data, (bytes, bytearray)):
            data = memoryview(data).cast('B')
        self._ensure_header_written(len(data))
        nframes = len(data) // (self._sampwidth * self._nchannels)
        if self._convert:
            data = self._convert(data)
        self._file.write(data)
        self._nframeswritten = self._nframeswritten + nframes
        self._datawritten = self._datawritten + len(data)
    def writeframes(self, data):
        self.writeframesraw(data)
        if self._nframeswritten != self._nframes or \
              self._datalength != self._datawritten:
            self._patchheader()
    def close(self):
        if self._file is None:
            return
        try:
            self._ensure_header_written(0)
            if self._datawritten & 1:
                # quick pad to even size
                self._file.write(b'\x00')
                self._datawritten = self._datawritten + 1
            self._writemarkers()
            if self._nframeswritten != self._nframes or \
                  self._datalength != self._datawritten or \
                  self._marklength:
                self._patchheader()
        finally:
            # Prevent ref cycles
            self._convert = None
            f = self._file
            self._file = None
            f.close()
    #
    # Internal methods.
    #
    def _lin2alaw(self, data):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=DeprecationWarning)
            import audioop
        return audioop.lin2alaw(data, 2)
    def _lin2ulaw(self, data):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=DeprecationWarning)
            import audioop
        return audioop.lin2ulaw(data, 2)
    def _lin2adpcm(self, data):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=DeprecationWarning)
            import audioop
        if not hasattr(self, '_adpcmstate'):
            self._adpcmstate = None
        data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
        return data
    def _lin2sowt(self, data):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=DeprecationWarning)
            import audioop
        return audioop.byteswap(data, 2)
    def _ensure_header_written(self, datasize):
        if not self._nframeswritten:
            if self._comptype in (b'ULAW', b'ulaw',
                b'ALAW', b'alaw', b'G722',
                b'sowt', b'SOWT'):
                if not self._sampwidth:
                    self._sampwidth = 2
                if self._sampwidth != 2:
                    raise Error('sample width must be 2 when compressing '
                                'with ulaw/ULAW, alaw/ALAW, sowt/SOWT '
                                'or G7.22 (ADPCM)')
            if not self._nchannels:
                raise Error('# channels not specified')
            if not self._sampwidth:
                raise Error('sample width not specified')
            if not self._framerate:
                raise Error('sampling rate not specified')
            self._write_header(datasize)
    def _init_compression(self):
        if self._comptype == b'G722':
            self._convert = self._lin2adpcm
        elif self._comptype in (b'ulaw', b'ULAW'):
            self._convert = self._lin2ulaw
        elif self._comptype in (b'alaw', b'ALAW'):
            self._convert = self._lin2alaw
        elif self._comptype in (b'sowt', b'SOWT'):
            self._convert = self._lin2sowt
    def _write_header(self, initlength):
        if self._aifc and self._comptype != b'NONE':
            self._init_compression()
        self._file.write(b'FORM')
        if not self._nframes:
            self._nframes = initlength // (self._nchannels * self._sampwidth)
        self._datalength = self._nframes * self._nchannels * self._sampwidth
        if self._datalength & 1:
            self._datalength = self._datalength + 1
        if self._aifc:
            if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'):
                self._datalength = self._datalength // 2
                if self._datalength & 1:
                    self._datalength = self._datalength + 1
            elif self._comptype == b'G722':
                self._datalength = (self._datalength + 3) // 4
                if self._datalength & 1:
                    self._datalength = self._datalength + 1
        try:
            self._form_length_pos = self._file.tell()
        except (AttributeError, OSError):
            self._form_length_pos = None
        commlength = self._write_form_length(self._datalength)
        if self._aifc:
            self._file.write(b'AIFC')
            self._file.write(b'FVER')
            _write_ulong(self._file, 4)
            _write_ulong(self._file, self._version)
        else:
            self._file.write(b'AIFF')
        self._file.write(b'COMM')
        _write_ulong(self._file, commlength)
        _write_short(self._file, self._nchannels)
        if self._form_length_pos is not None:
            self._nframes_pos = self._file.tell()
        _write_ulong(self._file, self._nframes)
        if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
            _write_short(self._file, 8)
        else:
            _write_short(self._file, self._sampwidth * 8)
        _write_float(self._file, self._framerate)
        if self._aifc:
            self._file.write(self._comptype)
            _write_string(self._file, self._compname)
        self._file.write(b'SSND')
        if self._form_length_pos is not None:
            self._ssnd_length_pos = self._file.tell()
        _write_ulong(self._file, self._datalength + 8)
        _write_ulong(self._file, 0)
        _write_ulong(self._file, 0)
    def _write_form_length(self, datalength):
        if self._aifc:
            commlength = 18 + 5 + len(self._compname)
            if commlength & 1:
                commlength = commlength + 1
            verslength = 12
        else:
            commlength = 18
            verslength = 0
        _write_ulong(self._file, 4 + verslength + self._marklength + \
                     8 + commlength + 16 + datalength)
        return commlength
    def _patchheader(self):
        curpos = self._file.tell()
        if self._datawritten & 1:
            datalength = self._datawritten + 1
            self._file.write(b'\x00')
        else:
            datalength = self._datawritten
        if datalength == self._datalength and \
              self._nframes == self._nframeswritten and \
              self._marklength == 0:
            self._file.seek(curpos, 0)
            return
        self._file.seek(self._form_length_pos, 0)
        dummy = self._write_form_length(datalength)
        self._file.seek(self._nframes_pos, 0)
        _write_ulong(self._file, self._nframeswritten)
        self._file.seek(self._ssnd_length_pos, 0)
        _write_ulong(self._file, datalength + 8)
        self._file.seek(curpos, 0)
        self._nframes = self._nframeswritten
        self._datalength = datalength
    def _writemarkers(self):
        if len(self._markers) == 0:
            return
        self._file.write(b'MARK')
        length = 2
        for marker in self._markers:
            id, pos, name = marker
            length = length + len(name) + 1 + 6
            if len(name) & 1 == 0:
                length = length + 1
        _write_ulong(self._file, length)
        self._marklength = length + 8
        _write_short(self._file, len(self._markers))
        for marker in self._markers:
            id, pos, name = marker
            _write_short(self._file, id)
            _write_ulong(self._file, pos)
            _write_string(self._file, name)
 def open(f, mode=None):
    if mode is None:
        if hasattr(f, 'mode'):
            mode = f.mode
        else:
            mode = 'rb'
    if mode in ('r', 'rb'):
        return Aifc_read(f)
    elif mode in ('w', 'wb'):
        return Aifc_write(f)
    else:
        raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
 if __name__ == '__main__':
    import sys
    if not sys.argv[1:]:
        sys.argv.append('/usr/demos/data/audio/bach.aiff')
    fn = sys.argv[1]
    with open(fn, 'r') as f:
        print("Reading", fn)
        print("nchannels =", f.getnchannels())
        print("nframes   =", f.getnframes())
        print("sampwidth =", f.getsampwidth())
        print("framerate =", f.getframerate())
        print("comptype  =", f.getcomptype())
        print("compname  =", f.getcompname())
        if sys.argv[2:]:
            gn = sys.argv[2]
            print("Writing", gn)
            with open(gn, 'w') as g:
                g.setparams(f.getparams())
                while 1:
                    data = f.readframes(1024)
                    if not data:
                        break
                    g.writeframes(data)
            print("Done.")
--- a/Lib/asynchat.py
+++ b/Lib/asynchat.py
@@ -1,307 +0,0 @@
 # -*- Mode: Python; tab-width: 4 -*-
 #       Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp
 #       Author: Sam Rushing <rushing@nightmare.com>
 # ======================================================================
 # Copyright 1996 by Sam Rushing
 #
 #                         All Rights Reserved
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose and without fee is hereby
 # granted, provided that the above copyright notice appear in all
 # copies and that both that copyright notice and this permission
 # notice appear in supporting documentation, and that the name of Sam
 # Rushing not be used in advertising or publicity pertaining to
 # distribution of the software without specific, written prior
 # permission.
 #
 # SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
 # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
 # NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
 # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
 # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 # ======================================================================
 r"""A class supporting chat-style (command/response) protocols.
 This class adds support for 'chat' style protocols - where one side
 sends a 'command', and the other sends a response (examples would be
 the common internet protocols - smtp, nntp, ftp, etc..).
 The handle_read() method looks at the input stream for the current
 'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n'
 for multi-line output), calling self.found_terminator() on its
 receipt.
 for example:
 Say you build an async nntp client using this class.  At the start
 of the connection, you'll have self.terminator set to '\r\n', in
 order to process the single-line greeting.  Just before issuing a
 'LIST' command you'll set it to '\r\n.\r\n'.  The output of the LIST
 command will be accumulated (using your own 'collect_incoming_data'
 method) up to the terminator, and then control will be returned to
 you - by calling your self.found_terminator() method.
 """
 import asyncore
 from collections import deque
 class async_chat(asyncore.dispatcher):
    """This is an abstract class.  You must derive from this class, and add
    the two methods collect_incoming_data() and found_terminator()"""
    # these are overridable defaults
    ac_in_buffer_size = 65536
    ac_out_buffer_size = 65536
    # we don't want to enable the use of encoding by default, because that is a
    # sign of an application bug that we don't want to pass silently
    use_encoding = 0
    encoding = 'latin-1'
    def __init__(self, sock=None, map=None):
        # for string terminator matching
        self.ac_in_buffer = b''
        # we use a list here rather than io.BytesIO for a few reasons...
        # del lst[:] is faster than bio.truncate(0)
        # lst = [] is faster than bio.truncate(0)
        self.incoming = []
        # we toss the use of the "simple producer" and replace it with
        # a pure deque, which the original fifo was a wrapping of
        self.producer_fifo = deque()
        asyncore.dispatcher.__init__(self, sock, map)
    def collect_incoming_data(self, data):
        raise NotImplementedError("must be implemented in subclass")
    def _collect_incoming_data(self, data):
        self.incoming.append(data)
    def _get_data(self):
        d = b''.join(self.incoming)
        del self.incoming[:]
        return d
    def found_terminator(self):
        raise NotImplementedError("must be implemented in subclass")
    def set_terminator(self, term):
        """Set the input delimiter.
        Can be a fixed string of any length, an integer, or None.
        """
        if isinstance(term, str) and self.use_encoding:
            term = bytes(term, self.encoding)
        elif isinstance(term, int) and term < 0:
            raise ValueError('the number of received bytes must be positive')
        self.terminator = term
    def get_terminator(self):
        return self.terminator
    # grab some more data from the socket,
    # throw it to the collector method,
    # check for the terminator,
    # if found, transition to the next state.
    def handle_read(self):
        try:
            data = self.recv(self.ac_in_buffer_size)
        except BlockingIOError:
            return
        except OSError as why:
            self.handle_error()
            return
        if isinstance(data, str) and self.use_encoding:
            data = bytes(str, self.encoding)
        self.ac_in_buffer = self.ac_in_buffer + data
        # Continue to search for self.terminator in self.ac_in_buffer,
        # while calling self.collect_incoming_data.  The while loop
        # is necessary because we might read several data+terminator
        # combos with a single recv(4096).
        while self.ac_in_buffer:
            lb = len(self.ac_in_buffer)
            terminator = self.get_terminator()
            if not terminator:
                # no terminator, collect it all
                self.collect_incoming_data(self.ac_in_buffer)
                self.ac_in_buffer = b''
            elif isinstance(terminator, int):
                # numeric terminator
                n = terminator
                if lb < n:
                    self.collect_incoming_data(self.ac_in_buffer)
                    self.ac_in_buffer = b''
                    self.terminator = self.terminator - lb
                else:
                    self.collect_incoming_data(self.ac_in_buffer[:n])
                    self.ac_in_buffer = self.ac_in_buffer[n:]
                    self.terminator = 0
                    self.found_terminator()
            else:
                # 3 cases:
                # 1) end of buffer matches terminator exactly:
                #    collect data, transition
                # 2) end of buffer matches some prefix:
                #    collect data to the prefix
                # 3) end of buffer does not match any prefix:
                #    collect data
                terminator_len = len(terminator)
                index = self.ac_in_buffer.find(terminator)
                if index != -1:
                    # we found the terminator
                    if index > 0:
                        # don't bother reporting the empty string
                        # (source of subtle bugs)
                        self.collect_incoming_data(self.ac_in_buffer[:index])
                    self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
                    # This does the Right Thing if the terminator
                    # is changed here.
                    self.found_terminator()
                else:
                    # check for a prefix of the terminator
                    index = find_prefix_at_end(self.ac_in_buffer, terminator)
                    if index:
                        if index != lb:
                            # we found a prefix, collect up to the prefix
                            self.collect_incoming_data(self.ac_in_buffer[:-index])
                            self.ac_in_buffer = self.ac_in_buffer[-index:]
                        break
                    else:
                        # no prefix, collect it all
                        self.collect_incoming_data(self.ac_in_buffer)
                        self.ac_in_buffer = b''
    def handle_write(self):
        self.initiate_send()
    def handle_close(self):
        self.close()
    def push(self, data):
        if not isinstance(data, (bytes, bytearray, memoryview)):
            raise TypeError('data argument must be byte-ish (%r)',
                            type(data))
        sabs = self.ac_out_buffer_size
        if len(data) > sabs:
            for i in range(0, len(data), sabs):
                self.producer_fifo.append(data[i:i+sabs])
        else:
            self.producer_fifo.append(data)
        self.initiate_send()
    def push_with_producer(self, producer):
        self.producer_fifo.append(producer)
        self.initiate_send()
    def readable(self):
        "predicate for inclusion in the readable for select()"
        # cannot use the old predicate, it violates the claim of the
        # set_terminator method.
        # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
        return 1
    def writable(self):
        "predicate for inclusion in the writable for select()"
        return self.producer_fifo or (not self.connected)
    def close_when_done(self):
        "automatically close this channel once the outgoing queue is empty"
        self.producer_fifo.append(None)
    def initiate_send(self):
        while self.producer_fifo and self.connected:
            first = self.producer_fifo[0]
            # handle empty string/buffer or None entry
            if not first:
                del self.producer_fifo[0]
                if first is None:
                    self.handle_close()
                    return
            # handle classic producer behavior
            obs = self.ac_out_buffer_size
            try:
                data = first[:obs]
            except TypeError:
                data = first.more()
                if data:
                    self.producer_fifo.appendleft(data)
                else:
                    del self.producer_fifo[0]
                continue
            if isinstance(data, str) and self.use_encoding:
                data = bytes(data, self.encoding)
            # send the data
            try:
                num_sent = self.send(data)
            except OSError:
                self.handle_error()
                return
            if num_sent:
                if num_sent < len(data) or obs < len(first):
                    self.producer_fifo[0] = first[num_sent:]
                else:
                    del self.producer_fifo[0]
            # we tried to send some actual data
            return
    def discard_buffers(self):
        # Emergencies only!
        self.ac_in_buffer = b''
        del self.incoming[:]
        self.producer_fifo.clear()
 class simple_producer:
    def __init__(self, data, buffer_size=512):
        self.data = data
        self.buffer_size = buffer_size
    def more(self):
        if len(self.data) > self.buffer_size:
            result = self.data[:self.buffer_size]
            self.data = self.data[self.buffer_size:]
            return result
        else:
            result = self.data
            self.data = b''
            return result
 # Given 'haystack', see if any prefix of 'needle' is at its end.  This
 # assumes an exact match has already been checked.  Return the number of
 # characters matched.
 # for example:
 # f_p_a_e("qwerty\r", "\r\n") => 1
 # f_p_a_e("qwertydkjf", "\r\n") => 0
 # f_p_a_e("qwerty\r\n", "\r\n") => <undefined>
 # this could maybe be made faster with a computed regex?
 # [answer: no; circa Python-2.0, Jan 2001]
 # new python:   28961/s
 # old python:   18307/s
 # re:        12820/s
 # regex:     14035/s
 def find_prefix_at_end(haystack, needle):
    l = len(needle) - 1
    while l and not haystack.endswith(needle[:l]):
        l -= 1
    return l
--- a/Lib/asyncore.py
+++ b/Lib/asyncore.py
@@ -1,642 +0,0 @@
 # -*- Mode: Python -*-
 #   Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp
 #   Author: Sam Rushing <rushing@nightmare.com>
 # ======================================================================
 # Copyright 1996 by Sam Rushing
 #
 #                         All Rights Reserved
 #
 # Permission to use, copy, modify, and distribute this software and
 # its documentation for any purpose and without fee is hereby
 # granted, provided that the above copyright notice appear in all
 # copies and that both that copyright notice and this permission
 # notice appear in supporting documentation, and that the name of Sam
 # Rushing not be used in advertising or publicity pertaining to
 # distribution of the software without specific, written prior
 # permission.
 #
 # SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
 # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
 # NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
 # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
 # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 # ======================================================================
 """Basic infrastructure for asynchronous socket service clients and servers.
 There are only two ways to have a program on a single processor do "more
 than one thing at a time".  Multi-threaded programming is the simplest and
 most popular way to do it, but there is another very different technique,
 that lets you have nearly all the advantages of multi-threading, without
 actually using multiple threads. it's really only practical if your program
 is largely I/O bound. If your program is CPU bound, then pre-emptive
 scheduled threads are probably what you really need. Network servers are
 rarely CPU-bound, however.
 If your operating system supports the select() system call in its I/O
 library (and nearly all do), then you can use it to juggle multiple
 communication channels at once; doing other work while your I/O is taking
 place in the "background."  Although this strategy can seem strange and
 complex, especially at first, it is in many ways easier to understand and
 control than multi-threaded programming. The module documented here solves
 many of the difficult problems for you, making the task of building
 sophisticated high-performance network servers and clients a snap.
 """
 import select
 import socket
 import sys
 import time
 import warnings
 import os
 from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
     ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
     errorcode
 _DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
                           EBADF})
 try:
    socket_map
 except NameError:
    socket_map = {}
 def _strerror(err):
    try:
        return os.strerror(err)
    except (ValueError, OverflowError, NameError):
        if err in errorcode:
            return errorcode[err]
        return "Unknown error %s" %err
 class ExitNow(Exception):
    pass
 _reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit)
 def read(obj):
    try:
        obj.handle_read_event()
    except _reraised_exceptions:
        raise
    except:
        obj.handle_error()
 def write(obj):
    try:
        obj.handle_write_event()
    except _reraised_exceptions:
        raise
    except:
        obj.handle_error()
 def _exception(obj):
    try:
        obj.handle_expt_event()
    except _reraised_exceptions:
        raise
    except:
        obj.handle_error()
 def readwrite(obj, flags):
    try:
        if flags & select.POLLIN:
            obj.handle_read_event()
        if flags & select.POLLOUT:
            obj.handle_write_event()
        if flags & select.POLLPRI:
            obj.handle_expt_event()
        if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
            obj.handle_close()
    except OSError as e:
        if e.args[0] not in _DISCONNECTED:
            obj.handle_error()
        else:
            obj.handle_close()
    except _reraised_exceptions:
        raise
    except:
        obj.handle_error()
 def poll(timeout=0.0, map=None):
    if map is None:
        map = socket_map
    if map:
        r = []; w = []; e = []
        for fd, obj in list(map.items()):
            is_r = obj.readable()
            is_w = obj.writable()
            if is_r:
                r.append(fd)
            # accepting sockets should not be writable
            if is_w and not obj.accepting:
                w.append(fd)
            if is_r or is_w:
                e.append(fd)
        if [] == r == w == e:
            time.sleep(timeout)
            return
        r, w, e = select.select(r, w, e, timeout)
        for fd in r:
            obj = map.get(fd)
            if obj is None:
                continue
            read(obj)
        for fd in w:
            obj = map.get(fd)
            if obj is None:
                continue
            write(obj)
        for fd in e:
            obj = map.get(fd)
            if obj is None:
                continue
            _exception(obj)
 def poll2(timeout=0.0, map=None):
    # Use the poll() support added to the select module in Python 2.0
    if map is None:
        map = socket_map
    if timeout is not None:
        # timeout is in milliseconds
        timeout = int(timeout*1000)
    pollster = select.poll()
    if map:
        for fd, obj in list(map.items()):
            flags = 0
            if obj.readable():
                flags |= select.POLLIN | select.POLLPRI
            # accepting sockets should not be writable
            if obj.writable() and not obj.accepting:
                flags |= select.POLLOUT
            if flags:
                pollster.register(fd, flags)
        r = pollster.poll(timeout)
        for fd, flags in r:
            obj = map.get(fd)
            if obj is None:
                continue
            readwrite(obj, flags)
 poll3 = poll2                           # Alias for backward compatibility
 def loop(timeout=30.0, use_poll=False, map=None, count=None):
    if map is None:
        map = socket_map
    if use_poll and hasattr(select, 'poll'):
        poll_fun = poll2
    else:
        poll_fun = poll
    if count is None:
        while map:
            poll_fun(timeout, map)
    else:
        while map and count > 0:
            poll_fun(timeout, map)
            count = count - 1
 class dispatcher:
    debug = False
    connected = False
    accepting = False
    connecting = False
    closing = False
    addr = None
    ignore_log_types = frozenset({'warning'})
    def __init__(self, sock=None, map=None):
        if map is None:
            self._map = socket_map
        else:
            self._map = map
        self._fileno = None
        if sock:
            # Set to nonblocking just to make sure for cases where we
            # get a socket from a blocking source.
            sock.setblocking(0)
            self.set_socket(sock, map)
            self.connected = True
            # The constructor no longer requires that the socket
            # passed be connected.
            try:
                self.addr = sock.getpeername()
            except OSError as err:
                if err.args[0] in (ENOTCONN, EINVAL):
                    # To handle the case where we got an unconnected
                    # socket.
                    self.connected = False
                else:
                    # The socket is broken in some unknown way, alert
                    # the user and remove it from the map (to prevent
                    # polling of broken sockets).
                    self.del_channel(map)
                    raise
        else:
            self.socket = None
    def __repr__(self):
        status = [self.__class__.__module__+"."+self.__class__.__qualname__]
        if self.accepting and self.addr:
            status.append('listening')
        elif self.connected:
            status.append('connected')
        if self.addr is not None:
            try:
                status.append('%s:%d' % self.addr)
            except TypeError:
                status.append(repr(self.addr))
        return '<%s at %#x>' % (' '.join(status), id(self))
    def add_channel(self, map=None):
        #self.log_info('adding channel %s' % self)
        if map is None:
            map = self._map
        map[self._fileno] = self
    def del_channel(self, map=None):
        fd = self._fileno
        if map is None:
            map = self._map
        if fd in map:
            #self.log_info('closing channel %d:%s' % (fd, self))
            del map[fd]
        self._fileno = None
    def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM):
        self.family_and_type = family, type
        sock = socket.socket(family, type)
        sock.setblocking(0)
        self.set_socket(sock)
    def set_socket(self, sock, map=None):
        self.socket = sock
        self._fileno = sock.fileno()
        self.add_channel(map)
    def set_reuse_addr(self):
        # try to re-use a server port if possible
        try:
            self.socket.setsockopt(
                socket.SOL_SOCKET, socket.SO_REUSEADDR,
                self.socket.getsockopt(socket.SOL_SOCKET,
                                       socket.SO_REUSEADDR) | 1
                )
        except OSError:
            pass
    # ==================================================
    # predicates for select()
    # these are used as filters for the lists of sockets
    # to pass to select().
    # ==================================================
    def readable(self):
        return True
    def writable(self):
        return True
    # ==================================================
    # socket object methods.
    # ==================================================
    def listen(self, num):
        self.accepting = True
        if os.name == 'nt' and num > 5:
            num = 5
        return self.socket.listen(num)
    def bind(self, addr):
        self.addr = addr
        return self.socket.bind(addr)
    def connect(self, address):
        self.connected = False
        self.connecting = True
        err = self.socket.connect_ex(address)
        if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \
        or err == EINVAL and os.name == 'nt':
            self.addr = address
            return
        if err in (0, EISCONN):
            self.addr = address
            self.handle_connect_event()
        else:
            raise OSError(err, errorcode[err])
    def accept(self):
        # XXX can return either an address pair or None
        try:
            conn, addr = self.socket.accept()
        except TypeError:
            return None
        except OSError as why:
            if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
                return None
            else:
                raise
        else:
            return conn, addr
    def send(self, data):
        try:
            result = self.socket.send(data)
            return result
        except OSError as why:
            if why.args[0] == EWOULDBLOCK:
                return 0
            elif why.args[0] in _DISCONNECTED:
                self.handle_close()
                return 0
            else:
                raise
    def recv(self, buffer_size):
        try:
            data = self.socket.recv(buffer_size)
            if not data:
                # a closed connection is indicated by signaling
                # a read condition, and having recv() return 0.
                self.handle_close()
                return b''
            else:
                return data
        except OSError as why:
            # winsock sometimes raises ENOTCONN
            if why.args[0] in _DISCONNECTED:
                self.handle_close()
                return b''
            else:
                raise
    def close(self):
        self.connected = False
        self.accepting = False
        self.connecting = False
        self.del_channel()
        if self.socket is not None:
            try:
                self.socket.close()
            except OSError as why:
                if why.args[0] not in (ENOTCONN, EBADF):
                    raise
    # log and log_info may be overridden to provide more sophisticated
    # logging and warning methods. In general, log is for 'hit' logging
    # and 'log_info' is for informational, warning and error logging.
    def log(self, message):
        sys.stderr.write('log: %s\n' % str(message))
    def log_info(self, message, type='info'):
        if type not in self.ignore_log_types:
            print('%s: %s' % (type, message))
    def handle_read_event(self):
        if self.accepting:
            # accepting sockets are never connected, they "spawn" new
            # sockets that are connected
            self.handle_accept()
        elif not self.connected:
            if self.connecting:
                self.handle_connect_event()
            self.handle_read()
        else:
            self.handle_read()
    def handle_connect_event(self):
        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
        if err != 0:
            raise OSError(err, _strerror(err))
        self.handle_connect()
        self.connected = True
        self.connecting = False
    def handle_write_event(self):
        if self.accepting:
            # Accepting sockets shouldn't get a write event.
            # We will pretend it didn't happen.
            return
        if not self.connected:
            if self.connecting:
                self.handle_connect_event()
        self.handle_write()
    def handle_expt_event(self):
        # handle_expt_event() is called if there might be an error on the
        # socket, or if there is OOB data
        # check for the error condition first
        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
        if err != 0:
            # we can get here when select.select() says that there is an
            # exceptional condition on the socket
            # since there is an error, we'll go ahead and close the socket
            # like we would in a subclassed handle_read() that received no
            # data
            self.handle_close()
        else:
            self.handle_expt()
    def handle_error(self):
        nil, t, v, tbinfo = compact_traceback()
        # sometimes a user repr method will crash.
        try:
            self_repr = repr(self)
        except:
            self_repr = '<__repr__(self) failed for object at %0x>' % id(self)
        self.log_info(
            'uncaptured python exception, closing channel %s (%s:%s %s)' % (
                self_repr,
                t,
                v,
                tbinfo
                ),
            'error'
            )
        self.handle_close()
    def handle_expt(self):
        self.log_info('unhandled incoming priority event', 'warning')
    def handle_read(self):
        self.log_info('unhandled read event', 'warning')
    def handle_write(self):
        self.log_info('unhandled write event', 'warning')
    def handle_connect(self):
        self.log_info('unhandled connect event', 'warning')
    def handle_accept(self):
        pair = self.accept()
        if pair is not None:
            self.handle_accepted(*pair)
    def handle_accepted(self, sock, addr):
        sock.close()
        self.log_info('unhandled accepted event', 'warning')
    def handle_close(self):
        self.log_info('unhandled close event', 'warning')
        self.close()
 # ---------------------------------------------------------------------------
 # adds simple buffered output capability, useful for simple clients.
 # [for more sophisticated usage use asynchat.async_chat]
 # ---------------------------------------------------------------------------
 class dispatcher_with_send(dispatcher):
    def __init__(self, sock=None, map=None):
        dispatcher.__init__(self, sock, map)
        self.out_buffer = b''
    def initiate_send(self):
        num_sent = 0
        num_sent = dispatcher.send(self, self.out_buffer[:65536])
        self.out_buffer = self.out_buffer[num_sent:]
    def handle_write(self):
        self.initiate_send()
    def writable(self):
        return (not self.connected) or len(self.out_buffer)
    def send(self, data):
        if self.debug:
            self.log_info('sending %s' % repr(data))
        self.out_buffer = self.out_buffer + data
        self.initiate_send()
 # ---------------------------------------------------------------------------
 # used for debugging.
 # ---------------------------------------------------------------------------
 def compact_traceback():
    t, v, tb = sys.exc_info()
    tbinfo = []
    if not tb: # Must have a traceback
        raise AssertionError("traceback does not exist")
    while tb:
        tbinfo.append((
            tb.tb_frame.f_code.co_filename,
            tb.tb_frame.f_code.co_name,
            str(tb.tb_lineno)
            ))
        tb = tb.tb_next
    # just to be safe
    del tb
    file, function, line = tbinfo[-1]
    info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])
    return (file, function, line), t, v, info
 def close_all(map=None, ignore_all=False):
    if map is None:
        map = socket_map
    for x in list(map.values()):
        try:
            x.close()
        except OSError as x:
            if x.args[0] == EBADF:
                pass
            elif not ignore_all:
                raise
        except _reraised_exceptions:
            raise
        except:
            if not ignore_all:
                raise
    map.clear()
 # Asynchronous File I/O:
 #
 # After a little research (reading man pages on various unixen, and
 # digging through the linux kernel), I've determined that select()
 # isn't meant for doing asynchronous file i/o.
 # Heartening, though - reading linux/mm/filemap.c shows that linux
 # supports asynchronous read-ahead.  So _MOST_ of the time, the data
 # will be sitting in memory for us already when we go to read it.
 #
 # What other OS's (besides NT) support async file i/o?  [VMS?]
 #
 # Regardless, this is useful for pipes, and stdin/stdout...
 if os.name == 'posix':
    class file_wrapper:
        # Here we override just enough to make a file
        # look like a socket for the purposes of asyncore.
        # The passed fd is automatically os.dup()'d
        def __init__(self, fd):
            self.fd = os.dup(fd)
        def __del__(self):
            if self.fd >= 0:
                warnings.warn("unclosed file %r" % self, ResourceWarning,
                              source=self)
            self.close()
        def recv(self, *args):
            return os.read(self.fd, *args)
        def send(self, *args):
            return os.write(self.fd, *args)
        def getsockopt(self, level, optname, buflen=None):
            if (level == socket.SOL_SOCKET and
                optname == socket.SO_ERROR and
                not buflen):
                return 0
            raise NotImplementedError("Only asyncore specific behaviour "
                                      "implemented.")
        read = recv
        write = send
        def close(self):
            if self.fd < 0:
                return
            fd = self.fd
            self.fd = -1
            os.close(fd)
        def fileno(self):
            return self.fd
    class file_dispatcher(dispatcher):
        def __init__(self, fd, map=None):
            dispatcher.__init__(self, None, map)
            self.connected = True
            try:
                fd = fd.fileno()
            except AttributeError:
                pass
            self.set_file(fd)
            # set it to non-blocking mode
            os.set_blocking(fd, False)
        def set_file(self, fd):
            self.socket = file_wrapper(fd)
            self._fileno = self.socket.fileno()
            self.add_channel()
--- a/Lib/calendar.py
+++ b/Lib/calendar.py
@@ -10,7 +10,6 @@ import datetime
 from enum import IntEnum, global_enum
 import locale as _locale
 from itertools import repeat
 import warnings
 __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
           "firstweekday", "isleap", "leapdays", "weekday", "monthrange",
@@ -28,7 +27,9 @@ __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
 error = ValueError
 # Exceptions raised for bad input
-class IllegalMonthError(ValueError):
+# This is trick for backward compatibility. Since 3.13, we will raise IllegalMonthError instead of
 # IndexError for bad month number(out of 1-12). But we can't remove IndexError for backward compatibility.
 class IllegalMonthError(ValueError, IndexError):
    def __init__(self, month):
        self.month = month
    def __str__(self):
@@ -44,6 +45,7 @@ class IllegalWeekdayError(ValueError):
 def __getattr__(name):
    if name in ('January', 'February'):
        import warnings
        warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead",
                      DeprecationWarning, stacklevel=2)
        if name == 'January':
@@ -158,11 +160,14 @@ def weekday(year, month, day):
    return Day(datetime.date(year, month, day).weekday())
-def monthrange(year, month):
+def _validate_month(month):
    """Return weekday (0-6 ~ Mon-Sun) and number of days (28-31) for
       year, month."""
    if not 1 <= month <= 12:
        raise IllegalMonthError(month)
 def monthrange(year, month):
    """Return weekday of first day of month (0-6 ~ Mon-Sun)
       and number of days (28-31) for year, month."""
    _validate_month(month)
    day1 = weekday(year, month, 1)
    ndays = mdays[month] + (month == FEBRUARY and isleap(year))
    return day1, ndays
@@ -370,6 +375,8 @@ class TextCalendar(Calendar):
        """
        Return a formatted month name.
        """
        _validate_month(themonth)
        s = month_name[themonth]
        if withyear:
            s = "%s %r" % (s, theyear)
@@ -500,6 +507,7 @@ class HTMLCalendar(Calendar):
        """
        Return a month name as a table row.
        """
        _validate_month(themonth)
        if withyear:
            s = '%s %s' % (month_name[themonth], theyear)
        else:
@@ -585,8 +593,6 @@ class different_locale:
        _locale.setlocale(_locale.LC_TIME, self.locale)
    def __exit__(self, *args):
        if self.oldlocale is None:
            return
        _locale.setlocale(_locale.LC_TIME, self.oldlocale)
@@ -690,7 +696,7 @@ def timegm(tuple):
    return seconds
-def main(args):
+def main(args=None):
    import argparse
    parser = argparse.ArgumentParser()
    textgroup = parser.add_argument_group('text only arguments')
@@ -736,10 +742,15 @@ def main(args):
        choices=("text", "html"),
        help="output type (text or html)"
    )
    parser.add_argument(
        "-f", "--first-weekday",
        type=int, default=0,
        help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)"
    )
    parser.add_argument(
        "year",
        nargs='?', type=int,
-        help="year number (1-9999)"
+        help="year number"
    )
    parser.add_argument(
        "month",
@@ -747,7 +758,7 @@ def main(args):
        help="month number (1-12, text only)"
    )
-    options = parser.parse_args(args[1:])
+    options = parser.parse_args(args)
    if options.locale and not options.encoding:
        parser.error("if --locale is specified --encoding is required")
@@ -756,10 +767,14 @@ def main(args):
    locale = options.locale, options.encoding
    if options.type == "html":
        if options.month:
            parser.error("incorrect number of arguments")
            sys.exit(1)
        if options.locale:
            cal = LocaleHTMLCalendar(locale=locale)
        else:
            cal = HTMLCalendar()
        cal.setfirstweekday(options.first_weekday)
        encoding = options.encoding
        if encoding is None:
            encoding = sys.getdefaultencoding()
@@ -767,20 +782,20 @@ def main(args):
        write = sys.stdout.buffer.write
        if options.year is None:
            write(cal.formatyearpage(datetime.date.today().year, **optdict))
        elif options.month is None:
            write(cal.formatyearpage(options.year, **optdict))
        else:
-            parser.error("incorrect number of arguments")
+            write(cal.formatyearpage(options.year, **optdict))
            sys.exit(1)
    else:
        if options.locale:
            cal = LocaleTextCalendar(locale=locale)
        else:
            cal = TextCalendar()
        cal.setfirstweekday(options.first_weekday)
        optdict = dict(w=options.width, l=options.lines)
        if options.month is None:
            optdict["c"] = options.spacing
            optdict["m"] = options.months
        if options.month is not None:
            _validate_month(options.month)
        if options.year is None:
            result = cal.formatyear(datetime.date.today().year, **optdict)
        elif options.month is None:
@@ -795,4 +810,4 @@ def main(args):
 if __name__ == "__main__":
-    main(sys.argv)
+    main()
--- a/Lib/cgitb.py
+++ b/Lib/cgitb.py
@@ -1,332 +0,0 @@
 """More comprehensive traceback formatting for Python scripts.
 To enable this module, do:
    import cgitb; cgitb.enable()
 at the top of your script.  The optional arguments to enable() are:
    display     - if true, tracebacks are displayed in the web browser
    logdir      - if set, tracebacks are written to files in this directory
    context     - number of lines of source code to show for each stack frame
    format      - 'text' or 'html' controls the output format
 By default, tracebacks are displayed but not saved, the context is 5 lines
 and the output format is 'html' (for backwards compatibility with the
 original use of this module)
 Alternatively, if you have caught an exception and want cgitb to display it
 for you, call cgitb.handler().  The optional argument to handler() is a
 3-item tuple (etype, evalue, etb) just like the value of sys.exc_info().
 The default handler displays output as HTML.
 """
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import sys
 import tempfile
 import time
 import tokenize
 import traceback
 import warnings
 from html import escape as html_escape
 warnings._deprecated(__name__, remove=(3, 13))
 def reset():
    """Return a string that resets the CGI and browser to a known state."""
    return '''<!--: spam
 Content-Type: text/html
 <body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> -->
 <body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> --> -->
 </font> </font> </font> </script> </object> </blockquote> </pre>
 </table> </table> </table> </table> </table> </font> </font> </font>'''
 __UNDEF__ = []                          # a special sentinel object
 def small(text):
    if text:
        return '<small>' + text + '</small>'
    else:
        return ''
 def strong(text):
    if text:
        return '<strong>' + text + '</strong>'
    else:
        return ''
 def grey(text):
    if text:
        return '<font color="#909090">' + text + '</font>'
    else:
        return ''
 def lookup(name, frame, locals):
    """Find the value for a given name in the given environment."""
    if name in locals:
        return 'local', locals[name]
    if name in frame.f_globals:
        return 'global', frame.f_globals[name]
    if '__builtins__' in frame.f_globals:
        builtins = frame.f_globals['__builtins__']
        if isinstance(builtins, dict):
            if name in builtins:
                return 'builtin', builtins[name]
        else:
            if hasattr(builtins, name):
                return 'builtin', getattr(builtins, name)
    return None, __UNDEF__
 def scanvars(reader, frame, locals):
    """Scan one logical line of Python and look up values of variables used."""
    vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__
    for ttype, token, start, end, line in tokenize.generate_tokens(reader):
        if ttype == tokenize.NEWLINE: break
        if ttype == tokenize.NAME and token not in keyword.kwlist:
            if lasttoken == '.':
                if parent is not __UNDEF__:
                    value = getattr(parent, token, __UNDEF__)
                    vars.append((prefix + token, prefix, value))
            else:
                where, value = lookup(token, frame, locals)
                vars.append((token, where, value))
        elif token == '.':
            prefix += lasttoken + '.'
            parent = value
        else:
            parent, prefix = None, ''
        lasttoken = token
    return vars
 def html(einfo, context=5):
    """Return a nice HTML document describing a given traceback."""
    etype, evalue, etb = einfo
    if isinstance(etype, type):
        etype = etype.__name__
    pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
    date = time.ctime(time.time())
    head = f'''
 <body bgcolor="#f0f0f8">
 <table width="100%" cellspacing=0 cellpadding=2 border=0 summary="heading">
 <tr bgcolor="#6622aa">
 <td valign=bottom>&nbsp;<br>
 <font color="#ffffff" face="helvetica, arial">&nbsp;<br>
 <big><big><strong>{html_escape(str(etype))}</strong></big></big></font></td>
 <td align=right valign=bottom>
 <font color="#ffffff" face="helvetica, arial">{pyver}<br>{date}</font></td>
 </tr></table>
 <p>A problem occurred in a Python script.  Here is the sequence of
 function calls leading up to the error, in the order they occurred.</p>'''
    indent = '<tt>' + small('&nbsp;' * 5) + '&nbsp;</tt>'
    frames = []
    records = inspect.getinnerframes(etb, context)
    for frame, file, lnum, func, lines, index in records:
        if file:
            file = os.path.abspath(file)
            link = '<a href="file://%s">%s</a>' % (file, pydoc.html.escape(file))
        else:
            file = link = '?'
        args, varargs, varkw, locals = inspect.getargvalues(frame)
        call = ''
        if func != '?':
            call = 'in ' + strong(pydoc.html.escape(func))
            if func != "<module>":
                call += inspect.formatargvalues(args, varargs, varkw, locals,
                    formatvalue=lambda value: '=' + pydoc.html.repr(value))
        highlight = {}
        def reader(lnum=[lnum]):
            highlight[lnum[0]] = 1
            try: return linecache.getline(file, lnum[0])
            finally: lnum[0] += 1
        vars = scanvars(reader, frame, locals)
        rows = ['<tr><td bgcolor="#d8bbff">%s%s %s</td></tr>' %
                ('<big>&nbsp;</big>', link, call)]
        if index is not None:
            i = lnum - index
            for line in lines:
                num = small('&nbsp;' * (5-len(str(i))) + str(i)) + '&nbsp;'
                if i in highlight:
                    line = '<tt>=&gt;%s%s</tt>' % (num, pydoc.html.preformat(line))
                    rows.append('<tr><td bgcolor="#ffccee">%s</td></tr>' % line)
                else:
                    line = '<tt>&nbsp;&nbsp;%s%s</tt>' % (num, pydoc.html.preformat(line))
                    rows.append('<tr><td>%s</td></tr>' % grey(line))
                i += 1
        done, dump = {}, []
        for name, where, value in vars:
            if name in done: continue
            done[name] = 1
            if value is not __UNDEF__:
                if where in ('global', 'builtin'):
                    name = ('<em>%s</em> ' % where) + strong(name)
                elif where == 'local':
                    name = strong(name)
                else:
                    name = where + strong(name.split('.')[-1])
                dump.append('%s&nbsp;= %s' % (name, pydoc.html.repr(value)))
            else:
                dump.append(name + ' <em>undefined</em>')
        rows.append('<tr><td>%s</td></tr>' % small(grey(', '.join(dump))))
        frames.append('''
 <table width="100%%" cellspacing=0 cellpadding=0 border=0>
 %s</table>''' % '\n'.join(rows))
    exception = ['<p>%s: %s' % (strong(pydoc.html.escape(str(etype))),
                                pydoc.html.escape(str(evalue)))]
    for name in dir(evalue):
        if name[:1] == '_': continue
        value = pydoc.html.repr(getattr(evalue, name))
        exception.append('\n<br>%s%s&nbsp;=\n%s' % (indent, name, value))
    return head + ''.join(frames) + ''.join(exception) + '''
 <!-- The above is a description of an error in a Python program, formatted
     for a web browser because the 'cgitb' module was enabled.  In case you
     are not reading this in a web browser, here is the original traceback:
 %s
 -->
 ''' % pydoc.html.escape(
          ''.join(traceback.format_exception(etype, evalue, etb)))
 def text(einfo, context=5):
    """Return a plain text document describing a given traceback."""
    etype, evalue, etb = einfo
    if isinstance(etype, type):
        etype = etype.__name__
    pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
    date = time.ctime(time.time())
    head = "%s\n%s\n%s\n" % (str(etype), pyver, date) + '''
 A problem occurred in a Python script.  Here is the sequence of
 function calls leading up to the error, in the order they occurred.
 '''
    frames = []
    records = inspect.getinnerframes(etb, context)
    for frame, file, lnum, func, lines, index in records:
        file = file and os.path.abspath(file) or '?'
        args, varargs, varkw, locals = inspect.getargvalues(frame)
        call = ''
        if func != '?':
            call = 'in ' + func
            if func != "<module>":
                call += inspect.formatargvalues(args, varargs, varkw, locals,
                    formatvalue=lambda value: '=' + pydoc.text.repr(value))
        highlight = {}
        def reader(lnum=[lnum]):
            highlight[lnum[0]] = 1
            try: return linecache.getline(file, lnum[0])
            finally: lnum[0] += 1
        vars = scanvars(reader, frame, locals)
        rows = [' %s %s' % (file, call)]
        if index is not None:
            i = lnum - index
            for line in lines:
                num = '%5d ' % i
                rows.append(num+line.rstrip())
                i += 1
        done, dump = {}, []
        for name, where, value in vars:
            if name in done: continue
            done[name] = 1
            if value is not __UNDEF__:
                if where == 'global': name = 'global ' + name
                elif where != 'local': name = where + name.split('.')[-1]
                dump.append('%s = %s' % (name, pydoc.text.repr(value)))
            else:
                dump.append(name + ' undefined')
        rows.append('\n'.join(dump))
        frames.append('\n%s\n' % '\n'.join(rows))
    exception = ['%s: %s' % (str(etype), str(evalue))]
    for name in dir(evalue):
        value = pydoc.text.repr(getattr(evalue, name))
        exception.append('\n%s%s = %s' % (" "*4, name, value))
    return head + ''.join(frames) + ''.join(exception) + '''
 The above is a description of an error in a Python program.  Here is
 the original traceback:
 %s
 ''' % ''.join(traceback.format_exception(etype, evalue, etb))
 class Hook:
    """A hook to replace sys.excepthook that shows tracebacks in HTML."""
    def __init__(self, display=1, logdir=None, context=5, file=None,
                 format="html"):
        self.display = display          # send tracebacks to browser if true
        self.logdir = logdir            # log tracebacks to files if not None
        self.context = context          # number of source code lines per frame
        self.file = file or sys.stdout  # place to send the output
        self.format = format
    def __call__(self, etype, evalue, etb):
        self.handle((etype, evalue, etb))
    def handle(self, info=None):
        info = info or sys.exc_info()
        if self.format == "html":
            self.file.write(reset())
        formatter = (self.format=="html") and html or text
        plain = False
        try:
            doc = formatter(info, self.context)
        except:                         # just in case something goes wrong
            doc = ''.join(traceback.format_exception(*info))
            plain = True
        if self.display:
            if plain:
                doc = pydoc.html.escape(doc)
                self.file.write('<pre>' + doc + '</pre>\n')
            else:
                self.file.write(doc + '\n')
        else:
            self.file.write('<p>A problem occurred in a Python script.\n')
        if self.logdir is not None:
            suffix = ['.txt', '.html'][self.format=="html"]
            (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir)
            try:
                with os.fdopen(fd, 'w') as file:
                    file.write(doc)
                msg = '%s contains the description of this error.' % path
            except:
                msg = 'Tried to save traceback to %s, but failed.' % path
            if self.format == 'html':
                self.file.write('<p>%s</p>\n' % msg)
            else:
                self.file.write(msg + '\n')
        try:
            self.file.flush()
        except: pass
 handler = Hook().handle
 def enable(display=1, logdir=None, context=5, format="html"):
    """Install an exception handler that formats tracebacks as HTML.
    The optional argument 'display' can be set to 0 to suppress sending the
    traceback to the browser, and 'logdir' can be set to a directory to cause
    tracebacks to be written to files there."""
    sys.excepthook = Hook(display=display, logdir=logdir,
                          context=context, format=format)
--- a/Lib/chunk.py
+++ b/Lib/chunk.py
@@ -1,173 +0,0 @@
 """Simple class to read IFF chunks.
 An IFF chunk (used in formats such as AIFF, TIFF, RMFF (RealMedia File
 Format)) has the following structure:
 +----------------+
 | ID (4 bytes)   |
 +----------------+
 | size (4 bytes) |
 +----------------+
 | data           |
 | ...            |
 +----------------+
 The ID is a 4-byte string which identifies the type of chunk.
 The size field (a 32-bit value, encoded using big-endian byte order)
 gives the size of the whole chunk, including the 8-byte header.
 Usually an IFF-type file consists of one or more chunks.  The proposed
 usage of the Chunk class defined here is to instantiate an instance at
 the start of each chunk and read from the instance until it reaches
 the end, after which a new instance can be instantiated.  At the end
 of the file, creating a new instance will fail with an EOFError
 exception.
 Usage:
 while True:
    try:
        chunk = Chunk(file)
    except EOFError:
        break
    chunktype = chunk.getname()
    while True:
        data = chunk.read(nbytes)
        if not data:
            pass
        # do something with data
 The interface is file-like.  The implemented methods are:
 read, close, seek, tell, isatty.
 Extra methods are: skip() (called by close, skips to the end of the chunk),
 getname() (returns the name (ID) of the chunk)
 The __init__ method has one required argument, a file-like object
 (including a chunk instance), and one optional argument, a flag which
 specifies whether or not chunks are aligned on 2-byte boundaries.  The
 default is 1, i.e. aligned.
 """
 import warnings
 warnings._deprecated(__name__, remove=(3, 13))
 class Chunk:
    def __init__(self, file, align=True, bigendian=True, inclheader=False):
        import struct
        self.closed = False
        self.align = align      # whether to align to word (2-byte) boundaries
        if bigendian:
            strflag = '>'
        else:
            strflag = '<'
        self.file = file
        self.chunkname = file.read(4)
        if len(self.chunkname) < 4:
            raise EOFError
        try:
            self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0]
        except struct.error:
            raise EOFError from None
        if inclheader:
            self.chunksize = self.chunksize - 8 # subtract header
        self.size_read = 0
        try:
            self.offset = self.file.tell()
        except (AttributeError, OSError):
            self.seekable = False
        else:
            self.seekable = True
    def getname(self):
        """Return the name (ID) of the current chunk."""
        return self.chunkname
    def getsize(self):
        """Return the size of the current chunk."""
        return self.chunksize
    def close(self):
        if not self.closed:
            try:
                self.skip()
            finally:
                self.closed = True
    def isatty(self):
        if self.closed:
            raise ValueError("I/O operation on closed file")
        return False
    def seek(self, pos, whence=0):
        """Seek to specified position into the chunk.
        Default position is 0 (start of chunk).
        If the file is not seekable, this will result in an error.
        """
        if self.closed:
            raise ValueError("I/O operation on closed file")
        if not self.seekable:
            raise OSError("cannot seek")
        if whence == 1:
            pos = pos + self.size_read
        elif whence == 2:
            pos = pos + self.chunksize
        if pos < 0 or pos > self.chunksize:
            raise RuntimeError
        self.file.seek(self.offset + pos, 0)
        self.size_read = pos
    def tell(self):
        if self.closed:
            raise ValueError("I/O operation on closed file")
        return self.size_read
    def read(self, size=-1):
        """Read at most size bytes from the chunk.
        If size is omitted or negative, read until the end
        of the chunk.
        """
        if self.closed:
            raise ValueError("I/O operation on closed file")
        if self.size_read >= self.chunksize:
            return b''
        if size < 0:
            size = self.chunksize - self.size_read
        if size > self.chunksize - self.size_read:
            size = self.chunksize - self.size_read
        data = self.file.read(size)
        self.size_read = self.size_read + len(data)
        if self.size_read == self.chunksize and \
           self.align and \
           (self.chunksize & 1):
            dummy = self.file.read(1)
            self.size_read = self.size_read + len(dummy)
        return data
    def skip(self):
        """Skip the rest of the chunk.
        If you are not interested in the contents of the chunk,
        this method should be called so that the file points to
        the start of the next chunk.
        """
        if self.closed:
            raise ValueError("I/O operation on closed file")
        if self.seekable:
            try:
                n = self.chunksize - self.size_read
                # maybe fix alignment
                if self.align and (self.chunksize & 1):
                    n = n + 1
                self.file.seek(n, 1)
                self.size_read = self.size_read + n
                return
            except OSError:
                pass
        while self.size_read < self.chunksize:
            n = min(8192, self.chunksize - self.size_read)
            dummy = self.read(n)
            if not dummy:
                raise EOFError
--- a/Lib/codeop.py
+++ b/Lib/codeop.py
@@ -66,7 +66,12 @@ def _maybe_compile(compiler, source, filename, symbol):
                compiler(source + "\n", filename, symbol)
                return None
            except SyntaxError as e:
-                if "incomplete input" in str(e):
+                # XXX: RustPython; support multiline definitions in REPL
                # See also: https://github.com/RustPython/RustPython/pull/5743
                strerr = str(e)
                if source.endswith(":") and "expected an indented block" in strerr:
                    return None
                elif "incomplete input" in str(e):
                    return None
                # fallthrough
--- a/Lib/colorsys.py
+++ b/Lib/colorsys.py
@@ -24,7 +24,7 @@ HSV: Hue, Saturation, Value
 __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb",
           "rgb_to_hsv","hsv_to_rgb"]
-# Some floating point constants
+# Some floating-point constants
 ONE_THIRD = 1.0/3.0
 ONE_SIXTH = 1.0/6.0
--- a/Lib/ctypes/init.py
+++ b/Lib/ctypes/init.py
@@ -36,6 +36,9 @@ from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \
     FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \
     FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR
 # TODO: RUSTPYTHON remove this
 from _ctypes import _non_existing_function
 # WINOLEAPI -> HRESULT
 # WINOLEAPI_(type)
 #
@@ -296,7 +299,9 @@ def create_unicode_buffer(init, size=None):
        return buf
    elif isinstance(init, int):
        _sys.audit("ctypes.create_unicode_buffer", None, init)
-        buftype = c_wchar * init
+        # XXX: RUSTPYTHON
        # buftype = c_wchar * init
        buftype = c_wchar.__mul__(init)
        buf = buftype()
        return buf
    raise TypeError(init)
@@ -495,14 +500,15 @@ elif sizeof(c_ulonglong) == sizeof(c_void_p):
    c_ssize_t = c_longlong
 # functions
 from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr
 ## void *memmove(void *, const void *, size_t);
-memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr)
+# XXX: RUSTPYTHON
 # memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr)
 ## void *memset(void *, int, size_t)
-memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr)
+# XXX: RUSTPYTHON
 # memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr)
 def PYFUNCTYPE(restype, *argtypes):
    class CFunctionType(_CFuncPtr):
@@ -511,11 +517,13 @@ def PYFUNCTYPE(restype, *argtypes):
        _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
    return CFunctionType
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
+# XXX: RUSTPYTHON
 # _cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
    return _cast(obj, obj, typ)
-_string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
+# XXX: RUSTPYTHON
 # _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=-1):
    """string_at(addr[, size]) -> string
@@ -527,7 +535,8 @@ try:
 except ImportError:
    pass
 else:
-    _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr)
+    # XXX: RUSTPYTHON
    # _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr)
    def wstring_at(ptr, size=-1):
        """wstring_at(addr[, size]) -> string
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -102,7 +102,7 @@ import re
 import sys
 import traceback
 import unittest
-from io import StringIO  # XXX: RUSTPYTHON; , IncrementalNewlineDecoder
+from io import StringIO, IncrementalNewlineDecoder
 from collections import namedtuple
 TestResults = namedtuple('TestResults', 'failed attempted')
@@ -230,9 +230,7 @@ def _load_testfile(filename, package, module_relative, encoding):
            # get_data() opens files as 'rb', so one must do the equivalent
            # conversion as universal newlines would do.
-            # TODO: RUSTPYTHON; use _newline_convert once io.IncrementalNewlineDecoder is implemented
+            return _newline_convert(file_contents), filename
            return file_contents.replace(os.linesep, '\n'), filename
            # return _newline_convert(file_contents), filename
    with open(filename, encoding=encoding) as f:
        return f.read(), filename
--- a/Lib/email/init.py
+++ b/Lib/email/init.py
@@ -25,7 +25,6 @@ __all__ = [
    ]
 # Some convenience routines.  Don't import Parser and Message as side-effects
 # of importing email since those cascadingly import most of the rest of the
 # email package.
--- a/Lib/email/_encoded_words.py
+++ b/Lib/email/_encoded_words.py
@@ -62,7 +62,7 @@ __all__ = ['decode_q',
 # regex based decoder.
 _q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub,
-        lambda m: bytes([int(m.group(1), 16)]))
+        lambda m: bytes.fromhex(m.group(1).decode()))
 def decode_q(encoded):
    encoded = encoded.replace(b'_', b' ')
@@ -98,30 +98,42 @@ def len_q(bstring):
 #
 def decode_b(encoded):
-    defects = []
+    # First try encoding with validate=True, fixing the padding if needed.
    # This will succeed only if encoded includes no invalid characters.
    pad_err = len(encoded) % 4
-    if pad_err:
+    missing_padding = b'==='[:4-pad_err] if pad_err else b''
        defects.append(errors.InvalidBase64PaddingDefect())
        padded_encoded = encoded + b'==='[:4-pad_err]
    else:
        padded_encoded = encoded
    try:
-        return base64.b64decode(padded_encoded, validate=True), defects
+        return (
            base64.b64decode(encoded + missing_padding, validate=True),
            [errors.InvalidBase64PaddingDefect()] if pad_err else [],
        )
    except binascii.Error:
-        # Since we had correct padding, this must an invalid char error.
+        # Since we had correct padding, this is likely an invalid char error.
-        defects = [errors.InvalidBase64CharactersDefect()]
+        #
        # The non-alphabet characters are ignored as far as padding
-        # goes, but we don't know how many there are.  So we'll just
+        # goes, but we don't know how many there are.  So try without adding
-        # try various padding lengths until something works.
+        # padding to see if it works.
-        for i in 0, 1, 2, 3:
+        try:
            return (
                base64.b64decode(encoded, validate=False),
                [errors.InvalidBase64CharactersDefect()],
            )
        except binascii.Error:
            # Add as much padding as could possibly be necessary (extra padding
            # is ignored).
            try:
-                return base64.b64decode(encoded+b'='*i, validate=False), defects
+                return (
                    base64.b64decode(encoded + b'==', validate=False),
                    [errors.InvalidBase64CharactersDefect(),
                     errors.InvalidBase64PaddingDefect()],
                )
            except binascii.Error:
-                if i==0:
+                # This only happens when the encoded string's length is 1 more
-                    defects.append(errors.InvalidBase64PaddingDefect())
+                # than a multiple of 4, which is invalid.
-        else:
+                #
-            # This should never happen.
+                # bpo-27397: Just return the encoded string since there's no
-            raise AssertionError("unexpected binascii.Error")
+                # way to decode.
                return encoded, [errors.InvalidBase64LengthDefect()]
 def encode_b(bstring):
    return base64.b64encode(bstring).decode('ascii')
@@ -167,15 +179,15 @@ def decode(ew):
    # Turn the CTE decoded bytes into unicode.
    try:
        string = bstring.decode(charset)
-    except UnicodeError:
+    except UnicodeDecodeError:
        defects.append(errors.UndecodableBytesDefect("Encoded word "
-            "contains bytes not decodable using {} charset".format(charset)))
+            f"contains bytes not decodable using {charset!r} charset"))
        string = bstring.decode(charset, 'surrogateescape')
-    except LookupError:
+    except (LookupError, UnicodeEncodeError):
        string = bstring.decode('ascii', 'surrogateescape')
        if charset.lower() != 'unknown-8bit':
-            defects.append(errors.CharsetError("Unknown charset {} "
+            defects.append(errors.CharsetError(f"Unknown charset {charset!r} "
-                "in encoded word; decoded as unknown bytes".format(charset)))
+                f"in encoded word; decoded as unknown bytes"))
    return string, charset, lang, defects
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
--- a/Lib/email/_parseaddr.py
+++ b/Lib/email/_parseaddr.py
@@ -13,7 +13,7 @@ __all__ = [
    'quote',
    ]
-import time, calendar
+import time
 SPACE = ' '
 EMPTYSTRING = ''
@@ -65,8 +65,10 @@ def _parsedate_tz(data):
    """
    if not data:
-        return
+        return None
    data = data.split()
    if not data:  # This happens for whitespace-only input.
        return None
    # The FWS after the comma after the day-of-week is optional, so search and
    # adjust for this.
    if data[0].endswith(',') or data[0].lower() in _daynames:
@@ -93,6 +95,8 @@ def _parsedate_tz(data):
        return None
    data = data[:5]
    [dd, mm, yy, tm, tz] = data
    if not (dd and mm and yy):
        return None
    mm = mm.lower()
    if mm not in _monthnames:
        dd, mm = mm, dd.lower()
@@ -108,6 +112,8 @@ def _parsedate_tz(data):
        yy, tm = tm, yy
    if yy[-1] == ',':
        yy = yy[:-1]
        if not yy:
            return None
    if not yy[0].isdigit():
        yy, tz = tz, yy
    if tm[-1] == ',':
@@ -126,6 +132,8 @@ def _parsedate_tz(data):
            tss = 0
        elif len(tm) == 3:
            [thh, tmm, tss] = tm
        else:
            return None
    else:
        return None
    try:
@@ -186,6 +194,9 @@ def mktime_tz(data):
        # No zone info, so localtime is better assumption than GMT
        return time.mktime(data[:8] + (-1,))
    else:
        # Delay the import, since mktime_tz is rarely used
        import calendar
        t = calendar.timegm(data)
        return t - data[9]
@@ -379,7 +390,12 @@ class AddrlistClass:
        aslist.append('@')
        self.pos += 1
        self.gotonext()
-        return EMPTYSTRING.join(aslist) + self.getdomain()
+        domain = self.getdomain()
        if not domain:
            # Invalid domain, return an empty address instead of returning a
            # local part to denote failed parsing.
            return EMPTYSTRING
        return EMPTYSTRING.join(aslist) + domain
    def getdomain(self):
        """Get the complete domain name from an address."""
@@ -394,6 +410,10 @@ class AddrlistClass:
            elif self.field[self.pos] == '.':
                self.pos += 1
                sdlist.append('.')
            elif self.field[self.pos] == '@':
                # bpo-34155: Don't parse domains with two `@` like
                # `a@malicious.org@important.com`.
                return EMPTYSTRING
            elif self.field[self.pos] in self.atomends:
                break
            else:
--- a/Lib/email/_policybase.py
+++ b/Lib/email/_policybase.py
@@ -152,11 +152,18 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
    mangle_from_        -- a flag that, when True escapes From_ lines in the
                           body of the message by putting a `>' in front of
                           them. This is used when the message is being
-                           serialized by a generator. Default: True.
+                           serialized by a generator. Default: False.
    message_factory     -- the class to use to create new message objects.
                           If the value is None, the default is Message.
    verify_generated_headers
                        -- if true, the generator verifies that each header
                           they are properly folded, so that a parser won't
                           treat it as multiple headers, start-of-body, or
                           part of another header.
                           This is a check against custom Header & fold()
                           implementations.
    """
    raise_on_defect = False
@@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
    max_line_length = 78
    mangle_from_ = False
    message_factory = None
    verify_generated_headers = True
    def handle_defect(self, obj, defect):
        """Based on policy, either raise defect or call register_defect.
@@ -294,12 +302,12 @@ class Compat32(Policy):
        """+
        The name is parsed as everything up to the ':' and returned unmodified.
        The value is determined by stripping leading whitespace off the
-        remainder of the first line, joining all subsequent lines together, and
+        remainder of the first line joined with all subsequent lines, and
        stripping any trailing carriage return or linefeed characters.
        """
        name, value = sourcelines[0].split(':', 1)
-        value = value.lstrip(' \t') + ''.join(sourcelines[1:])
+        value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n')
        return (name, value.rstrip('\r\n'))
    def header_store_parse(self, name, value):
@@ -361,8 +369,12 @@ class Compat32(Policy):
            # Assume it is a Header-like object.
            h = value
        if h is not None:
-            parts.append(h.encode(linesep=self.linesep,
+            # The Header class interprets a value of None for maxlinelen as the
-                                  maxlinelen=self.max_line_length))
+            # default value of 78, as recommended by RFC 2822.
            maxlinelen = 0
            if self.max_line_length is not None:
                maxlinelen = self.max_line_length
            parts.append(h.encode(linesep=self.linesep, maxlinelen=maxlinelen))
        parts.append(self.linesep)
        return ''.join(parts)
--- a/Lib/email/architecture.rst
+++ b/Lib/email/architecture.rst
@@ -66,7 +66,7 @@ data payloads.
 Message Lifecycle
 -----------------
-The general lifecyle of a message is:
+The general lifecycle of a message is:
    Creation
        A `Message` object can be created by a Parser, or it can be
--- a/Lib/email/base64mime.py
+++ b/Lib/email/base64mime.py
@@ -45,7 +45,6 @@ EMPTYSTRING = ''
 MISC_LEN = 7
 # Helpers
 def header_length(bytearray):
    """Return the length of s when it is encoded with base64."""
@@ -57,7 +56,6 @@ def header_length(bytearray):
    return n
 def header_encode(header_bytes, charset='iso-8859-1'):
    """Encode a single header line with Base64 encoding in a given charset.
@@ -72,7 +70,6 @@ def header_encode(header_bytes, charset='iso-8859-1'):
    return '=?%s?b?%s?=' % (charset, encoded)
 def body_encode(s, maxlinelen=76, eol=NL):
    r"""Encode a string with base64.
@@ -84,7 +81,7 @@ def body_encode(s, maxlinelen=76, eol=NL):
    in an email.
    """
    if not s:
-        return s
+        return ""
    encvec = []
    max_unencoded = maxlinelen * 3 // 4
@@ -98,7 +95,6 @@ def body_encode(s, maxlinelen=76, eol=NL):
    return EMPTYSTRING.join(encvec)
 def decode(string):
    """Decode a raw base64 string, returning a bytes object.
--- a/Lib/email/charset.py
+++ b/Lib/email/charset.py
@@ -18,7 +18,6 @@ from email import errors
 from email.encoders import encode_7or8bit
 # Flags for types of header encodings
 QP          = 1 # Quoted-Printable
 BASE64      = 2 # Base64
@@ -32,7 +31,6 @@ UNKNOWN8BIT = 'unknown-8bit'
 EMPTYSTRING = ''
 # Defaults
 CHARSETS = {
    # input        header enc  body enc output conv
@@ -104,7 +102,6 @@ CODEC_MAP = {
    }
 # Convenience functions for extending the above mappings
 def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
    """Add character set properties to the global registry.
@@ -112,8 +109,8 @@ def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
    charset is the input character set, and must be the canonical name of a
    character set.
-    Optional header_enc and body_enc is either Charset.QP for
+    Optional header_enc and body_enc is either charset.QP for
-    quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
+    quoted-printable, charset.BASE64 for base64 encoding, charset.SHORTEST for
    the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST
    is only valid for header_enc.  It describes how message headers and
    message bodies in the input charset are to be encoded.  Default is no
@@ -153,7 +150,6 @@ def add_codec(charset, codecname):
    CODEC_MAP[charset] = codecname
 # Convenience function for encoding strings, taking into account
 # that they might be unknown-8bit (ie: have surrogate-escaped bytes)
 def _encode(string, codec):
@@ -163,7 +159,6 @@ def _encode(string, codec):
        return string.encode(codec)
 class Charset:
    """Map character sets to their email properties.
@@ -185,13 +180,13 @@ class Charset:
    header_encoding: If the character set must be encoded before it can be
                     used in an email header, this attribute will be set to
-                     Charset.QP (for quoted-printable), Charset.BASE64 (for
+                     charset.QP (for quoted-printable), charset.BASE64 (for
-                     base64 encoding), or Charset.SHORTEST for the shortest of
+                     base64 encoding), or charset.SHORTEST for the shortest of
                     QP or BASE64 encoding.  Otherwise, it will be None.
    body_encoding: Same as header_encoding, but describes the encoding for the
                   mail message's body, which indeed may be different than the
-                   header encoding.  Charset.SHORTEST is not allowed for
+                   header encoding.  charset.SHORTEST is not allowed for
                   body_encoding.
    output_charset: Some character sets must be converted before they can be
@@ -241,11 +236,9 @@ class Charset:
        self.output_codec = CODEC_MAP.get(self.output_charset,
                                          self.output_charset)
-    def __str__(self):
+    def __repr__(self):
        return self.input_charset.lower()
    __repr__ = __str__
    def __eq__(self, other):
        return str(self) == str(other).lower()
@@ -348,7 +341,6 @@ class Charset:
                if not lines and not current_line:
                    lines.append(None)
                else:
                    separator = (' ' if lines else '')
                    joined_line = EMPTYSTRING.join(current_line)
                    header_bytes = _encode(joined_line, codec)
                    lines.append(encoder(header_bytes))
--- a/Lib/email/contentmanager.py
+++ b/Lib/email/contentmanager.py
@@ -72,12 +72,14 @@ def get_non_text_content(msg):
    return msg.get_payload(decode=True)
 for maintype in 'audio image video application'.split():
    raw_data_manager.add_get_handler(maintype, get_non_text_content)
 del maintype
 def get_message_content(msg):
    return msg.get_payload(0)
 for subtype in 'rfc822 external-body'.split():
    raw_data_manager.add_get_handler('message/'+subtype, get_message_content)
 del subtype
 def get_and_fixup_unknown_message_content(msg):
@@ -144,15 +146,15 @@ def _encode_text(string, charset, cte, policy):
    linesep = policy.linesep.encode('ascii')
    def embedded_body(lines): return linesep.join(lines) + linesep
    def normal_body(lines): return b'\n'.join(lines) + b'\n'
-    if cte==None:
+    if cte is None:
        # Use heuristics to decide on the "best" encoding.
-        try:
+        if max((len(x) for x in lines), default=0) <= policy.max_line_length:
-            return '7bit', normal_body(lines).decode('ascii')
+            try:
-        except UnicodeDecodeError:
+                return '7bit', normal_body(lines).decode('ascii')
-            pass
+            except UnicodeDecodeError:
-        if (policy.cte_type == '8bit' and
+                pass
-                max(len(x) for x in lines) <= policy.max_line_length):
+            if policy.cte_type == '8bit':
-            return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
+                return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
        sniff = embedded_body(lines[:10])
        sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'),
                                          policy.max_line_length)
@@ -238,9 +240,7 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64',
        data = binascii.b2a_qp(data, istext=False, header=False, quotetabs=True)
        data = data.decode('ascii')
    elif cte == '7bit':
-        # Make sure it really is only ASCII.  The early warning here seems
+        data = data.decode('ascii')
        # worth the overhead...if you care write your own content manager :).
        data.encode('ascii')
    elif cte in ('8bit', 'binary'):
        data = data.decode('ascii', 'surrogateescape')
    msg.set_payload(data)
@@ -248,3 +248,4 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64',
    _finalize_set(msg, disposition, filename, cid, params)
 for typ in (bytes, bytearray, memoryview):
    raw_data_manager.add_set_handler(typ, set_bytes_content)
 del typ
--- a/Lib/email/encoders.py
+++ b/Lib/email/encoders.py
@@ -16,7 +16,6 @@ from base64 import encodebytes as _bencode
 from quopri import encodestring as _encodestring
 def _qencode(s):
    enc = _encodestring(s, quotetabs=True)
    # Must encode spaces, which quopri.encodestring() doesn't do
@@ -34,7 +33,6 @@ def encode_base64(msg):
    msg['Content-Transfer-Encoding'] = 'base64'
 def encode_quopri(msg):
    """Encode the message's payload in quoted-printable.
@@ -46,7 +44,6 @@ def encode_quopri(msg):
    msg['Content-Transfer-Encoding'] = 'quoted-printable'
 def encode_7or8bit(msg):
    """Set the Content-Transfer-Encoding header to 7bit or 8bit."""
    orig = msg.get_payload(decode=True)
@@ -64,6 +61,5 @@ def encode_7or8bit(msg):
        msg['Content-Transfer-Encoding'] = '7bit'
 def encode_noop(msg):
    """Do nothing."""
--- a/Lib/email/errors.py
+++ b/Lib/email/errors.py
@@ -29,6 +29,10 @@ class CharsetError(MessageError):
    """An illegal charset was given."""
 class HeaderWriteError(MessageError):
    """Error while writing headers."""
 # These are parsing defects which the parser was able to work around.
 class MessageDefect(ValueError):
    """Base class for a message defect."""
@@ -73,6 +77,9 @@ class InvalidBase64PaddingDefect(MessageDefect):
 class InvalidBase64CharactersDefect(MessageDefect):
    """base64 encoded sequence had characters not in base64 alphabet"""
 class InvalidBase64LengthDefect(MessageDefect):
    """base64 encoded sequence had invalid length (1 mod 4)"""
 # These errors are specific to header parsing.
 class HeaderDefect(MessageDefect):
@@ -105,3 +112,6 @@ class NonASCIILocalPartDefect(HeaderDefect):
    """local_part contains non-ASCII characters"""
    # This defect only occurs during unicode parsing, not when
    # parsing messages decoded from binary.
 class InvalidDateDefect(HeaderDefect):
    """Header has unparsable or invalid date"""
--- a/Lib/email/feedparser.py
+++ b/Lib/email/feedparser.py
@@ -37,11 +37,12 @@ NLCRE_crack = re.compile(r'(\r\n|\r|\n)')
 headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])')
 EMPTYSTRING = ''
 NL = '\n'
 boundaryendRE = re.compile(
    r'(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
 NeedMoreData = object()
 class BufferedSubFile(object):
    """A file-ish object that can have new data loaded into it.
@@ -132,7 +133,6 @@ class BufferedSubFile(object):
        return line
 class FeedParser:
    """A feed-style parser of email."""
@@ -189,7 +189,7 @@ class FeedParser:
        assert not self._msgstack
        # Look for final set of defects
        if root.get_content_maintype() == 'multipart' \
-               and not root.is_multipart():
+               and not root.is_multipart() and not self._headersonly:
            defect = errors.MultipartInvariantViolationDefect()
            self.policy.handle_defect(root, defect)
        return root
@@ -266,7 +266,7 @@ class FeedParser:
                        yield NeedMoreData
                        continue
                    break
-                msg = self._pop_message()
+                self._pop_message()
                # We need to pop the EOF matcher in order to tell if we're at
                # the end of the current file, not the end of the last block
                # of message headers.
@@ -320,7 +320,7 @@ class FeedParser:
                self._cur.set_payload(EMPTYSTRING.join(lines))
                return
            # Make sure a valid content type was specified per RFC 2045:6.4.
-            if (self._cur.get('content-transfer-encoding', '8bit').lower()
+            if (str(self._cur.get('content-transfer-encoding', '8bit')).lower()
                    not in ('7bit', '8bit', 'binary')):
                defect = errors.InvalidMultipartContentTransferEncodingDefect()
                self.policy.handle_defect(self._cur, defect)
@@ -329,9 +329,10 @@ class FeedParser:
            # this onto the input stream until we've scanned past the
            # preamble.
            separator = '--' + boundary
-            boundaryre = re.compile(
+            def boundarymatch(line):
-                '(?P<sep>' + re.escape(separator) +
+                if not line.startswith(separator):
-                r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
+                    return None
                return boundaryendRE.match(line, len(separator))
            capturing_preamble = True
            preamble = []
            linesep = False
@@ -343,7 +344,7 @@ class FeedParser:
                    continue
                if line == '':
                    break
-                mo = boundaryre.match(line)
+                mo = boundarymatch(line)
                if mo:
                    # If we're looking at the end boundary, we're done with
                    # this multipart.  If there was a newline at the end of
@@ -375,13 +376,13 @@ class FeedParser:
                        if line is NeedMoreData:
                            yield NeedMoreData
                            continue
-                        mo = boundaryre.match(line)
+                        mo = boundarymatch(line)
                        if not mo:
                            self._input.unreadline(line)
                            break
                    # Recurse to parse this subpart; the input stream points
                    # at the subpart's first line.
-                    self._input.push_eof_matcher(boundaryre.match)
+                    self._input.push_eof_matcher(boundarymatch)
                    for retval in self._parsegen():
                        if retval is NeedMoreData:
                            yield NeedMoreData
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -14,15 +14,16 @@ import random
 from copy import deepcopy
 from io import StringIO, BytesIO
 from email.utils import _has_surrogates
 from email.errors import HeaderWriteError
 UNDERSCORE = '_'
 NL = '\n'  # XXX: no longer used by the code below.
 NLCRE = re.compile(r'\r\n|\r|\n')
 fcre = re.compile(r'^From ', re.MULTILINE)
 NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
 class Generator:
    """Generates output from a Message object tree.
@@ -170,7 +171,7 @@ class Generator:
        # parameter.
        #
        # The way we do this, so as to make the _handle_*() methods simpler,
-        # is to cache any subpart writes into a buffer.  The we write the
+        # is to cache any subpart writes into a buffer.  Then we write the
        # headers and the buffer contents.  That way, subpart handlers can
        # Do The Right Thing, and can still modify the Content-Type: header if
        # necessary.
@@ -186,7 +187,11 @@ class Generator:
        # If we munged the cte, copy the message again and re-fix the CTE.
        if munge_cte:
            msg = deepcopy(msg)
-            msg.replace_header('content-transfer-encoding', munge_cte[0])
+            # Preserve the header order if the CTE header already exists.
            if msg.get('content-transfer-encoding') is None:
                msg['Content-Transfer-Encoding'] = munge_cte[0]
            else:
                msg.replace_header('content-transfer-encoding', munge_cte[0])
            msg.replace_header('content-type', munge_cte[1])
        # Write the headers.  First we see if the message object wants to
        # handle that itself.  If not, we'll do it generically.
@@ -219,7 +224,16 @@ class Generator:
    def _write_headers(self, msg):
        for h, v in msg.raw_items():
-            self.write(self.policy.fold(h, v))
+            folded = self.policy.fold(h, v)
            if self.policy.verify_generated_headers:
                linesep = self.policy.linesep
                if not folded.endswith(self.policy.linesep):
                    raise HeaderWriteError(
                        f'folded header does not end with {linesep!r}: {folded!r}')
                if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)):
                    raise HeaderWriteError(
                        f'folded header contains newline: {folded!r}')
            self.write(folded)
        # A blank line always separates headers from body
        self.write(self._NL)
@@ -240,7 +254,7 @@ class Generator:
                # existing message.
                msg = deepcopy(msg)
                del msg['content-transfer-encoding']
-                msg.set_payload(payload, charset)
+                msg.set_payload(msg._payload, charset)
                payload = msg.get_payload()
                self._munge_cte = (msg['content-transfer-encoding'],
                                   msg['content-type'])
@@ -388,7 +402,7 @@ class Generator:
    def _compile_re(cls, s, flags):
        return re.compile(s, flags)
-
+
 class BytesGenerator(Generator):
    """Generates a bytes version of a Message object tree.
@@ -439,7 +453,6 @@ class BytesGenerator(Generator):
        return re.compile(s.encode('ascii'), flags)
 _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]'
 class DecodedGenerator(Generator):
@@ -499,7 +512,6 @@ class DecodedGenerator(Generator):
                    }, file=self)
 # Helper used by Generator._make_boundary
 _width = len(repr(sys.maxsize-1))
 _fmt = '%%0%dd' % _width
--- a/Lib/email/header.py
+++ b/Lib/email/header.py
@@ -36,11 +36,11 @@ ecre = re.compile(r'''
  =\?                   # literal =?
  (?P<charset>[^?]*?)   # non-greedy up to the next ? is the charset
  \?                    # literal ?
-  (?P<encoding>[qb])    # either a "q" or a "b", case insensitive
+  (?P<encoding>[qQbB])  # either a "q" or a "b", case insensitive
  \?                    # literal ?
  (?P<encoded>.*?)      # non-greedy up to the next ?= is the encoded string
  \?=                   # literal ?=
-  ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE)
+  ''', re.VERBOSE | re.MULTILINE)
 # Field name regexp, including trailing colon, but not separating whitespace,
 # according to RFC 2822.  Character range is from tilde to exclamation mark.
@@ -52,12 +52,10 @@ fcre = re.compile(r'[\041-\176]+:$')
 _embedded_header = re.compile(r'\n[^ \t]+:')
 # Helpers
 _max_append = email.quoprimime._max_append
 def decode_header(header):
    """Decode a message header value without converting charset.
@@ -152,7 +150,6 @@ def decode_header(header):
    return collapsed
 def make_header(decoded_seq, maxlinelen=None, header_name=None,
                continuation_ws=' '):
    """Create a Header from a sequence of pairs as returned by decode_header()
@@ -175,7 +172,6 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None,
    return h
 class Header:
    def __init__(self, s=None, charset=None,
                 maxlinelen=None, header_name=None,
@@ -409,7 +405,6 @@ class Header:
        self._chunks = chunks
 class _ValueFormatter:
    def __init__(self, headerlen, maxlen, continuation_ws, splitchars):
        self._maxlen = maxlen
@@ -431,7 +426,7 @@ class _ValueFormatter:
        if end_of_line != (' ', ''):
            self._current_line.push(*end_of_line)
        if len(self._current_line) > 0:
-            if self._current_line.is_onlyws():
+            if self._current_line.is_onlyws() and self._lines:
                self._lines[-1] += str(self._current_line)
            else:
                self._lines.append(str(self._current_line))
--- a/Lib/email/headerregistry.py
+++ b/Lib/email/headerregistry.py
@@ -2,10 +2,6 @@
 This module provides an implementation of the HeaderRegistry API.
 The implementation is designed to flexibly follow RFC5322 rules.
 Eventually HeaderRegistry will be a public API, but it isn't yet,
 and will probably change some before that happens.
 """
 from types import MappingProxyType
@@ -31,6 +27,11 @@ class Address:
        without any Content Transfer Encoding.
        """
        inputs = ''.join(filter(None, (display_name, username, domain, addr_spec)))
        if '\r' in inputs or '\n' in inputs:
            raise ValueError("invalid arguments; address parts cannot contain CR or LF")
        # This clause with its potential 'raise' may only happen when an
        # application program creates an Address object using an addr_spec
        # keyword.  The email library code itself must always supply username
@@ -69,11 +70,9 @@ class Address:
        """The addr_spec (username@domain) portion of the address, quoted
        according to RFC 5322 rules, but with no Content Transfer Encoding.
        """
-        nameset = set(self.username)
+        lp = self.username
-        if len(nameset) > len(nameset-parser.DOT_ATOM_ENDS):
+        if not parser.DOT_ATOM_ENDS.isdisjoint(lp):
-            lp = parser.quote_string(self.username)
+            lp = parser.quote_string(lp)
        else:
            lp = self.username
        if self.domain:
            return lp + '@' + self.domain
        if not lp:
@@ -86,19 +85,17 @@ class Address:
                        self.display_name, self.username, self.domain)
    def __str__(self):
-        nameset = set(self.display_name)
+        disp = self.display_name
-        if len(nameset) > len(nameset-parser.SPECIALS):
+        if not parser.SPECIALS.isdisjoint(disp):
-            disp = parser.quote_string(self.display_name)
+            disp = parser.quote_string(disp)
        else:
            disp = self.display_name
        if disp:
            addr_spec = '' if self.addr_spec=='<>' else self.addr_spec
            return "{} <{}>".format(disp, addr_spec)
        return self.addr_spec
    def __eq__(self, other):
-        if type(other) != type(self):
+        if not isinstance(other, Address):
-            return False
+            return NotImplemented
        return (self.display_name == other.display_name and
                self.username == other.username and
                self.domain == other.domain)
@@ -141,17 +138,15 @@ class Group:
        if self.display_name is None and len(self.addresses)==1:
            return str(self.addresses[0])
        disp = self.display_name
-        if disp is not None:
+        if disp is not None and not parser.SPECIALS.isdisjoint(disp):
-            nameset = set(disp)
+            disp = parser.quote_string(disp)
            if len(nameset) > len(nameset-parser.SPECIALS):
                disp = parser.quote_string(disp)
        adrstr = ", ".join(str(x) for x in self.addresses)
        adrstr = ' ' + adrstr if adrstr else adrstr
        return "{}:{};".format(disp, adrstr)
    def __eq__(self, other):
-        if type(other) != type(self):
+        if not isinstance(other, Group):
-            return False
+            return NotImplemented
        return (self.display_name == other.display_name and
                self.addresses == other.addresses)
@@ -223,7 +218,7 @@ class BaseHeader(str):
                self.__class__.__bases__,
                str(self),
            ),
-            self.__dict__)
+            self.__getstate__())
    @classmethod
    def _reconstruct(cls, value):
@@ -245,13 +240,16 @@ class BaseHeader(str):
        the header name and the ': ' separator.
        """
-        # At some point we need to only put fws here if it was in the source.
+        # At some point we need to put fws here if it was in the source.
        header = parser.Header([
            parser.HeaderLabel([
                parser.ValueTerminal(self.name, 'header-name'),
                parser.ValueTerminal(':', 'header-sep')]),
-            parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]),
+            ])
-                             self._parse_tree])
+        if self._parse_tree:
            header.append(
                parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]))
        header.append(self._parse_tree)
        return header.fold(policy=policy)
@@ -300,7 +298,14 @@ class DateHeader:
            kwds['parse_tree'] = parser.TokenList()
            return
        if isinstance(value, str):
-            value = utils.parsedate_to_datetime(value)
+            kwds['decoded'] = value
            try:
                value = utils.parsedate_to_datetime(value)
            except ValueError:
                kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format'))
                kwds['datetime'] = None
                kwds['parse_tree'] = parser.TokenList()
                return
        kwds['datetime'] = value
        kwds['decoded'] = utils.format_datetime(kwds['datetime'])
        kwds['parse_tree'] = cls.value_parser(kwds['decoded'])
@@ -369,8 +374,8 @@ class AddressHeader:
    @property
    def addresses(self):
        if self._addresses is None:
-            self._addresses = tuple([address for group in self._groups
+            self._addresses = tuple(address for group in self._groups
-                                             for address in group.addresses])
+                                            for address in group.addresses)
        return self._addresses
@@ -517,6 +522,18 @@ class ContentTransferEncodingHeader:
        return self._cte
 class MessageIDHeader:
    max_count = 1
    value_parser = staticmethod(parser.parse_message_id)
    @classmethod
    def parse(cls, value, kwds):
        kwds['parse_tree'] = parse_tree = cls.value_parser(value)
        kwds['decoded'] = str(parse_tree)
        kwds['defects'].extend(parse_tree.all_defects)
 # The header factory #
 _default_header_map = {
@@ -539,6 +556,7 @@ _default_header_map = {
    'content-type':                 ContentTypeHeader,
    'content-disposition':          ContentDispositionHeader,
    'content-transfer-encoding':    ContentTransferEncodingHeader,
    'message-id':                   MessageIDHeader,
    }
 class HeaderRegistry:
--- a/Lib/email/iterators.py
+++ b/Lib/email/iterators.py
@@ -15,7 +15,6 @@ import sys
 from io import StringIO
 # This function will become a method of the Message class
 def walk(self):
    """Walk over the message tree, yielding each subpart.
@@ -29,7 +28,6 @@ def walk(self):
            yield from subpart.walk()
 # These two functions are imported into the Iterators.py interface module.
 def body_line_iterator(msg, decode=False):
    """Iterate over the parts, returning string payloads line-by-line.
@@ -55,7 +53,6 @@ def typed_subpart_iterator(msg, maintype='text', subtype=None):
                yield subpart
 def _structure(msg, fp=None, level=0, include_default=False):
    """A handy debugging aid"""
    if fp is None:
--- a/Lib/email/message.py
+++ b/Lib/email/message.py
@@ -6,6 +6,7 @@
 __all__ = ['Message', 'EmailMessage']
 import binascii
 import re
 import quopri
 from io import BytesIO, StringIO
@@ -13,7 +14,7 @@ from io import BytesIO, StringIO
 # Intrapackage imports
 from email import utils
 from email import errors
-from email._policybase import Policy, compat32
+from email._policybase import compat32
 from email import charset as _charset
 from email._encoded_words import decode_b
 Charset = _charset.Charset
@@ -34,7 +35,7 @@ def _splitparam(param):
    if not sep:
        return a.strip(), None
    return a.strip(), b.strip()
-
+
 def _formatparam(param, value=None, quote=True):
    """Convenience function to format and return a key=value pair.
@@ -129,7 +130,8 @@ def _decode_uu(encoded):
        decoded_lines.append(decoded_line)
    return b''.join(decoded_lines)
-
+
 class Message:
    """Basic message object.
@@ -169,7 +171,7 @@ class Message:
        header.  For backward compatibility reasons, if maxheaderlen is
        not specified it defaults to 0, so you must override it explicitly
        if you want a different maxheaderlen.  'policy' is passed to the
-        Generator instance used to serialize the mesasge; if it is not
+        Generator instance used to serialize the message; if it is not
        specified the policy associated with the message instance is used.
        If the message object contains binary data that is not encoded
@@ -287,25 +289,26 @@ class Message:
        # cte might be a Header, so for now stringify it.
        cte = str(self.get('content-transfer-encoding', '')).lower()
        # payload may be bytes here.
-        if isinstance(payload, str):
+        if not decode:
-            if utils._has_surrogates(payload):
+            if isinstance(payload, str) and utils._has_surrogates(payload):
-                bpayload = payload.encode('ascii', 'surrogateescape')
+                try:
-                if not decode:
+                    bpayload = payload.encode('ascii', 'surrogateescape')
                    try:
-                        payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace')
+                        payload = bpayload.decode(self.get_content_charset('ascii'), 'replace')
                    except LookupError:
                        payload = bpayload.decode('ascii', 'replace')
-            elif decode:
+                except UnicodeEncodeError:
-                try:
+                    pass
                    bpayload = payload.encode('ascii')
                except UnicodeError:
                    # This won't happen for RFC compliant messages (messages
                    # containing only ASCII code points in the unicode input).
                    # If it does happen, turn the string into bytes in a way
                    # guaranteed not to fail.
                    bpayload = payload.encode('raw-unicode-escape')
        if not decode:
            return payload
        if isinstance(payload, str):
            try:
                bpayload = payload.encode('ascii', 'surrogateescape')
            except UnicodeEncodeError:
                # This won't happen for RFC compliant messages (messages
                # containing only ASCII code points in the unicode input).
                # If it does happen, turn the string into bytes in a way
                # guaranteed not to fail.
                bpayload = payload.encode('raw-unicode-escape')
        if cte == 'quoted-printable':
            return quopri.decodestring(bpayload)
        elif cte == 'base64':
@@ -337,7 +340,7 @@ class Message:
                return
            if not isinstance(charset, Charset):
                charset = Charset(charset)
-            payload = payload.encode(charset.output_charset)
+            payload = payload.encode(charset.output_charset, 'surrogateescape')
        if hasattr(payload, 'decode'):
            self._payload = payload.decode('ascii', 'surrogateescape')
        else:
@@ -446,7 +449,11 @@ class Message:
        self._headers = newheaders
    def __contains__(self, name):
-        return name.lower() in [k.lower() for k, v in self._headers]
+        name_lower = name.lower()
        for k, v in self._headers:
            if name_lower == k.lower():
                return True
        return False
    def __iter__(self):
        for field, value in self._headers:
@@ -973,7 +980,7 @@ class MIMEPart(Message):
        if policy is None:
            from email.policy import default
            policy = default
-        Message.__init__(self, policy)
+        super().__init__(policy)
    def as_string(self, unixfrom=False, maxheaderlen=None, policy=None):
@@ -983,14 +990,14 @@ class MIMEPart(Message):
        header.  maxheaderlen is retained for backward compatibility with the
        base Message class, but defaults to None, meaning that the policy value
        for max_line_length controls the header maximum length.  'policy' is
-        passed to the Generator instance used to serialize the mesasge; if it
+        passed to the Generator instance used to serialize the message; if it
        is not specified the policy associated with the message instance is
        used.
        """
        policy = self.policy if policy is None else policy
        if maxheaderlen is None:
            maxheaderlen = policy.max_line_length
-        return super().as_string(maxheaderlen=maxheaderlen, policy=policy)
+        return super().as_string(unixfrom, maxheaderlen, policy)
    def __str__(self):
        return self.as_string(policy=self.policy.clone(utf8=True))
@@ -1007,7 +1014,7 @@ class MIMEPart(Message):
            if subtype in preferencelist:
                yield (preferencelist.index(subtype), part)
            return
-        if maintype != 'multipart':
+        if maintype != 'multipart' or not self.is_multipart():
            return
        if subtype != 'related':
            for subpart in part.iter_parts():
@@ -1066,7 +1073,16 @@ class MIMEPart(Message):
        maintype, subtype = self.get_content_type().split('/')
        if maintype != 'multipart' or subtype == 'alternative':
            return
-        parts = self.get_payload().copy()
+        payload = self.get_payload()
        # Certain malformed messages can have content type set to `multipart/*`
        # but still have single part body, in which case payload.copy() can
        # fail with AttributeError.
        try:
            parts = payload.copy()
        except AttributeError:
            # payload is not a list, it is most probably a string.
            return
        if maintype == 'multipart' and subtype == 'related':
            # For related, we treat everything but the root as an attachment.
            # The root may be indicated by 'start'; if there's no start or we
@@ -1103,7 +1119,7 @@ class MIMEPart(Message):
        Return an empty iterator for a non-multipart.
        """
-        if self.get_content_maintype() == 'multipart':
+        if self.is_multipart():
            yield from self.get_payload()
    def get_content(self, *args, content_manager=None, **kw):
--- a/Lib/email/mime/application.py
+++ b/Lib/email/mime/application.py
@@ -17,7 +17,7 @@ class MIMEApplication(MIMENonMultipart):
                 _encoder=encoders.encode_base64, *, policy=None, **_params):
        """Create an application/* type MIME document.
-        _data is a string containing the raw application data.
+        _data contains the bytes for the raw application data.
        _subtype is the MIME content type subtype, defaulting to
        'octet-stream'.
--- a/Lib/email/mime/audio.py
+++ b/Lib/email/mime/audio.py
@@ -6,39 +6,10 @@
 __all__ = ['MIMEAudio']
 import sndhdr
 from io import BytesIO
 from email import encoders
 from email.mime.nonmultipart import MIMENonMultipart
 _sndhdr_MIMEmap = {'au'  : 'basic',
                   'wav' :'x-wav',
                   'aiff':'x-aiff',
                   'aifc':'x-aiff',
                   }
 # There are others in sndhdr that don't have MIME types. :(
 # Additional ones to be added to sndhdr? midi, mp3, realaudio, wma??
 def _whatsnd(data):
    """Try to identify a sound file type.
    sndhdr.what() has a pretty cruddy interface, unfortunately.  This is why
    we re-do it here.  It would be easier to reverse engineer the Unix 'file'
    command and use the standard 'magic' file, as shipped with a modern Unix.
    """
    hdr = data[:512]
    fakefile = BytesIO(hdr)
    for testfn in sndhdr.tests:
        res = testfn(hdr, fakefile)
        if res is not None:
            return _sndhdr_MIMEmap.get(res[0])
    return None
 class MIMEAudio(MIMENonMultipart):
    """Class for generating audio/* MIME documents."""
@@ -46,8 +17,8 @@ class MIMEAudio(MIMENonMultipart):
                 _encoder=encoders.encode_base64, *, policy=None, **_params):
        """Create an audio/* type MIME document.
-        _audiodata is a string containing the raw audio data.  If this data
+        _audiodata contains the bytes for the raw audio data.  If this data
-        can be decoded by the standard Python `sndhdr' module, then the
+        can be decoded as au, wav, aiff, or aifc, then the
        subtype will be automatically included in the Content-Type header.
        Otherwise, you can specify  the specific audio subtype via the
        _subtype parameter.  If _subtype is not given, and no subtype can be
@@ -65,10 +36,62 @@ class MIMEAudio(MIMENonMultipart):
        header.
        """
        if _subtype is None:
-            _subtype = _whatsnd(_audiodata)
+            _subtype = _what(_audiodata)
        if _subtype is None:
            raise TypeError('Could not find audio MIME subtype')
        MIMENonMultipart.__init__(self, 'audio', _subtype, policy=policy,
                                  **_params)
        self.set_payload(_audiodata)
        _encoder(self)
 _rules = []
 # Originally from the sndhdr module.
 #
 # There are others in sndhdr that don't have MIME types. :(
 # Additional ones to be added to sndhdr? midi, mp3, realaudio, wma??
 def _what(data):
    # Try to identify a sound file type.
    #
    # sndhdr.what() had a pretty cruddy interface, unfortunately.  This is why
    # we re-do it here.  It would be easier to reverse engineer the Unix 'file'
    # command and use the standard 'magic' file, as shipped with a modern Unix.
    for testfn in _rules:
        if res := testfn(data):
            return res
    else:
        return None
 def rule(rulefunc):
    _rules.append(rulefunc)
    return rulefunc
@rule
 def _aiff(h):
    if not h.startswith(b'FORM'):
        return None
    if h[8:12] in {b'AIFC', b'AIFF'}:
        return 'x-aiff'
    else:
        return None
@rule
 def _au(h):
    if h.startswith(b'.snd'):
        return 'basic'
    else:
        return None
@rule
 def _wav(h):
    # 'RIFF' <len> 'WAVE' 'fmt ' <len>
    if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ':
        return None
    else:
        return "x-wav"
--- a/Lib/email/mime/base.py
+++ b/Lib/email/mime/base.py
@@ -11,7 +11,6 @@ import email.policy
 from email import message
 class MIMEBase(message.Message):
    """Base class for MIME specializations."""
--- a/Lib/email/mime/image.py
+++ b/Lib/email/mime/image.py
@@ -6,13 +6,10 @@
 __all__ = ['MIMEImage']
 import imghdr
 from email import encoders
 from email.mime.nonmultipart import MIMENonMultipart
 class MIMEImage(MIMENonMultipart):
    """Class for generating image/* type MIME documents."""
@@ -20,11 +17,11 @@ class MIMEImage(MIMENonMultipart):
                 _encoder=encoders.encode_base64, *, policy=None, **_params):
        """Create an image/* type MIME document.
-        _imagedata is a string containing the raw image data.  If this data
+        _imagedata contains the bytes for the raw image data.  If the data
-        can be decoded by the standard Python `imghdr' module, then the
+        type can be detected (jpeg, png, gif, tiff, rgb, pbm, pgm, ppm,
-        subtype will be automatically included in the Content-Type header.
+        rast, xbm, bmp, webp, and exr attempted), then the subtype will be
-        Otherwise, you can specify the specific image subtype via the _subtype
+        automatically included in the Content-Type header. Otherwise, you can
-        parameter.
+        specify the specific image subtype via the _subtype parameter.
        _encoder is a function which will perform the actual encoding for
        transport of the image data.  It takes one argument, which is this
@@ -37,11 +34,119 @@ class MIMEImage(MIMENonMultipart):
        constructor, which turns them into parameters on the Content-Type
        header.
        """
-        if _subtype is None:
+        _subtype = _what(_imagedata) if _subtype is None else _subtype
            _subtype = imghdr.what(None, _imagedata)
        if _subtype is None:
            raise TypeError('Could not guess image MIME subtype')
        MIMENonMultipart.__init__(self, 'image', _subtype, policy=policy,
                                  **_params)
        self.set_payload(_imagedata)
        _encoder(self)
 _rules = []
 # Originally from the imghdr module.
 def _what(data):
    for rule in _rules:
        if res := rule(data):
            return res
    else:
        return None
 def rule(rulefunc):
    _rules.append(rulefunc)
    return rulefunc
@rule
 def _jpeg(h):
    """JPEG data with JFIF or Exif markers; and raw JPEG"""
    if h[6:10] in (b'JFIF', b'Exif'):
        return 'jpeg'
    elif h[:4] == b'\xff\xd8\xff\xdb':
        return 'jpeg'
@rule
 def _png(h):
    if h.startswith(b'\211PNG\r\n\032\n'):
        return 'png'
@rule
 def _gif(h):
    """GIF ('87 and '89 variants)"""
    if h[:6] in (b'GIF87a', b'GIF89a'):
        return 'gif'
@rule
 def _tiff(h):
    """TIFF (can be in Motorola or Intel byte order)"""
    if h[:2] in (b'MM', b'II'):
        return 'tiff'
@rule
 def _rgb(h):
    """SGI image library"""
    if h.startswith(b'\001\332'):
        return 'rgb'
@rule
 def _pbm(h):
    """PBM (portable bitmap)"""
    if len(h) >= 3 and \
            h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r':
        return 'pbm'
@rule
 def _pgm(h):
    """PGM (portable graymap)"""
    if len(h) >= 3 and \
            h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r':
        return 'pgm'
@rule
 def _ppm(h):
    """PPM (portable pixmap)"""
    if len(h) >= 3 and \
            h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r':
        return 'ppm'
@rule
 def _rast(h):
    """Sun raster file"""
    if h.startswith(b'\x59\xA6\x6A\x95'):
        return 'rast'
@rule
 def _xbm(h):
    """X bitmap (X10 or X11)"""
    if h.startswith(b'#define '):
        return 'xbm'
@rule
 def _bmp(h):
    if h.startswith(b'BM'):
        return 'bmp'
@rule
 def _webp(h):
    if h.startswith(b'RIFF') and h[8:12] == b'WEBP':
        return 'webp'
@rule
 def _exr(h):
    if h.startswith(b'\x76\x2f\x31\x01'):
        return 'exr'
--- a/Lib/email/mime/message.py
+++ b/Lib/email/mime/message.py
@@ -10,7 +10,6 @@ from email import message
 from email.mime.nonmultipart import MIMENonMultipart
 class MIMEMessage(MIMENonMultipart):
    """Class representing message/* MIME documents."""
--- a/Lib/email/mime/multipart.py
+++ b/Lib/email/mime/multipart.py
@@ -9,7 +9,6 @@ __all__ = ['MIMEMultipart']
 from email.mime.base import MIMEBase
 class MIMEMultipart(MIMEBase):
    """Base class for MIME multipart/* type messages."""
--- a/Lib/email/mime/nonmultipart.py
+++ b/Lib/email/mime/nonmultipart.py
@@ -10,7 +10,6 @@ from email import errors
 from email.mime.base import MIMEBase
 class MIMENonMultipart(MIMEBase):
    """Base class for MIME non-multipart type messages."""
--- a/Lib/email/mime/text.py
+++ b/Lib/email/mime/text.py
@@ -6,11 +6,9 @@
 __all__ = ['MIMEText']
 from email.charset import Charset
 from email.mime.nonmultipart import MIMENonMultipart
 class MIMEText(MIMENonMultipart):
    """Class for generating text/* type MIME documents."""
@@ -37,6 +35,6 @@ class MIMEText(MIMENonMultipart):
                _charset = 'utf-8'
        MIMENonMultipart.__init__(self, 'text', _subtype, policy=policy,
-                                  **{'charset': str(_charset)})
+                                  charset=str(_charset))
        self.set_payload(_text, _charset)
--- a/Lib/email/parser.py
+++ b/Lib/email/parser.py
@@ -13,7 +13,6 @@ from email.feedparser import FeedParser, BytesFeedParser
 from email._policybase import compat32
 class Parser:
    def __init__(self, _class=None, *, policy=compat32):
        """Parser of RFC 2822 and MIME email messages.
@@ -50,10 +49,7 @@ class Parser:
        feedparser = FeedParser(self._class, policy=self.policy)
        if headersonly:
            feedparser._set_headersonly()
-        while True:
+        while data := fp.read(8192):
            data = fp.read(8192)
            if not data:
                break
            feedparser.feed(data)
        return feedparser.close()
@@ -68,7 +64,6 @@ class Parser:
        return self.parse(StringIO(text), headersonly=headersonly)
 class HeaderParser(Parser):
    def parse(self, fp, headersonly=True):
        return Parser.parse(self, fp, True)
@@ -76,7 +71,7 @@ class HeaderParser(Parser):
    def parsestr(self, text, headersonly=True):
        return Parser.parsestr(self, text, True)
-
+
 class BytesParser:
    def __init__(self, *args, **kw):
--- a/Lib/email/policy.py
+++ b/Lib/email/policy.py
@@ -3,6 +3,7 @@ code that adds all the email6 features.
 """
 import re
 import sys
 from email._policybase import Policy, Compat32, compat32, _extend_docstrings
 from email.utils import _has_surrogates
 from email.headerregistry import HeaderRegistry as HeaderRegistry
@@ -20,7 +21,7 @@ __all__ = [
    'HTTP',
    ]
-linesep_splitter = re.compile(r'\n|\r')
+linesep_splitter = re.compile(r'\n|\r\n?')
@_extend_docstrings
 class EmailPolicy(Policy):
@@ -118,13 +119,13 @@ class EmailPolicy(Policy):
        """+
        The name is parsed as everything up to the ':' and returned unmodified.
        The value is determined by stripping leading whitespace off the
-        remainder of the first line, joining all subsequent lines together, and
+        remainder of the first line joined with all subsequent lines, and
        stripping any trailing carriage return or linefeed characters.  (This
        is the same as Compat32).
        """
        name, value = sourcelines[0].split(':', 1)
-        value = value.lstrip(' \t') + ''.join(sourcelines[1:])
+        value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n')
        return (name, value.rstrip('\r\n'))
    def header_store_parse(self, name, value):
@@ -203,14 +204,22 @@ class EmailPolicy(Policy):
    def _fold(self, name, value, refold_binary=False):
        if hasattr(value, 'name'):
            return value.fold(policy=self)
-        maxlen = self.max_line_length if self.max_line_length else float('inf')
+        maxlen = self.max_line_length if self.max_line_length else sys.maxsize
-        lines = value.splitlines()
+        # We can't use splitlines here because it splits on more than \r and \n.
        lines = linesep_splitter.split(value)
        refold = (self.refold_source == 'all' or
                  self.refold_source == 'long' and
                    (lines and len(lines[0])+len(name)+2 > maxlen or
                     any(len(x) > maxlen for x in lines[1:])))
-        if refold or refold_binary and _has_surrogates(value):
+
        if not refold:
            if not self.utf8:
                refold = not value.isascii()
            elif refold_binary:
                refold = _has_surrogates(value)
        if refold:
            return self.header_factory(name, ''.join(lines)).fold(policy=self)
        return name + ': ' + self.linesep.join(lines) + self.linesep
--- a/Lib/email/quoprimime.py
+++ b/Lib/email/quoprimime.py
@@ -148,6 +148,7 @@ def header_encode(header_bytes, charset='iso-8859-1'):
 _QUOPRI_BODY_ENCODE_MAP = _QUOPRI_BODY_MAP[:]
 for c in b'\r\n':
    _QUOPRI_BODY_ENCODE_MAP[c] = chr(c)
 del c
 def body_encode(body, maxlinelen=76, eol=NL):
    """Encode with quoted-printable, wrapping at maxlinelen characters.
@@ -173,7 +174,7 @@ def body_encode(body, maxlinelen=76, eol=NL):
    if not body:
        return body
-    # quote speacial characters
+    # quote special characters
    body = body.translate(_QUOPRI_BODY_ENCODE_MAP)
    soft_break = '=' + eol
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -25,8 +25,6 @@ __all__ = [
 import os
 import re
 import time
 import random
 import socket
 import datetime
 import urllib.parse
@@ -36,9 +34,6 @@ from email._parseaddr import mktime_tz
 from email._parseaddr import parsedate, parsedate_tz, _parsedate_tz
 # Intrapackage imports
 from email.charset import Charset
 COMMASPACE = ', '
 EMPTYSTRING = ''
 UEMPTYSTRING = ''
@@ -48,11 +43,12 @@ TICK = "'"
 specialsre = re.compile(r'[][\\()<>@,:;".]')
 escapesre = re.compile(r'[\\"]')
 def _has_surrogates(s):
-    """Return True if s contains surrogate-escaped binary data."""
+    """Return True if s may contain surrogate-escaped binary data."""
    # This check is based on the fact that unless there are surrogates, utf8
    # (Python's default encoding) can encode any string.  This is the fastest
-    # way to check for surrogates, see issue 11454 for timings.
+    # way to check for surrogates, see bpo-11454 (moved to gh-55663) for timings.
    try:
        s.encode()
        return False
@@ -81,7 +77,7 @@ def formataddr(pair, charset='utf-8'):
    If the first element of pair is false, then the second element is
    returned unmodified.
-    Optional charset if given is the character set that is used to encode
+    The optional charset is the character set that is used to encode
    realname in case realname is not ASCII safe.  Can be an instance of str or
    a Charset-like object which has a header_encode method.  Default is
    'utf-8'.
@@ -94,6 +90,8 @@ def formataddr(pair, charset='utf-8'):
            name.encode('ascii')
        except UnicodeEncodeError:
            if isinstance(charset, str):
                # lazy import to improve module import time
                from email.charset import Charset
                charset = Charset(charset)
            encoded_name = charset.header_encode(name)
            return "%s <%s>" % (encoded_name, address)
@@ -106,24 +104,127 @@ def formataddr(pair, charset='utf-8'):
    return address
-
+def _iter_escaped_chars(addr):
-def getaddresses(fieldvalues):
+    pos = 0
-    """Return a list of (REALNAME, EMAIL) for each fieldvalue."""
+    escape = False
-    all = COMMASPACE.join(fieldvalues)
+    for pos, ch in enumerate(addr):
-    a = _AddressList(all)
+        if escape:
-    return a.addresslist
+            yield (pos, '\\' + ch)
            escape = False
        elif ch == '\\':
            escape = True
        else:
            yield (pos, ch)
    if escape:
        yield (pos, '\\')
 def _strip_quoted_realnames(addr):
    """Strip real names between quotes."""
    if '"' not in addr:
        # Fast path
        return addr
-ecre = re.compile(r'''
+    start = 0
-  =\?                   # literal =?
+    open_pos = None
-  (?P<charset>[^?]*?)   # non-greedy up to the next ? is the charset
+    result = []
-  \?                    # literal ?
+    for pos, ch in _iter_escaped_chars(addr):
-  (?P<encoding>[qb])    # either a "q" or a "b", case insensitive
+        if ch == '"':
-  \?                    # literal ?
+            if open_pos is None:
-  (?P<atom>.*?)         # non-greedy up to the next ?= is the atom
+                open_pos = pos
-  \?=                   # literal ?=
+            else:
-  ''', re.VERBOSE | re.IGNORECASE)
+                if start != open_pos:
                    result.append(addr[start:open_pos])
                start = pos + 1
                open_pos = None
    if start < len(addr):
        result.append(addr[start:])
    return ''.join(result)
 supports_strict_parsing = True
 def getaddresses(fieldvalues, *, strict=True):
    """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
    When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
    its place.
    If strict is true, use a strict parser which rejects malformed inputs.
    """
    # If strict is true, if the resulting list of parsed addresses is greater
    # than the number of fieldvalues in the input list, a parsing error has
    # occurred and consequently a list containing a single empty 2-tuple [('',
    # '')] is returned in its place. This is done to avoid invalid output.
    #
    # Malformed input: getaddresses(['alice@example.com <bob@example.com>'])
    # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')]
    # Safe output: [('', '')]
    if not strict:
        all = COMMASPACE.join(str(v) for v in fieldvalues)
        a = _AddressList(all)
        return a.addresslist
    fieldvalues = [str(v) for v in fieldvalues]
    fieldvalues = _pre_parse_validation(fieldvalues)
    addr = COMMASPACE.join(fieldvalues)
    a = _AddressList(addr)
    result = _post_parse_validation(a.addresslist)
    # Treat output as invalid if the number of addresses is not equal to the
    # expected number of addresses.
    n = 0
    for v in fieldvalues:
        # When a comma is used in the Real Name part it is not a deliminator.
        # So strip those out before counting the commas.
        v = _strip_quoted_realnames(v)
        # Expected number of addresses: 1 + number of commas
        n += 1 + v.count(',')
    if len(result) != n:
        return [('', '')]
    return result
 def _check_parenthesis(addr):
    # Ignore parenthesis in quoted real names.
    addr = _strip_quoted_realnames(addr)
    opens = 0
    for pos, ch in _iter_escaped_chars(addr):
        if ch == '(':
            opens += 1
        elif ch == ')':
            opens -= 1
            if opens < 0:
                return False
    return (opens == 0)
 def _pre_parse_validation(email_header_fields):
    accepted_values = []
    for v in email_header_fields:
        if not _check_parenthesis(v):
            v = "('', '')"
        accepted_values.append(v)
    return accepted_values
 def _post_parse_validation(parsed_email_header_tuples):
    accepted_values = []
    # The parser would have parsed a correctly formatted domain-literal
    # The existence of an [ after parsing indicates a parsing failure
    for v in parsed_email_header_tuples:
        if '[' in v[1]:
            v = ('', '')
        accepted_values.append(v)
    return accepted_values
 def _format_timetuple_and_zone(timetuple, zone):
@@ -140,7 +241,7 @@ def formatdate(timeval=None, localtime=False, usegmt=False):
    Fri, 09 Nov 2001 01:08:47 -0000
-    Optional timeval if given is a floating point time value as accepted by
+    Optional timeval if given is a floating-point time value as accepted by
    gmtime() and localtime(), otherwise the current time is used.
    Optional localtime is a flag that when True, interprets timeval, and
@@ -155,13 +256,13 @@ def formatdate(timeval=None, localtime=False, usegmt=False):
    # 2822 requires that day and month names be the English abbreviations.
    if timeval is None:
        timeval = time.time()
-    if localtime or usegmt:
+    dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc)
-        dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc)
+
    else:
        dt = datetime.datetime.utcfromtimestamp(timeval)
    if localtime:
        dt = dt.astimezone()
        usegmt = False
    elif not usegmt:
        dt = dt.replace(tzinfo=None)
    return format_datetime(dt, usegmt)
 def format_datetime(dt, usegmt=False):
@@ -193,6 +294,11 @@ def make_msgid(idstring=None, domain=None):
    portion of the message id after the '@'.  It defaults to the locally
    defined hostname.
    """
    # Lazy imports to speedup module import time
    # (no other functions in email.utils need these modules)
    import random
    import socket
    timeval = int(time.time()*100)
    pid = os.getpid()
    randint = random.getrandbits(64)
@@ -207,17 +313,43 @@ def make_msgid(idstring=None, domain=None):
 def parsedate_to_datetime(data):
-    *dtuple, tz = _parsedate_tz(data)
+    parsed_date_tz = _parsedate_tz(data)
    if parsed_date_tz is None:
        raise ValueError('Invalid date value or format "%s"' % str(data))
    *dtuple, tz = parsed_date_tz
    if tz is None:
        return datetime.datetime(*dtuple[:6])
    return datetime.datetime(*dtuple[:6],
            tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
-def parseaddr(addr):
+def parseaddr(addr, *, strict=True):
-    addrs = _AddressList(addr).addresslist
+    """
-    if not addrs:
+    Parse addr into its constituent realname and email address parts.
-        return '', ''
+
    Return a tuple of realname and email address, unless the parse fails, in
    which case return a 2-tuple of ('', '').
    If strict is True, use a strict parser which rejects malformed inputs.
    """
    if not strict:
        addrs = _AddressList(addr).addresslist
        if not addrs:
            return ('', '')
        return addrs[0]
    if isinstance(addr, list):
        addr = addr[0]
    if not isinstance(addr, str):
        return ('', '')
    addr = _pre_parse_validation([addr])[0]
    addrs = _post_parse_validation(_AddressList(addr).addresslist)
    if not addrs or len(addrs) > 1:
        return ('', '')
    return addrs[0]
@@ -265,21 +397,13 @@ def decode_params(params):
    params is a sequence of 2-tuples containing (param name, string value).
    """
-    # Copy params so we don't mess with the original
+    new_params = [params[0]]
    params = params[:]
    new_params = []
    # Map parameter's name to a list of continuations.  The values are a
    # 3-tuple of the continuation number, the string value, and a flag
    # specifying whether a particular segment is %-encoded.
    rfc2231_params = {}
-    name, value = params.pop(0)
+    for name, value in params[1:]:
-    new_params.append((name, value))
+        encoded = name.endswith('*')
    while params:
        name, value = params.pop(0)
        if name.endswith('*'):
            encoded = True
        else:
            encoded = False
        value = unquote(value)
        mo = rfc2231_continuation.match(name)
        if mo:
@@ -342,41 +466,23 @@ def collapse_rfc2231_value(value, errors='replace',
 # better than not having it.
 #
-def localtime(dt=None, isdst=-1):
+def localtime(dt=None, isdst=None):
    """Return local time as an aware datetime object.
    If called without arguments, return current time.  Otherwise *dt*
    argument should be a datetime instance, and it is converted to the
    local time zone according to the system time zone database.  If *dt* is
    naive (that is, dt.tzinfo is None), it is assumed to be in local time.
-    In this case, a positive or zero value for *isdst* causes localtime to
+    The isdst parameter is ignored.
    presume initially that summer time (for example, Daylight Saving Time)
    is or is not (respectively) in effect for the specified time.  A
    negative value for *isdst* causes the localtime() function to attempt
    to divine whether summer time is in effect for the specified time.
    """
    if isdst is not None:
        import warnings
        warnings._deprecated(
            "The 'isdst' parameter to 'localtime'",
            message='{name} is deprecated and slated for removal in Python {remove}',
            remove=(3, 14),
            )
    if dt is None:
-        return datetime.datetime.now(datetime.timezone.utc).astimezone()
+        dt = datetime.datetime.now()
-    if dt.tzinfo is not None:
+    return dt.astimezone()
        return dt.astimezone()
    # We have a naive datetime.  Convert to a (localtime) timetuple and pass to
    # system mktime together with the isdst hint.  System mktime will return
    # seconds since epoch.
    tm = dt.timetuple()[:-1] + (isdst,)
    seconds = time.mktime(tm)
    localtm = time.localtime(seconds)
    try:
        delta = datetime.timedelta(seconds=localtm.tm_gmtoff)
        tz = datetime.timezone(delta, localtm.tm_zone)
    except AttributeError:
        # Compute UTC offset and compare with the value implied by tm_isdst.
        # If the values match, use the zone name implied by tm_isdst.
        delta = dt - datetime.datetime(*time.gmtime(seconds)[:6])
        dst = time.daylight and localtm.tm_isdst > 0
        gmtoff = -(time.altzone if dst else time.timezone)
        if delta == datetime.timedelta(seconds=gmtoff):
            tz = datetime.timezone(delta, time.tzname[dst])
        else:
            tz = datetime.timezone(delta)
    return dt.replace(tzinfo=tz)
--- a/Lib/fileinput.py
+++ b/Lib/fileinput.py
@@ -53,7 +53,7 @@ __getitem__() method which implements the sequence behavior.  The
 sequence must be accessed in strictly sequential order; sequence
 access and readline() cannot be mixed.
-Optional in-place filtering: if the keyword argument inplace=1 is
+Optional in-place filtering: if the keyword argument inplace=True is
 passed to input() or to the FileInput constructor, the file is moved
 to a backup file and standard output is directed to the input file.
 This makes it possible to write a filter that rewrites its input file
@@ -399,7 +399,7 @@ class FileInput:
 def hook_compressed(filename, mode, *, encoding=None, errors=None):
-    if encoding is None:  # EncodingWarning is emitted in FileInput() already.
+    if encoding is None and "b" not in mode:  # EncodingWarning is emitted in FileInput() already.
        encoding = "locale"
    ext = os.path.splitext(filename)[1]
    if ext == '.gz':
--- a/Lib/getpass.py
+++ b/Lib/getpass.py
@@ -18,7 +18,6 @@ import contextlib
 import io
 import os
 import sys
 import warnings
 __all__ = ["getpass","getuser","GetPassWarning"]
@@ -118,6 +117,7 @@ def win_getpass(prompt='Password: ', stream=None):
 def fallback_getpass(prompt='Password: ', stream=None):
    import warnings
    warnings.warn("Can not control echo on the terminal.", GetPassWarning,
                  stacklevel=2)
    if not stream:
@@ -156,7 +156,11 @@ def getuser():
    First try various environment variables, then the password
    database.  This works on Windows as long as USERNAME is set.
    Any failure to find a username raises OSError.
    .. versionchanged:: 3.13
        Previously, various exceptions beyond just :exc:`OSError`
        were raised.
    """
    for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'):
@@ -164,9 +168,12 @@ def getuser():
        if user:
            return user
-    # If this fails, the exception will "explain" why
+    try:
-    import pwd
+        import pwd
-    return pwd.getpwuid(os.getuid())[0]
+        return pwd.getpwuid(os.getuid())[0]
    except (ImportError, KeyError) as e:
        raise OSError('No username set in the environment') from e
 # Bind the name getpass to the appropriate function
 try:
--- a/Lib/graphlib.py
+++ b/Lib/graphlib.py
@@ -154,7 +154,7 @@ class TopologicalSorter:
        This method unblocks any successor of each node in *nodes* for being returned
        in the future by a call to "get_ready".
-        Raises :exec:`ValueError` if any node in *nodes* has already been marked as
+        Raises ValueError if any node in *nodes* has already been marked as
        processed by a previous call to this method, if a node was not added to the
        graph by using "add" or if called without calling "prepare" previously or if
        node has not yet been returned by "get_ready".
--- a/Lib/imghdr.py
+++ b/Lib/imghdr.py
@@ -1,175 +0,0 @@
 """Recognize image file formats based on their first few bytes."""
 from os import PathLike
 import warnings
 __all__ = ["what"]
 warnings._deprecated(__name__, remove=(3, 13))
 #-------------------------#
 # Recognize image headers #
 #-------------------------#
 def what(file, h=None):
    f = None
    try:
        if h is None:
            if isinstance(file, (str, PathLike)):
                f = open(file, 'rb')
                h = f.read(32)
            else:
                location = file.tell()
                h = file.read(32)
                file.seek(location)
        for tf in tests:
            res = tf(h, f)
            if res:
                return res
    finally:
        if f: f.close()
    return None
 #---------------------------------#
 # Subroutines per image file type #
 #---------------------------------#
 tests = []
 def test_jpeg(h, f):
    """JPEG data with JFIF or Exif markers; and raw JPEG"""
    if h[6:10] in (b'JFIF', b'Exif'):
        return 'jpeg'
    elif h[:4] == b'\xff\xd8\xff\xdb':
        return 'jpeg'
 tests.append(test_jpeg)
 def test_png(h, f):
    if h.startswith(b'\211PNG\r\n\032\n'):
        return 'png'
 tests.append(test_png)
 def test_gif(h, f):
    """GIF ('87 and '89 variants)"""
    if h[:6] in (b'GIF87a', b'GIF89a'):
        return 'gif'
 tests.append(test_gif)
 def test_tiff(h, f):
    """TIFF (can be in Motorola or Intel byte order)"""
    if h[:2] in (b'MM', b'II'):
        return 'tiff'
 tests.append(test_tiff)
 def test_rgb(h, f):
    """SGI image library"""
    if h.startswith(b'\001\332'):
        return 'rgb'
 tests.append(test_rgb)
 def test_pbm(h, f):
    """PBM (portable bitmap)"""
    if len(h) >= 3 and \
        h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r':
        return 'pbm'
 tests.append(test_pbm)
 def test_pgm(h, f):
    """PGM (portable graymap)"""
    if len(h) >= 3 and \
        h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r':
        return 'pgm'
 tests.append(test_pgm)
 def test_ppm(h, f):
    """PPM (portable pixmap)"""
    if len(h) >= 3 and \
        h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r':
        return 'ppm'
 tests.append(test_ppm)
 def test_rast(h, f):
    """Sun raster file"""
    if h.startswith(b'\x59\xA6\x6A\x95'):
        return 'rast'
 tests.append(test_rast)
 def test_xbm(h, f):
    """X bitmap (X10 or X11)"""
    if h.startswith(b'#define '):
        return 'xbm'
 tests.append(test_xbm)
 def test_bmp(h, f):
    if h.startswith(b'BM'):
        return 'bmp'
 tests.append(test_bmp)
 def test_webp(h, f):
    if h.startswith(b'RIFF') and h[8:12] == b'WEBP':
        return 'webp'
 tests.append(test_webp)
 def test_exr(h, f):
    if h.startswith(b'\x76\x2f\x31\x01'):
        return 'exr'
 tests.append(test_exr)
 #--------------------#
 # Small test program #
 #--------------------#
 def test():
    import sys
    recursive = 0
    if sys.argv[1:] and sys.argv[1] == '-r':
        del sys.argv[1:2]
        recursive = 1
    try:
        if sys.argv[1:]:
            testall(sys.argv[1:], recursive, 1)
        else:
            testall(['.'], recursive, 1)
    except KeyboardInterrupt:
        sys.stderr.write('\n[Interrupted]\n')
        sys.exit(1)
 def testall(list, recursive, toplevel):
    import sys
    import os
    for filename in list:
        if os.path.isdir(filename):
            print(filename + '/:', end=' ')
            if recursive or toplevel:
                print('recursing down:')
                import glob
                names = glob.glob(os.path.join(glob.escape(filename), '*'))
                testall(names, recursive, 0)
            else:
                print('*** directory (use -r) ***')
        else:
            print(filename + ':', end=' ')
            sys.stdout.flush()
            try:
                print(what(filename))
            except OSError:
                print('*** not found ***')
 if __name__ == '__main__':
    test()
--- a/Lib/imp.py
+++ b/Lib/imp.py
@@ -1,346 +0,0 @@
 """This module provides the components needed to build your own __import__
 function.  Undocumented functions are obsolete.
 In most cases it is preferred you consider using the importlib module's
 functionality over this module.
 """
 # (Probably) need to stay in _imp
 from _imp import (lock_held, acquire_lock, release_lock,
                  get_frozen_object, is_frozen_package,
                  init_frozen, is_builtin, is_frozen,
                  _fix_co_filename, _frozen_module_names)
 try:
    from _imp import create_dynamic
 except ImportError:
    # Platform doesn't support dynamic loading.
    create_dynamic = None
 from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name
 from importlib._bootstrap_external import SourcelessFileLoader
 from importlib import machinery
 from importlib import util
 import importlib
 import os
 import sys
 import tokenize
 import types
 import warnings
 warnings.warn("the imp module is deprecated in favour of importlib and slated "
              "for removal in Python 3.12; "
              "see the module's documentation for alternative uses",
              DeprecationWarning, stacklevel=2)
 # DEPRECATED
 SEARCH_ERROR = 0
 PY_SOURCE = 1
 PY_COMPILED = 2
 C_EXTENSION = 3
 PY_RESOURCE = 4
 PKG_DIRECTORY = 5
 C_BUILTIN = 6
 PY_FROZEN = 7
 PY_CODERESOURCE = 8
 IMP_HOOK = 9
 def new_module(name):
    """**DEPRECATED**
    Create a new module.
    The module is not entered into sys.modules.
    """
    return types.ModuleType(name)
 def get_magic():
    """**DEPRECATED**
    Return the magic number for .pyc files.
    """
    return util.MAGIC_NUMBER
 def get_tag():
    """Return the magic tag for .pyc files."""
    return sys.implementation.cache_tag
 def cache_from_source(path, debug_override=None):
    """**DEPRECATED**
    Given the path to a .py file, return the path to its .pyc file.
    The .py file does not need to exist; this simply returns the path to the
    .pyc file calculated as if the .py file were imported.
    If debug_override is not None, then it must be a boolean and is used in
    place of sys.flags.optimize.
    If sys.implementation.cache_tag is None then NotImplementedError is raised.
    """
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')
        return util.cache_from_source(path, debug_override)
 def source_from_cache(path):
    """**DEPRECATED**
    Given the path to a .pyc. file, return the path to its .py file.
    The .pyc file does not need to exist; this simply returns the path to
    the .py file calculated to correspond to the .pyc file.  If path does
    not conform to PEP 3147 format, ValueError will be raised. If
    sys.implementation.cache_tag is None then NotImplementedError is raised.
    """
    return util.source_from_cache(path)
 def get_suffixes():
    """**DEPRECATED**"""
    extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES]
    source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
    bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]
    return extensions + source + bytecode
 class NullImporter:
    """**DEPRECATED**
    Null import object.
    """
    def __init__(self, path):
        if path == '':
            raise ImportError('empty pathname', path='')
        elif os.path.isdir(path):
            raise ImportError('existing directory', path=path)
    def find_module(self, fullname):
        """Always returns None."""
        return None
 class _HackedGetData:
    """Compatibility support for 'file' arguments of various load_*()
    functions."""
    def __init__(self, fullname, path, file=None):
        super().__init__(fullname, path)
        self.file = file
    def get_data(self, path):
        """Gross hack to contort loader to deal w/ load_*()'s bad API."""
        if self.file and path == self.path:
            # The contract of get_data() requires us to return bytes. Reopen the
            # file in binary mode if needed.
            if not self.file.closed:
                file = self.file
                if 'b' not in file.mode:
                    file.close()
            if self.file.closed:
                self.file = file = open(self.path, 'rb')
            with file:
                return file.read()
        else:
            return super().get_data(path)
 class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader):
    """Compatibility support for implementing load_source()."""
 def load_source(name, pathname, file=None):
    loader = _LoadSourceCompatibility(name, pathname, file)
    spec = util.spec_from_file_location(name, pathname, loader=loader)
    if name in sys.modules:
        module = _exec(spec, sys.modules[name])
    else:
        module = _load(spec)
    # To allow reloading to potentially work, use a non-hacked loader which
    # won't rely on a now-closed file object.
    module.__loader__ = machinery.SourceFileLoader(name, pathname)
    module.__spec__.loader = module.__loader__
    return module
 class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader):
    """Compatibility support for implementing load_compiled()."""
 def load_compiled(name, pathname, file=None):
    """**DEPRECATED**"""
    loader = _LoadCompiledCompatibility(name, pathname, file)
    spec = util.spec_from_file_location(name, pathname, loader=loader)
    if name in sys.modules:
        module = _exec(spec, sys.modules[name])
    else:
        module = _load(spec)
    # To allow reloading to potentially work, use a non-hacked loader which
    # won't rely on a now-closed file object.
    module.__loader__ = SourcelessFileLoader(name, pathname)
    module.__spec__.loader = module.__loader__
    return module
 def load_package(name, path):
    """**DEPRECATED**"""
    if os.path.isdir(path):
        extensions = (machinery.SOURCE_SUFFIXES[:] +
                      machinery.BYTECODE_SUFFIXES[:])
        for extension in extensions:
            init_path = os.path.join(path, '__init__' + extension)
            if os.path.exists(init_path):
                path = init_path
                break
        else:
            raise ValueError('{!r} is not a package'.format(path))
    spec = util.spec_from_file_location(name, path,
                                        submodule_search_locations=[])
    if name in sys.modules:
        return _exec(spec, sys.modules[name])
    else:
        return _load(spec)
 def load_module(name, file, filename, details):
    """**DEPRECATED**
    Load a module, given information returned by find_module().
    The module name must include the full package name, if any.
    """
    suffix, mode, type_ = details
    if mode and (not mode.startswith('r') or '+' in mode):
        raise ValueError('invalid file open mode {!r}'.format(mode))
    elif file is None and type_ in {PY_SOURCE, PY_COMPILED}:
        msg = 'file object required for import (type code {})'.format(type_)
        raise ValueError(msg)
    elif type_ == PY_SOURCE:
        return load_source(name, filename, file)
    elif type_ == PY_COMPILED:
        return load_compiled(name, filename, file)
    elif type_ == C_EXTENSION and load_dynamic is not None:
        if file is None:
            with open(filename, 'rb') as opened_file:
                return load_dynamic(name, filename, opened_file)
        else:
            return load_dynamic(name, filename, file)
    elif type_ == PKG_DIRECTORY:
        return load_package(name, filename)
    elif type_ == C_BUILTIN:
        return init_builtin(name)
    elif type_ == PY_FROZEN:
        return init_frozen(name)
    else:
        msg =  "Don't know how to import {} (type code {})".format(name, type_)
        raise ImportError(msg, name=name)
 def find_module(name, path=None):
    """**DEPRECATED**
    Search for a module.
    If path is omitted or None, search for a built-in, frozen or special
    module and continue search in sys.path. The module name cannot
    contain '.'; to search for a submodule of a package, pass the
    submodule name and the package's __path__.
    """
    if not isinstance(name, str):
        raise TypeError("'name' must be a str, not {}".format(type(name)))
    elif not isinstance(path, (type(None), list)):
        # Backwards-compatibility
        raise RuntimeError("'path' must be None or a list, "
                           "not {}".format(type(path)))
    if path is None:
        if is_builtin(name):
            return None, None, ('', '', C_BUILTIN)
        elif is_frozen(name):
            return None, None, ('', '', PY_FROZEN)
        else:
            path = sys.path
    for entry in path:
        package_directory = os.path.join(entry, name)
        for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]:
            package_file_name = '__init__' + suffix
            file_path = os.path.join(package_directory, package_file_name)
            if os.path.isfile(file_path):
                return None, package_directory, ('', '', PKG_DIRECTORY)
        for suffix, mode, type_ in get_suffixes():
            file_name = name + suffix
            file_path = os.path.join(entry, file_name)
            if os.path.isfile(file_path):
                break
        else:
            continue
        break  # Break out of outer loop when breaking out of inner loop.
    else:
        raise ImportError(_ERR_MSG.format(name), name=name)
    encoding = None
    if 'b' not in mode:
        with open(file_path, 'rb') as file:
            encoding = tokenize.detect_encoding(file.readline)[0]
    file = open(file_path, mode, encoding=encoding)
    return file, file_path, (suffix, mode, type_)
 def reload(module):
    """**DEPRECATED**
    Reload the module and return it.
    The module must have been successfully imported before.
    """
    return importlib.reload(module)
 def init_builtin(name):
    """**DEPRECATED**
    Load and return a built-in module by name, or None is such module doesn't
    exist
    """
    try:
        return _builtin_from_name(name)
    except ImportError:
        return None
 if create_dynamic:
    def load_dynamic(name, path, file=None):
        """**DEPRECATED**
        Load an extension module.
        """
        import importlib.machinery
        loader = importlib.machinery.ExtensionFileLoader(name, path)
        # Issue #24748: Skip the sys.modules check in _load_module_shim;
        # always load new extension
        spec = importlib.machinery.ModuleSpec(
            name=name, loader=loader, origin=path)
        return _load(spec)
 else:
    load_dynamic = None
--- a/Lib/linecache.py
+++ b/Lib/linecache.py
@@ -5,17 +5,13 @@ is not found, it will look down the module search path for a file by
 that name.
 """
 import functools
 import sys
 import os
 import tokenize
 __all__ = ["getline", "clearcache", "checkcache", "lazycache"]
 # The cache. Maps filenames to either a thunk which will provide source code,
 # or a tuple (size, mtime, lines, fullname) once loaded.
 cache = {}
 _interactive_cache = {}
 def clearcache():
@@ -49,28 +45,54 @@ def getlines(filename, module_globals=None):
        return []
 def _getline_from_code(filename, lineno):
    lines = _getlines_from_code(filename)
    if 1 <= lineno <= len(lines):
        return lines[lineno - 1]
    return ''
 def _make_key(code):
    return (code.co_filename, code.co_qualname, code.co_firstlineno)
 def _getlines_from_code(code):
    code_id = _make_key(code)
    if code_id in _interactive_cache:
        entry = _interactive_cache[code_id]
        if len(entry) != 1:
            return _interactive_cache[code_id][2]
    return []
 def checkcache(filename=None):
    """Discard cache entries that are out of date.
    (This is not checked upon each call!)"""
    if filename is None:
-        filenames = list(cache.keys())
+        # get keys atomically
-    elif filename in cache:
+        filenames = cache.copy().keys()
        filenames = [filename]
    else:
-        return
+        filenames = [filename]
    for filename in filenames:
-        entry = cache[filename]
+        try:
            entry = cache[filename]
        except KeyError:
            continue
        if len(entry) == 1:
            # lazy cache entry, leave it lazy.
            continue
        size, mtime, lines, fullname = entry
        if mtime is None:
            continue   # no-op for files loaded via a __loader__
        try:
            # This import can fail if the interpreter is shutting down
            import os
        except ImportError:
            return
        try:
            stat = os.stat(fullname)
-        except OSError:
+        except (OSError, ValueError):
            cache.pop(filename, None)
            continue
        if size != stat.st_size or mtime != stat.st_mtime:
@@ -82,6 +104,17 @@ def updatecache(filename, module_globals=None):
    If something's wrong, print a message, discard the cache entry,
    and return an empty list."""
    # These imports are not at top level because linecache is in the critical
    # path of the interpreter startup and importing os and sys take a lot of time
    # and slows down the startup sequence.
    try:
        import os
        import sys
        import tokenize
    except ImportError:
        # These import can fail if the interpreter is shutting down
        return []
    if filename in cache:
        if len(cache[filename]) != 1:
            cache.pop(filename, None)
@@ -128,16 +161,20 @@ def updatecache(filename, module_globals=None):
            try:
                stat = os.stat(fullname)
                break
-            except OSError:
+            except (OSError, ValueError):
                pass
        else:
            return []
    except ValueError:  # may be raised by os.stat()
        return []
    try:
        with tokenize.open(fullname) as fp:
            lines = fp.readlines()
    except (OSError, UnicodeDecodeError, SyntaxError):
        return []
-    if lines and not lines[-1].endswith('\n'):
+    if not lines:
        lines = ['\n']
    elif not lines[-1].endswith('\n'):
        lines[-1] += '\n'
    size, mtime = stat.st_size, stat.st_mtime
    cache[filename] = size, mtime, lines, fullname
@@ -166,17 +203,29 @@ def lazycache(filename, module_globals):
        return False
    # Try for a __loader__, if available
    if module_globals and '__name__' in module_globals:
-        name = module_globals['__name__']
+        spec = module_globals.get('__spec__')
-        if (loader := module_globals.get('__loader__')) is None:
+        name = getattr(spec, 'name', None) or module_globals['__name__']
-            if spec := module_globals.get('__spec__'):
+        loader = getattr(spec, 'loader', None)
-                try:
+        if loader is None:
-                    loader = spec.loader
+            loader = module_globals.get('__loader__')
                except AttributeError:
                    pass
        get_source = getattr(loader, 'get_source', None)
        if name and get_source:
-            get_lines = functools.partial(get_source, name)
+            def get_lines(name=name, *args, **kwargs):
                return get_source(name, *args, **kwargs)
            cache[filename] = (get_lines,)
            return True
    return False
 def _register_code(code, string, name):
    entry = (len(string),
             None,
             [line + '\n' for line in string.splitlines()],
             name)
    stack = [code]
    while stack:
        code = stack.pop()
        for const in code.co_consts:
            if isinstance(const, type(code)):
                stack.append(const)
        _interactive_cache[_make_key(code)] = entry
--- a/Lib/lzma.py
+++ b/Lib/lzma.py
@@ -0,0 +1,364 @@
 """Interface to the liblzma compression library.
 This module provides a class for reading and writing compressed files,
 classes for incremental (de)compression, and convenience functions for
 one-shot (de)compression.
 These classes and functions support both the XZ and legacy LZMA
 container formats, as well as raw compressed data streams.
 """
 __all__ = [
    "CHECK_NONE", "CHECK_CRC32", "CHECK_CRC64", "CHECK_SHA256",
    "CHECK_ID_MAX", "CHECK_UNKNOWN",
    "FILTER_LZMA1", "FILTER_LZMA2", "FILTER_DELTA", "FILTER_X86", "FILTER_IA64",
    "FILTER_ARM", "FILTER_ARMTHUMB", "FILTER_POWERPC", "FILTER_SPARC",
    "FORMAT_AUTO", "FORMAT_XZ", "FORMAT_ALONE", "FORMAT_RAW",
    "MF_HC3", "MF_HC4", "MF_BT2", "MF_BT3", "MF_BT4",
    "MODE_FAST", "MODE_NORMAL", "PRESET_DEFAULT", "PRESET_EXTREME",
    "LZMACompressor", "LZMADecompressor", "LZMAFile", "LZMAError",
    "open", "compress", "decompress", "is_check_supported",
 ]
 import builtins
 import io
 import os
 from _lzma import *
 from _lzma import _encode_filter_properties, _decode_filter_properties
 import _compression
 # Value 0 no longer used
 _MODE_READ     = 1
 # Value 2 no longer used
 _MODE_WRITE    = 3
 class LZMAFile(_compression.BaseStream):
    """A file object providing transparent LZMA (de)compression.
    An LZMAFile can act as a wrapper for an existing file object, or
    refer directly to a named file on disk.
    Note that LZMAFile provides a *binary* file interface - data read
    is returned as bytes, and data to be written must be given as bytes.
    """
    def __init__(self, filename=None, mode="r", *,
                 format=None, check=-1, preset=None, filters=None):
        """Open an LZMA-compressed file in binary mode.
        filename can be either an actual file name (given as a str,
        bytes, or PathLike object), in which case the named file is
        opened, or it can be an existing file object to read from or
        write to.
        mode can be "r" for reading (default), "w" for (over)writing,
        "x" for creating exclusively, or "a" for appending. These can
        equivalently be given as "rb", "wb", "xb" and "ab" respectively.
        format specifies the container format to use for the file.
        If mode is "r", this defaults to FORMAT_AUTO. Otherwise, the
        default is FORMAT_XZ.
        check specifies the integrity check to use. This argument can
        only be used when opening a file for writing. For FORMAT_XZ,
        the default is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not
        support integrity checks - for these formats, check must be
        omitted, or be CHECK_NONE.
        When opening a file for reading, the *preset* argument is not
        meaningful, and should be omitted. The *filters* argument should
        also be omitted, except when format is FORMAT_RAW (in which case
        it is required).
        When opening a file for writing, the settings used by the
        compressor can be specified either as a preset compression
        level (with the *preset* argument), or in detail as a custom
        filter chain (with the *filters* argument). For FORMAT_XZ and
        FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset
        level. For FORMAT_RAW, the caller must always specify a filter
        chain; the raw compressor does not support preset compression
        levels.
        preset (if provided) should be an integer in the range 0-9,
        optionally OR-ed with the constant PRESET_EXTREME.
        filters (if provided) should be a sequence of dicts. Each dict
        should have an entry for "id" indicating ID of the filter, plus
        additional entries for options to the filter.
        """
        self._fp = None
        self._closefp = False
        self._mode = None
        if mode in ("r", "rb"):
            if check != -1:
                raise ValueError("Cannot specify an integrity check "
                                 "when opening a file for reading")
            if preset is not None:
                raise ValueError("Cannot specify a preset compression "
                                 "level when opening a file for reading")
            if format is None:
                format = FORMAT_AUTO
            mode_code = _MODE_READ
        elif mode in ("w", "wb", "a", "ab", "x", "xb"):
            if format is None:
                format = FORMAT_XZ
            mode_code = _MODE_WRITE
            self._compressor = LZMACompressor(format=format, check=check,
                                              preset=preset, filters=filters)
            self._pos = 0
        else:
            raise ValueError("Invalid mode: {!r}".format(mode))
        if isinstance(filename, (str, bytes, os.PathLike)):
            if "b" not in mode:
                mode += "b"
            self._fp = builtins.open(filename, mode)
            self._closefp = True
            self._mode = mode_code
        elif hasattr(filename, "read") or hasattr(filename, "write"):
            self._fp = filename
            self._mode = mode_code
        else:
            raise TypeError("filename must be a str, bytes, file or PathLike object")
        if self._mode == _MODE_READ:
            raw = _compression.DecompressReader(self._fp, LZMADecompressor,
                                                trailing_error=LZMAError, format=format, filters=filters)
            self._buffer = io.BufferedReader(raw)
    def close(self):
        """Flush and close the file.
        May be called more than once without error. Once the file is
        closed, any other operation on it will raise a ValueError.
        """
        if self.closed:
            return
        try:
            if self._mode == _MODE_READ:
                self._buffer.close()
                self._buffer = None
            elif self._mode == _MODE_WRITE:
                self._fp.write(self._compressor.flush())
                self._compressor = None
        finally:
            try:
                if self._closefp:
                    self._fp.close()
            finally:
                self._fp = None
                self._closefp = False
    @property
    def closed(self):
        """True if this file is closed."""
        return self._fp is None
    @property
    def name(self):
        self._check_not_closed()
        return self._fp.name
    @property
    def mode(self):
        return 'wb' if self._mode == _MODE_WRITE else 'rb'
    def fileno(self):
        """Return the file descriptor for the underlying file."""
        self._check_not_closed()
        return self._fp.fileno()
    def seekable(self):
        """Return whether the file supports seeking."""
        return self.readable() and self._buffer.seekable()
    def readable(self):
        """Return whether the file was opened for reading."""
        self._check_not_closed()
        return self._mode == _MODE_READ
    def writable(self):
        """Return whether the file was opened for writing."""
        self._check_not_closed()
        return self._mode == _MODE_WRITE
    def peek(self, size=-1):
        """Return buffered data without advancing the file position.
        Always returns at least one byte of data, unless at EOF.
        The exact number of bytes returned is unspecified.
        """
        self._check_can_read()
        # Relies on the undocumented fact that BufferedReader.peek() always
        # returns at least one byte (except at EOF)
        return self._buffer.peek(size)
    def read(self, size=-1):
        """Read up to size uncompressed bytes from the file.
        If size is negative or omitted, read until EOF is reached.
        Returns b"" if the file is already at EOF.
        """
        self._check_can_read()
        return self._buffer.read(size)
    def read1(self, size=-1):
        """Read up to size uncompressed bytes, while trying to avoid
        making multiple reads from the underlying stream. Reads up to a
        buffer's worth of data if size is negative.
        Returns b"" if the file is at EOF.
        """
        self._check_can_read()
        if size < 0:
            size = io.DEFAULT_BUFFER_SIZE
        return self._buffer.read1(size)
    def readline(self, size=-1):
        """Read a line of uncompressed bytes from the file.
        The terminating newline (if present) is retained. If size is
        non-negative, no more than size bytes will be read (in which
        case the line may be incomplete). Returns b'' if already at EOF.
        """
        self._check_can_read()
        return self._buffer.readline(size)
    def write(self, data):
        """Write a bytes object to the file.
        Returns the number of uncompressed bytes written, which is
        always the length of data in bytes. Note that due to buffering,
        the file on disk may not reflect the data written until close()
        is called.
        """
        self._check_can_write()
        if isinstance(data, (bytes, bytearray)):
            length = len(data)
        else:
            # accept any data that supports the buffer protocol
            data = memoryview(data)
            length = data.nbytes
        compressed = self._compressor.compress(data)
        self._fp.write(compressed)
        self._pos += length
        return length
    def seek(self, offset, whence=io.SEEK_SET):
        """Change the file position.
        The new position is specified by offset, relative to the
        position indicated by whence. Possible values for whence are:
            0: start of stream (default): offset must not be negative
            1: current stream position
            2: end of stream; offset must not be positive
        Returns the new file position.
        Note that seeking is emulated, so depending on the parameters,
        this operation may be extremely slow.
        """
        self._check_can_seek()
        return self._buffer.seek(offset, whence)
    def tell(self):
        """Return the current file position."""
        self._check_not_closed()
        if self._mode == _MODE_READ:
            return self._buffer.tell()
        return self._pos
 def open(filename, mode="rb", *,
         format=None, check=-1, preset=None, filters=None,
         encoding=None, errors=None, newline=None):
    """Open an LZMA-compressed file in binary or text mode.
    filename can be either an actual file name (given as a str, bytes,
    or PathLike object), in which case the named file is opened, or it
    can be an existing file object to read from or write to.
    The mode argument can be "r", "rb" (default), "w", "wb", "x", "xb",
    "a", or "ab" for binary mode, or "rt", "wt", "xt", or "at" for text
    mode.
    The format, check, preset and filters arguments specify the
    compression settings, as for LZMACompressor, LZMADecompressor and
    LZMAFile.
    For binary mode, this function is equivalent to the LZMAFile
    constructor: LZMAFile(filename, mode, ...). In this case, the
    encoding, errors and newline arguments must not be provided.
    For text mode, an LZMAFile object is created, and wrapped in an
    io.TextIOWrapper instance with the specified encoding, error
    handling behavior, and line ending(s).
    """
    if "t" in mode:
        if "b" in mode:
            raise ValueError("Invalid mode: %r" % (mode,))
    else:
        if encoding is not None:
            raise ValueError("Argument 'encoding' not supported in binary mode")
        if errors is not None:
            raise ValueError("Argument 'errors' not supported in binary mode")
        if newline is not None:
            raise ValueError("Argument 'newline' not supported in binary mode")
    lz_mode = mode.replace("t", "")
    binary_file = LZMAFile(filename, lz_mode, format=format, check=check,
                           preset=preset, filters=filters)
    if "t" in mode:
        encoding = io.text_encoding(encoding)
        return io.TextIOWrapper(binary_file, encoding, errors, newline)
    else:
        return binary_file
 def compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None):
    """Compress a block of data.
    Refer to LZMACompressor's docstring for a description of the
    optional arguments *format*, *check*, *preset* and *filters*.
    For incremental compression, use an LZMACompressor instead.
    """
    comp = LZMACompressor(format, check, preset, filters)
    return comp.compress(data) + comp.flush()
 def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None):
    """Decompress a block of data.
    Refer to LZMADecompressor's docstring for a description of the
    optional arguments *format*, *check* and *filters*.
    For incremental decompression, use an LZMADecompressor instead.
    """
    results = []
    while True:
        decomp = LZMADecompressor(format, memlimit, filters)
        try:
            res = decomp.decompress(data)
        except LZMAError:
            if results:
                break  # Leftover data is not a valid LZMA/XZ stream; ignore it.
            else:
                raise  # Error on the first iteration; bail out.
        results.append(res)
        if not decomp.eof:
            raise LZMAError("Compressed data ended before the "
                            "end-of-stream marker was reached")
        data = decomp.unused_data
        if not data:
            break
    return b"".join(results)
--- a/Lib/nntplib.py
+++ b/Lib/nntplib.py
--- a/Lib/pkgutil.py
+++ b/Lib/pkgutil.py
@@ -184,188 +184,6 @@ def _iter_file_finder_modules(importer, prefix=''):
 iter_importer_modules.register(
    importlib.machinery.FileFinder, _iter_file_finder_modules)
 def _import_imp():
    global imp
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', DeprecationWarning)
        imp = importlib.import_module('imp')
 class ImpImporter:
    """PEP 302 Finder that wraps Python's "classic" import algorithm
    ImpImporter(dirname) produces a PEP 302 finder that searches that
    directory.  ImpImporter(None) produces a PEP 302 finder that searches
    the current sys.path, plus any modules that are frozen or built-in.
    Note that ImpImporter does not currently support being used by placement
    on sys.meta_path.
    """
    def __init__(self, path=None):
        global imp
        warnings.warn("This emulation is deprecated and slated for removal "
                      "in Python 3.12; use 'importlib' instead",
             DeprecationWarning)
        _import_imp()
        self.path = path
    def find_module(self, fullname, path=None):
        # Note: we ignore 'path' argument since it is only used via meta_path
        subname = fullname.split(".")[-1]
        if subname != fullname and self.path is None:
            return None
        if self.path is None:
            path = None
        else:
            path = [os.path.realpath(self.path)]
        try:
            file, filename, etc = imp.find_module(subname, path)
        except ImportError:
            return None
        return ImpLoader(fullname, file, filename, etc)
    def iter_modules(self, prefix=''):
        if self.path is None or not os.path.isdir(self.path):
            return
        yielded = {}
        import inspect
        try:
            filenames = os.listdir(self.path)
        except OSError:
            # ignore unreadable directories like import does
            filenames = []
        filenames.sort()  # handle packages before same-named modules
        for fn in filenames:
            modname = inspect.getmodulename(fn)
            if modname=='__init__' or modname in yielded:
                continue
            path = os.path.join(self.path, fn)
            ispkg = False
            if not modname and os.path.isdir(path) and '.' not in fn:
                modname = fn
                try:
                    dircontents = os.listdir(path)
                except OSError:
                    # ignore unreadable directories like import does
                    dircontents = []
                for fn in dircontents:
                    subname = inspect.getmodulename(fn)
                    if subname=='__init__':
                        ispkg = True
                        break
                else:
                    continue    # not a package
            if modname and '.' not in modname:
                yielded[modname] = 1
                yield prefix + modname, ispkg
 class ImpLoader:
    """PEP 302 Loader that wraps Python's "classic" import algorithm
    """
    code = source = None
    def __init__(self, fullname, file, filename, etc):
        warnings.warn("This emulation is deprecated and slated for removal in "
                      "Python 3.12; use 'importlib' instead",
                      DeprecationWarning)
        _import_imp()
        self.file = file
        self.filename = filename
        self.fullname = fullname
        self.etc = etc
    def load_module(self, fullname):
        self._reopen()
        try:
            mod = imp.load_module(fullname, self.file, self.filename, self.etc)
        finally:
            if self.file:
                self.file.close()
        # Note: we don't set __loader__ because we want the module to look
        # normal; i.e. this is just a wrapper for standard import machinery
        return mod
    def get_data(self, pathname):
        with open(pathname, "rb") as file:
            return file.read()
    def _reopen(self):
        if self.file and self.file.closed:
            mod_type = self.etc[2]
            if mod_type==imp.PY_SOURCE:
                self.file = open(self.filename, 'r')
            elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
                self.file = open(self.filename, 'rb')
    def _fix_name(self, fullname):
        if fullname is None:
            fullname = self.fullname
        elif fullname != self.fullname:
            raise ImportError("Loader for module %s cannot handle "
                              "module %s" % (self.fullname, fullname))
        return fullname
    def is_package(self, fullname):
        fullname = self._fix_name(fullname)
        return self.etc[2]==imp.PKG_DIRECTORY
    def get_code(self, fullname=None):
        fullname = self._fix_name(fullname)
        if self.code is None:
            mod_type = self.etc[2]
            if mod_type==imp.PY_SOURCE:
                source = self.get_source(fullname)
                self.code = compile(source, self.filename, 'exec')
            elif mod_type==imp.PY_COMPILED:
                self._reopen()
                try:
                    self.code = read_code(self.file)
                finally:
                    self.file.close()
            elif mod_type==imp.PKG_DIRECTORY:
                self.code = self._get_delegate().get_code()
        return self.code
    def get_source(self, fullname=None):
        fullname = self._fix_name(fullname)
        if self.source is None:
            mod_type = self.etc[2]
            if mod_type==imp.PY_SOURCE:
                self._reopen()
                try:
                    self.source = self.file.read()
                finally:
                    self.file.close()
            elif mod_type==imp.PY_COMPILED:
                if os.path.exists(self.filename[:-1]):
                    with open(self.filename[:-1], 'r') as f:
                        self.source = f.read()
            elif mod_type==imp.PKG_DIRECTORY:
                self.source = self._get_delegate().get_source()
        return self.source
    def _get_delegate(self):
        finder = ImpImporter(self.filename)
        spec = _get_spec(finder, '__init__')
        return spec.loader
    def get_filename(self, fullname=None):
        fullname = self._fix_name(fullname)
        mod_type = self.etc[2]
        if mod_type==imp.PKG_DIRECTORY:
            return self._get_delegate().get_filename()
        elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
            return self.filename
        return None
 try:
    import zipimport
    from zipimport import zipimporter
--- a/Lib/pprint.py
+++ b/Lib/pprint.py
@@ -128,6 +128,9 @@ class PrettyPrinter:
        sort_dicts
            If true, dict keys are sorted.
        underscore_numbers
            If true, digit groups are separated with underscores.
        """
        indent = int(indent)
        width = int(width)
--- a/Lib/queue.py
+++ b/Lib/queue.py
@@ -10,7 +10,15 @@ try:
 except ImportError:
    SimpleQueue = None
-__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue']
+__all__ = [
    'Empty',
    'Full',
    'ShutDown',
    'Queue',
    'PriorityQueue',
    'LifoQueue',
    'SimpleQueue',
 ]
 try:
@@ -25,6 +33,10 @@ class Full(Exception):
    pass
 class ShutDown(Exception):
    '''Raised when put/get with shut-down queue.'''
 class Queue:
    '''Create a queue object with a given maximum size.
@@ -54,6 +66,9 @@ class Queue:
        self.all_tasks_done = threading.Condition(self.mutex)
        self.unfinished_tasks = 0
        # Queue shutdown state
        self.is_shutdown = False
    def task_done(self):
        '''Indicate that a formerly enqueued task is complete.
@@ -65,6 +80,9 @@ class Queue:
        have been processed (meaning that a task_done() call was received
        for every item that had been put() into the queue).
        shutdown(immediate=True) calls task_done() for each remaining item in
        the queue.
        Raises a ValueError if called more times than there were items
        placed in the queue.
        '''
@@ -129,8 +147,12 @@ class Queue:
        Otherwise ('block' is false), put an item on the queue if a free slot
        is immediately available, else raise the Full exception ('timeout'
        is ignored in that case).
        Raises ShutDown if the queue has been shut down.
        '''
        with self.not_full:
            if self.is_shutdown:
                raise ShutDown
            if self.maxsize > 0:
                if not block:
                    if self._qsize() >= self.maxsize:
@@ -138,6 +160,8 @@ class Queue:
                elif timeout is None:
                    while self._qsize() >= self.maxsize:
                        self.not_full.wait()
                        if self.is_shutdown:
                            raise ShutDown
                elif timeout < 0:
                    raise ValueError("'timeout' must be a non-negative number")
                else:
@@ -147,6 +171,8 @@ class Queue:
                        if remaining <= 0.0:
                            raise Full
                        self.not_full.wait(remaining)
                        if self.is_shutdown:
                            raise ShutDown
            self._put(item)
            self.unfinished_tasks += 1
            self.not_empty.notify()
@@ -161,14 +187,21 @@ class Queue:
        Otherwise ('block' is false), return an item if one is immediately
        available, else raise the Empty exception ('timeout' is ignored
        in that case).
        Raises ShutDown if the queue has been shut down and is empty,
        or if the queue has been shut down immediately.
        '''
        with self.not_empty:
            if self.is_shutdown and not self._qsize():
                raise ShutDown
            if not block:
                if not self._qsize():
                    raise Empty
            elif timeout is None:
                while not self._qsize():
                    self.not_empty.wait()
                    if self.is_shutdown and not self._qsize():
                        raise ShutDown
            elif timeout < 0:
                raise ValueError("'timeout' must be a non-negative number")
            else:
@@ -178,6 +211,8 @@ class Queue:
                    if remaining <= 0.0:
                        raise Empty
                    self.not_empty.wait(remaining)
                    if self.is_shutdown and not self._qsize():
                        raise ShutDown
            item = self._get()
            self.not_full.notify()
            return item
@@ -198,6 +233,29 @@ class Queue:
        '''
        return self.get(block=False)
    def shutdown(self, immediate=False):
        '''Shut-down the queue, making queue gets and puts raise ShutDown.
        By default, gets will only raise once the queue is empty. Set
        'immediate' to True to make gets raise immediately instead.
        All blocked callers of put() and get() will be unblocked. If
        'immediate', a task is marked as done for each item remaining in
        the queue, which may unblock callers of join().
        '''
        with self.mutex:
            self.is_shutdown = True
            if immediate:
                while self._qsize():
                    self._get()
                    if self.unfinished_tasks > 0:
                        self.unfinished_tasks -= 1
                # release all blocked threads in `join()`
                self.all_tasks_done.notify_all()
            # All getters need to re-check queue-empty to raise ShutDown
            self.not_empty.notify_all()
            self.not_full.notify_all()
    # Override these methods to implement other queue organizations
    # (e.g. stack or priority queue).
    # These will only be called with appropriate locks held
--- a/Lib/reprlib.py
+++ b/Lib/reprlib.py
@@ -29,49 +29,100 @@ def recursive_repr(fillvalue='...'):
        wrapper.__name__ = getattr(user_function, '__name__')
        wrapper.__qualname__ = getattr(user_function, '__qualname__')
        wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
        wrapper.__type_params__ = getattr(user_function, '__type_params__', ())
        wrapper.__wrapped__ = user_function
        return wrapper
    return decorating_function
 class Repr:
    _lookup = {
        'tuple': 'builtins',
        'list': 'builtins',
        'array': 'array',
        'set': 'builtins',
        'frozenset': 'builtins',
        'deque': 'collections',
        'dict': 'builtins',
        'str': 'builtins',
        'int': 'builtins'
    }
-    def __init__(self):
+    def __init__(
-        self.maxlevel = 6
+        self, *, maxlevel=6, maxtuple=6, maxlist=6, maxarray=5, maxdict=4,
-        self.maxtuple = 6
+        maxset=6, maxfrozenset=6, maxdeque=6, maxstring=30, maxlong=40,
-        self.maxlist = 6
+        maxother=30, fillvalue='...', indent=None,
-        self.maxarray = 5
+    ):
-        self.maxdict = 4
+        self.maxlevel = maxlevel
-        self.maxset = 6
+        self.maxtuple = maxtuple
-        self.maxfrozenset = 6
+        self.maxlist = maxlist
-        self.maxdeque = 6
+        self.maxarray = maxarray
-        self.maxstring = 30
+        self.maxdict = maxdict
-        self.maxlong = 40
+        self.maxset = maxset
-        self.maxother = 30
+        self.maxfrozenset = maxfrozenset
        self.maxdeque = maxdeque
        self.maxstring = maxstring
        self.maxlong = maxlong
        self.maxother = maxother
        self.fillvalue = fillvalue
        self.indent = indent
    def repr(self, x):
        return self.repr1(x, self.maxlevel)
    def repr1(self, x, level):
-        typename = type(x).__name__
+        cls = type(x)
        typename = cls.__name__
        if ' ' in typename:
            parts = typename.split()
            typename = '_'.join(parts)
-        if hasattr(self, 'repr_' + typename):
+
-            return getattr(self, 'repr_' + typename)(x, level)
+        method = getattr(self, 'repr_' + typename, None)
-        else:
+        if method:
-            return self.repr_instance(x, level)
+            # not defined in this class
            if typename not in self._lookup:
                return method(x, level)
            module = getattr(cls, '__module__', None)
            # defined in this class and is the module intended
            if module == self._lookup[typename]:
                return method(x, level)
        return self.repr_instance(x, level)
    def _join(self, pieces, level):
        if self.indent is None:
            return ', '.join(pieces)
        if not pieces:
            return ''
        indent = self.indent
        if isinstance(indent, int):
            if indent < 0:
                raise ValueError(
                    f'Repr.indent cannot be negative int (was {indent!r})'
                )
            indent *= ' '
        try:
            sep = ',\n' + (self.maxlevel - level + 1) * indent
        except TypeError as error:
            raise TypeError(
                f'Repr.indent must be a str, int or None, not {type(indent)}'
            ) from error
        return sep.join(('', *pieces, ''))[1:-len(indent) or None]
    def _repr_iterable(self, x, level, left, right, maxiter, trail=''):
        n = len(x)
        if level <= 0 and n:
-            s = '...'
+            s = self.fillvalue
        else:
            newlevel = level - 1
            repr1 = self.repr1
            pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)]
-            if n > maxiter:  pieces.append('...')
+            if n > maxiter:
-            s = ', '.join(pieces)
+                pieces.append(self.fillvalue)
-            if n == 1 and trail:  right = trail + right
+            s = self._join(pieces, level)
            if n == 1 and trail and self.indent is None:
                right = trail + right
        return '%s%s%s' % (left, s, right)
    def repr_tuple(self, x, level):
@@ -104,8 +155,10 @@ class Repr:
    def repr_dict(self, x, level):
        n = len(x)
-        if n == 0: return '{}'
+        if n == 0:
-        if level <= 0: return '{...}'
+            return '{}'
        if level <= 0:
            return '{' + self.fillvalue + '}'
        newlevel = level - 1
        repr1 = self.repr1
        pieces = []
@@ -113,8 +166,9 @@ class Repr:
            keyrepr = repr1(key, newlevel)
            valrepr = repr1(x[key], newlevel)
            pieces.append('%s: %s' % (keyrepr, valrepr))
-        if n > self.maxdict: pieces.append('...')
+        if n > self.maxdict:
-        s = ', '.join(pieces)
+            pieces.append(self.fillvalue)
        s = self._join(pieces, level)
        return '{%s}' % (s,)
    def repr_str(self, x, level):
@@ -123,7 +177,7 @@ class Repr:
            i = max(0, (self.maxstring-3)//2)
            j = max(0, self.maxstring-3-i)
            s = builtins.repr(x[:i] + x[len(x)-j:])
-            s = s[:i] + '...' + s[len(s)-j:]
+            s = s[:i] + self.fillvalue + s[len(s)-j:]
        return s
    def repr_int(self, x, level):
@@ -131,7 +185,7 @@ class Repr:
        if len(s) > self.maxlong:
            i = max(0, (self.maxlong-3)//2)
            j = max(0, self.maxlong-3-i)
-            s = s[:i] + '...' + s[len(s)-j:]
+            s = s[:i] + self.fillvalue + s[len(s)-j:]
        return s
    def repr_instance(self, x, level):
@@ -144,7 +198,7 @@ class Repr:
        if len(s) > self.maxother:
            i = max(0, (self.maxother-3)//2)
            j = max(0, self.maxother-3-i)
-            s = s[:i] + '...' + s[len(s)-j:]
+            s = s[:i] + self.fillvalue + s[len(s)-j:]
        return s
--- a/Lib/sched.py
+++ b/Lib/sched.py
@@ -11,7 +11,7 @@ substituting time and sleep from built-in module time, or you can
 implement simulated time by writing your own functions.  This can
 also be used to integrate scheduling with STDWIN events; the delay
 function is allowed to modify the queue.  Time can be expressed as
-integers or floating point numbers, as long as it is consistent.
+integers or floating-point numbers, as long as it is consistent.
 Events are specified by tuples (time, priority, action, argument, kwargs).
 As in UNIX, lower priority numbers mean higher priority; in this
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -679,5 +679,17 @@ def _script():
        print(textwrap.dedent(help % (sys.argv[0], os.pathsep)))
        sys.exit(10)
 def gethistoryfile():
    """Check if the PYTHON_HISTORY environment variable is set and define
    it as the .python_history file.  If PYTHON_HISTORY is not set, use the
    default .python_history file.
    """
    if not sys.flags.ignore_environment:
        history = os.environ.get("PYTHON_HISTORY")
        if history:
            return history
    return os.path.join(os.path.expanduser('~'),
        '.python_history')
 if __name__ == '__main__':
    _script()
--- a/Lib/smtpd.py
+++ b/Lib/smtpd.py
@@ -1,979 +0,0 @@
 #! /usr/bin/env python3
 """An RFC 5321 smtp proxy with optional RFC 1870 and RFC 6531 extensions.
 Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]]
 Options:
    --nosetuid
    -n
        This program generally tries to setuid `nobody', unless this flag is
        set.  The setuid call will fail if this program is not run as root (in
        which case, use this flag).
    --version
    -V
        Print the version number and exit.
    --class classname
    -c classname
        Use `classname' as the concrete SMTP proxy class.  Uses `PureProxy' by
        default.
    --size limit
    -s limit
        Restrict the total size of the incoming message to "limit" number of
        bytes via the RFC 1870 SIZE extension.  Defaults to 33554432 bytes.
    --smtputf8
    -u
        Enable the SMTPUTF8 extension and behave as an RFC 6531 smtp proxy.
    --debug
    -d
        Turn on debugging prints.
    --help
    -h
        Print this message and exit.
 Version: %(__version__)s
 If localhost is not given then `localhost' is used, and if localport is not
 given then 8025 is used.  If remotehost is not given then `localhost' is used,
 and if remoteport is not given, then 25 is used.
 """
 # Overview:
 #
 # This file implements the minimal SMTP protocol as defined in RFC 5321.  It
 # has a hierarchy of classes which implement the backend functionality for the
 # smtpd.  A number of classes are provided:
 #
 #   SMTPServer - the base class for the backend.  Raises NotImplementedError
 #   if you try to use it.
 #
 #   DebuggingServer - simply prints each message it receives on stdout.
 #
 #   PureProxy - Proxies all messages to a real smtpd which does final
 #   delivery.  One known problem with this class is that it doesn't handle
 #   SMTP errors from the backend server at all.  This should be fixed
 #   (contributions are welcome!).
 #
 #   MailmanProxy - An experimental hack to work with GNU Mailman
 #   <www.list.org>.  Using this server as your real incoming smtpd, your
 #   mailhost will automatically recognize and accept mail destined to Mailman
 #   lists when those lists are created.  Every message not destined for a list
 #   gets forwarded to a real backend smtpd, as with PureProxy.  Again, errors
 #   are not handled correctly yet.
 #
 #
 # Author: Barry Warsaw <barry@python.org>
 #
 # TODO:
 #
 # - support mailbox delivery
 # - alias files
 # - Handle more ESMTP extensions
 # - handle error codes from the backend smtpd
 import sys
 import os
 import errno
 import getopt
 import time
 import socket
 import collections
 from warnings import warn
 from email._header_value_parser import get_addr_spec, get_angle_addr
 __all__ = [
    "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy",
    "MailmanProxy",
 ]
 warn(
    'The smtpd module is deprecated and unmaintained and will be removed '
    'in Python 3.12.  Please see aiosmtpd '
    '(https://aiosmtpd.readthedocs.io/) for the recommended replacement.',
    DeprecationWarning,
    stacklevel=2)
 # These are imported after the above warning so that users get the correct
 # deprecation warning.
 import asyncore
 import asynchat
 program = sys.argv[0]
 __version__ = 'Python SMTP proxy version 0.3'
 class Devnull:
    def write(self, msg): pass
    def flush(self): pass
 DEBUGSTREAM = Devnull()
 NEWLINE = '\n'
 COMMASPACE = ', '
 DATA_SIZE_DEFAULT = 33554432
 def usage(code, msg=''):
    print(__doc__ % globals(), file=sys.stderr)
    if msg:
        print(msg, file=sys.stderr)
    sys.exit(code)
 class SMTPChannel(asynchat.async_chat):
    COMMAND = 0
    DATA = 1
    command_size_limit = 512
    command_size_limits = collections.defaultdict(lambda x=command_size_limit: x)
    @property
    def max_command_size_limit(self):
        try:
            return max(self.command_size_limits.values())
        except ValueError:
            return self.command_size_limit
    def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT,
                 map=None, enable_SMTPUTF8=False, decode_data=False):
        asynchat.async_chat.__init__(self, conn, map=map)
        self.smtp_server = server
        self.conn = conn
        self.addr = addr
        self.data_size_limit = data_size_limit
        self.enable_SMTPUTF8 = enable_SMTPUTF8
        self._decode_data = decode_data
        if enable_SMTPUTF8 and decode_data:
            raise ValueError("decode_data and enable_SMTPUTF8 cannot"
                             " be set to True at the same time")
        if decode_data:
            self._emptystring = ''
            self._linesep = '\r\n'
            self._dotsep = '.'
            self._newline = NEWLINE
        else:
            self._emptystring = b''
            self._linesep = b'\r\n'
            self._dotsep = ord(b'.')
            self._newline = b'\n'
        self._set_rset_state()
        self.seen_greeting = ''
        self.extended_smtp = False
        self.command_size_limits.clear()
        self.fqdn = socket.getfqdn()
        try:
            self.peer = conn.getpeername()
        except OSError as err:
            # a race condition  may occur if the other end is closing
            # before we can get the peername
            self.close()
            if err.errno != errno.ENOTCONN:
                raise
            return
        print('Peer:', repr(self.peer), file=DEBUGSTREAM)
        self.push('220 %s %s' % (self.fqdn, __version__))
    def _set_post_data_state(self):
        """Reset state variables to their post-DATA state."""
        self.smtp_state = self.COMMAND
        self.mailfrom = None
        self.rcpttos = []
        self.require_SMTPUTF8 = False
        self.num_bytes = 0
        self.set_terminator(b'\r\n')
    def _set_rset_state(self):
        """Reset all state variables except the greeting."""
        self._set_post_data_state()
        self.received_data = ''
        self.received_lines = []
    # properties for backwards-compatibility
    @property
    def __server(self):
        warn("Access to __server attribute on SMTPChannel is deprecated, "
            "use 'smtp_server' instead", DeprecationWarning, 2)
        return self.smtp_server
    @__server.setter
    def __server(self, value):
        warn("Setting __server attribute on SMTPChannel is deprecated, "
            "set 'smtp_server' instead", DeprecationWarning, 2)
        self.smtp_server = value
    @property
    def __line(self):
        warn("Access to __line attribute on SMTPChannel is deprecated, "
            "use 'received_lines' instead", DeprecationWarning, 2)
        return self.received_lines
    @__line.setter
    def __line(self, value):
        warn("Setting __line attribute on SMTPChannel is deprecated, "
            "set 'received_lines' instead", DeprecationWarning, 2)
        self.received_lines = value
    @property
    def __state(self):
        warn("Access to __state attribute on SMTPChannel is deprecated, "
            "use 'smtp_state' instead", DeprecationWarning, 2)
        return self.smtp_state
    @__state.setter
    def __state(self, value):
        warn("Setting __state attribute on SMTPChannel is deprecated, "
            "set 'smtp_state' instead", DeprecationWarning, 2)
        self.smtp_state = value
    @property
    def __greeting(self):
        warn("Access to __greeting attribute on SMTPChannel is deprecated, "
            "use 'seen_greeting' instead", DeprecationWarning, 2)
        return self.seen_greeting
    @__greeting.setter
    def __greeting(self, value):
        warn("Setting __greeting attribute on SMTPChannel is deprecated, "
            "set 'seen_greeting' instead", DeprecationWarning, 2)
        self.seen_greeting = value
    @property
    def __mailfrom(self):
        warn("Access to __mailfrom attribute on SMTPChannel is deprecated, "
            "use 'mailfrom' instead", DeprecationWarning, 2)
        return self.mailfrom
    @__mailfrom.setter
    def __mailfrom(self, value):
        warn("Setting __mailfrom attribute on SMTPChannel is deprecated, "
            "set 'mailfrom' instead", DeprecationWarning, 2)
        self.mailfrom = value
    @property
    def __rcpttos(self):
        warn("Access to __rcpttos attribute on SMTPChannel is deprecated, "
            "use 'rcpttos' instead", DeprecationWarning, 2)
        return self.rcpttos
    @__rcpttos.setter
    def __rcpttos(self, value):
        warn("Setting __rcpttos attribute on SMTPChannel is deprecated, "
            "set 'rcpttos' instead", DeprecationWarning, 2)
        self.rcpttos = value
    @property
    def __data(self):
        warn("Access to __data attribute on SMTPChannel is deprecated, "
            "use 'received_data' instead", DeprecationWarning, 2)
        return self.received_data
    @__data.setter
    def __data(self, value):
        warn("Setting __data attribute on SMTPChannel is deprecated, "
            "set 'received_data' instead", DeprecationWarning, 2)
        self.received_data = value
    @property
    def __fqdn(self):
        warn("Access to __fqdn attribute on SMTPChannel is deprecated, "
            "use 'fqdn' instead", DeprecationWarning, 2)
        return self.fqdn
    @__fqdn.setter
    def __fqdn(self, value):
        warn("Setting __fqdn attribute on SMTPChannel is deprecated, "
            "set 'fqdn' instead", DeprecationWarning, 2)
        self.fqdn = value
    @property
    def __peer(self):
        warn("Access to __peer attribute on SMTPChannel is deprecated, "
            "use 'peer' instead", DeprecationWarning, 2)
        return self.peer
    @__peer.setter
    def __peer(self, value):
        warn("Setting __peer attribute on SMTPChannel is deprecated, "
            "set 'peer' instead", DeprecationWarning, 2)
        self.peer = value
    @property
    def __conn(self):
        warn("Access to __conn attribute on SMTPChannel is deprecated, "
            "use 'conn' instead", DeprecationWarning, 2)
        return self.conn
    @__conn.setter
    def __conn(self, value):
        warn("Setting __conn attribute on SMTPChannel is deprecated, "
            "set 'conn' instead", DeprecationWarning, 2)
        self.conn = value
    @property
    def __addr(self):
        warn("Access to __addr attribute on SMTPChannel is deprecated, "
            "use 'addr' instead", DeprecationWarning, 2)
        return self.addr
    @__addr.setter
    def __addr(self, value):
        warn("Setting __addr attribute on SMTPChannel is deprecated, "
            "set 'addr' instead", DeprecationWarning, 2)
        self.addr = value
    # Overrides base class for convenience.
    def push(self, msg):
        asynchat.async_chat.push(self, bytes(
            msg + '\r\n', 'utf-8' if self.require_SMTPUTF8 else 'ascii'))
    # Implementation of base class abstract method
    def collect_incoming_data(self, data):
        limit = None
        if self.smtp_state == self.COMMAND:
            limit = self.max_command_size_limit
        elif self.smtp_state == self.DATA:
            limit = self.data_size_limit
        if limit and self.num_bytes > limit:
            return
        elif limit:
            self.num_bytes += len(data)
        if self._decode_data:
            self.received_lines.append(str(data, 'utf-8'))
        else:
            self.received_lines.append(data)
    # Implementation of base class abstract method
    def found_terminator(self):
        line = self._emptystring.join(self.received_lines)
        print('Data:', repr(line), file=DEBUGSTREAM)
        self.received_lines = []
        if self.smtp_state == self.COMMAND:
            sz, self.num_bytes = self.num_bytes, 0
            if not line:
                self.push('500 Error: bad syntax')
                return
            if not self._decode_data:
                line = str(line, 'utf-8')
            i = line.find(' ')
            if i < 0:
                command = line.upper()
                arg = None
            else:
                command = line[:i].upper()
                arg = line[i+1:].strip()
            max_sz = (self.command_size_limits[command]
                        if self.extended_smtp else self.command_size_limit)
            if sz > max_sz:
                self.push('500 Error: line too long')
                return
            method = getattr(self, 'smtp_' + command, None)
            if not method:
                self.push('500 Error: command "%s" not recognized' % command)
                return
            method(arg)
            return
        else:
            if self.smtp_state != self.DATA:
                self.push('451 Internal confusion')
                self.num_bytes = 0
                return
            if self.data_size_limit and self.num_bytes > self.data_size_limit:
                self.push('552 Error: Too much mail data')
                self.num_bytes = 0
                return
            # Remove extraneous carriage returns and de-transparency according
            # to RFC 5321, Section 4.5.2.
            data = []
            for text in line.split(self._linesep):
                if text and text[0] == self._dotsep:
                    data.append(text[1:])
                else:
                    data.append(text)
            self.received_data = self._newline.join(data)
            args = (self.peer, self.mailfrom, self.rcpttos, self.received_data)
            kwargs = {}
            if not self._decode_data:
                kwargs = {
                    'mail_options': self.mail_options,
                    'rcpt_options': self.rcpt_options,
                }
            status = self.smtp_server.process_message(*args, **kwargs)
            self._set_post_data_state()
            if not status:
                self.push('250 OK')
            else:
                self.push(status)
    # SMTP and ESMTP commands
    def smtp_HELO(self, arg):
        if not arg:
            self.push('501 Syntax: HELO hostname')
            return
        # See issue #21783 for a discussion of this behavior.
        if self.seen_greeting:
            self.push('503 Duplicate HELO/EHLO')
            return
        self._set_rset_state()
        self.seen_greeting = arg
        self.push('250 %s' % self.fqdn)
    def smtp_EHLO(self, arg):
        if not arg:
            self.push('501 Syntax: EHLO hostname')
            return
        # See issue #21783 for a discussion of this behavior.
        if self.seen_greeting:
            self.push('503 Duplicate HELO/EHLO')
            return
        self._set_rset_state()
        self.seen_greeting = arg
        self.extended_smtp = True
        self.push('250-%s' % self.fqdn)
        if self.data_size_limit:
            self.push('250-SIZE %s' % self.data_size_limit)
            self.command_size_limits['MAIL'] += 26
        if not self._decode_data:
            self.push('250-8BITMIME')
        if self.enable_SMTPUTF8:
            self.push('250-SMTPUTF8')
            self.command_size_limits['MAIL'] += 10
        self.push('250 HELP')
    def smtp_NOOP(self, arg):
        if arg:
            self.push('501 Syntax: NOOP')
        else:
            self.push('250 OK')
    def smtp_QUIT(self, arg):
        # args is ignored
        self.push('221 Bye')
        self.close_when_done()
    def _strip_command_keyword(self, keyword, arg):
        keylen = len(keyword)
        if arg[:keylen].upper() == keyword:
            return arg[keylen:].strip()
        return ''
    def _getaddr(self, arg):
        if not arg:
            return '', ''
        if arg.lstrip().startswith('<'):
            address, rest = get_angle_addr(arg)
        else:
            address, rest = get_addr_spec(arg)
        if not address:
            return address, rest
        return address.addr_spec, rest
    def _getparams(self, params):
        # Return params as dictionary. Return None if not all parameters
        # appear to be syntactically valid according to RFC 1869.
        result = {}
        for param in params:
            param, eq, value = param.partition('=')
            if not param.isalnum() or eq and not value:
                return None
            result[param] = value if eq else True
        return result
    def smtp_HELP(self, arg):
        if arg:
            extended = ' [SP <mail-parameters>]'
            lc_arg = arg.upper()
            if lc_arg == 'EHLO':
                self.push('250 Syntax: EHLO hostname')
            elif lc_arg == 'HELO':
                self.push('250 Syntax: HELO hostname')
            elif lc_arg == 'MAIL':
                msg = '250 Syntax: MAIL FROM: <address>'
                if self.extended_smtp:
                    msg += extended
                self.push(msg)
            elif lc_arg == 'RCPT':
                msg = '250 Syntax: RCPT TO: <address>'
                if self.extended_smtp:
                    msg += extended
                self.push(msg)
            elif lc_arg == 'DATA':
                self.push('250 Syntax: DATA')
            elif lc_arg == 'RSET':
                self.push('250 Syntax: RSET')
            elif lc_arg == 'NOOP':
                self.push('250 Syntax: NOOP')
            elif lc_arg == 'QUIT':
                self.push('250 Syntax: QUIT')
            elif lc_arg == 'VRFY':
                self.push('250 Syntax: VRFY <address>')
            else:
                self.push('501 Supported commands: EHLO HELO MAIL RCPT '
                          'DATA RSET NOOP QUIT VRFY')
        else:
            self.push('250 Supported commands: EHLO HELO MAIL RCPT DATA '
                      'RSET NOOP QUIT VRFY')
    def smtp_VRFY(self, arg):
        if arg:
            address, params = self._getaddr(arg)
            if address:
                self.push('252 Cannot VRFY user, but will accept message '
                          'and attempt delivery')
            else:
                self.push('502 Could not VRFY %s' % arg)
        else:
            self.push('501 Syntax: VRFY <address>')
    def smtp_MAIL(self, arg):
        if not self.seen_greeting:
            self.push('503 Error: send HELO first')
            return
        print('===> MAIL', arg, file=DEBUGSTREAM)
        syntaxerr = '501 Syntax: MAIL FROM: <address>'
        if self.extended_smtp:
            syntaxerr += ' [SP <mail-parameters>]'
        if arg is None:
            self.push(syntaxerr)
            return
        arg = self._strip_command_keyword('FROM:', arg)
        address, params = self._getaddr(arg)
        if not address:
            self.push(syntaxerr)
            return
        if not self.extended_smtp and params:
            self.push(syntaxerr)
            return
        if self.mailfrom:
            self.push('503 Error: nested MAIL command')
            return
        self.mail_options = params.upper().split()
        params = self._getparams(self.mail_options)
        if params is None:
            self.push(syntaxerr)
            return
        if not self._decode_data:
            body = params.pop('BODY', '7BIT')
            if body not in ['7BIT', '8BITMIME']:
                self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME')
                return
        if self.enable_SMTPUTF8:
            smtputf8 = params.pop('SMTPUTF8', False)
            if smtputf8 is True:
                self.require_SMTPUTF8 = True
            elif smtputf8 is not False:
                self.push('501 Error: SMTPUTF8 takes no arguments')
                return
        size = params.pop('SIZE', None)
        if size:
            if not size.isdigit():
                self.push(syntaxerr)
                return
            elif self.data_size_limit and int(size) > self.data_size_limit:
                self.push('552 Error: message size exceeds fixed maximum message size')
                return
        if len(params.keys()) > 0:
            self.push('555 MAIL FROM parameters not recognized or not implemented')
            return
        self.mailfrom = address
        print('sender:', self.mailfrom, file=DEBUGSTREAM)
        self.push('250 OK')
    def smtp_RCPT(self, arg):
        if not self.seen_greeting:
            self.push('503 Error: send HELO first');
            return
        print('===> RCPT', arg, file=DEBUGSTREAM)
        if not self.mailfrom:
            self.push('503 Error: need MAIL command')
            return
        syntaxerr = '501 Syntax: RCPT TO: <address>'
        if self.extended_smtp:
            syntaxerr += ' [SP <mail-parameters>]'
        if arg is None:
            self.push(syntaxerr)
            return
        arg = self._strip_command_keyword('TO:', arg)
        address, params = self._getaddr(arg)
        if not address:
            self.push(syntaxerr)
            return
        if not self.extended_smtp and params:
            self.push(syntaxerr)
            return
        self.rcpt_options = params.upper().split()
        params = self._getparams(self.rcpt_options)
        if params is None:
            self.push(syntaxerr)
            return
        # XXX currently there are no options we recognize.
        if len(params.keys()) > 0:
            self.push('555 RCPT TO parameters not recognized or not implemented')
            return
        self.rcpttos.append(address)
        print('recips:', self.rcpttos, file=DEBUGSTREAM)
        self.push('250 OK')
    def smtp_RSET(self, arg):
        if arg:
            self.push('501 Syntax: RSET')
            return
        self._set_rset_state()
        self.push('250 OK')
    def smtp_DATA(self, arg):
        if not self.seen_greeting:
            self.push('503 Error: send HELO first');
            return
        if not self.rcpttos:
            self.push('503 Error: need RCPT command')
            return
        if arg:
            self.push('501 Syntax: DATA')
            return
        self.smtp_state = self.DATA
        self.set_terminator(b'\r\n.\r\n')
        self.push('354 End data with <CR><LF>.<CR><LF>')
    # Commands that have not been implemented
    def smtp_EXPN(self, arg):
        self.push('502 EXPN not implemented')
 class SMTPServer(asyncore.dispatcher):
    # SMTPChannel class to use for managing client connections
    channel_class = SMTPChannel
    def __init__(self, localaddr, remoteaddr,
                 data_size_limit=DATA_SIZE_DEFAULT, map=None,
                 enable_SMTPUTF8=False, decode_data=False):
        self._localaddr = localaddr
        self._remoteaddr = remoteaddr
        self.data_size_limit = data_size_limit
        self.enable_SMTPUTF8 = enable_SMTPUTF8
        self._decode_data = decode_data
        if enable_SMTPUTF8 and decode_data:
            raise ValueError("decode_data and enable_SMTPUTF8 cannot"
                             " be set to True at the same time")
        asyncore.dispatcher.__init__(self, map=map)
        try:
            gai_results = socket.getaddrinfo(*localaddr,
                                             type=socket.SOCK_STREAM)
            self.create_socket(gai_results[0][0], gai_results[0][1])
            # try to re-use a server port if possible
            self.set_reuse_addr()
            self.bind(localaddr)
            self.listen(5)
        except:
            self.close()
            raise
        else:
            print('%s started at %s\n\tLocal addr: %s\n\tRemote addr:%s' % (
                self.__class__.__name__, time.ctime(time.time()),
                localaddr, remoteaddr), file=DEBUGSTREAM)
    def handle_accepted(self, conn, addr):
        print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM)
        channel = self.channel_class(self,
                                     conn,
                                     addr,
                                     self.data_size_limit,
                                     self._map,
                                     self.enable_SMTPUTF8,
                                     self._decode_data)
    # API for "doing something useful with the message"
    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
        """Override this abstract method to handle messages from the client.
        peer is a tuple containing (ipaddr, port) of the client that made the
        socket connection to our smtp port.
        mailfrom is the raw address the client claims the message is coming
        from.
        rcpttos is a list of raw addresses the client wishes to deliver the
        message to.
        data is a string containing the entire full text of the message,
        headers (if supplied) and all.  It has been `de-transparencied'
        according to RFC 821, Section 4.5.2.  In other words, a line
        containing a `.' followed by other text has had the leading dot
        removed.
        kwargs is a dictionary containing additional information.  It is
        empty if decode_data=True was given as init parameter, otherwise
        it will contain the following keys:
            'mail_options': list of parameters to the mail command.  All
                            elements are uppercase strings.  Example:
                            ['BODY=8BITMIME', 'SMTPUTF8'].
            'rcpt_options': same, for the rcpt command.
        This function should return None for a normal `250 Ok' response;
        otherwise, it should return the desired response string in RFC 821
        format.
        """
        raise NotImplementedError
 class DebuggingServer(SMTPServer):
    def _print_message_content(self, peer, data):
        inheaders = 1
        lines = data.splitlines()
        for line in lines:
            # headers first
            if inheaders and not line:
                peerheader = 'X-Peer: ' + peer[0]
                if not isinstance(data, str):
                    # decoded_data=false; make header match other binary output
                    peerheader = repr(peerheader.encode('utf-8'))
                print(peerheader)
                inheaders = 0
            if not isinstance(data, str):
                # Avoid spurious 'str on bytes instance' warning.
                line = repr(line)
            print(line)
    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
        print('---------- MESSAGE FOLLOWS ----------')
        if kwargs:
            if kwargs.get('mail_options'):
                print('mail options: %s' % kwargs['mail_options'])
            if kwargs.get('rcpt_options'):
                print('rcpt options: %s\n' % kwargs['rcpt_options'])
        self._print_message_content(peer, data)
        print('------------ END MESSAGE ------------')
 class PureProxy(SMTPServer):
    def __init__(self, *args, **kwargs):
        if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']:
            raise ValueError("PureProxy does not support SMTPUTF8.")
        super(PureProxy, self).__init__(*args, **kwargs)
    def process_message(self, peer, mailfrom, rcpttos, data):
        lines = data.split('\n')
        # Look for the last header
        i = 0
        for line in lines:
            if not line:
                break
            i += 1
        lines.insert(i, 'X-Peer: %s' % peer[0])
        data = NEWLINE.join(lines)
        refused = self._deliver(mailfrom, rcpttos, data)
        # TBD: what to do with refused addresses?
        print('we got some refusals:', refused, file=DEBUGSTREAM)
    def _deliver(self, mailfrom, rcpttos, data):
        import smtplib
        refused = {}
        try:
            s = smtplib.SMTP()
            s.connect(self._remoteaddr[0], self._remoteaddr[1])
            try:
                refused = s.sendmail(mailfrom, rcpttos, data)
            finally:
                s.quit()
        except smtplib.SMTPRecipientsRefused as e:
            print('got SMTPRecipientsRefused', file=DEBUGSTREAM)
            refused = e.recipients
        except (OSError, smtplib.SMTPException) as e:
            print('got', e.__class__, file=DEBUGSTREAM)
            # All recipients were refused.  If the exception had an associated
            # error code, use it.  Otherwise,fake it with a non-triggering
            # exception code.
            errcode = getattr(e, 'smtp_code', -1)
            errmsg = getattr(e, 'smtp_error', 'ignore')
            for r in rcpttos:
                refused[r] = (errcode, errmsg)
        return refused
 class MailmanProxy(PureProxy):
    def __init__(self, *args, **kwargs):
        warn('MailmanProxy is deprecated and will be removed '
             'in future', DeprecationWarning, 2)
        if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']:
            raise ValueError("MailmanProxy does not support SMTPUTF8.")
        super(PureProxy, self).__init__(*args, **kwargs)
    def process_message(self, peer, mailfrom, rcpttos, data):
        from io import StringIO
        from Mailman import Utils
        from Mailman import Message
        from Mailman import MailList
        # If the message is to a Mailman mailing list, then we'll invoke the
        # Mailman script directly, without going through the real smtpd.
        # Otherwise we'll forward it to the local proxy for disposition.
        listnames = []
        for rcpt in rcpttos:
            local = rcpt.lower().split('@')[0]
            # We allow the following variations on the theme
            #   listname
            #   listname-admin
            #   listname-owner
            #   listname-request
            #   listname-join
            #   listname-leave
            parts = local.split('-')
            if len(parts) > 2:
                continue
            listname = parts[0]
            if len(parts) == 2:
                command = parts[1]
            else:
                command = ''
            if not Utils.list_exists(listname) or command not in (
                    '', 'admin', 'owner', 'request', 'join', 'leave'):
                continue
            listnames.append((rcpt, listname, command))
        # Remove all list recipients from rcpttos and forward what we're not
        # going to take care of ourselves.  Linear removal should be fine
        # since we don't expect a large number of recipients.
        for rcpt, listname, command in listnames:
            rcpttos.remove(rcpt)
        # If there's any non-list destined recipients left,
        print('forwarding recips:', ' '.join(rcpttos), file=DEBUGSTREAM)
        if rcpttos:
            refused = self._deliver(mailfrom, rcpttos, data)
            # TBD: what to do with refused addresses?
            print('we got refusals:', refused, file=DEBUGSTREAM)
        # Now deliver directly to the list commands
        mlists = {}
        s = StringIO(data)
        msg = Message.Message(s)
        # These headers are required for the proper execution of Mailman.  All
        # MTAs in existence seem to add these if the original message doesn't
        # have them.
        if not msg.get('from'):
            msg['From'] = mailfrom
        if not msg.get('date'):
            msg['Date'] = time.ctime(time.time())
        for rcpt, listname, command in listnames:
            print('sending message to', rcpt, file=DEBUGSTREAM)
            mlist = mlists.get(listname)
            if not mlist:
                mlist = MailList.MailList(listname, lock=0)
                mlists[listname] = mlist
            # dispatch on the type of command
            if command == '':
                # post
                msg.Enqueue(mlist, tolist=1)
            elif command == 'admin':
                msg.Enqueue(mlist, toadmin=1)
            elif command == 'owner':
                msg.Enqueue(mlist, toowner=1)
            elif command == 'request':
                msg.Enqueue(mlist, torequest=1)
            elif command in ('join', 'leave'):
                # TBD: this is a hack!
                if command == 'join':
                    msg['Subject'] = 'subscribe'
                else:
                    msg['Subject'] = 'unsubscribe'
                msg.Enqueue(mlist, torequest=1)
 class Options:
    setuid = True
    classname = 'PureProxy'
    size_limit = None
    enable_SMTPUTF8 = False
 def parseargs():
    global DEBUGSTREAM
    try:
        opts, args = getopt.getopt(
            sys.argv[1:], 'nVhc:s:du',
            ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug',
             'smtputf8'])
    except getopt.error as e:
        usage(1, e)
    options = Options()
    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage(0)
        elif opt in ('-V', '--version'):
            print(__version__)
            sys.exit(0)
        elif opt in ('-n', '--nosetuid'):
            options.setuid = False
        elif opt in ('-c', '--class'):
            options.classname = arg
        elif opt in ('-d', '--debug'):
            DEBUGSTREAM = sys.stderr
        elif opt in ('-u', '--smtputf8'):
            options.enable_SMTPUTF8 = True
        elif opt in ('-s', '--size'):
            try:
                int_size = int(arg)
                options.size_limit = int_size
            except:
                print('Invalid size: ' + arg, file=sys.stderr)
                sys.exit(1)
    # parse the rest of the arguments
    if len(args) < 1:
        localspec = 'localhost:8025'
        remotespec = 'localhost:25'
    elif len(args) < 2:
        localspec = args[0]
        remotespec = 'localhost:25'
    elif len(args) < 3:
        localspec = args[0]
        remotespec = args[1]
    else:
        usage(1, 'Invalid arguments: %s' % COMMASPACE.join(args))
    # split into host/port pairs
    i = localspec.find(':')
    if i < 0:
        usage(1, 'Bad local spec: %s' % localspec)
    options.localhost = localspec[:i]
    try:
        options.localport = int(localspec[i+1:])
    except ValueError:
        usage(1, 'Bad local port: %s' % localspec)
    i = remotespec.find(':')
    if i < 0:
        usage(1, 'Bad remote spec: %s' % remotespec)
    options.remotehost = remotespec[:i]
    try:
        options.remoteport = int(remotespec[i+1:])
    except ValueError:
        usage(1, 'Bad remote port: %s' % remotespec)
    return options
 if __name__ == '__main__':
    options = parseargs()
    # Become nobody
    classname = options.classname
    if "." in classname:
        lastdot = classname.rfind(".")
        mod = __import__(classname[:lastdot], globals(), locals(), [""])
        classname = classname[lastdot+1:]
    else:
        import __main__ as mod
    class_ = getattr(mod, classname)
    proxy = class_((options.localhost, options.localport),
                   (options.remotehost, options.remoteport),
                   options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8)
    if options.setuid:
        try:
            import pwd
        except ImportError:
            print('Cannot import module "pwd"; try running with -n option.', file=sys.stderr)
            sys.exit(1)
        nobody = pwd.getpwnam('nobody')[2]
        try:
            os.setuid(nobody)
        except PermissionError:
            print('Cannot setuid "nobody"; try running with -n option.', file=sys.stderr)
            sys.exit(1)
    try:
        asyncore.loop()
    except KeyboardInterrupt:
        pass
--- a/Lib/sndhdr.py
+++ b/Lib/sndhdr.py
@@ -1,257 +0,0 @@
 """Routines to help recognizing sound files.
 Function whathdr() recognizes various types of sound file headers.
 It understands almost all headers that SOX can decode.
 The return tuple contains the following items, in this order:
 - file type (as SOX understands it)
 - sampling rate (0 if unknown or hard to decode)
 - number of channels (0 if unknown or hard to decode)
 - number of frames in the file (-1 if unknown or hard to decode)
 - number of bits/sample, or 'U' for U-LAW, or 'A' for A-LAW
 If the file doesn't have a recognizable type, it returns None.
 If the file can't be opened, OSError is raised.
 To compute the total time, divide the number of frames by the
 sampling rate (a frame contains a sample for each channel).
 Function what() calls whathdr().  (It used to also use some
 heuristics for raw data, but this doesn't work very well.)
 Finally, the function test() is a simple main program that calls
 what() for all files mentioned on the argument list.  For directory
 arguments it calls what() for all files in that directory.  Default
 argument is "." (testing all files in the current directory).  The
 option -r tells it to recurse down directories found inside
 explicitly given directories.
 """
 # The file structure is top-down except that the test program and its
 # subroutine come last.
 __all__ = ['what', 'whathdr']
 from collections import namedtuple
 SndHeaders = namedtuple('SndHeaders',
                        'filetype framerate nchannels nframes sampwidth')
 SndHeaders.filetype.__doc__ = ("""The value for type indicates the data type
 and will be one of the strings 'aifc', 'aiff', 'au','hcom',
 'sndr', 'sndt', 'voc', 'wav', '8svx', 'sb', 'ub', or 'ul'.""")
 SndHeaders.framerate.__doc__ = ("""The sampling_rate will be either the actual
 value or 0 if unknown or difficult to decode.""")
 SndHeaders.nchannels.__doc__ = ("""The number of channels or 0 if it cannot be
 determined or if the value is difficult to decode.""")
 SndHeaders.nframes.__doc__ = ("""The value for frames will be either the number
 of frames or -1.""")
 SndHeaders.sampwidth.__doc__ = ("""Either the sample size in bits or
 'A' for A-LAW or 'U' for u-LAW.""")
 def what(filename):
    """Guess the type of a sound file."""
    res = whathdr(filename)
    return res
 def whathdr(filename):
    """Recognize sound headers."""
    with open(filename, 'rb') as f:
        h = f.read(512)
        for tf in tests:
            res = tf(h, f)
            if res:
                return SndHeaders(*res)
        return None
 #-----------------------------------#
 # Subroutines per sound header type #
 #-----------------------------------#
 tests = []
 def test_aifc(h, f):
    import aifc
    if not h.startswith(b'FORM'):
        return None
    if h[8:12] == b'AIFC':
        fmt = 'aifc'
    elif h[8:12] == b'AIFF':
        fmt = 'aiff'
    else:
        return None
    f.seek(0)
    try:
        a = aifc.open(f, 'r')
    except (EOFError, aifc.Error):
        return None
    return (fmt, a.getframerate(), a.getnchannels(),
            a.getnframes(), 8 * a.getsampwidth())
 tests.append(test_aifc)
 def test_au(h, f):
    if h.startswith(b'.snd'):
        func = get_long_be
    elif h[:4] in (b'\0ds.', b'dns.'):
        func = get_long_le
    else:
        return None
    filetype = 'au'
    hdr_size = func(h[4:8])
    data_size = func(h[8:12])
    encoding = func(h[12:16])
    rate = func(h[16:20])
    nchannels = func(h[20:24])
    sample_size = 1 # default
    if encoding == 1:
        sample_bits = 'U'
    elif encoding == 2:
        sample_bits = 8
    elif encoding == 3:
        sample_bits = 16
        sample_size = 2
    else:
        sample_bits = '?'
    frame_size = sample_size * nchannels
    if frame_size:
        nframe = data_size / frame_size
    else:
        nframe = -1
    return filetype, rate, nchannels, nframe, sample_bits
 tests.append(test_au)
 def test_hcom(h, f):
    if h[65:69] != b'FSSD' or h[128:132] != b'HCOM':
        return None
    divisor = get_long_be(h[144:148])
    if divisor:
        rate = 22050 / divisor
    else:
        rate = 0
    return 'hcom', rate, 1, -1, 8
 tests.append(test_hcom)
 def test_voc(h, f):
    if not h.startswith(b'Creative Voice File\032'):
        return None
    sbseek = get_short_le(h[20:22])
    rate = 0
    if 0 <= sbseek < 500 and h[sbseek] == 1:
        ratecode = 256 - h[sbseek+4]
        if ratecode:
            rate = int(1000000.0 / ratecode)
    return 'voc', rate, 1, -1, 8
 tests.append(test_voc)
 def test_wav(h, f):
    import wave
    # 'RIFF' <len> 'WAVE' 'fmt ' <len>
    if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ':
        return None
    f.seek(0)
    try:
        w = wave.open(f, 'r')
    except (EOFError, wave.Error):
        return None
    return ('wav', w.getframerate(), w.getnchannels(),
                   w.getnframes(), 8*w.getsampwidth())
 tests.append(test_wav)
 def test_8svx(h, f):
    if not h.startswith(b'FORM') or h[8:12] != b'8SVX':
        return None
    # Should decode it to get #channels -- assume always 1
    return '8svx', 0, 1, 0, 8
 tests.append(test_8svx)
 def test_sndt(h, f):
    if h.startswith(b'SOUND'):
        nsamples = get_long_le(h[8:12])
        rate = get_short_le(h[20:22])
        return 'sndt', rate, 1, nsamples, 8
 tests.append(test_sndt)
 def test_sndr(h, f):
    if h.startswith(b'\0\0'):
        rate = get_short_le(h[2:4])
        if 4000 <= rate <= 25000:
            return 'sndr', rate, 1, -1, 8
 tests.append(test_sndr)
 #-------------------------------------------#
 # Subroutines to extract numbers from bytes #
 #-------------------------------------------#
 def get_long_be(b):
    return (b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]
 def get_long_le(b):
    return (b[3] << 24) | (b[2] << 16) | (b[1] << 8) | b[0]
 def get_short_be(b):
    return (b[0] << 8) | b[1]
 def get_short_le(b):
    return (b[1] << 8) | b[0]
 #--------------------#
 # Small test program #
 #--------------------#
 def test():
    import sys
    recursive = 0
    if sys.argv[1:] and sys.argv[1] == '-r':
        del sys.argv[1:2]
        recursive = 1
    try:
        if sys.argv[1:]:
            testall(sys.argv[1:], recursive, 1)
        else:
            testall(['.'], recursive, 1)
    except KeyboardInterrupt:
        sys.stderr.write('\n[Interrupted]\n')
        sys.exit(1)
 def testall(list, recursive, toplevel):
    import sys
    import os
    for filename in list:
        if os.path.isdir(filename):
            print(filename + '/:', end=' ')
            if recursive or toplevel:
                print('recursing down:')
                import glob
                names = glob.glob(os.path.join(filename, '*'))
                testall(names, recursive, 0)
            else:
                print('*** directory (use -r) ***')
        else:
            print(filename + ':', end=' ')
            sys.stdout.flush()
            try:
                print(what(filename))
            except OSError:
                print('*** not found ***')
 if __name__ == '__main__':
    test()
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -13,7 +13,7 @@ socket() -- create a new socket object
 socketpair() -- create a pair of new socket objects [*]
 fromfd() -- create a socket object from an open file descriptor [*]
 send_fds() -- Send file descriptor to the socket.
-recv_fds() -- Recieve file descriptors from the socket.
+recv_fds() -- Receive file descriptors from the socket.
 fromshare() -- create a socket object from data received from socket.share() [*]
 gethostname() -- return the current hostname
 gethostbyname() -- map a hostname to its IP number
@@ -28,6 +28,7 @@ socket.getdefaulttimeout() -- get the default timeout value
 socket.setdefaulttimeout() -- set the default timeout value
 create_connection() -- connects to an address, with an optional timeout and
                       optional source address.
 create_server() -- create a TCP socket and bind it to a specified address.
 [*] not available on all platforms!
@@ -122,7 +123,7 @@ if sys.platform.lower().startswith("win"):
    errorTab[10014] = "A fault occurred on the network??"  # WSAEFAULT
    errorTab[10022] = "An invalid operation was attempted."
    errorTab[10024] = "Too many open files."
-    errorTab[10035] = "The socket operation would block"
+    errorTab[10035] = "The socket operation would block."
    errorTab[10036] = "A blocking operation is already in progress."
    errorTab[10037] = "Operation already in progress."
    errorTab[10038] = "Socket operation on nonsocket."
@@ -254,17 +255,18 @@ class socket(_socket.socket):
               self.type,
               self.proto)
        if not closed:
            # getsockname and getpeername may not be available on WASI.
            try:
                laddr = self.getsockname()
                if laddr:
                    s += ", laddr=%s" % str(laddr)
-            except error:
+            except (error, AttributeError):
                pass
            try:
                raddr = self.getpeername()
                if raddr:
                    s += ", raddr=%s" % str(raddr)
-            except error:
+            except (error, AttributeError):
                pass
        s += '>'
        return s
@@ -380,7 +382,7 @@ class socket(_socket.socket):
                    if timeout and not selector_select(timeout):
                        raise TimeoutError('timed out')
                    if count:
-                        blocksize = count - total_sent
+                        blocksize = min(count - total_sent, blocksize)
                        if blocksize <= 0:
                            break
                    try:
@@ -783,11 +785,11 @@ def getfqdn(name=''):
    First the hostname returned by gethostbyaddr() is checked, then
    possibly existing aliases. In case no FQDN is available and `name`
-    was given, it is returned unchanged. If `name` was empty or '0.0.0.0',
+    was given, it is returned unchanged. If `name` was empty, '0.0.0.0' or '::',
    hostname from gethostname() is returned.
    """
    name = name.strip()
-    if not name or name == '0.0.0.0':
+    if not name or name in ('0.0.0.0', '::'):
        name = gethostname()
    try:
        hostname, aliases, ipaddrs = gethostbyaddr(name)
@@ -806,7 +808,7 @@ def getfqdn(name=''):
 _GLOBAL_DEFAULT_TIMEOUT = object()
 def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
-                      source_address=None):
+                      source_address=None, *, all_errors=False):
    """Connect to *address* and return the socket object.
    Convenience function.  Connect to *address* (a 2-tuple ``(host,
@@ -816,11 +818,13 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
    global default timeout setting returned by :func:`getdefaulttimeout`
    is used.  If *source_address* is set it must be a tuple of (host, port)
    for the socket to bind as a source address before making the connection.
-    A host of '' or port 0 tells the OS to use the default.
+    A host of '' or port 0 tells the OS to use the default. When a connection
    cannot be created, raises the last error if *all_errors* is False,
    and an ExceptionGroup of all errors if *all_errors* is True.
    """
    host, port = address
-    err = None
+    exceptions = []
    for res in getaddrinfo(host, port, 0, SOCK_STREAM):
        af, socktype, proto, canonname, sa = res
        sock = None
@@ -832,20 +836,24 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
                sock.bind(source_address)
            sock.connect(sa)
            # Break explicitly a reference cycle
-            err = None
+            exceptions.clear()
            return sock
-        except error as _:
+        except error as exc:
-            err = _
+            if not all_errors:
                exceptions.clear()  # raise only the last error
            exceptions.append(exc)
            if sock is not None:
                sock.close()
-    if err is not None:
+    if len(exceptions):
        try:
-            raise err
+            if not all_errors:
                raise exceptions[0]
            raise ExceptionGroup("create_connection failed", exceptions)
        finally:
            # Break explicitly a reference cycle
-            err = None
+            exceptions.clear()
    else:
        raise error("getaddrinfo returns an empty list")
@@ -902,7 +910,7 @@ def create_server(address, *, family=AF_INET, backlog=None, reuse_port=False,
        # address, effectively preventing this one from accepting
        # connections. Also, it may set the process in a state where
        # it'll no longer respond to any signals or graceful kills.
-        # See: msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx
+        # See: https://learn.microsoft.com/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse
        if os.name not in ('nt', 'cygwin') and \
                hasattr(_socket, 'SO_REUSEADDR'):
            try:
--- a/Lib/sunau.py
+++ b/Lib/sunau.py
@@ -1,531 +0,0 @@
 """Stuff to parse Sun and NeXT audio files.
 An audio file consists of a header followed by the data.  The structure
 of the header is as follows.
        +---------------+
        | magic word    |
        +---------------+
        | header size   |
        +---------------+
        | data size     |
        +---------------+
        | encoding      |
        +---------------+
        | sample rate   |
        +---------------+
        | # of channels |
        +---------------+
        | info          |
        |               |
        +---------------+
 The magic word consists of the 4 characters '.snd'.  Apart from the
 info field, all header fields are 4 bytes in size.  They are all
 32-bit unsigned integers encoded in big-endian byte order.
 The header size really gives the start of the data.
 The data size is the physical size of the data.  From the other
 parameters the number of frames can be calculated.
 The encoding gives the way in which audio samples are encoded.
 Possible values are listed below.
 The info field currently consists of an ASCII string giving a
 human-readable description of the audio file.  The info field is
 padded with NUL bytes to the header size.
 Usage.
 Reading audio files:
        f = sunau.open(file, 'r')
 where file is either the name of a file or an open file pointer.
 The open file pointer must have methods read(), seek(), and close().
 When the setpos() and rewind() methods are not used, the seek()
 method is not  necessary.
 This returns an instance of a class with the following public methods:
        getnchannels()  -- returns number of audio channels (1 for
                           mono, 2 for stereo)
        getsampwidth()  -- returns sample width in bytes
        getframerate()  -- returns sampling frequency
        getnframes()    -- returns number of audio frames
        getcomptype()   -- returns compression type ('NONE' or 'ULAW')
        getcompname()   -- returns human-readable version of
                           compression type ('not compressed' matches 'NONE')
        getparams()     -- returns a namedtuple consisting of all of the
                           above in the above order
        getmarkers()    -- returns None (for compatibility with the
                           aifc module)
        getmark(id)     -- raises an error since the mark does not
                           exist (for compatibility with the aifc module)
        readframes(n)   -- returns at most n frames of audio
        rewind()        -- rewind to the beginning of the audio stream
        setpos(pos)     -- seek to the specified position
        tell()          -- return the current position
        close()         -- close the instance (make it unusable)
 The position returned by tell() and the position given to setpos()
 are compatible and have nothing to do with the actual position in the
 file.
 The close() method is called automatically when the class instance
 is destroyed.
 Writing audio files:
        f = sunau.open(file, 'w')
 where file is either the name of a file or an open file pointer.
 The open file pointer must have methods write(), tell(), seek(), and
 close().
 This returns an instance of a class with the following public methods:
        setnchannels(n) -- set the number of channels
        setsampwidth(n) -- set the sample width
        setframerate(n) -- set the frame rate
        setnframes(n)   -- set the number of frames
        setcomptype(type, name)
                        -- set the compression type and the
                           human-readable compression type
        setparams(tuple)-- set all parameters at once
        tell()          -- return current position in output file
        writeframesraw(data)
                        -- write audio frames without pathing up the
                           file header
        writeframes(data)
                        -- write audio frames and patch up the file header
        close()         -- patch up the file header and close the
                           output file
 You should set the parameters before the first writeframesraw or
 writeframes.  The total number of frames does not need to be set,
 but when it is set to the correct value, the header does not have to
 be patched up.
 It is best to first set all parameters, perhaps possibly the
 compression type, and then write audio frames using writeframesraw.
 When all frames have been written, either call writeframes(b'') or
 close() to patch up the sizes in the header.
 The close() method is called automatically when the class instance
 is destroyed.
 """
 from collections import namedtuple
 import warnings
 _sunau_params = namedtuple('_sunau_params',
                           'nchannels sampwidth framerate nframes comptype compname')
 # from <multimedia/audio_filehdr.h>
 AUDIO_FILE_MAGIC = 0x2e736e64
 AUDIO_FILE_ENCODING_MULAW_8 = 1
 AUDIO_FILE_ENCODING_LINEAR_8 = 2
 AUDIO_FILE_ENCODING_LINEAR_16 = 3
 AUDIO_FILE_ENCODING_LINEAR_24 = 4
 AUDIO_FILE_ENCODING_LINEAR_32 = 5
 AUDIO_FILE_ENCODING_FLOAT = 6
 AUDIO_FILE_ENCODING_DOUBLE = 7
 AUDIO_FILE_ENCODING_ADPCM_G721 = 23
 AUDIO_FILE_ENCODING_ADPCM_G722 = 24
 AUDIO_FILE_ENCODING_ADPCM_G723_3 = 25
 AUDIO_FILE_ENCODING_ADPCM_G723_5 = 26
 AUDIO_FILE_ENCODING_ALAW_8 = 27
 # from <multimedia/audio_hdr.h>
 AUDIO_UNKNOWN_SIZE = 0xFFFFFFFF        # ((unsigned)(~0))
 _simple_encodings = [AUDIO_FILE_ENCODING_MULAW_8,
                     AUDIO_FILE_ENCODING_LINEAR_8,
                     AUDIO_FILE_ENCODING_LINEAR_16,
                     AUDIO_FILE_ENCODING_LINEAR_24,
                     AUDIO_FILE_ENCODING_LINEAR_32,
                     AUDIO_FILE_ENCODING_ALAW_8]
 class Error(Exception):
    pass
 def _read_u32(file):
    x = 0
    for i in range(4):
        byte = file.read(1)
        if not byte:
            raise EOFError
        x = x*256 + ord(byte)
    return x
 def _write_u32(file, x):
    data = []
    for i in range(4):
        d, m = divmod(x, 256)
        data.insert(0, int(m))
        x = d
    file.write(bytes(data))
 class Au_read:
    def __init__(self, f):
        if type(f) == type(''):
            import builtins
            f = builtins.open(f, 'rb')
            self._opened = True
        else:
            self._opened = False
        self.initfp(f)
    def __del__(self):
        if self._file:
            self.close()
    def __enter__(self):
        return self
    def __exit__(self, *args):
        self.close()
    def initfp(self, file):
        self._file = file
        self._soundpos = 0
        magic = int(_read_u32(file))
        if magic != AUDIO_FILE_MAGIC:
            raise Error('bad magic number')
        self._hdr_size = int(_read_u32(file))
        if self._hdr_size < 24:
            raise Error('header size too small')
        if self._hdr_size > 100:
            raise Error('header size ridiculously large')
        self._data_size = _read_u32(file)
        if self._data_size != AUDIO_UNKNOWN_SIZE:
            self._data_size = int(self._data_size)
        self._encoding = int(_read_u32(file))
        if self._encoding not in _simple_encodings:
            raise Error('encoding not (yet) supported')
        if self._encoding in (AUDIO_FILE_ENCODING_MULAW_8,
                  AUDIO_FILE_ENCODING_ALAW_8):
            self._sampwidth = 2
            self._framesize = 1
        elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_8:
            self._framesize = self._sampwidth = 1
        elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_16:
            self._framesize = self._sampwidth = 2
        elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_24:
            self._framesize = self._sampwidth = 3
        elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_32:
            self._framesize = self._sampwidth = 4
        else:
            raise Error('unknown encoding')
        self._framerate = int(_read_u32(file))
        self._nchannels = int(_read_u32(file))
        if not self._nchannels:
            raise Error('bad # of channels')
        self._framesize = self._framesize * self._nchannels
        if self._hdr_size > 24:
            self._info = file.read(self._hdr_size - 24)
            self._info, _, _ = self._info.partition(b'\0')
        else:
            self._info = b''
        try:
            self._data_pos = file.tell()
        except (AttributeError, OSError):
            self._data_pos = None
    def getfp(self):
        return self._file
    def getnchannels(self):
        return self._nchannels
    def getsampwidth(self):
        return self._sampwidth
    def getframerate(self):
        return self._framerate
    def getnframes(self):
        if self._data_size == AUDIO_UNKNOWN_SIZE:
            return AUDIO_UNKNOWN_SIZE
        if self._encoding in _simple_encodings:
            return self._data_size // self._framesize
        return 0                # XXX--must do some arithmetic here
    def getcomptype(self):
        if self._encoding == AUDIO_FILE_ENCODING_MULAW_8:
            return 'ULAW'
        elif self._encoding == AUDIO_FILE_ENCODING_ALAW_8:
            return 'ALAW'
        else:
            return 'NONE'
    def getcompname(self):
        if self._encoding == AUDIO_FILE_ENCODING_MULAW_8:
            return 'CCITT G.711 u-law'
        elif self._encoding == AUDIO_FILE_ENCODING_ALAW_8:
            return 'CCITT G.711 A-law'
        else:
            return 'not compressed'
    def getparams(self):
        return _sunau_params(self.getnchannels(), self.getsampwidth(),
                  self.getframerate(), self.getnframes(),
                  self.getcomptype(), self.getcompname())
    def getmarkers(self):
        return None
    def getmark(self, id):
        raise Error('no marks')
    def readframes(self, nframes):
        if self._encoding in _simple_encodings:
            if nframes == AUDIO_UNKNOWN_SIZE:
                data = self._file.read()
            else:
                data = self._file.read(nframes * self._framesize)
            self._soundpos += len(data) // self._framesize
            if self._encoding == AUDIO_FILE_ENCODING_MULAW_8:
                import audioop
                data = audioop.ulaw2lin(data, self._sampwidth)
            return data
        return None             # XXX--not implemented yet
    def rewind(self):
        if self._data_pos is None:
            raise OSError('cannot seek')
        self._file.seek(self._data_pos)
        self._soundpos = 0
    def tell(self):
        return self._soundpos
    def setpos(self, pos):
        if pos < 0 or pos > self.getnframes():
            raise Error('position not in range')
        if self._data_pos is None:
            raise OSError('cannot seek')
        self._file.seek(self._data_pos + pos * self._framesize)
        self._soundpos = pos
    def close(self):
        file = self._file
        if file:
            self._file = None
            if self._opened:
                file.close()
 class Au_write:
    def __init__(self, f):
        if type(f) == type(''):
            import builtins
            f = builtins.open(f, 'wb')
            self._opened = True
        else:
            self._opened = False
        self.initfp(f)
    def __del__(self):
        if self._file:
            self.close()
        self._file = None
    def __enter__(self):
        return self
    def __exit__(self, *args):
        self.close()
    def initfp(self, file):
        self._file = file
        self._framerate = 0
        self._nchannels = 0
        self._sampwidth = 0
        self._framesize = 0
        self._nframes = AUDIO_UNKNOWN_SIZE
        self._nframeswritten = 0
        self._datawritten = 0
        self._datalength = 0
        self._info = b''
        self._comptype = 'ULAW' # default is U-law
    def setnchannels(self, nchannels):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        if nchannels not in (1, 2, 4):
            raise Error('only 1, 2, or 4 channels supported')
        self._nchannels = nchannels
    def getnchannels(self):
        if not self._nchannels:
            raise Error('number of channels not set')
        return self._nchannels
    def setsampwidth(self, sampwidth):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        if sampwidth not in (1, 2, 3, 4):
            raise Error('bad sample width')
        self._sampwidth = sampwidth
    def getsampwidth(self):
        if not self._framerate:
            raise Error('sample width not specified')
        return self._sampwidth
    def setframerate(self, framerate):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        self._framerate = framerate
    def getframerate(self):
        if not self._framerate:
            raise Error('frame rate not set')
        return self._framerate
    def setnframes(self, nframes):
        if self._nframeswritten:
            raise Error('cannot change parameters after starting to write')
        if nframes < 0:
            raise Error('# of frames cannot be negative')
        self._nframes = nframes
    def getnframes(self):
        return self._nframeswritten
    def setcomptype(self, type, name):
        if type in ('NONE', 'ULAW'):
            self._comptype = type
        else:
            raise Error('unknown compression type')
    def getcomptype(self):
        return self._comptype
    def getcompname(self):
        if self._comptype == 'ULAW':
            return 'CCITT G.711 u-law'
        elif self._comptype == 'ALAW':
            return 'CCITT G.711 A-law'
        else:
            return 'not compressed'
    def setparams(self, params):
        nchannels, sampwidth, framerate, nframes, comptype, compname = params
        self.setnchannels(nchannels)
        self.setsampwidth(sampwidth)
        self.setframerate(framerate)
        self.setnframes(nframes)
        self.setcomptype(comptype, compname)
    def getparams(self):
        return _sunau_params(self.getnchannels(), self.getsampwidth(),
                  self.getframerate(), self.getnframes(),
                  self.getcomptype(), self.getcompname())
    def tell(self):
        return self._nframeswritten
    def writeframesraw(self, data):
        if not isinstance(data, (bytes, bytearray)):
            data = memoryview(data).cast('B')
        self._ensure_header_written()
        if self._comptype == 'ULAW':
            import audioop
            data = audioop.lin2ulaw(data, self._sampwidth)
        nframes = len(data) // self._framesize
        self._file.write(data)
        self._nframeswritten = self._nframeswritten + nframes
        self._datawritten = self._datawritten + len(data)
    def writeframes(self, data):
        self.writeframesraw(data)
        if self._nframeswritten != self._nframes or \
                  self._datalength != self._datawritten:
            self._patchheader()
    def close(self):
        if self._file:
            try:
                self._ensure_header_written()
                if self._nframeswritten != self._nframes or \
                        self._datalength != self._datawritten:
                    self._patchheader()
                self._file.flush()
            finally:
                file = self._file
                self._file = None
                if self._opened:
                    file.close()
    #
    # private methods
    #
    def _ensure_header_written(self):
        if not self._nframeswritten:
            if not self._nchannels:
                raise Error('# of channels not specified')
            if not self._sampwidth:
                raise Error('sample width not specified')
            if not self._framerate:
                raise Error('frame rate not specified')
            self._write_header()
    def _write_header(self):
        if self._comptype == 'NONE':
            if self._sampwidth == 1:
                encoding = AUDIO_FILE_ENCODING_LINEAR_8
                self._framesize = 1
            elif self._sampwidth == 2:
                encoding = AUDIO_FILE_ENCODING_LINEAR_16
                self._framesize = 2
            elif self._sampwidth == 3:
                encoding = AUDIO_FILE_ENCODING_LINEAR_24
                self._framesize = 3
            elif self._sampwidth == 4:
                encoding = AUDIO_FILE_ENCODING_LINEAR_32
                self._framesize = 4
            else:
                raise Error('internal error')
        elif self._comptype == 'ULAW':
            encoding = AUDIO_FILE_ENCODING_MULAW_8
            self._framesize = 1
        else:
            raise Error('internal error')
        self._framesize = self._framesize * self._nchannels
        _write_u32(self._file, AUDIO_FILE_MAGIC)
        header_size = 25 + len(self._info)
        header_size = (header_size + 7) & ~7
        _write_u32(self._file, header_size)
        if self._nframes == AUDIO_UNKNOWN_SIZE:
            length = AUDIO_UNKNOWN_SIZE
        else:
            length = self._nframes * self._framesize
        try:
            self._form_length_pos = self._file.tell()
        except (AttributeError, OSError):
            self._form_length_pos = None
        _write_u32(self._file, length)
        self._datalength = length
        _write_u32(self._file, encoding)
        _write_u32(self._file, self._framerate)
        _write_u32(self._file, self._nchannels)
        self._file.write(self._info)
        self._file.write(b'\0'*(header_size - len(self._info) - 24))
    def _patchheader(self):
        if self._form_length_pos is None:
            raise OSError('cannot seek')
        self._file.seek(self._form_length_pos)
        _write_u32(self._file, self._datawritten)
        self._datalength = self._datawritten
        self._file.seek(0, 2)
 def open(f, mode=None):
    if mode is None:
        if hasattr(f, 'mode'):
            mode = f.mode
        else:
            mode = 'rb'
    if mode in ('r', 'rb'):
        return Au_read(f)
    elif mode in ('w', 'wb'):
        return Au_write(f)
    else:
        raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
 def openfp(f, mode=None):
    warnings.warn("sunau.openfp is deprecated since Python 3.7. "
                  "Use sunau.open instead.", DeprecationWarning, stacklevel=2)
    return open(f, mode=mode)
--- a/Lib/tarfile.py
+++ b/Lib/tarfile.py
@@ -46,6 +46,7 @@ import time
 import struct
 import copy
 import re
 import warnings
 try:
    import pwd
@@ -57,19 +58,19 @@ except ImportError:
    grp = None
 # os.symlink on Windows prior to 6.0 raises NotImplementedError
-symlink_exception = (AttributeError, NotImplementedError)
+# OSError (winerror=1314) will be raised if the caller does not hold the
-try:
+# SeCreateSymbolicLinkPrivilege privilege
-    # OSError (winerror=1314) will be raised if the caller does not hold the
+symlink_exception = (AttributeError, NotImplementedError, OSError)
    # SeCreateSymbolicLinkPrivilege privilege
    symlink_exception += (OSError,)
 except NameError:
    pass
 # from tarfile import *
 __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError",
           "CompressionError", "StreamError", "ExtractError", "HeaderError",
           "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT",
-           "DEFAULT_FORMAT", "open"]
+           "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter",
           "tar_filter", "FilterError", "AbsoluteLinkError",
           "OutsideDestinationError", "SpecialFileError", "AbsolutePathError",
           "LinkOutsideDestinationError"]
 #---------------------------------------------------------
 # tar constants
@@ -158,6 +159,8 @@ else:
 def stn(s, length, encoding, errors):
    """Convert a string to a null-terminated bytes object.
    """
    if s is None:
        raise ValueError("metadata cannot contain None")
    s = s.encode(encoding, errors)
    return s[:length] + (length - len(s)) * NUL
@@ -328,15 +331,17 @@ class _LowLevelFile:
 class _Stream:
    """Class that serves as an adapter between TarFile and
       a stream-like object.  The stream-like object only
-       needs to have a read() or write() method and is accessed
+       needs to have a read() or write() method that works with bytes,
-       blockwise.  Use of gzip or bzip2 compression is possible.
+       and the method is accessed blockwise.
-       A stream-like object could be for example: sys.stdin,
+       Use of gzip or bzip2 compression is possible.
-       sys.stdout, a socket, a tape device etc.
+       A stream-like object could be for example: sys.stdin.buffer,
       sys.stdout.buffer, a socket, a tape device etc.
       _Stream is intended to be used only internally.
    """
-    def __init__(self, name, mode, comptype, fileobj, bufsize):
+    def __init__(self, name, mode, comptype, fileobj, bufsize,
                 compresslevel):
        """Construct a _Stream object.
        """
        self._extfileobj = True
@@ -368,10 +373,10 @@ class _Stream:
                self.zlib = zlib
                self.crc = zlib.crc32(b"")
                if mode == "r":
                    self._init_read_gz()
                    self.exception = zlib.error
                    self._init_read_gz()
                else:
-                    self._init_write_gz()
+                    self._init_write_gz(compresslevel)
            elif comptype == "bz2":
                try:
@@ -383,13 +388,17 @@ class _Stream:
                    self.cmp = bz2.BZ2Decompressor()
                    self.exception = OSError
                else:
-                    self.cmp = bz2.BZ2Compressor()
+                    self.cmp = bz2.BZ2Compressor(compresslevel)
            elif comptype == "xz":
                try:
                    import lzma
                except ImportError:
                    raise CompressionError("lzma module is not available") from None
                # XXX: RUSTPYTHON; xz is not supported yet
                raise CompressionError("lzma module is not available") from None
                if mode == "r":
                    self.dbuf = b""
                    self.cmp = lzma.LZMADecompressor()
@@ -410,13 +419,14 @@ class _Stream:
        if hasattr(self, "closed") and not self.closed:
            self.close()
-    def _init_write_gz(self):
+    def _init_write_gz(self, compresslevel):
        """Initialize for writing with gzip compression.
        """
-        self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED,
+        self.cmp = self.zlib.compressobj(compresslevel,
-                                            -self.zlib.MAX_WBITS,
+                                         self.zlib.DEFLATED,
-                                            self.zlib.DEF_MEM_LEVEL,
+                                         -self.zlib.MAX_WBITS,
-                                            0)
+                                         self.zlib.DEF_MEM_LEVEL,
                                         0)
        timestamp = struct.pack("<L", int(time.time()))
        self.__write(b"\037\213\010\010" + timestamp + b"\002\377")
        if self.name.endswith(".gz"):
@@ -603,12 +613,12 @@ class _FileInFile(object):
       object.
    """
-    def __init__(self, fileobj, offset, size, blockinfo=None):
+    def __init__(self, fileobj, offset, size, name, blockinfo=None):
        self.fileobj = fileobj
        self.offset = offset
        self.size = size
        self.position = 0
-        self.name = getattr(fileobj, "name", None)
+        self.name = name
        self.closed = False
        if blockinfo is None:
@@ -705,13 +715,138 @@ class ExFileObject(io.BufferedReader):
    def __init__(self, tarfile, tarinfo):
        fileobj = _FileInFile(tarfile.fileobj, tarinfo.offset_data,
-                tarinfo.size, tarinfo.sparse)
+                tarinfo.size, tarinfo.name, tarinfo.sparse)
        super().__init__(fileobj)
 #class ExFileObject
 #-----------------------------
 # extraction filters (PEP 706)
 #-----------------------------
 class FilterError(TarError):
    pass
 class AbsolutePathError(FilterError):
    def __init__(self, tarinfo):
        self.tarinfo = tarinfo
        super().__init__(f'member {tarinfo.name!r} has an absolute path')
 class OutsideDestinationError(FilterError):
    def __init__(self, tarinfo, path):
        self.tarinfo = tarinfo
        self._path = path
        super().__init__(f'{tarinfo.name!r} would be extracted to {path!r}, '
                         + 'which is outside the destination')
 class SpecialFileError(FilterError):
    def __init__(self, tarinfo):
        self.tarinfo = tarinfo
        super().__init__(f'{tarinfo.name!r} is a special file')
 class AbsoluteLinkError(FilterError):
    def __init__(self, tarinfo):
        self.tarinfo = tarinfo
        super().__init__(f'{tarinfo.name!r} is a link to an absolute path')
 class LinkOutsideDestinationError(FilterError):
    def __init__(self, tarinfo, path):
        self.tarinfo = tarinfo
        self._path = path
        super().__init__(f'{tarinfo.name!r} would link to {path!r}, '
                         + 'which is outside the destination')
 def _get_filtered_attrs(member, dest_path, for_data=True):
    new_attrs = {}
    name = member.name
    dest_path = os.path.realpath(dest_path)
    # Strip leading / (tar's directory separator) from filenames.
    # Include os.sep (target OS directory separator) as well.
    if name.startswith(('/', os.sep)):
        name = new_attrs['name'] = member.path.lstrip('/' + os.sep)
    if os.path.isabs(name):
        # Path is absolute even after stripping.
        # For example, 'C:/foo' on Windows.
        raise AbsolutePathError(member)
    # Ensure we stay in the destination
    target_path = os.path.realpath(os.path.join(dest_path, name))
    if os.path.commonpath([target_path, dest_path]) != dest_path:
        raise OutsideDestinationError(member, target_path)
    # Limit permissions (no high bits, and go-w)
    mode = member.mode
    if mode is not None:
        # Strip high bits & group/other write bits
        mode = mode & 0o755
        if for_data:
            # For data, handle permissions & file types
            if member.isreg() or member.islnk():
                if not mode & 0o100:
                    # Clear executable bits if not executable by user
                    mode &= ~0o111
                # Ensure owner can read & write
                mode |= 0o600
            elif member.isdir() or member.issym():
                # Ignore mode for directories & symlinks
                mode = None
            else:
                # Reject special files
                raise SpecialFileError(member)
        if mode != member.mode:
            new_attrs['mode'] = mode
    if for_data:
        # Ignore ownership for 'data'
        if member.uid is not None:
            new_attrs['uid'] = None
        if member.gid is not None:
            new_attrs['gid'] = None
        if member.uname is not None:
            new_attrs['uname'] = None
        if member.gname is not None:
            new_attrs['gname'] = None
        # Check link destination for 'data'
        if member.islnk() or member.issym():
            if os.path.isabs(member.linkname):
                raise AbsoluteLinkError(member)
            if member.issym():
                target_path = os.path.join(dest_path,
                                           os.path.dirname(name),
                                           member.linkname)
            else:
                target_path = os.path.join(dest_path,
                                           member.linkname)
            target_path = os.path.realpath(target_path)
            if os.path.commonpath([target_path, dest_path]) != dest_path:
                raise LinkOutsideDestinationError(member, target_path)
    return new_attrs
 def fully_trusted_filter(member, dest_path):
    return member
 def tar_filter(member, dest_path):
    new_attrs = _get_filtered_attrs(member, dest_path, False)
    if new_attrs:
        return member.replace(**new_attrs, deep=False)
    return member
 def data_filter(member, dest_path):
    new_attrs = _get_filtered_attrs(member, dest_path, True)
    if new_attrs:
        return member.replace(**new_attrs, deep=False)
    return member
 _NAMED_FILTERS = {
    "fully_trusted": fully_trusted_filter,
    "tar": tar_filter,
    "data": data_filter,
 }
 #------------------
 # Exported Classes
 #------------------
 # Sentinel for replace() defaults, meaning "don't change the attribute"
 _KEEP = object()
 class TarInfo(object):
    """Informational class which holds the details about an
       archive member given by a tar header block.
@@ -792,12 +927,44 @@ class TarInfo(object):
    def __repr__(self):
        return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self))
    def replace(self, *,
                name=_KEEP, mtime=_KEEP, mode=_KEEP, linkname=_KEEP,
                uid=_KEEP, gid=_KEEP, uname=_KEEP, gname=_KEEP,
                deep=True, _KEEP=_KEEP):
        """Return a deep copy of self with the given attributes replaced.
        """
        if deep:
            result = copy.deepcopy(self)
        else:
            result = copy.copy(self)
        if name is not _KEEP:
            result.name = name
        if mtime is not _KEEP:
            result.mtime = mtime
        if mode is not _KEEP:
            result.mode = mode
        if linkname is not _KEEP:
            result.linkname = linkname
        if uid is not _KEEP:
            result.uid = uid
        if gid is not _KEEP:
            result.gid = gid
        if uname is not _KEEP:
            result.uname = uname
        if gname is not _KEEP:
            result.gname = gname
        return result
    def get_info(self):
        """Return the TarInfo's attributes as a dictionary.
        """
        if self.mode is None:
            mode = None
        else:
            mode = self.mode & 0o7777
        info = {
            "name":     self.name,
-            "mode":     self.mode & 0o7777,
+            "mode":     mode,
            "uid":      self.uid,
            "gid":      self.gid,
            "size":     self.size,
@@ -820,6 +987,9 @@ class TarInfo(object):
        """Return a tar header as a string of 512 byte blocks.
        """
        info = self.get_info()
        for name, value in info.items():
            if value is None:
                raise ValueError("%s may not be None" % name)
        if format == USTAR_FORMAT:
            return self.create_ustar_header(info, encoding, errors)
@@ -950,6 +1120,12 @@ class TarInfo(object):
            devmajor = stn("", 8, encoding, errors)
            devminor = stn("", 8, encoding, errors)
        # None values in metadata should cause ValueError.
        # itn()/stn() do this for all fields except type.
        filetype = info.get("type", REGTYPE)
        if filetype is None:
            raise ValueError("TarInfo.type must not be None")
        parts = [
            stn(info.get("name", ""), 100, encoding, errors),
            itn(info.get("mode", 0) & 0o7777, 8, format),
@@ -958,7 +1134,7 @@ class TarInfo(object):
            itn(info.get("size", 0), 12, format),
            itn(info.get("mtime", 0), 12, format),
            b"        ", # checksum field
-            info.get("type", REGTYPE),
+            filetype,
            stn(info.get("linkname", ""), 100, encoding, errors),
            info.get("magic", POSIX_MAGIC),
            stn(info.get("uname", ""), 32, encoding, errors),
@@ -1264,11 +1440,7 @@ class TarInfo(object):
        # the newline. keyword and value are both UTF-8 encoded strings.
        regex = re.compile(br"(\d+) ([^=]+)=")
        pos = 0
-        while True:
+        while match := regex.match(buf, pos):
            match = regex.match(buf, pos)
            if not match:
                break
            length, keyword = match.groups()
            length = int(length)
            if length == 0:
@@ -1468,6 +1640,8 @@ class TarFile(object):
    fileobject = ExFileObject   # The file-object for extractfile().
    extraction_filter = None    # The default filter for extraction.
    def __init__(self, name=None, mode="r", fileobj=None, format=None,
            tarinfo=None, dereference=None, ignore_zeros=None, encoding=None,
            errors="surrogateescape", pax_headers=None, debug=None,
@@ -1659,7 +1833,9 @@ class TarFile(object):
            if filemode not in ("r", "w"):
                raise ValueError("mode must be 'r' or 'w'")
-            stream = _Stream(name, filemode, comptype, fileobj, bufsize)
+            compresslevel = kwargs.pop("compresslevel", 9)
            stream = _Stream(name, filemode, comptype, fileobj, bufsize,
                             compresslevel)
            try:
                t = cls(name, filemode, stream, **kwargs)
            except:
@@ -1755,6 +1931,9 @@ class TarFile(object):
        except ImportError:
            raise CompressionError("lzma module is not available") from None
        # XXX: RUSTPYTHON; xz is not supported yet
        raise CompressionError("lzma module is not available") from None
        fileobj = LZMAFile(fileobj or name, mode, preset=preset)
        try:
@@ -1940,7 +2119,10 @@ class TarFile(object):
            members = self
        for tarinfo in members:
            if verbose:
-                _safe_print(stat.filemode(tarinfo.mode))
+                if tarinfo.mode is None:
                    _safe_print("??????????")
                else:
                    _safe_print(stat.filemode(tarinfo.mode))
                _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid,
                                       tarinfo.gname or tarinfo.gid))
                if tarinfo.ischr() or tarinfo.isblk():
@@ -1948,8 +2130,11 @@ class TarFile(object):
                            ("%d,%d" % (tarinfo.devmajor, tarinfo.devminor)))
                else:
                    _safe_print("%10d" % tarinfo.size)
-                _safe_print("%d-%02d-%02d %02d:%02d:%02d" \
+                if tarinfo.mtime is None:
-                            % time.localtime(tarinfo.mtime)[:6])
+                    _safe_print("????-??-?? ??:??:??")
                else:
                    _safe_print("%d-%02d-%02d %02d:%02d:%02d" \
                                % time.localtime(tarinfo.mtime)[:6])
            _safe_print(tarinfo.name + ("/" if tarinfo.isdir() else ""))
@@ -2036,32 +2221,63 @@ class TarFile(object):
        self.members.append(tarinfo)
-    def extractall(self, path=".", members=None, *, numeric_owner=False):
+    def _get_filter_function(self, filter):
        if filter is None:
            filter = self.extraction_filter
            if filter is None:
                warnings.warn(
                    'Python 3.14 will, by default, filter extracted tar '
                    + 'archives and reject files or modify their metadata. '
                    + 'Use the filter argument to control this behavior.',
                    DeprecationWarning)
                return fully_trusted_filter
            if isinstance(filter, str):
                raise TypeError(
                    'String names are not supported for '
                    + 'TarFile.extraction_filter. Use a function such as '
                    + 'tarfile.data_filter directly.')
            return filter
        if callable(filter):
            return filter
        try:
            return _NAMED_FILTERS[filter]
        except KeyError:
            raise ValueError(f"filter {filter!r} not found") from None
    def extractall(self, path=".", members=None, *, numeric_owner=False,
                   filter=None):
        """Extract all members from the archive to the current working
           directory and set owner, modification time and permissions on
           directories afterwards. `path' specifies a different directory
           to extract to. `members' is optional and must be a subset of the
           list returned by getmembers(). If `numeric_owner` is True, only
           the numbers for user/group names are used and not the names.
           The `filter` function will be called on each member just
           before extraction.
           It can return a changed TarInfo or None to skip the member.
           String names of common filters are accepted.
        """
        directories = []
        filter_function = self._get_filter_function(filter)
        if members is None:
            members = self
-        for tarinfo in members:
+        for member in members:
            tarinfo = self._get_extract_tarinfo(member, filter_function, path)
            if tarinfo is None:
                continue
            if tarinfo.isdir():
-                # Extract directories with a safe mode.
+                # For directories, delay setting attributes until later,
                # since permissions can interfere with extraction and
                # extracting contents can reset mtime.
                directories.append(tarinfo)
-                tarinfo = copy.copy(tarinfo)
+            self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(),
-                tarinfo.mode = 0o700
+                              numeric_owner=numeric_owner)
            # Do not set_attrs directories, as we will do that further down
            self.extract(tarinfo, path, set_attrs=not tarinfo.isdir(),
                         numeric_owner=numeric_owner)
        # Reverse sort directories.
-        directories.sort(key=lambda a: a.name)
+        directories.sort(key=lambda a: a.name, reverse=True)
        directories.reverse()
        # Set correct owner, mtime and filemode on directories.
        for tarinfo in directories:
@@ -2071,12 +2287,10 @@ class TarFile(object):
                self.utime(tarinfo, dirpath)
                self.chmod(tarinfo, dirpath)
            except ExtractError as e:
-                if self.errorlevel > 1:
+                self._handle_nonfatal_error(e)
                    raise
                else:
                    self._dbg(1, "tarfile: %s" % e)
-    def extract(self, member, path="", set_attrs=True, *, numeric_owner=False):
+    def extract(self, member, path="", set_attrs=True, *, numeric_owner=False,
                filter=None):
        """Extract a member from the archive to the current working directory,
           using its full name. Its file information is extracted as accurately
           as possible. `member' may be a filename or a TarInfo object. You can
@@ -2084,35 +2298,70 @@ class TarFile(object):
           mtime, mode) are set unless `set_attrs' is False. If `numeric_owner`
           is True, only the numbers for user/group names are used and not
           the names.
        """
        self._check("r")
           The `filter` function will be called before extraction.
           It can return a changed TarInfo or None to skip the member.
           String names of common filters are accepted.
        """
        filter_function = self._get_filter_function(filter)
        tarinfo = self._get_extract_tarinfo(member, filter_function, path)
        if tarinfo is not None:
            self._extract_one(tarinfo, path, set_attrs, numeric_owner)
    def _get_extract_tarinfo(self, member, filter_function, path):
        """Get filtered TarInfo (or None) from member, which might be a str"""
        if isinstance(member, str):
            tarinfo = self.getmember(member)
        else:
            tarinfo = member
        unfiltered = tarinfo
        try:
            tarinfo = filter_function(tarinfo, path)
        except (OSError, FilterError) as e:
            self._handle_fatal_error(e)
        except ExtractError as e:
            self._handle_nonfatal_error(e)
        if tarinfo is None:
            self._dbg(2, "tarfile: Excluded %r" % unfiltered.name)
            return None
        # Prepare the link target for makelink().
        if tarinfo.islnk():
            tarinfo = copy.copy(tarinfo)
            tarinfo._link_target = os.path.join(path, tarinfo.linkname)
        return tarinfo
    def _extract_one(self, tarinfo, path, set_attrs, numeric_owner):
        """Extract from filtered tarinfo to disk"""
        self._check("r")
        try:
            self._extract_member(tarinfo, os.path.join(path, tarinfo.name),
                                 set_attrs=set_attrs,
                                 numeric_owner=numeric_owner)
        except OSError as e:
-            if self.errorlevel > 0:
+            self._handle_fatal_error(e)
                raise
            else:
                if e.filename is None:
                    self._dbg(1, "tarfile: %s" % e.strerror)
                else:
                    self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
        except ExtractError as e:
-            if self.errorlevel > 1:
+            self._handle_nonfatal_error(e)
-                raise
+
    def _handle_nonfatal_error(self, e):
        """Handle non-fatal error (ExtractError) according to errorlevel"""
        if self.errorlevel > 1:
            raise
        else:
            self._dbg(1, "tarfile: %s" % e)
    def _handle_fatal_error(self, e):
        """Handle "fatal" error according to self.errorlevel"""
        if self.errorlevel > 0:
            raise
        elif isinstance(e, OSError):
            if e.filename is None:
                self._dbg(1, "tarfile: %s" % e.strerror)
            else:
-                self._dbg(1, "tarfile: %s" % e)
+                self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
        else:
            self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e))
    def extractfile(self, member):
        """Extract a member from the archive as a file object. `member' may be
@@ -2199,11 +2448,16 @@ class TarFile(object):
        """Make a directory called targetpath.
        """
        try:
-            # Use a safe mode for the directory, the real mode is set
+            if tarinfo.mode is None:
-            # later in _extract_member().
+                # Use the system's default mode
-            os.mkdir(targetpath, 0o700)
+                os.mkdir(targetpath)
            else:
                # Use a safe mode for the directory, the real mode is set
                # later in _extract_member().
                os.mkdir(targetpath, 0o700)
        except FileExistsError:
-            pass
+            if not os.path.isdir(targetpath):
                raise
    def makefile(self, tarinfo, targetpath):
        """Make a file called targetpath.
@@ -2244,6 +2498,9 @@ class TarFile(object):
            raise ExtractError("special devices not supported by system")
        mode = tarinfo.mode
        if mode is None:
            # Use mknod's default
            mode = 0o600
        if tarinfo.isblk():
            mode |= stat.S_IFBLK
        else:
@@ -2265,7 +2522,6 @@ class TarFile(object):
                    os.unlink(targetpath)
                os.symlink(tarinfo.linkname, targetpath)
            else:
                # See extract().
                if os.path.exists(tarinfo._link_target):
                    os.link(tarinfo._link_target, targetpath)
                else:
@@ -2290,15 +2546,19 @@ class TarFile(object):
            u = tarinfo.uid
            if not numeric_owner:
                try:
-                    if grp:
+                    if grp and tarinfo.gname:
                        g = grp.getgrnam(tarinfo.gname)[2]
                except KeyError:
                    pass
                try:
-                    if pwd:
+                    if pwd and tarinfo.uname:
                        u = pwd.getpwnam(tarinfo.uname)[2]
                except KeyError:
                    pass
            if g is None:
                g = -1
            if u is None:
                u = -1
            try:
                if tarinfo.issym() and hasattr(os, "lchown"):
                    os.lchown(targetpath, u, g)
@@ -2310,6 +2570,8 @@ class TarFile(object):
    def chmod(self, tarinfo, targetpath):
        """Set file permissions of targetpath according to tarinfo.
        """
        if tarinfo.mode is None:
            return
        try:
            os.chmod(targetpath, tarinfo.mode)
        except OSError as e:
@@ -2318,10 +2580,13 @@ class TarFile(object):
    def utime(self, tarinfo, targetpath):
        """Set modification time of targetpath according to tarinfo.
        """
        mtime = tarinfo.mtime
        if mtime is None:
            return
        if not hasattr(os, 'utime'):
            return
        try:
-            os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
+            os.utime(targetpath, (mtime, mtime))
        except OSError as e:
            raise ExtractError("could not change modification time") from e
@@ -2339,6 +2604,8 @@ class TarFile(object):
        # Advance the file pointer.
        if self.offset != self.fileobj.tell():
            if self.offset == 0:
                return None
            self.fileobj.seek(self.offset - 1)
            if not self.fileobj.read(1):
                raise ReadError("unexpected end of data")
@@ -2397,13 +2664,26 @@ class TarFile(object):
        members = self.getmembers()
        # Limit the member search list up to tarinfo.
        skipping = False
        if tarinfo is not None:
-            members = members[:members.index(tarinfo)]
+            try:
                index = members.index(tarinfo)
            except ValueError:
                # The given starting point might be a (modified) copy.
                # We'll later skip members until we find an equivalent.
                skipping = True
            else:
                # Happy fast path
                members = members[:index]
        if normalize:
            name = os.path.normpath(name)
        for member in reversed(members):
            if skipping:
                if tarinfo.offset == member.offset:
                    skipping = False
                continue
            if normalize:
                member_name = os.path.normpath(member.name)
            else:
@@ -2412,14 +2692,16 @@ class TarFile(object):
            if name == member_name:
                return member
        if skipping:
            # Starting point was not found
            raise ValueError(tarinfo)
    def _load(self):
        """Read through the entire archive file and look for readable
           members.
        """
-        while True:
+        while self.next() is not None:
-            tarinfo = self.next()
+            pass
            if tarinfo is None:
                break
        self._loaded = True
    def _check(self, mode=None):
@@ -2504,6 +2786,7 @@ class TarFile(object):
 #--------------------
 # exported functions
 #--------------------
 def is_tarfile(name):
    """Return True if name points to a tar archive that we
       are able to handle, else return False.
@@ -2512,7 +2795,9 @@ def is_tarfile(name):
    """
    try:
        if hasattr(name, "read"):
            pos = name.tell()
            t = open(fileobj=name)
            name.seek(pos)
        else:
            t = open(name)
        t.close()
@@ -2530,6 +2815,10 @@ def main():
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-v', '--verbose', action='store_true', default=False,
                        help='Verbose output')
    parser.add_argument('--filter', metavar='<filtername>',
                        choices=_NAMED_FILTERS,
                        help='Filter for extraction')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-l', '--list', metavar='<tarfile>',
                       help='Show listing of a tarfile')
@@ -2541,8 +2830,12 @@ def main():
                       help='Create tarfile from sources')
    group.add_argument('-t', '--test', metavar='<tarfile>',
                       help='Test if a tarfile is valid')
    args = parser.parse_args()
    if args.filter and args.extract is None:
        parser.exit(1, '--filter is only valid for extraction\n')
    if args.test is not None:
        src = args.test
        if is_tarfile(src):
@@ -2573,7 +2866,7 @@ def main():
        if is_tarfile(src):
            with TarFile.open(src, 'r:*') as tf:
-                tf.extractall(path=curdir)
+                tf.extractall(path=curdir, filter=args.filter)
            if args.verbose:
                if curdir == '.':
                    msg = '{!r} file is extracted.'.format(src)
--- a/Show More
+++ b/Show More