Merge pull request #5744 from coolreader18/upd-malachite

Update to malachite 0.6

.cspell.dict/cpython.txt

@@ -0,0 +1,59 @@
+argtypes
+asdl
+asname
+augassign
+badsyntax
+basetype
+boolop
+bxor
+cached_tsver
+cellarg
+cellvar
+cellvars
+cmpop
+denom
+dictoffset
+elts
+excepthandler
+fileutils
+finalbody
+formatfloat
+freevar
+freevars
+fromlist
+heaptype
+HIGHRES
+IMMUTABLETYPE
+kwonlyarg
+kwonlyargs
+lasti
+linearise
+maxdepth
+mult
+nkwargs
+noraise
+numer
+orelse
+pathconfig
+patma
+posonlyarg
+posonlyargs
+prec
+preinitialized
+PYTHREAD_NAME
+SA_ONSTACK
+stackdepth
+stringlib
+structseq
+tok_oldval
+unaryop
+unparse
+unparser
+VARKEYWORDS
+varkwarg
+wbits
+weakreflist
+withitem
+withs
+xstat
+XXPRIME

.cspell.dict/python-more.txt

@@ -0,0 +1,257 @@
+abiflags
+abstractmethods
+aenter
+aexit
+aiter
+anext
+appendleft
+argcount
+arrayiterator
+arraytype
+asend
+asyncgen
+athrow
+backslashreplace
+baserepl
+basicsize
+bdfl
+bigcharset
+bignum
+breakpointhook
+cformat
+chunksize
+classcell
+closefd
+closesocket
+codepoint
+codepoints
+codesize
+contextvar
+cpython
+cratio
+dealloc
+debugbuild
+decompressor
+defaultaction
+descr
+dictcomp
+dictitems
+dictkeys
+dictview
+digestmod
+dllhandle
+docstring
+docstrings
+dunder
+endianness
+endpos
+eventmask
+excepthook
+exceptiongroup
+exitfuncs
+extendleft
+fastlocals
+fdel
+fedcba
+fget
+fileencoding
+fillchar
+fillvalue
+finallyhandler
+firstiter
+firstlineno
+fnctl
+frombytes
+fromhex
+fromunicode
+fset
+fspath
+fstring
+fstrings
+ftruncate
+genexpr
+getattro
+getcodesize
+getdefaultencoding
+getfilesystemencodeerrors
+getfilesystemencoding
+getformat
+getframe
+getnewargs
+getpip
+getrandom
+getrecursionlimit
+getrefcount
+getsizeof
+getweakrefcount
+getweakrefs
+getwindowsversion
+gmtoff
+groupdict
+groupindex
+hamt
+hostnames
+idfunc
+idiv
+idxs
+impls
+indexgroup
+infj
+instancecheck
+instanceof
+irepeat
+isabstractmethod
+isbytes
+iscased
+isfinal
+istext
+itemiterator
+itemsize
+iternext
+keepends
+keyfunc
+keyiterator
+kwarg
+kwargs
+kwdefaults
+kwonlyargcount
+lastgroup
+lastindex
+linearization
+linearize
+listcomp
+longrange
+lvalue
+mappingproxy
+maskpri
+maxdigits
+MAXGROUPS
+MAXREPEAT
+maxsplit
+maxunicode
+memoryview
+memoryviewiterator
+metaclass
+metaclasses
+metatype
+mformat
+mro
+mros
+multiarch
+namereplace
+nanj
+nbytes
+ncallbacks
+ndigits
+ndim
+nldecoder
+nlocals
+NOARGS
+nonbytes
+Nonprintable
+origname
+ospath
+pendingcr
+phello
+platlibdir
+popleft
+posixsubprocess
+posonly
+posonlyargcount
+prepending
+profilefunc
+pycache
+pycodecs
+pycs
+pyexpat
+PYTHONBREAKPOINT
+PYTHONDEBUG
+PYTHONHASHSEED
+PYTHONHOME
+PYTHONINSPECT
+PYTHONOPTIMIZE
+PYTHONPATH
+PYTHONPATH
+PYTHONSAFEPATH
+PYTHONVERBOSE
+PYTHONWARNDEFAULTENCODING
+PYTHONWARNINGS
+pytraverse
+PYVENV
+qualname
+quotetabs
+radd
+rdiv
+rdivmod
+readall
+readbuffer
+reconstructor
+refcnt
+releaselevel
+reverseitemiterator
+reverseiterator
+reversekeyiterator
+reversevalueiterator
+rfloordiv
+rlshift
+rmod
+rpow
+rrshift
+rsub
+rtruediv
+rvalue
+scproxy
+seennl
+setattro
+setcomp
+setrecursionlimit
+showwarnmsg
+signum
+slotnames
+STACKLESS
+stacklevel
+stacksize
+startpos
+subclassable
+subclasscheck
+subclasshook
+suboffset
+suboffsets
+SUBPATTERN
+sumprod
+surrogateescape
+surrogatepass
+sysconf
+sysconfigdata
+sysvars
+teedata
+thisclass
+titlecased
+tkapp
+tobytes
+tolist
+toreadonly
+TPFLAGS
+tracefunc
+unimportable
+unionable
+unraisablehook
+unsliceable
+urandom
+valueiterator
+vararg
+varargs
+varnames
+warningregistry
+warnmsg
+warnoptions
+warnopts
+weaklist
+weakproxy
+weakrefs
+winver
+withdata
+xmlcharrefreplace
+xoptions
+xopts
+yieldfrom

.cspell.dict/rust-more.txt

@@ -0,0 +1,83 @@
+ahash
+arrayvec
+bidi
+biguint
+bindgen
+bitflags
+bitor
+bstr
+byteorder
+byteset
+caseless
+chrono
+consts
+cranelift
+cstring
+datelike
+deserializer
+fdiv
+flamescope
+flate2
+fract
+getres
+hasher
+hexf
+hexversion
+idents
+illumos
+indexmap
+insta
+keccak
+lalrpop
+lexopt
+libc
+libloading
+libz
+longlong
+Manually
+maplit
+memmap
+memmem
+metas
+modpow
+msvc
+muldiv
+nanos
+nonoverlapping
+objclass
+peekable
+powc
+powf
+powi
+prepended
+punct
+puruspe
+replacen
+rmatch
+rposition
+rsplitn
+rustc
+rustfmt
+rustyline
+seedable
+seekfrom
+siphash
+siphasher
+splitn
+subsec
+thiserror
+timelike
+timsort
+trai
+ulonglong
+unic
+unistd
+unraw
+unsync
+wasip1
+wasip2
+wasmbind
+wasmtime
+widestring
+winapi
+winsock

.cspell.json

@@ -1,210 +1,81 @@
 // See: https://github.com/streetsidesoftware/cspell/tree/master/packages/cspell
 {
   "version": "0.2",
+  "import": [
+    "@cspell/dict-en_us/cspell-ext.json",
+    // "@cspell/dict-cpp/cspell-ext.json",
+    "@cspell/dict-python/cspell-ext.json",
+    "@cspell/dict-rust/cspell-ext.json",
+    "@cspell/dict-win32/cspell-ext.json",
+    "@cspell/dict-shell/cspell-ext.json",
+  ],
   // language - current active spelling language
   "language": "en",
   // dictionaries - list of the names of the dictionaries to use
   "dictionaries": [
+    "cpython",  // Sometimes keeping same terms with cpython is easy
+    "python-more",  // Python API terms not listed in python
+    "rust-more",  // Rust API terms not listed in rust
     "en_US",
     "softwareTerms",
     "c",
     "cpp",
     "python",
-    "python-custom",
     "rust",
-    "unix",
-    "posix",
-    "winapi"
+    "shell",
+    "win32"
   ],
   // dictionaryDefinitions - this list defines any custom dictionaries to use
-  "dictionaryDefinitions": [],
+  "dictionaryDefinitions": [
+    {
+      "name": "cpython",
+      "path": "./.cspell.dict/cpython.txt"
+    },
+    {
+      "name": "python-more",
+      "path": "./.cspell.dict/python-more.txt"
+    },
+    {
+      "name": "rust-more",
+      "path": "./.cspell.dict/rust-more.txt"
+    }
+  ],
   "ignorePaths": [
     "**/__pycache__/**",
     "Lib/**"
   ],
   // words - list of words to be always considered correct
   "words": [
-    // Rust
-    "ahash",
-    "bidi",
-    "biguint",
-    "bindgen",
-    "bitflags",
-    "bstr",
-    "byteorder",
-    "chrono",
-    "consts",
-    "cstring",
-    "flate2",
-    "fract",
-    "hasher",
-    "idents",
-    "indexmap",
-    "insta",
-    "keccak",
-    "lalrpop",
-    "libc",
-    "libz",
-    "longlong",
-    "Manually",
-    "maplit",
-    "memmap",
-    "metas",
-    "modpow",
-    "nanos",
-    "objclass",
-    "peekable",
-    "powc",
-    "powf",
-    "prepended",
-    "punct",
-    "replacen",
-    "rsplitn",
-    "rustc",
-    "rustfmt",
-    "seekfrom",
-    "splitn",
-    "subsec",
-    "timsort",
-    "trai",
-    "ulonglong",
-    "unic",
-    "unistd",
-    "winapi",
-    "winsock",
-    // Python
-    "abstractmethods",
-    "aiter",
-    "anext",
-    "arrayiterator",
-    "arraytype",
-    "asend",
-    "athrow",
-    "basicsize",
-    "cformat",
-    "classcell",
-    "closesocket",
-    "codepoint",
-    "codepoints",
-    "cpython",
-    "decompressor",
-    "defaultaction",
-    "descr",
-    "dictcomp",
-    "dictitems",
-    "dictkeys",
-    "dictview",
-    "docstring",
-    "docstrings",
-    "dunder",
-    "eventmask",
-    "fdel",
-    "fget",
-    "fileencoding",
-    "fillchar",
-    "finallyhandler",
-    "frombytes",
-    "fromhex",
-    "fromunicode",
-    "fset",
-    "fspath",
-    "fstring",
-    "fstrings",
-    "genexpr",
-    "getattro",
-    "getformat",
-    "getnewargs",
-    "getweakrefcount",
-    "getweakrefs",
-    "hostnames",
-    "idiv",
-    "impls",
-    "infj",
-    "instancecheck",
-    "instanceof",
-    "isabstractmethod",
-    "itemiterator",
-    "itemsize",
-    "iternext",
-    "keyiterator",
-    "kwarg",
-    "kwargs",
-    "linearization",
-    "linearize",
-    "listcomp",
-    "mappingproxy",
-    "maxsplit",
-    "memoryview",
-    "memoryviewiterator",
-    "metaclass",
-    "metaclasses",
-    "metatype",
-    "mro",
-    "mros",
-    "nanj",
-    "ndigits",
-    "ndim",
-    "nonbytes",
-    "origname",
-    "posixsubprocess",
-    "pyexpat",
-    "PYTHONDEBUG",
-    "PYTHONHOME",
-    "PYTHONINSPECT",
-    "PYTHONOPTIMIZE",
-    "PYTHONPATH",
-    "PYTHONPATH",
-    "PYTHONVERBOSE",
-    "PYTHONWARNINGS",
-    "qualname",
-    "radd",
-    "rdiv",
-    "rdivmod",
-    "reconstructor",
-    "reversevalueiterator",
-    "rfloordiv",
-    "rlshift",
-    "rmod",
-    "rpow",
-    "rrshift",
-    "rsub",
-    "rtruediv",
-    "scproxy",
-    "setattro",
-    "setcomp",
-    "showwarnmsg",
-    "warnmsg",
-    "stacklevel",
-    "subclasscheck",
-    "subclasshook",
-    "unionable",
-    "unraisablehook",
-    "valueiterator",
-    "vararg",
-    "varargs",
-    "varnames",
-    "warningregistry",
-    "warnopts",
-    "weakproxy",
-    "xopts",
-    // RustPython
+    "RUSTPYTHONPATH",
+    // RustPython terms
+    "aiterable",
+    "alnum",
     "baseclass",
+    "boxvec",
     "Bytecode",
     "cfgs",
     "codegen",
+    "coro",
     "dedentations",
     "dedents",
     "deduped",
     "downcasted",
     "dumpable",
+    "emscripten",
+    "excs",
+    "finalizer",
     "GetSet",
+    "groupref",
     "internable",
+    "lossily",
     "makeunicodedata",
     "miri",
     "notrace",
+    "openat",
     "pyarg",
     "pyarg",
     "pyargs",
+    "pyast",
     "PyAttr",
     "pyc",
     "PyClass",
@@ -213,6 +84,7 @@
     "PyFunction",
     "pygetset",
     "pyimpl",
+    "pylib",
     "pymember",
     "PyMethod",
     "PyModule",
@@ -225,6 +97,7 @@
     "PyResult",
     "pyslot",
     "PyStaticMethod",
+    "pystone",
     "pystr",
     "pystruct",
     "pystructseq",
@@ -232,57 +105,31 @@
     "reducelib",
     "richcompare",
     "RustPython",
+    "significand",
     "struc",
+    "summands", // plural of summand
+    "sysmodule",
     "tracebacks",
     "typealiases",
-    "Unconstructible",
+    "unconstructible",
     "unhashable",
     "uninit",
     "unraisable",
+    "unresizable",
     "wasi",
     "zelf",
-    // cpython
-    "argtypes",
-    "asdl",
-    "asname",
-    "augassign",
-    "badsyntax",
-    "basetype",
-    "boolop",
-    "bxor",
-    "cellarg",
-    "cellvar",
-    "cellvars",
-    "cmpop",
-    "dictoffset",
-    "elts",
-    "excepthandler",
-    "finalbody",
-    "freevar",
-    "freevars",
-    "fromlist",
-    "heaptype",
-    "IMMUTABLETYPE",
-    "kwonlyarg",
-    "kwonlyargs",
-    "linearise",
-    "maxdepth",
-    "mult",
-    "nkwargs",
-    "orelse",
-    "patma",
-    "posonlyarg",
-    "posonlyargs",
-    "prec",
-    "stackdepth",
-    "unaryop",
-    "unparse",
-    "unparser",
-    "VARKEYWORDS",
-    "varkwarg",
-    "wbits",
-    "withitem",
-    "withs"
+    // unix
+    "CLOEXEC",
+    "codeset",
+    "endgrent",
+    "gethrvtime",
+    "getrusage",
+    "nanosleep",
+    "sigaction",
+    "WRLCK",
+    // win32
+    "birthtime",
+    "IFEXEC",
   ],
   // flagWords - list of words to be always considered incorrect
   "flagWords": [

.devcontainer/devcontainer.json

@@ -1,6 +1,4 @@
 {
-  "image": "mcr.microsoft.com/devcontainers/universal:2",
-  "features": {
-    "ghcr.io/devcontainers/features/rust:1": {}
-  }
-}
+  "image": "mcr.microsoft.com/devcontainers/base:jammy",
+  "onCreateCommand": "curl https://sh.rustup.rs -sSf | sh -s -- -y"
+}

.gitattributes

@@ -1,6 +1,7 @@
 Lib/**      linguist-vendored
-Cargo.lock  linguist-generated -merge
+Cargo.lock  linguist-generated
 *.snap      linguist-generated -merge
 vm/src/stdlib/ast/gen.rs    linguist-generated -merge
 Lib/*.py    text working-tree-encoding=UTF-8 eol=LF
 **/*.rs     text working-tree-encoding=UTF-8 eol=LF
+*.pck       binary

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Keep GitHub Actions up to date with GitHub's Dependabot...
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: weekly

.github/workflows/ci.yaml

@@ -4,6 +4,7 @@ on:
   pull_request:
     types: [unlabeled, opened, synchronize, reopened]
   merge_group:
+  workflow_dispatch:
 
 name: CI
 
@@ -15,14 +16,19 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl
   # Skip additional tests on Windows. They are checked on Linux and MacOS.
+  # test_glob: many failing tests
+  # test_io: many failing tests
+  # test_os: many failing tests
+  # test_pathlib: support.rmtree() failing
+  # test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)')
+  # test_venv: couple of failing tests
   WINDOWS_SKIPS: >-
-    test_datetime
     test_glob
-    test_importlib
     test_io
     test_os
+    test_rlcompleter
     test_pathlib
     test_posixpath
     test_venv
@@ -34,7 +40,7 @@ env:
   # PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently
   # only run on Linux to speed up the CI.
   PLATFORM_INDEPENDENT_TESTS: >-
-    test_argparse
+    test__colorize
     test_array
     test_asyncgen
     test_binop
@@ -59,7 +65,6 @@ env:
     test_dis
     test_enumerate
     test_exception_variations
-    test_exceptions
     test_float
     test_format
     test_fractions
@@ -100,12 +105,11 @@ env:
     test_tuple
     test_types
     test_unary
-    test_unicode
     test_unpack
     test_weakref
     test_yield_from
   # Python version targeted by the CI.
-  PYTHON_VERSION: "3.12.3"
+  PYTHON_VERSION: "3.13.1"
 
 jobs:
   rust_tests:
@@ -119,7 +123,7 @@ jobs:
         os: [macos-latest, ubuntu-latest, windows-latest]
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
@@ -128,6 +132,7 @@ jobs:
       - name: Set up the Windows environment
         shell: bash
         run: |
+          git config --system core.longpaths true
           cargo install --target-dir=target -v cargo-vcpkg
           cargo vcpkg -v build
         if: runner.os == 'Windows'
@@ -136,7 +141,7 @@ jobs:
         if: runner.os == 'macOS'
 
       - name: run clippy
-        run: cargo clippy ${{ env.CARGO_ARGS }} --workspace  --exclude rustpython_wasm -- -Dwarnings
+        run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets --exclude rustpython_wasm -- -Dwarnings
 
       - name: run rust tests
         run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }}
@@ -176,7 +181,7 @@ jobs:
     name: Ensure compilation on various targets
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: i686-unknown-linux-gnu
@@ -229,6 +234,7 @@ jobs:
         uses: coolreader18/redoxer-action@v1
         with:
           command: check
+          args: --ignore-rust-version
 
   snippets_cpython:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
@@ -241,15 +247,16 @@ jobs:
         os: [macos-latest, ubuntu-latest, windows-latest]
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Set up the Windows environment
         shell: bash
         run: |
+          git config --system core.longpaths true
           cargo install cargo-vcpkg
           cargo vcpkg build
         if: runner.os == 'Windows'
@@ -262,7 +269,7 @@ jobs:
       - name: build rustpython
         run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit
         if: runner.os != 'macOS'
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: run snippets
@@ -305,7 +312,7 @@ jobs:
     name: Check Rust code with rustfmt and clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
             components: rustfmt, clippy
@@ -313,11 +320,13 @@ jobs:
         run: cargo fmt --check
       - name: run clippy on wasm
         run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: install ruff
         run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778
+      - name: Ensure docs generate no warnings
+        run: cargo doc
       - name: run python lint
         run: ruff extra_tests wasm examples  --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source
       - name: install prettier
@@ -325,13 +334,21 @@ jobs:
       - name: check wasm code with prettier
         # prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506
         run: cd wasm && git ls-files -z | xargs -0 prettier --check -u
+      # Keep cspell check as the last step. This is optional test.
+      - name: install extra dictionaries
+        run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell
+      - name: spell checker
+        uses: streetsidesoftware/cspell-action@v6
+        with:
+          files: '**/*.rs'
+          incremental_files_only: true
 
   miri:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Run tests under miri
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@master
         with:
             toolchain: nightly
@@ -348,7 +365,7 @@ jobs:
     name: Check the WASM package and demo
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: Swatinem/rust-cache@v2
@@ -356,15 +373,18 @@ jobs:
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: install geckodriver
         run: |
-          wget https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz
+          wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz
           mkdir geckodriver
-          tar -xzf geckodriver-v0.34.0-linux64.tar.gz -C geckodriver
-      - uses: actions/setup-python@v4
+          tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - run: python -m pip install -r requirements.txt
         working-directory: ./wasm/tests
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
+        with:
+          cache: "npm"
+          cache-dependency-path: "wasm/demo/package-lock.json"
       - name: run test
         run: |
           export PATH=$PATH:`pwd`/../../geckodriver
@@ -373,11 +393,12 @@ jobs:
         env:
           NODE_OPTIONS: "--openssl-legacy-provider"
         working-directory: ./wasm/demo
-      - uses: mwilliamson/setup-wabt-action@v1
-        with: { wabt-version: "1.0.30" }
+      - uses: mwilliamson/setup-wabt-action@v3
+        with: { wabt-version: "1.0.36" }
       - name: check wasm32-unknown without js
         run: |
-          cargo build --release --manifest-path wasm/wasm-unknown-test/Cargo.toml --target wasm32-unknown-unknown --verbose
+          cd wasm/wasm-unknown-test
+          cargo build --release --verbose
           if wasm-objdump -xj Import target/wasm32-unknown-unknown/release/wasm_unknown_test.wasm; then
             echo "ERROR: wasm32-unknown module expects imports from the host environment" >2
           fi
@@ -392,7 +413,7 @@ jobs:
         working-directory: ./wasm/notebook
       - name: Deploy demo to Github Pages
         if: success() && github.ref == 'refs/heads/release'
-        uses: peaceiris/actions-gh-pages@v2
+        uses: peaceiris/actions-gh-pages@v4
         env:
           ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
           PUBLISH_DIR: ./wasm/demo/dist
@@ -404,19 +425,19 @@ jobs:
     name: Run snippets and cpython tests on wasm-wasi
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
-          target: wasm32-wasi
+          target: wasm32-wasip1
 
       - uses: Swatinem/rust-cache@v2
       - name: Setup Wasmer
-        uses: wasmerio/setup-wasmer@v2
+        uses: wasmerio/setup-wasmer@v3
       - name: Install clang
         run: sudo apt-get update && sudo apt-get install clang -y
       - name: build rustpython
-        run: cargo build --release --target wasm32-wasi --features freeze-stdlib,stdlib --verbose
+        run: cargo build --release --target wasm32-wasip1 --features freeze-stdlib,stdlib --verbose
       - name: run snippets
-        run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py
+        run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py
       - name: run cpython unittest
-        run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py
+        run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py

.github/workflows/cron-ci.yaml

@@ -2,12 +2,15 @@ on:
   schedule:
     - cron: '0 0 * * 6'
   workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/cron-ci.yaml
 
 name: Periodic checks/tasks
 
 env:
-  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
-  PYTHON_VERSION: "3.12.0"
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl,jit
+  PYTHON_VERSION: "3.13.1"
 
 jobs:
   # codecov collects code coverage data from the rust tests, python snippets and python test suite.
@@ -16,15 +19,15 @@ jobs:
     name: Collect code coverage data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: taiki-e/install-action@cargo-llvm-cov
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - run: sudo apt-get update && sudo apt-get -y install lcov
       - name: Run cargo-llvm-cov with Rust tests.
-        run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
+        run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,importlib,encodings,ssl,jit
       - name: Run cargo-llvm-cov with Python snippets.
         run: python scripts/cargo-llvm-cov.py
         continue-on-error: true
@@ -34,7 +37,7 @@ jobs:
       - name: Prepare code coverage data
         run: cargo llvm-cov report --lcov --output-path='codecov.lcov'
       - name: Upload to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           file: ./codecov.lcov
 
@@ -42,7 +45,7 @@ jobs:
     name: Collect regression test data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - name: build rustpython
         run: cargo build --release --verbose
@@ -71,9 +74,9 @@ jobs:
     name: Collect what is left data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: build rustpython
@@ -81,7 +84,7 @@ jobs:
       - name: Collect what is left data
         run: |
           chmod +x ./whats_left.py
-          ./whats_left.py > whats_left.temp
+          ./whats_left.py --features "ssl,sqlite" > whats_left.temp
         env:
           RUSTPYTHONPATH: ${{ github.workspace }}/Lib
       - name: Upload data to the website
@@ -97,6 +100,9 @@ jobs:
           cd website
           [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp
           cp ../whats_left.temp ./_data/whats_left.temp
+          rm ./_data/whats_left/modules.csv
+          echo -e "module" > ./_data/whats_left/modules.csv
+          cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv
           git add -A
           if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then
             git push
@@ -106,9 +112,9 @@ jobs:
     name: Collect benchmark data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - run: cargo install cargo-criterion

.github/workflows/release.yml

@@ -0,0 +1,173 @@
+name: Release
+
+on:
+  schedule:
+    # 9 AM UTC on every Monday
+    - cron: "0 9 * * Mon"
+  workflow_dispatch:
+    inputs:
+      pre-release:
+        type: boolean
+        description: Mark "Pre-Release"
+        required: false
+        default: true
+
+permissions:
+  contents: write
+
+env:
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl
+
+jobs:
+  build:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: i686-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: aarch64-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: armv7-unknown-linux-gnueabi
+#          - runner: ubuntu-latest
+#            target: s390x-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: powerpc64le-unknown-linux-gnu
+          - runner: macos-latest
+            target: aarch64-apple-darwin
+#          - runner: macos-latest
+#            target: x86_64-apple-darwin
+          - runner: windows-latest
+            target: x86_64-pc-windows-msvc
+#          - runner: windows-latest
+#            target: i686-pc-windows-msvc
+#          - runner: windows-latest
+#            target: aarch64-pc-windows-msvc
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: cargo-bins/cargo-binstall@main
+
+      - name: Set up Environment
+        shell: bash
+        run: rustup target add ${{ matrix.platform.target }}
+      - name: Set up Windows Environment
+        shell: bash
+        run: |
+          git config --global core.longpaths true
+          cargo install --target-dir=target -v cargo-vcpkg
+          cargo vcpkg -v build
+        if: runner.os == 'Windows'
+      - name: Set up MacOS Environment
+        run: brew install autoconf automake libtool
+        if: runner.os == 'macOS'
+
+      - name: Build RustPython
+        run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }}
+        if: runner.os == 'macOS'
+      - name: Build RustPython
+        run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }},jit
+        if: runner.os != 'macOS'
+
+      - name: Rename Binary
+        run: cp target/${{ matrix.platform.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}
+        if: runner.os != 'Windows'
+      - name: Rename Binary
+        run: cp target/${{ matrix.platform.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}.exe
+        if: runner.os == 'Windows'
+
+      - name: Upload Binary Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}
+          path: target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}*
+
+  build-wasm:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: wasm32-wasip1
+
+      - name: Build RustPython
+        run: cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release
+
+      - name: Rename Binary
+        run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm
+
+      - name: Upload Binary Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rustpython-release-wasm32-wasip1
+          path: target/rustpython-release-wasm32-wasip1.wasm
+
+      - name: install wasm-pack
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+      - uses: actions/setup-node@v4
+      - uses: mwilliamson/setup-wabt-action@v3
+        with: { wabt-version: "1.0.30" }
+      - name: build demo
+        run: |
+          npm install
+          npm run dist
+        env:
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        working-directory: ./wasm/demo
+      - name: build notebook demo
+        run: |
+          npm install
+          npm run dist
+          mv dist ../demo/dist/notebook
+        env:
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        working-directory: ./wasm/notebook
+      - name: Deploy demo to Github Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
+          publish_dir: ./wasm/demo/dist
+          external_repository: RustPython/demo
+          publish_branch: master
+
+  release:
+    runs-on: ubuntu-latest
+    needs: [build, build-wasm]
+    steps:
+      - name: Download Binary Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: bin
+          pattern: rustpython-*
+          merge-multiple: true
+
+      - name: List Binaries
+        run: |
+          ls -lah bin/
+          file bin/*
+      - name: Create Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref_name }}
+          run: ${{ github.run_number }}
+        run: |
+          if [[ "${{ github.event.inputs.pre-release }}" == "false" ]]; then
+            RELEASE_TYPE_NAME=Release
+            PRERELEASE_ARG=
+          else
+            RELEASE_TYPE_NAME=Pre-Release
+            PRERELEASE_ARG=--prerelease
+          fi
+          
+          today=$(date '+%Y-%m-%d')
+          gh release create "$today-$tag-$run" \
+              --repo="$GITHUB_REPOSITORY" \
+              --title="RustPython $RELEASE_TYPE_NAME $today-$tag #$run" \
+              --target="$tag" \
+              --generate-notes \
+              $PRERELEASE_ARG \
+              bin/rustpython-release-*

Cargo.toml

@@ -10,35 +10,34 @@ repository.workspace = true
 license.workspace = true
 
 [features]
-default = ["threading", "stdlib", "zlib", "importlib"]
+default = ["threading", "stdlib", "stdio", "importlib"]
 importlib = ["rustpython-vm/importlib"]
 encodings = ["rustpython-vm/encodings"]
+stdio = ["rustpython-vm/stdio"]
 stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"]
 flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
 freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
 threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
-zlib = ["stdlib", "rustpython-stdlib/zlib"]
-bz2 = ["stdlib", "rustpython-stdlib/bz2"]
 sqlite = ["rustpython-stdlib/sqlite"]
 ssl = ["rustpython-stdlib/ssl"]
 ssl-vendor = ["ssl", "rustpython-stdlib/ssl-vendor"]
+tkinter = ["rustpython-stdlib/tkinter"]
 
 [dependencies]
 rustpython-compiler = { workspace = true }
 rustpython-pylib = { workspace = true, optional = true }
 rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
 rustpython-vm = { workspace = true, features = ["compiler"] }
-rustpython-parser = { workspace = true }
+ruff_python_parser = { workspace = true }
 
-atty = { workspace = true }
 cfg-if = { workspace = true }
 log = { workspace = true }
 flame = { workspace = true, optional = true }
 
-clap = "2.34"
-dirs = { package = "dirs-next", version = "2.0.0" }
-env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
+lexopt = "0.3"
+dirs = { package = "dirs-next", version = "2.0" }
+env_logger = "0.11"
 flamescope = { version = "0.1.2", optional = true }
 
 [target.'cfg(windows)'.dependencies]
@@ -48,8 +47,8 @@ libc = { workspace = true }
 rustyline = { workspace = true }
 
 [dev-dependencies]
-criterion = { version = "0.3.5", features = ["html_reports"] }
-pyo3 = { version = "0.20.2", features = ["auto-initialize"] }
+criterion = { workspace = true }
+pyo3 = { version = "0.24", features = ["auto-initialize"] }
 
 [[bench]]
 name = "execution"
@@ -93,23 +92,43 @@ rev = "2024.02.14"
 [package.metadata.vcpkg.target]
 x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] }
 
+[package.metadata.packager]
+product-name = "RustPython"
+identifier = "com.rustpython.rustpython"
+description = "An open source Python 3 interpreter written in Rust"
+homepage = "https://rustpython.github.io/"
+license_file = "LICENSE"
+authors = ["RustPython Team"]
+publisher = "RustPython Team"
+resources = ["LICENSE", "README.md", "Lib"]
+icons = ["32x32.png"]
+
+[package.metadata.packager.nsis]
+installer_mode = "both"
+template = "installer-config/installer.nsi"
+
+[package.metadata.packager.wix]
+template = "installer-config/installer.wxs"
+
+
 [workspace]
 resolver = "2"
 members = [
-    "compiler", "compiler/core", "compiler/codegen",
-    ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl",
+    "compiler", "compiler/core", "compiler/codegen", "compiler/literal", "compiler/source",
+    ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", "wtf8",
     "wasm/lib",
 ]
 
 [workspace.package]
 version = "0.4.0"
 authors = ["RustPython Team"]
-edition = "2021"
-rust-version = "1.78.0"
+edition = "2024"
+rust-version = "1.85.0"
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [workspace.dependencies]
+rustpython-compiler-source = { path = "compiler/source" }
 rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
 rustpython-compiler = { path = "compiler", version = "0.4.0" }
 rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }
@@ -117,77 +136,85 @@ rustpython-common = { path = "common", version = "0.4.0" }
 rustpython-derive = { path = "derive", version = "0.4.0" }
 rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" }
 rustpython-jit = { path = "jit", version = "0.4.0" }
+rustpython-literal = { path = "compiler/literal", version = "0.4.0" }
 rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" }
 rustpython-pylib = { path = "pylib", version = "0.4.0" }
 rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" }
 rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
+rustpython-wtf8 = { path = "wtf8", version = "0.4.0" }
 rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
 
-rustpython-literal = { version = "0.4.0" }
-rustpython-parser-core = { version = "0.4.0" }
-rustpython-parser = { version = "0.4.0" }
-rustpython-ast = { version = "0.4.0" }
-rustpython-format= { version = "0.4.0" }
-# rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-literal = { path = "../RustPython-parser/literal" }
-# rustpython-parser-core = { path = "../RustPython-parser/core" }
-# rustpython-parser = { path = "../RustPython-parser/parser" }
-# rustpython-ast = { path = "../RustPython-parser/ast" }
-# rustpython-format = { path = "../RustPython-parser/format" }
+ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
 
 ahash = "0.8.11"
-ascii = "1.0"
-atty = "0.2.14"
-bitflags = "2.4.1"
-bstr = "0.2.17"
+ascii = "1.1"
+bitflags = "2.4.2"
+bstr = "1"
 cfg-if = "1.0"
-chrono = "0.4.37"
-crossbeam-utils = "0.8.19"
+chrono = "0.4.39"
+constant_time_eq = "0.4"
+criterion = { version = "0.5", features = ["html_reports"] }
+crossbeam-utils = "0.8.21"
 flame = "0.2.2"
-getrandom = "0.2.12"
+getrandom = { version = "0.3", features = ["std"] }
 glob = "0.3"
 hex = "0.4.3"
 indexmap = { version = "2.2.6", features = ["std"] }
-insta = "1.38.0"
-itertools = "0.11.0"
-is-macro = "0.3.0"
-junction = "1.0.0"
-libc = "0.2.153"
-log = "0.4.16"
-nix = { version = "0.27", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
-malachite-bigint = "0.2.0"
-malachite-q = "0.4.4"
-malachite-base = "0.4.4"
-memchr = "2.7.2"
-num-complex = "0.4.0"
-num-integer = "0.1.44"
+insta = "1.42"
+itertools = "0.14.0"
+is-macro = "0.3.7"
+junction = "1.2.0"
+libc = "0.2.169"
+libffi = "3.2"
+log = "0.4.27"
+nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
+malachite-bigint = "0.6"
+malachite-q = "0.6"
+malachite-base = "0.6"
+memchr = "2.7.4"
+num-complex = "0.4.6"
+num-integer = "0.1.46"
 num-traits = "0.2"
-num_enum = "0.7"
-once_cell = "1.19.0"
-parking_lot = "0.12.1"
-paste = "1.0.7"
-rand = "0.8.5"
-rustix = { version = "0.38", features = ["event"] }
-rustyline = "14.0.0"
+num_enum = { version = "0.7", default-features = false }
+optional = "0.5"
+once_cell = "1.20.3"
+parking_lot = "0.12.3"
+paste = "1.0.15"
+proc-macro2 = "1.0.93"
+quote = "1.0.38"
+rand = "0.9"
+rand_core = { version = "0.9", features = ["os_rng"] }
+rustix = { version = "1.0", features = ["event"] }
+rustyline = "15.0.0"
 serde = { version = "1.0.133", default-features = false }
-schannel = "0.1.22"
+schannel = "0.1.27"
 static_assertions = "1.1"
-syn = "1.0.109"
-thiserror = "1.0"
-thread_local = "1.1.4"
-unicode_names2 = "1.2.0"
+strum = "0.27"
+strum_macros = "0.27"
+syn = "2"
+thiserror = "2.0"
+thread_local = "1.1.8"
+unicode-casing = "0.1.0"
+unic-char-property = "0.9.0"
+unic-normal = "0.9.0"
+unic-ucd-age = "0.9.0"
+unic-ucd-bidi = "0.9.0"
+unic-ucd-category = "0.9.0"
+unic-ucd-ident = "0.9.0"
+unicode_names2 = "1.3.0"
 widestring = "1.1.0"
-windows-sys = "0.52.0"
-wasm-bindgen = "0.2.92"
+windows-sys = "0.59.0"
+wasm-bindgen = "0.2.100"
 
 # Lints
 
 [workspace.lints.rust]
 unsafe_code = "allow"
+unsafe_op_in_unsafe_fn = "deny"
+elided_lifetimes_in_paths = "warn"
 
 [workspace.lints.clippy]
 perf = "warn"

DEVELOPMENT.md

@@ -25,7 +25,7 @@ RustPython requires the following:
       stable version: `rustup update stable`
     - If you do not have Rust installed, use [rustup](https://rustup.rs/) to
       do so.
-- CPython version 3.12 or higher
+- CPython version 3.13 or higher
     - CPython can be installed by your operating system's package manager,
       from the [Python website](https://www.python.org/downloads/), or
       using a third-party distribution, such as 

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 RustPython Team
+Copyright (c) 2025 RustPython Team
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Lib/_aix_support.py

@@ -0,0 +1,108 @@
+"""Shared AIX support functions."""
+
+import sys
+import sysconfig
+
+
+# Taken from _osx_support _read_output function
+def _read_cmd_output(commandstring, capture_stderr=False):
+    """Output from successful command execution or None"""
+    # Similar to os.popen(commandstring, "r").read(),
+    # but without actually using os.popen because that
+    # function is not usable during python bootstrap.
+    import os
+    import contextlib
+    fp = open("/tmp/_aix_support.%s"%(
+        os.getpid(),), "w+b")
+
+    with contextlib.closing(fp) as fp:
+        if capture_stderr:
+            cmd = "%s >'%s' 2>&1" % (commandstring, fp.name)
+        else:
+            cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
+        return fp.read() if not os.system(cmd) else None
+
+
+def _aix_tag(vrtl, bd):
+    # type: (List[int], int) -> str
+    # Infer the ABI bitwidth from maxsize (assuming 64 bit as the default)
+    _sz = 32 if sys.maxsize == (2**31-1) else 64
+    _bd = bd if bd != 0 else 9988
+    # vrtl[version, release, technology_level]
+    return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], _bd, _sz)
+
+
+# extract version, release and technology level from a VRMF string
+def _aix_vrtl(vrmf):
+    # type: (str) -> List[int]
+    v, r, tl = vrmf.split(".")[:3]
+    return [int(v[-1]), int(r), int(tl)]
+
+
+def _aix_bos_rte():
+    # type: () -> Tuple[str, int]
+    """
+    Return a Tuple[str, int] e.g., ['7.1.4.34', 1806]
+    The fileset bos.rte represents the current AIX run-time level. It's VRMF and
+    builddate reflect the current ABI levels of the runtime environment.
+    If no builddate is found give a value that will satisfy pep425 related queries
+    """
+    # All AIX systems to have lslpp installed in this location
+    # subprocess may not be available during python bootstrap
+    try:
+        import subprocess
+        out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.rte"])
+    except ImportError:
+        out = _read_cmd_output("/usr/bin/lslpp -Lqc bos.rte")
+    out = out.decode("utf-8")
+    out = out.strip().split(":")  # type: ignore
+    _bd = int(out[-1]) if out[-1] != '' else 9988
+    return (str(out[2]), _bd)
+
+
+def aix_platform():
+    # type: () -> str
+    """
+    AIX filesets are identified by four decimal values: V.R.M.F.
+    V (version) and R (release) can be retrieved using ``uname``
+    Since 2007, starting with AIX 5.3 TL7, the M value has been
+    included with the fileset bos.rte and represents the Technology
+    Level (TL) of AIX. The F (Fix) value also increases, but is not
+    relevant for comparing releases and binary compatibility.
+    For binary compatibility the so-called builddate is needed.
+    Again, the builddate of an AIX release is associated with bos.rte.
+    AIX ABI compatibility is described  as guaranteed at: https://www.ibm.com/\
+    support/knowledgecenter/en/ssw_aix_72/install/binary_compatability.html
+
+    For pep425 purposes the AIX platform tag becomes:
+    "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(v, r, tl, builddate, bitsize)
+    e.g., "aix-6107-1415-32" for AIX 6.1 TL7 bd 1415, 32-bit
+    and, "aix-6107-1415-64" for AIX 6.1 TL7 bd 1415, 64-bit
+    """
+    vrmf, bd = _aix_bos_rte()
+    return _aix_tag(_aix_vrtl(vrmf), bd)
+
+
+# extract vrtl from the BUILD_GNU_TYPE as an int
+def _aix_bgt():
+    # type: () -> List[int]
+    gnu_type = sysconfig.get_config_var("BUILD_GNU_TYPE")
+    if not gnu_type:
+        raise ValueError("BUILD_GNU_TYPE is not defined")
+    return _aix_vrtl(vrmf=gnu_type)
+
+
+def aix_buildtag():
+    # type: () -> str
+    """
+    Return the platform_tag of the system Python was built on.
+    """
+    # AIX_BUILDDATE is defined by configure with:
+    # lslpp -Lcq bos.rte | awk -F:  '{ print $NF }'
+    build_date = sysconfig.get_config_var("AIX_BUILDDATE")
+    try:
+        build_date = int(build_date)
+    except (ValueError, TypeError):
+        raise ValueError(f"AIX_BUILDDATE is not defined or invalid: "
+                         f"{build_date!r}")
+    return _aix_tag(_aix_bgt(), build_date)

Lib/_android_support.py

@@ -0,0 +1,181 @@
+import io
+import sys
+from threading import RLock
+from time import sleep, time
+
+# The maximum length of a log message in bytes, including the level marker and
+# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71.
+# Messages longer than this will be truncated by logcat. This limit has already
+# been reduced at least once in the history of Android (from 4076 to 4068 between
+# API level 23 and 26), so leave some headroom.
+MAX_BYTES_PER_WRITE = 4000
+
+# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this
+# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE.
+# However, if the actual number of bytes per character is smaller than that,
+# then we may still join multiple consecutive text writes into binary
+# writes containing a larger number of characters.
+MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4
+
+
+# When embedded in an app on current versions of Android, there's no easy way to
+# monitor the C-level stdout and stderr. The testbed comes with a .c file to
+# redirect them to the system log using a pipe, but that wouldn't be convenient
+# or appropriate for all apps. So we redirect at the Python level instead.
+def init_streams(android_log_write, stdout_prio, stderr_prio):
+    if sys.executable:
+        return  # Not embedded in an app.
+
+    global logcat
+    logcat = Logcat(android_log_write)
+
+    sys.stdout = TextLogStream(
+        stdout_prio, "python.stdout", sys.stdout.fileno())
+    sys.stderr = TextLogStream(
+        stderr_prio, "python.stderr", sys.stderr.fileno())
+
+
+class TextLogStream(io.TextIOWrapper):
+    def __init__(self, prio, tag, fileno=None, **kwargs):
+        # The default is surrogateescape for stdout and backslashreplace for
+        # stderr, but in the context of an Android log, readability is more
+        # important than reversibility.
+        kwargs.setdefault("encoding", "UTF-8")
+        kwargs.setdefault("errors", "backslashreplace")
+
+        super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs)
+        self._lock = RLock()
+        self._pending_bytes = []
+        self._pending_bytes_count = 0
+
+    def __repr__(self):
+        return f"<TextLogStream {self.buffer.tag!r}>"
+
+    def write(self, s):
+        if not isinstance(s, str):
+            raise TypeError(
+                f"write() argument must be str, not {type(s).__name__}")
+
+        # In case `s` is a str subclass that writes itself to stdout or stderr
+        # when we call its methods, convert it to an actual str.
+        s = str.__str__(s)
+
+        # We want to emit one log message per line wherever possible, so split
+        # the string into lines first. Note that "".splitlines() == [], so
+        # nothing will be logged for an empty string.
+        with self._lock:
+            for line in s.splitlines(keepends=True):
+                while line:
+                    chunk = line[:MAX_CHARS_PER_WRITE]
+                    line = line[MAX_CHARS_PER_WRITE:]
+                    self._write_chunk(chunk)
+
+        return len(s)
+
+    # The size and behavior of TextIOWrapper's buffer is not part of its public
+    # API, so we handle buffering ourselves to avoid truncation.
+    def _write_chunk(self, s):
+        b = s.encode(self.encoding, self.errors)
+        if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE:
+            self.flush()
+
+        self._pending_bytes.append(b)
+        self._pending_bytes_count += len(b)
+        if (
+            self.write_through
+            or b.endswith(b"\n")
+            or self._pending_bytes_count > MAX_BYTES_PER_WRITE
+        ):
+            self.flush()
+
+    def flush(self):
+        with self._lock:
+            self.buffer.write(b"".join(self._pending_bytes))
+            self._pending_bytes.clear()
+            self._pending_bytes_count = 0
+
+    # Since this is a line-based logging system, line buffering cannot be turned
+    # off, i.e. a newline always causes a flush.
+    @property
+    def line_buffering(self):
+        return True
+
+
+class BinaryLogStream(io.RawIOBase):
+    def __init__(self, prio, tag, fileno=None):
+        self.prio = prio
+        self.tag = tag
+        self._fileno = fileno
+
+    def __repr__(self):
+        return f"<BinaryLogStream {self.tag!r}>"
+
+    def writable(self):
+        return True
+
+    def write(self, b):
+        if type(b) is not bytes:
+            try:
+                b = bytes(memoryview(b))
+            except TypeError:
+                raise TypeError(
+                    f"write() argument must be bytes-like, not {type(b).__name__}"
+                ) from None
+
+        # Writing an empty string to the stream should have no effect.
+        if b:
+            logcat.write(self.prio, self.tag, b)
+        return len(b)
+
+    # This is needed by the test suite --timeout option, which uses faulthandler.
+    def fileno(self):
+        if self._fileno is None:
+            raise io.UnsupportedOperation("fileno")
+        return self._fileno
+
+
+# When a large volume of data is written to logcat at once, e.g. when a test
+# module fails in --verbose3 mode, there's a risk of overflowing logcat's own
+# buffer and losing messages. We avoid this by imposing a rate limit using the
+# token bucket algorithm, based on a conservative estimate of how fast `adb
+# logcat` can consume data.
+MAX_BYTES_PER_SECOND = 1024 * 1024
+
+# The logcat buffer size of a device can be determined by running `logcat -g`.
+# We set the token bucket size to half of the buffer size of our current minimum
+# API level, because other things on the system will be producing messages as
+# well.
+BUCKET_SIZE = 128 * 1024
+
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39
+PER_MESSAGE_OVERHEAD = 28
+
+
+class Logcat:
+    def __init__(self, android_log_write):
+        self.android_log_write = android_log_write
+        self._lock = RLock()
+        self._bucket_level = 0
+        self._prev_write_time = time()
+
+    def write(self, prio, tag, message):
+        # Encode null bytes using "modified UTF-8" to avoid them truncating the
+        # message.
+        message = message.replace(b"\x00", b"\xc0\x80")
+
+        with self._lock:
+            now = time()
+            self._bucket_level += (
+                (now - self._prev_write_time) * MAX_BYTES_PER_SECOND)
+
+            # If the bucket level is still below zero, the clock must have gone
+            # backwards, so reset it to zero and continue.
+            self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE))
+            self._prev_write_time = now
+
+            self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message)
+            if self._bucket_level < 0:
+                sleep(-self._bucket_level / MAX_BYTES_PER_SECOND)
+
+            self.android_log_write(prio, tag, message)

Lib/_apple_support.py

@@ -0,0 +1,66 @@
+import io
+import sys
+
+
+def init_streams(log_write, stdout_level, stderr_level):
+    # Redirect stdout and stderr to the Apple system log. This method is
+    # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger
+    # is enabled.
+    sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors)
+    sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors)
+
+
+class SystemLog(io.TextIOWrapper):
+    def __init__(self, log_write, level, **kwargs):
+        kwargs.setdefault("encoding", "UTF-8")
+        kwargs.setdefault("line_buffering", True)
+        super().__init__(LogStream(log_write, level), **kwargs)
+
+    def __repr__(self):
+        return f"<SystemLog (level {self.buffer.level})>"
+
+    def write(self, s):
+        if not isinstance(s, str):
+            raise TypeError(
+                f"write() argument must be str, not {type(s).__name__}")
+
+        # In case `s` is a str subclass that writes itself to stdout or stderr
+        # when we call its methods, convert it to an actual str.
+        s = str.__str__(s)
+
+        # We want to emit one log message per line, so split
+        # the string before sending it to the superclass.
+        for line in s.splitlines(keepends=True):
+            super().write(line)
+
+        return len(s)
+
+
+class LogStream(io.RawIOBase):
+    def __init__(self, log_write, level):
+        self.log_write = log_write
+        self.level = level
+
+    def __repr__(self):
+        return f"<LogStream (level {self.level!r})>"
+
+    def writable(self):
+        return True
+
+    def write(self, b):
+        if type(b) is not bytes:
+            try:
+                b = bytes(memoryview(b))
+            except TypeError:
+                raise TypeError(
+                    f"write() argument must be bytes-like, not {type(b).__name__}"
+                ) from None
+
+        # Writing an empty string to the stream should have no effect.
+        if b:
+            # Encode null bytes using "modified UTF-8" to avoid truncating the
+            # message. This should not affect the return value, as the caller
+            # may be expecting it to match the length of the input.
+            self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80"))
+
+        return len(b)

Lib/_colorize.py

@@ -0,0 +1,67 @@
+import io
+import os
+import sys
+
+COLORIZE = True
+
+
+class ANSIColors:
+    BOLD_GREEN = "\x1b[1;32m"
+    BOLD_MAGENTA = "\x1b[1;35m"
+    BOLD_RED = "\x1b[1;31m"
+    GREEN = "\x1b[32m"
+    GREY = "\x1b[90m"
+    MAGENTA = "\x1b[35m"
+    RED = "\x1b[31m"
+    RESET = "\x1b[0m"
+    YELLOW = "\x1b[33m"
+
+
+NoColors = ANSIColors()
+
+for attr in dir(NoColors):
+    if not attr.startswith("__"):
+        setattr(NoColors, attr, "")
+
+
+def get_colors(colorize: bool = False, *, file=None) -> ANSIColors:
+    if colorize or can_colorize(file=file):
+        return ANSIColors()
+    else:
+        return NoColors
+
+
+def can_colorize(*, file=None) -> bool:
+    if file is None:
+        file = sys.stdout
+
+    if not sys.flags.ignore_environment:
+        if os.environ.get("PYTHON_COLORS") == "0":
+            return False
+        if os.environ.get("PYTHON_COLORS") == "1":
+            return True
+    if os.environ.get("NO_COLOR"):
+        return False
+    if not COLORIZE:
+        return False
+    if os.environ.get("FORCE_COLOR"):
+        return True
+    if os.environ.get("TERM") == "dumb":
+        return False
+
+    if not hasattr(file, "fileno"):
+        return False
+
+    if sys.platform == "win32":
+        try:
+            import nt
+
+            if not nt._supports_virtual_terminal():
+                return False
+        except (ImportError, AttributeError):
+            return False
+
+    try:
+        return os.isatty(file.fileno())
+    except io.UnsupportedOperation:
+        return file.isatty()

Lib/_dummy_os.py

@@ -5,22 +5,30 @@ A shim of the os module containing only simple path-related utilities
 try:
     from os import *
 except ImportError:
-    import abc
+    import abc, sys
 
     def __getattr__(name):
-        raise OSError("no os specific module found")
+        if name in {"_path_normpath", "__path__"}:
+            raise AttributeError(name)
+        if name.isupper():
+            return 0
+        def dummy(*args, **kwargs):
+            import io
+            return io.UnsupportedOperation(f"{name}: no os specific module found")
+        dummy.__name__ = f"dummy_{name}"
+        return dummy
 
-    def _shim():
-        import _dummy_os, sys
-        sys.modules['os'] = _dummy_os
-        sys.modules['os.path'] = _dummy_os.path
+    sys.modules['os'] = sys.modules['posix'] = sys.modules[__name__]
 
     import posixpath as path
-    import sys
     sys.modules['os.path'] = path
     del sys
 
     sep = path.sep
+    supports_dir_fd = set()
+    supports_effective_ids = set()
+    supports_fd = set()
+    supports_follow_symlinks = set()
 
 
     def fspath(path):

Lib/_ios_support.py

@@ -0,0 +1,71 @@
+import sys
+try:
+    from ctypes import cdll, c_void_p, c_char_p, util
+except ImportError:
+    # ctypes is an optional module. If it's not present, we're limited in what
+    # we can tell about the system, but we don't want to prevent the module
+    # from working.
+    print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr)
+    objc = None
+else:
+    # ctypes is available. Load the ObjC library, and wrap the objc_getClass,
+    # sel_registerName methods
+    lib = util.find_library("objc")
+    if lib is None:
+        # Failed to load the objc library
+        raise ImportError("ObjC runtime library couldn't be loaded")
+
+    objc = cdll.LoadLibrary(lib)
+    objc.objc_getClass.restype = c_void_p
+    objc.objc_getClass.argtypes = [c_char_p]
+    objc.sel_registerName.restype = c_void_p
+    objc.sel_registerName.argtypes = [c_char_p]
+
+
+def get_platform_ios():
+    # Determine if this is a simulator using the multiarch value
+    is_simulator = sys.implementation._multiarch.endswith("simulator")
+
+    # We can't use ctypes; abort
+    if not objc:
+        return None
+
+    # Most of the methods return ObjC objects
+    objc.objc_msgSend.restype = c_void_p
+    # All the methods used have no arguments.
+    objc.objc_msgSend.argtypes = [c_void_p, c_void_p]
+
+    # Equivalent of:
+    #   device = [UIDevice currentDevice]
+    UIDevice = objc.objc_getClass(b"UIDevice")
+    SEL_currentDevice = objc.sel_registerName(b"currentDevice")
+    device = objc.objc_msgSend(UIDevice, SEL_currentDevice)
+
+    # Equivalent of:
+    #   device_systemVersion = [device systemVersion]
+    SEL_systemVersion = objc.sel_registerName(b"systemVersion")
+    device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion)
+
+    # Equivalent of:
+    #   device_systemName = [device systemName]
+    SEL_systemName = objc.sel_registerName(b"systemName")
+    device_systemName = objc.objc_msgSend(device, SEL_systemName)
+
+    # Equivalent of:
+    #   device_model = [device model]
+    SEL_model = objc.sel_registerName(b"model")
+    device_model = objc.objc_msgSend(device, SEL_model)
+
+    # UTF8String returns a const char*;
+    SEL_UTF8String = objc.sel_registerName(b"UTF8String")
+    objc.objc_msgSend.restype = c_char_p
+
+    # Equivalent of:
+    #   system = [device_systemName UTF8String]
+    #   release = [device_systemVersion UTF8String]
+    #   model = [device_model UTF8String]
+    system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode()
+    release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode()
+    model = objc.objc_msgSend(device_model, SEL_UTF8String).decode()
+
+    return system, release, model, is_simulator

Lib/_osx_support.py

@@ -507,6 +507,11 @@ def get_platform_osx(_config_vars, osname, release, machine):
     # MACOSX_DEPLOYMENT_TARGET.
 
     macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '')
+    if macver and '.' not in macver:
+        # Ensure that the version includes at least a major
+        # and minor version, even if MACOSX_DEPLOYMENT_TARGET
+        # is set to a single-label version like "14".
+        macver += '.0'
     macrelease = _get_system_version() or macver
     macver = macver or macrelease
 

Lib/_py_abc.py

@@ -33,6 +33,8 @@ class ABCMeta(type):
     _abc_invalidation_counter = 0
 
     def __new__(mcls, name, bases, namespace, /, **kwargs):
+        # TODO: RUSTPYTHON remove this line (prevents duplicate bases)
+        bases = tuple(dict.fromkeys(bases))
         cls = super().__new__(mcls, name, bases, namespace, **kwargs)
         # Compute set of abstract method names
         abstracts = {name
@@ -98,8 +100,8 @@ class ABCMeta(type):
         subtype = type(instance)
         if subtype is subclass:
             if (cls._abc_negative_cache_version ==
-                ABCMeta._abc_invalidation_counter and
-                subclass in cls._abc_negative_cache):
+                    ABCMeta._abc_invalidation_counter and
+                    subclass in cls._abc_negative_cache):
                 return False
             # Fall back to the subclass check.
             return cls.__subclasscheck__(subclass)

Lib/_pycodecs.py

@@ -1086,11 +1086,13 @@ def charmapencode_output(c, mapping):
     rep = mapping[c]
     if isinstance(rep, int) or isinstance(rep, int):
         if rep < 256:
-            return rep
+            return [rep]
         else:
             raise TypeError("character mapping must be in range(256)")
     elif isinstance(rep, str):
-        return ord(rep)
+        return [ord(rep)]
+    elif isinstance(rep, bytes):
+        return rep
     elif rep == None:
         raise KeyError("character maps to <undefined>")
     else:
@@ -1113,12 +1115,13 @@ def PyUnicode_EncodeCharmap(p, size, mapping='latin-1', errors='strict'):
         #/* try to encode it */
         try:
             x = charmapencode_output(ord(p[inpos]), mapping)
-            res += [x]
+            res += x
         except KeyError:
             x = unicode_call_errorhandler(errors, "charmap",
             "character maps to <undefined>", p, inpos, inpos+1, False)
             try:
-                res += [charmapencode_output(ord(y), mapping) for y in x[0]]
+                for y in x[0]:
+                    res += charmapencode_output(ord(y), mapping)
             except KeyError:
                 raise UnicodeEncodeError("charmap", p, inpos, inpos+1,
                                         "character maps to <undefined>")

Lib/_pylong.py

@@ -0,0 +1,363 @@
+"""Python implementations of some algorithms for use by longobject.c.
+The goal is to provide asymptotically faster algorithms that can be
+used for operations on integers with many digits.  In those cases, the
+performance overhead of the Python implementation is not significant
+since the asymptotic behavior is what dominates runtime. Functions
+provided by this module should be considered private and not part of any
+public API.
+
+Note: for ease of maintainability, please prefer clear code and avoid
+"micro-optimizations".  This module will only be imported and used for
+integers with a huge number of digits.  Saving a few microseconds with
+tricky or non-obvious code is not worth it.  For people looking for
+maximum performance, they should use something like gmpy2."""
+
+import re
+import decimal
+try:
+    import _decimal
+except ImportError:
+    _decimal = None
+
+# A number of functions have this form, where `w` is a desired number of
+# digits in base `base`:
+#
+#    def inner(...w...):
+#        if w <= LIMIT:
+#            return something
+#        lo = w >> 1
+#        hi = w - lo
+#        something involving base**lo, inner(...lo...), j, and inner(...hi...)
+#    figure out largest w needed
+#    result = inner(w)
+#
+# They all had some on-the-fly scheme to cache `base**lo` results for reuse.
+# Power is costly.
+#
+# This routine aims to compute all amd only the needed powers in advance, as
+# efficiently as reasonably possible. This isn't trivial, and all the
+# on-the-fly methods did needless work in many cases. The driving code above
+# changes to:
+#
+#    figure out largest w needed
+#    mycache = compute_powers(w, base, LIMIT)
+#    result = inner(w)
+#
+# and `mycache[lo]` replaces `base**lo` in the inner function.
+#
+# While this does give minor speedups (a few percent at best), the primary
+# intent is to simplify the functions using this, by eliminating the need for
+# them to craft their own ad-hoc caching schemes.
+def compute_powers(w, base, more_than, show=False):
+    seen = set()
+    need = set()
+    ws = {w}
+    while ws:
+        w = ws.pop() # any element is fine to use next
+        if w in seen or w <= more_than:
+            continue
+        seen.add(w)
+        lo = w >> 1
+        # only _need_ lo here; some other path may, or may not, need hi
+        need.add(lo)
+        ws.add(lo)
+        if w & 1:
+            ws.add(lo + 1)
+
+    d = {}
+    if not need:
+        return d
+    it = iter(sorted(need))
+    first = next(it)
+    if show:
+        print("pow at", first)
+    d[first] = base ** first
+    for this in it:
+        if this - 1 in d:
+            if show:
+                print("* base at", this)
+            d[this] = d[this - 1] * base # cheap
+        else:
+            lo = this >> 1
+            hi = this - lo
+            assert lo in d
+            if show:
+                print("square at", this)
+            # Multiplying a bigint by itself (same object!) is about twice
+            # as fast in CPython.
+            sq = d[lo] * d[lo]
+            if hi != lo:
+                assert hi == lo + 1
+                if show:
+                    print("    and * base")
+                sq *= base
+            d[this] = sq
+    return d
+
+_unbounded_dec_context = decimal.getcontext().copy()
+_unbounded_dec_context.prec = decimal.MAX_PREC
+_unbounded_dec_context.Emax = decimal.MAX_EMAX
+_unbounded_dec_context.Emin = decimal.MIN_EMIN
+_unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check
+
+def int_to_decimal(n):
+    """Asymptotically fast conversion of an 'int' to Decimal."""
+
+    # Function due to Tim Peters.  See GH issue #90716 for details.
+    # https://github.com/python/cpython/issues/90716
+    #
+    # The implementation in longobject.c of base conversion algorithms
+    # between power-of-2 and non-power-of-2 bases are quadratic time.
+    # This function implements a divide-and-conquer algorithm that is
+    # faster for large numbers.  Builds an equal decimal.Decimal in a
+    # "clever" recursive way.  If we want a string representation, we
+    # apply str to _that_.
+
+    from decimal import Decimal as D
+    BITLIM = 200
+
+    # Don't bother caching the "lo" mask in this; the time to compute it is
+    # tiny compared to the multiply.
+    def inner(n, w):
+        if w <= BITLIM:
+            return D(n)
+        w2 = w >> 1
+        hi = n >> w2
+        lo = n & ((1 << w2) - 1)
+        return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2]
+
+    with decimal.localcontext(_unbounded_dec_context):
+        nbits = n.bit_length()
+        w2pow = compute_powers(nbits, D(2), BITLIM)
+        if n < 0:
+            negate = True
+            n = -n
+        else:
+            negate = False
+        result = inner(n, nbits)
+        if negate:
+            result = -result
+    return result
+
+def int_to_decimal_string(n):
+    """Asymptotically fast conversion of an 'int' to a decimal string."""
+    w = n.bit_length()
+    if w > 450_000 and _decimal is not None:
+        # It is only usable with the C decimal implementation.
+        # _pydecimal.py calls str() on very large integers, which in its
+        # turn calls int_to_decimal_string(), causing very deep recursion.
+        return str(int_to_decimal(n))
+
+    # Fallback algorithm for the case when the C decimal module isn't
+    # available.  This algorithm is asymptotically worse than the algorithm
+    # using the decimal module, but better than the quadratic time
+    # implementation in longobject.c.
+
+    DIGLIM = 1000
+    def inner(n, w):
+        if w <= DIGLIM:
+            return str(n)
+        w2 = w >> 1
+        hi, lo = divmod(n, pow10[w2])
+        return inner(hi, w - w2) + inner(lo, w2).zfill(w2)
+
+    # The estimation of the number of decimal digits.
+    # There is no harm in small error.  If we guess too large, there may
+    # be leading 0's that need to be stripped.  If we guess too small, we
+    # may need to call str() recursively for the remaining highest digits,
+    # which can still potentially be a large integer. This is manifested
+    # only if the number has way more than 10**15 digits, that exceeds
+    # the 52-bit physical address limit in both Intel64 and AMD64.
+    w = int(w * 0.3010299956639812 + 1)  # log10(2)
+    pow10 = compute_powers(w, 5, DIGLIM)
+    for k, v in pow10.items():
+        pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k
+    if n < 0:
+        n = -n
+        sign = '-'
+    else:
+        sign = ''
+    s = inner(n, w)
+    if s[0] == '0' and n:
+        # If our guess of w is too large, there may be leading 0's that
+        # need to be stripped.
+        s = s.lstrip('0')
+    return sign + s
+
+def _str_to_int_inner(s):
+    """Asymptotically fast conversion of a 'str' to an 'int'."""
+
+    # Function due to Bjorn Martinsson.  See GH issue #90716 for details.
+    # https://github.com/python/cpython/issues/90716
+    #
+    # The implementation in longobject.c of base conversion algorithms
+    # between power-of-2 and non-power-of-2 bases are quadratic time.
+    # This function implements a divide-and-conquer algorithm making use
+    # of Python's built in big int multiplication. Since Python uses the
+    # Karatsuba algorithm for multiplication, the time complexity
+    # of this function is O(len(s)**1.58).
+
+    DIGLIM = 2048
+
+    def inner(a, b):
+        if b - a <= DIGLIM:
+            return int(s[a:b])
+        mid = (a + b + 1) >> 1
+        return (inner(mid, b)
+                + ((inner(a, mid) * w5pow[b - mid])
+                    << (b - mid)))
+
+    w5pow = compute_powers(len(s), 5, DIGLIM)
+    return inner(0, len(s))
+
+
+def int_from_string(s):
+    """Asymptotically fast version of PyLong_FromString(), conversion
+    of a string of decimal digits into an 'int'."""
+    # PyLong_FromString() has already removed leading +/-, checked for invalid
+    # use of underscore characters, checked that string consists of only digits
+    # and underscores, and stripped leading whitespace.  The input can still
+    # contain underscores and have trailing whitespace.
+    s = s.rstrip().replace('_', '')
+    return _str_to_int_inner(s)
+
+def str_to_int(s):
+    """Asymptotically fast version of decimal string to 'int' conversion."""
+    # FIXME: this doesn't support the full syntax that int() supports.
+    m = re.match(r'\s*([+-]?)([0-9_]+)\s*', s)
+    if not m:
+        raise ValueError('invalid literal for int() with base 10')
+    v = int_from_string(m.group(2))
+    if m.group(1) == '-':
+        v = -v
+    return v
+
+
+# Fast integer division, based on code from Mark Dickinson, fast_div.py
+# GH-47701. Additional refinements and optimizations by Bjorn Martinsson.  The
+# algorithm is due to Burnikel and Ziegler, in their paper "Fast Recursive
+# Division".
+
+_DIV_LIMIT = 4000
+
+
+def _div2n1n(a, b, n):
+    """Divide a 2n-bit nonnegative integer a by an n-bit positive integer
+    b, using a recursive divide-and-conquer algorithm.
+
+    Inputs:
+      n is a positive integer
+      b is a positive integer with exactly n bits
+      a is a nonnegative integer such that a < 2**n * b
+
+    Output:
+      (q, r) such that a = b*q+r and 0 <= r < b.
+
+    """
+    if a.bit_length() - n <= _DIV_LIMIT:
+        return divmod(a, b)
+    pad = n & 1
+    if pad:
+        a <<= 1
+        b <<= 1
+        n += 1
+    half_n = n >> 1
+    mask = (1 << half_n) - 1
+    b1, b2 = b >> half_n, b & mask
+    q1, r = _div3n2n(a >> n, (a >> half_n) & mask, b, b1, b2, half_n)
+    q2, r = _div3n2n(r, a & mask, b, b1, b2, half_n)
+    if pad:
+        r >>= 1
+    return q1 << half_n | q2, r
+
+
+def _div3n2n(a12, a3, b, b1, b2, n):
+    """Helper function for _div2n1n; not intended to be called directly."""
+    if a12 >> n == b1:
+        q, r = (1 << n) - 1, a12 - (b1 << n) + b1
+    else:
+        q, r = _div2n1n(a12, b1, n)
+    r = (r << n | a3) - q * b2
+    while r < 0:
+        q -= 1
+        r += b
+    return q, r
+
+
+def _int2digits(a, n):
+    """Decompose non-negative int a into base 2**n
+
+    Input:
+      a is a non-negative integer
+
+    Output:
+      List of the digits of a in base 2**n in little-endian order,
+      meaning the most significant digit is last. The most
+      significant digit is guaranteed to be non-zero.
+      If a is 0 then the output is an empty list.
+
+    """
+    a_digits = [0] * ((a.bit_length() + n - 1) // n)
+
+    def inner(x, L, R):
+        if L + 1 == R:
+            a_digits[L] = x
+            return
+        mid = (L + R) >> 1
+        shift = (mid - L) * n
+        upper = x >> shift
+        lower = x ^ (upper << shift)
+        inner(lower, L, mid)
+        inner(upper, mid, R)
+
+    if a:
+        inner(a, 0, len(a_digits))
+    return a_digits
+
+
+def _digits2int(digits, n):
+    """Combine base-2**n digits into an int. This function is the
+    inverse of `_int2digits`. For more details, see _int2digits.
+    """
+
+    def inner(L, R):
+        if L + 1 == R:
+            return digits[L]
+        mid = (L + R) >> 1
+        shift = (mid - L) * n
+        return (inner(mid, R) << shift) + inner(L, mid)
+
+    return inner(0, len(digits)) if digits else 0
+
+
+def _divmod_pos(a, b):
+    """Divide a non-negative integer a by a positive integer b, giving
+    quotient and remainder."""
+    # Use grade-school algorithm in base 2**n, n = nbits(b)
+    n = b.bit_length()
+    a_digits = _int2digits(a, n)
+
+    r = 0
+    q_digits = []
+    for a_digit in reversed(a_digits):
+        q_digit, r = _div2n1n((r << n) + a_digit, b, n)
+        q_digits.append(q_digit)
+    q_digits.reverse()
+    q = _digits2int(q_digits, n)
+    return q, r
+
+
+def int_divmod(a, b):
+    """Asymptotically fast replacement for divmod, for 'int'.
+    Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b).
+    """
+    if b == 0:
+        raise ZeroDivisionError
+    elif b < 0:
+        q, r = int_divmod(-a, -b)
+        return q, -r
+    elif a < 0:
+        q, r = int_divmod(~a, b)
+        return ~q, b + ~r
+    else:
+        return _divmod_pos(a, b)

Lib/_pyrepl/__init__.py

@@ -0,0 +1,19 @@
+#   Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

Lib/_pyrepl/__main__.py

@@ -0,0 +1,6 @@
+# Important: don't add things to this module, as they will end up in the REPL's
+# default globals.  Use _pyrepl.main instead.
+
+if __name__ == "__main__":
+    from .main import interactive_console as __pyrepl_interactive_console
+    __pyrepl_interactive_console()

Lib/_pyrepl/_minimal_curses.py

@@ -0,0 +1,68 @@
+"""Minimal '_curses' module, the low-level interface for curses module
+which is not meant to be used directly.
+
+Based on ctypes.  It's too incomplete to be really called '_curses', so
+to use it, you have to import it and stick it in sys.modules['_curses']
+manually.
+
+Note that there is also a built-in module _minimal_curses which will
+hide this one if compiled in.
+"""
+
+import ctypes
+import ctypes.util
+
+
+class error(Exception):
+    pass
+
+
+def _find_clib() -> str:
+    trylibs = ["ncursesw", "ncurses", "curses"]
+
+    for lib in trylibs:
+        path = ctypes.util.find_library(lib)
+        if path:
+            return path
+    raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses")
+
+
+_clibpath = _find_clib()
+clib = ctypes.cdll.LoadLibrary(_clibpath)
+
+clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
+clib.setupterm.restype = ctypes.c_int
+
+clib.tigetstr.argtypes = [ctypes.c_char_p]
+clib.tigetstr.restype = ctypes.c_ssize_t
+
+clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int]  # type: ignore[operator]
+clib.tparm.restype = ctypes.c_char_p
+
+OK = 0
+ERR = -1
+
+# ____________________________________________________________
+
+
+def setupterm(termstr, fd):
+    err = ctypes.c_int(0)
+    result = clib.setupterm(termstr, fd, ctypes.byref(err))
+    if result == ERR:
+        raise error("setupterm() failed (err=%d)" % err.value)
+
+
+def tigetstr(cap):
+    if not isinstance(cap, bytes):
+        cap = cap.encode("ascii")
+    result = clib.tigetstr(cap)
+    if result == ERR:
+        return None
+    return ctypes.cast(result, ctypes.c_char_p).value
+
+
+def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0):
+    result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9)
+    if result is None:
+        raise error("tparm() returned NULL")
+    return result

Lib/_pyrepl/_threading_handler.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import traceback
+
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from threading import Thread
+    from types import TracebackType
+    from typing import Protocol
+
+    class ExceptHookArgs(Protocol):
+        @property
+        def exc_type(self) -> type[BaseException]: ...
+        @property
+        def exc_value(self) -> BaseException | None: ...
+        @property
+        def exc_traceback(self) -> TracebackType | None: ...
+        @property
+        def thread(self) -> Thread | None: ...
+
+    class ShowExceptions(Protocol):
+        def __call__(self) -> int: ...
+        def add(self, s: str) -> None: ...
+
+    from .reader import Reader
+
+
+def install_threading_hook(reader: Reader) -> None:
+    import threading
+
+    @dataclass
+    class ExceptHookHandler:
+        lock: threading.Lock = field(default_factory=threading.Lock)
+        messages: list[str] = field(default_factory=list)
+
+        def show(self) -> int:
+            count = 0
+            with self.lock:
+                if not self.messages:
+                    return 0
+                reader.restore()
+                for tb in self.messages:
+                    count += 1
+                    if tb:
+                        print(tb)
+                self.messages.clear()
+                reader.scheduled_commands.append("ctrl-c")
+                reader.prepare()
+            return count
+
+        def add(self, s: str) -> None:
+            with self.lock:
+                self.messages.append(s)
+
+        def exception(self, args: ExceptHookArgs) -> None:
+            lines = traceback.format_exception(
+                args.exc_type,
+                args.exc_value,
+                args.exc_traceback,
+                colorize=reader.can_colorize,
+            )  # type: ignore[call-overload]
+            pre = f"\nException in {args.thread.name}:\n" if args.thread else "\n"
+            tb = pre + "".join(lines)
+            self.add(tb)
+
+        def __call__(self) -> int:
+            return self.show()
+
+
+    handler = ExceptHookHandler()
+    reader.threading_hook = handler
+    threading.excepthook = handler.exception

Lib/_pyrepl/commands.py

@@ -0,0 +1,489 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Antonio Cuni
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+import os
+
+# Categories of actions:
+#  killing
+#  yanking
+#  motion
+#  editing
+#  history
+#  finishing
+# [completion]
+
+
+# types
+if False:
+    from .historical_reader import HistoricalReader
+
+
+class Command:
+    finish: bool = False
+    kills_digit_arg: bool = True
+
+    def __init__(
+        self, reader: HistoricalReader, event_name: str, event: list[str]
+    ) -> None:
+        # Reader should really be "any reader" but there's too much usage of
+        # HistoricalReader methods and fields in the code below for us to
+        # refactor at the moment.
+
+        self.reader = reader
+        self.event = event
+        self.event_name = event_name
+
+    def do(self) -> None:
+        pass
+
+
+class KillCommand(Command):
+    def kill_range(self, start: int, end: int) -> None:
+        if start == end:
+            return
+        r = self.reader
+        b = r.buffer
+        text = b[start:end]
+        del b[start:end]
+        if is_kill(r.last_command):
+            if start < r.pos:
+                r.kill_ring[-1] = text + r.kill_ring[-1]
+            else:
+                r.kill_ring[-1] = r.kill_ring[-1] + text
+        else:
+            r.kill_ring.append(text)
+        r.pos = start
+        r.dirty = True
+
+
+class YankCommand(Command):
+    pass
+
+
+class MotionCommand(Command):
+    pass
+
+
+class EditCommand(Command):
+    pass
+
+
+class FinishCommand(Command):
+    finish = True
+    pass
+
+
+def is_kill(command: type[Command] | None) -> bool:
+    return command is not None and issubclass(command, KillCommand)
+
+
+def is_yank(command: type[Command] | None) -> bool:
+    return command is not None and issubclass(command, YankCommand)
+
+
+# etc
+
+
+class digit_arg(Command):
+    kills_digit_arg = False
+
+    def do(self) -> None:
+        r = self.reader
+        c = self.event[-1]
+        if c == "-":
+            if r.arg is not None:
+                r.arg = -r.arg
+            else:
+                r.arg = -1
+        else:
+            d = int(c)
+            if r.arg is None:
+                r.arg = d
+            else:
+                if r.arg < 0:
+                    r.arg = 10 * r.arg - d
+                else:
+                    r.arg = 10 * r.arg + d
+        r.dirty = True
+
+
+class clear_screen(Command):
+    def do(self) -> None:
+        r = self.reader
+        r.console.clear()
+        r.dirty = True
+
+
+class refresh(Command):
+    def do(self) -> None:
+        self.reader.dirty = True
+
+
+class repaint(Command):
+    def do(self) -> None:
+        self.reader.dirty = True
+        self.reader.console.repaint()
+
+
+class kill_line(KillCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        eol = r.eol()
+        for c in b[r.pos : eol]:
+            if not c.isspace():
+                self.kill_range(r.pos, eol)
+                return
+        else:
+            self.kill_range(r.pos, eol + 1)
+
+
+class unix_line_discard(KillCommand):
+    def do(self) -> None:
+        r = self.reader
+        self.kill_range(r.bol(), r.pos)
+
+
+class unix_word_rubout(KillCommand):
+    def do(self) -> None:
+        r = self.reader
+        for i in range(r.get_arg()):
+            self.kill_range(r.bow(), r.pos)
+
+
+class kill_word(KillCommand):
+    def do(self) -> None:
+        r = self.reader
+        for i in range(r.get_arg()):
+            self.kill_range(r.pos, r.eow())
+
+
+class backward_kill_word(KillCommand):
+    def do(self) -> None:
+        r = self.reader
+        for i in range(r.get_arg()):
+            self.kill_range(r.bow(), r.pos)
+
+
+class yank(YankCommand):
+    def do(self) -> None:
+        r = self.reader
+        if not r.kill_ring:
+            r.error("nothing to yank")
+            return
+        r.insert(r.kill_ring[-1])
+
+
+class yank_pop(YankCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        if not r.kill_ring:
+            r.error("nothing to yank")
+            return
+        if not is_yank(r.last_command):
+            r.error("previous command was not a yank")
+            return
+        repl = len(r.kill_ring[-1])
+        r.kill_ring.insert(0, r.kill_ring.pop())
+        t = r.kill_ring[-1]
+        b[r.pos - repl : r.pos] = t
+        r.pos = r.pos - repl + len(t)
+        r.dirty = True
+
+
+class interrupt(FinishCommand):
+    def do(self) -> None:
+        import signal
+
+        self.reader.console.finish()
+        self.reader.finish()
+        os.kill(os.getpid(), signal.SIGINT)
+
+
+class ctrl_c(Command):
+    def do(self) -> None:
+        self.reader.console.finish()
+        self.reader.finish()
+        raise KeyboardInterrupt
+
+
+class suspend(Command):
+    def do(self) -> None:
+        import signal
+
+        r = self.reader
+        p = r.pos
+        r.console.finish()
+        os.kill(os.getpid(), signal.SIGSTOP)
+        ## this should probably be done
+        ## in a handler for SIGCONT?
+        r.console.prepare()
+        r.pos = p
+        # r.posxy = 0, 0  # XXX this is invalid
+        r.dirty = True
+        r.console.screen = []
+
+
+class up(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        for _ in range(r.get_arg()):
+            x, y = r.pos2xy()
+            new_y = y - 1
+
+            if r.bol() == 0:
+                if r.historyi > 0:
+                    r.select_item(r.historyi - 1)
+                    return
+                r.pos = 0
+                r.error("start of buffer")
+                return
+
+            if (
+                x
+                > (
+                    new_x := r.max_column(new_y)
+                )  # we're past the end of the previous line
+                or x == r.max_column(y)
+                and any(
+                    not i.isspace() for i in r.buffer[r.bol() :]
+                )  # move between eols
+            ):
+                x = new_x
+
+            r.setpos_from_xy(x, new_y)
+
+
+class down(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        for _ in range(r.get_arg()):
+            x, y = r.pos2xy()
+            new_y = y + 1
+
+            if r.eol() == len(b):
+                if r.historyi < len(r.history):
+                    r.select_item(r.historyi + 1)
+                    r.pos = r.eol(0)
+                    return
+                r.pos = len(b)
+                r.error("end of buffer")
+                return
+
+            if (
+                x
+                > (
+                    new_x := r.max_column(new_y)
+                )  # we're past the end of the previous line
+                or x == r.max_column(y)
+                and any(
+                    not i.isspace() for i in r.buffer[r.bol() :]
+                )  # move between eols
+            ):
+                x = new_x
+
+            r.setpos_from_xy(x, new_y)
+
+
+class left(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        for _ in range(r.get_arg()):
+            p = r.pos - 1
+            if p >= 0:
+                r.pos = p
+            else:
+                self.reader.error("start of buffer")
+
+
+class right(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        for _ in range(r.get_arg()):
+            p = r.pos + 1
+            if p <= len(b):
+                r.pos = p
+            else:
+                self.reader.error("end of buffer")
+
+
+class beginning_of_line(MotionCommand):
+    def do(self) -> None:
+        self.reader.pos = self.reader.bol()
+
+
+class end_of_line(MotionCommand):
+    def do(self) -> None:
+        self.reader.pos = self.reader.eol()
+
+
+class home(MotionCommand):
+    def do(self) -> None:
+        self.reader.pos = 0
+
+
+class end(MotionCommand):
+    def do(self) -> None:
+        self.reader.pos = len(self.reader.buffer)
+
+
+class forward_word(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        for i in range(r.get_arg()):
+            r.pos = r.eow()
+
+
+class backward_word(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        for i in range(r.get_arg()):
+            r.pos = r.bow()
+
+
+class self_insert(EditCommand):
+    def do(self) -> None:
+        r = self.reader
+        text = self.event * r.get_arg()
+        r.insert(text)
+
+
+class insert_nl(EditCommand):
+    def do(self) -> None:
+        r = self.reader
+        r.insert("\n" * r.get_arg())
+
+
+class transpose_characters(EditCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        s = r.pos - 1
+        if s < 0:
+            r.error("cannot transpose at start of buffer")
+        else:
+            if s == len(b):
+                s -= 1
+            t = min(s + r.get_arg(), len(b) - 1)
+            c = b[s]
+            del b[s]
+            b.insert(t, c)
+            r.pos = t
+            r.dirty = True
+
+
+class backspace(EditCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        for i in range(r.get_arg()):
+            if r.pos > 0:
+                r.pos -= 1
+                del b[r.pos]
+                r.dirty = True
+            else:
+                self.reader.error("can't backspace at start")
+
+
+class delete(EditCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        if (
+            r.pos == 0
+            and len(b) == 0  # this is something of a hack
+            and self.event[-1] == "\004"
+        ):
+            r.update_screen()
+            r.console.finish()
+            raise EOFError
+        for i in range(r.get_arg()):
+            if r.pos != len(b):
+                del b[r.pos]
+                r.dirty = True
+            else:
+                self.reader.error("end of buffer")
+
+
+class accept(FinishCommand):
+    def do(self) -> None:
+        pass
+
+
+class help(Command):
+    def do(self) -> None:
+        import _sitebuiltins
+
+        with self.reader.suspend():
+            self.reader.msg = _sitebuiltins._Helper()()  # type: ignore[assignment, call-arg]
+
+
+class invalid_key(Command):
+    def do(self) -> None:
+        pending = self.reader.console.getpending()
+        s = "".join(self.event) + pending.data
+        self.reader.error("`%r' not bound" % s)
+
+
+class invalid_command(Command):
+    def do(self) -> None:
+        s = self.event_name
+        self.reader.error("command `%s' not known" % s)
+
+
+class show_history(Command):
+    def do(self) -> None:
+        from .pager import get_pager
+        from site import gethistoryfile  # type: ignore[attr-defined]
+
+        history = os.linesep.join(self.reader.history[:])
+        self.reader.console.restore()
+        pager = get_pager()
+        pager(history, gethistoryfile())
+        self.reader.console.prepare()
+
+        # We need to copy over the state so that it's consistent between
+        # console and reader, and console does not overwrite/append stuff
+        self.reader.console.screen = self.reader.screen.copy()
+        self.reader.console.posxy = self.reader.cxy
+
+
+class paste_mode(Command):
+
+    def do(self) -> None:
+        self.reader.paste_mode = not self.reader.paste_mode
+        self.reader.dirty = True
+
+
+class enable_bracketed_paste(Command):
+    def do(self) -> None:
+        self.reader.paste_mode = True
+        self.reader.in_bracketed_paste = True
+
+class disable_bracketed_paste(Command):
+    def do(self) -> None:
+        self.reader.paste_mode = False
+        self.reader.in_bracketed_paste = False
+        self.reader.dirty = True

Lib/_pyrepl/completing_reader.py

@@ -0,0 +1,295 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Antonio Cuni
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import re
+from . import commands, console, reader
+from .reader import Reader
+
+
+# types
+Command = commands.Command
+if False:
+    from .types import KeySpec, CommandName
+
+
+def prefix(wordlist: list[str], j: int = 0) -> str:
+    d = {}
+    i = j
+    try:
+        while 1:
+            for word in wordlist:
+                d[word[i]] = 1
+            if len(d) > 1:
+                return wordlist[0][j:i]
+            i += 1
+            d = {}
+    except IndexError:
+        return wordlist[0][j:i]
+    return ""
+
+
+STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]")
+
+def stripcolor(s: str) -> str:
+    return STRIPCOLOR_REGEX.sub('', s)
+
+
+def real_len(s: str) -> int:
+    return len(stripcolor(s))
+
+
+def left_align(s: str, maxlen: int) -> str:
+    stripped = stripcolor(s)
+    if len(stripped) > maxlen:
+        # too bad, we remove the color
+        return stripped[:maxlen]
+    padding = maxlen - len(stripped)
+    return s + ' '*padding
+
+
+def build_menu(
+        cons: console.Console,
+        wordlist: list[str],
+        start: int,
+        use_brackets: bool,
+        sort_in_column: bool,
+) -> tuple[list[str], int]:
+    if use_brackets:
+        item = "[ %s ]"
+        padding = 4
+    else:
+        item = "%s  "
+        padding = 2
+    maxlen = min(max(map(real_len, wordlist)), cons.width - padding)
+    cols = int(cons.width / (maxlen + padding))
+    rows = int((len(wordlist) - 1)/cols + 1)
+
+    if sort_in_column:
+        # sort_in_column=False (default)     sort_in_column=True
+        #          A B C                       A D G
+        #          D E F                       B E
+        #          G                           C F
+        #
+        # "fill" the table with empty words, so we always have the same amout
+        # of rows for each column
+        missing = cols*rows - len(wordlist)
+        wordlist = wordlist + ['']*missing
+        indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))]
+        wordlist = [wordlist[i] for i in indexes]
+    menu = []
+    i = start
+    for r in range(rows):
+        row = []
+        for col in range(cols):
+            row.append(item % left_align(wordlist[i], maxlen))
+            i += 1
+            if i >= len(wordlist):
+                break
+        menu.append(''.join(row))
+        if i >= len(wordlist):
+            i = 0
+            break
+        if r + 5 > cons.height:
+            menu.append("   %d more... " % (len(wordlist) - i))
+            break
+    return menu, i
+
+# this gets somewhat user interface-y, and as a result the logic gets
+# very convoluted.
+#
+#  To summarise the summary of the summary:- people are a problem.
+#                  -- The Hitch-Hikers Guide to the Galaxy, Episode 12
+
+#### Desired behaviour of the completions commands.
+# the considerations are:
+# (1) how many completions are possible
+# (2) whether the last command was a completion
+# (3) if we can assume that the completer is going to return the same set of
+#     completions: this is controlled by the ``assume_immutable_completions``
+#     variable on the reader, which is True by default to match the historical
+#     behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match
+#     more closely readline's semantics (this is needed e.g. by
+#     fancycompleter)
+#
+# if there's no possible completion, beep at the user and point this out.
+# this is easy.
+#
+# if there's only one possible completion, stick it in.  if the last thing
+# user did was a completion, point out that he isn't getting anywhere, but
+# only if the ``assume_immutable_completions`` is True.
+#
+# now it gets complicated.
+#
+# for the first press of a completion key:
+#  if there's a common prefix, stick it in.
+
+#  irrespective of whether anything got stuck in, if the word is now
+#  complete, show the "complete but not unique" message
+
+#  if there's no common prefix and if the word is not now complete,
+#  beep.
+
+#        common prefix ->    yes          no
+#        word complete \/
+#            yes           "cbnu"      "cbnu"
+#            no              -          beep
+
+# for the second bang on the completion key
+#  there will necessarily be no common prefix
+#  show a menu of the choices.
+
+# for subsequent bangs, rotate the menu around (if there are sufficient
+# choices).
+
+
+class complete(commands.Command):
+    def do(self) -> None:
+        r: CompletingReader
+        r = self.reader  # type: ignore[assignment]
+        last_is_completer = r.last_command_is(self.__class__)
+        immutable_completions = r.assume_immutable_completions
+        completions_unchangable = last_is_completer and immutable_completions
+        stem = r.get_stem()
+        if not completions_unchangable:
+            r.cmpltn_menu_choices = r.get_completions(stem)
+
+        completions = r.cmpltn_menu_choices
+        if not completions:
+            r.error("no matches")
+        elif len(completions) == 1:
+            if completions_unchangable and len(completions[0]) == len(stem):
+                r.msg = "[ sole completion ]"
+                r.dirty = True
+            r.insert(completions[0][len(stem):])
+        else:
+            p = prefix(completions, len(stem))
+            if p:
+                r.insert(p)
+            if last_is_completer:
+                r.cmpltn_menu_visible = True
+                r.cmpltn_message_visible = False
+                r.cmpltn_menu, r.cmpltn_menu_end = build_menu(
+                    r.console, completions, r.cmpltn_menu_end,
+                    r.use_brackets, r.sort_in_column)
+                r.dirty = True
+            elif not r.cmpltn_menu_visible:
+                r.cmpltn_message_visible = True
+                if stem + p in completions:
+                    r.msg = "[ complete but not unique ]"
+                    r.dirty = True
+                else:
+                    r.msg = "[ not unique ]"
+                    r.dirty = True
+
+
+class self_insert(commands.self_insert):
+    def do(self) -> None:
+        r: CompletingReader
+        r = self.reader  # type: ignore[assignment]
+
+        commands.self_insert.do(self)
+        if r.cmpltn_menu_visible:
+            stem = r.get_stem()
+            if len(stem) < 1:
+                r.cmpltn_reset()
+            else:
+                completions = [w for w in r.cmpltn_menu_choices
+                               if w.startswith(stem)]
+                if completions:
+                    r.cmpltn_menu, r.cmpltn_menu_end = build_menu(
+                        r.console, completions, 0,
+                        r.use_brackets, r.sort_in_column)
+                else:
+                    r.cmpltn_reset()
+
+
+@dataclass
+class CompletingReader(Reader):
+    """Adds completion support"""
+
+    ### Class variables
+    # see the comment for the complete command
+    assume_immutable_completions = True
+    use_brackets = True  # display completions inside []
+    sort_in_column = False
+
+    ### Instance variables
+    cmpltn_menu: list[str] = field(init=False)
+    cmpltn_menu_visible: bool = field(init=False)
+    cmpltn_message_visible: bool = field(init=False)
+    cmpltn_menu_end: int = field(init=False)
+    cmpltn_menu_choices: list[str] = field(init=False)
+
+    def __post_init__(self) -> None:
+        super().__post_init__()
+        self.cmpltn_reset()
+        for c in (complete, self_insert):
+            self.commands[c.__name__] = c
+            self.commands[c.__name__.replace('_', '-')] = c
+
+    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+        return super().collect_keymap() + (
+            (r'\t', 'complete'),)
+
+    def after_command(self, cmd: Command) -> None:
+        super().after_command(cmd)
+        if not isinstance(cmd, (complete, self_insert)):
+            self.cmpltn_reset()
+
+    def calc_screen(self) -> list[str]:
+        screen = super().calc_screen()
+        if self.cmpltn_menu_visible:
+            # We display the completions menu below the current prompt
+            ly = self.lxy[1] + 1
+            screen[ly:ly] = self.cmpltn_menu
+            # If we're not in the middle of multiline edit, don't append to screeninfo
+            # since that screws up the position calculation in pos2xy function.
+            # This is a hack to prevent the cursor jumping
+            # into the completions menu when pressing left or down arrow.
+            if self.pos != len(self.buffer):
+                self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu)
+        return screen
+
+    def finish(self) -> None:
+        super().finish()
+        self.cmpltn_reset()
+
+    def cmpltn_reset(self) -> None:
+        self.cmpltn_menu = []
+        self.cmpltn_menu_visible = False
+        self.cmpltn_message_visible = False
+        self.cmpltn_menu_end = 0
+        self.cmpltn_menu_choices = []
+
+    def get_stem(self) -> str:
+        st = self.syntax_table
+        SW = reader.SYNTAX_WORD
+        b = self.buffer
+        p = self.pos - 1
+        while p >= 0 and st.get(b[p], SW) == SW:
+            p -= 1
+        return ''.join(b[p+1:self.pos])
+
+    def get_completions(self, stem: str) -> list[str]:
+        return []

Lib/_pyrepl/console.py

@@ -0,0 +1,213 @@
+#   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import _colorize  # type: ignore[import-not-found]
+
+from abc import ABC, abstractmethod
+import ast
+import code
+from dataclasses import dataclass, field
+import os.path
+import sys
+
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+    from typing import IO
+    from typing import Callable
+
+
+@dataclass
+class Event:
+    evt: str
+    data: str
+    raw: bytes = b""
+
+
+@dataclass
+class Console(ABC):
+    posxy: tuple[int, int]
+    screen: list[str] = field(default_factory=list)
+    height: int = 25
+    width: int = 80
+
+    def __init__(
+        self,
+        f_in: IO[bytes] | int = 0,
+        f_out: IO[bytes] | int = 1,
+        term: str = "",
+        encoding: str = "",
+    ):
+        self.encoding = encoding or sys.getdefaultencoding()
+
+        if isinstance(f_in, int):
+            self.input_fd = f_in
+        else:
+            self.input_fd = f_in.fileno()
+
+        if isinstance(f_out, int):
+            self.output_fd = f_out
+        else:
+            self.output_fd = f_out.fileno()
+
+    @abstractmethod
+    def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ...
+
+    @abstractmethod
+    def prepare(self) -> None: ...
+
+    @abstractmethod
+    def restore(self) -> None: ...
+
+    @abstractmethod
+    def move_cursor(self, x: int, y: int) -> None: ...
+
+    @abstractmethod
+    def set_cursor_vis(self, visible: bool) -> None: ...
+
+    @abstractmethod
+    def getheightwidth(self) -> tuple[int, int]:
+        """Return (height, width) where height and width are the height
+        and width of the terminal window in characters."""
+        ...
+
+    @abstractmethod
+    def get_event(self, block: bool = True) -> Event | None:
+        """Return an Event instance.  Returns None if |block| is false
+        and there is no event pending, otherwise waits for the
+        completion of an event."""
+        ...
+
+    @abstractmethod
+    def push_char(self, char: int | bytes) -> None:
+        """
+        Push a character to the console event queue.
+        """
+        ...
+
+    @abstractmethod
+    def beep(self) -> None: ...
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Wipe the screen"""
+        ...
+
+    @abstractmethod
+    def finish(self) -> None:
+        """Move the cursor to the end of the display and otherwise get
+        ready for end.  XXX could be merged with restore?  Hmm."""
+        ...
+
+    @abstractmethod
+    def flushoutput(self) -> None:
+        """Flush all output to the screen (assuming there's some
+        buffering going on somewhere)."""
+        ...
+
+    @abstractmethod
+    def forgetinput(self) -> None:
+        """Forget all pending, but not yet processed input."""
+        ...
+
+    @abstractmethod
+    def getpending(self) -> Event:
+        """Return the characters that have been typed but not yet
+        processed."""
+        ...
+
+    @abstractmethod
+    def wait(self, timeout: float | None) -> bool:
+        """Wait for an event. The return value is True if an event is
+        available, False if the timeout has been reached. If timeout is
+        None, wait forever. The timeout is in milliseconds."""
+        ...
+
+    @property
+    def input_hook(self) -> Callable[[], int] | None:
+        """Returns the current input hook."""
+        ...
+
+    @abstractmethod
+    def repaint(self) -> None: ...
+
+
+class InteractiveColoredConsole(code.InteractiveConsole):
+    def __init__(
+        self,
+        locals: dict[str, object] | None = None,
+        filename: str = "<console>",
+        *,
+        local_exit: bool = False,
+    ) -> None:
+        super().__init__(locals=locals, filename=filename, local_exit=local_exit)  # type: ignore[call-arg]
+        self.can_colorize = _colorize.can_colorize()
+
+    def showsyntaxerror(self, filename=None, **kwargs):
+        super().showsyntaxerror(filename=filename, **kwargs)
+
+    def _excepthook(self, typ, value, tb):
+        import traceback
+        lines = traceback.format_exception(
+                typ, value, tb,
+                colorize=self.can_colorize,
+                limit=traceback.BUILTIN_EXCEPTION_LIMIT)
+        self.write(''.join(lines))
+
+    def runsource(self, source, filename="<input>", symbol="single"):
+        try:
+            tree = self.compile.compiler(
+                source,
+                filename,
+                "exec",
+                ast.PyCF_ONLY_AST,
+                incomplete_input=False,
+            )
+        except (SyntaxError, OverflowError, ValueError):
+            self.showsyntaxerror(filename, source=source)
+            return False
+        if tree.body:
+            *_, last_stmt = tree.body
+        for stmt in tree.body:
+            wrapper = ast.Interactive if stmt is last_stmt else ast.Module
+            the_symbol = symbol if stmt is last_stmt else "exec"
+            item = wrapper([stmt])
+            try:
+                code = self.compile.compiler(item, filename, the_symbol)
+            except SyntaxError as e:
+                if e.args[0] == "'await' outside function":
+                    python = os.path.basename(sys.executable)
+                    e.add_note(
+                        f"Try the asyncio REPL ({python} -m asyncio) to use"
+                        f" top-level 'await' and run background asyncio tasks."
+                    )
+                self.showsyntaxerror(filename, source=source)
+                return False
+            except (OverflowError, ValueError):
+                self.showsyntaxerror(filename, source=source)
+                return False
+
+            if code is None:
+                return True
+
+            self.runcode(code)
+        return False

Lib/_pyrepl/curses.py

@@ -0,0 +1,33 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+try:
+    import _curses
+except ImportError:
+    try:
+        import curses as _curses  # type: ignore[no-redef]
+    except ImportError:
+        from . import _minimal_curses as _curses  # type: ignore[no-redef]
+
+setupterm = _curses.setupterm
+tigetstr = _curses.tigetstr
+tparm = _curses.tparm
+error = _curses.error

Lib/_pyrepl/fancy_termios.py

@@ -0,0 +1,76 @@
+#   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import termios
+
+
+class TermState:
+    def __init__(self, tuples):
+        (
+            self.iflag,
+            self.oflag,
+            self.cflag,
+            self.lflag,
+            self.ispeed,
+            self.ospeed,
+            self.cc,
+        ) = tuples
+
+    def as_list(self):
+        return [
+            self.iflag,
+            self.oflag,
+            self.cflag,
+            self.lflag,
+            self.ispeed,
+            self.ospeed,
+            # Always return a copy of the control characters list to ensure
+            # there are not any additional references to self.cc
+            self.cc[:],
+        ]
+
+    def copy(self):
+        return self.__class__(self.as_list())
+
+
+def tcgetattr(fd):
+    return TermState(termios.tcgetattr(fd))
+
+
+def tcsetattr(fd, when, attrs):
+    termios.tcsetattr(fd, when, attrs.as_list())
+
+
+class Term(TermState):
+    TS__init__ = TermState.__init__
+
+    def __init__(self, fd=0):
+        self.TS__init__(termios.tcgetattr(fd))
+        self.fd = fd
+        self.stack = []
+
+    def save(self):
+        self.stack.append(self.as_list())
+
+    def set(self, when=termios.TCSANOW):
+        termios.tcsetattr(self.fd, when, self.as_list())
+
+    def restore(self):
+        self.TS__init__(self.stack.pop())
+        self.set()

Lib/_pyrepl/historical_reader.py

@@ -0,0 +1,419 @@
+#   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+
+from . import commands, input
+from .reader import Reader
+
+
+if False:
+    from .types import SimpleContextManager, KeySpec, CommandName
+
+
+isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple(
+    [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"]
+    + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"]
+    + [
+        ("\\%03o" % c, "isearch-add-character")
+        for c in range(256)
+        if chr(c).isalpha() and chr(c) != "\\"
+    ]
+    + [
+        ("\\\\", "self-insert"),
+        (r"\C-r", "isearch-backwards"),
+        (r"\C-s", "isearch-forwards"),
+        (r"\C-c", "isearch-cancel"),
+        (r"\C-g", "isearch-cancel"),
+        (r"\<backspace>", "isearch-backspace"),
+    ]
+)
+
+ISEARCH_DIRECTION_NONE = ""
+ISEARCH_DIRECTION_BACKWARDS = "r"
+ISEARCH_DIRECTION_FORWARDS = "f"
+
+
+class next_history(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        if r.historyi == len(r.history):
+            r.error("end of history list")
+            return
+        r.select_item(r.historyi + 1)
+
+
+class previous_history(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        if r.historyi == 0:
+            r.error("start of history list")
+            return
+        r.select_item(r.historyi - 1)
+
+
+class history_search_backward(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.search_next(forwards=False)
+
+
+class history_search_forward(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.search_next(forwards=True)
+
+
+class restore_history(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        if r.historyi != len(r.history):
+            if r.get_unicode() != r.history[r.historyi]:
+                r.buffer = list(r.history[r.historyi])
+                r.pos = len(r.buffer)
+                r.dirty = True
+
+
+class first_history(commands.Command):
+    def do(self) -> None:
+        self.reader.select_item(0)
+
+
+class last_history(commands.Command):
+    def do(self) -> None:
+        self.reader.select_item(len(self.reader.history))
+
+
+class operate_and_get_next(commands.FinishCommand):
+    def do(self) -> None:
+        self.reader.next_history = self.reader.historyi + 1
+
+
+class yank_arg(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        if r.last_command is self.__class__:
+            r.yank_arg_i += 1
+        else:
+            r.yank_arg_i = 0
+        if r.historyi < r.yank_arg_i:
+            r.error("beginning of history list")
+            return
+        a = r.get_arg(-1)
+        # XXX how to split?
+        words = r.get_item(r.historyi - r.yank_arg_i - 1).split()
+        if a < -len(words) or a >= len(words):
+            r.error("no such arg")
+            return
+        w = words[a]
+        b = r.buffer
+        if r.yank_arg_i > 0:
+            o = len(r.yank_arg_yanked)
+        else:
+            o = 0
+        b[r.pos - o : r.pos] = list(w)
+        r.yank_arg_yanked = w
+        r.pos += len(w) - o
+        r.dirty = True
+
+
+class forward_history_isearch(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_FORWARDS
+        r.isearch_start = r.historyi, r.pos
+        r.isearch_term = ""
+        r.dirty = True
+        r.push_input_trans(r.isearch_trans)
+
+
+class reverse_history_isearch(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS
+        r.dirty = True
+        r.isearch_term = ""
+        r.push_input_trans(r.isearch_trans)
+        r.isearch_start = r.historyi, r.pos
+
+
+class isearch_cancel(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_NONE
+        r.pop_input_trans()
+        r.select_item(r.isearch_start[0])
+        r.pos = r.isearch_start[1]
+        r.dirty = True
+
+
+class isearch_add_character(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        r.isearch_term += self.event[-1]
+        r.dirty = True
+        p = r.pos + len(r.isearch_term) - 1
+        if b[p : p + 1] != [r.isearch_term[-1]]:
+            r.isearch_next()
+
+
+class isearch_backspace(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        if len(r.isearch_term) > 0:
+            r.isearch_term = r.isearch_term[:-1]
+            r.dirty = True
+        else:
+            r.error("nothing to rubout")
+
+
+class isearch_forwards(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_FORWARDS
+        r.isearch_next()
+
+
+class isearch_backwards(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS
+        r.isearch_next()
+
+
+class isearch_end(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_NONE
+        r.console.forgetinput()
+        r.pop_input_trans()
+        r.dirty = True
+
+
+@dataclass
+class HistoricalReader(Reader):
+    """Adds history support (with incremental history searching) to the
+    Reader class.
+    """
+
+    history: list[str] = field(default_factory=list)
+    historyi: int = 0
+    next_history: int | None = None
+    transient_history: dict[int, str] = field(default_factory=dict)
+    isearch_term: str = ""
+    isearch_direction: str = ISEARCH_DIRECTION_NONE
+    isearch_start: tuple[int, int] = field(init=False)
+    isearch_trans: input.KeymapTranslator = field(init=False)
+    yank_arg_i: int = 0
+    yank_arg_yanked: str = ""
+
+    def __post_init__(self) -> None:
+        super().__post_init__()
+        for c in [
+            next_history,
+            previous_history,
+            restore_history,
+            first_history,
+            last_history,
+            yank_arg,
+            forward_history_isearch,
+            reverse_history_isearch,
+            isearch_end,
+            isearch_add_character,
+            isearch_cancel,
+            isearch_add_character,
+            isearch_backspace,
+            isearch_forwards,
+            isearch_backwards,
+            operate_and_get_next,
+            history_search_backward,
+            history_search_forward,
+        ]:
+            self.commands[c.__name__] = c
+            self.commands[c.__name__.replace("_", "-")] = c
+        self.isearch_start = self.historyi, self.pos
+        self.isearch_trans = input.KeymapTranslator(
+            isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character
+        )
+
+    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+        return super().collect_keymap() + (
+            (r"\C-n", "next-history"),
+            (r"\C-p", "previous-history"),
+            (r"\C-o", "operate-and-get-next"),
+            (r"\C-r", "reverse-history-isearch"),
+            (r"\C-s", "forward-history-isearch"),
+            (r"\M-r", "restore-history"),
+            (r"\M-.", "yank-arg"),
+            (r"\<page down>", "history-search-forward"),
+            (r"\x1b[6~", "history-search-forward"),
+            (r"\<page up>", "history-search-backward"),
+            (r"\x1b[5~", "history-search-backward"),
+        )
+
+    def select_item(self, i: int) -> None:
+        self.transient_history[self.historyi] = self.get_unicode()
+        buf = self.transient_history.get(i)
+        if buf is None:
+            buf = self.history[i].rstrip()
+        self.buffer = list(buf)
+        self.historyi = i
+        self.pos = len(self.buffer)
+        self.dirty = True
+        self.last_refresh_cache.invalidated = True
+
+    def get_item(self, i: int) -> str:
+        if i != len(self.history):
+            return self.transient_history.get(i, self.history[i])
+        else:
+            return self.transient_history.get(i, self.get_unicode())
+
+    @contextmanager
+    def suspend(self) -> SimpleContextManager:
+        with super().suspend(), self.suspend_history():
+            yield
+
+    @contextmanager
+    def suspend_history(self) -> SimpleContextManager:
+        try:
+            old_history = self.history[:]
+            del self.history[:]
+            yield
+        finally:
+            self.history[:] = old_history
+
+    def prepare(self) -> None:
+        super().prepare()
+        try:
+            self.transient_history = {}
+            if self.next_history is not None and self.next_history < len(self.history):
+                self.historyi = self.next_history
+                self.buffer[:] = list(self.history[self.next_history])
+                self.pos = len(self.buffer)
+                self.transient_history[len(self.history)] = ""
+            else:
+                self.historyi = len(self.history)
+            self.next_history = None
+        except:
+            self.restore()
+            raise
+
+    def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
+        if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE:
+            d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS]
+            return "(%s-search `%s') " % (d, self.isearch_term)
+        else:
+            return super().get_prompt(lineno, cursor_on_line)
+
+    def search_next(self, *, forwards: bool) -> None:
+        """Search history for the current line contents up to the cursor.
+
+        Selects the first item found. If nothing is under the cursor, any next
+        item in history is selected.
+        """
+        pos = self.pos
+        s = self.get_unicode()
+        history_index = self.historyi
+
+        # In multiline contexts, we're only interested in the current line.
+        nl_index = s.rfind('\n', 0, pos)
+        prefix = s[nl_index + 1:pos]
+        pos = len(prefix)
+
+        match_prefix = len(prefix)
+        len_item = 0
+        if history_index < len(self.history):
+            len_item = len(self.get_item(history_index))
+        if len_item and pos == len_item:
+            match_prefix = False
+        elif not pos:
+            match_prefix = False
+
+        while 1:
+            if forwards:
+                out_of_bounds = history_index >= len(self.history) - 1
+            else:
+                out_of_bounds = history_index == 0
+            if out_of_bounds:
+                if forwards and not match_prefix:
+                    self.pos = 0
+                    self.buffer = []
+                    self.dirty = True
+                else:
+                    self.error("not found")
+                return
+
+            history_index += 1 if forwards else -1
+            s = self.get_item(history_index)
+
+            if not match_prefix:
+                self.select_item(history_index)
+                return
+
+            len_acc = 0
+            for i, line in enumerate(s.splitlines(keepends=True)):
+                if line.startswith(prefix):
+                    self.select_item(history_index)
+                    self.pos = pos + len_acc
+                    return
+                len_acc += len(line)
+
+    def isearch_next(self) -> None:
+        st = self.isearch_term
+        p = self.pos
+        i = self.historyi
+        s = self.get_unicode()
+        forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS
+        while 1:
+            if forwards:
+                p = s.find(st, p + 1)
+            else:
+                p = s.rfind(st, 0, p + len(st) - 1)
+            if p != -1:
+                self.select_item(i)
+                self.pos = p
+                return
+            elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0):
+                self.error("not found")
+                return
+            else:
+                if forwards:
+                    i += 1
+                    s = self.get_item(i)
+                    p = -1
+                else:
+                    i -= 1
+                    s = self.get_item(i)
+                    p = len(s)
+
+    def finish(self) -> None:
+        super().finish()
+        ret = self.get_unicode()
+        for i, t in self.transient_history.items():
+            if i < len(self.history) and i != self.historyi:
+                self.history[i] = t
+        if ret and should_auto_add_history:
+            self.history.append(ret)
+
+
+should_auto_add_history = True

Lib/_pyrepl/input.py

@@ -0,0 +1,114 @@
+#   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+# (naming modules after builtin functions is not such a hot idea...)
+
+# an KeyTrans instance translates Event objects into Command objects
+
+# hmm, at what level do we want [C-i] and [tab] to be equivalent?
+# [meta-a] and [esc a]?  obviously, these are going to be equivalent
+# for the UnixConsole, but should they be for PygameConsole?
+
+# it would in any situation seem to be a bad idea to bind, say, [tab]
+# and [C-i] to *different* things... but should binding one bind the
+# other?
+
+# executive, temporary decision: [tab] and [C-i] are distinct, but
+# [meta-key] is identified with [esc key].  We demand that any console
+# class does quite a lot towards emulating a unix terminal.
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+import unicodedata
+from collections import deque
+
+
+# types
+if False:
+    from .types import EventTuple
+
+
+class InputTranslator(ABC):
+    @abstractmethod
+    def push(self, evt: EventTuple) -> None:
+        pass
+
+    @abstractmethod
+    def get(self) -> EventTuple | None:
+        return None
+
+    @abstractmethod
+    def empty(self) -> bool:
+        return True
+
+
+class KeymapTranslator(InputTranslator):
+    def __init__(self, keymap, verbose=False, invalid_cls=None, character_cls=None):
+        self.verbose = verbose
+        from .keymap import compile_keymap, parse_keys
+
+        self.keymap = keymap
+        self.invalid_cls = invalid_cls
+        self.character_cls = character_cls
+        d = {}
+        for keyspec, command in keymap:
+            keyseq = tuple(parse_keys(keyspec))
+            d[keyseq] = command
+        if self.verbose:
+            print(d)
+        self.k = self.ck = compile_keymap(d, ())
+        self.results = deque()
+        self.stack = []
+
+    def push(self, evt):
+        if self.verbose:
+            print("pushed", evt.data, end="")
+        key = evt.data
+        d = self.k.get(key)
+        if isinstance(d, dict):
+            if self.verbose:
+                print("transition")
+            self.stack.append(key)
+            self.k = d
+        else:
+            if d is None:
+                if self.verbose:
+                    print("invalid")
+                if self.stack or len(key) > 1 or unicodedata.category(key) == "C":
+                    self.results.append((self.invalid_cls, self.stack + [key]))
+                else:
+                    # small optimization:
+                    self.k[key] = self.character_cls
+                    self.results.append((self.character_cls, [key]))
+            else:
+                if self.verbose:
+                    print("matched", d)
+                self.results.append((d, self.stack + [key]))
+            self.stack = []
+            self.k = self.ck
+
+    def get(self):
+        if self.results:
+            return self.results.popleft()
+        else:
+            return None
+
+    def empty(self) -> bool:
+        return not self.results

Lib/_pyrepl/keymap.py

@@ -0,0 +1,213 @@
+#   Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Keymap contains functions for parsing keyspecs and turning keyspecs into
+appropriate sequences.
+
+A keyspec is a string representing a sequence of key presses that can
+be bound to a command. All characters other than the backslash represent
+themselves. In the traditional manner, a backslash introduces an escape
+sequence.
+
+pyrepl uses its own keyspec format that is meant to be a strict superset of
+readline's KEYSEQ format. This means that if a spec is found that readline
+accepts that this doesn't, it should be logged as a bug. Note that this means
+we're using the `\\C-o' style of readline's keyspec, not the `Control-o' sort.
+
+The extension to readline is that the sequence \\<KEY> denotes the
+sequence of characters produced by hitting KEY.
+
+Examples:
+`a'      - what you get when you hit the `a' key
+`\\EOA'  - Escape - O - A (up, on my terminal)
+`\\<UP>' - the up arrow key
+`\\<up>' - ditto (keynames are case-insensitive)
+`\\C-o', `\\c-o'  - control-o
+`\\M-.'  - meta-period
+`\\E.'   - ditto (that's how meta works for pyrepl)
+`\\<tab>', `\\<TAB>', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I'
+   - all of these are the tab character.
+"""
+
+_escapes = {
+    "\\": "\\",
+    "'": "'",
+    '"': '"',
+    "a": "\a",
+    "b": "\b",
+    "e": "\033",
+    "f": "\f",
+    "n": "\n",
+    "r": "\r",
+    "t": "\t",
+    "v": "\v",
+}
+
+_keynames = {
+    "backspace": "backspace",
+    "delete": "delete",
+    "down": "down",
+    "end": "end",
+    "enter": "\r",
+    "escape": "\033",
+    "f1": "f1",
+    "f2": "f2",
+    "f3": "f3",
+    "f4": "f4",
+    "f5": "f5",
+    "f6": "f6",
+    "f7": "f7",
+    "f8": "f8",
+    "f9": "f9",
+    "f10": "f10",
+    "f11": "f11",
+    "f12": "f12",
+    "f13": "f13",
+    "f14": "f14",
+    "f15": "f15",
+    "f16": "f16",
+    "f17": "f17",
+    "f18": "f18",
+    "f19": "f19",
+    "f20": "f20",
+    "home": "home",
+    "insert": "insert",
+    "left": "left",
+    "page down": "page down",
+    "page up": "page up",
+    "return": "\r",
+    "right": "right",
+    "space": " ",
+    "tab": "\t",
+    "up": "up",
+}
+
+
+class KeySpecError(Exception):
+    pass
+
+
+def parse_keys(keys: str) -> list[str]:
+    """Parse keys in keyspec format to a sequence of keys."""
+    s = 0
+    r: list[str] = []
+    while s < len(keys):
+        k, s = _parse_single_key_sequence(keys, s)
+        r.extend(k)
+    return r
+
+
+def _parse_single_key_sequence(key: str, s: int) -> tuple[list[str], int]:
+    ctrl = 0
+    meta = 0
+    ret = ""
+    while not ret and s < len(key):
+        if key[s] == "\\":
+            c = key[s + 1].lower()
+            if c in _escapes:
+                ret = _escapes[c]
+                s += 2
+            elif c == "c":
+                if key[s + 2] != "-":
+                    raise KeySpecError(
+                        "\\C must be followed by `-' (char %d of %s)"
+                        % (s + 2, repr(key))
+                    )
+                if ctrl:
+                    raise KeySpecError(
+                        "doubled \\C- (char %d of %s)" % (s + 1, repr(key))
+                    )
+                ctrl = 1
+                s += 3
+            elif c == "m":
+                if key[s + 2] != "-":
+                    raise KeySpecError(
+                        "\\M must be followed by `-' (char %d of %s)"
+                        % (s + 2, repr(key))
+                    )
+                if meta:
+                    raise KeySpecError(
+                        "doubled \\M- (char %d of %s)" % (s + 1, repr(key))
+                    )
+                meta = 1
+                s += 3
+            elif c.isdigit():
+                n = key[s + 1 : s + 4]
+                ret = chr(int(n, 8))
+                s += 4
+            elif c == "x":
+                n = key[s + 2 : s + 4]
+                ret = chr(int(n, 16))
+                s += 4
+            elif c == "<":
+                t = key.find(">", s)
+                if t == -1:
+                    raise KeySpecError(
+                        "unterminated \\< starting at char %d of %s"
+                        % (s + 1, repr(key))
+                    )
+                ret = key[s + 2 : t].lower()
+                if ret not in _keynames:
+                    raise KeySpecError(
+                        "unrecognised keyname `%s' at char %d of %s"
+                        % (ret, s + 2, repr(key))
+                    )
+                ret = _keynames[ret]
+                s = t + 1
+            else:
+                raise KeySpecError(
+                    "unknown backslash escape %s at char %d of %s"
+                    % (repr(c), s + 2, repr(key))
+                )
+        else:
+            ret = key[s]
+            s += 1
+    if ctrl:
+        if len(ret) == 1:
+            ret = chr(ord(ret) & 0x1F)  # curses.ascii.ctrl()
+        elif ret in {"left", "right"}:
+            ret = f"ctrl {ret}"
+        else:
+            raise KeySpecError("\\C- followed by invalid key")
+
+    result = [ret], s
+    if meta:
+        result[0].insert(0, "\033")
+    return result
+
+
+def compile_keymap(keymap, empty=b""):
+    r = {}
+    for key, value in keymap.items():
+        if isinstance(key, bytes):
+            first = key[:1]
+        else:
+            first = key[0]
+        r.setdefault(first, {})[key[1:]] = value
+    for key, value in r.items():
+        if empty in value:
+            if len(value) != 1:
+                raise KeySpecError("key definitions for %s clash" % (value.values(),))
+            else:
+                r[key] = value[empty]
+        else:
+            r[key] = compile_keymap(value, empty)
+    return r

Lib/_pyrepl/main.py

@@ -0,0 +1,59 @@
+import errno
+import os
+import sys
+
+
+CAN_USE_PYREPL: bool
+FAIL_REASON: str
+try:
+    if sys.platform == "win32" and sys.getwindowsversion().build < 10586:
+        raise RuntimeError("Windows 10 TH2 or later required")
+    if not os.isatty(sys.stdin.fileno()):
+        raise OSError(errno.ENOTTY, "tty required", "stdin")
+    from .simple_interact import check
+    if err := check():
+        raise RuntimeError(err)
+except Exception as e:
+    CAN_USE_PYREPL = False
+    FAIL_REASON = f"warning: can't use pyrepl: {e}"
+else:
+    CAN_USE_PYREPL = True
+    FAIL_REASON = ""
+
+
+def interactive_console(mainmodule=None, quiet=False, pythonstartup=False):
+    if not CAN_USE_PYREPL:
+        if not os.getenv('PYTHON_BASIC_REPL') and FAIL_REASON:
+            from .trace import trace
+            trace(FAIL_REASON)
+            print(FAIL_REASON, file=sys.stderr)
+        return sys._baserepl()
+
+    if mainmodule:
+        namespace = mainmodule.__dict__
+    else:
+        import __main__
+        namespace = __main__.__dict__
+        namespace.pop("__pyrepl_interactive_console", None)
+
+    # sys._baserepl() above does this internally, we do it here
+    startup_path = os.getenv("PYTHONSTARTUP")
+    if pythonstartup and startup_path:
+        sys.audit("cpython.run_startup", startup_path)
+
+        import tokenize
+        with tokenize.open(startup_path) as f:
+            startup_code = compile(f.read(), startup_path, "exec")
+            exec(startup_code, namespace)
+
+    # set sys.{ps1,ps2} just before invoking the interactive interpreter. This
+    # mimics what CPython does in pythonrun.c
+    if not hasattr(sys, "ps1"):
+        sys.ps1 = ">>> "
+    if not hasattr(sys, "ps2"):
+        sys.ps2 = "... "
+
+    from .console import InteractiveColoredConsole
+    from .simple_interact import run_multiline_interactive_console
+    console = InteractiveColoredConsole(namespace, filename="<stdin>")
+    run_multiline_interactive_console(console)

Lib/_pyrepl/mypy.ini

@@ -0,0 +1,24 @@
+# Config file for running mypy on _pyrepl.
+# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini`
+# on the command-line from the repo root
+
+[mypy]
+files = Lib/_pyrepl
+explicit_package_bases = True
+python_version = 3.12
+platform = linux
+pretty = True
+
+# Enable most stricter settings
+enable_error_code = ignore-without-code,redundant-expr
+strict = True
+
+# Various stricter settings that we can't yet enable
+# Try to enable these in the following order:
+disallow_untyped_calls = False
+disallow_untyped_defs = False
+check_untyped_defs = False
+
+# Various internal modules that typeshed deliberately doesn't have stubs for:
+[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*]
+ignore_missing_imports = True

Lib/_pyrepl/pager.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import io
+import os
+import re
+import sys
+
+
+# types
+if False:
+    from typing import Protocol
+    class Pager(Protocol):
+        def __call__(self, text: str, title: str = "") -> None:
+            ...
+
+
+def get_pager() -> Pager:
+    """Decide what method to use for paging through text."""
+    if not hasattr(sys.stdin, "isatty"):
+        return plain_pager
+    if not hasattr(sys.stdout, "isatty"):
+        return plain_pager
+    if not sys.stdin.isatty() or not sys.stdout.isatty():
+        return plain_pager
+    if sys.platform == "emscripten":
+        return plain_pager
+    use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER')
+    if use_pager:
+        if sys.platform == 'win32': # pipes completely broken in Windows
+            return lambda text, title='': tempfile_pager(plain(text), use_pager)
+        elif os.environ.get('TERM') in ('dumb', 'emacs'):
+            return lambda text, title='': pipe_pager(plain(text), use_pager, title)
+        else:
+            return lambda text, title='': pipe_pager(text, use_pager, title)
+    if os.environ.get('TERM') in ('dumb', 'emacs'):
+        return plain_pager
+    if sys.platform == 'win32':
+        return lambda text, title='': tempfile_pager(plain(text), 'more <')
+    if hasattr(os, 'system') and os.system('(pager) 2>/dev/null') == 0:
+        return lambda text, title='': pipe_pager(text, 'pager', title)
+    if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0:
+        return lambda text, title='': pipe_pager(text, 'less', title)
+
+    import tempfile
+    (fd, filename) = tempfile.mkstemp()
+    os.close(fd)
+    try:
+        if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0:
+            return lambda text, title='': pipe_pager(text, 'more', title)
+        else:
+            return tty_pager
+    finally:
+        os.unlink(filename)
+
+
+def escape_stdout(text: str) -> str:
+    # Escape non-encodable characters to avoid encoding errors later
+    encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8'
+    return text.encode(encoding, 'backslashreplace').decode(encoding)
+
+
+def escape_less(s: str) -> str:
+    return re.sub(r'([?:.%\\])', r'\\\1', s)
+
+
+def plain(text: str) -> str:
+    """Remove boldface formatting from text."""
+    return re.sub('.\b', '', text)
+
+
+def tty_pager(text: str, title: str = '') -> None:
+    """Page through text on a text terminal."""
+    lines = plain(escape_stdout(text)).split('\n')
+    has_tty = False
+    try:
+        import tty
+        import termios
+        fd = sys.stdin.fileno()
+        old = termios.tcgetattr(fd)
+        tty.setcbreak(fd)
+        has_tty = True
+
+        def getchar() -> str:
+            return sys.stdin.read(1)
+
+    except (ImportError, AttributeError, io.UnsupportedOperation):
+        def getchar() -> str:
+            return sys.stdin.readline()[:-1][:1]
+
+    try:
+        try:
+            h = int(os.environ.get('LINES', 0))
+        except ValueError:
+            h = 0
+        if h <= 1:
+            h = 25
+        r = inc = h - 1
+        sys.stdout.write('\n'.join(lines[:inc]) + '\n')
+        while lines[r:]:
+            sys.stdout.write('-- more --')
+            sys.stdout.flush()
+            c = getchar()
+
+            if c in ('q', 'Q'):
+                sys.stdout.write('\r          \r')
+                break
+            elif c in ('\r', '\n'):
+                sys.stdout.write('\r          \r' + lines[r] + '\n')
+                r = r + 1
+                continue
+            if c in ('b', 'B', '\x1b'):
+                r = r - inc - inc
+                if r < 0: r = 0
+            sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n')
+            r = r + inc
+
+    finally:
+        if has_tty:
+            termios.tcsetattr(fd, termios.TCSAFLUSH, old)
+
+
+def plain_pager(text: str, title: str = '') -> None:
+    """Simply print unformatted text.  This is the ultimate fallback."""
+    sys.stdout.write(plain(escape_stdout(text)))
+
+
+def pipe_pager(text: str, cmd: str, title: str = '') -> None:
+    """Page through text by feeding it to another program."""
+    import subprocess
+    env = os.environ.copy()
+    if title:
+        title += ' '
+    esc_title = escape_less(title)
+    prompt_string = (
+        f' {esc_title}' +
+        '?ltline %lt?L/%L.'
+        ':byte %bB?s/%s.'
+        '.'
+        '?e (END):?pB %pB\\%..'
+        ' (press h for help or q to quit)')
+    env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string)
+    proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
+                            errors='backslashreplace', env=env)
+    assert proc.stdin is not None
+    try:
+        with proc.stdin as pipe:
+            try:
+                pipe.write(text)
+            except KeyboardInterrupt:
+                # We've hereby abandoned whatever text hasn't been written,
+                # but the pager is still in control of the terminal.
+                pass
+    except OSError:
+        pass # Ignore broken pipes caused by quitting the pager program.
+    while True:
+        try:
+            proc.wait()
+            break
+        except KeyboardInterrupt:
+            # Ignore ctl-c like the pager itself does.  Otherwise the pager is
+            # left running and the terminal is in raw mode and unusable.
+            pass
+
+
+def tempfile_pager(text: str, cmd: str, title: str = '') -> None:
+    """Page through text by invoking a program on a temporary file."""
+    import tempfile
+    with tempfile.TemporaryDirectory() as tempdir:
+        filename = os.path.join(tempdir, 'pydoc.out')
+        with open(filename, 'w', errors='backslashreplace',
+                  encoding=os.device_encoding(0) if
+                  sys.platform == 'win32' else None
+                  ) as file:
+            file.write(text)
+        os.system(cmd + ' "' + filename + '"')

Lib/_pyrepl/reader.py

@@ -0,0 +1,816 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Antonio Cuni
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import sys
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field, fields
+import unicodedata
+from _colorize import can_colorize, ANSIColors  # type: ignore[import-not-found]
+
+
+from . import commands, console, input
+from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width
+from .trace import trace
+
+
+# types
+Command = commands.Command
+from .types import Callback, SimpleContextManager, KeySpec, CommandName
+
+
+def disp_str(buffer: str) -> tuple[str, list[int]]:
+    """disp_str(buffer:string) -> (string, [int])
+
+    Return the string that should be the printed representation of
+    |buffer| and a list detailing where the characters of |buffer|
+    get used up.  E.g.:
+
+    >>> disp_str(chr(3))
+    ('^C', [1, 0])
+
+    """
+    b: list[int] = []
+    s: list[str] = []
+    for c in buffer:
+        if c == '\x1a':
+            s.append(c)
+            b.append(2)
+        elif ord(c) < 128:
+            s.append(c)
+            b.append(1)
+        elif unicodedata.category(c).startswith("C"):
+            c = r"\u%04x" % ord(c)
+            s.append(c)
+            b.extend([0] * (len(c) - 1))
+        else:
+            s.append(c)
+            b.append(str_width(c))
+    return "".join(s), b
+
+
+# syntax classes:
+
+SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3)
+
+
+def make_default_syntax_table() -> dict[str, int]:
+    # XXX perhaps should use some unicodedata here?
+    st: dict[str, int] = {}
+    for c in map(chr, range(256)):
+        st[c] = SYNTAX_SYMBOL
+    for c in [a for a in map(chr, range(256)) if a.isalnum()]:
+        st[c] = SYNTAX_WORD
+    st["\n"] = st[" "] = SYNTAX_WHITESPACE
+    return st
+
+
+def make_default_commands() -> dict[CommandName, type[Command]]:
+    result: dict[CommandName, type[Command]] = {}
+    for v in vars(commands).values():
+        if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower():
+            result[v.__name__] = v
+            result[v.__name__.replace("_", "-")] = v
+    return result
+
+
+default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple(
+    [
+        (r"\C-a", "beginning-of-line"),
+        (r"\C-b", "left"),
+        (r"\C-c", "interrupt"),
+        (r"\C-d", "delete"),
+        (r"\C-e", "end-of-line"),
+        (r"\C-f", "right"),
+        (r"\C-g", "cancel"),
+        (r"\C-h", "backspace"),
+        (r"\C-j", "accept"),
+        (r"\<return>", "accept"),
+        (r"\C-k", "kill-line"),
+        (r"\C-l", "clear-screen"),
+        (r"\C-m", "accept"),
+        (r"\C-t", "transpose-characters"),
+        (r"\C-u", "unix-line-discard"),
+        (r"\C-w", "unix-word-rubout"),
+        (r"\C-x\C-u", "upcase-region"),
+        (r"\C-y", "yank"),
+        *(() if sys.platform == "win32" else ((r"\C-z", "suspend"), )),
+        (r"\M-b", "backward-word"),
+        (r"\M-c", "capitalize-word"),
+        (r"\M-d", "kill-word"),
+        (r"\M-f", "forward-word"),
+        (r"\M-l", "downcase-word"),
+        (r"\M-t", "transpose-words"),
+        (r"\M-u", "upcase-word"),
+        (r"\M-y", "yank-pop"),
+        (r"\M--", "digit-arg"),
+        (r"\M-0", "digit-arg"),
+        (r"\M-1", "digit-arg"),
+        (r"\M-2", "digit-arg"),
+        (r"\M-3", "digit-arg"),
+        (r"\M-4", "digit-arg"),
+        (r"\M-5", "digit-arg"),
+        (r"\M-6", "digit-arg"),
+        (r"\M-7", "digit-arg"),
+        (r"\M-8", "digit-arg"),
+        (r"\M-9", "digit-arg"),
+        (r"\M-\n", "accept"),
+        ("\\\\", "self-insert"),
+        (r"\x1b[200~", "enable_bracketed_paste"),
+        (r"\x1b[201~", "disable_bracketed_paste"),
+        (r"\x03", "ctrl-c"),
+    ]
+    + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"]
+    + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()]
+    + [
+        (r"\<up>", "up"),
+        (r"\<down>", "down"),
+        (r"\<left>", "left"),
+        (r"\C-\<left>", "backward-word"),
+        (r"\<right>", "right"),
+        (r"\C-\<right>", "forward-word"),
+        (r"\<delete>", "delete"),
+        (r"\x1b[3~", "delete"),
+        (r"\<backspace>", "backspace"),
+        (r"\M-\<backspace>", "backward-kill-word"),
+        (r"\<end>", "end-of-line"),  # was 'end'
+        (r"\<home>", "beginning-of-line"),  # was 'home'
+        (r"\<f1>", "help"),
+        (r"\<f2>", "show-history"),
+        (r"\<f3>", "paste-mode"),
+        (r"\EOF", "end"),  # the entries in the terminfo database for xterms
+        (r"\EOH", "home"),  # seem to be wrong.  this is a less than ideal
+        # workaround
+    ]
+)
+
+
+@dataclass(slots=True)
+class Reader:
+    """The Reader class implements the bare bones of a command reader,
+    handling such details as editing and cursor motion.  What it does
+    not support are such things as completion or history support -
+    these are implemented elsewhere.
+
+    Instance variables of note include:
+
+      * buffer:
+        A *list* (*not* a string at the moment :-) containing all the
+        characters that have been entered.
+      * console:
+        Hopefully encapsulates the OS dependent stuff.
+      * pos:
+        A 0-based index into `buffer' for where the insertion point
+        is.
+      * screeninfo:
+        Ahem.  This list contains some info needed to move the
+        insertion point around reasonably efficiently.
+      * cxy, lxy:
+        the position of the insertion point in screen ...
+      * syntax_table:
+        Dictionary mapping characters to `syntax class'; read the
+        emacs docs to see what this means :-)
+      * commands:
+        Dictionary mapping command names to command classes.
+      * arg:
+        The emacs-style prefix argument.  It will be None if no such
+        argument has been provided.
+      * dirty:
+        True if we need to refresh the display.
+      * kill_ring:
+        The emacs-style kill-ring; manipulated with yank & yank-pop
+      * ps1, ps2, ps3, ps4:
+        prompts.  ps1 is the prompt for a one-line input; for a
+        multiline input it looks like:
+            ps2> first line of input goes here
+            ps3> second and further
+            ps3> lines get ps3
+            ...
+            ps4> and the last one gets ps4
+        As with the usual top-level, you can set these to instances if
+        you like; str() will be called on them (once) at the beginning
+        of each command.  Don't put really long or newline containing
+        strings here, please!
+        This is just the default policy; you can change it freely by
+        overriding get_prompt() (and indeed some standard subclasses
+        do).
+      * finished:
+        handle1 will set this to a true value if a command signals
+        that we're done.
+    """
+
+    console: console.Console
+
+    ## state
+    buffer: list[str] = field(default_factory=list)
+    pos: int = 0
+    ps1: str = "->> "
+    ps2: str = "/>> "
+    ps3: str = "|.. "
+    ps4: str = R"\__ "
+    kill_ring: list[list[str]] = field(default_factory=list)
+    msg: str = ""
+    arg: int | None = None
+    dirty: bool = False
+    finished: bool = False
+    paste_mode: bool = False
+    in_bracketed_paste: bool = False
+    commands: dict[str, type[Command]] = field(default_factory=make_default_commands)
+    last_command: type[Command] | None = None
+    syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table)
+    keymap: tuple[tuple[str, str], ...] = ()
+    input_trans: input.KeymapTranslator = field(init=False)
+    input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list)
+    screen: list[str] = field(default_factory=list)
+    screeninfo: list[tuple[int, list[int]]] = field(init=False)
+    cxy: tuple[int, int] = field(init=False)
+    lxy: tuple[int, int] = field(init=False)
+    scheduled_commands: list[str] = field(default_factory=list)
+    can_colorize: bool = False
+    threading_hook: Callback | None = None
+
+    ## cached metadata to speed up screen refreshes
+    @dataclass
+    class RefreshCache:
+        in_bracketed_paste: bool = False
+        screen: list[str] = field(default_factory=list)
+        screeninfo: list[tuple[int, list[int]]] = field(init=False)
+        line_end_offsets: list[int] = field(default_factory=list)
+        pos: int = field(init=False)
+        cxy: tuple[int, int] = field(init=False)
+        dimensions: tuple[int, int] = field(init=False)
+        invalidated: bool = False
+
+        def update_cache(self,
+                         reader: Reader,
+                         screen: list[str],
+                         screeninfo: list[tuple[int, list[int]]],
+            ) -> None:
+            self.in_bracketed_paste = reader.in_bracketed_paste
+            self.screen = screen.copy()
+            self.screeninfo = screeninfo.copy()
+            self.pos = reader.pos
+            self.cxy = reader.cxy
+            self.dimensions = reader.console.width, reader.console.height
+            self.invalidated = False
+
+        def valid(self, reader: Reader) -> bool:
+            if self.invalidated:
+                return False
+            dimensions = reader.console.width, reader.console.height
+            dimensions_changed = dimensions != self.dimensions
+            paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste
+            return not (dimensions_changed or paste_changed)
+
+        def get_cached_location(self, reader: Reader) -> tuple[int, int]:
+            if self.invalidated:
+                raise ValueError("Cache is invalidated")
+            offset = 0
+            earliest_common_pos = min(reader.pos, self.pos)
+            num_common_lines = len(self.line_end_offsets)
+            while num_common_lines > 0:
+                offset = self.line_end_offsets[num_common_lines - 1]
+                if earliest_common_pos > offset:
+                    break
+                num_common_lines -= 1
+            else:
+                offset = 0
+            return offset, num_common_lines
+
+    last_refresh_cache: RefreshCache = field(default_factory=RefreshCache)
+
+    def __post_init__(self) -> None:
+        # Enable the use of `insert` without a `prepare` call - necessary to
+        # facilitate the tab completion hack implemented for
+        # <https://bugs.python.org/issue25660>.
+        self.keymap = self.collect_keymap()
+        self.input_trans = input.KeymapTranslator(
+            self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
+        )
+        self.screeninfo = [(0, [])]
+        self.cxy = self.pos2xy()
+        self.lxy = (self.pos, 0)
+        self.can_colorize = can_colorize()
+
+        self.last_refresh_cache.screeninfo = self.screeninfo
+        self.last_refresh_cache.pos = self.pos
+        self.last_refresh_cache.cxy = self.cxy
+        self.last_refresh_cache.dimensions = (0, 0)
+
+    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+        return default_keymap
+
+    def calc_screen(self) -> list[str]:
+        """Translate changes in self.buffer into changes in self.console.screen."""
+        # Since the last call to calc_screen:
+        # screen and screeninfo may differ due to a completion menu being shown
+        # pos and cxy may differ due to edits, cursor movements, or completion menus
+
+        # Lines that are above both the old and new cursor position can't have changed,
+        # unless the terminal has been resized (which might cause reflowing) or we've
+        # entered or left paste mode (which changes prompts, causing reflowing).
+        num_common_lines = 0
+        offset = 0
+        if self.last_refresh_cache.valid(self):
+            offset, num_common_lines = self.last_refresh_cache.get_cached_location(self)
+
+        screen = self.last_refresh_cache.screen
+        del screen[num_common_lines:]
+
+        screeninfo = self.last_refresh_cache.screeninfo
+        del screeninfo[num_common_lines:]
+
+        last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets
+        del last_refresh_line_end_offsets[num_common_lines:]
+
+        pos = self.pos
+        pos -= offset
+
+        prompt_from_cache = (offset and self.buffer[offset - 1] != "\n")
+
+        lines = "".join(self.buffer[offset:]).split("\n")
+
+        cursor_found = False
+        lines_beyond_cursor = 0
+        for ln, line in enumerate(lines, num_common_lines):
+            ll = len(line)
+            if 0 <= pos <= ll:
+                self.lxy = pos, ln
+                cursor_found = True
+            elif cursor_found:
+                lines_beyond_cursor += 1
+                if lines_beyond_cursor > self.console.height:
+                    # No need to keep formatting lines.
+                    # The console can't show them.
+                    break
+            if prompt_from_cache:
+                # Only the first line's prompt can come from the cache
+                prompt_from_cache = False
+                prompt = ""
+            else:
+                prompt = self.get_prompt(ln, ll >= pos >= 0)
+            while "\n" in prompt:
+                pre_prompt, _, prompt = prompt.partition("\n")
+                last_refresh_line_end_offsets.append(offset)
+                screen.append(pre_prompt)
+                screeninfo.append((0, []))
+            pos -= ll + 1
+            prompt, lp = self.process_prompt(prompt)
+            l, l2 = disp_str(line)
+            wrapcount = (wlen(l) + lp) // self.console.width
+            if wrapcount == 0:
+                offset += ll + 1  # Takes all of the line plus the newline
+                last_refresh_line_end_offsets.append(offset)
+                screen.append(prompt + l)
+                screeninfo.append((lp, l2))
+            else:
+                i = 0
+                while l:
+                    prelen = lp if i == 0 else 0
+                    index_to_wrap_before = 0
+                    column = 0
+                    for character_width in l2:
+                        if column + character_width >= self.console.width - prelen:
+                            break
+                        index_to_wrap_before += 1
+                        column += character_width
+                    pre = prompt if i == 0 else ""
+                    if len(l) > index_to_wrap_before:
+                        offset += index_to_wrap_before
+                        post = "\\"
+                        after = [1]
+                    else:
+                        offset += index_to_wrap_before + 1  # Takes the newline
+                        post = ""
+                        after = []
+                    last_refresh_line_end_offsets.append(offset)
+                    screen.append(pre + l[:index_to_wrap_before] + post)
+                    screeninfo.append((prelen, l2[:index_to_wrap_before] + after))
+                    l = l[index_to_wrap_before:]
+                    l2 = l2[index_to_wrap_before:]
+                    i += 1
+        self.screeninfo = screeninfo
+        self.cxy = self.pos2xy()
+        if self.msg:
+            for mline in self.msg.split("\n"):
+                screen.append(mline)
+                screeninfo.append((0, []))
+
+        self.last_refresh_cache.update_cache(self, screen, screeninfo)
+        return screen
+
+    @staticmethod
+    def process_prompt(prompt: str) -> tuple[str, int]:
+        """Process the prompt.
+
+        This means calculate the length of the prompt. The character \x01
+        and \x02 are used to bracket ANSI control sequences and need to be
+        excluded from the length calculation.  So also a copy of the prompt
+        is returned with these control characters removed."""
+
+        # The logic below also ignores the length of common escape
+        # sequences if they were not explicitly within \x01...\x02.
+        # They are CSI (or ANSI) sequences  ( ESC [ ... LETTER )
+
+        # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars,
+        # which breaks the logic below so we redefine it here.
+        def wlen(s: str) -> int:
+            return sum(str_width(i) for i in s)
+
+        out_prompt = ""
+        l = wlen(prompt)
+        pos = 0
+        while True:
+            s = prompt.find("\x01", pos)
+            if s == -1:
+                break
+            e = prompt.find("\x02", s)
+            if e == -1:
+                break
+            # Found start and end brackets, subtract from string length
+            l = l - (e - s + 1)
+            keep = prompt[pos:s]
+            l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep)))
+            out_prompt += keep + prompt[s + 1 : e]
+            pos = e + 1
+        keep = prompt[pos:]
+        l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep)))
+        out_prompt += keep
+        return out_prompt, l
+
+    def bow(self, p: int | None = None) -> int:
+        """Return the 0-based index of the word break preceding p most
+        immediately.
+
+        p defaults to self.pos; word boundaries are determined using
+        self.syntax_table."""
+        if p is None:
+            p = self.pos
+        st = self.syntax_table
+        b = self.buffer
+        p -= 1
+        while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD:
+            p -= 1
+        while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
+            p -= 1
+        return p + 1
+
+    def eow(self, p: int | None = None) -> int:
+        """Return the 0-based index of the word break following p most
+        immediately.
+
+        p defaults to self.pos; word boundaries are determined using
+        self.syntax_table."""
+        if p is None:
+            p = self.pos
+        st = self.syntax_table
+        b = self.buffer
+        while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD:
+            p += 1
+        while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
+            p += 1
+        return p
+
+    def bol(self, p: int | None = None) -> int:
+        """Return the 0-based index of the line break preceding p most
+        immediately.
+
+        p defaults to self.pos."""
+        if p is None:
+            p = self.pos
+        b = self.buffer
+        p -= 1
+        while p >= 0 and b[p] != "\n":
+            p -= 1
+        return p + 1
+
+    def eol(self, p: int | None = None) -> int:
+        """Return the 0-based index of the line break following p most
+        immediately.
+
+        p defaults to self.pos."""
+        if p is None:
+            p = self.pos
+        b = self.buffer
+        while p < len(b) and b[p] != "\n":
+            p += 1
+        return p
+
+    def max_column(self, y: int) -> int:
+        """Return the last x-offset for line y"""
+        return self.screeninfo[y][0] + sum(self.screeninfo[y][1])
+
+    def max_row(self) -> int:
+        return len(self.screeninfo) - 1
+
+    def get_arg(self, default: int = 1) -> int:
+        """Return any prefix argument that the user has supplied,
+        returning `default' if there is None.  Defaults to 1.
+        """
+        if self.arg is None:
+            return default
+        return self.arg
+
+    def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
+        """Return what should be in the left-hand margin for line
+        `lineno'."""
+        if self.arg is not None and cursor_on_line:
+            prompt = f"(arg: {self.arg}) "
+        elif self.paste_mode and not self.in_bracketed_paste:
+            prompt = "(paste) "
+        elif "\n" in self.buffer:
+            if lineno == 0:
+                prompt = self.ps2
+            elif self.ps4 and lineno == self.buffer.count("\n"):
+                prompt = self.ps4
+            else:
+                prompt = self.ps3
+        else:
+            prompt = self.ps1
+
+        if self.can_colorize:
+            prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}"
+        return prompt
+
+    def push_input_trans(self, itrans: input.KeymapTranslator) -> None:
+        self.input_trans_stack.append(self.input_trans)
+        self.input_trans = itrans
+
+    def pop_input_trans(self) -> None:
+        self.input_trans = self.input_trans_stack.pop()
+
+    def setpos_from_xy(self, x: int, y: int) -> None:
+        """Set pos according to coordinates x, y"""
+        pos = 0
+        i = 0
+        while i < y:
+            prompt_len, character_widths = self.screeninfo[i]
+            offset = len(character_widths) - character_widths.count(0)
+            in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width
+            if in_wrapped_line:
+                pos += offset - 1  # -1 cause backslash is not in buffer
+            else:
+                pos += offset + 1  # +1 cause newline is in buffer
+            i += 1
+
+        j = 0
+        cur_x = self.screeninfo[i][0]
+        while cur_x < x:
+            if self.screeninfo[i][1][j] == 0:
+                continue
+            cur_x += self.screeninfo[i][1][j]
+            j += 1
+            pos += 1
+
+        self.pos = pos
+
+    def pos2xy(self) -> tuple[int, int]:
+        """Return the x, y coordinates of position 'pos'."""
+        # this *is* incomprehensible, yes.
+        p, y = 0, 0
+        l2: list[int] = []
+        pos = self.pos
+        assert 0 <= pos <= len(self.buffer)
+        if pos == len(self.buffer) and len(self.screeninfo) > 0:
+            y = len(self.screeninfo) - 1
+            p, l2 = self.screeninfo[y]
+            return p + sum(l2) + l2.count(0), y
+
+        for p, l2 in self.screeninfo:
+            l = len(l2) - l2.count(0)
+            in_wrapped_line = p + sum(l2) >= self.console.width
+            offset = l - 1 if in_wrapped_line else l  # need to remove backslash
+            if offset >= pos:
+                break
+
+            if p + sum(l2) >= self.console.width:
+                pos -= l - 1  # -1 cause backslash is not in buffer
+            else:
+                pos -= l + 1  # +1 cause newline is in buffer
+            y += 1
+        return p + sum(l2[:pos]), y
+
+    def insert(self, text: str | list[str]) -> None:
+        """Insert 'text' at the insertion point."""
+        self.buffer[self.pos : self.pos] = list(text)
+        self.pos += len(text)
+        self.dirty = True
+
+    def update_cursor(self) -> None:
+        """Move the cursor to reflect changes in self.pos"""
+        self.cxy = self.pos2xy()
+        self.console.move_cursor(*self.cxy)
+
+    def after_command(self, cmd: Command) -> None:
+        """This function is called to allow post command cleanup."""
+        if getattr(cmd, "kills_digit_arg", True):
+            if self.arg is not None:
+                self.dirty = True
+            self.arg = None
+
+    def prepare(self) -> None:
+        """Get ready to run.  Call restore when finished.  You must not
+        write to the console in between the calls to prepare and
+        restore."""
+        try:
+            self.console.prepare()
+            self.arg = None
+            self.finished = False
+            del self.buffer[:]
+            self.pos = 0
+            self.dirty = True
+            self.last_command = None
+            self.calc_screen()
+        except BaseException:
+            self.restore()
+            raise
+
+        while self.scheduled_commands:
+            cmd = self.scheduled_commands.pop()
+            self.do_cmd((cmd, []))
+
+    def last_command_is(self, cls: type) -> bool:
+        if not self.last_command:
+            return False
+        return issubclass(cls, self.last_command)
+
+    def restore(self) -> None:
+        """Clean up after a run."""
+        self.console.restore()
+
+    @contextmanager
+    def suspend(self) -> SimpleContextManager:
+        """A context manager to delegate to another reader."""
+        prev_state = {f.name: getattr(self, f.name) for f in fields(self)}
+        try:
+            self.restore()
+            yield
+        finally:
+            for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"):
+                setattr(self, arg, prev_state[arg])
+            self.prepare()
+
+    def finish(self) -> None:
+        """Called when a command signals that we're finished."""
+        pass
+
+    def error(self, msg: str = "none") -> None:
+        self.msg = "! " + msg + " "
+        self.dirty = True
+        self.console.beep()
+
+    def update_screen(self) -> None:
+        if self.dirty:
+            self.refresh()
+
+    def refresh(self) -> None:
+        """Recalculate and refresh the screen."""
+        if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n":
+            return
+
+        # this call sets up self.cxy, so call it first.
+        self.screen = self.calc_screen()
+        self.console.refresh(self.screen, self.cxy)
+        self.dirty = False
+
+    def do_cmd(self, cmd: tuple[str, list[str]]) -> None:
+        """`cmd` is a tuple of "event_name" and "event", which in the current
+        implementation is always just the "buffer" which happens to be a list
+        of single-character strings."""
+
+        trace("received command {cmd}", cmd=cmd)
+        if isinstance(cmd[0], str):
+            command_type = self.commands.get(cmd[0], commands.invalid_command)
+        elif isinstance(cmd[0], type):
+            command_type = cmd[0]
+        else:
+            return  # nothing to do
+
+        command = command_type(self, *cmd)  # type: ignore[arg-type]
+        command.do()
+
+        self.after_command(command)
+
+        if self.dirty:
+            self.refresh()
+        else:
+            self.update_cursor()
+
+        if not isinstance(cmd, commands.digit_arg):
+            self.last_command = command_type
+
+        self.finished = bool(command.finish)
+        if self.finished:
+            self.console.finish()
+            self.finish()
+
+    def run_hooks(self) -> None:
+        threading_hook = self.threading_hook
+        if threading_hook is None and 'threading' in sys.modules:
+            from ._threading_handler import install_threading_hook
+            install_threading_hook(self)
+        if threading_hook is not None:
+            try:
+                threading_hook()
+            except Exception:
+                pass
+
+        input_hook = self.console.input_hook
+        if input_hook:
+            try:
+                input_hook()
+            except Exception:
+                pass
+
+    def handle1(self, block: bool = True) -> bool:
+        """Handle a single event.  Wait as long as it takes if block
+        is true (the default), otherwise return False if no event is
+        pending."""
+
+        if self.msg:
+            self.msg = ""
+            self.dirty = True
+
+        while True:
+            # We use the same timeout as in readline.c: 100ms
+            self.run_hooks()
+            self.console.wait(100)
+            event = self.console.get_event(block=False)
+            if not event:
+                if block:
+                    continue
+                return False
+
+            translate = True
+
+            if event.evt == "key":
+                self.input_trans.push(event)
+            elif event.evt == "scroll":
+                self.refresh()
+            elif event.evt == "resize":
+                self.refresh()
+            else:
+                translate = False
+
+            if translate:
+                cmd = self.input_trans.get()
+            else:
+                cmd = [event.evt, event.data]
+
+            if cmd is None:
+                if block:
+                    continue
+                return False
+
+            self.do_cmd(cmd)
+            return True
+
+    def push_char(self, char: int | bytes) -> None:
+        self.console.push_char(char)
+        self.handle1(block=False)
+
+    def readline(self, startup_hook: Callback | None = None) -> str:
+        """Read a line.  The implementation of this method also shows
+        how to drive Reader if you want more control over the event
+        loop."""
+        self.prepare()
+        try:
+            if startup_hook is not None:
+                startup_hook()
+            self.refresh()
+            while not self.finished:
+                self.handle1()
+            return self.get_unicode()
+
+        finally:
+            self.restore()
+
+    def bind(self, spec: KeySpec, command: CommandName) -> None:
+        self.keymap = self.keymap + ((spec, command),)
+        self.input_trans = input.KeymapTranslator(
+            self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
+        )
+
+    def get_unicode(self) -> str:
+        """Return the current buffer as a unicode string."""
+        return "".join(self.buffer)

Lib/_pyrepl/readline.py

@@ -0,0 +1,598 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Alex Gaynor
+#                       Antonio Cuni
+#                       Armin Rigo
+#                       Holger Krekel
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""A compatibility wrapper reimplementing the 'readline' standard module
+on top of pyrepl.  Not all functionalities are supported.  Contains
+extensions for multiline input.
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass, field
+
+import os
+from site import gethistoryfile   # type: ignore[attr-defined]
+import sys
+from rlcompleter import Completer as RLCompleter
+
+from . import commands, historical_reader
+from .completing_reader import CompletingReader
+from .console import Console as ConsoleType
+
+Console: type[ConsoleType]
+_error: tuple[type[Exception], ...] | type[Exception]
+try:
+    from .unix_console import UnixConsole as Console, _error
+except ImportError:
+    from .windows_console import WindowsConsole as Console, _error
+
+ENCODING = sys.getdefaultencoding() or "latin1"
+
+
+# types
+Command = commands.Command
+from collections.abc import Callable, Collection
+from .types import Callback, Completer, KeySpec, CommandName
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+    from typing import Any, Mapping
+
+
+MoreLinesCallable = Callable[[str], bool]
+
+
+__all__ = [
+    "add_history",
+    "clear_history",
+    "get_begidx",
+    "get_completer",
+    "get_completer_delims",
+    "get_current_history_length",
+    "get_endidx",
+    "get_history_item",
+    "get_history_length",
+    "get_line_buffer",
+    "insert_text",
+    "parse_and_bind",
+    "read_history_file",
+    # "read_init_file",
+    # "redisplay",
+    "remove_history_item",
+    "replace_history_item",
+    "set_auto_history",
+    "set_completer",
+    "set_completer_delims",
+    "set_history_length",
+    # "set_pre_input_hook",
+    "set_startup_hook",
+    "write_history_file",
+    # ---- multiline extensions ----
+    "multiline_input",
+]
+
+# ____________________________________________________________
+
+@dataclass
+class ReadlineConfig:
+    readline_completer: Completer | None = None
+    completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?")
+
+
+@dataclass(kw_only=True)
+class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader):
+    # Class fields
+    assume_immutable_completions = False
+    use_brackets = False
+    sort_in_column = True
+
+    # Instance fields
+    config: ReadlineConfig
+    more_lines: MoreLinesCallable | None = None
+    last_used_indentation: str | None = None
+
+    def __post_init__(self) -> None:
+        super().__post_init__()
+        self.commands["maybe_accept"] = maybe_accept
+        self.commands["maybe-accept"] = maybe_accept
+        self.commands["backspace_dedent"] = backspace_dedent
+        self.commands["backspace-dedent"] = backspace_dedent
+
+    def error(self, msg: str = "none") -> None:
+        pass  # don't show error messages by default
+
+    def get_stem(self) -> str:
+        b = self.buffer
+        p = self.pos - 1
+        completer_delims = self.config.completer_delims
+        while p >= 0 and b[p] not in completer_delims:
+            p -= 1
+        return "".join(b[p + 1 : self.pos])
+
+    def get_completions(self, stem: str) -> list[str]:
+        if len(stem) == 0 and self.more_lines is not None:
+            b = self.buffer
+            p = self.pos
+            while p > 0 and b[p - 1] != "\n":
+                p -= 1
+            num_spaces = 4 - ((self.pos - p) % 4)
+            return [" " * num_spaces]
+        result = []
+        function = self.config.readline_completer
+        if function is not None:
+            try:
+                stem = str(stem)  # rlcompleter.py seems to not like unicode
+            except UnicodeEncodeError:
+                pass  # but feed unicode anyway if we have no choice
+            state = 0
+            while True:
+                try:
+                    next = function(stem, state)
+                except Exception:
+                    break
+                if not isinstance(next, str):
+                    break
+                result.append(next)
+                state += 1
+            # emulate the behavior of the standard readline that sorts
+            # the completions before displaying them.
+            result.sort()
+        return result
+
+    def get_trimmed_history(self, maxlength: int) -> list[str]:
+        if maxlength >= 0:
+            cut = len(self.history) - maxlength
+            if cut < 0:
+                cut = 0
+        else:
+            cut = 0
+        return self.history[cut:]
+
+    def update_last_used_indentation(self) -> None:
+        indentation = _get_first_indentation(self.buffer)
+        if indentation is not None:
+            self.last_used_indentation = indentation
+
+    # --- simplified support for reading multiline Python statements ---
+
+    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+        return super().collect_keymap() + (
+            (r"\n", "maybe-accept"),
+            (r"\<backspace>", "backspace-dedent"),
+        )
+
+    def after_command(self, cmd: Command) -> None:
+        super().after_command(cmd)
+        if self.more_lines is None:
+            # Force single-line input if we are in raw_input() mode.
+            # Although there is no direct way to add a \n in this mode,
+            # multiline buffers can still show up using various
+            # commands, e.g. navigating the history.
+            try:
+                index = self.buffer.index("\n")
+            except ValueError:
+                pass
+            else:
+                self.buffer = self.buffer[:index]
+                if self.pos > len(self.buffer):
+                    self.pos = len(self.buffer)
+
+
+def set_auto_history(_should_auto_add_history: bool) -> None:
+    """Enable or disable automatic history"""
+    historical_reader.should_auto_add_history = bool(_should_auto_add_history)
+
+
+def _get_this_line_indent(buffer: list[str], pos: int) -> int:
+    indent = 0
+    while pos > 0 and buffer[pos - 1] in " \t":
+        indent += 1
+        pos -= 1
+    if pos > 0 and buffer[pos - 1] == "\n":
+        return indent
+    return 0
+
+
+def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]:
+    prevlinestart = pos
+    while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n":
+        prevlinestart -= 1
+    prevlinetext = prevlinestart
+    while prevlinetext < pos and buffer[prevlinetext] in " \t":
+        prevlinetext += 1
+    if prevlinetext == pos:
+        indent = None
+    else:
+        indent = prevlinetext - prevlinestart
+    return prevlinestart, indent
+
+
+def _get_first_indentation(buffer: list[str]) -> str | None:
+    indented_line_start = None
+    for i in range(len(buffer)):
+        if (i < len(buffer) - 1
+            and buffer[i] == "\n"
+            and buffer[i + 1] in " \t"
+        ):
+            indented_line_start = i + 1
+        elif indented_line_start is not None and buffer[i] not in " \t\n":
+            return ''.join(buffer[indented_line_start : i])
+    return None
+
+
+def _should_auto_indent(buffer: list[str], pos: int) -> bool:
+    # check if last character before "pos" is a colon, ignoring
+    # whitespaces and comments.
+    last_char = None
+    while pos > 0:
+        pos -= 1
+        if last_char is None:
+            if buffer[pos] not in " \t\n#":  # ignore whitespaces and comments
+                last_char = buffer[pos]
+        else:
+            # even if we found a non-whitespace character before
+            # original pos, we keep going back until newline is reached
+            # to make sure we ignore comments
+            if buffer[pos] == "\n":
+                break
+            if buffer[pos] == "#":
+                last_char = None
+    return last_char == ":"
+
+
+class maybe_accept(commands.Command):
+    def do(self) -> None:
+        r: ReadlineAlikeReader
+        r = self.reader  # type: ignore[assignment]
+        r.dirty = True  # this is needed to hide the completion menu, if visible
+
+        if self.reader.in_bracketed_paste:
+            r.insert("\n")
+            return
+
+        # if there are already several lines and the cursor
+        # is not on the last one, always insert a new \n.
+        text = r.get_unicode()
+
+        if "\n" in r.buffer[r.pos :] or (
+            r.more_lines is not None and r.more_lines(text)
+        ):
+            def _newline_before_pos():
+                before_idx = r.pos - 1
+                while before_idx > 0 and text[before_idx].isspace():
+                    before_idx -= 1
+                return text[before_idx : r.pos].count("\n") > 0
+
+            # if there's already a new line before the cursor then
+            # even if the cursor is followed by whitespace, we assume
+            # the user is trying to terminate the block
+            if _newline_before_pos() and text[r.pos:].isspace():
+                self.finish = True
+                return
+
+            # auto-indent the next line like the previous line
+            prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos)
+            r.insert("\n")
+            if not self.reader.paste_mode:
+                if indent:
+                    for i in range(prevlinestart, prevlinestart + indent):
+                        r.insert(r.buffer[i])
+                r.update_last_used_indentation()
+                if _should_auto_indent(r.buffer, r.pos):
+                    if r.last_used_indentation is not None:
+                        indentation = r.last_used_indentation
+                    else:
+                        # default
+                        indentation = " " * 4
+                    r.insert(indentation)
+        elif not self.reader.paste_mode:
+            self.finish = True
+        else:
+            r.insert("\n")
+
+
+class backspace_dedent(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        if r.pos > 0:
+            repeat = 1
+            if b[r.pos - 1] != "\n":
+                indent = _get_this_line_indent(b, r.pos)
+                if indent > 0:
+                    ls = r.pos - indent
+                    while ls > 0:
+                        ls, pi = _get_previous_line_indent(b, ls - 1)
+                        if pi is not None and pi < indent:
+                            repeat = indent - pi
+                            break
+            r.pos -= repeat
+            del b[r.pos : r.pos + repeat]
+            r.dirty = True
+        else:
+            self.reader.error("can't backspace at start")
+
+
+# ____________________________________________________________
+
+
+@dataclass(slots=True)
+class _ReadlineWrapper:
+    f_in: int = -1
+    f_out: int = -1
+    reader: ReadlineAlikeReader | None = field(default=None, repr=False)
+    saved_history_length: int = -1
+    startup_hook: Callback | None = None
+    config: ReadlineConfig = field(default_factory=ReadlineConfig, repr=False)
+
+    def __post_init__(self) -> None:
+        if self.f_in == -1:
+            self.f_in = os.dup(0)
+        if self.f_out == -1:
+            self.f_out = os.dup(1)
+
+    def get_reader(self) -> ReadlineAlikeReader:
+        if self.reader is None:
+            console = Console(self.f_in, self.f_out, encoding=ENCODING)
+            self.reader = ReadlineAlikeReader(console=console, config=self.config)
+        return self.reader
+
+    def input(self, prompt: object = "") -> str:
+        try:
+            reader = self.get_reader()
+        except _error:
+            assert raw_input is not None
+            return raw_input(prompt)
+        prompt_str = str(prompt)
+        reader.ps1 = prompt_str
+        sys.audit("builtins.input", prompt_str)
+        result = reader.readline(startup_hook=self.startup_hook)
+        sys.audit("builtins.input/result", result)
+        return result
+
+    def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str:
+        """Read an input on possibly multiple lines, asking for more
+        lines as long as 'more_lines(unicodetext)' returns an object whose
+        boolean value is true.
+        """
+        reader = self.get_reader()
+        saved = reader.more_lines
+        try:
+            reader.more_lines = more_lines
+            reader.ps1 = ps1
+            reader.ps2 = ps1
+            reader.ps3 = ps2
+            reader.ps4 = ""
+            with warnings.catch_warnings(action="ignore"):
+                return reader.readline()
+        finally:
+            reader.more_lines = saved
+            reader.paste_mode = False
+
+    def parse_and_bind(self, string: str) -> None:
+        pass  # XXX we don't support parsing GNU-readline-style init files
+
+    def set_completer(self, function: Completer | None = None) -> None:
+        self.config.readline_completer = function
+
+    def get_completer(self) -> Completer | None:
+        return self.config.readline_completer
+
+    def set_completer_delims(self, delimiters: Collection[str]) -> None:
+        self.config.completer_delims = frozenset(delimiters)
+
+    def get_completer_delims(self) -> str:
+        return "".join(sorted(self.config.completer_delims))
+
+    def _histline(self, line: str) -> str:
+        line = line.rstrip("\n")
+        return line
+
+    def get_history_length(self) -> int:
+        return self.saved_history_length
+
+    def set_history_length(self, length: int) -> None:
+        self.saved_history_length = length
+
+    def get_current_history_length(self) -> int:
+        return len(self.get_reader().history)
+
+    def read_history_file(self, filename: str = gethistoryfile()) -> None:
+        # multiline extension (really a hack) for the end of lines that
+        # are actually continuations inside a single multiline_input()
+        # history item: we use \r\n instead of just \n.  If the history
+        # file is passed to GNU readline, the extra \r are just ignored.
+        history = self.get_reader().history
+
+        with open(os.path.expanduser(filename), 'rb') as f:
+            is_editline = f.readline().startswith(b"_HiStOrY_V2_")
+            if is_editline:
+                encoding = "unicode-escape"
+            else:
+                f.seek(0)
+                encoding = "utf-8"
+
+            lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')]
+            buffer = []
+            for line in lines:
+                if line.endswith("\r"):
+                    buffer.append(line+'\n')
+                else:
+                    line = self._histline(line)
+                    if buffer:
+                        line = self._histline("".join(buffer).replace("\r", "") + line)
+                        del buffer[:]
+                    if line:
+                        history.append(line)
+
+    def write_history_file(self, filename: str = gethistoryfile()) -> None:
+        maxlength = self.saved_history_length
+        history = self.get_reader().get_trimmed_history(maxlength)
+        f = open(os.path.expanduser(filename), "w",
+                 encoding="utf-8", newline="\n")
+        with f:
+            for entry in history:
+                entry = entry.replace("\n", "\r\n")  # multiline history support
+                f.write(entry + "\n")
+
+    def clear_history(self) -> None:
+        del self.get_reader().history[:]
+
+    def get_history_item(self, index: int) -> str | None:
+        history = self.get_reader().history
+        if 1 <= index <= len(history):
+            return history[index - 1]
+        else:
+            return None  # like readline.c
+
+    def remove_history_item(self, index: int) -> None:
+        history = self.get_reader().history
+        if 0 <= index < len(history):
+            del history[index]
+        else:
+            raise ValueError("No history item at position %d" % index)
+            # like readline.c
+
+    def replace_history_item(self, index: int, line: str) -> None:
+        history = self.get_reader().history
+        if 0 <= index < len(history):
+            history[index] = self._histline(line)
+        else:
+            raise ValueError("No history item at position %d" % index)
+            # like readline.c
+
+    def add_history(self, line: str) -> None:
+        self.get_reader().history.append(self._histline(line))
+
+    def set_startup_hook(self, function: Callback | None = None) -> None:
+        self.startup_hook = function
+
+    def get_line_buffer(self) -> str:
+        return self.get_reader().get_unicode()
+
+    def _get_idxs(self) -> tuple[int, int]:
+        start = cursor = self.get_reader().pos
+        buf = self.get_line_buffer()
+        for i in range(cursor - 1, -1, -1):
+            if buf[i] in self.get_completer_delims():
+                break
+            start = i
+        return start, cursor
+
+    def get_begidx(self) -> int:
+        return self._get_idxs()[0]
+
+    def get_endidx(self) -> int:
+        return self._get_idxs()[1]
+
+    def insert_text(self, text: str) -> None:
+        self.get_reader().insert(text)
+
+
+_wrapper = _ReadlineWrapper()
+
+# ____________________________________________________________
+# Public API
+
+parse_and_bind = _wrapper.parse_and_bind
+set_completer = _wrapper.set_completer
+get_completer = _wrapper.get_completer
+set_completer_delims = _wrapper.set_completer_delims
+get_completer_delims = _wrapper.get_completer_delims
+get_history_length = _wrapper.get_history_length
+set_history_length = _wrapper.set_history_length
+get_current_history_length = _wrapper.get_current_history_length
+read_history_file = _wrapper.read_history_file
+write_history_file = _wrapper.write_history_file
+clear_history = _wrapper.clear_history
+get_history_item = _wrapper.get_history_item
+remove_history_item = _wrapper.remove_history_item
+replace_history_item = _wrapper.replace_history_item
+add_history = _wrapper.add_history
+set_startup_hook = _wrapper.set_startup_hook
+get_line_buffer = _wrapper.get_line_buffer
+get_begidx = _wrapper.get_begidx
+get_endidx = _wrapper.get_endidx
+insert_text = _wrapper.insert_text
+
+# Extension
+multiline_input = _wrapper.multiline_input
+
+# Internal hook
+_get_reader = _wrapper.get_reader
+
+# ____________________________________________________________
+# Stubs
+
+
+def _make_stub(_name: str, _ret: object) -> None:
+    def stub(*args: object, **kwds: object) -> None:
+        import warnings
+
+        warnings.warn("readline.%s() not implemented" % _name, stacklevel=2)
+
+    stub.__name__ = _name
+    globals()[_name] = stub
+
+
+for _name, _ret in [
+    ("read_init_file", None),
+    ("redisplay", None),
+    ("set_pre_input_hook", None),
+]:
+    assert _name not in globals(), _name
+    _make_stub(_name, _ret)
+
+# ____________________________________________________________
+
+
+def _setup(namespace: Mapping[str, Any]) -> None:
+    global raw_input
+    if raw_input is not None:
+        return  # don't run _setup twice
+
+    try:
+        f_in = sys.stdin.fileno()
+        f_out = sys.stdout.fileno()
+    except (AttributeError, ValueError):
+        return
+    if not os.isatty(f_in) or not os.isatty(f_out):
+        return
+
+    _wrapper.f_in = f_in
+    _wrapper.f_out = f_out
+
+    # set up namespace in rlcompleter, which requires it to be a bona fide dict
+    if not isinstance(namespace, dict):
+        namespace = dict(namespace)
+    _wrapper.config.readline_completer = RLCompleter(namespace).complete
+
+    # this is not really what readline.c does.  Better than nothing I guess
+    import builtins
+    raw_input = builtins.input
+    builtins.input = _wrapper.input
+
+
+raw_input: Callable[[object], str] | None = None

Lib/_pyrepl/simple_interact.py

@@ -0,0 +1,167 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""This is an alternative to python_reader which tries to emulate
+the CPython prompt as closely as possible, with the exception of
+allowing multiline input and multiline history entries.
+"""
+
+from __future__ import annotations
+
+import _sitebuiltins
+import linecache
+import functools
+import os
+import sys
+import code
+
+from .readline import _get_reader, multiline_input
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+_error: tuple[type[Exception], ...] | type[Exception]
+try:
+    from .unix_console import _error
+except ModuleNotFoundError:
+    from .windows_console import _error
+
+def check() -> str:
+    """Returns the error message if there is a problem initializing the state."""
+    try:
+        _get_reader()
+    except _error as e:
+        if term := os.environ.get("TERM", ""):
+            term = f"; TERM={term}"
+        return str(str(e) or repr(e) or "unknown error") + term
+    return ""
+
+
+def _strip_final_indent(text: str) -> str:
+    # kill spaces and tabs at the end, but only if they follow '\n'.
+    # meant to remove the auto-indentation only (although it would of
+    # course also remove explicitly-added indentation).
+    short = text.rstrip(" \t")
+    n = len(short)
+    if n > 0 and text[n - 1] == "\n":
+        return short
+    return text
+
+
+def _clear_screen():
+    reader = _get_reader()
+    reader.scheduled_commands.append("clear_screen")
+
+
+REPL_COMMANDS = {
+    "exit": _sitebuiltins.Quitter('exit', ''),
+    "quit": _sitebuiltins.Quitter('quit' ,''),
+    "copyright": _sitebuiltins._Printer('copyright', sys.copyright),
+    "help": _sitebuiltins._Helper(),
+    "clear": _clear_screen,
+    "\x1a": _sitebuiltins.Quitter('\x1a', ''),
+}
+
+
+def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool:
+    # ooh, look at the hack:
+    src = _strip_final_indent(unicodetext)
+    try:
+        code = console.compile(src, "<stdin>", "single")
+    except (OverflowError, SyntaxError, ValueError):
+        lines = src.splitlines(keepends=True)
+        if len(lines) == 1:
+            return False
+
+        last_line = lines[-1]
+        was_indented = last_line.startswith((" ", "\t"))
+        not_empty = last_line.strip() != ""
+        incomplete = not last_line.endswith("\n")
+        return (was_indented or not_empty) and incomplete
+    else:
+        return code is None
+
+
+def run_multiline_interactive_console(
+    console: code.InteractiveConsole,
+    *,
+    future_flags: int = 0,
+) -> None:
+    from .readline import _setup
+    _setup(console.locals)
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
+
+    more_lines = functools.partial(_more_lines, console)
+    input_n = 0
+
+    def maybe_run_command(statement: str) -> bool:
+        statement = statement.strip()
+        if statement in console.locals or statement not in REPL_COMMANDS:
+            return False
+
+        reader = _get_reader()
+        reader.history.pop()  # skip internal commands in history
+        command = REPL_COMMANDS[statement]
+        if callable(command):
+            # Make sure that history does not change because of commands
+            with reader.suspend_history():
+                command()
+            return True
+        return False
+
+    while 1:
+        try:
+            try:
+                sys.stdout.flush()
+            except Exception:
+                pass
+
+            ps1 = getattr(sys, "ps1", ">>> ")
+            ps2 = getattr(sys, "ps2", "... ")
+            try:
+                statement = multiline_input(more_lines, ps1, ps2)
+            except EOFError:
+                break
+
+            if maybe_run_command(statement):
+                continue
+
+            input_name = f"<python-input-{input_n}>"
+            linecache._register_code(input_name, statement, "<stdin>")  # type: ignore[attr-defined]
+            more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single")  # type: ignore[call-arg]
+            assert not more
+            input_n += 1
+        except KeyboardInterrupt:
+            r = _get_reader()
+            if r.input_trans is r.isearch_trans:
+                r.do_cmd(("isearch-end", [""]))
+            r.pos = len(r.get_unicode())
+            r.dirty = True
+            r.refresh()
+            r.in_bracketed_paste = False
+            console.write("\nKeyboardInterrupt\n")
+            console.resetbuffer()
+        except MemoryError:
+            console.write("\nMemoryError\n")
+            console.resetbuffer()

Lib/_pyrepl/trace.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import os
+
+# types
+if False:
+    from typing import IO
+
+
+trace_file: IO[str] | None = None
+if trace_filename := os.environ.get("PYREPL_TRACE"):
+    trace_file = open(trace_filename, "a")
+
+
+def trace(line: str, *k: object, **kw: object) -> None:
+    if trace_file is None:
+        return
+    if k or kw:
+        line = line.format(*k, **kw)
+    trace_file.write(line + "\n")
+    trace_file.flush()

Lib/_pyrepl/types.py

@@ -0,0 +1,8 @@
+from collections.abc import Callable, Iterator
+
+Callback = Callable[[], object]
+SimpleContextManager = Iterator[None]
+KeySpec = str  # like r"\C-c"
+CommandName = str  # like "interrupt"
+EventTuple = tuple[CommandName, str]
+Completer = Callable[[str, int], str | None]

Lib/_pyrepl/unix_console.py

@@ -0,0 +1,810 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Antonio Cuni
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import errno
+import os
+import re
+import select
+import signal
+import struct
+import termios
+import time
+import platform
+from fcntl import ioctl
+
+from . import curses
+from .console import Console, Event
+from .fancy_termios import tcgetattr, tcsetattr
+from .trace import trace
+from .unix_eventqueue import EventQueue
+from .utils import wlen
+
+
+TYPE_CHECKING = False
+
+# types
+if TYPE_CHECKING:
+    from typing import IO, Literal, overload
+else:
+    overload = lambda func: None
+
+
+class InvalidTerminal(RuntimeError):
+    pass
+
+
+_error = (termios.error, curses.error, InvalidTerminal)
+
+SIGWINCH_EVENT = "repaint"
+
+FIONREAD = getattr(termios, "FIONREAD", None)
+TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None)
+
+# ------------ start of baudrate definitions ------------
+
+# Add (possibly) missing baudrates (check termios man page) to termios
+
+
+def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None:
+    baudrate_name = "B%d" % rate
+    if hasattr(termios, baudrate_name):
+        dictionary[getattr(termios, baudrate_name)] = rate
+
+
+# Check the termios man page (Line speed) to know where these
+# values come from.
+potential_baudrates = [
+    0,
+    110,
+    115200,
+    1200,
+    134,
+    150,
+    1800,
+    19200,
+    200,
+    230400,
+    2400,
+    300,
+    38400,
+    460800,
+    4800,
+    50,
+    57600,
+    600,
+    75,
+    9600,
+]
+
+ratedict: dict[int, int] = {}
+for rate in potential_baudrates:
+    add_baudrate_if_supported(ratedict, rate)
+
+# Clean up variables to avoid unintended usage
+del rate, add_baudrate_if_supported
+
+# ------------ end of baudrate definitions ------------
+
+delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>")
+
+try:
+    poll: type[select.poll] = select.poll
+except AttributeError:
+    # this is exactly the minumum necessary to support what we
+    # do with poll objects
+    class MinimalPoll:
+        def __init__(self):
+            pass
+
+        def register(self, fd, flag):
+            self.fd = fd
+        # note: The 'timeout' argument is received as *milliseconds*
+        def poll(self, timeout: float | None = None) -> list[int]:
+            if timeout is None:
+                r, w, e = select.select([self.fd], [], [])
+            else:
+                r, w, e = select.select([self.fd], [], [], timeout/1000)
+            return r
+
+    poll = MinimalPoll  # type: ignore[assignment]
+
+
+class UnixConsole(Console):
+    def __init__(
+        self,
+        f_in: IO[bytes] | int = 0,
+        f_out: IO[bytes] | int = 1,
+        term: str = "",
+        encoding: str = "",
+    ):
+        """
+        Initialize the UnixConsole.
+
+        Parameters:
+        - f_in (int or file-like object): Input file descriptor or object.
+        - f_out (int or file-like object): Output file descriptor or object.
+        - term (str): Terminal name.
+        - encoding (str): Encoding to use for I/O operations.
+        """
+        super().__init__(f_in, f_out, term, encoding)
+
+        self.pollob = poll()
+        self.pollob.register(self.input_fd, select.POLLIN)
+        self.input_buffer = b""
+        self.input_buffer_pos = 0
+        curses.setupterm(term or None, self.output_fd)
+        self.term = term
+
+        @overload
+        def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ...
+
+        @overload
+        def _my_getstr(cap: str, optional: bool) -> bytes | None: ...
+
+        def _my_getstr(cap: str, optional: bool = False) -> bytes | None:
+            r = curses.tigetstr(cap)
+            if not optional and r is None:
+                raise InvalidTerminal(
+                    f"terminal doesn't have the required {cap} capability"
+                )
+            return r
+
+        self._bel = _my_getstr("bel")
+        self._civis = _my_getstr("civis", optional=True)
+        self._clear = _my_getstr("clear")
+        self._cnorm = _my_getstr("cnorm", optional=True)
+        self._cub = _my_getstr("cub", optional=True)
+        self._cub1 = _my_getstr("cub1", optional=True)
+        self._cud = _my_getstr("cud", optional=True)
+        self._cud1 = _my_getstr("cud1", optional=True)
+        self._cuf = _my_getstr("cuf", optional=True)
+        self._cuf1 = _my_getstr("cuf1", optional=True)
+        self._cup = _my_getstr("cup")
+        self._cuu = _my_getstr("cuu", optional=True)
+        self._cuu1 = _my_getstr("cuu1", optional=True)
+        self._dch1 = _my_getstr("dch1", optional=True)
+        self._dch = _my_getstr("dch", optional=True)
+        self._el = _my_getstr("el")
+        self._hpa = _my_getstr("hpa", optional=True)
+        self._ich = _my_getstr("ich", optional=True)
+        self._ich1 = _my_getstr("ich1", optional=True)
+        self._ind = _my_getstr("ind", optional=True)
+        self._pad = _my_getstr("pad", optional=True)
+        self._ri = _my_getstr("ri", optional=True)
+        self._rmkx = _my_getstr("rmkx", optional=True)
+        self._smkx = _my_getstr("smkx", optional=True)
+
+        self.__setup_movement()
+
+        self.event_queue = EventQueue(self.input_fd, self.encoding)
+        self.cursor_visible = 1
+
+    def more_in_buffer(self) -> bool:
+        return bool(
+            self.input_buffer
+            and self.input_buffer_pos < len(self.input_buffer)
+        )
+
+    def __read(self, n: int) -> bytes:
+        if not self.more_in_buffer():
+            self.input_buffer = os.read(self.input_fd, 10000)
+
+        ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n]
+        self.input_buffer_pos += len(ret)
+        if self.input_buffer_pos >= len(self.input_buffer):
+            self.input_buffer = b""
+            self.input_buffer_pos = 0
+        return ret
+
+
+    def change_encoding(self, encoding: str) -> None:
+        """
+        Change the encoding used for I/O operations.
+
+        Parameters:
+        - encoding (str): New encoding to use.
+        """
+        self.encoding = encoding
+
+    def refresh(self, screen, c_xy):
+        """
+        Refresh the console screen.
+
+        Parameters:
+        - screen (list): List of strings representing the screen contents.
+        - c_xy (tuple): Cursor position (x, y) on the screen.
+        """
+        cx, cy = c_xy
+        if not self.__gone_tall:
+            while len(self.screen) < min(len(screen), self.height):
+                self.__hide_cursor()
+                self.__move(0, len(self.screen) - 1)
+                self.__write("\n")
+                self.posxy = 0, len(self.screen)
+                self.screen.append("")
+        else:
+            while len(self.screen) < len(screen):
+                self.screen.append("")
+
+        if len(screen) > self.height:
+            self.__gone_tall = 1
+            self.__move = self.__move_tall
+
+        px, py = self.posxy
+        old_offset = offset = self.__offset
+        height = self.height
+
+        # we make sure the cursor is on the screen, and that we're
+        # using all of the screen if we can
+        if cy < offset:
+            offset = cy
+        elif cy >= offset + height:
+            offset = cy - height + 1
+        elif offset > 0 and len(screen) < offset + height:
+            offset = max(len(screen) - height, 0)
+            screen.append("")
+
+        oldscr = self.screen[old_offset : old_offset + height]
+        newscr = screen[offset : offset + height]
+
+        # use hardware scrolling if we have it.
+        if old_offset > offset and self._ri:
+            self.__hide_cursor()
+            self.__write_code(self._cup, 0, 0)
+            self.posxy = 0, old_offset
+            for i in range(old_offset - offset):
+                self.__write_code(self._ri)
+                oldscr.pop(-1)
+                oldscr.insert(0, "")
+        elif old_offset < offset and self._ind:
+            self.__hide_cursor()
+            self.__write_code(self._cup, self.height - 1, 0)
+            self.posxy = 0, old_offset + self.height - 1
+            for i in range(offset - old_offset):
+                self.__write_code(self._ind)
+                oldscr.pop(0)
+                oldscr.append("")
+
+        self.__offset = offset
+
+        for (
+            y,
+            oldline,
+            newline,
+        ) in zip(range(offset, offset + height), oldscr, newscr):
+            if oldline != newline:
+                self.__write_changed_line(y, oldline, newline, px)
+
+        y = len(newscr)
+        while y < len(oldscr):
+            self.__hide_cursor()
+            self.__move(0, y)
+            self.posxy = 0, y
+            self.__write_code(self._el)
+            y += 1
+
+        self.__show_cursor()
+
+        self.screen = screen.copy()
+        self.move_cursor(cx, cy)
+        self.flushoutput()
+
+    def move_cursor(self, x, y):
+        """
+        Move the cursor to the specified position on the screen.
+
+        Parameters:
+        - x (int): X coordinate.
+        - y (int): Y coordinate.
+        """
+        if y < self.__offset or y >= self.__offset + self.height:
+            self.event_queue.insert(Event("scroll", None))
+        else:
+            self.__move(x, y)
+            self.posxy = x, y
+            self.flushoutput()
+
+    def prepare(self):
+        """
+        Prepare the console for input/output operations.
+        """
+        self.__svtermstate = tcgetattr(self.input_fd)
+        raw = self.__svtermstate.copy()
+        raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON)
+        raw.oflag &= ~(termios.OPOST)
+        raw.cflag &= ~(termios.CSIZE | termios.PARENB)
+        raw.cflag |= termios.CS8
+        raw.iflag |= termios.BRKINT
+        raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN)
+        raw.lflag |= termios.ISIG
+        raw.cc[termios.VMIN] = 1
+        raw.cc[termios.VTIME] = 0
+        tcsetattr(self.input_fd, termios.TCSADRAIN, raw)
+
+        # In macOS terminal we need to deactivate line wrap via ANSI escape code
+        if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal":
+            os.write(self.output_fd, b"\033[?7l")
+
+        self.screen = []
+        self.height, self.width = self.getheightwidth()
+
+        self.__buffer = []
+
+        self.posxy = 0, 0
+        self.__gone_tall = 0
+        self.__move = self.__move_short
+        self.__offset = 0
+
+        self.__maybe_write_code(self._smkx)
+
+        try:
+            self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch)
+        except ValueError:
+            pass
+
+        self.__enable_bracketed_paste()
+
+    def restore(self):
+        """
+        Restore the console to the default state
+        """
+        self.__disable_bracketed_paste()
+        self.__maybe_write_code(self._rmkx)
+        self.flushoutput()
+        tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate)
+
+        if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal":
+            os.write(self.output_fd, b"\033[?7h")
+
+        if hasattr(self, "old_sigwinch"):
+            signal.signal(signal.SIGWINCH, self.old_sigwinch)
+            del self.old_sigwinch
+
+    def push_char(self, char: int | bytes) -> None:
+        """
+        Push a character to the console event queue.
+        """
+        trace("push char {char!r}", char=char)
+        self.event_queue.push(char)
+
+    def get_event(self, block: bool = True) -> Event | None:
+        """
+        Get an event from the console event queue.
+
+        Parameters:
+        - block (bool): Whether to block until an event is available.
+
+        Returns:
+        - Event: Event object from the event queue.
+        """
+        if not block and not self.wait(timeout=0):
+            return None
+
+        while self.event_queue.empty():
+            while True:
+                try:
+                    self.push_char(self.__read(1))
+                except OSError as err:
+                    if err.errno == errno.EINTR:
+                        if not self.event_queue.empty():
+                            return self.event_queue.get()
+                        else:
+                            continue
+                    else:
+                        raise
+                else:
+                    break
+        return self.event_queue.get()
+
+    def wait(self, timeout: float | None = None) -> bool:
+        """
+        Wait for events on the console.
+        """
+        return (
+            not self.event_queue.empty()
+            or self.more_in_buffer()
+            or bool(self.pollob.poll(timeout))
+        )
+
+    def set_cursor_vis(self, visible):
+        """
+        Set the visibility of the cursor.
+
+        Parameters:
+        - visible (bool): Visibility flag.
+        """
+        if visible:
+            self.__show_cursor()
+        else:
+            self.__hide_cursor()
+
+    if TIOCGWINSZ:
+
+        def getheightwidth(self):
+            """
+            Get the height and width of the console.
+
+            Returns:
+            - tuple: Height and width of the console.
+            """
+            try:
+                return int(os.environ["LINES"]), int(os.environ["COLUMNS"])
+            except (KeyError, TypeError, ValueError):
+                try:
+                    size = ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8)
+                except OSError:
+                    return 25, 80
+                height, width = struct.unpack("hhhh", size)[0:2]
+                if not height:
+                    return 25, 80
+                return height, width
+
+    else:
+
+        def getheightwidth(self):
+            """
+            Get the height and width of the console.
+
+            Returns:
+            - tuple: Height and width of the console.
+            """
+            try:
+                return int(os.environ["LINES"]), int(os.environ["COLUMNS"])
+            except (KeyError, TypeError, ValueError):
+                return 25, 80
+
+    def forgetinput(self):
+        """
+        Discard any pending input on the console.
+        """
+        termios.tcflush(self.input_fd, termios.TCIFLUSH)
+
+    def flushoutput(self):
+        """
+        Flush the output buffer.
+        """
+        for text, iscode in self.__buffer:
+            if iscode:
+                self.__tputs(text)
+            else:
+                os.write(self.output_fd, text.encode(self.encoding, "replace"))
+        del self.__buffer[:]
+
+    def finish(self):
+        """
+        Finish console operations and flush the output buffer.
+        """
+        y = len(self.screen) - 1
+        while y >= 0 and not self.screen[y]:
+            y -= 1
+        self.__move(0, min(y, self.height + self.__offset - 1))
+        self.__write("\n\r")
+        self.flushoutput()
+
+    def beep(self):
+        """
+        Emit a beep sound.
+        """
+        self.__maybe_write_code(self._bel)
+        self.flushoutput()
+
+    if FIONREAD:
+
+        def getpending(self):
+            """
+            Get pending events from the console event queue.
+
+            Returns:
+            - Event: Pending event from the event queue.
+            """
+            e = Event("key", "", b"")
+
+            while not self.event_queue.empty():
+                e2 = self.event_queue.get()
+                e.data += e2.data
+                e.raw += e.raw
+
+            amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0]
+            raw = self.__read(amount)
+            data = str(raw, self.encoding, "replace")
+            e.data += data
+            e.raw += raw
+            return e
+
+    else:
+
+        def getpending(self):
+            """
+            Get pending events from the console event queue.
+
+            Returns:
+            - Event: Pending event from the event queue.
+            """
+            e = Event("key", "", b"")
+
+            while not self.event_queue.empty():
+                e2 = self.event_queue.get()
+                e.data += e2.data
+                e.raw += e.raw
+
+            amount = 10000
+            raw = self.__read(amount)
+            data = str(raw, self.encoding, "replace")
+            e.data += data
+            e.raw += raw
+            return e
+
+    def clear(self):
+        """
+        Clear the console screen.
+        """
+        self.__write_code(self._clear)
+        self.__gone_tall = 1
+        self.__move = self.__move_tall
+        self.posxy = 0, 0
+        self.screen = []
+
+    @property
+    def input_hook(self):
+        try:
+            import posix
+        except ImportError:
+            return None
+        if posix._is_inputhook_installed():
+            return posix._inputhook
+
+    def __enable_bracketed_paste(self) -> None:
+        os.write(self.output_fd, b"\x1b[?2004h")
+
+    def __disable_bracketed_paste(self) -> None:
+        os.write(self.output_fd, b"\x1b[?2004l")
+
+    def __setup_movement(self):
+        """
+        Set up the movement functions based on the terminal capabilities.
+        """
+        if 0 and self._hpa:  # hpa don't work in windows telnet :-(
+            self.__move_x = self.__move_x_hpa
+        elif self._cub and self._cuf:
+            self.__move_x = self.__move_x_cub_cuf
+        elif self._cub1 and self._cuf1:
+            self.__move_x = self.__move_x_cub1_cuf1
+        else:
+            raise RuntimeError("insufficient terminal (horizontal)")
+
+        if self._cuu and self._cud:
+            self.__move_y = self.__move_y_cuu_cud
+        elif self._cuu1 and self._cud1:
+            self.__move_y = self.__move_y_cuu1_cud1
+        else:
+            raise RuntimeError("insufficient terminal (vertical)")
+
+        if self._dch1:
+            self.dch1 = self._dch1
+        elif self._dch:
+            self.dch1 = curses.tparm(self._dch, 1)
+        else:
+            self.dch1 = None
+
+        if self._ich1:
+            self.ich1 = self._ich1
+        elif self._ich:
+            self.ich1 = curses.tparm(self._ich, 1)
+        else:
+            self.ich1 = None
+
+        self.__move = self.__move_short
+
+    def __write_changed_line(self, y, oldline, newline, px_coord):
+        # this is frustrating; there's no reason to test (say)
+        # self.dch1 inside the loop -- but alternative ways of
+        # structuring this function are equally painful (I'm trying to
+        # avoid writing code generators these days...)
+        minlen = min(wlen(oldline), wlen(newline))
+        x_pos = 0
+        x_coord = 0
+
+        px_pos = 0
+        j = 0
+        for c in oldline:
+            if j >= px_coord:
+                break
+            j += wlen(c)
+            px_pos += 1
+
+        # reuse the oldline as much as possible, but stop as soon as we
+        # encounter an ESCAPE, because it might be the start of an escape
+        # sequene
+        while (
+            x_coord < minlen
+            and oldline[x_pos] == newline[x_pos]
+            and newline[x_pos] != "\x1b"
+        ):
+            x_coord += wlen(newline[x_pos])
+            x_pos += 1
+
+        # if we need to insert a single character right after the first detected change
+        if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1:
+            if (
+                y == self.posxy[1]
+                and x_coord > self.posxy[0]
+                and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1]
+            ):
+                x_pos = px_pos
+                x_coord = px_coord
+            character_width = wlen(newline[x_pos])
+            self.__move(x_coord, y)
+            self.__write_code(self.ich1)
+            self.__write(newline[x_pos])
+            self.posxy = x_coord + character_width, y
+
+        # if it's a single character change in the middle of the line
+        elif (
+            x_coord < minlen
+            and oldline[x_pos + 1 :] == newline[x_pos + 1 :]
+            and wlen(oldline[x_pos]) == wlen(newline[x_pos])
+        ):
+            character_width = wlen(newline[x_pos])
+            self.__move(x_coord, y)
+            self.__write(newline[x_pos])
+            self.posxy = x_coord + character_width, y
+
+        # if this is the last character to fit in the line and we edit in the middle of the line
+        elif (
+            self.dch1
+            and self.ich1
+            and wlen(newline) == self.width
+            and x_coord < wlen(newline) - 2
+            and newline[x_pos + 1 : -1] == oldline[x_pos:-2]
+        ):
+            self.__hide_cursor()
+            self.__move(self.width - 2, y)
+            self.posxy = self.width - 2, y
+            self.__write_code(self.dch1)
+
+            character_width = wlen(newline[x_pos])
+            self.__move(x_coord, y)
+            self.__write_code(self.ich1)
+            self.__write(newline[x_pos])
+            self.posxy = character_width + 1, y
+
+        else:
+            self.__hide_cursor()
+            self.__move(x_coord, y)
+            if wlen(oldline) > wlen(newline):
+                self.__write_code(self._el)
+            self.__write(newline[x_pos:])
+            self.posxy = wlen(newline), y
+
+        if "\x1b" in newline:
+            # ANSI escape characters are present, so we can't assume
+            # anything about the position of the cursor.  Moving the cursor
+            # to the left margin should work to get to a known position.
+            self.move_cursor(0, y)
+
+    def __write(self, text):
+        self.__buffer.append((text, 0))
+
+    def __write_code(self, fmt, *args):
+        self.__buffer.append((curses.tparm(fmt, *args), 1))
+
+    def __maybe_write_code(self, fmt, *args):
+        if fmt:
+            self.__write_code(fmt, *args)
+
+    def __move_y_cuu1_cud1(self, y):
+        assert self._cud1 is not None
+        assert self._cuu1 is not None
+        dy = y - self.posxy[1]
+        if dy > 0:
+            self.__write_code(dy * self._cud1)
+        elif dy < 0:
+            self.__write_code((-dy) * self._cuu1)
+
+    def __move_y_cuu_cud(self, y):
+        dy = y - self.posxy[1]
+        if dy > 0:
+            self.__write_code(self._cud, dy)
+        elif dy < 0:
+            self.__write_code(self._cuu, -dy)
+
+    def __move_x_hpa(self, x: int) -> None:
+        if x != self.posxy[0]:
+            self.__write_code(self._hpa, x)
+
+    def __move_x_cub1_cuf1(self, x: int) -> None:
+        assert self._cuf1 is not None
+        assert self._cub1 is not None
+        dx = x - self.posxy[0]
+        if dx > 0:
+            self.__write_code(self._cuf1 * dx)
+        elif dx < 0:
+            self.__write_code(self._cub1 * (-dx))
+
+    def __move_x_cub_cuf(self, x: int) -> None:
+        dx = x - self.posxy[0]
+        if dx > 0:
+            self.__write_code(self._cuf, dx)
+        elif dx < 0:
+            self.__write_code(self._cub, -dx)
+
+    def __move_short(self, x, y):
+        self.__move_x(x)
+        self.__move_y(y)
+
+    def __move_tall(self, x, y):
+        assert 0 <= y - self.__offset < self.height, y - self.__offset
+        self.__write_code(self._cup, y - self.__offset, x)
+
+    def __sigwinch(self, signum, frame):
+        self.height, self.width = self.getheightwidth()
+        self.event_queue.insert(Event("resize", None))
+
+    def __hide_cursor(self):
+        if self.cursor_visible:
+            self.__maybe_write_code(self._civis)
+            self.cursor_visible = 0
+
+    def __show_cursor(self):
+        if not self.cursor_visible:
+            self.__maybe_write_code(self._cnorm)
+            self.cursor_visible = 1
+
+    def repaint(self):
+        if not self.__gone_tall:
+            self.posxy = 0, self.posxy[1]
+            self.__write("\r")
+            ns = len(self.screen) * ["\000" * self.width]
+            self.screen = ns
+        else:
+            self.posxy = 0, self.__offset
+            self.__move(0, self.__offset)
+            ns = self.height * ["\000" * self.width]
+            self.screen = ns
+
+    def __tputs(self, fmt, prog=delayprog):
+        """A Python implementation of the curses tputs function; the
+        curses one can't really be wrapped in a sane manner.
+
+        I have the strong suspicion that this is complexity that
+        will never do anyone any good."""
+        # using .get() means that things will blow up
+        # only if the bps is actually needed (which I'm
+        # betting is pretty unlkely)
+        bps = ratedict.get(self.__svtermstate.ospeed)
+        while 1:
+            m = prog.search(fmt)
+            if not m:
+                os.write(self.output_fd, fmt)
+                break
+            x, y = m.span()
+            os.write(self.output_fd, fmt[:x])
+            fmt = fmt[y:]
+            delay = int(m.group(1))
+            if b"*" in m.group(2):
+                delay *= self.height
+            if self._pad and bps is not None:
+                nchars = (bps * delay) / 1000
+                os.write(self.output_fd, self._pad * nchars)
+            else:
+                time.sleep(float(delay) / 1000.0)

Lib/_pyrepl/unix_eventqueue.py

@@ -0,0 +1,152 @@
+#   Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from collections import deque
+
+from . import keymap
+from .console import Event
+from . import curses
+from .trace import trace
+from termios import tcgetattr, VERASE
+import os
+
+
+# Mapping of human-readable key names to their terminal-specific codes
+TERMINAL_KEYNAMES = {
+    "delete": "kdch1",
+    "down": "kcud1",
+    "end": "kend",
+    "enter": "kent",
+    "home": "khome",
+    "insert": "kich1",
+    "left": "kcub1",
+    "page down": "knp",
+    "page up": "kpp",
+    "right": "kcuf1",
+    "up": "kcuu1",
+}
+
+
+# Function keys F1-F20 mapping
+TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21))
+
+# Known CTRL-arrow keycodes
+CTRL_ARROW_KEYCODES= {
+    # for xterm, gnome-terminal, xfce terminal, etc.
+    b'\033[1;5D': 'ctrl left',
+    b'\033[1;5C': 'ctrl right',
+    # for rxvt
+    b'\033Od': 'ctrl left',
+    b'\033Oc': 'ctrl right',
+}
+
+def get_terminal_keycodes() -> dict[bytes, str]:
+    """
+    Generates a dictionary mapping terminal keycodes to human-readable names.
+    """
+    keycodes = {}
+    for key, terminal_code in TERMINAL_KEYNAMES.items():
+        keycode = curses.tigetstr(terminal_code)
+        trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals())
+        if keycode:
+            keycodes[keycode] = key
+    keycodes.update(CTRL_ARROW_KEYCODES)
+    return keycodes
+
+class EventQueue:
+    def __init__(self, fd: int, encoding: str) -> None:
+        self.keycodes = get_terminal_keycodes()
+        if os.isatty(fd):
+            backspace = tcgetattr(fd)[6][VERASE]
+            self.keycodes[backspace] = "backspace"
+        self.compiled_keymap = keymap.compile_keymap(self.keycodes)
+        self.keymap = self.compiled_keymap
+        trace("keymap {k!r}", k=self.keymap)
+        self.encoding = encoding
+        self.events: deque[Event] = deque()
+        self.buf = bytearray()
+
+    def get(self) -> Event | None:
+        """
+        Retrieves the next event from the queue.
+        """
+        if self.events:
+            return self.events.popleft()
+        else:
+            return None
+
+    def empty(self) -> bool:
+        """
+        Checks if the queue is empty.
+        """
+        return not self.events
+
+    def flush_buf(self) -> bytearray:
+        """
+        Flushes the buffer and returns its contents.
+        """
+        old = self.buf
+        self.buf = bytearray()
+        return old
+
+    def insert(self, event: Event) -> None:
+        """
+        Inserts an event into the queue.
+        """
+        trace('added event {event}', event=event)
+        self.events.append(event)
+
+    def push(self, char: int | bytes) -> None:
+        """
+        Processes a character by updating the buffer and handling special key mappings.
+        """
+        ord_char = char if isinstance(char, int) else ord(char)
+        char = bytes(bytearray((ord_char,)))
+        self.buf.append(ord_char)
+        if char in self.keymap:
+            if self.keymap is self.compiled_keymap:
+                #sanity check, buffer is empty when a special key comes
+                assert len(self.buf) == 1
+            k = self.keymap[char]
+            trace('found map {k!r}', k=k)
+            if isinstance(k, dict):
+                self.keymap = k
+            else:
+                self.insert(Event('key', k, self.flush_buf()))
+                self.keymap = self.compiled_keymap
+
+        elif self.buf and self.buf[0] == 27:  # escape
+            # escape sequence not recognized by our keymap: propagate it
+            # outside so that i can be recognized as an M-... key (see also
+            # the docstring in keymap.py
+            trace('unrecognized escape sequence, propagating...')
+            self.keymap = self.compiled_keymap
+            self.insert(Event('key', '\033', bytearray(b'\033')))
+            for _c in self.flush_buf()[1:]:
+                self.push(_c)
+
+        else:
+            try:
+                decoded = bytes(self.buf).decode(self.encoding)
+            except UnicodeError:
+                return
+            else:
+                self.insert(Event('key', decoded, self.flush_buf()))
+            self.keymap = self.compiled_keymap

Lib/_pyrepl/utils.py

@@ -0,0 +1,25 @@
+import re
+import unicodedata
+import functools
+
+ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]")
+
+
+@functools.cache
+def str_width(c: str) -> int:
+    if ord(c) < 128:
+        return 1
+    w = unicodedata.east_asian_width(c)
+    if w in ('N', 'Na', 'H', 'A'):
+        return 1
+    return 2
+
+
+def wlen(s: str) -> int:
+    if len(s) == 1 and s != '\x1a':
+        return str_width(s)
+    length = sum(str_width(i) for i in s)
+    # remove lengths of any escape sequences
+    sequence = ANSI_ESCAPE_SEQUENCE.findall(s)
+    ctrl_z_cnt = s.count('\x1a')
+    return length - sum(len(i) for i in sequence) + ctrl_z_cnt

Lib/_pyrepl/windows_console.py

@@ -0,0 +1,618 @@
+#   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import io
+import os
+import sys
+import time
+import msvcrt
+
+from collections import deque
+import ctypes
+from ctypes.wintypes import (
+    _COORD,
+    WORD,
+    SMALL_RECT,
+    BOOL,
+    HANDLE,
+    CHAR,
+    DWORD,
+    WCHAR,
+    SHORT,
+)
+from ctypes import Structure, POINTER, Union
+from .console import Event, Console
+from .trace import trace
+from .utils import wlen
+
+try:
+    from ctypes import GetLastError, WinDLL, windll, WinError  # type: ignore[attr-defined]
+except:
+    # Keep MyPy happy off Windows
+    from ctypes import CDLL as WinDLL, cdll as windll
+
+    def GetLastError() -> int:
+        return 42
+
+    class WinError(OSError):  # type: ignore[no-redef]
+        def __init__(self, err: int | None, descr: str | None = None) -> None:
+            self.err = err
+            self.descr = descr
+
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+    from typing import IO
+
+# Virtual-Key Codes: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
+VK_MAP: dict[int, str] = {
+    0x23: "end",  # VK_END
+    0x24: "home",  # VK_HOME
+    0x25: "left",  # VK_LEFT
+    0x26: "up",  # VK_UP
+    0x27: "right",  # VK_RIGHT
+    0x28: "down",  # VK_DOWN
+    0x2E: "delete",  # VK_DELETE
+    0x70: "f1",  # VK_F1
+    0x71: "f2",  # VK_F2
+    0x72: "f3",  # VK_F3
+    0x73: "f4",  # VK_F4
+    0x74: "f5",  # VK_F5
+    0x75: "f6",  # VK_F6
+    0x76: "f7",  # VK_F7
+    0x77: "f8",  # VK_F8
+    0x78: "f9",  # VK_F9
+    0x79: "f10",  # VK_F10
+    0x7A: "f11",  # VK_F11
+    0x7B: "f12",  # VK_F12
+    0x7C: "f13",  # VK_F13
+    0x7D: "f14",  # VK_F14
+    0x7E: "f15",  # VK_F15
+    0x7F: "f16",  # VK_F16
+    0x80: "f17",  # VK_F17
+    0x81: "f18",  # VK_F18
+    0x82: "f19",  # VK_F19
+    0x83: "f20",  # VK_F20
+}
+
+# Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
+ERASE_IN_LINE = "\x1b[K"
+MOVE_LEFT = "\x1b[{}D"
+MOVE_RIGHT = "\x1b[{}C"
+MOVE_UP = "\x1b[{}A"
+MOVE_DOWN = "\x1b[{}B"
+CLEAR = "\x1b[H\x1b[J"
+
+
+class _error(Exception):
+    pass
+
+
+class WindowsConsole(Console):
+    def __init__(
+        self,
+        f_in: IO[bytes] | int = 0,
+        f_out: IO[bytes] | int = 1,
+        term: str = "",
+        encoding: str = "",
+    ):
+        super().__init__(f_in, f_out, term, encoding)
+
+        SetConsoleMode(
+            OutHandle,
+            ENABLE_WRAP_AT_EOL_OUTPUT
+            | ENABLE_PROCESSED_OUTPUT
+            | ENABLE_VIRTUAL_TERMINAL_PROCESSING,
+        )
+        self.screen: list[str] = []
+        self.width = 80
+        self.height = 25
+        self.__offset = 0
+        self.event_queue: deque[Event] = deque()
+        try:
+            self.out = io._WindowsConsoleIO(self.output_fd, "w")  # type: ignore[attr-defined]
+        except ValueError:
+            # Console I/O is redirected, fallback...
+            self.out = None
+
+    def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None:
+        """
+        Refresh the console screen.
+
+        Parameters:
+        - screen (list): List of strings representing the screen contents.
+        - c_xy (tuple): Cursor position (x, y) on the screen.
+        """
+        cx, cy = c_xy
+
+        while len(self.screen) < min(len(screen), self.height):
+            self._hide_cursor()
+            self._move_relative(0, len(self.screen) - 1)
+            self.__write("\n")
+            self.posxy = 0, len(self.screen)
+            self.screen.append("")
+
+        px, py = self.posxy
+        old_offset = offset = self.__offset
+        height = self.height
+
+        # we make sure the cursor is on the screen, and that we're
+        # using all of the screen if we can
+        if cy < offset:
+            offset = cy
+        elif cy >= offset + height:
+            offset = cy - height + 1
+            scroll_lines = offset - old_offset
+
+            # Scrolling the buffer as the current input is greater than the visible
+            # portion of the window.  We need to scroll the visible portion and the
+            # entire history
+            self._scroll(scroll_lines, self._getscrollbacksize())
+            self.posxy = self.posxy[0], self.posxy[1] + scroll_lines
+            self.__offset += scroll_lines
+
+            for i in range(scroll_lines):
+                self.screen.append("")
+        elif offset > 0 and len(screen) < offset + height:
+            offset = max(len(screen) - height, 0)
+            screen.append("")
+
+        oldscr = self.screen[old_offset : old_offset + height]
+        newscr = screen[offset : offset + height]
+
+        self.__offset = offset
+
+        self._hide_cursor()
+        for (
+            y,
+            oldline,
+            newline,
+        ) in zip(range(offset, offset + height), oldscr, newscr):
+            if oldline != newline:
+                self.__write_changed_line(y, oldline, newline, px)
+
+        y = len(newscr)
+        while y < len(oldscr):
+            self._move_relative(0, y)
+            self.posxy = 0, y
+            self._erase_to_end()
+            y += 1
+
+        self._show_cursor()
+
+        self.screen = screen
+        self.move_cursor(cx, cy)
+
+    @property
+    def input_hook(self):
+        try:
+            import nt
+        except ImportError:
+            return None
+        if nt._is_inputhook_installed():
+            return nt._inputhook
+
+    def __write_changed_line(
+        self, y: int, oldline: str, newline: str, px_coord: int
+    ) -> None:
+        # this is frustrating; there's no reason to test (say)
+        # self.dch1 inside the loop -- but alternative ways of
+        # structuring this function are equally painful (I'm trying to
+        # avoid writing code generators these days...)
+        minlen = min(wlen(oldline), wlen(newline))
+        x_pos = 0
+        x_coord = 0
+
+        px_pos = 0
+        j = 0
+        for c in oldline:
+            if j >= px_coord:
+                break
+            j += wlen(c)
+            px_pos += 1
+
+        # reuse the oldline as much as possible, but stop as soon as we
+        # encounter an ESCAPE, because it might be the start of an escape
+        # sequene
+        while (
+            x_coord < minlen
+            and oldline[x_pos] == newline[x_pos]
+            and newline[x_pos] != "\x1b"
+        ):
+            x_coord += wlen(newline[x_pos])
+            x_pos += 1
+
+        self._hide_cursor()
+        self._move_relative(x_coord, y)
+        if wlen(oldline) > wlen(newline):
+            self._erase_to_end()
+
+        self.__write(newline[x_pos:])
+        if wlen(newline) == self.width:
+            # If we wrapped we want to start at the next line
+            self._move_relative(0, y + 1)
+            self.posxy = 0, y + 1
+        else:
+            self.posxy = wlen(newline), y
+
+            if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline:
+                # ANSI escape characters are present, so we can't assume
+                # anything about the position of the cursor.  Moving the cursor
+                # to the left margin should work to get to a known position.
+                self.move_cursor(0, y)
+
+    def _scroll(
+        self, top: int, bottom: int, left: int | None = None, right: int | None = None
+    ) -> None:
+        scroll_rect = SMALL_RECT()
+        scroll_rect.Top = SHORT(top)
+        scroll_rect.Bottom = SHORT(bottom)
+        scroll_rect.Left = SHORT(0 if left is None else left)
+        scroll_rect.Right = SHORT(
+            self.getheightwidth()[1] - 1 if right is None else right
+        )
+        destination_origin = _COORD()
+        fill_info = CHAR_INFO()
+        fill_info.UnicodeChar = " "
+
+        if not ScrollConsoleScreenBuffer(
+            OutHandle, scroll_rect, None, destination_origin, fill_info
+        ):
+            raise WinError(GetLastError())
+
+    def _hide_cursor(self):
+        self.__write("\x1b[?25l")
+
+    def _show_cursor(self):
+        self.__write("\x1b[?25h")
+
+    def _enable_blinking(self):
+        self.__write("\x1b[?12h")
+
+    def _disable_blinking(self):
+        self.__write("\x1b[?12l")
+
+    def __write(self, text: str) -> None:
+        if "\x1a" in text:
+            text = ''.join(["^Z" if x == '\x1a' else x for x in text])
+
+        if self.out is not None:
+            self.out.write(text.encode(self.encoding, "replace"))
+            self.out.flush()
+        else:
+            os.write(self.output_fd, text.encode(self.encoding, "replace"))
+
+    @property
+    def screen_xy(self) -> tuple[int, int]:
+        info = CONSOLE_SCREEN_BUFFER_INFO()
+        if not GetConsoleScreenBufferInfo(OutHandle, info):
+            raise WinError(GetLastError())
+        return info.dwCursorPosition.X, info.dwCursorPosition.Y
+
+    def _erase_to_end(self) -> None:
+        self.__write(ERASE_IN_LINE)
+
+    def prepare(self) -> None:
+        trace("prepare")
+        self.screen = []
+        self.height, self.width = self.getheightwidth()
+
+        self.posxy = 0, 0
+        self.__gone_tall = 0
+        self.__offset = 0
+
+    def restore(self) -> None:
+        pass
+
+    def _move_relative(self, x: int, y: int) -> None:
+        """Moves relative to the current posxy"""
+        dx = x - self.posxy[0]
+        dy = y - self.posxy[1]
+        if dx < 0:
+            self.__write(MOVE_LEFT.format(-dx))
+        elif dx > 0:
+            self.__write(MOVE_RIGHT.format(dx))
+
+        if dy < 0:
+            self.__write(MOVE_UP.format(-dy))
+        elif dy > 0:
+            self.__write(MOVE_DOWN.format(dy))
+
+    def move_cursor(self, x: int, y: int) -> None:
+        if x < 0 or y < 0:
+            raise ValueError(f"Bad cursor position {x}, {y}")
+
+        if y < self.__offset or y >= self.__offset + self.height:
+            self.event_queue.insert(0, Event("scroll", ""))
+        else:
+            self._move_relative(x, y)
+            self.posxy = x, y
+
+    def set_cursor_vis(self, visible: bool) -> None:
+        if visible:
+            self._show_cursor()
+        else:
+            self._hide_cursor()
+
+    def getheightwidth(self) -> tuple[int, int]:
+        """Return (height, width) where height and width are the height
+        and width of the terminal window in characters."""
+        info = CONSOLE_SCREEN_BUFFER_INFO()
+        if not GetConsoleScreenBufferInfo(OutHandle, info):
+            raise WinError(GetLastError())
+        return (
+            info.srWindow.Bottom - info.srWindow.Top + 1,
+            info.srWindow.Right - info.srWindow.Left + 1,
+        )
+
+    def _getscrollbacksize(self) -> int:
+        info = CONSOLE_SCREEN_BUFFER_INFO()
+        if not GetConsoleScreenBufferInfo(OutHandle, info):
+            raise WinError(GetLastError())
+
+        return info.srWindow.Bottom  # type: ignore[no-any-return]
+
+    def _read_input(self, block: bool = True) -> INPUT_RECORD | None:
+        if not block:
+            events = DWORD()
+            if not GetNumberOfConsoleInputEvents(InHandle, events):
+                raise WinError(GetLastError())
+            if not events.value:
+                return None
+
+        rec = INPUT_RECORD()
+        read = DWORD()
+        if not ReadConsoleInput(InHandle, rec, 1, read):
+            raise WinError(GetLastError())
+
+        return rec
+
+    def get_event(self, block: bool = True) -> Event | None:
+        """Return an Event instance.  Returns None if |block| is false
+        and there is no event pending, otherwise waits for the
+        completion of an event."""
+        if self.event_queue:
+            return self.event_queue.pop()
+
+        while True:
+            rec = self._read_input(block)
+            if rec is None:
+                return None
+
+            if rec.EventType == WINDOW_BUFFER_SIZE_EVENT:
+                return Event("resize", "")
+
+            if rec.EventType != KEY_EVENT or not rec.Event.KeyEvent.bKeyDown:
+                # Only process keys and keydown events
+                if block:
+                    continue
+                return None
+
+            key = rec.Event.KeyEvent.uChar.UnicodeChar
+
+            if rec.Event.KeyEvent.uChar.UnicodeChar == "\r":
+                # Make enter make unix-like
+                return Event(evt="key", data="\n", raw=b"\n")
+            elif rec.Event.KeyEvent.wVirtualKeyCode == 8:
+                # Turn backspace directly into the command
+                return Event(
+                    evt="key",
+                    data="backspace",
+                    raw=rec.Event.KeyEvent.uChar.UnicodeChar,
+                )
+            elif rec.Event.KeyEvent.uChar.UnicodeChar == "\x00":
+                # Handle special keys like arrow keys and translate them into the appropriate command
+                code = VK_MAP.get(rec.Event.KeyEvent.wVirtualKeyCode)
+                if code:
+                    return Event(
+                        evt="key", data=code, raw=rec.Event.KeyEvent.uChar.UnicodeChar
+                    )
+                if block:
+                    continue
+
+                return None
+
+            return Event(evt="key", data=key, raw=rec.Event.KeyEvent.uChar.UnicodeChar)
+
+    def push_char(self, char: int | bytes) -> None:
+        """
+        Push a character to the console event queue.
+        """
+        raise NotImplementedError("push_char not supported on Windows")
+
+    def beep(self) -> None:
+        self.__write("\x07")
+
+    def clear(self) -> None:
+        """Wipe the screen"""
+        self.__write(CLEAR)
+        self.posxy = 0, 0
+        self.screen = [""]
+
+    def finish(self) -> None:
+        """Move the cursor to the end of the display and otherwise get
+        ready for end.  XXX could be merged with restore?  Hmm."""
+        y = len(self.screen) - 1
+        while y >= 0 and not self.screen[y]:
+            y -= 1
+        self._move_relative(0, min(y, self.height + self.__offset - 1))
+        self.__write("\r\n")
+
+    def flushoutput(self) -> None:
+        """Flush all output to the screen (assuming there's some
+        buffering going on somewhere).
+
+        All output on Windows is unbuffered so this is a nop"""
+        pass
+
+    def forgetinput(self) -> None:
+        """Forget all pending, but not yet processed input."""
+        if not FlushConsoleInputBuffer(InHandle):
+            raise WinError(GetLastError())
+
+    def getpending(self) -> Event:
+        """Return the characters that have been typed but not yet
+        processed."""
+        return Event("key", "", b"")
+
+    def wait(self, timeout: float | None) -> bool:
+        """Wait for an event."""
+        # Poor man's Windows select loop
+        start_time = time.time()
+        while True:
+            if msvcrt.kbhit(): # type: ignore[attr-defined]
+                return True
+            if timeout and time.time() - start_time > timeout / 1000:
+                return False
+            time.sleep(0.01)
+
+    def repaint(self) -> None:
+        raise NotImplementedError("No repaint support")
+
+
+# Windows interop
+class CONSOLE_SCREEN_BUFFER_INFO(Structure):
+    _fields_ = [
+        ("dwSize", _COORD),
+        ("dwCursorPosition", _COORD),
+        ("wAttributes", WORD),
+        ("srWindow", SMALL_RECT),
+        ("dwMaximumWindowSize", _COORD),
+    ]
+
+
+class CONSOLE_CURSOR_INFO(Structure):
+    _fields_ = [
+        ("dwSize", DWORD),
+        ("bVisible", BOOL),
+    ]
+
+
+class CHAR_INFO(Structure):
+    _fields_ = [
+        ("UnicodeChar", WCHAR),
+        ("Attributes", WORD),
+    ]
+
+
+class Char(Union):
+    _fields_ = [
+        ("UnicodeChar", WCHAR),
+        ("Char", CHAR),
+    ]
+
+
+class KeyEvent(ctypes.Structure):
+    _fields_ = [
+        ("bKeyDown", BOOL),
+        ("wRepeatCount", WORD),
+        ("wVirtualKeyCode", WORD),
+        ("wVirtualScanCode", WORD),
+        ("uChar", Char),
+        ("dwControlKeyState", DWORD),
+    ]
+
+
+class WindowsBufferSizeEvent(ctypes.Structure):
+    _fields_ = [("dwSize", _COORD)]
+
+
+class ConsoleEvent(ctypes.Union):
+    _fields_ = [
+        ("KeyEvent", KeyEvent),
+        ("WindowsBufferSizeEvent", WindowsBufferSizeEvent),
+    ]
+
+
+class INPUT_RECORD(Structure):
+    _fields_ = [("EventType", WORD), ("Event", ConsoleEvent)]
+
+
+KEY_EVENT = 0x01
+FOCUS_EVENT = 0x10
+MENU_EVENT = 0x08
+MOUSE_EVENT = 0x02
+WINDOW_BUFFER_SIZE_EVENT = 0x04
+
+ENABLE_PROCESSED_OUTPUT = 0x01
+ENABLE_WRAP_AT_EOL_OUTPUT = 0x02
+ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04
+
+STD_INPUT_HANDLE = -10
+STD_OUTPUT_HANDLE = -11
+
+if sys.platform == "win32":
+    _KERNEL32 = WinDLL("kernel32", use_last_error=True)
+
+    GetStdHandle = windll.kernel32.GetStdHandle
+    GetStdHandle.argtypes = [DWORD]
+    GetStdHandle.restype = HANDLE
+
+    GetConsoleScreenBufferInfo = _KERNEL32.GetConsoleScreenBufferInfo
+    GetConsoleScreenBufferInfo.argtypes = [
+        HANDLE,
+        ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO),
+    ]
+    GetConsoleScreenBufferInfo.restype = BOOL
+
+    ScrollConsoleScreenBuffer = _KERNEL32.ScrollConsoleScreenBufferW
+    ScrollConsoleScreenBuffer.argtypes = [
+        HANDLE,
+        POINTER(SMALL_RECT),
+        POINTER(SMALL_RECT),
+        _COORD,
+        POINTER(CHAR_INFO),
+    ]
+    ScrollConsoleScreenBuffer.restype = BOOL
+
+    SetConsoleMode = _KERNEL32.SetConsoleMode
+    SetConsoleMode.argtypes = [HANDLE, DWORD]
+    SetConsoleMode.restype = BOOL
+
+    ReadConsoleInput = _KERNEL32.ReadConsoleInputW
+    ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)]
+    ReadConsoleInput.restype = BOOL
+
+    GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents
+    GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)]
+    GetNumberOfConsoleInputEvents.restype = BOOL
+
+    FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer
+    FlushConsoleInputBuffer.argtypes = [HANDLE]
+    FlushConsoleInputBuffer.restype = BOOL
+
+    OutHandle = GetStdHandle(STD_OUTPUT_HANDLE)
+    InHandle = GetStdHandle(STD_INPUT_HANDLE)
+else:
+
+    def _win_only(*args, **kwargs):
+        raise NotImplementedError("Windows only")
+
+    GetStdHandle = _win_only
+    GetConsoleScreenBufferInfo = _win_only
+    ScrollConsoleScreenBuffer = _win_only
+    SetConsoleMode = _win_only
+    ReadConsoleInput = _win_only
+    GetNumberOfConsoleInputEvents = _win_only
+    FlushConsoleInputBuffer = _win_only
+    OutHandle = 0
+    InHandle = 0

Lib/aifc.py

@@ -1,984 +0,0 @@
-"""Stuff to parse AIFF-C and AIFF files.
-
-Unless explicitly stated otherwise, the description below is true
-both for AIFF-C files and AIFF files.
-
-An AIFF-C file has the following structure.
-
-  +-----------------+
-  | FORM            |
-  +-----------------+
-  | <size>          |
-  +----+------------+
-  |    | AIFC       |
-  |    +------------+
-  |    | <chunks>   |
-  |    |    .       |
-  |    |    .       |
-  |    |    .       |
-  +----+------------+
-
-An AIFF file has the string "AIFF" instead of "AIFC".
-
-A chunk consists of an identifier (4 bytes) followed by a size (4 bytes,
-big endian order), followed by the data.  The size field does not include
-the size of the 8 byte header.
-
-The following chunk types are recognized.
-
-  FVER
-      <version number of AIFF-C defining document> (AIFF-C only).
-  MARK
-      <# of markers> (2 bytes)
-      list of markers:
-          <marker ID> (2 bytes, must be > 0)
-          <position> (4 bytes)
-          <marker name> ("pstring")
-  COMM
-      <# of channels> (2 bytes)
-      <# of sound frames> (4 bytes)
-      <size of the samples> (2 bytes)
-      <sampling frequency> (10 bytes, IEEE 80-bit extended
-          floating point)
-      in AIFF-C files only:
-      <compression type> (4 bytes)
-      <human-readable version of compression type> ("pstring")
-  SSND
-      <offset> (4 bytes, not used by this program)
-      <blocksize> (4 bytes, not used by this program)
-      <sound data>
-
-A pstring consists of 1 byte length, a string of characters, and 0 or 1
-byte pad to make the total length even.
-
-Usage.
-
-Reading AIFF files:
-  f = aifc.open(file, 'r')
-where file is either the name of a file or an open file pointer.
-The open file pointer must have methods read(), seek(), and close().
-In some types of audio files, if the setpos() method is not used,
-the seek() method is not necessary.
-
-This returns an instance of a class with the following public methods:
-  getnchannels()  -- returns number of audio channels (1 for
-             mono, 2 for stereo)
-  getsampwidth()  -- returns sample width in bytes
-  getframerate()  -- returns sampling frequency
-  getnframes()    -- returns number of audio frames
-  getcomptype()   -- returns compression type ('NONE' for AIFF files)
-  getcompname()   -- returns human-readable version of
-             compression type ('not compressed' for AIFF files)
-  getparams() -- returns a namedtuple consisting of all of the
-             above in the above order
-  getmarkers()    -- get the list of marks in the audio file or None
-             if there are no marks
-  getmark(id) -- get mark with the specified id (raises an error
-             if the mark does not exist)
-  readframes(n)   -- returns at most n frames of audio
-  rewind()    -- rewind to the beginning of the audio stream
-  setpos(pos) -- seek to the specified position
-  tell()      -- return the current position
-  close()     -- close the instance (make it unusable)
-The position returned by tell(), the position given to setpos() and
-the position of marks are all compatible and have nothing to do with
-the actual position in the file.
-The close() method is called automatically when the class instance
-is destroyed.
-
-Writing AIFF files:
-  f = aifc.open(file, 'w')
-where file is either the name of a file or an open file pointer.
-The open file pointer must have methods write(), tell(), seek(), and
-close().
-
-This returns an instance of a class with the following public methods:
-  aiff()      -- create an AIFF file (AIFF-C default)
-  aifc()      -- create an AIFF-C file
-  setnchannels(n) -- set the number of channels
-  setsampwidth(n) -- set the sample width
-  setframerate(n) -- set the frame rate
-  setnframes(n)   -- set the number of frames
-  setcomptype(type, name)
-          -- set the compression type and the
-             human-readable compression type
-  setparams(tuple)
-          -- set all parameters at once
-  setmark(id, pos, name)
-          -- add specified mark to the list of marks
-  tell()      -- return current position in output file (useful
-             in combination with setmark())
-  writeframesraw(data)
-          -- write audio frames without pathing up the
-             file header
-  writeframes(data)
-          -- write audio frames and patch up the file header
-  close()     -- patch up the file header and close the
-             output file
-You should set the parameters before the first writeframesraw or
-writeframes.  The total number of frames does not need to be set,
-but when it is set to the correct value, the header does not have to
-be patched up.
-It is best to first set all parameters, perhaps possibly the
-compression type, and then write audio frames using writeframesraw.
-When all frames have been written, either call writeframes(b'') or
-close() to patch up the sizes in the header.
-Marks can be added anytime.  If there are any marks, you must call
-close() after all frames have been written.
-The close() method is called automatically when the class instance
-is destroyed.
-
-When a file is opened with the extension '.aiff', an AIFF file is
-written, otherwise an AIFF-C file is written.  This default can be
-changed by calling aiff() or aifc() before the first writeframes or
-writeframesraw.
-"""
-
-import struct
-import builtins
-import warnings
-
-__all__ = ["Error", "open"]
-
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-
-class Error(Exception):
-    pass
-
-_AIFC_version = 0xA2805140     # Version 1 of AIFF-C
-
-def _read_long(file):
-    try:
-        return struct.unpack('>l', file.read(4))[0]
-    except struct.error:
-        raise EOFError from None
-
-def _read_ulong(file):
-    try:
-        return struct.unpack('>L', file.read(4))[0]
-    except struct.error:
-        raise EOFError from None
-
-def _read_short(file):
-    try:
-        return struct.unpack('>h', file.read(2))[0]
-    except struct.error:
-        raise EOFError from None
-
-def _read_ushort(file):
-    try:
-        return struct.unpack('>H', file.read(2))[0]
-    except struct.error:
-        raise EOFError from None
-
-def _read_string(file):
-    length = ord(file.read(1))
-    if length == 0:
-        data = b''
-    else:
-        data = file.read(length)
-    if length & 1 == 0:
-        dummy = file.read(1)
-    return data
-
-_HUGE_VAL = 1.79769313486231e+308 # See <limits.h>
-
-def _read_float(f): # 10 bytes
-    expon = _read_short(f) # 2 bytes
-    sign = 1
-    if expon < 0:
-        sign = -1
-        expon = expon + 0x8000
-    himant = _read_ulong(f) # 4 bytes
-    lomant = _read_ulong(f) # 4 bytes
-    if expon == himant == lomant == 0:
-        f = 0.0
-    elif expon == 0x7FFF:
-        f = _HUGE_VAL
-    else:
-        expon = expon - 16383
-        f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63)
-    return sign * f
-
-def _write_short(f, x):
-    f.write(struct.pack('>h', x))
-
-def _write_ushort(f, x):
-    f.write(struct.pack('>H', x))
-
-def _write_long(f, x):
-    f.write(struct.pack('>l', x))
-
-def _write_ulong(f, x):
-    f.write(struct.pack('>L', x))
-
-def _write_string(f, s):
-    if len(s) > 255:
-        raise ValueError("string exceeds maximum pstring length")
-    f.write(struct.pack('B', len(s)))
-    f.write(s)
-    if len(s) & 1 == 0:
-        f.write(b'\x00')
-
-def _write_float(f, x):
-    import math
-    if x < 0:
-        sign = 0x8000
-        x = x * -1
-    else:
-        sign = 0
-    if x == 0:
-        expon = 0
-        himant = 0
-        lomant = 0
-    else:
-        fmant, expon = math.frexp(x)
-        if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN
-            expon = sign|0x7FFF
-            himant = 0
-            lomant = 0
-        else:                   # Finite
-            expon = expon + 16382
-            if expon < 0:           # denormalized
-                fmant = math.ldexp(fmant, expon)
-                expon = 0
-            expon = expon | sign
-            fmant = math.ldexp(fmant, 32)
-            fsmant = math.floor(fmant)
-            himant = int(fsmant)
-            fmant = math.ldexp(fmant - fsmant, 32)
-            fsmant = math.floor(fmant)
-            lomant = int(fsmant)
-    _write_ushort(f, expon)
-    _write_ulong(f, himant)
-    _write_ulong(f, lomant)
-
-with warnings.catch_warnings():
-    warnings.simplefilter("ignore", DeprecationWarning)
-    from chunk import Chunk
-from collections import namedtuple
-
-_aifc_params = namedtuple('_aifc_params',
-                          'nchannels sampwidth framerate nframes comptype compname')
-
-_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)'
-_aifc_params.sampwidth.__doc__ = 'Sample width in bytes'
-_aifc_params.framerate.__doc__ = 'Sampling frequency'
-_aifc_params.nframes.__doc__ = 'Number of audio frames'
-_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)'
-_aifc_params.compname.__doc__ = ("""\
-A human-readable version of the compression type
-('not compressed' for AIFF files)""")
-
-
-class Aifc_read:
-    # Variables used in this class:
-    #
-    # These variables are available to the user though appropriate
-    # methods of this class:
-    # _file -- the open file with methods read(), close(), and seek()
-    #       set through the __init__() method
-    # _nchannels -- the number of audio channels
-    #       available through the getnchannels() method
-    # _nframes -- the number of audio frames
-    #       available through the getnframes() method
-    # _sampwidth -- the number of bytes per audio sample
-    #       available through the getsampwidth() method
-    # _framerate -- the sampling frequency
-    #       available through the getframerate() method
-    # _comptype -- the AIFF-C compression type ('NONE' if AIFF)
-    #       available through the getcomptype() method
-    # _compname -- the human-readable AIFF-C compression type
-    #       available through the getcomptype() method
-    # _markers -- the marks in the audio file
-    #       available through the getmarkers() and getmark()
-    #       methods
-    # _soundpos -- the position in the audio stream
-    #       available through the tell() method, set through the
-    #       setpos() method
-    #
-    # These variables are used internally only:
-    # _version -- the AIFF-C version number
-    # _decomp -- the decompressor from builtin module cl
-    # _comm_chunk_read -- 1 iff the COMM chunk has been read
-    # _aifc -- 1 iff reading an AIFF-C file
-    # _ssnd_seek_needed -- 1 iff positioned correctly in audio
-    #       file for readframes()
-    # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
-    # _framesize -- size of one frame in the file
-
-    _file = None  # Set here since __del__ checks it
-
-    def initfp(self, file):
-        self._version = 0
-        self._convert = None
-        self._markers = []
-        self._soundpos = 0
-        self._file = file
-        chunk = Chunk(file)
-        if chunk.getname() != b'FORM':
-            raise Error('file does not start with FORM id')
-        formdata = chunk.read(4)
-        if formdata == b'AIFF':
-            self._aifc = 0
-        elif formdata == b'AIFC':
-            self._aifc = 1
-        else:
-            raise Error('not an AIFF or AIFF-C file')
-        self._comm_chunk_read = 0
-        self._ssnd_chunk = None
-        while 1:
-            self._ssnd_seek_needed = 1
-            try:
-                chunk = Chunk(self._file)
-            except EOFError:
-                break
-            chunkname = chunk.getname()
-            if chunkname == b'COMM':
-                self._read_comm_chunk(chunk)
-                self._comm_chunk_read = 1
-            elif chunkname == b'SSND':
-                self._ssnd_chunk = chunk
-                dummy = chunk.read(8)
-                self._ssnd_seek_needed = 0
-            elif chunkname == b'FVER':
-                self._version = _read_ulong(chunk)
-            elif chunkname == b'MARK':
-                self._readmark(chunk)
-            chunk.skip()
-        if not self._comm_chunk_read or not self._ssnd_chunk:
-            raise Error('COMM chunk and/or SSND chunk missing')
-
-    def __init__(self, f):
-        if isinstance(f, str):
-            file_object = builtins.open(f, 'rb')
-            try:
-                self.initfp(file_object)
-            except:
-                file_object.close()
-                raise
-        else:
-            # assume it is an open file object already
-            self.initfp(f)
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        self.close()
-
-    #
-    # User visible methods.
-    #
-    def getfp(self):
-        return self._file
-
-    def rewind(self):
-        self._ssnd_seek_needed = 1
-        self._soundpos = 0
-
-    def close(self):
-        file = self._file
-        if file is not None:
-            self._file = None
-            file.close()
-
-    def tell(self):
-        return self._soundpos
-
-    def getnchannels(self):
-        return self._nchannels
-
-    def getnframes(self):
-        return self._nframes
-
-    def getsampwidth(self):
-        return self._sampwidth
-
-    def getframerate(self):
-        return self._framerate
-
-    def getcomptype(self):
-        return self._comptype
-
-    def getcompname(self):
-        return self._compname
-
-##  def getversion(self):
-##      return self._version
-
-    def getparams(self):
-        return _aifc_params(self.getnchannels(), self.getsampwidth(),
-                            self.getframerate(), self.getnframes(),
-                            self.getcomptype(), self.getcompname())
-
-    def getmarkers(self):
-        if len(self._markers) == 0:
-            return None
-        return self._markers
-
-    def getmark(self, id):
-        for marker in self._markers:
-            if id == marker[0]:
-                return marker
-        raise Error('marker {0!r} does not exist'.format(id))
-
-    def setpos(self, pos):
-        if pos < 0 or pos > self._nframes:
-            raise Error('position not in range')
-        self._soundpos = pos
-        self._ssnd_seek_needed = 1
-
-    def readframes(self, nframes):
-        if self._ssnd_seek_needed:
-            self._ssnd_chunk.seek(0)
-            dummy = self._ssnd_chunk.read(8)
-            pos = self._soundpos * self._framesize
-            if pos:
-                self._ssnd_chunk.seek(pos + 8)
-            self._ssnd_seek_needed = 0
-        if nframes == 0:
-            return b''
-        data = self._ssnd_chunk.read(nframes * self._framesize)
-        if self._convert and data:
-            data = self._convert(data)
-        self._soundpos = self._soundpos + len(data) // (self._nchannels
-                                                        * self._sampwidth)
-        return data
-
-    #
-    # Internal methods.
-    #
-
-    def _alaw2lin(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.alaw2lin(data, 2)
-
-    def _ulaw2lin(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.ulaw2lin(data, 2)
-
-    def _adpcm2lin(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        if not hasattr(self, '_adpcmstate'):
-            # first time
-            self._adpcmstate = None
-        data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
-        return data
-
-    def _sowt2lin(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.byteswap(data, 2)
-
-    def _read_comm_chunk(self, chunk):
-        self._nchannels = _read_short(chunk)
-        self._nframes = _read_long(chunk)
-        self._sampwidth = (_read_short(chunk) + 7) // 8
-        self._framerate = int(_read_float(chunk))
-        if self._sampwidth <= 0:
-            raise Error('bad sample width')
-        if self._nchannels <= 0:
-            raise Error('bad # of channels')
-        self._framesize = self._nchannels * self._sampwidth
-        if self._aifc:
-            #DEBUG: SGI's soundeditor produces a bad size :-(
-            kludge = 0
-            if chunk.chunksize == 18:
-                kludge = 1
-                warnings.warn('Warning: bad COMM chunk size')
-                chunk.chunksize = 23
-            #DEBUG end
-            self._comptype = chunk.read(4)
-            #DEBUG start
-            if kludge:
-                length = ord(chunk.file.read(1))
-                if length & 1 == 0:
-                    length = length + 1
-                chunk.chunksize = chunk.chunksize + length
-                chunk.file.seek(-1, 1)
-            #DEBUG end
-            self._compname = _read_string(chunk)
-            if self._comptype != b'NONE':
-                if self._comptype == b'G722':
-                    self._convert = self._adpcm2lin
-                elif self._comptype in (b'ulaw', b'ULAW'):
-                    self._convert = self._ulaw2lin
-                elif self._comptype in (b'alaw', b'ALAW'):
-                    self._convert = self._alaw2lin
-                elif self._comptype in (b'sowt', b'SOWT'):
-                    self._convert = self._sowt2lin
-                else:
-                    raise Error('unsupported compression type')
-                self._sampwidth = 2
-        else:
-            self._comptype = b'NONE'
-            self._compname = b'not compressed'
-
-    def _readmark(self, chunk):
-        nmarkers = _read_short(chunk)
-        # Some files appear to contain invalid counts.
-        # Cope with this by testing for EOF.
-        try:
-            for i in range(nmarkers):
-                id = _read_short(chunk)
-                pos = _read_long(chunk)
-                name = _read_string(chunk)
-                if pos or name:
-                    # some files appear to have
-                    # dummy markers consisting of
-                    # a position 0 and name ''
-                    self._markers.append((id, pos, name))
-        except EOFError:
-            w = ('Warning: MARK chunk contains only %s marker%s instead of %s' %
-                 (len(self._markers), '' if len(self._markers) == 1 else 's',
-                  nmarkers))
-            warnings.warn(w)
-
-class Aifc_write:
-    # Variables used in this class:
-    #
-    # These variables are user settable through appropriate methods
-    # of this class:
-    # _file -- the open file with methods write(), close(), tell(), seek()
-    #       set through the __init__() method
-    # _comptype -- the AIFF-C compression type ('NONE' in AIFF)
-    #       set through the setcomptype() or setparams() method
-    # _compname -- the human-readable AIFF-C compression type
-    #       set through the setcomptype() or setparams() method
-    # _nchannels -- the number of audio channels
-    #       set through the setnchannels() or setparams() method
-    # _sampwidth -- the number of bytes per audio sample
-    #       set through the setsampwidth() or setparams() method
-    # _framerate -- the sampling frequency
-    #       set through the setframerate() or setparams() method
-    # _nframes -- the number of audio frames written to the header
-    #       set through the setnframes() or setparams() method
-    # _aifc -- whether we're writing an AIFF-C file or an AIFF file
-    #       set through the aifc() method, reset through the
-    #       aiff() method
-    #
-    # These variables are used internally only:
-    # _version -- the AIFF-C version number
-    # _comp -- the compressor from builtin module cl
-    # _nframeswritten -- the number of audio frames actually written
-    # _datalength -- the size of the audio samples written to the header
-    # _datawritten -- the size of the audio samples actually written
-
-    _file = None  # Set here since __del__ checks it
-
-    def __init__(self, f):
-        if isinstance(f, str):
-            file_object = builtins.open(f, 'wb')
-            try:
-                self.initfp(file_object)
-            except:
-                file_object.close()
-                raise
-
-            # treat .aiff file extensions as non-compressed audio
-            if f.endswith('.aiff'):
-                self._aifc = 0
-        else:
-            # assume it is an open file object already
-            self.initfp(f)
-
-    def initfp(self, file):
-        self._file = file
-        self._version = _AIFC_version
-        self._comptype = b'NONE'
-        self._compname = b'not compressed'
-        self._convert = None
-        self._nchannels = 0
-        self._sampwidth = 0
-        self._framerate = 0
-        self._nframes = 0
-        self._nframeswritten = 0
-        self._datawritten = 0
-        self._datalength = 0
-        self._markers = []
-        self._marklength = 0
-        self._aifc = 1      # AIFF-C is default
-
-    def __del__(self):
-        self.close()
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        self.close()
-
-    #
-    # User visible methods.
-    #
-    def aiff(self):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        self._aifc = 0
-
-    def aifc(self):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        self._aifc = 1
-
-    def setnchannels(self, nchannels):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        if nchannels < 1:
-            raise Error('bad # of channels')
-        self._nchannels = nchannels
-
-    def getnchannels(self):
-        if not self._nchannels:
-            raise Error('number of channels not set')
-        return self._nchannels
-
-    def setsampwidth(self, sampwidth):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        if sampwidth < 1 or sampwidth > 4:
-            raise Error('bad sample width')
-        self._sampwidth = sampwidth
-
-    def getsampwidth(self):
-        if not self._sampwidth:
-            raise Error('sample width not set')
-        return self._sampwidth
-
-    def setframerate(self, framerate):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        if framerate <= 0:
-            raise Error('bad frame rate')
-        self._framerate = framerate
-
-    def getframerate(self):
-        if not self._framerate:
-            raise Error('frame rate not set')
-        return self._framerate
-
-    def setnframes(self, nframes):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        self._nframes = nframes
-
-    def getnframes(self):
-        return self._nframeswritten
-
-    def setcomptype(self, comptype, compname):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        if comptype not in (b'NONE', b'ulaw', b'ULAW',
-                            b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
-            raise Error('unsupported compression type')
-        self._comptype = comptype
-        self._compname = compname
-
-    def getcomptype(self):
-        return self._comptype
-
-    def getcompname(self):
-        return self._compname
-
-##  def setversion(self, version):
-##      if self._nframeswritten:
-##          raise Error, 'cannot change parameters after starting to write'
-##      self._version = version
-
-    def setparams(self, params):
-        nchannels, sampwidth, framerate, nframes, comptype, compname = params
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        if comptype not in (b'NONE', b'ulaw', b'ULAW',
-                            b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
-            raise Error('unsupported compression type')
-        self.setnchannels(nchannels)
-        self.setsampwidth(sampwidth)
-        self.setframerate(framerate)
-        self.setnframes(nframes)
-        self.setcomptype(comptype, compname)
-
-    def getparams(self):
-        if not self._nchannels or not self._sampwidth or not self._framerate:
-            raise Error('not all parameters set')
-        return _aifc_params(self._nchannels, self._sampwidth, self._framerate,
-                            self._nframes, self._comptype, self._compname)
-
-    def setmark(self, id, pos, name):
-        if id <= 0:
-            raise Error('marker ID must be > 0')
-        if pos < 0:
-            raise Error('marker position must be >= 0')
-        if not isinstance(name, bytes):
-            raise Error('marker name must be bytes')
-        for i in range(len(self._markers)):
-            if id == self._markers[i][0]:
-                self._markers[i] = id, pos, name
-                return
-        self._markers.append((id, pos, name))
-
-    def getmark(self, id):
-        for marker in self._markers:
-            if id == marker[0]:
-                return marker
-        raise Error('marker {0!r} does not exist'.format(id))
-
-    def getmarkers(self):
-        if len(self._markers) == 0:
-            return None
-        return self._markers
-
-    def tell(self):
-        return self._nframeswritten
-
-    def writeframesraw(self, data):
-        if not isinstance(data, (bytes, bytearray)):
-            data = memoryview(data).cast('B')
-        self._ensure_header_written(len(data))
-        nframes = len(data) // (self._sampwidth * self._nchannels)
-        if self._convert:
-            data = self._convert(data)
-        self._file.write(data)
-        self._nframeswritten = self._nframeswritten + nframes
-        self._datawritten = self._datawritten + len(data)
-
-    def writeframes(self, data):
-        self.writeframesraw(data)
-        if self._nframeswritten != self._nframes or \
-              self._datalength != self._datawritten:
-            self._patchheader()
-
-    def close(self):
-        if self._file is None:
-            return
-        try:
-            self._ensure_header_written(0)
-            if self._datawritten & 1:
-                # quick pad to even size
-                self._file.write(b'\x00')
-                self._datawritten = self._datawritten + 1
-            self._writemarkers()
-            if self._nframeswritten != self._nframes or \
-                  self._datalength != self._datawritten or \
-                  self._marklength:
-                self._patchheader()
-        finally:
-            # Prevent ref cycles
-            self._convert = None
-            f = self._file
-            self._file = None
-            f.close()
-
-    #
-    # Internal methods.
-    #
-
-    def _lin2alaw(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.lin2alaw(data, 2)
-
-    def _lin2ulaw(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.lin2ulaw(data, 2)
-
-    def _lin2adpcm(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        if not hasattr(self, '_adpcmstate'):
-            self._adpcmstate = None
-        data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
-        return data
-
-    def _lin2sowt(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.byteswap(data, 2)
-
-    def _ensure_header_written(self, datasize):
-        if not self._nframeswritten:
-            if self._comptype in (b'ULAW', b'ulaw',
-                b'ALAW', b'alaw', b'G722',
-                b'sowt', b'SOWT'):
-                if not self._sampwidth:
-                    self._sampwidth = 2
-                if self._sampwidth != 2:
-                    raise Error('sample width must be 2 when compressing '
-                                'with ulaw/ULAW, alaw/ALAW, sowt/SOWT '
-                                'or G7.22 (ADPCM)')
-            if not self._nchannels:
-                raise Error('# channels not specified')
-            if not self._sampwidth:
-                raise Error('sample width not specified')
-            if not self._framerate:
-                raise Error('sampling rate not specified')
-            self._write_header(datasize)
-
-    def _init_compression(self):
-        if self._comptype == b'G722':
-            self._convert = self._lin2adpcm
-        elif self._comptype in (b'ulaw', b'ULAW'):
-            self._convert = self._lin2ulaw
-        elif self._comptype in (b'alaw', b'ALAW'):
-            self._convert = self._lin2alaw
-        elif self._comptype in (b'sowt', b'SOWT'):
-            self._convert = self._lin2sowt
-
-    def _write_header(self, initlength):
-        if self._aifc and self._comptype != b'NONE':
-            self._init_compression()
-        self._file.write(b'FORM')
-        if not self._nframes:
-            self._nframes = initlength // (self._nchannels * self._sampwidth)
-        self._datalength = self._nframes * self._nchannels * self._sampwidth
-        if self._datalength & 1:
-            self._datalength = self._datalength + 1
-        if self._aifc:
-            if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'):
-                self._datalength = self._datalength // 2
-                if self._datalength & 1:
-                    self._datalength = self._datalength + 1
-            elif self._comptype == b'G722':
-                self._datalength = (self._datalength + 3) // 4
-                if self._datalength & 1:
-                    self._datalength = self._datalength + 1
-        try:
-            self._form_length_pos = self._file.tell()
-        except (AttributeError, OSError):
-            self._form_length_pos = None
-        commlength = self._write_form_length(self._datalength)
-        if self._aifc:
-            self._file.write(b'AIFC')
-            self._file.write(b'FVER')
-            _write_ulong(self._file, 4)
-            _write_ulong(self._file, self._version)
-        else:
-            self._file.write(b'AIFF')
-        self._file.write(b'COMM')
-        _write_ulong(self._file, commlength)
-        _write_short(self._file, self._nchannels)
-        if self._form_length_pos is not None:
-            self._nframes_pos = self._file.tell()
-        _write_ulong(self._file, self._nframes)
-        if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
-            _write_short(self._file, 8)
-        else:
-            _write_short(self._file, self._sampwidth * 8)
-        _write_float(self._file, self._framerate)
-        if self._aifc:
-            self._file.write(self._comptype)
-            _write_string(self._file, self._compname)
-        self._file.write(b'SSND')
-        if self._form_length_pos is not None:
-            self._ssnd_length_pos = self._file.tell()
-        _write_ulong(self._file, self._datalength + 8)
-        _write_ulong(self._file, 0)
-        _write_ulong(self._file, 0)
-
-    def _write_form_length(self, datalength):
-        if self._aifc:
-            commlength = 18 + 5 + len(self._compname)
-            if commlength & 1:
-                commlength = commlength + 1
-            verslength = 12
-        else:
-            commlength = 18
-            verslength = 0
-        _write_ulong(self._file, 4 + verslength + self._marklength + \
-                     8 + commlength + 16 + datalength)
-        return commlength
-
-    def _patchheader(self):
-        curpos = self._file.tell()
-        if self._datawritten & 1:
-            datalength = self._datawritten + 1
-            self._file.write(b'\x00')
-        else:
-            datalength = self._datawritten
-        if datalength == self._datalength and \
-              self._nframes == self._nframeswritten and \
-              self._marklength == 0:
-            self._file.seek(curpos, 0)
-            return
-        self._file.seek(self._form_length_pos, 0)
-        dummy = self._write_form_length(datalength)
-        self._file.seek(self._nframes_pos, 0)
-        _write_ulong(self._file, self._nframeswritten)
-        self._file.seek(self._ssnd_length_pos, 0)
-        _write_ulong(self._file, datalength + 8)
-        self._file.seek(curpos, 0)
-        self._nframes = self._nframeswritten
-        self._datalength = datalength
-
-    def _writemarkers(self):
-        if len(self._markers) == 0:
-            return
-        self._file.write(b'MARK')
-        length = 2
-        for marker in self._markers:
-            id, pos, name = marker
-            length = length + len(name) + 1 + 6
-            if len(name) & 1 == 0:
-                length = length + 1
-        _write_ulong(self._file, length)
-        self._marklength = length + 8
-        _write_short(self._file, len(self._markers))
-        for marker in self._markers:
-            id, pos, name = marker
-            _write_short(self._file, id)
-            _write_ulong(self._file, pos)
-            _write_string(self._file, name)
-
-def open(f, mode=None):
-    if mode is None:
-        if hasattr(f, 'mode'):
-            mode = f.mode
-        else:
-            mode = 'rb'
-    if mode in ('r', 'rb'):
-        return Aifc_read(f)
-    elif mode in ('w', 'wb'):
-        return Aifc_write(f)
-    else:
-        raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
-
-
-if __name__ == '__main__':
-    import sys
-    if not sys.argv[1:]:
-        sys.argv.append('/usr/demos/data/audio/bach.aiff')
-    fn = sys.argv[1]
-    with open(fn, 'r') as f:
-        print("Reading", fn)
-        print("nchannels =", f.getnchannels())
-        print("nframes   =", f.getnframes())
-        print("sampwidth =", f.getsampwidth())
-        print("framerate =", f.getframerate())
-        print("comptype  =", f.getcomptype())
-        print("compname  =", f.getcompname())
-        if sys.argv[2:]:
-            gn = sys.argv[2]
-            print("Writing", gn)
-            with open(gn, 'w') as g:
-                g.setparams(f.getparams())
-                while 1:
-                    data = f.readframes(1024)
-                    if not data:
-                        break
-                    g.writeframes(data)
-            print("Done.")

Lib/asynchat.py

@@ -1,307 +0,0 @@
-# -*- Mode: Python; tab-width: 4 -*-
-#       Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp
-#       Author: Sam Rushing <rushing@nightmare.com>
-
-# ======================================================================
-# Copyright 1996 by Sam Rushing
-#
-#                         All Rights Reserved
-#
-# Permission to use, copy, modify, and distribute this software and
-# its documentation for any purpose and without fee is hereby
-# granted, provided that the above copyright notice appear in all
-# copies and that both that copyright notice and this permission
-# notice appear in supporting documentation, and that the name of Sam
-# Rushing not be used in advertising or publicity pertaining to
-# distribution of the software without specific, written prior
-# permission.
-#
-# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
-# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
-# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
-# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
-# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-# ======================================================================
-
-r"""A class supporting chat-style (command/response) protocols.
-
-This class adds support for 'chat' style protocols - where one side
-sends a 'command', and the other sends a response (examples would be
-the common internet protocols - smtp, nntp, ftp, etc..).
-
-The handle_read() method looks at the input stream for the current
-'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n'
-for multi-line output), calling self.found_terminator() on its
-receipt.
-
-for example:
-Say you build an async nntp client using this class.  At the start
-of the connection, you'll have self.terminator set to '\r\n', in
-order to process the single-line greeting.  Just before issuing a
-'LIST' command you'll set it to '\r\n.\r\n'.  The output of the LIST
-command will be accumulated (using your own 'collect_incoming_data'
-method) up to the terminator, and then control will be returned to
-you - by calling your self.found_terminator() method.
-"""
-import asyncore
-from collections import deque
-
-
-class async_chat(asyncore.dispatcher):
-    """This is an abstract class.  You must derive from this class, and add
-    the two methods collect_incoming_data() and found_terminator()"""
-
-    # these are overridable defaults
-
-    ac_in_buffer_size = 65536
-    ac_out_buffer_size = 65536
-
-    # we don't want to enable the use of encoding by default, because that is a
-    # sign of an application bug that we don't want to pass silently
-
-    use_encoding = 0
-    encoding = 'latin-1'
-
-    def __init__(self, sock=None, map=None):
-        # for string terminator matching
-        self.ac_in_buffer = b''
-
-        # we use a list here rather than io.BytesIO for a few reasons...
-        # del lst[:] is faster than bio.truncate(0)
-        # lst = [] is faster than bio.truncate(0)
-        self.incoming = []
-
-        # we toss the use of the "simple producer" and replace it with
-        # a pure deque, which the original fifo was a wrapping of
-        self.producer_fifo = deque()
-        asyncore.dispatcher.__init__(self, sock, map)
-
-    def collect_incoming_data(self, data):
-        raise NotImplementedError("must be implemented in subclass")
-
-    def _collect_incoming_data(self, data):
-        self.incoming.append(data)
-
-    def _get_data(self):
-        d = b''.join(self.incoming)
-        del self.incoming[:]
-        return d
-
-    def found_terminator(self):
-        raise NotImplementedError("must be implemented in subclass")
-
-    def set_terminator(self, term):
-        """Set the input delimiter.
-
-        Can be a fixed string of any length, an integer, or None.
-        """
-        if isinstance(term, str) and self.use_encoding:
-            term = bytes(term, self.encoding)
-        elif isinstance(term, int) and term < 0:
-            raise ValueError('the number of received bytes must be positive')
-        self.terminator = term
-
-    def get_terminator(self):
-        return self.terminator
-
-    # grab some more data from the socket,
-    # throw it to the collector method,
-    # check for the terminator,
-    # if found, transition to the next state.
-
-    def handle_read(self):
-
-        try:
-            data = self.recv(self.ac_in_buffer_size)
-        except BlockingIOError:
-            return
-        except OSError as why:
-            self.handle_error()
-            return
-
-        if isinstance(data, str) and self.use_encoding:
-            data = bytes(str, self.encoding)
-        self.ac_in_buffer = self.ac_in_buffer + data
-
-        # Continue to search for self.terminator in self.ac_in_buffer,
-        # while calling self.collect_incoming_data.  The while loop
-        # is necessary because we might read several data+terminator
-        # combos with a single recv(4096).
-
-        while self.ac_in_buffer:
-            lb = len(self.ac_in_buffer)
-            terminator = self.get_terminator()
-            if not terminator:
-                # no terminator, collect it all
-                self.collect_incoming_data(self.ac_in_buffer)
-                self.ac_in_buffer = b''
-            elif isinstance(terminator, int):
-                # numeric terminator
-                n = terminator
-                if lb < n:
-                    self.collect_incoming_data(self.ac_in_buffer)
-                    self.ac_in_buffer = b''
-                    self.terminator = self.terminator - lb
-                else:
-                    self.collect_incoming_data(self.ac_in_buffer[:n])
-                    self.ac_in_buffer = self.ac_in_buffer[n:]
-                    self.terminator = 0
-                    self.found_terminator()
-            else:
-                # 3 cases:
-                # 1) end of buffer matches terminator exactly:
-                #    collect data, transition
-                # 2) end of buffer matches some prefix:
-                #    collect data to the prefix
-                # 3) end of buffer does not match any prefix:
-                #    collect data
-                terminator_len = len(terminator)
-                index = self.ac_in_buffer.find(terminator)
-                if index != -1:
-                    # we found the terminator
-                    if index > 0:
-                        # don't bother reporting the empty string
-                        # (source of subtle bugs)
-                        self.collect_incoming_data(self.ac_in_buffer[:index])
-                    self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
-                    # This does the Right Thing if the terminator
-                    # is changed here.
-                    self.found_terminator()
-                else:
-                    # check for a prefix of the terminator
-                    index = find_prefix_at_end(self.ac_in_buffer, terminator)
-                    if index:
-                        if index != lb:
-                            # we found a prefix, collect up to the prefix
-                            self.collect_incoming_data(self.ac_in_buffer[:-index])
-                            self.ac_in_buffer = self.ac_in_buffer[-index:]
-                        break
-                    else:
-                        # no prefix, collect it all
-                        self.collect_incoming_data(self.ac_in_buffer)
-                        self.ac_in_buffer = b''
-
-    def handle_write(self):
-        self.initiate_send()
-
-    def handle_close(self):
-        self.close()
-
-    def push(self, data):
-        if not isinstance(data, (bytes, bytearray, memoryview)):
-            raise TypeError('data argument must be byte-ish (%r)',
-                            type(data))
-        sabs = self.ac_out_buffer_size
-        if len(data) > sabs:
-            for i in range(0, len(data), sabs):
-                self.producer_fifo.append(data[i:i+sabs])
-        else:
-            self.producer_fifo.append(data)
-        self.initiate_send()
-
-    def push_with_producer(self, producer):
-        self.producer_fifo.append(producer)
-        self.initiate_send()
-
-    def readable(self):
-        "predicate for inclusion in the readable for select()"
-        # cannot use the old predicate, it violates the claim of the
-        # set_terminator method.
-
-        # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
-        return 1
-
-    def writable(self):
-        "predicate for inclusion in the writable for select()"
-        return self.producer_fifo or (not self.connected)
-
-    def close_when_done(self):
-        "automatically close this channel once the outgoing queue is empty"
-        self.producer_fifo.append(None)
-
-    def initiate_send(self):
-        while self.producer_fifo and self.connected:
-            first = self.producer_fifo[0]
-            # handle empty string/buffer or None entry
-            if not first:
-                del self.producer_fifo[0]
-                if first is None:
-                    self.handle_close()
-                    return
-
-            # handle classic producer behavior
-            obs = self.ac_out_buffer_size
-            try:
-                data = first[:obs]
-            except TypeError:
-                data = first.more()
-                if data:
-                    self.producer_fifo.appendleft(data)
-                else:
-                    del self.producer_fifo[0]
-                continue
-
-            if isinstance(data, str) and self.use_encoding:
-                data = bytes(data, self.encoding)
-
-            # send the data
-            try:
-                num_sent = self.send(data)
-            except OSError:
-                self.handle_error()
-                return
-
-            if num_sent:
-                if num_sent < len(data) or obs < len(first):
-                    self.producer_fifo[0] = first[num_sent:]
-                else:
-                    del self.producer_fifo[0]
-            # we tried to send some actual data
-            return
-
-    def discard_buffers(self):
-        # Emergencies only!
-        self.ac_in_buffer = b''
-        del self.incoming[:]
-        self.producer_fifo.clear()
-
-
-class simple_producer:
-
-    def __init__(self, data, buffer_size=512):
-        self.data = data
-        self.buffer_size = buffer_size
-
-    def more(self):
-        if len(self.data) > self.buffer_size:
-            result = self.data[:self.buffer_size]
-            self.data = self.data[self.buffer_size:]
-            return result
-        else:
-            result = self.data
-            self.data = b''
-            return result
-
-
-# Given 'haystack', see if any prefix of 'needle' is at its end.  This
-# assumes an exact match has already been checked.  Return the number of
-# characters matched.
-# for example:
-# f_p_a_e("qwerty\r", "\r\n") => 1
-# f_p_a_e("qwertydkjf", "\r\n") => 0
-# f_p_a_e("qwerty\r\n", "\r\n") => <undefined>
-
-# this could maybe be made faster with a computed regex?
-# [answer: no; circa Python-2.0, Jan 2001]
-# new python:   28961/s
-# old python:   18307/s
-# re:        12820/s
-# regex:     14035/s
-
-def find_prefix_at_end(haystack, needle):
-    l = len(needle) - 1
-    while l and not haystack.endswith(needle[:l]):
-        l -= 1
-    return l

Lib/asyncore.py

@@ -1,642 +0,0 @@
-# -*- Mode: Python -*-
-#   Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp
-#   Author: Sam Rushing <rushing@nightmare.com>
-
-# ======================================================================
-# Copyright 1996 by Sam Rushing
-#
-#                         All Rights Reserved
-#
-# Permission to use, copy, modify, and distribute this software and
-# its documentation for any purpose and without fee is hereby
-# granted, provided that the above copyright notice appear in all
-# copies and that both that copyright notice and this permission
-# notice appear in supporting documentation, and that the name of Sam
-# Rushing not be used in advertising or publicity pertaining to
-# distribution of the software without specific, written prior
-# permission.
-#
-# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
-# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
-# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
-# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
-# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-# ======================================================================
-
-"""Basic infrastructure for asynchronous socket service clients and servers.
-
-There are only two ways to have a program on a single processor do "more
-than one thing at a time".  Multi-threaded programming is the simplest and
-most popular way to do it, but there is another very different technique,
-that lets you have nearly all the advantages of multi-threading, without
-actually using multiple threads. it's really only practical if your program
-is largely I/O bound. If your program is CPU bound, then pre-emptive
-scheduled threads are probably what you really need. Network servers are
-rarely CPU-bound, however.
-
-If your operating system supports the select() system call in its I/O
-library (and nearly all do), then you can use it to juggle multiple
-communication channels at once; doing other work while your I/O is taking
-place in the "background."  Although this strategy can seem strange and
-complex, especially at first, it is in many ways easier to understand and
-control than multi-threaded programming. The module documented here solves
-many of the difficult problems for you, making the task of building
-sophisticated high-performance network servers and clients a snap.
-"""
-
-import select
-import socket
-import sys
-import time
-import warnings
-
-import os
-from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
-     ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
-     errorcode
-
-_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
-                           EBADF})
-
-try:
-    socket_map
-except NameError:
-    socket_map = {}
-
-def _strerror(err):
-    try:
-        return os.strerror(err)
-    except (ValueError, OverflowError, NameError):
-        if err in errorcode:
-            return errorcode[err]
-        return "Unknown error %s" %err
-
-class ExitNow(Exception):
-    pass
-
-_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit)
-
-def read(obj):
-    try:
-        obj.handle_read_event()
-    except _reraised_exceptions:
-        raise
-    except:
-        obj.handle_error()
-
-def write(obj):
-    try:
-        obj.handle_write_event()
-    except _reraised_exceptions:
-        raise
-    except:
-        obj.handle_error()
-
-def _exception(obj):
-    try:
-        obj.handle_expt_event()
-    except _reraised_exceptions:
-        raise
-    except:
-        obj.handle_error()
-
-def readwrite(obj, flags):
-    try:
-        if flags & select.POLLIN:
-            obj.handle_read_event()
-        if flags & select.POLLOUT:
-            obj.handle_write_event()
-        if flags & select.POLLPRI:
-            obj.handle_expt_event()
-        if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
-            obj.handle_close()
-    except OSError as e:
-        if e.args[0] not in _DISCONNECTED:
-            obj.handle_error()
-        else:
-            obj.handle_close()
-    except _reraised_exceptions:
-        raise
-    except:
-        obj.handle_error()
-
-def poll(timeout=0.0, map=None):
-    if map is None:
-        map = socket_map
-    if map:
-        r = []; w = []; e = []
-        for fd, obj in list(map.items()):
-            is_r = obj.readable()
-            is_w = obj.writable()
-            if is_r:
-                r.append(fd)
-            # accepting sockets should not be writable
-            if is_w and not obj.accepting:
-                w.append(fd)
-            if is_r or is_w:
-                e.append(fd)
-        if [] == r == w == e:
-            time.sleep(timeout)
-            return
-
-        r, w, e = select.select(r, w, e, timeout)
-
-        for fd in r:
-            obj = map.get(fd)
-            if obj is None:
-                continue
-            read(obj)
-
-        for fd in w:
-            obj = map.get(fd)
-            if obj is None:
-                continue
-            write(obj)
-
-        for fd in e:
-            obj = map.get(fd)
-            if obj is None:
-                continue
-            _exception(obj)
-
-def poll2(timeout=0.0, map=None):
-    # Use the poll() support added to the select module in Python 2.0
-    if map is None:
-        map = socket_map
-    if timeout is not None:
-        # timeout is in milliseconds
-        timeout = int(timeout*1000)
-    pollster = select.poll()
-    if map:
-        for fd, obj in list(map.items()):
-            flags = 0
-            if obj.readable():
-                flags |= select.POLLIN | select.POLLPRI
-            # accepting sockets should not be writable
-            if obj.writable() and not obj.accepting:
-                flags |= select.POLLOUT
-            if flags:
-                pollster.register(fd, flags)
-
-        r = pollster.poll(timeout)
-        for fd, flags in r:
-            obj = map.get(fd)
-            if obj is None:
-                continue
-            readwrite(obj, flags)
-
-poll3 = poll2                           # Alias for backward compatibility
-
-def loop(timeout=30.0, use_poll=False, map=None, count=None):
-    if map is None:
-        map = socket_map
-
-    if use_poll and hasattr(select, 'poll'):
-        poll_fun = poll2
-    else:
-        poll_fun = poll
-
-    if count is None:
-        while map:
-            poll_fun(timeout, map)
-
-    else:
-        while map and count > 0:
-            poll_fun(timeout, map)
-            count = count - 1
-
-class dispatcher:
-
-    debug = False
-    connected = False
-    accepting = False
-    connecting = False
-    closing = False
-    addr = None
-    ignore_log_types = frozenset({'warning'})
-
-    def __init__(self, sock=None, map=None):
-        if map is None:
-            self._map = socket_map
-        else:
-            self._map = map
-
-        self._fileno = None
-
-        if sock:
-            # Set to nonblocking just to make sure for cases where we
-            # get a socket from a blocking source.
-            sock.setblocking(0)
-            self.set_socket(sock, map)
-            self.connected = True
-            # The constructor no longer requires that the socket
-            # passed be connected.
-            try:
-                self.addr = sock.getpeername()
-            except OSError as err:
-                if err.args[0] in (ENOTCONN, EINVAL):
-                    # To handle the case where we got an unconnected
-                    # socket.
-                    self.connected = False
-                else:
-                    # The socket is broken in some unknown way, alert
-                    # the user and remove it from the map (to prevent
-                    # polling of broken sockets).
-                    self.del_channel(map)
-                    raise
-        else:
-            self.socket = None
-
-    def __repr__(self):
-        status = [self.__class__.__module__+"."+self.__class__.__qualname__]
-        if self.accepting and self.addr:
-            status.append('listening')
-        elif self.connected:
-            status.append('connected')
-        if self.addr is not None:
-            try:
-                status.append('%s:%d' % self.addr)
-            except TypeError:
-                status.append(repr(self.addr))
-        return '<%s at %#x>' % (' '.join(status), id(self))
-
-    def add_channel(self, map=None):
-        #self.log_info('adding channel %s' % self)
-        if map is None:
-            map = self._map
-        map[self._fileno] = self
-
-    def del_channel(self, map=None):
-        fd = self._fileno
-        if map is None:
-            map = self._map
-        if fd in map:
-            #self.log_info('closing channel %d:%s' % (fd, self))
-            del map[fd]
-        self._fileno = None
-
-    def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM):
-        self.family_and_type = family, type
-        sock = socket.socket(family, type)
-        sock.setblocking(0)
-        self.set_socket(sock)
-
-    def set_socket(self, sock, map=None):
-        self.socket = sock
-        self._fileno = sock.fileno()
-        self.add_channel(map)
-
-    def set_reuse_addr(self):
-        # try to re-use a server port if possible
-        try:
-            self.socket.setsockopt(
-                socket.SOL_SOCKET, socket.SO_REUSEADDR,
-                self.socket.getsockopt(socket.SOL_SOCKET,
-                                       socket.SO_REUSEADDR) | 1
-                )
-        except OSError:
-            pass
-
-    # ==================================================
-    # predicates for select()
-    # these are used as filters for the lists of sockets
-    # to pass to select().
-    # ==================================================
-
-    def readable(self):
-        return True
-
-    def writable(self):
-        return True
-
-    # ==================================================
-    # socket object methods.
-    # ==================================================
-
-    def listen(self, num):
-        self.accepting = True
-        if os.name == 'nt' and num > 5:
-            num = 5
-        return self.socket.listen(num)
-
-    def bind(self, addr):
-        self.addr = addr
-        return self.socket.bind(addr)
-
-    def connect(self, address):
-        self.connected = False
-        self.connecting = True
-        err = self.socket.connect_ex(address)
-        if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \
-        or err == EINVAL and os.name == 'nt':
-            self.addr = address
-            return
-        if err in (0, EISCONN):
-            self.addr = address
-            self.handle_connect_event()
-        else:
-            raise OSError(err, errorcode[err])
-
-    def accept(self):
-        # XXX can return either an address pair or None
-        try:
-            conn, addr = self.socket.accept()
-        except TypeError:
-            return None
-        except OSError as why:
-            if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
-                return None
-            else:
-                raise
-        else:
-            return conn, addr
-
-    def send(self, data):
-        try:
-            result = self.socket.send(data)
-            return result
-        except OSError as why:
-            if why.args[0] == EWOULDBLOCK:
-                return 0
-            elif why.args[0] in _DISCONNECTED:
-                self.handle_close()
-                return 0
-            else:
-                raise
-
-    def recv(self, buffer_size):
-        try:
-            data = self.socket.recv(buffer_size)
-            if not data:
-                # a closed connection is indicated by signaling
-                # a read condition, and having recv() return 0.
-                self.handle_close()
-                return b''
-            else:
-                return data
-        except OSError as why:
-            # winsock sometimes raises ENOTCONN
-            if why.args[0] in _DISCONNECTED:
-                self.handle_close()
-                return b''
-            else:
-                raise
-
-    def close(self):
-        self.connected = False
-        self.accepting = False
-        self.connecting = False
-        self.del_channel()
-        if self.socket is not None:
-            try:
-                self.socket.close()
-            except OSError as why:
-                if why.args[0] not in (ENOTCONN, EBADF):
-                    raise
-
-    # log and log_info may be overridden to provide more sophisticated
-    # logging and warning methods. In general, log is for 'hit' logging
-    # and 'log_info' is for informational, warning and error logging.
-
-    def log(self, message):
-        sys.stderr.write('log: %s\n' % str(message))
-
-    def log_info(self, message, type='info'):
-        if type not in self.ignore_log_types:
-            print('%s: %s' % (type, message))
-
-    def handle_read_event(self):
-        if self.accepting:
-            # accepting sockets are never connected, they "spawn" new
-            # sockets that are connected
-            self.handle_accept()
-        elif not self.connected:
-            if self.connecting:
-                self.handle_connect_event()
-            self.handle_read()
-        else:
-            self.handle_read()
-
-    def handle_connect_event(self):
-        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
-        if err != 0:
-            raise OSError(err, _strerror(err))
-        self.handle_connect()
-        self.connected = True
-        self.connecting = False
-
-    def handle_write_event(self):
-        if self.accepting:
-            # Accepting sockets shouldn't get a write event.
-            # We will pretend it didn't happen.
-            return
-
-        if not self.connected:
-            if self.connecting:
-                self.handle_connect_event()
-        self.handle_write()
-
-    def handle_expt_event(self):
-        # handle_expt_event() is called if there might be an error on the
-        # socket, or if there is OOB data
-        # check for the error condition first
-        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
-        if err != 0:
-            # we can get here when select.select() says that there is an
-            # exceptional condition on the socket
-            # since there is an error, we'll go ahead and close the socket
-            # like we would in a subclassed handle_read() that received no
-            # data
-            self.handle_close()
-        else:
-            self.handle_expt()
-
-    def handle_error(self):
-        nil, t, v, tbinfo = compact_traceback()
-
-        # sometimes a user repr method will crash.
-        try:
-            self_repr = repr(self)
-        except:
-            self_repr = '<__repr__(self) failed for object at %0x>' % id(self)
-
-        self.log_info(
-            'uncaptured python exception, closing channel %s (%s:%s %s)' % (
-                self_repr,
-                t,
-                v,
-                tbinfo
-                ),
-            'error'
-            )
-        self.handle_close()
-
-    def handle_expt(self):
-        self.log_info('unhandled incoming priority event', 'warning')
-
-    def handle_read(self):
-        self.log_info('unhandled read event', 'warning')
-
-    def handle_write(self):
-        self.log_info('unhandled write event', 'warning')
-
-    def handle_connect(self):
-        self.log_info('unhandled connect event', 'warning')
-
-    def handle_accept(self):
-        pair = self.accept()
-        if pair is not None:
-            self.handle_accepted(*pair)
-
-    def handle_accepted(self, sock, addr):
-        sock.close()
-        self.log_info('unhandled accepted event', 'warning')
-
-    def handle_close(self):
-        self.log_info('unhandled close event', 'warning')
-        self.close()
-
-# ---------------------------------------------------------------------------
-# adds simple buffered output capability, useful for simple clients.
-# [for more sophisticated usage use asynchat.async_chat]
-# ---------------------------------------------------------------------------
-
-class dispatcher_with_send(dispatcher):
-
-    def __init__(self, sock=None, map=None):
-        dispatcher.__init__(self, sock, map)
-        self.out_buffer = b''
-
-    def initiate_send(self):
-        num_sent = 0
-        num_sent = dispatcher.send(self, self.out_buffer[:65536])
-        self.out_buffer = self.out_buffer[num_sent:]
-
-    def handle_write(self):
-        self.initiate_send()
-
-    def writable(self):
-        return (not self.connected) or len(self.out_buffer)
-
-    def send(self, data):
-        if self.debug:
-            self.log_info('sending %s' % repr(data))
-        self.out_buffer = self.out_buffer + data
-        self.initiate_send()
-
-# ---------------------------------------------------------------------------
-# used for debugging.
-# ---------------------------------------------------------------------------
-
-def compact_traceback():
-    t, v, tb = sys.exc_info()
-    tbinfo = []
-    if not tb: # Must have a traceback
-        raise AssertionError("traceback does not exist")
-    while tb:
-        tbinfo.append((
-            tb.tb_frame.f_code.co_filename,
-            tb.tb_frame.f_code.co_name,
-            str(tb.tb_lineno)
-            ))
-        tb = tb.tb_next
-
-    # just to be safe
-    del tb
-
-    file, function, line = tbinfo[-1]
-    info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])
-    return (file, function, line), t, v, info
-
-def close_all(map=None, ignore_all=False):
-    if map is None:
-        map = socket_map
-    for x in list(map.values()):
-        try:
-            x.close()
-        except OSError as x:
-            if x.args[0] == EBADF:
-                pass
-            elif not ignore_all:
-                raise
-        except _reraised_exceptions:
-            raise
-        except:
-            if not ignore_all:
-                raise
-    map.clear()
-
-# Asynchronous File I/O:
-#
-# After a little research (reading man pages on various unixen, and
-# digging through the linux kernel), I've determined that select()
-# isn't meant for doing asynchronous file i/o.
-# Heartening, though - reading linux/mm/filemap.c shows that linux
-# supports asynchronous read-ahead.  So _MOST_ of the time, the data
-# will be sitting in memory for us already when we go to read it.
-#
-# What other OS's (besides NT) support async file i/o?  [VMS?]
-#
-# Regardless, this is useful for pipes, and stdin/stdout...
-
-if os.name == 'posix':
-    class file_wrapper:
-        # Here we override just enough to make a file
-        # look like a socket for the purposes of asyncore.
-        # The passed fd is automatically os.dup()'d
-
-        def __init__(self, fd):
-            self.fd = os.dup(fd)
-
-        def __del__(self):
-            if self.fd >= 0:
-                warnings.warn("unclosed file %r" % self, ResourceWarning,
-                              source=self)
-            self.close()
-
-        def recv(self, *args):
-            return os.read(self.fd, *args)
-
-        def send(self, *args):
-            return os.write(self.fd, *args)
-
-        def getsockopt(self, level, optname, buflen=None):
-            if (level == socket.SOL_SOCKET and
-                optname == socket.SO_ERROR and
-                not buflen):
-                return 0
-            raise NotImplementedError("Only asyncore specific behaviour "
-                                      "implemented.")
-
-        read = recv
-        write = send
-
-        def close(self):
-            if self.fd < 0:
-                return
-            fd = self.fd
-            self.fd = -1
-            os.close(fd)
-
-        def fileno(self):
-            return self.fd
-
-    class file_dispatcher(dispatcher):
-
-        def __init__(self, fd, map=None):
-            dispatcher.__init__(self, None, map)
-            self.connected = True
-            try:
-                fd = fd.fileno()
-            except AttributeError:
-                pass
-            self.set_file(fd)
-            # set it to non-blocking mode
-            os.set_blocking(fd, False)
-
-        def set_file(self, fd):
-            self.socket = file_wrapper(fd)
-            self._fileno = self.socket.fileno()
-            self.add_channel()

Lib/calendar.py

@@ -10,7 +10,6 @@ import datetime
 from enum import IntEnum, global_enum
 import locale as _locale
 from itertools import repeat
-import warnings
 
 __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
            "firstweekday", "isleap", "leapdays", "weekday", "monthrange",
@@ -28,7 +27,9 @@ __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
 error = ValueError
 
 # Exceptions raised for bad input
-class IllegalMonthError(ValueError):
+# This is trick for backward compatibility. Since 3.13, we will raise IllegalMonthError instead of
+# IndexError for bad month number(out of 1-12). But we can't remove IndexError for backward compatibility.
+class IllegalMonthError(ValueError, IndexError):
     def __init__(self, month):
         self.month = month
     def __str__(self):
@@ -44,6 +45,7 @@ class IllegalWeekdayError(ValueError):
 
 def __getattr__(name):
     if name in ('January', 'February'):
+        import warnings
         warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead",
                       DeprecationWarning, stacklevel=2)
         if name == 'January':
@@ -158,11 +160,14 @@ def weekday(year, month, day):
     return Day(datetime.date(year, month, day).weekday())
 
 
-def monthrange(year, month):
-    """Return weekday (0-6 ~ Mon-Sun) and number of days (28-31) for
-       year, month."""
+def _validate_month(month):
     if not 1 <= month <= 12:
         raise IllegalMonthError(month)
+
+def monthrange(year, month):
+    """Return weekday of first day of month (0-6 ~ Mon-Sun)
+       and number of days (28-31) for year, month."""
+    _validate_month(month)
     day1 = weekday(year, month, 1)
     ndays = mdays[month] + (month == FEBRUARY and isleap(year))
     return day1, ndays
@@ -370,6 +375,8 @@ class TextCalendar(Calendar):
         """
         Return a formatted month name.
         """
+        _validate_month(themonth)
+
         s = month_name[themonth]
         if withyear:
             s = "%s %r" % (s, theyear)
@@ -500,6 +507,7 @@ class HTMLCalendar(Calendar):
         """
         Return a month name as a table row.
         """
+        _validate_month(themonth)
         if withyear:
             s = '%s %s' % (month_name[themonth], theyear)
         else:
@@ -585,8 +593,6 @@ class different_locale:
         _locale.setlocale(_locale.LC_TIME, self.locale)
 
     def __exit__(self, *args):
-        if self.oldlocale is None:
-            return
         _locale.setlocale(_locale.LC_TIME, self.oldlocale)
 
 
@@ -690,7 +696,7 @@ def timegm(tuple):
     return seconds
 
 
-def main(args):
+def main(args=None):
     import argparse
     parser = argparse.ArgumentParser()
     textgroup = parser.add_argument_group('text only arguments')
@@ -736,10 +742,15 @@ def main(args):
         choices=("text", "html"),
         help="output type (text or html)"
     )
+    parser.add_argument(
+        "-f", "--first-weekday",
+        type=int, default=0,
+        help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)"
+    )
     parser.add_argument(
         "year",
         nargs='?', type=int,
-        help="year number (1-9999)"
+        help="year number"
     )
     parser.add_argument(
         "month",
@@ -747,7 +758,7 @@ def main(args):
         help="month number (1-12, text only)"
     )
 
-    options = parser.parse_args(args[1:])
+    options = parser.parse_args(args)
 
     if options.locale and not options.encoding:
         parser.error("if --locale is specified --encoding is required")
@@ -756,10 +767,14 @@ def main(args):
     locale = options.locale, options.encoding
 
     if options.type == "html":
+        if options.month:
+            parser.error("incorrect number of arguments")
+            sys.exit(1)
         if options.locale:
             cal = LocaleHTMLCalendar(locale=locale)
         else:
             cal = HTMLCalendar()
+        cal.setfirstweekday(options.first_weekday)
         encoding = options.encoding
         if encoding is None:
             encoding = sys.getdefaultencoding()
@@ -767,20 +782,20 @@ def main(args):
         write = sys.stdout.buffer.write
         if options.year is None:
             write(cal.formatyearpage(datetime.date.today().year, **optdict))
-        elif options.month is None:
-            write(cal.formatyearpage(options.year, **optdict))
         else:
-            parser.error("incorrect number of arguments")
-            sys.exit(1)
+            write(cal.formatyearpage(options.year, **optdict))
     else:
         if options.locale:
             cal = LocaleTextCalendar(locale=locale)
         else:
             cal = TextCalendar()
+        cal.setfirstweekday(options.first_weekday)
         optdict = dict(w=options.width, l=options.lines)
         if options.month is None:
             optdict["c"] = options.spacing
             optdict["m"] = options.months
+        if options.month is not None:
+            _validate_month(options.month)
         if options.year is None:
             result = cal.formatyear(datetime.date.today().year, **optdict)
         elif options.month is None:
@@ -795,4 +810,4 @@ def main(args):
 
 
 if __name__ == "__main__":
-    main(sys.argv)
+    main()

Lib/cgitb.py

@@ -1,332 +0,0 @@
-"""More comprehensive traceback formatting for Python scripts.
-
-To enable this module, do:
-
-    import cgitb; cgitb.enable()
-
-at the top of your script.  The optional arguments to enable() are:
-
-    display     - if true, tracebacks are displayed in the web browser
-    logdir      - if set, tracebacks are written to files in this directory
-    context     - number of lines of source code to show for each stack frame
-    format      - 'text' or 'html' controls the output format
-
-By default, tracebacks are displayed but not saved, the context is 5 lines
-and the output format is 'html' (for backwards compatibility with the
-original use of this module)
-
-Alternatively, if you have caught an exception and want cgitb to display it
-for you, call cgitb.handler().  The optional argument to handler() is a
-3-item tuple (etype, evalue, etb) just like the value of sys.exc_info().
-The default handler displays output as HTML.
-
-"""
-import inspect
-import keyword
-import linecache
-import os
-import pydoc
-import sys
-import tempfile
-import time
-import tokenize
-import traceback
-import warnings
-from html import escape as html_escape
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-
-def reset():
-    """Return a string that resets the CGI and browser to a known state."""
-    return '''<!--: spam
-Content-Type: text/html
-
-<body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> -->
-<body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> --> -->
-</font> </font> </font> </script> </object> </blockquote> </pre>
-</table> </table> </table> </table> </table> </font> </font> </font>'''
-
-__UNDEF__ = []                          # a special sentinel object
-def small(text):
-    if text:
-        return '<small>' + text + '</small>'
-    else:
-        return ''
-
-def strong(text):
-    if text:
-        return '<strong>' + text + '</strong>'
-    else:
-        return ''
-
-def grey(text):
-    if text:
-        return '<font color="#909090">' + text + '</font>'
-    else:
-        return ''
-
-def lookup(name, frame, locals):
-    """Find the value for a given name in the given environment."""
-    if name in locals:
-        return 'local', locals[name]
-    if name in frame.f_globals:
-        return 'global', frame.f_globals[name]
-    if '__builtins__' in frame.f_globals:
-        builtins = frame.f_globals['__builtins__']
-        if isinstance(builtins, dict):
-            if name in builtins:
-                return 'builtin', builtins[name]
-        else:
-            if hasattr(builtins, name):
-                return 'builtin', getattr(builtins, name)
-    return None, __UNDEF__
-
-def scanvars(reader, frame, locals):
-    """Scan one logical line of Python and look up values of variables used."""
-    vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__
-    for ttype, token, start, end, line in tokenize.generate_tokens(reader):
-        if ttype == tokenize.NEWLINE: break
-        if ttype == tokenize.NAME and token not in keyword.kwlist:
-            if lasttoken == '.':
-                if parent is not __UNDEF__:
-                    value = getattr(parent, token, __UNDEF__)
-                    vars.append((prefix + token, prefix, value))
-            else:
-                where, value = lookup(token, frame, locals)
-                vars.append((token, where, value))
-        elif token == '.':
-            prefix += lasttoken + '.'
-            parent = value
-        else:
-            parent, prefix = None, ''
-        lasttoken = token
-    return vars
-
-def html(einfo, context=5):
-    """Return a nice HTML document describing a given traceback."""
-    etype, evalue, etb = einfo
-    if isinstance(etype, type):
-        etype = etype.__name__
-    pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
-    date = time.ctime(time.time())
-    head = f'''
-<body bgcolor="#f0f0f8">
-<table width="100%" cellspacing=0 cellpadding=2 border=0 summary="heading">
-<tr bgcolor="#6622aa">
-<td valign=bottom>&nbsp;<br>
-<font color="#ffffff" face="helvetica, arial">&nbsp;<br>
-<big><big><strong>{html_escape(str(etype))}</strong></big></big></font></td>
-<td align=right valign=bottom>
-<font color="#ffffff" face="helvetica, arial">{pyver}<br>{date}</font></td>
-</tr></table>
-<p>A problem occurred in a Python script.  Here is the sequence of
-function calls leading up to the error, in the order they occurred.</p>'''
-
-    indent = '<tt>' + small('&nbsp;' * 5) + '&nbsp;</tt>'
-    frames = []
-    records = inspect.getinnerframes(etb, context)
-    for frame, file, lnum, func, lines, index in records:
-        if file:
-            file = os.path.abspath(file)
-            link = '<a href="file://%s">%s</a>' % (file, pydoc.html.escape(file))
-        else:
-            file = link = '?'
-        args, varargs, varkw, locals = inspect.getargvalues(frame)
-        call = ''
-        if func != '?':
-            call = 'in ' + strong(pydoc.html.escape(func))
-            if func != "<module>":
-                call += inspect.formatargvalues(args, varargs, varkw, locals,
-                    formatvalue=lambda value: '=' + pydoc.html.repr(value))
-
-        highlight = {}
-        def reader(lnum=[lnum]):
-            highlight[lnum[0]] = 1
-            try: return linecache.getline(file, lnum[0])
-            finally: lnum[0] += 1
-        vars = scanvars(reader, frame, locals)
-
-        rows = ['<tr><td bgcolor="#d8bbff">%s%s %s</td></tr>' %
-                ('<big>&nbsp;</big>', link, call)]
-        if index is not None:
-            i = lnum - index
-            for line in lines:
-                num = small('&nbsp;' * (5-len(str(i))) + str(i)) + '&nbsp;'
-                if i in highlight:
-                    line = '<tt>=&gt;%s%s</tt>' % (num, pydoc.html.preformat(line))
-                    rows.append('<tr><td bgcolor="#ffccee">%s</td></tr>' % line)
-                else:
-                    line = '<tt>&nbsp;&nbsp;%s%s</tt>' % (num, pydoc.html.preformat(line))
-                    rows.append('<tr><td>%s</td></tr>' % grey(line))
-                i += 1
-
-        done, dump = {}, []
-        for name, where, value in vars:
-            if name in done: continue
-            done[name] = 1
-            if value is not __UNDEF__:
-                if where in ('global', 'builtin'):
-                    name = ('<em>%s</em> ' % where) + strong(name)
-                elif where == 'local':
-                    name = strong(name)
-                else:
-                    name = where + strong(name.split('.')[-1])
-                dump.append('%s&nbsp;= %s' % (name, pydoc.html.repr(value)))
-            else:
-                dump.append(name + ' <em>undefined</em>')
-
-        rows.append('<tr><td>%s</td></tr>' % small(grey(', '.join(dump))))
-        frames.append('''
-<table width="100%%" cellspacing=0 cellpadding=0 border=0>
-%s</table>''' % '\n'.join(rows))
-
-    exception = ['<p>%s: %s' % (strong(pydoc.html.escape(str(etype))),
-                                pydoc.html.escape(str(evalue)))]
-    for name in dir(evalue):
-        if name[:1] == '_': continue
-        value = pydoc.html.repr(getattr(evalue, name))
-        exception.append('\n<br>%s%s&nbsp;=\n%s' % (indent, name, value))
-
-    return head + ''.join(frames) + ''.join(exception) + '''
-
-
-<!-- The above is a description of an error in a Python program, formatted
-     for a web browser because the 'cgitb' module was enabled.  In case you
-     are not reading this in a web browser, here is the original traceback:
-
-%s
--->
-''' % pydoc.html.escape(
-          ''.join(traceback.format_exception(etype, evalue, etb)))
-
-def text(einfo, context=5):
-    """Return a plain text document describing a given traceback."""
-    etype, evalue, etb = einfo
-    if isinstance(etype, type):
-        etype = etype.__name__
-    pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
-    date = time.ctime(time.time())
-    head = "%s\n%s\n%s\n" % (str(etype), pyver, date) + '''
-A problem occurred in a Python script.  Here is the sequence of
-function calls leading up to the error, in the order they occurred.
-'''
-
-    frames = []
-    records = inspect.getinnerframes(etb, context)
-    for frame, file, lnum, func, lines, index in records:
-        file = file and os.path.abspath(file) or '?'
-        args, varargs, varkw, locals = inspect.getargvalues(frame)
-        call = ''
-        if func != '?':
-            call = 'in ' + func
-            if func != "<module>":
-                call += inspect.formatargvalues(args, varargs, varkw, locals,
-                    formatvalue=lambda value: '=' + pydoc.text.repr(value))
-
-        highlight = {}
-        def reader(lnum=[lnum]):
-            highlight[lnum[0]] = 1
-            try: return linecache.getline(file, lnum[0])
-            finally: lnum[0] += 1
-        vars = scanvars(reader, frame, locals)
-
-        rows = [' %s %s' % (file, call)]
-        if index is not None:
-            i = lnum - index
-            for line in lines:
-                num = '%5d ' % i
-                rows.append(num+line.rstrip())
-                i += 1
-
-        done, dump = {}, []
-        for name, where, value in vars:
-            if name in done: continue
-            done[name] = 1
-            if value is not __UNDEF__:
-                if where == 'global': name = 'global ' + name
-                elif where != 'local': name = where + name.split('.')[-1]
-                dump.append('%s = %s' % (name, pydoc.text.repr(value)))
-            else:
-                dump.append(name + ' undefined')
-
-        rows.append('\n'.join(dump))
-        frames.append('\n%s\n' % '\n'.join(rows))
-
-    exception = ['%s: %s' % (str(etype), str(evalue))]
-    for name in dir(evalue):
-        value = pydoc.text.repr(getattr(evalue, name))
-        exception.append('\n%s%s = %s' % (" "*4, name, value))
-
-    return head + ''.join(frames) + ''.join(exception) + '''
-
-The above is a description of an error in a Python program.  Here is
-the original traceback:
-
-%s
-''' % ''.join(traceback.format_exception(etype, evalue, etb))
-
-class Hook:
-    """A hook to replace sys.excepthook that shows tracebacks in HTML."""
-
-    def __init__(self, display=1, logdir=None, context=5, file=None,
-                 format="html"):
-        self.display = display          # send tracebacks to browser if true
-        self.logdir = logdir            # log tracebacks to files if not None
-        self.context = context          # number of source code lines per frame
-        self.file = file or sys.stdout  # place to send the output
-        self.format = format
-
-    def __call__(self, etype, evalue, etb):
-        self.handle((etype, evalue, etb))
-
-    def handle(self, info=None):
-        info = info or sys.exc_info()
-        if self.format == "html":
-            self.file.write(reset())
-
-        formatter = (self.format=="html") and html or text
-        plain = False
-        try:
-            doc = formatter(info, self.context)
-        except:                         # just in case something goes wrong
-            doc = ''.join(traceback.format_exception(*info))
-            plain = True
-
-        if self.display:
-            if plain:
-                doc = pydoc.html.escape(doc)
-                self.file.write('<pre>' + doc + '</pre>\n')
-            else:
-                self.file.write(doc + '\n')
-        else:
-            self.file.write('<p>A problem occurred in a Python script.\n')
-
-        if self.logdir is not None:
-            suffix = ['.txt', '.html'][self.format=="html"]
-            (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir)
-
-            try:
-                with os.fdopen(fd, 'w') as file:
-                    file.write(doc)
-                msg = '%s contains the description of this error.' % path
-            except:
-                msg = 'Tried to save traceback to %s, but failed.' % path
-
-            if self.format == 'html':
-                self.file.write('<p>%s</p>\n' % msg)
-            else:
-                self.file.write(msg + '\n')
-        try:
-            self.file.flush()
-        except: pass
-
-handler = Hook().handle
-def enable(display=1, logdir=None, context=5, format="html"):
-    """Install an exception handler that formats tracebacks as HTML.
-
-    The optional argument 'display' can be set to 0 to suppress sending the
-    traceback to the browser, and 'logdir' can be set to a directory to cause
-    tracebacks to be written to files there."""
-    sys.excepthook = Hook(display=display, logdir=logdir,
-                          context=context, format=format)

Lib/chunk.py

@@ -1,173 +0,0 @@
-"""Simple class to read IFF chunks.
-
-An IFF chunk (used in formats such as AIFF, TIFF, RMFF (RealMedia File
-Format)) has the following structure:
-
-+----------------+
-| ID (4 bytes)   |
-+----------------+
-| size (4 bytes) |
-+----------------+
-| data           |
-| ...            |
-+----------------+
-
-The ID is a 4-byte string which identifies the type of chunk.
-
-The size field (a 32-bit value, encoded using big-endian byte order)
-gives the size of the whole chunk, including the 8-byte header.
-
-Usually an IFF-type file consists of one or more chunks.  The proposed
-usage of the Chunk class defined here is to instantiate an instance at
-the start of each chunk and read from the instance until it reaches
-the end, after which a new instance can be instantiated.  At the end
-of the file, creating a new instance will fail with an EOFError
-exception.
-
-Usage:
-while True:
-    try:
-        chunk = Chunk(file)
-    except EOFError:
-        break
-    chunktype = chunk.getname()
-    while True:
-        data = chunk.read(nbytes)
-        if not data:
-            pass
-        # do something with data
-
-The interface is file-like.  The implemented methods are:
-read, close, seek, tell, isatty.
-Extra methods are: skip() (called by close, skips to the end of the chunk),
-getname() (returns the name (ID) of the chunk)
-
-The __init__ method has one required argument, a file-like object
-(including a chunk instance), and one optional argument, a flag which
-specifies whether or not chunks are aligned on 2-byte boundaries.  The
-default is 1, i.e. aligned.
-"""
-
-import warnings
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-class Chunk:
-    def __init__(self, file, align=True, bigendian=True, inclheader=False):
-        import struct
-        self.closed = False
-        self.align = align      # whether to align to word (2-byte) boundaries
-        if bigendian:
-            strflag = '>'
-        else:
-            strflag = '<'
-        self.file = file
-        self.chunkname = file.read(4)
-        if len(self.chunkname) < 4:
-            raise EOFError
-        try:
-            self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0]
-        except struct.error:
-            raise EOFError from None
-        if inclheader:
-            self.chunksize = self.chunksize - 8 # subtract header
-        self.size_read = 0
-        try:
-            self.offset = self.file.tell()
-        except (AttributeError, OSError):
-            self.seekable = False
-        else:
-            self.seekable = True
-
-    def getname(self):
-        """Return the name (ID) of the current chunk."""
-        return self.chunkname
-
-    def getsize(self):
-        """Return the size of the current chunk."""
-        return self.chunksize
-
-    def close(self):
-        if not self.closed:
-            try:
-                self.skip()
-            finally:
-                self.closed = True
-
-    def isatty(self):
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        return False
-
-    def seek(self, pos, whence=0):
-        """Seek to specified position into the chunk.
-        Default position is 0 (start of chunk).
-        If the file is not seekable, this will result in an error.
-        """
-
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        if not self.seekable:
-            raise OSError("cannot seek")
-        if whence == 1:
-            pos = pos + self.size_read
-        elif whence == 2:
-            pos = pos + self.chunksize
-        if pos < 0 or pos > self.chunksize:
-            raise RuntimeError
-        self.file.seek(self.offset + pos, 0)
-        self.size_read = pos
-
-    def tell(self):
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        return self.size_read
-
-    def read(self, size=-1):
-        """Read at most size bytes from the chunk.
-        If size is omitted or negative, read until the end
-        of the chunk.
-        """
-
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        if self.size_read >= self.chunksize:
-            return b''
-        if size < 0:
-            size = self.chunksize - self.size_read
-        if size > self.chunksize - self.size_read:
-            size = self.chunksize - self.size_read
-        data = self.file.read(size)
-        self.size_read = self.size_read + len(data)
-        if self.size_read == self.chunksize and \
-           self.align and \
-           (self.chunksize & 1):
-            dummy = self.file.read(1)
-            self.size_read = self.size_read + len(dummy)
-        return data
-
-    def skip(self):
-        """Skip the rest of the chunk.
-        If you are not interested in the contents of the chunk,
-        this method should be called so that the file points to
-        the start of the next chunk.
-        """
-
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        if self.seekable:
-            try:
-                n = self.chunksize - self.size_read
-                # maybe fix alignment
-                if self.align and (self.chunksize & 1):
-                    n = n + 1
-                self.file.seek(n, 1)
-                self.size_read = self.size_read + n
-                return
-            except OSError:
-                pass
-        while self.size_read < self.chunksize:
-            n = min(8192, self.chunksize - self.size_read)
-            dummy = self.read(n)
-            if not dummy:
-                raise EOFError

Lib/colorsys.py

@@ -24,7 +24,7 @@ HSV: Hue, Saturation, Value
 __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb",
            "rgb_to_hsv","hsv_to_rgb"]
 
-# Some floating point constants
+# Some floating-point constants
 
 ONE_THIRD = 1.0/3.0
 ONE_SIXTH = 1.0/6.0

Lib/contextlib.py

@@ -145,14 +145,17 @@ class _GeneratorContextManager(
             except StopIteration:
                 return False
             else:
-                raise RuntimeError("generator didn't stop")
+                try:
+                    raise RuntimeError("generator didn't stop")
+                finally:
+                    self.gen.close()
         else:
             if value is None:
                 # Need to force instantiation so we can reliably
                 # tell if we get the same exception back
                 value = typ()
             try:
-                self.gen.throw(typ, value, traceback)
+                self.gen.throw(value)
             except StopIteration as exc:
                 # Suppress StopIteration *unless* it's the same exception that
                 # was passed to throw().  This prevents a StopIteration
@@ -187,7 +190,10 @@ class _GeneratorContextManager(
                     raise
                 exc.__traceback__ = traceback
                 return False
-            raise RuntimeError("generator didn't stop after throw()")
+            try:
+                raise RuntimeError("generator didn't stop after throw()")
+            finally:
+                self.gen.close()
 
 class _AsyncGeneratorContextManager(
     _GeneratorContextManagerBase,
@@ -212,14 +218,17 @@ class _AsyncGeneratorContextManager(
             except StopAsyncIteration:
                 return False
             else:
-                raise RuntimeError("generator didn't stop")
+                try:
+                    raise RuntimeError("generator didn't stop")
+                finally:
+                    await self.gen.aclose()
         else:
             if value is None:
                 # Need to force instantiation so we can reliably
                 # tell if we get the same exception back
                 value = typ()
             try:
-                await self.gen.athrow(typ, value, traceback)
+                await self.gen.athrow(value)
             except StopAsyncIteration as exc:
                 # Suppress StopIteration *unless* it's the same exception that
                 # was passed to throw().  This prevents a StopIteration
@@ -254,7 +263,10 @@ class _AsyncGeneratorContextManager(
                     raise
                 exc.__traceback__ = traceback
                 return False
-            raise RuntimeError("generator didn't stop after athrow()")
+            try:
+                raise RuntimeError("generator didn't stop after athrow()")
+            finally:
+                await self.gen.aclose()
 
 
 def contextmanager(func):
@@ -441,7 +453,16 @@ class suppress(AbstractContextManager):
         # exactly reproduce the limitations of the CPython interpreter.
         #
         # See http://bugs.python.org/issue12029 for more details
-        return exctype is not None and issubclass(exctype, self._exceptions)
+        if exctype is None:
+            return
+        if issubclass(exctype, self._exceptions):
+            return True
+        if issubclass(exctype, BaseExceptionGroup):
+            match, rest = excinst.split(self._exceptions)
+            if rest is None:
+                return True
+            raise rest
+        return False
 
 
 class _BaseExitStack:

Lib/ctypes/__init__.py

@@ -36,6 +36,9 @@ from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \
      FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \
      FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR
 
+# TODO: RUSTPYTHON remove this
+from _ctypes import _non_existing_function
+
 # WINOLEAPI -> HRESULT
 # WINOLEAPI_(type)
 #
@@ -296,7 +299,9 @@ def create_unicode_buffer(init, size=None):
         return buf
     elif isinstance(init, int):
         _sys.audit("ctypes.create_unicode_buffer", None, init)
-        buftype = c_wchar * init
+        # XXX: RUSTPYTHON
+        # buftype = c_wchar * init
+        buftype = c_wchar.__mul__(init)
         buf = buftype()
         return buf
     raise TypeError(init)
@@ -495,14 +500,15 @@ elif sizeof(c_ulonglong) == sizeof(c_void_p):
     c_ssize_t = c_longlong
 
 # functions
-
 from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr
 
 ## void *memmove(void *, const void *, size_t);
-memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr)
+# XXX: RUSTPYTHON
+# memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr)
 
 ## void *memset(void *, int, size_t)
-memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr)
+# XXX: RUSTPYTHON
+# memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr)
 
 def PYFUNCTYPE(restype, *argtypes):
     class CFunctionType(_CFuncPtr):
@@ -511,11 +517,13 @@ def PYFUNCTYPE(restype, *argtypes):
         _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
     return CFunctionType
 
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
+# XXX: RUSTPYTHON
+# _cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
     return _cast(obj, obj, typ)
 
-_string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
+# XXX: RUSTPYTHON
+# _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=-1):
     """string_at(addr[, size]) -> string
 
@@ -527,7 +535,8 @@ try:
 except ImportError:
     pass
 else:
-    _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr)
+    # XXX: RUSTPYTHON
+    # _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr)
     def wstring_at(ptr, size=-1):
         """wstring_at(addr[, size]) -> string
 

Lib/ctypes/test/test_numbers.py

@@ -82,14 +82,6 @@ class NumberTestCase(unittest.TestCase):
             self.assertRaises(TypeError, t, "")
             self.assertRaises(TypeError, t, None)
 
-    @unittest.skip('test disabled')
-    def test_valid_ranges(self):
-        # invalid values of the correct type
-        # raise ValueError (not OverflowError)
-        for t, (l, h) in zip(unsigned_types, unsigned_ranges):
-            self.assertRaises(ValueError, t, l-1)
-            self.assertRaises(ValueError, t, h+1)
-
     def test_from_param(self):
         # the from_param class method attribute always
         # returns PyCArgObject instances
@@ -106,7 +98,7 @@ class NumberTestCase(unittest.TestCase):
     def test_floats(self):
         # c_float and c_double can be created from
         # Python int and float
-        class FloatLike(object):
+        class FloatLike:
             def __float__(self):
                 return 2.0
         f = FloatLike()
@@ -117,15 +109,15 @@ class NumberTestCase(unittest.TestCase):
             self.assertEqual(t(f).value, 2.0)
 
     def test_integers(self):
-        class FloatLike(object):
+        class FloatLike:
             def __float__(self):
                 return 2.0
         f = FloatLike()
-        class IntLike(object):
+        class IntLike:
             def __int__(self):
                 return 2
         d = IntLike()
-        class IndexLike(object):
+        class IndexLike:
             def __index__(self):
                 return 2
         i = IndexLike()
@@ -155,10 +147,10 @@ class NumberTestCase(unittest.TestCase):
 
             # alignment of the type...
             self.assertEqual((code, alignment(t)),
-                                 (code, align))
+                             (code, align))
             # and alignment of an instance
             self.assertEqual((code, alignment(t())),
-                                 (code, align))
+                             (code, align))
 
     def test_int_from_address(self):
         from array import array
@@ -205,19 +197,6 @@ class NumberTestCase(unittest.TestCase):
         a[0] = ord('?')
         self.assertEqual(v.value, b'?')
 
-    # array does not support c_bool / 't'
-    @unittest.skip('test disabled')
-    def test_bool_from_address(self):
-        from ctypes import c_bool
-        from array import array
-        a = array(c_bool._type_, [True])
-        v = t.from_address(a.buffer_info()[0])
-        self.assertEqual(v.value, a[0])
-        self.assertEqual(type(v) is t)
-        a[0] = False
-        self.assertEqual(v.value, a[0])
-        self.assertEqual(type(v) is t)
-
     def test_init(self):
         # c_int() can be initialized from Python's int, and c_int.
         # Not from c_long or so, which seems strange, abc should
@@ -234,62 +213,6 @@ class NumberTestCase(unittest.TestCase):
             if (hasattr(t, "__ctype_le__")):
                 self.assertRaises(OverflowError, t.__ctype_le__, big_int)
 
-    @unittest.skip('test disabled')
-    def test_perf(self):
-        check_perf()
-
-from ctypes import _SimpleCData
-class c_int_S(_SimpleCData):
-    _type_ = "i"
-    __slots__ = []
-
-def run_test(rep, msg, func, arg=None):
-##    items = [None] * rep
-    items = range(rep)
-    from time import perf_counter as clock
-    if arg is not None:
-        start = clock()
-        for i in items:
-            func(arg); func(arg); func(arg); func(arg); func(arg)
-        stop = clock()
-    else:
-        start = clock()
-        for i in items:
-            func(); func(); func(); func(); func()
-        stop = clock()
-    print("%15s: %.2f us" % (msg, ((stop-start)*1e6/5/rep)))
-
-def check_perf():
-    # Construct 5 objects
-    from ctypes import c_int
-
-    REP = 200000
-
-    run_test(REP, "int()", int)
-    run_test(REP, "int(999)", int)
-    run_test(REP, "c_int()", c_int)
-    run_test(REP, "c_int(999)", c_int)
-    run_test(REP, "c_int_S()", c_int_S)
-    run_test(REP, "c_int_S(999)", c_int_S)
-
-# Python 2.3 -OO, win2k, P4 700 MHz:
-#
-#          int(): 0.87 us
-#       int(999): 0.87 us
-#        c_int(): 3.35 us
-#     c_int(999): 3.34 us
-#      c_int_S(): 3.23 us
-#   c_int_S(999): 3.24 us
-
-# Python 2.2 -OO, win2k, P4 700 MHz:
-#
-#          int(): 0.89 us
-#       int(999): 0.89 us
-#        c_int(): 9.99 us
-#     c_int(999): 10.02 us
-#      c_int_S(): 9.87 us
-#   c_int_S(999): 9.85 us
 
 if __name__ == '__main__':
-##    check_perf()
     unittest.main()

Lib/difflib.py

@@ -1200,25 +1200,6 @@ def context_diff(a, b, fromfile='', tofile='',
     strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'.
     The modification times are normally expressed in the ISO 8601 format.
     If not specified, the strings default to blanks.
-
-    Example:
-
-    >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True),
-    ...       'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')),
-    ...       end="")
-    *** Original
-    --- Current
-    ***************
-    *** 1,4 ****
-      one
-    ! two
-    ! three
-      four
-    --- 1,4 ----
-    + zero
-      one
-    ! tree
-      four
     """
 
     _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm)

Lib/doctest.py

@@ -102,7 +102,7 @@ import re
 import sys
 import traceback
 import unittest
-from io import StringIO  # XXX: RUSTPYTHON; , IncrementalNewlineDecoder
+from io import StringIO, IncrementalNewlineDecoder
 from collections import namedtuple
 
 TestResults = namedtuple('TestResults', 'failed attempted')
@@ -230,9 +230,7 @@ def _load_testfile(filename, package, module_relative, encoding):
             # get_data() opens files as 'rb', so one must do the equivalent
             # conversion as universal newlines would do.
 
-            # TODO: RUSTPYTHON; use _newline_convert once io.IncrementalNewlineDecoder is implemented
-            return file_contents.replace(os.linesep, '\n'), filename
-            # return _newline_convert(file_contents), filename
+            return _newline_convert(file_contents), filename
     with open(filename, encoding=encoding) as f:
         return f.read(), filename
 

Lib/email/__init__.py

@@ -25,7 +25,6 @@ __all__ = [
     ]
 
 
-
 # Some convenience routines.  Don't import Parser and Message as side-effects
 # of importing email since those cascadingly import most of the rest of the
 # email package.

Lib/email/_encoded_words.py

@@ -62,7 +62,7 @@ __all__ = ['decode_q',
 
 # regex based decoder.
 _q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub,
-        lambda m: bytes([int(m.group(1), 16)]))
+        lambda m: bytes.fromhex(m.group(1).decode()))
 
 def decode_q(encoded):
     encoded = encoded.replace(b'_', b' ')
@@ -98,30 +98,42 @@ def len_q(bstring):
 #
 
 def decode_b(encoded):
-    defects = []
+    # First try encoding with validate=True, fixing the padding if needed.
+    # This will succeed only if encoded includes no invalid characters.
     pad_err = len(encoded) % 4
-    if pad_err:
-        defects.append(errors.InvalidBase64PaddingDefect())
-        padded_encoded = encoded + b'==='[:4-pad_err]
-    else:
-        padded_encoded = encoded
+    missing_padding = b'==='[:4-pad_err] if pad_err else b''
     try:
-        return base64.b64decode(padded_encoded, validate=True), defects
+        return (
+            base64.b64decode(encoded + missing_padding, validate=True),
+            [errors.InvalidBase64PaddingDefect()] if pad_err else [],
+        )
     except binascii.Error:
-        # Since we had correct padding, this must an invalid char error.
-        defects = [errors.InvalidBase64CharactersDefect()]
+        # Since we had correct padding, this is likely an invalid char error.
+        #
         # The non-alphabet characters are ignored as far as padding
-        # goes, but we don't know how many there are.  So we'll just
-        # try various padding lengths until something works.
-        for i in 0, 1, 2, 3:
+        # goes, but we don't know how many there are.  So try without adding
+        # padding to see if it works.
+        try:
+            return (
+                base64.b64decode(encoded, validate=False),
+                [errors.InvalidBase64CharactersDefect()],
+            )
+        except binascii.Error:
+            # Add as much padding as could possibly be necessary (extra padding
+            # is ignored).
             try:
-                return base64.b64decode(encoded+b'='*i, validate=False), defects
+                return (
+                    base64.b64decode(encoded + b'==', validate=False),
+                    [errors.InvalidBase64CharactersDefect(),
+                     errors.InvalidBase64PaddingDefect()],
+                )
             except binascii.Error:
-                if i==0:
-                    defects.append(errors.InvalidBase64PaddingDefect())
-        else:
-            # This should never happen.
-            raise AssertionError("unexpected binascii.Error")
+                # This only happens when the encoded string's length is 1 more
+                # than a multiple of 4, which is invalid.
+                #
+                # bpo-27397: Just return the encoded string since there's no
+                # way to decode.
+                return encoded, [errors.InvalidBase64LengthDefect()]
 
 def encode_b(bstring):
     return base64.b64encode(bstring).decode('ascii')
@@ -167,15 +179,15 @@ def decode(ew):
     # Turn the CTE decoded bytes into unicode.
     try:
         string = bstring.decode(charset)
-    except UnicodeError:
+    except UnicodeDecodeError:
         defects.append(errors.UndecodableBytesDefect("Encoded word "
-            "contains bytes not decodable using {} charset".format(charset)))
+            f"contains bytes not decodable using {charset!r} charset"))
         string = bstring.decode(charset, 'surrogateescape')
-    except LookupError:
+    except (LookupError, UnicodeEncodeError):
         string = bstring.decode('ascii', 'surrogateescape')
         if charset.lower() != 'unknown-8bit':
-            defects.append(errors.CharsetError("Unknown charset {} "
-                "in encoded word; decoded as unknown bytes".format(charset)))
+            defects.append(errors.CharsetError(f"Unknown charset {charset!r} "
+                f"in encoded word; decoded as unknown bytes"))
     return string, charset, lang, defects
 
 

Lib/email/_parseaddr.py

@@ -13,7 +13,7 @@ __all__ = [
     'quote',
     ]
 
-import time, calendar
+import time
 
 SPACE = ' '
 EMPTYSTRING = ''
@@ -65,8 +65,10 @@ def _parsedate_tz(data):
 
     """
     if not data:
-        return
+        return None
     data = data.split()
+    if not data:  # This happens for whitespace-only input.
+        return None
     # The FWS after the comma after the day-of-week is optional, so search and
     # adjust for this.
     if data[0].endswith(',') or data[0].lower() in _daynames:
@@ -93,6 +95,8 @@ def _parsedate_tz(data):
         return None
     data = data[:5]
     [dd, mm, yy, tm, tz] = data
+    if not (dd and mm and yy):
+        return None
     mm = mm.lower()
     if mm not in _monthnames:
         dd, mm = mm, dd.lower()
@@ -108,6 +112,8 @@ def _parsedate_tz(data):
         yy, tm = tm, yy
     if yy[-1] == ',':
         yy = yy[:-1]
+        if not yy:
+            return None
     if not yy[0].isdigit():
         yy, tz = tz, yy
     if tm[-1] == ',':
@@ -126,6 +132,8 @@ def _parsedate_tz(data):
             tss = 0
         elif len(tm) == 3:
             [thh, tmm, tss] = tm
+        else:
+            return None
     else:
         return None
     try:
@@ -186,6 +194,9 @@ def mktime_tz(data):
         # No zone info, so localtime is better assumption than GMT
         return time.mktime(data[:8] + (-1,))
     else:
+        # Delay the import, since mktime_tz is rarely used
+        import calendar
+
         t = calendar.timegm(data)
         return t - data[9]
 
@@ -379,7 +390,12 @@ class AddrlistClass:
         aslist.append('@')
         self.pos += 1
         self.gotonext()
-        return EMPTYSTRING.join(aslist) + self.getdomain()
+        domain = self.getdomain()
+        if not domain:
+            # Invalid domain, return an empty address instead of returning a
+            # local part to denote failed parsing.
+            return EMPTYSTRING
+        return EMPTYSTRING.join(aslist) + domain
 
     def getdomain(self):
         """Get the complete domain name from an address."""
@@ -394,6 +410,10 @@ class AddrlistClass:
             elif self.field[self.pos] == '.':
                 self.pos += 1
                 sdlist.append('.')
+            elif self.field[self.pos] == '@':
+                # bpo-34155: Don't parse domains with two `@` like
+                # `a@malicious.org@important.com`.
+                return EMPTYSTRING
             elif self.field[self.pos] in self.atomends:
                 break
             else:

Lib/email/_policybase.py

@@ -152,11 +152,18 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
     mangle_from_        -- a flag that, when True escapes From_ lines in the
                            body of the message by putting a `>' in front of
                            them. This is used when the message is being
-                           serialized by a generator. Default: True.
+                           serialized by a generator. Default: False.
 
     message_factory     -- the class to use to create new message objects.
                            If the value is None, the default is Message.
 
+    verify_generated_headers
+                        -- if true, the generator verifies that each header
+                           they are properly folded, so that a parser won't
+                           treat it as multiple headers, start-of-body, or
+                           part of another header.
+                           This is a check against custom Header & fold()
+                           implementations.
     """
 
     raise_on_defect = False
@@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
     max_line_length = 78
     mangle_from_ = False
     message_factory = None
+    verify_generated_headers = True
 
     def handle_defect(self, obj, defect):
         """Based on policy, either raise defect or call register_defect.
@@ -294,12 +302,12 @@ class Compat32(Policy):
         """+
         The name is parsed as everything up to the ':' and returned unmodified.
         The value is determined by stripping leading whitespace off the
-        remainder of the first line, joining all subsequent lines together, and
+        remainder of the first line joined with all subsequent lines, and
         stripping any trailing carriage return or linefeed characters.
 
         """
         name, value = sourcelines[0].split(':', 1)
-        value = value.lstrip(' \t') + ''.join(sourcelines[1:])
+        value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n')
         return (name, value.rstrip('\r\n'))
 
     def header_store_parse(self, name, value):
@@ -361,8 +369,12 @@ class Compat32(Policy):
             # Assume it is a Header-like object.
             h = value
         if h is not None:
-            parts.append(h.encode(linesep=self.linesep,
-                                  maxlinelen=self.max_line_length))
+            # The Header class interprets a value of None for maxlinelen as the
+            # default value of 78, as recommended by RFC 2822.
+            maxlinelen = 0
+            if self.max_line_length is not None:
+                maxlinelen = self.max_line_length
+            parts.append(h.encode(linesep=self.linesep, maxlinelen=maxlinelen))
         parts.append(self.linesep)
         return ''.join(parts)
 

Lib/email/architecture.rst

@@ -66,7 +66,7 @@ data payloads.
 Message Lifecycle
 -----------------
 
-The general lifecyle of a message is:
+The general lifecycle of a message is:
 
     Creation
         A `Message` object can be created by a Parser, or it can be

Lib/email/base64mime.py

@@ -45,7 +45,6 @@ EMPTYSTRING = ''
 MISC_LEN = 7
 
 
-
 # Helpers
 def header_length(bytearray):
     """Return the length of s when it is encoded with base64."""
@@ -57,7 +56,6 @@ def header_length(bytearray):
     return n
 
 
-
 def header_encode(header_bytes, charset='iso-8859-1'):
     """Encode a single header line with Base64 encoding in a given charset.
 
@@ -72,7 +70,6 @@ def header_encode(header_bytes, charset='iso-8859-1'):
     return '=?%s?b?%s?=' % (charset, encoded)
 
 
-
 def body_encode(s, maxlinelen=76, eol=NL):
     r"""Encode a string with base64.
 
@@ -84,7 +81,7 @@ def body_encode(s, maxlinelen=76, eol=NL):
     in an email.
     """
     if not s:
-        return s
+        return ""
 
     encvec = []
     max_unencoded = maxlinelen * 3 // 4
@@ -98,7 +95,6 @@ def body_encode(s, maxlinelen=76, eol=NL):
     return EMPTYSTRING.join(encvec)
 
 
-
 def decode(string):
     """Decode a raw base64 string, returning a bytes object.
 

Lib/email/charset.py

@@ -18,7 +18,6 @@ from email import errors
 from email.encoders import encode_7or8bit
 
 
-
 # Flags for types of header encodings
 QP          = 1 # Quoted-Printable
 BASE64      = 2 # Base64
@@ -32,7 +31,6 @@ UNKNOWN8BIT = 'unknown-8bit'
 EMPTYSTRING = ''
 
 
-
 # Defaults
 CHARSETS = {
     # input        header enc  body enc output conv
@@ -104,7 +102,6 @@ CODEC_MAP = {
     }
 
 
-
 # Convenience functions for extending the above mappings
 def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
     """Add character set properties to the global registry.
@@ -112,8 +109,8 @@ def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
     charset is the input character set, and must be the canonical name of a
     character set.
 
-    Optional header_enc and body_enc is either Charset.QP for
-    quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
+    Optional header_enc and body_enc is either charset.QP for
+    quoted-printable, charset.BASE64 for base64 encoding, charset.SHORTEST for
     the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST
     is only valid for header_enc.  It describes how message headers and
     message bodies in the input charset are to be encoded.  Default is no
@@ -153,7 +150,6 @@ def add_codec(charset, codecname):
     CODEC_MAP[charset] = codecname
 
 
-
 # Convenience function for encoding strings, taking into account
 # that they might be unknown-8bit (ie: have surrogate-escaped bytes)
 def _encode(string, codec):
@@ -163,7 +159,6 @@ def _encode(string, codec):
         return string.encode(codec)
 
 
-
 class Charset:
     """Map character sets to their email properties.
 
@@ -185,13 +180,13 @@ class Charset:
 
     header_encoding: If the character set must be encoded before it can be
                      used in an email header, this attribute will be set to
-                     Charset.QP (for quoted-printable), Charset.BASE64 (for
-                     base64 encoding), or Charset.SHORTEST for the shortest of
+                     charset.QP (for quoted-printable), charset.BASE64 (for
+                     base64 encoding), or charset.SHORTEST for the shortest of
                      QP or BASE64 encoding.  Otherwise, it will be None.
 
     body_encoding: Same as header_encoding, but describes the encoding for the
                    mail message's body, which indeed may be different than the
-                   header encoding.  Charset.SHORTEST is not allowed for
+                   header encoding.  charset.SHORTEST is not allowed for
                    body_encoding.
 
     output_charset: Some character sets must be converted before they can be
@@ -241,11 +236,9 @@ class Charset:
         self.output_codec = CODEC_MAP.get(self.output_charset,
                                           self.output_charset)
 
-    def __str__(self):
+    def __repr__(self):
         return self.input_charset.lower()
 
-    __repr__ = __str__
-
     def __eq__(self, other):
         return str(self) == str(other).lower()
 
@@ -348,7 +341,6 @@ class Charset:
                 if not lines and not current_line:
                     lines.append(None)
                 else:
-                    separator = (' ' if lines else '')
                     joined_line = EMPTYSTRING.join(current_line)
                     header_bytes = _encode(joined_line, codec)
                     lines.append(encoder(header_bytes))

Lib/email/contentmanager.py

@@ -72,12 +72,14 @@ def get_non_text_content(msg):
     return msg.get_payload(decode=True)
 for maintype in 'audio image video application'.split():
     raw_data_manager.add_get_handler(maintype, get_non_text_content)
+del maintype
 
 
 def get_message_content(msg):
     return msg.get_payload(0)
 for subtype in 'rfc822 external-body'.split():
     raw_data_manager.add_get_handler('message/'+subtype, get_message_content)
+del subtype
 
 
 def get_and_fixup_unknown_message_content(msg):
@@ -144,15 +146,15 @@ def _encode_text(string, charset, cte, policy):
     linesep = policy.linesep.encode('ascii')
     def embedded_body(lines): return linesep.join(lines) + linesep
     def normal_body(lines): return b'\n'.join(lines) + b'\n'
-    if cte==None:
+    if cte is None:
         # Use heuristics to decide on the "best" encoding.
-        try:
-            return '7bit', normal_body(lines).decode('ascii')
-        except UnicodeDecodeError:
-            pass
-        if (policy.cte_type == '8bit' and
-                max(len(x) for x in lines) <= policy.max_line_length):
-            return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
+        if max((len(x) for x in lines), default=0) <= policy.max_line_length:
+            try:
+                return '7bit', normal_body(lines).decode('ascii')
+            except UnicodeDecodeError:
+                pass
+            if policy.cte_type == '8bit':
+                return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
         sniff = embedded_body(lines[:10])
         sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'),
                                           policy.max_line_length)
@@ -238,9 +240,7 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64',
         data = binascii.b2a_qp(data, istext=False, header=False, quotetabs=True)
         data = data.decode('ascii')
     elif cte == '7bit':
-        # Make sure it really is only ASCII.  The early warning here seems
-        # worth the overhead...if you care write your own content manager :).
-        data.encode('ascii')
+        data = data.decode('ascii')
     elif cte in ('8bit', 'binary'):
         data = data.decode('ascii', 'surrogateescape')
     msg.set_payload(data)
@@ -248,3 +248,4 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64',
     _finalize_set(msg, disposition, filename, cid, params)
 for typ in (bytes, bytearray, memoryview):
     raw_data_manager.add_set_handler(typ, set_bytes_content)
+del typ

Lib/email/encoders.py

@@ -16,7 +16,6 @@ from base64 import encodebytes as _bencode
 from quopri import encodestring as _encodestring
 
 
-
 def _qencode(s):
     enc = _encodestring(s, quotetabs=True)
     # Must encode spaces, which quopri.encodestring() doesn't do
@@ -34,7 +33,6 @@ def encode_base64(msg):
     msg['Content-Transfer-Encoding'] = 'base64'
 
 
-
 def encode_quopri(msg):
     """Encode the message's payload in quoted-printable.
 
@@ -46,7 +44,6 @@ def encode_quopri(msg):
     msg['Content-Transfer-Encoding'] = 'quoted-printable'
 
 
-
 def encode_7or8bit(msg):
     """Set the Content-Transfer-Encoding header to 7bit or 8bit."""
     orig = msg.get_payload(decode=True)
@@ -64,6 +61,5 @@ def encode_7or8bit(msg):
         msg['Content-Transfer-Encoding'] = '7bit'
 
 
-
 def encode_noop(msg):
     """Do nothing."""

Lib/email/errors.py

@@ -29,6 +29,10 @@ class CharsetError(MessageError):
     """An illegal charset was given."""
 
 
+class HeaderWriteError(MessageError):
+    """Error while writing headers."""
+
+
 # These are parsing defects which the parser was able to work around.
 class MessageDefect(ValueError):
     """Base class for a message defect."""
@@ -73,6 +77,9 @@ class InvalidBase64PaddingDefect(MessageDefect):
 class InvalidBase64CharactersDefect(MessageDefect):
     """base64 encoded sequence had characters not in base64 alphabet"""
 
+class InvalidBase64LengthDefect(MessageDefect):
+    """base64 encoded sequence had invalid length (1 mod 4)"""
+
 # These errors are specific to header parsing.
 
 class HeaderDefect(MessageDefect):
@@ -105,3 +112,6 @@ class NonASCIILocalPartDefect(HeaderDefect):
     """local_part contains non-ASCII characters"""
     # This defect only occurs during unicode parsing, not when
     # parsing messages decoded from binary.
+
+class InvalidDateDefect(HeaderDefect):
+    """Header has unparsable or invalid date"""

Lib/email/feedparser.py

@@ -37,11 +37,12 @@ NLCRE_crack = re.compile(r'(\r\n|\r|\n)')
 headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])')
 EMPTYSTRING = ''
 NL = '\n'
+boundaryendRE = re.compile(
+    r'(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
 
 NeedMoreData = object()
 
 
-
 class BufferedSubFile(object):
     """A file-ish object that can have new data loaded into it.
 
@@ -132,7 +133,6 @@ class BufferedSubFile(object):
         return line
 
 
-
 class FeedParser:
     """A feed-style parser of email."""
 
@@ -189,7 +189,7 @@ class FeedParser:
         assert not self._msgstack
         # Look for final set of defects
         if root.get_content_maintype() == 'multipart' \
-               and not root.is_multipart():
+               and not root.is_multipart() and not self._headersonly:
             defect = errors.MultipartInvariantViolationDefect()
             self.policy.handle_defect(root, defect)
         return root
@@ -266,7 +266,7 @@ class FeedParser:
                         yield NeedMoreData
                         continue
                     break
-                msg = self._pop_message()
+                self._pop_message()
                 # We need to pop the EOF matcher in order to tell if we're at
                 # the end of the current file, not the end of the last block
                 # of message headers.
@@ -320,7 +320,7 @@ class FeedParser:
                 self._cur.set_payload(EMPTYSTRING.join(lines))
                 return
             # Make sure a valid content type was specified per RFC 2045:6.4.
-            if (self._cur.get('content-transfer-encoding', '8bit').lower()
+            if (str(self._cur.get('content-transfer-encoding', '8bit')).lower()
                     not in ('7bit', '8bit', 'binary')):
                 defect = errors.InvalidMultipartContentTransferEncodingDefect()
                 self.policy.handle_defect(self._cur, defect)
@@ -329,9 +329,10 @@ class FeedParser:
             # this onto the input stream until we've scanned past the
             # preamble.
             separator = '--' + boundary
-            boundaryre = re.compile(
-                '(?P<sep>' + re.escape(separator) +
-                r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
+            def boundarymatch(line):
+                if not line.startswith(separator):
+                    return None
+                return boundaryendRE.match(line, len(separator))
             capturing_preamble = True
             preamble = []
             linesep = False
@@ -343,7 +344,7 @@ class FeedParser:
                     continue
                 if line == '':
                     break
-                mo = boundaryre.match(line)
+                mo = boundarymatch(line)
                 if mo:
                     # If we're looking at the end boundary, we're done with
                     # this multipart.  If there was a newline at the end of
@@ -375,13 +376,13 @@ class FeedParser:
                         if line is NeedMoreData:
                             yield NeedMoreData
                             continue
-                        mo = boundaryre.match(line)
+                        mo = boundarymatch(line)
                         if not mo:
                             self._input.unreadline(line)
                             break
                     # Recurse to parse this subpart; the input stream points
                     # at the subpart's first line.
-                    self._input.push_eof_matcher(boundaryre.match)
+                    self._input.push_eof_matcher(boundarymatch)
                     for retval in self._parsegen():
                         if retval is NeedMoreData:
                             yield NeedMoreData

Lib/email/generator.py

@@ -14,15 +14,16 @@ import random
 from copy import deepcopy
 from io import StringIO, BytesIO
 from email.utils import _has_surrogates
+from email.errors import HeaderWriteError
 
 UNDERSCORE = '_'
 NL = '\n'  # XXX: no longer used by the code below.
 
 NLCRE = re.compile(r'\r\n|\r|\n')
 fcre = re.compile(r'^From ', re.MULTILINE)
+NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
 
 
-
 class Generator:
     """Generates output from a Message object tree.
 
@@ -170,7 +171,7 @@ class Generator:
         # parameter.
         #
         # The way we do this, so as to make the _handle_*() methods simpler,
-        # is to cache any subpart writes into a buffer.  The we write the
+        # is to cache any subpart writes into a buffer.  Then we write the
         # headers and the buffer contents.  That way, subpart handlers can
         # Do The Right Thing, and can still modify the Content-Type: header if
         # necessary.
@@ -186,7 +187,11 @@ class Generator:
         # If we munged the cte, copy the message again and re-fix the CTE.
         if munge_cte:
             msg = deepcopy(msg)
-            msg.replace_header('content-transfer-encoding', munge_cte[0])
+            # Preserve the header order if the CTE header already exists.
+            if msg.get('content-transfer-encoding') is None:
+                msg['Content-Transfer-Encoding'] = munge_cte[0]
+            else:
+                msg.replace_header('content-transfer-encoding', munge_cte[0])
             msg.replace_header('content-type', munge_cte[1])
         # Write the headers.  First we see if the message object wants to
         # handle that itself.  If not, we'll do it generically.
@@ -219,7 +224,16 @@ class Generator:
 
     def _write_headers(self, msg):
         for h, v in msg.raw_items():
-            self.write(self.policy.fold(h, v))
+            folded = self.policy.fold(h, v)
+            if self.policy.verify_generated_headers:
+                linesep = self.policy.linesep
+                if not folded.endswith(self.policy.linesep):
+                    raise HeaderWriteError(
+                        f'folded header does not end with {linesep!r}: {folded!r}')
+                if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)):
+                    raise HeaderWriteError(
+                        f'folded header contains newline: {folded!r}')
+            self.write(folded)
         # A blank line always separates headers from body
         self.write(self._NL)
 
@@ -240,7 +254,7 @@ class Generator:
                 # existing message.
                 msg = deepcopy(msg)
                 del msg['content-transfer-encoding']
-                msg.set_payload(payload, charset)
+                msg.set_payload(msg._payload, charset)
                 payload = msg.get_payload()
                 self._munge_cte = (msg['content-transfer-encoding'],
                                    msg['content-type'])
@@ -388,7 +402,7 @@ class Generator:
     def _compile_re(cls, s, flags):
         return re.compile(s, flags)
 
-
+
 class BytesGenerator(Generator):
     """Generates a bytes version of a Message object tree.
 
@@ -439,7 +453,6 @@ class BytesGenerator(Generator):
         return re.compile(s.encode('ascii'), flags)
 
 
-
 _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]'
 
 class DecodedGenerator(Generator):
@@ -499,7 +512,6 @@ class DecodedGenerator(Generator):
                     }, file=self)
 
 
-
 # Helper used by Generator._make_boundary
 _width = len(repr(sys.maxsize-1))
 _fmt = '%%0%dd' % _width

Lib/email/header.py

@@ -36,11 +36,11 @@ ecre = re.compile(r'''
   =\?                   # literal =?
   (?P<charset>[^?]*?)   # non-greedy up to the next ? is the charset
   \?                    # literal ?
-  (?P<encoding>[qb])    # either a "q" or a "b", case insensitive
+  (?P<encoding>[qQbB])  # either a "q" or a "b", case insensitive
   \?                    # literal ?
   (?P<encoded>.*?)      # non-greedy up to the next ?= is the encoded string
   \?=                   # literal ?=
-  ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE)
+  ''', re.VERBOSE | re.MULTILINE)
 
 # Field name regexp, including trailing colon, but not separating whitespace,
 # according to RFC 2822.  Character range is from tilde to exclamation mark.
@@ -52,12 +52,10 @@ fcre = re.compile(r'[\041-\176]+:$')
 _embedded_header = re.compile(r'\n[^ \t]+:')
 
 
-
 # Helpers
 _max_append = email.quoprimime._max_append
 
 
-
 def decode_header(header):
     """Decode a message header value without converting charset.
 
@@ -152,7 +150,6 @@ def decode_header(header):
     return collapsed
 
 
-
 def make_header(decoded_seq, maxlinelen=None, header_name=None,
                 continuation_ws=' '):
     """Create a Header from a sequence of pairs as returned by decode_header()
@@ -175,7 +172,6 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None,
     return h
 
 
-
 class Header:
     def __init__(self, s=None, charset=None,
                  maxlinelen=None, header_name=None,
@@ -409,7 +405,6 @@ class Header:
         self._chunks = chunks
 
 
-
 class _ValueFormatter:
     def __init__(self, headerlen, maxlen, continuation_ws, splitchars):
         self._maxlen = maxlen
@@ -431,7 +426,7 @@ class _ValueFormatter:
         if end_of_line != (' ', ''):
             self._current_line.push(*end_of_line)
         if len(self._current_line) > 0:
-            if self._current_line.is_onlyws():
+            if self._current_line.is_onlyws() and self._lines:
                 self._lines[-1] += str(self._current_line)
             else:
                 self._lines.append(str(self._current_line))

Lib/email/headerregistry.py

@@ -2,10 +2,6 @@
 
 This module provides an implementation of the HeaderRegistry API.
 The implementation is designed to flexibly follow RFC5322 rules.
-
-Eventually HeaderRegistry will be a public API, but it isn't yet,
-and will probably change some before that happens.
-
 """
 from types import MappingProxyType
 
@@ -31,6 +27,11 @@ class Address:
         without any Content Transfer Encoding.
 
         """
+
+        inputs = ''.join(filter(None, (display_name, username, domain, addr_spec)))
+        if '\r' in inputs or '\n' in inputs:
+            raise ValueError("invalid arguments; address parts cannot contain CR or LF")
+
         # This clause with its potential 'raise' may only happen when an
         # application program creates an Address object using an addr_spec
         # keyword.  The email library code itself must always supply username
@@ -69,11 +70,9 @@ class Address:
         """The addr_spec (username@domain) portion of the address, quoted
         according to RFC 5322 rules, but with no Content Transfer Encoding.
         """
-        nameset = set(self.username)
-        if len(nameset) > len(nameset-parser.DOT_ATOM_ENDS):
-            lp = parser.quote_string(self.username)
-        else:
-            lp = self.username
+        lp = self.username
+        if not parser.DOT_ATOM_ENDS.isdisjoint(lp):
+            lp = parser.quote_string(lp)
         if self.domain:
             return lp + '@' + self.domain
         if not lp:
@@ -86,19 +85,17 @@ class Address:
                         self.display_name, self.username, self.domain)
 
     def __str__(self):
-        nameset = set(self.display_name)
-        if len(nameset) > len(nameset-parser.SPECIALS):
-            disp = parser.quote_string(self.display_name)
-        else:
-            disp = self.display_name
+        disp = self.display_name
+        if not parser.SPECIALS.isdisjoint(disp):
+            disp = parser.quote_string(disp)
         if disp:
             addr_spec = '' if self.addr_spec=='<>' else self.addr_spec
             return "{} <{}>".format(disp, addr_spec)
         return self.addr_spec
 
     def __eq__(self, other):
-        if type(other) != type(self):
-            return False
+        if not isinstance(other, Address):
+            return NotImplemented
         return (self.display_name == other.display_name and
                 self.username == other.username and
                 self.domain == other.domain)
@@ -141,17 +138,15 @@ class Group:
         if self.display_name is None and len(self.addresses)==1:
             return str(self.addresses[0])
         disp = self.display_name
-        if disp is not None:
-            nameset = set(disp)
-            if len(nameset) > len(nameset-parser.SPECIALS):
-                disp = parser.quote_string(disp)
+        if disp is not None and not parser.SPECIALS.isdisjoint(disp):
+            disp = parser.quote_string(disp)
         adrstr = ", ".join(str(x) for x in self.addresses)
         adrstr = ' ' + adrstr if adrstr else adrstr
         return "{}:{};".format(disp, adrstr)
 
     def __eq__(self, other):
-        if type(other) != type(self):
-            return False
+        if not isinstance(other, Group):
+            return NotImplemented
         return (self.display_name == other.display_name and
                 self.addresses == other.addresses)
 
@@ -223,7 +218,7 @@ class BaseHeader(str):
                 self.__class__.__bases__,
                 str(self),
             ),
-            self.__dict__)
+            self.__getstate__())
 
     @classmethod
     def _reconstruct(cls, value):
@@ -245,13 +240,16 @@ class BaseHeader(str):
         the header name and the ': ' separator.
 
         """
-        # At some point we need to only put fws here if it was in the source.
+        # At some point we need to put fws here if it was in the source.
         header = parser.Header([
             parser.HeaderLabel([
                 parser.ValueTerminal(self.name, 'header-name'),
                 parser.ValueTerminal(':', 'header-sep')]),
-            parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]),
-                             self._parse_tree])
+            ])
+        if self._parse_tree:
+            header.append(
+                parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]))
+        header.append(self._parse_tree)
         return header.fold(policy=policy)
 
 
@@ -300,7 +298,14 @@ class DateHeader:
             kwds['parse_tree'] = parser.TokenList()
             return
         if isinstance(value, str):
-            value = utils.parsedate_to_datetime(value)
+            kwds['decoded'] = value
+            try:
+                value = utils.parsedate_to_datetime(value)
+            except ValueError:
+                kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format'))
+                kwds['datetime'] = None
+                kwds['parse_tree'] = parser.TokenList()
+                return
         kwds['datetime'] = value
         kwds['decoded'] = utils.format_datetime(kwds['datetime'])
         kwds['parse_tree'] = cls.value_parser(kwds['decoded'])
@@ -369,8 +374,8 @@ class AddressHeader:
     @property
     def addresses(self):
         if self._addresses is None:
-            self._addresses = tuple([address for group in self._groups
-                                             for address in group.addresses])
+            self._addresses = tuple(address for group in self._groups
+                                            for address in group.addresses)
         return self._addresses
 
 
@@ -517,6 +522,18 @@ class ContentTransferEncodingHeader:
         return self._cte
 
 
+class MessageIDHeader:
+
+    max_count = 1
+    value_parser = staticmethod(parser.parse_message_id)
+
+    @classmethod
+    def parse(cls, value, kwds):
+        kwds['parse_tree'] = parse_tree = cls.value_parser(value)
+        kwds['decoded'] = str(parse_tree)
+        kwds['defects'].extend(parse_tree.all_defects)
+
+
 # The header factory #
 
 _default_header_map = {
@@ -539,6 +556,7 @@ _default_header_map = {
     'content-type':                 ContentTypeHeader,
     'content-disposition':          ContentDispositionHeader,
     'content-transfer-encoding':    ContentTransferEncodingHeader,
+    'message-id':                   MessageIDHeader,
     }
 
 class HeaderRegistry:

Lib/email/iterators.py

@@ -15,7 +15,6 @@ import sys
 from io import StringIO
 
 
-
 # This function will become a method of the Message class
 def walk(self):
     """Walk over the message tree, yielding each subpart.
@@ -29,7 +28,6 @@ def walk(self):
             yield from subpart.walk()
 
 
-
 # These two functions are imported into the Iterators.py interface module.
 def body_line_iterator(msg, decode=False):
     """Iterate over the parts, returning string payloads line-by-line.
@@ -55,7 +53,6 @@ def typed_subpart_iterator(msg, maintype='text', subtype=None):
                 yield subpart
 
 
-
 def _structure(msg, fp=None, level=0, include_default=False):
     """A handy debugging aid"""
     if fp is None:

Lib/email/message.py

@@ -6,15 +6,15 @@
 
 __all__ = ['Message', 'EmailMessage']
 
+import binascii
 import re
-import uu
 import quopri
 from io import BytesIO, StringIO
 
 # Intrapackage imports
 from email import utils
 from email import errors
-from email._policybase import Policy, compat32
+from email._policybase import compat32
 from email import charset as _charset
 from email._encoded_words import decode_b
 Charset = _charset.Charset
@@ -35,7 +35,7 @@ def _splitparam(param):
     if not sep:
         return a.strip(), None
     return a.strip(), b.strip()
-
+
 def _formatparam(param, value=None, quote=True):
     """Convenience function to format and return a key=value pair.
 
@@ -101,7 +101,37 @@ def _unquotevalue(value):
         return utils.unquote(value)
 
 
-
+def _decode_uu(encoded):
+    """Decode uuencoded data."""
+    decoded_lines = []
+    encoded_lines_iter = iter(encoded.splitlines())
+    for line in encoded_lines_iter:
+        if line.startswith(b"begin "):
+            mode, _, path = line.removeprefix(b"begin ").partition(b" ")
+            try:
+                int(mode, base=8)
+            except ValueError:
+                continue
+            else:
+                break
+    else:
+        raise ValueError("`begin` line not found")
+    for line in encoded_lines_iter:
+        if not line:
+            raise ValueError("Truncated input")
+        elif line.strip(b' \t\r\n\f') == b'end':
+            break
+        try:
+            decoded_line = binascii.a2b_uu(line)
+        except binascii.Error:
+            # Workaround for broken uuencoders by /Fredrik Lundh
+            nbytes = (((line[0]-32) & 63) * 4 + 5) // 3
+            decoded_line = binascii.a2b_uu(line[:nbytes])
+        decoded_lines.append(decoded_line)
+
+    return b''.join(decoded_lines)
+
+
 class Message:
     """Basic message object.
 
@@ -141,7 +171,7 @@ class Message:
         header.  For backward compatibility reasons, if maxheaderlen is
         not specified it defaults to 0, so you must override it explicitly
         if you want a different maxheaderlen.  'policy' is passed to the
-        Generator instance used to serialize the mesasge; if it is not
+        Generator instance used to serialize the message; if it is not
         specified the policy associated with the message instance is used.
 
         If the message object contains binary data that is not encoded
@@ -259,25 +289,26 @@ class Message:
         # cte might be a Header, so for now stringify it.
         cte = str(self.get('content-transfer-encoding', '')).lower()
         # payload may be bytes here.
-        if isinstance(payload, str):
-            if utils._has_surrogates(payload):
-                bpayload = payload.encode('ascii', 'surrogateescape')
-                if not decode:
+        if not decode:
+            if isinstance(payload, str) and utils._has_surrogates(payload):
+                try:
+                    bpayload = payload.encode('ascii', 'surrogateescape')
                     try:
-                        payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace')
+                        payload = bpayload.decode(self.get_content_charset('ascii'), 'replace')
                     except LookupError:
                         payload = bpayload.decode('ascii', 'replace')
-            elif decode:
-                try:
-                    bpayload = payload.encode('ascii')
-                except UnicodeError:
-                    # This won't happen for RFC compliant messages (messages
-                    # containing only ASCII code points in the unicode input).
-                    # If it does happen, turn the string into bytes in a way
-                    # guaranteed not to fail.
-                    bpayload = payload.encode('raw-unicode-escape')
-        if not decode:
+                except UnicodeEncodeError:
+                    pass
             return payload
+        if isinstance(payload, str):
+            try:
+                bpayload = payload.encode('ascii', 'surrogateescape')
+            except UnicodeEncodeError:
+                # This won't happen for RFC compliant messages (messages
+                # containing only ASCII code points in the unicode input).
+                # If it does happen, turn the string into bytes in a way
+                # guaranteed not to fail.
+                bpayload = payload.encode('raw-unicode-escape')
         if cte == 'quoted-printable':
             return quopri.decodestring(bpayload)
         elif cte == 'base64':
@@ -288,13 +319,10 @@ class Message:
                 self.policy.handle_defect(self, defect)
             return value
         elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'):
-            in_file = BytesIO(bpayload)
-            out_file = BytesIO()
             try:
-                uu.decode(in_file, out_file, quiet=True)
-                return out_file.getvalue()
-            except uu.Error:
-                # Some decoding problem
+                return _decode_uu(bpayload)
+            except ValueError:
+                # Some decoding problem.
                 return bpayload
         if isinstance(payload, str):
             return bpayload
@@ -312,7 +340,7 @@ class Message:
                 return
             if not isinstance(charset, Charset):
                 charset = Charset(charset)
-            payload = payload.encode(charset.output_charset)
+            payload = payload.encode(charset.output_charset, 'surrogateescape')
         if hasattr(payload, 'decode'):
             self._payload = payload.decode('ascii', 'surrogateescape')
         else:
@@ -421,7 +449,11 @@ class Message:
         self._headers = newheaders
 
     def __contains__(self, name):
-        return name.lower() in [k.lower() for k, v in self._headers]
+        name_lower = name.lower()
+        for k, v in self._headers:
+            if name_lower == k.lower():
+                return True
+        return False
 
     def __iter__(self):
         for field, value in self._headers:
@@ -948,7 +980,7 @@ class MIMEPart(Message):
         if policy is None:
             from email.policy import default
             policy = default
-        Message.__init__(self, policy)
+        super().__init__(policy)
 
 
     def as_string(self, unixfrom=False, maxheaderlen=None, policy=None):
@@ -958,14 +990,14 @@ class MIMEPart(Message):
         header.  maxheaderlen is retained for backward compatibility with the
         base Message class, but defaults to None, meaning that the policy value
         for max_line_length controls the header maximum length.  'policy' is
-        passed to the Generator instance used to serialize the mesasge; if it
+        passed to the Generator instance used to serialize the message; if it
         is not specified the policy associated with the message instance is
         used.
         """
         policy = self.policy if policy is None else policy
         if maxheaderlen is None:
             maxheaderlen = policy.max_line_length
-        return super().as_string(maxheaderlen=maxheaderlen, policy=policy)
+        return super().as_string(unixfrom, maxheaderlen, policy)
 
     def __str__(self):
         return self.as_string(policy=self.policy.clone(utf8=True))
@@ -982,7 +1014,7 @@ class MIMEPart(Message):
             if subtype in preferencelist:
                 yield (preferencelist.index(subtype), part)
             return
-        if maintype != 'multipart':
+        if maintype != 'multipart' or not self.is_multipart():
             return
         if subtype != 'related':
             for subpart in part.iter_parts():
@@ -1041,7 +1073,16 @@ class MIMEPart(Message):
         maintype, subtype = self.get_content_type().split('/')
         if maintype != 'multipart' or subtype == 'alternative':
             return
-        parts = self.get_payload().copy()
+        payload = self.get_payload()
+        # Certain malformed messages can have content type set to `multipart/*`
+        # but still have single part body, in which case payload.copy() can
+        # fail with AttributeError.
+        try:
+            parts = payload.copy()
+        except AttributeError:
+            # payload is not a list, it is most probably a string.
+            return
+
         if maintype == 'multipart' and subtype == 'related':
             # For related, we treat everything but the root as an attachment.
             # The root may be indicated by 'start'; if there's no start or we
@@ -1078,7 +1119,7 @@ class MIMEPart(Message):
 
         Return an empty iterator for a non-multipart.
         """
-        if self.get_content_maintype() == 'multipart':
+        if self.is_multipart():
             yield from self.get_payload()
 
     def get_content(self, *args, content_manager=None, **kw):

Lib/email/mime/application.py

@@ -17,7 +17,7 @@ class MIMEApplication(MIMENonMultipart):
                  _encoder=encoders.encode_base64, *, policy=None, **_params):
         """Create an application/* type MIME document.
 
-        _data is a string containing the raw application data.
+        _data contains the bytes for the raw application data.
 
         _subtype is the MIME content type subtype, defaulting to
         'octet-stream'.

Lib/email/mime/audio.py

@@ -6,39 +6,10 @@
 
 __all__ = ['MIMEAudio']
 
-import sndhdr
-
-from io import BytesIO
 from email import encoders
 from email.mime.nonmultipart import MIMENonMultipart
 
 
-
-_sndhdr_MIMEmap = {'au'  : 'basic',
-                   'wav' :'x-wav',
-                   'aiff':'x-aiff',
-                   'aifc':'x-aiff',
-                   }
-
-# There are others in sndhdr that don't have MIME types. :(
-# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma??
-def _whatsnd(data):
-    """Try to identify a sound file type.
-
-    sndhdr.what() has a pretty cruddy interface, unfortunately.  This is why
-    we re-do it here.  It would be easier to reverse engineer the Unix 'file'
-    command and use the standard 'magic' file, as shipped with a modern Unix.
-    """
-    hdr = data[:512]
-    fakefile = BytesIO(hdr)
-    for testfn in sndhdr.tests:
-        res = testfn(hdr, fakefile)
-        if res is not None:
-            return _sndhdr_MIMEmap.get(res[0])
-    return None
-
-
-
 class MIMEAudio(MIMENonMultipart):
     """Class for generating audio/* MIME documents."""
 
@@ -46,8 +17,8 @@ class MIMEAudio(MIMENonMultipart):
                  _encoder=encoders.encode_base64, *, policy=None, **_params):
         """Create an audio/* type MIME document.
 
-        _audiodata is a string containing the raw audio data.  If this data
-        can be decoded by the standard Python `sndhdr' module, then the
+        _audiodata contains the bytes for the raw audio data.  If this data
+        can be decoded as au, wav, aiff, or aifc, then the
         subtype will be automatically included in the Content-Type header.
         Otherwise, you can specify  the specific audio subtype via the
         _subtype parameter.  If _subtype is not given, and no subtype can be
@@ -65,10 +36,62 @@ class MIMEAudio(MIMENonMultipart):
         header.
         """
         if _subtype is None:
-            _subtype = _whatsnd(_audiodata)
+            _subtype = _what(_audiodata)
         if _subtype is None:
             raise TypeError('Could not find audio MIME subtype')
         MIMENonMultipart.__init__(self, 'audio', _subtype, policy=policy,
                                   **_params)
         self.set_payload(_audiodata)
         _encoder(self)
+
+
+_rules = []
+
+
+# Originally from the sndhdr module.
+#
+# There are others in sndhdr that don't have MIME types. :(
+# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma??
+def _what(data):
+    # Try to identify a sound file type.
+    #
+    # sndhdr.what() had a pretty cruddy interface, unfortunately.  This is why
+    # we re-do it here.  It would be easier to reverse engineer the Unix 'file'
+    # command and use the standard 'magic' file, as shipped with a modern Unix.
+    for testfn in _rules:
+        if res := testfn(data):
+            return res
+    else:
+        return None
+
+
+def rule(rulefunc):
+    _rules.append(rulefunc)
+    return rulefunc
+
+
+@rule
+def _aiff(h):
+    if not h.startswith(b'FORM'):
+        return None
+    if h[8:12] in {b'AIFC', b'AIFF'}:
+        return 'x-aiff'
+    else:
+        return None
+
+
+@rule
+def _au(h):
+    if h.startswith(b'.snd'):
+        return 'basic'
+    else:
+        return None
+
+
+@rule
+def _wav(h):
+    # 'RIFF' <len> 'WAVE' 'fmt ' <len>
+    if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ':
+        return None
+    else:
+        return "x-wav"

Lib/email/mime/base.py

@@ -11,7 +11,6 @@ import email.policy
 from email import message
 
 
-
 class MIMEBase(message.Message):
     """Base class for MIME specializations."""
 

Lib/email/mime/image.py

@@ -6,13 +6,10 @@
 
 __all__ = ['MIMEImage']
 
-import imghdr
-
 from email import encoders
 from email.mime.nonmultipart import MIMENonMultipart
 
 
-
 class MIMEImage(MIMENonMultipart):
     """Class for generating image/* type MIME documents."""
 
@@ -20,11 +17,11 @@ class MIMEImage(MIMENonMultipart):
                  _encoder=encoders.encode_base64, *, policy=None, **_params):
         """Create an image/* type MIME document.
 
-        _imagedata is a string containing the raw image data.  If this data
-        can be decoded by the standard Python `imghdr' module, then the
-        subtype will be automatically included in the Content-Type header.
-        Otherwise, you can specify the specific image subtype via the _subtype
-        parameter.
+        _imagedata contains the bytes for the raw image data.  If the data
+        type can be detected (jpeg, png, gif, tiff, rgb, pbm, pgm, ppm,
+        rast, xbm, bmp, webp, and exr attempted), then the subtype will be
+        automatically included in the Content-Type header. Otherwise, you can
+        specify the specific image subtype via the _subtype parameter.
 
         _encoder is a function which will perform the actual encoding for
         transport of the image data.  It takes one argument, which is this
@@ -37,11 +34,119 @@ class MIMEImage(MIMENonMultipart):
         constructor, which turns them into parameters on the Content-Type
         header.
         """
-        if _subtype is None:
-            _subtype = imghdr.what(None, _imagedata)
+        _subtype = _what(_imagedata) if _subtype is None else _subtype
         if _subtype is None:
             raise TypeError('Could not guess image MIME subtype')
         MIMENonMultipart.__init__(self, 'image', _subtype, policy=policy,
                                   **_params)
         self.set_payload(_imagedata)
         _encoder(self)
+
+
+_rules = []
+
+
+# Originally from the imghdr module.
+def _what(data):
+    for rule in _rules:
+        if res := rule(data):
+            return res
+    else:
+        return None
+
+
+def rule(rulefunc):
+    _rules.append(rulefunc)
+    return rulefunc
+
+
+@rule
+def _jpeg(h):
+    """JPEG data with JFIF or Exif markers; and raw JPEG"""
+    if h[6:10] in (b'JFIF', b'Exif'):
+        return 'jpeg'
+    elif h[:4] == b'\xff\xd8\xff\xdb':
+        return 'jpeg'
+
+
+@rule
+def _png(h):
+    if h.startswith(b'\211PNG\r\n\032\n'):
+        return 'png'
+
+
+@rule
+def _gif(h):
+    """GIF ('87 and '89 variants)"""
+    if h[:6] in (b'GIF87a', b'GIF89a'):
+        return 'gif'
+
+
+@rule
+def _tiff(h):
+    """TIFF (can be in Motorola or Intel byte order)"""
+    if h[:2] in (b'MM', b'II'):
+        return 'tiff'
+
+
+@rule
+def _rgb(h):
+    """SGI image library"""
+    if h.startswith(b'\001\332'):
+        return 'rgb'
+
+
+@rule
+def _pbm(h):
+    """PBM (portable bitmap)"""
+    if len(h) >= 3 and \
+            h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r':
+        return 'pbm'
+
+
+@rule
+def _pgm(h):
+    """PGM (portable graymap)"""
+    if len(h) >= 3 and \
+            h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r':
+        return 'pgm'
+
+
+@rule
+def _ppm(h):
+    """PPM (portable pixmap)"""
+    if len(h) >= 3 and \
+            h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r':
+        return 'ppm'
+
+
+@rule
+def _rast(h):
+    """Sun raster file"""
+    if h.startswith(b'\x59\xA6\x6A\x95'):
+        return 'rast'
+
+
+@rule
+def _xbm(h):
+    """X bitmap (X10 or X11)"""
+    if h.startswith(b'#define '):
+        return 'xbm'
+
+
+@rule
+def _bmp(h):
+    if h.startswith(b'BM'):
+        return 'bmp'
+
+
+@rule
+def _webp(h):
+    if h.startswith(b'RIFF') and h[8:12] == b'WEBP':
+        return 'webp'
+
+
+@rule
+def _exr(h):
+    if h.startswith(b'\x76\x2f\x31\x01'):
+        return 'exr'

Lib/email/mime/message.py

@@ -10,7 +10,6 @@ from email import message
 from email.mime.nonmultipart import MIMENonMultipart
 
 
-
 class MIMEMessage(MIMENonMultipart):
     """Class representing message/* MIME documents."""
 

Lib/email/mime/multipart.py

@@ -9,7 +9,6 @@ __all__ = ['MIMEMultipart']
 from email.mime.base import MIMEBase
 
 
-
 class MIMEMultipart(MIMEBase):
     """Base class for MIME multipart/* type messages."""
 

Lib/email/mime/nonmultipart.py

@@ -10,7 +10,6 @@ from email import errors
 from email.mime.base import MIMEBase
 
 
-
 class MIMENonMultipart(MIMEBase):
     """Base class for MIME non-multipart type messages."""
 

Lib/email/mime/text.py

@@ -6,11 +6,9 @@
 
 __all__ = ['MIMEText']
 
-from email.charset import Charset
 from email.mime.nonmultipart import MIMENonMultipart
 
 
-
 class MIMEText(MIMENonMultipart):
     """Class for generating text/* type MIME documents."""
 
@@ -37,6 +35,6 @@ class MIMEText(MIMENonMultipart):
                 _charset = 'utf-8'
 
         MIMENonMultipart.__init__(self, 'text', _subtype, policy=policy,
-                                  **{'charset': str(_charset)})
+                                  charset=str(_charset))
 
         self.set_payload(_text, _charset)

Lib/email/parser.py

@@ -13,7 +13,6 @@ from email.feedparser import FeedParser, BytesFeedParser
 from email._policybase import compat32
 
 
-
 class Parser:
     def __init__(self, _class=None, *, policy=compat32):
         """Parser of RFC 2822 and MIME email messages.
@@ -50,10 +49,7 @@ class Parser:
         feedparser = FeedParser(self._class, policy=self.policy)
         if headersonly:
             feedparser._set_headersonly()
-        while True:
-            data = fp.read(8192)
-            if not data:
-                break
+        while data := fp.read(8192):
             feedparser.feed(data)
         return feedparser.close()
 
@@ -68,7 +64,6 @@ class Parser:
         return self.parse(StringIO(text), headersonly=headersonly)
 
 
-
 class HeaderParser(Parser):
     def parse(self, fp, headersonly=True):
         return Parser.parse(self, fp, True)
@@ -76,7 +71,7 @@ class HeaderParser(Parser):
     def parsestr(self, text, headersonly=True):
         return Parser.parsestr(self, text, True)
 
-
+
 class BytesParser:
 
     def __init__(self, *args, **kw):

Lib/email/policy.py

@@ -3,6 +3,7 @@ code that adds all the email6 features.
 """
 
 import re
+import sys
 from email._policybase import Policy, Compat32, compat32, _extend_docstrings
 from email.utils import _has_surrogates
 from email.headerregistry import HeaderRegistry as HeaderRegistry
@@ -20,7 +21,7 @@ __all__ = [
     'HTTP',
     ]
 
-linesep_splitter = re.compile(r'\n|\r')
+linesep_splitter = re.compile(r'\n|\r\n?')
 
 @_extend_docstrings
 class EmailPolicy(Policy):
@@ -118,13 +119,13 @@ class EmailPolicy(Policy):
         """+
         The name is parsed as everything up to the ':' and returned unmodified.
         The value is determined by stripping leading whitespace off the
-        remainder of the first line, joining all subsequent lines together, and
+        remainder of the first line joined with all subsequent lines, and
         stripping any trailing carriage return or linefeed characters.  (This
         is the same as Compat32).
 
         """
         name, value = sourcelines[0].split(':', 1)
-        value = value.lstrip(' \t') + ''.join(sourcelines[1:])
+        value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n')
         return (name, value.rstrip('\r\n'))
 
     def header_store_parse(self, name, value):
@@ -203,14 +204,22 @@ class EmailPolicy(Policy):
     def _fold(self, name, value, refold_binary=False):
         if hasattr(value, 'name'):
             return value.fold(policy=self)
-        maxlen = self.max_line_length if self.max_line_length else float('inf')
-        lines = value.splitlines()
+        maxlen = self.max_line_length if self.max_line_length else sys.maxsize
+        # We can't use splitlines here because it splits on more than \r and \n.
+        lines = linesep_splitter.split(value)
         refold = (self.refold_source == 'all' or
                   self.refold_source == 'long' and
                     (lines and len(lines[0])+len(name)+2 > maxlen or
                      any(len(x) > maxlen for x in lines[1:])))
-        if refold or refold_binary and _has_surrogates(value):
+
+        if not refold:
+            if not self.utf8:
+                refold = not value.isascii()
+            elif refold_binary:
+                refold = _has_surrogates(value)
+        if refold:
             return self.header_factory(name, ''.join(lines)).fold(policy=self)
+
         return name + ': ' + self.linesep.join(lines) + self.linesep
 
 

Lib/email/quoprimime.py

@@ -148,6 +148,7 @@ def header_encode(header_bytes, charset='iso-8859-1'):
 _QUOPRI_BODY_ENCODE_MAP = _QUOPRI_BODY_MAP[:]
 for c in b'\r\n':
     _QUOPRI_BODY_ENCODE_MAP[c] = chr(c)
+del c
 
 def body_encode(body, maxlinelen=76, eol=NL):
     """Encode with quoted-printable, wrapping at maxlinelen characters.
@@ -173,7 +174,7 @@ def body_encode(body, maxlinelen=76, eol=NL):
     if not body:
         return body
 
-    # quote speacial characters
+    # quote special characters
     body = body.translate(_QUOPRI_BODY_ENCODE_MAP)
 
     soft_break = '=' + eol

Lib/email/utils.py

@@ -25,8 +25,6 @@ __all__ = [
 import os
 import re
 import time
-import random
-import socket
 import datetime
 import urllib.parse
 
@@ -36,9 +34,6 @@ from email._parseaddr import mktime_tz
 
 from email._parseaddr import parsedate, parsedate_tz, _parsedate_tz
 
-# Intrapackage imports
-from email.charset import Charset
-
 COMMASPACE = ', '
 EMPTYSTRING = ''
 UEMPTYSTRING = ''
@@ -48,11 +43,12 @@ TICK = "'"
 specialsre = re.compile(r'[][\\()<>@,:;".]')
 escapesre = re.compile(r'[\\"]')
 
+
 def _has_surrogates(s):
-    """Return True if s contains surrogate-escaped binary data."""
+    """Return True if s may contain surrogate-escaped binary data."""
     # This check is based on the fact that unless there are surrogates, utf8
     # (Python's default encoding) can encode any string.  This is the fastest
-    # way to check for surrogates, see issue 11454 for timings.
+    # way to check for surrogates, see bpo-11454 (moved to gh-55663) for timings.
     try:
         s.encode()
         return False
@@ -81,7 +77,7 @@ def formataddr(pair, charset='utf-8'):
     If the first element of pair is false, then the second element is
     returned unmodified.
 
-    Optional charset if given is the character set that is used to encode
+    The optional charset is the character set that is used to encode
     realname in case realname is not ASCII safe.  Can be an instance of str or
     a Charset-like object which has a header_encode method.  Default is
     'utf-8'.
@@ -94,6 +90,8 @@ def formataddr(pair, charset='utf-8'):
             name.encode('ascii')
         except UnicodeEncodeError:
             if isinstance(charset, str):
+                # lazy import to improve module import time
+                from email.charset import Charset
                 charset = Charset(charset)
             encoded_name = charset.header_encode(name)
             return "%s <%s>" % (encoded_name, address)
@@ -106,24 +104,127 @@ def formataddr(pair, charset='utf-8'):
     return address
 
 
-
-def getaddresses(fieldvalues):
-    """Return a list of (REALNAME, EMAIL) for each fieldvalue."""
-    all = COMMASPACE.join(fieldvalues)
-    a = _AddressList(all)
-    return a.addresslist
+def _iter_escaped_chars(addr):
+    pos = 0
+    escape = False
+    for pos, ch in enumerate(addr):
+        if escape:
+            yield (pos, '\\' + ch)
+            escape = False
+        elif ch == '\\':
+            escape = True
+        else:
+            yield (pos, ch)
+    if escape:
+        yield (pos, '\\')
 
 
+def _strip_quoted_realnames(addr):
+    """Strip real names between quotes."""
+    if '"' not in addr:
+        # Fast path
+        return addr
 
-ecre = re.compile(r'''
-  =\?                   # literal =?
-  (?P<charset>[^?]*?)   # non-greedy up to the next ? is the charset
-  \?                    # literal ?
-  (?P<encoding>[qb])    # either a "q" or a "b", case insensitive
-  \?                    # literal ?
-  (?P<atom>.*?)         # non-greedy up to the next ?= is the atom
-  \?=                   # literal ?=
-  ''', re.VERBOSE | re.IGNORECASE)
+    start = 0
+    open_pos = None
+    result = []
+    for pos, ch in _iter_escaped_chars(addr):
+        if ch == '"':
+            if open_pos is None:
+                open_pos = pos
+            else:
+                if start != open_pos:
+                    result.append(addr[start:open_pos])
+                start = pos + 1
+                open_pos = None
+
+    if start < len(addr):
+        result.append(addr[start:])
+
+    return ''.join(result)
+
+
+supports_strict_parsing = True
+
+def getaddresses(fieldvalues, *, strict=True):
+    """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
+
+    When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
+    its place.
+
+    If strict is true, use a strict parser which rejects malformed inputs.
+    """
+
+    # If strict is true, if the resulting list of parsed addresses is greater
+    # than the number of fieldvalues in the input list, a parsing error has
+    # occurred and consequently a list containing a single empty 2-tuple [('',
+    # '')] is returned in its place. This is done to avoid invalid output.
+    #
+    # Malformed input: getaddresses(['alice@example.com <bob@example.com>'])
+    # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')]
+    # Safe output: [('', '')]
+
+    if not strict:
+        all = COMMASPACE.join(str(v) for v in fieldvalues)
+        a = _AddressList(all)
+        return a.addresslist
+
+    fieldvalues = [str(v) for v in fieldvalues]
+    fieldvalues = _pre_parse_validation(fieldvalues)
+    addr = COMMASPACE.join(fieldvalues)
+    a = _AddressList(addr)
+    result = _post_parse_validation(a.addresslist)
+
+    # Treat output as invalid if the number of addresses is not equal to the
+    # expected number of addresses.
+    n = 0
+    for v in fieldvalues:
+        # When a comma is used in the Real Name part it is not a deliminator.
+        # So strip those out before counting the commas.
+        v = _strip_quoted_realnames(v)
+        # Expected number of addresses: 1 + number of commas
+        n += 1 + v.count(',')
+    if len(result) != n:
+        return [('', '')]
+
+    return result
+
+
+def _check_parenthesis(addr):
+    # Ignore parenthesis in quoted real names.
+    addr = _strip_quoted_realnames(addr)
+
+    opens = 0
+    for pos, ch in _iter_escaped_chars(addr):
+        if ch == '(':
+            opens += 1
+        elif ch == ')':
+            opens -= 1
+            if opens < 0:
+                return False
+    return (opens == 0)
+
+
+def _pre_parse_validation(email_header_fields):
+    accepted_values = []
+    for v in email_header_fields:
+        if not _check_parenthesis(v):
+            v = "('', '')"
+        accepted_values.append(v)
+
+    return accepted_values
+
+
+def _post_parse_validation(parsed_email_header_tuples):
+    accepted_values = []
+    # The parser would have parsed a correctly formatted domain-literal
+    # The existence of an [ after parsing indicates a parsing failure
+    for v in parsed_email_header_tuples:
+        if '[' in v[1]:
+            v = ('', '')
+        accepted_values.append(v)
+
+    return accepted_values
 
 
 def _format_timetuple_and_zone(timetuple, zone):
@@ -140,7 +241,7 @@ def formatdate(timeval=None, localtime=False, usegmt=False):
 
     Fri, 09 Nov 2001 01:08:47 -0000
 
-    Optional timeval if given is a floating point time value as accepted by
+    Optional timeval if given is a floating-point time value as accepted by
     gmtime() and localtime(), otherwise the current time is used.
 
     Optional localtime is a flag that when True, interprets timeval, and
@@ -155,13 +256,13 @@ def formatdate(timeval=None, localtime=False, usegmt=False):
     # 2822 requires that day and month names be the English abbreviations.
     if timeval is None:
         timeval = time.time()
-    if localtime or usegmt:
-        dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc)
-    else:
-        dt = datetime.datetime.utcfromtimestamp(timeval)
+    dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc)
+
     if localtime:
         dt = dt.astimezone()
         usegmt = False
+    elif not usegmt:
+        dt = dt.replace(tzinfo=None)
     return format_datetime(dt, usegmt)
 
 def format_datetime(dt, usegmt=False):
@@ -193,6 +294,11 @@ def make_msgid(idstring=None, domain=None):
     portion of the message id after the '@'.  It defaults to the locally
     defined hostname.
     """
+    # Lazy imports to speedup module import time
+    # (no other functions in email.utils need these modules)
+    import random
+    import socket
+
     timeval = int(time.time()*100)
     pid = os.getpid()
     randint = random.getrandbits(64)
@@ -207,17 +313,43 @@ def make_msgid(idstring=None, domain=None):
 
 
 def parsedate_to_datetime(data):
-    *dtuple, tz = _parsedate_tz(data)
+    parsed_date_tz = _parsedate_tz(data)
+    if parsed_date_tz is None:
+        raise ValueError('Invalid date value or format "%s"' % str(data))
+    *dtuple, tz = parsed_date_tz
     if tz is None:
         return datetime.datetime(*dtuple[:6])
     return datetime.datetime(*dtuple[:6],
             tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
 
 
-def parseaddr(addr):
-    addrs = _AddressList(addr).addresslist
-    if not addrs:
-        return '', ''
+def parseaddr(addr, *, strict=True):
+    """
+    Parse addr into its constituent realname and email address parts.
+
+    Return a tuple of realname and email address, unless the parse fails, in
+    which case return a 2-tuple of ('', '').
+
+    If strict is True, use a strict parser which rejects malformed inputs.
+    """
+    if not strict:
+        addrs = _AddressList(addr).addresslist
+        if not addrs:
+            return ('', '')
+        return addrs[0]
+
+    if isinstance(addr, list):
+        addr = addr[0]
+
+    if not isinstance(addr, str):
+        return ('', '')
+
+    addr = _pre_parse_validation([addr])[0]
+    addrs = _post_parse_validation(_AddressList(addr).addresslist)
+
+    if not addrs or len(addrs) > 1:
+        return ('', '')
+
     return addrs[0]
 
 
@@ -265,21 +397,13 @@ def decode_params(params):
 
     params is a sequence of 2-tuples containing (param name, string value).
     """
-    # Copy params so we don't mess with the original
-    params = params[:]
-    new_params = []
+    new_params = [params[0]]
     # Map parameter's name to a list of continuations.  The values are a
     # 3-tuple of the continuation number, the string value, and a flag
     # specifying whether a particular segment is %-encoded.
     rfc2231_params = {}
-    name, value = params.pop(0)
-    new_params.append((name, value))
-    while params:
-        name, value = params.pop(0)
-        if name.endswith('*'):
-            encoded = True
-        else:
-            encoded = False
+    for name, value in params[1:]:
+        encoded = name.endswith('*')
         value = unquote(value)
         mo = rfc2231_continuation.match(name)
         if mo:
@@ -342,41 +466,23 @@ def collapse_rfc2231_value(value, errors='replace',
 # better than not having it.
 #
 
-def localtime(dt=None, isdst=-1):
+def localtime(dt=None, isdst=None):
     """Return local time as an aware datetime object.
 
     If called without arguments, return current time.  Otherwise *dt*
     argument should be a datetime instance, and it is converted to the
     local time zone according to the system time zone database.  If *dt* is
     naive (that is, dt.tzinfo is None), it is assumed to be in local time.
-    In this case, a positive or zero value for *isdst* causes localtime to
-    presume initially that summer time (for example, Daylight Saving Time)
-    is or is not (respectively) in effect for the specified time.  A
-    negative value for *isdst* causes the localtime() function to attempt
-    to divine whether summer time is in effect for the specified time.
+    The isdst parameter is ignored.
 
     """
+    if isdst is not None:
+        import warnings
+        warnings._deprecated(
+            "The 'isdst' parameter to 'localtime'",
+            message='{name} is deprecated and slated for removal in Python {remove}',
+            remove=(3, 14),
+            )
     if dt is None:
-        return datetime.datetime.now(datetime.timezone.utc).astimezone()
-    if dt.tzinfo is not None:
-        return dt.astimezone()
-    # We have a naive datetime.  Convert to a (localtime) timetuple and pass to
-    # system mktime together with the isdst hint.  System mktime will return
-    # seconds since epoch.
-    tm = dt.timetuple()[:-1] + (isdst,)
-    seconds = time.mktime(tm)
-    localtm = time.localtime(seconds)
-    try:
-        delta = datetime.timedelta(seconds=localtm.tm_gmtoff)
-        tz = datetime.timezone(delta, localtm.tm_zone)
-    except AttributeError:
-        # Compute UTC offset and compare with the value implied by tm_isdst.
-        # If the values match, use the zone name implied by tm_isdst.
-        delta = dt - datetime.datetime(*time.gmtime(seconds)[:6])
-        dst = time.daylight and localtm.tm_isdst > 0
-        gmtoff = -(time.altzone if dst else time.timezone)
-        if delta == datetime.timedelta(seconds=gmtoff):
-            tz = datetime.timezone(delta, time.tzname[dst])
-        else:
-            tz = datetime.timezone(delta)
-    return dt.replace(tzinfo=tz)
+        dt = datetime.datetime.now()
+    return dt.astimezone()

Lib/graphlib.py

@@ -154,7 +154,7 @@ class TopologicalSorter:
         This method unblocks any successor of each node in *nodes* for being returned
         in the future by a call to "get_ready".
 
-        Raises :exec:`ValueError` if any node in *nodes* has already been marked as
+        Raises ValueError if any node in *nodes* has already been marked as
         processed by a previous call to this method, if a node was not added to the
         graph by using "add" or if called without calling "prepare" previously or if
         node has not yet been returned by "get_ready".

Lib/gzip.py

@@ -15,12 +15,16 @@ __all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"]
 
 FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
 
-READ, WRITE = 1, 2
+READ = 'rb'
+WRITE = 'wb'
 
 _COMPRESS_LEVEL_FAST = 1
 _COMPRESS_LEVEL_TRADEOFF = 6
 _COMPRESS_LEVEL_BEST = 9
 
+READ_BUFFER_SIZE = 128 * 1024
+_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE
+
 
 def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST,
          encoding=None, errors=None, newline=None):
@@ -118,6 +122,21 @@ class BadGzipFile(OSError):
     """Exception raised in some cases for invalid gzip files."""
 
 
+class _WriteBufferStream(io.RawIOBase):
+    """Minimal object to pass WriteBuffer flushes into GzipFile"""
+    def __init__(self, gzip_file):
+        self.gzip_file = gzip_file
+
+    def write(self, data):
+        return self.gzip_file._write_raw(data)
+
+    def seekable(self):
+        return False
+
+    def writable(self):
+        return True
+
+
 class GzipFile(_compression.BaseStream):
     """The GzipFile class simulates most of the methods of a file object with
     the exception of the truncate() method.
@@ -160,9 +179,10 @@ class GzipFile(_compression.BaseStream):
         and 9 is slowest and produces the most compression. 0 is no compression
         at all. The default is 9.
 
-        The mtime argument is an optional numeric timestamp to be written
-        to the last modification time field in the stream when compressing.
-        If omitted or None, the current time is used.
+        The optional mtime argument is the timestamp requested by gzip. The time
+        is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970.
+        If mtime is omitted or None, the current time is used. Use mtime = 0
+        to generate a compressed stream that does not depend on creation time.
 
         """
 
@@ -182,6 +202,7 @@ class GzipFile(_compression.BaseStream):
         if mode is None:
             mode = getattr(fileobj, 'mode', 'rb')
 
+
         if mode.startswith('r'):
             self.mode = READ
             raw = _GzipReader(fileobj)
@@ -204,6 +225,9 @@ class GzipFile(_compression.BaseStream):
                                              zlib.DEF_MEM_LEVEL,
                                              0)
             self._write_mtime = mtime
+            self._buffer_size = _WRITE_BUFFER_SIZE
+            self._buffer = io.BufferedWriter(_WriteBufferStream(self),
+                                             buffer_size=self._buffer_size)
         else:
             raise ValueError("Invalid mode: {!r}".format(mode))
 
@@ -212,14 +236,6 @@ class GzipFile(_compression.BaseStream):
         if self.mode == WRITE:
             self._write_gzip_header(compresslevel)
 
-    @property
-    def filename(self):
-        import warnings
-        warnings.warn("use the name attribute", DeprecationWarning, 2)
-        if self.mode == WRITE and self.name[-3:] != ".gz":
-            return self.name + ".gz"
-        return self.name
-
     @property
     def mtime(self):
         """Last modification time read from stream, or None"""
@@ -237,6 +253,11 @@ class GzipFile(_compression.BaseStream):
         self.bufsize = 0
         self.offset = 0  # Current file offset for seek(), tell(), etc
 
+    def tell(self):
+        self._check_not_closed()
+        self._buffer.flush()
+        return super().tell()
+
     def _write_gzip_header(self, compresslevel):
         self.fileobj.write(b'\037\213')             # magic header
         self.fileobj.write(b'\010')                 # compression method
@@ -278,6 +299,10 @@ class GzipFile(_compression.BaseStream):
         if self.fileobj is None:
             raise ValueError("write() on closed GzipFile object")
 
+        return self._buffer.write(data)
+
+    def _write_raw(self, data):
+        # Called by our self._buffer underlying WriteBufferStream.
         if isinstance(data, (bytes, bytearray)):
             length = len(data)
         else:
@@ -326,11 +351,11 @@ class GzipFile(_compression.BaseStream):
 
     def close(self):
         fileobj = self.fileobj
-        if fileobj is None:
+        if fileobj is None or self._buffer.closed:
             return
-        self.fileobj = None
         try:
             if self.mode == WRITE:
+                self._buffer.flush()
                 fileobj.write(self.compress.flush())
                 write32u(fileobj, self.crc)
                 # self.size may exceed 2 GiB, or even 4 GiB
@@ -338,6 +363,7 @@ class GzipFile(_compression.BaseStream):
             elif self.mode == READ:
                 self._buffer.close()
         finally:
+            self.fileobj = None
             myfileobj = self.myfileobj
             if myfileobj:
                 self.myfileobj = None
@@ -346,6 +372,7 @@ class GzipFile(_compression.BaseStream):
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_not_closed()
         if self.mode == WRITE:
+            self._buffer.flush()
             # Ensure the compressor's buffer is flushed
             self.fileobj.write(self.compress.flush(zlib_mode))
             self.fileobj.flush()
@@ -376,6 +403,9 @@ class GzipFile(_compression.BaseStream):
 
     def seek(self, offset, whence=io.SEEK_SET):
         if self.mode == WRITE:
+            self._check_not_closed()
+            # Flush buffer to ensure validity of self.offset
+            self._buffer.flush()
             if whence != io.SEEK_SET:
                 if whence == io.SEEK_CUR:
                     offset = self.offset + offset
@@ -384,10 +414,10 @@ class GzipFile(_compression.BaseStream):
             if offset < self.offset:
                 raise OSError('Negative seek in write mode')
             count = offset - self.offset
-            chunk = b'\0' * 1024
-            for i in range(count // 1024):
+            chunk = b'\0' * self._buffer_size
+            for i in range(count // self._buffer_size):
                 self.write(chunk)
-            self.write(b'\0' * (count % 1024))
+            self.write(b'\0' * (count % self._buffer_size))
         elif self.mode == READ:
             self._check_not_closed()
             return self._buffer.seek(offset, whence)
@@ -454,7 +484,7 @@ def _read_gzip_header(fp):
 
 class _GzipReader(_compression.DecompressReader):
     def __init__(self, fp):
-        super().__init__(_PaddedFile(fp), zlib.decompressobj,
+        super().__init__(_PaddedFile(fp), zlib._ZlibDecompressor,
                          wbits=-zlib.MAX_WBITS)
         # Set flag indicating start of a new member
         self._new_member = True
@@ -502,12 +532,13 @@ class _GzipReader(_compression.DecompressReader):
                 self._new_member = False
 
             # Read a chunk of data from the file
-            buf = self._fp.read(io.DEFAULT_BUFFER_SIZE)
+            if self._decompressor.needs_input:
+                buf = self._fp.read(READ_BUFFER_SIZE)
+                uncompress = self._decompressor.decompress(buf, size)
+            else:
+                uncompress = self._decompressor.decompress(b"", size)
 
-            uncompress = self._decompressor.decompress(buf, size)
-            if self._decompressor.unconsumed_tail != b"":
-                self._fp.prepend(self._decompressor.unconsumed_tail)
-            elif self._decompressor.unused_data != b"":
+            if self._decompressor.unused_data != b"":
                 # Prepend the already read bytes to the fileobj so they can
                 # be seen by _read_eof() and _read_gzip_header()
                 self._fp.prepend(self._decompressor.unused_data)
@@ -518,14 +549,11 @@ class _GzipReader(_compression.DecompressReader):
                 raise EOFError("Compressed file ended before the "
                                "end-of-stream marker was reached")
 
-        self._add_read_data( uncompress )
+        self._crc = zlib.crc32(uncompress, self._crc)
+        self._stream_size += len(uncompress)
         self._pos += len(uncompress)
         return uncompress
 
-    def _add_read_data(self, data):
-        self._crc = zlib.crc32(data, self._crc)
-        self._stream_size = self._stream_size + len(data)
-
     def _read_eof(self):
         # We've read to the end of the file
         # We check that the computed CRC and size of the
@@ -552,43 +580,21 @@ class _GzipReader(_compression.DecompressReader):
         self._new_member = True
 
 
-def _create_simple_gzip_header(compresslevel: int,
-                               mtime = None) -> bytes:
-    """
-    Write a simple gzip header with no extra fields.
-    :param compresslevel: Compresslevel used to determine the xfl bytes.
-    :param mtime: The mtime (must support conversion to a 32-bit integer).
-    :return: A bytes object representing the gzip header.
-    """
-    if mtime is None:
-        mtime = time.time()
-    if compresslevel == _COMPRESS_LEVEL_BEST:
-        xfl = 2
-    elif compresslevel == _COMPRESS_LEVEL_FAST:
-        xfl = 4
-    else:
-        xfl = 0
-    # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra
-    # fields added to header), mtime, xfl and os (255 for unknown OS).
-    return struct.pack("<BBBBLBB", 0x1f, 0x8b, 8, 0, int(mtime), xfl, 255)
-
-
-def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
+def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=0):
     """Compress data in one shot and return the compressed string.
 
     compresslevel sets the compression level in range of 0-9.
-    mtime can be used to set the modification time. The modification time is
-    set to the current time by default.
+    mtime can be used to set the modification time.
+    The modification time is set to 0 by default, for reproducibility.
     """
-    if mtime == 0:
-        # Use zlib as it creates the header with 0 mtime by default.
-        # This is faster and with less overhead.
-        return zlib.compress(data, level=compresslevel, wbits=31)
-    header = _create_simple_gzip_header(compresslevel, mtime)
-    trailer = struct.pack("<LL", zlib.crc32(data), (len(data) & 0xffffffff))
-    # Wbits=-15 creates a raw deflate block.
-    return (header + zlib.compress(data, level=compresslevel, wbits=-15) +
-            trailer)
+    # Wbits=31 automatically includes a gzip header and trailer.
+    gzip_data = zlib.compress(data, level=compresslevel, wbits=31)
+    if mtime is None:
+        mtime = time.time()
+    # Reuse gzip header created by zlib, replace mtime and OS byte for
+    # consistency.
+    header = struct.pack("<4sLBB", gzip_data, int(mtime), gzip_data[8], 255)
+    return header + gzip_data[10:]
 
 
 def decompress(data):
@@ -655,7 +661,7 @@ def main():
                 f = builtins.open(arg, "rb")
                 g = open(arg + ".gz", "wb")
         while True:
-            chunk = f.read(io.DEFAULT_BUFFER_SIZE)
+            chunk = f.read(READ_BUFFER_SIZE)
             if not chunk:
                 break
             g.write(chunk)

Lib/imghdr.py

@@ -1,175 +0,0 @@
-"""Recognize image file formats based on their first few bytes."""
-
-from os import PathLike
-import warnings
-
-__all__ = ["what"]
-
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-
-#-------------------------#
-# Recognize image headers #
-#-------------------------#
-
-def what(file, h=None):
-    f = None
-    try:
-        if h is None:
-            if isinstance(file, (str, PathLike)):
-                f = open(file, 'rb')
-                h = f.read(32)
-            else:
-                location = file.tell()
-                h = file.read(32)
-                file.seek(location)
-        for tf in tests:
-            res = tf(h, f)
-            if res:
-                return res
-    finally:
-        if f: f.close()
-    return None
-
-
-#---------------------------------#
-# Subroutines per image file type #
-#---------------------------------#
-
-tests = []
-
-def test_jpeg(h, f):
-    """JPEG data with JFIF or Exif markers; and raw JPEG"""
-    if h[6:10] in (b'JFIF', b'Exif'):
-        return 'jpeg'
-    elif h[:4] == b'\xff\xd8\xff\xdb':
-        return 'jpeg'
-
-tests.append(test_jpeg)
-
-def test_png(h, f):
-    if h.startswith(b'\211PNG\r\n\032\n'):
-        return 'png'
-
-tests.append(test_png)
-
-def test_gif(h, f):
-    """GIF ('87 and '89 variants)"""
-    if h[:6] in (b'GIF87a', b'GIF89a'):
-        return 'gif'
-
-tests.append(test_gif)
-
-def test_tiff(h, f):
-    """TIFF (can be in Motorola or Intel byte order)"""
-    if h[:2] in (b'MM', b'II'):
-        return 'tiff'
-
-tests.append(test_tiff)
-
-def test_rgb(h, f):
-    """SGI image library"""
-    if h.startswith(b'\001\332'):
-        return 'rgb'
-
-tests.append(test_rgb)
-
-def test_pbm(h, f):
-    """PBM (portable bitmap)"""
-    if len(h) >= 3 and \
-        h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r':
-        return 'pbm'
-
-tests.append(test_pbm)
-
-def test_pgm(h, f):
-    """PGM (portable graymap)"""
-    if len(h) >= 3 and \
-        h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r':
-        return 'pgm'
-
-tests.append(test_pgm)
-
-def test_ppm(h, f):
-    """PPM (portable pixmap)"""
-    if len(h) >= 3 and \
-        h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r':
-        return 'ppm'
-
-tests.append(test_ppm)
-
-def test_rast(h, f):
-    """Sun raster file"""
-    if h.startswith(b'\x59\xA6\x6A\x95'):
-        return 'rast'
-
-tests.append(test_rast)
-
-def test_xbm(h, f):
-    """X bitmap (X10 or X11)"""
-    if h.startswith(b'#define '):
-        return 'xbm'
-
-tests.append(test_xbm)
-
-def test_bmp(h, f):
-    if h.startswith(b'BM'):
-        return 'bmp'
-
-tests.append(test_bmp)
-
-def test_webp(h, f):
-    if h.startswith(b'RIFF') and h[8:12] == b'WEBP':
-        return 'webp'
-
-tests.append(test_webp)
-
-def test_exr(h, f):
-    if h.startswith(b'\x76\x2f\x31\x01'):
-        return 'exr'
-
-tests.append(test_exr)
-
-#--------------------#
-# Small test program #
-#--------------------#
-
-def test():
-    import sys
-    recursive = 0
-    if sys.argv[1:] and sys.argv[1] == '-r':
-        del sys.argv[1:2]
-        recursive = 1
-    try:
-        if sys.argv[1:]:
-            testall(sys.argv[1:], recursive, 1)
-        else:
-            testall(['.'], recursive, 1)
-    except KeyboardInterrupt:
-        sys.stderr.write('\n[Interrupted]\n')
-        sys.exit(1)
-
-def testall(list, recursive, toplevel):
-    import sys
-    import os
-    for filename in list:
-        if os.path.isdir(filename):
-            print(filename + '/:', end=' ')
-            if recursive or toplevel:
-                print('recursing down:')
-                import glob
-                names = glob.glob(os.path.join(glob.escape(filename), '*'))
-                testall(names, recursive, 0)
-            else:
-                print('*** directory (use -r) ***')
-        else:
-            print(filename + ':', end=' ')
-            sys.stdout.flush()
-            try:
-                print(what(filename))
-            except OSError:
-                print('*** not found ***')
-
-if __name__ == '__main__':
-    test()

Lib/imp.py

@@ -1,346 +0,0 @@
-"""This module provides the components needed to build your own __import__
-function.  Undocumented functions are obsolete.
-
-In most cases it is preferred you consider using the importlib module's
-functionality over this module.
-
-"""
-# (Probably) need to stay in _imp
-from _imp import (lock_held, acquire_lock, release_lock,
-                  get_frozen_object, is_frozen_package,
-                  init_frozen, is_builtin, is_frozen,
-                  _fix_co_filename, _frozen_module_names)
-try:
-    from _imp import create_dynamic
-except ImportError:
-    # Platform doesn't support dynamic loading.
-    create_dynamic = None
-
-from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name
-from importlib._bootstrap_external import SourcelessFileLoader
-
-from importlib import machinery
-from importlib import util
-import importlib
-import os
-import sys
-import tokenize
-import types
-import warnings
-
-warnings.warn("the imp module is deprecated in favour of importlib and slated "
-              "for removal in Python 3.12; "
-              "see the module's documentation for alternative uses",
-              DeprecationWarning, stacklevel=2)
-
-# DEPRECATED
-SEARCH_ERROR = 0
-PY_SOURCE = 1
-PY_COMPILED = 2
-C_EXTENSION = 3
-PY_RESOURCE = 4
-PKG_DIRECTORY = 5
-C_BUILTIN = 6
-PY_FROZEN = 7
-PY_CODERESOURCE = 8
-IMP_HOOK = 9
-
-
-def new_module(name):
-    """**DEPRECATED**
-
-    Create a new module.
-
-    The module is not entered into sys.modules.
-
-    """
-    return types.ModuleType(name)
-
-
-def get_magic():
-    """**DEPRECATED**
-
-    Return the magic number for .pyc files.
-    """
-    return util.MAGIC_NUMBER
-
-
-def get_tag():
-    """Return the magic tag for .pyc files."""
-    return sys.implementation.cache_tag
-
-
-def cache_from_source(path, debug_override=None):
-    """**DEPRECATED**
-
-    Given the path to a .py file, return the path to its .pyc file.
-
-    The .py file does not need to exist; this simply returns the path to the
-    .pyc file calculated as if the .py file were imported.
-
-    If debug_override is not None, then it must be a boolean and is used in
-    place of sys.flags.optimize.
-
-    If sys.implementation.cache_tag is None then NotImplementedError is raised.
-
-    """
-    with warnings.catch_warnings():
-        warnings.simplefilter('ignore')
-        return util.cache_from_source(path, debug_override)
-
-
-def source_from_cache(path):
-    """**DEPRECATED**
-
-    Given the path to a .pyc. file, return the path to its .py file.
-
-    The .pyc file does not need to exist; this simply returns the path to
-    the .py file calculated to correspond to the .pyc file.  If path does
-    not conform to PEP 3147 format, ValueError will be raised. If
-    sys.implementation.cache_tag is None then NotImplementedError is raised.
-
-    """
-    return util.source_from_cache(path)
-
-
-def get_suffixes():
-    """**DEPRECATED**"""
-    extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES]
-    source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
-    bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]
-
-    return extensions + source + bytecode
-
-
-class NullImporter:
-
-    """**DEPRECATED**
-
-    Null import object.
-
-    """
-
-    def __init__(self, path):
-        if path == '':
-            raise ImportError('empty pathname', path='')
-        elif os.path.isdir(path):
-            raise ImportError('existing directory', path=path)
-
-    def find_module(self, fullname):
-        """Always returns None."""
-        return None
-
-
-class _HackedGetData:
-
-    """Compatibility support for 'file' arguments of various load_*()
-    functions."""
-
-    def __init__(self, fullname, path, file=None):
-        super().__init__(fullname, path)
-        self.file = file
-
-    def get_data(self, path):
-        """Gross hack to contort loader to deal w/ load_*()'s bad API."""
-        if self.file and path == self.path:
-            # The contract of get_data() requires us to return bytes. Reopen the
-            # file in binary mode if needed.
-            if not self.file.closed:
-                file = self.file
-                if 'b' not in file.mode:
-                    file.close()
-            if self.file.closed:
-                self.file = file = open(self.path, 'rb')
-
-            with file:
-                return file.read()
-        else:
-            return super().get_data(path)
-
-
-class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader):
-
-    """Compatibility support for implementing load_source()."""
-
-
-def load_source(name, pathname, file=None):
-    loader = _LoadSourceCompatibility(name, pathname, file)
-    spec = util.spec_from_file_location(name, pathname, loader=loader)
-    if name in sys.modules:
-        module = _exec(spec, sys.modules[name])
-    else:
-        module = _load(spec)
-    # To allow reloading to potentially work, use a non-hacked loader which
-    # won't rely on a now-closed file object.
-    module.__loader__ = machinery.SourceFileLoader(name, pathname)
-    module.__spec__.loader = module.__loader__
-    return module
-
-
-class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader):
-
-    """Compatibility support for implementing load_compiled()."""
-
-
-def load_compiled(name, pathname, file=None):
-    """**DEPRECATED**"""
-    loader = _LoadCompiledCompatibility(name, pathname, file)
-    spec = util.spec_from_file_location(name, pathname, loader=loader)
-    if name in sys.modules:
-        module = _exec(spec, sys.modules[name])
-    else:
-        module = _load(spec)
-    # To allow reloading to potentially work, use a non-hacked loader which
-    # won't rely on a now-closed file object.
-    module.__loader__ = SourcelessFileLoader(name, pathname)
-    module.__spec__.loader = module.__loader__
-    return module
-
-
-def load_package(name, path):
-    """**DEPRECATED**"""
-    if os.path.isdir(path):
-        extensions = (machinery.SOURCE_SUFFIXES[:] +
-                      machinery.BYTECODE_SUFFIXES[:])
-        for extension in extensions:
-            init_path = os.path.join(path, '__init__' + extension)
-            if os.path.exists(init_path):
-                path = init_path
-                break
-        else:
-            raise ValueError('{!r} is not a package'.format(path))
-    spec = util.spec_from_file_location(name, path,
-                                        submodule_search_locations=[])
-    if name in sys.modules:
-        return _exec(spec, sys.modules[name])
-    else:
-        return _load(spec)
-
-
-def load_module(name, file, filename, details):
-    """**DEPRECATED**
-
-    Load a module, given information returned by find_module().
-
-    The module name must include the full package name, if any.
-
-    """
-    suffix, mode, type_ = details
-    if mode and (not mode.startswith('r') or '+' in mode):
-        raise ValueError('invalid file open mode {!r}'.format(mode))
-    elif file is None and type_ in {PY_SOURCE, PY_COMPILED}:
-        msg = 'file object required for import (type code {})'.format(type_)
-        raise ValueError(msg)
-    elif type_ == PY_SOURCE:
-        return load_source(name, filename, file)
-    elif type_ == PY_COMPILED:
-        return load_compiled(name, filename, file)
-    elif type_ == C_EXTENSION and load_dynamic is not None:
-        if file is None:
-            with open(filename, 'rb') as opened_file:
-                return load_dynamic(name, filename, opened_file)
-        else:
-            return load_dynamic(name, filename, file)
-    elif type_ == PKG_DIRECTORY:
-        return load_package(name, filename)
-    elif type_ == C_BUILTIN:
-        return init_builtin(name)
-    elif type_ == PY_FROZEN:
-        return init_frozen(name)
-    else:
-        msg =  "Don't know how to import {} (type code {})".format(name, type_)
-        raise ImportError(msg, name=name)
-
-
-def find_module(name, path=None):
-    """**DEPRECATED**
-
-    Search for a module.
-
-    If path is omitted or None, search for a built-in, frozen or special
-    module and continue search in sys.path. The module name cannot
-    contain '.'; to search for a submodule of a package, pass the
-    submodule name and the package's __path__.
-
-    """
-    if not isinstance(name, str):
-        raise TypeError("'name' must be a str, not {}".format(type(name)))
-    elif not isinstance(path, (type(None), list)):
-        # Backwards-compatibility
-        raise RuntimeError("'path' must be None or a list, "
-                           "not {}".format(type(path)))
-
-    if path is None:
-        if is_builtin(name):
-            return None, None, ('', '', C_BUILTIN)
-        elif is_frozen(name):
-            return None, None, ('', '', PY_FROZEN)
-        else:
-            path = sys.path
-
-    for entry in path:
-        package_directory = os.path.join(entry, name)
-        for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]:
-            package_file_name = '__init__' + suffix
-            file_path = os.path.join(package_directory, package_file_name)
-            if os.path.isfile(file_path):
-                return None, package_directory, ('', '', PKG_DIRECTORY)
-        for suffix, mode, type_ in get_suffixes():
-            file_name = name + suffix
-            file_path = os.path.join(entry, file_name)
-            if os.path.isfile(file_path):
-                break
-        else:
-            continue
-        break  # Break out of outer loop when breaking out of inner loop.
-    else:
-        raise ImportError(_ERR_MSG.format(name), name=name)
-
-    encoding = None
-    if 'b' not in mode:
-        with open(file_path, 'rb') as file:
-            encoding = tokenize.detect_encoding(file.readline)[0]
-    file = open(file_path, mode, encoding=encoding)
-    return file, file_path, (suffix, mode, type_)
-
-
-def reload(module):
-    """**DEPRECATED**
-
-    Reload the module and return it.
-
-    The module must have been successfully imported before.
-
-    """
-    return importlib.reload(module)
-
-
-def init_builtin(name):
-    """**DEPRECATED**
-
-    Load and return a built-in module by name, or None is such module doesn't
-    exist
-    """
-    try:
-        return _builtin_from_name(name)
-    except ImportError:
-        return None
-
-
-if create_dynamic:
-    def load_dynamic(name, path, file=None):
-        """**DEPRECATED**
-
-        Load an extension module.
-        """
-        import importlib.machinery
-        loader = importlib.machinery.ExtensionFileLoader(name, path)
-
-        # Issue #24748: Skip the sys.modules check in _load_module_shim;
-        # always load new extension
-        spec = importlib.machinery.ModuleSpec(
-            name=name, loader=loader, origin=path)
-        return _load(spec)
-
-else:
-    load_dynamic = None

Lib/io.py

@@ -55,11 +55,14 @@ import _io
 import abc
 
 from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
-                 open, open_code, FileIO, BytesIO, StringIO, BufferedReader,
+                 open, open_code, BytesIO, StringIO, BufferedReader,
                  BufferedWriter, BufferedRWPair, BufferedRandom,
-                 # XXX RUSTPYTHON TODO: IncrementalNewlineDecoder
-                 # IncrementalNewlineDecoder,
-                 text_encoding, TextIOWrapper)
+                 IncrementalNewlineDecoder, text_encoding, TextIOWrapper)
+
+try:
+    from _io import FileIO
+except ImportError:
+    pass
 
 # Pretend this exception was created here.
 UnsupportedOperation.__module__ = "io"
@@ -84,7 +87,10 @@ class BufferedIOBase(_io._BufferedIOBase, IOBase):
 class TextIOBase(_io._TextIOBase, IOBase):
     __doc__ = _io._TextIOBase.__doc__
 
-RawIOBase.register(FileIO)
+try:
+    RawIOBase.register(FileIO)
+except NameError:
+    pass
 
 for klass in (BytesIO, BufferedReader, BufferedWriter, BufferedRandom,
               BufferedRWPair):
@@ -100,10 +106,3 @@ except ImportError:
     pass
 else:
     RawIOBase.register(_WindowsConsoleIO)
-
-
-# XXX: RUSTPYTHON; borrow IncrementalNewlineDecoder from _pyio
-try:
-    from _pyio import IncrementalNewlineDecoder
-except ImportError:
-    pass

Lib/linecache.py

@@ -5,17 +5,13 @@ is not found, it will look down the module search path for a file by
 that name.
 """
 
-import functools
-import sys
-import os
-import tokenize
-
 __all__ = ["getline", "clearcache", "checkcache", "lazycache"]
 
 
 # The cache. Maps filenames to either a thunk which will provide source code,
 # or a tuple (size, mtime, lines, fullname) once loaded.
 cache = {}
+_interactive_cache = {}
 
 
 def clearcache():
@@ -49,28 +45,54 @@ def getlines(filename, module_globals=None):
         return []
 
 
+def _getline_from_code(filename, lineno):
+    lines = _getlines_from_code(filename)
+    if 1 <= lineno <= len(lines):
+        return lines[lineno - 1]
+    return ''
+
+def _make_key(code):
+    return (code.co_filename, code.co_qualname, code.co_firstlineno)
+
+def _getlines_from_code(code):
+    code_id = _make_key(code)
+    if code_id in _interactive_cache:
+        entry = _interactive_cache[code_id]
+        if len(entry) != 1:
+            return _interactive_cache[code_id][2]
+    return []
+
+
 def checkcache(filename=None):
     """Discard cache entries that are out of date.
     (This is not checked upon each call!)"""
 
     if filename is None:
-        filenames = list(cache.keys())
-    elif filename in cache:
-        filenames = [filename]
+        # get keys atomically
+        filenames = cache.copy().keys()
     else:
-        return
+        filenames = [filename]
 
     for filename in filenames:
-        entry = cache[filename]
+        try:
+            entry = cache[filename]
+        except KeyError:
+            continue
+
         if len(entry) == 1:
             # lazy cache entry, leave it lazy.
             continue
         size, mtime, lines, fullname = entry
         if mtime is None:
             continue   # no-op for files loaded via a __loader__
+        try:
+            # This import can fail if the interpreter is shutting down
+            import os
+        except ImportError:
+            return
         try:
             stat = os.stat(fullname)
-        except OSError:
+        except (OSError, ValueError):
             cache.pop(filename, None)
             continue
         if size != stat.st_size or mtime != stat.st_mtime:
@@ -82,6 +104,17 @@ def updatecache(filename, module_globals=None):
     If something's wrong, print a message, discard the cache entry,
     and return an empty list."""
 
+    # These imports are not at top level because linecache is in the critical
+    # path of the interpreter startup and importing os and sys take a lot of time
+    # and slows down the startup sequence.
+    try:
+        import os
+        import sys
+        import tokenize
+    except ImportError:
+        # These import can fail if the interpreter is shutting down
+        return []
+
     if filename in cache:
         if len(cache[filename]) != 1:
             cache.pop(filename, None)
@@ -128,16 +161,20 @@ def updatecache(filename, module_globals=None):
             try:
                 stat = os.stat(fullname)
                 break
-            except OSError:
+            except (OSError, ValueError):
                 pass
         else:
             return []
+    except ValueError:  # may be raised by os.stat()
+        return []
     try:
         with tokenize.open(fullname) as fp:
             lines = fp.readlines()
     except (OSError, UnicodeDecodeError, SyntaxError):
         return []
-    if lines and not lines[-1].endswith('\n'):
+    if not lines:
+        lines = ['\n']
+    elif not lines[-1].endswith('\n'):
         lines[-1] += '\n'
     size, mtime = stat.st_size, stat.st_mtime
     cache[filename] = size, mtime, lines, fullname
@@ -166,17 +203,29 @@ def lazycache(filename, module_globals):
         return False
     # Try for a __loader__, if available
     if module_globals and '__name__' in module_globals:
-        name = module_globals['__name__']
-        if (loader := module_globals.get('__loader__')) is None:
-            if spec := module_globals.get('__spec__'):
-                try:
-                    loader = spec.loader
-                except AttributeError:
-                    pass
+        spec = module_globals.get('__spec__')
+        name = getattr(spec, 'name', None) or module_globals['__name__']
+        loader = getattr(spec, 'loader', None)
+        if loader is None:
+            loader = module_globals.get('__loader__')
         get_source = getattr(loader, 'get_source', None)
 
         if name and get_source:
-            get_lines = functools.partial(get_source, name)
+            def get_lines(name=name, *args, **kwargs):
+                return get_source(name, *args, **kwargs)
             cache[filename] = (get_lines,)
             return True
     return False
+
+def _register_code(code, string, name):
+    entry = (len(string),
+             None,
+             [line + '\n' for line in string.splitlines()],
+             name)
+    stack = [code]
+    while stack:
+        code = stack.pop()
+        for const in code.co_consts:
+            if isinstance(const, type(code)):
+                stack.append(const)
+        _interactive_cache[_make_key(code)] = entry

Lib/logging/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -18,13 +18,14 @@
 Logging package for Python. Based on PEP 282 and comments thereto in
 comp.lang.python.
 
-Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
 
 import sys, os, time, io, re, traceback, warnings, weakref, collections.abc
 
+from types import GenericAlias
 from string import Template
 from string import Formatter as StrFormatter
 
@@ -37,7 +38,8 @@ __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR',
            'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass',
            'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown',
            'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory',
-           'lastResort', 'raiseExceptions']
+           'lastResort', 'raiseExceptions', 'getLevelNamesMapping',
+           'getHandlerByName', 'getHandlerNames']
 
 import threading
 
@@ -63,20 +65,25 @@ _startTime = time.time()
 raiseExceptions = True
 
 #
-# If you don't want threading information in the log, set this to zero
+# If you don't want threading information in the log, set this to False
 #
 logThreads = True
 
 #
-# If you don't want multiprocessing information in the log, set this to zero
+# If you don't want multiprocessing information in the log, set this to False
 #
 logMultiprocessing = True
 
 #
-# If you don't want process information in the log, set this to zero
+# If you don't want process information in the log, set this to False
 #
 logProcesses = True
 
+#
+# If you don't want asyncio task information in the log, set this to False
+#
+logAsyncioTasks = True
+
 #---------------------------------------------------------------------------
 #   Level related stuff
 #---------------------------------------------------------------------------
@@ -116,6 +123,9 @@ _nameToLevel = {
     'NOTSET': NOTSET,
 }
 
+def getLevelNamesMapping():
+    return _nameToLevel.copy()
+
 def getLevelName(level):
     """
     Return the textual or numeric representation of logging level 'level'.
@@ -156,15 +166,15 @@ def addLevelName(level, levelName):
     finally:
         _releaseLock()
 
-if hasattr(sys, '_getframe'):
-    currentframe = lambda: sys._getframe(3)
+if hasattr(sys, "_getframe"):
+    currentframe = lambda: sys._getframe(1)
 else: #pragma: no cover
     def currentframe():
         """Return the frame object for the caller's stack frame."""
         try:
             raise Exception
-        except Exception:
-            return sys.exc_info()[2].tb_frame.f_back
+        except Exception as exc:
+            return exc.__traceback__.tb_frame.f_back
 
 #
 # _srcfile is used when walking the stack to check when we've got the first
@@ -181,13 +191,18 @@ else: #pragma: no cover
 _srcfile = os.path.normcase(addLevelName.__code__.co_filename)
 
 # _srcfile is only used in conjunction with sys._getframe().
-# To provide compatibility with older versions of Python, set _srcfile
-# to None if _getframe() is not available; this value will prevent
-# findCaller() from being called. You can also do this if you want to avoid
-# the overhead of fetching caller information, even when _getframe() is
-# available.
-#if not hasattr(sys, '_getframe'):
-#    _srcfile = None
+# Setting _srcfile to None will prevent findCaller() from being called. This
+# way, you can avoid the overhead of fetching caller information.
+
+# The following is based on warnings._is_internal_frame. It makes sure that
+# frames of the import mechanism are skipped when logging at module level and
+# using a stacklevel value greater than one.
+def _is_internal_frame(frame):
+    """Signal whether the frame is a CPython or logging module internal."""
+    filename = os.path.normcase(frame.f_code.co_filename)
+    return filename == _srcfile or (
+            "importlib" in filename and "_bootstrap" in filename
+    )
 
 
 def _checkLevel(level):
@@ -307,7 +322,7 @@ class LogRecord(object):
         # Thus, while not removing the isinstance check, it does now look
         # for collections.abc.Mapping rather than, as before, dict.
         if (args and len(args) == 1 and isinstance(args[0], collections.abc.Mapping)
-            and args[0]):
+                and args[0]):
             args = args[0]
         self.args = args
         self.levelname = getLevelName(level)
@@ -325,7 +340,7 @@ class LogRecord(object):
         self.lineno = lineno
         self.funcName = func
         self.created = ct
-        self.msecs = (ct - int(ct)) * 1000
+        self.msecs = int((ct - int(ct)) * 1000) + 0.0  # see gh-89047
         self.relativeCreated = (self.created - _startTime) * 1000
         if logThreads:
             self.thread = threading.get_ident()
@@ -352,9 +367,18 @@ class LogRecord(object):
         else:
             self.process = None
 
+        self.taskName = None
+        if logAsyncioTasks:
+            asyncio = sys.modules.get('asyncio')
+            if asyncio:
+                try:
+                    self.taskName = asyncio.current_task().get_name()
+                except Exception:
+                    pass
+
     def __repr__(self):
         return '<LogRecord: %s, %s, %s, %s, "%s">'%(self.name, self.levelno,
-            self.pathname, self.lineno, self.msg)
+                                                    self.pathname, self.lineno, self.msg)
 
     def getMessage(self):
         """
@@ -487,7 +511,7 @@ class StringTemplateStyle(PercentStyle):
 
     def usesTime(self):
         fmt = self._fmt
-        return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0
+        return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_search) >= 0
 
     def validate(self):
         pattern = Template.pattern
@@ -557,6 +581,7 @@ class Formatter(object):
                         (typically at application startup time)
     %(thread)d          Thread ID (if available)
     %(threadName)s      Thread name (if available)
+    %(taskName)s        Task name (if available)
     %(process)d         Process ID (if available)
     %(message)s         The result of record.getMessage(), computed just as
                         the record is emitted
@@ -583,7 +608,7 @@ class Formatter(object):
         """
         if style not in _STYLES:
             raise ValueError('Style must be one of: %s' % ','.join(
-                             _STYLES.keys()))
+                _STYLES.keys()))
         self._style = _STYLES[style][0](fmt, defaults=defaults)
         if validate:
             self._style.validate()
@@ -808,23 +833,36 @@ class Filterer(object):
         Determine if a record is loggable by consulting all the filters.
 
         The default is to allow the record to be logged; any filter can veto
-        this and the record is then dropped. Returns a zero value if a record
-        is to be dropped, else non-zero.
+        this by returning a false value.
+        If a filter attached to a handler returns a log record instance,
+        then that instance is used in place of the original log record in
+        any further processing of the event by that handler.
+        If a filter returns any other true value, the original log record
+        is used in any further processing of the event by that handler.
+
+        If none of the filters return false values, this method returns
+        a log record.
+        If any of the filters return a false value, this method returns
+        a false value.
 
         .. versionchanged:: 3.2
 
            Allow filters to be just callables.
+
+        .. versionchanged:: 3.12
+           Allow filters to return a LogRecord instead of
+           modifying it in place.
         """
-        rv = True
         for f in self.filters:
             if hasattr(f, 'filter'):
                 result = f.filter(record)
             else:
                 result = f(record) # assume callable - will raise if not
             if not result:
-                rv = False
-                break
-        return rv
+                return False
+            if isinstance(result, LogRecord):
+                record = result
+        return record
 
 #---------------------------------------------------------------------------
 #   Handler classes and functions
@@ -845,8 +883,9 @@ def _removeHandlerRef(wr):
     if acquire and release and handlers:
         acquire()
         try:
-            if wr in handlers:
-                handlers.remove(wr)
+            handlers.remove(wr)
+        except ValueError:
+            pass
         finally:
             release()
 
@@ -860,6 +899,23 @@ def _addHandlerRef(handler):
     finally:
         _releaseLock()
 
+
+def getHandlerByName(name):
+    """
+    Get a handler with the specified *name*, or None if there isn't one with
+    that name.
+    """
+    return _handlers.get(name)
+
+
+def getHandlerNames():
+    """
+    Return all known handler names as an immutable set.
+    """
+    result = set(_handlers.keys())
+    return frozenset(result)
+
+
 class Handler(Filterer):
     """
     Handler instances dispatch logging events to specific destinations.
@@ -958,10 +1014,14 @@ class Handler(Filterer):
 
         Emission depends on filters which may have been added to the handler.
         Wrap the actual emission of the record with acquisition/release of
-        the I/O thread lock. Returns whether the filter passed the record for
-        emission.
+        the I/O thread lock.
+
+        Returns an instance of the log record that was emitted
+        if it passed all filters, otherwise a false value is returned.
         """
         rv = self.filter(record)
+        if isinstance(rv, LogRecord):
+            record = rv
         if rv:
             self.acquire()
             try:
@@ -1032,7 +1092,7 @@ class Handler(Filterer):
                 else:
                     # couldn't find the right stack frame, for some reason
                     sys.stderr.write('Logged from file %s, line %s\n' % (
-                                     record.filename, record.lineno))
+                        record.filename, record.lineno))
                 # Issue 18671: output logging message and arguments
                 try:
                     sys.stderr.write('Message: %r\n'
@@ -1044,7 +1104,7 @@ class Handler(Filterer):
                     sys.stderr.write('Unable to print the message and arguments'
                                      ' - possible formatting error.\nUse the'
                                      ' traceback above to help find the error.\n'
-                                    )
+                                     )
             except OSError: #pragma: no cover
                 pass    # see issue 5971
             finally:
@@ -1136,6 +1196,8 @@ class StreamHandler(Handler):
             name += ' '
         return '<%s %s(%s)>' % (self.__class__.__name__, name, level)
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class FileHandler(StreamHandler):
     """
@@ -1459,7 +1521,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.debug("Houston, we have a %s", "thorny problem", exc_info=1)
+        logger.debug("Houston, we have a %s", "thorny problem", exc_info=True)
         """
         if self.isEnabledFor(DEBUG):
             self._log(DEBUG, msg, args, **kwargs)
@@ -1471,7 +1533,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.info("Houston, we have a %s", "interesting problem", exc_info=1)
+        logger.info("Houston, we have a %s", "notable problem", exc_info=True)
         """
         if self.isEnabledFor(INFO):
             self._log(INFO, msg, args, **kwargs)
@@ -1483,14 +1545,14 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1)
+        logger.warning("Houston, we have a %s", "bit of a problem", exc_info=True)
         """
         if self.isEnabledFor(WARNING):
             self._log(WARNING, msg, args, **kwargs)
 
     def warn(self, msg, *args, **kwargs):
         warnings.warn("The 'warn' method is deprecated, "
-            "use 'warning' instead", DeprecationWarning, 2)
+                      "use 'warning' instead", DeprecationWarning, 2)
         self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
@@ -1500,7 +1562,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.error("Houston, we have a %s", "major problem", exc_info=1)
+        logger.error("Houston, we have a %s", "major problem", exc_info=True)
         """
         if self.isEnabledFor(ERROR):
             self._log(ERROR, msg, args, **kwargs)
@@ -1518,7 +1580,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.critical("Houston, we have a %s", "major disaster", exc_info=1)
+        logger.critical("Houston, we have a %s", "major disaster", exc_info=True)
         """
         if self.isEnabledFor(CRITICAL):
             self._log(CRITICAL, msg, args, **kwargs)
@@ -1536,7 +1598,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.log(level, "We have a %s", "mysterious problem", exc_info=1)
+        logger.log(level, "We have a %s", "mysterious problem", exc_info=True)
         """
         if not isinstance(level, int):
             if raiseExceptions:
@@ -1554,33 +1616,31 @@ class Logger(Filterer):
         f = currentframe()
         #On some versions of IronPython, currentframe() returns None if
         #IronPython isn't run with -X:Frames.
-        if f is not None:
-            f = f.f_back
-        orig_f = f
-        while f and stacklevel > 1:
-            f = f.f_back
-            stacklevel -= 1
-        if not f:
-            f = orig_f
-        rv = "(unknown file)", 0, "(unknown function)", None
-        while hasattr(f, "f_code"):
-            co = f.f_code
-            filename = os.path.normcase(co.co_filename)
-            if filename == _srcfile:
-                f = f.f_back
-                continue
-            sinfo = None
-            if stack_info:
-                sio = io.StringIO()
-                sio.write('Stack (most recent call last):\n')
+        if f is None:
+            return "(unknown file)", 0, "(unknown function)", None
+        while stacklevel > 0:
+            next_f = f.f_back
+            if next_f is None:
+                ## We've got options here.
+                ## If we want to use the last (deepest) frame:
+                break
+                ## If we want to mimic the warnings module:
+                #return ("sys", 1, "(unknown function)", None)
+                ## If we want to be pedantic:
+                #raise ValueError("call stack is not deep enough")
+            f = next_f
+            if not _is_internal_frame(f):
+                stacklevel -= 1
+        co = f.f_code
+        sinfo = None
+        if stack_info:
+            with io.StringIO() as sio:
+                sio.write("Stack (most recent call last):\n")
                 traceback.print_stack(f, file=sio)
                 sinfo = sio.getvalue()
                 if sinfo[-1] == '\n':
                     sinfo = sinfo[:-1]
-                sio.close()
-            rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
-            break
-        return rv
+        return co.co_filename, f.f_lineno, co.co_name, sinfo
 
     def makeRecord(self, name, level, fn, lno, msg, args, exc_info,
                    func=None, extra=None, sinfo=None):
@@ -1589,7 +1649,7 @@ class Logger(Filterer):
         specialized LogRecords.
         """
         rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func,
-                             sinfo)
+                               sinfo)
         if extra is not None:
             for key in extra:
                 if (key in ["message", "asctime"]) or (key in rv.__dict__):
@@ -1630,8 +1690,14 @@ class Logger(Filterer):
         This method is used for unpickled records received from a socket, as
         well as those created locally. Logger-level filtering is applied.
         """
-        if (not self.disabled) and self.filter(record):
-            self.callHandlers(record)
+        if self.disabled:
+            return
+        maybe_record = self.filter(record)
+        if not maybe_record:
+            return
+        if isinstance(maybe_record, LogRecord):
+            record = maybe_record
+        self.callHandlers(record)
 
     def addHandler(self, hdlr):
         """
@@ -1737,7 +1803,7 @@ class Logger(Filterer):
                     is_enabled = self._cache[level] = False
                 else:
                     is_enabled = self._cache[level] = (
-                        level >= self.getEffectiveLevel()
+                            level >= self.getEffectiveLevel()
                     )
             finally:
                 _releaseLock()
@@ -1762,13 +1828,30 @@ class Logger(Filterer):
             suffix = '.'.join((self.name, suffix))
         return self.manager.getLogger(suffix)
 
+    def getChildren(self):
+
+        def _hierlevel(logger):
+            if logger is logger.manager.root:
+                return 0
+            return 1 + logger.name.count('.')
+
+        d = self.manager.loggerDict
+        _acquireLock()
+        try:
+            # exclude PlaceHolders - the last check is to ensure that lower-level
+            # descendants aren't returned - if there are placeholders, a logger's
+            # parent field might point to a grandparent or ancestor thereof.
+            return set(item for item in d.values()
+                       if isinstance(item, Logger) and item.parent is self and
+                       _hierlevel(item) == 1 + _hierlevel(item.parent))
+        finally:
+            _releaseLock()
+
     def __repr__(self):
         level = getLevelName(self.getEffectiveLevel())
         return '<%s %s (%s)>' % (self.__class__.__name__, self.name, level)
 
     def __reduce__(self):
-        # In general, only the root logger will not be accessible via its name.
-        # However, the root logger's class has its own __reduce__ method.
         if getLogger(self.name) is not self:
             import pickle
             raise pickle.PicklingError('logger cannot be pickled')
@@ -1848,7 +1931,7 @@ class LoggerAdapter(object):
 
     def warn(self, msg, *args, **kwargs):
         warnings.warn("The 'warn' method is deprecated, "
-            "use 'warning' instead", DeprecationWarning, 2)
+                      "use 'warning' instead", DeprecationWarning, 2)
         self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
@@ -1902,18 +1985,11 @@ class LoggerAdapter(object):
         """
         return self.logger.hasHandlers()
 
-    def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False):
+    def _log(self, level, msg, args, **kwargs):
         """
         Low-level log implementation, proxied to allow nested logger adapters.
         """
-        return self.logger._log(
-            level,
-            msg,
-            args,
-            exc_info=exc_info,
-            extra=extra,
-            stack_info=stack_info,
-        )
+        return self.logger._log(level, msg, args, **kwargs)
 
     @property
     def manager(self):
@@ -1932,6 +2008,8 @@ class LoggerAdapter(object):
         level = getLevelName(logger.getEffectiveLevel())
         return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level)
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 root = RootLogger(WARNING)
 Logger.root = root
 Logger.manager = Manager(Logger.root)
@@ -1971,7 +2049,7 @@ def basicConfig(**kwargs):
               that this argument is incompatible with 'filename' - if both
               are present, 'stream' is ignored.
     handlers  If specified, this should be an iterable of already created
-              handlers, which will be added to the root handler. Any handler
+              handlers, which will be added to the root logger. Any handler
               in the list which does not have a formatter assigned will be
               assigned the formatter created in this function.
     force     If this keyword  is specified as true, any existing handlers
@@ -2047,7 +2125,7 @@ def basicConfig(**kwargs):
             style = kwargs.pop("style", '%')
             if style not in _STYLES:
                 raise ValueError('Style must be one of: %s' % ','.join(
-                                 _STYLES.keys()))
+                    _STYLES.keys()))
             fs = kwargs.pop("format", _STYLES[style][1])
             fmt = Formatter(fs, dfs, style)
             for h in handlers:
@@ -2124,7 +2202,7 @@ def warning(msg, *args, **kwargs):
 
 def warn(msg, *args, **kwargs):
     warnings.warn("The 'warn' function is deprecated, "
-        "use 'warning' instead", DeprecationWarning, 2)
+                  "use 'warning' instead", DeprecationWarning, 2)
     warning(msg, *args, **kwargs)
 
 def info(msg, *args, **kwargs):
@@ -2179,7 +2257,11 @@ def shutdown(handlerList=_handlerList):
             if h:
                 try:
                     h.acquire()
-                    h.flush()
+                    # MemoryHandlers might not want to be flushed on close,
+                    # but circular imports prevent us scoping this to just
+                    # those handlers.  hence the default to True.
+                    if getattr(h, 'flushOnClose', True):
+                        h.flush()
                     h.close()
                 except (OSError, ValueError):
                     # Ignore errors which might be caused
@@ -2242,7 +2324,9 @@ def _showwarning(message, category, filename, lineno, file=None, line=None):
         logger = getLogger("py.warnings")
         if not logger.handlers:
             logger.addHandler(NullHandler())
-        logger.warning("%s", s)
+        # bpo-46557: Log str(s) as msg instead of logger.warning("%s", s)
+        # since some log aggregation tools group logs by the msg arg
+        logger.warning(str(s))
 
 def captureWarnings(capture):
     """

Lib/logging/config.py

@@ -1,4 +1,4 @@
-# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2023 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -19,18 +19,20 @@ Configuration functions for the logging package for Python. The core package
 is based on PEP 282 and comments thereto in comp.lang.python, and influenced
 by Apache's log4j system.
 
-Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
 
 import errno
+import functools
 import io
 import logging
 import logging.handlers
+import os
+import queue
 import re
 import struct
-import sys
 import threading
 import traceback
 
@@ -59,15 +61,24 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=Non
     """
     import configparser
 
+    if isinstance(fname, str):
+        if not os.path.exists(fname):
+            raise FileNotFoundError(f"{fname} doesn't exist")
+        elif not os.path.getsize(fname):
+            raise RuntimeError(f'{fname} is an empty file')
+
     if isinstance(fname, configparser.RawConfigParser):
         cp = fname
     else:
-        cp = configparser.ConfigParser(defaults)
-        if hasattr(fname, 'readline'):
-            cp.read_file(fname)
-        else:
-            encoding = io.text_encoding(encoding)
-            cp.read(fname, encoding=encoding)
+        try:
+            cp = configparser.ConfigParser(defaults)
+            if hasattr(fname, 'readline'):
+                cp.read_file(fname)
+            else:
+                encoding = io.text_encoding(encoding)
+                cp.read(fname, encoding=encoding)
+        except configparser.ParsingError as e:
+            raise RuntimeError(f'{fname} is invalid: {e}')
 
     formatters = _create_formatters(cp)
 
@@ -113,11 +124,18 @@ def _create_formatters(cp):
         fs = cp.get(sectname, "format", raw=True, fallback=None)
         dfs = cp.get(sectname, "datefmt", raw=True, fallback=None)
         stl = cp.get(sectname, "style", raw=True, fallback='%')
+        defaults = cp.get(sectname, "defaults", raw=True, fallback=None)
+
         c = logging.Formatter
         class_name = cp[sectname].get("class")
         if class_name:
             c = _resolve(class_name)
-        f = c(fs, dfs, stl)
+
+        if defaults is not None:
+            defaults = eval(defaults, vars(logging))
+            f = c(fs, dfs, stl, defaults=defaults)
+        else:
+            f = c(fs, dfs, stl)
         formatters[form] = f
     return formatters
 
@@ -296,7 +314,7 @@ class ConvertingMixin(object):
             if replace:
                 self[key] = result
             if type(result) in (ConvertingDict, ConvertingList,
-                               ConvertingTuple):
+                                ConvertingTuple):
                 result.parent = self
                 result.key = key
         return result
@@ -305,7 +323,7 @@ class ConvertingMixin(object):
         result = self.configurator.convert(value)
         if value is not result:
             if type(result) in (ConvertingDict, ConvertingList,
-                               ConvertingTuple):
+                                ConvertingTuple):
                 result.parent = self
         return result
 
@@ -392,11 +410,9 @@ class BaseConfigurator(object):
                     self.importer(used)
                     found = getattr(found, frag)
             return found
-        except ImportError:
-            e, tb = sys.exc_info()[1:]
+        except ImportError as e:
             v = ValueError('Cannot resolve %r: %s' % (s, e))
-            v.__cause__, v.__traceback__ = e, tb
-            raise v
+            raise v from e
 
     def ext_convert(self, value):
         """Default converter for the ext:// protocol."""
@@ -448,8 +464,8 @@ class BaseConfigurator(object):
         elif not isinstance(value, ConvertingList) and isinstance(value, list):
             value = ConvertingList(value)
             value.configurator = self
-        elif not isinstance(value, ConvertingTuple) and\
-                 isinstance(value, tuple) and not hasattr(value, '_fields'):
+        elif not isinstance(value, ConvertingTuple) and \
+                isinstance(value, tuple) and not hasattr(value, '_fields'):
             value = ConvertingTuple(value)
             value.configurator = self
         elif isinstance(value, str): # str for py3k
@@ -469,10 +485,10 @@ class BaseConfigurator(object):
         c = config.pop('()')
         if not callable(c):
             c = self.resolve(c)
-        props = config.pop('.', None)
         # Check for valid identifiers
-        kwargs = {k: config[k] for k in config if valid_ident(k)}
+        kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))}
         result = c(**kwargs)
+        props = config.pop('.', None)
         if props:
             for name, value in props.items():
                 setattr(result, name, value)
@@ -484,6 +500,33 @@ class BaseConfigurator(object):
             value = tuple(value)
         return value
 
+def _is_queue_like_object(obj):
+    """Check that *obj* implements the Queue API."""
+    if isinstance(obj, (queue.Queue, queue.SimpleQueue)):
+        return True
+    # defer importing multiprocessing as much as possible
+    from multiprocessing.queues import Queue as MPQueue
+    if isinstance(obj, MPQueue):
+        return True
+    # Depending on the multiprocessing start context, we cannot create
+    # a multiprocessing.managers.BaseManager instance 'mm' to get the
+    # runtime type of mm.Queue() or mm.JoinableQueue() (see gh-119819).
+    #
+    # Since we only need an object implementing the Queue API, we only
+    # do a protocol check, but we do not use typing.runtime_checkable()
+    # and typing.Protocol to reduce import time (see gh-121723).
+    #
+    # Ideally, we would have wanted to simply use strict type checking
+    # instead of a protocol-based type checking since the latter does
+    # not check the method signatures.
+    #
+    # Note that only 'put_nowait' and 'get' are required by the logging
+    # queue handler and queue listener (see gh-124653) and that other
+    # methods are either optional or unused.
+    minimal_queue_interface = ['put_nowait', 'get']
+    return all(callable(getattr(obj, method, None))
+               for method in minimal_queue_interface)
+
 class DictConfigurator(BaseConfigurator):
     """
     Configure logging using a dictionary-like object to describe the
@@ -542,7 +585,7 @@ class DictConfigurator(BaseConfigurator):
                 for name in formatters:
                     try:
                         formatters[name] = self.configure_formatter(
-                                                            formatters[name])
+                            formatters[name])
                     except Exception as e:
                         raise ValueError('Unable to configure '
                                          'formatter %r' % name) from e
@@ -566,7 +609,7 @@ class DictConfigurator(BaseConfigurator):
                         handler.name = name
                         handlers[name] = handler
                     except Exception as e:
-                        if 'target not configured yet' in str(e.__cause__):
+                        if ' not configured yet' in str(e.__cause__):
                             deferred.append(name)
                         else:
                             raise ValueError('Unable to configure handler '
@@ -669,18 +712,27 @@ class DictConfigurator(BaseConfigurator):
             dfmt = config.get('datefmt', None)
             style = config.get('style', '%')
             cname = config.get('class', None)
+            defaults = config.get('defaults', None)
 
             if not cname:
                 c = logging.Formatter
             else:
                 c = _resolve(cname)
 
+            kwargs  = {}
+
+            # Add defaults only if it exists.
+            # Prevents TypeError in custom formatter callables that do not
+            # accept it.
+            if defaults is not None:
+                kwargs['defaults'] = defaults
+
             # A TypeError would be raised if "validate" key is passed in with a formatter callable
             # that does not accept "validate" as a parameter
             if 'validate' in config:  # if user hasn't mentioned it, the default will be fine
-                result = c(fmt, dfmt, style, config['validate'])
+                result = c(fmt, dfmt, style, config['validate'], **kwargs)
             else:
-                result = c(fmt, dfmt, style)
+                result = c(fmt, dfmt, style, **kwargs)
 
         return result
 
@@ -697,10 +749,29 @@ class DictConfigurator(BaseConfigurator):
         """Add filters to a filterer from a list of names."""
         for f in filters:
             try:
-                filterer.addFilter(self.config['filters'][f])
+                if callable(f) or callable(getattr(f, 'filter', None)):
+                    filter_ = f
+                else:
+                    filter_ = self.config['filters'][f]
+                filterer.addFilter(filter_)
             except Exception as e:
                 raise ValueError('Unable to add filter %r' % f) from e
 
+    def _configure_queue_handler(self, klass, **kwargs):
+        if 'queue' in kwargs:
+            q = kwargs.pop('queue')
+        else:
+            q = queue.Queue()  # unbounded
+
+        rhl = kwargs.pop('respect_handler_level', False)
+        lklass = kwargs.pop('listener', logging.handlers.QueueListener)
+        handlers = kwargs.pop('handlers', [])
+
+        listener = lklass(q, *handlers, respect_handler_level=rhl)
+        handler = klass(q, **kwargs)
+        handler.listener = listener
+        return handler
+
     def configure_handler(self, config):
         """Configure a handler from a dictionary."""
         config_copy = dict(config)  # for restoring in case of error
@@ -720,28 +791,87 @@ class DictConfigurator(BaseConfigurator):
             factory = c
         else:
             cname = config.pop('class')
-            klass = self.resolve(cname)
-            #Special case for handler which refers to another handler
-            if issubclass(klass, logging.handlers.MemoryHandler) and\
-                'target' in config:
-                try:
-                    th = self.config['handlers'][config['target']]
-                    if not isinstance(th, logging.Handler):
-                        config.update(config_copy)  # restore for deferred cfg
-                        raise TypeError('target not configured yet')
-                    config['target'] = th
-                except Exception as e:
-                    raise ValueError('Unable to set target handler '
-                                     '%r' % config['target']) from e
-            elif issubclass(klass, logging.handlers.SMTPHandler) and\
-                'mailhost' in config:
+            if callable(cname):
+                klass = cname
+            else:
+                klass = self.resolve(cname)
+            if issubclass(klass, logging.handlers.MemoryHandler):
+                if 'flushLevel' in config:
+                    config['flushLevel'] = logging._checkLevel(config['flushLevel'])
+                if 'target' in config:
+                    # Special case for handler which refers to another handler
+                    try:
+                        tn = config['target']
+                        th = self.config['handlers'][tn]
+                        if not isinstance(th, logging.Handler):
+                            config.update(config_copy)  # restore for deferred cfg
+                            raise TypeError('target not configured yet')
+                        config['target'] = th
+                    except Exception as e:
+                        raise ValueError('Unable to set target handler %r' % tn) from e
+            elif issubclass(klass, logging.handlers.QueueHandler):
+                # Another special case for handler which refers to other handlers
+                # if 'handlers' not in config:
+                # raise ValueError('No handlers specified for a QueueHandler')
+                if 'queue' in config:
+                    qspec = config['queue']
+
+                    if isinstance(qspec, str):
+                        q = self.resolve(qspec)
+                        if not callable(q):
+                            raise TypeError('Invalid queue specifier %r' % qspec)
+                        config['queue'] = q()
+                    elif isinstance(qspec, dict):
+                        if '()' not in qspec:
+                            raise TypeError('Invalid queue specifier %r' % qspec)
+                        config['queue'] = self.configure_custom(dict(qspec))
+                    elif not _is_queue_like_object(qspec):
+                        raise TypeError('Invalid queue specifier %r' % qspec)
+
+                if 'listener' in config:
+                    lspec = config['listener']
+                    if isinstance(lspec, type):
+                        if not issubclass(lspec, logging.handlers.QueueListener):
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                    else:
+                        if isinstance(lspec, str):
+                            listener = self.resolve(lspec)
+                            if isinstance(listener, type) and \
+                                    not issubclass(listener, logging.handlers.QueueListener):
+                                raise TypeError('Invalid listener specifier %r' % lspec)
+                        elif isinstance(lspec, dict):
+                            if '()' not in lspec:
+                                raise TypeError('Invalid listener specifier %r' % lspec)
+                            listener = self.configure_custom(dict(lspec))
+                        else:
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                        if not callable(listener):
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                        config['listener'] = listener
+                if 'handlers' in config:
+                    hlist = []
+                    try:
+                        for hn in config['handlers']:
+                            h = self.config['handlers'][hn]
+                            if not isinstance(h, logging.Handler):
+                                config.update(config_copy)  # restore for deferred cfg
+                                raise TypeError('Required handler %r '
+                                                'is not configured yet' % hn)
+                            hlist.append(h)
+                    except Exception as e:
+                        raise ValueError('Unable to set required handler %r' % hn) from e
+                    config['handlers'] = hlist
+            elif issubclass(klass, logging.handlers.SMTPHandler) and \
+                    'mailhost' in config:
                 config['mailhost'] = self.as_tuple(config['mailhost'])
-            elif issubclass(klass, logging.handlers.SysLogHandler) and\
-                'address' in config:
+            elif issubclass(klass, logging.handlers.SysLogHandler) and \
+                    'address' in config:
                 config['address'] = self.as_tuple(config['address'])
-            factory = klass
-        props = config.pop('.', None)
-        kwargs = {k: config[k] for k in config if valid_ident(k)}
+            if issubclass(klass, logging.handlers.QueueHandler):
+                factory = functools.partial(self._configure_queue_handler, klass)
+            else:
+                factory = klass
+        kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))}
         try:
             result = factory(**kwargs)
         except TypeError as te:
@@ -759,6 +889,7 @@ class DictConfigurator(BaseConfigurator):
             result.setLevel(logging._checkLevel(level))
         if filters:
             self.add_filters(result, filters)
+        props = config.pop('.', None)
         if props:
             for name, value in props.items():
                 setattr(result, name, value)
@@ -794,6 +925,7 @@ class DictConfigurator(BaseConfigurator):
         """Configure a non-root logger from a dictionary."""
         logger = logging.getLogger(name)
         self.common_logger_config(logger, config, incremental)
+        logger.disabled = False
         propagate = config.get('propagate', None)
         if propagate is not None:
             logger.propagate = propagate

Lib/logging/handlers.py

@@ -187,15 +187,18 @@ class RotatingFileHandler(BaseRotatingHandler):
         Basically, see if the supplied record would cause the file to exceed
         the size limit we have.
         """
-        # See bpo-45401: Never rollover anything other than regular files
-        if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
-            return False
         if self.stream is None:                 # delay was set...
             self.stream = self._open()
         if self.maxBytes > 0:                   # are we rolling over?
+            pos = self.stream.tell()
+            if not pos:
+                # gh-116263: Never rollover an empty file
+                return False
             msg = "%s\n" % self.format(record)
-            self.stream.seek(0, 2)  #due to non-posix-compliant Windows feature
-            if self.stream.tell() + len(msg) >= self.maxBytes:
+            if pos + len(msg) >= self.maxBytes:
+                # See bpo-45401: Never rollover anything other than regular files
+                if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
+                    return False
                 return True
         return False
 
@@ -232,19 +235,19 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         if self.when == 'S':
             self.interval = 1 # one second
             self.suffix = "%Y-%m-%d_%H-%M-%S"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(?!\d)"
         elif self.when == 'M':
             self.interval = 60 # one minute
             self.suffix = "%Y-%m-%d_%H-%M"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(?!\d)"
         elif self.when == 'H':
             self.interval = 60 * 60 # one hour
             self.suffix = "%Y-%m-%d_%H"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}_\d{2}(?!\d)"
         elif self.when == 'D' or self.when == 'MIDNIGHT':
             self.interval = 60 * 60 * 24 # one day
             self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}(?!\d)"
         elif self.when.startswith('W'):
             self.interval = 60 * 60 * 24 * 7 # one week
             if len(self.when) != 2:
@@ -253,11 +256,17 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                 raise ValueError("Invalid day specified for weekly rollover: %s" % self.when)
             self.dayOfWeek = int(self.when[1])
             self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}(?!\d)"
         else:
             raise ValueError("Invalid rollover interval specified: %s" % self.when)
 
-        self.extMatch = re.compile(self.extMatch, re.ASCII)
+        # extMatch is a pattern for matching a datetime suffix in a file name.
+        # After custom naming, it is no longer guaranteed to be separated by
+        # periods from other parts of the filename.  The lookup statements
+        # (?<!\d) and (?!\d) ensure that the datetime suffix (which itself
+        # starts and ends with digits) is not preceded or followed by digits.
+        # This reduces the number of false matches and improves performance.
+        self.extMatch = re.compile(extMatch, re.ASCII)
         self.interval = self.interval * interval # multiply by units requested
         # The following line added because the filename passed in could be a
         # path object (see Issue #27493), but self.baseFilename will be a string
@@ -295,11 +304,11 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                 rotate_ts = _MIDNIGHT
             else:
                 rotate_ts = ((self.atTime.hour * 60 + self.atTime.minute)*60 +
-                    self.atTime.second)
+                             self.atTime.second)
 
             r = rotate_ts - ((currentHour * 60 + currentMinute) * 60 +
-                currentSecond)
-            if r < 0:
+                             currentSecond)
+            if r <= 0:
                 # Rotate time is before the current time (for example when
                 # self.rotateAt is 13:45 and it now 14:15), rotation is
                 # tomorrow.
@@ -328,17 +337,21 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                         daysToWait = self.dayOfWeek - day
                     else:
                         daysToWait = 6 - day + self.dayOfWeek + 1
-                    newRolloverAt = result + (daysToWait * (60 * 60 * 24))
-                    if not self.utc:
-                        dstNow = t[-1]
-                        dstAtRollover = time.localtime(newRolloverAt)[-1]
-                        if dstNow != dstAtRollover:
-                            if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
-                                addend = -3600
-                            else:           # DST bows out before next rollover, so we need to add an hour
-                                addend = 3600
-                            newRolloverAt += addend
-                    result = newRolloverAt
+                    result += daysToWait * _MIDNIGHT
+                result += self.interval - _MIDNIGHT * 7
+            else:
+                result += self.interval - _MIDNIGHT
+            if not self.utc:
+                dstNow = t[-1]
+                dstAtRollover = time.localtime(result)[-1]
+                if dstNow != dstAtRollover:
+                    if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
+                        addend = -3600
+                        if not time.localtime(result-3600)[-1]:
+                            addend = 0
+                    else:           # DST bows out before next rollover, so we need to add an hour
+                        addend = 3600
+                    result += addend
         return result
 
     def shouldRollover(self, record):
@@ -348,11 +361,15 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         record is not used, as we are just comparing times, but it is needed so
         the method signatures are the same
         """
-        # See bpo-45401: Never rollover anything other than regular files
-        if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
-            return False
         t = int(time.time())
         if t >= self.rolloverAt:
+            # See #89564: Never rollover anything other than regular files
+            if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
+                # The file is not a regular file, so do not rollover, but do
+                # set the next rollover time to avoid repeated checks.
+                self.rolloverAt = self.computeRollover(t)
+                return False
+
             return True
         return False
 
@@ -365,32 +382,28 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         dirName, baseName = os.path.split(self.baseFilename)
         fileNames = os.listdir(dirName)
         result = []
-        # See bpo-44753: Don't use the extension when computing the prefix.
-        n, e = os.path.splitext(baseName)
-        prefix = n + '.'
-        plen = len(prefix)
-        for fileName in fileNames:
-            if self.namer is None:
-                # Our files will always start with baseName
-                if not fileName.startswith(baseName):
-                    continue
-            else:
-                # Our files could be just about anything after custom naming, but
-                # likely candidates are of the form
-                # foo.log.DATETIME_SUFFIX or foo.DATETIME_SUFFIX.log
-                if (not fileName.startswith(baseName) and fileName.endswith(e) and
-                    len(fileName) > (plen + 1) and not fileName[plen+1].isdigit()):
-                    continue
-
-            if fileName[:plen] == prefix:
-                suffix = fileName[plen:]
-                # See bpo-45628: The date/time suffix could be anywhere in the
-                # filename
-                parts = suffix.split('.')
-                for part in parts:
-                    if self.extMatch.match(part):
+        if self.namer is None:
+            prefix = baseName + '.'
+            plen = len(prefix)
+            for fileName in fileNames:
+                if fileName[:plen] == prefix:
+                    suffix = fileName[plen:]
+                    if self.extMatch.fullmatch(suffix):
+                        result.append(os.path.join(dirName, fileName))
+        else:
+            for fileName in fileNames:
+                # Our files could be just about anything after custom naming,
+                # but they should contain the datetime suffix.
+                # Try to find the datetime suffix in the file name and verify
+                # that the file name can be generated by this handler.
+                m = self.extMatch.search(fileName)
+                while m:
+                    dfn = self.namer(self.baseFilename + "." + m[0])
+                    if os.path.basename(dfn) == fileName:
                         result.append(os.path.join(dirName, fileName))
                         break
+                    m = self.extMatch.search(fileName, m.start() + 1)
+
         if len(result) < self.backupCount:
             result = []
         else:
@@ -406,17 +419,14 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         then we have to get a list of matching filenames, sort them and remove
         the one with the oldest suffix.
         """
-        if self.stream:
-            self.stream.close()
-            self.stream = None
         # get the time that this sequence started at and make it a TimeTuple
         currentTime = int(time.time())
-        dstNow = time.localtime(currentTime)[-1]
         t = self.rolloverAt - self.interval
         if self.utc:
             timeTuple = time.gmtime(t)
         else:
             timeTuple = time.localtime(t)
+            dstNow = time.localtime(currentTime)[-1]
             dstThen = timeTuple[-1]
             if dstNow != dstThen:
                 if dstNow:
@@ -427,26 +437,19 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         dfn = self.rotation_filename(self.baseFilename + "." +
                                      time.strftime(self.suffix, timeTuple))
         if os.path.exists(dfn):
-            os.remove(dfn)
+            # Already rolled over.
+            return
+
+        if self.stream:
+            self.stream.close()
+            self.stream = None
         self.rotate(self.baseFilename, dfn)
         if self.backupCount > 0:
             for s in self.getFilesToDelete():
                 os.remove(s)
         if not self.delay:
             self.stream = self._open()
-        newRolloverAt = self.computeRollover(currentTime)
-        while newRolloverAt <= currentTime:
-            newRolloverAt = newRolloverAt + self.interval
-        #If DST changes and midnight or weekly rollover, adjust for this.
-        if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc:
-            dstAtRollover = time.localtime(newRolloverAt)[-1]
-            if dstNow != dstAtRollover:
-                if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
-                    addend = -3600
-                else:           # DST bows out before next rollover, so we need to add an hour
-                    addend = 3600
-                newRolloverAt += addend
-        self.rolloverAt = newRolloverAt
+        self.rolloverAt = self.computeRollover(currentTime)
 
 class WatchedFileHandler(logging.FileHandler):
     """
@@ -800,7 +803,7 @@ class SysLogHandler(logging.Handler):
         "panic":    LOG_EMERG,      #  DEPRECATED
         "warn":     LOG_WARNING,    #  DEPRECATED
         "warning":  LOG_WARNING,
-        }
+    }
 
     facility_names = {
         "auth":         LOG_AUTH,
@@ -827,12 +830,10 @@ class SysLogHandler(logging.Handler):
         "local5":       LOG_LOCAL5,
         "local6":       LOG_LOCAL6,
         "local7":       LOG_LOCAL7,
-        }
+    }
 
-    #The map below appears to be trivially lowercasing the key. However,
-    #there's more to it than meets the eye - in some locales, lowercasing
-    #gives unexpected results. See SF #1524081: in the Turkish locale,
-    #"INFO".lower() != "info"
+    # Originally added to work around GH-43683. Unnecessary since GH-50043 but kept
+    # for backwards compatibility.
     priority_map = {
         "DEBUG" : "debug",
         "INFO" : "info",
@@ -859,12 +860,49 @@ class SysLogHandler(logging.Handler):
         self.address = address
         self.facility = facility
         self.socktype = socktype
+        self.socket = None
+        self.createSocket()
+
+    def _connect_unixsocket(self, address):
+        use_socktype = self.socktype
+        if use_socktype is None:
+            use_socktype = socket.SOCK_DGRAM
+        self.socket = socket.socket(socket.AF_UNIX, use_socktype)
+        try:
+            self.socket.connect(address)
+            # it worked, so set self.socktype to the used type
+            self.socktype = use_socktype
+        except OSError:
+            self.socket.close()
+            if self.socktype is not None:
+                # user didn't specify falling back, so fail
+                raise
+            use_socktype = socket.SOCK_STREAM
+            self.socket = socket.socket(socket.AF_UNIX, use_socktype)
+            try:
+                self.socket.connect(address)
+                # it worked, so set self.socktype to the used type
+                self.socktype = use_socktype
+            except OSError:
+                self.socket.close()
+                raise
+
+    def createSocket(self):
+        """
+        Try to create a socket and, if it's not a datagram socket, connect it
+        to the other end. This method is called during handler initialization,
+        but it's not regarded as an error if the other end isn't listening yet
+        --- the method will be called again when emitting an event,
+        if there is no socket at that point.
+        """
+        address = self.address
+        socktype = self.socktype
 
         if isinstance(address, str):
             self.unixsocket = True
             # Syslog server may be unavailable during handler initialisation.
             # C's openlog() function also ignores connection errors.
-            # Moreover, we ignore these errors while logging, so it not worse
+            # Moreover, we ignore these errors while logging, so it's not worse
             # to ignore it also here.
             try:
                 self._connect_unixsocket(address)
@@ -895,30 +933,6 @@ class SysLogHandler(logging.Handler):
             self.socket = sock
             self.socktype = socktype
 
-    def _connect_unixsocket(self, address):
-        use_socktype = self.socktype
-        if use_socktype is None:
-            use_socktype = socket.SOCK_DGRAM
-        self.socket = socket.socket(socket.AF_UNIX, use_socktype)
-        try:
-            self.socket.connect(address)
-            # it worked, so set self.socktype to the used type
-            self.socktype = use_socktype
-        except OSError:
-            self.socket.close()
-            if self.socktype is not None:
-                # user didn't specify falling back, so fail
-                raise
-            use_socktype = socket.SOCK_STREAM
-            self.socket = socket.socket(socket.AF_UNIX, use_socktype)
-            try:
-                self.socket.connect(address)
-                # it worked, so set self.socktype to the used type
-                self.socktype = use_socktype
-            except OSError:
-                self.socket.close()
-                raise
-
     def encodePriority(self, facility, priority):
         """
         Encode the facility and priority. You can pass in strings or
@@ -938,7 +952,10 @@ class SysLogHandler(logging.Handler):
         """
         self.acquire()
         try:
-            self.socket.close()
+            sock = self.socket
+            if sock:
+                self.socket = None
+                sock.close()
             logging.Handler.close(self)
         finally:
             self.release()
@@ -978,6 +995,10 @@ class SysLogHandler(logging.Handler):
             # Message is a string. Convert to bytes as required by RFC 5424
             msg = msg.encode('utf-8')
             msg = prio + msg
+
+            if not self.socket:
+                self.createSocket()
+
             if self.unixsocket:
                 try:
                     self.socket.send(msg)
@@ -1094,7 +1115,16 @@ class NTEventLogHandler(logging.Handler):
                 dllname = os.path.join(dllname[0], r'win32service.pyd')
             self.dllname = dllname
             self.logtype = logtype
-            self._welu.AddSourceToRegistry(appname, dllname, logtype)
+            # Administrative privileges are required to add a source to the registry.
+            # This may not be available for a user that just wants to add to an
+            # existing source - handle this specific case.
+            try:
+                self._welu.AddSourceToRegistry(appname, dllname, logtype)
+            except Exception as e:
+                # This will probably be a pywintypes.error. Only raise if it's not
+                # an "access denied" error, else let it pass
+                if getattr(e, 'winerror', None) != 5:  # not access denied
+                    raise
             self.deftype = win32evtlog.EVENTLOG_ERROR_TYPE
             self.typemap = {
                 logging.DEBUG   : win32evtlog.EVENTLOG_INFORMATION_TYPE,
@@ -1102,10 +1132,10 @@ class NTEventLogHandler(logging.Handler):
                 logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE,
                 logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
                 logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
-         }
+            }
         except ImportError:
-            print("The Python Win32 extensions for NT (service, event "\
-                        "logging) appear not to be available.")
+            print("The Python Win32 extensions for NT (service, event " \
+                  "logging) appear not to be available.")
             self._welu = None
 
     def getMessageID(self, record):
@@ -1348,7 +1378,7 @@ class MemoryHandler(BufferingHandler):
         Check for buffer full or a record at the flushLevel or higher.
         """
         return (len(self.buffer) >= self.capacity) or \
-                (record.levelno >= self.flushLevel)
+            (record.levelno >= self.flushLevel)
 
     def setTarget(self, target):
         """
@@ -1366,7 +1396,7 @@ class MemoryHandler(BufferingHandler):
         records to the target, if there is one. Override if you want
         different behaviour.
 
-        The record buffer is also cleared by this operation.
+        The record buffer is only cleared if a target has been set.
         """
         self.acquire()
         try:
@@ -1411,6 +1441,7 @@ class QueueHandler(logging.Handler):
         """
         logging.Handler.__init__(self)
         self.queue = queue
+        self.listener = None  # will be set to listener if configured via dictConfig()
 
     def enqueue(self, record):
         """
@@ -1424,12 +1455,15 @@ class QueueHandler(logging.Handler):
 
     def prepare(self, record):
         """
-        Prepares a record for queuing. The object returned by this method is
+        Prepare a record for queuing. The object returned by this method is
         enqueued.
 
-        The base implementation formats the record to merge the message
-        and arguments, and removes unpickleable items from the record
-        in-place.
+        The base implementation formats the record to merge the message and
+        arguments, and removes unpickleable items from the record in-place.
+        Specifically, it overwrites the record's `msg` and
+        `message` attributes with the merged message (obtained by
+        calling the handler's `format` method), and sets the `args`,
+        `exc_info` and `exc_text` attributes to None.
 
         You might want to override this method if you want to convert
         the record to a dict or JSON string, or send a modified copy
@@ -1439,7 +1473,7 @@ class QueueHandler(logging.Handler):
         # (if there's exception data), and also returns the formatted
         # message. We can then use this to replace the original
         # msg + args, as these might be unpickleable. We also zap the
-        # exc_info and exc_text attributes, as they are no longer
+        # exc_info, exc_text and stack_info attributes, as they are no longer
         # needed and, if not None, will typically not be pickleable.
         msg = self.format(record)
         # bpo-35726: make copy of record to avoid affecting other handlers in the chain.
@@ -1449,6 +1483,7 @@ class QueueHandler(logging.Handler):
         record.args = None
         record.exc_info = None
         record.exc_text = None
+        record.stack_info = None
         return record
 
     def emit(self, record):