Update some tests from 3.13.7 (#6108)

* Update `test_opcache.py`

* Update test_optparse.py

* Add some missing folders & test files

* Update `test_long.py` and impl "is_integer" for int

* Update `support/hypothesis_helper.py` from 3.13.7

* Update test_binascii

* Update test_math

* Add `test_math_property.py`

* Update `test_property.py` from 3.13.7

* Update `test_cmath.py` from 3.13.7

* Unmark passing tests

* Update `test_ucn.py` from 3.13.7

* Mark failing tests

* Add `site-packages` dir

.coderabbit.yml

@@ -0,0 +1,3 @@
+reviews:
+  path_filters:
+    - "!Lib/**"

.cspell.dict/cpython.txt

@@ -0,0 +1,61 @@
+argtypes
+asdl
+asname
+augassign
+badsyntax
+basetype
+boolop
+bxor
+cached_tsver
+cellarg
+cellvar
+cellvars
+cmpop
+denom
+dictoffset
+elts
+excepthandler
+fileutils
+finalbody
+formatfloat
+freevar
+freevars
+fromlist
+heaptype
+HIGHRES
+IMMUTABLETYPE
+kwonlyarg
+kwonlyargs
+lasti
+linearise
+maxdepth
+mult
+nkwargs
+noraise
+numer
+orelse
+pathconfig
+patma
+posonlyarg
+posonlyargs
+prec
+preinitialized
+PYTHREAD_NAME
+SA_ONSTACK
+stackdepth
+stringlib
+structseq
+subparams
+tok_oldval
+tvars
+unaryop
+unparse
+unparser
+VARKEYWORDS
+varkwarg
+wbits
+weakreflist
+withitem
+withs
+xstat
+XXPRIME

.cspell.dict/python-more.txt

@@ -0,0 +1,265 @@
+abiflags
+abstractmethods
+aenter
+aexit
+aiter
+anext
+appendleft
+argcount
+arrayiterator
+arraytype
+asend
+asyncgen
+athrow
+backslashreplace
+baserepl
+basicsize
+bdfl
+bigcharset
+bignum
+bivariant
+breakpointhook
+cformat
+chunksize
+classcell
+closefd
+closesocket
+codepoint
+codepoints
+codesize
+contextvar
+cpython
+cratio
+dealloc
+debugbuild
+decompressor
+defaultaction
+descr
+dictcomp
+dictitems
+dictkeys
+dictview
+digestmod
+dllhandle
+docstring
+docstrings
+dunder
+endianness
+endpos
+eventmask
+excepthook
+exceptiongroup
+exitfuncs
+extendleft
+fastlocals
+fdel
+fedcba
+fget
+fileencoding
+fillchar
+fillvalue
+finallyhandler
+firstiter
+firstlineno
+fnctl
+frombytes
+fromhex
+fromunicode
+fset
+fspath
+fstring
+fstrings
+ftruncate
+genexpr
+getattro
+getcodesize
+getdefaultencoding
+getfilesystemencodeerrors
+getfilesystemencoding
+getformat
+getframe
+getframemodulename
+getnewargs
+getpip
+getrandom
+getrecursionlimit
+getrefcount
+getsizeof
+getswitchinterval
+getweakrefcount
+getweakrefs
+getwindowsversion
+gmtoff
+groupdict
+groupindex
+hamt
+hostnames
+idfunc
+idiv
+idxs
+impls
+indexgroup
+infj
+instancecheck
+instanceof
+irepeat
+isabstractmethod
+isbytes
+iscased
+isfinal
+istext
+itemiterator
+itemsize
+iternext
+keepends
+keyfunc
+keyiterator
+kwarg
+kwargs
+kwdefaults
+kwonlyargcount
+lastgroup
+lastindex
+linearization
+linearize
+listcomp
+longrange
+lvalue
+mappingproxy
+maskpri
+maxdigits
+MAXGROUPS
+MAXREPEAT
+maxsplit
+maxunicode
+memoryview
+memoryviewiterator
+metaclass
+metaclasses
+metatype
+mformat
+mro
+mros
+multiarch
+namereplace
+nanj
+nbytes
+ncallbacks
+ndigits
+ndim
+nldecoder
+nlocals
+NOARGS
+nonbytes
+Nonprintable
+origname
+ospath
+pendingcr
+phello
+platlibdir
+popleft
+posixsubprocess
+posonly
+posonlyargcount
+prepending
+profilefunc
+pycache
+pycodecs
+pycs
+pyexpat
+PYTHONBREAKPOINT
+PYTHONDEBUG
+PYTHONDONTWRITEBYTECODE
+PYTHONHASHSEED
+PYTHONHOME
+PYTHONINSPECT
+PYTHONINTMAXSTRDIGITS
+PYTHONNOUSERSITE
+PYTHONOPTIMIZE
+PYTHONPATH
+PYTHONPATH
+PYTHONSAFEPATH
+PYTHONUNBUFFERED
+PYTHONVERBOSE
+PYTHONWARNDEFAULTENCODING
+PYTHONWARNINGS
+pytraverse
+PYVENV
+qualname
+quotetabs
+radd
+rdiv
+rdivmod
+readall
+readbuffer
+reconstructor
+refcnt
+releaselevel
+reverseitemiterator
+reverseiterator
+reversekeyiterator
+reversevalueiterator
+rfloordiv
+rlshift
+rmod
+rpow
+rrshift
+rsub
+rtruediv
+rvalue
+scproxy
+seennl
+setattro
+setcomp
+setrecursionlimit
+setswitchinterval
+showwarnmsg
+signum
+slotnames
+STACKLESS
+stacklevel
+stacksize
+startpos
+subclassable
+subclasscheck
+subclasshook
+suboffset
+suboffsets
+SUBPATTERN
+sumprod
+surrogateescape
+surrogatepass
+sysconf
+sysconfigdata
+sysvars
+teedata
+thisclass
+titlecased
+tkapp
+tobytes
+tolist
+toreadonly
+TPFLAGS
+tracefunc
+unimportable
+unionable
+unraisablehook
+unsliceable
+urandom
+valueiterator
+vararg
+varargs
+varnames
+warningregistry
+warnmsg
+warnoptions
+warnopts
+weaklist
+weakproxy
+weakrefs
+winver
+withdata
+xmlcharrefreplace
+xoptions
+xopts
+yieldfrom

.cspell.dict/rust-more.txt

@@ -0,0 +1,86 @@
+ahash
+arrayvec
+bidi
+biguint
+bindgen
+bitand
+bitflags
+bitor
+bitxor
+bstr
+byteorder
+byteset
+caseless
+chrono
+consts
+cranelift
+cstring
+datelike
+deserializer
+deserializers
+fdiv
+flamescope
+flate2
+fract
+getres
+hasher
+hexf
+hexversion
+idents
+illumos
+indexmap
+insta
+keccak
+lalrpop
+lexopt
+libc
+libcall
+libloading
+libz
+longlong
+Manually
+maplit
+memmap
+memmem
+metas
+modpow
+msvc
+muldiv
+nanos
+nonoverlapping
+objclass
+peekable
+powc
+powf
+powi
+prepended
+punct
+replacen
+rmatch
+rposition
+rsplitn
+rustc
+rustfmt
+rustyline
+seedable
+seekfrom
+siphash
+siphasher
+splitn
+subsec
+thiserror
+timelike
+timsort
+trai
+ulonglong
+unic
+unistd
+unraw
+unsync
+wasip1
+wasip2
+wasmbind
+wasmtime
+widestring
+winapi
+winsock

.cspell.json

@@ -1,210 +1,85 @@
 // See: https://github.com/streetsidesoftware/cspell/tree/master/packages/cspell
 {
   "version": "0.2",
+  "import": [
+    "@cspell/dict-en_us/cspell-ext.json",
+    // "@cspell/dict-cpp/cspell-ext.json",
+    "@cspell/dict-python/cspell-ext.json",
+    "@cspell/dict-rust/cspell-ext.json",
+    "@cspell/dict-win32/cspell-ext.json",
+    "@cspell/dict-shell/cspell-ext.json",
+  ],
   // language - current active spelling language
   "language": "en",
   // dictionaries - list of the names of the dictionaries to use
   "dictionaries": [
+    "cpython",  // Sometimes keeping same terms with cpython is easy
+    "python-more",  // Python API terms not listed in python
+    "rust-more",  // Rust API terms not listed in rust
     "en_US",
     "softwareTerms",
     "c",
     "cpp",
     "python",
-    "python-custom",
     "rust",
-    "unix",
-    "posix",
-    "winapi"
+    "shell",
+    "win32"
   ],
   // dictionaryDefinitions - this list defines any custom dictionaries to use
-  "dictionaryDefinitions": [],
+  "dictionaryDefinitions": [
+    {
+      "name": "cpython",
+      "path": "./.cspell.dict/cpython.txt"
+    },
+    {
+      "name": "python-more",
+      "path": "./.cspell.dict/python-more.txt"
+    },
+    {
+      "name": "rust-more",
+      "path": "./.cspell.dict/rust-more.txt"
+    }
+  ],
   "ignorePaths": [
     "**/__pycache__/**",
+    "target/**",
     "Lib/**"
   ],
   // words - list of words to be always considered correct
   "words": [
-    // Rust
-    "ahash",
-    "bidi",
-    "biguint",
-    "bindgen",
-    "bitflags",
-    "bstr",
-    "byteorder",
-    "chrono",
-    "consts",
-    "cstring",
-    "flate2",
-    "fract",
-    "hasher",
-    "idents",
-    "indexmap",
-    "insta",
-    "keccak",
-    "lalrpop",
-    "libc",
-    "libz",
-    "longlong",
-    "Manually",
-    "maplit",
-    "memmap",
-    "metas",
-    "modpow",
-    "nanos",
-    "objclass",
-    "peekable",
-    "powc",
-    "powf",
-    "prepended",
-    "punct",
-    "replacen",
-    "rsplitn",
-    "rustc",
-    "rustfmt",
-    "seekfrom",
-    "splitn",
-    "subsec",
-    "timsort",
-    "trai",
-    "ulonglong",
-    "unic",
-    "unistd",
-    "winapi",
-    "winsock",
-    // Python
-    "abstractmethods",
-    "aiter",
-    "anext",
-    "arrayiterator",
-    "arraytype",
-    "asend",
-    "athrow",
-    "basicsize",
-    "cformat",
-    "classcell",
-    "closesocket",
-    "codepoint",
-    "codepoints",
-    "cpython",
-    "decompressor",
-    "defaultaction",
-    "descr",
-    "dictcomp",
-    "dictitems",
-    "dictkeys",
-    "dictview",
-    "docstring",
-    "docstrings",
-    "dunder",
-    "eventmask",
-    "fdel",
-    "fget",
-    "fileencoding",
-    "fillchar",
-    "finallyhandler",
-    "frombytes",
-    "fromhex",
-    "fromunicode",
-    "fset",
-    "fspath",
-    "fstring",
-    "fstrings",
-    "genexpr",
-    "getattro",
-    "getformat",
-    "getnewargs",
-    "getweakrefcount",
-    "getweakrefs",
-    "hostnames",
-    "idiv",
-    "impls",
-    "infj",
-    "instancecheck",
-    "instanceof",
-    "isabstractmethod",
-    "itemiterator",
-    "itemsize",
-    "iternext",
-    "keyiterator",
-    "kwarg",
-    "kwargs",
-    "linearization",
-    "linearize",
-    "listcomp",
-    "mappingproxy",
-    "maxsplit",
-    "memoryview",
-    "memoryviewiterator",
-    "metaclass",
-    "metaclasses",
-    "metatype",
-    "mro",
-    "mros",
-    "nanj",
-    "ndigits",
-    "ndim",
-    "nonbytes",
-    "origname",
-    "posixsubprocess",
-    "pyexpat",
-    "PYTHONDEBUG",
-    "PYTHONHOME",
-    "PYTHONINSPECT",
-    "PYTHONOPTIMIZE",
-    "PYTHONPATH",
-    "PYTHONPATH",
-    "PYTHONVERBOSE",
-    "PYTHONWARNINGS",
-    "qualname",
-    "radd",
-    "rdiv",
-    "rdivmod",
-    "reconstructor",
-    "reversevalueiterator",
-    "rfloordiv",
-    "rlshift",
-    "rmod",
-    "rpow",
-    "rrshift",
-    "rsub",
-    "rtruediv",
-    "scproxy",
-    "setattro",
-    "setcomp",
-    "showwarnmsg",
-    "warnmsg",
-    "stacklevel",
-    "subclasscheck",
-    "subclasshook",
-    "unionable",
-    "unraisablehook",
-    "valueiterator",
-    "vararg",
-    "varargs",
-    "varnames",
-    "warningregistry",
-    "warnopts",
-    "weakproxy",
-    "xopts",
-    // RustPython
+    "RUSTPYTHONPATH",
+    // RustPython terms
+    "aiterable",
+    "alnum",
     "baseclass",
+    "boxvec",
     "Bytecode",
     "cfgs",
     "codegen",
+    "coro",
     "dedentations",
     "dedents",
     "deduped",
+    "downcastable",
     "downcasted",
     "dumpable",
+    "emscripten",
+    "excs",
+    "finalizer",
     "GetSet",
+    "groupref",
     "internable",
+    "jitted",
+    "jitting",
+    "lossily",
     "makeunicodedata",
     "miri",
     "notrace",
+    "openat",
     "pyarg",
     "pyarg",
     "pyargs",
+    "pyast",
     "PyAttr",
     "pyc",
     "PyClass",
@@ -213,6 +88,8 @@
     "PyFunction",
     "pygetset",
     "pyimpl",
+    "pylib",
+    "pymath",
     "pymember",
     "PyMethod",
     "PyModule",
@@ -225,6 +102,7 @@
     "PyResult",
     "pyslot",
     "PyStaticMethod",
+    "pystone",
     "pystr",
     "pystruct",
     "pystructseq",
@@ -232,57 +110,33 @@
     "reducelib",
     "richcompare",
     "RustPython",
+    "significand",
     "struc",
+    "summands", // plural of summand
+    "sysmodule",
     "tracebacks",
     "typealiases",
-    "Unconstructible",
+    "unconstructible",
     "unhashable",
     "uninit",
     "unraisable",
+    "unresizable",
     "wasi",
     "zelf",
-    // cpython
-    "argtypes",
-    "asdl",
-    "asname",
-    "augassign",
-    "badsyntax",
-    "basetype",
-    "boolop",
-    "bxor",
-    "cellarg",
-    "cellvar",
-    "cellvars",
-    "cmpop",
-    "dictoffset",
-    "elts",
-    "excepthandler",
-    "finalbody",
-    "freevar",
-    "freevars",
-    "fromlist",
-    "heaptype",
-    "IMMUTABLETYPE",
-    "kwonlyarg",
-    "kwonlyargs",
-    "linearise",
-    "maxdepth",
-    "mult",
-    "nkwargs",
-    "orelse",
-    "patma",
-    "posonlyarg",
-    "posonlyargs",
-    "prec",
-    "stackdepth",
-    "unaryop",
-    "unparse",
-    "unparser",
-    "VARKEYWORDS",
-    "varkwarg",
-    "wbits",
-    "withitem",
-    "withs"
+    // unix
+    "CLOEXEC",
+    "codeset",
+    "endgrent",
+    "gethrvtime",
+    "getrusage",
+    "nanosleep",
+    "sigaction",
+    "WRLCK",
+    // win32
+    "birthtime",
+    "IFEXEC",
+    // "stat"
+    "FIRMLINK"
   ],
   // flagWords - list of words to be always considered incorrect
   "flagWords": [

.devcontainer/Dockerfile

@@ -0,0 +1,6 @@
+FROM mcr.microsoft.com/vscode/devcontainers/rust:1-bullseye
+
+# Install clang
+RUN apt-get update \
+  && apt-get install -y clang \
+  && rm -rf /var/lib/apt/lists/*

.devcontainer/devcontainer.json

@@ -1,6 +1,25 @@
 {
-  "image": "mcr.microsoft.com/devcontainers/universal:2",
-  "features": {
-    "ghcr.io/devcontainers/features/rust:1": {}
-  }
+  "name": "Rust",
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+  "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "lldb.executable": "/usr/bin/lldb",
+        // VS Code don't watch files under ./target
+        "files.watcherExclude": {
+          "**/target/**": true
+        },
+        "extensions": [
+          "rust-lang.rust-analyzer",
+          "tamasfe.even-better-toml",
+          "vadimcn.vscode-lldb",
+          "mutantdino.resourcemonitor"
+        ]
+      }
+    }
+  },
+  "remoteUser": "vscode"
 }

.gemini/config.yaml

@@ -0,0 +1,2 @@
+ignore_patterns:
+  - "Lib/**"

.gitattributes

@@ -1,6 +1,7 @@
 Lib/**      linguist-vendored
-Cargo.lock  linguist-generated -merge
+Cargo.lock  linguist-generated
 *.snap      linguist-generated -merge
 vm/src/stdlib/ast/gen.rs    linguist-generated -merge
 Lib/*.py    text working-tree-encoding=UTF-8 eol=LF
 **/*.rs     text working-tree-encoding=UTF-8 eol=LF
+*.pck       binary

.github/copilot-instructions.md

@@ -0,0 +1,212 @@
+# GitHub Copilot Instructions for RustPython
+
+This document provides guidelines for working with GitHub Copilot when contributing to the RustPython project.
+
+## Project Overview
+
+RustPython is a Python 3 interpreter written in Rust, implementing Python 3.13.0+ compatibility. The project aims to provide:
+
+- A complete Python-3 environment entirely in Rust (not CPython bindings)
+- A clean implementation without compatibility hacks
+- Cross-platform support, including WebAssembly compilation
+- The ability to embed Python scripting in Rust applications
+
+## Repository Structure
+
+- `src/` - Top-level code for the RustPython binary
+- `vm/` - The Python virtual machine implementation
+  - `builtins/` - Python built-in types and functions
+  - `stdlib/` - Essential standard library modules implemented in Rust, required to run the Python core
+- `compiler/` - Python compiler components
+  - `parser/` - Parser for converting Python source to AST
+  - `core/` - Bytecode representation in Rust structures
+  - `codegen/` - AST to bytecode compiler
+- `Lib/` - CPython's standard library in Python (copied from CPython). **IMPORTANT**: Do not edit this directory directly; The only allowed operation is copying files from CPython.
+- `derive/` - Rust macros for RustPython
+- `common/` - Common utilities
+- `extra_tests/` - Integration tests and snippets
+- `stdlib/` - Non-essential Python standard library modules implemented in Rust (useful but not required for core functionality)
+- `wasm/` - WebAssembly support
+- `jit/` - Experimental JIT compiler implementation
+- `pylib/` - Python standard library packaging (do not modify this directory directly - its contents are generated automatically)
+
+## Important Development Notes
+
+### Running Python Code
+
+When testing Python code, always use RustPython instead of the standard `python` command:
+
+```bash
+# Use this instead of python script.py
+cargo run -- script.py
+
+# For interactive REPL
+cargo run
+
+# With specific features
+cargo run --features ssl
+
+# Release mode (recommended for better performance)
+cargo run --release -- script.py
+```
+
+### Comparing with CPython
+
+When you need to compare behavior with CPython or run test suites:
+
+```bash
+# Use python command to explicitly run CPython
+python my_test_script.py
+
+# Run RustPython
+cargo run -- my_test_script.py
+```
+
+### Working with the Lib Directory
+
+The `Lib/` directory contains Python standard library files copied from the CPython repository. Important notes:
+
+- These files should be edited very conservatively
+- Modifications should be minimal and only to work around RustPython limitations
+- Tests in `Lib/test` often use one of the following markers:
+  - Add a `# TODO: RUSTPYTHON` comment when modifications are made
+  - `unittest.skip("TODO: RustPython <reason>")`
+  - `unittest.expectedFailure` with `# TODO: RUSTPYTHON <reason>` comment
+
+### Testing
+
+```bash
+# Run Rust unit tests
+cargo test --workspace --exclude rustpython_wasm
+
+# Run Python snippets tests
+cd extra_tests
+pytest -v
+
+# Run the Python test module
+cargo run --release -- -m test ${TEST_MODULE}
+cargo run --release -- -m test test_unicode # to test test_unicode.py
+
+# Run the Python test module with specific function
+cargo run --release -- -m test test_unicode -k test_unicode_escape
+```
+
+### Determining What to Implement
+
+Run `./whats_left.py` to get a list of unimplemented methods, which is helpful when looking for contribution opportunities.
+
+## Coding Guidelines
+
+### Rust Code
+
+- Follow the default rustfmt code style (`cargo fmt` to format)
+- **IMPORTANT**: Always run clippy to lint code (`cargo clippy`) before completing tasks. Fix any warnings or lints that are introduced by your changes
+- Follow Rust best practices for error handling and memory management
+- Use the macro system (`pyclass`, `pymodule`, `pyfunction`, etc.) when implementing Python functionality in Rust
+
+### Python Code
+
+- **IMPORTANT**: In most cases, Python code should not be edited. Bug fixes should be made through Rust code modifications only
+- Follow PEP 8 style for custom Python code
+- Use ruff for linting Python code
+- Minimize modifications to CPython standard library files
+
+## Integration Between Rust and Python
+
+The project provides several mechanisms for integration:
+
+- `pymodule` macro for creating Python modules in Rust
+- `pyclass` macro for implementing Python classes in Rust
+- `pyfunction` macro for exposing Rust functions to Python
+- `PyObjectRef` and other types for working with Python objects in Rust
+
+## Common Patterns
+
+### Implementing a Python Module in Rust
+
+```rust
+#[pymodule]
+mod mymodule {
+    use rustpython_vm::prelude::*;
+
+    #[pyfunction]
+    fn my_function(value: i32) -> i32 {
+        value * 2
+    }
+
+    #[pyattr]
+    #[pyclass(name = "MyClass")]
+    #[derive(Debug, PyPayload)]
+    struct MyClass {
+        value: usize,
+    }
+
+    #[pyclass]
+    impl MyClass {
+        #[pymethod]
+        fn get_value(&self) -> usize {
+            self.value
+        }
+    }
+}
+```
+
+### Adding a Python Module to the Interpreter
+
+```rust
+vm.add_native_module(
+    "my_module_name".to_owned(),
+    Box::new(my_module::make_module),
+);
+```
+
+## Building for Different Targets
+
+### WebAssembly
+
+```bash
+# Build for WASM
+cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release
+```
+
+### JIT Support
+
+```bash
+# Enable JIT support
+cargo run --features jit
+```
+
+### SSL Support
+
+```bash
+# Enable SSL support
+cargo run --features ssl
+```
+
+## Test Code Modification Rules
+
+**CRITICAL: Test code modification restrictions**
+- NEVER comment out or delete any test code lines except for removing `@unittest.expectedFailure` decorators and upper TODO comments
+- NEVER modify test assertions, test logic, or test data
+- When a test cannot pass due to missing language features, keep it as expectedFailure and document the reason
+- The only acceptable modifications to test files are:
+  1. Removing `@unittest.expectedFailure` decorators and the upper TODO comments when tests actually pass
+  2. Adding `@unittest.expectedFailure` decorators when tests cannot be fixed
+
+**Examples of FORBIDDEN modifications:**
+- Commenting out test lines
+- Changing test assertions
+- Modifying test data or expected results
+- Removing test logic
+
+**Correct approach when tests fail due to unsupported syntax:**
+- Keep the test as `@unittest.expectedFailure`
+- Document that it requires PEP 695 support
+- Focus on tests that can be fixed through Rust code changes only
+
+## Documentation
+
+- Check the [architecture document](architecture/architecture.md) for a high-level overview
+- Read the [development guide](DEVELOPMENT.md) for detailed setup instructions
+- Generate documentation with `cargo doc --no-deps --all`
+- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/)

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Keep GitHub Actions up to date with GitHub's Dependabot...
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: weekly

.github/workflows/ci.yaml

@@ -4,6 +4,7 @@ on:
   pull_request:
     types: [unlabeled, opened, synchronize, reopened]
   merge_group:
+  workflow_dispatch:
 
 name: CI
 
@@ -15,26 +16,26 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl
   # Skip additional tests on Windows. They are checked on Linux and MacOS.
+  # test_glob: many failing tests
+  # test_io: many failing tests
+  # test_os: many failing tests
+  # test_pathlib: support.rmtree() failing
+  # test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)')
+  # test_venv: couple of failing tests
   WINDOWS_SKIPS: >-
-    test_datetime
     test_glob
-    test_importlib
     test_io
     test_os
+    test_rlcompleter
     test_pathlib
     test_posixpath
     test_venv
-  # configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417
-  # socketserver: seems related to configparser crash.
-  MACOS_SKIPS: >-
-    test_configparser
-    test_socketserver
   # PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently
   # only run on Linux to speed up the CI.
   PLATFORM_INDEPENDENT_TESTS: >-
-    test_argparse
+    test__colorize
     test_array
     test_asyncgen
     test_binop
@@ -59,7 +60,6 @@ env:
     test_dis
     test_enumerate
     test_exception_variations
-    test_exceptions
     test_float
     test_format
     test_fractions
@@ -100,12 +100,11 @@ env:
     test_tuple
     test_types
     test_unary
-    test_unicode
     test_unpack
     test_weakref
     test_yield_from
   # Python version targeted by the CI.
-  PYTHON_VERSION: "3.12.3"
+  PYTHON_VERSION: "3.13.1"
 
 jobs:
   rust_tests:
@@ -114,12 +113,13 @@ jobs:
       RUST_BACKTRACE: full
     name: Run rust tests
     runs-on:  ${{ matrix.os }}
+    timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 30 }}
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
@@ -128,6 +128,7 @@ jobs:
       - name: Set up the Windows environment
         shell: bash
         run: |
+          git config --system core.longpaths true
           cargo install --target-dir=target -v cargo-vcpkg
           cargo vcpkg -v build
         if: runner.os == 'Windows'
@@ -136,7 +137,7 @@ jobs:
         if: runner.os == 'macOS'
 
       - name: run clippy
-        run: cargo clippy ${{ env.CARGO_ARGS }} --workspace  --exclude rustpython_wasm -- -Dwarnings
+        run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets --exclude rustpython_wasm -- -Dwarnings
 
       - name: run rust tests
         run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }}
@@ -175,8 +176,9 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Ensure compilation on various targets
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: i686-unknown-linux-gnu
@@ -216,13 +218,6 @@ jobs:
       - name: Check compilation for freebsd
         run: cargo check --target x86_64-unknown-freebsd
 
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          target: wasm32-unknown-unknown
-
-      - name: Check compilation for wasm32
-        run: cargo check --target wasm32-unknown-unknown --no-default-features
-
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: x86_64-unknown-freebsd
@@ -230,12 +225,13 @@ jobs:
       - name: Check compilation for freeBSD
         run: cargo check --target x86_64-unknown-freebsd
 
-      - name: Prepare repository for redox compilation
-        run: bash scripts/redox/uncomment-cargo.sh
-      - name: Check compilation for Redox
-        uses: coolreader18/redoxer-action@v1
-        with:
-          command: check
+      # - name: Prepare repository for redox compilation
+      #   run: bash scripts/redox/uncomment-cargo.sh
+      # - name: Check compilation for Redox
+      #   uses: coolreader18/redoxer-action@v1
+      #   with:
+      #     command: check
+      #     args: --ignore-rust-version
 
   snippets_cpython:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
@@ -243,20 +239,22 @@ jobs:
       RUST_BACKTRACE: full
     name: Run snippets and cpython tests
     runs-on:  ${{ matrix.os }}
+    timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 30 }}
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Set up the Windows environment
         shell: bash
         run: |
+          git config --system core.longpaths true
           cargo install cargo-vcpkg
           cargo vcpkg build
         if: runner.os == 'Windows'
@@ -269,7 +267,7 @@ jobs:
       - name: build rustpython
         run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit
         if: runner.os != 'macOS'
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: run snippets
@@ -284,7 +282,7 @@ jobs:
         run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }}
       - if: runner.os == 'macOS'
         name: run cpython platform-dependent tests (MacOS)
-        run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.MACOS_SKIPS }}
+        run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }}
       - if: runner.os == 'Windows'
         name: run cpython platform-dependent tests (windows partial - fixme)
         run:
@@ -312,7 +310,7 @@ jobs:
     name: Check Rust code with rustfmt and clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
             components: rustfmt, clippy
@@ -320,42 +318,62 @@ jobs:
         run: cargo fmt --check
       - name: run clippy on wasm
         run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: install ruff
-        run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778
-      - name: run python lint
-        run: ruff extra_tests wasm examples  --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source
+        run: python -m pip install ruff==0.11.8
+      - name: Ensure docs generate no warnings
+        run: cargo doc
+      - name: run ruff check
+        run: ruff check --diff
+      - name: run ruff format
+        run: ruff format --check
       - name: install prettier
         run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH
       - name: check wasm code with prettier
         # prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506
         run: cd wasm && git ls-files -z | xargs -0 prettier --check -u
+      # Keep cspell check as the last step. This is optional test.
+      - name: install extra dictionaries
+        run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell
+      - name: spell checker
+        uses: streetsidesoftware/cspell-action@v7
+        with:
+          files: '**/*.rs'
+          incremental_files_only: true
 
   miri:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Run tests under miri
     runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      NIGHTLY_CHANNEL: nightly
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
       - uses: dtolnay/rust-toolchain@master
         with:
-            toolchain: nightly
+            toolchain: ${{ env.NIGHTLY_CHANNEL }}
             components: miri
 
       - uses: Swatinem/rust-cache@v2
+
       - name: Run tests under miri
+        run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test
+        env:
         # miri-ignore-leaks because the type-object circular reference means that there will always be
         # a memory leak, at least until we have proper cyclic gc
-        run: MIRIFLAGS='-Zmiri-ignore-leaks' cargo +nightly miri test -p rustpython-vm -- miri_test
+          MIRIFLAGS: '-Zmiri-ignore-leaks'
 
   wasm:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Check the WASM package and demo
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: Swatinem/rust-cache@v2
@@ -363,15 +381,18 @@ jobs:
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: install geckodriver
         run: |
-          wget https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz
+          wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz
           mkdir geckodriver
-          tar -xzf geckodriver-v0.34.0-linux64.tar.gz -C geckodriver
-      - uses: actions/setup-python@v4
+          tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - run: python -m pip install -r requirements.txt
         working-directory: ./wasm/tests
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
+        with:
+          cache: "npm"
+          cache-dependency-path: "wasm/demo/package-lock.json"
       - name: run test
         run: |
           export PATH=$PATH:`pwd`/../../geckodriver
@@ -380,6 +401,15 @@ jobs:
         env:
           NODE_OPTIONS: "--openssl-legacy-provider"
         working-directory: ./wasm/demo
+      - uses: mwilliamson/setup-wabt-action@v3
+        with: { wabt-version: "1.0.36" }
+      - name: check wasm32-unknown without js
+        run: |
+          cd wasm/wasm-unknown-test
+          cargo build --release --verbose
+          if wasm-objdump -xj Import target/wasm32-unknown-unknown/release/wasm_unknown_test.wasm; then
+            echo "ERROR: wasm32-unknown module expects imports from the host environment" >2
+          fi
       - name: build notebook demo
         if: github.ref == 'refs/heads/release'
         run: |
@@ -391,7 +421,7 @@ jobs:
         working-directory: ./wasm/notebook
       - name: Deploy demo to Github Pages
         if: success() && github.ref == 'refs/heads/release'
-        uses: peaceiris/actions-gh-pages@v2
+        uses: peaceiris/actions-gh-pages@v4
         env:
           ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
           PUBLISH_DIR: ./wasm/demo/dist
@@ -402,20 +432,21 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Run snippets and cpython tests on wasm-wasi
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
-          target: wasm32-wasi
+          target: wasm32-wasip1
 
       - uses: Swatinem/rust-cache@v2
       - name: Setup Wasmer
-        uses: wasmerio/setup-wasmer@v2
+        uses: wasmerio/setup-wasmer@v3
       - name: Install clang
         run: sudo apt-get update && sudo apt-get install clang -y
       - name: build rustpython
-        run: cargo build --release --target wasm32-wasi --features freeze-stdlib,stdlib --verbose
+        run: cargo build --release --target wasm32-wasip1 --features freeze-stdlib,stdlib --verbose
       - name: run snippets
-        run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py
+        run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py
       - name: run cpython unittest
-        run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py
+        run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py

.github/workflows/comment-commands.yml

@@ -0,0 +1,21 @@
+name: Comment Commands
+
+on:
+  issue_comment:
+    types: created
+
+jobs:
+  issue_assign:
+    if: (!github.event.issue.pull_request) && github.event.comment.body == 'take'
+    runs-on: ubuntu-latest
+
+    concurrency:
+      group: ${{ github.actor }}-issue-assign
+
+    permissions:
+      issues: write
+
+    steps:
+      # Using REST API and not `gh issue edit`. https://github.com/cli/cli/issues/6235#issuecomment-1243487651
+      - run: |
+          curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees

.github/workflows/cron-ci.yaml

@@ -2,12 +2,15 @@ on:
   schedule:
     - cron: '0 0 * * 6'
   workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/cron-ci.yaml
 
 name: Periodic checks/tasks
 
 env:
-  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
-  PYTHON_VERSION: "3.12.0"
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl,jit
+  PYTHON_VERSION: "3.13.1"
 
 jobs:
   # codecov collects code coverage data from the rust tests, python snippets and python test suite.
@@ -16,15 +19,15 @@ jobs:
     name: Collect code coverage data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: taiki-e/install-action@cargo-llvm-cov
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - run: sudo apt-get update && sudo apt-get -y install lcov
       - name: Run cargo-llvm-cov with Rust tests.
-        run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
+        run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,importlib,encodings,ssl,jit
       - name: Run cargo-llvm-cov with Python snippets.
         run: python scripts/cargo-llvm-cov.py
         continue-on-error: true
@@ -34,7 +37,7 @@ jobs:
       - name: Prepare code coverage data
         run: cargo llvm-cov report --lcov --output-path='codecov.lcov'
       - name: Upload to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           file: ./codecov.lcov
 
@@ -42,7 +45,7 @@ jobs:
     name: Collect regression test data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - name: build rustpython
         run: cargo build --release --verbose
@@ -71,9 +74,9 @@ jobs:
     name: Collect what is left data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: build rustpython
@@ -81,7 +84,7 @@ jobs:
       - name: Collect what is left data
         run: |
           chmod +x ./whats_left.py
-          ./whats_left.py > whats_left.temp
+          ./whats_left.py --features "ssl,sqlite" > whats_left.temp
         env:
           RUSTPYTHONPATH: ${{ github.workspace }}/Lib
       - name: Upload data to the website
@@ -97,6 +100,9 @@ jobs:
           cd website
           [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp
           cp ../whats_left.temp ./_data/whats_left.temp
+          rm ./_data/whats_left/modules.csv
+          echo -e "module" > ./_data/whats_left/modules.csv
+          cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv
           git add -A
           if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then
             git push
@@ -106,9 +112,9 @@ jobs:
     name: Collect benchmark data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - run: cargo install cargo-criterion

.github/workflows/release.yml

@@ -0,0 +1,173 @@
+name: Release
+
+on:
+  schedule:
+    # 9 AM UTC on every Monday
+    - cron: "0 9 * * Mon"
+  workflow_dispatch:
+    inputs:
+      pre-release:
+        type: boolean
+        description: Mark "Pre-Release"
+        required: false
+        default: true
+
+permissions:
+  contents: write
+
+env:
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl
+
+jobs:
+  build:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: i686-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: aarch64-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: armv7-unknown-linux-gnueabi
+#          - runner: ubuntu-latest
+#            target: s390x-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: powerpc64le-unknown-linux-gnu
+          - runner: macos-latest
+            target: aarch64-apple-darwin
+#          - runner: macos-latest
+#            target: x86_64-apple-darwin
+          - runner: windows-latest
+            target: x86_64-pc-windows-msvc
+#          - runner: windows-latest
+#            target: i686-pc-windows-msvc
+#          - runner: windows-latest
+#            target: aarch64-pc-windows-msvc
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: cargo-bins/cargo-binstall@main
+
+      - name: Set up Environment
+        shell: bash
+        run: rustup target add ${{ matrix.platform.target }}
+      - name: Set up Windows Environment
+        shell: bash
+        run: |
+          git config --global core.longpaths true
+          cargo install --target-dir=target -v cargo-vcpkg
+          cargo vcpkg -v build
+        if: runner.os == 'Windows'
+      - name: Set up MacOS Environment
+        run: brew install autoconf automake libtool
+        if: runner.os == 'macOS'
+
+      - name: Build RustPython
+        run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }}
+        if: runner.os == 'macOS'
+      - name: Build RustPython
+        run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }},jit
+        if: runner.os != 'macOS'
+
+      - name: Rename Binary
+        run: cp target/${{ matrix.platform.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}
+        if: runner.os != 'Windows'
+      - name: Rename Binary
+        run: cp target/${{ matrix.platform.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}.exe
+        if: runner.os == 'Windows'
+
+      - name: Upload Binary Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}
+          path: target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}*
+
+  build-wasm:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: wasm32-wasip1
+
+      - name: Build RustPython
+        run: cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release
+
+      - name: Rename Binary
+        run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm
+
+      - name: Upload Binary Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rustpython-release-wasm32-wasip1
+          path: target/rustpython-release-wasm32-wasip1.wasm
+
+      - name: install wasm-pack
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+      - uses: actions/setup-node@v4
+      - uses: mwilliamson/setup-wabt-action@v3
+        with: { wabt-version: "1.0.30" }
+      - name: build demo
+        run: |
+          npm install
+          npm run dist
+        env:
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        working-directory: ./wasm/demo
+      - name: build notebook demo
+        run: |
+          npm install
+          npm run dist
+          mv dist ../demo/dist/notebook
+        env:
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        working-directory: ./wasm/notebook
+      - name: Deploy demo to Github Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
+          publish_dir: ./wasm/demo/dist
+          external_repository: RustPython/demo
+          publish_branch: master
+
+  release:
+    runs-on: ubuntu-latest
+    needs: [build, build-wasm]
+    steps:
+      - name: Download Binary Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: bin
+          pattern: rustpython-*
+          merge-multiple: true
+
+      - name: List Binaries
+        run: |
+          ls -lah bin/
+          file bin/*
+      - name: Create Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref_name }}
+          run: ${{ github.run_number }}
+        run: |
+          if [[ "${{ github.event.inputs.pre-release }}" == "false" ]]; then
+            RELEASE_TYPE_NAME=Release
+            PRERELEASE_ARG=
+          else
+            RELEASE_TYPE_NAME=Pre-Release
+            PRERELEASE_ARG=--prerelease
+          fi
+          
+          today=$(date '+%Y-%m-%d')
+          gh release create "$today-$tag-$run" \
+              --repo="$GITHUB_REPOSITORY" \
+              --title="RustPython $RELEASE_TYPE_NAME $today-$tag #$run" \
+              --target="$tag" \
+              --generate-notes \
+              $PRERELEASE_ARG \
+              bin/rustpython-release-*

.gitignore

@@ -2,11 +2,11 @@
 /*/target
 **/*.rs.bk
 **/*.bytecode
-__pycache__
+__pycache__/
 **/*.pytest_cache
 .*sw*
 .repl_history.txt
-.vscode
+.vscode/
 wasm-pack.log
 .idea/
 .envrc
@@ -21,3 +21,8 @@ flamescope.json
 
 extra_tests/snippets/resources
 extra_tests/not_impl.py
+
+Lib/site-packages/*
+!Lib/site-packages/README.txt
+Lib/test/data/*
+!Lib/test/data/README

Cargo.toml

@@ -1,122 +1,46 @@
 [package]
 name = "rustpython"
-version = "0.4.0"
-authors = ["RustPython Team"]
-edition = "2021"
-rust-version = "1.75.0"
 description = "A python interpreter written in rust."
-repository = "https://github.com/RustPython/RustPython"
-license = "MIT"
 include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
-
-[workspace]
-resolver = "2"
-members = [
-    "compiler", "compiler/core", "compiler/codegen",
-    ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "wasm/lib", "derive-impl",
-]
-
-[workspace.dependencies]
-rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
-rustpython-compiler = { path = "compiler", version = "0.4.0" }
-rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }
-rustpython-common = { path = "common", version = "0.4.0" }
-rustpython-derive = { path = "derive", version = "0.4.0" }
-rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" }
-rustpython-jit = { path = "jit", version = "0.4.0" }
-rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" }
-rustpython-pylib = { path = "pylib", version = "0.4.0" }
-rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" }
-rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
-rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
-
-rustpython-literal = { version = "0.4.0" }
-rustpython-parser-core = { version = "0.4.0" }
-rustpython-parser = { version = "0.4.0" }
-rustpython-ast = { version = "0.4.0" }
-rustpython-format= { version = "0.4.0" }
-# rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
-# rustpython-literal = { path = "../RustPython-parser/literal" }
-# rustpython-parser-core = { path = "../RustPython-parser/core" }
-# rustpython-parser = { path = "../RustPython-parser/parser" }
-# rustpython-ast = { path = "../RustPython-parser/ast" }
-# rustpython-format = { path = "../RustPython-parser/format" }
-
-ahash = "0.8.11"
-ascii = "1.0"
-atty = "0.2.14"
-bitflags = "2.4.1"
-bstr = "0.2.17"
-cfg-if = "1.0"
-chrono = "0.4.37"
-crossbeam-utils = "0.8.19"
-flame = "0.2.2"
-glob = "0.3"
-hex = "0.4.3"
-indexmap = { version = "2.2.6", features = ["std"] }
-insta = "1.38.0"
-itertools = "0.11.0"
-is-macro = "0.3.0"
-junction = "1.0.0"
-libc = "0.2.153"
-log = "0.4.16"
-nix = { version = "0.27", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
-malachite-bigint = "0.2.0"
-malachite-q = "0.4.4"
-malachite-base = "0.4.4"
-memchr = "2.7.2"
-num-complex = "0.4.0"
-num-integer = "0.1.44"
-num-traits = "0.2"
-num_enum = "0.7"
-once_cell = "1.19.0"
-parking_lot = "0.12.1"
-paste = "1.0.7"
-rand = "0.8.5"
-rustyline = "14.0.0"
-serde = { version = "1.0.133", default-features = false }
-schannel = "0.1.22"
-static_assertions = "1.1"
-syn = "1.0.109"
-thiserror = "1.0"
-thread_local = "1.1.4"
-unicode_names2 = "1.2.0"
-widestring = "1.1.0"
-windows-sys = "0.52.0"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+license.workspace = true
 
 [features]
-default = ["threading", "stdlib", "zlib", "importlib"]
+default = ["threading", "stdlib", "stdio", "importlib"]
 importlib = ["rustpython-vm/importlib"]
 encodings = ["rustpython-vm/encodings"]
+stdio = ["rustpython-vm/stdio"]
 stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"]
 flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
 freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
 threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
-zlib = ["stdlib", "rustpython-stdlib/zlib"]
-bz2 = ["stdlib", "rustpython-stdlib/bz2"]
+sqlite = ["rustpython-stdlib/sqlite"]
 ssl = ["rustpython-stdlib/ssl"]
 ssl-vendor = ["ssl", "rustpython-stdlib/ssl-vendor"]
+tkinter = ["rustpython-stdlib/tkinter"]
+
+[build-dependencies]
+winresource = "0.1"
 
 [dependencies]
 rustpython-compiler = { workspace = true }
 rustpython-pylib = { workspace = true, optional = true }
 rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
 rustpython-vm = { workspace = true, features = ["compiler"] }
-rustpython-parser = { workspace = true }
+ruff_python_parser = { workspace = true }
 
-atty = { workspace = true }
 cfg-if = { workspace = true }
 log = { workspace = true }
 flame = { workspace = true, optional = true }
 
-clap = "2.34"
-dirs = { package = "dirs-next", version = "2.0.0" }
-env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
+lexopt = "0.3"
+dirs = { package = "dirs-next", version = "2.0" }
+env_logger = "0.11"
 flamescope = { version = "0.1.2", optional = true }
 
 [target.'cfg(windows)'.dependencies]
@@ -126,8 +50,8 @@ libc = { workspace = true }
 rustyline = { workspace = true }
 
 [dev-dependencies]
-criterion = { version = "0.3.5", features = ["html_reports"] }
-pyo3 = { version = "0.20.2", features = ["auto-initialize"] }
+criterion = { workspace = true }
+pyo3 = { version = "0.24", features = ["auto-initialize"] }
 
 [[bench]]
 name = "execution"
@@ -171,10 +95,143 @@ rev = "2024.02.14"
 [package.metadata.vcpkg.target]
 x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] }
 
+[package.metadata.packager]
+product-name = "RustPython"
+identifier = "com.rustpython.rustpython"
+description = "An open source Python 3 interpreter written in Rust"
+homepage = "https://rustpython.github.io/"
+license_file = "LICENSE"
+authors = ["RustPython Team"]
+publisher = "RustPython Team"
+resources = ["LICENSE", "README.md", "Lib"]
+icons = ["32x32.png"]
+
+[package.metadata.packager.nsis]
+installer_mode = "both"
+template = "installer-config/installer.nsi"
+
+[package.metadata.packager.wix]
+template = "installer-config/installer.wxs"
+
+
+[workspace]
+resolver = "2"
+members = [
+    "compiler",
+    "compiler/core",
+    "compiler/codegen",
+    "compiler/literal",
+    ".",
+    "common",
+    "derive",
+    "jit",
+    "vm",
+    "vm/sre_engine",
+    "pylib",
+    "stdlib",
+    "derive-impl",
+    "wtf8",
+    "wasm/lib",
+]
+
+[workspace.package]
+version = "0.4.0"
+authors = ["RustPython Team"]
+edition = "2024"
+rust-version = "1.87.0"
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+
+[workspace.dependencies]
+rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
+rustpython-compiler = { path = "compiler", version = "0.4.0" }
+rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }
+rustpython-common = { path = "common", version = "0.4.0" }
+rustpython-derive = { path = "derive", version = "0.4.0" }
+rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" }
+rustpython-jit = { path = "jit", version = "0.4.0" }
+rustpython-literal = { path = "compiler/literal", version = "0.4.0" }
+rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" }
+rustpython-pylib = { path = "pylib", version = "0.4.0" }
+rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" }
+rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
+rustpython-wtf8 = { path = "wtf8", version = "0.4.0" }
+rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
+
+ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+
+ahash = "0.8.11"
+ascii = "1.1"
+bitflags = "2.9.1"
+bstr = "1"
+cfg-if = "1.0"
+chrono = "0.4.39"
+constant_time_eq = "0.4"
+criterion = { version = "0.5", features = ["html_reports"] }
+crossbeam-utils = "0.8.21"
+flame = "0.2.2"
+getrandom = { version = "0.3", features = ["std"] }
+glob = "0.3"
+hex = "0.4.3"
+indexmap = { version = "2.10.0", features = ["std"] }
+insta = "1.42"
+itertools = "0.14.0"
+is-macro = "0.3.7"
+junction = "1.2.0"
+libc = "0.2.169"
+libffi = "4.1"
+log = "0.4.27"
+nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
+malachite-bigint = "0.6"
+malachite-q = "0.6"
+malachite-base = "0.6"
+memchr = "2.7.4"
+num-complex = "0.4.6"
+num-integer = "0.1.46"
+num-traits = "0.2"
+num_enum = { version = "0.7", default-features = false }
+optional = "0.5"
+once_cell = "1.20.3"
+parking_lot = "0.12.3"
+paste = "1.0.15"
+proc-macro2 = "1.0.93"
+pymath = "0.0.2"
+quote = "1.0.38"
+radium = "1.1.1"
+rand = "0.9"
+rand_core = { version = "0.9", features = ["os_rng"] }
+rustix = { version = "1.0", features = ["event"] }
+rustyline = "17.0.0"
+serde = { version = "1.0.133", default-features = false }
+schannel = "0.1.27"
+static_assertions = "1.1"
+strum = "0.27"
+strum_macros = "0.27"
+syn = "2"
+thiserror = "2.0"
+thread_local = "1.1.8"
+unicode-casing = "0.1.0"
+unic-char-property = "0.9.0"
+unic-normal = "0.9.0"
+unic-ucd-age = "0.9.0"
+unic-ucd-bidi = "0.9.0"
+unic-ucd-category = "0.9.0"
+unic-ucd-ident = "0.9.0"
+unicode_names2 = "1.3.0"
+unicode-bidi-mirroring = "0.2"
+widestring = "1.2.0"
+windows-sys = "0.59.0"
+wasm-bindgen = "0.2.100"
+
 # Lints
 
 [workspace.lints.rust]
 unsafe_code = "allow"
+unsafe_op_in_unsafe_fn = "deny"
+elided_lifetimes_in_paths = "warn"
 
 [workspace.lints.clippy]
 perf = "warn"

DEVELOPMENT.md

@@ -25,7 +25,7 @@ RustPython requires the following:
       stable version: `rustup update stable`
     - If you do not have Rust installed, use [rustup](https://rustup.rs/) to
       do so.
-- CPython version 3.12 or higher
+- CPython version 3.13 or higher
     - CPython can be installed by your operating system's package manager,
       from the [Python website](https://www.python.org/downloads/), or
       using a third-party distribution, such as 

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 RustPython Team
+Copyright (c) 2025 RustPython Team
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Lib/_aix_support.py

@@ -0,0 +1,108 @@
+"""Shared AIX support functions."""
+
+import sys
+import sysconfig
+
+
+# Taken from _osx_support _read_output function
+def _read_cmd_output(commandstring, capture_stderr=False):
+    """Output from successful command execution or None"""
+    # Similar to os.popen(commandstring, "r").read(),
+    # but without actually using os.popen because that
+    # function is not usable during python bootstrap.
+    import os
+    import contextlib
+    fp = open("/tmp/_aix_support.%s"%(
+        os.getpid(),), "w+b")
+
+    with contextlib.closing(fp) as fp:
+        if capture_stderr:
+            cmd = "%s >'%s' 2>&1" % (commandstring, fp.name)
+        else:
+            cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
+        return fp.read() if not os.system(cmd) else None
+
+
+def _aix_tag(vrtl, bd):
+    # type: (List[int], int) -> str
+    # Infer the ABI bitwidth from maxsize (assuming 64 bit as the default)
+    _sz = 32 if sys.maxsize == (2**31-1) else 64
+    _bd = bd if bd != 0 else 9988
+    # vrtl[version, release, technology_level]
+    return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], _bd, _sz)
+
+
+# extract version, release and technology level from a VRMF string
+def _aix_vrtl(vrmf):
+    # type: (str) -> List[int]
+    v, r, tl = vrmf.split(".")[:3]
+    return [int(v[-1]), int(r), int(tl)]
+
+
+def _aix_bos_rte():
+    # type: () -> Tuple[str, int]
+    """
+    Return a Tuple[str, int] e.g., ['7.1.4.34', 1806]
+    The fileset bos.rte represents the current AIX run-time level. It's VRMF and
+    builddate reflect the current ABI levels of the runtime environment.
+    If no builddate is found give a value that will satisfy pep425 related queries
+    """
+    # All AIX systems to have lslpp installed in this location
+    # subprocess may not be available during python bootstrap
+    try:
+        import subprocess
+        out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.rte"])
+    except ImportError:
+        out = _read_cmd_output("/usr/bin/lslpp -Lqc bos.rte")
+    out = out.decode("utf-8")
+    out = out.strip().split(":")  # type: ignore
+    _bd = int(out[-1]) if out[-1] != '' else 9988
+    return (str(out[2]), _bd)
+
+
+def aix_platform():
+    # type: () -> str
+    """
+    AIX filesets are identified by four decimal values: V.R.M.F.
+    V (version) and R (release) can be retrieved using ``uname``
+    Since 2007, starting with AIX 5.3 TL7, the M value has been
+    included with the fileset bos.rte and represents the Technology
+    Level (TL) of AIX. The F (Fix) value also increases, but is not
+    relevant for comparing releases and binary compatibility.
+    For binary compatibility the so-called builddate is needed.
+    Again, the builddate of an AIX release is associated with bos.rte.
+    AIX ABI compatibility is described  as guaranteed at: https://www.ibm.com/\
+    support/knowledgecenter/en/ssw_aix_72/install/binary_compatability.html
+
+    For pep425 purposes the AIX platform tag becomes:
+    "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(v, r, tl, builddate, bitsize)
+    e.g., "aix-6107-1415-32" for AIX 6.1 TL7 bd 1415, 32-bit
+    and, "aix-6107-1415-64" for AIX 6.1 TL7 bd 1415, 64-bit
+    """
+    vrmf, bd = _aix_bos_rte()
+    return _aix_tag(_aix_vrtl(vrmf), bd)
+
+
+# extract vrtl from the BUILD_GNU_TYPE as an int
+def _aix_bgt():
+    # type: () -> List[int]
+    gnu_type = sysconfig.get_config_var("BUILD_GNU_TYPE")
+    if not gnu_type:
+        raise ValueError("BUILD_GNU_TYPE is not defined")
+    return _aix_vrtl(vrmf=gnu_type)
+
+
+def aix_buildtag():
+    # type: () -> str
+    """
+    Return the platform_tag of the system Python was built on.
+    """
+    # AIX_BUILDDATE is defined by configure with:
+    # lslpp -Lcq bos.rte | awk -F:  '{ print $NF }'
+    build_date = sysconfig.get_config_var("AIX_BUILDDATE")
+    try:
+        build_date = int(build_date)
+    except (ValueError, TypeError):
+        raise ValueError(f"AIX_BUILDDATE is not defined or invalid: "
+                         f"{build_date!r}")
+    return _aix_tag(_aix_bgt(), build_date)

Lib/_android_support.py

@@ -0,0 +1,181 @@
+import io
+import sys
+from threading import RLock
+from time import sleep, time
+
+# The maximum length of a log message in bytes, including the level marker and
+# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71.
+# Messages longer than this will be truncated by logcat. This limit has already
+# been reduced at least once in the history of Android (from 4076 to 4068 between
+# API level 23 and 26), so leave some headroom.
+MAX_BYTES_PER_WRITE = 4000
+
+# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this
+# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE.
+# However, if the actual number of bytes per character is smaller than that,
+# then we may still join multiple consecutive text writes into binary
+# writes containing a larger number of characters.
+MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4
+
+
+# When embedded in an app on current versions of Android, there's no easy way to
+# monitor the C-level stdout and stderr. The testbed comes with a .c file to
+# redirect them to the system log using a pipe, but that wouldn't be convenient
+# or appropriate for all apps. So we redirect at the Python level instead.
+def init_streams(android_log_write, stdout_prio, stderr_prio):
+    if sys.executable:
+        return  # Not embedded in an app.
+
+    global logcat
+    logcat = Logcat(android_log_write)
+
+    sys.stdout = TextLogStream(
+        stdout_prio, "python.stdout", sys.stdout.fileno())
+    sys.stderr = TextLogStream(
+        stderr_prio, "python.stderr", sys.stderr.fileno())
+
+
+class TextLogStream(io.TextIOWrapper):
+    def __init__(self, prio, tag, fileno=None, **kwargs):
+        # The default is surrogateescape for stdout and backslashreplace for
+        # stderr, but in the context of an Android log, readability is more
+        # important than reversibility.
+        kwargs.setdefault("encoding", "UTF-8")
+        kwargs.setdefault("errors", "backslashreplace")
+
+        super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs)
+        self._lock = RLock()
+        self._pending_bytes = []
+        self._pending_bytes_count = 0
+
+    def __repr__(self):
+        return f"<TextLogStream {self.buffer.tag!r}>"
+
+    def write(self, s):
+        if not isinstance(s, str):
+            raise TypeError(
+                f"write() argument must be str, not {type(s).__name__}")
+
+        # In case `s` is a str subclass that writes itself to stdout or stderr
+        # when we call its methods, convert it to an actual str.
+        s = str.__str__(s)
+
+        # We want to emit one log message per line wherever possible, so split
+        # the string into lines first. Note that "".splitlines() == [], so
+        # nothing will be logged for an empty string.
+        with self._lock:
+            for line in s.splitlines(keepends=True):
+                while line:
+                    chunk = line[:MAX_CHARS_PER_WRITE]
+                    line = line[MAX_CHARS_PER_WRITE:]
+                    self._write_chunk(chunk)
+
+        return len(s)
+
+    # The size and behavior of TextIOWrapper's buffer is not part of its public
+    # API, so we handle buffering ourselves to avoid truncation.
+    def _write_chunk(self, s):
+        b = s.encode(self.encoding, self.errors)
+        if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE:
+            self.flush()
+
+        self._pending_bytes.append(b)
+        self._pending_bytes_count += len(b)
+        if (
+            self.write_through
+            or b.endswith(b"\n")
+            or self._pending_bytes_count > MAX_BYTES_PER_WRITE
+        ):
+            self.flush()
+
+    def flush(self):
+        with self._lock:
+            self.buffer.write(b"".join(self._pending_bytes))
+            self._pending_bytes.clear()
+            self._pending_bytes_count = 0
+
+    # Since this is a line-based logging system, line buffering cannot be turned
+    # off, i.e. a newline always causes a flush.
+    @property
+    def line_buffering(self):
+        return True
+
+
+class BinaryLogStream(io.RawIOBase):
+    def __init__(self, prio, tag, fileno=None):
+        self.prio = prio
+        self.tag = tag
+        self._fileno = fileno
+
+    def __repr__(self):
+        return f"<BinaryLogStream {self.tag!r}>"
+
+    def writable(self):
+        return True
+
+    def write(self, b):
+        if type(b) is not bytes:
+            try:
+                b = bytes(memoryview(b))
+            except TypeError:
+                raise TypeError(
+                    f"write() argument must be bytes-like, not {type(b).__name__}"
+                ) from None
+
+        # Writing an empty string to the stream should have no effect.
+        if b:
+            logcat.write(self.prio, self.tag, b)
+        return len(b)
+
+    # This is needed by the test suite --timeout option, which uses faulthandler.
+    def fileno(self):
+        if self._fileno is None:
+            raise io.UnsupportedOperation("fileno")
+        return self._fileno
+
+
+# When a large volume of data is written to logcat at once, e.g. when a test
+# module fails in --verbose3 mode, there's a risk of overflowing logcat's own
+# buffer and losing messages. We avoid this by imposing a rate limit using the
+# token bucket algorithm, based on a conservative estimate of how fast `adb
+# logcat` can consume data.
+MAX_BYTES_PER_SECOND = 1024 * 1024
+
+# The logcat buffer size of a device can be determined by running `logcat -g`.
+# We set the token bucket size to half of the buffer size of our current minimum
+# API level, because other things on the system will be producing messages as
+# well.
+BUCKET_SIZE = 128 * 1024
+
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39
+PER_MESSAGE_OVERHEAD = 28
+
+
+class Logcat:
+    def __init__(self, android_log_write):
+        self.android_log_write = android_log_write
+        self._lock = RLock()
+        self._bucket_level = 0
+        self._prev_write_time = time()
+
+    def write(self, prio, tag, message):
+        # Encode null bytes using "modified UTF-8" to avoid them truncating the
+        # message.
+        message = message.replace(b"\x00", b"\xc0\x80")
+
+        with self._lock:
+            now = time()
+            self._bucket_level += (
+                (now - self._prev_write_time) * MAX_BYTES_PER_SECOND)
+
+            # If the bucket level is still below zero, the clock must have gone
+            # backwards, so reset it to zero and continue.
+            self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE))
+            self._prev_write_time = now
+
+            self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message)
+            if self._bucket_level < 0:
+                sleep(-self._bucket_level / MAX_BYTES_PER_SECOND)
+
+            self.android_log_write(prio, tag, message)

Lib/_apple_support.py

@@ -0,0 +1,66 @@
+import io
+import sys
+
+
+def init_streams(log_write, stdout_level, stderr_level):
+    # Redirect stdout and stderr to the Apple system log. This method is
+    # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger
+    # is enabled.
+    sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors)
+    sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors)
+
+
+class SystemLog(io.TextIOWrapper):
+    def __init__(self, log_write, level, **kwargs):
+        kwargs.setdefault("encoding", "UTF-8")
+        kwargs.setdefault("line_buffering", True)
+        super().__init__(LogStream(log_write, level), **kwargs)
+
+    def __repr__(self):
+        return f"<SystemLog (level {self.buffer.level})>"
+
+    def write(self, s):
+        if not isinstance(s, str):
+            raise TypeError(
+                f"write() argument must be str, not {type(s).__name__}")
+
+        # In case `s` is a str subclass that writes itself to stdout or stderr
+        # when we call its methods, convert it to an actual str.
+        s = str.__str__(s)
+
+        # We want to emit one log message per line, so split
+        # the string before sending it to the superclass.
+        for line in s.splitlines(keepends=True):
+            super().write(line)
+
+        return len(s)
+
+
+class LogStream(io.RawIOBase):
+    def __init__(self, log_write, level):
+        self.log_write = log_write
+        self.level = level
+
+    def __repr__(self):
+        return f"<LogStream (level {self.level!r})>"
+
+    def writable(self):
+        return True
+
+    def write(self, b):
+        if type(b) is not bytes:
+            try:
+                b = bytes(memoryview(b))
+            except TypeError:
+                raise TypeError(
+                    f"write() argument must be bytes-like, not {type(b).__name__}"
+                ) from None
+
+        # Writing an empty string to the stream should have no effect.
+        if b:
+            # Encode null bytes using "modified UTF-8" to avoid truncating the
+            # message. This should not affect the return value, as the caller
+            # may be expecting it to match the length of the input.
+            self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80"))
+
+        return len(b)

Lib/_collections_abc.py

@@ -85,6 +85,10 @@ dict_values = type({}.values())
 dict_items = type({}.items())
 ## misc ##
 mappingproxy = type(type.__dict__)
+def _get_framelocalsproxy():
+    return type(sys._getframe().f_locals)
+framelocalsproxy = _get_framelocalsproxy()
+del _get_framelocalsproxy
 generator = type((lambda: (yield))())
 ## coroutine ##
 async def _coro(): pass
@@ -508,6 +512,10 @@ class _CallableGenericAlias(GenericAlias):
             new_args = (t_args, t_result)
         return _CallableGenericAlias(Callable, tuple(new_args))
 
+    # TODO: RUSTPYTHON patch for common call
+    def __or__(self, other):
+        super().__or__(other)
+
 def _is_param_expr(obj):
     """Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or
     ``_ConcatenateGenericAlias`` from typing.py
@@ -836,6 +844,7 @@ class Mapping(Collection):
     __reversed__ = None
 
 Mapping.register(mappingproxy)
+Mapping.register(framelocalsproxy)
 
 
 class MappingView(Sized):
@@ -973,7 +982,7 @@ class MutableMapping(Mapping):
 
     def update(self, other=(), /, **kwds):
         ''' D.update([E, ]**F) -> None.  Update D from mapping/iterable E and F.
-            If E present and has a .keys() method, does:     for k in E: D[k] = E[k]
+            If E present and has a .keys() method, does:     for k in E.keys(): D[k] = E[k]
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''

Lib/_colorize.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+import io
+import os
+import sys
+
+COLORIZE = True
+
+# types
+if False:
+    from typing import IO
+
+
+class ANSIColors:
+    RESET = "\x1b[0m"
+
+    BLACK = "\x1b[30m"
+    BLUE = "\x1b[34m"
+    CYAN = "\x1b[36m"
+    GREEN = "\x1b[32m"
+    MAGENTA = "\x1b[35m"
+    RED = "\x1b[31m"
+    WHITE = "\x1b[37m"  # more like LIGHT GRAY
+    YELLOW = "\x1b[33m"
+
+    BOLD_BLACK = "\x1b[1;30m"  # DARK GRAY
+    BOLD_BLUE = "\x1b[1;34m"
+    BOLD_CYAN = "\x1b[1;36m"
+    BOLD_GREEN = "\x1b[1;32m"
+    BOLD_MAGENTA = "\x1b[1;35m"
+    BOLD_RED = "\x1b[1;31m"
+    BOLD_WHITE = "\x1b[1;37m"  # actual WHITE
+    BOLD_YELLOW = "\x1b[1;33m"
+
+    # intense = like bold but without being bold
+    INTENSE_BLACK = "\x1b[90m"
+    INTENSE_BLUE = "\x1b[94m"
+    INTENSE_CYAN = "\x1b[96m"
+    INTENSE_GREEN = "\x1b[92m"
+    INTENSE_MAGENTA = "\x1b[95m"
+    INTENSE_RED = "\x1b[91m"
+    INTENSE_WHITE = "\x1b[97m"
+    INTENSE_YELLOW = "\x1b[93m"
+
+    BACKGROUND_BLACK = "\x1b[40m"
+    BACKGROUND_BLUE = "\x1b[44m"
+    BACKGROUND_CYAN = "\x1b[46m"
+    BACKGROUND_GREEN = "\x1b[42m"
+    BACKGROUND_MAGENTA = "\x1b[45m"
+    BACKGROUND_RED = "\x1b[41m"
+    BACKGROUND_WHITE = "\x1b[47m"
+    BACKGROUND_YELLOW = "\x1b[43m"
+
+    INTENSE_BACKGROUND_BLACK = "\x1b[100m"
+    INTENSE_BACKGROUND_BLUE = "\x1b[104m"
+    INTENSE_BACKGROUND_CYAN = "\x1b[106m"
+    INTENSE_BACKGROUND_GREEN = "\x1b[102m"
+    INTENSE_BACKGROUND_MAGENTA = "\x1b[105m"
+    INTENSE_BACKGROUND_RED = "\x1b[101m"
+    INTENSE_BACKGROUND_WHITE = "\x1b[107m"
+    INTENSE_BACKGROUND_YELLOW = "\x1b[103m"
+
+
+NoColors = ANSIColors()
+
+for attr in dir(NoColors):
+    if not attr.startswith("__"):
+        setattr(NoColors, attr, "")
+
+
+def get_colors(
+    colorize: bool = False, *, file: IO[str] | IO[bytes] | None = None
+) -> ANSIColors:
+    if colorize or can_colorize(file=file):
+        return ANSIColors()
+    else:
+        return NoColors
+
+
+def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool:
+    if file is None:
+        file = sys.stdout
+
+    if not sys.flags.ignore_environment:
+        if os.environ.get("PYTHON_COLORS") == "0":
+            return False
+        if os.environ.get("PYTHON_COLORS") == "1":
+            return True
+    if os.environ.get("NO_COLOR"):
+        return False
+    if not COLORIZE:
+        return False
+    if os.environ.get("FORCE_COLOR"):
+        return True
+    if os.environ.get("TERM") == "dumb":
+        return False
+
+    if not hasattr(file, "fileno"):
+        return False
+
+    if sys.platform == "win32":
+        try:
+            import nt
+
+            if not nt._supports_virtual_terminal():
+                return False
+        except (ImportError, AttributeError):
+            return False
+
+    try:
+        return os.isatty(file.fileno())
+    except io.UnsupportedOperation:
+        return hasattr(file, "isatty") and file.isatty()

Lib/_dummy_os.py

@@ -5,22 +5,30 @@ A shim of the os module containing only simple path-related utilities
 try:
     from os import *
 except ImportError:
-    import abc
+    import abc, sys
 
     def __getattr__(name):
-        raise OSError("no os specific module found")
+        if name in {"_path_normpath", "__path__"}:
+            raise AttributeError(name)
+        if name.isupper():
+            return 0
+        def dummy(*args, **kwargs):
+            import io
+            return io.UnsupportedOperation(f"{name}: no os specific module found")
+        dummy.__name__ = f"dummy_{name}"
+        return dummy
 
-    def _shim():
-        import _dummy_os, sys
-        sys.modules['os'] = _dummy_os
-        sys.modules['os.path'] = _dummy_os.path
+    sys.modules['os'] = sys.modules['posix'] = sys.modules[__name__]
 
     import posixpath as path
-    import sys
     sys.modules['os.path'] = path
     del sys
 
     sep = path.sep
+    supports_dir_fd = set()
+    supports_effective_ids = set()
+    supports_fd = set()
+    supports_follow_symlinks = set()
 
 
     def fspath(path):

Lib/_ios_support.py

@@ -0,0 +1,71 @@
+import sys
+try:
+    from ctypes import cdll, c_void_p, c_char_p, util
+except ImportError:
+    # ctypes is an optional module. If it's not present, we're limited in what
+    # we can tell about the system, but we don't want to prevent the module
+    # from working.
+    print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr)
+    objc = None
+else:
+    # ctypes is available. Load the ObjC library, and wrap the objc_getClass,
+    # sel_registerName methods
+    lib = util.find_library("objc")
+    if lib is None:
+        # Failed to load the objc library
+        raise ImportError("ObjC runtime library couldn't be loaded")
+
+    objc = cdll.LoadLibrary(lib)
+    objc.objc_getClass.restype = c_void_p
+    objc.objc_getClass.argtypes = [c_char_p]
+    objc.sel_registerName.restype = c_void_p
+    objc.sel_registerName.argtypes = [c_char_p]
+
+
+def get_platform_ios():
+    # Determine if this is a simulator using the multiarch value
+    is_simulator = sys.implementation._multiarch.endswith("simulator")
+
+    # We can't use ctypes; abort
+    if not objc:
+        return None
+
+    # Most of the methods return ObjC objects
+    objc.objc_msgSend.restype = c_void_p
+    # All the methods used have no arguments.
+    objc.objc_msgSend.argtypes = [c_void_p, c_void_p]
+
+    # Equivalent of:
+    #   device = [UIDevice currentDevice]
+    UIDevice = objc.objc_getClass(b"UIDevice")
+    SEL_currentDevice = objc.sel_registerName(b"currentDevice")
+    device = objc.objc_msgSend(UIDevice, SEL_currentDevice)
+
+    # Equivalent of:
+    #   device_systemVersion = [device systemVersion]
+    SEL_systemVersion = objc.sel_registerName(b"systemVersion")
+    device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion)
+
+    # Equivalent of:
+    #   device_systemName = [device systemName]
+    SEL_systemName = objc.sel_registerName(b"systemName")
+    device_systemName = objc.objc_msgSend(device, SEL_systemName)
+
+    # Equivalent of:
+    #   device_model = [device model]
+    SEL_model = objc.sel_registerName(b"model")
+    device_model = objc.objc_msgSend(device, SEL_model)
+
+    # UTF8String returns a const char*;
+    SEL_UTF8String = objc.sel_registerName(b"UTF8String")
+    objc.objc_msgSend.restype = c_char_p
+
+    # Equivalent of:
+    #   system = [device_systemName UTF8String]
+    #   release = [device_systemVersion UTF8String]
+    #   model = [device_model UTF8String]
+    system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode()
+    release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode()
+    model = objc.objc_msgSend(device_model, SEL_UTF8String).decode()
+
+    return system, release, model, is_simulator

Lib/_osx_support.py

@@ -507,6 +507,11 @@ def get_platform_osx(_config_vars, osname, release, machine):
     # MACOSX_DEPLOYMENT_TARGET.
 
     macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '')
+    if macver and '.' not in macver:
+        # Ensure that the version includes at least a major
+        # and minor version, even if MACOSX_DEPLOYMENT_TARGET
+        # is set to a single-label version like "14".
+        macver += '.0'
     macrelease = _get_system_version() or macver
     macver = macver or macrelease
 

Lib/_pycodecs.py

@@ -1086,11 +1086,13 @@ def charmapencode_output(c, mapping):
     rep = mapping[c]
     if isinstance(rep, int) or isinstance(rep, int):
         if rep < 256:
-            return rep
+            return [rep]
         else:
             raise TypeError("character mapping must be in range(256)")
     elif isinstance(rep, str):
-        return ord(rep)
+        return [ord(rep)]
+    elif isinstance(rep, bytes):
+        return rep
     elif rep == None:
         raise KeyError("character maps to <undefined>")
     else:
@@ -1113,12 +1115,13 @@ def PyUnicode_EncodeCharmap(p, size, mapping='latin-1', errors='strict'):
         #/* try to encode it */
         try:
             x = charmapencode_output(ord(p[inpos]), mapping)
-            res += [x]
+            res += x
         except KeyError:
             x = unicode_call_errorhandler(errors, "charmap",
             "character maps to <undefined>", p, inpos, inpos+1, False)
             try:
-                res += [charmapencode_output(ord(y), mapping) for y in x[0]]
+                for y in x[0]:
+                    res += charmapencode_output(ord(y), mapping)
             except KeyError:
                 raise UnicodeEncodeError("charmap", p, inpos, inpos+1,
                                         "character maps to <undefined>")

Lib/_pydecimal.py

@@ -13,104 +13,7 @@
 # bug) and will be backported.  At this point the spec is stabilizing
 # and the updates are becoming fewer, smaller, and less significant.
 
-"""
-This is an implementation of decimal floating point arithmetic based on
-the General Decimal Arithmetic Specification:
-
-    http://speleotrove.com/decimal/decarith.html
-
-and IEEE standard 854-1987:
-
-    http://en.wikipedia.org/wiki/IEEE_854-1987
-
-Decimal floating point has finite precision with arbitrarily large bounds.
-
-The purpose of this module is to support arithmetic using familiar
-"schoolhouse" rules and to avoid some of the tricky representation
-issues associated with binary floating point.  The package is especially
-useful for financial applications or for contexts where users have
-expectations that are at odds with binary floating point (for instance,
-in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead
-of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected
-Decimal('0.00')).
-
-Here are some examples of using the decimal module:
-
->>> from decimal import *
->>> setcontext(ExtendedContext)
->>> Decimal(0)
-Decimal('0')
->>> Decimal('1')
-Decimal('1')
->>> Decimal('-.0123')
-Decimal('-0.0123')
->>> Decimal(123456)
-Decimal('123456')
->>> Decimal('123.45e12345678')
-Decimal('1.2345E+12345680')
->>> Decimal('1.33') + Decimal('1.27')
-Decimal('2.60')
->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')
-Decimal('-2.20')
->>> dig = Decimal(1)
->>> print(dig / Decimal(3))
-0.333333333
->>> getcontext().prec = 18
->>> print(dig / Decimal(3))
-0.333333333333333333
->>> print(dig.sqrt())
-1
->>> print(Decimal(3).sqrt())
-1.73205080756887729
->>> print(Decimal(3) ** 123)
-4.85192780976896427E+58
->>> inf = Decimal(1) / Decimal(0)
->>> print(inf)
-Infinity
->>> neginf = Decimal(-1) / Decimal(0)
->>> print(neginf)
--Infinity
->>> print(neginf + inf)
-NaN
->>> print(neginf * inf)
--Infinity
->>> print(dig / 0)
-Infinity
->>> getcontext().traps[DivisionByZero] = 1
->>> print(dig / 0)
-Traceback (most recent call last):
-  ...
-  ...
-  ...
-decimal.DivisionByZero: x / 0
->>> c = Context()
->>> c.traps[InvalidOperation] = 0
->>> print(c.flags[InvalidOperation])
-0
->>> c.divide(Decimal(0), Decimal(0))
-Decimal('NaN')
->>> c.traps[InvalidOperation] = 1
->>> print(c.flags[InvalidOperation])
-1
->>> c.flags[InvalidOperation] = 0
->>> print(c.flags[InvalidOperation])
-0
->>> print(c.divide(Decimal(0), Decimal(0)))
-Traceback (most recent call last):
-  ...
-  ...
-  ...
-decimal.InvalidOperation: 0 / 0
->>> print(c.flags[InvalidOperation])
-1
->>> c.flags[InvalidOperation] = 0
->>> c.traps[InvalidOperation] = 0
->>> print(c.divide(Decimal(0), Decimal(0)))
-NaN
->>> print(c.flags[InvalidOperation])
-1
->>>
-"""
+"""Python decimal arithmetic module"""
 
 __all__ = [
     # Two major classes
@@ -140,8 +43,11 @@ __all__ = [
     # Limits for the C version for compatibility
     'MAX_PREC',  'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',
 
-    # C version: compile time choice that enables the thread local context
-    'HAVE_THREADS'
+    # C version: compile time choice that enables the thread local context (deprecated, now always true)
+    'HAVE_THREADS',
+
+    # C version: compile time choice that enables the coroutine local context
+    'HAVE_CONTEXTVAR'
 ]
 
 __xname__ = __name__    # sys.modules lookup (--without-threads)
@@ -156,7 +62,7 @@ import sys
 
 try:
     from collections import namedtuple as _namedtuple
-    DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent')
+    DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent', module='decimal')
 except ImportError:
     DecimalTuple = lambda *args: args
 
@@ -172,6 +78,7 @@ ROUND_05UP = 'ROUND_05UP'
 
 # Compatibility with the C version
 HAVE_THREADS = True
+HAVE_CONTEXTVAR = True
 if sys.maxsize == 2**63-1:
     MAX_PREC = 999999999999999999
     MAX_EMAX = 999999999999999999
@@ -190,7 +97,7 @@ class DecimalException(ArithmeticError):
 
     Used exceptions derive from this.
     If an exception derives from another exception besides this (such as
-    Underflow (Inexact, Rounded, Subnormal) that indicates that it is only
+    Underflow (Inexact, Rounded, Subnormal)) that indicates that it is only
     called if the others are present.  This isn't actually used for
     anything, though.
 
@@ -238,7 +145,7 @@ class InvalidOperation(DecimalException):
     x ** (+-)INF
     An operand is invalid
 
-    The result of the operation after these is a quiet positive NaN,
+    The result of the operation after this is a quiet positive NaN,
     except when the cause is a signaling NaN, in which case the result is
     also a quiet NaN, but with the original sign, and an optional
     diagnostic information.
@@ -431,82 +338,40 @@ _rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING,
 ##### Context Functions ##################################################
 
 # The getcontext() and setcontext() function manage access to a thread-local
-# current context.  Py2.4 offers direct support for thread locals.  If that
-# is not available, use threading.current_thread() which is slower but will
-# work for older Pythons.  If threads are not part of the build, create a
-# mock threading object with threading.local() returning the module namespace.
+# current context.
 
-try:
-    import threading
-except ImportError:
-    # Python was compiled without threads; create a mock object instead
-    class MockThreading(object):
-        def local(self, sys=sys):
-            return sys.modules[__xname__]
-    threading = MockThreading()
-    del MockThreading
+import contextvars
 
-try:
-    threading.local
+_current_context_var = contextvars.ContextVar('decimal_context')
 
-except AttributeError:
+_context_attributes = frozenset(
+    ['prec', 'Emin', 'Emax', 'capitals', 'clamp', 'rounding', 'flags', 'traps']
+)
 
-    # To fix reloading, force it to create a new context
-    # Old contexts have different exceptions in their dicts, making problems.
-    if hasattr(threading.current_thread(), '__decimal_context__'):
-        del threading.current_thread().__decimal_context__
+def getcontext():
+    """Returns this thread's context.
 
-    def setcontext(context):
-        """Set this thread's context to context."""
-        if context in (DefaultContext, BasicContext, ExtendedContext):
-            context = context.copy()
-            context.clear_flags()
-        threading.current_thread().__decimal_context__ = context
+    If this thread does not yet have a context, returns
+    a new context and sets this thread's context.
+    New contexts are copies of DefaultContext.
+    """
+    try:
+        return _current_context_var.get()
+    except LookupError:
+        context = Context()
+        _current_context_var.set(context)
+        return context
 
-    def getcontext():
-        """Returns this thread's context.
+def setcontext(context):
+    """Set this thread's context to context."""
+    if context in (DefaultContext, BasicContext, ExtendedContext):
+        context = context.copy()
+        context.clear_flags()
+    _current_context_var.set(context)
 
-        If this thread does not yet have a context, returns
-        a new context and sets this thread's context.
-        New contexts are copies of DefaultContext.
-        """
-        try:
-            return threading.current_thread().__decimal_context__
-        except AttributeError:
-            context = Context()
-            threading.current_thread().__decimal_context__ = context
-            return context
+del contextvars        # Don't contaminate the namespace
 
-else:
-
-    local = threading.local()
-    if hasattr(local, '__decimal_context__'):
-        del local.__decimal_context__
-
-    def getcontext(_local=local):
-        """Returns this thread's context.
-
-        If this thread does not yet have a context, returns
-        a new context and sets this thread's context.
-        New contexts are copies of DefaultContext.
-        """
-        try:
-            return _local.__decimal_context__
-        except AttributeError:
-            context = Context()
-            _local.__decimal_context__ = context
-            return context
-
-    def setcontext(context, _local=local):
-        """Set this thread's context to context."""
-        if context in (DefaultContext, BasicContext, ExtendedContext):
-            context = context.copy()
-            context.clear_flags()
-        _local.__decimal_context__ = context
-
-    del threading, local        # Don't contaminate the namespace
-
-def localcontext(ctx=None):
+def localcontext(ctx=None, **kwargs):
     """Return a context manager for a copy of the supplied context
 
     Uses a copy of the current context if no context is specified
@@ -542,8 +407,14 @@ def localcontext(ctx=None):
     >>> print(getcontext().prec)
     28
     """
-    if ctx is None: ctx = getcontext()
-    return _ContextManager(ctx)
+    if ctx is None:
+        ctx = getcontext()
+    ctx_manager = _ContextManager(ctx)
+    for key, value in kwargs.items():
+        if key not in _context_attributes:
+            raise TypeError(f"'{key}' is an invalid keyword argument for this function")
+        setattr(ctx_manager.new_context, key, value)
+    return ctx_manager
 
 
 ##### Decimal class #######################################################
@@ -553,7 +424,7 @@ def localcontext(ctx=None):
 # numbers.py for more detail.
 
 class Decimal(object):
-    """Floating point class for decimal arithmetic."""
+    """Floating-point class for decimal arithmetic."""
 
     __slots__ = ('_exp','_int','_sign', '_is_special')
     # Generally, the value of the Decimal instance is given by
@@ -734,18 +605,23 @@ class Decimal(object):
 
         """
         if isinstance(f, int):                # handle integer inputs
-            return cls(f)
-        if not isinstance(f, float):
-            raise TypeError("argument must be int or float.")
-        if _math.isinf(f) or _math.isnan(f):
-            return cls(repr(f))
-        if _math.copysign(1.0, f) == 1.0:
-            sign = 0
+            sign = 0 if f >= 0 else 1
+            k = 0
+            coeff = str(abs(f))
+        elif isinstance(f, float):
+            if _math.isinf(f) or _math.isnan(f):
+                return cls(repr(f))
+            if _math.copysign(1.0, f) == 1.0:
+                sign = 0
+            else:
+                sign = 1
+            n, d = abs(f).as_integer_ratio()
+            k = d.bit_length() - 1
+            coeff = str(n*5**k)
         else:
-            sign = 1
-        n, d = abs(f).as_integer_ratio()
-        k = d.bit_length() - 1
-        result = _dec_from_triple(sign, str(n*5**k), -k)
+            raise TypeError("argument must be int or float.")
+
+        result = _dec_from_triple(sign, coeff, -k)
         if cls is Decimal:
             return result
         else:
@@ -988,7 +864,7 @@ class Decimal(object):
             if self.is_snan():
                 raise TypeError('Cannot hash a signaling NaN value.')
             elif self.is_nan():
-                return _PyHASH_NAN
+                return object.__hash__(self)
             else:
                 if self._sign:
                     return -_PyHASH_INF
@@ -1669,13 +1545,13 @@ class Decimal(object):
 
     __trunc__ = __int__
 
+    @property
     def real(self):
         return self
-    real = property(real)
 
+    @property
     def imag(self):
         return Decimal(0)
-    imag = property(imag)
 
     def conjugate(self):
         return self
@@ -2255,10 +2131,16 @@ class Decimal(object):
             else:
                 return None
 
-            if xc >= 10**p:
+            # An exact power of 10 is representable, but can convert to a
+            # string of any length. But an exact power of 10 shouldn't be
+            # possible at this point.
+            assert xc > 1, self
+            assert xc % 10 != 0, self
+            strxc = str(xc)
+            if len(strxc) > p:
                 return None
             xe = -e-xe
-            return _dec_from_triple(0, str(xc), xe)
+            return _dec_from_triple(0, strxc, xe)
 
         # now y is positive; find m and n such that y = m/n
         if ye >= 0:
@@ -2267,7 +2149,7 @@ class Decimal(object):
             if xe != 0 and len(str(abs(yc*xe))) <= -ye:
                 return None
             xc_bits = _nbits(xc)
-            if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye:
+            if len(str(abs(yc)*xc_bits)) <= -ye:
                 return None
             m, n = yc, 10**(-ye)
             while m % 2 == n % 2 == 0:
@@ -2280,7 +2162,7 @@ class Decimal(object):
         # compute nth root of xc*10**xe
         if n > 1:
             # if 1 < xc < 2**n then xc isn't an nth power
-            if xc != 1 and xc_bits <= n:
+            if xc_bits <= n:
                 return None
 
             xe, rem = divmod(xe, n)
@@ -2308,13 +2190,18 @@ class Decimal(object):
             return None
         xc = xc**m
         xe *= m
-        if xc > 10**p:
+        # An exact power of 10 is representable, but can convert to a string
+        # of any length. But an exact power of 10 shouldn't be possible at
+        # this point.
+        assert xc > 1, self
+        assert xc % 10 != 0, self
+        str_xc = str(xc)
+        if len(str_xc) > p:
             return None
 
         # by this point the result *is* exactly representable
         # adjust the exponent to get as close as possible to the ideal
         # exponent, if necessary
-        str_xc = str(xc)
         if other._isinteger() and other._sign == 0:
             ideal_exponent = self._exp*int(other)
             zeros = min(xe-ideal_exponent, p-len(str_xc))
@@ -3832,6 +3719,10 @@ class Decimal(object):
         # represented in fixed point; rescale them to 0e0.
         if not self and self._exp > 0 and spec['type'] in 'fF%':
             self = self._rescale(0, rounding)
+        if not self and spec['no_neg_0'] and self._sign:
+            adjusted_sign = 0
+        else:
+            adjusted_sign = self._sign
 
         # figure out placement of the decimal point
         leftdigits = self._exp + len(self._int)
@@ -3862,7 +3753,7 @@ class Decimal(object):
 
         # done with the decimal-specific stuff;  hand over the rest
         # of the formatting to the _format_number function
-        return _format_number(self._sign, intpart, fracpart, exp, spec)
+        return _format_number(adjusted_sign, intpart, fracpart, exp, spec)
 
 def _dec_from_triple(sign, coefficient, exponent, special=False):
     """Create a decimal instance directly, without any validation,
@@ -5672,8 +5563,6 @@ class _WorkRep(object):
     def __repr__(self):
         return "(%r, %r, %r)" % (self.sign, self.int, self.exp)
 
-    __str__ = __repr__
-
 
 
 def _normalize(op1, op2, prec = 0):
@@ -6182,7 +6071,7 @@ _exact_half = re.compile('50*$').match
 #
 # A format specifier for Decimal looks like:
 #
-#   [[fill]align][sign][#][0][minimumwidth][,][.precision][type]
+#   [[fill]align][sign][z][#][0][minimumwidth][,][.precision][type]
 
 _parse_format_specifier_regex = re.compile(r"""\A
 (?:
@@ -6190,6 +6079,7 @@ _parse_format_specifier_regex = re.compile(r"""\A
    (?P<align>[<>=^])
 )?
 (?P<sign>[-+ ])?
+(?P<no_neg_0>z)?
 (?P<alt>\#)?
 (?P<zeropad>0)?
 (?P<minimumwidth>(?!0)\d+)?

Lib/_pylong.py

@@ -0,0 +1,363 @@
+"""Python implementations of some algorithms for use by longobject.c.
+The goal is to provide asymptotically faster algorithms that can be
+used for operations on integers with many digits.  In those cases, the
+performance overhead of the Python implementation is not significant
+since the asymptotic behavior is what dominates runtime. Functions
+provided by this module should be considered private and not part of any
+public API.
+
+Note: for ease of maintainability, please prefer clear code and avoid
+"micro-optimizations".  This module will only be imported and used for
+integers with a huge number of digits.  Saving a few microseconds with
+tricky or non-obvious code is not worth it.  For people looking for
+maximum performance, they should use something like gmpy2."""
+
+import re
+import decimal
+try:
+    import _decimal
+except ImportError:
+    _decimal = None
+
+# A number of functions have this form, where `w` is a desired number of
+# digits in base `base`:
+#
+#    def inner(...w...):
+#        if w <= LIMIT:
+#            return something
+#        lo = w >> 1
+#        hi = w - lo
+#        something involving base**lo, inner(...lo...), j, and inner(...hi...)
+#    figure out largest w needed
+#    result = inner(w)
+#
+# They all had some on-the-fly scheme to cache `base**lo` results for reuse.
+# Power is costly.
+#
+# This routine aims to compute all amd only the needed powers in advance, as
+# efficiently as reasonably possible. This isn't trivial, and all the
+# on-the-fly methods did needless work in many cases. The driving code above
+# changes to:
+#
+#    figure out largest w needed
+#    mycache = compute_powers(w, base, LIMIT)
+#    result = inner(w)
+#
+# and `mycache[lo]` replaces `base**lo` in the inner function.
+#
+# While this does give minor speedups (a few percent at best), the primary
+# intent is to simplify the functions using this, by eliminating the need for
+# them to craft their own ad-hoc caching schemes.
+def compute_powers(w, base, more_than, show=False):
+    seen = set()
+    need = set()
+    ws = {w}
+    while ws:
+        w = ws.pop() # any element is fine to use next
+        if w in seen or w <= more_than:
+            continue
+        seen.add(w)
+        lo = w >> 1
+        # only _need_ lo here; some other path may, or may not, need hi
+        need.add(lo)
+        ws.add(lo)
+        if w & 1:
+            ws.add(lo + 1)
+
+    d = {}
+    if not need:
+        return d
+    it = iter(sorted(need))
+    first = next(it)
+    if show:
+        print("pow at", first)
+    d[first] = base ** first
+    for this in it:
+        if this - 1 in d:
+            if show:
+                print("* base at", this)
+            d[this] = d[this - 1] * base # cheap
+        else:
+            lo = this >> 1
+            hi = this - lo
+            assert lo in d
+            if show:
+                print("square at", this)
+            # Multiplying a bigint by itself (same object!) is about twice
+            # as fast in CPython.
+            sq = d[lo] * d[lo]
+            if hi != lo:
+                assert hi == lo + 1
+                if show:
+                    print("    and * base")
+                sq *= base
+            d[this] = sq
+    return d
+
+_unbounded_dec_context = decimal.getcontext().copy()
+_unbounded_dec_context.prec = decimal.MAX_PREC
+_unbounded_dec_context.Emax = decimal.MAX_EMAX
+_unbounded_dec_context.Emin = decimal.MIN_EMIN
+_unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check
+
+def int_to_decimal(n):
+    """Asymptotically fast conversion of an 'int' to Decimal."""
+
+    # Function due to Tim Peters.  See GH issue #90716 for details.
+    # https://github.com/python/cpython/issues/90716
+    #
+    # The implementation in longobject.c of base conversion algorithms
+    # between power-of-2 and non-power-of-2 bases are quadratic time.
+    # This function implements a divide-and-conquer algorithm that is
+    # faster for large numbers.  Builds an equal decimal.Decimal in a
+    # "clever" recursive way.  If we want a string representation, we
+    # apply str to _that_.
+
+    from decimal import Decimal as D
+    BITLIM = 200
+
+    # Don't bother caching the "lo" mask in this; the time to compute it is
+    # tiny compared to the multiply.
+    def inner(n, w):
+        if w <= BITLIM:
+            return D(n)
+        w2 = w >> 1
+        hi = n >> w2
+        lo = n & ((1 << w2) - 1)
+        return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2]
+
+    with decimal.localcontext(_unbounded_dec_context):
+        nbits = n.bit_length()
+        w2pow = compute_powers(nbits, D(2), BITLIM)
+        if n < 0:
+            negate = True
+            n = -n
+        else:
+            negate = False
+        result = inner(n, nbits)
+        if negate:
+            result = -result
+    return result
+
+def int_to_decimal_string(n):
+    """Asymptotically fast conversion of an 'int' to a decimal string."""
+    w = n.bit_length()
+    if w > 450_000 and _decimal is not None:
+        # It is only usable with the C decimal implementation.
+        # _pydecimal.py calls str() on very large integers, which in its
+        # turn calls int_to_decimal_string(), causing very deep recursion.
+        return str(int_to_decimal(n))
+
+    # Fallback algorithm for the case when the C decimal module isn't
+    # available.  This algorithm is asymptotically worse than the algorithm
+    # using the decimal module, but better than the quadratic time
+    # implementation in longobject.c.
+
+    DIGLIM = 1000
+    def inner(n, w):
+        if w <= DIGLIM:
+            return str(n)
+        w2 = w >> 1
+        hi, lo = divmod(n, pow10[w2])
+        return inner(hi, w - w2) + inner(lo, w2).zfill(w2)
+
+    # The estimation of the number of decimal digits.
+    # There is no harm in small error.  If we guess too large, there may
+    # be leading 0's that need to be stripped.  If we guess too small, we
+    # may need to call str() recursively for the remaining highest digits,
+    # which can still potentially be a large integer. This is manifested
+    # only if the number has way more than 10**15 digits, that exceeds
+    # the 52-bit physical address limit in both Intel64 and AMD64.
+    w = int(w * 0.3010299956639812 + 1)  # log10(2)
+    pow10 = compute_powers(w, 5, DIGLIM)
+    for k, v in pow10.items():
+        pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k
+    if n < 0:
+        n = -n
+        sign = '-'
+    else:
+        sign = ''
+    s = inner(n, w)
+    if s[0] == '0' and n:
+        # If our guess of w is too large, there may be leading 0's that
+        # need to be stripped.
+        s = s.lstrip('0')
+    return sign + s
+
+def _str_to_int_inner(s):
+    """Asymptotically fast conversion of a 'str' to an 'int'."""
+
+    # Function due to Bjorn Martinsson.  See GH issue #90716 for details.
+    # https://github.com/python/cpython/issues/90716
+    #
+    # The implementation in longobject.c of base conversion algorithms
+    # between power-of-2 and non-power-of-2 bases are quadratic time.
+    # This function implements a divide-and-conquer algorithm making use
+    # of Python's built in big int multiplication. Since Python uses the
+    # Karatsuba algorithm for multiplication, the time complexity
+    # of this function is O(len(s)**1.58).
+
+    DIGLIM = 2048
+
+    def inner(a, b):
+        if b - a <= DIGLIM:
+            return int(s[a:b])
+        mid = (a + b + 1) >> 1
+        return (inner(mid, b)
+                + ((inner(a, mid) * w5pow[b - mid])
+                    << (b - mid)))
+
+    w5pow = compute_powers(len(s), 5, DIGLIM)
+    return inner(0, len(s))
+
+
+def int_from_string(s):
+    """Asymptotically fast version of PyLong_FromString(), conversion
+    of a string of decimal digits into an 'int'."""
+    # PyLong_FromString() has already removed leading +/-, checked for invalid
+    # use of underscore characters, checked that string consists of only digits
+    # and underscores, and stripped leading whitespace.  The input can still
+    # contain underscores and have trailing whitespace.
+    s = s.rstrip().replace('_', '')
+    return _str_to_int_inner(s)
+
+def str_to_int(s):
+    """Asymptotically fast version of decimal string to 'int' conversion."""
+    # FIXME: this doesn't support the full syntax that int() supports.
+    m = re.match(r'\s*([+-]?)([0-9_]+)\s*', s)
+    if not m:
+        raise ValueError('invalid literal for int() with base 10')
+    v = int_from_string(m.group(2))
+    if m.group(1) == '-':
+        v = -v
+    return v
+
+
+# Fast integer division, based on code from Mark Dickinson, fast_div.py
+# GH-47701. Additional refinements and optimizations by Bjorn Martinsson.  The
+# algorithm is due to Burnikel and Ziegler, in their paper "Fast Recursive
+# Division".
+
+_DIV_LIMIT = 4000
+
+
+def _div2n1n(a, b, n):
+    """Divide a 2n-bit nonnegative integer a by an n-bit positive integer
+    b, using a recursive divide-and-conquer algorithm.
+
+    Inputs:
+      n is a positive integer
+      b is a positive integer with exactly n bits
+      a is a nonnegative integer such that a < 2**n * b
+
+    Output:
+      (q, r) such that a = b*q+r and 0 <= r < b.
+
+    """
+    if a.bit_length() - n <= _DIV_LIMIT:
+        return divmod(a, b)
+    pad = n & 1
+    if pad:
+        a <<= 1
+        b <<= 1
+        n += 1
+    half_n = n >> 1
+    mask = (1 << half_n) - 1
+    b1, b2 = b >> half_n, b & mask
+    q1, r = _div3n2n(a >> n, (a >> half_n) & mask, b, b1, b2, half_n)
+    q2, r = _div3n2n(r, a & mask, b, b1, b2, half_n)
+    if pad:
+        r >>= 1
+    return q1 << half_n | q2, r
+
+
+def _div3n2n(a12, a3, b, b1, b2, n):
+    """Helper function for _div2n1n; not intended to be called directly."""
+    if a12 >> n == b1:
+        q, r = (1 << n) - 1, a12 - (b1 << n) + b1
+    else:
+        q, r = _div2n1n(a12, b1, n)
+    r = (r << n | a3) - q * b2
+    while r < 0:
+        q -= 1
+        r += b
+    return q, r
+
+
+def _int2digits(a, n):
+    """Decompose non-negative int a into base 2**n
+
+    Input:
+      a is a non-negative integer
+
+    Output:
+      List of the digits of a in base 2**n in little-endian order,
+      meaning the most significant digit is last. The most
+      significant digit is guaranteed to be non-zero.
+      If a is 0 then the output is an empty list.
+
+    """
+    a_digits = [0] * ((a.bit_length() + n - 1) // n)
+
+    def inner(x, L, R):
+        if L + 1 == R:
+            a_digits[L] = x
+            return
+        mid = (L + R) >> 1
+        shift = (mid - L) * n
+        upper = x >> shift
+        lower = x ^ (upper << shift)
+        inner(lower, L, mid)
+        inner(upper, mid, R)
+
+    if a:
+        inner(a, 0, len(a_digits))
+    return a_digits
+
+
+def _digits2int(digits, n):
+    """Combine base-2**n digits into an int. This function is the
+    inverse of `_int2digits`. For more details, see _int2digits.
+    """
+
+    def inner(L, R):
+        if L + 1 == R:
+            return digits[L]
+        mid = (L + R) >> 1
+        shift = (mid - L) * n
+        return (inner(mid, R) << shift) + inner(L, mid)
+
+    return inner(0, len(digits)) if digits else 0
+
+
+def _divmod_pos(a, b):
+    """Divide a non-negative integer a by a positive integer b, giving
+    quotient and remainder."""
+    # Use grade-school algorithm in base 2**n, n = nbits(b)
+    n = b.bit_length()
+    a_digits = _int2digits(a, n)
+
+    r = 0
+    q_digits = []
+    for a_digit in reversed(a_digits):
+        q_digit, r = _div2n1n((r << n) + a_digit, b, n)
+        q_digits.append(q_digit)
+    q_digits.reverse()
+    q = _digits2int(q_digits, n)
+    return q, r
+
+
+def int_divmod(a, b):
+    """Asymptotically fast replacement for divmod, for 'int'.
+    Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b).
+    """
+    if b == 0:
+        raise ZeroDivisionError
+    elif b < 0:
+        q, r = int_divmod(-a, -b)
+        return q, -r
+    elif a < 0:
+        q, r = int_divmod(~a, b)
+        return ~q, b + ~r
+    else:
+        return _divmod_pos(a, b)

Lib/_pyrepl/__init__.py

@@ -0,0 +1,19 @@
+#   Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

Lib/_pyrepl/__main__.py

@@ -0,0 +1,6 @@
+# Important: don't add things to this module, as they will end up in the REPL's
+# default globals.  Use _pyrepl.main instead.
+
+if __name__ == "__main__":
+    from .main import interactive_console as __pyrepl_interactive_console
+    __pyrepl_interactive_console()

Lib/_pyrepl/_minimal_curses.py

@@ -0,0 +1,68 @@
+"""Minimal '_curses' module, the low-level interface for curses module
+which is not meant to be used directly.
+
+Based on ctypes.  It's too incomplete to be really called '_curses', so
+to use it, you have to import it and stick it in sys.modules['_curses']
+manually.
+
+Note that there is also a built-in module _minimal_curses which will
+hide this one if compiled in.
+"""
+
+import ctypes
+import ctypes.util
+
+
+class error(Exception):
+    pass
+
+
+def _find_clib() -> str:
+    trylibs = ["ncursesw", "ncurses", "curses"]
+
+    for lib in trylibs:
+        path = ctypes.util.find_library(lib)
+        if path:
+            return path
+    raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses")
+
+
+_clibpath = _find_clib()
+clib = ctypes.cdll.LoadLibrary(_clibpath)
+
+clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
+clib.setupterm.restype = ctypes.c_int
+
+clib.tigetstr.argtypes = [ctypes.c_char_p]
+clib.tigetstr.restype = ctypes.c_ssize_t
+
+clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int]  # type: ignore[operator]
+clib.tparm.restype = ctypes.c_char_p
+
+OK = 0
+ERR = -1
+
+# ____________________________________________________________
+
+
+def setupterm(termstr, fd):
+    err = ctypes.c_int(0)
+    result = clib.setupterm(termstr, fd, ctypes.byref(err))
+    if result == ERR:
+        raise error("setupterm() failed (err=%d)" % err.value)
+
+
+def tigetstr(cap):
+    if not isinstance(cap, bytes):
+        cap = cap.encode("ascii")
+    result = clib.tigetstr(cap)
+    if result == ERR:
+        return None
+    return ctypes.cast(result, ctypes.c_char_p).value
+
+
+def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0):
+    result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9)
+    if result is None:
+        raise error("tparm() returned NULL")
+    return result

Lib/_pyrepl/_threading_handler.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import traceback
+
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from threading import Thread
+    from types import TracebackType
+    from typing import Protocol
+
+    class ExceptHookArgs(Protocol):
+        @property
+        def exc_type(self) -> type[BaseException]: ...
+        @property
+        def exc_value(self) -> BaseException | None: ...
+        @property
+        def exc_traceback(self) -> TracebackType | None: ...
+        @property
+        def thread(self) -> Thread | None: ...
+
+    class ShowExceptions(Protocol):
+        def __call__(self) -> int: ...
+        def add(self, s: str) -> None: ...
+
+    from .reader import Reader
+
+
+def install_threading_hook(reader: Reader) -> None:
+    import threading
+
+    @dataclass
+    class ExceptHookHandler:
+        lock: threading.Lock = field(default_factory=threading.Lock)
+        messages: list[str] = field(default_factory=list)
+
+        def show(self) -> int:
+            count = 0
+            with self.lock:
+                if not self.messages:
+                    return 0
+                reader.restore()
+                for tb in self.messages:
+                    count += 1
+                    if tb:
+                        print(tb)
+                self.messages.clear()
+                reader.scheduled_commands.append("ctrl-c")
+                reader.prepare()
+            return count
+
+        def add(self, s: str) -> None:
+            with self.lock:
+                self.messages.append(s)
+
+        def exception(self, args: ExceptHookArgs) -> None:
+            lines = traceback.format_exception(
+                args.exc_type,
+                args.exc_value,
+                args.exc_traceback,
+                colorize=reader.can_colorize,
+            )  # type: ignore[call-overload]
+            pre = f"\nException in {args.thread.name}:\n" if args.thread else "\n"
+            tb = pre + "".join(lines)
+            self.add(tb)
+
+        def __call__(self) -> int:
+            return self.show()
+
+
+    handler = ExceptHookHandler()
+    reader.threading_hook = handler
+    threading.excepthook = handler.exception

Lib/_pyrepl/commands.py

@@ -0,0 +1,489 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Antonio Cuni
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+import os
+
+# Categories of actions:
+#  killing
+#  yanking
+#  motion
+#  editing
+#  history
+#  finishing
+# [completion]
+
+
+# types
+if False:
+    from .historical_reader import HistoricalReader
+
+
+class Command:
+    finish: bool = False
+    kills_digit_arg: bool = True
+
+    def __init__(
+        self, reader: HistoricalReader, event_name: str, event: list[str]
+    ) -> None:
+        # Reader should really be "any reader" but there's too much usage of
+        # HistoricalReader methods and fields in the code below for us to
+        # refactor at the moment.
+
+        self.reader = reader
+        self.event = event
+        self.event_name = event_name
+
+    def do(self) -> None:
+        pass
+
+
+class KillCommand(Command):
+    def kill_range(self, start: int, end: int) -> None:
+        if start == end:
+            return
+        r = self.reader
+        b = r.buffer
+        text = b[start:end]
+        del b[start:end]
+        if is_kill(r.last_command):
+            if start < r.pos:
+                r.kill_ring[-1] = text + r.kill_ring[-1]
+            else:
+                r.kill_ring[-1] = r.kill_ring[-1] + text
+        else:
+            r.kill_ring.append(text)
+        r.pos = start
+        r.dirty = True
+
+
+class YankCommand(Command):
+    pass
+
+
+class MotionCommand(Command):
+    pass
+
+
+class EditCommand(Command):
+    pass
+
+
+class FinishCommand(Command):
+    finish = True
+    pass
+
+
+def is_kill(command: type[Command] | None) -> bool:
+    return command is not None and issubclass(command, KillCommand)
+
+
+def is_yank(command: type[Command] | None) -> bool:
+    return command is not None and issubclass(command, YankCommand)
+
+
+# etc
+
+
+class digit_arg(Command):
+    kills_digit_arg = False
+
+    def do(self) -> None:
+        r = self.reader
+        c = self.event[-1]
+        if c == "-":
+            if r.arg is not None:
+                r.arg = -r.arg
+            else:
+                r.arg = -1
+        else:
+            d = int(c)
+            if r.arg is None:
+                r.arg = d
+            else:
+                if r.arg < 0:
+                    r.arg = 10 * r.arg - d
+                else:
+                    r.arg = 10 * r.arg + d
+        r.dirty = True
+
+
+class clear_screen(Command):
+    def do(self) -> None:
+        r = self.reader
+        r.console.clear()
+        r.dirty = True
+
+
+class refresh(Command):
+    def do(self) -> None:
+        self.reader.dirty = True
+
+
+class repaint(Command):
+    def do(self) -> None:
+        self.reader.dirty = True
+        self.reader.console.repaint()
+
+
+class kill_line(KillCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        eol = r.eol()
+        for c in b[r.pos : eol]:
+            if not c.isspace():
+                self.kill_range(r.pos, eol)
+                return
+        else:
+            self.kill_range(r.pos, eol + 1)
+
+
+class unix_line_discard(KillCommand):
+    def do(self) -> None:
+        r = self.reader
+        self.kill_range(r.bol(), r.pos)
+
+
+class unix_word_rubout(KillCommand):
+    def do(self) -> None:
+        r = self.reader
+        for i in range(r.get_arg()):
+            self.kill_range(r.bow(), r.pos)
+
+
+class kill_word(KillCommand):
+    def do(self) -> None:
+        r = self.reader
+        for i in range(r.get_arg()):
+            self.kill_range(r.pos, r.eow())
+
+
+class backward_kill_word(KillCommand):
+    def do(self) -> None:
+        r = self.reader
+        for i in range(r.get_arg()):
+            self.kill_range(r.bow(), r.pos)
+
+
+class yank(YankCommand):
+    def do(self) -> None:
+        r = self.reader
+        if not r.kill_ring:
+            r.error("nothing to yank")
+            return
+        r.insert(r.kill_ring[-1])
+
+
+class yank_pop(YankCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        if not r.kill_ring:
+            r.error("nothing to yank")
+            return
+        if not is_yank(r.last_command):
+            r.error("previous command was not a yank")
+            return
+        repl = len(r.kill_ring[-1])
+        r.kill_ring.insert(0, r.kill_ring.pop())
+        t = r.kill_ring[-1]
+        b[r.pos - repl : r.pos] = t
+        r.pos = r.pos - repl + len(t)
+        r.dirty = True
+
+
+class interrupt(FinishCommand):
+    def do(self) -> None:
+        import signal
+
+        self.reader.console.finish()
+        self.reader.finish()
+        os.kill(os.getpid(), signal.SIGINT)
+
+
+class ctrl_c(Command):
+    def do(self) -> None:
+        self.reader.console.finish()
+        self.reader.finish()
+        raise KeyboardInterrupt
+
+
+class suspend(Command):
+    def do(self) -> None:
+        import signal
+
+        r = self.reader
+        p = r.pos
+        r.console.finish()
+        os.kill(os.getpid(), signal.SIGSTOP)
+        ## this should probably be done
+        ## in a handler for SIGCONT?
+        r.console.prepare()
+        r.pos = p
+        # r.posxy = 0, 0  # XXX this is invalid
+        r.dirty = True
+        r.console.screen = []
+
+
+class up(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        for _ in range(r.get_arg()):
+            x, y = r.pos2xy()
+            new_y = y - 1
+
+            if r.bol() == 0:
+                if r.historyi > 0:
+                    r.select_item(r.historyi - 1)
+                    return
+                r.pos = 0
+                r.error("start of buffer")
+                return
+
+            if (
+                x
+                > (
+                    new_x := r.max_column(new_y)
+                )  # we're past the end of the previous line
+                or x == r.max_column(y)
+                and any(
+                    not i.isspace() for i in r.buffer[r.bol() :]
+                )  # move between eols
+            ):
+                x = new_x
+
+            r.setpos_from_xy(x, new_y)
+
+
+class down(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        for _ in range(r.get_arg()):
+            x, y = r.pos2xy()
+            new_y = y + 1
+
+            if r.eol() == len(b):
+                if r.historyi < len(r.history):
+                    r.select_item(r.historyi + 1)
+                    r.pos = r.eol(0)
+                    return
+                r.pos = len(b)
+                r.error("end of buffer")
+                return
+
+            if (
+                x
+                > (
+                    new_x := r.max_column(new_y)
+                )  # we're past the end of the previous line
+                or x == r.max_column(y)
+                and any(
+                    not i.isspace() for i in r.buffer[r.bol() :]
+                )  # move between eols
+            ):
+                x = new_x
+
+            r.setpos_from_xy(x, new_y)
+
+
+class left(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        for _ in range(r.get_arg()):
+            p = r.pos - 1
+            if p >= 0:
+                r.pos = p
+            else:
+                self.reader.error("start of buffer")
+
+
+class right(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        for _ in range(r.get_arg()):
+            p = r.pos + 1
+            if p <= len(b):
+                r.pos = p
+            else:
+                self.reader.error("end of buffer")
+
+
+class beginning_of_line(MotionCommand):
+    def do(self) -> None:
+        self.reader.pos = self.reader.bol()
+
+
+class end_of_line(MotionCommand):
+    def do(self) -> None:
+        self.reader.pos = self.reader.eol()
+
+
+class home(MotionCommand):
+    def do(self) -> None:
+        self.reader.pos = 0
+
+
+class end(MotionCommand):
+    def do(self) -> None:
+        self.reader.pos = len(self.reader.buffer)
+
+
+class forward_word(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        for i in range(r.get_arg()):
+            r.pos = r.eow()
+
+
+class backward_word(MotionCommand):
+    def do(self) -> None:
+        r = self.reader
+        for i in range(r.get_arg()):
+            r.pos = r.bow()
+
+
+class self_insert(EditCommand):
+    def do(self) -> None:
+        r = self.reader
+        text = self.event * r.get_arg()
+        r.insert(text)
+
+
+class insert_nl(EditCommand):
+    def do(self) -> None:
+        r = self.reader
+        r.insert("\n" * r.get_arg())
+
+
+class transpose_characters(EditCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        s = r.pos - 1
+        if s < 0:
+            r.error("cannot transpose at start of buffer")
+        else:
+            if s == len(b):
+                s -= 1
+            t = min(s + r.get_arg(), len(b) - 1)
+            c = b[s]
+            del b[s]
+            b.insert(t, c)
+            r.pos = t
+            r.dirty = True
+
+
+class backspace(EditCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        for i in range(r.get_arg()):
+            if r.pos > 0:
+                r.pos -= 1
+                del b[r.pos]
+                r.dirty = True
+            else:
+                self.reader.error("can't backspace at start")
+
+
+class delete(EditCommand):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        if (
+            r.pos == 0
+            and len(b) == 0  # this is something of a hack
+            and self.event[-1] == "\004"
+        ):
+            r.update_screen()
+            r.console.finish()
+            raise EOFError
+        for i in range(r.get_arg()):
+            if r.pos != len(b):
+                del b[r.pos]
+                r.dirty = True
+            else:
+                self.reader.error("end of buffer")
+
+
+class accept(FinishCommand):
+    def do(self) -> None:
+        pass
+
+
+class help(Command):
+    def do(self) -> None:
+        import _sitebuiltins
+
+        with self.reader.suspend():
+            self.reader.msg = _sitebuiltins._Helper()()  # type: ignore[assignment, call-arg]
+
+
+class invalid_key(Command):
+    def do(self) -> None:
+        pending = self.reader.console.getpending()
+        s = "".join(self.event) + pending.data
+        self.reader.error("`%r' not bound" % s)
+
+
+class invalid_command(Command):
+    def do(self) -> None:
+        s = self.event_name
+        self.reader.error("command `%s' not known" % s)
+
+
+class show_history(Command):
+    def do(self) -> None:
+        from .pager import get_pager
+        from site import gethistoryfile  # type: ignore[attr-defined]
+
+        history = os.linesep.join(self.reader.history[:])
+        self.reader.console.restore()
+        pager = get_pager()
+        pager(history, gethistoryfile())
+        self.reader.console.prepare()
+
+        # We need to copy over the state so that it's consistent between
+        # console and reader, and console does not overwrite/append stuff
+        self.reader.console.screen = self.reader.screen.copy()
+        self.reader.console.posxy = self.reader.cxy
+
+
+class paste_mode(Command):
+
+    def do(self) -> None:
+        self.reader.paste_mode = not self.reader.paste_mode
+        self.reader.dirty = True
+
+
+class enable_bracketed_paste(Command):
+    def do(self) -> None:
+        self.reader.paste_mode = True
+        self.reader.in_bracketed_paste = True
+
+class disable_bracketed_paste(Command):
+    def do(self) -> None:
+        self.reader.paste_mode = False
+        self.reader.in_bracketed_paste = False
+        self.reader.dirty = True

Lib/_pyrepl/completing_reader.py

@@ -0,0 +1,295 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Antonio Cuni
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import re
+from . import commands, console, reader
+from .reader import Reader
+
+
+# types
+Command = commands.Command
+if False:
+    from .types import KeySpec, CommandName
+
+
+def prefix(wordlist: list[str], j: int = 0) -> str:
+    d = {}
+    i = j
+    try:
+        while 1:
+            for word in wordlist:
+                d[word[i]] = 1
+            if len(d) > 1:
+                return wordlist[0][j:i]
+            i += 1
+            d = {}
+    except IndexError:
+        return wordlist[0][j:i]
+    return ""
+
+
+STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]")
+
+def stripcolor(s: str) -> str:
+    return STRIPCOLOR_REGEX.sub('', s)
+
+
+def real_len(s: str) -> int:
+    return len(stripcolor(s))
+
+
+def left_align(s: str, maxlen: int) -> str:
+    stripped = stripcolor(s)
+    if len(stripped) > maxlen:
+        # too bad, we remove the color
+        return stripped[:maxlen]
+    padding = maxlen - len(stripped)
+    return s + ' '*padding
+
+
+def build_menu(
+        cons: console.Console,
+        wordlist: list[str],
+        start: int,
+        use_brackets: bool,
+        sort_in_column: bool,
+) -> tuple[list[str], int]:
+    if use_brackets:
+        item = "[ %s ]"
+        padding = 4
+    else:
+        item = "%s  "
+        padding = 2
+    maxlen = min(max(map(real_len, wordlist)), cons.width - padding)
+    cols = int(cons.width / (maxlen + padding))
+    rows = int((len(wordlist) - 1)/cols + 1)
+
+    if sort_in_column:
+        # sort_in_column=False (default)     sort_in_column=True
+        #          A B C                       A D G
+        #          D E F                       B E
+        #          G                           C F
+        #
+        # "fill" the table with empty words, so we always have the same amout
+        # of rows for each column
+        missing = cols*rows - len(wordlist)
+        wordlist = wordlist + ['']*missing
+        indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))]
+        wordlist = [wordlist[i] for i in indexes]
+    menu = []
+    i = start
+    for r in range(rows):
+        row = []
+        for col in range(cols):
+            row.append(item % left_align(wordlist[i], maxlen))
+            i += 1
+            if i >= len(wordlist):
+                break
+        menu.append(''.join(row))
+        if i >= len(wordlist):
+            i = 0
+            break
+        if r + 5 > cons.height:
+            menu.append("   %d more... " % (len(wordlist) - i))
+            break
+    return menu, i
+
+# this gets somewhat user interface-y, and as a result the logic gets
+# very convoluted.
+#
+#  To summarise the summary of the summary:- people are a problem.
+#                  -- The Hitch-Hikers Guide to the Galaxy, Episode 12
+
+#### Desired behaviour of the completions commands.
+# the considerations are:
+# (1) how many completions are possible
+# (2) whether the last command was a completion
+# (3) if we can assume that the completer is going to return the same set of
+#     completions: this is controlled by the ``assume_immutable_completions``
+#     variable on the reader, which is True by default to match the historical
+#     behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match
+#     more closely readline's semantics (this is needed e.g. by
+#     fancycompleter)
+#
+# if there's no possible completion, beep at the user and point this out.
+# this is easy.
+#
+# if there's only one possible completion, stick it in.  if the last thing
+# user did was a completion, point out that he isn't getting anywhere, but
+# only if the ``assume_immutable_completions`` is True.
+#
+# now it gets complicated.
+#
+# for the first press of a completion key:
+#  if there's a common prefix, stick it in.
+
+#  irrespective of whether anything got stuck in, if the word is now
+#  complete, show the "complete but not unique" message
+
+#  if there's no common prefix and if the word is not now complete,
+#  beep.
+
+#        common prefix ->    yes          no
+#        word complete \/
+#            yes           "cbnu"      "cbnu"
+#            no              -          beep
+
+# for the second bang on the completion key
+#  there will necessarily be no common prefix
+#  show a menu of the choices.
+
+# for subsequent bangs, rotate the menu around (if there are sufficient
+# choices).
+
+
+class complete(commands.Command):
+    def do(self) -> None:
+        r: CompletingReader
+        r = self.reader  # type: ignore[assignment]
+        last_is_completer = r.last_command_is(self.__class__)
+        immutable_completions = r.assume_immutable_completions
+        completions_unchangable = last_is_completer and immutable_completions
+        stem = r.get_stem()
+        if not completions_unchangable:
+            r.cmpltn_menu_choices = r.get_completions(stem)
+
+        completions = r.cmpltn_menu_choices
+        if not completions:
+            r.error("no matches")
+        elif len(completions) == 1:
+            if completions_unchangable and len(completions[0]) == len(stem):
+                r.msg = "[ sole completion ]"
+                r.dirty = True
+            r.insert(completions[0][len(stem):])
+        else:
+            p = prefix(completions, len(stem))
+            if p:
+                r.insert(p)
+            if last_is_completer:
+                r.cmpltn_menu_visible = True
+                r.cmpltn_message_visible = False
+                r.cmpltn_menu, r.cmpltn_menu_end = build_menu(
+                    r.console, completions, r.cmpltn_menu_end,
+                    r.use_brackets, r.sort_in_column)
+                r.dirty = True
+            elif not r.cmpltn_menu_visible:
+                r.cmpltn_message_visible = True
+                if stem + p in completions:
+                    r.msg = "[ complete but not unique ]"
+                    r.dirty = True
+                else:
+                    r.msg = "[ not unique ]"
+                    r.dirty = True
+
+
+class self_insert(commands.self_insert):
+    def do(self) -> None:
+        r: CompletingReader
+        r = self.reader  # type: ignore[assignment]
+
+        commands.self_insert.do(self)
+        if r.cmpltn_menu_visible:
+            stem = r.get_stem()
+            if len(stem) < 1:
+                r.cmpltn_reset()
+            else:
+                completions = [w for w in r.cmpltn_menu_choices
+                               if w.startswith(stem)]
+                if completions:
+                    r.cmpltn_menu, r.cmpltn_menu_end = build_menu(
+                        r.console, completions, 0,
+                        r.use_brackets, r.sort_in_column)
+                else:
+                    r.cmpltn_reset()
+
+
+@dataclass
+class CompletingReader(Reader):
+    """Adds completion support"""
+
+    ### Class variables
+    # see the comment for the complete command
+    assume_immutable_completions = True
+    use_brackets = True  # display completions inside []
+    sort_in_column = False
+
+    ### Instance variables
+    cmpltn_menu: list[str] = field(init=False)
+    cmpltn_menu_visible: bool = field(init=False)
+    cmpltn_message_visible: bool = field(init=False)
+    cmpltn_menu_end: int = field(init=False)
+    cmpltn_menu_choices: list[str] = field(init=False)
+
+    def __post_init__(self) -> None:
+        super().__post_init__()
+        self.cmpltn_reset()
+        for c in (complete, self_insert):
+            self.commands[c.__name__] = c
+            self.commands[c.__name__.replace('_', '-')] = c
+
+    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+        return super().collect_keymap() + (
+            (r'\t', 'complete'),)
+
+    def after_command(self, cmd: Command) -> None:
+        super().after_command(cmd)
+        if not isinstance(cmd, (complete, self_insert)):
+            self.cmpltn_reset()
+
+    def calc_screen(self) -> list[str]:
+        screen = super().calc_screen()
+        if self.cmpltn_menu_visible:
+            # We display the completions menu below the current prompt
+            ly = self.lxy[1] + 1
+            screen[ly:ly] = self.cmpltn_menu
+            # If we're not in the middle of multiline edit, don't append to screeninfo
+            # since that screws up the position calculation in pos2xy function.
+            # This is a hack to prevent the cursor jumping
+            # into the completions menu when pressing left or down arrow.
+            if self.pos != len(self.buffer):
+                self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu)
+        return screen
+
+    def finish(self) -> None:
+        super().finish()
+        self.cmpltn_reset()
+
+    def cmpltn_reset(self) -> None:
+        self.cmpltn_menu = []
+        self.cmpltn_menu_visible = False
+        self.cmpltn_message_visible = False
+        self.cmpltn_menu_end = 0
+        self.cmpltn_menu_choices = []
+
+    def get_stem(self) -> str:
+        st = self.syntax_table
+        SW = reader.SYNTAX_WORD
+        b = self.buffer
+        p = self.pos - 1
+        while p >= 0 and st.get(b[p], SW) == SW:
+            p -= 1
+        return ''.join(b[p+1:self.pos])
+
+    def get_completions(self, stem: str) -> list[str]:
+        return []

Lib/_pyrepl/console.py

@@ -0,0 +1,213 @@
+#   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import _colorize  # type: ignore[import-not-found]
+
+from abc import ABC, abstractmethod
+import ast
+import code
+from dataclasses import dataclass, field
+import os.path
+import sys
+
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+    from typing import IO
+    from typing import Callable
+
+
+@dataclass
+class Event:
+    evt: str
+    data: str
+    raw: bytes = b""
+
+
+@dataclass
+class Console(ABC):
+    posxy: tuple[int, int]
+    screen: list[str] = field(default_factory=list)
+    height: int = 25
+    width: int = 80
+
+    def __init__(
+        self,
+        f_in: IO[bytes] | int = 0,
+        f_out: IO[bytes] | int = 1,
+        term: str = "",
+        encoding: str = "",
+    ):
+        self.encoding = encoding or sys.getdefaultencoding()
+
+        if isinstance(f_in, int):
+            self.input_fd = f_in
+        else:
+            self.input_fd = f_in.fileno()
+
+        if isinstance(f_out, int):
+            self.output_fd = f_out
+        else:
+            self.output_fd = f_out.fileno()
+
+    @abstractmethod
+    def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ...
+
+    @abstractmethod
+    def prepare(self) -> None: ...
+
+    @abstractmethod
+    def restore(self) -> None: ...
+
+    @abstractmethod
+    def move_cursor(self, x: int, y: int) -> None: ...
+
+    @abstractmethod
+    def set_cursor_vis(self, visible: bool) -> None: ...
+
+    @abstractmethod
+    def getheightwidth(self) -> tuple[int, int]:
+        """Return (height, width) where height and width are the height
+        and width of the terminal window in characters."""
+        ...
+
+    @abstractmethod
+    def get_event(self, block: bool = True) -> Event | None:
+        """Return an Event instance.  Returns None if |block| is false
+        and there is no event pending, otherwise waits for the
+        completion of an event."""
+        ...
+
+    @abstractmethod
+    def push_char(self, char: int | bytes) -> None:
+        """
+        Push a character to the console event queue.
+        """
+        ...
+
+    @abstractmethod
+    def beep(self) -> None: ...
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Wipe the screen"""
+        ...
+
+    @abstractmethod
+    def finish(self) -> None:
+        """Move the cursor to the end of the display and otherwise get
+        ready for end.  XXX could be merged with restore?  Hmm."""
+        ...
+
+    @abstractmethod
+    def flushoutput(self) -> None:
+        """Flush all output to the screen (assuming there's some
+        buffering going on somewhere)."""
+        ...
+
+    @abstractmethod
+    def forgetinput(self) -> None:
+        """Forget all pending, but not yet processed input."""
+        ...
+
+    @abstractmethod
+    def getpending(self) -> Event:
+        """Return the characters that have been typed but not yet
+        processed."""
+        ...
+
+    @abstractmethod
+    def wait(self, timeout: float | None) -> bool:
+        """Wait for an event. The return value is True if an event is
+        available, False if the timeout has been reached. If timeout is
+        None, wait forever. The timeout is in milliseconds."""
+        ...
+
+    @property
+    def input_hook(self) -> Callable[[], int] | None:
+        """Returns the current input hook."""
+        ...
+
+    @abstractmethod
+    def repaint(self) -> None: ...
+
+
+class InteractiveColoredConsole(code.InteractiveConsole):
+    def __init__(
+        self,
+        locals: dict[str, object] | None = None,
+        filename: str = "<console>",
+        *,
+        local_exit: bool = False,
+    ) -> None:
+        super().__init__(locals=locals, filename=filename, local_exit=local_exit)  # type: ignore[call-arg]
+        self.can_colorize = _colorize.can_colorize()
+
+    def showsyntaxerror(self, filename=None, **kwargs):
+        super().showsyntaxerror(filename=filename, **kwargs)
+
+    def _excepthook(self, typ, value, tb):
+        import traceback
+        lines = traceback.format_exception(
+                typ, value, tb,
+                colorize=self.can_colorize,
+                limit=traceback.BUILTIN_EXCEPTION_LIMIT)
+        self.write(''.join(lines))
+
+    def runsource(self, source, filename="<input>", symbol="single"):
+        try:
+            tree = self.compile.compiler(
+                source,
+                filename,
+                "exec",
+                ast.PyCF_ONLY_AST,
+                incomplete_input=False,
+            )
+        except (SyntaxError, OverflowError, ValueError):
+            self.showsyntaxerror(filename, source=source)
+            return False
+        if tree.body:
+            *_, last_stmt = tree.body
+        for stmt in tree.body:
+            wrapper = ast.Interactive if stmt is last_stmt else ast.Module
+            the_symbol = symbol if stmt is last_stmt else "exec"
+            item = wrapper([stmt])
+            try:
+                code = self.compile.compiler(item, filename, the_symbol)
+            except SyntaxError as e:
+                if e.args[0] == "'await' outside function":
+                    python = os.path.basename(sys.executable)
+                    e.add_note(
+                        f"Try the asyncio REPL ({python} -m asyncio) to use"
+                        f" top-level 'await' and run background asyncio tasks."
+                    )
+                self.showsyntaxerror(filename, source=source)
+                return False
+            except (OverflowError, ValueError):
+                self.showsyntaxerror(filename, source=source)
+                return False
+
+            if code is None:
+                return True
+
+            self.runcode(code)
+        return False

Lib/_pyrepl/curses.py

@@ -0,0 +1,33 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+try:
+    import _curses
+except ImportError:
+    try:
+        import curses as _curses  # type: ignore[no-redef]
+    except ImportError:
+        from . import _minimal_curses as _curses  # type: ignore[no-redef]
+
+setupterm = _curses.setupterm
+tigetstr = _curses.tigetstr
+tparm = _curses.tparm
+error = _curses.error

Lib/_pyrepl/fancy_termios.py

@@ -0,0 +1,76 @@
+#   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import termios
+
+
+class TermState:
+    def __init__(self, tuples):
+        (
+            self.iflag,
+            self.oflag,
+            self.cflag,
+            self.lflag,
+            self.ispeed,
+            self.ospeed,
+            self.cc,
+        ) = tuples
+
+    def as_list(self):
+        return [
+            self.iflag,
+            self.oflag,
+            self.cflag,
+            self.lflag,
+            self.ispeed,
+            self.ospeed,
+            # Always return a copy of the control characters list to ensure
+            # there are not any additional references to self.cc
+            self.cc[:],
+        ]
+
+    def copy(self):
+        return self.__class__(self.as_list())
+
+
+def tcgetattr(fd):
+    return TermState(termios.tcgetattr(fd))
+
+
+def tcsetattr(fd, when, attrs):
+    termios.tcsetattr(fd, when, attrs.as_list())
+
+
+class Term(TermState):
+    TS__init__ = TermState.__init__
+
+    def __init__(self, fd=0):
+        self.TS__init__(termios.tcgetattr(fd))
+        self.fd = fd
+        self.stack = []
+
+    def save(self):
+        self.stack.append(self.as_list())
+
+    def set(self, when=termios.TCSANOW):
+        termios.tcsetattr(self.fd, when, self.as_list())
+
+    def restore(self):
+        self.TS__init__(self.stack.pop())
+        self.set()

Lib/_pyrepl/historical_reader.py

@@ -0,0 +1,419 @@
+#   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+
+from . import commands, input
+from .reader import Reader
+
+
+if False:
+    from .types import SimpleContextManager, KeySpec, CommandName
+
+
+isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple(
+    [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"]
+    + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"]
+    + [
+        ("\\%03o" % c, "isearch-add-character")
+        for c in range(256)
+        if chr(c).isalpha() and chr(c) != "\\"
+    ]
+    + [
+        ("\\\\", "self-insert"),
+        (r"\C-r", "isearch-backwards"),
+        (r"\C-s", "isearch-forwards"),
+        (r"\C-c", "isearch-cancel"),
+        (r"\C-g", "isearch-cancel"),
+        (r"\<backspace>", "isearch-backspace"),
+    ]
+)
+
+ISEARCH_DIRECTION_NONE = ""
+ISEARCH_DIRECTION_BACKWARDS = "r"
+ISEARCH_DIRECTION_FORWARDS = "f"
+
+
+class next_history(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        if r.historyi == len(r.history):
+            r.error("end of history list")
+            return
+        r.select_item(r.historyi + 1)
+
+
+class previous_history(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        if r.historyi == 0:
+            r.error("start of history list")
+            return
+        r.select_item(r.historyi - 1)
+
+
+class history_search_backward(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.search_next(forwards=False)
+
+
+class history_search_forward(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.search_next(forwards=True)
+
+
+class restore_history(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        if r.historyi != len(r.history):
+            if r.get_unicode() != r.history[r.historyi]:
+                r.buffer = list(r.history[r.historyi])
+                r.pos = len(r.buffer)
+                r.dirty = True
+
+
+class first_history(commands.Command):
+    def do(self) -> None:
+        self.reader.select_item(0)
+
+
+class last_history(commands.Command):
+    def do(self) -> None:
+        self.reader.select_item(len(self.reader.history))
+
+
+class operate_and_get_next(commands.FinishCommand):
+    def do(self) -> None:
+        self.reader.next_history = self.reader.historyi + 1
+
+
+class yank_arg(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        if r.last_command is self.__class__:
+            r.yank_arg_i += 1
+        else:
+            r.yank_arg_i = 0
+        if r.historyi < r.yank_arg_i:
+            r.error("beginning of history list")
+            return
+        a = r.get_arg(-1)
+        # XXX how to split?
+        words = r.get_item(r.historyi - r.yank_arg_i - 1).split()
+        if a < -len(words) or a >= len(words):
+            r.error("no such arg")
+            return
+        w = words[a]
+        b = r.buffer
+        if r.yank_arg_i > 0:
+            o = len(r.yank_arg_yanked)
+        else:
+            o = 0
+        b[r.pos - o : r.pos] = list(w)
+        r.yank_arg_yanked = w
+        r.pos += len(w) - o
+        r.dirty = True
+
+
+class forward_history_isearch(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_FORWARDS
+        r.isearch_start = r.historyi, r.pos
+        r.isearch_term = ""
+        r.dirty = True
+        r.push_input_trans(r.isearch_trans)
+
+
+class reverse_history_isearch(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS
+        r.dirty = True
+        r.isearch_term = ""
+        r.push_input_trans(r.isearch_trans)
+        r.isearch_start = r.historyi, r.pos
+
+
+class isearch_cancel(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_NONE
+        r.pop_input_trans()
+        r.select_item(r.isearch_start[0])
+        r.pos = r.isearch_start[1]
+        r.dirty = True
+
+
+class isearch_add_character(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        r.isearch_term += self.event[-1]
+        r.dirty = True
+        p = r.pos + len(r.isearch_term) - 1
+        if b[p : p + 1] != [r.isearch_term[-1]]:
+            r.isearch_next()
+
+
+class isearch_backspace(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        if len(r.isearch_term) > 0:
+            r.isearch_term = r.isearch_term[:-1]
+            r.dirty = True
+        else:
+            r.error("nothing to rubout")
+
+
+class isearch_forwards(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_FORWARDS
+        r.isearch_next()
+
+
+class isearch_backwards(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS
+        r.isearch_next()
+
+
+class isearch_end(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        r.isearch_direction = ISEARCH_DIRECTION_NONE
+        r.console.forgetinput()
+        r.pop_input_trans()
+        r.dirty = True
+
+
+@dataclass
+class HistoricalReader(Reader):
+    """Adds history support (with incremental history searching) to the
+    Reader class.
+    """
+
+    history: list[str] = field(default_factory=list)
+    historyi: int = 0
+    next_history: int | None = None
+    transient_history: dict[int, str] = field(default_factory=dict)
+    isearch_term: str = ""
+    isearch_direction: str = ISEARCH_DIRECTION_NONE
+    isearch_start: tuple[int, int] = field(init=False)
+    isearch_trans: input.KeymapTranslator = field(init=False)
+    yank_arg_i: int = 0
+    yank_arg_yanked: str = ""
+
+    def __post_init__(self) -> None:
+        super().__post_init__()
+        for c in [
+            next_history,
+            previous_history,
+            restore_history,
+            first_history,
+            last_history,
+            yank_arg,
+            forward_history_isearch,
+            reverse_history_isearch,
+            isearch_end,
+            isearch_add_character,
+            isearch_cancel,
+            isearch_add_character,
+            isearch_backspace,
+            isearch_forwards,
+            isearch_backwards,
+            operate_and_get_next,
+            history_search_backward,
+            history_search_forward,
+        ]:
+            self.commands[c.__name__] = c
+            self.commands[c.__name__.replace("_", "-")] = c
+        self.isearch_start = self.historyi, self.pos
+        self.isearch_trans = input.KeymapTranslator(
+            isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character
+        )
+
+    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+        return super().collect_keymap() + (
+            (r"\C-n", "next-history"),
+            (r"\C-p", "previous-history"),
+            (r"\C-o", "operate-and-get-next"),
+            (r"\C-r", "reverse-history-isearch"),
+            (r"\C-s", "forward-history-isearch"),
+            (r"\M-r", "restore-history"),
+            (r"\M-.", "yank-arg"),
+            (r"\<page down>", "history-search-forward"),
+            (r"\x1b[6~", "history-search-forward"),
+            (r"\<page up>", "history-search-backward"),
+            (r"\x1b[5~", "history-search-backward"),
+        )
+
+    def select_item(self, i: int) -> None:
+        self.transient_history[self.historyi] = self.get_unicode()
+        buf = self.transient_history.get(i)
+        if buf is None:
+            buf = self.history[i].rstrip()
+        self.buffer = list(buf)
+        self.historyi = i
+        self.pos = len(self.buffer)
+        self.dirty = True
+        self.last_refresh_cache.invalidated = True
+
+    def get_item(self, i: int) -> str:
+        if i != len(self.history):
+            return self.transient_history.get(i, self.history[i])
+        else:
+            return self.transient_history.get(i, self.get_unicode())
+
+    @contextmanager
+    def suspend(self) -> SimpleContextManager:
+        with super().suspend(), self.suspend_history():
+            yield
+
+    @contextmanager
+    def suspend_history(self) -> SimpleContextManager:
+        try:
+            old_history = self.history[:]
+            del self.history[:]
+            yield
+        finally:
+            self.history[:] = old_history
+
+    def prepare(self) -> None:
+        super().prepare()
+        try:
+            self.transient_history = {}
+            if self.next_history is not None and self.next_history < len(self.history):
+                self.historyi = self.next_history
+                self.buffer[:] = list(self.history[self.next_history])
+                self.pos = len(self.buffer)
+                self.transient_history[len(self.history)] = ""
+            else:
+                self.historyi = len(self.history)
+            self.next_history = None
+        except:
+            self.restore()
+            raise
+
+    def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
+        if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE:
+            d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS]
+            return "(%s-search `%s') " % (d, self.isearch_term)
+        else:
+            return super().get_prompt(lineno, cursor_on_line)
+
+    def search_next(self, *, forwards: bool) -> None:
+        """Search history for the current line contents up to the cursor.
+
+        Selects the first item found. If nothing is under the cursor, any next
+        item in history is selected.
+        """
+        pos = self.pos
+        s = self.get_unicode()
+        history_index = self.historyi
+
+        # In multiline contexts, we're only interested in the current line.
+        nl_index = s.rfind('\n', 0, pos)
+        prefix = s[nl_index + 1:pos]
+        pos = len(prefix)
+
+        match_prefix = len(prefix)
+        len_item = 0
+        if history_index < len(self.history):
+            len_item = len(self.get_item(history_index))
+        if len_item and pos == len_item:
+            match_prefix = False
+        elif not pos:
+            match_prefix = False
+
+        while 1:
+            if forwards:
+                out_of_bounds = history_index >= len(self.history) - 1
+            else:
+                out_of_bounds = history_index == 0
+            if out_of_bounds:
+                if forwards and not match_prefix:
+                    self.pos = 0
+                    self.buffer = []
+                    self.dirty = True
+                else:
+                    self.error("not found")
+                return
+
+            history_index += 1 if forwards else -1
+            s = self.get_item(history_index)
+
+            if not match_prefix:
+                self.select_item(history_index)
+                return
+
+            len_acc = 0
+            for i, line in enumerate(s.splitlines(keepends=True)):
+                if line.startswith(prefix):
+                    self.select_item(history_index)
+                    self.pos = pos + len_acc
+                    return
+                len_acc += len(line)
+
+    def isearch_next(self) -> None:
+        st = self.isearch_term
+        p = self.pos
+        i = self.historyi
+        s = self.get_unicode()
+        forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS
+        while 1:
+            if forwards:
+                p = s.find(st, p + 1)
+            else:
+                p = s.rfind(st, 0, p + len(st) - 1)
+            if p != -1:
+                self.select_item(i)
+                self.pos = p
+                return
+            elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0):
+                self.error("not found")
+                return
+            else:
+                if forwards:
+                    i += 1
+                    s = self.get_item(i)
+                    p = -1
+                else:
+                    i -= 1
+                    s = self.get_item(i)
+                    p = len(s)
+
+    def finish(self) -> None:
+        super().finish()
+        ret = self.get_unicode()
+        for i, t in self.transient_history.items():
+            if i < len(self.history) and i != self.historyi:
+                self.history[i] = t
+        if ret and should_auto_add_history:
+            self.history.append(ret)
+
+
+should_auto_add_history = True

Lib/_pyrepl/input.py

@@ -0,0 +1,114 @@
+#   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+# (naming modules after builtin functions is not such a hot idea...)
+
+# an KeyTrans instance translates Event objects into Command objects
+
+# hmm, at what level do we want [C-i] and [tab] to be equivalent?
+# [meta-a] and [esc a]?  obviously, these are going to be equivalent
+# for the UnixConsole, but should they be for PygameConsole?
+
+# it would in any situation seem to be a bad idea to bind, say, [tab]
+# and [C-i] to *different* things... but should binding one bind the
+# other?
+
+# executive, temporary decision: [tab] and [C-i] are distinct, but
+# [meta-key] is identified with [esc key].  We demand that any console
+# class does quite a lot towards emulating a unix terminal.
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+import unicodedata
+from collections import deque
+
+
+# types
+if False:
+    from .types import EventTuple
+
+
+class InputTranslator(ABC):
+    @abstractmethod
+    def push(self, evt: EventTuple) -> None:
+        pass
+
+    @abstractmethod
+    def get(self) -> EventTuple | None:
+        return None
+
+    @abstractmethod
+    def empty(self) -> bool:
+        return True
+
+
+class KeymapTranslator(InputTranslator):
+    def __init__(self, keymap, verbose=False, invalid_cls=None, character_cls=None):
+        self.verbose = verbose
+        from .keymap import compile_keymap, parse_keys
+
+        self.keymap = keymap
+        self.invalid_cls = invalid_cls
+        self.character_cls = character_cls
+        d = {}
+        for keyspec, command in keymap:
+            keyseq = tuple(parse_keys(keyspec))
+            d[keyseq] = command
+        if self.verbose:
+            print(d)
+        self.k = self.ck = compile_keymap(d, ())
+        self.results = deque()
+        self.stack = []
+
+    def push(self, evt):
+        if self.verbose:
+            print("pushed", evt.data, end="")
+        key = evt.data
+        d = self.k.get(key)
+        if isinstance(d, dict):
+            if self.verbose:
+                print("transition")
+            self.stack.append(key)
+            self.k = d
+        else:
+            if d is None:
+                if self.verbose:
+                    print("invalid")
+                if self.stack or len(key) > 1 or unicodedata.category(key) == "C":
+                    self.results.append((self.invalid_cls, self.stack + [key]))
+                else:
+                    # small optimization:
+                    self.k[key] = self.character_cls
+                    self.results.append((self.character_cls, [key]))
+            else:
+                if self.verbose:
+                    print("matched", d)
+                self.results.append((d, self.stack + [key]))
+            self.stack = []
+            self.k = self.ck
+
+    def get(self):
+        if self.results:
+            return self.results.popleft()
+        else:
+            return None
+
+    def empty(self) -> bool:
+        return not self.results

Lib/_pyrepl/keymap.py

@@ -0,0 +1,213 @@
+#   Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Keymap contains functions for parsing keyspecs and turning keyspecs into
+appropriate sequences.
+
+A keyspec is a string representing a sequence of key presses that can
+be bound to a command. All characters other than the backslash represent
+themselves. In the traditional manner, a backslash introduces an escape
+sequence.
+
+pyrepl uses its own keyspec format that is meant to be a strict superset of
+readline's KEYSEQ format. This means that if a spec is found that readline
+accepts that this doesn't, it should be logged as a bug. Note that this means
+we're using the `\\C-o' style of readline's keyspec, not the `Control-o' sort.
+
+The extension to readline is that the sequence \\<KEY> denotes the
+sequence of characters produced by hitting KEY.
+
+Examples:
+`a'      - what you get when you hit the `a' key
+`\\EOA'  - Escape - O - A (up, on my terminal)
+`\\<UP>' - the up arrow key
+`\\<up>' - ditto (keynames are case-insensitive)
+`\\C-o', `\\c-o'  - control-o
+`\\M-.'  - meta-period
+`\\E.'   - ditto (that's how meta works for pyrepl)
+`\\<tab>', `\\<TAB>', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I'
+   - all of these are the tab character.
+"""
+
+_escapes = {
+    "\\": "\\",
+    "'": "'",
+    '"': '"',
+    "a": "\a",
+    "b": "\b",
+    "e": "\033",
+    "f": "\f",
+    "n": "\n",
+    "r": "\r",
+    "t": "\t",
+    "v": "\v",
+}
+
+_keynames = {
+    "backspace": "backspace",
+    "delete": "delete",
+    "down": "down",
+    "end": "end",
+    "enter": "\r",
+    "escape": "\033",
+    "f1": "f1",
+    "f2": "f2",
+    "f3": "f3",
+    "f4": "f4",
+    "f5": "f5",
+    "f6": "f6",
+    "f7": "f7",
+    "f8": "f8",
+    "f9": "f9",
+    "f10": "f10",
+    "f11": "f11",
+    "f12": "f12",
+    "f13": "f13",
+    "f14": "f14",
+    "f15": "f15",
+    "f16": "f16",
+    "f17": "f17",
+    "f18": "f18",
+    "f19": "f19",
+    "f20": "f20",
+    "home": "home",
+    "insert": "insert",
+    "left": "left",
+    "page down": "page down",
+    "page up": "page up",
+    "return": "\r",
+    "right": "right",
+    "space": " ",
+    "tab": "\t",
+    "up": "up",
+}
+
+
+class KeySpecError(Exception):
+    pass
+
+
+def parse_keys(keys: str) -> list[str]:
+    """Parse keys in keyspec format to a sequence of keys."""
+    s = 0
+    r: list[str] = []
+    while s < len(keys):
+        k, s = _parse_single_key_sequence(keys, s)
+        r.extend(k)
+    return r
+
+
+def _parse_single_key_sequence(key: str, s: int) -> tuple[list[str], int]:
+    ctrl = 0
+    meta = 0
+    ret = ""
+    while not ret and s < len(key):
+        if key[s] == "\\":
+            c = key[s + 1].lower()
+            if c in _escapes:
+                ret = _escapes[c]
+                s += 2
+            elif c == "c":
+                if key[s + 2] != "-":
+                    raise KeySpecError(
+                        "\\C must be followed by `-' (char %d of %s)"
+                        % (s + 2, repr(key))
+                    )
+                if ctrl:
+                    raise KeySpecError(
+                        "doubled \\C- (char %d of %s)" % (s + 1, repr(key))
+                    )
+                ctrl = 1
+                s += 3
+            elif c == "m":
+                if key[s + 2] != "-":
+                    raise KeySpecError(
+                        "\\M must be followed by `-' (char %d of %s)"
+                        % (s + 2, repr(key))
+                    )
+                if meta:
+                    raise KeySpecError(
+                        "doubled \\M- (char %d of %s)" % (s + 1, repr(key))
+                    )
+                meta = 1
+                s += 3
+            elif c.isdigit():
+                n = key[s + 1 : s + 4]
+                ret = chr(int(n, 8))
+                s += 4
+            elif c == "x":
+                n = key[s + 2 : s + 4]
+                ret = chr(int(n, 16))
+                s += 4
+            elif c == "<":
+                t = key.find(">", s)
+                if t == -1:
+                    raise KeySpecError(
+                        "unterminated \\< starting at char %d of %s"
+                        % (s + 1, repr(key))
+                    )
+                ret = key[s + 2 : t].lower()
+                if ret not in _keynames:
+                    raise KeySpecError(
+                        "unrecognised keyname `%s' at char %d of %s"
+                        % (ret, s + 2, repr(key))
+                    )
+                ret = _keynames[ret]
+                s = t + 1
+            else:
+                raise KeySpecError(
+                    "unknown backslash escape %s at char %d of %s"
+                    % (repr(c), s + 2, repr(key))
+                )
+        else:
+            ret = key[s]
+            s += 1
+    if ctrl:
+        if len(ret) == 1:
+            ret = chr(ord(ret) & 0x1F)  # curses.ascii.ctrl()
+        elif ret in {"left", "right"}:
+            ret = f"ctrl {ret}"
+        else:
+            raise KeySpecError("\\C- followed by invalid key")
+
+    result = [ret], s
+    if meta:
+        result[0].insert(0, "\033")
+    return result
+
+
+def compile_keymap(keymap, empty=b""):
+    r = {}
+    for key, value in keymap.items():
+        if isinstance(key, bytes):
+            first = key[:1]
+        else:
+            first = key[0]
+        r.setdefault(first, {})[key[1:]] = value
+    for key, value in r.items():
+        if empty in value:
+            if len(value) != 1:
+                raise KeySpecError("key definitions for %s clash" % (value.values(),))
+            else:
+                r[key] = value[empty]
+        else:
+            r[key] = compile_keymap(value, empty)
+    return r

Lib/_pyrepl/main.py

@@ -0,0 +1,59 @@
+import errno
+import os
+import sys
+
+
+CAN_USE_PYREPL: bool
+FAIL_REASON: str
+try:
+    if sys.platform == "win32" and sys.getwindowsversion().build < 10586:
+        raise RuntimeError("Windows 10 TH2 or later required")
+    if not os.isatty(sys.stdin.fileno()):
+        raise OSError(errno.ENOTTY, "tty required", "stdin")
+    from .simple_interact import check
+    if err := check():
+        raise RuntimeError(err)
+except Exception as e:
+    CAN_USE_PYREPL = False
+    FAIL_REASON = f"warning: can't use pyrepl: {e}"
+else:
+    CAN_USE_PYREPL = True
+    FAIL_REASON = ""
+
+
+def interactive_console(mainmodule=None, quiet=False, pythonstartup=False):
+    if not CAN_USE_PYREPL:
+        if not os.getenv('PYTHON_BASIC_REPL') and FAIL_REASON:
+            from .trace import trace
+            trace(FAIL_REASON)
+            print(FAIL_REASON, file=sys.stderr)
+        return sys._baserepl()
+
+    if mainmodule:
+        namespace = mainmodule.__dict__
+    else:
+        import __main__
+        namespace = __main__.__dict__
+        namespace.pop("__pyrepl_interactive_console", None)
+
+    # sys._baserepl() above does this internally, we do it here
+    startup_path = os.getenv("PYTHONSTARTUP")
+    if pythonstartup and startup_path:
+        sys.audit("cpython.run_startup", startup_path)
+
+        import tokenize
+        with tokenize.open(startup_path) as f:
+            startup_code = compile(f.read(), startup_path, "exec")
+            exec(startup_code, namespace)
+
+    # set sys.{ps1,ps2} just before invoking the interactive interpreter. This
+    # mimics what CPython does in pythonrun.c
+    if not hasattr(sys, "ps1"):
+        sys.ps1 = ">>> "
+    if not hasattr(sys, "ps2"):
+        sys.ps2 = "... "
+
+    from .console import InteractiveColoredConsole
+    from .simple_interact import run_multiline_interactive_console
+    console = InteractiveColoredConsole(namespace, filename="<stdin>")
+    run_multiline_interactive_console(console)

Lib/_pyrepl/mypy.ini

@@ -0,0 +1,24 @@
+# Config file for running mypy on _pyrepl.
+# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini`
+# on the command-line from the repo root
+
+[mypy]
+files = Lib/_pyrepl
+explicit_package_bases = True
+python_version = 3.12
+platform = linux
+pretty = True
+
+# Enable most stricter settings
+enable_error_code = ignore-without-code,redundant-expr
+strict = True
+
+# Various stricter settings that we can't yet enable
+# Try to enable these in the following order:
+disallow_untyped_calls = False
+disallow_untyped_defs = False
+check_untyped_defs = False
+
+# Various internal modules that typeshed deliberately doesn't have stubs for:
+[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*]
+ignore_missing_imports = True

Lib/_pyrepl/pager.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import io
+import os
+import re
+import sys
+
+
+# types
+if False:
+    from typing import Protocol
+    class Pager(Protocol):
+        def __call__(self, text: str, title: str = "") -> None:
+            ...
+
+
+def get_pager() -> Pager:
+    """Decide what method to use for paging through text."""
+    if not hasattr(sys.stdin, "isatty"):
+        return plain_pager
+    if not hasattr(sys.stdout, "isatty"):
+        return plain_pager
+    if not sys.stdin.isatty() or not sys.stdout.isatty():
+        return plain_pager
+    if sys.platform == "emscripten":
+        return plain_pager
+    use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER')
+    if use_pager:
+        if sys.platform == 'win32': # pipes completely broken in Windows
+            return lambda text, title='': tempfile_pager(plain(text), use_pager)
+        elif os.environ.get('TERM') in ('dumb', 'emacs'):
+            return lambda text, title='': pipe_pager(plain(text), use_pager, title)
+        else:
+            return lambda text, title='': pipe_pager(text, use_pager, title)
+    if os.environ.get('TERM') in ('dumb', 'emacs'):
+        return plain_pager
+    if sys.platform == 'win32':
+        return lambda text, title='': tempfile_pager(plain(text), 'more <')
+    if hasattr(os, 'system') and os.system('(pager) 2>/dev/null') == 0:
+        return lambda text, title='': pipe_pager(text, 'pager', title)
+    if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0:
+        return lambda text, title='': pipe_pager(text, 'less', title)
+
+    import tempfile
+    (fd, filename) = tempfile.mkstemp()
+    os.close(fd)
+    try:
+        if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0:
+            return lambda text, title='': pipe_pager(text, 'more', title)
+        else:
+            return tty_pager
+    finally:
+        os.unlink(filename)
+
+
+def escape_stdout(text: str) -> str:
+    # Escape non-encodable characters to avoid encoding errors later
+    encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8'
+    return text.encode(encoding, 'backslashreplace').decode(encoding)
+
+
+def escape_less(s: str) -> str:
+    return re.sub(r'([?:.%\\])', r'\\\1', s)
+
+
+def plain(text: str) -> str:
+    """Remove boldface formatting from text."""
+    return re.sub('.\b', '', text)
+
+
+def tty_pager(text: str, title: str = '') -> None:
+    """Page through text on a text terminal."""
+    lines = plain(escape_stdout(text)).split('\n')
+    has_tty = False
+    try:
+        import tty
+        import termios
+        fd = sys.stdin.fileno()
+        old = termios.tcgetattr(fd)
+        tty.setcbreak(fd)
+        has_tty = True
+
+        def getchar() -> str:
+            return sys.stdin.read(1)
+
+    except (ImportError, AttributeError, io.UnsupportedOperation):
+        def getchar() -> str:
+            return sys.stdin.readline()[:-1][:1]
+
+    try:
+        try:
+            h = int(os.environ.get('LINES', 0))
+        except ValueError:
+            h = 0
+        if h <= 1:
+            h = 25
+        r = inc = h - 1
+        sys.stdout.write('\n'.join(lines[:inc]) + '\n')
+        while lines[r:]:
+            sys.stdout.write('-- more --')
+            sys.stdout.flush()
+            c = getchar()
+
+            if c in ('q', 'Q'):
+                sys.stdout.write('\r          \r')
+                break
+            elif c in ('\r', '\n'):
+                sys.stdout.write('\r          \r' + lines[r] + '\n')
+                r = r + 1
+                continue
+            if c in ('b', 'B', '\x1b'):
+                r = r - inc - inc
+                if r < 0: r = 0
+            sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n')
+            r = r + inc
+
+    finally:
+        if has_tty:
+            termios.tcsetattr(fd, termios.TCSAFLUSH, old)
+
+
+def plain_pager(text: str, title: str = '') -> None:
+    """Simply print unformatted text.  This is the ultimate fallback."""
+    sys.stdout.write(plain(escape_stdout(text)))
+
+
+def pipe_pager(text: str, cmd: str, title: str = '') -> None:
+    """Page through text by feeding it to another program."""
+    import subprocess
+    env = os.environ.copy()
+    if title:
+        title += ' '
+    esc_title = escape_less(title)
+    prompt_string = (
+        f' {esc_title}' +
+        '?ltline %lt?L/%L.'
+        ':byte %bB?s/%s.'
+        '.'
+        '?e (END):?pB %pB\\%..'
+        ' (press h for help or q to quit)')
+    env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string)
+    proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
+                            errors='backslashreplace', env=env)
+    assert proc.stdin is not None
+    try:
+        with proc.stdin as pipe:
+            try:
+                pipe.write(text)
+            except KeyboardInterrupt:
+                # We've hereby abandoned whatever text hasn't been written,
+                # but the pager is still in control of the terminal.
+                pass
+    except OSError:
+        pass # Ignore broken pipes caused by quitting the pager program.
+    while True:
+        try:
+            proc.wait()
+            break
+        except KeyboardInterrupt:
+            # Ignore ctl-c like the pager itself does.  Otherwise the pager is
+            # left running and the terminal is in raw mode and unusable.
+            pass
+
+
+def tempfile_pager(text: str, cmd: str, title: str = '') -> None:
+    """Page through text by invoking a program on a temporary file."""
+    import tempfile
+    with tempfile.TemporaryDirectory() as tempdir:
+        filename = os.path.join(tempdir, 'pydoc.out')
+        with open(filename, 'w', errors='backslashreplace',
+                  encoding=os.device_encoding(0) if
+                  sys.platform == 'win32' else None
+                  ) as file:
+            file.write(text)
+        os.system(cmd + ' "' + filename + '"')

Lib/_pyrepl/reader.py

@@ -0,0 +1,816 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Antonio Cuni
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import sys
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field, fields
+import unicodedata
+from _colorize import can_colorize, ANSIColors  # type: ignore[import-not-found]
+
+
+from . import commands, console, input
+from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width
+from .trace import trace
+
+
+# types
+Command = commands.Command
+from .types import Callback, SimpleContextManager, KeySpec, CommandName
+
+
+def disp_str(buffer: str) -> tuple[str, list[int]]:
+    """disp_str(buffer:string) -> (string, [int])
+
+    Return the string that should be the printed representation of
+    |buffer| and a list detailing where the characters of |buffer|
+    get used up.  E.g.:
+
+    >>> disp_str(chr(3))
+    ('^C', [1, 0])
+
+    """
+    b: list[int] = []
+    s: list[str] = []
+    for c in buffer:
+        if c == '\x1a':
+            s.append(c)
+            b.append(2)
+        elif ord(c) < 128:
+            s.append(c)
+            b.append(1)
+        elif unicodedata.category(c).startswith("C"):
+            c = r"\u%04x" % ord(c)
+            s.append(c)
+            b.extend([0] * (len(c) - 1))
+        else:
+            s.append(c)
+            b.append(str_width(c))
+    return "".join(s), b
+
+
+# syntax classes:
+
+SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3)
+
+
+def make_default_syntax_table() -> dict[str, int]:
+    # XXX perhaps should use some unicodedata here?
+    st: dict[str, int] = {}
+    for c in map(chr, range(256)):
+        st[c] = SYNTAX_SYMBOL
+    for c in [a for a in map(chr, range(256)) if a.isalnum()]:
+        st[c] = SYNTAX_WORD
+    st["\n"] = st[" "] = SYNTAX_WHITESPACE
+    return st
+
+
+def make_default_commands() -> dict[CommandName, type[Command]]:
+    result: dict[CommandName, type[Command]] = {}
+    for v in vars(commands).values():
+        if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower():
+            result[v.__name__] = v
+            result[v.__name__.replace("_", "-")] = v
+    return result
+
+
+default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple(
+    [
+        (r"\C-a", "beginning-of-line"),
+        (r"\C-b", "left"),
+        (r"\C-c", "interrupt"),
+        (r"\C-d", "delete"),
+        (r"\C-e", "end-of-line"),
+        (r"\C-f", "right"),
+        (r"\C-g", "cancel"),
+        (r"\C-h", "backspace"),
+        (r"\C-j", "accept"),
+        (r"\<return>", "accept"),
+        (r"\C-k", "kill-line"),
+        (r"\C-l", "clear-screen"),
+        (r"\C-m", "accept"),
+        (r"\C-t", "transpose-characters"),
+        (r"\C-u", "unix-line-discard"),
+        (r"\C-w", "unix-word-rubout"),
+        (r"\C-x\C-u", "upcase-region"),
+        (r"\C-y", "yank"),
+        *(() if sys.platform == "win32" else ((r"\C-z", "suspend"), )),
+        (r"\M-b", "backward-word"),
+        (r"\M-c", "capitalize-word"),
+        (r"\M-d", "kill-word"),
+        (r"\M-f", "forward-word"),
+        (r"\M-l", "downcase-word"),
+        (r"\M-t", "transpose-words"),
+        (r"\M-u", "upcase-word"),
+        (r"\M-y", "yank-pop"),
+        (r"\M--", "digit-arg"),
+        (r"\M-0", "digit-arg"),
+        (r"\M-1", "digit-arg"),
+        (r"\M-2", "digit-arg"),
+        (r"\M-3", "digit-arg"),
+        (r"\M-4", "digit-arg"),
+        (r"\M-5", "digit-arg"),
+        (r"\M-6", "digit-arg"),
+        (r"\M-7", "digit-arg"),
+        (r"\M-8", "digit-arg"),
+        (r"\M-9", "digit-arg"),
+        (r"\M-\n", "accept"),
+        ("\\\\", "self-insert"),
+        (r"\x1b[200~", "enable_bracketed_paste"),
+        (r"\x1b[201~", "disable_bracketed_paste"),
+        (r"\x03", "ctrl-c"),
+    ]
+    + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"]
+    + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()]
+    + [
+        (r"\<up>", "up"),
+        (r"\<down>", "down"),
+        (r"\<left>", "left"),
+        (r"\C-\<left>", "backward-word"),
+        (r"\<right>", "right"),
+        (r"\C-\<right>", "forward-word"),
+        (r"\<delete>", "delete"),
+        (r"\x1b[3~", "delete"),
+        (r"\<backspace>", "backspace"),
+        (r"\M-\<backspace>", "backward-kill-word"),
+        (r"\<end>", "end-of-line"),  # was 'end'
+        (r"\<home>", "beginning-of-line"),  # was 'home'
+        (r"\<f1>", "help"),
+        (r"\<f2>", "show-history"),
+        (r"\<f3>", "paste-mode"),
+        (r"\EOF", "end"),  # the entries in the terminfo database for xterms
+        (r"\EOH", "home"),  # seem to be wrong.  this is a less than ideal
+        # workaround
+    ]
+)
+
+
+@dataclass(slots=True)
+class Reader:
+    """The Reader class implements the bare bones of a command reader,
+    handling such details as editing and cursor motion.  What it does
+    not support are such things as completion or history support -
+    these are implemented elsewhere.
+
+    Instance variables of note include:
+
+      * buffer:
+        A *list* (*not* a string at the moment :-) containing all the
+        characters that have been entered.
+      * console:
+        Hopefully encapsulates the OS dependent stuff.
+      * pos:
+        A 0-based index into `buffer' for where the insertion point
+        is.
+      * screeninfo:
+        Ahem.  This list contains some info needed to move the
+        insertion point around reasonably efficiently.
+      * cxy, lxy:
+        the position of the insertion point in screen ...
+      * syntax_table:
+        Dictionary mapping characters to `syntax class'; read the
+        emacs docs to see what this means :-)
+      * commands:
+        Dictionary mapping command names to command classes.
+      * arg:
+        The emacs-style prefix argument.  It will be None if no such
+        argument has been provided.
+      * dirty:
+        True if we need to refresh the display.
+      * kill_ring:
+        The emacs-style kill-ring; manipulated with yank & yank-pop
+      * ps1, ps2, ps3, ps4:
+        prompts.  ps1 is the prompt for a one-line input; for a
+        multiline input it looks like:
+            ps2> first line of input goes here
+            ps3> second and further
+            ps3> lines get ps3
+            ...
+            ps4> and the last one gets ps4
+        As with the usual top-level, you can set these to instances if
+        you like; str() will be called on them (once) at the beginning
+        of each command.  Don't put really long or newline containing
+        strings here, please!
+        This is just the default policy; you can change it freely by
+        overriding get_prompt() (and indeed some standard subclasses
+        do).
+      * finished:
+        handle1 will set this to a true value if a command signals
+        that we're done.
+    """
+
+    console: console.Console
+
+    ## state
+    buffer: list[str] = field(default_factory=list)
+    pos: int = 0
+    ps1: str = "->> "
+    ps2: str = "/>> "
+    ps3: str = "|.. "
+    ps4: str = R"\__ "
+    kill_ring: list[list[str]] = field(default_factory=list)
+    msg: str = ""
+    arg: int | None = None
+    dirty: bool = False
+    finished: bool = False
+    paste_mode: bool = False
+    in_bracketed_paste: bool = False
+    commands: dict[str, type[Command]] = field(default_factory=make_default_commands)
+    last_command: type[Command] | None = None
+    syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table)
+    keymap: tuple[tuple[str, str], ...] = ()
+    input_trans: input.KeymapTranslator = field(init=False)
+    input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list)
+    screen: list[str] = field(default_factory=list)
+    screeninfo: list[tuple[int, list[int]]] = field(init=False)
+    cxy: tuple[int, int] = field(init=False)
+    lxy: tuple[int, int] = field(init=False)
+    scheduled_commands: list[str] = field(default_factory=list)
+    can_colorize: bool = False
+    threading_hook: Callback | None = None
+
+    ## cached metadata to speed up screen refreshes
+    @dataclass
+    class RefreshCache:
+        in_bracketed_paste: bool = False
+        screen: list[str] = field(default_factory=list)
+        screeninfo: list[tuple[int, list[int]]] = field(init=False)
+        line_end_offsets: list[int] = field(default_factory=list)
+        pos: int = field(init=False)
+        cxy: tuple[int, int] = field(init=False)
+        dimensions: tuple[int, int] = field(init=False)
+        invalidated: bool = False
+
+        def update_cache(self,
+                         reader: Reader,
+                         screen: list[str],
+                         screeninfo: list[tuple[int, list[int]]],
+            ) -> None:
+            self.in_bracketed_paste = reader.in_bracketed_paste
+            self.screen = screen.copy()
+            self.screeninfo = screeninfo.copy()
+            self.pos = reader.pos
+            self.cxy = reader.cxy
+            self.dimensions = reader.console.width, reader.console.height
+            self.invalidated = False
+
+        def valid(self, reader: Reader) -> bool:
+            if self.invalidated:
+                return False
+            dimensions = reader.console.width, reader.console.height
+            dimensions_changed = dimensions != self.dimensions
+            paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste
+            return not (dimensions_changed or paste_changed)
+
+        def get_cached_location(self, reader: Reader) -> tuple[int, int]:
+            if self.invalidated:
+                raise ValueError("Cache is invalidated")
+            offset = 0
+            earliest_common_pos = min(reader.pos, self.pos)
+            num_common_lines = len(self.line_end_offsets)
+            while num_common_lines > 0:
+                offset = self.line_end_offsets[num_common_lines - 1]
+                if earliest_common_pos > offset:
+                    break
+                num_common_lines -= 1
+            else:
+                offset = 0
+            return offset, num_common_lines
+
+    last_refresh_cache: RefreshCache = field(default_factory=RefreshCache)
+
+    def __post_init__(self) -> None:
+        # Enable the use of `insert` without a `prepare` call - necessary to
+        # facilitate the tab completion hack implemented for
+        # <https://bugs.python.org/issue25660>.
+        self.keymap = self.collect_keymap()
+        self.input_trans = input.KeymapTranslator(
+            self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
+        )
+        self.screeninfo = [(0, [])]
+        self.cxy = self.pos2xy()
+        self.lxy = (self.pos, 0)
+        self.can_colorize = can_colorize()
+
+        self.last_refresh_cache.screeninfo = self.screeninfo
+        self.last_refresh_cache.pos = self.pos
+        self.last_refresh_cache.cxy = self.cxy
+        self.last_refresh_cache.dimensions = (0, 0)
+
+    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+        return default_keymap
+
+    def calc_screen(self) -> list[str]:
+        """Translate changes in self.buffer into changes in self.console.screen."""
+        # Since the last call to calc_screen:
+        # screen and screeninfo may differ due to a completion menu being shown
+        # pos and cxy may differ due to edits, cursor movements, or completion menus
+
+        # Lines that are above both the old and new cursor position can't have changed,
+        # unless the terminal has been resized (which might cause reflowing) or we've
+        # entered or left paste mode (which changes prompts, causing reflowing).
+        num_common_lines = 0
+        offset = 0
+        if self.last_refresh_cache.valid(self):
+            offset, num_common_lines = self.last_refresh_cache.get_cached_location(self)
+
+        screen = self.last_refresh_cache.screen
+        del screen[num_common_lines:]
+
+        screeninfo = self.last_refresh_cache.screeninfo
+        del screeninfo[num_common_lines:]
+
+        last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets
+        del last_refresh_line_end_offsets[num_common_lines:]
+
+        pos = self.pos
+        pos -= offset
+
+        prompt_from_cache = (offset and self.buffer[offset - 1] != "\n")
+
+        lines = "".join(self.buffer[offset:]).split("\n")
+
+        cursor_found = False
+        lines_beyond_cursor = 0
+        for ln, line in enumerate(lines, num_common_lines):
+            ll = len(line)
+            if 0 <= pos <= ll:
+                self.lxy = pos, ln
+                cursor_found = True
+            elif cursor_found:
+                lines_beyond_cursor += 1
+                if lines_beyond_cursor > self.console.height:
+                    # No need to keep formatting lines.
+                    # The console can't show them.
+                    break
+            if prompt_from_cache:
+                # Only the first line's prompt can come from the cache
+                prompt_from_cache = False
+                prompt = ""
+            else:
+                prompt = self.get_prompt(ln, ll >= pos >= 0)
+            while "\n" in prompt:
+                pre_prompt, _, prompt = prompt.partition("\n")
+                last_refresh_line_end_offsets.append(offset)
+                screen.append(pre_prompt)
+                screeninfo.append((0, []))
+            pos -= ll + 1
+            prompt, lp = self.process_prompt(prompt)
+            l, l2 = disp_str(line)
+            wrapcount = (wlen(l) + lp) // self.console.width
+            if wrapcount == 0:
+                offset += ll + 1  # Takes all of the line plus the newline
+                last_refresh_line_end_offsets.append(offset)
+                screen.append(prompt + l)
+                screeninfo.append((lp, l2))
+            else:
+                i = 0
+                while l:
+                    prelen = lp if i == 0 else 0
+                    index_to_wrap_before = 0
+                    column = 0
+                    for character_width in l2:
+                        if column + character_width >= self.console.width - prelen:
+                            break
+                        index_to_wrap_before += 1
+                        column += character_width
+                    pre = prompt if i == 0 else ""
+                    if len(l) > index_to_wrap_before:
+                        offset += index_to_wrap_before
+                        post = "\\"
+                        after = [1]
+                    else:
+                        offset += index_to_wrap_before + 1  # Takes the newline
+                        post = ""
+                        after = []
+                    last_refresh_line_end_offsets.append(offset)
+                    screen.append(pre + l[:index_to_wrap_before] + post)
+                    screeninfo.append((prelen, l2[:index_to_wrap_before] + after))
+                    l = l[index_to_wrap_before:]
+                    l2 = l2[index_to_wrap_before:]
+                    i += 1
+        self.screeninfo = screeninfo
+        self.cxy = self.pos2xy()
+        if self.msg:
+            for mline in self.msg.split("\n"):
+                screen.append(mline)
+                screeninfo.append((0, []))
+
+        self.last_refresh_cache.update_cache(self, screen, screeninfo)
+        return screen
+
+    @staticmethod
+    def process_prompt(prompt: str) -> tuple[str, int]:
+        """Process the prompt.
+
+        This means calculate the length of the prompt. The character \x01
+        and \x02 are used to bracket ANSI control sequences and need to be
+        excluded from the length calculation.  So also a copy of the prompt
+        is returned with these control characters removed."""
+
+        # The logic below also ignores the length of common escape
+        # sequences if they were not explicitly within \x01...\x02.
+        # They are CSI (or ANSI) sequences  ( ESC [ ... LETTER )
+
+        # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars,
+        # which breaks the logic below so we redefine it here.
+        def wlen(s: str) -> int:
+            return sum(str_width(i) for i in s)
+
+        out_prompt = ""
+        l = wlen(prompt)
+        pos = 0
+        while True:
+            s = prompt.find("\x01", pos)
+            if s == -1:
+                break
+            e = prompt.find("\x02", s)
+            if e == -1:
+                break
+            # Found start and end brackets, subtract from string length
+            l = l - (e - s + 1)
+            keep = prompt[pos:s]
+            l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep)))
+            out_prompt += keep + prompt[s + 1 : e]
+            pos = e + 1
+        keep = prompt[pos:]
+        l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep)))
+        out_prompt += keep
+        return out_prompt, l
+
+    def bow(self, p: int | None = None) -> int:
+        """Return the 0-based index of the word break preceding p most
+        immediately.
+
+        p defaults to self.pos; word boundaries are determined using
+        self.syntax_table."""
+        if p is None:
+            p = self.pos
+        st = self.syntax_table
+        b = self.buffer
+        p -= 1
+        while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD:
+            p -= 1
+        while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
+            p -= 1
+        return p + 1
+
+    def eow(self, p: int | None = None) -> int:
+        """Return the 0-based index of the word break following p most
+        immediately.
+
+        p defaults to self.pos; word boundaries are determined using
+        self.syntax_table."""
+        if p is None:
+            p = self.pos
+        st = self.syntax_table
+        b = self.buffer
+        while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD:
+            p += 1
+        while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
+            p += 1
+        return p
+
+    def bol(self, p: int | None = None) -> int:
+        """Return the 0-based index of the line break preceding p most
+        immediately.
+
+        p defaults to self.pos."""
+        if p is None:
+            p = self.pos
+        b = self.buffer
+        p -= 1
+        while p >= 0 and b[p] != "\n":
+            p -= 1
+        return p + 1
+
+    def eol(self, p: int | None = None) -> int:
+        """Return the 0-based index of the line break following p most
+        immediately.
+
+        p defaults to self.pos."""
+        if p is None:
+            p = self.pos
+        b = self.buffer
+        while p < len(b) and b[p] != "\n":
+            p += 1
+        return p
+
+    def max_column(self, y: int) -> int:
+        """Return the last x-offset for line y"""
+        return self.screeninfo[y][0] + sum(self.screeninfo[y][1])
+
+    def max_row(self) -> int:
+        return len(self.screeninfo) - 1
+
+    def get_arg(self, default: int = 1) -> int:
+        """Return any prefix argument that the user has supplied,
+        returning `default' if there is None.  Defaults to 1.
+        """
+        if self.arg is None:
+            return default
+        return self.arg
+
+    def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
+        """Return what should be in the left-hand margin for line
+        `lineno'."""
+        if self.arg is not None and cursor_on_line:
+            prompt = f"(arg: {self.arg}) "
+        elif self.paste_mode and not self.in_bracketed_paste:
+            prompt = "(paste) "
+        elif "\n" in self.buffer:
+            if lineno == 0:
+                prompt = self.ps2
+            elif self.ps4 and lineno == self.buffer.count("\n"):
+                prompt = self.ps4
+            else:
+                prompt = self.ps3
+        else:
+            prompt = self.ps1
+
+        if self.can_colorize:
+            prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}"
+        return prompt
+
+    def push_input_trans(self, itrans: input.KeymapTranslator) -> None:
+        self.input_trans_stack.append(self.input_trans)
+        self.input_trans = itrans
+
+    def pop_input_trans(self) -> None:
+        self.input_trans = self.input_trans_stack.pop()
+
+    def setpos_from_xy(self, x: int, y: int) -> None:
+        """Set pos according to coordinates x, y"""
+        pos = 0
+        i = 0
+        while i < y:
+            prompt_len, character_widths = self.screeninfo[i]
+            offset = len(character_widths) - character_widths.count(0)
+            in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width
+            if in_wrapped_line:
+                pos += offset - 1  # -1 cause backslash is not in buffer
+            else:
+                pos += offset + 1  # +1 cause newline is in buffer
+            i += 1
+
+        j = 0
+        cur_x = self.screeninfo[i][0]
+        while cur_x < x:
+            if self.screeninfo[i][1][j] == 0:
+                continue
+            cur_x += self.screeninfo[i][1][j]
+            j += 1
+            pos += 1
+
+        self.pos = pos
+
+    def pos2xy(self) -> tuple[int, int]:
+        """Return the x, y coordinates of position 'pos'."""
+        # this *is* incomprehensible, yes.
+        p, y = 0, 0
+        l2: list[int] = []
+        pos = self.pos
+        assert 0 <= pos <= len(self.buffer)
+        if pos == len(self.buffer) and len(self.screeninfo) > 0:
+            y = len(self.screeninfo) - 1
+            p, l2 = self.screeninfo[y]
+            return p + sum(l2) + l2.count(0), y
+
+        for p, l2 in self.screeninfo:
+            l = len(l2) - l2.count(0)
+            in_wrapped_line = p + sum(l2) >= self.console.width
+            offset = l - 1 if in_wrapped_line else l  # need to remove backslash
+            if offset >= pos:
+                break
+
+            if p + sum(l2) >= self.console.width:
+                pos -= l - 1  # -1 cause backslash is not in buffer
+            else:
+                pos -= l + 1  # +1 cause newline is in buffer
+            y += 1
+        return p + sum(l2[:pos]), y
+
+    def insert(self, text: str | list[str]) -> None:
+        """Insert 'text' at the insertion point."""
+        self.buffer[self.pos : self.pos] = list(text)
+        self.pos += len(text)
+        self.dirty = True
+
+    def update_cursor(self) -> None:
+        """Move the cursor to reflect changes in self.pos"""
+        self.cxy = self.pos2xy()
+        self.console.move_cursor(*self.cxy)
+
+    def after_command(self, cmd: Command) -> None:
+        """This function is called to allow post command cleanup."""
+        if getattr(cmd, "kills_digit_arg", True):
+            if self.arg is not None:
+                self.dirty = True
+            self.arg = None
+
+    def prepare(self) -> None:
+        """Get ready to run.  Call restore when finished.  You must not
+        write to the console in between the calls to prepare and
+        restore."""
+        try:
+            self.console.prepare()
+            self.arg = None
+            self.finished = False
+            del self.buffer[:]
+            self.pos = 0
+            self.dirty = True
+            self.last_command = None
+            self.calc_screen()
+        except BaseException:
+            self.restore()
+            raise
+
+        while self.scheduled_commands:
+            cmd = self.scheduled_commands.pop()
+            self.do_cmd((cmd, []))
+
+    def last_command_is(self, cls: type) -> bool:
+        if not self.last_command:
+            return False
+        return issubclass(cls, self.last_command)
+
+    def restore(self) -> None:
+        """Clean up after a run."""
+        self.console.restore()
+
+    @contextmanager
+    def suspend(self) -> SimpleContextManager:
+        """A context manager to delegate to another reader."""
+        prev_state = {f.name: getattr(self, f.name) for f in fields(self)}
+        try:
+            self.restore()
+            yield
+        finally:
+            for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"):
+                setattr(self, arg, prev_state[arg])
+            self.prepare()
+
+    def finish(self) -> None:
+        """Called when a command signals that we're finished."""
+        pass
+
+    def error(self, msg: str = "none") -> None:
+        self.msg = "! " + msg + " "
+        self.dirty = True
+        self.console.beep()
+
+    def update_screen(self) -> None:
+        if self.dirty:
+            self.refresh()
+
+    def refresh(self) -> None:
+        """Recalculate and refresh the screen."""
+        if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n":
+            return
+
+        # this call sets up self.cxy, so call it first.
+        self.screen = self.calc_screen()
+        self.console.refresh(self.screen, self.cxy)
+        self.dirty = False
+
+    def do_cmd(self, cmd: tuple[str, list[str]]) -> None:
+        """`cmd` is a tuple of "event_name" and "event", which in the current
+        implementation is always just the "buffer" which happens to be a list
+        of single-character strings."""
+
+        trace("received command {cmd}", cmd=cmd)
+        if isinstance(cmd[0], str):
+            command_type = self.commands.get(cmd[0], commands.invalid_command)
+        elif isinstance(cmd[0], type):
+            command_type = cmd[0]
+        else:
+            return  # nothing to do
+
+        command = command_type(self, *cmd)  # type: ignore[arg-type]
+        command.do()
+
+        self.after_command(command)
+
+        if self.dirty:
+            self.refresh()
+        else:
+            self.update_cursor()
+
+        if not isinstance(cmd, commands.digit_arg):
+            self.last_command = command_type
+
+        self.finished = bool(command.finish)
+        if self.finished:
+            self.console.finish()
+            self.finish()
+
+    def run_hooks(self) -> None:
+        threading_hook = self.threading_hook
+        if threading_hook is None and 'threading' in sys.modules:
+            from ._threading_handler import install_threading_hook
+            install_threading_hook(self)
+        if threading_hook is not None:
+            try:
+                threading_hook()
+            except Exception:
+                pass
+
+        input_hook = self.console.input_hook
+        if input_hook:
+            try:
+                input_hook()
+            except Exception:
+                pass
+
+    def handle1(self, block: bool = True) -> bool:
+        """Handle a single event.  Wait as long as it takes if block
+        is true (the default), otherwise return False if no event is
+        pending."""
+
+        if self.msg:
+            self.msg = ""
+            self.dirty = True
+
+        while True:
+            # We use the same timeout as in readline.c: 100ms
+            self.run_hooks()
+            self.console.wait(100)
+            event = self.console.get_event(block=False)
+            if not event:
+                if block:
+                    continue
+                return False
+
+            translate = True
+
+            if event.evt == "key":
+                self.input_trans.push(event)
+            elif event.evt == "scroll":
+                self.refresh()
+            elif event.evt == "resize":
+                self.refresh()
+            else:
+                translate = False
+
+            if translate:
+                cmd = self.input_trans.get()
+            else:
+                cmd = [event.evt, event.data]
+
+            if cmd is None:
+                if block:
+                    continue
+                return False
+
+            self.do_cmd(cmd)
+            return True
+
+    def push_char(self, char: int | bytes) -> None:
+        self.console.push_char(char)
+        self.handle1(block=False)
+
+    def readline(self, startup_hook: Callback | None = None) -> str:
+        """Read a line.  The implementation of this method also shows
+        how to drive Reader if you want more control over the event
+        loop."""
+        self.prepare()
+        try:
+            if startup_hook is not None:
+                startup_hook()
+            self.refresh()
+            while not self.finished:
+                self.handle1()
+            return self.get_unicode()
+
+        finally:
+            self.restore()
+
+    def bind(self, spec: KeySpec, command: CommandName) -> None:
+        self.keymap = self.keymap + ((spec, command),)
+        self.input_trans = input.KeymapTranslator(
+            self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
+        )
+
+    def get_unicode(self) -> str:
+        """Return the current buffer as a unicode string."""
+        return "".join(self.buffer)

Lib/_pyrepl/readline.py

@@ -0,0 +1,598 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Alex Gaynor
+#                       Antonio Cuni
+#                       Armin Rigo
+#                       Holger Krekel
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""A compatibility wrapper reimplementing the 'readline' standard module
+on top of pyrepl.  Not all functionalities are supported.  Contains
+extensions for multiline input.
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass, field
+
+import os
+from site import gethistoryfile   # type: ignore[attr-defined]
+import sys
+from rlcompleter import Completer as RLCompleter
+
+from . import commands, historical_reader
+from .completing_reader import CompletingReader
+from .console import Console as ConsoleType
+
+Console: type[ConsoleType]
+_error: tuple[type[Exception], ...] | type[Exception]
+try:
+    from .unix_console import UnixConsole as Console, _error
+except ImportError:
+    from .windows_console import WindowsConsole as Console, _error
+
+ENCODING = sys.getdefaultencoding() or "latin1"
+
+
+# types
+Command = commands.Command
+from collections.abc import Callable, Collection
+from .types import Callback, Completer, KeySpec, CommandName
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+    from typing import Any, Mapping
+
+
+MoreLinesCallable = Callable[[str], bool]
+
+
+__all__ = [
+    "add_history",
+    "clear_history",
+    "get_begidx",
+    "get_completer",
+    "get_completer_delims",
+    "get_current_history_length",
+    "get_endidx",
+    "get_history_item",
+    "get_history_length",
+    "get_line_buffer",
+    "insert_text",
+    "parse_and_bind",
+    "read_history_file",
+    # "read_init_file",
+    # "redisplay",
+    "remove_history_item",
+    "replace_history_item",
+    "set_auto_history",
+    "set_completer",
+    "set_completer_delims",
+    "set_history_length",
+    # "set_pre_input_hook",
+    "set_startup_hook",
+    "write_history_file",
+    # ---- multiline extensions ----
+    "multiline_input",
+]
+
+# ____________________________________________________________
+
+@dataclass
+class ReadlineConfig:
+    readline_completer: Completer | None = None
+    completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?")
+
+
+@dataclass(kw_only=True)
+class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader):
+    # Class fields
+    assume_immutable_completions = False
+    use_brackets = False
+    sort_in_column = True
+
+    # Instance fields
+    config: ReadlineConfig
+    more_lines: MoreLinesCallable | None = None
+    last_used_indentation: str | None = None
+
+    def __post_init__(self) -> None:
+        super().__post_init__()
+        self.commands["maybe_accept"] = maybe_accept
+        self.commands["maybe-accept"] = maybe_accept
+        self.commands["backspace_dedent"] = backspace_dedent
+        self.commands["backspace-dedent"] = backspace_dedent
+
+    def error(self, msg: str = "none") -> None:
+        pass  # don't show error messages by default
+
+    def get_stem(self) -> str:
+        b = self.buffer
+        p = self.pos - 1
+        completer_delims = self.config.completer_delims
+        while p >= 0 and b[p] not in completer_delims:
+            p -= 1
+        return "".join(b[p + 1 : self.pos])
+
+    def get_completions(self, stem: str) -> list[str]:
+        if len(stem) == 0 and self.more_lines is not None:
+            b = self.buffer
+            p = self.pos
+            while p > 0 and b[p - 1] != "\n":
+                p -= 1
+            num_spaces = 4 - ((self.pos - p) % 4)
+            return [" " * num_spaces]
+        result = []
+        function = self.config.readline_completer
+        if function is not None:
+            try:
+                stem = str(stem)  # rlcompleter.py seems to not like unicode
+            except UnicodeEncodeError:
+                pass  # but feed unicode anyway if we have no choice
+            state = 0
+            while True:
+                try:
+                    next = function(stem, state)
+                except Exception:
+                    break
+                if not isinstance(next, str):
+                    break
+                result.append(next)
+                state += 1
+            # emulate the behavior of the standard readline that sorts
+            # the completions before displaying them.
+            result.sort()
+        return result
+
+    def get_trimmed_history(self, maxlength: int) -> list[str]:
+        if maxlength >= 0:
+            cut = len(self.history) - maxlength
+            if cut < 0:
+                cut = 0
+        else:
+            cut = 0
+        return self.history[cut:]
+
+    def update_last_used_indentation(self) -> None:
+        indentation = _get_first_indentation(self.buffer)
+        if indentation is not None:
+            self.last_used_indentation = indentation
+
+    # --- simplified support for reading multiline Python statements ---
+
+    def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+        return super().collect_keymap() + (
+            (r"\n", "maybe-accept"),
+            (r"\<backspace>", "backspace-dedent"),
+        )
+
+    def after_command(self, cmd: Command) -> None:
+        super().after_command(cmd)
+        if self.more_lines is None:
+            # Force single-line input if we are in raw_input() mode.
+            # Although there is no direct way to add a \n in this mode,
+            # multiline buffers can still show up using various
+            # commands, e.g. navigating the history.
+            try:
+                index = self.buffer.index("\n")
+            except ValueError:
+                pass
+            else:
+                self.buffer = self.buffer[:index]
+                if self.pos > len(self.buffer):
+                    self.pos = len(self.buffer)
+
+
+def set_auto_history(_should_auto_add_history: bool) -> None:
+    """Enable or disable automatic history"""
+    historical_reader.should_auto_add_history = bool(_should_auto_add_history)
+
+
+def _get_this_line_indent(buffer: list[str], pos: int) -> int:
+    indent = 0
+    while pos > 0 and buffer[pos - 1] in " \t":
+        indent += 1
+        pos -= 1
+    if pos > 0 and buffer[pos - 1] == "\n":
+        return indent
+    return 0
+
+
+def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]:
+    prevlinestart = pos
+    while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n":
+        prevlinestart -= 1
+    prevlinetext = prevlinestart
+    while prevlinetext < pos and buffer[prevlinetext] in " \t":
+        prevlinetext += 1
+    if prevlinetext == pos:
+        indent = None
+    else:
+        indent = prevlinetext - prevlinestart
+    return prevlinestart, indent
+
+
+def _get_first_indentation(buffer: list[str]) -> str | None:
+    indented_line_start = None
+    for i in range(len(buffer)):
+        if (i < len(buffer) - 1
+            and buffer[i] == "\n"
+            and buffer[i + 1] in " \t"
+        ):
+            indented_line_start = i + 1
+        elif indented_line_start is not None and buffer[i] not in " \t\n":
+            return ''.join(buffer[indented_line_start : i])
+    return None
+
+
+def _should_auto_indent(buffer: list[str], pos: int) -> bool:
+    # check if last character before "pos" is a colon, ignoring
+    # whitespaces and comments.
+    last_char = None
+    while pos > 0:
+        pos -= 1
+        if last_char is None:
+            if buffer[pos] not in " \t\n#":  # ignore whitespaces and comments
+                last_char = buffer[pos]
+        else:
+            # even if we found a non-whitespace character before
+            # original pos, we keep going back until newline is reached
+            # to make sure we ignore comments
+            if buffer[pos] == "\n":
+                break
+            if buffer[pos] == "#":
+                last_char = None
+    return last_char == ":"
+
+
+class maybe_accept(commands.Command):
+    def do(self) -> None:
+        r: ReadlineAlikeReader
+        r = self.reader  # type: ignore[assignment]
+        r.dirty = True  # this is needed to hide the completion menu, if visible
+
+        if self.reader.in_bracketed_paste:
+            r.insert("\n")
+            return
+
+        # if there are already several lines and the cursor
+        # is not on the last one, always insert a new \n.
+        text = r.get_unicode()
+
+        if "\n" in r.buffer[r.pos :] or (
+            r.more_lines is not None and r.more_lines(text)
+        ):
+            def _newline_before_pos():
+                before_idx = r.pos - 1
+                while before_idx > 0 and text[before_idx].isspace():
+                    before_idx -= 1
+                return text[before_idx : r.pos].count("\n") > 0
+
+            # if there's already a new line before the cursor then
+            # even if the cursor is followed by whitespace, we assume
+            # the user is trying to terminate the block
+            if _newline_before_pos() and text[r.pos:].isspace():
+                self.finish = True
+                return
+
+            # auto-indent the next line like the previous line
+            prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos)
+            r.insert("\n")
+            if not self.reader.paste_mode:
+                if indent:
+                    for i in range(prevlinestart, prevlinestart + indent):
+                        r.insert(r.buffer[i])
+                r.update_last_used_indentation()
+                if _should_auto_indent(r.buffer, r.pos):
+                    if r.last_used_indentation is not None:
+                        indentation = r.last_used_indentation
+                    else:
+                        # default
+                        indentation = " " * 4
+                    r.insert(indentation)
+        elif not self.reader.paste_mode:
+            self.finish = True
+        else:
+            r.insert("\n")
+
+
+class backspace_dedent(commands.Command):
+    def do(self) -> None:
+        r = self.reader
+        b = r.buffer
+        if r.pos > 0:
+            repeat = 1
+            if b[r.pos - 1] != "\n":
+                indent = _get_this_line_indent(b, r.pos)
+                if indent > 0:
+                    ls = r.pos - indent
+                    while ls > 0:
+                        ls, pi = _get_previous_line_indent(b, ls - 1)
+                        if pi is not None and pi < indent:
+                            repeat = indent - pi
+                            break
+            r.pos -= repeat
+            del b[r.pos : r.pos + repeat]
+            r.dirty = True
+        else:
+            self.reader.error("can't backspace at start")
+
+
+# ____________________________________________________________
+
+
+@dataclass(slots=True)
+class _ReadlineWrapper:
+    f_in: int = -1
+    f_out: int = -1
+    reader: ReadlineAlikeReader | None = field(default=None, repr=False)
+    saved_history_length: int = -1
+    startup_hook: Callback | None = None
+    config: ReadlineConfig = field(default_factory=ReadlineConfig, repr=False)
+
+    def __post_init__(self) -> None:
+        if self.f_in == -1:
+            self.f_in = os.dup(0)
+        if self.f_out == -1:
+            self.f_out = os.dup(1)
+
+    def get_reader(self) -> ReadlineAlikeReader:
+        if self.reader is None:
+            console = Console(self.f_in, self.f_out, encoding=ENCODING)
+            self.reader = ReadlineAlikeReader(console=console, config=self.config)
+        return self.reader
+
+    def input(self, prompt: object = "") -> str:
+        try:
+            reader = self.get_reader()
+        except _error:
+            assert raw_input is not None
+            return raw_input(prompt)
+        prompt_str = str(prompt)
+        reader.ps1 = prompt_str
+        sys.audit("builtins.input", prompt_str)
+        result = reader.readline(startup_hook=self.startup_hook)
+        sys.audit("builtins.input/result", result)
+        return result
+
+    def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str:
+        """Read an input on possibly multiple lines, asking for more
+        lines as long as 'more_lines(unicodetext)' returns an object whose
+        boolean value is true.
+        """
+        reader = self.get_reader()
+        saved = reader.more_lines
+        try:
+            reader.more_lines = more_lines
+            reader.ps1 = ps1
+            reader.ps2 = ps1
+            reader.ps3 = ps2
+            reader.ps4 = ""
+            with warnings.catch_warnings(action="ignore"):
+                return reader.readline()
+        finally:
+            reader.more_lines = saved
+            reader.paste_mode = False
+
+    def parse_and_bind(self, string: str) -> None:
+        pass  # XXX we don't support parsing GNU-readline-style init files
+
+    def set_completer(self, function: Completer | None = None) -> None:
+        self.config.readline_completer = function
+
+    def get_completer(self) -> Completer | None:
+        return self.config.readline_completer
+
+    def set_completer_delims(self, delimiters: Collection[str]) -> None:
+        self.config.completer_delims = frozenset(delimiters)
+
+    def get_completer_delims(self) -> str:
+        return "".join(sorted(self.config.completer_delims))
+
+    def _histline(self, line: str) -> str:
+        line = line.rstrip("\n")
+        return line
+
+    def get_history_length(self) -> int:
+        return self.saved_history_length
+
+    def set_history_length(self, length: int) -> None:
+        self.saved_history_length = length
+
+    def get_current_history_length(self) -> int:
+        return len(self.get_reader().history)
+
+    def read_history_file(self, filename: str = gethistoryfile()) -> None:
+        # multiline extension (really a hack) for the end of lines that
+        # are actually continuations inside a single multiline_input()
+        # history item: we use \r\n instead of just \n.  If the history
+        # file is passed to GNU readline, the extra \r are just ignored.
+        history = self.get_reader().history
+
+        with open(os.path.expanduser(filename), 'rb') as f:
+            is_editline = f.readline().startswith(b"_HiStOrY_V2_")
+            if is_editline:
+                encoding = "unicode-escape"
+            else:
+                f.seek(0)
+                encoding = "utf-8"
+
+            lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')]
+            buffer = []
+            for line in lines:
+                if line.endswith("\r"):
+                    buffer.append(line+'\n')
+                else:
+                    line = self._histline(line)
+                    if buffer:
+                        line = self._histline("".join(buffer).replace("\r", "") + line)
+                        del buffer[:]
+                    if line:
+                        history.append(line)
+
+    def write_history_file(self, filename: str = gethistoryfile()) -> None:
+        maxlength = self.saved_history_length
+        history = self.get_reader().get_trimmed_history(maxlength)
+        f = open(os.path.expanduser(filename), "w",
+                 encoding="utf-8", newline="\n")
+        with f:
+            for entry in history:
+                entry = entry.replace("\n", "\r\n")  # multiline history support
+                f.write(entry + "\n")
+
+    def clear_history(self) -> None:
+        del self.get_reader().history[:]
+
+    def get_history_item(self, index: int) -> str | None:
+        history = self.get_reader().history
+        if 1 <= index <= len(history):
+            return history[index - 1]
+        else:
+            return None  # like readline.c
+
+    def remove_history_item(self, index: int) -> None:
+        history = self.get_reader().history
+        if 0 <= index < len(history):
+            del history[index]
+        else:
+            raise ValueError("No history item at position %d" % index)
+            # like readline.c
+
+    def replace_history_item(self, index: int, line: str) -> None:
+        history = self.get_reader().history
+        if 0 <= index < len(history):
+            history[index] = self._histline(line)
+        else:
+            raise ValueError("No history item at position %d" % index)
+            # like readline.c
+
+    def add_history(self, line: str) -> None:
+        self.get_reader().history.append(self._histline(line))
+
+    def set_startup_hook(self, function: Callback | None = None) -> None:
+        self.startup_hook = function
+
+    def get_line_buffer(self) -> str:
+        return self.get_reader().get_unicode()
+
+    def _get_idxs(self) -> tuple[int, int]:
+        start = cursor = self.get_reader().pos
+        buf = self.get_line_buffer()
+        for i in range(cursor - 1, -1, -1):
+            if buf[i] in self.get_completer_delims():
+                break
+            start = i
+        return start, cursor
+
+    def get_begidx(self) -> int:
+        return self._get_idxs()[0]
+
+    def get_endidx(self) -> int:
+        return self._get_idxs()[1]
+
+    def insert_text(self, text: str) -> None:
+        self.get_reader().insert(text)
+
+
+_wrapper = _ReadlineWrapper()
+
+# ____________________________________________________________
+# Public API
+
+parse_and_bind = _wrapper.parse_and_bind
+set_completer = _wrapper.set_completer
+get_completer = _wrapper.get_completer
+set_completer_delims = _wrapper.set_completer_delims
+get_completer_delims = _wrapper.get_completer_delims
+get_history_length = _wrapper.get_history_length
+set_history_length = _wrapper.set_history_length
+get_current_history_length = _wrapper.get_current_history_length
+read_history_file = _wrapper.read_history_file
+write_history_file = _wrapper.write_history_file
+clear_history = _wrapper.clear_history
+get_history_item = _wrapper.get_history_item
+remove_history_item = _wrapper.remove_history_item
+replace_history_item = _wrapper.replace_history_item
+add_history = _wrapper.add_history
+set_startup_hook = _wrapper.set_startup_hook
+get_line_buffer = _wrapper.get_line_buffer
+get_begidx = _wrapper.get_begidx
+get_endidx = _wrapper.get_endidx
+insert_text = _wrapper.insert_text
+
+# Extension
+multiline_input = _wrapper.multiline_input
+
+# Internal hook
+_get_reader = _wrapper.get_reader
+
+# ____________________________________________________________
+# Stubs
+
+
+def _make_stub(_name: str, _ret: object) -> None:
+    def stub(*args: object, **kwds: object) -> None:
+        import warnings
+
+        warnings.warn("readline.%s() not implemented" % _name, stacklevel=2)
+
+    stub.__name__ = _name
+    globals()[_name] = stub
+
+
+for _name, _ret in [
+    ("read_init_file", None),
+    ("redisplay", None),
+    ("set_pre_input_hook", None),
+]:
+    assert _name not in globals(), _name
+    _make_stub(_name, _ret)
+
+# ____________________________________________________________
+
+
+def _setup(namespace: Mapping[str, Any]) -> None:
+    global raw_input
+    if raw_input is not None:
+        return  # don't run _setup twice
+
+    try:
+        f_in = sys.stdin.fileno()
+        f_out = sys.stdout.fileno()
+    except (AttributeError, ValueError):
+        return
+    if not os.isatty(f_in) or not os.isatty(f_out):
+        return
+
+    _wrapper.f_in = f_in
+    _wrapper.f_out = f_out
+
+    # set up namespace in rlcompleter, which requires it to be a bona fide dict
+    if not isinstance(namespace, dict):
+        namespace = dict(namespace)
+    _wrapper.config.readline_completer = RLCompleter(namespace).complete
+
+    # this is not really what readline.c does.  Better than nothing I guess
+    import builtins
+    raw_input = builtins.input
+    builtins.input = _wrapper.input
+
+
+raw_input: Callable[[object], str] | None = None

Lib/_pyrepl/simple_interact.py

@@ -0,0 +1,167 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""This is an alternative to python_reader which tries to emulate
+the CPython prompt as closely as possible, with the exception of
+allowing multiline input and multiline history entries.
+"""
+
+from __future__ import annotations
+
+import _sitebuiltins
+import linecache
+import functools
+import os
+import sys
+import code
+
+from .readline import _get_reader, multiline_input
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+_error: tuple[type[Exception], ...] | type[Exception]
+try:
+    from .unix_console import _error
+except ModuleNotFoundError:
+    from .windows_console import _error
+
+def check() -> str:
+    """Returns the error message if there is a problem initializing the state."""
+    try:
+        _get_reader()
+    except _error as e:
+        if term := os.environ.get("TERM", ""):
+            term = f"; TERM={term}"
+        return str(str(e) or repr(e) or "unknown error") + term
+    return ""
+
+
+def _strip_final_indent(text: str) -> str:
+    # kill spaces and tabs at the end, but only if they follow '\n'.
+    # meant to remove the auto-indentation only (although it would of
+    # course also remove explicitly-added indentation).
+    short = text.rstrip(" \t")
+    n = len(short)
+    if n > 0 and text[n - 1] == "\n":
+        return short
+    return text
+
+
+def _clear_screen():
+    reader = _get_reader()
+    reader.scheduled_commands.append("clear_screen")
+
+
+REPL_COMMANDS = {
+    "exit": _sitebuiltins.Quitter('exit', ''),
+    "quit": _sitebuiltins.Quitter('quit' ,''),
+    "copyright": _sitebuiltins._Printer('copyright', sys.copyright),
+    "help": _sitebuiltins._Helper(),
+    "clear": _clear_screen,
+    "\x1a": _sitebuiltins.Quitter('\x1a', ''),
+}
+
+
+def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool:
+    # ooh, look at the hack:
+    src = _strip_final_indent(unicodetext)
+    try:
+        code = console.compile(src, "<stdin>", "single")
+    except (OverflowError, SyntaxError, ValueError):
+        lines = src.splitlines(keepends=True)
+        if len(lines) == 1:
+            return False
+
+        last_line = lines[-1]
+        was_indented = last_line.startswith((" ", "\t"))
+        not_empty = last_line.strip() != ""
+        incomplete = not last_line.endswith("\n")
+        return (was_indented or not_empty) and incomplete
+    else:
+        return code is None
+
+
+def run_multiline_interactive_console(
+    console: code.InteractiveConsole,
+    *,
+    future_flags: int = 0,
+) -> None:
+    from .readline import _setup
+    _setup(console.locals)
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
+
+    more_lines = functools.partial(_more_lines, console)
+    input_n = 0
+
+    def maybe_run_command(statement: str) -> bool:
+        statement = statement.strip()
+        if statement in console.locals or statement not in REPL_COMMANDS:
+            return False
+
+        reader = _get_reader()
+        reader.history.pop()  # skip internal commands in history
+        command = REPL_COMMANDS[statement]
+        if callable(command):
+            # Make sure that history does not change because of commands
+            with reader.suspend_history():
+                command()
+            return True
+        return False
+
+    while 1:
+        try:
+            try:
+                sys.stdout.flush()
+            except Exception:
+                pass
+
+            ps1 = getattr(sys, "ps1", ">>> ")
+            ps2 = getattr(sys, "ps2", "... ")
+            try:
+                statement = multiline_input(more_lines, ps1, ps2)
+            except EOFError:
+                break
+
+            if maybe_run_command(statement):
+                continue
+
+            input_name = f"<python-input-{input_n}>"
+            linecache._register_code(input_name, statement, "<stdin>")  # type: ignore[attr-defined]
+            more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single")  # type: ignore[call-arg]
+            assert not more
+            input_n += 1
+        except KeyboardInterrupt:
+            r = _get_reader()
+            if r.input_trans is r.isearch_trans:
+                r.do_cmd(("isearch-end", [""]))
+            r.pos = len(r.get_unicode())
+            r.dirty = True
+            r.refresh()
+            r.in_bracketed_paste = False
+            console.write("\nKeyboardInterrupt\n")
+            console.resetbuffer()
+        except MemoryError:
+            console.write("\nMemoryError\n")
+            console.resetbuffer()

Lib/_pyrepl/trace.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import os
+
+# types
+if False:
+    from typing import IO
+
+
+trace_file: IO[str] | None = None
+if trace_filename := os.environ.get("PYREPL_TRACE"):
+    trace_file = open(trace_filename, "a")
+
+
+def trace(line: str, *k: object, **kw: object) -> None:
+    if trace_file is None:
+        return
+    if k or kw:
+        line = line.format(*k, **kw)
+    trace_file.write(line + "\n")
+    trace_file.flush()

Lib/_pyrepl/types.py

@@ -0,0 +1,8 @@
+from collections.abc import Callable, Iterator
+
+Callback = Callable[[], object]
+SimpleContextManager = Iterator[None]
+KeySpec = str  # like r"\C-c"
+CommandName = str  # like "interrupt"
+EventTuple = tuple[CommandName, str]
+Completer = Callable[[str, int], str | None]

Lib/_pyrepl/unix_console.py

@@ -0,0 +1,810 @@
+#   Copyright 2000-2010 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Antonio Cuni
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import errno
+import os
+import re
+import select
+import signal
+import struct
+import termios
+import time
+import platform
+from fcntl import ioctl
+
+from . import curses
+from .console import Console, Event
+from .fancy_termios import tcgetattr, tcsetattr
+from .trace import trace
+from .unix_eventqueue import EventQueue
+from .utils import wlen
+
+
+TYPE_CHECKING = False
+
+# types
+if TYPE_CHECKING:
+    from typing import IO, Literal, overload
+else:
+    overload = lambda func: None
+
+
+class InvalidTerminal(RuntimeError):
+    pass
+
+
+_error = (termios.error, curses.error, InvalidTerminal)
+
+SIGWINCH_EVENT = "repaint"
+
+FIONREAD = getattr(termios, "FIONREAD", None)
+TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None)
+
+# ------------ start of baudrate definitions ------------
+
+# Add (possibly) missing baudrates (check termios man page) to termios
+
+
+def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None:
+    baudrate_name = "B%d" % rate
+    if hasattr(termios, baudrate_name):
+        dictionary[getattr(termios, baudrate_name)] = rate
+
+
+# Check the termios man page (Line speed) to know where these
+# values come from.
+potential_baudrates = [
+    0,
+    110,
+    115200,
+    1200,
+    134,
+    150,
+    1800,
+    19200,
+    200,
+    230400,
+    2400,
+    300,
+    38400,
+    460800,
+    4800,
+    50,
+    57600,
+    600,
+    75,
+    9600,
+]
+
+ratedict: dict[int, int] = {}
+for rate in potential_baudrates:
+    add_baudrate_if_supported(ratedict, rate)
+
+# Clean up variables to avoid unintended usage
+del rate, add_baudrate_if_supported
+
+# ------------ end of baudrate definitions ------------
+
+delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>")
+
+try:
+    poll: type[select.poll] = select.poll
+except AttributeError:
+    # this is exactly the minumum necessary to support what we
+    # do with poll objects
+    class MinimalPoll:
+        def __init__(self):
+            pass
+
+        def register(self, fd, flag):
+            self.fd = fd
+        # note: The 'timeout' argument is received as *milliseconds*
+        def poll(self, timeout: float | None = None) -> list[int]:
+            if timeout is None:
+                r, w, e = select.select([self.fd], [], [])
+            else:
+                r, w, e = select.select([self.fd], [], [], timeout/1000)
+            return r
+
+    poll = MinimalPoll  # type: ignore[assignment]
+
+
+class UnixConsole(Console):
+    def __init__(
+        self,
+        f_in: IO[bytes] | int = 0,
+        f_out: IO[bytes] | int = 1,
+        term: str = "",
+        encoding: str = "",
+    ):
+        """
+        Initialize the UnixConsole.
+
+        Parameters:
+        - f_in (int or file-like object): Input file descriptor or object.
+        - f_out (int or file-like object): Output file descriptor or object.
+        - term (str): Terminal name.
+        - encoding (str): Encoding to use for I/O operations.
+        """
+        super().__init__(f_in, f_out, term, encoding)
+
+        self.pollob = poll()
+        self.pollob.register(self.input_fd, select.POLLIN)
+        self.input_buffer = b""
+        self.input_buffer_pos = 0
+        curses.setupterm(term or None, self.output_fd)
+        self.term = term
+
+        @overload
+        def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ...
+
+        @overload
+        def _my_getstr(cap: str, optional: bool) -> bytes | None: ...
+
+        def _my_getstr(cap: str, optional: bool = False) -> bytes | None:
+            r = curses.tigetstr(cap)
+            if not optional and r is None:
+                raise InvalidTerminal(
+                    f"terminal doesn't have the required {cap} capability"
+                )
+            return r
+
+        self._bel = _my_getstr("bel")
+        self._civis = _my_getstr("civis", optional=True)
+        self._clear = _my_getstr("clear")
+        self._cnorm = _my_getstr("cnorm", optional=True)
+        self._cub = _my_getstr("cub", optional=True)
+        self._cub1 = _my_getstr("cub1", optional=True)
+        self._cud = _my_getstr("cud", optional=True)
+        self._cud1 = _my_getstr("cud1", optional=True)
+        self._cuf = _my_getstr("cuf", optional=True)
+        self._cuf1 = _my_getstr("cuf1", optional=True)
+        self._cup = _my_getstr("cup")
+        self._cuu = _my_getstr("cuu", optional=True)
+        self._cuu1 = _my_getstr("cuu1", optional=True)
+        self._dch1 = _my_getstr("dch1", optional=True)
+        self._dch = _my_getstr("dch", optional=True)
+        self._el = _my_getstr("el")
+        self._hpa = _my_getstr("hpa", optional=True)
+        self._ich = _my_getstr("ich", optional=True)
+        self._ich1 = _my_getstr("ich1", optional=True)
+        self._ind = _my_getstr("ind", optional=True)
+        self._pad = _my_getstr("pad", optional=True)
+        self._ri = _my_getstr("ri", optional=True)
+        self._rmkx = _my_getstr("rmkx", optional=True)
+        self._smkx = _my_getstr("smkx", optional=True)
+
+        self.__setup_movement()
+
+        self.event_queue = EventQueue(self.input_fd, self.encoding)
+        self.cursor_visible = 1
+
+    def more_in_buffer(self) -> bool:
+        return bool(
+            self.input_buffer
+            and self.input_buffer_pos < len(self.input_buffer)
+        )
+
+    def __read(self, n: int) -> bytes:
+        if not self.more_in_buffer():
+            self.input_buffer = os.read(self.input_fd, 10000)
+
+        ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n]
+        self.input_buffer_pos += len(ret)
+        if self.input_buffer_pos >= len(self.input_buffer):
+            self.input_buffer = b""
+            self.input_buffer_pos = 0
+        return ret
+
+
+    def change_encoding(self, encoding: str) -> None:
+        """
+        Change the encoding used for I/O operations.
+
+        Parameters:
+        - encoding (str): New encoding to use.
+        """
+        self.encoding = encoding
+
+    def refresh(self, screen, c_xy):
+        """
+        Refresh the console screen.
+
+        Parameters:
+        - screen (list): List of strings representing the screen contents.
+        - c_xy (tuple): Cursor position (x, y) on the screen.
+        """
+        cx, cy = c_xy
+        if not self.__gone_tall:
+            while len(self.screen) < min(len(screen), self.height):
+                self.__hide_cursor()
+                self.__move(0, len(self.screen) - 1)
+                self.__write("\n")
+                self.posxy = 0, len(self.screen)
+                self.screen.append("")
+        else:
+            while len(self.screen) < len(screen):
+                self.screen.append("")
+
+        if len(screen) > self.height:
+            self.__gone_tall = 1
+            self.__move = self.__move_tall
+
+        px, py = self.posxy
+        old_offset = offset = self.__offset
+        height = self.height
+
+        # we make sure the cursor is on the screen, and that we're
+        # using all of the screen if we can
+        if cy < offset:
+            offset = cy
+        elif cy >= offset + height:
+            offset = cy - height + 1
+        elif offset > 0 and len(screen) < offset + height:
+            offset = max(len(screen) - height, 0)
+            screen.append("")
+
+        oldscr = self.screen[old_offset : old_offset + height]
+        newscr = screen[offset : offset + height]
+
+        # use hardware scrolling if we have it.
+        if old_offset > offset and self._ri:
+            self.__hide_cursor()
+            self.__write_code(self._cup, 0, 0)
+            self.posxy = 0, old_offset
+            for i in range(old_offset - offset):
+                self.__write_code(self._ri)
+                oldscr.pop(-1)
+                oldscr.insert(0, "")
+        elif old_offset < offset and self._ind:
+            self.__hide_cursor()
+            self.__write_code(self._cup, self.height - 1, 0)
+            self.posxy = 0, old_offset + self.height - 1
+            for i in range(offset - old_offset):
+                self.__write_code(self._ind)
+                oldscr.pop(0)
+                oldscr.append("")
+
+        self.__offset = offset
+
+        for (
+            y,
+            oldline,
+            newline,
+        ) in zip(range(offset, offset + height), oldscr, newscr):
+            if oldline != newline:
+                self.__write_changed_line(y, oldline, newline, px)
+
+        y = len(newscr)
+        while y < len(oldscr):
+            self.__hide_cursor()
+            self.__move(0, y)
+            self.posxy = 0, y
+            self.__write_code(self._el)
+            y += 1
+
+        self.__show_cursor()
+
+        self.screen = screen.copy()
+        self.move_cursor(cx, cy)
+        self.flushoutput()
+
+    def move_cursor(self, x, y):
+        """
+        Move the cursor to the specified position on the screen.
+
+        Parameters:
+        - x (int): X coordinate.
+        - y (int): Y coordinate.
+        """
+        if y < self.__offset or y >= self.__offset + self.height:
+            self.event_queue.insert(Event("scroll", None))
+        else:
+            self.__move(x, y)
+            self.posxy = x, y
+            self.flushoutput()
+
+    def prepare(self):
+        """
+        Prepare the console for input/output operations.
+        """
+        self.__svtermstate = tcgetattr(self.input_fd)
+        raw = self.__svtermstate.copy()
+        raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON)
+        raw.oflag &= ~(termios.OPOST)
+        raw.cflag &= ~(termios.CSIZE | termios.PARENB)
+        raw.cflag |= termios.CS8
+        raw.iflag |= termios.BRKINT
+        raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN)
+        raw.lflag |= termios.ISIG
+        raw.cc[termios.VMIN] = 1
+        raw.cc[termios.VTIME] = 0
+        tcsetattr(self.input_fd, termios.TCSADRAIN, raw)
+
+        # In macOS terminal we need to deactivate line wrap via ANSI escape code
+        if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal":
+            os.write(self.output_fd, b"\033[?7l")
+
+        self.screen = []
+        self.height, self.width = self.getheightwidth()
+
+        self.__buffer = []
+
+        self.posxy = 0, 0
+        self.__gone_tall = 0
+        self.__move = self.__move_short
+        self.__offset = 0
+
+        self.__maybe_write_code(self._smkx)
+
+        try:
+            self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch)
+        except ValueError:
+            pass
+
+        self.__enable_bracketed_paste()
+
+    def restore(self):
+        """
+        Restore the console to the default state
+        """
+        self.__disable_bracketed_paste()
+        self.__maybe_write_code(self._rmkx)
+        self.flushoutput()
+        tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate)
+
+        if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal":
+            os.write(self.output_fd, b"\033[?7h")
+
+        if hasattr(self, "old_sigwinch"):
+            signal.signal(signal.SIGWINCH, self.old_sigwinch)
+            del self.old_sigwinch
+
+    def push_char(self, char: int | bytes) -> None:
+        """
+        Push a character to the console event queue.
+        """
+        trace("push char {char!r}", char=char)
+        self.event_queue.push(char)
+
+    def get_event(self, block: bool = True) -> Event | None:
+        """
+        Get an event from the console event queue.
+
+        Parameters:
+        - block (bool): Whether to block until an event is available.
+
+        Returns:
+        - Event: Event object from the event queue.
+        """
+        if not block and not self.wait(timeout=0):
+            return None
+
+        while self.event_queue.empty():
+            while True:
+                try:
+                    self.push_char(self.__read(1))
+                except OSError as err:
+                    if err.errno == errno.EINTR:
+                        if not self.event_queue.empty():
+                            return self.event_queue.get()
+                        else:
+                            continue
+                    else:
+                        raise
+                else:
+                    break
+        return self.event_queue.get()
+
+    def wait(self, timeout: float | None = None) -> bool:
+        """
+        Wait for events on the console.
+        """
+        return (
+            not self.event_queue.empty()
+            or self.more_in_buffer()
+            or bool(self.pollob.poll(timeout))
+        )
+
+    def set_cursor_vis(self, visible):
+        """
+        Set the visibility of the cursor.
+
+        Parameters:
+        - visible (bool): Visibility flag.
+        """
+        if visible:
+            self.__show_cursor()
+        else:
+            self.__hide_cursor()
+
+    if TIOCGWINSZ:
+
+        def getheightwidth(self):
+            """
+            Get the height and width of the console.
+
+            Returns:
+            - tuple: Height and width of the console.
+            """
+            try:
+                return int(os.environ["LINES"]), int(os.environ["COLUMNS"])
+            except (KeyError, TypeError, ValueError):
+                try:
+                    size = ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8)
+                except OSError:
+                    return 25, 80
+                height, width = struct.unpack("hhhh", size)[0:2]
+                if not height:
+                    return 25, 80
+                return height, width
+
+    else:
+
+        def getheightwidth(self):
+            """
+            Get the height and width of the console.
+
+            Returns:
+            - tuple: Height and width of the console.
+            """
+            try:
+                return int(os.environ["LINES"]), int(os.environ["COLUMNS"])
+            except (KeyError, TypeError, ValueError):
+                return 25, 80
+
+    def forgetinput(self):
+        """
+        Discard any pending input on the console.
+        """
+        termios.tcflush(self.input_fd, termios.TCIFLUSH)
+
+    def flushoutput(self):
+        """
+        Flush the output buffer.
+        """
+        for text, iscode in self.__buffer:
+            if iscode:
+                self.__tputs(text)
+            else:
+                os.write(self.output_fd, text.encode(self.encoding, "replace"))
+        del self.__buffer[:]
+
+    def finish(self):
+        """
+        Finish console operations and flush the output buffer.
+        """
+        y = len(self.screen) - 1
+        while y >= 0 and not self.screen[y]:
+            y -= 1
+        self.__move(0, min(y, self.height + self.__offset - 1))
+        self.__write("\n\r")
+        self.flushoutput()
+
+    def beep(self):
+        """
+        Emit a beep sound.
+        """
+        self.__maybe_write_code(self._bel)
+        self.flushoutput()
+
+    if FIONREAD:
+
+        def getpending(self):
+            """
+            Get pending events from the console event queue.
+
+            Returns:
+            - Event: Pending event from the event queue.
+            """
+            e = Event("key", "", b"")
+
+            while not self.event_queue.empty():
+                e2 = self.event_queue.get()
+                e.data += e2.data
+                e.raw += e.raw
+
+            amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0]
+            raw = self.__read(amount)
+            data = str(raw, self.encoding, "replace")
+            e.data += data
+            e.raw += raw
+            return e
+
+    else:
+
+        def getpending(self):
+            """
+            Get pending events from the console event queue.
+
+            Returns:
+            - Event: Pending event from the event queue.
+            """
+            e = Event("key", "", b"")
+
+            while not self.event_queue.empty():
+                e2 = self.event_queue.get()
+                e.data += e2.data
+                e.raw += e.raw
+
+            amount = 10000
+            raw = self.__read(amount)
+            data = str(raw, self.encoding, "replace")
+            e.data += data
+            e.raw += raw
+            return e
+
+    def clear(self):
+        """
+        Clear the console screen.
+        """
+        self.__write_code(self._clear)
+        self.__gone_tall = 1
+        self.__move = self.__move_tall
+        self.posxy = 0, 0
+        self.screen = []
+
+    @property
+    def input_hook(self):
+        try:
+            import posix
+        except ImportError:
+            return None
+        if posix._is_inputhook_installed():
+            return posix._inputhook
+
+    def __enable_bracketed_paste(self) -> None:
+        os.write(self.output_fd, b"\x1b[?2004h")
+
+    def __disable_bracketed_paste(self) -> None:
+        os.write(self.output_fd, b"\x1b[?2004l")
+
+    def __setup_movement(self):
+        """
+        Set up the movement functions based on the terminal capabilities.
+        """
+        if 0 and self._hpa:  # hpa don't work in windows telnet :-(
+            self.__move_x = self.__move_x_hpa
+        elif self._cub and self._cuf:
+            self.__move_x = self.__move_x_cub_cuf
+        elif self._cub1 and self._cuf1:
+            self.__move_x = self.__move_x_cub1_cuf1
+        else:
+            raise RuntimeError("insufficient terminal (horizontal)")
+
+        if self._cuu and self._cud:
+            self.__move_y = self.__move_y_cuu_cud
+        elif self._cuu1 and self._cud1:
+            self.__move_y = self.__move_y_cuu1_cud1
+        else:
+            raise RuntimeError("insufficient terminal (vertical)")
+
+        if self._dch1:
+            self.dch1 = self._dch1
+        elif self._dch:
+            self.dch1 = curses.tparm(self._dch, 1)
+        else:
+            self.dch1 = None
+
+        if self._ich1:
+            self.ich1 = self._ich1
+        elif self._ich:
+            self.ich1 = curses.tparm(self._ich, 1)
+        else:
+            self.ich1 = None
+
+        self.__move = self.__move_short
+
+    def __write_changed_line(self, y, oldline, newline, px_coord):
+        # this is frustrating; there's no reason to test (say)
+        # self.dch1 inside the loop -- but alternative ways of
+        # structuring this function are equally painful (I'm trying to
+        # avoid writing code generators these days...)
+        minlen = min(wlen(oldline), wlen(newline))
+        x_pos = 0
+        x_coord = 0
+
+        px_pos = 0
+        j = 0
+        for c in oldline:
+            if j >= px_coord:
+                break
+            j += wlen(c)
+            px_pos += 1
+
+        # reuse the oldline as much as possible, but stop as soon as we
+        # encounter an ESCAPE, because it might be the start of an escape
+        # sequene
+        while (
+            x_coord < minlen
+            and oldline[x_pos] == newline[x_pos]
+            and newline[x_pos] != "\x1b"
+        ):
+            x_coord += wlen(newline[x_pos])
+            x_pos += 1
+
+        # if we need to insert a single character right after the first detected change
+        if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1:
+            if (
+                y == self.posxy[1]
+                and x_coord > self.posxy[0]
+                and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1]
+            ):
+                x_pos = px_pos
+                x_coord = px_coord
+            character_width = wlen(newline[x_pos])
+            self.__move(x_coord, y)
+            self.__write_code(self.ich1)
+            self.__write(newline[x_pos])
+            self.posxy = x_coord + character_width, y
+
+        # if it's a single character change in the middle of the line
+        elif (
+            x_coord < minlen
+            and oldline[x_pos + 1 :] == newline[x_pos + 1 :]
+            and wlen(oldline[x_pos]) == wlen(newline[x_pos])
+        ):
+            character_width = wlen(newline[x_pos])
+            self.__move(x_coord, y)
+            self.__write(newline[x_pos])
+            self.posxy = x_coord + character_width, y
+
+        # if this is the last character to fit in the line and we edit in the middle of the line
+        elif (
+            self.dch1
+            and self.ich1
+            and wlen(newline) == self.width
+            and x_coord < wlen(newline) - 2
+            and newline[x_pos + 1 : -1] == oldline[x_pos:-2]
+        ):
+            self.__hide_cursor()
+            self.__move(self.width - 2, y)
+            self.posxy = self.width - 2, y
+            self.__write_code(self.dch1)
+
+            character_width = wlen(newline[x_pos])
+            self.__move(x_coord, y)
+            self.__write_code(self.ich1)
+            self.__write(newline[x_pos])
+            self.posxy = character_width + 1, y
+
+        else:
+            self.__hide_cursor()
+            self.__move(x_coord, y)
+            if wlen(oldline) > wlen(newline):
+                self.__write_code(self._el)
+            self.__write(newline[x_pos:])
+            self.posxy = wlen(newline), y
+
+        if "\x1b" in newline:
+            # ANSI escape characters are present, so we can't assume
+            # anything about the position of the cursor.  Moving the cursor
+            # to the left margin should work to get to a known position.
+            self.move_cursor(0, y)
+
+    def __write(self, text):
+        self.__buffer.append((text, 0))
+
+    def __write_code(self, fmt, *args):
+        self.__buffer.append((curses.tparm(fmt, *args), 1))
+
+    def __maybe_write_code(self, fmt, *args):
+        if fmt:
+            self.__write_code(fmt, *args)
+
+    def __move_y_cuu1_cud1(self, y):
+        assert self._cud1 is not None
+        assert self._cuu1 is not None
+        dy = y - self.posxy[1]
+        if dy > 0:
+            self.__write_code(dy * self._cud1)
+        elif dy < 0:
+            self.__write_code((-dy) * self._cuu1)
+
+    def __move_y_cuu_cud(self, y):
+        dy = y - self.posxy[1]
+        if dy > 0:
+            self.__write_code(self._cud, dy)
+        elif dy < 0:
+            self.__write_code(self._cuu, -dy)
+
+    def __move_x_hpa(self, x: int) -> None:
+        if x != self.posxy[0]:
+            self.__write_code(self._hpa, x)
+
+    def __move_x_cub1_cuf1(self, x: int) -> None:
+        assert self._cuf1 is not None
+        assert self._cub1 is not None
+        dx = x - self.posxy[0]
+        if dx > 0:
+            self.__write_code(self._cuf1 * dx)
+        elif dx < 0:
+            self.__write_code(self._cub1 * (-dx))
+
+    def __move_x_cub_cuf(self, x: int) -> None:
+        dx = x - self.posxy[0]
+        if dx > 0:
+            self.__write_code(self._cuf, dx)
+        elif dx < 0:
+            self.__write_code(self._cub, -dx)
+
+    def __move_short(self, x, y):
+        self.__move_x(x)
+        self.__move_y(y)
+
+    def __move_tall(self, x, y):
+        assert 0 <= y - self.__offset < self.height, y - self.__offset
+        self.__write_code(self._cup, y - self.__offset, x)
+
+    def __sigwinch(self, signum, frame):
+        self.height, self.width = self.getheightwidth()
+        self.event_queue.insert(Event("resize", None))
+
+    def __hide_cursor(self):
+        if self.cursor_visible:
+            self.__maybe_write_code(self._civis)
+            self.cursor_visible = 0
+
+    def __show_cursor(self):
+        if not self.cursor_visible:
+            self.__maybe_write_code(self._cnorm)
+            self.cursor_visible = 1
+
+    def repaint(self):
+        if not self.__gone_tall:
+            self.posxy = 0, self.posxy[1]
+            self.__write("\r")
+            ns = len(self.screen) * ["\000" * self.width]
+            self.screen = ns
+        else:
+            self.posxy = 0, self.__offset
+            self.__move(0, self.__offset)
+            ns = self.height * ["\000" * self.width]
+            self.screen = ns
+
+    def __tputs(self, fmt, prog=delayprog):
+        """A Python implementation of the curses tputs function; the
+        curses one can't really be wrapped in a sane manner.
+
+        I have the strong suspicion that this is complexity that
+        will never do anyone any good."""
+        # using .get() means that things will blow up
+        # only if the bps is actually needed (which I'm
+        # betting is pretty unlkely)
+        bps = ratedict.get(self.__svtermstate.ospeed)
+        while 1:
+            m = prog.search(fmt)
+            if not m:
+                os.write(self.output_fd, fmt)
+                break
+            x, y = m.span()
+            os.write(self.output_fd, fmt[:x])
+            fmt = fmt[y:]
+            delay = int(m.group(1))
+            if b"*" in m.group(2):
+                delay *= self.height
+            if self._pad and bps is not None:
+                nchars = (bps * delay) / 1000
+                os.write(self.output_fd, self._pad * nchars)
+            else:
+                time.sleep(float(delay) / 1000.0)

Lib/_pyrepl/unix_eventqueue.py

@@ -0,0 +1,152 @@
+#   Copyright 2000-2008 Michael Hudson-Doyle <micahel@gmail.com>
+#                       Armin Rigo
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from collections import deque
+
+from . import keymap
+from .console import Event
+from . import curses
+from .trace import trace
+from termios import tcgetattr, VERASE
+import os
+
+
+# Mapping of human-readable key names to their terminal-specific codes
+TERMINAL_KEYNAMES = {
+    "delete": "kdch1",
+    "down": "kcud1",
+    "end": "kend",
+    "enter": "kent",
+    "home": "khome",
+    "insert": "kich1",
+    "left": "kcub1",
+    "page down": "knp",
+    "page up": "kpp",
+    "right": "kcuf1",
+    "up": "kcuu1",
+}
+
+
+# Function keys F1-F20 mapping
+TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21))
+
+# Known CTRL-arrow keycodes
+CTRL_ARROW_KEYCODES= {
+    # for xterm, gnome-terminal, xfce terminal, etc.
+    b'\033[1;5D': 'ctrl left',
+    b'\033[1;5C': 'ctrl right',
+    # for rxvt
+    b'\033Od': 'ctrl left',
+    b'\033Oc': 'ctrl right',
+}
+
+def get_terminal_keycodes() -> dict[bytes, str]:
+    """
+    Generates a dictionary mapping terminal keycodes to human-readable names.
+    """
+    keycodes = {}
+    for key, terminal_code in TERMINAL_KEYNAMES.items():
+        keycode = curses.tigetstr(terminal_code)
+        trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals())
+        if keycode:
+            keycodes[keycode] = key
+    keycodes.update(CTRL_ARROW_KEYCODES)
+    return keycodes
+
+class EventQueue:
+    def __init__(self, fd: int, encoding: str) -> None:
+        self.keycodes = get_terminal_keycodes()
+        if os.isatty(fd):
+            backspace = tcgetattr(fd)[6][VERASE]
+            self.keycodes[backspace] = "backspace"
+        self.compiled_keymap = keymap.compile_keymap(self.keycodes)
+        self.keymap = self.compiled_keymap
+        trace("keymap {k!r}", k=self.keymap)
+        self.encoding = encoding
+        self.events: deque[Event] = deque()
+        self.buf = bytearray()
+
+    def get(self) -> Event | None:
+        """
+        Retrieves the next event from the queue.
+        """
+        if self.events:
+            return self.events.popleft()
+        else:
+            return None
+
+    def empty(self) -> bool:
+        """
+        Checks if the queue is empty.
+        """
+        return not self.events
+
+    def flush_buf(self) -> bytearray:
+        """
+        Flushes the buffer and returns its contents.
+        """
+        old = self.buf
+        self.buf = bytearray()
+        return old
+
+    def insert(self, event: Event) -> None:
+        """
+        Inserts an event into the queue.
+        """
+        trace('added event {event}', event=event)
+        self.events.append(event)
+
+    def push(self, char: int | bytes) -> None:
+        """
+        Processes a character by updating the buffer and handling special key mappings.
+        """
+        ord_char = char if isinstance(char, int) else ord(char)
+        char = bytes(bytearray((ord_char,)))
+        self.buf.append(ord_char)
+        if char in self.keymap:
+            if self.keymap is self.compiled_keymap:
+                #sanity check, buffer is empty when a special key comes
+                assert len(self.buf) == 1
+            k = self.keymap[char]
+            trace('found map {k!r}', k=k)
+            if isinstance(k, dict):
+                self.keymap = k
+            else:
+                self.insert(Event('key', k, self.flush_buf()))
+                self.keymap = self.compiled_keymap
+
+        elif self.buf and self.buf[0] == 27:  # escape
+            # escape sequence not recognized by our keymap: propagate it
+            # outside so that i can be recognized as an M-... key (see also
+            # the docstring in keymap.py
+            trace('unrecognized escape sequence, propagating...')
+            self.keymap = self.compiled_keymap
+            self.insert(Event('key', '\033', bytearray(b'\033')))
+            for _c in self.flush_buf()[1:]:
+                self.push(_c)
+
+        else:
+            try:
+                decoded = bytes(self.buf).decode(self.encoding)
+            except UnicodeError:
+                return
+            else:
+                self.insert(Event('key', decoded, self.flush_buf()))
+            self.keymap = self.compiled_keymap

Lib/_pyrepl/utils.py

@@ -0,0 +1,25 @@
+import re
+import unicodedata
+import functools
+
+ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]")
+
+
+@functools.cache
+def str_width(c: str) -> int:
+    if ord(c) < 128:
+        return 1
+    w = unicodedata.east_asian_width(c)
+    if w in ('N', 'Na', 'H', 'A'):
+        return 1
+    return 2
+
+
+def wlen(s: str) -> int:
+    if len(s) == 1 and s != '\x1a':
+        return str_width(s)
+    length = sum(str_width(i) for i in s)
+    # remove lengths of any escape sequences
+    sequence = ANSI_ESCAPE_SEQUENCE.findall(s)
+    ctrl_z_cnt = s.count('\x1a')
+    return length - sum(len(i) for i in sequence) + ctrl_z_cnt

Lib/_pyrepl/windows_console.py

@@ -0,0 +1,618 @@
+#   Copyright 2000-2004 Michael Hudson-Doyle <micahel@gmail.com>
+#
+#                        All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import io
+import os
+import sys
+import time
+import msvcrt
+
+from collections import deque
+import ctypes
+from ctypes.wintypes import (
+    _COORD,
+    WORD,
+    SMALL_RECT,
+    BOOL,
+    HANDLE,
+    CHAR,
+    DWORD,
+    WCHAR,
+    SHORT,
+)
+from ctypes import Structure, POINTER, Union
+from .console import Event, Console
+from .trace import trace
+from .utils import wlen
+
+try:
+    from ctypes import GetLastError, WinDLL, windll, WinError  # type: ignore[attr-defined]
+except:
+    # Keep MyPy happy off Windows
+    from ctypes import CDLL as WinDLL, cdll as windll
+
+    def GetLastError() -> int:
+        return 42
+
+    class WinError(OSError):  # type: ignore[no-redef]
+        def __init__(self, err: int | None, descr: str | None = None) -> None:
+            self.err = err
+            self.descr = descr
+
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+    from typing import IO
+
+# Virtual-Key Codes: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
+VK_MAP: dict[int, str] = {
+    0x23: "end",  # VK_END
+    0x24: "home",  # VK_HOME
+    0x25: "left",  # VK_LEFT
+    0x26: "up",  # VK_UP
+    0x27: "right",  # VK_RIGHT
+    0x28: "down",  # VK_DOWN
+    0x2E: "delete",  # VK_DELETE
+    0x70: "f1",  # VK_F1
+    0x71: "f2",  # VK_F2
+    0x72: "f3",  # VK_F3
+    0x73: "f4",  # VK_F4
+    0x74: "f5",  # VK_F5
+    0x75: "f6",  # VK_F6
+    0x76: "f7",  # VK_F7
+    0x77: "f8",  # VK_F8
+    0x78: "f9",  # VK_F9
+    0x79: "f10",  # VK_F10
+    0x7A: "f11",  # VK_F11
+    0x7B: "f12",  # VK_F12
+    0x7C: "f13",  # VK_F13
+    0x7D: "f14",  # VK_F14
+    0x7E: "f15",  # VK_F15
+    0x7F: "f16",  # VK_F16
+    0x80: "f17",  # VK_F17
+    0x81: "f18",  # VK_F18
+    0x82: "f19",  # VK_F19
+    0x83: "f20",  # VK_F20
+}
+
+# Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
+ERASE_IN_LINE = "\x1b[K"
+MOVE_LEFT = "\x1b[{}D"
+MOVE_RIGHT = "\x1b[{}C"
+MOVE_UP = "\x1b[{}A"
+MOVE_DOWN = "\x1b[{}B"
+CLEAR = "\x1b[H\x1b[J"
+
+
+class _error(Exception):
+    pass
+
+
+class WindowsConsole(Console):
+    def __init__(
+        self,
+        f_in: IO[bytes] | int = 0,
+        f_out: IO[bytes] | int = 1,
+        term: str = "",
+        encoding: str = "",
+    ):
+        super().__init__(f_in, f_out, term, encoding)
+
+        SetConsoleMode(
+            OutHandle,
+            ENABLE_WRAP_AT_EOL_OUTPUT
+            | ENABLE_PROCESSED_OUTPUT
+            | ENABLE_VIRTUAL_TERMINAL_PROCESSING,
+        )
+        self.screen: list[str] = []
+        self.width = 80
+        self.height = 25
+        self.__offset = 0
+        self.event_queue: deque[Event] = deque()
+        try:
+            self.out = io._WindowsConsoleIO(self.output_fd, "w")  # type: ignore[attr-defined]
+        except ValueError:
+            # Console I/O is redirected, fallback...
+            self.out = None
+
+    def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None:
+        """
+        Refresh the console screen.
+
+        Parameters:
+        - screen (list): List of strings representing the screen contents.
+        - c_xy (tuple): Cursor position (x, y) on the screen.
+        """
+        cx, cy = c_xy
+
+        while len(self.screen) < min(len(screen), self.height):
+            self._hide_cursor()
+            self._move_relative(0, len(self.screen) - 1)
+            self.__write("\n")
+            self.posxy = 0, len(self.screen)
+            self.screen.append("")
+
+        px, py = self.posxy
+        old_offset = offset = self.__offset
+        height = self.height
+
+        # we make sure the cursor is on the screen, and that we're
+        # using all of the screen if we can
+        if cy < offset:
+            offset = cy
+        elif cy >= offset + height:
+            offset = cy - height + 1
+            scroll_lines = offset - old_offset
+
+            # Scrolling the buffer as the current input is greater than the visible
+            # portion of the window.  We need to scroll the visible portion and the
+            # entire history
+            self._scroll(scroll_lines, self._getscrollbacksize())
+            self.posxy = self.posxy[0], self.posxy[1] + scroll_lines
+            self.__offset += scroll_lines
+
+            for i in range(scroll_lines):
+                self.screen.append("")
+        elif offset > 0 and len(screen) < offset + height:
+            offset = max(len(screen) - height, 0)
+            screen.append("")
+
+        oldscr = self.screen[old_offset : old_offset + height]
+        newscr = screen[offset : offset + height]
+
+        self.__offset = offset
+
+        self._hide_cursor()
+        for (
+            y,
+            oldline,
+            newline,
+        ) in zip(range(offset, offset + height), oldscr, newscr):
+            if oldline != newline:
+                self.__write_changed_line(y, oldline, newline, px)
+
+        y = len(newscr)
+        while y < len(oldscr):
+            self._move_relative(0, y)
+            self.posxy = 0, y
+            self._erase_to_end()
+            y += 1
+
+        self._show_cursor()
+
+        self.screen = screen
+        self.move_cursor(cx, cy)
+
+    @property
+    def input_hook(self):
+        try:
+            import nt
+        except ImportError:
+            return None
+        if nt._is_inputhook_installed():
+            return nt._inputhook
+
+    def __write_changed_line(
+        self, y: int, oldline: str, newline: str, px_coord: int
+    ) -> None:
+        # this is frustrating; there's no reason to test (say)
+        # self.dch1 inside the loop -- but alternative ways of
+        # structuring this function are equally painful (I'm trying to
+        # avoid writing code generators these days...)
+        minlen = min(wlen(oldline), wlen(newline))
+        x_pos = 0
+        x_coord = 0
+
+        px_pos = 0
+        j = 0
+        for c in oldline:
+            if j >= px_coord:
+                break
+            j += wlen(c)
+            px_pos += 1
+
+        # reuse the oldline as much as possible, but stop as soon as we
+        # encounter an ESCAPE, because it might be the start of an escape
+        # sequene
+        while (
+            x_coord < minlen
+            and oldline[x_pos] == newline[x_pos]
+            and newline[x_pos] != "\x1b"
+        ):
+            x_coord += wlen(newline[x_pos])
+            x_pos += 1
+
+        self._hide_cursor()
+        self._move_relative(x_coord, y)
+        if wlen(oldline) > wlen(newline):
+            self._erase_to_end()
+
+        self.__write(newline[x_pos:])
+        if wlen(newline) == self.width:
+            # If we wrapped we want to start at the next line
+            self._move_relative(0, y + 1)
+            self.posxy = 0, y + 1
+        else:
+            self.posxy = wlen(newline), y
+
+            if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline:
+                # ANSI escape characters are present, so we can't assume
+                # anything about the position of the cursor.  Moving the cursor
+                # to the left margin should work to get to a known position.
+                self.move_cursor(0, y)
+
+    def _scroll(
+        self, top: int, bottom: int, left: int | None = None, right: int | None = None
+    ) -> None:
+        scroll_rect = SMALL_RECT()
+        scroll_rect.Top = SHORT(top)
+        scroll_rect.Bottom = SHORT(bottom)
+        scroll_rect.Left = SHORT(0 if left is None else left)
+        scroll_rect.Right = SHORT(
+            self.getheightwidth()[1] - 1 if right is None else right
+        )
+        destination_origin = _COORD()
+        fill_info = CHAR_INFO()
+        fill_info.UnicodeChar = " "
+
+        if not ScrollConsoleScreenBuffer(
+            OutHandle, scroll_rect, None, destination_origin, fill_info
+        ):
+            raise WinError(GetLastError())
+
+    def _hide_cursor(self):
+        self.__write("\x1b[?25l")
+
+    def _show_cursor(self):
+        self.__write("\x1b[?25h")
+
+    def _enable_blinking(self):
+        self.__write("\x1b[?12h")
+
+    def _disable_blinking(self):
+        self.__write("\x1b[?12l")
+
+    def __write(self, text: str) -> None:
+        if "\x1a" in text:
+            text = ''.join(["^Z" if x == '\x1a' else x for x in text])
+
+        if self.out is not None:
+            self.out.write(text.encode(self.encoding, "replace"))
+            self.out.flush()
+        else:
+            os.write(self.output_fd, text.encode(self.encoding, "replace"))
+
+    @property
+    def screen_xy(self) -> tuple[int, int]:
+        info = CONSOLE_SCREEN_BUFFER_INFO()
+        if not GetConsoleScreenBufferInfo(OutHandle, info):
+            raise WinError(GetLastError())
+        return info.dwCursorPosition.X, info.dwCursorPosition.Y
+
+    def _erase_to_end(self) -> None:
+        self.__write(ERASE_IN_LINE)
+
+    def prepare(self) -> None:
+        trace("prepare")
+        self.screen = []
+        self.height, self.width = self.getheightwidth()
+
+        self.posxy = 0, 0
+        self.__gone_tall = 0
+        self.__offset = 0
+
+    def restore(self) -> None:
+        pass
+
+    def _move_relative(self, x: int, y: int) -> None:
+        """Moves relative to the current posxy"""
+        dx = x - self.posxy[0]
+        dy = y - self.posxy[1]
+        if dx < 0:
+            self.__write(MOVE_LEFT.format(-dx))
+        elif dx > 0:
+            self.__write(MOVE_RIGHT.format(dx))
+
+        if dy < 0:
+            self.__write(MOVE_UP.format(-dy))
+        elif dy > 0:
+            self.__write(MOVE_DOWN.format(dy))
+
+    def move_cursor(self, x: int, y: int) -> None:
+        if x < 0 or y < 0:
+            raise ValueError(f"Bad cursor position {x}, {y}")
+
+        if y < self.__offset or y >= self.__offset + self.height:
+            self.event_queue.insert(0, Event("scroll", ""))
+        else:
+            self._move_relative(x, y)
+            self.posxy = x, y
+
+    def set_cursor_vis(self, visible: bool) -> None:
+        if visible:
+            self._show_cursor()
+        else:
+            self._hide_cursor()
+
+    def getheightwidth(self) -> tuple[int, int]:
+        """Return (height, width) where height and width are the height
+        and width of the terminal window in characters."""
+        info = CONSOLE_SCREEN_BUFFER_INFO()
+        if not GetConsoleScreenBufferInfo(OutHandle, info):
+            raise WinError(GetLastError())
+        return (
+            info.srWindow.Bottom - info.srWindow.Top + 1,
+            info.srWindow.Right - info.srWindow.Left + 1,
+        )
+
+    def _getscrollbacksize(self) -> int:
+        info = CONSOLE_SCREEN_BUFFER_INFO()
+        if not GetConsoleScreenBufferInfo(OutHandle, info):
+            raise WinError(GetLastError())
+
+        return info.srWindow.Bottom  # type: ignore[no-any-return]
+
+    def _read_input(self, block: bool = True) -> INPUT_RECORD | None:
+        if not block:
+            events = DWORD()
+            if not GetNumberOfConsoleInputEvents(InHandle, events):
+                raise WinError(GetLastError())
+            if not events.value:
+                return None
+
+        rec = INPUT_RECORD()
+        read = DWORD()
+        if not ReadConsoleInput(InHandle, rec, 1, read):
+            raise WinError(GetLastError())
+
+        return rec
+
+    def get_event(self, block: bool = True) -> Event | None:
+        """Return an Event instance.  Returns None if |block| is false
+        and there is no event pending, otherwise waits for the
+        completion of an event."""
+        if self.event_queue:
+            return self.event_queue.pop()
+
+        while True:
+            rec = self._read_input(block)
+            if rec is None:
+                return None
+
+            if rec.EventType == WINDOW_BUFFER_SIZE_EVENT:
+                return Event("resize", "")
+
+            if rec.EventType != KEY_EVENT or not rec.Event.KeyEvent.bKeyDown:
+                # Only process keys and keydown events
+                if block:
+                    continue
+                return None
+
+            key = rec.Event.KeyEvent.uChar.UnicodeChar
+
+            if rec.Event.KeyEvent.uChar.UnicodeChar == "\r":
+                # Make enter make unix-like
+                return Event(evt="key", data="\n", raw=b"\n")
+            elif rec.Event.KeyEvent.wVirtualKeyCode == 8:
+                # Turn backspace directly into the command
+                return Event(
+                    evt="key",
+                    data="backspace",
+                    raw=rec.Event.KeyEvent.uChar.UnicodeChar,
+                )
+            elif rec.Event.KeyEvent.uChar.UnicodeChar == "\x00":
+                # Handle special keys like arrow keys and translate them into the appropriate command
+                code = VK_MAP.get(rec.Event.KeyEvent.wVirtualKeyCode)
+                if code:
+                    return Event(
+                        evt="key", data=code, raw=rec.Event.KeyEvent.uChar.UnicodeChar
+                    )
+                if block:
+                    continue
+
+                return None
+
+            return Event(evt="key", data=key, raw=rec.Event.KeyEvent.uChar.UnicodeChar)
+
+    def push_char(self, char: int | bytes) -> None:
+        """
+        Push a character to the console event queue.
+        """
+        raise NotImplementedError("push_char not supported on Windows")
+
+    def beep(self) -> None:
+        self.__write("\x07")
+
+    def clear(self) -> None:
+        """Wipe the screen"""
+        self.__write(CLEAR)
+        self.posxy = 0, 0
+        self.screen = [""]
+
+    def finish(self) -> None:
+        """Move the cursor to the end of the display and otherwise get
+        ready for end.  XXX could be merged with restore?  Hmm."""
+        y = len(self.screen) - 1
+        while y >= 0 and not self.screen[y]:
+            y -= 1
+        self._move_relative(0, min(y, self.height + self.__offset - 1))
+        self.__write("\r\n")
+
+    def flushoutput(self) -> None:
+        """Flush all output to the screen (assuming there's some
+        buffering going on somewhere).
+
+        All output on Windows is unbuffered so this is a nop"""
+        pass
+
+    def forgetinput(self) -> None:
+        """Forget all pending, but not yet processed input."""
+        if not FlushConsoleInputBuffer(InHandle):
+            raise WinError(GetLastError())
+
+    def getpending(self) -> Event:
+        """Return the characters that have been typed but not yet
+        processed."""
+        return Event("key", "", b"")
+
+    def wait(self, timeout: float | None) -> bool:
+        """Wait for an event."""
+        # Poor man's Windows select loop
+        start_time = time.time()
+        while True:
+            if msvcrt.kbhit(): # type: ignore[attr-defined]
+                return True
+            if timeout and time.time() - start_time > timeout / 1000:
+                return False
+            time.sleep(0.01)
+
+    def repaint(self) -> None:
+        raise NotImplementedError("No repaint support")
+
+
+# Windows interop
+class CONSOLE_SCREEN_BUFFER_INFO(Structure):
+    _fields_ = [
+        ("dwSize", _COORD),
+        ("dwCursorPosition", _COORD),
+        ("wAttributes", WORD),
+        ("srWindow", SMALL_RECT),
+        ("dwMaximumWindowSize", _COORD),
+    ]
+
+
+class CONSOLE_CURSOR_INFO(Structure):
+    _fields_ = [
+        ("dwSize", DWORD),
+        ("bVisible", BOOL),
+    ]
+
+
+class CHAR_INFO(Structure):
+    _fields_ = [
+        ("UnicodeChar", WCHAR),
+        ("Attributes", WORD),
+    ]
+
+
+class Char(Union):
+    _fields_ = [
+        ("UnicodeChar", WCHAR),
+        ("Char", CHAR),
+    ]
+
+
+class KeyEvent(ctypes.Structure):
+    _fields_ = [
+        ("bKeyDown", BOOL),
+        ("wRepeatCount", WORD),
+        ("wVirtualKeyCode", WORD),
+        ("wVirtualScanCode", WORD),
+        ("uChar", Char),
+        ("dwControlKeyState", DWORD),
+    ]
+
+
+class WindowsBufferSizeEvent(ctypes.Structure):
+    _fields_ = [("dwSize", _COORD)]
+
+
+class ConsoleEvent(ctypes.Union):
+    _fields_ = [
+        ("KeyEvent", KeyEvent),
+        ("WindowsBufferSizeEvent", WindowsBufferSizeEvent),
+    ]
+
+
+class INPUT_RECORD(Structure):
+    _fields_ = [("EventType", WORD), ("Event", ConsoleEvent)]
+
+
+KEY_EVENT = 0x01
+FOCUS_EVENT = 0x10
+MENU_EVENT = 0x08
+MOUSE_EVENT = 0x02
+WINDOW_BUFFER_SIZE_EVENT = 0x04
+
+ENABLE_PROCESSED_OUTPUT = 0x01
+ENABLE_WRAP_AT_EOL_OUTPUT = 0x02
+ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04
+
+STD_INPUT_HANDLE = -10
+STD_OUTPUT_HANDLE = -11
+
+if sys.platform == "win32":
+    _KERNEL32 = WinDLL("kernel32", use_last_error=True)
+
+    GetStdHandle = windll.kernel32.GetStdHandle
+    GetStdHandle.argtypes = [DWORD]
+    GetStdHandle.restype = HANDLE
+
+    GetConsoleScreenBufferInfo = _KERNEL32.GetConsoleScreenBufferInfo
+    GetConsoleScreenBufferInfo.argtypes = [
+        HANDLE,
+        ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO),
+    ]
+    GetConsoleScreenBufferInfo.restype = BOOL
+
+    ScrollConsoleScreenBuffer = _KERNEL32.ScrollConsoleScreenBufferW
+    ScrollConsoleScreenBuffer.argtypes = [
+        HANDLE,
+        POINTER(SMALL_RECT),
+        POINTER(SMALL_RECT),
+        _COORD,
+        POINTER(CHAR_INFO),
+    ]
+    ScrollConsoleScreenBuffer.restype = BOOL
+
+    SetConsoleMode = _KERNEL32.SetConsoleMode
+    SetConsoleMode.argtypes = [HANDLE, DWORD]
+    SetConsoleMode.restype = BOOL
+
+    ReadConsoleInput = _KERNEL32.ReadConsoleInputW
+    ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)]
+    ReadConsoleInput.restype = BOOL
+
+    GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents
+    GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)]
+    GetNumberOfConsoleInputEvents.restype = BOOL
+
+    FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer
+    FlushConsoleInputBuffer.argtypes = [HANDLE]
+    FlushConsoleInputBuffer.restype = BOOL
+
+    OutHandle = GetStdHandle(STD_OUTPUT_HANDLE)
+    InHandle = GetStdHandle(STD_INPUT_HANDLE)
+else:
+
+    def _win_only(*args, **kwargs):
+        raise NotImplementedError("Windows only")
+
+    GetStdHandle = _win_only
+    GetConsoleScreenBufferInfo = _win_only
+    ScrollConsoleScreenBuffer = _win_only
+    SetConsoleMode = _win_only
+    ReadConsoleInput = _win_only
+    GetNumberOfConsoleInputEvents = _win_only
+    FlushConsoleInputBuffer = _win_only
+    OutHandle = 0
+    InHandle = 0

Lib/_threading_local.py

@@ -4,133 +4,6 @@
  class.  Depending on the version of Python you're using, there may be a
  faster one available.  You should always import the `local` class from
  `threading`.)
-
-Thread-local objects support the management of thread-local data.
-If you have data that you want to be local to a thread, simply create
-a thread-local object and use its attributes:
-
-  >>> mydata = local()
-  >>> mydata.number = 42
-  >>> mydata.number
-  42
-
-You can also access the local-object's dictionary:
-
-  >>> mydata.__dict__
-  {'number': 42}
-  >>> mydata.__dict__.setdefault('widgets', [])
-  []
-  >>> mydata.widgets
-  []
-
-What's important about thread-local objects is that their data are
-local to a thread. If we access the data in a different thread:
-
-  >>> log = []
-  >>> def f():
-  ...     items = sorted(mydata.__dict__.items())
-  ...     log.append(items)
-  ...     mydata.number = 11
-  ...     log.append(mydata.number)
-
-  >>> import threading
-  >>> thread = threading.Thread(target=f)
-  >>> thread.start()
-  >>> thread.join()
-  >>> log
-  [[], 11]
-
-we get different data.  Furthermore, changes made in the other thread
-don't affect data seen in this thread:
-
-  >>> mydata.number
-  42
-
-Of course, values you get from a local object, including a __dict__
-attribute, are for whatever thread was current at the time the
-attribute was read.  For that reason, you generally don't want to save
-these values across threads, as they apply only to the thread they
-came from.
-
-You can create custom local objects by subclassing the local class:
-
-  >>> class MyLocal(local):
-  ...     number = 2
-  ...     initialized = False
-  ...     def __init__(self, **kw):
-  ...         if self.initialized:
-  ...             raise SystemError('__init__ called too many times')
-  ...         self.initialized = True
-  ...         self.__dict__.update(kw)
-  ...     def squared(self):
-  ...         return self.number ** 2
-
-This can be useful to support default values, methods and
-initialization.  Note that if you define an __init__ method, it will be
-called each time the local object is used in a separate thread.  This
-is necessary to initialize each thread's dictionary.
-
-Now if we create a local object:
-
-  >>> mydata = MyLocal(color='red')
-
-Now we have a default number:
-
-  >>> mydata.number
-  2
-
-an initial color:
-
-  >>> mydata.color
-  'red'
-  >>> del mydata.color
-
-And a method that operates on the data:
-
-  >>> mydata.squared()
-  4
-
-As before, we can access the data in a separate thread:
-
-  >>> log = []
-  >>> thread = threading.Thread(target=f)
-  >>> thread.start()
-  >>> thread.join()
-  >>> log
-  [[('color', 'red'), ('initialized', True)], 11]
-
-without affecting this thread's data:
-
-  >>> mydata.number
-  2
-  >>> mydata.color
-  Traceback (most recent call last):
-  ...
-  AttributeError: 'MyLocal' object has no attribute 'color'
-
-Note that subclasses can define slots, but they are not thread
-local. They are shared across threads:
-
-  >>> class MyLocal(local):
-  ...     __slots__ = 'number'
-
-  >>> mydata = MyLocal()
-  >>> mydata.number = 42
-  >>> mydata.color = 'red'
-
-So, the separate thread:
-
-  >>> thread = threading.Thread(target=f)
-  >>> thread.start()
-  >>> thread.join()
-
-affects what we see:
-
-  >>> # TODO: RUSTPYTHON, __slots__
-  >>> mydata.number #doctest: +SKIP
-  11
-
->>> del mydata
 """
 
 from weakref import ref
@@ -194,7 +67,6 @@ class _localimpl:
 
 @contextmanager
 def _patch(self):
-    old = object.__getattribute__(self, '__dict__')
     impl = object.__getattribute__(self, '_local__impl')
     try:
         dct = impl.get_dict()
@@ -205,13 +77,12 @@ def _patch(self):
     with impl.locallock:
         object.__setattr__(self, '__dict__', dct)
         yield
-        object.__setattr__(self, '__dict__', old)
 
 
 class local:
     __slots__ = '_local__impl', '__dict__'
 
-    def __new__(cls, *args, **kw):
+    def __new__(cls, /, *args, **kw):
         if (args or kw) and (cls.__init__ is object.__init__):
             raise TypeError("Initialization arguments are not supported")
         self = object.__new__(cls)

Lib/_weakrefset.py

@@ -80,8 +80,7 @@ class WeakSet:
         return wr in self.data
 
     def __reduce__(self):
-        return (self.__class__, (list(self),),
-                getattr(self, '__dict__', None))
+        return self.__class__, (list(self),), self.__getstate__()
 
     def add(self, item):
         if self._pending_removals:

Lib/abc.py

@@ -85,10 +85,6 @@ try:
     from _abc import (get_cache_token, _abc_init, _abc_register,
                       _abc_instancecheck, _abc_subclasscheck, _get_dump,
                       _reset_registry, _reset_caches)
-# TODO: RUSTPYTHON missing _abc module implementation.
-except ModuleNotFoundError:
-    from _py_abc import ABCMeta, get_cache_token
-    ABCMeta.__module__ = 'abc'
 except ImportError:
     from _py_abc import ABCMeta, get_cache_token
     ABCMeta.__module__ = 'abc'

Lib/aifc.py

@@ -1,984 +0,0 @@
-"""Stuff to parse AIFF-C and AIFF files.
-
-Unless explicitly stated otherwise, the description below is true
-both for AIFF-C files and AIFF files.
-
-An AIFF-C file has the following structure.
-
-  +-----------------+
-  | FORM            |
-  +-----------------+
-  | <size>          |
-  +----+------------+
-  |    | AIFC       |
-  |    +------------+
-  |    | <chunks>   |
-  |    |    .       |
-  |    |    .       |
-  |    |    .       |
-  +----+------------+
-
-An AIFF file has the string "AIFF" instead of "AIFC".
-
-A chunk consists of an identifier (4 bytes) followed by a size (4 bytes,
-big endian order), followed by the data.  The size field does not include
-the size of the 8 byte header.
-
-The following chunk types are recognized.
-
-  FVER
-      <version number of AIFF-C defining document> (AIFF-C only).
-  MARK
-      <# of markers> (2 bytes)
-      list of markers:
-          <marker ID> (2 bytes, must be > 0)
-          <position> (4 bytes)
-          <marker name> ("pstring")
-  COMM
-      <# of channels> (2 bytes)
-      <# of sound frames> (4 bytes)
-      <size of the samples> (2 bytes)
-      <sampling frequency> (10 bytes, IEEE 80-bit extended
-          floating point)
-      in AIFF-C files only:
-      <compression type> (4 bytes)
-      <human-readable version of compression type> ("pstring")
-  SSND
-      <offset> (4 bytes, not used by this program)
-      <blocksize> (4 bytes, not used by this program)
-      <sound data>
-
-A pstring consists of 1 byte length, a string of characters, and 0 or 1
-byte pad to make the total length even.
-
-Usage.
-
-Reading AIFF files:
-  f = aifc.open(file, 'r')
-where file is either the name of a file or an open file pointer.
-The open file pointer must have methods read(), seek(), and close().
-In some types of audio files, if the setpos() method is not used,
-the seek() method is not necessary.
-
-This returns an instance of a class with the following public methods:
-  getnchannels()  -- returns number of audio channels (1 for
-             mono, 2 for stereo)
-  getsampwidth()  -- returns sample width in bytes
-  getframerate()  -- returns sampling frequency
-  getnframes()    -- returns number of audio frames
-  getcomptype()   -- returns compression type ('NONE' for AIFF files)
-  getcompname()   -- returns human-readable version of
-             compression type ('not compressed' for AIFF files)
-  getparams() -- returns a namedtuple consisting of all of the
-             above in the above order
-  getmarkers()    -- get the list of marks in the audio file or None
-             if there are no marks
-  getmark(id) -- get mark with the specified id (raises an error
-             if the mark does not exist)
-  readframes(n)   -- returns at most n frames of audio
-  rewind()    -- rewind to the beginning of the audio stream
-  setpos(pos) -- seek to the specified position
-  tell()      -- return the current position
-  close()     -- close the instance (make it unusable)
-The position returned by tell(), the position given to setpos() and
-the position of marks are all compatible and have nothing to do with
-the actual position in the file.
-The close() method is called automatically when the class instance
-is destroyed.
-
-Writing AIFF files:
-  f = aifc.open(file, 'w')
-where file is either the name of a file or an open file pointer.
-The open file pointer must have methods write(), tell(), seek(), and
-close().
-
-This returns an instance of a class with the following public methods:
-  aiff()      -- create an AIFF file (AIFF-C default)
-  aifc()      -- create an AIFF-C file
-  setnchannels(n) -- set the number of channels
-  setsampwidth(n) -- set the sample width
-  setframerate(n) -- set the frame rate
-  setnframes(n)   -- set the number of frames
-  setcomptype(type, name)
-          -- set the compression type and the
-             human-readable compression type
-  setparams(tuple)
-          -- set all parameters at once
-  setmark(id, pos, name)
-          -- add specified mark to the list of marks
-  tell()      -- return current position in output file (useful
-             in combination with setmark())
-  writeframesraw(data)
-          -- write audio frames without pathing up the
-             file header
-  writeframes(data)
-          -- write audio frames and patch up the file header
-  close()     -- patch up the file header and close the
-             output file
-You should set the parameters before the first writeframesraw or
-writeframes.  The total number of frames does not need to be set,
-but when it is set to the correct value, the header does not have to
-be patched up.
-It is best to first set all parameters, perhaps possibly the
-compression type, and then write audio frames using writeframesraw.
-When all frames have been written, either call writeframes(b'') or
-close() to patch up the sizes in the header.
-Marks can be added anytime.  If there are any marks, you must call
-close() after all frames have been written.
-The close() method is called automatically when the class instance
-is destroyed.
-
-When a file is opened with the extension '.aiff', an AIFF file is
-written, otherwise an AIFF-C file is written.  This default can be
-changed by calling aiff() or aifc() before the first writeframes or
-writeframesraw.
-"""
-
-import struct
-import builtins
-import warnings
-
-__all__ = ["Error", "open"]
-
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-
-class Error(Exception):
-    pass
-
-_AIFC_version = 0xA2805140     # Version 1 of AIFF-C
-
-def _read_long(file):
-    try:
-        return struct.unpack('>l', file.read(4))[0]
-    except struct.error:
-        raise EOFError from None
-
-def _read_ulong(file):
-    try:
-        return struct.unpack('>L', file.read(4))[0]
-    except struct.error:
-        raise EOFError from None
-
-def _read_short(file):
-    try:
-        return struct.unpack('>h', file.read(2))[0]
-    except struct.error:
-        raise EOFError from None
-
-def _read_ushort(file):
-    try:
-        return struct.unpack('>H', file.read(2))[0]
-    except struct.error:
-        raise EOFError from None
-
-def _read_string(file):
-    length = ord(file.read(1))
-    if length == 0:
-        data = b''
-    else:
-        data = file.read(length)
-    if length & 1 == 0:
-        dummy = file.read(1)
-    return data
-
-_HUGE_VAL = 1.79769313486231e+308 # See <limits.h>
-
-def _read_float(f): # 10 bytes
-    expon = _read_short(f) # 2 bytes
-    sign = 1
-    if expon < 0:
-        sign = -1
-        expon = expon + 0x8000
-    himant = _read_ulong(f) # 4 bytes
-    lomant = _read_ulong(f) # 4 bytes
-    if expon == himant == lomant == 0:
-        f = 0.0
-    elif expon == 0x7FFF:
-        f = _HUGE_VAL
-    else:
-        expon = expon - 16383
-        f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63)
-    return sign * f
-
-def _write_short(f, x):
-    f.write(struct.pack('>h', x))
-
-def _write_ushort(f, x):
-    f.write(struct.pack('>H', x))
-
-def _write_long(f, x):
-    f.write(struct.pack('>l', x))
-
-def _write_ulong(f, x):
-    f.write(struct.pack('>L', x))
-
-def _write_string(f, s):
-    if len(s) > 255:
-        raise ValueError("string exceeds maximum pstring length")
-    f.write(struct.pack('B', len(s)))
-    f.write(s)
-    if len(s) & 1 == 0:
-        f.write(b'\x00')
-
-def _write_float(f, x):
-    import math
-    if x < 0:
-        sign = 0x8000
-        x = x * -1
-    else:
-        sign = 0
-    if x == 0:
-        expon = 0
-        himant = 0
-        lomant = 0
-    else:
-        fmant, expon = math.frexp(x)
-        if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN
-            expon = sign|0x7FFF
-            himant = 0
-            lomant = 0
-        else:                   # Finite
-            expon = expon + 16382
-            if expon < 0:           # denormalized
-                fmant = math.ldexp(fmant, expon)
-                expon = 0
-            expon = expon | sign
-            fmant = math.ldexp(fmant, 32)
-            fsmant = math.floor(fmant)
-            himant = int(fsmant)
-            fmant = math.ldexp(fmant - fsmant, 32)
-            fsmant = math.floor(fmant)
-            lomant = int(fsmant)
-    _write_ushort(f, expon)
-    _write_ulong(f, himant)
-    _write_ulong(f, lomant)
-
-with warnings.catch_warnings():
-    warnings.simplefilter("ignore", DeprecationWarning)
-    from chunk import Chunk
-from collections import namedtuple
-
-_aifc_params = namedtuple('_aifc_params',
-                          'nchannels sampwidth framerate nframes comptype compname')
-
-_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)'
-_aifc_params.sampwidth.__doc__ = 'Sample width in bytes'
-_aifc_params.framerate.__doc__ = 'Sampling frequency'
-_aifc_params.nframes.__doc__ = 'Number of audio frames'
-_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)'
-_aifc_params.compname.__doc__ = ("""\
-A human-readable version of the compression type
-('not compressed' for AIFF files)""")
-
-
-class Aifc_read:
-    # Variables used in this class:
-    #
-    # These variables are available to the user though appropriate
-    # methods of this class:
-    # _file -- the open file with methods read(), close(), and seek()
-    #       set through the __init__() method
-    # _nchannels -- the number of audio channels
-    #       available through the getnchannels() method
-    # _nframes -- the number of audio frames
-    #       available through the getnframes() method
-    # _sampwidth -- the number of bytes per audio sample
-    #       available through the getsampwidth() method
-    # _framerate -- the sampling frequency
-    #       available through the getframerate() method
-    # _comptype -- the AIFF-C compression type ('NONE' if AIFF)
-    #       available through the getcomptype() method
-    # _compname -- the human-readable AIFF-C compression type
-    #       available through the getcomptype() method
-    # _markers -- the marks in the audio file
-    #       available through the getmarkers() and getmark()
-    #       methods
-    # _soundpos -- the position in the audio stream
-    #       available through the tell() method, set through the
-    #       setpos() method
-    #
-    # These variables are used internally only:
-    # _version -- the AIFF-C version number
-    # _decomp -- the decompressor from builtin module cl
-    # _comm_chunk_read -- 1 iff the COMM chunk has been read
-    # _aifc -- 1 iff reading an AIFF-C file
-    # _ssnd_seek_needed -- 1 iff positioned correctly in audio
-    #       file for readframes()
-    # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
-    # _framesize -- size of one frame in the file
-
-    _file = None  # Set here since __del__ checks it
-
-    def initfp(self, file):
-        self._version = 0
-        self._convert = None
-        self._markers = []
-        self._soundpos = 0
-        self._file = file
-        chunk = Chunk(file)
-        if chunk.getname() != b'FORM':
-            raise Error('file does not start with FORM id')
-        formdata = chunk.read(4)
-        if formdata == b'AIFF':
-            self._aifc = 0
-        elif formdata == b'AIFC':
-            self._aifc = 1
-        else:
-            raise Error('not an AIFF or AIFF-C file')
-        self._comm_chunk_read = 0
-        self._ssnd_chunk = None
-        while 1:
-            self._ssnd_seek_needed = 1
-            try:
-                chunk = Chunk(self._file)
-            except EOFError:
-                break
-            chunkname = chunk.getname()
-            if chunkname == b'COMM':
-                self._read_comm_chunk(chunk)
-                self._comm_chunk_read = 1
-            elif chunkname == b'SSND':
-                self._ssnd_chunk = chunk
-                dummy = chunk.read(8)
-                self._ssnd_seek_needed = 0
-            elif chunkname == b'FVER':
-                self._version = _read_ulong(chunk)
-            elif chunkname == b'MARK':
-                self._readmark(chunk)
-            chunk.skip()
-        if not self._comm_chunk_read or not self._ssnd_chunk:
-            raise Error('COMM chunk and/or SSND chunk missing')
-
-    def __init__(self, f):
-        if isinstance(f, str):
-            file_object = builtins.open(f, 'rb')
-            try:
-                self.initfp(file_object)
-            except:
-                file_object.close()
-                raise
-        else:
-            # assume it is an open file object already
-            self.initfp(f)
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        self.close()
-
-    #
-    # User visible methods.
-    #
-    def getfp(self):
-        return self._file
-
-    def rewind(self):
-        self._ssnd_seek_needed = 1
-        self._soundpos = 0
-
-    def close(self):
-        file = self._file
-        if file is not None:
-            self._file = None
-            file.close()
-
-    def tell(self):
-        return self._soundpos
-
-    def getnchannels(self):
-        return self._nchannels
-
-    def getnframes(self):
-        return self._nframes
-
-    def getsampwidth(self):
-        return self._sampwidth
-
-    def getframerate(self):
-        return self._framerate
-
-    def getcomptype(self):
-        return self._comptype
-
-    def getcompname(self):
-        return self._compname
-
-##  def getversion(self):
-##      return self._version
-
-    def getparams(self):
-        return _aifc_params(self.getnchannels(), self.getsampwidth(),
-                            self.getframerate(), self.getnframes(),
-                            self.getcomptype(), self.getcompname())
-
-    def getmarkers(self):
-        if len(self._markers) == 0:
-            return None
-        return self._markers
-
-    def getmark(self, id):
-        for marker in self._markers:
-            if id == marker[0]:
-                return marker
-        raise Error('marker {0!r} does not exist'.format(id))
-
-    def setpos(self, pos):
-        if pos < 0 or pos > self._nframes:
-            raise Error('position not in range')
-        self._soundpos = pos
-        self._ssnd_seek_needed = 1
-
-    def readframes(self, nframes):
-        if self._ssnd_seek_needed:
-            self._ssnd_chunk.seek(0)
-            dummy = self._ssnd_chunk.read(8)
-            pos = self._soundpos * self._framesize
-            if pos:
-                self._ssnd_chunk.seek(pos + 8)
-            self._ssnd_seek_needed = 0
-        if nframes == 0:
-            return b''
-        data = self._ssnd_chunk.read(nframes * self._framesize)
-        if self._convert and data:
-            data = self._convert(data)
-        self._soundpos = self._soundpos + len(data) // (self._nchannels
-                                                        * self._sampwidth)
-        return data
-
-    #
-    # Internal methods.
-    #
-
-    def _alaw2lin(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.alaw2lin(data, 2)
-
-    def _ulaw2lin(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.ulaw2lin(data, 2)
-
-    def _adpcm2lin(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        if not hasattr(self, '_adpcmstate'):
-            # first time
-            self._adpcmstate = None
-        data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
-        return data
-
-    def _sowt2lin(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.byteswap(data, 2)
-
-    def _read_comm_chunk(self, chunk):
-        self._nchannels = _read_short(chunk)
-        self._nframes = _read_long(chunk)
-        self._sampwidth = (_read_short(chunk) + 7) // 8
-        self._framerate = int(_read_float(chunk))
-        if self._sampwidth <= 0:
-            raise Error('bad sample width')
-        if self._nchannels <= 0:
-            raise Error('bad # of channels')
-        self._framesize = self._nchannels * self._sampwidth
-        if self._aifc:
-            #DEBUG: SGI's soundeditor produces a bad size :-(
-            kludge = 0
-            if chunk.chunksize == 18:
-                kludge = 1
-                warnings.warn('Warning: bad COMM chunk size')
-                chunk.chunksize = 23
-            #DEBUG end
-            self._comptype = chunk.read(4)
-            #DEBUG start
-            if kludge:
-                length = ord(chunk.file.read(1))
-                if length & 1 == 0:
-                    length = length + 1
-                chunk.chunksize = chunk.chunksize + length
-                chunk.file.seek(-1, 1)
-            #DEBUG end
-            self._compname = _read_string(chunk)
-            if self._comptype != b'NONE':
-                if self._comptype == b'G722':
-                    self._convert = self._adpcm2lin
-                elif self._comptype in (b'ulaw', b'ULAW'):
-                    self._convert = self._ulaw2lin
-                elif self._comptype in (b'alaw', b'ALAW'):
-                    self._convert = self._alaw2lin
-                elif self._comptype in (b'sowt', b'SOWT'):
-                    self._convert = self._sowt2lin
-                else:
-                    raise Error('unsupported compression type')
-                self._sampwidth = 2
-        else:
-            self._comptype = b'NONE'
-            self._compname = b'not compressed'
-
-    def _readmark(self, chunk):
-        nmarkers = _read_short(chunk)
-        # Some files appear to contain invalid counts.
-        # Cope with this by testing for EOF.
-        try:
-            for i in range(nmarkers):
-                id = _read_short(chunk)
-                pos = _read_long(chunk)
-                name = _read_string(chunk)
-                if pos or name:
-                    # some files appear to have
-                    # dummy markers consisting of
-                    # a position 0 and name ''
-                    self._markers.append((id, pos, name))
-        except EOFError:
-            w = ('Warning: MARK chunk contains only %s marker%s instead of %s' %
-                 (len(self._markers), '' if len(self._markers) == 1 else 's',
-                  nmarkers))
-            warnings.warn(w)
-
-class Aifc_write:
-    # Variables used in this class:
-    #
-    # These variables are user settable through appropriate methods
-    # of this class:
-    # _file -- the open file with methods write(), close(), tell(), seek()
-    #       set through the __init__() method
-    # _comptype -- the AIFF-C compression type ('NONE' in AIFF)
-    #       set through the setcomptype() or setparams() method
-    # _compname -- the human-readable AIFF-C compression type
-    #       set through the setcomptype() or setparams() method
-    # _nchannels -- the number of audio channels
-    #       set through the setnchannels() or setparams() method
-    # _sampwidth -- the number of bytes per audio sample
-    #       set through the setsampwidth() or setparams() method
-    # _framerate -- the sampling frequency
-    #       set through the setframerate() or setparams() method
-    # _nframes -- the number of audio frames written to the header
-    #       set through the setnframes() or setparams() method
-    # _aifc -- whether we're writing an AIFF-C file or an AIFF file
-    #       set through the aifc() method, reset through the
-    #       aiff() method
-    #
-    # These variables are used internally only:
-    # _version -- the AIFF-C version number
-    # _comp -- the compressor from builtin module cl
-    # _nframeswritten -- the number of audio frames actually written
-    # _datalength -- the size of the audio samples written to the header
-    # _datawritten -- the size of the audio samples actually written
-
-    _file = None  # Set here since __del__ checks it
-
-    def __init__(self, f):
-        if isinstance(f, str):
-            file_object = builtins.open(f, 'wb')
-            try:
-                self.initfp(file_object)
-            except:
-                file_object.close()
-                raise
-
-            # treat .aiff file extensions as non-compressed audio
-            if f.endswith('.aiff'):
-                self._aifc = 0
-        else:
-            # assume it is an open file object already
-            self.initfp(f)
-
-    def initfp(self, file):
-        self._file = file
-        self._version = _AIFC_version
-        self._comptype = b'NONE'
-        self._compname = b'not compressed'
-        self._convert = None
-        self._nchannels = 0
-        self._sampwidth = 0
-        self._framerate = 0
-        self._nframes = 0
-        self._nframeswritten = 0
-        self._datawritten = 0
-        self._datalength = 0
-        self._markers = []
-        self._marklength = 0
-        self._aifc = 1      # AIFF-C is default
-
-    def __del__(self):
-        self.close()
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        self.close()
-
-    #
-    # User visible methods.
-    #
-    def aiff(self):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        self._aifc = 0
-
-    def aifc(self):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        self._aifc = 1
-
-    def setnchannels(self, nchannels):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        if nchannels < 1:
-            raise Error('bad # of channels')
-        self._nchannels = nchannels
-
-    def getnchannels(self):
-        if not self._nchannels:
-            raise Error('number of channels not set')
-        return self._nchannels
-
-    def setsampwidth(self, sampwidth):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        if sampwidth < 1 or sampwidth > 4:
-            raise Error('bad sample width')
-        self._sampwidth = sampwidth
-
-    def getsampwidth(self):
-        if not self._sampwidth:
-            raise Error('sample width not set')
-        return self._sampwidth
-
-    def setframerate(self, framerate):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        if framerate <= 0:
-            raise Error('bad frame rate')
-        self._framerate = framerate
-
-    def getframerate(self):
-        if not self._framerate:
-            raise Error('frame rate not set')
-        return self._framerate
-
-    def setnframes(self, nframes):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        self._nframes = nframes
-
-    def getnframes(self):
-        return self._nframeswritten
-
-    def setcomptype(self, comptype, compname):
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        if comptype not in (b'NONE', b'ulaw', b'ULAW',
-                            b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
-            raise Error('unsupported compression type')
-        self._comptype = comptype
-        self._compname = compname
-
-    def getcomptype(self):
-        return self._comptype
-
-    def getcompname(self):
-        return self._compname
-
-##  def setversion(self, version):
-##      if self._nframeswritten:
-##          raise Error, 'cannot change parameters after starting to write'
-##      self._version = version
-
-    def setparams(self, params):
-        nchannels, sampwidth, framerate, nframes, comptype, compname = params
-        if self._nframeswritten:
-            raise Error('cannot change parameters after starting to write')
-        if comptype not in (b'NONE', b'ulaw', b'ULAW',
-                            b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
-            raise Error('unsupported compression type')
-        self.setnchannels(nchannels)
-        self.setsampwidth(sampwidth)
-        self.setframerate(framerate)
-        self.setnframes(nframes)
-        self.setcomptype(comptype, compname)
-
-    def getparams(self):
-        if not self._nchannels or not self._sampwidth or not self._framerate:
-            raise Error('not all parameters set')
-        return _aifc_params(self._nchannels, self._sampwidth, self._framerate,
-                            self._nframes, self._comptype, self._compname)
-
-    def setmark(self, id, pos, name):
-        if id <= 0:
-            raise Error('marker ID must be > 0')
-        if pos < 0:
-            raise Error('marker position must be >= 0')
-        if not isinstance(name, bytes):
-            raise Error('marker name must be bytes')
-        for i in range(len(self._markers)):
-            if id == self._markers[i][0]:
-                self._markers[i] = id, pos, name
-                return
-        self._markers.append((id, pos, name))
-
-    def getmark(self, id):
-        for marker in self._markers:
-            if id == marker[0]:
-                return marker
-        raise Error('marker {0!r} does not exist'.format(id))
-
-    def getmarkers(self):
-        if len(self._markers) == 0:
-            return None
-        return self._markers
-
-    def tell(self):
-        return self._nframeswritten
-
-    def writeframesraw(self, data):
-        if not isinstance(data, (bytes, bytearray)):
-            data = memoryview(data).cast('B')
-        self._ensure_header_written(len(data))
-        nframes = len(data) // (self._sampwidth * self._nchannels)
-        if self._convert:
-            data = self._convert(data)
-        self._file.write(data)
-        self._nframeswritten = self._nframeswritten + nframes
-        self._datawritten = self._datawritten + len(data)
-
-    def writeframes(self, data):
-        self.writeframesraw(data)
-        if self._nframeswritten != self._nframes or \
-              self._datalength != self._datawritten:
-            self._patchheader()
-
-    def close(self):
-        if self._file is None:
-            return
-        try:
-            self._ensure_header_written(0)
-            if self._datawritten & 1:
-                # quick pad to even size
-                self._file.write(b'\x00')
-                self._datawritten = self._datawritten + 1
-            self._writemarkers()
-            if self._nframeswritten != self._nframes or \
-                  self._datalength != self._datawritten or \
-                  self._marklength:
-                self._patchheader()
-        finally:
-            # Prevent ref cycles
-            self._convert = None
-            f = self._file
-            self._file = None
-            f.close()
-
-    #
-    # Internal methods.
-    #
-
-    def _lin2alaw(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.lin2alaw(data, 2)
-
-    def _lin2ulaw(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.lin2ulaw(data, 2)
-
-    def _lin2adpcm(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        if not hasattr(self, '_adpcmstate'):
-            self._adpcmstate = None
-        data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
-        return data
-
-    def _lin2sowt(self, data):
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', category=DeprecationWarning)
-            import audioop
-        return audioop.byteswap(data, 2)
-
-    def _ensure_header_written(self, datasize):
-        if not self._nframeswritten:
-            if self._comptype in (b'ULAW', b'ulaw',
-                b'ALAW', b'alaw', b'G722',
-                b'sowt', b'SOWT'):
-                if not self._sampwidth:
-                    self._sampwidth = 2
-                if self._sampwidth != 2:
-                    raise Error('sample width must be 2 when compressing '
-                                'with ulaw/ULAW, alaw/ALAW, sowt/SOWT '
-                                'or G7.22 (ADPCM)')
-            if not self._nchannels:
-                raise Error('# channels not specified')
-            if not self._sampwidth:
-                raise Error('sample width not specified')
-            if not self._framerate:
-                raise Error('sampling rate not specified')
-            self._write_header(datasize)
-
-    def _init_compression(self):
-        if self._comptype == b'G722':
-            self._convert = self._lin2adpcm
-        elif self._comptype in (b'ulaw', b'ULAW'):
-            self._convert = self._lin2ulaw
-        elif self._comptype in (b'alaw', b'ALAW'):
-            self._convert = self._lin2alaw
-        elif self._comptype in (b'sowt', b'SOWT'):
-            self._convert = self._lin2sowt
-
-    def _write_header(self, initlength):
-        if self._aifc and self._comptype != b'NONE':
-            self._init_compression()
-        self._file.write(b'FORM')
-        if not self._nframes:
-            self._nframes = initlength // (self._nchannels * self._sampwidth)
-        self._datalength = self._nframes * self._nchannels * self._sampwidth
-        if self._datalength & 1:
-            self._datalength = self._datalength + 1
-        if self._aifc:
-            if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'):
-                self._datalength = self._datalength // 2
-                if self._datalength & 1:
-                    self._datalength = self._datalength + 1
-            elif self._comptype == b'G722':
-                self._datalength = (self._datalength + 3) // 4
-                if self._datalength & 1:
-                    self._datalength = self._datalength + 1
-        try:
-            self._form_length_pos = self._file.tell()
-        except (AttributeError, OSError):
-            self._form_length_pos = None
-        commlength = self._write_form_length(self._datalength)
-        if self._aifc:
-            self._file.write(b'AIFC')
-            self._file.write(b'FVER')
-            _write_ulong(self._file, 4)
-            _write_ulong(self._file, self._version)
-        else:
-            self._file.write(b'AIFF')
-        self._file.write(b'COMM')
-        _write_ulong(self._file, commlength)
-        _write_short(self._file, self._nchannels)
-        if self._form_length_pos is not None:
-            self._nframes_pos = self._file.tell()
-        _write_ulong(self._file, self._nframes)
-        if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
-            _write_short(self._file, 8)
-        else:
-            _write_short(self._file, self._sampwidth * 8)
-        _write_float(self._file, self._framerate)
-        if self._aifc:
-            self._file.write(self._comptype)
-            _write_string(self._file, self._compname)
-        self._file.write(b'SSND')
-        if self._form_length_pos is not None:
-            self._ssnd_length_pos = self._file.tell()
-        _write_ulong(self._file, self._datalength + 8)
-        _write_ulong(self._file, 0)
-        _write_ulong(self._file, 0)
-
-    def _write_form_length(self, datalength):
-        if self._aifc:
-            commlength = 18 + 5 + len(self._compname)
-            if commlength & 1:
-                commlength = commlength + 1
-            verslength = 12
-        else:
-            commlength = 18
-            verslength = 0
-        _write_ulong(self._file, 4 + verslength + self._marklength + \
-                     8 + commlength + 16 + datalength)
-        return commlength
-
-    def _patchheader(self):
-        curpos = self._file.tell()
-        if self._datawritten & 1:
-            datalength = self._datawritten + 1
-            self._file.write(b'\x00')
-        else:
-            datalength = self._datawritten
-        if datalength == self._datalength and \
-              self._nframes == self._nframeswritten and \
-              self._marklength == 0:
-            self._file.seek(curpos, 0)
-            return
-        self._file.seek(self._form_length_pos, 0)
-        dummy = self._write_form_length(datalength)
-        self._file.seek(self._nframes_pos, 0)
-        _write_ulong(self._file, self._nframeswritten)
-        self._file.seek(self._ssnd_length_pos, 0)
-        _write_ulong(self._file, datalength + 8)
-        self._file.seek(curpos, 0)
-        self._nframes = self._nframeswritten
-        self._datalength = datalength
-
-    def _writemarkers(self):
-        if len(self._markers) == 0:
-            return
-        self._file.write(b'MARK')
-        length = 2
-        for marker in self._markers:
-            id, pos, name = marker
-            length = length + len(name) + 1 + 6
-            if len(name) & 1 == 0:
-                length = length + 1
-        _write_ulong(self._file, length)
-        self._marklength = length + 8
-        _write_short(self._file, len(self._markers))
-        for marker in self._markers:
-            id, pos, name = marker
-            _write_short(self._file, id)
-            _write_ulong(self._file, pos)
-            _write_string(self._file, name)
-
-def open(f, mode=None):
-    if mode is None:
-        if hasattr(f, 'mode'):
-            mode = f.mode
-        else:
-            mode = 'rb'
-    if mode in ('r', 'rb'):
-        return Aifc_read(f)
-    elif mode in ('w', 'wb'):
-        return Aifc_write(f)
-    else:
-        raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
-
-
-if __name__ == '__main__':
-    import sys
-    if not sys.argv[1:]:
-        sys.argv.append('/usr/demos/data/audio/bach.aiff')
-    fn = sys.argv[1]
-    with open(fn, 'r') as f:
-        print("Reading", fn)
-        print("nchannels =", f.getnchannels())
-        print("nframes   =", f.getnframes())
-        print("sampwidth =", f.getsampwidth())
-        print("framerate =", f.getframerate())
-        print("comptype  =", f.getcomptype())
-        print("compname  =", f.getcompname())
-        if sys.argv[2:]:
-            gn = sys.argv[2]
-            print("Writing", gn)
-            with open(gn, 'w') as g:
-                g.setparams(f.getparams())
-                while 1:
-                    data = f.readframes(1024)
-                    if not data:
-                        break
-                    g.writeframes(data)
-            print("Done.")

Lib/ast.py

@@ -1,28 +1,24 @@
 """
-    ast
-    ~~~
+The `ast` module helps Python applications to process trees of the Python
+abstract syntax grammar.  The abstract syntax itself might change with
+each Python release; this module helps to find out programmatically what
+the current grammar looks like and allows modifications of it.
 
-    The `ast` module helps Python applications to process trees of the Python
-    abstract syntax grammar.  The abstract syntax itself might change with
-    each Python release; this module helps to find out programmatically what
-    the current grammar looks like and allows modifications of it.
+An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
+a flag to the `compile()` builtin function or by using the `parse()`
+function from this module.  The result will be a tree of objects whose
+classes all inherit from `ast.AST`.
 
-    An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
-    a flag to the `compile()` builtin function or by using the `parse()`
-    function from this module.  The result will be a tree of objects whose
-    classes all inherit from `ast.AST`.
+A modified abstract syntax tree can be compiled into a Python code object
+using the built-in `compile()` function.
 
-    A modified abstract syntax tree can be compiled into a Python code object
-    using the built-in `compile()` function.
+Additionally various helper functions are provided that make working with
+the trees simpler.  The main intention of the helper functions and this
+module in general is to provide an easy to use interface for libraries
+that work tightly with the python syntax (template engines for example).
 
-    Additionally various helper functions are provided that make working with
-    the trees simpler.  The main intention of the helper functions and this
-    module in general is to provide an easy to use interface for libraries
-    that work tightly with the python syntax (template engines for example).
-
-
-    :copyright: Copyright 2008 by Armin Ronacher.
-    :license: Python License.
+:copyright: Copyright 2008 by Armin Ronacher.
+:license: Python License.
 """
 import sys
 import re
@@ -32,13 +28,15 @@ from enum import IntEnum, auto, _simple_enum
 
 
 def parse(source, filename='<unknown>', mode='exec', *,
-          type_comments=False, feature_version=None):
+          type_comments=False, feature_version=None, optimize=-1):
     """
     Parse the source into an AST node.
     Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
     Pass type_comments=True to get back type comments where the syntax allows.
     """
     flags = PyCF_ONLY_AST
+    if optimize > 0:
+        flags |= PyCF_OPTIMIZED_AST
     if type_comments:
         flags |= PyCF_TYPE_COMMENTS
     if feature_version is None:
@@ -50,7 +48,7 @@ def parse(source, filename='<unknown>', mode='exec', *,
         feature_version = minor
     # Else it should be an int giving the minor version for 3.x.
     return compile(source, filename, mode, flags,
-                   _feature_version=feature_version)
+                   _feature_version=feature_version, optimize=optimize)
 
 
 def literal_eval(node_or_string):
@@ -112,7 +110,11 @@ def literal_eval(node_or_string):
     return _convert(node_or_string)
 
 
-def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
+def dump(
+    node, annotate_fields=True, include_attributes=False,
+    *,
+    indent=None, show_empty=False,
+):
     """
     Return a formatted dump of the tree in node.  This is mainly useful for
     debugging purposes.  If annotate_fields is true (by default),
@@ -123,6 +125,8 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
     include_attributes can be set to true.  If indent is a non-negative
     integer or string, then the tree will be pretty-printed with that indent
     level. None (the default) selects the single line representation.
+    If show_empty is False, then empty lists and fields that are None
+    will be omitted from the output for better readability.
     """
     def _format(node, level=0):
         if indent is not None:
@@ -135,6 +139,7 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
         if isinstance(node, AST):
             cls = type(node)
             args = []
+            args_buffer = []
             allsimple = True
             keywords = annotate_fields
             for name in node._fields:
@@ -146,6 +151,16 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
                 if value is None and getattr(cls, name, ...) is None:
                     keywords = True
                     continue
+                if not show_empty:
+                    if value == []:
+                        field_type = cls._field_types.get(name, object)
+                        if getattr(field_type, '__origin__', ...) is list:
+                            if not keywords:
+                                args_buffer.append(repr(value))
+                            continue
+                    if not keywords:
+                        args.extend(args_buffer)
+                        args_buffer = []
                 value, simple = _format(value, level)
                 allsimple = allsimple and simple
                 if keywords:
@@ -726,12 +741,11 @@ class _Unparser(NodeVisitor):
     output source code for the abstract syntax; original formatting
     is disregarded."""
 
-    def __init__(self, *, _avoid_backslashes=False):
+    def __init__(self):
         self._source = []
         self._precedences = {}
         self._type_ignores = {}
         self._indent = 0
-        self._avoid_backslashes = _avoid_backslashes
         self._in_try_star = False
 
     def interleave(self, inter, f, seq):
@@ -1104,12 +1118,21 @@ class _Unparser(NodeVisitor):
         if node.bound:
             self.write(": ")
             self.traverse(node.bound)
+        if node.default_value:
+            self.write(" = ")
+            self.traverse(node.default_value)
 
     def visit_TypeVarTuple(self, node):
         self.write("*" + node.name)
+        if node.default_value:
+            self.write(" = ")
+            self.traverse(node.default_value)
 
     def visit_ParamSpec(self, node):
         self.write("**" + node.name)
+        if node.default_value:
+            self.write(" = ")
+            self.traverse(node.default_value)
 
     def visit_TypeAlias(self, node):
         self.fill("type ")
@@ -1246,9 +1269,14 @@ class _Unparser(NodeVisitor):
                     fallback_to_repr = True
                     break
                 quote_types = new_quote_types
-            elif "\n" in value:
-                quote_types = [q for q in quote_types if q in _MULTI_QUOTES]
-                assert quote_types
+            else:
+                if "\n" in value:
+                    quote_types = [q for q in quote_types if q in _MULTI_QUOTES]
+                    assert quote_types
+
+                new_quote_types = [q for q in quote_types if q not in value]
+                if new_quote_types:
+                    quote_types = new_quote_types
             new_fstring_parts.append(value)
 
         if fallback_to_repr:
@@ -1268,13 +1296,19 @@ class _Unparser(NodeVisitor):
         quote_type = quote_types[0]
         self.write(f"{quote_type}{value}{quote_type}")
 
-    def _write_fstring_inner(self, node):
+    def _write_fstring_inner(self, node, is_format_spec=False):
         if isinstance(node, JoinedStr):
             # for both the f-string itself, and format_spec
             for value in node.values:
-                self._write_fstring_inner(value)
+                self._write_fstring_inner(value, is_format_spec=is_format_spec)
         elif isinstance(node, Constant) and isinstance(node.value, str):
             value = node.value.replace("{", "{{").replace("}", "}}")
+
+            if is_format_spec:
+                value = value.replace("\\", "\\\\")
+                value = value.replace("'", "\\'")
+                value = value.replace('"', '\\"')
+                value = value.replace("\n", "\\n")
             self.write(value)
         elif isinstance(node, FormattedValue):
             self.visit_FormattedValue(node)
@@ -1297,7 +1331,7 @@ class _Unparser(NodeVisitor):
                 self.write(f"!{chr(node.conversion)}")
             if node.format_spec:
                 self.write(":")
-                self._write_fstring_inner(node.format_spec)
+                self._write_fstring_inner(node.format_spec, is_format_spec=True)
 
     def visit_Name(self, node):
         self.write(node.id)
@@ -1317,8 +1351,6 @@ class _Unparser(NodeVisitor):
                 .replace("inf", _INFSTR)
                 .replace("nan", f"({_INFSTR}-{_INFSTR})")
             )
-        elif self._avoid_backslashes and isinstance(value, str):
-            self._write_str_avoiding_backslashes(value)
         else:
             self.write(repr(value))
 
@@ -1805,8 +1837,7 @@ def main():
     import argparse
 
     parser = argparse.ArgumentParser(prog='python -m ast')
-    parser.add_argument('infile', type=argparse.FileType(mode='rb'), nargs='?',
-                        default='-',
+    parser.add_argument('infile', nargs='?', default='-',
                         help='the file to parse; defaults to stdin')
     parser.add_argument('-m', '--mode', default='exec',
                         choices=('exec', 'single', 'eval', 'func_type'),
@@ -1820,9 +1851,14 @@ def main():
                         help='indentation of nodes (number of spaces)')
     args = parser.parse_args()
 
-    with args.infile as infile:
-        source = infile.read()
-    tree = parse(source, args.infile.name, args.mode, type_comments=args.no_type_comments)
+    if args.infile == '-':
+        name = '<stdin>'
+        source = sys.stdin.buffer.read()
+    else:
+        name = args.infile
+        with open(args.infile, 'rb') as infile:
+            source = infile.read()
+    tree = parse(source, name, args.mode, type_comments=args.no_type_comments)
     print(dump(tree, include_attributes=args.include_attributes, indent=args.indent))
 
 if __name__ == '__main__':

Lib/asynchat.py

@@ -1,307 +0,0 @@
-# -*- Mode: Python; tab-width: 4 -*-
-#       Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp
-#       Author: Sam Rushing <rushing@nightmare.com>
-
-# ======================================================================
-# Copyright 1996 by Sam Rushing
-#
-#                         All Rights Reserved
-#
-# Permission to use, copy, modify, and distribute this software and
-# its documentation for any purpose and without fee is hereby
-# granted, provided that the above copyright notice appear in all
-# copies and that both that copyright notice and this permission
-# notice appear in supporting documentation, and that the name of Sam
-# Rushing not be used in advertising or publicity pertaining to
-# distribution of the software without specific, written prior
-# permission.
-#
-# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
-# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
-# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
-# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
-# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-# ======================================================================
-
-r"""A class supporting chat-style (command/response) protocols.
-
-This class adds support for 'chat' style protocols - where one side
-sends a 'command', and the other sends a response (examples would be
-the common internet protocols - smtp, nntp, ftp, etc..).
-
-The handle_read() method looks at the input stream for the current
-'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n'
-for multi-line output), calling self.found_terminator() on its
-receipt.
-
-for example:
-Say you build an async nntp client using this class.  At the start
-of the connection, you'll have self.terminator set to '\r\n', in
-order to process the single-line greeting.  Just before issuing a
-'LIST' command you'll set it to '\r\n.\r\n'.  The output of the LIST
-command will be accumulated (using your own 'collect_incoming_data'
-method) up to the terminator, and then control will be returned to
-you - by calling your self.found_terminator() method.
-"""
-import asyncore
-from collections import deque
-
-
-class async_chat(asyncore.dispatcher):
-    """This is an abstract class.  You must derive from this class, and add
-    the two methods collect_incoming_data() and found_terminator()"""
-
-    # these are overridable defaults
-
-    ac_in_buffer_size = 65536
-    ac_out_buffer_size = 65536
-
-    # we don't want to enable the use of encoding by default, because that is a
-    # sign of an application bug that we don't want to pass silently
-
-    use_encoding = 0
-    encoding = 'latin-1'
-
-    def __init__(self, sock=None, map=None):
-        # for string terminator matching
-        self.ac_in_buffer = b''
-
-        # we use a list here rather than io.BytesIO for a few reasons...
-        # del lst[:] is faster than bio.truncate(0)
-        # lst = [] is faster than bio.truncate(0)
-        self.incoming = []
-
-        # we toss the use of the "simple producer" and replace it with
-        # a pure deque, which the original fifo was a wrapping of
-        self.producer_fifo = deque()
-        asyncore.dispatcher.__init__(self, sock, map)
-
-    def collect_incoming_data(self, data):
-        raise NotImplementedError("must be implemented in subclass")
-
-    def _collect_incoming_data(self, data):
-        self.incoming.append(data)
-
-    def _get_data(self):
-        d = b''.join(self.incoming)
-        del self.incoming[:]
-        return d
-
-    def found_terminator(self):
-        raise NotImplementedError("must be implemented in subclass")
-
-    def set_terminator(self, term):
-        """Set the input delimiter.
-
-        Can be a fixed string of any length, an integer, or None.
-        """
-        if isinstance(term, str) and self.use_encoding:
-            term = bytes(term, self.encoding)
-        elif isinstance(term, int) and term < 0:
-            raise ValueError('the number of received bytes must be positive')
-        self.terminator = term
-
-    def get_terminator(self):
-        return self.terminator
-
-    # grab some more data from the socket,
-    # throw it to the collector method,
-    # check for the terminator,
-    # if found, transition to the next state.
-
-    def handle_read(self):
-
-        try:
-            data = self.recv(self.ac_in_buffer_size)
-        except BlockingIOError:
-            return
-        except OSError as why:
-            self.handle_error()
-            return
-
-        if isinstance(data, str) and self.use_encoding:
-            data = bytes(str, self.encoding)
-        self.ac_in_buffer = self.ac_in_buffer + data
-
-        # Continue to search for self.terminator in self.ac_in_buffer,
-        # while calling self.collect_incoming_data.  The while loop
-        # is necessary because we might read several data+terminator
-        # combos with a single recv(4096).
-
-        while self.ac_in_buffer:
-            lb = len(self.ac_in_buffer)
-            terminator = self.get_terminator()
-            if not terminator:
-                # no terminator, collect it all
-                self.collect_incoming_data(self.ac_in_buffer)
-                self.ac_in_buffer = b''
-            elif isinstance(terminator, int):
-                # numeric terminator
-                n = terminator
-                if lb < n:
-                    self.collect_incoming_data(self.ac_in_buffer)
-                    self.ac_in_buffer = b''
-                    self.terminator = self.terminator - lb
-                else:
-                    self.collect_incoming_data(self.ac_in_buffer[:n])
-                    self.ac_in_buffer = self.ac_in_buffer[n:]
-                    self.terminator = 0
-                    self.found_terminator()
-            else:
-                # 3 cases:
-                # 1) end of buffer matches terminator exactly:
-                #    collect data, transition
-                # 2) end of buffer matches some prefix:
-                #    collect data to the prefix
-                # 3) end of buffer does not match any prefix:
-                #    collect data
-                terminator_len = len(terminator)
-                index = self.ac_in_buffer.find(terminator)
-                if index != -1:
-                    # we found the terminator
-                    if index > 0:
-                        # don't bother reporting the empty string
-                        # (source of subtle bugs)
-                        self.collect_incoming_data(self.ac_in_buffer[:index])
-                    self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
-                    # This does the Right Thing if the terminator
-                    # is changed here.
-                    self.found_terminator()
-                else:
-                    # check for a prefix of the terminator
-                    index = find_prefix_at_end(self.ac_in_buffer, terminator)
-                    if index:
-                        if index != lb:
-                            # we found a prefix, collect up to the prefix
-                            self.collect_incoming_data(self.ac_in_buffer[:-index])
-                            self.ac_in_buffer = self.ac_in_buffer[-index:]
-                        break
-                    else:
-                        # no prefix, collect it all
-                        self.collect_incoming_data(self.ac_in_buffer)
-                        self.ac_in_buffer = b''
-
-    def handle_write(self):
-        self.initiate_send()
-
-    def handle_close(self):
-        self.close()
-
-    def push(self, data):
-        if not isinstance(data, (bytes, bytearray, memoryview)):
-            raise TypeError('data argument must be byte-ish (%r)',
-                            type(data))
-        sabs = self.ac_out_buffer_size
-        if len(data) > sabs:
-            for i in range(0, len(data), sabs):
-                self.producer_fifo.append(data[i:i+sabs])
-        else:
-            self.producer_fifo.append(data)
-        self.initiate_send()
-
-    def push_with_producer(self, producer):
-        self.producer_fifo.append(producer)
-        self.initiate_send()
-
-    def readable(self):
-        "predicate for inclusion in the readable for select()"
-        # cannot use the old predicate, it violates the claim of the
-        # set_terminator method.
-
-        # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
-        return 1
-
-    def writable(self):
-        "predicate for inclusion in the writable for select()"
-        return self.producer_fifo or (not self.connected)
-
-    def close_when_done(self):
-        "automatically close this channel once the outgoing queue is empty"
-        self.producer_fifo.append(None)
-
-    def initiate_send(self):
-        while self.producer_fifo and self.connected:
-            first = self.producer_fifo[0]
-            # handle empty string/buffer or None entry
-            if not first:
-                del self.producer_fifo[0]
-                if first is None:
-                    self.handle_close()
-                    return
-
-            # handle classic producer behavior
-            obs = self.ac_out_buffer_size
-            try:
-                data = first[:obs]
-            except TypeError:
-                data = first.more()
-                if data:
-                    self.producer_fifo.appendleft(data)
-                else:
-                    del self.producer_fifo[0]
-                continue
-
-            if isinstance(data, str) and self.use_encoding:
-                data = bytes(data, self.encoding)
-
-            # send the data
-            try:
-                num_sent = self.send(data)
-            except OSError:
-                self.handle_error()
-                return
-
-            if num_sent:
-                if num_sent < len(data) or obs < len(first):
-                    self.producer_fifo[0] = first[num_sent:]
-                else:
-                    del self.producer_fifo[0]
-            # we tried to send some actual data
-            return
-
-    def discard_buffers(self):
-        # Emergencies only!
-        self.ac_in_buffer = b''
-        del self.incoming[:]
-        self.producer_fifo.clear()
-
-
-class simple_producer:
-
-    def __init__(self, data, buffer_size=512):
-        self.data = data
-        self.buffer_size = buffer_size
-
-    def more(self):
-        if len(self.data) > self.buffer_size:
-            result = self.data[:self.buffer_size]
-            self.data = self.data[self.buffer_size:]
-            return result
-        else:
-            result = self.data
-            self.data = b''
-            return result
-
-
-# Given 'haystack', see if any prefix of 'needle' is at its end.  This
-# assumes an exact match has already been checked.  Return the number of
-# characters matched.
-# for example:
-# f_p_a_e("qwerty\r", "\r\n") => 1
-# f_p_a_e("qwertydkjf", "\r\n") => 0
-# f_p_a_e("qwerty\r\n", "\r\n") => <undefined>
-
-# this could maybe be made faster with a computed regex?
-# [answer: no; circa Python-2.0, Jan 2001]
-# new python:   28961/s
-# old python:   18307/s
-# re:        12820/s
-# regex:     14035/s
-
-def find_prefix_at_end(haystack, needle):
-    l = len(needle) - 1
-    while l and not haystack.endswith(needle[:l]):
-        l -= 1
-    return l

Lib/asyncore.py

@@ -1,642 +0,0 @@
-# -*- Mode: Python -*-
-#   Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp
-#   Author: Sam Rushing <rushing@nightmare.com>
-
-# ======================================================================
-# Copyright 1996 by Sam Rushing
-#
-#                         All Rights Reserved
-#
-# Permission to use, copy, modify, and distribute this software and
-# its documentation for any purpose and without fee is hereby
-# granted, provided that the above copyright notice appear in all
-# copies and that both that copyright notice and this permission
-# notice appear in supporting documentation, and that the name of Sam
-# Rushing not be used in advertising or publicity pertaining to
-# distribution of the software without specific, written prior
-# permission.
-#
-# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
-# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
-# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
-# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
-# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-# ======================================================================
-
-"""Basic infrastructure for asynchronous socket service clients and servers.
-
-There are only two ways to have a program on a single processor do "more
-than one thing at a time".  Multi-threaded programming is the simplest and
-most popular way to do it, but there is another very different technique,
-that lets you have nearly all the advantages of multi-threading, without
-actually using multiple threads. it's really only practical if your program
-is largely I/O bound. If your program is CPU bound, then pre-emptive
-scheduled threads are probably what you really need. Network servers are
-rarely CPU-bound, however.
-
-If your operating system supports the select() system call in its I/O
-library (and nearly all do), then you can use it to juggle multiple
-communication channels at once; doing other work while your I/O is taking
-place in the "background."  Although this strategy can seem strange and
-complex, especially at first, it is in many ways easier to understand and
-control than multi-threaded programming. The module documented here solves
-many of the difficult problems for you, making the task of building
-sophisticated high-performance network servers and clients a snap.
-"""
-
-import select
-import socket
-import sys
-import time
-import warnings
-
-import os
-from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
-     ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
-     errorcode
-
-_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
-                           EBADF})
-
-try:
-    socket_map
-except NameError:
-    socket_map = {}
-
-def _strerror(err):
-    try:
-        return os.strerror(err)
-    except (ValueError, OverflowError, NameError):
-        if err in errorcode:
-            return errorcode[err]
-        return "Unknown error %s" %err
-
-class ExitNow(Exception):
-    pass
-
-_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit)
-
-def read(obj):
-    try:
-        obj.handle_read_event()
-    except _reraised_exceptions:
-        raise
-    except:
-        obj.handle_error()
-
-def write(obj):
-    try:
-        obj.handle_write_event()
-    except _reraised_exceptions:
-        raise
-    except:
-        obj.handle_error()
-
-def _exception(obj):
-    try:
-        obj.handle_expt_event()
-    except _reraised_exceptions:
-        raise
-    except:
-        obj.handle_error()
-
-def readwrite(obj, flags):
-    try:
-        if flags & select.POLLIN:
-            obj.handle_read_event()
-        if flags & select.POLLOUT:
-            obj.handle_write_event()
-        if flags & select.POLLPRI:
-            obj.handle_expt_event()
-        if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
-            obj.handle_close()
-    except OSError as e:
-        if e.args[0] not in _DISCONNECTED:
-            obj.handle_error()
-        else:
-            obj.handle_close()
-    except _reraised_exceptions:
-        raise
-    except:
-        obj.handle_error()
-
-def poll(timeout=0.0, map=None):
-    if map is None:
-        map = socket_map
-    if map:
-        r = []; w = []; e = []
-        for fd, obj in list(map.items()):
-            is_r = obj.readable()
-            is_w = obj.writable()
-            if is_r:
-                r.append(fd)
-            # accepting sockets should not be writable
-            if is_w and not obj.accepting:
-                w.append(fd)
-            if is_r or is_w:
-                e.append(fd)
-        if [] == r == w == e:
-            time.sleep(timeout)
-            return
-
-        r, w, e = select.select(r, w, e, timeout)
-
-        for fd in r:
-            obj = map.get(fd)
-            if obj is None:
-                continue
-            read(obj)
-
-        for fd in w:
-            obj = map.get(fd)
-            if obj is None:
-                continue
-            write(obj)
-
-        for fd in e:
-            obj = map.get(fd)
-            if obj is None:
-                continue
-            _exception(obj)
-
-def poll2(timeout=0.0, map=None):
-    # Use the poll() support added to the select module in Python 2.0
-    if map is None:
-        map = socket_map
-    if timeout is not None:
-        # timeout is in milliseconds
-        timeout = int(timeout*1000)
-    pollster = select.poll()
-    if map:
-        for fd, obj in list(map.items()):
-            flags = 0
-            if obj.readable():
-                flags |= select.POLLIN | select.POLLPRI
-            # accepting sockets should not be writable
-            if obj.writable() and not obj.accepting:
-                flags |= select.POLLOUT
-            if flags:
-                pollster.register(fd, flags)
-
-        r = pollster.poll(timeout)
-        for fd, flags in r:
-            obj = map.get(fd)
-            if obj is None:
-                continue
-            readwrite(obj, flags)
-
-poll3 = poll2                           # Alias for backward compatibility
-
-def loop(timeout=30.0, use_poll=False, map=None, count=None):
-    if map is None:
-        map = socket_map
-
-    if use_poll and hasattr(select, 'poll'):
-        poll_fun = poll2
-    else:
-        poll_fun = poll
-
-    if count is None:
-        while map:
-            poll_fun(timeout, map)
-
-    else:
-        while map and count > 0:
-            poll_fun(timeout, map)
-            count = count - 1
-
-class dispatcher:
-
-    debug = False
-    connected = False
-    accepting = False
-    connecting = False
-    closing = False
-    addr = None
-    ignore_log_types = frozenset({'warning'})
-
-    def __init__(self, sock=None, map=None):
-        if map is None:
-            self._map = socket_map
-        else:
-            self._map = map
-
-        self._fileno = None
-
-        if sock:
-            # Set to nonblocking just to make sure for cases where we
-            # get a socket from a blocking source.
-            sock.setblocking(0)
-            self.set_socket(sock, map)
-            self.connected = True
-            # The constructor no longer requires that the socket
-            # passed be connected.
-            try:
-                self.addr = sock.getpeername()
-            except OSError as err:
-                if err.args[0] in (ENOTCONN, EINVAL):
-                    # To handle the case where we got an unconnected
-                    # socket.
-                    self.connected = False
-                else:
-                    # The socket is broken in some unknown way, alert
-                    # the user and remove it from the map (to prevent
-                    # polling of broken sockets).
-                    self.del_channel(map)
-                    raise
-        else:
-            self.socket = None
-
-    def __repr__(self):
-        status = [self.__class__.__module__+"."+self.__class__.__qualname__]
-        if self.accepting and self.addr:
-            status.append('listening')
-        elif self.connected:
-            status.append('connected')
-        if self.addr is not None:
-            try:
-                status.append('%s:%d' % self.addr)
-            except TypeError:
-                status.append(repr(self.addr))
-        return '<%s at %#x>' % (' '.join(status), id(self))
-
-    def add_channel(self, map=None):
-        #self.log_info('adding channel %s' % self)
-        if map is None:
-            map = self._map
-        map[self._fileno] = self
-
-    def del_channel(self, map=None):
-        fd = self._fileno
-        if map is None:
-            map = self._map
-        if fd in map:
-            #self.log_info('closing channel %d:%s' % (fd, self))
-            del map[fd]
-        self._fileno = None
-
-    def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM):
-        self.family_and_type = family, type
-        sock = socket.socket(family, type)
-        sock.setblocking(0)
-        self.set_socket(sock)
-
-    def set_socket(self, sock, map=None):
-        self.socket = sock
-        self._fileno = sock.fileno()
-        self.add_channel(map)
-
-    def set_reuse_addr(self):
-        # try to re-use a server port if possible
-        try:
-            self.socket.setsockopt(
-                socket.SOL_SOCKET, socket.SO_REUSEADDR,
-                self.socket.getsockopt(socket.SOL_SOCKET,
-                                       socket.SO_REUSEADDR) | 1
-                )
-        except OSError:
-            pass
-
-    # ==================================================
-    # predicates for select()
-    # these are used as filters for the lists of sockets
-    # to pass to select().
-    # ==================================================
-
-    def readable(self):
-        return True
-
-    def writable(self):
-        return True
-
-    # ==================================================
-    # socket object methods.
-    # ==================================================
-
-    def listen(self, num):
-        self.accepting = True
-        if os.name == 'nt' and num > 5:
-            num = 5
-        return self.socket.listen(num)
-
-    def bind(self, addr):
-        self.addr = addr
-        return self.socket.bind(addr)
-
-    def connect(self, address):
-        self.connected = False
-        self.connecting = True
-        err = self.socket.connect_ex(address)
-        if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \
-        or err == EINVAL and os.name == 'nt':
-            self.addr = address
-            return
-        if err in (0, EISCONN):
-            self.addr = address
-            self.handle_connect_event()
-        else:
-            raise OSError(err, errorcode[err])
-
-    def accept(self):
-        # XXX can return either an address pair or None
-        try:
-            conn, addr = self.socket.accept()
-        except TypeError:
-            return None
-        except OSError as why:
-            if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
-                return None
-            else:
-                raise
-        else:
-            return conn, addr
-
-    def send(self, data):
-        try:
-            result = self.socket.send(data)
-            return result
-        except OSError as why:
-            if why.args[0] == EWOULDBLOCK:
-                return 0
-            elif why.args[0] in _DISCONNECTED:
-                self.handle_close()
-                return 0
-            else:
-                raise
-
-    def recv(self, buffer_size):
-        try:
-            data = self.socket.recv(buffer_size)
-            if not data:
-                # a closed connection is indicated by signaling
-                # a read condition, and having recv() return 0.
-                self.handle_close()
-                return b''
-            else:
-                return data
-        except OSError as why:
-            # winsock sometimes raises ENOTCONN
-            if why.args[0] in _DISCONNECTED:
-                self.handle_close()
-                return b''
-            else:
-                raise
-
-    def close(self):
-        self.connected = False
-        self.accepting = False
-        self.connecting = False
-        self.del_channel()
-        if self.socket is not None:
-            try:
-                self.socket.close()
-            except OSError as why:
-                if why.args[0] not in (ENOTCONN, EBADF):
-                    raise
-
-    # log and log_info may be overridden to provide more sophisticated
-    # logging and warning methods. In general, log is for 'hit' logging
-    # and 'log_info' is for informational, warning and error logging.
-
-    def log(self, message):
-        sys.stderr.write('log: %s\n' % str(message))
-
-    def log_info(self, message, type='info'):
-        if type not in self.ignore_log_types:
-            print('%s: %s' % (type, message))
-
-    def handle_read_event(self):
-        if self.accepting:
-            # accepting sockets are never connected, they "spawn" new
-            # sockets that are connected
-            self.handle_accept()
-        elif not self.connected:
-            if self.connecting:
-                self.handle_connect_event()
-            self.handle_read()
-        else:
-            self.handle_read()
-
-    def handle_connect_event(self):
-        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
-        if err != 0:
-            raise OSError(err, _strerror(err))
-        self.handle_connect()
-        self.connected = True
-        self.connecting = False
-
-    def handle_write_event(self):
-        if self.accepting:
-            # Accepting sockets shouldn't get a write event.
-            # We will pretend it didn't happen.
-            return
-
-        if not self.connected:
-            if self.connecting:
-                self.handle_connect_event()
-        self.handle_write()
-
-    def handle_expt_event(self):
-        # handle_expt_event() is called if there might be an error on the
-        # socket, or if there is OOB data
-        # check for the error condition first
-        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
-        if err != 0:
-            # we can get here when select.select() says that there is an
-            # exceptional condition on the socket
-            # since there is an error, we'll go ahead and close the socket
-            # like we would in a subclassed handle_read() that received no
-            # data
-            self.handle_close()
-        else:
-            self.handle_expt()
-
-    def handle_error(self):
-        nil, t, v, tbinfo = compact_traceback()
-
-        # sometimes a user repr method will crash.
-        try:
-            self_repr = repr(self)
-        except:
-            self_repr = '<__repr__(self) failed for object at %0x>' % id(self)
-
-        self.log_info(
-            'uncaptured python exception, closing channel %s (%s:%s %s)' % (
-                self_repr,
-                t,
-                v,
-                tbinfo
-                ),
-            'error'
-            )
-        self.handle_close()
-
-    def handle_expt(self):
-        self.log_info('unhandled incoming priority event', 'warning')
-
-    def handle_read(self):
-        self.log_info('unhandled read event', 'warning')
-
-    def handle_write(self):
-        self.log_info('unhandled write event', 'warning')
-
-    def handle_connect(self):
-        self.log_info('unhandled connect event', 'warning')
-
-    def handle_accept(self):
-        pair = self.accept()
-        if pair is not None:
-            self.handle_accepted(*pair)
-
-    def handle_accepted(self, sock, addr):
-        sock.close()
-        self.log_info('unhandled accepted event', 'warning')
-
-    def handle_close(self):
-        self.log_info('unhandled close event', 'warning')
-        self.close()
-
-# ---------------------------------------------------------------------------
-# adds simple buffered output capability, useful for simple clients.
-# [for more sophisticated usage use asynchat.async_chat]
-# ---------------------------------------------------------------------------
-
-class dispatcher_with_send(dispatcher):
-
-    def __init__(self, sock=None, map=None):
-        dispatcher.__init__(self, sock, map)
-        self.out_buffer = b''
-
-    def initiate_send(self):
-        num_sent = 0
-        num_sent = dispatcher.send(self, self.out_buffer[:65536])
-        self.out_buffer = self.out_buffer[num_sent:]
-
-    def handle_write(self):
-        self.initiate_send()
-
-    def writable(self):
-        return (not self.connected) or len(self.out_buffer)
-
-    def send(self, data):
-        if self.debug:
-            self.log_info('sending %s' % repr(data))
-        self.out_buffer = self.out_buffer + data
-        self.initiate_send()
-
-# ---------------------------------------------------------------------------
-# used for debugging.
-# ---------------------------------------------------------------------------
-
-def compact_traceback():
-    t, v, tb = sys.exc_info()
-    tbinfo = []
-    if not tb: # Must have a traceback
-        raise AssertionError("traceback does not exist")
-    while tb:
-        tbinfo.append((
-            tb.tb_frame.f_code.co_filename,
-            tb.tb_frame.f_code.co_name,
-            str(tb.tb_lineno)
-            ))
-        tb = tb.tb_next
-
-    # just to be safe
-    del tb
-
-    file, function, line = tbinfo[-1]
-    info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])
-    return (file, function, line), t, v, info
-
-def close_all(map=None, ignore_all=False):
-    if map is None:
-        map = socket_map
-    for x in list(map.values()):
-        try:
-            x.close()
-        except OSError as x:
-            if x.args[0] == EBADF:
-                pass
-            elif not ignore_all:
-                raise
-        except _reraised_exceptions:
-            raise
-        except:
-            if not ignore_all:
-                raise
-    map.clear()
-
-# Asynchronous File I/O:
-#
-# After a little research (reading man pages on various unixen, and
-# digging through the linux kernel), I've determined that select()
-# isn't meant for doing asynchronous file i/o.
-# Heartening, though - reading linux/mm/filemap.c shows that linux
-# supports asynchronous read-ahead.  So _MOST_ of the time, the data
-# will be sitting in memory for us already when we go to read it.
-#
-# What other OS's (besides NT) support async file i/o?  [VMS?]
-#
-# Regardless, this is useful for pipes, and stdin/stdout...
-
-if os.name == 'posix':
-    class file_wrapper:
-        # Here we override just enough to make a file
-        # look like a socket for the purposes of asyncore.
-        # The passed fd is automatically os.dup()'d
-
-        def __init__(self, fd):
-            self.fd = os.dup(fd)
-
-        def __del__(self):
-            if self.fd >= 0:
-                warnings.warn("unclosed file %r" % self, ResourceWarning,
-                              source=self)
-            self.close()
-
-        def recv(self, *args):
-            return os.read(self.fd, *args)
-
-        def send(self, *args):
-            return os.write(self.fd, *args)
-
-        def getsockopt(self, level, optname, buflen=None):
-            if (level == socket.SOL_SOCKET and
-                optname == socket.SO_ERROR and
-                not buflen):
-                return 0
-            raise NotImplementedError("Only asyncore specific behaviour "
-                                      "implemented.")
-
-        read = recv
-        write = send
-
-        def close(self):
-            if self.fd < 0:
-                return
-            fd = self.fd
-            self.fd = -1
-            os.close(fd)
-
-        def fileno(self):
-            return self.fd
-
-    class file_dispatcher(dispatcher):
-
-        def __init__(self, fd, map=None):
-            dispatcher.__init__(self, None, map)
-            self.connected = True
-            try:
-                fd = fd.fileno()
-            except AttributeError:
-                pass
-            self.set_file(fd)
-            # set it to non-blocking mode
-            os.set_blocking(fd, False)
-
-        def set_file(self, fd):
-            self.socket = file_wrapper(fd)
-            self._fileno = self.socket.fileno()
-            self.add_channel()

Lib/base64.py

@@ -18,7 +18,7 @@ __all__ = [
     'b64encode', 'b64decode', 'b32encode', 'b32decode',
     'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode',
     # Base85 and Ascii85 encodings
-    'b85encode', 'b85decode', 'a85encode', 'a85decode',
+    'b85encode', 'b85decode', 'a85encode', 'a85decode', 'z85encode', 'z85decode',
     # Standard Base64 encoding
     'standard_b64encode', 'standard_b64decode',
     # Some common Base64 alternatives.  As referenced by RFC 3458, see thread
@@ -164,7 +164,6 @@ _b32tab2 = {}
 _b32rev = {}
 
 def _b32encode(alphabet, s):
-    global _b32tab2
     # Delay the initialization of the table to not waste memory
     # if the function is never called
     if alphabet not in _b32tab2:
@@ -200,7 +199,6 @@ def _b32encode(alphabet, s):
     return bytes(encoded)
 
 def _b32decode(alphabet, s, casefold=False, map01=None):
-    global _b32rev
     # Delay the initialization of the table to not waste memory
     # if the function is never called
     if alphabet not in _b32rev:
@@ -334,7 +332,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
 
     wrapcol controls whether the output should have newline (b'\\n') characters
     added to it. If this is non-zero, each output line will be at most this
-    many characters long.
+    many characters long, excluding the trailing newline.
 
     pad controls whether the input is padded to a multiple of 4 before
     encoding. Note that the btoa implementation always pads.
@@ -499,6 +497,33 @@ def b85decode(b):
         result = result[:-padding]
     return result
 
+_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz'
+                b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#')
+# Translating b85 valid but z85 invalid chars to b'\x00' is required
+# to prevent them from being decoded as b85 valid chars.
+_z85_b85_decode_diff = b';_`|~'
+_z85_decode_translation = bytes.maketrans(
+    _z85alphabet + _z85_b85_decode_diff,
+    _b85alphabet + b'\x00' * len(_z85_b85_decode_diff)
+)
+_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet)
+
+def z85encode(s):
+    """Encode bytes-like object b in z85 format and return a bytes object."""
+    return b85encode(s).translate(_z85_encode_translation)
+
+def z85decode(s):
+    """Decode the z85-encoded bytes-like object or ASCII string b
+
+    The result is returned as a bytes object.
+    """
+    s = _bytes_from_decode_data(s)
+    s = s.translate(_z85_decode_translation)
+    try:
+        return b85decode(s)
+    except ValueError as e:
+        raise ValueError(e.args[0].replace('base85', 'z85')) from None
+
 # Legacy interface.  This code could be cleaned up since I don't believe
 # binascii has any line length limitations.  It just doesn't seem worth it
 # though.  The files should be opened in binary mode.

Lib/bz2.py

@@ -17,7 +17,7 @@ import _compression
 from _bz2 import BZ2Compressor, BZ2Decompressor
 
 
-_MODE_CLOSED   = 0
+# Value 0 no longer used
 _MODE_READ     = 1
 # Value 2 no longer used
 _MODE_WRITE    = 3
@@ -54,7 +54,7 @@ class BZ2File(_compression.BaseStream):
         """
         self._fp = None
         self._closefp = False
-        self._mode = _MODE_CLOSED
+        self._mode = None
 
         if not (1 <= compresslevel <= 9):
             raise ValueError("compresslevel must be between 1 and 9")
@@ -100,7 +100,7 @@ class BZ2File(_compression.BaseStream):
         May be called more than once without error. Once the file is
         closed, any other operation on it will raise a ValueError.
         """
-        if self._mode == _MODE_CLOSED:
+        if self.closed:
             return
         try:
             if self._mode == _MODE_READ:
@@ -115,13 +115,21 @@ class BZ2File(_compression.BaseStream):
             finally:
                 self._fp = None
                 self._closefp = False
-                self._mode = _MODE_CLOSED
                 self._buffer = None
 
     @property
     def closed(self):
         """True if this file is closed."""
-        return self._mode == _MODE_CLOSED
+        return self._fp is None
+
+    @property
+    def name(self):
+        self._check_not_closed()
+        return self._fp.name
+
+    @property
+    def mode(self):
+        return 'wb' if self._mode == _MODE_WRITE else 'rb'
 
     def fileno(self):
         """Return the file descriptor for the underlying file."""

Lib/calendar.py

@@ -10,7 +10,6 @@ import datetime
 from enum import IntEnum, global_enum
 import locale as _locale
 from itertools import repeat
-import warnings
 
 __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
            "firstweekday", "isleap", "leapdays", "weekday", "monthrange",
@@ -28,7 +27,9 @@ __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
 error = ValueError
 
 # Exceptions raised for bad input
-class IllegalMonthError(ValueError):
+# This is trick for backward compatibility. Since 3.13, we will raise IllegalMonthError instead of
+# IndexError for bad month number(out of 1-12). But we can't remove IndexError for backward compatibility.
+class IllegalMonthError(ValueError, IndexError):
     def __init__(self, month):
         self.month = month
     def __str__(self):
@@ -44,6 +45,7 @@ class IllegalWeekdayError(ValueError):
 
 def __getattr__(name):
     if name in ('January', 'February'):
+        import warnings
         warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead",
                       DeprecationWarning, stacklevel=2)
         if name == 'January':
@@ -158,11 +160,14 @@ def weekday(year, month, day):
     return Day(datetime.date(year, month, day).weekday())
 
 
-def monthrange(year, month):
-    """Return weekday (0-6 ~ Mon-Sun) and number of days (28-31) for
-       year, month."""
+def _validate_month(month):
     if not 1 <= month <= 12:
         raise IllegalMonthError(month)
+
+def monthrange(year, month):
+    """Return weekday of first day of month (0-6 ~ Mon-Sun)
+       and number of days (28-31) for year, month."""
+    _validate_month(month)
     day1 = weekday(year, month, 1)
     ndays = mdays[month] + (month == FEBRUARY and isleap(year))
     return day1, ndays
@@ -370,6 +375,8 @@ class TextCalendar(Calendar):
         """
         Return a formatted month name.
         """
+        _validate_month(themonth)
+
         s = month_name[themonth]
         if withyear:
             s = "%s %r" % (s, theyear)
@@ -500,6 +507,7 @@ class HTMLCalendar(Calendar):
         """
         Return a month name as a table row.
         """
+        _validate_month(themonth)
         if withyear:
             s = '%s %s' % (month_name[themonth], theyear)
         else:
@@ -585,8 +593,6 @@ class different_locale:
         _locale.setlocale(_locale.LC_TIME, self.locale)
 
     def __exit__(self, *args):
-        if self.oldlocale is None:
-            return
         _locale.setlocale(_locale.LC_TIME, self.oldlocale)
 
 
@@ -690,7 +696,7 @@ def timegm(tuple):
     return seconds
 
 
-def main(args):
+def main(args=None):
     import argparse
     parser = argparse.ArgumentParser()
     textgroup = parser.add_argument_group('text only arguments')
@@ -736,10 +742,15 @@ def main(args):
         choices=("text", "html"),
         help="output type (text or html)"
     )
+    parser.add_argument(
+        "-f", "--first-weekday",
+        type=int, default=0,
+        help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)"
+    )
     parser.add_argument(
         "year",
         nargs='?', type=int,
-        help="year number (1-9999)"
+        help="year number"
     )
     parser.add_argument(
         "month",
@@ -747,7 +758,7 @@ def main(args):
         help="month number (1-12, text only)"
     )
 
-    options = parser.parse_args(args[1:])
+    options = parser.parse_args(args)
 
     if options.locale and not options.encoding:
         parser.error("if --locale is specified --encoding is required")
@@ -756,10 +767,14 @@ def main(args):
     locale = options.locale, options.encoding
 
     if options.type == "html":
+        if options.month:
+            parser.error("incorrect number of arguments")
+            sys.exit(1)
         if options.locale:
             cal = LocaleHTMLCalendar(locale=locale)
         else:
             cal = HTMLCalendar()
+        cal.setfirstweekday(options.first_weekday)
         encoding = options.encoding
         if encoding is None:
             encoding = sys.getdefaultencoding()
@@ -767,20 +782,20 @@ def main(args):
         write = sys.stdout.buffer.write
         if options.year is None:
             write(cal.formatyearpage(datetime.date.today().year, **optdict))
-        elif options.month is None:
-            write(cal.formatyearpage(options.year, **optdict))
         else:
-            parser.error("incorrect number of arguments")
-            sys.exit(1)
+            write(cal.formatyearpage(options.year, **optdict))
     else:
         if options.locale:
             cal = LocaleTextCalendar(locale=locale)
         else:
             cal = TextCalendar()
+        cal.setfirstweekday(options.first_weekday)
         optdict = dict(w=options.width, l=options.lines)
         if options.month is None:
             optdict["c"] = options.spacing
             optdict["m"] = options.months
+        if options.month is not None:
+            _validate_month(options.month)
         if options.year is None:
             result = cal.formatyear(datetime.date.today().year, **optdict)
         elif options.month is None:
@@ -795,4 +810,4 @@ def main(args):
 
 
 if __name__ == "__main__":
-    main(sys.argv)
+    main()

Lib/cgitb.py

@@ -1,332 +0,0 @@
-"""More comprehensive traceback formatting for Python scripts.
-
-To enable this module, do:
-
-    import cgitb; cgitb.enable()
-
-at the top of your script.  The optional arguments to enable() are:
-
-    display     - if true, tracebacks are displayed in the web browser
-    logdir      - if set, tracebacks are written to files in this directory
-    context     - number of lines of source code to show for each stack frame
-    format      - 'text' or 'html' controls the output format
-
-By default, tracebacks are displayed but not saved, the context is 5 lines
-and the output format is 'html' (for backwards compatibility with the
-original use of this module)
-
-Alternatively, if you have caught an exception and want cgitb to display it
-for you, call cgitb.handler().  The optional argument to handler() is a
-3-item tuple (etype, evalue, etb) just like the value of sys.exc_info().
-The default handler displays output as HTML.
-
-"""
-import inspect
-import keyword
-import linecache
-import os
-import pydoc
-import sys
-import tempfile
-import time
-import tokenize
-import traceback
-import warnings
-from html import escape as html_escape
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-
-def reset():
-    """Return a string that resets the CGI and browser to a known state."""
-    return '''<!--: spam
-Content-Type: text/html
-
-<body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> -->
-<body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> --> -->
-</font> </font> </font> </script> </object> </blockquote> </pre>
-</table> </table> </table> </table> </table> </font> </font> </font>'''
-
-__UNDEF__ = []                          # a special sentinel object
-def small(text):
-    if text:
-        return '<small>' + text + '</small>'
-    else:
-        return ''
-
-def strong(text):
-    if text:
-        return '<strong>' + text + '</strong>'
-    else:
-        return ''
-
-def grey(text):
-    if text:
-        return '<font color="#909090">' + text + '</font>'
-    else:
-        return ''
-
-def lookup(name, frame, locals):
-    """Find the value for a given name in the given environment."""
-    if name in locals:
-        return 'local', locals[name]
-    if name in frame.f_globals:
-        return 'global', frame.f_globals[name]
-    if '__builtins__' in frame.f_globals:
-        builtins = frame.f_globals['__builtins__']
-        if isinstance(builtins, dict):
-            if name in builtins:
-                return 'builtin', builtins[name]
-        else:
-            if hasattr(builtins, name):
-                return 'builtin', getattr(builtins, name)
-    return None, __UNDEF__
-
-def scanvars(reader, frame, locals):
-    """Scan one logical line of Python and look up values of variables used."""
-    vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__
-    for ttype, token, start, end, line in tokenize.generate_tokens(reader):
-        if ttype == tokenize.NEWLINE: break
-        if ttype == tokenize.NAME and token not in keyword.kwlist:
-            if lasttoken == '.':
-                if parent is not __UNDEF__:
-                    value = getattr(parent, token, __UNDEF__)
-                    vars.append((prefix + token, prefix, value))
-            else:
-                where, value = lookup(token, frame, locals)
-                vars.append((token, where, value))
-        elif token == '.':
-            prefix += lasttoken + '.'
-            parent = value
-        else:
-            parent, prefix = None, ''
-        lasttoken = token
-    return vars
-
-def html(einfo, context=5):
-    """Return a nice HTML document describing a given traceback."""
-    etype, evalue, etb = einfo
-    if isinstance(etype, type):
-        etype = etype.__name__
-    pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
-    date = time.ctime(time.time())
-    head = f'''
-<body bgcolor="#f0f0f8">
-<table width="100%" cellspacing=0 cellpadding=2 border=0 summary="heading">
-<tr bgcolor="#6622aa">
-<td valign=bottom>&nbsp;<br>
-<font color="#ffffff" face="helvetica, arial">&nbsp;<br>
-<big><big><strong>{html_escape(str(etype))}</strong></big></big></font></td>
-<td align=right valign=bottom>
-<font color="#ffffff" face="helvetica, arial">{pyver}<br>{date}</font></td>
-</tr></table>
-<p>A problem occurred in a Python script.  Here is the sequence of
-function calls leading up to the error, in the order they occurred.</p>'''
-
-    indent = '<tt>' + small('&nbsp;' * 5) + '&nbsp;</tt>'
-    frames = []
-    records = inspect.getinnerframes(etb, context)
-    for frame, file, lnum, func, lines, index in records:
-        if file:
-            file = os.path.abspath(file)
-            link = '<a href="file://%s">%s</a>' % (file, pydoc.html.escape(file))
-        else:
-            file = link = '?'
-        args, varargs, varkw, locals = inspect.getargvalues(frame)
-        call = ''
-        if func != '?':
-            call = 'in ' + strong(pydoc.html.escape(func))
-            if func != "<module>":
-                call += inspect.formatargvalues(args, varargs, varkw, locals,
-                    formatvalue=lambda value: '=' + pydoc.html.repr(value))
-
-        highlight = {}
-        def reader(lnum=[lnum]):
-            highlight[lnum[0]] = 1
-            try: return linecache.getline(file, lnum[0])
-            finally: lnum[0] += 1
-        vars = scanvars(reader, frame, locals)
-
-        rows = ['<tr><td bgcolor="#d8bbff">%s%s %s</td></tr>' %
-                ('<big>&nbsp;</big>', link, call)]
-        if index is not None:
-            i = lnum - index
-            for line in lines:
-                num = small('&nbsp;' * (5-len(str(i))) + str(i)) + '&nbsp;'
-                if i in highlight:
-                    line = '<tt>=&gt;%s%s</tt>' % (num, pydoc.html.preformat(line))
-                    rows.append('<tr><td bgcolor="#ffccee">%s</td></tr>' % line)
-                else:
-                    line = '<tt>&nbsp;&nbsp;%s%s</tt>' % (num, pydoc.html.preformat(line))
-                    rows.append('<tr><td>%s</td></tr>' % grey(line))
-                i += 1
-
-        done, dump = {}, []
-        for name, where, value in vars:
-            if name in done: continue
-            done[name] = 1
-            if value is not __UNDEF__:
-                if where in ('global', 'builtin'):
-                    name = ('<em>%s</em> ' % where) + strong(name)
-                elif where == 'local':
-                    name = strong(name)
-                else:
-                    name = where + strong(name.split('.')[-1])
-                dump.append('%s&nbsp;= %s' % (name, pydoc.html.repr(value)))
-            else:
-                dump.append(name + ' <em>undefined</em>')
-
-        rows.append('<tr><td>%s</td></tr>' % small(grey(', '.join(dump))))
-        frames.append('''
-<table width="100%%" cellspacing=0 cellpadding=0 border=0>
-%s</table>''' % '\n'.join(rows))
-
-    exception = ['<p>%s: %s' % (strong(pydoc.html.escape(str(etype))),
-                                pydoc.html.escape(str(evalue)))]
-    for name in dir(evalue):
-        if name[:1] == '_': continue
-        value = pydoc.html.repr(getattr(evalue, name))
-        exception.append('\n<br>%s%s&nbsp;=\n%s' % (indent, name, value))
-
-    return head + ''.join(frames) + ''.join(exception) + '''
-
-
-<!-- The above is a description of an error in a Python program, formatted
-     for a web browser because the 'cgitb' module was enabled.  In case you
-     are not reading this in a web browser, here is the original traceback:
-
-%s
--->
-''' % pydoc.html.escape(
-          ''.join(traceback.format_exception(etype, evalue, etb)))
-
-def text(einfo, context=5):
-    """Return a plain text document describing a given traceback."""
-    etype, evalue, etb = einfo
-    if isinstance(etype, type):
-        etype = etype.__name__
-    pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
-    date = time.ctime(time.time())
-    head = "%s\n%s\n%s\n" % (str(etype), pyver, date) + '''
-A problem occurred in a Python script.  Here is the sequence of
-function calls leading up to the error, in the order they occurred.
-'''
-
-    frames = []
-    records = inspect.getinnerframes(etb, context)
-    for frame, file, lnum, func, lines, index in records:
-        file = file and os.path.abspath(file) or '?'
-        args, varargs, varkw, locals = inspect.getargvalues(frame)
-        call = ''
-        if func != '?':
-            call = 'in ' + func
-            if func != "<module>":
-                call += inspect.formatargvalues(args, varargs, varkw, locals,
-                    formatvalue=lambda value: '=' + pydoc.text.repr(value))
-
-        highlight = {}
-        def reader(lnum=[lnum]):
-            highlight[lnum[0]] = 1
-            try: return linecache.getline(file, lnum[0])
-            finally: lnum[0] += 1
-        vars = scanvars(reader, frame, locals)
-
-        rows = [' %s %s' % (file, call)]
-        if index is not None:
-            i = lnum - index
-            for line in lines:
-                num = '%5d ' % i
-                rows.append(num+line.rstrip())
-                i += 1
-
-        done, dump = {}, []
-        for name, where, value in vars:
-            if name in done: continue
-            done[name] = 1
-            if value is not __UNDEF__:
-                if where == 'global': name = 'global ' + name
-                elif where != 'local': name = where + name.split('.')[-1]
-                dump.append('%s = %s' % (name, pydoc.text.repr(value)))
-            else:
-                dump.append(name + ' undefined')
-
-        rows.append('\n'.join(dump))
-        frames.append('\n%s\n' % '\n'.join(rows))
-
-    exception = ['%s: %s' % (str(etype), str(evalue))]
-    for name in dir(evalue):
-        value = pydoc.text.repr(getattr(evalue, name))
-        exception.append('\n%s%s = %s' % (" "*4, name, value))
-
-    return head + ''.join(frames) + ''.join(exception) + '''
-
-The above is a description of an error in a Python program.  Here is
-the original traceback:
-
-%s
-''' % ''.join(traceback.format_exception(etype, evalue, etb))
-
-class Hook:
-    """A hook to replace sys.excepthook that shows tracebacks in HTML."""
-
-    def __init__(self, display=1, logdir=None, context=5, file=None,
-                 format="html"):
-        self.display = display          # send tracebacks to browser if true
-        self.logdir = logdir            # log tracebacks to files if not None
-        self.context = context          # number of source code lines per frame
-        self.file = file or sys.stdout  # place to send the output
-        self.format = format
-
-    def __call__(self, etype, evalue, etb):
-        self.handle((etype, evalue, etb))
-
-    def handle(self, info=None):
-        info = info or sys.exc_info()
-        if self.format == "html":
-            self.file.write(reset())
-
-        formatter = (self.format=="html") and html or text
-        plain = False
-        try:
-            doc = formatter(info, self.context)
-        except:                         # just in case something goes wrong
-            doc = ''.join(traceback.format_exception(*info))
-            plain = True
-
-        if self.display:
-            if plain:
-                doc = pydoc.html.escape(doc)
-                self.file.write('<pre>' + doc + '</pre>\n')
-            else:
-                self.file.write(doc + '\n')
-        else:
-            self.file.write('<p>A problem occurred in a Python script.\n')
-
-        if self.logdir is not None:
-            suffix = ['.txt', '.html'][self.format=="html"]
-            (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir)
-
-            try:
-                with os.fdopen(fd, 'w') as file:
-                    file.write(doc)
-                msg = '%s contains the description of this error.' % path
-            except:
-                msg = 'Tried to save traceback to %s, but failed.' % path
-
-            if self.format == 'html':
-                self.file.write('<p>%s</p>\n' % msg)
-            else:
-                self.file.write(msg + '\n')
-        try:
-            self.file.flush()
-        except: pass
-
-handler = Hook().handle
-def enable(display=1, logdir=None, context=5, format="html"):
-    """Install an exception handler that formats tracebacks as HTML.
-
-    The optional argument 'display' can be set to 0 to suppress sending the
-    traceback to the browser, and 'logdir' can be set to a directory to cause
-    tracebacks to be written to files there."""
-    sys.excepthook = Hook(display=display, logdir=logdir,
-                          context=context, format=format)

Lib/chunk.py

@@ -1,173 +0,0 @@
-"""Simple class to read IFF chunks.
-
-An IFF chunk (used in formats such as AIFF, TIFF, RMFF (RealMedia File
-Format)) has the following structure:
-
-+----------------+
-| ID (4 bytes)   |
-+----------------+
-| size (4 bytes) |
-+----------------+
-| data           |
-| ...            |
-+----------------+
-
-The ID is a 4-byte string which identifies the type of chunk.
-
-The size field (a 32-bit value, encoded using big-endian byte order)
-gives the size of the whole chunk, including the 8-byte header.
-
-Usually an IFF-type file consists of one or more chunks.  The proposed
-usage of the Chunk class defined here is to instantiate an instance at
-the start of each chunk and read from the instance until it reaches
-the end, after which a new instance can be instantiated.  At the end
-of the file, creating a new instance will fail with an EOFError
-exception.
-
-Usage:
-while True:
-    try:
-        chunk = Chunk(file)
-    except EOFError:
-        break
-    chunktype = chunk.getname()
-    while True:
-        data = chunk.read(nbytes)
-        if not data:
-            pass
-        # do something with data
-
-The interface is file-like.  The implemented methods are:
-read, close, seek, tell, isatty.
-Extra methods are: skip() (called by close, skips to the end of the chunk),
-getname() (returns the name (ID) of the chunk)
-
-The __init__ method has one required argument, a file-like object
-(including a chunk instance), and one optional argument, a flag which
-specifies whether or not chunks are aligned on 2-byte boundaries.  The
-default is 1, i.e. aligned.
-"""
-
-import warnings
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-class Chunk:
-    def __init__(self, file, align=True, bigendian=True, inclheader=False):
-        import struct
-        self.closed = False
-        self.align = align      # whether to align to word (2-byte) boundaries
-        if bigendian:
-            strflag = '>'
-        else:
-            strflag = '<'
-        self.file = file
-        self.chunkname = file.read(4)
-        if len(self.chunkname) < 4:
-            raise EOFError
-        try:
-            self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0]
-        except struct.error:
-            raise EOFError from None
-        if inclheader:
-            self.chunksize = self.chunksize - 8 # subtract header
-        self.size_read = 0
-        try:
-            self.offset = self.file.tell()
-        except (AttributeError, OSError):
-            self.seekable = False
-        else:
-            self.seekable = True
-
-    def getname(self):
-        """Return the name (ID) of the current chunk."""
-        return self.chunkname
-
-    def getsize(self):
-        """Return the size of the current chunk."""
-        return self.chunksize
-
-    def close(self):
-        if not self.closed:
-            try:
-                self.skip()
-            finally:
-                self.closed = True
-
-    def isatty(self):
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        return False
-
-    def seek(self, pos, whence=0):
-        """Seek to specified position into the chunk.
-        Default position is 0 (start of chunk).
-        If the file is not seekable, this will result in an error.
-        """
-
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        if not self.seekable:
-            raise OSError("cannot seek")
-        if whence == 1:
-            pos = pos + self.size_read
-        elif whence == 2:
-            pos = pos + self.chunksize
-        if pos < 0 or pos > self.chunksize:
-            raise RuntimeError
-        self.file.seek(self.offset + pos, 0)
-        self.size_read = pos
-
-    def tell(self):
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        return self.size_read
-
-    def read(self, size=-1):
-        """Read at most size bytes from the chunk.
-        If size is omitted or negative, read until the end
-        of the chunk.
-        """
-
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        if self.size_read >= self.chunksize:
-            return b''
-        if size < 0:
-            size = self.chunksize - self.size_read
-        if size > self.chunksize - self.size_read:
-            size = self.chunksize - self.size_read
-        data = self.file.read(size)
-        self.size_read = self.size_read + len(data)
-        if self.size_read == self.chunksize and \
-           self.align and \
-           (self.chunksize & 1):
-            dummy = self.file.read(1)
-            self.size_read = self.size_read + len(dummy)
-        return data
-
-    def skip(self):
-        """Skip the rest of the chunk.
-        If you are not interested in the contents of the chunk,
-        this method should be called so that the file points to
-        the start of the next chunk.
-        """
-
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        if self.seekable:
-            try:
-                n = self.chunksize - self.size_read
-                # maybe fix alignment
-                if self.align and (self.chunksize & 1):
-                    n = n + 1
-                self.file.seek(n, 1)
-                self.size_read = self.size_read + n
-                return
-            except OSError:
-                pass
-        while self.size_read < self.chunksize:
-            n = min(8192, self.chunksize - self.size_read)
-            dummy = self.read(n)
-            if not dummy:
-                raise EOFError

Lib/cmd.py

@@ -42,7 +42,7 @@ listings of documented functions, miscellaneous topics, and undocumented
 functions respectively.
 """
 
-import string, sys
+import inspect, string, sys
 
 __all__ = ["Cmd"]
 
@@ -108,7 +108,15 @@ class Cmd:
                 import readline
                 self.old_completer = readline.get_completer()
                 readline.set_completer(self.complete)
-                readline.parse_and_bind(self.completekey+": complete")
+                if readline.backend == "editline":
+                    if self.completekey == 'tab':
+                        # libedit uses "^I" instead of "tab"
+                        command_string = "bind ^I rl_complete"
+                    else:
+                        command_string = f"bind {self.completekey} rl_complete"
+                else:
+                    command_string = f"{self.completekey}: complete"
+                readline.parse_and_bind(command_string)
             except ImportError:
                 pass
         try:
@@ -210,9 +218,8 @@ class Cmd:
         if cmd == '':
             return self.default(line)
         else:
-            try:
-                func = getattr(self, 'do_' + cmd)
-            except AttributeError:
+            func = getattr(self, 'do_' + cmd, None)
+            if func is None:
                 return self.default(line)
             return func(arg)
 
@@ -298,6 +305,7 @@ class Cmd:
             except AttributeError:
                 try:
                     doc=getattr(self, 'do_' + arg).__doc__
+                    doc = inspect.cleandoc(doc)
                     if doc:
                         self.stdout.write("%s\n"%str(doc))
                         return

Lib/codeop.py

@@ -44,6 +44,7 @@ __all__ = ["compile_command", "Compile", "CommandCompiler"]
 # Caveat emptor: These flags are undocumented on purpose and depending
 # on their effect outside the standard library is **unsupported**.
 PyCF_DONT_IMPLY_DEDENT = 0x200
+PyCF_ONLY_AST = 0x400
 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
 
 def _maybe_compile(compiler, source, filename, symbol):
@@ -65,22 +66,14 @@ def _maybe_compile(compiler, source, filename, symbol):
             try:
                 compiler(source + "\n", filename, symbol)
                 return None
+            except _IncompleteInputError as e:
+                return None
             except SyntaxError as e:
-                if "incomplete input" in str(e):
-                    return None
+                pass
                 # fallthrough
 
     return compiler(source, filename, symbol, incomplete_input=False)
 
-def _is_syntax_error(err1, err2):
-    rep1 = repr(err1)
-    rep2 = repr(err2)
-    if "was never closed" in rep1 and "was never closed" in rep2:
-        return False
-    if rep1 == rep2:
-        return True
-    return False
-
 def _compile(source, filename, symbol, incomplete_input=True):
     flags = 0
     if incomplete_input:
@@ -88,7 +81,6 @@ def _compile(source, filename, symbol, incomplete_input=True):
         flags |= PyCF_DONT_IMPLY_DEDENT
     return compile(source, filename, symbol, flags)
 
-
 def compile_command(source, filename="<input>", symbol="single"):
     r"""Compile a command and determine whether it is incomplete.
 
@@ -118,12 +110,14 @@ class Compile:
     def __init__(self):
         self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
 
-    def __call__(self, source, filename, symbol, **kwargs):
-        flags = self.flags
+    def __call__(self, source, filename, symbol, flags=0, **kwargs):
+        flags |= self.flags
         if kwargs.get('incomplete_input', True) is False:
             flags &= ~PyCF_DONT_IMPLY_DEDENT
             flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT
         codeob = compile(source, filename, symbol, flags, True)
+        if flags & PyCF_ONLY_AST:
+            return codeob  # this is an ast.Module in this case
         for feature in _features:
             if codeob.co_flags & feature.compiler_flag:
                 self.flags |= feature.compiler_flag

Lib/colorsys.py

@@ -24,7 +24,7 @@ HSV: Hue, Saturation, Value
 __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb",
            "rgb_to_hsv","hsv_to_rgb"]
 
-# Some floating point constants
+# Some floating-point constants
 
 ONE_THIRD = 1.0/3.0
 ONE_SIXTH = 1.0/6.0

Lib/compileall.py

@@ -97,9 +97,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
     success = True
     if workers != 1 and ProcessPoolExecutor is not None:
+        import multiprocessing
+        if multiprocessing.get_start_method() == 'fork':
+            mp_context = multiprocessing.get_context('forkserver')
+        else:
+            mp_context = None
         # If workers == 0, let ProcessPoolExecutor choose
         workers = workers or None
-        with ProcessPoolExecutor(max_workers=workers) as executor:
+        with ProcessPoolExecutor(max_workers=workers,
+                                 mp_context=mp_context) as executor:
             results = executor.map(partial(compile_file,
                                            ddir=ddir, force=force,
                                            rx=rx, quiet=quiet,
@@ -110,7 +116,8 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
                                            prependdir=prependdir,
                                            limit_sl_dest=limit_sl_dest,
                                            hardlink_dupes=hardlink_dupes),
-                                   files)
+                                   files,
+                                   chunksize=4)
             success = min(results, default=True)
     else:
         for file in files:
@@ -166,13 +173,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
     if stripdir is not None:
         fullname_parts = fullname.split(os.path.sep)
         stripdir_parts = stripdir.split(os.path.sep)
-        ddir_parts = list(fullname_parts)
 
-        for spart, opart in zip(stripdir_parts, fullname_parts):
-            if spart == opart:
-                ddir_parts.remove(spart)
-
-        dfile = os.path.join(*ddir_parts)
+        if stripdir_parts != fullname_parts[:len(stripdir_parts)]:
+            if quiet < 2:
+                print("The stripdir path {!r} is not a valid prefix for "
+                      "source path {!r}; ignoring".format(stripdir, fullname))
+        else:
+            dfile = os.path.join(*fullname_parts[len(stripdir_parts):])
 
     if prependdir is not None:
         if dfile is None:

Lib/configparser.py

@@ -18,8 +18,8 @@ ConfigParser -- responsible for parsing a list of
              delimiters=('=', ':'), comment_prefixes=('#', ';'),
              inline_comment_prefixes=None, strict=True,
              empty_lines_in_values=True, default_section='DEFAULT',
-             interpolation=<unset>, converters=<unset>):
-
+             interpolation=<unset>, converters=<unset>,
+             allow_unnamed_section=False):
         Create the parser. When `defaults` is given, it is initialized into the
         dictionary or intrinsic defaults. The keys must be strings, the values
         must be appropriate for %()s string interpolation.
@@ -68,6 +68,10 @@ ConfigParser -- responsible for parsing a list of
         converter gets its corresponding get*() method on the parser object and
         section proxies.
 
+        When `allow_unnamed_section` is True (default: False), options
+        without section are accepted: the section for these is
+        ``configparser.UNNAMED_SECTION``.
+
     sections()
         Return all the configuration section names, sans DEFAULT.
 
@@ -139,24 +143,28 @@ ConfigParser -- responsible for parsing a list of
         between keys and values are surrounded by spaces.
 """
 
-from collections.abc import MutableMapping
+# Do not import dataclasses; overhead is unacceptable (gh-117703)
+
+from collections.abc import Iterable, MutableMapping
 from collections import ChainMap as _ChainMap
+import contextlib
 import functools
 import io
 import itertools
 import os
 import re
 import sys
-import warnings
+import types
 
 __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError",
            "NoOptionError", "InterpolationError", "InterpolationDepthError",
            "InterpolationMissingOptionError", "InterpolationSyntaxError",
            "ParsingError", "MissingSectionHeaderError",
+           "MultilineContinuationError",
            "ConfigParser", "RawConfigParser",
            "Interpolation", "BasicInterpolation",  "ExtendedInterpolation",
-           "LegacyInterpolation", "SectionProxy", "ConverterMapping",
-           "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH")
+           "SectionProxy", "ConverterMapping",
+           "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION")
 
 _default_dict = dict
 DEFAULTSECT = "DEFAULT"
@@ -298,15 +306,33 @@ class InterpolationDepthError(InterpolationError):
 class ParsingError(Error):
     """Raised when a configuration file does not follow legal syntax."""
 
-    def __init__(self, source):
+    def __init__(self, source, *args):
         super().__init__(f'Source contains parsing errors: {source!r}')
         self.source = source
         self.errors = []
         self.args = (source, )
+        if args:
+            self.append(*args)
 
     def append(self, lineno, line):
         self.errors.append((lineno, line))
-        self.message += '\n\t[line %2d]: %s' % (lineno, line)
+        self.message += '\n\t[line %2d]: %s' % (lineno, repr(line))
+
+    def combine(self, others):
+        for other in others:
+            for error in other.errors:
+                self.append(*error)
+        return self
+
+    @staticmethod
+    def _raise_all(exceptions: Iterable['ParsingError']):
+        """
+        Combine any number of ParsingErrors into one and raise it.
+        """
+        exceptions = iter(exceptions)
+        with contextlib.suppress(StopIteration):
+            raise next(exceptions).combine(exceptions)
+
 
 
 class MissingSectionHeaderError(ParsingError):
@@ -323,6 +349,28 @@ class MissingSectionHeaderError(ParsingError):
         self.args = (filename, lineno, line)
 
 
+class MultilineContinuationError(ParsingError):
+    """Raised when a key without value is followed by continuation line"""
+    def __init__(self, filename, lineno, line):
+        Error.__init__(
+            self,
+            "Key without value continued with an indented line.\n"
+            "file: %r, line: %d\n%r"
+            %(filename, lineno, line))
+        self.source = filename
+        self.lineno = lineno
+        self.line = line
+        self.args = (filename, lineno, line)
+
+class _UnnamedSection:
+
+    def __repr__(self):
+        return "<UNNAMED_SECTION>"
+
+
+UNNAMED_SECTION = _UnnamedSection()
+
+
 # Used in parser getters to indicate the default behaviour when a specific
 # option is not found it to raise an exception. Created to enable `None` as
 # a valid fallback value.
@@ -478,6 +526,8 @@ class ExtendedInterpolation(Interpolation):
                 except (KeyError, NoSectionError, NoOptionError):
                     raise InterpolationMissingOptionError(
                         option, section, rawval, ":".join(path)) from None
+                if v is None:
+                    continue
                 if "$" in v:
                     self._interpolate_some(parser, opt, accum, v, sect,
                                            dict(parser.items(sect, raw=True)),
@@ -491,51 +541,50 @@ class ExtendedInterpolation(Interpolation):
                     "found: %r" % (rest,))
 
 
-class LegacyInterpolation(Interpolation):
-    """Deprecated interpolation used in old versions of ConfigParser.
-    Use BasicInterpolation or ExtendedInterpolation instead."""
+class _ReadState:
+    elements_added : set[str]
+    cursect : dict[str, str] | None = None
+    sectname : str | None = None
+    optname : str | None = None
+    lineno : int = 0
+    indent_level : int = 0
+    errors : list[ParsingError]
 
-    _KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
+    def __init__(self):
+        self.elements_added = set()
+        self.errors = list()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        warnings.warn(
-            "LegacyInterpolation has been deprecated since Python 3.2 "
-            "and will be removed from the configparser module in Python 3.13. "
-            "Use BasicInterpolation or ExtendedInterpolation instead.",
-            DeprecationWarning, stacklevel=2
+
+class _Line(str):
+
+    def __new__(cls, val, *args, **kwargs):
+        return super().__new__(cls, val)
+
+    def __init__(self, val, prefixes):
+        self.prefixes = prefixes
+
+    @functools.cached_property
+    def clean(self):
+        return self._strip_full() and self._strip_inline()
+
+    @property
+    def has_comments(self):
+        return self.strip() != self.clean
+
+    def _strip_inline(self):
+        """
+        Search for the earliest prefix at the beginning of the line or following a space.
+        """
+        matcher = re.compile(
+            '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline)
+            # match nothing if no prefixes
+            or '(?!)'
         )
+        match = matcher.search(self)
+        return self[:match.start() if match else None].strip()
 
-    def before_get(self, parser, section, option, value, vars):
-        rawval = value
-        depth = MAX_INTERPOLATION_DEPTH
-        while depth:                    # Loop through this until it's done
-            depth -= 1
-            if value and "%(" in value:
-                replace = functools.partial(self._interpolation_replace,
-                                            parser=parser)
-                value = self._KEYCRE.sub(replace, value)
-                try:
-                    value = value % vars
-                except KeyError as e:
-                    raise InterpolationMissingOptionError(
-                        option, section, rawval, e.args[0]) from None
-            else:
-                break
-        if value and "%(" in value:
-            raise InterpolationDepthError(option, section, rawval)
-        return value
-
-    def before_set(self, parser, section, option, value):
-        return value
-
-    @staticmethod
-    def _interpolation_replace(match, parser):
-        s = match.group(1)
-        if s is None:
-            return match.group()
-        else:
-            return "%%(%s)s" % parser.optionxform(s)
+    def _strip_full(self):
+        return '' if any(map(self.strip().startswith, self.prefixes.full)) else True
 
 
 class RawConfigParser(MutableMapping):
@@ -584,7 +633,8 @@ class RawConfigParser(MutableMapping):
                  comment_prefixes=('#', ';'), inline_comment_prefixes=None,
                  strict=True, empty_lines_in_values=True,
                  default_section=DEFAULTSECT,
-                 interpolation=_UNSET, converters=_UNSET):
+                 interpolation=_UNSET, converters=_UNSET,
+                 allow_unnamed_section=False,):
 
         self._dict = dict_type
         self._sections = self._dict()
@@ -603,8 +653,10 @@ class RawConfigParser(MutableMapping):
             else:
                 self._optcre = re.compile(self._OPT_TMPL.format(delim=d),
                                           re.VERBOSE)
-        self._comment_prefixes = tuple(comment_prefixes or ())
-        self._inline_comment_prefixes = tuple(inline_comment_prefixes or ())
+        self._prefixes = types.SimpleNamespace(
+            full=tuple(comment_prefixes or ()),
+            inline=tuple(inline_comment_prefixes or ()),
+        )
         self._strict = strict
         self._allow_no_value = allow_no_value
         self._empty_lines_in_values = empty_lines_in_values
@@ -623,6 +675,7 @@ class RawConfigParser(MutableMapping):
             self._converters.update(converters)
         if defaults:
             self._read_defaults(defaults)
+        self._allow_unnamed_section = allow_unnamed_section
 
     def defaults(self):
         return self._defaults
@@ -896,13 +949,19 @@ class RawConfigParser(MutableMapping):
         if self._defaults:
             self._write_section(fp, self.default_section,
                                     self._defaults.items(), d)
+        if UNNAMED_SECTION in self._sections:
+            self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True)
+
         for section in self._sections:
+            if section is UNNAMED_SECTION:
+                continue
             self._write_section(fp, section,
                                 self._sections[section].items(), d)
 
-    def _write_section(self, fp, section_name, section_items, delimiter):
-        """Write a single section to the specified `fp`."""
-        fp.write("[{}]\n".format(section_name))
+    def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False):
+        """Write a single section to the specified `fp'."""
+        if not unnamed:
+            fp.write("[{}]\n".format(section_name))
         for key, value in section_items:
             value = self._interpolation.before_write(self, section_name, key,
                                                      value)
@@ -988,110 +1047,113 @@ class RawConfigParser(MutableMapping):
         in an otherwise empty line or may be entered in lines holding values or
         section names. Please note that comments get stripped off when reading configuration files.
         """
-        elements_added = set()
-        cursect = None                        # None, or a dictionary
-        sectname = None
-        optname = None
-        lineno = 0
-        indent_level = 0
-        e = None                              # None, or an exception
-        for lineno, line in enumerate(fp, start=1):
-            comment_start = sys.maxsize
-            # strip inline comments
-            inline_prefixes = {p: -1 for p in self._inline_comment_prefixes}
-            while comment_start == sys.maxsize and inline_prefixes:
-                next_prefixes = {}
-                for prefix, index in inline_prefixes.items():
-                    index = line.find(prefix, index+1)
-                    if index == -1:
-                        continue
-                    next_prefixes[prefix] = index
-                    if index == 0 or (index > 0 and line[index-1].isspace()):
-                        comment_start = min(comment_start, index)
-                inline_prefixes = next_prefixes
-            # strip full line comments
-            for prefix in self._comment_prefixes:
-                if line.strip().startswith(prefix):
-                    comment_start = 0
-                    break
-            if comment_start == sys.maxsize:
-                comment_start = None
-            value = line[:comment_start].strip()
-            if not value:
+
+        try:
+            ParsingError._raise_all(self._read_inner(fp, fpname))
+        finally:
+            self._join_multiline_values()
+
+    def _read_inner(self, fp, fpname):
+        st = _ReadState()
+
+        Line = functools.partial(_Line, prefixes=self._prefixes)
+        for st.lineno, line in enumerate(map(Line, fp), start=1):
+            if not line.clean:
                 if self._empty_lines_in_values:
                     # add empty line to the value, but only if there was no
                     # comment on the line
-                    if (comment_start is None and
-                        cursect is not None and
-                        optname and
-                        cursect[optname] is not None):
-                        cursect[optname].append('') # newlines added at join
+                    if (not line.has_comments and
+                        st.cursect is not None and
+                        st.optname and
+                        st.cursect[st.optname] is not None):
+                        st.cursect[st.optname].append('') # newlines added at join
                 else:
                     # empty line marks end of value
-                    indent_level = sys.maxsize
+                    st.indent_level = sys.maxsize
                 continue
-            # continuation line?
+
             first_nonspace = self.NONSPACECRE.search(line)
-            cur_indent_level = first_nonspace.start() if first_nonspace else 0
-            if (cursect is not None and optname and
-                cur_indent_level > indent_level):
-                cursect[optname].append(value)
-            # a section header or option header?
-            else:
-                indent_level = cur_indent_level
-                # is it a section header?
-                mo = self.SECTCRE.match(value)
-                if mo:
-                    sectname = mo.group('header')
-                    if sectname in self._sections:
-                        if self._strict and sectname in elements_added:
-                            raise DuplicateSectionError(sectname, fpname,
-                                                        lineno)
-                        cursect = self._sections[sectname]
-                        elements_added.add(sectname)
-                    elif sectname == self.default_section:
-                        cursect = self._defaults
-                    else:
-                        cursect = self._dict()
-                        self._sections[sectname] = cursect
-                        self._proxies[sectname] = SectionProxy(self, sectname)
-                        elements_added.add(sectname)
-                    # So sections can't start with a continuation line
-                    optname = None
-                # no section header in the file?
-                elif cursect is None:
-                    raise MissingSectionHeaderError(fpname, lineno, line)
-                # an option line?
-                else:
-                    mo = self._optcre.match(value)
-                    if mo:
-                        optname, vi, optval = mo.group('option', 'vi', 'value')
-                        if not optname:
-                            e = self._handle_error(e, fpname, lineno, line)
-                        optname = self.optionxform(optname.rstrip())
-                        if (self._strict and
-                            (sectname, optname) in elements_added):
-                            raise DuplicateOptionError(sectname, optname,
-                                                       fpname, lineno)
-                        elements_added.add((sectname, optname))
-                        # This check is fine because the OPTCRE cannot
-                        # match if it would set optval to None
-                        if optval is not None:
-                            optval = optval.strip()
-                            cursect[optname] = [optval]
-                        else:
-                            # valueless option handling
-                            cursect[optname] = None
-                    else:
-                        # a non-fatal parsing error occurred. set up the
-                        # exception but keep going. the exception will be
-                        # raised at the end of the file and will contain a
-                        # list of all bogus lines
-                        e = self._handle_error(e, fpname, lineno, line)
-        self._join_multiline_values()
-        # if any parsing errors occurred, raise an exception
-        if e:
-            raise e
+            st.cur_indent_level = first_nonspace.start() if first_nonspace else 0
+
+            if self._handle_continuation_line(st, line, fpname):
+                continue
+
+            self._handle_rest(st, line, fpname)
+
+        return st.errors
+
+    def _handle_continuation_line(self, st, line, fpname):
+        # continuation line?
+        is_continue = (st.cursect is not None and st.optname and
+            st.cur_indent_level > st.indent_level)
+        if is_continue:
+            if st.cursect[st.optname] is None:
+                raise MultilineContinuationError(fpname, st.lineno, line)
+            st.cursect[st.optname].append(line.clean)
+        return is_continue
+
+    def _handle_rest(self, st, line, fpname):
+        # a section header or option header?
+        if self._allow_unnamed_section and st.cursect is None:
+            self._handle_header(st, UNNAMED_SECTION, fpname)
+
+        st.indent_level = st.cur_indent_level
+        # is it a section header?
+        mo = self.SECTCRE.match(line.clean)
+
+        if not mo and st.cursect is None:
+            raise MissingSectionHeaderError(fpname, st.lineno, line)
+
+        self._handle_header(st, mo.group('header'), fpname) if mo else self._handle_option(st, line, fpname)
+
+    def _handle_header(self, st, sectname, fpname):
+        st.sectname = sectname
+        if st.sectname in self._sections:
+            if self._strict and st.sectname in st.elements_added:
+                raise DuplicateSectionError(st.sectname, fpname,
+                                            st.lineno)
+            st.cursect = self._sections[st.sectname]
+            st.elements_added.add(st.sectname)
+        elif st.sectname == self.default_section:
+            st.cursect = self._defaults
+        else:
+            st.cursect = self._dict()
+            self._sections[st.sectname] = st.cursect
+            self._proxies[st.sectname] = SectionProxy(self, st.sectname)
+            st.elements_added.add(st.sectname)
+        # So sections can't start with a continuation line
+        st.optname = None
+
+    def _handle_option(self, st, line, fpname):
+        # an option line?
+        st.indent_level = st.cur_indent_level
+
+        mo = self._optcre.match(line.clean)
+        if not mo:
+            # a non-fatal parsing error occurred. set up the
+            # exception but keep going. the exception will be
+            # raised at the end of the file and will contain a
+            # list of all bogus lines
+            st.errors.append(ParsingError(fpname, st.lineno, line))
+            return
+
+        st.optname, vi, optval = mo.group('option', 'vi', 'value')
+        if not st.optname:
+            st.errors.append(ParsingError(fpname, st.lineno, line))
+        st.optname = self.optionxform(st.optname.rstrip())
+        if (self._strict and
+            (st.sectname, st.optname) in st.elements_added):
+            raise DuplicateOptionError(st.sectname, st.optname,
+                                    fpname, st.lineno)
+        st.elements_added.add((st.sectname, st.optname))
+        # This check is fine because the OPTCRE cannot
+        # match if it would set optval to None
+        if optval is not None:
+            optval = optval.strip()
+            st.cursect[st.optname] = [optval]
+        else:
+            # valueless option handling
+            st.cursect[st.optname] = None
 
     def _join_multiline_values(self):
         defaults = self.default_section, self._defaults
@@ -1111,12 +1173,6 @@ class RawConfigParser(MutableMapping):
         for key, value in defaults.items():
             self._defaults[self.optionxform(key)] = value
 
-    def _handle_error(self, exc, fpname, lineno, line):
-        if not exc:
-            exc = ParsingError(fpname)
-        exc.append(lineno, repr(line))
-        return exc
-
     def _unify_values(self, section, vars):
         """Create a sequence of lookups with 'vars' taking priority over
         the 'section' which takes priority over the DEFAULTSECT.

Lib/contextlib.py

@@ -20,6 +20,8 @@ class AbstractContextManager(abc.ABC):
 
     __class_getitem__ = classmethod(GenericAlias)
 
+    __slots__ = ()
+
     def __enter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -42,6 +44,8 @@ class AbstractAsyncContextManager(abc.ABC):
 
     __class_getitem__ = classmethod(GenericAlias)
 
+    __slots__ = ()
+
     async def __aenter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -145,14 +149,17 @@ class _GeneratorContextManager(
             except StopIteration:
                 return False
             else:
-                raise RuntimeError("generator didn't stop")
+                try:
+                    raise RuntimeError("generator didn't stop")
+                finally:
+                    self.gen.close()
         else:
             if value is None:
                 # Need to force instantiation so we can reliably
                 # tell if we get the same exception back
                 value = typ()
             try:
-                self.gen.throw(typ, value, traceback)
+                self.gen.throw(value)
             except StopIteration as exc:
                 # Suppress StopIteration *unless* it's the same exception that
                 # was passed to throw().  This prevents a StopIteration
@@ -187,7 +194,10 @@ class _GeneratorContextManager(
                     raise
                 exc.__traceback__ = traceback
                 return False
-            raise RuntimeError("generator didn't stop after throw()")
+            try:
+                raise RuntimeError("generator didn't stop after throw()")
+            finally:
+                self.gen.close()
 
 class _AsyncGeneratorContextManager(
     _GeneratorContextManagerBase,
@@ -212,14 +222,17 @@ class _AsyncGeneratorContextManager(
             except StopAsyncIteration:
                 return False
             else:
-                raise RuntimeError("generator didn't stop")
+                try:
+                    raise RuntimeError("generator didn't stop")
+                finally:
+                    await self.gen.aclose()
         else:
             if value is None:
                 # Need to force instantiation so we can reliably
                 # tell if we get the same exception back
                 value = typ()
             try:
-                await self.gen.athrow(typ, value, traceback)
+                await self.gen.athrow(value)
             except StopAsyncIteration as exc:
                 # Suppress StopIteration *unless* it's the same exception that
                 # was passed to throw().  This prevents a StopIteration
@@ -254,7 +267,10 @@ class _AsyncGeneratorContextManager(
                     raise
                 exc.__traceback__ = traceback
                 return False
-            raise RuntimeError("generator didn't stop after athrow()")
+            try:
+                raise RuntimeError("generator didn't stop after athrow()")
+            finally:
+                await self.gen.aclose()
 
 
 def contextmanager(func):
@@ -441,7 +457,16 @@ class suppress(AbstractContextManager):
         # exactly reproduce the limitations of the CPython interpreter.
         #
         # See http://bugs.python.org/issue12029 for more details
-        return exctype is not None and issubclass(exctype, self._exceptions)
+        if exctype is None:
+            return
+        if issubclass(exctype, self._exceptions):
+            return True
+        if issubclass(exctype, BaseExceptionGroup):
+            match, rest = excinst.split(self._exceptions)
+            if rest is None:
+                return True
+            raise rest
+        return False
 
 
 class _BaseExitStack:
@@ -544,11 +569,12 @@ class ExitStack(_BaseExitStack, AbstractContextManager):
         return self
 
     def __exit__(self, *exc_details):
-        received_exc = exc_details[0] is not None
+        exc = exc_details[1]
+        received_exc = exc is not None
 
         # We manipulate the exception state so it behaves as though
         # we were actually nesting multiple with statements
-        frame_exc = sys.exc_info()[1]
+        frame_exc = sys.exception()
         def _fix_exception_context(new_exc, old_exc):
             # Context may not be correct, so find the end of the chain
             while 1:
@@ -571,24 +597,28 @@ class ExitStack(_BaseExitStack, AbstractContextManager):
             is_sync, cb = self._exit_callbacks.pop()
             assert is_sync
             try:
+                if exc is None:
+                    exc_details = None, None, None
+                else:
+                    exc_details = type(exc), exc, exc.__traceback__
                 if cb(*exc_details):
                     suppressed_exc = True
                     pending_raise = False
-                    exc_details = (None, None, None)
-            except:
-                new_exc_details = sys.exc_info()
+                    exc = None
+            except BaseException as new_exc:
                 # simulate the stack of exceptions by setting the context
-                _fix_exception_context(new_exc_details[1], exc_details[1])
+                _fix_exception_context(new_exc, exc)
                 pending_raise = True
-                exc_details = new_exc_details
+                exc = new_exc
+
         if pending_raise:
             try:
-                # bare "raise exc_details[1]" replaces our carefully
+                # bare "raise exc" replaces our carefully
                 # set-up context
-                fixed_ctx = exc_details[1].__context__
-                raise exc_details[1]
+                fixed_ctx = exc.__context__
+                raise exc
             except BaseException:
-                exc_details[1].__context__ = fixed_ctx
+                exc.__context__ = fixed_ctx
                 raise
         return received_exc and suppressed_exc
 
@@ -684,11 +714,12 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         return self
 
     async def __aexit__(self, *exc_details):
-        received_exc = exc_details[0] is not None
+        exc = exc_details[1]
+        received_exc = exc is not None
 
         # We manipulate the exception state so it behaves as though
         # we were actually nesting multiple with statements
-        frame_exc = sys.exc_info()[1]
+        frame_exc = sys.exception()
         def _fix_exception_context(new_exc, old_exc):
             # Context may not be correct, so find the end of the chain
             while 1:
@@ -710,6 +741,10 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         while self._exit_callbacks:
             is_sync, cb = self._exit_callbacks.pop()
             try:
+                if exc is None:
+                    exc_details = None, None, None
+                else:
+                    exc_details = type(exc), exc, exc.__traceback__
                 if is_sync:
                     cb_suppress = cb(*exc_details)
                 else:
@@ -718,21 +753,21 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
                 if cb_suppress:
                     suppressed_exc = True
                     pending_raise = False
-                    exc_details = (None, None, None)
-            except:
-                new_exc_details = sys.exc_info()
+                    exc = None
+            except BaseException as new_exc:
                 # simulate the stack of exceptions by setting the context
-                _fix_exception_context(new_exc_details[1], exc_details[1])
+                _fix_exception_context(new_exc, exc)
                 pending_raise = True
-                exc_details = new_exc_details
+                exc = new_exc
+
         if pending_raise:
             try:
-                # bare "raise exc_details[1]" replaces our carefully
+                # bare "raise exc" replaces our carefully
                 # set-up context
-                fixed_ctx = exc_details[1].__context__
-                raise exc_details[1]
+                fixed_ctx = exc.__context__
+                raise exc
             except BaseException:
-                exc_details[1].__context__ = fixed_ctx
+                exc.__context__ = fixed_ctx
                 raise
         return received_exc and suppressed_exc
 

Lib/copy.py

@@ -4,8 +4,9 @@ Interface summary:
 
         import copy
 
-        x = copy.copy(y)        # make a shallow copy of y
-        x = copy.deepcopy(y)    # make a deep copy of y
+        x = copy.copy(y)                # make a shallow copy of y
+        x = copy.deepcopy(y)            # make a deep copy of y
+        x = copy.replace(y, a=1, b=2)   # new object with fields replaced, as defined by `__replace__`
 
 For module specific errors, copy.Error is raised.
 
@@ -56,7 +57,7 @@ class Error(Exception):
     pass
 error = Error   # backward compatibility
 
-__all__ = ["Error", "copy", "deepcopy"]
+__all__ = ["Error", "copy", "deepcopy", "replace"]
 
 def copy(x):
     """Shallow copy operation on arbitrary Python objects.
@@ -121,13 +122,13 @@ def deepcopy(x, memo=None, _nil=[]):
     See the module's __doc__ string for more info.
     """
 
+    d = id(x)
     if memo is None:
         memo = {}
-
-    d = id(x)
-    y = memo.get(d, _nil)
-    if y is not _nil:
-        return y
+    else:
+        y = memo.get(d, _nil)
+        if y is not _nil:
+            return y
 
     cls = type(x)
 
@@ -290,3 +291,16 @@ def _reconstruct(x, memo, func, args,
     return y
 
 del types, weakref
+
+
+def replace(obj, /, **changes):
+    """Return a new object replacing specified fields with new values.
+
+    This is especially useful for immutable objects, like named tuples or
+    frozen dataclasses.
+    """
+    cls = obj.__class__
+    func = getattr(cls, '__replace__', None)
+    if func is None:
+        raise TypeError(f"replace() does not support {cls.__name__} objects")
+    return func(obj, **changes)

Lib/csv.py

@@ -1,28 +1,90 @@
 
-"""
-csv.py - read/write/investigate CSV files
+r"""
+CSV parsing and writing.
+
+This module provides classes that assist in the reading and writing
+of Comma Separated Value (CSV) files, and implements the interface
+described by PEP 305.  Although many CSV files are simple to parse,
+the format is not formally defined by a stable specification and
+is subtle enough that parsing lines of a CSV file with something
+like line.split(",") is bound to fail.  The module supports three
+basic APIs: reading, writing, and registration of dialects.
+
+
+DIALECT REGISTRATION:
+
+Readers and writers support a dialect argument, which is a convenient
+handle on a group of settings.  When the dialect argument is a string,
+it identifies one of the dialects previously registered with the module.
+If it is a class or instance, the attributes of the argument are used as
+the settings for the reader or writer:
+
+    class excel:
+        delimiter = ','
+        quotechar = '"'
+        escapechar = None
+        doublequote = True
+        skipinitialspace = False
+        lineterminator = '\r\n'
+        quoting = QUOTE_MINIMAL
+
+SETTINGS:
+
+    * quotechar - specifies a one-character string to use as the
+        quoting character.  It defaults to '"'.
+    * delimiter - specifies a one-character string to use as the
+        field separator.  It defaults to ','.
+    * skipinitialspace - specifies how to interpret spaces which
+        immediately follow a delimiter.  It defaults to False, which
+        means that spaces immediately following a delimiter is part
+        of the following field.
+    * lineterminator - specifies the character sequence which should
+        terminate rows.
+    * quoting - controls when quotes should be generated by the writer.
+        It can take on any of the following module constants:
+
+        csv.QUOTE_MINIMAL means only when required, for example, when a
+            field contains either the quotechar or the delimiter
+        csv.QUOTE_ALL means that quotes are always placed around fields.
+        csv.QUOTE_NONNUMERIC means that quotes are always placed around
+            fields which do not parse as integers or floating-point
+            numbers.
+        csv.QUOTE_STRINGS means that quotes are always placed around
+            fields which are strings.  Note that the Python value None
+            is not a string.
+        csv.QUOTE_NOTNULL means that quotes are only placed around fields
+            that are not the Python value None.
+        csv.QUOTE_NONE means that quotes are never placed around fields.
+    * escapechar - specifies a one-character string used to escape
+        the delimiter when quoting is set to QUOTE_NONE.
+    * doublequote - controls the handling of quotes inside fields.  When
+        True, two consecutive quotes are interpreted as one during read,
+        and when writing, each quote character embedded in the data is
+        written as two quotes
 """
 
 import re
 import types
-from _csv import Error, __version__, writer, reader, register_dialect, \
+from _csv import Error, writer, reader, register_dialect, \
                  unregister_dialect, get_dialect, list_dialects, \
                  field_size_limit, \
                  QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \
-                 QUOTE_STRINGS, QUOTE_NOTNULL, \
-                 __doc__
+                 QUOTE_STRINGS, QUOTE_NOTNULL
 from _csv import Dialect as _Dialect
 
 from io import StringIO
 
 __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE",
            "QUOTE_STRINGS", "QUOTE_NOTNULL",
-           "Error", "Dialect", "__doc__", "excel", "excel_tab",
+           "Error", "Dialect", "excel", "excel_tab",
            "field_size_limit", "reader", "writer",
            "register_dialect", "get_dialect", "list_dialects", "Sniffer",
-           "unregister_dialect", "__version__", "DictReader", "DictWriter",
+           "unregister_dialect", "DictReader", "DictWriter",
            "unix_dialect"]
 
+__version__ = "1.0"
+
+
 class Dialect:
     """Describe a CSV dialect.
 
@@ -51,8 +113,8 @@ class Dialect:
         try:
             _Dialect(self)
         except TypeError as e:
-            # We do this for compatibility with py2.3
-            raise Error(str(e))
+            # Re-raise to get a traceback showing more user code.
+            raise Error(str(e)) from None
 
 class excel(Dialect):
     """Describe the usual properties of Excel-generated CSV files."""

Lib/ctypes/__init__.py

@@ -36,6 +36,9 @@ from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \
      FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \
      FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR
 
+# TODO: RUSTPYTHON remove this
+from _ctypes import _non_existing_function
+
 # WINOLEAPI -> HRESULT
 # WINOLEAPI_(type)
 #
@@ -296,7 +299,9 @@ def create_unicode_buffer(init, size=None):
         return buf
     elif isinstance(init, int):
         _sys.audit("ctypes.create_unicode_buffer", None, init)
-        buftype = c_wchar * init
+        # XXX: RUSTPYTHON
+        # buftype = c_wchar * init
+        buftype = c_wchar.__mul__(init)
         buf = buftype()
         return buf
     raise TypeError(init)
@@ -495,14 +500,15 @@ elif sizeof(c_ulonglong) == sizeof(c_void_p):
     c_ssize_t = c_longlong
 
 # functions
-
 from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr
 
 ## void *memmove(void *, const void *, size_t);
-memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr)
+# XXX: RUSTPYTHON
+# memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr)
 
 ## void *memset(void *, int, size_t)
-memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr)
+# XXX: RUSTPYTHON
+# memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr)
 
 def PYFUNCTYPE(restype, *argtypes):
     class CFunctionType(_CFuncPtr):
@@ -511,11 +517,13 @@ def PYFUNCTYPE(restype, *argtypes):
         _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
     return CFunctionType
 
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
+# XXX: RUSTPYTHON
+# _cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
     return _cast(obj, obj, typ)
 
-_string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
+# XXX: RUSTPYTHON
+# _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=-1):
     """string_at(addr[, size]) -> string
 
@@ -527,7 +535,8 @@ try:
 except ImportError:
     pass
 else:
-    _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr)
+    # XXX: RUSTPYTHON
+    # _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr)
     def wstring_at(ptr, size=-1):
         """wstring_at(addr[, size]) -> string
 

Lib/ctypes/test/test_numbers.py

@@ -82,14 +82,6 @@ class NumberTestCase(unittest.TestCase):
             self.assertRaises(TypeError, t, "")
             self.assertRaises(TypeError, t, None)
 
-    @unittest.skip('test disabled')
-    def test_valid_ranges(self):
-        # invalid values of the correct type
-        # raise ValueError (not OverflowError)
-        for t, (l, h) in zip(unsigned_types, unsigned_ranges):
-            self.assertRaises(ValueError, t, l-1)
-            self.assertRaises(ValueError, t, h+1)
-
     def test_from_param(self):
         # the from_param class method attribute always
         # returns PyCArgObject instances
@@ -106,7 +98,7 @@ class NumberTestCase(unittest.TestCase):
     def test_floats(self):
         # c_float and c_double can be created from
         # Python int and float
-        class FloatLike(object):
+        class FloatLike:
             def __float__(self):
                 return 2.0
         f = FloatLike()
@@ -117,15 +109,15 @@ class NumberTestCase(unittest.TestCase):
             self.assertEqual(t(f).value, 2.0)
 
     def test_integers(self):
-        class FloatLike(object):
+        class FloatLike:
             def __float__(self):
                 return 2.0
         f = FloatLike()
-        class IntLike(object):
+        class IntLike:
             def __int__(self):
                 return 2
         d = IntLike()
-        class IndexLike(object):
+        class IndexLike:
             def __index__(self):
                 return 2
         i = IndexLike()
@@ -155,10 +147,10 @@ class NumberTestCase(unittest.TestCase):
 
             # alignment of the type...
             self.assertEqual((code, alignment(t)),
-                                 (code, align))
+                             (code, align))
             # and alignment of an instance
             self.assertEqual((code, alignment(t())),
-                                 (code, align))
+                             (code, align))
 
     def test_int_from_address(self):
         from array import array
@@ -205,19 +197,6 @@ class NumberTestCase(unittest.TestCase):
         a[0] = ord('?')
         self.assertEqual(v.value, b'?')
 
-    # array does not support c_bool / 't'
-    @unittest.skip('test disabled')
-    def test_bool_from_address(self):
-        from ctypes import c_bool
-        from array import array
-        a = array(c_bool._type_, [True])
-        v = t.from_address(a.buffer_info()[0])
-        self.assertEqual(v.value, a[0])
-        self.assertEqual(type(v) is t)
-        a[0] = False
-        self.assertEqual(v.value, a[0])
-        self.assertEqual(type(v) is t)
-
     def test_init(self):
         # c_int() can be initialized from Python's int, and c_int.
         # Not from c_long or so, which seems strange, abc should
@@ -234,62 +213,6 @@ class NumberTestCase(unittest.TestCase):
             if (hasattr(t, "__ctype_le__")):
                 self.assertRaises(OverflowError, t.__ctype_le__, big_int)
 
-    @unittest.skip('test disabled')
-    def test_perf(self):
-        check_perf()
-
-from ctypes import _SimpleCData
-class c_int_S(_SimpleCData):
-    _type_ = "i"
-    __slots__ = []
-
-def run_test(rep, msg, func, arg=None):
-##    items = [None] * rep
-    items = range(rep)
-    from time import perf_counter as clock
-    if arg is not None:
-        start = clock()
-        for i in items:
-            func(arg); func(arg); func(arg); func(arg); func(arg)
-        stop = clock()
-    else:
-        start = clock()
-        for i in items:
-            func(); func(); func(); func(); func()
-        stop = clock()
-    print("%15s: %.2f us" % (msg, ((stop-start)*1e6/5/rep)))
-
-def check_perf():
-    # Construct 5 objects
-    from ctypes import c_int
-
-    REP = 200000
-
-    run_test(REP, "int()", int)
-    run_test(REP, "int(999)", int)
-    run_test(REP, "c_int()", c_int)
-    run_test(REP, "c_int(999)", c_int)
-    run_test(REP, "c_int_S()", c_int_S)
-    run_test(REP, "c_int_S(999)", c_int_S)
-
-# Python 2.3 -OO, win2k, P4 700 MHz:
-#
-#          int(): 0.87 us
-#       int(999): 0.87 us
-#        c_int(): 3.35 us
-#     c_int(999): 3.34 us
-#      c_int_S(): 3.23 us
-#   c_int_S(999): 3.24 us
-
-# Python 2.2 -OO, win2k, P4 700 MHz:
-#
-#          int(): 0.89 us
-#       int(999): 0.89 us
-#        c_int(): 9.99 us
-#     c_int(999): 10.02 us
-#      c_int_S(): 9.87 us
-#   c_int_S(999): 9.85 us
 
 if __name__ == '__main__':
-##    check_perf()
     unittest.main()

Lib/decimal.py

@@ -1,11 +1,109 @@
+"""Decimal fixed-point and floating-point arithmetic.
+
+This is an implementation of decimal floating-point arithmetic based on
+the General Decimal Arithmetic Specification:
+
+    http://speleotrove.com/decimal/decarith.html
+
+and IEEE standard 854-1987:
+
+    http://en.wikipedia.org/wiki/IEEE_854-1987
+
+Decimal floating point has finite precision with arbitrarily large bounds.
+
+The purpose of this module is to support arithmetic using familiar
+"schoolhouse" rules and to avoid some of the tricky representation
+issues associated with binary floating point.  The package is especially
+useful for financial applications or for contexts where users have
+expectations that are at odds with binary floating point (for instance,
+in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead
+of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected
+Decimal('0.00')).
+
+Here are some examples of using the decimal module:
+
+>>> from decimal import *
+>>> setcontext(ExtendedContext)
+>>> Decimal(0)
+Decimal('0')
+>>> Decimal('1')
+Decimal('1')
+>>> Decimal('-.0123')
+Decimal('-0.0123')
+>>> Decimal(123456)
+Decimal('123456')
+>>> Decimal('123.45e12345678')
+Decimal('1.2345E+12345680')
+>>> Decimal('1.33') + Decimal('1.27')
+Decimal('2.60')
+>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')
+Decimal('-2.20')
+>>> dig = Decimal(1)
+>>> print(dig / Decimal(3))
+0.333333333
+>>> getcontext().prec = 18
+>>> print(dig / Decimal(3))
+0.333333333333333333
+>>> print(dig.sqrt())
+1
+>>> print(Decimal(3).sqrt())
+1.73205080756887729
+>>> print(Decimal(3) ** 123)
+4.85192780976896427E+58
+>>> inf = Decimal(1) / Decimal(0)
+>>> print(inf)
+Infinity
+>>> neginf = Decimal(-1) / Decimal(0)
+>>> print(neginf)
+-Infinity
+>>> print(neginf + inf)
+NaN
+>>> print(neginf * inf)
+-Infinity
+>>> print(dig / 0)
+Infinity
+>>> getcontext().traps[DivisionByZero] = 1
+>>> print(dig / 0)
+Traceback (most recent call last):
+  ...
+  ...
+  ...
+decimal.DivisionByZero: x / 0
+>>> c = Context()
+>>> c.traps[InvalidOperation] = 0
+>>> print(c.flags[InvalidOperation])
+0
+>>> c.divide(Decimal(0), Decimal(0))
+Decimal('NaN')
+>>> c.traps[InvalidOperation] = 1
+>>> print(c.flags[InvalidOperation])
+1
+>>> c.flags[InvalidOperation] = 0
+>>> print(c.flags[InvalidOperation])
+0
+>>> print(c.divide(Decimal(0), Decimal(0)))
+Traceback (most recent call last):
+  ...
+  ...
+  ...
+decimal.InvalidOperation: 0 / 0
+>>> print(c.flags[InvalidOperation])
+1
+>>> c.flags[InvalidOperation] = 0
+>>> c.traps[InvalidOperation] = 0
+>>> print(c.divide(Decimal(0), Decimal(0)))
+NaN
+>>> print(c.flags[InvalidOperation])
+1
+>>>
+"""
 
 try:
     from _decimal import *
-    from _decimal import __doc__
     from _decimal import __version__
     from _decimal import __libmpdec_version__
 except ImportError:
-    from _pydecimal import *
-    from _pydecimal import __doc__
-    from _pydecimal import __version__
-    from _pydecimal import __libmpdec_version__
+    import _pydecimal
+    import sys
+    _pydecimal.__doc__ = __doc__
+    sys.modules[__name__] = _pydecimal

Lib/difflib.py

@@ -1628,7 +1628,7 @@ _file_template = """
 </html>"""
 
 _styles = """
-        table.diff {font-family:Courier; border:medium;}
+        table.diff {font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; border:medium}
         .diff_header {background-color:#e0e0e0}
         td.diff_header {text-align:right}
         .diff_next {background-color:#c0c0c0}

Lib/doctest.py

@@ -102,7 +102,7 @@ import re
 import sys
 import traceback
 import unittest
-from io import StringIO  # XXX: RUSTPYTHON; , IncrementalNewlineDecoder
+from io import StringIO, IncrementalNewlineDecoder
 from collections import namedtuple
 
 TestResults = namedtuple('TestResults', 'failed attempted')
@@ -230,9 +230,7 @@ def _load_testfile(filename, package, module_relative, encoding):
             # get_data() opens files as 'rb', so one must do the equivalent
             # conversion as universal newlines would do.
 
-            # TODO: RUSTPYTHON; use _newline_convert once io.IncrementalNewlineDecoder is implemented
-            return file_contents.replace(os.linesep, '\n'), filename
-            # return _newline_convert(file_contents), filename
+            return _newline_convert(file_contents), filename
     with open(filename, encoding=encoding) as f:
         return f.read(), filename
 

Lib/email/__init__.py

@@ -25,7 +25,6 @@ __all__ = [
     ]
 
 
-
 # Some convenience routines.  Don't import Parser and Message as side-effects
 # of importing email since those cascadingly import most of the rest of the
 # email package.

Lib/email/_encoded_words.py

@@ -62,7 +62,7 @@ __all__ = ['decode_q',
 
 # regex based decoder.
 _q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub,
-        lambda m: bytes([int(m.group(1), 16)]))
+        lambda m: bytes.fromhex(m.group(1).decode()))
 
 def decode_q(encoded):
     encoded = encoded.replace(b'_', b' ')
@@ -98,30 +98,42 @@ def len_q(bstring):
 #
 
 def decode_b(encoded):
-    defects = []
+    # First try encoding with validate=True, fixing the padding if needed.
+    # This will succeed only if encoded includes no invalid characters.
     pad_err = len(encoded) % 4
-    if pad_err:
-        defects.append(errors.InvalidBase64PaddingDefect())
-        padded_encoded = encoded + b'==='[:4-pad_err]
-    else:
-        padded_encoded = encoded
+    missing_padding = b'==='[:4-pad_err] if pad_err else b''
     try:
-        return base64.b64decode(padded_encoded, validate=True), defects
+        return (
+            base64.b64decode(encoded + missing_padding, validate=True),
+            [errors.InvalidBase64PaddingDefect()] if pad_err else [],
+        )
     except binascii.Error:
-        # Since we had correct padding, this must an invalid char error.
-        defects = [errors.InvalidBase64CharactersDefect()]
+        # Since we had correct padding, this is likely an invalid char error.
+        #
         # The non-alphabet characters are ignored as far as padding
-        # goes, but we don't know how many there are.  So we'll just
-        # try various padding lengths until something works.
-        for i in 0, 1, 2, 3:
+        # goes, but we don't know how many there are.  So try without adding
+        # padding to see if it works.
+        try:
+            return (
+                base64.b64decode(encoded, validate=False),
+                [errors.InvalidBase64CharactersDefect()],
+            )
+        except binascii.Error:
+            # Add as much padding as could possibly be necessary (extra padding
+            # is ignored).
             try:
-                return base64.b64decode(encoded+b'='*i, validate=False), defects
+                return (
+                    base64.b64decode(encoded + b'==', validate=False),
+                    [errors.InvalidBase64CharactersDefect(),
+                     errors.InvalidBase64PaddingDefect()],
+                )
             except binascii.Error:
-                if i==0:
-                    defects.append(errors.InvalidBase64PaddingDefect())
-        else:
-            # This should never happen.
-            raise AssertionError("unexpected binascii.Error")
+                # This only happens when the encoded string's length is 1 more
+                # than a multiple of 4, which is invalid.
+                #
+                # bpo-27397: Just return the encoded string since there's no
+                # way to decode.
+                return encoded, [errors.InvalidBase64LengthDefect()]
 
 def encode_b(bstring):
     return base64.b64encode(bstring).decode('ascii')
@@ -167,15 +179,15 @@ def decode(ew):
     # Turn the CTE decoded bytes into unicode.
     try:
         string = bstring.decode(charset)
-    except UnicodeError:
+    except UnicodeDecodeError:
         defects.append(errors.UndecodableBytesDefect("Encoded word "
-            "contains bytes not decodable using {} charset".format(charset)))
+            f"contains bytes not decodable using {charset!r} charset"))
         string = bstring.decode(charset, 'surrogateescape')
-    except LookupError:
+    except (LookupError, UnicodeEncodeError):
         string = bstring.decode('ascii', 'surrogateescape')
         if charset.lower() != 'unknown-8bit':
-            defects.append(errors.CharsetError("Unknown charset {} "
-                "in encoded word; decoded as unknown bytes".format(charset)))
+            defects.append(errors.CharsetError(f"Unknown charset {charset!r} "
+                f"in encoded word; decoded as unknown bytes"))
     return string, charset, lang, defects
 
 

Lib/email/_parseaddr.py

@@ -13,7 +13,7 @@ __all__ = [
     'quote',
     ]
 
-import time, calendar
+import time
 
 SPACE = ' '
 EMPTYSTRING = ''
@@ -65,8 +65,10 @@ def _parsedate_tz(data):
 
     """
     if not data:
-        return
+        return None
     data = data.split()
+    if not data:  # This happens for whitespace-only input.
+        return None
     # The FWS after the comma after the day-of-week is optional, so search and
     # adjust for this.
     if data[0].endswith(',') or data[0].lower() in _daynames:
@@ -93,6 +95,8 @@ def _parsedate_tz(data):
         return None
     data = data[:5]
     [dd, mm, yy, tm, tz] = data
+    if not (dd and mm and yy):
+        return None
     mm = mm.lower()
     if mm not in _monthnames:
         dd, mm = mm, dd.lower()
@@ -108,6 +112,8 @@ def _parsedate_tz(data):
         yy, tm = tm, yy
     if yy[-1] == ',':
         yy = yy[:-1]
+        if not yy:
+            return None
     if not yy[0].isdigit():
         yy, tz = tz, yy
     if tm[-1] == ',':
@@ -126,6 +132,8 @@ def _parsedate_tz(data):
             tss = 0
         elif len(tm) == 3:
             [thh, tmm, tss] = tm
+        else:
+            return None
     else:
         return None
     try:
@@ -186,6 +194,9 @@ def mktime_tz(data):
         # No zone info, so localtime is better assumption than GMT
         return time.mktime(data[:8] + (-1,))
     else:
+        # Delay the import, since mktime_tz is rarely used
+        import calendar
+
         t = calendar.timegm(data)
         return t - data[9]
 
@@ -379,7 +390,12 @@ class AddrlistClass:
         aslist.append('@')
         self.pos += 1
         self.gotonext()
-        return EMPTYSTRING.join(aslist) + self.getdomain()
+        domain = self.getdomain()
+        if not domain:
+            # Invalid domain, return an empty address instead of returning a
+            # local part to denote failed parsing.
+            return EMPTYSTRING
+        return EMPTYSTRING.join(aslist) + domain
 
     def getdomain(self):
         """Get the complete domain name from an address."""
@@ -394,6 +410,10 @@ class AddrlistClass:
             elif self.field[self.pos] == '.':
                 self.pos += 1
                 sdlist.append('.')
+            elif self.field[self.pos] == '@':
+                # bpo-34155: Don't parse domains with two `@` like
+                # `a@malicious.org@important.com`.
+                return EMPTYSTRING
             elif self.field[self.pos] in self.atomends:
                 break
             else:

Lib/email/_policybase.py

@@ -152,11 +152,18 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
     mangle_from_        -- a flag that, when True escapes From_ lines in the
                            body of the message by putting a `>' in front of
                            them. This is used when the message is being
-                           serialized by a generator. Default: True.
+                           serialized by a generator. Default: False.
 
     message_factory     -- the class to use to create new message objects.
                            If the value is None, the default is Message.
 
+    verify_generated_headers
+                        -- if true, the generator verifies that each header
+                           they are properly folded, so that a parser won't
+                           treat it as multiple headers, start-of-body, or
+                           part of another header.
+                           This is a check against custom Header & fold()
+                           implementations.
     """
 
     raise_on_defect = False
@@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
     max_line_length = 78
     mangle_from_ = False
     message_factory = None
+    verify_generated_headers = True
 
     def handle_defect(self, obj, defect):
         """Based on policy, either raise defect or call register_defect.
@@ -294,12 +302,12 @@ class Compat32(Policy):
         """+
         The name is parsed as everything up to the ':' and returned unmodified.
         The value is determined by stripping leading whitespace off the
-        remainder of the first line, joining all subsequent lines together, and
+        remainder of the first line joined with all subsequent lines, and
         stripping any trailing carriage return or linefeed characters.
 
         """
         name, value = sourcelines[0].split(':', 1)
-        value = value.lstrip(' \t') + ''.join(sourcelines[1:])
+        value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n')
         return (name, value.rstrip('\r\n'))
 
     def header_store_parse(self, name, value):
@@ -361,8 +369,12 @@ class Compat32(Policy):
             # Assume it is a Header-like object.
             h = value
         if h is not None:
-            parts.append(h.encode(linesep=self.linesep,
-                                  maxlinelen=self.max_line_length))
+            # The Header class interprets a value of None for maxlinelen as the
+            # default value of 78, as recommended by RFC 2822.
+            maxlinelen = 0
+            if self.max_line_length is not None:
+                maxlinelen = self.max_line_length
+            parts.append(h.encode(linesep=self.linesep, maxlinelen=maxlinelen))
         parts.append(self.linesep)
         return ''.join(parts)
 

Lib/email/architecture.rst

@@ -66,7 +66,7 @@ data payloads.
 Message Lifecycle
 -----------------
 
-The general lifecyle of a message is:
+The general lifecycle of a message is:
 
     Creation
         A `Message` object can be created by a Parser, or it can be

Lib/email/base64mime.py

@@ -45,7 +45,6 @@ EMPTYSTRING = ''
 MISC_LEN = 7
 
 
-
 # Helpers
 def header_length(bytearray):
     """Return the length of s when it is encoded with base64."""
@@ -57,7 +56,6 @@ def header_length(bytearray):
     return n
 
 
-
 def header_encode(header_bytes, charset='iso-8859-1'):
     """Encode a single header line with Base64 encoding in a given charset.
 
@@ -72,7 +70,6 @@ def header_encode(header_bytes, charset='iso-8859-1'):
     return '=?%s?b?%s?=' % (charset, encoded)
 
 
-
 def body_encode(s, maxlinelen=76, eol=NL):
     r"""Encode a string with base64.
 
@@ -84,7 +81,7 @@ def body_encode(s, maxlinelen=76, eol=NL):
     in an email.
     """
     if not s:
-        return s
+        return ""
 
     encvec = []
     max_unencoded = maxlinelen * 3 // 4
@@ -98,7 +95,6 @@ def body_encode(s, maxlinelen=76, eol=NL):
     return EMPTYSTRING.join(encvec)
 
 
-
 def decode(string):
     """Decode a raw base64 string, returning a bytes object.
 

Lib/email/charset.py

@@ -18,7 +18,6 @@ from email import errors
 from email.encoders import encode_7or8bit
 
 
-
 # Flags for types of header encodings
 QP          = 1 # Quoted-Printable
 BASE64      = 2 # Base64
@@ -32,7 +31,6 @@ UNKNOWN8BIT = 'unknown-8bit'
 EMPTYSTRING = ''
 
 
-
 # Defaults
 CHARSETS = {
     # input        header enc  body enc output conv
@@ -104,7 +102,6 @@ CODEC_MAP = {
     }
 
 
-
 # Convenience functions for extending the above mappings
 def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
     """Add character set properties to the global registry.
@@ -112,8 +109,8 @@ def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
     charset is the input character set, and must be the canonical name of a
     character set.
 
-    Optional header_enc and body_enc is either Charset.QP for
-    quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
+    Optional header_enc and body_enc is either charset.QP for
+    quoted-printable, charset.BASE64 for base64 encoding, charset.SHORTEST for
     the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST
     is only valid for header_enc.  It describes how message headers and
     message bodies in the input charset are to be encoded.  Default is no
@@ -153,7 +150,6 @@ def add_codec(charset, codecname):
     CODEC_MAP[charset] = codecname
 
 
-
 # Convenience function for encoding strings, taking into account
 # that they might be unknown-8bit (ie: have surrogate-escaped bytes)
 def _encode(string, codec):
@@ -163,7 +159,6 @@ def _encode(string, codec):
         return string.encode(codec)
 
 
-
 class Charset:
     """Map character sets to their email properties.
 
@@ -185,13 +180,13 @@ class Charset:
 
     header_encoding: If the character set must be encoded before it can be
                      used in an email header, this attribute will be set to
-                     Charset.QP (for quoted-printable), Charset.BASE64 (for
-                     base64 encoding), or Charset.SHORTEST for the shortest of
+                     charset.QP (for quoted-printable), charset.BASE64 (for
+                     base64 encoding), or charset.SHORTEST for the shortest of
                      QP or BASE64 encoding.  Otherwise, it will be None.
 
     body_encoding: Same as header_encoding, but describes the encoding for the
                    mail message's body, which indeed may be different than the
-                   header encoding.  Charset.SHORTEST is not allowed for
+                   header encoding.  charset.SHORTEST is not allowed for
                    body_encoding.
 
     output_charset: Some character sets must be converted before they can be
@@ -241,11 +236,9 @@ class Charset:
         self.output_codec = CODEC_MAP.get(self.output_charset,
                                           self.output_charset)
 
-    def __str__(self):
+    def __repr__(self):
         return self.input_charset.lower()
 
-    __repr__ = __str__
-
     def __eq__(self, other):
         return str(self) == str(other).lower()
 
@@ -348,7 +341,6 @@ class Charset:
                 if not lines and not current_line:
                     lines.append(None)
                 else:
-                    separator = (' ' if lines else '')
                     joined_line = EMPTYSTRING.join(current_line)
                     header_bytes = _encode(joined_line, codec)
                     lines.append(encoder(header_bytes))

Lib/email/contentmanager.py

@@ -72,12 +72,14 @@ def get_non_text_content(msg):
     return msg.get_payload(decode=True)
 for maintype in 'audio image video application'.split():
     raw_data_manager.add_get_handler(maintype, get_non_text_content)
+del maintype
 
 
 def get_message_content(msg):
     return msg.get_payload(0)
 for subtype in 'rfc822 external-body'.split():
     raw_data_manager.add_get_handler('message/'+subtype, get_message_content)
+del subtype
 
 
 def get_and_fixup_unknown_message_content(msg):
@@ -144,15 +146,15 @@ def _encode_text(string, charset, cte, policy):
     linesep = policy.linesep.encode('ascii')
     def embedded_body(lines): return linesep.join(lines) + linesep
     def normal_body(lines): return b'\n'.join(lines) + b'\n'
-    if cte==None:
+    if cte is None:
         # Use heuristics to decide on the "best" encoding.
-        try:
-            return '7bit', normal_body(lines).decode('ascii')
-        except UnicodeDecodeError:
-            pass
-        if (policy.cte_type == '8bit' and
-                max(len(x) for x in lines) <= policy.max_line_length):
-            return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
+        if max((len(x) for x in lines), default=0) <= policy.max_line_length:
+            try:
+                return '7bit', normal_body(lines).decode('ascii')
+            except UnicodeDecodeError:
+                pass
+            if policy.cte_type == '8bit':
+                return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
         sniff = embedded_body(lines[:10])
         sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'),
                                           policy.max_line_length)
@@ -238,9 +240,7 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64',
         data = binascii.b2a_qp(data, istext=False, header=False, quotetabs=True)
         data = data.decode('ascii')
     elif cte == '7bit':
-        # Make sure it really is only ASCII.  The early warning here seems
-        # worth the overhead...if you care write your own content manager :).
-        data.encode('ascii')
+        data = data.decode('ascii')
     elif cte in ('8bit', 'binary'):
         data = data.decode('ascii', 'surrogateescape')
     msg.set_payload(data)
@@ -248,3 +248,4 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64',
     _finalize_set(msg, disposition, filename, cid, params)
 for typ in (bytes, bytearray, memoryview):
     raw_data_manager.add_set_handler(typ, set_bytes_content)
+del typ

Lib/email/encoders.py

@@ -16,7 +16,6 @@ from base64 import encodebytes as _bencode
 from quopri import encodestring as _encodestring
 
 
-
 def _qencode(s):
     enc = _encodestring(s, quotetabs=True)
     # Must encode spaces, which quopri.encodestring() doesn't do
@@ -34,7 +33,6 @@ def encode_base64(msg):
     msg['Content-Transfer-Encoding'] = 'base64'
 
 
-
 def encode_quopri(msg):
     """Encode the message's payload in quoted-printable.
 
@@ -46,7 +44,6 @@ def encode_quopri(msg):
     msg['Content-Transfer-Encoding'] = 'quoted-printable'
 
 
-
 def encode_7or8bit(msg):
     """Set the Content-Transfer-Encoding header to 7bit or 8bit."""
     orig = msg.get_payload(decode=True)
@@ -64,6 +61,5 @@ def encode_7or8bit(msg):
         msg['Content-Transfer-Encoding'] = '7bit'
 
 
-
 def encode_noop(msg):
     """Do nothing."""

Lib/email/errors.py

@@ -29,6 +29,10 @@ class CharsetError(MessageError):
     """An illegal charset was given."""
 
 
+class HeaderWriteError(MessageError):
+    """Error while writing headers."""
+
+
 # These are parsing defects which the parser was able to work around.
 class MessageDefect(ValueError):
     """Base class for a message defect."""
@@ -73,6 +77,9 @@ class InvalidBase64PaddingDefect(MessageDefect):
 class InvalidBase64CharactersDefect(MessageDefect):
     """base64 encoded sequence had characters not in base64 alphabet"""
 
+class InvalidBase64LengthDefect(MessageDefect):
+    """base64 encoded sequence had invalid length (1 mod 4)"""
+
 # These errors are specific to header parsing.
 
 class HeaderDefect(MessageDefect):
@@ -105,3 +112,6 @@ class NonASCIILocalPartDefect(HeaderDefect):
     """local_part contains non-ASCII characters"""
     # This defect only occurs during unicode parsing, not when
     # parsing messages decoded from binary.
+
+class InvalidDateDefect(HeaderDefect):
+    """Header has unparsable or invalid date"""

Lib/email/feedparser.py

@@ -37,11 +37,12 @@ NLCRE_crack = re.compile(r'(\r\n|\r|\n)')
 headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])')
 EMPTYSTRING = ''
 NL = '\n'
+boundaryendRE = re.compile(
+    r'(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
 
 NeedMoreData = object()
 
 
-
 class BufferedSubFile(object):
     """A file-ish object that can have new data loaded into it.
 
@@ -132,7 +133,6 @@ class BufferedSubFile(object):
         return line
 
 
-
 class FeedParser:
     """A feed-style parser of email."""
 
@@ -189,7 +189,7 @@ class FeedParser:
         assert not self._msgstack
         # Look for final set of defects
         if root.get_content_maintype() == 'multipart' \
-               and not root.is_multipart():
+               and not root.is_multipart() and not self._headersonly:
             defect = errors.MultipartInvariantViolationDefect()
             self.policy.handle_defect(root, defect)
         return root
@@ -266,7 +266,7 @@ class FeedParser:
                         yield NeedMoreData
                         continue
                     break
-                msg = self._pop_message()
+                self._pop_message()
                 # We need to pop the EOF matcher in order to tell if we're at
                 # the end of the current file, not the end of the last block
                 # of message headers.
@@ -320,7 +320,7 @@ class FeedParser:
                 self._cur.set_payload(EMPTYSTRING.join(lines))
                 return
             # Make sure a valid content type was specified per RFC 2045:6.4.
-            if (self._cur.get('content-transfer-encoding', '8bit').lower()
+            if (str(self._cur.get('content-transfer-encoding', '8bit')).lower()
                     not in ('7bit', '8bit', 'binary')):
                 defect = errors.InvalidMultipartContentTransferEncodingDefect()
                 self.policy.handle_defect(self._cur, defect)
@@ -329,9 +329,10 @@ class FeedParser:
             # this onto the input stream until we've scanned past the
             # preamble.
             separator = '--' + boundary
-            boundaryre = re.compile(
-                '(?P<sep>' + re.escape(separator) +
-                r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
+            def boundarymatch(line):
+                if not line.startswith(separator):
+                    return None
+                return boundaryendRE.match(line, len(separator))
             capturing_preamble = True
             preamble = []
             linesep = False
@@ -343,7 +344,7 @@ class FeedParser:
                     continue
                 if line == '':
                     break
-                mo = boundaryre.match(line)
+                mo = boundarymatch(line)
                 if mo:
                     # If we're looking at the end boundary, we're done with
                     # this multipart.  If there was a newline at the end of
@@ -375,13 +376,13 @@ class FeedParser:
                         if line is NeedMoreData:
                             yield NeedMoreData
                             continue
-                        mo = boundaryre.match(line)
+                        mo = boundarymatch(line)
                         if not mo:
                             self._input.unreadline(line)
                             break
                     # Recurse to parse this subpart; the input stream points
                     # at the subpart's first line.
-                    self._input.push_eof_matcher(boundaryre.match)
+                    self._input.push_eof_matcher(boundarymatch)
                     for retval in self._parsegen():
                         if retval is NeedMoreData:
                             yield NeedMoreData

Lib/email/generator.py

@@ -14,15 +14,16 @@ import random
 from copy import deepcopy
 from io import StringIO, BytesIO
 from email.utils import _has_surrogates
+from email.errors import HeaderWriteError
 
 UNDERSCORE = '_'
 NL = '\n'  # XXX: no longer used by the code below.
 
 NLCRE = re.compile(r'\r\n|\r|\n')
 fcre = re.compile(r'^From ', re.MULTILINE)
+NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
 
 
-
 class Generator:
     """Generates output from a Message object tree.
 
@@ -170,7 +171,7 @@ class Generator:
         # parameter.
         #
         # The way we do this, so as to make the _handle_*() methods simpler,
-        # is to cache any subpart writes into a buffer.  The we write the
+        # is to cache any subpart writes into a buffer.  Then we write the
         # headers and the buffer contents.  That way, subpart handlers can
         # Do The Right Thing, and can still modify the Content-Type: header if
         # necessary.
@@ -186,7 +187,11 @@ class Generator:
         # If we munged the cte, copy the message again and re-fix the CTE.
         if munge_cte:
             msg = deepcopy(msg)
-            msg.replace_header('content-transfer-encoding', munge_cte[0])
+            # Preserve the header order if the CTE header already exists.
+            if msg.get('content-transfer-encoding') is None:
+                msg['Content-Transfer-Encoding'] = munge_cte[0]
+            else:
+                msg.replace_header('content-transfer-encoding', munge_cte[0])
             msg.replace_header('content-type', munge_cte[1])
         # Write the headers.  First we see if the message object wants to
         # handle that itself.  If not, we'll do it generically.
@@ -219,7 +224,16 @@ class Generator:
 
     def _write_headers(self, msg):
         for h, v in msg.raw_items():
-            self.write(self.policy.fold(h, v))
+            folded = self.policy.fold(h, v)
+            if self.policy.verify_generated_headers:
+                linesep = self.policy.linesep
+                if not folded.endswith(self.policy.linesep):
+                    raise HeaderWriteError(
+                        f'folded header does not end with {linesep!r}: {folded!r}')
+                if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)):
+                    raise HeaderWriteError(
+                        f'folded header contains newline: {folded!r}')
+            self.write(folded)
         # A blank line always separates headers from body
         self.write(self._NL)
 
@@ -240,7 +254,7 @@ class Generator:
                 # existing message.
                 msg = deepcopy(msg)
                 del msg['content-transfer-encoding']
-                msg.set_payload(payload, charset)
+                msg.set_payload(msg._payload, charset)
                 payload = msg.get_payload()
                 self._munge_cte = (msg['content-transfer-encoding'],
                                    msg['content-type'])
@@ -388,7 +402,7 @@ class Generator:
     def _compile_re(cls, s, flags):
         return re.compile(s, flags)
 
-
+
 class BytesGenerator(Generator):
     """Generates a bytes version of a Message object tree.
 
@@ -439,7 +453,6 @@ class BytesGenerator(Generator):
         return re.compile(s.encode('ascii'), flags)
 
 
-
 _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]'
 
 class DecodedGenerator(Generator):
@@ -499,7 +512,6 @@ class DecodedGenerator(Generator):
                     }, file=self)
 
 
-
 # Helper used by Generator._make_boundary
 _width = len(repr(sys.maxsize-1))
 _fmt = '%%0%dd' % _width

Lib/email/header.py

@@ -36,11 +36,11 @@ ecre = re.compile(r'''
   =\?                   # literal =?
   (?P<charset>[^?]*?)   # non-greedy up to the next ? is the charset
   \?                    # literal ?
-  (?P<encoding>[qb])    # either a "q" or a "b", case insensitive
+  (?P<encoding>[qQbB])  # either a "q" or a "b", case insensitive
   \?                    # literal ?
   (?P<encoded>.*?)      # non-greedy up to the next ?= is the encoded string
   \?=                   # literal ?=
-  ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE)
+  ''', re.VERBOSE | re.MULTILINE)
 
 # Field name regexp, including trailing colon, but not separating whitespace,
 # according to RFC 2822.  Character range is from tilde to exclamation mark.
@@ -52,12 +52,10 @@ fcre = re.compile(r'[\041-\176]+:$')
 _embedded_header = re.compile(r'\n[^ \t]+:')
 
 
-
 # Helpers
 _max_append = email.quoprimime._max_append
 
 
-
 def decode_header(header):
     """Decode a message header value without converting charset.
 
@@ -152,7 +150,6 @@ def decode_header(header):
     return collapsed
 
 
-
 def make_header(decoded_seq, maxlinelen=None, header_name=None,
                 continuation_ws=' '):
     """Create a Header from a sequence of pairs as returned by decode_header()
@@ -175,7 +172,6 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None,
     return h
 
 
-
 class Header:
     def __init__(self, s=None, charset=None,
                  maxlinelen=None, header_name=None,
@@ -409,7 +405,6 @@ class Header:
         self._chunks = chunks
 
 
-
 class _ValueFormatter:
     def __init__(self, headerlen, maxlen, continuation_ws, splitchars):
         self._maxlen = maxlen
@@ -431,7 +426,7 @@ class _ValueFormatter:
         if end_of_line != (' ', ''):
             self._current_line.push(*end_of_line)
         if len(self._current_line) > 0:
-            if self._current_line.is_onlyws():
+            if self._current_line.is_onlyws() and self._lines:
                 self._lines[-1] += str(self._current_line)
             else:
                 self._lines.append(str(self._current_line))

Lib/email/headerregistry.py

@@ -2,10 +2,6 @@
 
 This module provides an implementation of the HeaderRegistry API.
 The implementation is designed to flexibly follow RFC5322 rules.
-
-Eventually HeaderRegistry will be a public API, but it isn't yet,
-and will probably change some before that happens.
-
 """
 from types import MappingProxyType
 
@@ -31,6 +27,11 @@ class Address:
         without any Content Transfer Encoding.
 
         """
+
+        inputs = ''.join(filter(None, (display_name, username, domain, addr_spec)))
+        if '\r' in inputs or '\n' in inputs:
+            raise ValueError("invalid arguments; address parts cannot contain CR or LF")
+
         # This clause with its potential 'raise' may only happen when an
         # application program creates an Address object using an addr_spec
         # keyword.  The email library code itself must always supply username
@@ -69,11 +70,9 @@ class Address:
         """The addr_spec (username@domain) portion of the address, quoted
         according to RFC 5322 rules, but with no Content Transfer Encoding.
         """
-        nameset = set(self.username)
-        if len(nameset) > len(nameset-parser.DOT_ATOM_ENDS):
-            lp = parser.quote_string(self.username)
-        else:
-            lp = self.username
+        lp = self.username
+        if not parser.DOT_ATOM_ENDS.isdisjoint(lp):
+            lp = parser.quote_string(lp)
         if self.domain:
             return lp + '@' + self.domain
         if not lp:
@@ -86,19 +85,17 @@ class Address:
                         self.display_name, self.username, self.domain)
 
     def __str__(self):
-        nameset = set(self.display_name)
-        if len(nameset) > len(nameset-parser.SPECIALS):
-            disp = parser.quote_string(self.display_name)
-        else:
-            disp = self.display_name
+        disp = self.display_name
+        if not parser.SPECIALS.isdisjoint(disp):
+            disp = parser.quote_string(disp)
         if disp:
             addr_spec = '' if self.addr_spec=='<>' else self.addr_spec
             return "{} <{}>".format(disp, addr_spec)
         return self.addr_spec
 
     def __eq__(self, other):
-        if type(other) != type(self):
-            return False
+        if not isinstance(other, Address):
+            return NotImplemented
         return (self.display_name == other.display_name and
                 self.username == other.username and
                 self.domain == other.domain)
@@ -141,17 +138,15 @@ class Group:
         if self.display_name is None and len(self.addresses)==1:
             return str(self.addresses[0])
         disp = self.display_name
-        if disp is not None:
-            nameset = set(disp)
-            if len(nameset) > len(nameset-parser.SPECIALS):
-                disp = parser.quote_string(disp)
+        if disp is not None and not parser.SPECIALS.isdisjoint(disp):
+            disp = parser.quote_string(disp)
         adrstr = ", ".join(str(x) for x in self.addresses)
         adrstr = ' ' + adrstr if adrstr else adrstr
         return "{}:{};".format(disp, adrstr)
 
     def __eq__(self, other):
-        if type(other) != type(self):
-            return False
+        if not isinstance(other, Group):
+            return NotImplemented
         return (self.display_name == other.display_name and
                 self.addresses == other.addresses)
 
@@ -223,7 +218,7 @@ class BaseHeader(str):
                 self.__class__.__bases__,
                 str(self),
             ),
-            self.__dict__)
+            self.__getstate__())
 
     @classmethod
     def _reconstruct(cls, value):
@@ -245,13 +240,16 @@ class BaseHeader(str):
         the header name and the ': ' separator.
 
         """
-        # At some point we need to only put fws here if it was in the source.
+        # At some point we need to put fws here if it was in the source.
         header = parser.Header([
             parser.HeaderLabel([
                 parser.ValueTerminal(self.name, 'header-name'),
                 parser.ValueTerminal(':', 'header-sep')]),
-            parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]),
-                             self._parse_tree])
+            ])
+        if self._parse_tree:
+            header.append(
+                parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]))
+        header.append(self._parse_tree)
         return header.fold(policy=policy)
 
 
@@ -300,7 +298,14 @@ class DateHeader:
             kwds['parse_tree'] = parser.TokenList()
             return
         if isinstance(value, str):
-            value = utils.parsedate_to_datetime(value)
+            kwds['decoded'] = value
+            try:
+                value = utils.parsedate_to_datetime(value)
+            except ValueError:
+                kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format'))
+                kwds['datetime'] = None
+                kwds['parse_tree'] = parser.TokenList()
+                return
         kwds['datetime'] = value
         kwds['decoded'] = utils.format_datetime(kwds['datetime'])
         kwds['parse_tree'] = cls.value_parser(kwds['decoded'])
@@ -369,8 +374,8 @@ class AddressHeader:
     @property
     def addresses(self):
         if self._addresses is None:
-            self._addresses = tuple([address for group in self._groups
-                                             for address in group.addresses])
+            self._addresses = tuple(address for group in self._groups
+                                            for address in group.addresses)
         return self._addresses
 
 
@@ -517,6 +522,18 @@ class ContentTransferEncodingHeader:
         return self._cte
 
 
+class MessageIDHeader:
+
+    max_count = 1
+    value_parser = staticmethod(parser.parse_message_id)
+
+    @classmethod
+    def parse(cls, value, kwds):
+        kwds['parse_tree'] = parse_tree = cls.value_parser(value)
+        kwds['decoded'] = str(parse_tree)
+        kwds['defects'].extend(parse_tree.all_defects)
+
+
 # The header factory #
 
 _default_header_map = {
@@ -539,6 +556,7 @@ _default_header_map = {
     'content-type':                 ContentTypeHeader,
     'content-disposition':          ContentDispositionHeader,
     'content-transfer-encoding':    ContentTransferEncodingHeader,
+    'message-id':                   MessageIDHeader,
     }
 
 class HeaderRegistry:

Lib/email/iterators.py

@@ -15,7 +15,6 @@ import sys
 from io import StringIO
 
 
-
 # This function will become a method of the Message class
 def walk(self):
     """Walk over the message tree, yielding each subpart.
@@ -29,7 +28,6 @@ def walk(self):
             yield from subpart.walk()
 
 
-
 # These two functions are imported into the Iterators.py interface module.
 def body_line_iterator(msg, decode=False):
     """Iterate over the parts, returning string payloads line-by-line.
@@ -55,7 +53,6 @@ def typed_subpart_iterator(msg, maintype='text', subtype=None):
                 yield subpart
 
 
-
 def _structure(msg, fp=None, level=0, include_default=False):
     """A handy debugging aid"""
     if fp is None:

Lib/email/message.py

@@ -6,15 +6,15 @@
 
 __all__ = ['Message', 'EmailMessage']
 
+import binascii
 import re
-import uu
 import quopri
 from io import BytesIO, StringIO
 
 # Intrapackage imports
 from email import utils
 from email import errors
-from email._policybase import Policy, compat32
+from email._policybase import compat32
 from email import charset as _charset
 from email._encoded_words import decode_b
 Charset = _charset.Charset
@@ -35,7 +35,7 @@ def _splitparam(param):
     if not sep:
         return a.strip(), None
     return a.strip(), b.strip()
-
+
 def _formatparam(param, value=None, quote=True):
     """Convenience function to format and return a key=value pair.
 
@@ -101,7 +101,37 @@ def _unquotevalue(value):
         return utils.unquote(value)
 
 
-
+def _decode_uu(encoded):
+    """Decode uuencoded data."""
+    decoded_lines = []
+    encoded_lines_iter = iter(encoded.splitlines())
+    for line in encoded_lines_iter:
+        if line.startswith(b"begin "):
+            mode, _, path = line.removeprefix(b"begin ").partition(b" ")
+            try:
+                int(mode, base=8)
+            except ValueError:
+                continue
+            else:
+                break
+    else:
+        raise ValueError("`begin` line not found")
+    for line in encoded_lines_iter:
+        if not line:
+            raise ValueError("Truncated input")
+        elif line.strip(b' \t\r\n\f') == b'end':
+            break
+        try:
+            decoded_line = binascii.a2b_uu(line)
+        except binascii.Error:
+            # Workaround for broken uuencoders by /Fredrik Lundh
+            nbytes = (((line[0]-32) & 63) * 4 + 5) // 3
+            decoded_line = binascii.a2b_uu(line[:nbytes])
+        decoded_lines.append(decoded_line)
+
+    return b''.join(decoded_lines)
+
+
 class Message:
     """Basic message object.
 
@@ -141,7 +171,7 @@ class Message:
         header.  For backward compatibility reasons, if maxheaderlen is
         not specified it defaults to 0, so you must override it explicitly
         if you want a different maxheaderlen.  'policy' is passed to the
-        Generator instance used to serialize the mesasge; if it is not
+        Generator instance used to serialize the message; if it is not
         specified the policy associated with the message instance is used.
 
         If the message object contains binary data that is not encoded
@@ -259,25 +289,26 @@ class Message:
         # cte might be a Header, so for now stringify it.
         cte = str(self.get('content-transfer-encoding', '')).lower()
         # payload may be bytes here.
-        if isinstance(payload, str):
-            if utils._has_surrogates(payload):
-                bpayload = payload.encode('ascii', 'surrogateescape')
-                if not decode:
+        if not decode:
+            if isinstance(payload, str) and utils._has_surrogates(payload):
+                try:
+                    bpayload = payload.encode('ascii', 'surrogateescape')
                     try:
-                        payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace')
+                        payload = bpayload.decode(self.get_content_charset('ascii'), 'replace')
                     except LookupError:
                         payload = bpayload.decode('ascii', 'replace')
-            elif decode:
-                try:
-                    bpayload = payload.encode('ascii')
-                except UnicodeError:
-                    # This won't happen for RFC compliant messages (messages
-                    # containing only ASCII code points in the unicode input).
-                    # If it does happen, turn the string into bytes in a way
-                    # guaranteed not to fail.
-                    bpayload = payload.encode('raw-unicode-escape')
-        if not decode:
+                except UnicodeEncodeError:
+                    pass
             return payload
+        if isinstance(payload, str):
+            try:
+                bpayload = payload.encode('ascii', 'surrogateescape')
+            except UnicodeEncodeError:
+                # This won't happen for RFC compliant messages (messages
+                # containing only ASCII code points in the unicode input).
+                # If it does happen, turn the string into bytes in a way
+                # guaranteed not to fail.
+                bpayload = payload.encode('raw-unicode-escape')
         if cte == 'quoted-printable':
             return quopri.decodestring(bpayload)
         elif cte == 'base64':
@@ -288,13 +319,10 @@ class Message:
                 self.policy.handle_defect(self, defect)
             return value
         elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'):
-            in_file = BytesIO(bpayload)
-            out_file = BytesIO()
             try:
-                uu.decode(in_file, out_file, quiet=True)
-                return out_file.getvalue()
-            except uu.Error:
-                # Some decoding problem
+                return _decode_uu(bpayload)
+            except ValueError:
+                # Some decoding problem.
                 return bpayload
         if isinstance(payload, str):
             return bpayload
@@ -312,7 +340,7 @@ class Message:
                 return
             if not isinstance(charset, Charset):
                 charset = Charset(charset)
-            payload = payload.encode(charset.output_charset)
+            payload = payload.encode(charset.output_charset, 'surrogateescape')
         if hasattr(payload, 'decode'):
             self._payload = payload.decode('ascii', 'surrogateescape')
         else:
@@ -421,7 +449,11 @@ class Message:
         self._headers = newheaders
 
     def __contains__(self, name):
-        return name.lower() in [k.lower() for k, v in self._headers]
+        name_lower = name.lower()
+        for k, v in self._headers:
+            if name_lower == k.lower():
+                return True
+        return False
 
     def __iter__(self):
         for field, value in self._headers:
@@ -948,7 +980,7 @@ class MIMEPart(Message):
         if policy is None:
             from email.policy import default
             policy = default
-        Message.__init__(self, policy)
+        super().__init__(policy)
 
 
     def as_string(self, unixfrom=False, maxheaderlen=None, policy=None):
@@ -958,14 +990,14 @@ class MIMEPart(Message):
         header.  maxheaderlen is retained for backward compatibility with the
         base Message class, but defaults to None, meaning that the policy value
         for max_line_length controls the header maximum length.  'policy' is
-        passed to the Generator instance used to serialize the mesasge; if it
+        passed to the Generator instance used to serialize the message; if it
         is not specified the policy associated with the message instance is
         used.
         """
         policy = self.policy if policy is None else policy
         if maxheaderlen is None:
             maxheaderlen = policy.max_line_length
-        return super().as_string(maxheaderlen=maxheaderlen, policy=policy)
+        return super().as_string(unixfrom, maxheaderlen, policy)
 
     def __str__(self):
         return self.as_string(policy=self.policy.clone(utf8=True))
@@ -982,7 +1014,7 @@ class MIMEPart(Message):
             if subtype in preferencelist:
                 yield (preferencelist.index(subtype), part)
             return
-        if maintype != 'multipart':
+        if maintype != 'multipart' or not self.is_multipart():
             return
         if subtype != 'related':
             for subpart in part.iter_parts():
@@ -1041,7 +1073,16 @@ class MIMEPart(Message):
         maintype, subtype = self.get_content_type().split('/')
         if maintype != 'multipart' or subtype == 'alternative':
             return
-        parts = self.get_payload().copy()
+        payload = self.get_payload()
+        # Certain malformed messages can have content type set to `multipart/*`
+        # but still have single part body, in which case payload.copy() can
+        # fail with AttributeError.
+        try:
+            parts = payload.copy()
+        except AttributeError:
+            # payload is not a list, it is most probably a string.
+            return
+
         if maintype == 'multipart' and subtype == 'related':
             # For related, we treat everything but the root as an attachment.
             # The root may be indicated by 'start'; if there's no start or we
@@ -1078,7 +1119,7 @@ class MIMEPart(Message):
 
         Return an empty iterator for a non-multipart.
         """
-        if self.get_content_maintype() == 'multipart':
+        if self.is_multipart():
             yield from self.get_payload()
 
     def get_content(self, *args, content_manager=None, **kw):

Lib/email/mime/application.py

@@ -17,7 +17,7 @@ class MIMEApplication(MIMENonMultipart):
                  _encoder=encoders.encode_base64, *, policy=None, **_params):
         """Create an application/* type MIME document.
 
-        _data is a string containing the raw application data.
+        _data contains the bytes for the raw application data.
 
         _subtype is the MIME content type subtype, defaulting to
         'octet-stream'.