0.3.1

Update indexmap

.cspell.json

@@ -26,109 +26,262 @@
   "words": [
     // Rust
     "ahash",
-    "bitflags",
+    "bidi",
+    "biguint",
     "bindgen",
-    "cstring",
+    "bitflags",
+    "bstr",
+    "byteorder",
     "chrono",
-    "peekable",
+    "consts",
+    "cstring",
+    "flate2",
+    "fract",
+    "hasher",
+    "idents",
+    "indexmap",
+    "insta",
+    "keccak",
     "lalrpop",
-    "memmap",
+    "libc",
+    "libz",
+    "longlong",
     "Manually",
+    "maplit",
+    "memmap",
+    "metas",
+    "modpow",
+    "nanos",
+    "peekable",
+    "powc",
+    "powf",
+    "prepended",
+    "punct",
+    "replacen",
+    "rsplitn",
     "rustc",
-    "unistd",
+    "rustfmt",
+    "seekfrom",
+    "splitn",
+    "subsec",
+    "timsort",
+    "trai",
+    "ulonglong",
     "unic",
+    "unistd",
+    "winapi",
+    "winsock",
     // Python
-    "cformat",
-    "cpython",
-    "fspath",
-    "kwarg",
-    "kwargs",
-    "vararg",
-    "varargs",
-    "metaclass",
-    "metaclasses",
-    "fstring",
-    "fstrings",
-    "docstring",
-    "docstrings",
-    "fileencoding",
-    "linearization",
-    "linearize",
-    "PYTHONDEBUG",
-    "PYTHONINSPECT",
-    "PYTHONPATH",
-    "PYTHONHOME",
-    "PYTHONPATH",
-    "PYTHONVERBOSE",
-    "PYTHONOPTIMIZE",
-    "PYTHONWARNINGS",
-    "basicsize",
-    "itemsize",
-    "getattro",
-    "setattro",
-    "iternext",
-    "maxsplit",
-    "fdel",
-    "subclasscheck",
-    "qualname",
-    "eventmask",
-    "instanceof",
     "abstractmethods",
     "aiter",
     "anext",
-    "rdiv",
+    "arrayiterator",
-    "idiv",
+    "arraytype",
-    "ndim",
+    "asend",
-    "varnames",
+    "athrow",
-    "getweakrefs",
+    "basicsize",
-    "getweakrefcount",
+    "cformat",
-    "stacklevel",
+    "classcell",
-    "MemoryView",
+    "closesocket",
-    "warningregistry",
+    "codepoint",
+    "codepoints",
+    "cpython",
+    "decompressor",
     "defaultaction",
-    "unraisablehook",
     "descr",
-    "xopts",
+    "dictcomp",
+    "dictitems",
+    "dictkeys",
+    "dictview",
+    "docstring",
+    "docstrings",
+    "dunder",
+    "eventmask",
+    "fdel",
+    "fget",
+    "fileencoding",
+    "fillchar",
+    "finallyhandler",
+    "frombytes",
+    "fromhex",
+    "fromunicode",
+    "fset",
+    "fspath",
+    "fstring",
+    "fstrings",
+    "genexpr",
+    "getattro",
+    "getformat",
+    "getnewargs",
+    "getweakrefcount",
+    "getweakrefs",
+    "hostnames",
+    "idiv",
+    "impls",
+    "infj",
+    "instancecheck",
+    "instanceof",
+    "isabstractmethod",
+    "itemiterator",
+    "itemsize",
+    "iternext",
+    "keyiterator",
+    "kwarg",
+    "kwargs",
+    "linearization",
+    "linearize",
+    "listcomp",
+    "mappingproxy",
+    "maxsplit",
+    "memoryview",
+    "memoryviewiterator",
+    "metaclass",
+    "metaclasses",
+    "metatype",
+    "mro",
+    "mros",
+    "nanj",
+    "ndigits",
+    "ndim",
+    "nonbytes",
+    "origname",
+    "posixsubprocess",
+    "pyexpat",
+    "PYTHONDEBUG",
+    "PYTHONHOME",
+    "PYTHONINSPECT",
+    "PYTHONOPTIMIZE",
+    "PYTHONPATH",
+    "PYTHONPATH",
+    "PYTHONVERBOSE",
+    "PYTHONWARNINGS",
+    "qualname",
+    "radd",
+    "rdiv",
+    "rdivmod",
+    "reconstructor",
+    "reversevalueiterator",
+    "rfloordiv",
+    "rlshift",
+    "rmod",
+    "rpow",
+    "rrshift",
+    "rsub",
+    "rtruediv",
+    "scproxy",
+    "setattro",
+    "setcomp",
+    "showwarnmsg",
+    "warnmsg",
+    "stacklevel",
+    "subclasscheck",
+    "subclasshook",
+    "unionable",
+    "unraisablehook",
+    "valueiterator",
+    "vararg",
+    "varargs",
+    "varnames",
+    "warningregistry",
     "warnopts",
     "weakproxy",
-    "mappingproxy",
+    "xopts",
     // RustPython
-    "RustPython",
+    "baseclass",
+    "Bytecode",
+    "cfgs",
+    "codegen",
+    "dedentations",
+    "dedents",
+    "deduped",
+    "downcasted",
+    "dumpable",
+    "GetSet",
+    "internable",
+    "makeunicodedata",
+    "miri",
+    "notrace",
+    "pyarg",
     "pyarg",
     "pyargs",
-    "pygetset",
-    "pyobj",
-    "pystr",
-    "pyc",
-    "pyref",
-    "pyslot",
-    "PyFunction",
-    "PyMethod",
-    "PyClassMethod",
-    "PyStaticMethod",
-    "PyProperty",
-    "PyClass",
-    "pyimpl",
-    "pyarg",
-    "PyModule",
     "PyAttr",
-    "PyResult",
+    "pyc",
-    "PyObject",
+    "PyClass",
+    "PyClassMethod",
     "PyException",
-    "GetSet",
+    "PyFunction",
-    "zelf",
+    "pygetset",
-    "wasi",
+    "pyimpl",
-    "Bytecode",
+    "pymember",
+    "PyMethod",
+    "PyModule",
+    "pyname",
+    "pyobj",
+    "PyObject",
+    "pypayload",
+    "PyProperty",
+    "pyref",
+    "PyResult",
+    "pyslot",
+    "PyStaticMethod",
+    "pystr",
+    "pystruct",
+    "pystructseq",
+    "pytrace",
+    "reducelib",
     "richcompare",
-    "makeunicodedata",
+    "RustPython",
-    "unhashable",
+    "struc",
-    "unraisable",
+    "tracebacks",
     "typealiases",
     "Unconstructible",
-    "posonlyargs",
+    "unhashable",
-    "kwonlyargs",
     "uninit",
-    "miri"
+    "unraisable",
+    "wasi",
+    "zelf",
+    // cpython
+    "argtypes",
+    "asdl",
+    "asname",
+    "augassign",
+    "badsyntax",
+    "basetype",
+    "boolop",
+    "bxor",
+    "cellarg",
+    "cellvar",
+    "cellvars",
+    "cmpop",
+    "dictoffset",
+    "elts",
+    "excepthandler",
+    "finalbody",
+    "freevar",
+    "freevars",
+    "fromlist",
+    "heaptype",
+    "IMMUTABLETYPE",
+    "kwonlyarg",
+    "kwonlyargs",
+    "linearise",
+    "maxdepth",
+    "mult",
+    "nkwargs",
+    "orelse",
+    "patma",
+    "posonlyarg",
+    "posonlyargs",
+    "prec",
+    "stackdepth",
+    "unaryop",
+    "unparse",
+    "unparser",
+    "VARKEYWORDS",
+    "varkwarg",
+    "wbits",
+    "withitem",
+    "withs"
   ],
   // flagWords - list of words to be always considered incorrect
   "flagWords": [

.gitattributes

@@ -1,8 +1,6 @@
 Lib/**      linguist-vendored
 Cargo.lock  linguist-generated -merge
 *.snap      linguist-generated -merge
-ast/src/ast_gen.rs          linguist-generated -merge
 vm/src/stdlib/ast/gen.rs    linguist-generated -merge
-compiler/parser/python.lalrpop       text eol=LF
 Lib/*.py    text working-tree-encoding=UTF-8 eol=LF
 **/*.rs     text working-tree-encoding=UTF-8 eol=LF

.github/ISSUE_TEMPLATE/empty.md

@@ -0,0 +1,16 @@
+---
+name: Generic issue template
+about: which is not covered by other templates
+title: ''
+labels: 
+assignees: ''
+
+---
+
+## Summary
+
+<!-- Short description of the issue. -->
+
+## Details
+
+<!-- Whatever you want to share -->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,16 @@
+---
+name: Feature request
+about: Request a feature to use RustPython (as a Rust library)
+title: ''
+labels: C-enhancement
+assignees: 'youknowone'
+
+---
+
+## Summary
+
+<!-- Short description of the request. Please use incompatibility form to report missing features as Python interpreter -->
+
+## Expected use case
+
+<!-- By sharing detailed use case, we can understand the requirements better! If it will be used by open source projects, please also share the project URL. -->

.github/ISSUE_TEMPLATE/report-bug.md

@@ -0,0 +1,24 @@
+---
+name: Report bugs
+about: Report a bug not related to CPython compatibility
+title: ''
+labels: C-bug
+assignees: ''
+
+---
+
+## Summary
+
+<!-- Short description of the bug -->
+
+## Expected
+
+<!-- What's the expected result? Using ``` ``` block is preferred for text. -->
+
+## Actual
+
+<!-- What's the actual result? Using ``` ``` block is preferred for text. -->
+
+## Python Documentation
+
+<!-- If applicable. -->

.github/ISSUE_TEMPLATE/report-incompatibility.md

@@ -2,7 +2,7 @@
 name: Report incompatibility
 about: Report an incompatibility between RustPython and CPython
 title: ''
-labels: feat
+labels: C-compat
 assignees: ''
 
 ---
@@ -11,6 +11,6 @@ assignees: ''
 
 <!-- What Python feature is missing from RustPython? Give a short description of the feature and how you ran into its absence. -->
 
-## Python Documentation
+## Python Documentation or reference to CPython source code
 
 <!-- Give a link to the feature in the CPython documentation (https://docs.python.org/3/) in order to assist in its implementation. -->

.github/workflows/ci.yaml

@@ -2,7 +2,8 @@ on:
   push:
     branches: [main, release]
   pull_request:
-    types: [labeled, unlabeled, opened, synchronize, reopened]
+    types: [unlabeled, opened, synchronize, reopened]
+  merge_group:
 
 name: CI
 
@@ -15,17 +16,23 @@ concurrency:
 
 env:
   CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
-  NON_WASM_PACKAGES: >-
+  # Skip additional tests on Windows. They are checked on Linux and MacOS.
-    -p rustpython-common
+  WINDOWS_SKIPS: >-
-    -p rustpython-compiler-core
+    test_glob
-    -p rustpython-compiler
+    test_importlib
-    -p rustpython-codegen
+    test_io
-    -p rustpython-parser
+    test_os
-    -p rustpython-vm
+    test_pathlib
-    -p rustpython-stdlib
+    test_posixpath
-    -p rustpython-jit
+    test_shutil
-    -p rustpython-derive
+    test_venv
-    -p rustpython
+  # configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417
+  # socketserver: seems related to configparser crash.
+  MACOS_SKIPS: >-
+    test_configparser
+    test_socketserver
+  # PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently
+  # only run on Linux to speed up the CI.
   PLATFORM_INDEPENDENT_TESTS: >-
     test_argparse
     test_array
@@ -97,6 +104,8 @@ env:
     test_unpack
     test_weakref
     test_yield_from
+  # Python version targeted by the CI.
+  PYTHON_VERSION: "3.12.0"
 
 jobs:
   rust_tests:
@@ -104,38 +113,39 @@ jobs:
     env:
       RUST_BACKTRACE: full
     name: Run rust tests
-    needs: lalrpop
     runs-on:  ${{ matrix.os }}
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
+      - uses: Swatinem/rust-cache@v2
+
       - name: Set up the Windows environment
         shell: bash
         run: |
-          choco install llvm openssl
+          cargo install --target-dir=target -v cargo-vcpkg
-          echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >>$GITHUB_ENV
+          cargo vcpkg -v build
         if: runner.os == 'Windows'
       - name: Set up the Mac environment
         run: brew install autoconf automake libtool
         if: runner.os == 'macOS'
-      - uses: Swatinem/rust-cache@v1
 
       - name: run clippy
-        run: cargo clippy ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }} -- -Dwarnings
+        run: cargo clippy ${{ env.CARGO_ARGS }} --workspace  --exclude rustpython_wasm -- -Dwarnings
 
       - name: run rust tests
-        run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }}
+        run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }}
+        if: runner.os != 'macOS'  
+      # temp skip ssl linking for Mac to avoid CI failure
+      - name: run rust tests (MacOS no ssl)
+        run: cargo test --workspace --exclude rustpython_wasm --verbose --no-default-features --features threading,stdlib,zlib,importlib,encodings,jit
+        if: runner.os == 'macOS'
+
       - name: check compilation without threading
         run: cargo check ${{ env.CARGO_ARGS }}
 
@@ -159,16 +169,9 @@ jobs:
   exotic_targets:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Ensure compilation on various targets
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: i686-unknown-linux-gnu
@@ -185,6 +188,15 @@ jobs:
       - name: Check compilation for android
         run: cargo check --target aarch64-linux-android
 
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          target: aarch64-unknown-linux-gnu
+
+      - name: Install gcc-aarch64-linux-gnu
+        run: sudo apt install gcc-aarch64-linux-gnu
+      - name: Check compilation for aarch64 linux gnu
+        run: cargo check --target aarch64-unknown-linux-gnu
+
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: i686-unknown-linux-musl
@@ -216,14 +228,12 @@ jobs:
       - name: Prepare repository for redox compilation
         run: bash scripts/redox/uncomment-cargo.sh
       - name: Check compilation for Redox
-        if: false # FIXME: redoxer toolchain is from ~july 2021, edition2021 isn't stabilized
         uses: coolreader18/redoxer-action@v1
         with:
           command: check
 
   snippets_cpython:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    needs: lalrpop
     env:
       RUST_BACKTRACE: full
     name: Run snippets and cpython tests
@@ -233,32 +243,27 @@ jobs:
         os: [macos-latest, ubuntu-latest, windows-latest]
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-
       - uses: dtolnay/rust-toolchain@stable
-      - uses: actions/setup-python@v2
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: Set up the Windows environment
         shell: bash
         run: |
-          choco install llvm openssl
+          cargo install cargo-vcpkg
-          echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >>$GITHUB_ENV
+          cargo vcpkg build
         if: runner.os == 'Windows'
       - name: Set up the Mac environment
-        run: brew install autoconf automake libtool
+        run: brew install autoconf automake libtool openssl@3
         if: runner.os == 'macOS'
-      - uses: Swatinem/rust-cache@v1
+
       - name: build rustpython
         run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }}
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: run snippets
         run: python -m pip install -r requirements.txt && pytest -v
         working-directory: ./extra_tests
@@ -266,113 +271,71 @@ jobs:
         name: run cpython platform-independent tests
         run:
           target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }}
-      - if: runner.os != 'Windows'
+      - if: runner.os == 'Linux'
-        name: run cpython platform-dependent tests
+        name: run cpython platform-dependent tests (Linux)
         run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }}
+      - if: runner.os == 'macOS'
+        name: run cpython platform-dependent tests (MacOS)
+        run: target/release/rustpython -m test -j 1 all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.MACOS_SKIPS }}
       - if: runner.os == 'Windows'
         name: run cpython platform-dependent tests (windows partial - fixme)
         run:
-          target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }}
+          target/release/rustpython -m test -j 1 all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }}
-            test_glob
-            test_importlib
-            test_io
-            test_iter
-            test_os
-            test_pathlib
-            test_posixpath
-            test_shutil
-            test_venv
       - if: runner.os != 'Windows'
         name: check that --install-pip succeeds
         run: |
           mkdir site-packages
           target/release/rustpython --install-pip ensurepip --user
-
+      - if: runner.os != 'Windows'
-  lalrpop:
+        name: Check that ensurepip succeeds.
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    name: Generate parser with lalrpop
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest]
-    runs-on:  ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - name: Check if cached generated parser exists
-        id: generated_parser
-        uses: andstor/file-existence-action@v1
-        with:
-          files: "compiler/parser/python.rs"
-      - if: runner.os == 'Windows'
-        name: Force python.lalrpop to be lf  # actions@checkout ignore .gitattributes
         run: |
-          set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
+          target/release/rustpython -m ensurepip
-      - name: Install lalrpop
+          target/release/rustpython -c "import pip"
-        if: steps.generated_parser.outputs.files_exists == 'false'
+      - if: runner.os != 'Windows'
-        uses: baptiste0928/cargo-install@v1
+        name: Check if pip inside venv is functional
-        with:
+        run: |
-          crate: lalrpop
+          target/release/rustpython -m venv testvenv
-          version: "0.19.8"
+          testvenv/bin/rustpython -m pip install wheel
-      - name: Run lalrpop
+      - name: Check whats_left is not broken
-        if: steps.generated_parser.outputs.files_exists == 'false'
+        run: python -I whats_left.py
-        run: lalrpop compiler/parser/python.lalrpop
 
   lint:
     name: Check Rust code with rustfmt and clippy
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
         with:
             components: rustfmt, clippy
       - name: run rustfmt
-        run: cargo fmt --all -- --check
+        run: cargo fmt --check
       - name: run clippy on wasm
         run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: ${{ env.PYTHON_VERSION }}
-      - name: install flake8
+      - name: install ruff
-        run: python -m pip install flake8
+        run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778
-      - name: run lint
+      - name: run python lint
-        run: flake8 . --count --exclude=./.*,./Lib,./vm/Lib,./benches/  --select=E9,F63,F7,F82 --show-source --statistics
+        run: ruff extra_tests wasm examples  --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source
       - name: install prettier
         run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH
       - name: check wasm code with prettier
         # prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506
         run: cd wasm && git ls-files -z | xargs -0 prettier --check -u
-      - name: Check update_asdl.sh consistency
-        run: bash scripts/update_asdl.sh && git diff --exit-code
-      - name: Check whats_left is not broken
-        run: python -I whats_left.py
 
   miri:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Run tests under miri
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@master
         with:
             toolchain: nightly
             components: miri
-      - uses: Swatinem/rust-cache@v1
+
+      - uses: Swatinem/rust-cache@v2
       - name: Run tests under miri
         # miri-ignore-leaks because the type-object circular reference means that there will always be
         # a memory leak, at least until we have proper cyclic gc
@@ -381,17 +344,12 @@ jobs:
   wasm:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Check the WASM package and demo
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v1
+
+      - uses: Swatinem/rust-cache@v2
       - name: install wasm-pack
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: install geckodriver
@@ -399,17 +357,19 @@ jobs:
           wget https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz
           mkdir geckodriver
           tar -xzf geckodriver-v0.30.0-linux64.tar.gz -C geckodriver
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: ${{ env.PYTHON_VERSION }}
       - run: python -m pip install -r requirements.txt
         working-directory: ./wasm/tests
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
       - name: run test
         run: |
           export PATH=$PATH:`pwd`/../../geckodriver
           npm install
           npm run test
+        env:
+          NODE_OPTIONS: "--openssl-legacy-provider"
         working-directory: ./wasm/demo
       - name: build notebook demo
         if: github.ref == 'refs/heads/release'
@@ -417,6 +377,8 @@ jobs:
           npm install
           npm run dist
           mv dist ../demo/dist/notebook
+        env:
+          NODE_OPTIONS: "--openssl-legacy-provider"
         working-directory: ./wasm/notebook
       - name: Deploy demo to Github Pages
         if: success() && github.ref == 'refs/heads/release'
@@ -430,24 +392,19 @@ jobs:
   wasm-wasi:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Run snippets and cpython tests on wasm-wasi
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: wasm32-wasi
-      - uses: Swatinem/rust-cache@v1
+
+      - uses: Swatinem/rust-cache@v2
       - name: Setup Wasmer
-        uses: wasmerio/setup-wasmer@v1
+        uses: wasmerio/setup-wasmer@v2
       - name: Install clang
         run: sudo apt-get update && sudo apt-get install clang -y
       - name: build rustpython
         run: cargo build --release --target wasm32-wasi --features freeze-stdlib,stdlib --verbose
       - name: run snippets
-        run: wasmer run --dir . target/wasm32-wasi/release/rustpython.wasm -- extra_tests/snippets/stdlib_random.py
+        run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py

.github/workflows/cron-ci.yaml

@@ -6,71 +6,43 @@ on:
 name: Periodic checks/tasks
 
 env:
-  CARGO_ARGS: --features ssl,jit
+  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
+  PYTHON_VERSION: "3.12.0"
 
 jobs:
+  # codecov collects code coverage data from the rust tests, python snippets and python test suite.
+  # This is done using cargo-llvm-cov, which is a wrapper around llvm-cov.
   codecov:
     name: Collect code coverage data
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
+      - uses: taiki-e/install-action@cargo-llvm-cov
+      - uses: actions/setup-python@v4
         with:
-          components: llvm-tools-preview
+          python-version: ${{ env.PYTHON_VERSION }}
       - run: sudo apt-get update && sudo apt-get -y install lcov
-      - run: cargo build --release --verbose ${{ env.CARGO_ARGS }}
+      - name: Run cargo-llvm-cov with Rust tests.
-        env:
+        run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
-          RUSTC_WRAPPER: './scripts/codecoverage-rustc-wrapper.sh'
+      - name: Run cargo-llvm-cov with Python snippets.
-      - uses: actions/setup-python@v2
+        run: python scripts/cargo-llvm-cov.py
-        with:
-          python-version: "3.10"
-      - run: python -m pip install pytest
-        working-directory: ./extra_tests
-      - name: run snippets
-        run: LLVM_PROFILE_FILE="$PWD/snippet-%p.profraw" pytest -v
-        working-directory: ./extra_tests
         continue-on-error: true
-      - name: run cpython tests
+      - name: Run cargo-llvm-cov with Python test suite.
-        run: |
+        run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed 
-          alltests=($(target/release/rustpython -c 'from test.libregrtest.runtest import findtests; print(*findtests())'))
-          i=0
-          # chunk into chunks of 10 tests each. idk at this point
-          while subtests=("${alltests[@]:$i:10}"); [[ ${#subtests[@]} -ne 0 ]]; do
-            LLVM_PROFILE_FILE="$PWD/regrtest-%p.profraw" target/release/rustpython -m test -v "${subtests[@]}" || true
-            ((i+=10))
-          done
         continue-on-error: true
-      - name: prepare code coverage data
+      - name: Prepare code coverage data
-        run: |
+        run: cargo llvm-cov report --lcov --output-path='codecov.lcov'
-          rusttool() {
+      - name: Upload to Codecov
-            local tool=$1; shift; "$(rustc --print target-libdir)/../bin/llvm-$tool" "$@"
-          }
-          rusttool profdata merge extra_tests/snippet-*.profraw regrtest-*.profraw --output codecov.profdata
-          rusttool cov export --instr-profile codecov.profdata target/release/rustpython --format lcov > codecov_tmp.lcov
-          lcov -e codecov_tmp.lcov "$PWD"/'*' -o codecov_tmp2.lcov
-          lcov -r codecov_tmp2.lcov "$PWD"/target/'*' -o codecov.lcov  # remove LALRPOP-generated parser
-      - name: upload to Codecov
         uses: codecov/codecov-action@v3
         with:
           file: ./codecov.lcov
 
   testdata:
     name: Collect regression test data
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
       - name: build rustpython
         run: cargo build --release --verbose
@@ -97,16 +69,13 @@ jobs:
 
   whatsleft:
     name: Collect what is left data
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: build rustpython
         run: cargo build --release --verbose
       - name: Collect what is left data
@@ -135,17 +104,11 @@ jobs:
 
   benchmark:
     name: Collect benchmark data
-    needs: lalrpop
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.9
       - run: cargo install cargo-criterion
@@ -183,35 +146,3 @@ jobs:
           if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then
             git push
           fi
-
-  lalrpop:
-    name: Generate parser with lalrpop
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest]
-    runs-on:  ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Cache generated parser
-        uses: actions/cache@v2
-        with:
-          path: compiler/parser/python.rs
-          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
-      - name: Check if cached generated parser exists
-        id: generated_parser
-        uses: andstor/file-existence-action@v1
-        with:
-          files: "compiler/parser/python.rs"
-      - if: runner.os == 'Windows'
-        name: Force python.lalrpop to be lf  # actions@checkout ignore .gitattributes
-        run: |
-          set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
-      - name: Install lalrpop
-        if: steps.generated_parser.outputs.files_exists == 'false'
-        uses: baptiste0928/cargo-install@v1
-        with:
-          crate: lalrpop
-          version: "0.19.8"
-      - name: Run lalrpop
-        if: steps.generated_parser.outputs.files_exists == 'false'
-        run: lalrpop compiler/parser/python.lalrpop

.gitignore

@@ -19,5 +19,3 @@ flamescope.json
 
 extra_tests/snippets/resources
 extra_tests/not_impl.py
-
-compiler/parser/python.rs

.vscode/launch.json

@@ -8,15 +8,24 @@
             "type": "lldb",
             "request": "launch",
             "name": "Debug executable 'rustpython'",
-            "cargo": {
-                "args": [
-                    "build",
-                    "--package=rustpython"
-                ],
-            },
             "preLaunchTask": "Build RustPython Debug",
             "program": "target/debug/rustpython",
             "args": [],
+            "env": {
+                "RUST_BACKTRACE": "1"
+            },
+            "cwd": "${workspaceFolder}"
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug executable 'rustpython' without SSL",
+            "preLaunchTask": "Build RustPython Debug without SSL",
+            "program": "target/debug/rustpython",
+            "args": [],
+            "env": {
+                "RUST_BACKTRACE": "1"
+            },
             "cwd": "${workspaceFolder}"
         },
         {

.vscode/tasks.json

@@ -2,7 +2,7 @@
     "version": "2.0.0",
     "tasks": [
         {
-            "label": "Build RustPython Debug",
+            "label": "Build RustPython Debug without SSL",
             "type": "shell",
             "command": "cargo",
             "args": [
@@ -15,6 +15,22 @@
                 "kind": "build",
                 "isDefault": true,
             },
-        }
+        },
+        {
+            "label": "Build RustPython Debug",
+            "type": "shell",
+            "command": "cargo",
+            "args": [
+                "build",
+                "--features=ssl"
+            ],
+            "problemMatcher": [
+                "$rustc",
+            ],
+            "group": {
+                "kind": "build",
+                "isDefault": true,
+            },
+        },
     ],
 }

Cargo.toml

@@ -1,11 +1,9 @@
-# REDOX START
-# cargo-features = ["edition2021"]
-# REDOX END
 [package]
 name = "rustpython"
-version = "0.2.0"
+version = "0.3.1"
 authors = ["RustPython Team"]
 edition = "2021"
+rust-version = "1.67.1"
 description = "A python interpreter written in rust."
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
@@ -14,15 +12,80 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 [workspace]
 resolver = "2"
 members = [
-    "compiler", "compiler/ast", "compiler/core", "compiler/codegen", "compiler/parser",
+    "compiler", "compiler/core", "compiler/codegen",
-    ".", "common", "derive", "jit", "vm", "pylib", "stdlib", "wasm/lib", "derive-impl",
+    ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "wasm/lib", "derive-impl",
 ]
 
+[workspace.dependencies]
+rustpython-compiler-core = { path = "compiler/core", version = "0.3.1" }
+rustpython-compiler = { path = "compiler", version = "0.3.1" }
+rustpython-codegen = { path = "compiler/codegen", version = "0.3.1" }
+rustpython-common = { path = "common", version = "0.3.1" }
+rustpython-derive = { path = "derive", version = "0.3.1" }
+rustpython-derive-impl = { path = "derive-impl", version = "0.3.1" }
+rustpython-jit = { path = "jit", version = "0.3.1" }
+rustpython-vm = { path = "vm", default-features = false, version = "0.3.1" }
+rustpython-pylib = { path = "pylib", version = "0.3.1" }
+rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.3.1" }
+rustpython-sre_engine = { path = "vm/sre_engine", version = "0.3.1" }
+rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
+
+rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
+rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
+rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
+rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
+rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
+# rustpython-literal = { path = "../RustPython-parser/literal" }
+# rustpython-parser-core = { path = "../RustPython-parser/core" }
+# rustpython-parser = { path = "../RustPython-parser/parser" }
+# rustpython-ast = { path = "../RustPython-parser/ast" }
+# rustpython-format = { path = "../RustPython-parser/format" }
+
+ahash = "0.8.11"
+ascii = "1.0"
+atty = "0.2.14"
+bitflags = "2.4.1"
+bstr = "0.2.17"
+cfg-if = "1.0"
+chrono = "0.4.37"
+crossbeam-utils = "0.8.19"
+flame = "0.2.2"
+glob = "0.3"
+hex = "0.4.3"
+indexmap = { version = "2.2.6", features = ["std"] }
+insta = "1.38.0"
+itertools = "0.11.0"
+is-macro = "0.3.0"
+libc = "0.2.153"
+log = "0.4.16"
+nix = { version = "0.27", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
+malachite-bigint = "0.2.0"
+malachite-q = "0.4.4"
+malachite-base = "0.4.4"
+memchr = "2.7.2"
+num-complex = "0.4.0"
+num-integer = "0.1.44"
+num-traits = "0.2"
+num_enum = "0.7"
+once_cell = "1.19.0"
+parking_lot = "0.12.1"
+paste = "1.0.7"
+rand = "0.8.5"
+rustyline = "14.0.0"
+serde = { version = "1.0.133", default-features = false }
+schannel = "0.1.22"
+static_assertions = "1.1"
+syn = "1.0.109"
+thiserror = "1.0"
+thread_local = "1.1.4"
+unicode_names2 = "1.1.0"
+widestring = "1.1.0"
+
 [features]
-default = ["threading", "stdlib", "zlib", "importlib", "encodings", "rustpython-parser/lalrpop"]
+default = ["threading", "stdlib", "zlib", "importlib"]
 importlib = ["rustpython-vm/importlib"]
 encodings = ["rustpython-vm/encodings"]
-stdlib = ["rustpython-stdlib", "rustpython-pylib"]
+stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"]
 flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
 freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
@@ -33,30 +96,31 @@ ssl = ["rustpython-stdlib/ssl"]
 ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
 
 [dependencies]
-rustpython-compiler = { path = "compiler", version = "0.2.0" }
+rustpython-compiler = { workspace = true }
-rustpython-parser = { path = "compiler/parser", version = "0.2.0" }
+rustpython-pylib = { workspace = true, optional = true }
-rustpython-pylib = { path = "pylib", optional = true, default-features = false }
+rustpython-stdlib = { workspace = true, optional = true }
-rustpython-stdlib = { path = "stdlib", optional = true, default-features = false }
+rustpython-vm = { workspace = true, features = ["compiler"] }
-rustpython-vm = { path = "vm", version = "0.2.0", default-features = false, features = ["compiler"] }
+rustpython-parser = { workspace = true }
+
+atty = { workspace = true }
+cfg-if = { workspace = true }
+log = { workspace = true }
+flame = { workspace = true, optional = true }
 
-cfg-if = "1.0.0"
 clap = "2.34"
 dirs = { package = "dirs-next", version = "2.0.0" }
 env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
-flame = { version = "0.2.2", optional = true }
 flamescope = { version = "0.1.2", optional = true }
-libc = "0.2.133"
+
-log = "0.4.16"
+[target.'cfg(windows)'.dependencies]
-num-traits = "0.2.14"
+libc = { workspace = true }
-atty = "0.2.14"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-rustyline = "10.0.0"
+rustyline = { workspace = true }
 
 [dev-dependencies]
-cpython = "0.7.0"
+criterion = { version = "0.3.5", features = ["html_reports"] }
-criterion = "0.3.5"
+pyo3 = { version = "0.20.2", features = ["auto-initialize"] }
-python3-sys = "0.7.0"
 
 [[bench]]
 name = "execution"
@@ -87,5 +151,15 @@ opt-level = 3
 lto = "thin"
 
 [patch.crates-io]
-# REDOX START, Uncommment when you want to compile/check with redoxer
+# REDOX START, Uncomment when you want to compile/check with redoxer
 # REDOX END
+
+# Used only on Windows to build the vcpkg dependencies
+[package.metadata.vcpkg]
+git = "https://github.com/microsoft/vcpkg"
+# The revision of the vcpkg repository to use
+# https://github.com/microsoft/vcpkg/tags
+rev = "2024.02.14"
+
+[package.metadata.vcpkg.target]
+x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] }

DEVELOPMENT.md

@@ -19,13 +19,13 @@ The contents of the Development Guide include:
 
 RustPython requires the following:
 
-- Rust latest stable version (e.g 1.51.0 as of Apr 2 2021)
+- Rust latest stable version (e.g 1.69.0 as of Apr 20 2023)
     - To check Rust version: `rustc --version` 
     - If you have `rustup` on your system, enter to update to the latest
       stable version: `rustup update stable`
     - If you do not have Rust installed, use [rustup](https://rustup.rs/) to
       do so.
-- CPython version 3.10 or higher
+- CPython version 3.12 or higher
     - CPython can be installed by your operating system's package manager,
       from the [Python website](https://www.python.org/downloads/), or
       using a third-party distribution, such as 
@@ -47,7 +47,10 @@ you can check yourself with `cargo clippy`.
 
 Custom Python code (i.e. code not copied from CPython's standard library) should
 follow the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style. We also use
-[flake8](http://flake8.pycqa.org/en/latest/) to check Python code style.
+[ruff](https://beta.ruff.rs/docs/) to check Python code style.
+
+In addition to language specific tools, [cspell](https://github.com/streetsidesoftware/cspell),
+a code spell checker, is used in order to ensure correct spellings for code.
 
 ## Testing
 

Lib/__hello__.py

@@ -0,0 +1,16 @@
+initialized = True
+
+class TestFrozenUtf8_1:
+    """\u00b6"""
+
+class TestFrozenUtf8_2:
+    """\u03c0"""
+
+class TestFrozenUtf8_4:
+    """\U0001f600"""
+
+def main():
+    print("Hello world!")
+
+if __name__ == '__main__':
+    main()

Lib/__phello__/__init__.py

@@ -0,0 +1,7 @@
+initialized = True
+
+def main():
+    print("Hello world!")
+
+if __name__ == '__main__':
+    main()

Lib/__phello__/spam.py

@@ -0,0 +1,7 @@
+initialized = True
+
+def main():
+    print("Hello world!")
+
+if __name__ == '__main__':
+    main()

Lib/_collections_abc.py

@@ -6,6 +6,32 @@
 Unit tests are in test_collections.
 """
 
+############ Maintenance notes #########################################
+#
+# ABCs are different from other standard library modules in that they
+# specify compliance tests.  In general, once an ABC has been published,
+# new methods (either abstract or concrete) cannot be added.
+#
+# Though classes that inherit from an ABC would automatically receive a
+# new mixin method, registered classes would become non-compliant and
+# violate the contract promised by ``isinstance(someobj, SomeABC)``.
+#
+# Though irritating, the correct procedure for adding new abstract or
+# mixin methods is to create a new ABC as a subclass of the previous
+# ABC.  For example, union(), intersection(), and difference() cannot
+# be added to Set but could go into a new ABC that extends Set.
+#
+# Because they are so hard to change, new ABCs should have their APIs
+# carefully thought through prior to publication.
+#
+# Since ABCMeta only checks for the presence of methods, it is possible
+# to alter the signature of a method by adding optional arguments
+# or changing parameters names.  This is still a bit dubious but at
+# least it won't cause isinstance() to return an incorrect result.
+#
+#
+#######################################################################
+
 from abc import ABCMeta, abstractmethod
 import sys
 
@@ -23,7 +49,7 @@ __all__ = ["Awaitable", "Coroutine",
            "Mapping", "MutableMapping",
            "MappingView", "KeysView", "ItemsView", "ValuesView",
            "Sequence", "MutableSequence",
-           "ByteString",
+           "ByteString", "Buffer",
            ]
 
 # This module has been renamed from collections.abc to _collections_abc to
@@ -413,6 +439,21 @@ class Collection(Sized, Iterable, Container):
         return NotImplemented
 
 
+class Buffer(metaclass=ABCMeta):
+
+    __slots__ = ()
+
+    @abstractmethod
+    def __buffer__(self, flags: int, /) -> memoryview:
+        raise NotImplementedError
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Buffer:
+            return _check_methods(C, "__buffer__")
+        return NotImplemented
+
+
 class _CallableGenericAlias(GenericAlias):
     """ Represent `Callable[argtypes, resulttype]`.
 
@@ -430,25 +471,13 @@ class _CallableGenericAlias(GenericAlias):
             raise TypeError(
                 "Callable must be used as Callable[[arg, ...], result].")
         t_args, t_result = args
-        if isinstance(t_args, list):
+        if isinstance(t_args, (tuple, list)):
             args = (*t_args, t_result)
         elif not _is_param_expr(t_args):
             raise TypeError(f"Expected a list of types, an ellipsis, "
                             f"ParamSpec, or Concatenate. Got {t_args}")
         return super().__new__(cls, origin, args)
 
-    @property
-    def __parameters__(self):
-        params = []
-        for arg in self.__args__:
-            # Looks like a genericalias
-            if hasattr(arg, "__parameters__") and isinstance(arg.__parameters__, tuple):
-                params.extend(arg.__parameters__)
-            else:
-                if _is_typevarlike(arg):
-                    params.append(arg)
-        return tuple(dict.fromkeys(params))
-
     def __repr__(self):
         if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]):
             return super().__repr__()
@@ -467,55 +496,18 @@ class _CallableGenericAlias(GenericAlias):
         # rather than the default types.GenericAlias object.  Most of the
         # code is copied from typing's _GenericAlias and the builtin
         # types.GenericAlias.
-
-        # A special case in PEP 612 where if X = Callable[P, int],
-        # then X[int, str] == X[[int, str]].
-        param_len = len(self.__parameters__)
-        if param_len == 0:
-            raise TypeError(f'{self} is not a generic class')
         if not isinstance(item, tuple):
             item = (item,)
-        if (param_len == 1 and _is_param_expr(self.__parameters__[0])
+
-                and item and not _is_param_expr(item[0])):
+        new_args = super().__getitem__(item).__args__
-            item = (list(item),)
-        item_len = len(item)
-        if item_len != param_len:
-            raise TypeError(f'Too {"many" if item_len > param_len else "few"}'
-                            f' arguments for {self};'
-                            f' actual {item_len}, expected {param_len}')
-        subst = dict(zip(self.__parameters__, item))
-        new_args = []
-        for arg in self.__args__:
-            if _is_typevarlike(arg):
-                if _is_param_expr(arg):
-                    arg = subst[arg]
-                    if not _is_param_expr(arg):
-                        raise TypeError(f"Expected a list of types, an ellipsis, "
-                                        f"ParamSpec, or Concatenate. Got {arg}")
-                else:
-                    arg = subst[arg]
-            # Looks like a GenericAlias
-            elif hasattr(arg, '__parameters__') and isinstance(arg.__parameters__, tuple):
-                subparams = arg.__parameters__
-                if subparams:
-                    subargs = tuple(subst[x] for x in subparams)
-                    arg = arg[subargs]
-            new_args.append(arg)
 
         # args[0] occurs due to things like Z[[int, str, bool]] from PEP 612
-        if not isinstance(new_args[0], list):
+        if not isinstance(new_args[0], (tuple, list)):
             t_result = new_args[-1]
             t_args = new_args[:-1]
             new_args = (t_args, t_result)
         return _CallableGenericAlias(Callable, tuple(new_args))
 
-
-def _is_typevarlike(arg):
-    obj = type(arg)
-    # looks like a TypeVar/ParamSpec
-    return (obj.__module__ == 'typing'
-            and obj.__name__ in {'ParamSpec', 'TypeVar'})
-
 def _is_param_expr(obj):
     """Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or
     ``_ConcatenateGenericAlias`` from typing.py
@@ -533,9 +525,8 @@ def _type_repr(obj):
 
     Copied from :mod:`typing` since collections.abc
     shouldn't depend on that module.
+    (Keep this roughly in sync with the typing version.)
     """
-    if isinstance(obj, GenericAlias):
-        return repr(obj)
     if isinstance(obj, type):
         if obj.__module__ == 'builtins':
             return obj.__qualname__
@@ -868,7 +859,7 @@ class KeysView(MappingView, Set):
     __slots__ = ()
 
     @classmethod
-    def _from_iterable(self, it):
+    def _from_iterable(cls, it):
         return set(it)
 
     def __contains__(self, key):
@@ -886,7 +877,7 @@ class ItemsView(MappingView, Set):
     __slots__ = ()
 
     @classmethod
-    def _from_iterable(self, it):
+    def _from_iterable(cls, it):
         return set(it)
 
     def __contains__(self, item):
@@ -1064,10 +1055,10 @@ class Sequence(Reversible, Collection):
         while stop is None or i < stop:
             try:
                 v = self[i]
-                if v is value or v == value:
-                    return i
             except IndexError:
                 break
+            if v is value or v == value:
+                return i
             i += 1
         raise ValueError
 
@@ -1080,8 +1071,27 @@ Sequence.register(str)
 Sequence.register(range)
 Sequence.register(memoryview)
 
+class _DeprecateByteStringMeta(ABCMeta):
+    def __new__(cls, name, bases, namespace, **kwargs):
+        if name != "ByteString":
+            import warnings
 
-class ByteString(Sequence):
+            warnings._deprecated(
+                "collections.abc.ByteString",
+                remove=(3, 14),
+            )
+        return super().__new__(cls, name, bases, namespace, **kwargs)
+
+    def __instancecheck__(cls, instance):
+        import warnings
+
+        warnings._deprecated(
+            "collections.abc.ByteString",
+            remove=(3, 14),
+        )
+        return super().__instancecheck__(instance)
+
+class ByteString(Sequence, metaclass=_DeprecateByteStringMeta):
     """This unifies bytes and bytearray.
 
     XXX Should add all their methods.

Lib/_compression.py

@@ -1,7 +1,7 @@
 """Internal classes used by the gzip, lzma and bz2 modules"""
 
 import io
-
+import sys
 
 BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE  # Compressed data read chunk size
 
@@ -110,6 +110,16 @@ class DecompressReader(io.RawIOBase):
         self._pos += len(data)
         return data
 
+    def readall(self):
+        chunks = []
+        # sys.maxsize means the max length of output buffer is unlimited,
+        # so that the whole input buffer can be decompressed within one
+        # .decompress() call.
+        while data := self.read(sys.maxsize):
+            chunks.append(data)
+
+        return b"".join(chunks)
+
     # Rewind the file to the beginning of the data stream.
     def _rewind(self):
         self._fp.seek(0)

Lib/_dummy_thread.py

@@ -145,6 +145,9 @@ class LockType(object):
     def locked(self):
         return self.locked_status
 
+    def _at_fork_reinit(self):
+        self.locked_status = False
+
     def __repr__(self):
         return "<%s %s.%s object at %s>" % (
             "locked" if self.locked_status else "unlocked",

Lib/_markupbase.py

@@ -29,10 +29,6 @@ class ParserBase:
             raise RuntimeError(
                 "_markupbase.ParserBase must be subclassed")
 
-    def error(self, message):
-        raise NotImplementedError(
-            "subclasses of ParserBase must override error()")
-
     def reset(self):
         self.lineno = 1
         self.offset = 0
@@ -131,12 +127,11 @@ class ParserBase:
                     # also in data attribute specifications of attlist declaration
                     # also link type declaration subsets in linktype declarations
                     # also link attribute specification lists in link declarations
-                    self.error("unsupported '[' char in %s declaration" % decltype)
+                    raise AssertionError("unsupported '[' char in %s declaration" % decltype)
                 else:
-                    self.error("unexpected '[' char in declaration")
+                    raise AssertionError("unexpected '[' char in declaration")
             else:
-                self.error(
+                raise AssertionError("unexpected %r char in declaration" % rawdata[j])
-                    "unexpected %r char in declaration" % rawdata[j])
             if j < 0:
                 return j
         return -1 # incomplete
@@ -156,7 +151,9 @@ class ParserBase:
             # look for MS Office ]> ending
             match= _msmarkedsectionclose.search(rawdata, i+3)
         else:
-            self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
+            raise AssertionError(
+                'unknown status keyword %r in marked section' % rawdata[i+3:j]
+            )
         if not match:
             return -1
         if report:
@@ -168,7 +165,7 @@ class ParserBase:
     def parse_comment(self, i, report=1):
         rawdata = self.rawdata
         if rawdata[i:i+4] != '<!--':
-            self.error('unexpected call to parse_comment()')
+            raise AssertionError('unexpected call to parse_comment()')
         match = _commentclose.search(rawdata, i+4)
         if not match:
             return -1
@@ -192,7 +189,9 @@ class ParserBase:
                     return -1
                 if s != "<!":
                     self.updatepos(declstartpos, j + 1)
-                    self.error("unexpected char in internal subset (in %r)" % s)
+                    raise AssertionError(
+                        "unexpected char in internal subset (in %r)" % s
+                    )
                 if (j + 2) == n:
                     # end of buffer; incomplete
                     return -1
@@ -209,8 +208,9 @@ class ParserBase:
                     return -1
                 if name not in {"attlist", "element", "entity", "notation"}:
                     self.updatepos(declstartpos, j + 2)
-                    self.error(
+                    raise AssertionError(
-                        "unknown declaration %r in internal subset" % name)
+                        "unknown declaration %r in internal subset" % name
+                    )
                 # handle the individual names
                 meth = getattr(self, "_parse_doctype_" + name)
                 j = meth(j, declstartpos)
@@ -234,14 +234,14 @@ class ParserBase:
                     if rawdata[j] == ">":
                         return j
                     self.updatepos(declstartpos, j)
-                    self.error("unexpected char after internal subset")
+                    raise AssertionError("unexpected char after internal subset")
                 else:
                     return -1
             elif c.isspace():
                 j = j + 1
             else:
                 self.updatepos(declstartpos, j)
-                self.error("unexpected char %r in internal subset" % c)
+                raise AssertionError("unexpected char %r in internal subset" % c)
         # end of buffer reached
         return -1
 
@@ -387,8 +387,9 @@ class ParserBase:
             return name.lower(), m.end()
         else:
             self.updatepos(declstartpos, i)
-            self.error("expected name token at %r"
+            raise AssertionError(
-                       % rawdata[declstartpos:declstartpos+20])
+                "expected name token at %r" % rawdata[declstartpos:declstartpos+20]
+            )
 
     # To be overridden -- handlers for unknown objects
     def unknown_decl(self, data):

Lib/abc.py

@@ -18,7 +18,7 @@ def abstractmethod(funcobj):
 
         class C(metaclass=ABCMeta):
             @abstractmethod
-            def my_abstract_method(self, ...):
+            def my_abstract_method(self, arg1, arg2, argN):
                 ...
     """
     funcobj.__isabstractmethod__ = True
@@ -106,7 +106,7 @@ else:
         implementations defined by the registering ABC be callable (not
         even via super()).
         """
-        def __new__(mcls, name, bases, namespace, **kwargs):
+        def __new__(mcls, name, bases, namespace, /, **kwargs):
             cls = super().__new__(mcls, name, bases, namespace, **kwargs)
             _abc_init(cls)
             return cls

Lib/aifc.py

@@ -138,7 +138,11 @@ import struct
 import builtins
 import warnings
 
-__all__ = ["Error", "open", "openfp"]
+__all__ = ["Error", "open"]
+
+
+warnings._deprecated(__name__, remove=(3, 13))
+
 
 class Error(Exception):
     pass
@@ -251,7 +255,9 @@ def _write_float(f, x):
     _write_ulong(f, himant)
     _write_ulong(f, lomant)
 
-from chunk import Chunk
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", DeprecationWarning)
+    from chunk import Chunk
 from collections import namedtuple
 
 _aifc_params = namedtuple('_aifc_params',
@@ -447,21 +453,33 @@ class Aifc_read:
     #
 
     def _alaw2lin(self, data):
-        import audioop
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', category=DeprecationWarning)
+            import audioop
         return audioop.alaw2lin(data, 2)
 
     def _ulaw2lin(self, data):
-        import audioop
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', category=DeprecationWarning)
+            import audioop
         return audioop.ulaw2lin(data, 2)
 
     def _adpcm2lin(self, data):
-        import audioop
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', category=DeprecationWarning)
+            import audioop
         if not hasattr(self, '_adpcmstate'):
             # first time
             self._adpcmstate = None
         data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
         return data
 
+    def _sowt2lin(self, data):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', category=DeprecationWarning)
+            import audioop
+        return audioop.byteswap(data, 2)
+
     def _read_comm_chunk(self, chunk):
         self._nchannels = _read_short(chunk)
         self._nframes = _read_long(chunk)
@@ -497,6 +515,8 @@ class Aifc_read:
                     self._convert = self._ulaw2lin
                 elif self._comptype in (b'alaw', b'ALAW'):
                     self._convert = self._alaw2lin
+                elif self._comptype in (b'sowt', b'SOWT'):
+                    self._convert = self._sowt2lin
                 else:
                     raise Error('unsupported compression type')
                 self._sampwidth = 2
@@ -659,7 +679,7 @@ class Aifc_write:
         if self._nframeswritten:
             raise Error('cannot change parameters after starting to write')
         if comptype not in (b'NONE', b'ulaw', b'ULAW',
-                            b'alaw', b'ALAW', b'G722'):
+                            b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
             raise Error('unsupported compression type')
         self._comptype = comptype
         self._compname = compname
@@ -680,7 +700,7 @@ class Aifc_write:
         if self._nframeswritten:
             raise Error('cannot change parameters after starting to write')
         if comptype not in (b'NONE', b'ulaw', b'ULAW',
-                            b'alaw', b'ALAW', b'G722'):
+                            b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
             raise Error('unsupported compression type')
         self.setnchannels(nchannels)
         self.setsampwidth(sampwidth)
@@ -764,28 +784,43 @@ class Aifc_write:
     #
 
     def _lin2alaw(self, data):
-        import audioop
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', category=DeprecationWarning)
+            import audioop
         return audioop.lin2alaw(data, 2)
 
     def _lin2ulaw(self, data):
-        import audioop
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', category=DeprecationWarning)
+            import audioop
         return audioop.lin2ulaw(data, 2)
 
     def _lin2adpcm(self, data):
-        import audioop
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', category=DeprecationWarning)
+            import audioop
         if not hasattr(self, '_adpcmstate'):
             self._adpcmstate = None
         data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
         return data
 
+    def _lin2sowt(self, data):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', category=DeprecationWarning)
+            import audioop
+        return audioop.byteswap(data, 2)
+
     def _ensure_header_written(self, datasize):
         if not self._nframeswritten:
-            if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
+            if self._comptype in (b'ULAW', b'ulaw',
+                b'ALAW', b'alaw', b'G722',
+                b'sowt', b'SOWT'):
                 if not self._sampwidth:
                     self._sampwidth = 2
                 if self._sampwidth != 2:
                     raise Error('sample width must be 2 when compressing '
-                                'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)')
+                                'with ulaw/ULAW, alaw/ALAW, sowt/SOWT '
+                                'or G7.22 (ADPCM)')
             if not self._nchannels:
                 raise Error('# channels not specified')
             if not self._sampwidth:
@@ -801,6 +836,8 @@ class Aifc_write:
             self._convert = self._lin2ulaw
         elif self._comptype in (b'alaw', b'ALAW'):
             self._convert = self._lin2alaw
+        elif self._comptype in (b'sowt', b'SOWT'):
+            self._convert = self._lin2sowt
 
     def _write_header(self, initlength):
         if self._aifc and self._comptype != b'NONE':
@@ -920,10 +957,6 @@ def open(f, mode=None):
     else:
         raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
 
-def openfp(f, mode=None):
-    warnings.warn("aifc.openfp is deprecated since Python 3.7. "
-                  "Use aifc.open instead.", DeprecationWarning, stacklevel=2)
-    return open(f, mode=mode)
 
 if __name__ == '__main__':
     import sys

Lib/antigravity.py

@@ -11,7 +11,7 @@ def geohash(latitude, longitude, datedow):
     37.857713 -122.544543
 
     '''
-    # http://xkcd.com/426/
+    # https://xkcd.com/426/
-    h = hashlib.md5(datedow).hexdigest()
+    h = hashlib.md5(datedow, usedforsecurity=False).hexdigest()
     p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])]
     print('%d%s %d%s' % (latitude, p[1:], longitude, q[1:]))

Lib/argparse.py

@@ -89,6 +89,8 @@ import os as _os
 import re as _re
 import sys as _sys
 
+import warnings
+
 from gettext import gettext as _, ngettext
 
 SUPPRESS = '==SUPPRESS=='
@@ -151,6 +153,7 @@ def _copy_items(items):
 # Formatting Help
 # ===============
 
+
 class HelpFormatter(object):
     """Formatter for generating usage messages and argument help strings.
 
@@ -342,21 +345,22 @@ class HelpFormatter(object):
                 def get_lines(parts, indent, prefix=None):
                     lines = []
                     line = []
+                    indent_length = len(indent)
                     if prefix is not None:
                         line_len = len(prefix) - 1
                     else:
-                        line_len = len(indent) - 1
+                        line_len = indent_length - 1
                     for part in parts:
                         if line_len + 1 + len(part) > text_width and line:
                             lines.append(indent + ' '.join(line))
                             line = []
-                            line_len = len(indent) - 1
+                            line_len = indent_length - 1
                         line.append(part)
                         line_len += len(part) + 1
                     if line:
                         lines.append(indent + ' '.join(line))
                     if prefix is not None:
-                        lines[0] = lines[0][len(indent):]
+                        lines[0] = lines[0][indent_length:]
                     return lines
 
                 # if prog is short, follow it with optionals or positionals
@@ -400,10 +404,18 @@ class HelpFormatter(object):
             except ValueError:
                 continue
             else:
-                end = start + len(group._group_actions)
+                group_action_count = len(group._group_actions)
+                end = start + group_action_count
                 if actions[start:end] == group._group_actions:
+
+                    suppressed_actions_count = 0
                     for action in group._group_actions:
                         group_actions.add(action)
+                        if action.help is SUPPRESS:
+                            suppressed_actions_count += 1
+
+                    exposed_actions_count = group_action_count - suppressed_actions_count
+
                     if not group.required:
                         if start in inserts:
                             inserts[start] += ' ['
@@ -413,7 +425,7 @@ class HelpFormatter(object):
                             inserts[end] += ']'
                         else:
                             inserts[end] = ']'
-                    else:
+                    elif exposed_actions_count > 1:
                         if start in inserts:
                             inserts[start] += ' ('
                         else:
@@ -487,7 +499,6 @@ class HelpFormatter(object):
         text = _re.sub(r'(%s) ' % open, r'\1', text)
         text = _re.sub(r' (%s)' % close, r'\1', text)
         text = _re.sub(r'%s *%s' % (open, close), r'', text)
-        text = _re.sub(r'\(([^|]*)\)', r'\1', text)
         text = text.strip()
 
         # return the text
@@ -693,8 +704,19 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
     """
 
     def _get_help_string(self, action):
+        """
+        Add the default value to the option help message.
+
+        ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't
+        already present. This code will do that, detecting cornercases to
+        prevent duplicates or cases where it wouldn't make sense to the end
+        user.
+        """
         help = action.help
-        if '%(default)' not in action.help:
+        if help is None:
+            help = ''
+
+        if '%(default)' not in help:
             if action.default is not SUPPRESS:
                 defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
                 if action.option_strings or action.nargs in defaulting_nargs:
@@ -702,6 +724,7 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
         return help
 
 
+
 class MetavarTypeHelpFormatter(HelpFormatter):
     """Help message formatter which uses the argument 'type' as the default
     metavar value (instead of the argument 'dest')
@@ -717,7 +740,6 @@ class MetavarTypeHelpFormatter(HelpFormatter):
         return action.type.__name__
 
 
-
 # =====================
 # Options and Arguments
 # =====================
@@ -752,7 +774,7 @@ class ArgumentError(Exception):
         if self.argument_name is None:
             format = '%(message)s'
         else:
-            format = 'argument %(argument_name)s: %(message)s'
+            format = _('argument %(argument_name)s: %(message)s')
         return format % dict(message=self.message,
                              argument_name=self.argument_name)
 
@@ -860,16 +882,20 @@ class Action(_AttributeHolder):
     def __call__(self, parser, namespace, values, option_string=None):
         raise NotImplementedError(_('.__call__() not defined'))
 
+
+# FIXME: remove together with `BooleanOptionalAction` deprecated arguments.
+_deprecated_default = object()
+
 class BooleanOptionalAction(Action):
     def __init__(self,
                  option_strings,
                  dest,
                  default=None,
-                 type=None,
+                 type=_deprecated_default,
-                 choices=None,
+                 choices=_deprecated_default,
                  required=False,
                  help=None,
-                 metavar=None):
+                 metavar=_deprecated_default):
 
         _option_strings = []
         for option_string in option_strings:
@@ -879,8 +905,23 @@ class BooleanOptionalAction(Action):
                 option_string = '--no-' + option_string[2:]
                 _option_strings.append(option_string)
 
-        if help is not None and default is not None and default is not SUPPRESS:
+        # We need `_deprecated` special value to ban explicit arguments that
-            help += " (default: %(default)s)"
+        # match default value. Like:
+        #   parser.add_argument('-f', action=BooleanOptionalAction, type=int)
+        for field_name in ('type', 'choices', 'metavar'):
+            if locals()[field_name] is not _deprecated_default:
+                warnings._deprecated(
+                    field_name,
+                    "{name!r} is deprecated as of Python 3.12 and will be "
+                    "removed in Python {remove}.",
+                    remove=(3, 14))
+
+        if type is _deprecated_default:
+            type = None
+        if choices is _deprecated_default:
+            choices = None
+        if metavar is _deprecated_default:
+            metavar = None
 
         super().__init__(
             option_strings=_option_strings,
@@ -893,6 +934,7 @@ class BooleanOptionalAction(Action):
             help=help,
             metavar=metavar)
 
+
     def __call__(self, parser, namespace, values, option_string=None):
         if option_string in self.option_strings:
             setattr(namespace, self.dest, not option_string.startswith('--no-'))
@@ -941,7 +983,7 @@ class _StoreConstAction(Action):
     def __init__(self,
                  option_strings,
                  dest,
-                 const,
+                 const=None,
                  default=None,
                  required=False,
                  help=None,
@@ -1036,7 +1078,7 @@ class _AppendConstAction(Action):
     def __init__(self,
                  option_strings,
                  dest,
-                 const,
+                 const=None,
                  default=None,
                  required=False,
                  help=None,
@@ -1168,6 +1210,13 @@ class _SubParsersAction(Action):
 
         aliases = kwargs.pop('aliases', ())
 
+        if name in self._name_parser_map:
+            raise ArgumentError(self, _('conflicting subparser: %s') % name)
+        for alias in aliases:
+            if alias in self._name_parser_map:
+                raise ArgumentError(
+                    self, _('conflicting subparser alias: %s') % alias)
+
         # create a pseudo-action to hold the choice help
         if 'help' in kwargs:
             help = kwargs.pop('help')
@@ -1648,6 +1697,14 @@ class _ArgumentGroup(_ActionsContainer):
         super(_ArgumentGroup, self)._remove_action(action)
         self._group_actions.remove(action)
 
+    def add_argument_group(self, *args, **kwargs):
+        warnings.warn(
+            "Nesting argument groups is deprecated.",
+            category=DeprecationWarning,
+            stacklevel=2
+        )
+        return super().add_argument_group(*args, **kwargs)
+
 
 class _MutuallyExclusiveGroup(_ArgumentGroup):
 
@@ -1668,6 +1725,14 @@ class _MutuallyExclusiveGroup(_ArgumentGroup):
         self._container._remove_action(action)
         self._group_actions.remove(action)
 
+    def add_mutually_exclusive_group(self, *args, **kwargs):
+        warnings.warn(
+            "Nesting mutually exclusive groups is deprecated.",
+            category=DeprecationWarning,
+            stacklevel=2
+        )
+        return super().add_mutually_exclusive_group(*args, **kwargs)
+
 
 class ArgumentParser(_AttributeHolder, _ActionsContainer):
     """Object for parsing command line strings into Python objects.
@@ -1857,8 +1922,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         if self.exit_on_error:
             try:
                 namespace, args = self._parse_known_args(args, namespace)
-            except ArgumentError:
+            except ArgumentError as err:
-                err = _sys.exc_info()[1]
                 self.error(str(err))
         else:
             namespace, args = self._parse_known_args(args, namespace)
@@ -1962,7 +2026,11 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                     # arguments, try to parse more single-dash options out
                     # of the tail of the option string
                     chars = self.prefix_chars
-                    if arg_count == 0 and option_string[1] not in chars:
+                    if (
+                        arg_count == 0
+                        and option_string[1] not in chars
+                        and explicit_arg != ''
+                    ):
                         action_tuples.append((action, [], option_string))
                         char = option_string[0]
                         option_string = char + explicit_arg[0]
@@ -2126,15 +2194,16 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
             # replace arguments referencing files with the file content
             else:
                 try:
-                    with open(arg_string[1:]) as args_file:
+                    with open(arg_string[1:],
+                              encoding=_sys.getfilesystemencoding(),
+                              errors=_sys.getfilesystemencodeerrors()) as args_file:
                         arg_strings = []
                         for arg_line in args_file.read().splitlines():
                             for arg in self.convert_arg_line_to_args(arg_line):
                                 arg_strings.append(arg)
                         arg_strings = self._read_args_from_files(arg_strings)
                         new_arg_strings.extend(arg_strings)
-                except OSError:
+                except OSError as err:
-                    err = _sys.exc_info()[1]
                     self.error(str(err))
 
         # return the modified argument list
@@ -2441,9 +2510,11 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
               not action.option_strings):
             if action.default is not None:
                 value = action.default
+                self._check_value(action, value)
             else:
+                # since arg_strings is always [] at this point
+                # there is no need to use self._check_value(action, value)
                 value = arg_strings
-            self._check_value(action, value)
 
         # single argument or optional argument produces a single value
         elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
@@ -2484,9 +2555,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
             result = type_func(arg_string)
 
         # ArgumentTypeErrors indicate errors
-        except ArgumentTypeError:
+        except ArgumentTypeError as err:
-            name = getattr(action.type, '__name__', repr(action.type))
+            msg = str(err)
-            msg = str(_sys.exc_info()[1])
             raise ArgumentError(action, msg)
 
         # TypeErrors or ValueErrors also indicate errors
@@ -2557,9 +2627,11 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
 
     def _print_message(self, message, file=None):
         if message:
-            if file is None:
+            file = file or _sys.stderr
-                file = _sys.stderr
+            try:
-            file.write(message)
+                file.write(message)
+            except (AttributeError, OSError):
+                pass
 
     # ===============
     # Exiting methods

Lib/ast.py

@@ -25,9 +25,10 @@
     :license: Python License.
 """
 import sys
+import re
 from _ast import *
 from contextlib import contextmanager, nullcontext
-from enum import IntEnum, auto
+from enum import IntEnum, auto, _simple_enum
 
 
 def parse(source, filename='<unknown>', mode='exec', *,
@@ -40,12 +41,13 @@ def parse(source, filename='<unknown>', mode='exec', *,
     flags = PyCF_ONLY_AST
     if type_comments:
         flags |= PyCF_TYPE_COMMENTS
-    if isinstance(feature_version, tuple):
+    if feature_version is None:
-        major, minor = feature_version  # Should be a 2-tuple.
-        assert major == 3
-        feature_version = minor
-    elif feature_version is None:
         feature_version = -1
+    elif isinstance(feature_version, tuple):
+        major, minor = feature_version  # Should be a 2-tuple.
+        if major != 3:
+            raise ValueError(f"Unsupported major version: {major}")
+        feature_version = minor
     # Else it should be an int giving the minor version for 3.x.
     return compile(source, filename, mode, flags,
                    _feature_version=feature_version)
@@ -53,10 +55,12 @@ def parse(source, filename='<unknown>', mode='exec', *,
 
 def literal_eval(node_or_string):
     """
-    Safely evaluate an expression node or a string containing a Python
+    Evaluate an expression node or a string containing only a Python
     expression.  The string or node provided may only consist of the following
     Python literal structures: strings, bytes, numbers, tuples, lists, dicts,
     sets, booleans, and None.
+
+    Caution: A complex expression can overflow the C stack and cause a crash.
     """
     if isinstance(node_or_string, str):
         node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval')
@@ -234,6 +238,12 @@ def increment_lineno(node, n=1):
     location in a file.
     """
     for child in walk(node):
+        # TypeIgnore is a special case where lineno is not an attribute
+        # but rather a field of the node itself.
+        if isinstance(child, TypeIgnore):
+            child.lineno = getattr(child, 'lineno', 0) + n
+            continue
+
         if 'lineno' in child._attributes:
             child.lineno = getattr(child, 'lineno', 0) + n
         if (
@@ -284,9 +294,7 @@ def get_docstring(node, clean=True):
     if not(node.body and isinstance(node.body[0], Expr)):
         return None
     node = node.body[0].value
-    if isinstance(node, Str):
+    if isinstance(node, Constant) and isinstance(node.value, str):
-        text = node.s
-    elif isinstance(node, Constant) and isinstance(node.value, str):
         text = node.value
     else:
         return None
@@ -296,28 +304,17 @@ def get_docstring(node, clean=True):
     return text
 
 
-def _splitlines_no_ff(source):
+_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))")
+def _splitlines_no_ff(source, maxlines=None):
     """Split a string into lines ignoring form feed and other chars.
 
     This mimics how the Python parser splits source code.
     """
-    idx = 0
     lines = []
-    next_line = ''
+    for lineno, match in enumerate(_line_pattern.finditer(source), 1):
-    while idx < len(source):
+        if maxlines is not None and lineno > maxlines:
-        c = source[idx]
+            break
-        next_line += c
+        lines.append(match[0])
-        idx += 1
-        # Keep \r\n together
-        if c == '\r' and idx < len(source) and source[idx] == '\n':
-            next_line += '\n'
-            idx += 1
-        if c in '\r\n':
-            lines.append(next_line)
-            next_line = ''
-
-    if next_line:
-        lines.append(next_line)
     return lines
 
 
@@ -351,7 +348,7 @@ def get_source_segment(source, node, *, padded=False):
     except AttributeError:
         return None
 
-    lines = _splitlines_no_ff(source)
+    lines = _splitlines_no_ff(source, maxlines=end_lineno+1)
     if end_lineno == lineno:
         return lines[lineno].encode()[col_offset:end_col_offset].decode()
 
@@ -500,20 +497,52 @@ class NodeTransformer(NodeVisitor):
         return node
 
 
+_DEPRECATED_VALUE_ALIAS_MESSAGE = (
+    "{name} is deprecated and will be removed in Python {remove}; use value instead"
+)
+_DEPRECATED_CLASS_MESSAGE = (
+    "{name} is deprecated and will be removed in Python {remove}; "
+    "use ast.Constant instead"
+)
+
+
 # If the ast module is loaded more than once, only add deprecated methods once
 if not hasattr(Constant, 'n'):
     # The following code is for backward compatibility.
     # It will be removed in future.
 
-    def _getter(self):
+    def _n_getter(self):
         """Deprecated. Use value instead."""
+        import warnings
+        warnings._deprecated(
+            "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14)
+        )
         return self.value
 
-    def _setter(self, value):
+    def _n_setter(self, value):
+        import warnings
+        warnings._deprecated(
+            "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14)
+        )
         self.value = value
 
-    Constant.n = property(_getter, _setter)
+    def _s_getter(self):
-    Constant.s = property(_getter, _setter)
+        """Deprecated. Use value instead."""
+        import warnings
+        warnings._deprecated(
+            "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14)
+        )
+        return self.value
+
+    def _s_setter(self, value):
+        import warnings
+        warnings._deprecated(
+            "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14)
+        )
+        self.value = value
+
+    Constant.n = property(_n_getter, _n_setter)
+    Constant.s = property(_s_getter, _s_setter)
 
 class _ABC(type):
 
@@ -521,6 +550,13 @@ class _ABC(type):
         cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead"""
 
     def __instancecheck__(cls, inst):
+        if cls in _const_types:
+            import warnings
+            warnings._deprecated(
+                f"ast.{cls.__qualname__}",
+                message=_DEPRECATED_CLASS_MESSAGE,
+                remove=(3, 14)
+            )
         if not isinstance(inst, Constant):
             return False
         if cls in _const_types:
@@ -544,6 +580,10 @@ def _new(cls, *args, **kwargs):
         if pos < len(args):
             raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}")
     if cls in _const_types:
+        import warnings
+        warnings._deprecated(
+            f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14)
+        )
         return Constant(*args, **kwargs)
     return Constant.__new__(cls, *args, **kwargs)
 
@@ -566,10 +606,19 @@ class Ellipsis(Constant, metaclass=_ABC):
     _fields = ()
 
     def __new__(cls, *args, **kwargs):
-        if cls is Ellipsis:
+        if cls is _ast_Ellipsis:
+            import warnings
+            warnings._deprecated(
+                "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14)
+            )
             return Constant(..., *args, **kwargs)
         return Constant.__new__(cls, *args, **kwargs)
 
+# Keep another reference to Ellipsis in the global namespace
+# so it can be referenced in Ellipsis.__new__
+# (The original "Ellipsis" name is removed from the global namespace later on)
+_ast_Ellipsis = Ellipsis
+
 _const_types = {
     Num: (int, float, complex),
     Str: (str,),
@@ -636,10 +685,12 @@ class Param(expr_context):
 # We unparse those infinities to INFSTR.
 _INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
 
-class _Precedence(IntEnum):
+@_simple_enum(IntEnum)
+class _Precedence:
     """Precedence table that originated from python grammar."""
 
-    TUPLE = auto()
+    NAMED_EXPR = auto()      # <target> := <expr1>
+    TUPLE = auto()           # <expr1>, <expr2>
     YIELD = auto()           # 'yield', 'yield from'
     TEST = auto()            # 'if'-'else', 'lambda'
     OR = auto()              # 'or'
@@ -677,11 +728,11 @@ class _Unparser(NodeVisitor):
 
     def __init__(self, *, _avoid_backslashes=False):
         self._source = []
-        self._buffer = []
         self._precedences = {}
         self._type_ignores = {}
         self._indent = 0
         self._avoid_backslashes = _avoid_backslashes
+        self._in_try_star = False
 
     def interleave(self, inter, f, seq):
         """Call f on each item in seq, calling inter() in between."""
@@ -716,18 +767,19 @@ class _Unparser(NodeVisitor):
         self.maybe_newline()
         self.write("    " * self._indent + text)
 
-    def write(self, text):
+    def write(self, *text):
-        """Append a piece of text"""
+        """Add new source parts"""
-        self._source.append(text)
+        self._source.extend(text)
 
-    def buffer_writer(self, text):
+    @contextmanager
-        self._buffer.append(text)
+    def buffered(self, buffer = None):
+        if buffer is None:
+            buffer = []
 
-    @property
+        original_source = self._source
-    def buffer(self):
+        self._source = buffer
-        value = "".join(self._buffer)
+        yield buffer
-        self._buffer.clear()
+        self._source = original_source
-        return value
 
     @contextmanager
     def block(self, *, extra = None):
@@ -837,7 +889,7 @@ class _Unparser(NodeVisitor):
         self.traverse(node.value)
 
     def visit_NamedExpr(self, node):
-        with self.require_parens(_Precedence.TUPLE, node):
+        with self.require_parens(_Precedence.NAMED_EXPR, node):
             self.set_precedence(_Precedence.ATOM, node.target, node.value)
             self.traverse(node.target)
             self.write(" := ")
@@ -849,7 +901,7 @@ class _Unparser(NodeVisitor):
 
     def visit_ImportFrom(self, node):
         self.fill("from ")
-        self.write("." * node.level)
+        self.write("." * (node.level or 0))
         if node.module:
             self.write(node.module)
         self.write(" import ")
@@ -858,6 +910,7 @@ class _Unparser(NodeVisitor):
     def visit_Assign(self, node):
         self.fill()
         for target in node.targets:
+            self.set_precedence(_Precedence.TUPLE, target)
             self.traverse(target)
             self.write(" = ")
         self.traverse(node.value)
@@ -950,7 +1003,7 @@ class _Unparser(NodeVisitor):
             self.write(" from ")
             self.traverse(node.cause)
 
-    def visit_Try(self, node):
+    def do_visit_try(self, node):
         self.fill("try")
         with self.block():
             self.traverse(node.body)
@@ -965,8 +1018,24 @@ class _Unparser(NodeVisitor):
             with self.block():
                 self.traverse(node.finalbody)
 
+    def visit_Try(self, node):
+        prev_in_try_star = self._in_try_star
+        try:
+            self._in_try_star = False
+            self.do_visit_try(node)
+        finally:
+            self._in_try_star = prev_in_try_star
+
+    def visit_TryStar(self, node):
+        prev_in_try_star = self._in_try_star
+        try:
+            self._in_try_star = True
+            self.do_visit_try(node)
+        finally:
+            self._in_try_star = prev_in_try_star
+
     def visit_ExceptHandler(self, node):
-        self.fill("except")
+        self.fill("except*" if self._in_try_star else "except")
         if node.type:
             self.write(" ")
             self.traverse(node.type)
@@ -982,6 +1051,8 @@ class _Unparser(NodeVisitor):
             self.fill("@")
             self.traverse(deco)
         self.fill("class " + node.name)
+        if hasattr(node, "type_params"):
+            self._type_params_helper(node.type_params)
         with self.delimit_if("(", ")", condition = node.bases or node.keywords):
             comma = False
             for e in node.bases:
@@ -1013,6 +1084,8 @@ class _Unparser(NodeVisitor):
             self.traverse(deco)
         def_str = fill_suffix + " " + node.name
         self.fill(def_str)
+        if hasattr(node, "type_params"):
+            self._type_params_helper(node.type_params)
         with self.delimit("(", ")"):
             self.traverse(node.args)
         if node.returns:
@@ -1021,6 +1094,30 @@ class _Unparser(NodeVisitor):
         with self.block(extra=self.get_type_comment(node)):
             self._write_docstring_and_traverse_body(node)
 
+    def _type_params_helper(self, type_params):
+        if type_params is not None and len(type_params) > 0:
+            with self.delimit("[", "]"):
+                self.interleave(lambda: self.write(", "), self.traverse, type_params)
+
+    def visit_TypeVar(self, node):
+        self.write(node.name)
+        if node.bound:
+            self.write(": ")
+            self.traverse(node.bound)
+
+    def visit_TypeVarTuple(self, node):
+        self.write("*" + node.name)
+
+    def visit_ParamSpec(self, node):
+        self.write("**" + node.name)
+
+    def visit_TypeAlias(self, node):
+        self.fill("type ")
+        self.traverse(node.name)
+        self._type_params_helper(node.type_params)
+        self.write(" = ")
+        self.traverse(node.value)
+
     def visit_For(self, node):
         self._for_helper("for ", node)
 
@@ -1029,6 +1126,7 @@ class _Unparser(NodeVisitor):
 
     def _for_helper(self, fill, node):
         self.fill(fill)
+        self.set_precedence(_Precedence.TUPLE, node.target)
         self.traverse(node.target)
         self.write(" in ")
         self.traverse(node.iter)
@@ -1125,71 +1223,81 @@ class _Unparser(NodeVisitor):
 
     def visit_JoinedStr(self, node):
         self.write("f")
-        if self._avoid_backslashes:
-            self._fstring_JoinedStr(node, self.buffer_writer)
-            self._write_str_avoiding_backslashes(self.buffer)
-            return
 
-        # If we don't need to avoid backslashes globally (i.e., we only need
+        fstring_parts = []
-        # to avoid them inside FormattedValues), it's cosmetically preferred
-        # to use escaped whitespace. That is, it's preferred to use backslashes
-        # for cases like: f"{x}\n". To accomplish this, we keep track of what
-        # in our buffer corresponds to FormattedValues and what corresponds to
-        # Constant parts of the f-string, and allow escapes accordingly.
-        buffer = []
         for value in node.values:
-            meth = getattr(self, "_fstring_" + type(value).__name__)
+            with self.buffered() as buffer:
-            meth(value, self.buffer_writer)
+                self._write_fstring_inner(value)
-            buffer.append((self.buffer, isinstance(value, Constant)))
+            fstring_parts.append(
-        new_buffer = []
+                ("".join(buffer), isinstance(value, Constant))
-        quote_types = _ALL_QUOTES
-        for value, is_constant in buffer:
-            # Repeatedly narrow down the list of possible quote_types
-            value, quote_types = self._str_literal_helper(
-                value, quote_types=quote_types,
-                escape_special_whitespace=is_constant
             )
-            new_buffer.append(value)
+
-        value = "".join(new_buffer)
+        new_fstring_parts = []
+        quote_types = list(_ALL_QUOTES)
+        fallback_to_repr = False
+        for value, is_constant in fstring_parts:
+            if is_constant:
+                value, new_quote_types = self._str_literal_helper(
+                    value,
+                    quote_types=quote_types,
+                    escape_special_whitespace=True,
+                )
+                if set(new_quote_types).isdisjoint(quote_types):
+                    fallback_to_repr = True
+                    break
+                quote_types = new_quote_types
+            elif "\n" in value:
+                quote_types = [q for q in quote_types if q in _MULTI_QUOTES]
+                assert quote_types
+            new_fstring_parts.append(value)
+
+        if fallback_to_repr:
+            # If we weren't able to find a quote type that works for all parts
+            # of the JoinedStr, fallback to using repr and triple single quotes.
+            quote_types = ["'''"]
+            new_fstring_parts.clear()
+            for value, is_constant in fstring_parts:
+                if is_constant:
+                    value = repr('"' + value)  # force repr to use single quotes
+                    expected_prefix = "'\""
+                    assert value.startswith(expected_prefix), repr(value)
+                    value = value[len(expected_prefix):-1]
+                new_fstring_parts.append(value)
+
+        value = "".join(new_fstring_parts)
         quote_type = quote_types[0]
         self.write(f"{quote_type}{value}{quote_type}")
 
+    def _write_fstring_inner(self, node):
+        if isinstance(node, JoinedStr):
+            # for both the f-string itself, and format_spec
+            for value in node.values:
+                self._write_fstring_inner(value)
+        elif isinstance(node, Constant) and isinstance(node.value, str):
+            value = node.value.replace("{", "{{").replace("}", "}}")
+            self.write(value)
+        elif isinstance(node, FormattedValue):
+            self.visit_FormattedValue(node)
+        else:
+            raise ValueError(f"Unexpected node inside JoinedStr, {node!r}")
+
     def visit_FormattedValue(self, node):
-        self.write("f")
+        def unparse_inner(inner):
-        self._fstring_FormattedValue(node, self.buffer_writer)
+            unparser = type(self)()
-        self._write_str_avoiding_backslashes(self.buffer)
+            unparser.set_precedence(_Precedence.TEST.next(), inner)
+            return unparser.visit(inner)
 
-    def _fstring_JoinedStr(self, node, write):
+        with self.delimit("{", "}"):
-        for value in node.values:
+            expr = unparse_inner(node.value)
-            meth = getattr(self, "_fstring_" + type(value).__name__)
+            if expr.startswith("{"):
-            meth(value, write)
+                # Separate pair of opening brackets as "{ {"
-
+                self.write(" ")
-    def _fstring_Constant(self, node, write):
+            self.write(expr)
-        if not isinstance(node.value, str):
+            if node.conversion != -1:
-            raise ValueError("Constants inside JoinedStr should be a string.")
+                self.write(f"!{chr(node.conversion)}")
-        value = node.value.replace("{", "{{").replace("}", "}}")
+            if node.format_spec:
-        write(value)
+                self.write(":")
-
+                self._write_fstring_inner(node.format_spec)
-    def _fstring_FormattedValue(self, node, write):
-        write("{")
-        unparser = type(self)(_avoid_backslashes=True)
-        unparser.set_precedence(_Precedence.TEST.next(), node.value)
-        expr = unparser.visit(node.value)
-        if expr.startswith("{"):
-            write(" ")  # Separate pair of opening brackets as "{ {"
-        if "\\" in expr:
-            raise ValueError("Unable to avoid backslash in f-string expression part")
-        write(expr)
-        if node.conversion != -1:
-            conversion = chr(node.conversion)
-            if conversion not in "sra":
-                raise ValueError("Unknown f-string conversion.")
-            write(f"!{conversion}")
-        if node.format_spec:
-            write(":")
-            meth = getattr(self, "_fstring_" + type(node.format_spec).__name__)
-            meth(node.format_spec, write)
-        write("}")
 
     def visit_Name(self, node):
         self.write(node.id)
@@ -1312,7 +1420,11 @@ class _Unparser(NodeVisitor):
             )
 
     def visit_Tuple(self, node):
-        with self.delimit("(", ")"):
+        with self.delimit_if(
+            "(",
+            ")",
+            len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE
+        ):
             self.items_view(self.traverse, node.elts)
 
     unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"}
@@ -1328,7 +1440,7 @@ class _Unparser(NodeVisitor):
         operator_precedence = self.unop_precedence[operator]
         with self.require_parens(operator_precedence, node):
             self.write(operator)
-            # factor prefixes (+, -, ~) shouldn't be seperated
+            # factor prefixes (+, -, ~) shouldn't be separated
             # from the value they belong, (e.g: +1 instead of + 1)
             if operator_precedence is not _Precedence.FACTOR:
                 self.write(" ")
@@ -1453,20 +1565,17 @@ class _Unparser(NodeVisitor):
                 self.traverse(e)
 
     def visit_Subscript(self, node):
-        def is_simple_tuple(slice_value):
+        def is_non_empty_tuple(slice_value):
-            # when unparsing a non-empty tuple, the parentheses can be safely
-            # omitted if there aren't any elements that explicitly requires
-            # parentheses (such as starred expressions).
             return (
                 isinstance(slice_value, Tuple)
                 and slice_value.elts
-                and not any(isinstance(elt, Starred) for elt in slice_value.elts)
             )
 
         self.set_precedence(_Precedence.ATOM, node.value)
         self.traverse(node.value)
         with self.delimit("[", "]"):
-            if is_simple_tuple(node.slice):
+            if is_non_empty_tuple(node.slice):
+                # parentheses can be omitted if the tuple isn't empty
                 self.items_view(self.traverse, node.slice.elts)
             else:
                 self.traverse(node.slice)
@@ -1563,8 +1672,11 @@ class _Unparser(NodeVisitor):
 
     def visit_Lambda(self, node):
         with self.require_parens(_Precedence.TEST, node):
-            self.write("lambda ")
+            self.write("lambda")
-            self.traverse(node.args)
+            with self.buffered() as buffer:
+                self.traverse(node.args)
+            if buffer:
+                self.write(" ", *buffer)
             self.write(": ")
             self.set_precedence(_Precedence.TEST, node.body)
             self.traverse(node.body)
@@ -1673,6 +1785,22 @@ def unparse(ast_obj):
     return unparser.visit(ast_obj)
 
 
+_deprecated_globals = {
+    name: globals().pop(name)
+    for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis')
+}
+
+def __getattr__(name):
+    if name in _deprecated_globals:
+        globals()[name] = value = _deprecated_globals[name]
+        import warnings
+        warnings._deprecated(
+            f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14)
+        )
+        return value
+    raise AttributeError(f"module 'ast' has no attribute '{name}'")
+
+
 def main():
     import argparse
 

Lib/base64.py

@@ -1,4 +1,4 @@
-#! /usr/bin/python3.6
+#! /usr/bin/env python3
 
 """Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings"""
 
@@ -16,7 +16,7 @@ __all__ = [
     'encode', 'decode', 'encodebytes', 'decodebytes',
     # Generalized interface for other encodings
     'b64encode', 'b64decode', 'b32encode', 'b32decode',
-    'b16encode', 'b16decode',
+    'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode',
     # Base85 and Ascii85 encodings
     'b85encode', 'b85decode', 'a85encode', 'a85decode',
     # Standard Base64 encoding
@@ -76,15 +76,16 @@ def b64decode(s, altchars=None, validate=False):
     normal base-64 alphabet nor the alternative alphabet are discarded prior
     to the padding check.  If validate is True, these non-alphabet characters
     in the input result in a binascii.Error.
+    For more information about the strict base64 check, see:
+
+    https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64
     """
     s = _bytes_from_decode_data(s)
     if altchars is not None:
         altchars = _bytes_from_decode_data(altchars)
         assert len(altchars) == 2, repr(altchars)
         s = s.translate(bytes.maketrans(altchars, b'+/'))
-    if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s):
+    return binascii.a2b_base64(s, strict_mode=validate)
-        raise binascii.Error('Non-base64 digit found')
-    return binascii.a2b_base64(s)
 
 
 def standard_b64encode(s):
@@ -135,19 +136,40 @@ def urlsafe_b64decode(s):
 
 
 # Base32 encoding/decoding must be done in Python
-_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
+_B32_ENCODE_DOCSTRING = '''
-_b32tab2 = None
+Encode the bytes-like objects using {encoding} and return a bytes object.
-_b32rev = None
+'''
+_B32_DECODE_DOCSTRING = '''
+Decode the {encoding} encoded bytes-like object or ASCII string s.
 
-def b32encode(s):
+Optional casefold is a flag specifying whether a lowercase alphabet is
-    """Encode the bytes-like object s using Base32 and return a bytes object.
+acceptable as input.  For security purposes, the default is False.
-    """
+{extra_args}
+The result is returned as a bytes object.  A binascii.Error is raised if
+the input is incorrectly padded or if there are non-alphabet
+characters present in the input.
+'''
+_B32_DECODE_MAP01_DOCSTRING = '''
+RFC 3548 allows for optional mapping of the digit 0 (zero) to the
+letter O (oh), and for optional mapping of the digit 1 (one) to
+either the letter I (eye) or letter L (el).  The optional argument
+map01 when not None, specifies which letter the digit 1 should be
+mapped to (when map01 is not None, the digit 0 is always mapped to
+the letter O).  For security purposes the default is None, so that
+0 and 1 are not allowed in the input.
+'''
+_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
+_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV'
+_b32tab2 = {}
+_b32rev = {}
+
+def _b32encode(alphabet, s):
     global _b32tab2
     # Delay the initialization of the table to not waste memory
     # if the function is never called
-    if _b32tab2 is None:
+    if alphabet not in _b32tab2:
-        b32tab = [bytes((i,)) for i in _b32alphabet]
+        b32tab = [bytes((i,)) for i in alphabet]
-        _b32tab2 = [a + b for a in b32tab for b in b32tab]
+        _b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab]
         b32tab = None
 
     if not isinstance(s, bytes_types):
@@ -158,9 +180,9 @@ def b32encode(s):
         s = s + b'\0' * (5 - leftover)  # Don't use += !
     encoded = bytearray()
     from_bytes = int.from_bytes
-    b32tab2 = _b32tab2
+    b32tab2 = _b32tab2[alphabet]
     for i in range(0, len(s), 5):
-        c = from_bytes(s[i: i + 5], 'big')
+        c = from_bytes(s[i: i + 5])              # big endian
         encoded += (b32tab2[c >> 30] +           # bits 1 - 10
                     b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20
                     b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30
@@ -177,29 +199,12 @@ def b32encode(s):
         encoded[-1:] = b'='
     return bytes(encoded)
 
-def b32decode(s, casefold=False, map01=None):
+def _b32decode(alphabet, s, casefold=False, map01=None):
-    """Decode the Base32 encoded bytes-like object or ASCII string s.
-
-    Optional casefold is a flag specifying whether a lowercase alphabet is
-    acceptable as input.  For security purposes, the default is False.
-
-    RFC 3548 allows for optional mapping of the digit 0 (zero) to the
-    letter O (oh), and for optional mapping of the digit 1 (one) to
-    either the letter I (eye) or letter L (el).  The optional argument
-    map01 when not None, specifies which letter the digit 1 should be
-    mapped to (when map01 is not None, the digit 0 is always mapped to
-    the letter O).  For security purposes the default is None, so that
-    0 and 1 are not allowed in the input.
-
-    The result is returned as a bytes object.  A binascii.Error is raised if
-    the input is incorrectly padded or if there are non-alphabet
-    characters present in the input.
-    """
     global _b32rev
     # Delay the initialization of the table to not waste memory
     # if the function is never called
-    if _b32rev is None:
+    if alphabet not in _b32rev:
-        _b32rev = {v: k for k, v in enumerate(_b32alphabet)}
+        _b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)}
     s = _bytes_from_decode_data(s)
     if len(s) % 8:
         raise binascii.Error('Incorrect padding')
@@ -220,7 +225,7 @@ def b32decode(s, casefold=False, map01=None):
     padchars = l - len(s)
     # Now decode the full quanta
     decoded = bytearray()
-    b32rev = _b32rev
+    b32rev = _b32rev[alphabet]
     for i in range(0, len(s), 8):
         quanta = s[i: i + 8]
         acc = 0
@@ -229,18 +234,38 @@ def b32decode(s, casefold=False, map01=None):
                 acc = (acc << 5) + b32rev[c]
         except KeyError:
             raise binascii.Error('Non-base32 digit found') from None
-        decoded += acc.to_bytes(5, 'big')
+        decoded += acc.to_bytes(5)  # big endian
     # Process the last, partial quanta
     if l % 8 or padchars not in {0, 1, 3, 4, 6}:
         raise binascii.Error('Incorrect padding')
     if padchars and decoded:
         acc <<= 5 * padchars
-        last = acc.to_bytes(5, 'big')
+        last = acc.to_bytes(5)  # big endian
         leftover = (43 - 5 * padchars) // 8  # 1: 4, 3: 3, 4: 2, 6: 1
         decoded[-5:] = last[:leftover]
     return bytes(decoded)
 
 
+def b32encode(s):
+    return _b32encode(_b32alphabet, s)
+b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32')
+
+def b32decode(s, casefold=False, map01=None):
+    return _b32decode(_b32alphabet, s, casefold, map01)
+b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32',
+                                        extra_args=_B32_DECODE_MAP01_DOCSTRING)
+
+def b32hexencode(s):
+    return _b32encode(_b32hexalphabet, s)
+b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex')
+
+def b32hexdecode(s, casefold=False):
+    # base32hex does not have the 01 mapping
+    return _b32decode(_b32hexalphabet, s, casefold)
+b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex',
+                                                    extra_args='')
+
+
 # RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
 # lowercase.  The RFC also recommends against accepting input case
 # insensitively.
@@ -320,7 +345,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
     global _a85chars, _a85chars2
     # Delay the initialization of tables to not waste memory
     # if the function is never called
-    if _a85chars is None:
+    if _a85chars2 is None:
         _a85chars = [bytes((i,)) for i in range(33, 118)]
         _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]
 
@@ -428,7 +453,7 @@ def b85encode(b, pad=False):
     global _b85chars, _b85chars2
     # Delay the initialization of tables to not waste memory
     # if the function is never called
-    if _b85chars is None:
+    if _b85chars2 is None:
         _b85chars = [bytes((i,)) for i in _b85alphabet]
         _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
     return _85encode(b, _b85chars, _b85chars2, pad)
@@ -483,14 +508,8 @@ MAXBINSIZE = (MAXLINESIZE//4)*3
 
 def encode(input, output):
     """Encode a file; input and output are binary files."""
-    while True:
+    while s := input.read(MAXBINSIZE):
-        s = input.read(MAXBINSIZE)
+        while len(s) < MAXBINSIZE and (ns := input.read(MAXBINSIZE-len(s))):
-        if not s:
-            break
-        while len(s) < MAXBINSIZE:
-            ns = input.read(MAXBINSIZE-len(s))
-            if not ns:
-                break
             s += ns
         line = binascii.b2a_base64(s)
         output.write(line)
@@ -498,10 +517,7 @@ def encode(input, output):
 
 def decode(input, output):
     """Decode a file; input and output are binary files."""
-    while True:
+    while line := input.readline():
-        line = input.readline()
-        if not line:
-            break
         s = binascii.a2b_base64(line)
         output.write(s)
 
@@ -531,49 +547,34 @@ def encodebytes(s):
         pieces.append(binascii.b2a_base64(chunk))
     return b"".join(pieces)
 
-def encodestring(s):
-    """Legacy alias of encodebytes()."""
-    import warnings
-    warnings.warn("encodestring() is a deprecated alias since 3.1, "
-                  "use encodebytes()",
-                  DeprecationWarning, 2)
-    return encodebytes(s)
-
 
 def decodebytes(s):
     """Decode a bytestring of base-64 data into a bytes object."""
     _input_type_check(s)
     return binascii.a2b_base64(s)
 
-def decodestring(s):
-    """Legacy alias of decodebytes()."""
-    import warnings
-    warnings.warn("decodestring() is a deprecated alias since Python 3.1, "
-                  "use decodebytes()",
-                  DeprecationWarning, 2)
-    return decodebytes(s)
-
 
 # Usable as a script...
 def main():
     """Small main program"""
     import sys, getopt
+    usage = f"""usage: {sys.argv[0]} [-h|-d|-e|-u] [file|-]
+        -h: print this help message and exit
+        -d, -u: decode
+        -e: encode (default)"""
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'deut')
+        opts, args = getopt.getopt(sys.argv[1:], 'hdeu')
     except getopt.error as msg:
         sys.stdout = sys.stderr
         print(msg)
-        print("""usage: %s [-d|-e|-u|-t] [file|-]
+        print(usage)
-        -d, -u: decode
-        -e: encode (default)
-        -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0])
         sys.exit(2)
     func = encode
     for o, a in opts:
         if o == '-e': func = encode
         if o == '-d': func = decode
         if o == '-u': func = decode
-        if o == '-t': test(); return
+        if o == '-h': print(usage); return
     if args and args[0] != '-':
         with open(args[0], 'rb') as f:
             func(f, sys.stdout.buffer)
@@ -581,15 +582,5 @@ def main():
         func(sys.stdin.buffer, sys.stdout.buffer)
 
 
-def test():
-    s0 = b"Aladdin:open sesame"
-    print(repr(s0))
-    s1 = encodebytes(s0)
-    print(repr(s1))
-    s2 = decodebytes(s1)
-    print(repr(s2))
-    assert s0 == s2
-
-
 if __name__ == '__main__':
     main()

Lib/bdb.py

@@ -570,9 +570,12 @@ class Bdb:
             rv = frame.f_locals['__return__']
             s += '->'
             s += reprlib.repr(rv)
-        line = linecache.getline(filename, lineno, frame.f_globals)
+        if lineno is not None:
-        if line:
+            line = linecache.getline(filename, lineno, frame.f_globals)
-            s += lprefix + line.strip()
+            if line:
+                s += lprefix + line.strip()
+        else:
+            s += f'{lprefix}Warning: lineno is None'
         return s
 
     # The following methods can be called by clients to use
@@ -805,15 +808,18 @@ def checkfuncname(b, frame):
     return True
 
 
-# Determines if there is an effective (active) breakpoint at this
-# line of code.  Returns breakpoint number or 0 if none
 def effective(file, line, frame):
-    """Determine which breakpoint for this file:line is to be acted upon.
+    """Return (active breakpoint, delete temporary flag) or (None, None) as
+       breakpoint to act upon.
 
-    Called only if we know there is a breakpoint at this location.  Return
+       The "active breakpoint" is the first entry in bplist[line, file] (which
-    the breakpoint that was triggered and a boolean that indicates if it is
+       must exist) that is enabled, for which checkfuncname is True, and that
-    ok to delete a temporary breakpoint.  Return (None, None) if there is no
+       has neither a False condition nor a positive ignore count.  The flag,
-    matching breakpoint.
+       meaning that a temporary breakpoint should be deleted, is False only
+       when the condiion cannot be evaluated (in which case, ignore count is
+       ignored).
+
+       If no such entry exists, then (None, None) is returned.
     """
     possibles = Breakpoint.bplist[file, line]
     for b in possibles:

Lib/binhex.py

@@ -1,502 +0,0 @@
-"""Macintosh binhex compression/decompression.
-
-easy interface:
-binhex(inputfilename, outputfilename)
-hexbin(inputfilename, outputfilename)
-"""
-
-#
-# Jack Jansen, CWI, August 1995.
-#
-# The module is supposed to be as compatible as possible. Especially the
-# easy interface should work "as expected" on any platform.
-# XXXX Note: currently, textfiles appear in mac-form on all platforms.
-# We seem to lack a simple character-translate in python.
-# (we should probably use ISO-Latin-1 on all but the mac platform).
-# XXXX The simple routines are too simple: they expect to hold the complete
-# files in-core. Should be fixed.
-# XXXX It would be nice to handle AppleDouble format on unix
-# (for servers serving macs).
-# XXXX I don't understand what happens when you get 0x90 times the same byte on
-# input. The resulting code (xx 90 90) would appear to be interpreted as an
-# escaped *value* of 0x90. All coders I've seen appear to ignore this nicety...
-#
-import binascii
-import contextlib
-import io
-import os
-import struct
-import warnings
-
-warnings.warn('the binhex module is deprecated', DeprecationWarning,
-              stacklevel=2)
-
-
-__all__ = ["binhex","hexbin","Error"]
-
-class Error(Exception):
-    pass
-
-# States (what have we written)
-_DID_HEADER = 0
-_DID_DATA = 1
-
-# Various constants
-REASONABLY_LARGE = 32768  # Minimal amount we pass the rle-coder
-LINELEN = 64
-RUNCHAR = b"\x90"
-
-#
-# This code is no longer byte-order dependent
-
-
-class FInfo:
-    def __init__(self):
-        self.Type = '????'
-        self.Creator = '????'
-        self.Flags = 0
-
-def getfileinfo(name):
-    finfo = FInfo()
-    with io.open(name, 'rb') as fp:
-        # Quick check for textfile
-        data = fp.read(512)
-        if 0 not in data:
-            finfo.Type = 'TEXT'
-        fp.seek(0, 2)
-        dsize = fp.tell()
-    dir, file = os.path.split(name)
-    file = file.replace(':', '-', 1)
-    return file, finfo, dsize, 0
-
-class openrsrc:
-    def __init__(self, *args):
-        pass
-
-    def read(self, *args):
-        return b''
-
-    def write(self, *args):
-        pass
-
-    def close(self):
-        pass
-
-
-# DeprecationWarning is already emitted on "import binhex". There is no need
-# to repeat the warning at each call to deprecated binascii functions.
-@contextlib.contextmanager
-def _ignore_deprecation_warning():
-    with warnings.catch_warnings():
-        warnings.filterwarnings('ignore', '', DeprecationWarning)
-        yield
-
-
-class _Hqxcoderengine:
-    """Write data to the coder in 3-byte chunks"""
-
-    def __init__(self, ofp):
-        self.ofp = ofp
-        self.data = b''
-        self.hqxdata = b''
-        self.linelen = LINELEN - 1
-
-    def write(self, data):
-        self.data = self.data + data
-        datalen = len(self.data)
-        todo = (datalen // 3) * 3
-        data = self.data[:todo]
-        self.data = self.data[todo:]
-        if not data:
-            return
-        with _ignore_deprecation_warning():
-            self.hqxdata = self.hqxdata + binascii.b2a_hqx(data)
-        self._flush(0)
-
-    def _flush(self, force):
-        first = 0
-        while first <= len(self.hqxdata) - self.linelen:
-            last = first + self.linelen
-            self.ofp.write(self.hqxdata[first:last] + b'\r')
-            self.linelen = LINELEN
-            first = last
-        self.hqxdata = self.hqxdata[first:]
-        if force:
-            self.ofp.write(self.hqxdata + b':\r')
-
-    def close(self):
-        if self.data:
-            with _ignore_deprecation_warning():
-                self.hqxdata = self.hqxdata + binascii.b2a_hqx(self.data)
-        self._flush(1)
-        self.ofp.close()
-        del self.ofp
-
-class _Rlecoderengine:
-    """Write data to the RLE-coder in suitably large chunks"""
-
-    def __init__(self, ofp):
-        self.ofp = ofp
-        self.data = b''
-
-    def write(self, data):
-        self.data = self.data + data
-        if len(self.data) < REASONABLY_LARGE:
-            return
-        with _ignore_deprecation_warning():
-            rledata = binascii.rlecode_hqx(self.data)
-        self.ofp.write(rledata)
-        self.data = b''
-
-    def close(self):
-        if self.data:
-            with _ignore_deprecation_warning():
-                rledata = binascii.rlecode_hqx(self.data)
-            self.ofp.write(rledata)
-        self.ofp.close()
-        del self.ofp
-
-class BinHex:
-    def __init__(self, name_finfo_dlen_rlen, ofp):
-        name, finfo, dlen, rlen = name_finfo_dlen_rlen
-        close_on_error = False
-        if isinstance(ofp, str):
-            ofname = ofp
-            ofp = io.open(ofname, 'wb')
-            close_on_error = True
-        try:
-            ofp.write(b'(This file must be converted with BinHex 4.0)\r\r:')
-            hqxer = _Hqxcoderengine(ofp)
-            self.ofp = _Rlecoderengine(hqxer)
-            self.crc = 0
-            if finfo is None:
-                finfo = FInfo()
-            self.dlen = dlen
-            self.rlen = rlen
-            self._writeinfo(name, finfo)
-            self.state = _DID_HEADER
-        except:
-            if close_on_error:
-                ofp.close()
-            raise
-
-    def _writeinfo(self, name, finfo):
-        nl = len(name)
-        if nl > 63:
-            raise Error('Filename too long')
-        d = bytes([nl]) + name.encode("latin-1") + b'\0'
-        tp, cr = finfo.Type, finfo.Creator
-        if isinstance(tp, str):
-            tp = tp.encode("latin-1")
-        if isinstance(cr, str):
-            cr = cr.encode("latin-1")
-        d2 = tp + cr
-
-        # Force all structs to be packed with big-endian
-        d3 = struct.pack('>h', finfo.Flags)
-        d4 = struct.pack('>ii', self.dlen, self.rlen)
-        info = d + d2 + d3 + d4
-        self._write(info)
-        self._writecrc()
-
-    def _write(self, data):
-        self.crc = binascii.crc_hqx(data, self.crc)
-        self.ofp.write(data)
-
-    def _writecrc(self):
-        # XXXX Should this be here??
-        # self.crc = binascii.crc_hqx('\0\0', self.crc)
-        if self.crc < 0:
-            fmt = '>h'
-        else:
-            fmt = '>H'
-        self.ofp.write(struct.pack(fmt, self.crc))
-        self.crc = 0
-
-    def write(self, data):
-        if self.state != _DID_HEADER:
-            raise Error('Writing data at the wrong time')
-        self.dlen = self.dlen - len(data)
-        self._write(data)
-
-    def close_data(self):
-        if self.dlen != 0:
-            raise Error('Incorrect data size, diff=%r' % (self.rlen,))
-        self._writecrc()
-        self.state = _DID_DATA
-
-    def write_rsrc(self, data):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error('Writing resource data at the wrong time')
-        self.rlen = self.rlen - len(data)
-        self._write(data)
-
-    def close(self):
-        if self.state is None:
-            return
-        try:
-            if self.state < _DID_DATA:
-                self.close_data()
-            if self.state != _DID_DATA:
-                raise Error('Close at the wrong time')
-            if self.rlen != 0:
-                raise Error("Incorrect resource-datasize, diff=%r" % (self.rlen,))
-            self._writecrc()
-        finally:
-            self.state = None
-            ofp = self.ofp
-            del self.ofp
-            ofp.close()
-
-def binhex(inp, out):
-    """binhex(infilename, outfilename): create binhex-encoded copy of a file"""
-    finfo = getfileinfo(inp)
-    ofp = BinHex(finfo, out)
-
-    with io.open(inp, 'rb') as ifp:
-        # XXXX Do textfile translation on non-mac systems
-        while True:
-            d = ifp.read(128000)
-            if not d: break
-            ofp.write(d)
-        ofp.close_data()
-
-    ifp = openrsrc(inp, 'rb')
-    while True:
-        d = ifp.read(128000)
-        if not d: break
-        ofp.write_rsrc(d)
-    ofp.close()
-    ifp.close()
-
-class _Hqxdecoderengine:
-    """Read data via the decoder in 4-byte chunks"""
-
-    def __init__(self, ifp):
-        self.ifp = ifp
-        self.eof = 0
-
-    def read(self, totalwtd):
-        """Read at least wtd bytes (or until EOF)"""
-        decdata = b''
-        wtd = totalwtd
-        #
-        # The loop here is convoluted, since we don't really now how
-        # much to decode: there may be newlines in the incoming data.
-        while wtd > 0:
-            if self.eof: return decdata
-            wtd = ((wtd + 2) // 3) * 4
-            data = self.ifp.read(wtd)
-            #
-            # Next problem: there may not be a complete number of
-            # bytes in what we pass to a2b. Solve by yet another
-            # loop.
-            #
-            while True:
-                try:
-                    with _ignore_deprecation_warning():
-                        decdatacur, self.eof = binascii.a2b_hqx(data)
-                    break
-                except binascii.Incomplete:
-                    pass
-                newdata = self.ifp.read(1)
-                if not newdata:
-                    raise Error('Premature EOF on binhex file')
-                data = data + newdata
-            decdata = decdata + decdatacur
-            wtd = totalwtd - len(decdata)
-            if not decdata and not self.eof:
-                raise Error('Premature EOF on binhex file')
-        return decdata
-
-    def close(self):
-        self.ifp.close()
-
-class _Rledecoderengine:
-    """Read data via the RLE-coder"""
-
-    def __init__(self, ifp):
-        self.ifp = ifp
-        self.pre_buffer = b''
-        self.post_buffer = b''
-        self.eof = 0
-
-    def read(self, wtd):
-        if wtd > len(self.post_buffer):
-            self._fill(wtd - len(self.post_buffer))
-        rv = self.post_buffer[:wtd]
-        self.post_buffer = self.post_buffer[wtd:]
-        return rv
-
-    def _fill(self, wtd):
-        self.pre_buffer = self.pre_buffer + self.ifp.read(wtd + 4)
-        if self.ifp.eof:
-            with _ignore_deprecation_warning():
-                self.post_buffer = self.post_buffer + \
-                    binascii.rledecode_hqx(self.pre_buffer)
-            self.pre_buffer = b''
-            return
-
-        #
-        # Obfuscated code ahead. We have to take care that we don't
-        # end up with an orphaned RUNCHAR later on. So, we keep a couple
-        # of bytes in the buffer, depending on what the end of
-        # the buffer looks like:
-        # '\220\0\220' - Keep 3 bytes: repeated \220 (escaped as \220\0)
-        # '?\220' - Keep 2 bytes: repeated something-else
-        # '\220\0' - Escaped \220: Keep 2 bytes.
-        # '?\220?' - Complete repeat sequence: decode all
-        # otherwise: keep 1 byte.
-        #
-        mark = len(self.pre_buffer)
-        if self.pre_buffer[-3:] == RUNCHAR + b'\0' + RUNCHAR:
-            mark = mark - 3
-        elif self.pre_buffer[-1:] == RUNCHAR:
-            mark = mark - 2
-        elif self.pre_buffer[-2:] == RUNCHAR + b'\0':
-            mark = mark - 2
-        elif self.pre_buffer[-2:-1] == RUNCHAR:
-            pass # Decode all
-        else:
-            mark = mark - 1
-
-        with _ignore_deprecation_warning():
-            self.post_buffer = self.post_buffer + \
-                binascii.rledecode_hqx(self.pre_buffer[:mark])
-        self.pre_buffer = self.pre_buffer[mark:]
-
-    def close(self):
-        self.ifp.close()
-
-class HexBin:
-    def __init__(self, ifp):
-        if isinstance(ifp, str):
-            ifp = io.open(ifp, 'rb')
-        #
-        # Find initial colon.
-        #
-        while True:
-            ch = ifp.read(1)
-            if not ch:
-                raise Error("No binhex data found")
-            # Cater for \r\n terminated lines (which show up as \n\r, hence
-            # all lines start with \r)
-            if ch == b'\r':
-                continue
-            if ch == b':':
-                break
-
-        hqxifp = _Hqxdecoderengine(ifp)
-        self.ifp = _Rledecoderengine(hqxifp)
-        self.crc = 0
-        self._readheader()
-
-    def _read(self, len):
-        data = self.ifp.read(len)
-        self.crc = binascii.crc_hqx(data, self.crc)
-        return data
-
-    def _checkcrc(self):
-        filecrc = struct.unpack('>h', self.ifp.read(2))[0] & 0xffff
-        #self.crc = binascii.crc_hqx('\0\0', self.crc)
-        # XXXX Is this needed??
-        self.crc = self.crc & 0xffff
-        if filecrc != self.crc:
-            raise Error('CRC error, computed %x, read %x'
-                        % (self.crc, filecrc))
-        self.crc = 0
-
-    def _readheader(self):
-        len = self._read(1)
-        fname = self._read(ord(len))
-        rest = self._read(1 + 4 + 4 + 2 + 4 + 4)
-        self._checkcrc()
-
-        type = rest[1:5]
-        creator = rest[5:9]
-        flags = struct.unpack('>h', rest[9:11])[0]
-        self.dlen = struct.unpack('>l', rest[11:15])[0]
-        self.rlen = struct.unpack('>l', rest[15:19])[0]
-
-        self.FName = fname
-        self.FInfo = FInfo()
-        self.FInfo.Creator = creator
-        self.FInfo.Type = type
-        self.FInfo.Flags = flags
-
-        self.state = _DID_HEADER
-
-    def read(self, *n):
-        if self.state != _DID_HEADER:
-            raise Error('Read data at wrong time')
-        if n:
-            n = n[0]
-            n = min(n, self.dlen)
-        else:
-            n = self.dlen
-        rv = b''
-        while len(rv) < n:
-            rv = rv + self._read(n-len(rv))
-        self.dlen = self.dlen - n
-        return rv
-
-    def close_data(self):
-        if self.state != _DID_HEADER:
-            raise Error('close_data at wrong time')
-        if self.dlen:
-            dummy = self._read(self.dlen)
-        self._checkcrc()
-        self.state = _DID_DATA
-
-    def read_rsrc(self, *n):
-        if self.state == _DID_HEADER:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error('Read resource data at wrong time')
-        if n:
-            n = n[0]
-            n = min(n, self.rlen)
-        else:
-            n = self.rlen
-        self.rlen = self.rlen - n
-        return self._read(n)
-
-    def close(self):
-        if self.state is None:
-            return
-        try:
-            if self.rlen:
-                dummy = self.read_rsrc(self.rlen)
-            self._checkcrc()
-        finally:
-            self.state = None
-            self.ifp.close()
-
-def hexbin(inp, out):
-    """hexbin(infilename, outfilename) - Decode binhexed file"""
-    ifp = HexBin(inp)
-    finfo = ifp.FInfo
-    if not out:
-        out = ifp.FName
-
-    with io.open(out, 'wb') as ofp:
-        # XXXX Do translation on non-mac systems
-        while True:
-            d = ifp.read(128000)
-            if not d: break
-            ofp.write(d)
-    ifp.close_data()
-
-    d = ifp.read_rsrc(128000)
-    if d:
-        ofp = openrsrc(out, 'wb')
-        ofp.write(d)
-        while True:
-            d = ifp.read_rsrc(128000)
-            if not d: break
-            ofp.write(d)
-        ofp.close()
-
-    ifp.close()

Lib/bisect.py

@@ -8,6 +8,8 @@ def insort_right(a, x, lo=0, hi=None, *, key=None):
 
     Optional args lo (default 0) and hi (default len(a)) bound the
     slice of a to be searched.
+
+    A custom key function can be supplied to customize the sort order.
     """
     if key is None:
         lo = bisect_right(a, x, lo, hi)
@@ -25,6 +27,8 @@ def bisect_right(a, x, lo=0, hi=None, *, key=None):
 
     Optional args lo (default 0) and hi (default len(a)) bound the
     slice of a to be searched.
+
+    A custom key function can be supplied to customize the sort order.
     """
 
     if lo < 0:
@@ -57,6 +61,8 @@ def insort_left(a, x, lo=0, hi=None, *, key=None):
 
     Optional args lo (default 0) and hi (default len(a)) bound the
     slice of a to be searched.
+
+    A custom key function can be supplied to customize the sort order.
     """
 
     if key is None:
@@ -74,6 +80,8 @@ def bisect_left(a, x, lo=0, hi=None, *, key=None):
 
     Optional args lo (default 0) and hi (default len(a)) bound the
     slice of a to be searched.
+
+    A custom key function can be supplied to customize the sort order.
     """
 
     if lo < 0:
@@ -107,4 +115,4 @@ except ImportError:
 
 # Create aliases
 bisect = bisect_right
-insort = insort_right
+insort = insort_right

Lib/calendar.py

@@ -7,15 +7,22 @@ set the first day of the week (0=Monday, 6=Sunday)."""
 
 import sys
 import datetime
+from enum import IntEnum, global_enum
 import locale as _locale
 from itertools import repeat
+import warnings
 
 __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
            "firstweekday", "isleap", "leapdays", "weekday", "monthrange",
            "monthcalendar", "prmonth", "month", "prcal", "calendar",
            "timegm", "month_name", "month_abbr", "day_name", "day_abbr",
            "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar",
-           "LocaleHTMLCalendar", "weekheader"]
+           "LocaleHTMLCalendar", "weekheader",
+           "Day", "Month", "JANUARY", "FEBRUARY", "MARCH",
+           "APRIL", "MAY", "JUNE", "JULY",
+           "AUGUST", "SEPTEMBER", "OCTOBER", "NOVEMBER", "DECEMBER",
+           "MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY",
+           "SATURDAY", "SUNDAY"]
 
 # Exception raised for bad input (with string parameter for details)
 error = ValueError
@@ -35,9 +42,46 @@ class IllegalWeekdayError(ValueError):
         return "bad weekday number %r; must be 0 (Monday) to 6 (Sunday)" % self.weekday
 
 
-# Constants for months referenced later
+def __getattr__(name):
-January = 1
+    if name in ('January', 'February'):
-February = 2
+        warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead",
+                      DeprecationWarning, stacklevel=2)
+        if name == 'January':
+            return 1
+        else:
+            return 2
+
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+# Constants for months
+@global_enum
+class Month(IntEnum):
+    JANUARY = 1
+    FEBRUARY = 2
+    MARCH = 3
+    APRIL = 4
+    MAY = 5
+    JUNE = 6
+    JULY = 7
+    AUGUST = 8
+    SEPTEMBER = 9
+    OCTOBER = 10
+    NOVEMBER = 11
+    DECEMBER = 12
+
+
+# Constants for days
+@global_enum
+class Day(IntEnum):
+    MONDAY = 0
+    TUESDAY = 1
+    WEDNESDAY = 2
+    THURSDAY = 3
+    FRIDAY = 4
+    SATURDAY = 5
+    SUNDAY = 6
+
 
 # Number of days per month (except for February in leap years)
 mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
@@ -93,9 +137,6 @@ day_abbr = _localized_day('%a')
 month_name = _localized_month('%B')
 month_abbr = _localized_month('%b')
 
-# Constants for weekdays
-(MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY) = range(7)
-
 
 def isleap(year):
     """Return True for leap years, False for non-leap years."""
@@ -114,7 +155,7 @@ def weekday(year, month, day):
     """Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31)."""
     if not datetime.MINYEAR <= year <= datetime.MAXYEAR:
         year = 2000 + year % 400
-    return datetime.date(year, month, day).weekday()
+    return Day(datetime.date(year, month, day).weekday())
 
 
 def monthrange(year, month):
@@ -123,12 +164,12 @@ def monthrange(year, month):
     if not 1 <= month <= 12:
         raise IllegalMonthError(month)
     day1 = weekday(year, month, 1)
-    ndays = mdays[month] + (month == February and isleap(year))
+    ndays = mdays[month] + (month == FEBRUARY and isleap(year))
     return day1, ndays
 
 
 def _monthlen(year, month):
-    return mdays[month] + (month == February and isleap(year))
+    return mdays[month] + (month == FEBRUARY and isleap(year))
 
 
 def _prevmonth(year, month):
@@ -258,10 +299,7 @@ class Calendar(object):
         Each month contains between 4 and 6 weeks and each week contains 1-7
         days. Days are datetime.date objects.
         """
-        months = [
+        months = [self.monthdatescalendar(year, m) for m in Month]
-            self.monthdatescalendar(year, i)
-            for i in range(January, January+12)
-        ]
         return [months[i:i+width] for i in range(0, len(months), width) ]
 
     def yeardays2calendar(self, year, width=3):
@@ -271,10 +309,7 @@ class Calendar(object):
         (day number, weekday number) tuples. Day numbers outside this month are
         zero.
         """
-        months = [
+        months = [self.monthdays2calendar(year, m) for m in Month]
-            self.monthdays2calendar(year, i)
-            for i in range(January, January+12)
-        ]
         return [months[i:i+width] for i in range(0, len(months), width) ]
 
     def yeardayscalendar(self, year, width=3):
@@ -283,10 +318,7 @@ class Calendar(object):
         yeardatescalendar()). Entries in the week lists are day numbers.
         Day numbers outside this month are zero.
         """
-        months = [
+        months = [self.monthdayscalendar(year, m) for m in Month]
-            self.monthdayscalendar(year, i)
-            for i in range(January, January+12)
-        ]
         return [months[i:i+width] for i in range(0, len(months), width) ]
 
 
@@ -507,7 +539,7 @@ class HTMLCalendar(Calendar):
         a('\n')
         a('<tr><th colspan="%d" class="%s">%s</th></tr>' % (
             width, self.cssclass_year_head, theyear))
-        for i in range(January, January+12, width):
+        for i in range(JANUARY, JANUARY+12, width):
             # months in this row
             months = range(i, min(i+width, 13))
             a('<tr>')
@@ -546,71 +578,67 @@ class HTMLCalendar(Calendar):
 class different_locale:
     def __init__(self, locale):
         self.locale = locale
+        self.oldlocale = None
 
     def __enter__(self):
-        self.oldlocale = _locale.getlocale(_locale.LC_TIME)
+        self.oldlocale = _locale.setlocale(_locale.LC_TIME, None)
         _locale.setlocale(_locale.LC_TIME, self.locale)
 
     def __exit__(self, *args):
+        if self.oldlocale is None:
+            return
         _locale.setlocale(_locale.LC_TIME, self.oldlocale)
 
 
+def _get_default_locale():
+    locale = _locale.setlocale(_locale.LC_TIME, None)
+    if locale == "C":
+        with different_locale(""):
+            # The LC_TIME locale does not seem to be configured:
+            # get the user preferred locale.
+            locale = _locale.setlocale(_locale.LC_TIME, None)
+    return locale
+
+
 class LocaleTextCalendar(TextCalendar):
     """
     This class can be passed a locale name in the constructor and will return
-    month and weekday names in the specified locale. If this locale includes
+    month and weekday names in the specified locale.
-    an encoding all strings containing month and weekday names will be returned
-    as unicode.
     """
 
     def __init__(self, firstweekday=0, locale=None):
         TextCalendar.__init__(self, firstweekday)
         if locale is None:
-            locale = _locale.getdefaultlocale()
+            locale = _get_default_locale()
         self.locale = locale
 
     def formatweekday(self, day, width):
         with different_locale(self.locale):
-            if width >= 9:
+            return super().formatweekday(day, width)
-                names = day_name
-            else:
-                names = day_abbr
-            name = names[day]
-            return name[:width].center(width)
 
     def formatmonthname(self, theyear, themonth, width, withyear=True):
         with different_locale(self.locale):
-            s = month_name[themonth]
+            return super().formatmonthname(theyear, themonth, width, withyear)
-            if withyear:
-                s = "%s %r" % (s, theyear)
-            return s.center(width)
 
 
 class LocaleHTMLCalendar(HTMLCalendar):
     """
     This class can be passed a locale name in the constructor and will return
-    month and weekday names in the specified locale. If this locale includes
+    month and weekday names in the specified locale.
-    an encoding all strings containing month and weekday names will be returned
-    as unicode.
     """
     def __init__(self, firstweekday=0, locale=None):
         HTMLCalendar.__init__(self, firstweekday)
         if locale is None:
-            locale = _locale.getdefaultlocale()
+            locale = _get_default_locale()
         self.locale = locale
 
     def formatweekday(self, day):
         with different_locale(self.locale):
-            s = day_abbr[day]
+            return super().formatweekday(day)
-            return '<th class="%s">%s</th>' % (self.cssclasses[day], s)
 
     def formatmonthname(self, theyear, themonth, withyear=True):
         with different_locale(self.locale):
-            s = month_name[themonth]
+            return super().formatmonthname(theyear, themonth, withyear)
-            if withyear:
-                s = '%s %s' % (s, theyear)
-            return '<tr><th colspan="7" class="month">%s</th></tr>' % s
-
 
 # Support for old module level interface
 c = TextCalendar()
@@ -695,7 +723,7 @@ def main(args):
     parser.add_argument(
         "-L", "--locale",
         default=None,
-        help="locale to be used from month and weekday names"
+        help="locale to use for month and weekday names"
     )
     parser.add_argument(
         "-e", "--encoding",

Lib/cgi.py

@@ -13,6 +13,11 @@
 
 This module defines a number of utilities for use by CGI scripts
 written in Python.
+
+The global variable maxlen can be set to an integer indicating the maximum size
+of a POST request. POST requests larger than this size will result in a
+ValueError being raised during parsing. The default value of this variable is 0,
+meaning the request size is unlimited.
 """
 
 # History
@@ -41,12 +46,16 @@ from email.message import Message
 import html
 import locale
 import tempfile
+import warnings
 
 __all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart",
            "parse_header", "test", "print_exception", "print_environ",
            "print_form", "print_directory", "print_arguments",
            "print_environ_usage"]
 
+
+warnings._deprecated(__name__, remove=(3,13))
+
 # Logging support
 # ===============
 
@@ -77,9 +86,11 @@ def initlog(*allargs):
 
     """
     global log, logfile, logfp
+    warnings.warn("cgi.log() is deprecated as of 3.10. Use logging instead",
+                  DeprecationWarning, stacklevel=2)
     if logfile and not logfp:
         try:
-            logfp = open(logfile, "a")
+            logfp = open(logfile, "a", encoding="locale")
         except OSError:
             pass
     if not logfp:
@@ -115,7 +126,8 @@ log = initlog           # The current logging function
 # 0 ==> unlimited input
 maxlen = 0
 
-def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
+def parse(fp=None, environ=os.environ, keep_blank_values=0,
+          strict_parsing=0, separator='&'):
     """Parse a query in the environment or from a file (default stdin)
 
         Arguments, all optional:
@@ -134,6 +146,9 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
         strict_parsing: flag indicating what to do with parsing errors.
             If false (the default), errors are silently ignored.
             If true, errors raise a ValueError exception.
+
+        separator: str. The symbol to use for separating the query arguments.
+            Defaults to &.
     """
     if fp is None:
         fp = sys.stdin
@@ -154,7 +169,7 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
     if environ['REQUEST_METHOD'] == 'POST':
         ctype, pdict = parse_header(environ['CONTENT_TYPE'])
         if ctype == 'multipart/form-data':
-            return parse_multipart(fp, pdict)
+            return parse_multipart(fp, pdict, separator=separator)
         elif ctype == 'application/x-www-form-urlencoded':
             clength = int(environ['CONTENT_LENGTH'])
             if maxlen and clength > maxlen:
@@ -178,10 +193,10 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
             qs = ""
         environ['QUERY_STRING'] = qs    # XXX Shouldn't, really
     return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
-                                 encoding=encoding)
+                                 encoding=encoding, separator=separator)
 
 
-def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"):
+def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'):
     """Parse multipart input.
 
     Arguments:
@@ -194,15 +209,18 @@ def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"):
     value is a list of values for that field. For non-file fields, the value
     is a list of strings.
     """
-    # RFC 2026, Section 5.1 : The "multipart" boundary delimiters are always
+    # RFC 2046, Section 5.1 : The "multipart" boundary delimiters are always
     # represented as 7bit US-ASCII.
     boundary = pdict['boundary'].decode('ascii')
     ctype = "multipart/form-data; boundary={}".format(boundary)
     headers = Message()
     headers.set_type(ctype)
-    headers['Content-Length'] = pdict['CONTENT-LENGTH']
+    try:
+        headers['Content-Length'] = pdict['CONTENT-LENGTH']
+    except KeyError:
+        pass
     fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors,
-        environ={'REQUEST_METHOD': 'POST'})
+        environ={'REQUEST_METHOD': 'POST'}, separator=separator)
     return {k: fs.getlist(k) for k in fs}
 
 def _parseparam(s):
@@ -312,7 +330,7 @@ class FieldStorage:
     def __init__(self, fp=None, headers=None, outerboundary=b'',
                  environ=os.environ, keep_blank_values=0, strict_parsing=0,
                  limit=None, encoding='utf-8', errors='replace',
-                 max_num_fields=None):
+                 max_num_fields=None, separator='&'):
         """Constructor.  Read multipart/* until last part.
 
         Arguments, all optional:
@@ -360,6 +378,7 @@ class FieldStorage:
         self.keep_blank_values = keep_blank_values
         self.strict_parsing = strict_parsing
         self.max_num_fields = max_num_fields
+        self.separator = separator
         if 'REQUEST_METHOD' in environ:
             method = environ['REQUEST_METHOD'].upper()
         self.qs_on_post = None
@@ -586,7 +605,7 @@ class FieldStorage:
         query = urllib.parse.parse_qsl(
             qs, self.keep_blank_values, self.strict_parsing,
             encoding=self.encoding, errors=self.errors,
-            max_num_fields=self.max_num_fields)
+            max_num_fields=self.max_num_fields, separator=self.separator)
         self.list = [MiniFieldStorage(key, value) for key, value in query]
         self.skip_lines()
 
@@ -602,7 +621,7 @@ class FieldStorage:
             query = urllib.parse.parse_qsl(
                 self.qs_on_post, self.keep_blank_values, self.strict_parsing,
                 encoding=self.encoding, errors=self.errors,
-                max_num_fields=self.max_num_fields)
+                max_num_fields=self.max_num_fields, separator=self.separator)
             self.list.extend(MiniFieldStorage(key, value) for key, value in query)
 
         klass = self.FieldStorageClass or self.__class__
@@ -646,7 +665,7 @@ class FieldStorage:
                 else self.limit - self.bytes_read
             part = klass(self.fp, headers, ib, environ, keep_blank_values,
                          strict_parsing, limit,
-                         self.encoding, self.errors, max_num_fields)
+                         self.encoding, self.errors, max_num_fields, self.separator)
 
             if max_num_fields is not None:
                 max_num_fields -= 1
@@ -736,7 +755,8 @@ class FieldStorage:
         last_line_lfend = True
         _read = 0
         while 1:
-            if self.limit is not None and _read >= self.limit:
+
+            if self.limit is not None and 0 <= self.limit <= _read:
                 break
             line = self.fp.readline(1<<16) # bytes
             self.bytes_read += len(line)

Lib/cgitb.py

@@ -31,6 +31,11 @@ import tempfile
 import time
 import tokenize
 import traceback
+import warnings
+from html import escape as html_escape
+
+warnings._deprecated(__name__, remove=(3, 13))
+
 
 def reset():
     """Return a string that resets the CGI and browser to a known state."""
@@ -69,7 +74,7 @@ def lookup(name, frame, locals):
         return 'global', frame.f_globals[name]
     if '__builtins__' in frame.f_globals:
         builtins = frame.f_globals['__builtins__']
-        if type(builtins) is type({}):
+        if isinstance(builtins, dict):
             if name in builtins:
                 return 'builtin', builtins[name]
         else:
@@ -105,10 +110,16 @@ def html(einfo, context=5):
         etype = etype.__name__
     pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
     date = time.ctime(time.time())
-    head = '<body bgcolor="#f0f0f8">' + pydoc.html.heading(
+    head = f'''
-        '<big><big>%s</big></big>' %
+<body bgcolor="#f0f0f8">
-        strong(pydoc.html.escape(str(etype))),
+<table width="100%" cellspacing=0 cellpadding=2 border=0 summary="heading">
-        '#ffffff', '#6622aa', pyver + '<br>' + date) + '''
+<tr bgcolor="#6622aa">
+<td valign=bottom>&nbsp;<br>
+<font color="#ffffff" face="helvetica, arial">&nbsp;<br>
+<big><big><strong>{html_escape(str(etype))}</strong></big></big></font></td>
+<td align=right valign=bottom>
+<font color="#ffffff" face="helvetica, arial">{pyver}<br>{date}</font></td>
+</tr></table>
 <p>A problem occurred in a Python script.  Here is the sequence of
 function calls leading up to the error, in the order they occurred.</p>'''
 
@@ -181,8 +192,8 @@ function calls leading up to the error, in the order they occurred.</p>'''
 
 
 <!-- The above is a description of an error in a Python program, formatted
-     for a Web browser because the 'cgitb' module was enabled.  In case you
+     for a web browser because the 'cgitb' module was enabled.  In case you
-     are not reading this in a Web browser, here is the original traceback:
+     are not reading this in a web browser, here is the original traceback:
 
 %s
 -->

Lib/chunk.py

@@ -48,6 +48,10 @@ specifies whether or not chunks are aligned on 2-byte boundaries.  The
 default is 1, i.e. aligned.
 """
 
+import warnings
+
+warnings._deprecated(__name__, remove=(3, 13))
+
 class Chunk:
     def __init__(self, file, align=True, bigendian=True, inclheader=False):
         import struct
@@ -64,7 +68,7 @@ class Chunk:
         try:
             self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0]
         except struct.error:
-            raise EOFError
+            raise EOFError from None
         if inclheader:
             self.chunksize = self.chunksize - 8 # subtract header
         self.size_read = 0

Lib/cmd.py

@@ -310,10 +310,10 @@ class Cmd:
             names = self.get_names()
             cmds_doc = []
             cmds_undoc = []
-            help = {}
+            topics = set()
             for name in names:
                 if name[:5] == 'help_':
-                    help[name[5:]]=1
+                    topics.add(name[5:])
             names.sort()
             # There can be duplicates if routines overridden
             prevname = ''
@@ -323,16 +323,16 @@ class Cmd:
                         continue
                     prevname = name
                     cmd=name[3:]
-                    if cmd in help:
+                    if cmd in topics:
                         cmds_doc.append(cmd)
-                        del help[cmd]
+                        topics.remove(cmd)
                     elif getattr(self, name).__doc__:
                         cmds_doc.append(cmd)
                     else:
                         cmds_undoc.append(cmd)
             self.stdout.write("%s\n"%str(self.doc_leader))
             self.print_topics(self.doc_header,   cmds_doc,   15,80)
-            self.print_topics(self.misc_header,  list(help.keys()),15,80)
+            self.print_topics(self.misc_header,  sorted(topics),15,80)
             self.print_topics(self.undoc_header, cmds_undoc, 15,80)
 
     def print_topics(self, header, cmds, cmdlen, maxcol):

Lib/code.py

@@ -7,7 +7,6 @@
 
 import sys
 import traceback
-import argparse
 from codeop import CommandCompiler, compile_command
 
 __all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact",
@@ -41,7 +40,7 @@ class InteractiveInterpreter:
 
         Arguments are as for compile_command().
 
-        One several things can happen:
+        One of several things can happen:
 
         1) The input is incorrect; compile_command() raised an
         exception (SyntaxError or OverflowError).  A syntax traceback
@@ -107,6 +106,7 @@ class InteractiveInterpreter:
 
         """
         type, value, tb = sys.exc_info()
+        sys.last_exc = value
         sys.last_type = type
         sys.last_value = value
         sys.last_traceback = tb
@@ -120,7 +120,7 @@ class InteractiveInterpreter:
             else:
                 # Stuff in the right filename
                 value = SyntaxError(msg, (filename, lineno, offset, line))
-                sys.last_value = value
+                sys.last_exc = sys.last_value = value
         if sys.excepthook is sys.__excepthook__:
             lines = traceback.format_exception_only(type, value)
             self.write(''.join(lines))
@@ -139,6 +139,7 @@ class InteractiveInterpreter:
         """
         sys.last_type, sys.last_value, last_tb = ei = sys.exc_info()
         sys.last_traceback = last_tb
+        sys.last_exc = ei[1]
         try:
             lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next)
             if sys.excepthook is sys.__excepthook__:
@@ -303,6 +304,8 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None):
 
 
 if __name__ == "__main__":
+    import argparse
+
     parser = argparse.ArgumentParser()
     parser.add_argument('-q', action='store_true',
                        help="don't print version and copyright messages")

Lib/codeop.py

@@ -10,30 +10,6 @@ and:
   syntax error (OverflowError and ValueError can be produced by
   malformed literals).
 
-Approach:
-
-First, check if the source consists entirely of blank lines and
-comments; if so, replace it with 'pass', because the built-in
-parser doesn't always do the right thing for these.
-
-Compile three times: as is, with \n, and with \n\n appended.  If it
-compiles as is, it's complete.  If it compiles with one \n appended,
-we expect more.  If it doesn't compile either way, we compare the
-error we get when compiling with \n or \n\n appended.  If the errors
-are the same, the code is broken.  But if the errors are different, we
-expect more.  Not intuitive; not even guaranteed to hold in future
-releases; but this matches the compiler's behavior from Python 1.4
-through 2.2, at least.
-
-Caveat:
-
-It is possible (but not likely) that the parser stops parsing with a
-successful outcome before reaching the end of the source; in this
-case, trailing symbols may be ignored instead of causing an error.
-For example, a backslash followed by two newlines may be followed by
-arbitrary garbage.  This will be fixed once the API for the parser is
-better.
-
 The two interfaces are:
 
 compile_command(source, filename, symbol):
@@ -64,53 +40,53 @@ _features = [getattr(__future__, fname)
 
 __all__ = ["compile_command", "Compile", "CommandCompiler"]
 
-PyCF_DONT_IMPLY_DEDENT = 0x200          # Matches pythonrun.h
+# The following flags match the values from Include/cpython/compile.h
-
+# Caveat emptor: These flags are undocumented on purpose and depending
+# on their effect outside the standard library is **unsupported**.
+PyCF_DONT_IMPLY_DEDENT = 0x200
+PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
 
 def _maybe_compile(compiler, source, filename, symbol):
-    # Check for source consisting of only blank lines and comments
+    # Check for source consisting of only blank lines and comments.
     for line in source.split("\n"):
         line = line.strip()
         if line and line[0] != '#':
-            break               # Leave it alone
+            break               # Leave it alone.
     else:
         if symbol != "eval":
             source = "pass"     # Replace it with a 'pass' statement
 
-    err = err1 = err2 = None
+    # Disable compiler warnings when checking for incomplete input.
-    code = code1 = code2 = None
-
-    try:
-        code = compiler(source, filename, symbol)
-    except SyntaxError:
-        pass
-
-    # Catch syntax warnings after the first compile
-    # to emit warnings (SyntaxWarning, DeprecationWarning) at most once.
     with warnings.catch_warnings():
-        warnings.simplefilter("error")
+        warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
-
         try:
-            code1 = compiler(source + "\n", filename, symbol)
+            compiler(source, filename, symbol)
-        except SyntaxError as e:
+        except SyntaxError:  # Let other compile() errors propagate.
-            err1 = e
+            try:
+                compiler(source + "\n", filename, symbol)
+                return None
+            except SyntaxError as e:
+                if "incomplete input" in str(e):
+                    return None
+                # fallthrough
 
-        try:
+    return compiler(source, filename, symbol, incomplete_input=False)
-            code2 = compiler(source + "\n\n", filename, symbol)
-        except SyntaxError as e:
-            err2 = e
 
-    try:
+def _is_syntax_error(err1, err2):
-        if code:
+    rep1 = repr(err1)
-            return code
+    rep2 = repr(err2)
-        if not code1 and repr(err1) == repr(err2):
+    if "was never closed" in rep1 and "was never closed" in rep2:
-            raise err1
+        return False
-    finally:
+    if rep1 == rep2:
-        err1 = err2 = None
+        return True
+    return False
 
-
+def _compile(source, filename, symbol, incomplete_input=True):
-def _compile(source, filename, symbol):
+    flags = 0
-    return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT)
+    if incomplete_input:
+        flags |= PyCF_ALLOW_INCOMPLETE_INPUT
+        flags |= PyCF_DONT_IMPLY_DEDENT
+    return compile(source, filename, symbol, flags)
 
 
 def compile_command(source, filename="<input>", symbol="single"):
@@ -134,24 +110,25 @@ def compile_command(source, filename="<input>", symbol="single"):
     """
     return _maybe_compile(_compile, source, filename, symbol)
 
-
 class Compile:
     """Instances of this class behave much like the built-in compile
     function, but if one is used to compile text containing a future
     statement, it "remembers" and compiles all subsequent program texts
     with the statement in force."""
-
     def __init__(self):
-        self.flags = PyCF_DONT_IMPLY_DEDENT
+        self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
 
-    def __call__(self, source, filename, symbol):
+    def __call__(self, source, filename, symbol, **kwargs):
-        codeob = compile(source, filename, symbol, self.flags, True)
+        flags = self.flags
+        if kwargs.get('incomplete_input', True) is False:
+            flags &= ~PyCF_DONT_IMPLY_DEDENT
+            flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT
+        codeob = compile(source, filename, symbol, flags, True)
         for feature in _features:
             if codeob.co_flags & feature.compiler_flag:
                 self.flags |= feature.compiler_flag
         return codeob
 
-
 class CommandCompiler:
     """Instances of this class have __call__ methods identical in
     signature to compile_command; the difference is that if the

Lib/collections/__init__.py

@@ -45,6 +45,11 @@ except ImportError:
 else:
     _collections_abc.MutableSequence.register(deque)
 
+try:
+    from _collections import _deque_iterator
+except ImportError:
+    pass
+
 try:
     from _collections import defaultdict
 except ImportError:
@@ -94,17 +99,19 @@ class OrderedDict(dict):
     # Individual links are kept alive by the hard reference in self.__map.
     # Those hard references disappear when a key is deleted from an OrderedDict.
 
+    def __new__(cls, /, *args, **kwds):
+        "Create the ordered dict object and set up the underlying structures."
+        self = dict.__new__(cls)
+        self.__hardroot = _Link()
+        self.__root = root = _proxy(self.__hardroot)
+        root.prev = root.next = root
+        self.__map = {}
+        return self
+
     def __init__(self, other=(), /, **kwds):
         '''Initialize an ordered dictionary.  The signature is the same as
         regular dictionaries.  Keyword argument order is preserved.
         '''
-        try:
-            self.__root
-        except AttributeError:
-            self.__hardroot = _Link()
-            self.__root = root = _proxy(self.__hardroot)
-            root.prev = root.next = root
-            self.__map = {}
         self.__update(other, **kwds)
 
     def __setitem__(self, key, value,
@@ -240,11 +247,19 @@ class OrderedDict(dict):
         is raised.
 
         '''
-        if key in self:
+        marker = self.__marker
-            result = self[key]
+        result = dict.pop(self, key, marker)
-            del self[key]
+        if result is not marker:
+            # The same as in __delitem__().
+            link = self.__map.pop(key)
+            link_prev = link.prev
+            link_next = link.next
+            link_prev.next = link_next
+            link_next.prev = link_prev
+            link.prev = None
+            link.next = None
             return result
-        if default is self.__marker:
+        if default is marker:
             raise KeyError(key)
         return default
 
@@ -263,14 +278,26 @@ class OrderedDict(dict):
         'od.__repr__() <==> repr(od)'
         if not self:
             return '%s()' % (self.__class__.__name__,)
-        return '%s(%r)' % (self.__class__.__name__, list(self.items()))
+        return '%s(%r)' % (self.__class__.__name__, dict(self.items()))
 
     def __reduce__(self):
         'Return state information for pickling'
-        inst_dict = vars(self).copy()
+        state = self.__getstate__()
-        for k in vars(OrderedDict()):
+        if state:
-            inst_dict.pop(k, None)
+            if isinstance(state, tuple):
-        return self.__class__, (), inst_dict or None, None, iter(self.items())
+                state, slots = state
+            else:
+                slots = {}
+            state = state.copy()
+            slots = slots.copy()
+            for k in vars(OrderedDict()):
+                state.pop(k, None)
+                slots.pop(k, None)
+            if slots:
+                state = state, slots
+            else:
+                state = state or None
+        return self.__class__, (), state, None, iter(self.items())
 
     def copy(self):
         'od.copy() -> a shallow copy of od'
@@ -491,9 +518,12 @@ def namedtuple(typename, field_names, *, rename=False, defaults=None, module=Non
     # specified a particular module.
     if module is None:
         try:
-            module = _sys._getframe(1).f_globals.get('__name__', '__main__')
+            module = _sys._getframemodulename(1) or '__main__'
-        except (AttributeError, ValueError):
+        except AttributeError:
-            pass
+            try:
+                module = _sys._getframe(1).f_globals.get('__name__', '__main__')
+            except (AttributeError, ValueError):
+                pass
     if module is not None:
         result.__module__ = module
 
@@ -613,11 +643,9 @@ class Counter(dict):
         ['A', 'A', 'B', 'B', 'C', 'C']
 
         # Knuth's example for prime factors of 1836:  2**2 * 3**3 * 17**1
+        >>> import math
         >>> prime_factors = Counter({2: 2, 3: 3, 17: 1})
-        >>> product = 1
+        >>> math.prod(prime_factors.elements())
-        >>> for factor in prime_factors.elements():     # loop over factors
-        ...     product *= factor                       # and multiply them
-        >>> product
         1836
 
         Note, if an element's count has been set to zero or is a negative
@@ -714,42 +742,6 @@ class Counter(dict):
         if elem in self:
             super().__delitem__(elem)
 
-    def __eq__(self, other):
-        'True if all counts agree. Missing counts are treated as zero.'
-        if not isinstance(other, Counter):
-            return NotImplemented
-        return all(self[e] == other[e] for c in (self, other) for e in c)
-
-    def __ne__(self, other):
-        'True if any counts disagree. Missing counts are treated as zero.'
-        if not isinstance(other, Counter):
-            return NotImplemented
-        return not self == other
-
-    def __le__(self, other):
-        'True if all counts in self are a subset of those in other.'
-        if not isinstance(other, Counter):
-            return NotImplemented
-        return all(self[e] <= other[e] for c in (self, other) for e in c)
-
-    def __lt__(self, other):
-        'True if all counts in self are a proper subset of those in other.'
-        if not isinstance(other, Counter):
-            return NotImplemented
-        return self <= other and self != other
-
-    def __ge__(self, other):
-        'True if all counts in self are a superset of those in other.'
-        if not isinstance(other, Counter):
-            return NotImplemented
-        return all(self[e] >= other[e] for c in (self, other) for e in c)
-
-    def __gt__(self, other):
-        'True if all counts in self are a proper superset of those in other.'
-        if not isinstance(other, Counter):
-            return NotImplemented
-        return self >= other and self != other
-
     def __repr__(self):
         if not self:
             return f'{self.__class__.__name__}()'
@@ -795,6 +787,42 @@ class Counter(dict):
     #         (cp >= cq) == (sp >= sq)
     #         (cp > cq) == (sp > sq)
 
+    def __eq__(self, other):
+        'True if all counts agree. Missing counts are treated as zero.'
+        if not isinstance(other, Counter):
+            return NotImplemented
+        return all(self[e] == other[e] for c in (self, other) for e in c)
+
+    def __ne__(self, other):
+        'True if any counts disagree. Missing counts are treated as zero.'
+        if not isinstance(other, Counter):
+            return NotImplemented
+        return not self == other
+
+    def __le__(self, other):
+        'True if all counts in self are a subset of those in other.'
+        if not isinstance(other, Counter):
+            return NotImplemented
+        return all(self[e] <= other[e] for c in (self, other) for e in c)
+
+    def __lt__(self, other):
+        'True if all counts in self are a proper subset of those in other.'
+        if not isinstance(other, Counter):
+            return NotImplemented
+        return self <= other and self != other
+
+    def __ge__(self, other):
+        'True if all counts in self are a superset of those in other.'
+        if not isinstance(other, Counter):
+            return NotImplemented
+        return all(self[e] >= other[e] for c in (self, other) for e in c)
+
+    def __gt__(self, other):
+        'True if all counts in self are a proper superset of those in other.'
+        if not isinstance(other, Counter):
+            return NotImplemented
+        return self >= other and self != other
+
     def __add__(self, other):
         '''Add counts from two counters.
 
@@ -997,8 +1025,8 @@ class ChainMap(_collections_abc.MutableMapping):
 
     def __iter__(self):
         d = {}
-        for mapping in reversed(self.maps):
+        for mapping in map(dict.fromkeys, reversed(self.maps)):
-            d.update(dict.fromkeys(mapping))    # reuses stored hash values if possible
+            d |= mapping                        # reuses stored hash values if possible
         return iter(d)
 
     def __contains__(self, key):
@@ -1118,10 +1146,17 @@ class UserDict(_collections_abc.MutableMapping):
     def __iter__(self):
         return iter(self.data)
 
-    # Modify __contains__ to work correctly when __missing__ is present
+    # Modify __contains__ and get() to work like dict
+    # does when __missing__ is present.
     def __contains__(self, key):
         return key in self.data
 
+    def get(self, key, default=None):
+        if key in self:
+            return self[key]
+        return default
+
+
     # Now, add the methods in dicts but not in MutableMapping
     def __repr__(self):
         return repr(self.data)

Lib/colorsys.py

@@ -1,10 +1,14 @@
 """Conversion functions between RGB and other color systems.
+
 This modules provides two functions for each color system ABC:
+
   rgb_to_abc(r, g, b) --> a, b, c
   abc_to_rgb(a, b, c) --> r, g, b
+
 All inputs and outputs are triples of floats in the range [0.0...1.0]
 (with the exception of I and Q, which covers a slightly larger range).
 Inputs outside the valid range may cause exceptions or invalid outputs.
+
 Supported color systems:
 RGB: Red, Green, Blue components
 YIQ: Luminance, Chrominance (used by composite video signals)
@@ -71,17 +75,18 @@ def yiq_to_rgb(y, i, q):
 def rgb_to_hls(r, g, b):
     maxc = max(r, g, b)
     minc = min(r, g, b)
-    # XXX Can optimize (maxc+minc) and (maxc-minc)
+    sumc = (maxc+minc)
-    l = (minc+maxc)/2.0
+    rangec = (maxc-minc)
+    l = sumc/2.0
     if minc == maxc:
         return 0.0, l, 0.0
     if l <= 0.5:
-        s = (maxc-minc) / (maxc+minc)
+        s = rangec / sumc
     else:
-        s = (maxc-minc) / (2.0-maxc-minc)
+        s = rangec / (2.0-maxc-minc)  # Not always 2.0-sumc: gh-106498.
-    rc = (maxc-r) / (maxc-minc)
+    rc = (maxc-r) / rangec
-    gc = (maxc-g) / (maxc-minc)
+    gc = (maxc-g) / rangec
-    bc = (maxc-b) / (maxc-minc)
+    bc = (maxc-b) / rangec
     if r == maxc:
         h = bc-gc
     elif g == maxc:
@@ -120,13 +125,14 @@ def _v(m1, m2, hue):
 def rgb_to_hsv(r, g, b):
     maxc = max(r, g, b)
     minc = min(r, g, b)
+    rangec = (maxc-minc)
     v = maxc
     if minc == maxc:
         return 0.0, 0.0, v
-    s = (maxc-minc) / maxc
+    s = rangec / maxc
-    rc = (maxc-r) / (maxc-minc)
+    rc = (maxc-r) / rangec
-    gc = (maxc-g) / (maxc-minc)
+    gc = (maxc-g) / rangec
-    bc = (maxc-b) / (maxc-minc)
+    bc = (maxc-b) / rangec
     if r == maxc:
         h = bc-gc
     elif g == maxc:

Lib/compileall.py

@@ -4,7 +4,7 @@ When called as a script with arguments, this compiles the directories
 given as arguments recursively; the -l option prevents it from
 recursing into directories.
 
-Without arguments, if compiles all modules on sys.path, without
+Without arguments, it compiles all modules on sys.path, without
 recursing into subdirectories.  (Even though it should do so for
 packages -- for now, you'll have to deal with packages separately.)
 
@@ -15,16 +15,14 @@ import sys
 import importlib.util
 import py_compile
 import struct
+import filecmp
 
-try:
-    from concurrent.futures import ProcessPoolExecutor
-except ImportError:
-    ProcessPoolExecutor = None
 from functools import partial
+from pathlib import Path
 
 __all__ = ["compile_dir","compile_file","compile_path"]
 
-def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
+def _walk_dir(dir, maxlevels, quiet=0):
     if quiet < 2 and isinstance(dir, os.PathLike):
         dir = os.fspath(dir)
     if not quiet:
@@ -40,59 +38,94 @@ def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
         if name == '__pycache__':
             continue
         fullname = os.path.join(dir, name)
-        if ddir is not None:
-            dfile = os.path.join(ddir, name)
-        else:
-            dfile = None
         if not os.path.isdir(fullname):
             yield fullname
         elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
               os.path.isdir(fullname) and not os.path.islink(fullname)):
-            yield from _walk_dir(fullname, ddir=dfile,
+            yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
-                                 maxlevels=maxlevels - 1, quiet=quiet)
+                                 quiet=quiet)
 
-def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
+def compile_dir(dir, maxlevels=None, ddir=None, force=False,
-                quiet=0, legacy=False, optimize=-1, workers=1):
+                rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
+                invalidation_mode=None, *, stripdir=None,
+                prependdir=None, limit_sl_dest=None, hardlink_dupes=False):
     """Byte-compile all modules in the given directory tree.
 
     Arguments (only dir is required):
 
     dir:       the directory to byte-compile
-    maxlevels: maximum recursion level (default 10)
+    maxlevels: maximum recursion level (default `sys.getrecursionlimit()`)
     ddir:      the directory that will be prepended to the path to the
                file as it is compiled into each byte-code file.
     force:     if True, force compilation, even if timestamps are up-to-date
     quiet:     full output with False or 0, errors only with 1,
                no output with 2
     legacy:    if True, produce legacy pyc paths instead of PEP 3147 paths
-    optimize:  optimization level or -1 for level of the interpreter
+    optimize:  int or list of optimization levels or -1 for level of
+               the interpreter. Multiple levels leads to multiple compiled
+               files each with one optimization level.
     workers:   maximum number of parallel workers
+    invalidation_mode: how the up-to-dateness of the pyc will be checked
+    stripdir:  part of path to left-strip from source file path
+    prependdir: path to prepend to beginning of original file path, applied
+               after stripdir
+    limit_sl_dest: ignore symlinks if they are pointing outside of
+                   the defined path
+    hardlink_dupes: hardlink duplicated pyc files
     """
-    if workers is not None and workers < 0:
+    ProcessPoolExecutor = None
+    if ddir is not None and (stripdir is not None or prependdir is not None):
+        raise ValueError(("Destination dir (ddir) cannot be used "
+                          "in combination with stripdir or prependdir"))
+    if ddir is not None:
+        stripdir = dir
+        prependdir = ddir
+        ddir = None
+    if workers < 0:
         raise ValueError('workers must be greater or equal to 0')
-
+    if workers != 1:
-    files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
+        # Check if this is a system where ProcessPoolExecutor can function.
-                      ddir=ddir)
+        from concurrent.futures.process import _check_system_limits
+        try:
+            _check_system_limits()
+        except NotImplementedError:
+            workers = 1
+        else:
+            from concurrent.futures import ProcessPoolExecutor
+    if maxlevels is None:
+        maxlevels = sys.getrecursionlimit()
+    files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
     success = True
-    if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
+    if workers != 1 and ProcessPoolExecutor is not None:
+        # If workers == 0, let ProcessPoolExecutor choose
         workers = workers or None
         with ProcessPoolExecutor(max_workers=workers) as executor:
             results = executor.map(partial(compile_file,
                                            ddir=ddir, force=force,
                                            rx=rx, quiet=quiet,
                                            legacy=legacy,
-                                           optimize=optimize),
+                                           optimize=optimize,
+                                           invalidation_mode=invalidation_mode,
+                                           stripdir=stripdir,
+                                           prependdir=prependdir,
+                                           limit_sl_dest=limit_sl_dest,
+                                           hardlink_dupes=hardlink_dupes),
                                    files)
             success = min(results, default=True)
     else:
         for file in files:
             if not compile_file(file, ddir, force, rx, quiet,
-                                legacy, optimize):
+                                legacy, optimize, invalidation_mode,
+                                stripdir=stripdir, prependdir=prependdir,
+                                limit_sl_dest=limit_sl_dest,
+                                hardlink_dupes=hardlink_dupes):
                 success = False
     return success
 
 def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
-                 legacy=False, optimize=-1):
+                 legacy=False, optimize=-1,
+                 invalidation_mode=None, *, stripdir=None, prependdir=None,
+                 limit_sl_dest=None, hardlink_dupes=False):
     """Byte-compile one file.
 
     Arguments (only fullname is required):
@@ -104,49 +137,114 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
     quiet:     full output with False or 0, errors only with 1,
                no output with 2
     legacy:    if True, produce legacy pyc paths instead of PEP 3147 paths
-    optimize:  optimization level or -1 for level of the interpreter
+    optimize:  int or list of optimization levels or -1 for level of
+               the interpreter. Multiple levels leads to multiple compiled
+               files each with one optimization level.
+    invalidation_mode: how the up-to-dateness of the pyc will be checked
+    stripdir:  part of path to left-strip from source file path
+    prependdir: path to prepend to beginning of original file path, applied
+               after stripdir
+    limit_sl_dest: ignore symlinks if they are pointing outside of
+                   the defined path.
+    hardlink_dupes: hardlink duplicated pyc files
     """
+
+    if ddir is not None and (stripdir is not None or prependdir is not None):
+        raise ValueError(("Destination dir (ddir) cannot be used "
+                          "in combination with stripdir or prependdir"))
+
     success = True
-    if quiet < 2 and isinstance(fullname, os.PathLike):
+    fullname = os.fspath(fullname)
-        fullname = os.fspath(fullname)
+    stripdir = os.fspath(stripdir) if stripdir is not None else None
     name = os.path.basename(fullname)
+
+    dfile = None
+
     if ddir is not None:
         dfile = os.path.join(ddir, name)
-    else:
+
-        dfile = None
+    if stripdir is not None:
+        fullname_parts = fullname.split(os.path.sep)
+        stripdir_parts = stripdir.split(os.path.sep)
+        ddir_parts = list(fullname_parts)
+
+        for spart, opart in zip(stripdir_parts, fullname_parts):
+            if spart == opart:
+                ddir_parts.remove(spart)
+
+        dfile = os.path.join(*ddir_parts)
+
+    if prependdir is not None:
+        if dfile is None:
+            dfile = os.path.join(prependdir, fullname)
+        else:
+            dfile = os.path.join(prependdir, dfile)
+
+    if isinstance(optimize, int):
+        optimize = [optimize]
+
+    # Use set() to remove duplicates.
+    # Use sorted() to create pyc files in a deterministic order.
+    optimize = sorted(set(optimize))
+
+    if hardlink_dupes and len(optimize) < 2:
+        raise ValueError("Hardlinking of duplicated bytecode makes sense "
+                          "only for more than one optimization level")
+
     if rx is not None:
         mo = rx.search(fullname)
         if mo:
             return success
+
+    if limit_sl_dest is not None and os.path.islink(fullname):
+        if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents:
+            return success
+
+    opt_cfiles = {}
+
     if os.path.isfile(fullname):
-        if legacy:
+        for opt_level in optimize:
-            cfile = fullname + 'c'
+            if legacy:
-        else:
+                opt_cfiles[opt_level] = fullname + 'c'
-            if optimize >= 0:
-                opt = optimize if optimize >= 1 else ''
-                cfile = importlib.util.cache_from_source(
-                                fullname, optimization=opt)
             else:
-                cfile = importlib.util.cache_from_source(fullname)
+                if opt_level >= 0:
-            cache_dir = os.path.dirname(cfile)
+                    opt = opt_level if opt_level >= 1 else ''
+                    cfile = (importlib.util.cache_from_source(
+                             fullname, optimization=opt))
+                    opt_cfiles[opt_level] = cfile
+                else:
+                    cfile = importlib.util.cache_from_source(fullname)
+                    opt_cfiles[opt_level] = cfile
+
         head, tail = name[:-3], name[-3:]
         if tail == '.py':
             if not force:
                 try:
                     mtime = int(os.stat(fullname).st_mtime)
-                    expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER,
+                    expect = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER,
-                                         mtime)
+                                         0, mtime & 0xFFFF_FFFF)
-                    with open(cfile, 'rb') as chandle:
+                    for cfile in opt_cfiles.values():
-                        actual = chandle.read(8)
+                        with open(cfile, 'rb') as chandle:
-                    if expect == actual:
+                            actual = chandle.read(12)
+                        if expect != actual:
+                            break
+                    else:
                         return success
                 except OSError:
                     pass
             if not quiet:
                 print('Compiling {!r}...'.format(fullname))
             try:
-                ok = py_compile.compile(fullname, cfile, dfile, True,
+                for index, opt_level in enumerate(optimize):
-                                        optimize=optimize)
+                    cfile = opt_cfiles[opt_level]
+                    ok = py_compile.compile(fullname, cfile, dfile, True,
+                                            optimize=opt_level,
+                                            invalidation_mode=invalidation_mode)
+                    if index > 0 and hardlink_dupes:
+                        previous_cfile = opt_cfiles[optimize[index - 1]]
+                        if filecmp.cmp(cfile, previous_cfile, shallow=False):
+                            os.unlink(cfile)
+                            os.link(previous_cfile, cfile)
             except py_compile.PyCompileError as err:
                 success = False
                 if quiet >= 2:
@@ -156,9 +254,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
                 else:
                     print('*** ', end='')
                 # escape non-printable characters in msg
-                msg = err.msg.encode(sys.stdout.encoding,
+                encoding = sys.stdout.encoding or sys.getdefaultencoding()
-                                     errors='backslashreplace')
+                msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding)
-                msg = msg.decode(sys.stdout.encoding)
                 print(msg)
             except (SyntaxError, UnicodeError, OSError) as e:
                 success = False
@@ -175,7 +272,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
     return success
 
 def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
-                 legacy=False, optimize=-1):
+                 legacy=False, optimize=-1,
+                 invalidation_mode=None):
     """Byte-compile all module on sys.path.
 
     Arguments (all optional):
@@ -186,6 +284,7 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
     quiet: as for compile_dir() (default 0)
     legacy: as for compile_dir() (default False)
     optimize: as for compile_dir() (default -1)
+    invalidation_mode: as for compiler_dir()
     """
     success = True
     for dir in sys.path:
@@ -193,9 +292,16 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
             if quiet < 2:
                 print('Skipping current directory')
         else:
-            success = success and compile_dir(dir, maxlevels, None,
+            success = success and compile_dir(
-                                              force, quiet=quiet,
+                dir,
-                                              legacy=legacy, optimize=optimize)
+                maxlevels,
+                None,
+                force,
+                quiet=quiet,
+                legacy=legacy,
+                optimize=optimize,
+                invalidation_mode=invalidation_mode,
+            )
     return success
 
 
@@ -206,7 +312,7 @@ def main():
     parser = argparse.ArgumentParser(
         description='Utilities to support installing Python libraries.')
     parser.add_argument('-l', action='store_const', const=0,
-                        default=10, dest='maxlevels',
+                        default=None, dest='maxlevels',
                         help="don't recurse into subdirectories")
     parser.add_argument('-r', type=int, dest='recursion',
                         help=('control the maximum recursion level. '
@@ -224,6 +330,20 @@ def main():
                               'compile-time tracebacks and in runtime '
                               'tracebacks in cases where the source file is '
                               'unavailable'))
+    parser.add_argument('-s', metavar='STRIPDIR',  dest='stripdir',
+                        default=None,
+                        help=('part of path to left-strip from path '
+                              'to source file - for example buildroot. '
+                              '`-d` and `-s` options cannot be '
+                              'specified together.'))
+    parser.add_argument('-p', metavar='PREPENDDIR',  dest='prependdir',
+                        default=None,
+                        help=('path to add as prefix to path '
+                              'to source file - for example / to make '
+                              'it absolute when some part is removed '
+                              'by `-s` option. '
+                              '`-d` and `-p` options cannot be '
+                              'specified together.'))
     parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
                         help=('skip files matching the regular expression; '
                               'the regexp is searched for in the full path '
@@ -238,6 +358,23 @@ def main():
                               'to the equivalent of -l sys.path'))
     parser.add_argument('-j', '--workers', default=1,
                         type=int, help='Run compileall concurrently')
+    invalidation_modes = [mode.name.lower().replace('_', '-')
+                          for mode in py_compile.PycInvalidationMode]
+    parser.add_argument('--invalidation-mode',
+                        choices=sorted(invalidation_modes),
+                        help=('set .pyc invalidation mode; defaults to '
+                              '"checked-hash" if the SOURCE_DATE_EPOCH '
+                              'environment variable is set, and '
+                              '"timestamp" otherwise.'))
+    parser.add_argument('-o', action='append', type=int, dest='opt_levels',
+                        help=('Optimization levels to run compilation with. '
+                              'Default is -1 which uses the optimization level '
+                              'of the Python interpreter itself (see -O).'))
+    parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',
+                        help='Ignore symlinks pointing outsite of the DIR')
+    parser.add_argument('--hardlink-dupes', action='store_true',
+                        dest='hardlink_dupes',
+                        help='Hardlink duplicated pyc files')
 
     args = parser.parse_args()
     compile_dests = args.compile_dest
@@ -246,16 +383,31 @@ def main():
         import re
         args.rx = re.compile(args.rx)
 
+    if args.limit_sl_dest == "":
+        args.limit_sl_dest = None
 
     if args.recursion is not None:
         maxlevels = args.recursion
     else:
         maxlevels = args.maxlevels
 
+    if args.opt_levels is None:
+        args.opt_levels = [-1]
+
+    if len(args.opt_levels) == 1 and args.hardlink_dupes:
+        parser.error(("Hardlinking of duplicated bytecode makes sense "
+                      "only for more than one optimization level."))
+
+    if args.ddir is not None and (
+        args.stripdir is not None or args.prependdir is not None
+    ):
+        parser.error("-d cannot be used in combination with -s or -p")
+
     # if flist is provided then load it
     if args.flist:
         try:
-            with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
+            with (sys.stdin if args.flist=='-' else
+                    open(args.flist, encoding="utf-8")) as f:
                 for line in f:
                     compile_dests.append(line.strip())
         except OSError:
@@ -263,8 +415,11 @@ def main():
                 print("Error reading file list {}".format(args.flist))
             return False
 
-    if args.workers is not None:
+    if args.invalidation_mode:
-        args.workers = args.workers or None
+        ivl_mode = args.invalidation_mode.replace('-', '_').upper()
+        invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]
+    else:
+        invalidation_mode = None
 
     success = True
     try:
@@ -272,17 +427,30 @@ def main():
             for dest in compile_dests:
                 if os.path.isfile(dest):
                     if not compile_file(dest, args.ddir, args.force, args.rx,
-                                        args.quiet, args.legacy):
+                                        args.quiet, args.legacy,
+                                        invalidation_mode=invalidation_mode,
+                                        stripdir=args.stripdir,
+                                        prependdir=args.prependdir,
+                                        optimize=args.opt_levels,
+                                        limit_sl_dest=args.limit_sl_dest,
+                                        hardlink_dupes=args.hardlink_dupes):
                         success = False
                 else:
                     if not compile_dir(dest, maxlevels, args.ddir,
                                        args.force, args.rx, args.quiet,
-                                       args.legacy, workers=args.workers):
+                                       args.legacy, workers=args.workers,
+                                       invalidation_mode=invalidation_mode,
+                                       stripdir=args.stripdir,
+                                       prependdir=args.prependdir,
+                                       optimize=args.opt_levels,
+                                       limit_sl_dest=args.limit_sl_dest,
+                                       hardlink_dupes=args.hardlink_dupes):
                         success = False
             return success
         else:
             return compile_path(legacy=args.legacy, force=args.force,
-                                quiet=args.quiet)
+                                quiet=args.quiet,
+                                invalidation_mode=invalidation_mode)
     except KeyboardInterrupt:
         if args.quiet < 2:
             print("\n[interrupted]")

Lib/concurrent/futures/thread.py

@@ -37,7 +37,8 @@ def _python_exit():
 threading._register_atexit(_python_exit)
 
 # At fork, reinitialize the `_global_shutdown_lock` lock in the child process
-if hasattr(os, 'register_at_fork'):
+# TODO RUSTPYTHON - _at_fork_reinit is not implemented yet
+if hasattr(os, 'register_at_fork') and hasattr(_global_shutdown_lock, '_at_fork_reinit'):
     os.register_at_fork(before=_global_shutdown_lock.acquire,
                         after_in_child=_global_shutdown_lock._at_fork_reinit,
                         after_in_parent=_global_shutdown_lock.release)

Lib/configparser.py

@@ -19,36 +19,37 @@ ConfigParser -- responsible for parsing a list of
              inline_comment_prefixes=None, strict=True,
              empty_lines_in_values=True, default_section='DEFAULT',
              interpolation=<unset>, converters=<unset>):
-        Create the parser. When `defaults' is given, it is initialized into the
+
+        Create the parser. When `defaults` is given, it is initialized into the
         dictionary or intrinsic defaults. The keys must be strings, the values
         must be appropriate for %()s string interpolation.
 
-        When `dict_type' is given, it will be used to create the dictionary
+        When `dict_type` is given, it will be used to create the dictionary
         objects for the list of sections, for the options within a section, and
         for the default values.
 
-        When `delimiters' is given, it will be used as the set of substrings
+        When `delimiters` is given, it will be used as the set of substrings
         that divide keys from values.
 
-        When `comment_prefixes' is given, it will be used as the set of
+        When `comment_prefixes` is given, it will be used as the set of
         substrings that prefix comments in empty lines. Comments can be
         indented.
 
-        When `inline_comment_prefixes' is given, it will be used as the set of
+        When `inline_comment_prefixes` is given, it will be used as the set of
         substrings that prefix comments in non-empty lines.
 
         When `strict` is True, the parser won't allow for any section or option
         duplicates while reading from a single source (file, string or
         dictionary). Default is True.
 
-        When `empty_lines_in_values' is False (default: True), each empty line
+        When `empty_lines_in_values` is False (default: True), each empty line
         marks the end of an option. Otherwise, internal empty lines of
         a multiline option are kept as part of the value.
 
-        When `allow_no_value' is True (default: False), options without
+        When `allow_no_value` is True (default: False), options without
         values are accepted; the value presented for these is None.
 
-        When `default_section' is given, the name of the special section is
+        When `default_section` is given, the name of the special section is
         named accordingly. By default it is called ``"DEFAULT"`` but this can
         be customized to point to any other valid section name. Its current
         value can be retrieved using the ``parser_instance.default_section``
@@ -56,9 +57,9 @@ ConfigParser -- responsible for parsing a list of
 
         When `interpolation` is given, it should be an Interpolation subclass
         instance. It will be used as the handler for option value
-        pre-processing when using getters. RawConfigParser object s don't do
+        pre-processing when using getters. RawConfigParser objects don't do
         any sort of interpolation, whereas ConfigParser uses an instance of
-        BasicInterpolation. The library also provides a ``zc.buildbot``
+        BasicInterpolation. The library also provides a ``zc.buildout``
         inspired ExtendedInterpolation implementation.
 
         When `converters` is given, it should be a dictionary where each key
@@ -80,14 +81,14 @@ ConfigParser -- responsible for parsing a list of
         Return list of configuration options for the named section.
 
     read(filenames, encoding=None)
-        Read and parse the list of named configuration files, given by
+        Read and parse the iterable of named configuration files, given by
         name.  A single filename is also allowed.  Non-existing files
         are ignored.  Return list of successfully read files.
 
     read_file(f, filename=None)
         Read and parse one configuration file, given as a file object.
         The filename defaults to f.name; it is only used in error
-        messages (if f has no `name' attribute, the string `<???>' is used).
+        messages (if f has no `name` attribute, the string `<???>` is used).
 
     read_string(string)
         Read configuration from a given string.
@@ -103,9 +104,9 @@ ConfigParser -- responsible for parsing a list of
         Return a string value for the named option.  All % interpolations are
         expanded in the return values, based on the defaults passed into the
         constructor and the DEFAULT section.  Additional substitutions may be
-        provided using the `vars' argument, which must be a dictionary whose
+        provided using the `vars` argument, which must be a dictionary whose
-        contents override any pre-existing defaults. If `option' is a key in
+        contents override any pre-existing defaults. If `option` is a key in
-        `vars', the value from `vars' is used.
+        `vars`, the value from `vars` is used.
 
     getint(section, options, raw=False, vars=None, fallback=_UNSET)
         Like get(), but convert value to an integer.
@@ -134,28 +135,30 @@ ConfigParser -- responsible for parsing a list of
 
     write(fp, space_around_delimiters=True)
         Write the configuration state in .ini format. If
-        `space_around_delimiters' is True (the default), delimiters
+        `space_around_delimiters` is True (the default), delimiters
         between keys and values are surrounded by spaces.
 """
 
 from collections.abc import MutableMapping
-from collections import OrderedDict as _default_dict, ChainMap as _ChainMap
+from collections import ChainMap as _ChainMap
 import functools
 import io
 import itertools
+import os
 import re
 import sys
 import warnings
 
-__all__ = ["NoSectionError", "DuplicateOptionError", "DuplicateSectionError",
+__all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError",
            "NoOptionError", "InterpolationError", "InterpolationDepthError",
            "InterpolationMissingOptionError", "InterpolationSyntaxError",
            "ParsingError", "MissingSectionHeaderError",
-           "ConfigParser", "SafeConfigParser", "RawConfigParser",
+           "ConfigParser", "RawConfigParser",
            "Interpolation", "BasicInterpolation",  "ExtendedInterpolation",
            "LegacyInterpolation", "SectionProxy", "ConverterMapping",
-           "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH"]
+           "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH")
 
+_default_dict = dict
 DEFAULTSECT = "DEFAULT"
 
 MAX_INTERPOLATION_DEPTH = 10
@@ -295,41 +298,12 @@ class InterpolationDepthError(InterpolationError):
 class ParsingError(Error):
     """Raised when a configuration file does not follow legal syntax."""
 
-    def __init__(self, source=None, filename=None):
+    def __init__(self, source):
-        # Exactly one of `source'/`filename' arguments has to be given.
+        super().__init__(f'Source contains parsing errors: {source!r}')
-        # `filename' kept for compatibility.
-        if filename and source:
-            raise ValueError("Cannot specify both `filename' and `source'. "
-                             "Use `source'.")
-        elif not filename and not source:
-            raise ValueError("Required argument `source' not given.")
-        elif filename:
-            source = filename
-        Error.__init__(self, 'Source contains parsing errors: %r' % source)
         self.source = source
         self.errors = []
         self.args = (source, )
 
-    @property
-    def filename(self):
-        """Deprecated, use `source'."""
-        warnings.warn(
-            "The 'filename' attribute will be removed in future versions.  "
-            "Use 'source' instead.",
-            DeprecationWarning, stacklevel=2
-        )
-        return self.source
-
-    @filename.setter
-    def filename(self, value):
-        """Deprecated, user `source'."""
-        warnings.warn(
-            "The 'filename' attribute will be removed in future versions.  "
-            "Use 'source' instead.",
-            DeprecationWarning, stacklevel=2
-        )
-        self.source = value
-
     def append(self, lineno, line):
         self.errors.append((lineno, line))
         self.message += '\n\t[line %2d]: %s' % (lineno, line)
@@ -350,7 +324,7 @@ class MissingSectionHeaderError(ParsingError):
 
 
 # Used in parser getters to indicate the default behaviour when a specific
-# option is not found it to raise an exception. Created to enable `None' as
+# option is not found it to raise an exception. Created to enable `None` as
 # a valid fallback value.
 _UNSET = object()
 
@@ -384,7 +358,7 @@ class BasicInterpolation(Interpolation):
     would resolve the "%(dir)s" to the value of dir.  All reference
     expansions are done late, on demand. If a user needs to use a bare % in
     a configuration file, she can escape it by writing %%. Other % usage
-    is considered a user error and raises `InterpolationSyntaxError'."""
+    is considered a user error and raises `InterpolationSyntaxError`."""
 
     _KEYCRE = re.compile(r"%\(([^)]+)\)s")
 
@@ -445,7 +419,7 @@ class BasicInterpolation(Interpolation):
 
 class ExtendedInterpolation(Interpolation):
     """Advanced variant of interpolation, supports the syntax used by
-    `zc.buildout'. Enables interpolation between sections."""
+    `zc.buildout`. Enables interpolation between sections."""
 
     _KEYCRE = re.compile(r"\$\{([^}]+)\}")
 
@@ -523,6 +497,15 @@ class LegacyInterpolation(Interpolation):
 
     _KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        warnings.warn(
+            "LegacyInterpolation has been deprecated since Python 3.2 "
+            "and will be removed from the configparser module in Python 3.13. "
+            "Use BasicInterpolation or ExtendedInterpolation instead.",
+            DeprecationWarning, stacklevel=2
+        )
+
     def before_get(self, parser, section, option, value, vars):
         rawval = value
         depth = MAX_INTERPOLATION_DEPTH
@@ -561,7 +544,7 @@ class RawConfigParser(MutableMapping):
     # Regular expressions for parsing section headers and options
     _SECT_TMPL = r"""
         \[                                 # [
-        (?P<header>[^]]+)                  # very permissive!
+        (?P<header>.+)                     # very permissive!
         \]                                 # ]
         """
     _OPT_TMPL = r"""
@@ -609,9 +592,6 @@ class RawConfigParser(MutableMapping):
         self._converters = ConverterMapping(self)
         self._proxies = self._dict()
         self._proxies[default_section] = SectionProxy(self, default_section)
-        if defaults:
-            for key, value in defaults.items():
-                self._defaults[self.optionxform(key)] = value
         self._delimiters = tuple(delimiters)
         if delimiters == ('=', ':'):
             self._optcre = self.OPTCRE_NV if allow_no_value else self.OPTCRE
@@ -634,8 +614,15 @@ class RawConfigParser(MutableMapping):
             self._interpolation = self._DEFAULT_INTERPOLATION
         if self._interpolation is None:
             self._interpolation = Interpolation()
+        if not isinstance(self._interpolation, Interpolation):
+            raise TypeError(
+                f"interpolation= must be None or an instance of Interpolation;"
+                f" got an object of type {type(self._interpolation)}"
+            )
         if converters is not _UNSET:
             self._converters.update(converters)
+        if defaults:
+            self._read_defaults(defaults)
 
     def defaults(self):
         return self._defaults
@@ -676,19 +663,20 @@ class RawConfigParser(MutableMapping):
         return list(opts.keys())
 
     def read(self, filenames, encoding=None):
-        """Read and parse a filename or a list of filenames.
+        """Read and parse a filename or an iterable of filenames.
 
         Files that cannot be opened are silently ignored; this is
-        designed so that you can specify a list of potential
+        designed so that you can specify an iterable of potential
         configuration file locations (e.g. current directory, user's
         home directory, systemwide directory), and all existing
-        configuration files in the list will be read.  A single
+        configuration files in the iterable will be read.  A single
         filename may also be given.
 
         Return list of successfully read files.
         """
-        if isinstance(filenames, str):
+        if isinstance(filenames, (str, bytes, os.PathLike)):
             filenames = [filenames]
+        encoding = io.text_encoding(encoding)
         read_ok = []
         for filename in filenames:
             try:
@@ -696,16 +684,18 @@ class RawConfigParser(MutableMapping):
                     self._read(fp, filename)
             except OSError:
                 continue
+            if isinstance(filename, os.PathLike):
+                filename = os.fspath(filename)
             read_ok.append(filename)
         return read_ok
 
     def read_file(self, f, source=None):
         """Like read() but the argument must be a file-like object.
 
-        The `f' argument must be iterable, returning one line at a time.
+        The `f` argument must be iterable, returning one line at a time.
-        Optional second argument is the `source' specifying the name of the
+        Optional second argument is the `source` specifying the name of the
-        file being read. If not given, it is taken from f.name. If `f' has no
+        file being read. If not given, it is taken from f.name. If `f` has no
-        `name' attribute, `<???>' is used.
+        `name` attribute, `<???>` is used.
         """
         if source is None:
             try:
@@ -729,7 +719,7 @@ class RawConfigParser(MutableMapping):
         All types held in the dictionary are converted to strings during
         reading, including section names, option names and keys.
 
-        Optional second argument is the `source' specifying the name of the
+        Optional second argument is the `source` specifying the name of the
         dictionary being read.
         """
         elements_added = set()
@@ -750,27 +740,18 @@ class RawConfigParser(MutableMapping):
                 elements_added.add((section, key))
                 self.set(section, key, value)
 
-    def readfp(self, fp, filename=None):
-        """Deprecated, use read_file instead."""
-        warnings.warn(
-            "This method will be removed in future versions.  "
-            "Use 'parser.read_file()' instead.",
-            DeprecationWarning, stacklevel=2
-        )
-        self.read_file(fp, source=filename)
-
     def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET):
         """Get an option value for a given section.
 
-        If `vars' is provided, it must be a dictionary. The option is looked up
+        If `vars` is provided, it must be a dictionary. The option is looked up
-        in `vars' (if provided), `section', and in `DEFAULTSECT' in that order.
+        in `vars` (if provided), `section`, and in `DEFAULTSECT` in that order.
-        If the key is not found and `fallback' is provided, it is used as
+        If the key is not found and `fallback` is provided, it is used as
-        a fallback value. `None' can be provided as a `fallback' value.
+        a fallback value. `None` can be provided as a `fallback` value.
 
-        If interpolation is enabled and the optional argument `raw' is False,
+        If interpolation is enabled and the optional argument `raw` is False,
         all interpolations are expanded in the return values.
 
-        Arguments `raw', `vars', and `fallback' are keyword only.
+        Arguments `raw`, `vars`, and `fallback` are keyword only.
 
         The section DEFAULT is special.
         """
@@ -830,8 +811,8 @@ class RawConfigParser(MutableMapping):
 
         All % interpolations are expanded in the return values, based on the
         defaults passed into the constructor, unless the optional argument
-        `raw' is true.  Additional substitutions may be provided using the
+        `raw` is true.  Additional substitutions may be provided using the
-        `vars' argument, which must be a dictionary whose contents overrides
+        `vars` argument, which must be a dictionary whose contents overrides
         any pre-existing defaults.
 
         The section DEFAULT is special.
@@ -844,6 +825,7 @@ class RawConfigParser(MutableMapping):
         except KeyError:
             if section != self.default_section:
                 raise NoSectionError(section)
+        orig_keys = list(d.keys())
         # Update with the entry specific variables
         if vars:
             for key, value in vars.items():
@@ -852,7 +834,7 @@ class RawConfigParser(MutableMapping):
             section, option, d[option], d)
         if raw:
             value_getter = lambda option: d[option]
-        return [(option, value_getter(option)) for option in d.keys()]
+        return [(option, value_getter(option)) for option in orig_keys]
 
     def popitem(self):
         """Remove a section from the parser and return it as
@@ -872,8 +854,8 @@ class RawConfigParser(MutableMapping):
 
     def has_option(self, section, option):
         """Check for the existence of a given option in a given section.
-        If the specified `section' is None or an empty string, DEFAULT is
+        If the specified `section` is None or an empty string, DEFAULT is
-        assumed. If the specified `section' does not exist, returns False."""
+        assumed. If the specified `section` does not exist, returns False."""
         if not section or section == self.default_section:
             option = self.optionxform(option)
             return option in self._defaults
@@ -901,8 +883,11 @@ class RawConfigParser(MutableMapping):
     def write(self, fp, space_around_delimiters=True):
         """Write an .ini-format representation of the configuration state.
 
-        If `space_around_delimiters' is True (the default), delimiters
+        If `space_around_delimiters` is True (the default), delimiters
         between keys and values are surrounded by spaces.
+
+        Please note that comments in the original configuration file are not
+        preserved when writing the configuration back.
         """
         if space_around_delimiters:
             d = " {} ".format(self._delimiters[0])
@@ -916,7 +901,7 @@ class RawConfigParser(MutableMapping):
                                 self._sections[section].items(), d)
 
     def _write_section(self, fp, section_name, section_items, delimiter):
-        """Write a single section to the specified `fp'."""
+        """Write a single section to the specified `fp`."""
         fp.write("[{}]\n".format(section_name))
         for key, value in section_items:
             value = self._interpolation.before_write(self, section_name, key,
@@ -959,7 +944,8 @@ class RawConfigParser(MutableMapping):
     def __setitem__(self, key, value):
         # To conform with the mapping protocol, overwrites existing values in
         # the section.
-
+        if key in self and self[key] is value:
+            return
         # XXX this is not atomic if read_dict fails at any point. Then again,
         # no update method in configparser is atomic in this implementation.
         if key == self.default_section:
@@ -989,8 +975,8 @@ class RawConfigParser(MutableMapping):
         """Parse a sectioned configuration file.
 
         Each section in a configuration file contains a header, indicated by
-        a name in square brackets (`[]'), plus key/value options, indicated by
+        a name in square brackets (`[]`), plus key/value options, indicated by
-        `name' and `value' delimited with a specific substring (`=' or `:' by
+        `name` and `value` delimited with a specific substring (`=` or `:` by
         default).
 
         Values can span multiple lines, as long as they are indented deeper
@@ -998,9 +984,9 @@ class RawConfigParser(MutableMapping):
         lines may be treated as parts of multiline values or ignored.
 
         Configuration files may include comments, prefixed by specific
-        characters (`#' and `;' by default). Comments may appear on their own
+        characters (`#` and `;` by default). Comments may appear on their own
         in an otherwise empty line or may be entered in lines holding values or
-        section names.
+        section names. Please note that comments get stripped off when reading configuration files.
         """
         elements_added = set()
         cursect = None                        # None, or a dictionary
@@ -1119,6 +1105,12 @@ class RawConfigParser(MutableMapping):
                                                                 section,
                                                                 name, val)
 
+    def _read_defaults(self, defaults):
+        """Read the defaults passed in the initializer.
+        Note: values can be non-string."""
+        for key, value in defaults.items():
+            self._defaults[self.optionxform(key)] = value
+
     def _handle_error(self, exc, fpname, lineno, line):
         if not exc:
             exc = ParsingError(fpname)
@@ -1135,7 +1127,7 @@ class RawConfigParser(MutableMapping):
             sectiondict = self._sections[section]
         except KeyError:
             if section != self.default_section:
-                raise NoSectionError(section)
+                raise NoSectionError(section) from None
         # Update with the entry specific variables
         vardict = {}
         if vars:
@@ -1196,18 +1188,18 @@ class ConfigParser(RawConfigParser):
         self._validate_value_types(section=section)
         super().add_section(section)
 
+    def _read_defaults(self, defaults):
+        """Reads the defaults passed in the initializer, implicitly converting
+        values to strings like the rest of the API.
 
-class SafeConfigParser(ConfigParser):
+        Does not perform interpolation for backwards compatibility.
-    """ConfigParser alias for backwards compatibility purposes."""
+        """
-
+        try:
-    def __init__(self, *args, **kwargs):
+            hold_interpolation = self._interpolation
-        super().__init__(*args, **kwargs)
+            self._interpolation = Interpolation()
-        warnings.warn(
+            self.read_dict({self.default_section: defaults})
-            "The SafeConfigParser class has been renamed to ConfigParser "
+        finally:
-            "in Python 3.2. This alias will be removed in future versions."
+            self._interpolation = hold_interpolation
-            " Use ConfigParser directly instead.",
-            DeprecationWarning, stacklevel=2
-        )
 
 
 class SectionProxy(MutableMapping):

Lib/contextlib.py

@@ -1,20 +1,25 @@
 """Utilities for with-statement contexts.  See PEP 343."""
 import abc
+import os
 import sys
 import _collections_abc
 from collections import deque
 from functools import wraps
+from types import MethodType, GenericAlias
 
 __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext",
            "AbstractContextManager", "AbstractAsyncContextManager",
            "AsyncExitStack", "ContextDecorator", "ExitStack",
-           "redirect_stdout", "redirect_stderr", "suppress"]
+           "redirect_stdout", "redirect_stderr", "suppress", "aclosing",
+           "chdir"]
 
 
 class AbstractContextManager(abc.ABC):
 
     """An abstract base class for context managers."""
 
+    __class_getitem__ = classmethod(GenericAlias)
+
     def __enter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -35,6 +40,8 @@ class AbstractAsyncContextManager(abc.ABC):
 
     """An abstract base class for asynchronous context managers."""
 
+    __class_getitem__ = classmethod(GenericAlias)
+
     async def __aenter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -75,6 +82,22 @@ class ContextDecorator(object):
         return inner
 
 
+class AsyncContextDecorator(object):
+    "A base class or mixin that enables async context managers to work as decorators."
+
+    def _recreate_cm(self):
+        """Return a recreated instance of self.
+        """
+        return self
+
+    def __call__(self, func):
+        @wraps(func)
+        async def inner(*args, **kwds):
+            async with self._recreate_cm():
+                return await func(*args, **kwds)
+        return inner
+
+
 class _GeneratorContextManagerBase:
     """Shared functionality for @contextmanager and @asynccontextmanager."""
 
@@ -92,18 +115,20 @@ class _GeneratorContextManagerBase:
         # for the class instead.
         # See http://bugs.python.org/issue19404 for more details.
 
-
-class _GeneratorContextManager(_GeneratorContextManagerBase,
-                               AbstractContextManager,
-                               ContextDecorator):
-    """Helper for @contextmanager decorator."""
-
     def _recreate_cm(self):
-        # _GCM instances are one-shot context managers, so the
+        # _GCMB instances are one-shot context managers, so the
         # CM must be recreated each time a decorated function is
         # called
         return self.__class__(self.func, self.args, self.kwds)
 
+
+class _GeneratorContextManager(
+    _GeneratorContextManagerBase,
+    AbstractContextManager,
+    ContextDecorator,
+):
+    """Helper for @contextmanager decorator."""
+
     def __enter__(self):
         # do not keep args and kwds alive unnecessarily
         # they are only needed for recreation, which is not possible anymore
@@ -113,8 +138,8 @@ class _GeneratorContextManager(_GeneratorContextManagerBase,
         except StopIteration:
             raise RuntimeError("generator didn't yield") from None
 
-    def __exit__(self, type, value, traceback):
+    def __exit__(self, typ, value, traceback):
-        if type is None:
+        if typ is None:
             try:
                 next(self.gen)
             except StopIteration:
@@ -125,9 +150,9 @@ class _GeneratorContextManager(_GeneratorContextManagerBase,
             if value is None:
                 # Need to force instantiation so we can reliably
                 # tell if we get the same exception back
-                value = type()
+                value = typ()
             try:
-                self.gen.throw(type, value, traceback)
+                self.gen.throw(typ, value, traceback)
             except StopIteration as exc:
                 # Suppress StopIteration *unless* it's the same exception that
                 # was passed to throw().  This prevents a StopIteration
@@ -136,75 +161,100 @@ class _GeneratorContextManager(_GeneratorContextManagerBase,
             except RuntimeError as exc:
                 # Don't re-raise the passed in exception. (issue27122)
                 if exc is value:
+                    exc.__traceback__ = traceback
                     return False
-                # Likewise, avoid suppressing if a StopIteration exception
+                # Avoid suppressing if a StopIteration exception
                 # was passed to throw() and later wrapped into a RuntimeError
-                # (see PEP 479).
+                # (see PEP 479 for sync generators; async generators also
-                if type is StopIteration and exc.__cause__ is value:
+                # have this behavior). But do this only if the exception wrapped
+                # by the RuntimeError is actually Stop(Async)Iteration (see
+                # issue29692).
+                if (
+                    isinstance(value, StopIteration)
+                    and exc.__cause__ is value
+                ):
+                    value.__traceback__ = traceback
                     return False
                 raise
-            except:
+            except BaseException as exc:
                 # only re-raise if it's *not* the exception that was
                 # passed to throw(), because __exit__() must not raise
                 # an exception unless __exit__() itself failed.  But throw()
                 # has to raise the exception to signal propagation, so this
                 # fixes the impedance mismatch between the throw() protocol
                 # and the __exit__() protocol.
-                #
+                if exc is not value:
-                # This cannot use 'except BaseException as exc' (as in the
+                    raise
-                # async implementation) to maintain compatibility with
+                exc.__traceback__ = traceback
-                # Python 2, where old-style class exceptions are not caught
+                return False
-                # by 'except BaseException'.
-                if sys.exc_info()[1] is value:
-                    return False
-                raise
             raise RuntimeError("generator didn't stop after throw()")
 
-
+class _AsyncGeneratorContextManager(
-class _AsyncGeneratorContextManager(_GeneratorContextManagerBase,
+    _GeneratorContextManagerBase,
-                                    AbstractAsyncContextManager):
+    AbstractAsyncContextManager,
-    """Helper for @asynccontextmanager."""
+    AsyncContextDecorator,
+):
+    """Helper for @asynccontextmanager decorator."""
 
     async def __aenter__(self):
+        # do not keep args and kwds alive unnecessarily
+        # they are only needed for recreation, which is not possible anymore
+        del self.args, self.kwds, self.func
         try:
-            return await self.gen.__anext__()
+            return await anext(self.gen)
         except StopAsyncIteration:
             raise RuntimeError("generator didn't yield") from None
 
     async def __aexit__(self, typ, value, traceback):
         if typ is None:
             try:
-                await self.gen.__anext__()
+                await anext(self.gen)
             except StopAsyncIteration:
-                return
+                return False
             else:
                 raise RuntimeError("generator didn't stop")
         else:
             if value is None:
+                # Need to force instantiation so we can reliably
+                # tell if we get the same exception back
                 value = typ()
-            # See _GeneratorContextManager.__exit__ for comments on subtleties
-            # in this implementation
             try:
                 await self.gen.athrow(typ, value, traceback)
-                raise RuntimeError("generator didn't stop after throw()")
             except StopAsyncIteration as exc:
+                # Suppress StopIteration *unless* it's the same exception that
+                # was passed to throw().  This prevents a StopIteration
+                # raised inside the "with" statement from being suppressed.
                 return exc is not value
             except RuntimeError as exc:
+                # Don't re-raise the passed in exception. (issue27122)
                 if exc is value:
+                    exc.__traceback__ = traceback
                     return False
-                # Avoid suppressing if a StopIteration exception
+                # Avoid suppressing if a Stop(Async)Iteration exception
-                # was passed to throw() and later wrapped into a RuntimeError
+                # was passed to athrow() and later wrapped into a RuntimeError
                 # (see PEP 479 for sync generators; async generators also
                 # have this behavior). But do this only if the exception wrapped
-                # by the RuntimeError is actully Stop(Async)Iteration (see
+                # by the RuntimeError is actually Stop(Async)Iteration (see
                 # issue29692).
-                if isinstance(value, (StopIteration, StopAsyncIteration)):
+                if (
-                    if exc.__cause__ is value:
+                    isinstance(value, (StopIteration, StopAsyncIteration))
-                        return False
+                    and exc.__cause__ is value
+                ):
+                    value.__traceback__ = traceback
+                    return False
                 raise
             except BaseException as exc:
+                # only re-raise if it's *not* the exception that was
+                # passed to throw(), because __exit__() must not raise
+                # an exception unless __exit__() itself failed.  But throw()
+                # has to raise the exception to signal propagation, so this
+                # fixes the impedance mismatch between the throw() protocol
+                # and the __exit__() protocol.
                 if exc is not value:
                     raise
+                exc.__traceback__ = traceback
+                return False
+            raise RuntimeError("generator didn't stop after athrow()")
 
 
 def contextmanager(func):
@@ -298,6 +348,32 @@ class closing(AbstractContextManager):
         self.thing.close()
 
 
+class aclosing(AbstractAsyncContextManager):
+    """Async context manager for safely finalizing an asynchronously cleaned-up
+    resource such as an async generator, calling its ``aclose()`` method.
+
+    Code like this:
+
+        async with aclosing(<module>.fetch(<arguments>)) as agen:
+            <block>
+
+    is equivalent to this:
+
+        agen = <module>.fetch(<arguments>)
+        try:
+            <block>
+        finally:
+            await agen.aclose()
+
+    """
+    def __init__(self, thing):
+        self.thing = thing
+    async def __aenter__(self):
+        return self.thing
+    async def __aexit__(self, *exc_info):
+        await self.thing.aclose()
+
+
 class _RedirectStream(AbstractContextManager):
 
     _stream = None
@@ -373,12 +449,10 @@ class _BaseExitStack:
 
     @staticmethod
     def _create_exit_wrapper(cm, cm_exit):
-        def _exit_wrapper(exc_type, exc, tb):
+        return MethodType(cm_exit, cm)
-            return cm_exit(cm, exc_type, exc, tb)
-        return _exit_wrapper
 
     @staticmethod
-    def _create_cb_wrapper(callback, *args, **kwds):
+    def _create_cb_wrapper(callback, /, *args, **kwds):
         def _exit_wrapper(exc_type, exc, tb):
             callback(*args, **kwds)
         return _exit_wrapper
@@ -421,13 +495,18 @@ class _BaseExitStack:
         """
         # We look up the special methods on the type to match the with
         # statement.
-        _cm_type = type(cm)
+        cls = type(cm)
-        _exit = _cm_type.__exit__
+        try:
-        result = _cm_type.__enter__(cm)
+            _enter = cls.__enter__
+            _exit = cls.__exit__
+        except AttributeError:
+            raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does "
+                            f"not support the context manager protocol") from None
+        result = _enter(cm)
         self._push_cm_exit(cm, _exit)
         return result
 
-    def callback(self, callback, *args, **kwds):
+    def callback(self, callback, /, *args, **kwds):
         """Registers an arbitrary callback and arguments.
 
         Cannot suppress exceptions.
@@ -443,7 +522,6 @@ class _BaseExitStack:
     def _push_cm_exit(self, cm, cm_exit):
         """Helper to correctly register callbacks to __exit__ methods."""
         _exit_wrapper = self._create_exit_wrapper(cm, cm_exit)
-        _exit_wrapper.__self__ = cm
         self._push_exit_callback(_exit_wrapper, True)
 
     def _push_exit_callback(self, callback, is_sync=True):
@@ -475,10 +553,10 @@ class ExitStack(_BaseExitStack, AbstractContextManager):
             # Context may not be correct, so find the end of the chain
             while 1:
                 exc_context = new_exc.__context__
-                if exc_context is old_exc:
+                if exc_context is None or exc_context is old_exc:
                     # Context is already set correctly (see issue 20317)
                     return
-                if exc_context is None or exc_context is frame_exc:
+                if exc_context is frame_exc:
                     break
                 new_exc = exc_context
             # Change the end of the chain to point to the exception
@@ -535,12 +613,10 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
 
     @staticmethod
     def _create_async_exit_wrapper(cm, cm_exit):
-        async def _exit_wrapper(exc_type, exc, tb):
+        return MethodType(cm_exit, cm)
-            return await cm_exit(cm, exc_type, exc, tb)
-        return _exit_wrapper
 
     @staticmethod
-    def _create_async_cb_wrapper(callback, *args, **kwds):
+    def _create_async_cb_wrapper(callback, /, *args, **kwds):
         async def _exit_wrapper(exc_type, exc, tb):
             await callback(*args, **kwds)
         return _exit_wrapper
@@ -551,9 +627,15 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         If successful, also pushes its __aexit__ method as a callback and
         returns the result of the __aenter__ method.
         """
-        _cm_type = type(cm)
+        cls = type(cm)
-        _exit = _cm_type.__aexit__
+        try:
-        result = await _cm_type.__aenter__(cm)
+            _enter = cls.__aenter__
+            _exit = cls.__aexit__
+        except AttributeError:
+            raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does "
+                            f"not support the asynchronous context manager protocol"
+                           ) from None
+        result = await _enter(cm)
         self._push_async_cm_exit(cm, _exit)
         return result
 
@@ -575,7 +657,7 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
             self._push_async_cm_exit(exit, exit_method)
         return exit  # Allow use as a decorator
 
-    def push_async_callback(self, callback, *args, **kwds):
+    def push_async_callback(self, callback, /, *args, **kwds):
         """Registers an arbitrary coroutine function and arguments.
 
         Cannot suppress exceptions.
@@ -596,7 +678,6 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         """Helper to correctly register coroutine function to __aexit__
         method."""
         _exit_wrapper = self._create_async_exit_wrapper(cm, cm_exit)
-        _exit_wrapper.__self__ = cm
         self._push_exit_callback(_exit_wrapper, False)
 
     async def __aenter__(self):
@@ -612,10 +693,10 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
             # Context may not be correct, so find the end of the chain
             while 1:
                 exc_context = new_exc.__context__
-                if exc_context is old_exc:
+                if exc_context is None or exc_context is old_exc:
                     # Context is already set correctly (see issue 20317)
                     return
-                if exc_context is None or exc_context is frame_exc:
+                if exc_context is frame_exc:
                     break
                 new_exc = exc_context
             # Change the end of the chain to point to the exception
@@ -656,7 +737,7 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         return received_exc and suppressed_exc
 
 
-class nullcontext(AbstractContextManager):
+class nullcontext(AbstractContextManager, AbstractAsyncContextManager):
     """Context manager that does no additional processing.
 
     Used as a stand-in for a normal context manager, when a particular
@@ -675,3 +756,24 @@ class nullcontext(AbstractContextManager):
 
     def __exit__(self, *excinfo):
         pass
+
+    async def __aenter__(self):
+        return self.enter_result
+
+    async def __aexit__(self, *excinfo):
+        pass
+
+
+class chdir(AbstractContextManager):
+    """Non thread-safe context manager to change the current working directory."""
+
+    def __init__(self, path):
+        self.path = path
+        self._old_cwd = []
+
+    def __enter__(self):
+        self._old_cwd.append(os.getcwd())
+        os.chdir(self.path)
+
+    def __exit__(self, *excinfo):
+        os.chdir(self._old_cwd.pop())

Lib/copy.py

@@ -39,8 +39,8 @@ Python's deep copy operation avoids these problems by:
     set of components copied
 
 This version does not copy types like module, class, function, method,
-nor stack trace, stack frame, nor file, socket, window, nor array, nor
+nor stack trace, stack frame, nor file, socket, window, nor any
-any similar types.
+similar types.
 
 Classes can use the same interfaces to control copying that they use
 to control pickling: they can define methods called __getinitargs__(),
@@ -56,11 +56,6 @@ class Error(Exception):
     pass
 error = Error   # backward compatibility
 
-try:
-    from org.python.core import PyStringMap
-except ImportError:
-    PyStringMap = None
-
 __all__ = ["Error", "copy", "deepcopy"]
 
 def copy(x):
@@ -106,13 +101,11 @@ _copy_dispatch = d = {}
 
 def _copy_immutable(x):
     return x
-for t in (type(None), int, float, bool, complex, str, tuple,
+for t in (types.NoneType, int, float, bool, complex, str, tuple,
           bytes, frozenset, type, range, slice, property,
-          types.BuiltinFunctionType, type(Ellipsis), type(NotImplemented),
+          types.BuiltinFunctionType, types.EllipsisType,
-          types.FunctionType, weakref.ref):
+          types.NotImplementedType, types.FunctionType, types.CodeType,
-    d[t] = _copy_immutable
+          weakref.ref):
-t = getattr(types, "CodeType", None)
-if t is not None:
     d[t] = _copy_immutable
 
 d[list] = list.copy
@@ -120,9 +113,6 @@ d[dict] = dict.copy
 d[set] = set.copy
 d[bytearray] = bytearray.copy
 
-if PyStringMap is not None:
-    d[PyStringMap] = PyStringMap.copy
-
 del d, t
 
 def deepcopy(x, memo=None, _nil=[]):
@@ -181,9 +171,9 @@ _deepcopy_dispatch = d = {}
 
 def _deepcopy_atomic(x, memo):
     return x
-d[type(None)] = _deepcopy_atomic
+d[types.NoneType] = _deepcopy_atomic
-d[type(Ellipsis)] = _deepcopy_atomic
+d[types.EllipsisType] = _deepcopy_atomic
-d[type(NotImplemented)] = _deepcopy_atomic
+d[types.NotImplementedType] = _deepcopy_atomic
 d[int] = _deepcopy_atomic
 d[float] = _deepcopy_atomic
 d[bool] = _deepcopy_atomic
@@ -192,6 +182,7 @@ d[bytes] = _deepcopy_atomic
 d[str] = _deepcopy_atomic
 d[types.CodeType] = _deepcopy_atomic
 d[type] = _deepcopy_atomic
+d[range] = _deepcopy_atomic
 d[types.BuiltinFunctionType] = _deepcopy_atomic
 d[types.FunctionType] = _deepcopy_atomic
 d[weakref.ref] = _deepcopy_atomic
@@ -230,8 +221,6 @@ def _deepcopy_dict(x, memo, deepcopy=deepcopy):
         y[deepcopy(key, memo)] = deepcopy(value, memo)
     return y
 d[dict] = _deepcopy_dict
-if PyStringMap is not None:
-    d[PyStringMap] = _deepcopy_dict
 
 def _deepcopy_method(x, memo): # Copy instance methods
     return type(x)(x.__func__, deepcopy(x.__self__, memo))
@@ -257,7 +246,7 @@ def _keep_alive(x, memo):
 
 def _reconstruct(x, memo, func, args,
                  state=None, listiter=None, dictiter=None,
-                 deepcopy=deepcopy):
+                 *, deepcopy=deepcopy):
     deep = memo is not None
     if deep and args:
         args = (deepcopy(arg, memo) for arg in args)
@@ -300,4 +289,4 @@ def _reconstruct(x, memo, func, args,
                 y[key] = value
     return y
 
-del types, weakref, PyStringMap
+del types, weakref

Lib/csv.py

@@ -4,17 +4,22 @@ csv.py - read/write/investigate CSV files
 """
 
 import re
-from _csv import Error, writer, reader, \
+import types
+from _csv import Error, __version__, writer, reader, register_dialect, \
+                 unregister_dialect, get_dialect, list_dialects, \
+                 field_size_limit, \
                  QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \
+                 QUOTE_STRINGS, QUOTE_NOTNULL, \
                  __doc__
+from _csv import Dialect as _Dialect
 
-from collections import OrderedDict
 from io import StringIO
 
 __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE",
+           "QUOTE_STRINGS", "QUOTE_NOTNULL",
            "Error", "Dialect", "__doc__", "excel", "excel_tab",
            "field_size_limit", "reader", "writer",
-           "Sniffer",
+           "register_dialect", "get_dialect", "list_dialects", "Sniffer",
            "unregister_dialect", "__version__", "DictReader", "DictWriter",
            "unix_dialect"]
 
@@ -57,10 +62,12 @@ class excel(Dialect):
     skipinitialspace = False
     lineterminator = '\r\n'
     quoting = QUOTE_MINIMAL
+register_dialect("excel", excel)
 
 class excel_tab(excel):
     """Describe the usual properties of Excel-generated TAB-delimited files."""
     delimiter = '\t'
+register_dialect("excel-tab", excel_tab)
 
 class unix_dialect(Dialect):
     """Describe the usual properties of Unix-generated CSV files."""
@@ -70,11 +77,14 @@ class unix_dialect(Dialect):
     skipinitialspace = False
     lineterminator = '\n'
     quoting = QUOTE_ALL
+register_dialect("unix", unix_dialect)
 
 
 class DictReader:
     def __init__(self, f, fieldnames=None, restkey=None, restval=None,
                  dialect="excel", *args, **kwds):
+        if fieldnames is not None and iter(fieldnames) is fieldnames:
+            fieldnames = list(fieldnames)
         self._fieldnames = fieldnames   # list of keys for the dict
         self.restkey = restkey          # key to catch long rows
         self.restval = restval          # default value for short rows
@@ -111,7 +121,7 @@ class DictReader:
         # values
         while row == []:
             row = next(self.reader)
-        d = OrderedDict(zip(self.fieldnames, row))
+        d = dict(zip(self.fieldnames, row))
         lf = len(self.fieldnames)
         lr = len(row)
         if lf < lr:
@@ -121,13 +131,18 @@ class DictReader:
                 d[key] = self.restval
         return d
 
+    __class_getitem__ = classmethod(types.GenericAlias)
+
 
 class DictWriter:
     def __init__(self, f, fieldnames, restval="", extrasaction="raise",
                  dialect="excel", *args, **kwds):
+        if fieldnames is not None and iter(fieldnames) is fieldnames:
+            fieldnames = list(fieldnames)
         self.fieldnames = fieldnames    # list of keys for the dict
         self.restval = restval          # for writing short dicts
-        if extrasaction.lower() not in ("raise", "ignore"):
+        extrasaction = extrasaction.lower()
+        if extrasaction not in ("raise", "ignore"):
             raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'"
                              % extrasaction)
         self.extrasaction = extrasaction
@@ -135,7 +150,7 @@ class DictWriter:
 
     def writeheader(self):
         header = dict(zip(self.fieldnames, self.fieldnames))
-        self.writerow(header)
+        return self.writerow(header)
 
     def _dict_to_list(self, rowdict):
         if self.extrasaction == "raise":
@@ -151,11 +166,8 @@ class DictWriter:
     def writerows(self, rowdicts):
         return self.writer.writerows(map(self._dict_to_list, rowdicts))
 
-# Guard Sniffer's type checking against builds that exclude complex()
+    __class_getitem__ = classmethod(types.GenericAlias)
-try:
+
-    complex
-except NameError:
-    complex = float
 
 class Sniffer:
     '''
@@ -404,14 +416,10 @@ class Sniffer:
                 continue # skip rows that have irregular number of columns
 
             for col in list(columnTypes.keys()):
-
+                thisType = complex
-                for thisType in [int, float, complex]:
+                try:
-                    try:
+                    thisType(row[col])
-                        thisType(row[col])
+                except (ValueError, OverflowError):
-                        break
-                    except (ValueError, OverflowError):
-                        pass
-                else:
                     # fallback to length of string
                     thisType = len(row[col])
 
@@ -427,7 +435,7 @@ class Sniffer:
         # on whether it's a header
         hasHeader = 0
         for col, colType in columnTypes.items():
-            if type(colType) == type(0): # it's a length
+            if isinstance(colType, int): # it's a length
                 if len(header[col]) != colType:
                     hasHeader += 1
                 else:

Lib/ctypes/__init__.py

@@ -0,0 +1,577 @@
+"""create and manipulate C data types in Python"""
+
+import os as _os, sys as _sys
+import types as _types
+
+__version__ = "1.1.0"
+
+from _ctypes import Union, Structure, Array
+from _ctypes import _Pointer
+from _ctypes import CFuncPtr as _CFuncPtr
+from _ctypes import __version__ as _ctypes_version
+from _ctypes import RTLD_LOCAL, RTLD_GLOBAL
+from _ctypes import ArgumentError
+from _ctypes import SIZEOF_TIME_T
+
+from struct import calcsize as _calcsize
+
+if __version__ != _ctypes_version:
+    raise Exception("Version number mismatch", __version__, _ctypes_version)
+
+if _os.name == "nt":
+    from _ctypes import FormatError
+
+DEFAULT_MODE = RTLD_LOCAL
+if _os.name == "posix" and _sys.platform == "darwin":
+    # On OS X 10.3, we use RTLD_GLOBAL as default mode
+    # because RTLD_LOCAL does not work at least on some
+    # libraries.  OS X 10.3 is Darwin 7, so we check for
+    # that.
+
+    if int(_os.uname().release.split('.')[0]) < 8:
+        DEFAULT_MODE = RTLD_GLOBAL
+
+from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \
+     FUNCFLAG_PYTHONAPI as _FUNCFLAG_PYTHONAPI, \
+     FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \
+     FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR
+
+# WINOLEAPI -> HRESULT
+# WINOLEAPI_(type)
+#
+# STDMETHODCALLTYPE
+#
+# STDMETHOD(name)
+# STDMETHOD_(type, name)
+#
+# STDAPICALLTYPE
+
+def create_string_buffer(init, size=None):
+    """create_string_buffer(aBytes) -> character array
+    create_string_buffer(anInteger) -> character array
+    create_string_buffer(aBytes, anInteger) -> character array
+    """
+    if isinstance(init, bytes):
+        if size is None:
+            size = len(init)+1
+        _sys.audit("ctypes.create_string_buffer", init, size)
+        buftype = c_char * size
+        buf = buftype()
+        buf.value = init
+        return buf
+    elif isinstance(init, int):
+        _sys.audit("ctypes.create_string_buffer", None, init)
+        buftype = c_char * init
+        buf = buftype()
+        return buf
+    raise TypeError(init)
+
+# Alias to create_string_buffer() for backward compatibility
+c_buffer = create_string_buffer
+
+_c_functype_cache = {}
+def CFUNCTYPE(restype, *argtypes, **kw):
+    """CFUNCTYPE(restype, *argtypes,
+                 use_errno=False, use_last_error=False) -> function prototype.
+
+    restype: the result type
+    argtypes: a sequence specifying the argument types
+
+    The function prototype can be called in different ways to create a
+    callable object:
+
+    prototype(integer address) -> foreign function
+    prototype(callable) -> create and return a C callable function from callable
+    prototype(integer index, method name[, paramflags]) -> foreign function calling a COM method
+    prototype((ordinal number, dll object)[, paramflags]) -> foreign function exported by ordinal
+    prototype((function name, dll object)[, paramflags]) -> foreign function exported by name
+    """
+    flags = _FUNCFLAG_CDECL
+    if kw.pop("use_errno", False):
+        flags |= _FUNCFLAG_USE_ERRNO
+    if kw.pop("use_last_error", False):
+        flags |= _FUNCFLAG_USE_LASTERROR
+    if kw:
+        raise ValueError("unexpected keyword argument(s) %s" % kw.keys())
+
+    try:
+        return _c_functype_cache[(restype, argtypes, flags)]
+    except KeyError:
+        pass
+
+    class CFunctionType(_CFuncPtr):
+        _argtypes_ = argtypes
+        _restype_ = restype
+        _flags_ = flags
+    _c_functype_cache[(restype, argtypes, flags)] = CFunctionType
+    return CFunctionType
+
+if _os.name == "nt":
+    from _ctypes import LoadLibrary as _dlopen
+    from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL
+
+    _win_functype_cache = {}
+    def WINFUNCTYPE(restype, *argtypes, **kw):
+        # docstring set later (very similar to CFUNCTYPE.__doc__)
+        flags = _FUNCFLAG_STDCALL
+        if kw.pop("use_errno", False):
+            flags |= _FUNCFLAG_USE_ERRNO
+        if kw.pop("use_last_error", False):
+            flags |= _FUNCFLAG_USE_LASTERROR
+        if kw:
+            raise ValueError("unexpected keyword argument(s) %s" % kw.keys())
+
+        try:
+            return _win_functype_cache[(restype, argtypes, flags)]
+        except KeyError:
+            pass
+
+        class WinFunctionType(_CFuncPtr):
+            _argtypes_ = argtypes
+            _restype_ = restype
+            _flags_ = flags
+        _win_functype_cache[(restype, argtypes, flags)] = WinFunctionType
+        return WinFunctionType
+    if WINFUNCTYPE.__doc__:
+        WINFUNCTYPE.__doc__ = CFUNCTYPE.__doc__.replace("CFUNCTYPE", "WINFUNCTYPE")
+
+elif _os.name == "posix":
+    from _ctypes import dlopen as _dlopen
+
+from _ctypes import sizeof, byref, addressof, alignment, resize
+from _ctypes import get_errno, set_errno
+from _ctypes import _SimpleCData
+
+def _check_size(typ, typecode=None):
+    # Check if sizeof(ctypes_type) against struct.calcsize.  This
+    # should protect somewhat against a misconfigured libffi.
+    from struct import calcsize
+    if typecode is None:
+        # Most _type_ codes are the same as used in struct
+        typecode = typ._type_
+    actual, required = sizeof(typ), calcsize(typecode)
+    if actual != required:
+        raise SystemError("sizeof(%s) wrong: %d instead of %d" % \
+                          (typ, actual, required))
+
+class py_object(_SimpleCData):
+    _type_ = "O"
+    def __repr__(self):
+        try:
+            return super().__repr__()
+        except ValueError:
+            return "%s(<NULL>)" % type(self).__name__
+_check_size(py_object, "P")
+
+class c_short(_SimpleCData):
+    _type_ = "h"
+_check_size(c_short)
+
+class c_ushort(_SimpleCData):
+    _type_ = "H"
+_check_size(c_ushort)
+
+class c_long(_SimpleCData):
+    _type_ = "l"
+_check_size(c_long)
+
+class c_ulong(_SimpleCData):
+    _type_ = "L"
+_check_size(c_ulong)
+
+if _calcsize("i") == _calcsize("l"):
+    # if int and long have the same size, make c_int an alias for c_long
+    c_int = c_long
+    c_uint = c_ulong
+else:
+    class c_int(_SimpleCData):
+        _type_ = "i"
+    _check_size(c_int)
+
+    class c_uint(_SimpleCData):
+        _type_ = "I"
+    _check_size(c_uint)
+
+class c_float(_SimpleCData):
+    _type_ = "f"
+_check_size(c_float)
+
+class c_double(_SimpleCData):
+    _type_ = "d"
+_check_size(c_double)
+
+class c_longdouble(_SimpleCData):
+    _type_ = "g"
+if sizeof(c_longdouble) == sizeof(c_double):
+    c_longdouble = c_double
+
+if _calcsize("l") == _calcsize("q"):
+    # if long and long long have the same size, make c_longlong an alias for c_long
+    c_longlong = c_long
+    c_ulonglong = c_ulong
+else:
+    class c_longlong(_SimpleCData):
+        _type_ = "q"
+    _check_size(c_longlong)
+
+    class c_ulonglong(_SimpleCData):
+        _type_ = "Q"
+    ##    def from_param(cls, val):
+    ##        return ('d', float(val), val)
+    ##    from_param = classmethod(from_param)
+    _check_size(c_ulonglong)
+
+class c_ubyte(_SimpleCData):
+    _type_ = "B"
+c_ubyte.__ctype_le__ = c_ubyte.__ctype_be__ = c_ubyte
+# backward compatibility:
+##c_uchar = c_ubyte
+_check_size(c_ubyte)
+
+class c_byte(_SimpleCData):
+    _type_ = "b"
+c_byte.__ctype_le__ = c_byte.__ctype_be__ = c_byte
+_check_size(c_byte)
+
+class c_char(_SimpleCData):
+    _type_ = "c"
+c_char.__ctype_le__ = c_char.__ctype_be__ = c_char
+_check_size(c_char)
+
+class c_char_p(_SimpleCData):
+    _type_ = "z"
+    def __repr__(self):
+        return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value)
+_check_size(c_char_p, "P")
+
+class c_void_p(_SimpleCData):
+    _type_ = "P"
+c_voidp = c_void_p # backwards compatibility (to a bug)
+_check_size(c_void_p)
+
+class c_bool(_SimpleCData):
+    _type_ = "?"
+
+from _ctypes import POINTER, pointer, _pointer_type_cache
+
+class c_wchar_p(_SimpleCData):
+    _type_ = "Z"
+    def __repr__(self):
+        return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value)
+
+class c_wchar(_SimpleCData):
+    _type_ = "u"
+
+def _reset_cache():
+    _pointer_type_cache.clear()
+    _c_functype_cache.clear()
+    if _os.name == "nt":
+        _win_functype_cache.clear()
+    # _SimpleCData.c_wchar_p_from_param
+    POINTER(c_wchar).from_param = c_wchar_p.from_param
+    # _SimpleCData.c_char_p_from_param
+    POINTER(c_char).from_param = c_char_p.from_param
+    _pointer_type_cache[None] = c_void_p
+
+def create_unicode_buffer(init, size=None):
+    """create_unicode_buffer(aString) -> character array
+    create_unicode_buffer(anInteger) -> character array
+    create_unicode_buffer(aString, anInteger) -> character array
+    """
+    if isinstance(init, str):
+        if size is None:
+            if sizeof(c_wchar) == 2:
+                # UTF-16 requires a surrogate pair (2 wchar_t) for non-BMP
+                # characters (outside [U+0000; U+FFFF] range). +1 for trailing
+                # NUL character.
+                size = sum(2 if ord(c) > 0xFFFF else 1 for c in init) + 1
+            else:
+                # 32-bit wchar_t (1 wchar_t per Unicode character). +1 for
+                # trailing NUL character.
+                size = len(init) + 1
+        _sys.audit("ctypes.create_unicode_buffer", init, size)
+        buftype = c_wchar * size
+        buf = buftype()
+        buf.value = init
+        return buf
+    elif isinstance(init, int):
+        _sys.audit("ctypes.create_unicode_buffer", None, init)
+        buftype = c_wchar * init
+        buf = buftype()
+        return buf
+    raise TypeError(init)
+
+
+# XXX Deprecated
+def SetPointerType(pointer, cls):
+    if _pointer_type_cache.get(cls, None) is not None:
+        raise RuntimeError("This type already exists in the cache")
+    if id(pointer) not in _pointer_type_cache:
+        raise RuntimeError("What's this???")
+    pointer.set_type(cls)
+    _pointer_type_cache[cls] = pointer
+    del _pointer_type_cache[id(pointer)]
+
+# XXX Deprecated
+def ARRAY(typ, len):
+    return typ * len
+
+################################################################
+
+
+class CDLL(object):
+    """An instance of this class represents a loaded dll/shared
+    library, exporting functions using the standard C calling
+    convention (named 'cdecl' on Windows).
+
+    The exported functions can be accessed as attributes, or by
+    indexing with the function name.  Examples:
+
+    <obj>.qsort -> callable object
+    <obj>['qsort'] -> callable object
+
+    Calling the functions releases the Python GIL during the call and
+    reacquires it afterwards.
+    """
+    _func_flags_ = _FUNCFLAG_CDECL
+    _func_restype_ = c_int
+    # default values for repr
+    _name = '<uninitialized>'
+    _handle = 0
+    _FuncPtr = None
+
+    def __init__(self, name, mode=DEFAULT_MODE, handle=None,
+                 use_errno=False,
+                 use_last_error=False,
+                 winmode=None):
+        self._name = name
+        flags = self._func_flags_
+        if use_errno:
+            flags |= _FUNCFLAG_USE_ERRNO
+        if use_last_error:
+            flags |= _FUNCFLAG_USE_LASTERROR
+        if _sys.platform.startswith("aix"):
+            """When the name contains ".a(" and ends with ")",
+               e.g., "libFOO.a(libFOO.so)" - this is taken to be an
+               archive(member) syntax for dlopen(), and the mode is adjusted.
+               Otherwise, name is presented to dlopen() as a file argument.
+            """
+            if name and name.endswith(")") and ".a(" in name:
+                mode |= ( _os.RTLD_MEMBER | _os.RTLD_NOW )
+        if _os.name == "nt":
+            if winmode is not None:
+                mode = winmode
+            else:
+                import nt
+                mode = nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS
+                if '/' in name or '\\' in name:
+                    self._name = nt._getfullpathname(self._name)
+                    mode |= nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR
+
+        class _FuncPtr(_CFuncPtr):
+            _flags_ = flags
+            _restype_ = self._func_restype_
+        self._FuncPtr = _FuncPtr
+
+        if handle is None:
+            self._handle = _dlopen(self._name, mode)
+        else:
+            self._handle = handle
+
+    def __repr__(self):
+        return "<%s '%s', handle %x at %#x>" % \
+               (self.__class__.__name__, self._name,
+                (self._handle & (_sys.maxsize*2 + 1)),
+                id(self) & (_sys.maxsize*2 + 1))
+
+    def __getattr__(self, name):
+        if name.startswith('__') and name.endswith('__'):
+            raise AttributeError(name)
+        func = self.__getitem__(name)
+        setattr(self, name, func)
+        return func
+
+    def __getitem__(self, name_or_ordinal):
+        func = self._FuncPtr((name_or_ordinal, self))
+        if not isinstance(name_or_ordinal, int):
+            func.__name__ = name_or_ordinal
+        return func
+
+class PyDLL(CDLL):
+    """This class represents the Python library itself.  It allows
+    accessing Python API functions.  The GIL is not released, and
+    Python exceptions are handled correctly.
+    """
+    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
+
+if _os.name == "nt":
+
+    class WinDLL(CDLL):
+        """This class represents a dll exporting functions using the
+        Windows stdcall calling convention.
+        """
+        _func_flags_ = _FUNCFLAG_STDCALL
+
+    # XXX Hm, what about HRESULT as normal parameter?
+    # Mustn't it derive from c_long then?
+    from _ctypes import _check_HRESULT, _SimpleCData
+    class HRESULT(_SimpleCData):
+        _type_ = "l"
+        # _check_retval_ is called with the function's result when it
+        # is used as restype.  It checks for the FAILED bit, and
+        # raises an OSError if it is set.
+        #
+        # The _check_retval_ method is implemented in C, so that the
+        # method definition itself is not included in the traceback
+        # when it raises an error - that is what we want (and Python
+        # doesn't have a way to raise an exception in the caller's
+        # frame).
+        _check_retval_ = _check_HRESULT
+
+    class OleDLL(CDLL):
+        """This class represents a dll exporting functions using the
+        Windows stdcall calling convention, and returning HRESULT.
+        HRESULT error values are automatically raised as OSError
+        exceptions.
+        """
+        _func_flags_ = _FUNCFLAG_STDCALL
+        _func_restype_ = HRESULT
+
+class LibraryLoader(object):
+    def __init__(self, dlltype):
+        self._dlltype = dlltype
+
+    def __getattr__(self, name):
+        if name[0] == '_':
+            raise AttributeError(name)
+        try:
+            dll = self._dlltype(name)
+        except OSError:
+            raise AttributeError(name)
+        setattr(self, name, dll)
+        return dll
+
+    def __getitem__(self, name):
+        return getattr(self, name)
+
+    def LoadLibrary(self, name):
+        return self._dlltype(name)
+
+    __class_getitem__ = classmethod(_types.GenericAlias)
+
+cdll = LibraryLoader(CDLL)
+pydll = LibraryLoader(PyDLL)
+
+if _os.name == "nt":
+    pythonapi = PyDLL("python dll", None, _sys.dllhandle)
+elif _sys.platform == "cygwin":
+    pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
+else:
+    pythonapi = PyDLL(None)
+
+
+if _os.name == "nt":
+    windll = LibraryLoader(WinDLL)
+    oledll = LibraryLoader(OleDLL)
+
+    GetLastError = windll.kernel32.GetLastError
+    from _ctypes import get_last_error, set_last_error
+
+    def WinError(code=None, descr=None):
+        if code is None:
+            code = GetLastError()
+        if descr is None:
+            descr = FormatError(code).strip()
+        return OSError(None, descr, None, code)
+
+if sizeof(c_uint) == sizeof(c_void_p):
+    c_size_t = c_uint
+    c_ssize_t = c_int
+elif sizeof(c_ulong) == sizeof(c_void_p):
+    c_size_t = c_ulong
+    c_ssize_t = c_long
+elif sizeof(c_ulonglong) == sizeof(c_void_p):
+    c_size_t = c_ulonglong
+    c_ssize_t = c_longlong
+
+# functions
+
+from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr
+
+## void *memmove(void *, const void *, size_t);
+memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr)
+
+## void *memset(void *, int, size_t)
+memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr)
+
+def PYFUNCTYPE(restype, *argtypes):
+    class CFunctionType(_CFuncPtr):
+        _argtypes_ = argtypes
+        _restype_ = restype
+        _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
+    return CFunctionType
+
+_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
+def cast(obj, typ):
+    return _cast(obj, obj, typ)
+
+_string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
+def string_at(ptr, size=-1):
+    """string_at(addr[, size]) -> string
+
+    Return the string at addr."""
+    return _string_at(ptr, size)
+
+try:
+    from _ctypes import _wstring_at_addr
+except ImportError:
+    pass
+else:
+    _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr)
+    def wstring_at(ptr, size=-1):
+        """wstring_at(addr[, size]) -> string
+
+        Return the string at addr."""
+        return _wstring_at(ptr, size)
+
+
+if _os.name == "nt": # COM stuff
+    def DllGetClassObject(rclsid, riid, ppv):
+        try:
+            ccom = __import__("comtypes.server.inprocserver", globals(), locals(), ['*'])
+        except ImportError:
+            return -2147221231 # CLASS_E_CLASSNOTAVAILABLE
+        else:
+            return ccom.DllGetClassObject(rclsid, riid, ppv)
+
+    def DllCanUnloadNow():
+        try:
+            ccom = __import__("comtypes.server.inprocserver", globals(), locals(), ['*'])
+        except ImportError:
+            return 0 # S_OK
+        return ccom.DllCanUnloadNow()
+
+from ctypes._endian import BigEndianStructure, LittleEndianStructure
+from ctypes._endian import BigEndianUnion, LittleEndianUnion
+
+# Fill in specifically-sized types
+c_int8 = c_byte
+c_uint8 = c_ubyte
+for kind in [c_short, c_int, c_long, c_longlong]:
+    if sizeof(kind) == 2: c_int16 = kind
+    elif sizeof(kind) == 4: c_int32 = kind
+    elif sizeof(kind) == 8: c_int64 = kind
+for kind in [c_ushort, c_uint, c_ulong, c_ulonglong]:
+    if sizeof(kind) == 2: c_uint16 = kind
+    elif sizeof(kind) == 4: c_uint32 = kind
+    elif sizeof(kind) == 8: c_uint64 = kind
+del(kind)
+
+if SIZEOF_TIME_T == 8:
+    c_time_t = c_int64
+elif SIZEOF_TIME_T == 4:
+    c_time_t = c_int32
+else:
+    raise SystemError(f"Unexpected sizeof(time_t): {SIZEOF_TIME_T=}")
+
+_reset_cache()

Lib/ctypes/_aix.py

@@ -0,0 +1,327 @@
+"""
+Lib/ctypes.util.find_library() support for AIX
+Similar approach as done for Darwin support by using separate files
+but unlike Darwin - no extension such as ctypes.macholib.*
+
+dlopen() is an interface to AIX initAndLoad() - primary documentation at:
+https://www.ibm.com/support/knowledgecenter/en/ssw_aix_61/com.ibm.aix.basetrf1/dlopen.htm
+https://www.ibm.com/support/knowledgecenter/en/ssw_aix_61/com.ibm.aix.basetrf1/load.htm
+
+AIX supports two styles for dlopen(): svr4 (System V Release 4) which is common on posix
+platforms, but also a BSD style - aka SVR3.
+
+From AIX 5.3 Difference Addendum (December 2004)
+2.9 SVR4 linking affinity
+Nowadays, there are two major object file formats used by the operating systems:
+XCOFF: The COFF enhanced by IBM and others. The original COFF (Common
+Object File Format) was the base of SVR3 and BSD 4.2 systems.
+ELF:   Executable and Linking Format that was developed by AT&T and is a
+base for SVR4 UNIX.
+
+While the shared library content is identical on AIX - one is located as a filepath name
+(svr4 style) and the other is located as a member of an archive (and the archive
+is located as a filepath name).
+
+The key difference arises when supporting multiple abi formats (i.e., 32 and 64 bit).
+For svr4 either only one ABI is supported, or there are two directories, or there
+are different file names. The most common solution for multiple ABI is multiple
+directories.
+
+For the XCOFF (aka AIX) style - one directory (one archive file) is sufficient
+as multiple shared libraries can be in the archive - even sharing the same name.
+In documentation the archive is also referred to as the "base" and the shared
+library object is referred to as the "member".
+
+For dlopen() on AIX (read initAndLoad()) the calls are similar.
+Default activity occurs when no path information is provided. When path
+information is provided dlopen() does not search any other directories.
+
+For SVR4 - the shared library name is the name of the file expected: libFOO.so
+For AIX - the shared library is expressed as base(member). The search is for the
+base (e.g., libFOO.a) and once the base is found the shared library - identified by
+member (e.g., libFOO.so, or shr.o) is located and loaded.
+
+The mode bit RTLD_MEMBER tells initAndLoad() that it needs to use the AIX (SVR3)
+naming style.
+"""
+__author__ = "Michael Felt <aixtools@felt.demon.nl>"
+
+import re
+from os import environ, path
+from sys import executable
+from ctypes import c_void_p, sizeof
+from subprocess import Popen, PIPE, DEVNULL
+
+# Executable bit size - 32 or 64
+# Used to filter the search in an archive by size, e.g., -X64
+AIX_ABI = sizeof(c_void_p) * 8
+
+
+from sys import maxsize
+def _last_version(libnames, sep):
+    def _num_version(libname):
+        # "libxyz.so.MAJOR.MINOR" => [MAJOR, MINOR]
+        parts = libname.split(sep)
+        nums = []
+        try:
+            while parts:
+                nums.insert(0, int(parts.pop()))
+        except ValueError:
+            pass
+        return nums or [maxsize]
+    return max(reversed(libnames), key=_num_version)
+
+def get_ld_header(p):
+    # "nested-function, but placed at module level
+    ld_header = None
+    for line in p.stdout:
+        if line.startswith(('/', './', '../')):
+            ld_header = line
+        elif "INDEX" in line:
+            return ld_header.rstrip('\n')
+    return None
+
+def get_ld_header_info(p):
+    # "nested-function, but placed at module level
+    # as an ld_header was found, return known paths, archives and members
+    # these lines start with a digit
+    info = []
+    for line in p.stdout:
+        if re.match("[0-9]", line):
+            info.append(line)
+        else:
+            # blank line (separator), consume line and end for loop
+            break
+    return info
+
+def get_ld_headers(file):
+    """
+    Parse the header of the loader section of executable and archives
+    This function calls /usr/bin/dump -H as a subprocess
+    and returns a list of (ld_header, ld_header_info) tuples.
+    """
+    # get_ld_headers parsing:
+    # 1. Find a line that starts with /, ./, or ../ - set as ld_header
+    # 2. If "INDEX" in occurs in a following line - return ld_header
+    # 3. get info (lines starting with [0-9])
+    ldr_headers = []
+    p = Popen(["/usr/bin/dump", f"-X{AIX_ABI}", "-H", file],
+        universal_newlines=True, stdout=PIPE, stderr=DEVNULL)
+    # be sure to read to the end-of-file - getting all entries
+    while ld_header := get_ld_header(p):
+        ldr_headers.append((ld_header, get_ld_header_info(p)))
+    p.stdout.close()
+    p.wait()
+    return ldr_headers
+
+def get_shared(ld_headers):
+    """
+    extract the shareable objects from ld_headers
+    character "[" is used to strip off the path information.
+    Note: the "[" and "]" characters that are part of dump -H output
+    are not removed here.
+    """
+    shared = []
+    for (line, _) in ld_headers:
+        # potential member lines contain "["
+        # otherwise, no processing needed
+        if "[" in line:
+            # Strip off trailing colon (:)
+            shared.append(line[line.index("["):-1])
+    return shared
+
+def get_one_match(expr, lines):
+    """
+    Must be only one match, otherwise result is None.
+    When there is a match, strip leading "[" and trailing "]"
+    """
+    # member names in the ld_headers output are between square brackets
+    expr = rf'\[({expr})\]'
+    matches = list(filter(None, (re.search(expr, line) for line in lines)))
+    if len(matches) == 1:
+        return matches[0].group(1)
+    else:
+        return None
+
+# additional processing to deal with AIX legacy names for 64-bit members
+def get_legacy(members):
+    """
+    This routine provides historical aka legacy naming schemes started
+    in AIX4 shared library support for library members names.
+    e.g., in /usr/lib/libc.a the member name shr.o for 32-bit binary and
+    shr_64.o for 64-bit binary.
+    """
+    if AIX_ABI == 64:
+        # AIX 64-bit member is one of shr64.o, shr_64.o, or shr4_64.o
+        expr = r'shr4?_?64\.o'
+        member = get_one_match(expr, members)
+        if member:
+            return member
+    else:
+        # 32-bit legacy names - both shr.o and shr4.o exist.
+        # shr.o is the preferred name so we look for shr.o first
+        #  i.e., shr4.o is returned only when shr.o does not exist
+        for name in ['shr.o', 'shr4.o']:
+            member = get_one_match(re.escape(name), members)
+            if member:
+                return member
+    return None
+
+def get_version(name, members):
+    """
+    Sort list of members and return highest numbered version - if it exists.
+    This function is called when an unversioned libFOO.a(libFOO.so) has
+    not been found.
+
+    Versioning for the member name is expected to follow
+    GNU LIBTOOL conventions: the highest version (x, then X.y, then X.Y.z)
+     * find [libFoo.so.X]
+     * find [libFoo.so.X.Y]
+     * find [libFoo.so.X.Y.Z]
+
+    Before the GNU convention became the standard scheme regardless of
+    binary size AIX packagers used GNU convention "as-is" for 32-bit
+    archive members but used an "distinguishing" name for 64-bit members.
+    This scheme inserted either 64 or _64 between libFOO and .so
+    - generally libFOO_64.so, but occasionally libFOO64.so
+    """
+    # the expression ending for versions must start as
+    # '.so.[0-9]', i.e., *.so.[at least one digit]
+    # while multiple, more specific expressions could be specified
+    # to search for .so.X, .so.X.Y and .so.X.Y.Z
+    # after the first required 'dot' digit
+    # any combination of additional 'dot' digits pairs are accepted
+    # anything more than libFOO.so.digits.digits.digits
+    # should be seen as a member name outside normal expectations
+    exprs = [rf'lib{name}\.so\.[0-9]+[0-9.]*',
+        rf'lib{name}_?64\.so\.[0-9]+[0-9.]*']
+    for expr in exprs:
+        versions = []
+        for line in members:
+            m = re.search(expr, line)
+            if m:
+                versions.append(m.group(0))
+        if versions:
+            return _last_version(versions, '.')
+    return None
+
+def get_member(name, members):
+    """
+    Return an archive member matching the request in name.
+    Name is the library name without any prefix like lib, suffix like .so,
+    or version number.
+    Given a list of members find and return the most appropriate result
+    Priority is given to generic libXXX.so, then a versioned libXXX.so.a.b.c
+    and finally, legacy AIX naming scheme.
+    """
+    # look first for a generic match - prepend lib and append .so
+    expr = rf'lib{name}\.so'
+    member = get_one_match(expr, members)
+    if member:
+        return member
+    elif AIX_ABI == 64:
+        expr = rf'lib{name}64\.so'
+        member = get_one_match(expr, members)
+    if member:
+        return member
+    # since an exact match with .so as suffix was not found
+    # look for a versioned name
+    # If a versioned name is not found, look for AIX legacy member name
+    member = get_version(name, members)
+    if member:
+        return member
+    else:
+        return get_legacy(members)
+
+def get_libpaths():
+    """
+    On AIX, the buildtime searchpath is stored in the executable.
+    as "loader header information".
+    The command /usr/bin/dump -H extracts this info.
+    Prefix searched libraries with LD_LIBRARY_PATH (preferred),
+    or LIBPATH if defined. These paths are appended to the paths
+    to libraries the python executable is linked with.
+    This mimics AIX dlopen() behavior.
+    """
+    libpaths = environ.get("LD_LIBRARY_PATH")
+    if libpaths is None:
+        libpaths = environ.get("LIBPATH")
+    if libpaths is None:
+        libpaths = []
+    else:
+        libpaths = libpaths.split(":")
+    objects = get_ld_headers(executable)
+    for (_, lines) in objects:
+        for line in lines:
+            # the second (optional) argument is PATH if it includes a /
+            path = line.split()[1]
+            if "/" in path:
+                libpaths.extend(path.split(":"))
+    return libpaths
+
+def find_shared(paths, name):
+    """
+    paths is a list of directories to search for an archive.
+    name is the abbreviated name given to find_library().
+    Process: search "paths" for archive, and if an archive is found
+    return the result of get_member().
+    If an archive is not found then return None
+    """
+    for dir in paths:
+        # /lib is a symbolic link to /usr/lib, skip it
+        if dir == "/lib":
+            continue
+        # "lib" is prefixed to emulate compiler name resolution,
+        # e.g., -lc to libc
+        base = f'lib{name}.a'
+        archive = path.join(dir, base)
+        if path.exists(archive):
+            members = get_shared(get_ld_headers(archive))
+            member = get_member(re.escape(name), members)
+            if member is not None:
+                return (base, member)
+            else:
+                return (None, None)
+    return (None, None)
+
+def find_library(name):
+    """AIX implementation of ctypes.util.find_library()
+    Find an archive member that will dlopen(). If not available,
+    also search for a file (or link) with a .so suffix.
+
+    AIX supports two types of schemes that can be used with dlopen().
+    The so-called SystemV Release4 (svr4) format is commonly suffixed
+    with .so while the (default) AIX scheme has the library (archive)
+    ending with the suffix .a
+    As an archive has multiple members (e.g., 32-bit and 64-bit) in one file
+    the argument passed to dlopen must include both the library and
+    the member names in a single string.
+
+    find_library() looks first for an archive (.a) with a suitable member.
+    If no archive+member pair is found, look for a .so file.
+    """
+
+    libpaths = get_libpaths()
+    (base, member) = find_shared(libpaths, name)
+    if base is not None:
+        return f"{base}({member})"
+
+    # To get here, a member in an archive has not been found
+    # In other words, either:
+    # a) a .a file was not found
+    # b) a .a file did not have a suitable member
+    # So, look for a .so file
+    # Check libpaths for .so file
+    # Note, the installation must prepare a link from a .so
+    # to a versioned file
+    # This is common practice by GNU libtool on other platforms
+    soname = f"lib{name}.so"
+    for dir in libpaths:
+        # /lib is a symbolic link to /usr/lib, skip it
+        if dir == "/lib":
+            continue
+        shlib = path.join(dir, soname)
+        if path.exists(shlib):
+            return soname
+    # if we are here, we have not found anything plausible
+    return None

Lib/ctypes/_endian.py

@@ -0,0 +1,78 @@
+import sys
+from ctypes import *
+
+_array_type = type(Array)
+
+def _other_endian(typ):
+    """Return the type with the 'other' byte order.  Simple types like
+    c_int and so on already have __ctype_be__ and __ctype_le__
+    attributes which contain the types, for more complicated types
+    arrays and structures are supported.
+    """
+    # check _OTHER_ENDIAN attribute (present if typ is primitive type)
+    if hasattr(typ, _OTHER_ENDIAN):
+        return getattr(typ, _OTHER_ENDIAN)
+    # if typ is array
+    if isinstance(typ, _array_type):
+        return _other_endian(typ._type_) * typ._length_
+    # if typ is structure
+    if issubclass(typ, Structure):
+        return typ
+    raise TypeError("This type does not support other endian: %s" % typ)
+
+class _swapped_meta:
+    def __setattr__(self, attrname, value):
+        if attrname == "_fields_":
+            fields = []
+            for desc in value:
+                name = desc[0]
+                typ = desc[1]
+                rest = desc[2:]
+                fields.append((name, _other_endian(typ)) + rest)
+            value = fields
+        super().__setattr__(attrname, value)
+class _swapped_struct_meta(_swapped_meta, type(Structure)): pass
+class _swapped_union_meta(_swapped_meta, type(Union)): pass
+
+################################################################
+
+# Note: The Structure metaclass checks for the *presence* (not the
+# value!) of a _swapped_bytes_ attribute to determine the bit order in
+# structures containing bit fields.
+
+if sys.byteorder == "little":
+    _OTHER_ENDIAN = "__ctype_be__"
+
+    LittleEndianStructure = Structure
+
+    class BigEndianStructure(Structure, metaclass=_swapped_struct_meta):
+        """Structure with big endian byte order"""
+        __slots__ = ()
+        _swappedbytes_ = None
+
+    LittleEndianUnion = Union
+
+    class BigEndianUnion(Union, metaclass=_swapped_union_meta):
+        """Union with big endian byte order"""
+        __slots__ = ()
+        _swappedbytes_ = None
+
+elif sys.byteorder == "big":
+    _OTHER_ENDIAN = "__ctype_le__"
+
+    BigEndianStructure = Structure
+
+    class LittleEndianStructure(Structure, metaclass=_swapped_struct_meta):
+        """Structure with little endian byte order"""
+        __slots__ = ()
+        _swappedbytes_ = None
+
+    BigEndianUnion = Union
+
+    class LittleEndianUnion(Union, metaclass=_swapped_union_meta):
+        """Union with little endian byte order"""
+        __slots__ = ()
+        _swappedbytes_ = None
+
+else:
+    raise RuntimeError("Invalid byteorder")

Lib/ctypes/macholib/README.ctypes

@@ -0,0 +1,7 @@
+Files in this directory come from Bob Ippolito's py2app.
+
+License: Any components of the py2app suite may be distributed under
+the MIT or PSF open source licenses.
+
+This is version 1.0, SVN revision 789, from 2006/01/25.
+The main repository is http://svn.red-bean.com/bob/macholib/trunk/macholib/

Lib/ctypes/macholib/__init__.py

@@ -0,0 +1,9 @@
+"""
+Enough Mach-O to make your head spin.
+
+See the relevant header files in /usr/include/mach-o
+
+And also Apple's documentation.
+"""
+
+__version__ = '1.0'

Lib/ctypes/macholib/dyld.py

@@ -0,0 +1,165 @@
+"""
+dyld emulation
+"""
+
+import os
+from ctypes.macholib.framework import framework_info
+from ctypes.macholib.dylib import dylib_info
+from itertools import *
+try:
+    from _ctypes import _dyld_shared_cache_contains_path
+except ImportError:
+    def _dyld_shared_cache_contains_path(*args):
+        raise NotImplementedError
+
+__all__ = [
+    'dyld_find', 'framework_find',
+    'framework_info', 'dylib_info',
+]
+
+# These are the defaults as per man dyld(1)
+#
+DEFAULT_FRAMEWORK_FALLBACK = [
+    os.path.expanduser("~/Library/Frameworks"),
+    "/Library/Frameworks",
+    "/Network/Library/Frameworks",
+    "/System/Library/Frameworks",
+]
+
+DEFAULT_LIBRARY_FALLBACK = [
+    os.path.expanduser("~/lib"),
+    "/usr/local/lib",
+    "/lib",
+    "/usr/lib",
+]
+
+def dyld_env(env, var):
+    if env is None:
+        env = os.environ
+    rval = env.get(var)
+    if rval is None:
+        return []
+    return rval.split(':')
+
+def dyld_image_suffix(env=None):
+    if env is None:
+        env = os.environ
+    return env.get('DYLD_IMAGE_SUFFIX')
+
+def dyld_framework_path(env=None):
+    return dyld_env(env, 'DYLD_FRAMEWORK_PATH')
+
+def dyld_library_path(env=None):
+    return dyld_env(env, 'DYLD_LIBRARY_PATH')
+
+def dyld_fallback_framework_path(env=None):
+    return dyld_env(env, 'DYLD_FALLBACK_FRAMEWORK_PATH')
+
+def dyld_fallback_library_path(env=None):
+    return dyld_env(env, 'DYLD_FALLBACK_LIBRARY_PATH')
+
+def dyld_image_suffix_search(iterator, env=None):
+    """For a potential path iterator, add DYLD_IMAGE_SUFFIX semantics"""
+    suffix = dyld_image_suffix(env)
+    if suffix is None:
+        return iterator
+    def _inject(iterator=iterator, suffix=suffix):
+        for path in iterator:
+            if path.endswith('.dylib'):
+                yield path[:-len('.dylib')] + suffix + '.dylib'
+            else:
+                yield path + suffix
+            yield path
+    return _inject()
+
+def dyld_override_search(name, env=None):
+    # If DYLD_FRAMEWORK_PATH is set and this dylib_name is a
+    # framework name, use the first file that exists in the framework
+    # path if any.  If there is none go on to search the DYLD_LIBRARY_PATH
+    # if any.
+
+    framework = framework_info(name)
+
+    if framework is not None:
+        for path in dyld_framework_path(env):
+            yield os.path.join(path, framework['name'])
+
+    # If DYLD_LIBRARY_PATH is set then use the first file that exists
+    # in the path.  If none use the original name.
+    for path in dyld_library_path(env):
+        yield os.path.join(path, os.path.basename(name))
+
+def dyld_executable_path_search(name, executable_path=None):
+    # If we haven't done any searching and found a library and the
+    # dylib_name starts with "@executable_path/" then construct the
+    # library name.
+    if name.startswith('@executable_path/') and executable_path is not None:
+        yield os.path.join(executable_path, name[len('@executable_path/'):])
+
+def dyld_default_search(name, env=None):
+    yield name
+
+    framework = framework_info(name)
+
+    if framework is not None:
+        fallback_framework_path = dyld_fallback_framework_path(env)
+        for path in fallback_framework_path:
+            yield os.path.join(path, framework['name'])
+
+    fallback_library_path = dyld_fallback_library_path(env)
+    for path in fallback_library_path:
+        yield os.path.join(path, os.path.basename(name))
+
+    if framework is not None and not fallback_framework_path:
+        for path in DEFAULT_FRAMEWORK_FALLBACK:
+            yield os.path.join(path, framework['name'])
+
+    if not fallback_library_path:
+        for path in DEFAULT_LIBRARY_FALLBACK:
+            yield os.path.join(path, os.path.basename(name))
+
+def dyld_find(name, executable_path=None, env=None):
+    """
+    Find a library or framework using dyld semantics
+    """
+    for path in dyld_image_suffix_search(chain(
+                dyld_override_search(name, env),
+                dyld_executable_path_search(name, executable_path),
+                dyld_default_search(name, env),
+            ), env):
+
+        if os.path.isfile(path):
+            return path
+        try:
+            if _dyld_shared_cache_contains_path(path):
+                return path
+        except NotImplementedError:
+            pass
+
+    raise ValueError("dylib %s could not be found" % (name,))
+
+def framework_find(fn, executable_path=None, env=None):
+    """
+    Find a framework using dyld semantics in a very loose manner.
+
+    Will take input such as:
+        Python
+        Python.framework
+        Python.framework/Versions/Current
+    """
+    error = None
+    try:
+        return dyld_find(fn, executable_path=executable_path, env=env)
+    except ValueError as e:
+        error = e
+    fmwk_index = fn.rfind('.framework')
+    if fmwk_index == -1:
+        fmwk_index = len(fn)
+        fn += '.framework'
+    fn = os.path.join(fn, os.path.basename(fn[:fmwk_index]))
+    try:
+        return dyld_find(fn, executable_path=executable_path, env=env)
+    except ValueError:
+        raise error
+    finally:
+        error = None

Lib/ctypes/macholib/dylib.py

@@ -0,0 +1,42 @@
+"""
+Generic dylib path manipulation
+"""
+
+import re
+
+__all__ = ['dylib_info']
+
+DYLIB_RE = re.compile(r"""(?x)
+(?P<location>^.*)(?:^|/)
+(?P<name>
+    (?P<shortname>\w+?)
+    (?:\.(?P<version>[^._]+))?
+    (?:_(?P<suffix>[^._]+))?
+    \.dylib$
+)
+""")
+
+def dylib_info(filename):
+    """
+    A dylib name can take one of the following four forms:
+        Location/Name.SomeVersion_Suffix.dylib
+        Location/Name.SomeVersion.dylib
+        Location/Name_Suffix.dylib
+        Location/Name.dylib
+
+    returns None if not found or a mapping equivalent to:
+        dict(
+            location='Location',
+            name='Name.SomeVersion_Suffix.dylib',
+            shortname='Name',
+            version='SomeVersion',
+            suffix='Suffix',
+        )
+
+    Note that SomeVersion and Suffix are optional and may be None
+    if not present.
+    """
+    is_dylib = DYLIB_RE.match(filename)
+    if not is_dylib:
+        return None
+    return is_dylib.groupdict()

Lib/ctypes/macholib/fetch_macholib

@@ -0,0 +1,2 @@
+#!/bin/sh
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .

Lib/ctypes/macholib/fetch_macholib.bat

@@ -0,0 +1 @@
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .

Lib/ctypes/macholib/framework.py

@@ -0,0 +1,42 @@
+"""
+Generic framework path manipulation
+"""
+
+import re
+
+__all__ = ['framework_info']
+
+STRICT_FRAMEWORK_RE = re.compile(r"""(?x)
+(?P<location>^.*)(?:^|/)
+(?P<name>
+    (?P<shortname>\w+).framework/
+    (?:Versions/(?P<version>[^/]+)/)?
+    (?P=shortname)
+    (?:_(?P<suffix>[^_]+))?
+)$
+""")
+
+def framework_info(filename):
+    """
+    A framework name can take one of the following four forms:
+        Location/Name.framework/Versions/SomeVersion/Name_Suffix
+        Location/Name.framework/Versions/SomeVersion/Name
+        Location/Name.framework/Name_Suffix
+        Location/Name.framework/Name
+
+    returns None if not found, or a mapping equivalent to:
+        dict(
+            location='Location',
+            name='Name.framework/Versions/SomeVersion/Name_Suffix',
+            shortname='Name',
+            version='SomeVersion',
+            suffix='Suffix',
+        )
+
+    Note that SomeVersion and Suffix are optional and may be None
+    if not present
+    """
+    is_framework = STRICT_FRAMEWORK_RE.match(filename)
+    if not is_framework:
+        return None
+    return is_framework.groupdict()

Lib/ctypes/test/__init__.py

@@ -0,0 +1,16 @@
+import os
+import unittest
+from test import support
+from test.support import import_helper
+
+
+# skip tests if _ctypes was not built
+ctypes = import_helper.import_module('ctypes')
+ctypes_symbols = dir(ctypes)
+
+def need_symbol(name):
+    return unittest.skipUnless(name in ctypes_symbols,
+                               '{!r} is required'.format(name))
+
+def load_tests(*args):
+    return support.load_package_tests(os.path.dirname(__file__), *args)

Lib/ctypes/test/__main__.py

@@ -0,0 +1,4 @@
+from ctypes.test import load_tests
+import unittest
+
+unittest.main()

Lib/ctypes/test/test_anon.py

@@ -0,0 +1,73 @@
+import unittest
+import test.support
+from ctypes import *
+
+class AnonTest(unittest.TestCase):
+
+    def test_anon(self):
+        class ANON(Union):
+            _fields_ = [("a", c_int),
+                        ("b", c_int)]
+
+        class Y(Structure):
+            _fields_ = [("x", c_int),
+                        ("_", ANON),
+                        ("y", c_int)]
+            _anonymous_ = ["_"]
+
+        self.assertEqual(Y.a.offset, sizeof(c_int))
+        self.assertEqual(Y.b.offset, sizeof(c_int))
+
+        self.assertEqual(ANON.a.offset, 0)
+        self.assertEqual(ANON.b.offset, 0)
+
+    def test_anon_nonseq(self):
+        # TypeError: _anonymous_ must be a sequence
+        self.assertRaises(TypeError,
+                              lambda: type(Structure)("Name",
+                                                      (Structure,),
+                                                      {"_fields_": [], "_anonymous_": 42}))
+
+    def test_anon_nonmember(self):
+        # AttributeError: type object 'Name' has no attribute 'x'
+        self.assertRaises(AttributeError,
+                              lambda: type(Structure)("Name",
+                                                      (Structure,),
+                                                      {"_fields_": [],
+                                                       "_anonymous_": ["x"]}))
+
+    @test.support.cpython_only
+    def test_issue31490(self):
+        # There shouldn't be an assertion failure in case the class has an
+        # attribute whose name is specified in _anonymous_ but not in _fields_.
+
+        # AttributeError: 'x' is specified in _anonymous_ but not in _fields_
+        with self.assertRaises(AttributeError):
+            class Name(Structure):
+                _fields_ = []
+                _anonymous_ = ["x"]
+                x = 42
+
+    def test_nested(self):
+        class ANON_S(Structure):
+            _fields_ = [("a", c_int)]
+
+        class ANON_U(Union):
+            _fields_ = [("_", ANON_S),
+                        ("b", c_int)]
+            _anonymous_ = ["_"]
+
+        class Y(Structure):
+            _fields_ = [("x", c_int),
+                        ("_", ANON_U),
+                        ("y", c_int)]
+            _anonymous_ = ["_"]
+
+        self.assertEqual(Y.x.offset, 0)
+        self.assertEqual(Y.a.offset, sizeof(c_int))
+        self.assertEqual(Y.b.offset, sizeof(c_int))
+        self.assertEqual(Y._.offset, sizeof(c_int))
+        self.assertEqual(Y.y.offset, sizeof(c_int) * 2)
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_array_in_pointer.py

@@ -0,0 +1,64 @@
+import unittest
+from ctypes import *
+from binascii import hexlify
+import re
+
+def dump(obj):
+    # helper function to dump memory contents in hex, with a hyphen
+    # between the bytes.
+    h = hexlify(memoryview(obj)).decode()
+    return re.sub(r"(..)", r"\1-", h)[:-1]
+
+
+class Value(Structure):
+    _fields_ = [("val", c_byte)]
+
+class Container(Structure):
+    _fields_ = [("pvalues", POINTER(Value))]
+
+class Test(unittest.TestCase):
+    def test(self):
+        # create an array of 4 values
+        val_array = (Value * 4)()
+
+        # create a container, which holds a pointer to the pvalues array.
+        c = Container()
+        c.pvalues = val_array
+
+        # memory contains 4 NUL bytes now, that's correct
+        self.assertEqual("00-00-00-00", dump(val_array))
+
+        # set the values of the array through the pointer:
+        for i in range(4):
+            c.pvalues[i].val = i + 1
+
+        values = [c.pvalues[i].val for i in range(4)]
+
+        # These are the expected results: here s the bug!
+        self.assertEqual(
+            (values, dump(val_array)),
+            ([1, 2, 3, 4], "01-02-03-04")
+        )
+
+    def test_2(self):
+
+        val_array = (Value * 4)()
+
+        # memory contains 4 NUL bytes now, that's correct
+        self.assertEqual("00-00-00-00", dump(val_array))
+
+        ptr = cast(val_array, POINTER(Value))
+        # set the values of the array through the pointer:
+        for i in range(4):
+            ptr[i].val = i + 1
+
+        values = [ptr[i].val for i in range(4)]
+
+        # These are the expected results: here s the bug!
+        self.assertEqual(
+            (values, dump(val_array)),
+            ([1, 2, 3, 4], "01-02-03-04")
+        )
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_arrays.py

@@ -0,0 +1,238 @@
+import unittest
+from test.support import bigmemtest, _2G
+import sys
+from ctypes import *
+
+from ctypes.test import need_symbol
+
+formats = "bBhHiIlLqQfd"
+
+formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \
+          c_long, c_ulonglong, c_float, c_double, c_longdouble
+
+class ArrayTestCase(unittest.TestCase):
+    def test_simple(self):
+        # create classes holding simple numeric types, and check
+        # various properties.
+
+        init = list(range(15, 25))
+
+        for fmt in formats:
+            alen = len(init)
+            int_array = ARRAY(fmt, alen)
+
+            ia = int_array(*init)
+            # length of instance ok?
+            self.assertEqual(len(ia), alen)
+
+            # slot values ok?
+            values = [ia[i] for i in range(alen)]
+            self.assertEqual(values, init)
+
+            # out-of-bounds accesses should be caught
+            with self.assertRaises(IndexError): ia[alen]
+            with self.assertRaises(IndexError): ia[-alen-1]
+
+            # change the items
+            from operator import setitem
+            new_values = list(range(42, 42+alen))
+            [setitem(ia, n, new_values[n]) for n in range(alen)]
+            values = [ia[i] for i in range(alen)]
+            self.assertEqual(values, new_values)
+
+            # are the items initialized to 0?
+            ia = int_array()
+            values = [ia[i] for i in range(alen)]
+            self.assertEqual(values, [0] * alen)
+
+            # Too many initializers should be caught
+            self.assertRaises(IndexError, int_array, *range(alen*2))
+
+        CharArray = ARRAY(c_char, 3)
+
+        ca = CharArray(b"a", b"b", b"c")
+
+        # Should this work? It doesn't:
+        # CharArray("abc")
+        self.assertRaises(TypeError, CharArray, "abc")
+
+        self.assertEqual(ca[0], b"a")
+        self.assertEqual(ca[1], b"b")
+        self.assertEqual(ca[2], b"c")
+        self.assertEqual(ca[-3], b"a")
+        self.assertEqual(ca[-2], b"b")
+        self.assertEqual(ca[-1], b"c")
+
+        self.assertEqual(len(ca), 3)
+
+        # cannot delete items
+        from operator import delitem
+        self.assertRaises(TypeError, delitem, ca, 0)
+
+    def test_step_overflow(self):
+        a = (c_int * 5)()
+        a[3::sys.maxsize] = (1,)
+        self.assertListEqual(a[3::sys.maxsize], [1])
+        a = (c_char * 5)()
+        a[3::sys.maxsize] = b"A"
+        self.assertEqual(a[3::sys.maxsize], b"A")
+        a = (c_wchar * 5)()
+        a[3::sys.maxsize] = u"X"
+        self.assertEqual(a[3::sys.maxsize], u"X")
+
+    def test_numeric_arrays(self):
+
+        alen = 5
+
+        numarray = ARRAY(c_int, alen)
+
+        na = numarray()
+        values = [na[i] for i in range(alen)]
+        self.assertEqual(values, [0] * alen)
+
+        na = numarray(*[c_int()] * alen)
+        values = [na[i] for i in range(alen)]
+        self.assertEqual(values, [0]*alen)
+
+        na = numarray(1, 2, 3, 4, 5)
+        values = [i for i in na]
+        self.assertEqual(values, [1, 2, 3, 4, 5])
+
+        na = numarray(*map(c_int, (1, 2, 3, 4, 5)))
+        values = [i for i in na]
+        self.assertEqual(values, [1, 2, 3, 4, 5])
+
+    def test_classcache(self):
+        self.assertIsNot(ARRAY(c_int, 3), ARRAY(c_int, 4))
+        self.assertIs(ARRAY(c_int, 3), ARRAY(c_int, 3))
+
+    def test_from_address(self):
+        # Failed with 0.9.8, reported by JUrner
+        p = create_string_buffer(b"foo")
+        sz = (c_char * 3).from_address(addressof(p))
+        self.assertEqual(sz[:], b"foo")
+        self.assertEqual(sz[::], b"foo")
+        self.assertEqual(sz[::-1], b"oof")
+        self.assertEqual(sz[::3], b"f")
+        self.assertEqual(sz[1:4:2], b"o")
+        self.assertEqual(sz.value, b"foo")
+
+    @need_symbol('create_unicode_buffer')
+    def test_from_addressW(self):
+        p = create_unicode_buffer("foo")
+        sz = (c_wchar * 3).from_address(addressof(p))
+        self.assertEqual(sz[:], "foo")
+        self.assertEqual(sz[::], "foo")
+        self.assertEqual(sz[::-1], "oof")
+        self.assertEqual(sz[::3], "f")
+        self.assertEqual(sz[1:4:2], "o")
+        self.assertEqual(sz.value, "foo")
+
+    def test_cache(self):
+        # Array types are cached internally in the _ctypes extension,
+        # in a WeakValueDictionary.  Make sure the array type is
+        # removed from the cache when the itemtype goes away.  This
+        # test will not fail, but will show a leak in the testsuite.
+
+        # Create a new type:
+        class my_int(c_int):
+            pass
+        # Create a new array type based on it:
+        t1 = my_int * 1
+        t2 = my_int * 1
+        self.assertIs(t1, t2)
+
+    def test_subclass(self):
+        class T(Array):
+            _type_ = c_int
+            _length_ = 13
+        class U(T):
+            pass
+        class V(U):
+            pass
+        class W(V):
+            pass
+        class X(T):
+            _type_ = c_short
+        class Y(T):
+            _length_ = 187
+
+        for c in [T, U, V, W]:
+            self.assertEqual(c._type_, c_int)
+            self.assertEqual(c._length_, 13)
+            self.assertEqual(c()._type_, c_int)
+            self.assertEqual(c()._length_, 13)
+
+        self.assertEqual(X._type_, c_short)
+        self.assertEqual(X._length_, 13)
+        self.assertEqual(X()._type_, c_short)
+        self.assertEqual(X()._length_, 13)
+
+        self.assertEqual(Y._type_, c_int)
+        self.assertEqual(Y._length_, 187)
+        self.assertEqual(Y()._type_, c_int)
+        self.assertEqual(Y()._length_, 187)
+
+    def test_bad_subclass(self):
+        with self.assertRaises(AttributeError):
+            class T(Array):
+                pass
+        with self.assertRaises(AttributeError):
+            class T(Array):
+                _type_ = c_int
+        with self.assertRaises(AttributeError):
+            class T(Array):
+                _length_ = 13
+
+    def test_bad_length(self):
+        with self.assertRaises(ValueError):
+            class T(Array):
+                _type_ = c_int
+                _length_ = - sys.maxsize * 2
+        with self.assertRaises(ValueError):
+            class T(Array):
+                _type_ = c_int
+                _length_ = -1
+        with self.assertRaises(TypeError):
+            class T(Array):
+                _type_ = c_int
+                _length_ = 1.87
+        with self.assertRaises(OverflowError):
+            class T(Array):
+                _type_ = c_int
+                _length_ = sys.maxsize * 2
+
+    def test_zero_length(self):
+        # _length_ can be zero.
+        class T(Array):
+            _type_ = c_int
+            _length_ = 0
+
+    def test_empty_element_struct(self):
+        class EmptyStruct(Structure):
+            _fields_ = []
+
+        obj = (EmptyStruct * 2)()  # bpo37188: Floating point exception
+        self.assertEqual(sizeof(obj), 0)
+
+    def test_empty_element_array(self):
+        class EmptyArray(Array):
+            _type_ = c_int
+            _length_ = 0
+
+        obj = (EmptyArray * 2)()  # bpo37188: Floating point exception
+        self.assertEqual(sizeof(obj), 0)
+
+    def test_bpo36504_signed_int_overflow(self):
+        # The overflow check in PyCArrayType_new() could cause signed integer
+        # overflow.
+        with self.assertRaises(OverflowError):
+            c_char * sys.maxsize * 2
+
+    @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+    @bigmemtest(size=_2G, memuse=1, dry_run=False)
+    def test_large_array(self, size):
+        c_char * size
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_as_parameter.py

@@ -0,0 +1,231 @@
+import unittest
+from ctypes import *
+from ctypes.test import need_symbol
+import _ctypes_test
+
+dll = CDLL(_ctypes_test.__file__)
+
+try:
+    CALLBACK_FUNCTYPE = WINFUNCTYPE
+except NameError:
+    # fake to enable this test on Linux
+    CALLBACK_FUNCTYPE = CFUNCTYPE
+
+class POINT(Structure):
+    _fields_ = [("x", c_int), ("y", c_int)]
+
+class BasicWrapTestCase(unittest.TestCase):
+    def wrap(self, param):
+        return param
+
+    @need_symbol('c_wchar')
+    def test_wchar_parm(self):
+        f = dll._testfunc_i_bhilfd
+        f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
+        result = f(self.wrap(1), self.wrap("x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0))
+        self.assertEqual(result, 139)
+        self.assertIs(type(result), int)
+
+    def test_pointers(self):
+        f = dll._testfunc_p_p
+        f.restype = POINTER(c_int)
+        f.argtypes = [POINTER(c_int)]
+
+        # This only works if the value c_int(42) passed to the
+        # function is still alive while the pointer (the result) is
+        # used.
+
+        v = c_int(42)
+
+        self.assertEqual(pointer(v).contents.value, 42)
+        result = f(self.wrap(pointer(v)))
+        self.assertEqual(type(result), POINTER(c_int))
+        self.assertEqual(result.contents.value, 42)
+
+        # This on works...
+        result = f(self.wrap(pointer(v)))
+        self.assertEqual(result.contents.value, v.value)
+
+        p = pointer(c_int(99))
+        result = f(self.wrap(p))
+        self.assertEqual(result.contents.value, 99)
+
+    def test_shorts(self):
+        f = dll._testfunc_callback_i_if
+
+        args = []
+        expected = [262144, 131072, 65536, 32768, 16384, 8192, 4096, 2048,
+                    1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1]
+
+        def callback(v):
+            args.append(v)
+            return v
+
+        CallBack = CFUNCTYPE(c_int, c_int)
+
+        cb = CallBack(callback)
+        f(self.wrap(2**18), self.wrap(cb))
+        self.assertEqual(args, expected)
+
+    ################################################################
+
+    def test_callbacks(self):
+        f = dll._testfunc_callback_i_if
+        f.restype = c_int
+        f.argtypes = None
+
+        MyCallback = CFUNCTYPE(c_int, c_int)
+
+        def callback(value):
+            #print "called back with", value
+            return value
+
+        cb = MyCallback(callback)
+
+        result = f(self.wrap(-10), self.wrap(cb))
+        self.assertEqual(result, -18)
+
+        # test with prototype
+        f.argtypes = [c_int, MyCallback]
+        cb = MyCallback(callback)
+
+        result = f(self.wrap(-10), self.wrap(cb))
+        self.assertEqual(result, -18)
+
+        result = f(self.wrap(-10), self.wrap(cb))
+        self.assertEqual(result, -18)
+
+        AnotherCallback = CALLBACK_FUNCTYPE(c_int, c_int, c_int, c_int, c_int)
+
+        # check that the prototype works: we call f with wrong
+        # argument types
+        cb = AnotherCallback(callback)
+        self.assertRaises(ArgumentError, f, self.wrap(-10), self.wrap(cb))
+
+    def test_callbacks_2(self):
+        # Can also use simple datatypes as argument type specifiers
+        # for the callback function.
+        # In this case the call receives an instance of that type
+        f = dll._testfunc_callback_i_if
+        f.restype = c_int
+
+        MyCallback = CFUNCTYPE(c_int, c_int)
+
+        f.argtypes = [c_int, MyCallback]
+
+        def callback(value):
+            #print "called back with", value
+            self.assertEqual(type(value), int)
+            return value
+
+        cb = MyCallback(callback)
+        result = f(self.wrap(-10), self.wrap(cb))
+        self.assertEqual(result, -18)
+
+    @need_symbol('c_longlong')
+    def test_longlong_callbacks(self):
+
+        f = dll._testfunc_callback_q_qf
+        f.restype = c_longlong
+
+        MyCallback = CFUNCTYPE(c_longlong, c_longlong)
+
+        f.argtypes = [c_longlong, MyCallback]
+
+        def callback(value):
+            self.assertIsInstance(value, int)
+            return value & 0x7FFFFFFF
+
+        cb = MyCallback(callback)
+
+        self.assertEqual(13577625587, int(f(self.wrap(1000000000000), self.wrap(cb))))
+
+    def test_byval(self):
+        # without prototype
+        ptin = POINT(1, 2)
+        ptout = POINT()
+        # EXPORT int _testfunc_byval(point in, point *pout)
+        result = dll._testfunc_byval(ptin, byref(ptout))
+        got = result, ptout.x, ptout.y
+        expected = 3, 1, 2
+        self.assertEqual(got, expected)
+
+        # with prototype
+        ptin = POINT(101, 102)
+        ptout = POINT()
+        dll._testfunc_byval.argtypes = (POINT, POINTER(POINT))
+        dll._testfunc_byval.restype = c_int
+        result = dll._testfunc_byval(self.wrap(ptin), byref(ptout))
+        got = result, ptout.x, ptout.y
+        expected = 203, 101, 102
+        self.assertEqual(got, expected)
+
+    def test_struct_return_2H(self):
+        class S2H(Structure):
+            _fields_ = [("x", c_short),
+                        ("y", c_short)]
+        dll.ret_2h_func.restype = S2H
+        dll.ret_2h_func.argtypes = [S2H]
+        inp = S2H(99, 88)
+        s2h = dll.ret_2h_func(self.wrap(inp))
+        self.assertEqual((s2h.x, s2h.y), (99*2, 88*3))
+
+        # Test also that the original struct was unmodified (i.e. was passed by
+        # value)
+        self.assertEqual((inp.x, inp.y), (99, 88))
+
+    def test_struct_return_8H(self):
+        class S8I(Structure):
+            _fields_ = [("a", c_int),
+                        ("b", c_int),
+                        ("c", c_int),
+                        ("d", c_int),
+                        ("e", c_int),
+                        ("f", c_int),
+                        ("g", c_int),
+                        ("h", c_int)]
+        dll.ret_8i_func.restype = S8I
+        dll.ret_8i_func.argtypes = [S8I]
+        inp = S8I(9, 8, 7, 6, 5, 4, 3, 2)
+        s8i = dll.ret_8i_func(self.wrap(inp))
+        self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
+                             (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
+
+    def test_recursive_as_param(self):
+        from ctypes import c_int
+
+        class A(object):
+            pass
+
+        a = A()
+        a._as_parameter_ = a
+        with self.assertRaises(RecursionError):
+            c_int.from_param(a)
+
+
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+class AsParamWrapper(object):
+    def __init__(self, param):
+        self._as_parameter_ = param
+
+class AsParamWrapperTestCase(BasicWrapTestCase):
+    wrap = AsParamWrapper
+
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+class AsParamPropertyWrapper(object):
+    def __init__(self, param):
+        self._param = param
+
+    def getParameter(self):
+        return self._param
+    _as_parameter_ = property(getParameter)
+
+class AsParamPropertyWrapperTestCase(BasicWrapTestCase):
+    wrap = AsParamPropertyWrapper
+
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_bitfields.py

@@ -0,0 +1,297 @@
+from ctypes import *
+from ctypes.test import need_symbol
+from test import support
+import unittest
+import os
+
+import _ctypes_test
+
+class BITS(Structure):
+    _fields_ = [("A", c_int, 1),
+                ("B", c_int, 2),
+                ("C", c_int, 3),
+                ("D", c_int, 4),
+                ("E", c_int, 5),
+                ("F", c_int, 6),
+                ("G", c_int, 7),
+                ("H", c_int, 8),
+                ("I", c_int, 9),
+
+                ("M", c_short, 1),
+                ("N", c_short, 2),
+                ("O", c_short, 3),
+                ("P", c_short, 4),
+                ("Q", c_short, 5),
+                ("R", c_short, 6),
+                ("S", c_short, 7)]
+
+func = CDLL(_ctypes_test.__file__).unpack_bitfields
+func.argtypes = POINTER(BITS), c_char
+
+##for n in "ABCDEFGHIMNOPQRS":
+##    print n, hex(getattr(BITS, n).size), getattr(BITS, n).offset
+
+class C_Test(unittest.TestCase):
+
+    def test_ints(self):
+        for i in range(512):
+            for name in "ABCDEFGHI":
+                b = BITS()
+                setattr(b, name, i)
+                self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii')))
+
+    # bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior
+    @support.skip_if_sanitizer(ub=True)
+    def test_shorts(self):
+        b = BITS()
+        name = "M"
+        if func(byref(b), name.encode('ascii')) == 999:
+            self.skipTest("Compiler does not support signed short bitfields")
+        for i in range(256):
+            for name in "MNOPQRS":
+                b = BITS()
+                setattr(b, name, i)
+                self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii')))
+
+signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong)
+unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong)
+int_types = unsigned_int_types + signed_int_types
+
+class BitFieldTest(unittest.TestCase):
+
+    def test_longlong(self):
+        class X(Structure):
+            _fields_ = [("a", c_longlong, 1),
+                        ("b", c_longlong, 62),
+                        ("c", c_longlong, 1)]
+
+        self.assertEqual(sizeof(X), sizeof(c_longlong))
+        x = X()
+        x.a, x.b, x.c = -1, 7, -1
+        self.assertEqual((x.a, x.b, x.c), (-1, 7, -1))
+
+    def test_ulonglong(self):
+        class X(Structure):
+            _fields_ = [("a", c_ulonglong, 1),
+                        ("b", c_ulonglong, 62),
+                        ("c", c_ulonglong, 1)]
+
+        self.assertEqual(sizeof(X), sizeof(c_longlong))
+        x = X()
+        self.assertEqual((x.a, x.b, x.c), (0, 0, 0))
+        x.a, x.b, x.c = 7, 7, 7
+        self.assertEqual((x.a, x.b, x.c), (1, 7, 1))
+
+    def test_signed(self):
+        for c_typ in signed_int_types:
+            class X(Structure):
+                _fields_ = [("dummy", c_typ),
+                            ("a", c_typ, 3),
+                            ("b", c_typ, 3),
+                            ("c", c_typ, 1)]
+            self.assertEqual(sizeof(X), sizeof(c_typ)*2)
+
+            x = X()
+            self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0))
+            x.a = -1
+            self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0))
+            x.a, x.b = 0, -1
+            self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0))
+
+
+    def test_unsigned(self):
+        for c_typ in unsigned_int_types:
+            class X(Structure):
+                _fields_ = [("a", c_typ, 3),
+                            ("b", c_typ, 3),
+                            ("c", c_typ, 1)]
+            self.assertEqual(sizeof(X), sizeof(c_typ))
+
+            x = X()
+            self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0))
+            x.a = -1
+            self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0))
+            x.a, x.b = 0, -1
+            self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0))
+
+
+    def fail_fields(self, *fields):
+        return self.get_except(type(Structure), "X", (),
+                               {"_fields_": fields})
+
+    def test_nonint_types(self):
+        # bit fields are not allowed on non-integer types.
+        result = self.fail_fields(("a", c_char_p, 1))
+        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char_p'))
+
+        result = self.fail_fields(("a", c_void_p, 1))
+        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_void_p'))
+
+        if c_int != c_long:
+            result = self.fail_fields(("a", POINTER(c_int), 1))
+            self.assertEqual(result, (TypeError, 'bit fields not allowed for type LP_c_int'))
+
+        result = self.fail_fields(("a", c_char, 1))
+        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char'))
+
+        class Dummy(Structure):
+            _fields_ = []
+
+        result = self.fail_fields(("a", Dummy, 1))
+        self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy'))
+
+    @need_symbol('c_wchar')
+    def test_c_wchar(self):
+        result = self.fail_fields(("a", c_wchar, 1))
+        self.assertEqual(result,
+                (TypeError, 'bit fields not allowed for type c_wchar'))
+
+    def test_single_bitfield_size(self):
+        for c_typ in int_types:
+            result = self.fail_fields(("a", c_typ, -1))
+            self.assertEqual(result, (ValueError, 'number of bits invalid for bit field'))
+
+            result = self.fail_fields(("a", c_typ, 0))
+            self.assertEqual(result, (ValueError, 'number of bits invalid for bit field'))
+
+            class X(Structure):
+                _fields_ = [("a", c_typ, 1)]
+            self.assertEqual(sizeof(X), sizeof(c_typ))
+
+            class X(Structure):
+                _fields_ = [("a", c_typ, sizeof(c_typ)*8)]
+            self.assertEqual(sizeof(X), sizeof(c_typ))
+
+            result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1))
+            self.assertEqual(result, (ValueError, 'number of bits invalid for bit field'))
+
+    def test_multi_bitfields_size(self):
+        class X(Structure):
+            _fields_ = [("a", c_short, 1),
+                        ("b", c_short, 14),
+                        ("c", c_short, 1)]
+        self.assertEqual(sizeof(X), sizeof(c_short))
+
+        class X(Structure):
+            _fields_ = [("a", c_short, 1),
+                        ("a1", c_short),
+                        ("b", c_short, 14),
+                        ("c", c_short, 1)]
+        self.assertEqual(sizeof(X), sizeof(c_short)*3)
+        self.assertEqual(X.a.offset, 0)
+        self.assertEqual(X.a1.offset, sizeof(c_short))
+        self.assertEqual(X.b.offset, sizeof(c_short)*2)
+        self.assertEqual(X.c.offset, sizeof(c_short)*2)
+
+        class X(Structure):
+            _fields_ = [("a", c_short, 3),
+                        ("b", c_short, 14),
+                        ("c", c_short, 14)]
+        self.assertEqual(sizeof(X), sizeof(c_short)*3)
+        self.assertEqual(X.a.offset, sizeof(c_short)*0)
+        self.assertEqual(X.b.offset, sizeof(c_short)*1)
+        self.assertEqual(X.c.offset, sizeof(c_short)*2)
+
+
+    def get_except(self, func, *args, **kw):
+        try:
+            func(*args, **kw)
+        except Exception as detail:
+            return detail.__class__, str(detail)
+
+    def test_mixed_1(self):
+        class X(Structure):
+            _fields_ = [("a", c_byte, 4),
+                        ("b", c_int, 4)]
+        if os.name == "nt":
+            self.assertEqual(sizeof(X), sizeof(c_int)*2)
+        else:
+            self.assertEqual(sizeof(X), sizeof(c_int))
+
+    def test_mixed_2(self):
+        class X(Structure):
+            _fields_ = [("a", c_byte, 4),
+                        ("b", c_int, 32)]
+        self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int))
+
+    def test_mixed_3(self):
+        class X(Structure):
+            _fields_ = [("a", c_byte, 4),
+                        ("b", c_ubyte, 4)]
+        self.assertEqual(sizeof(X), sizeof(c_byte))
+
+    def test_mixed_4(self):
+        class X(Structure):
+            _fields_ = [("a", c_short, 4),
+                        ("b", c_short, 4),
+                        ("c", c_int, 24),
+                        ("d", c_short, 4),
+                        ("e", c_short, 4),
+                        ("f", c_int, 24)]
+        # MSVC does NOT combine c_short and c_int into one field, GCC
+        # does (unless GCC is run with '-mms-bitfields' which
+        # produces code compatible with MSVC).
+        if os.name == "nt":
+            self.assertEqual(sizeof(X), sizeof(c_int) * 4)
+        else:
+            self.assertEqual(sizeof(X), sizeof(c_int) * 2)
+
+    def test_anon_bitfields(self):
+        # anonymous bit-fields gave a strange error message
+        class X(Structure):
+            _fields_ = [("a", c_byte, 4),
+                        ("b", c_ubyte, 4)]
+        class Y(Structure):
+            _anonymous_ = ["_"]
+            _fields_ = [("_", X)]
+
+    @need_symbol('c_uint32')
+    def test_uint32(self):
+        class X(Structure):
+            _fields_ = [("a", c_uint32, 32)]
+        x = X()
+        x.a = 10
+        self.assertEqual(x.a, 10)
+        x.a = 0xFDCBA987
+        self.assertEqual(x.a, 0xFDCBA987)
+
+    @need_symbol('c_uint64')
+    def test_uint64(self):
+        class X(Structure):
+            _fields_ = [("a", c_uint64, 64)]
+        x = X()
+        x.a = 10
+        self.assertEqual(x.a, 10)
+        x.a = 0xFEDCBA9876543211
+        self.assertEqual(x.a, 0xFEDCBA9876543211)
+
+    @need_symbol('c_uint32')
+    def test_uint32_swap_little_endian(self):
+        # Issue #23319
+        class Little(LittleEndianStructure):
+            _fields_ = [("a", c_uint32, 24),
+                        ("b", c_uint32, 4),
+                        ("c", c_uint32, 4)]
+        b = bytearray(4)
+        x = Little.from_buffer(b)
+        x.a = 0xabcdef
+        x.b = 1
+        x.c = 2
+        self.assertEqual(b, b'\xef\xcd\xab\x21')
+
+    @need_symbol('c_uint32')
+    def test_uint32_swap_big_endian(self):
+        # Issue #23319
+        class Big(BigEndianStructure):
+            _fields_ = [("a", c_uint32, 24),
+                        ("b", c_uint32, 4),
+                        ("c", c_uint32, 4)]
+        b = bytearray(4)
+        x = Big.from_buffer(b)
+        x.a = 0xabcdef
+        x.b = 1
+        x.c = 2
+        self.assertEqual(b, b'\xab\xcd\xef\x12')
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_buffers.py

@@ -0,0 +1,73 @@
+from ctypes import *
+from ctypes.test import need_symbol
+import unittest
+
+class StringBufferTestCase(unittest.TestCase):
+
+    def test_buffer(self):
+        b = create_string_buffer(32)
+        self.assertEqual(len(b), 32)
+        self.assertEqual(sizeof(b), 32 * sizeof(c_char))
+        self.assertIs(type(b[0]), bytes)
+
+        b = create_string_buffer(b"abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_char))
+        self.assertIs(type(b[0]), bytes)
+        self.assertEqual(b[0], b"a")
+        self.assertEqual(b[:], b"abc\0")
+        self.assertEqual(b[::], b"abc\0")
+        self.assertEqual(b[::-1], b"\0cba")
+        self.assertEqual(b[::2], b"ac")
+        self.assertEqual(b[::5], b"a")
+
+        self.assertRaises(TypeError, create_string_buffer, "abc")
+
+    def test_buffer_interface(self):
+        self.assertEqual(len(bytearray(create_string_buffer(0))), 0)
+        self.assertEqual(len(bytearray(create_string_buffer(1))), 1)
+
+    @need_symbol('c_wchar')
+    def test_unicode_buffer(self):
+        b = create_unicode_buffer(32)
+        self.assertEqual(len(b), 32)
+        self.assertEqual(sizeof(b), 32 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), str)
+
+        b = create_unicode_buffer("abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), str)
+        self.assertEqual(b[0], "a")
+        self.assertEqual(b[:], "abc\0")
+        self.assertEqual(b[::], "abc\0")
+        self.assertEqual(b[::-1], "\0cba")
+        self.assertEqual(b[::2], "ac")
+        self.assertEqual(b[::5], "a")
+
+        self.assertRaises(TypeError, create_unicode_buffer, b"abc")
+
+    @need_symbol('c_wchar')
+    def test_unicode_conversion(self):
+        b = create_unicode_buffer("abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), str)
+        self.assertEqual(b[0], "a")
+        self.assertEqual(b[:], "abc\0")
+        self.assertEqual(b[::], "abc\0")
+        self.assertEqual(b[::-1], "\0cba")
+        self.assertEqual(b[::2], "ac")
+        self.assertEqual(b[::5], "a")
+
+    @need_symbol('c_wchar')
+    def test_create_unicode_buffer_non_bmp(self):
+        expected = 5 if sizeof(c_wchar) == 2 else 3
+        for s in '\U00010000\U00100000', '\U00010000\U0010ffff':
+            b = create_unicode_buffer(s)
+            self.assertEqual(len(b), expected)
+            self.assertEqual(b[-1], '\0')
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_bytes.py

@@ -0,0 +1,66 @@
+"""Test where byte objects are accepted"""
+import unittest
+import sys
+from ctypes import *
+
+class BytesTest(unittest.TestCase):
+    def test_c_char(self):
+        x = c_char(b"x")
+        self.assertRaises(TypeError, c_char, "x")
+        x.value = b"y"
+        with self.assertRaises(TypeError):
+            x.value = "y"
+        c_char.from_param(b"x")
+        self.assertRaises(TypeError, c_char.from_param, "x")
+        self.assertIn('xbd', repr(c_char.from_param(b"\xbd")))
+        (c_char * 3)(b"a", b"b", b"c")
+        self.assertRaises(TypeError, c_char * 3, "a", "b", "c")
+
+    def test_c_wchar(self):
+        x = c_wchar("x")
+        self.assertRaises(TypeError, c_wchar, b"x")
+        x.value = "y"
+        with self.assertRaises(TypeError):
+            x.value = b"y"
+        c_wchar.from_param("x")
+        self.assertRaises(TypeError, c_wchar.from_param, b"x")
+        (c_wchar * 3)("a", "b", "c")
+        self.assertRaises(TypeError, c_wchar * 3, b"a", b"b", b"c")
+
+    def test_c_char_p(self):
+        c_char_p(b"foo bar")
+        self.assertRaises(TypeError, c_char_p, "foo bar")
+
+    def test_c_wchar_p(self):
+        c_wchar_p("foo bar")
+        self.assertRaises(TypeError, c_wchar_p, b"foo bar")
+
+    def test_struct(self):
+        class X(Structure):
+            _fields_ = [("a", c_char * 3)]
+
+        x = X(b"abc")
+        self.assertRaises(TypeError, X, "abc")
+        self.assertEqual(x.a, b"abc")
+        self.assertEqual(type(x.a), bytes)
+
+    def test_struct_W(self):
+        class X(Structure):
+            _fields_ = [("a", c_wchar * 3)]
+
+        x = X("abc")
+        self.assertRaises(TypeError, X, b"abc")
+        self.assertEqual(x.a, "abc")
+        self.assertEqual(type(x.a), str)
+
+    @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
+    def test_BSTR(self):
+        from _ctypes import _SimpleCData
+        class BSTR(_SimpleCData):
+            _type_ = "X"
+
+        BSTR("abc")
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_byteswap.py

@@ -0,0 +1,364 @@
+import sys, unittest, struct, math, ctypes
+from binascii import hexlify
+
+from ctypes import *
+
+def bin(s):
+    return hexlify(memoryview(s)).decode().upper()
+
+# Each *simple* type that supports different byte orders has an
+# __ctype_be__ attribute that specifies the same type in BIG ENDIAN
+# byte order, and a __ctype_le__ attribute that is the same type in
+# LITTLE ENDIAN byte order.
+#
+# For Structures and Unions, these types are created on demand.
+
+class Test(unittest.TestCase):
+    @unittest.skip('test disabled')
+    def test_X(self):
+        print(sys.byteorder, file=sys.stderr)
+        for i in range(32):
+            bits = BITS()
+            setattr(bits, "i%s" % i, 1)
+            dump(bits)
+
+    def test_slots(self):
+        class BigPoint(BigEndianStructure):
+            __slots__ = ()
+            _fields_ = [("x", c_int), ("y", c_int)]
+
+        class LowPoint(LittleEndianStructure):
+            __slots__ = ()
+            _fields_ = [("x", c_int), ("y", c_int)]
+
+        big = BigPoint()
+        little = LowPoint()
+        big.x = 4
+        big.y = 2
+        little.x = 2
+        little.y = 4
+        with self.assertRaises(AttributeError):
+            big.z = 42
+        with self.assertRaises(AttributeError):
+            little.z = 24
+
+    def test_endian_short(self):
+        if sys.byteorder == "little":
+            self.assertIs(c_short.__ctype_le__, c_short)
+            self.assertIs(c_short.__ctype_be__.__ctype_le__, c_short)
+        else:
+            self.assertIs(c_short.__ctype_be__, c_short)
+            self.assertIs(c_short.__ctype_le__.__ctype_be__, c_short)
+        s = c_short.__ctype_be__(0x1234)
+        self.assertEqual(bin(struct.pack(">h", 0x1234)), "1234")
+        self.assertEqual(bin(s), "1234")
+        self.assertEqual(s.value, 0x1234)
+
+        s = c_short.__ctype_le__(0x1234)
+        self.assertEqual(bin(struct.pack("<h", 0x1234)), "3412")
+        self.assertEqual(bin(s), "3412")
+        self.assertEqual(s.value, 0x1234)
+
+        s = c_ushort.__ctype_be__(0x1234)
+        self.assertEqual(bin(struct.pack(">h", 0x1234)), "1234")
+        self.assertEqual(bin(s), "1234")
+        self.assertEqual(s.value, 0x1234)
+
+        s = c_ushort.__ctype_le__(0x1234)
+        self.assertEqual(bin(struct.pack("<h", 0x1234)), "3412")
+        self.assertEqual(bin(s), "3412")
+        self.assertEqual(s.value, 0x1234)
+
+    def test_endian_int(self):
+        if sys.byteorder == "little":
+            self.assertIs(c_int.__ctype_le__, c_int)
+            self.assertIs(c_int.__ctype_be__.__ctype_le__, c_int)
+        else:
+            self.assertIs(c_int.__ctype_be__, c_int)
+            self.assertIs(c_int.__ctype_le__.__ctype_be__, c_int)
+
+        s = c_int.__ctype_be__(0x12345678)
+        self.assertEqual(bin(struct.pack(">i", 0x12345678)), "12345678")
+        self.assertEqual(bin(s), "12345678")
+        self.assertEqual(s.value, 0x12345678)
+
+        s = c_int.__ctype_le__(0x12345678)
+        self.assertEqual(bin(struct.pack("<i", 0x12345678)), "78563412")
+        self.assertEqual(bin(s), "78563412")
+        self.assertEqual(s.value, 0x12345678)
+
+        s = c_uint.__ctype_be__(0x12345678)
+        self.assertEqual(bin(struct.pack(">I", 0x12345678)), "12345678")
+        self.assertEqual(bin(s), "12345678")
+        self.assertEqual(s.value, 0x12345678)
+
+        s = c_uint.__ctype_le__(0x12345678)
+        self.assertEqual(bin(struct.pack("<I", 0x12345678)), "78563412")
+        self.assertEqual(bin(s), "78563412")
+        self.assertEqual(s.value, 0x12345678)
+
+    def test_endian_longlong(self):
+        if sys.byteorder == "little":
+            self.assertIs(c_longlong.__ctype_le__, c_longlong)
+            self.assertIs(c_longlong.__ctype_be__.__ctype_le__, c_longlong)
+        else:
+            self.assertIs(c_longlong.__ctype_be__, c_longlong)
+            self.assertIs(c_longlong.__ctype_le__.__ctype_be__, c_longlong)
+
+        s = c_longlong.__ctype_be__(0x1234567890ABCDEF)
+        self.assertEqual(bin(struct.pack(">q", 0x1234567890ABCDEF)), "1234567890ABCDEF")
+        self.assertEqual(bin(s), "1234567890ABCDEF")
+        self.assertEqual(s.value, 0x1234567890ABCDEF)
+
+        s = c_longlong.__ctype_le__(0x1234567890ABCDEF)
+        self.assertEqual(bin(struct.pack("<q", 0x1234567890ABCDEF)), "EFCDAB9078563412")
+        self.assertEqual(bin(s), "EFCDAB9078563412")
+        self.assertEqual(s.value, 0x1234567890ABCDEF)
+
+        s = c_ulonglong.__ctype_be__(0x1234567890ABCDEF)
+        self.assertEqual(bin(struct.pack(">Q", 0x1234567890ABCDEF)), "1234567890ABCDEF")
+        self.assertEqual(bin(s), "1234567890ABCDEF")
+        self.assertEqual(s.value, 0x1234567890ABCDEF)
+
+        s = c_ulonglong.__ctype_le__(0x1234567890ABCDEF)
+        self.assertEqual(bin(struct.pack("<Q", 0x1234567890ABCDEF)), "EFCDAB9078563412")
+        self.assertEqual(bin(s), "EFCDAB9078563412")
+        self.assertEqual(s.value, 0x1234567890ABCDEF)
+
+    def test_endian_float(self):
+        if sys.byteorder == "little":
+            self.assertIs(c_float.__ctype_le__, c_float)
+            self.assertIs(c_float.__ctype_be__.__ctype_le__, c_float)
+        else:
+            self.assertIs(c_float.__ctype_be__, c_float)
+            self.assertIs(c_float.__ctype_le__.__ctype_be__, c_float)
+        s = c_float(math.pi)
+        self.assertEqual(bin(struct.pack("f", math.pi)), bin(s))
+        # Hm, what's the precision of a float compared to a double?
+        self.assertAlmostEqual(s.value, math.pi, places=6)
+        s = c_float.__ctype_le__(math.pi)
+        self.assertAlmostEqual(s.value, math.pi, places=6)
+        self.assertEqual(bin(struct.pack("<f", math.pi)), bin(s))
+        s = c_float.__ctype_be__(math.pi)
+        self.assertAlmostEqual(s.value, math.pi, places=6)
+        self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
+
+    def test_endian_double(self):
+        if sys.byteorder == "little":
+            self.assertIs(c_double.__ctype_le__, c_double)
+            self.assertIs(c_double.__ctype_be__.__ctype_le__, c_double)
+        else:
+            self.assertIs(c_double.__ctype_be__, c_double)
+            self.assertIs(c_double.__ctype_le__.__ctype_be__, c_double)
+        s = c_double(math.pi)
+        self.assertEqual(s.value, math.pi)
+        self.assertEqual(bin(struct.pack("d", math.pi)), bin(s))
+        s = c_double.__ctype_le__(math.pi)
+        self.assertEqual(s.value, math.pi)
+        self.assertEqual(bin(struct.pack("<d", math.pi)), bin(s))
+        s = c_double.__ctype_be__(math.pi)
+        self.assertEqual(s.value, math.pi)
+        self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s))
+
+    def test_endian_other(self):
+        self.assertIs(c_byte.__ctype_le__, c_byte)
+        self.assertIs(c_byte.__ctype_be__, c_byte)
+
+        self.assertIs(c_ubyte.__ctype_le__, c_ubyte)
+        self.assertIs(c_ubyte.__ctype_be__, c_ubyte)
+
+        self.assertIs(c_char.__ctype_le__, c_char)
+        self.assertIs(c_char.__ctype_be__, c_char)
+
+    def test_struct_fields_unsupported_byte_order(self):
+
+        fields = [
+            ("a", c_ubyte),
+            ("b", c_byte),
+            ("c", c_short),
+            ("d", c_ushort),
+            ("e", c_int),
+            ("f", c_uint),
+            ("g", c_long),
+            ("h", c_ulong),
+            ("i", c_longlong),
+            ("k", c_ulonglong),
+            ("l", c_float),
+            ("m", c_double),
+            ("n", c_char),
+            ("b1", c_byte, 3),
+            ("b2", c_byte, 3),
+            ("b3", c_byte, 2),
+            ("a", c_int * 3 * 3 * 3)
+        ]
+
+        # these fields do not support different byte order:
+        for typ in c_wchar, c_void_p, POINTER(c_int):
+            with self.assertRaises(TypeError):
+                class T(BigEndianStructure if sys.byteorder == "little" else LittleEndianStructure):
+                    _fields_ = fields + [("x", typ)]
+
+
+    def test_struct_struct(self):
+        # nested structures with different byteorders
+
+        # create nested structures with given byteorders and set memory to data
+
+        for nested, data in (
+            (BigEndianStructure, b'\0\0\0\1\0\0\0\2'),
+            (LittleEndianStructure, b'\1\0\0\0\2\0\0\0'),
+        ):
+            for parent in (
+                BigEndianStructure,
+                LittleEndianStructure,
+                Structure,
+            ):
+                class NestedStructure(nested):
+                    _fields_ = [("x", c_uint32),
+                                ("y", c_uint32)]
+
+                class TestStructure(parent):
+                    _fields_ = [("point", NestedStructure)]
+
+                self.assertEqual(len(data), sizeof(TestStructure))
+                ptr = POINTER(TestStructure)
+                s = cast(data, ptr)[0]
+                del ctypes._pointer_type_cache[TestStructure]
+                self.assertEqual(s.point.x, 1)
+                self.assertEqual(s.point.y, 2)
+
+    def test_struct_field_alignment(self):
+        # standard packing in struct uses no alignment.
+        # So, we have to align using pad bytes.
+        #
+        # Unaligned accesses will crash Python (on those platforms that
+        # don't allow it, like sparc solaris).
+        if sys.byteorder == "little":
+            base = BigEndianStructure
+            fmt = ">bxhid"
+        else:
+            base = LittleEndianStructure
+            fmt = "<bxhid"
+
+        class S(base):
+            _fields_ = [("b", c_byte),
+                        ("h", c_short),
+                        ("i", c_int),
+                        ("d", c_double)]
+
+        s1 = S(0x12, 0x1234, 0x12345678, 3.14)
+        s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
+        self.assertEqual(bin(s1), bin(s2))
+
+    def test_unaligned_nonnative_struct_fields(self):
+        if sys.byteorder == "little":
+            base = BigEndianStructure
+            fmt = ">b h xi xd"
+        else:
+            base = LittleEndianStructure
+            fmt = "<b h xi xd"
+
+        class S(base):
+            _pack_ = 1
+            _fields_ = [("b", c_byte),
+                        ("h", c_short),
+
+                        ("_1", c_byte),
+                        ("i", c_int),
+
+                        ("_2", c_byte),
+                        ("d", c_double)]
+
+        s1 = S()
+        s1.b = 0x12
+        s1.h = 0x1234
+        s1.i = 0x12345678
+        s1.d = 3.14
+        s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
+        self.assertEqual(bin(s1), bin(s2))
+
+    def test_unaligned_native_struct_fields(self):
+        if sys.byteorder == "little":
+            fmt = "<b h xi xd"
+        else:
+            base = LittleEndianStructure
+            fmt = ">b h xi xd"
+
+        class S(Structure):
+            _pack_ = 1
+            _fields_ = [("b", c_byte),
+
+                        ("h", c_short),
+
+                        ("_1", c_byte),
+                        ("i", c_int),
+
+                        ("_2", c_byte),
+                        ("d", c_double)]
+
+        s1 = S()
+        s1.b = 0x12
+        s1.h = 0x1234
+        s1.i = 0x12345678
+        s1.d = 3.14
+        s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
+        self.assertEqual(bin(s1), bin(s2))
+
+    def test_union_fields_unsupported_byte_order(self):
+
+        fields = [
+            ("a", c_ubyte),
+            ("b", c_byte),
+            ("c", c_short),
+            ("d", c_ushort),
+            ("e", c_int),
+            ("f", c_uint),
+            ("g", c_long),
+            ("h", c_ulong),
+            ("i", c_longlong),
+            ("k", c_ulonglong),
+            ("l", c_float),
+            ("m", c_double),
+            ("n", c_char),
+            ("b1", c_byte, 3),
+            ("b2", c_byte, 3),
+            ("b3", c_byte, 2),
+            ("a", c_int * 3 * 3 * 3)
+        ]
+
+        # these fields do not support different byte order:
+        for typ in c_wchar, c_void_p, POINTER(c_int):
+            with self.assertRaises(TypeError):
+                class T(BigEndianUnion if sys.byteorder == "little" else LittleEndianUnion):
+                    _fields_ = fields + [("x", typ)]
+
+    def test_union_struct(self):
+        # nested structures in unions with different byteorders
+
+        # create nested structures in unions with given byteorders and set memory to data
+
+        for nested, data in (
+            (BigEndianStructure, b'\0\0\0\1\0\0\0\2'),
+            (LittleEndianStructure, b'\1\0\0\0\2\0\0\0'),
+        ):
+            for parent in (
+                BigEndianUnion,
+                LittleEndianUnion,
+                Union,
+            ):
+                class NestedStructure(nested):
+                    _fields_ = [("x", c_uint32),
+                                ("y", c_uint32)]
+
+                class TestUnion(parent):
+                    _fields_ = [("point", NestedStructure)]
+
+                self.assertEqual(len(data), sizeof(TestUnion))
+                ptr = POINTER(TestUnion)
+                s = cast(data, ptr)[0]
+                del ctypes._pointer_type_cache[TestUnion]
+                self.assertEqual(s.point.x, 1)
+                self.assertEqual(s.point.y, 2)
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_callbacks.py

@@ -0,0 +1,333 @@
+import functools
+import unittest
+from test import support
+
+from ctypes import *
+from ctypes.test import need_symbol
+from _ctypes import CTYPES_MAX_ARGCOUNT
+import _ctypes_test
+
+class Callbacks(unittest.TestCase):
+    functype = CFUNCTYPE
+
+##    def tearDown(self):
+##        import gc
+##        gc.collect()
+
+    def callback(self, *args):
+        self.got_args = args
+        return args[-1]
+
+    def check_type(self, typ, arg):
+        PROTO = self.functype.__func__(typ, typ)
+        result = PROTO(self.callback)(arg)
+        if typ == c_float:
+            self.assertAlmostEqual(result, arg, places=5)
+        else:
+            self.assertEqual(self.got_args, (arg,))
+            self.assertEqual(result, arg)
+
+        PROTO = self.functype.__func__(typ, c_byte, typ)
+        result = PROTO(self.callback)(-3, arg)
+        if typ == c_float:
+            self.assertAlmostEqual(result, arg, places=5)
+        else:
+            self.assertEqual(self.got_args, (-3, arg))
+            self.assertEqual(result, arg)
+
+    ################
+
+    def test_byte(self):
+        self.check_type(c_byte, 42)
+        self.check_type(c_byte, -42)
+
+    def test_ubyte(self):
+        self.check_type(c_ubyte, 42)
+
+    def test_short(self):
+        self.check_type(c_short, 42)
+        self.check_type(c_short, -42)
+
+    def test_ushort(self):
+        self.check_type(c_ushort, 42)
+
+    def test_int(self):
+        self.check_type(c_int, 42)
+        self.check_type(c_int, -42)
+
+    def test_uint(self):
+        self.check_type(c_uint, 42)
+
+    def test_long(self):
+        self.check_type(c_long, 42)
+        self.check_type(c_long, -42)
+
+    def test_ulong(self):
+        self.check_type(c_ulong, 42)
+
+    @need_symbol('c_longlong')
+    def test_longlong(self):
+        self.check_type(c_longlong, 42)
+        self.check_type(c_longlong, -42)
+
+    @need_symbol('c_ulonglong')
+    def test_ulonglong(self):
+        self.check_type(c_ulonglong, 42)
+
+    def test_float(self):
+        # only almost equal: double -> float -> double
+        import math
+        self.check_type(c_float, math.e)
+        self.check_type(c_float, -math.e)
+
+    def test_double(self):
+        self.check_type(c_double, 3.14)
+        self.check_type(c_double, -3.14)
+
+    @need_symbol('c_longdouble')
+    def test_longdouble(self):
+        self.check_type(c_longdouble, 3.14)
+        self.check_type(c_longdouble, -3.14)
+
+    def test_char(self):
+        self.check_type(c_char, b"x")
+        self.check_type(c_char, b"a")
+
+    # disabled: would now (correctly) raise a RuntimeWarning about
+    # a memory leak.  A callback function cannot return a non-integral
+    # C type without causing a memory leak.
+    @unittest.skip('test disabled')
+    def test_char_p(self):
+        self.check_type(c_char_p, "abc")
+        self.check_type(c_char_p, "def")
+
+    def test_pyobject(self):
+        o = ()
+        from sys import getrefcount as grc
+        for o in (), [], object():
+            initial = grc(o)
+            # This call leaks a reference to 'o'...
+            self.check_type(py_object, o)
+            before = grc(o)
+            # ...but this call doesn't leak any more.  Where is the refcount?
+            self.check_type(py_object, o)
+            after = grc(o)
+            self.assertEqual((after, o), (before, o))
+
+    def test_unsupported_restype_1(self):
+        # Only "fundamental" result types are supported for callback
+        # functions, the type must have a non-NULL stgdict->setfunc.
+        # POINTER(c_double), for example, is not supported.
+
+        prototype = self.functype.__func__(POINTER(c_double))
+        # The type is checked when the prototype is called
+        self.assertRaises(TypeError, prototype, lambda: None)
+
+    def test_unsupported_restype_2(self):
+        prototype = self.functype.__func__(object)
+        self.assertRaises(TypeError, prototype, lambda: None)
+
+    def test_issue_7959(self):
+        proto = self.functype.__func__(None)
+
+        class X(object):
+            def func(self): pass
+            def __init__(self):
+                self.v = proto(self.func)
+
+        import gc
+        for i in range(32):
+            X()
+        gc.collect()
+        live = [x for x in gc.get_objects()
+                if isinstance(x, X)]
+        self.assertEqual(len(live), 0)
+
+    def test_issue12483(self):
+        import gc
+        class Nasty:
+            def __del__(self):
+                gc.collect()
+        CFUNCTYPE(None)(lambda x=Nasty(): None)
+
+
+@need_symbol('WINFUNCTYPE')
+class StdcallCallbacks(Callbacks):
+    try:
+        functype = WINFUNCTYPE
+    except NameError:
+        pass
+
+################################################################
+
+class SampleCallbacksTestCase(unittest.TestCase):
+
+    def test_integrate(self):
+        # Derived from some then non-working code, posted by David Foster
+        dll = CDLL(_ctypes_test.__file__)
+
+        # The function prototype called by 'integrate': double func(double);
+        CALLBACK = CFUNCTYPE(c_double, c_double)
+
+        # The integrate function itself, exposed from the _ctypes_test dll
+        integrate = dll.integrate
+        integrate.argtypes = (c_double, c_double, CALLBACK, c_long)
+        integrate.restype = c_double
+
+        def func(x):
+            return x**2
+
+        result = integrate(0.0, 1.0, CALLBACK(func), 10)
+        diff = abs(result - 1./3.)
+
+        self.assertLess(diff, 0.01, "%s not less than 0.01" % diff)
+
+    def test_issue_8959_a(self):
+        from ctypes.util import find_library
+        libc_path = find_library("c")
+        if not libc_path:
+            self.skipTest('could not find libc')
+        libc = CDLL(libc_path)
+
+        @CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int))
+        def cmp_func(a, b):
+            return a[0] - b[0]
+
+        array = (c_int * 5)(5, 1, 99, 7, 33)
+
+        libc.qsort(array, len(array), sizeof(c_int), cmp_func)
+        self.assertEqual(array[:], [1, 5, 7, 33, 99])
+
+    @need_symbol('WINFUNCTYPE')
+    def test_issue_8959_b(self):
+        from ctypes.wintypes import BOOL, HWND, LPARAM
+        global windowCount
+        windowCount = 0
+
+        @WINFUNCTYPE(BOOL, HWND, LPARAM)
+        def EnumWindowsCallbackFunc(hwnd, lParam):
+            global windowCount
+            windowCount += 1
+            return True #Allow windows to keep enumerating
+
+        windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
+
+    def test_callback_register_int(self):
+        # Issue #8275: buggy handling of callback args under Win64
+        # NOTE: should be run on release builds as well
+        dll = CDLL(_ctypes_test.__file__)
+        CALLBACK = CFUNCTYPE(c_int, c_int, c_int, c_int, c_int, c_int)
+        # All this function does is call the callback with its args squared
+        func = dll._testfunc_cbk_reg_int
+        func.argtypes = (c_int, c_int, c_int, c_int, c_int, CALLBACK)
+        func.restype = c_int
+
+        def callback(a, b, c, d, e):
+            return a + b + c + d + e
+
+        result = func(2, 3, 4, 5, 6, CALLBACK(callback))
+        self.assertEqual(result, callback(2*2, 3*3, 4*4, 5*5, 6*6))
+
+    def test_callback_register_double(self):
+        # Issue #8275: buggy handling of callback args under Win64
+        # NOTE: should be run on release builds as well
+        dll = CDLL(_ctypes_test.__file__)
+        CALLBACK = CFUNCTYPE(c_double, c_double, c_double, c_double,
+                             c_double, c_double)
+        # All this function does is call the callback with its args squared
+        func = dll._testfunc_cbk_reg_double
+        func.argtypes = (c_double, c_double, c_double,
+                         c_double, c_double, CALLBACK)
+        func.restype = c_double
+
+        def callback(a, b, c, d, e):
+            return a + b + c + d + e
+
+        result = func(1.1, 2.2, 3.3, 4.4, 5.5, CALLBACK(callback))
+        self.assertEqual(result,
+                         callback(1.1*1.1, 2.2*2.2, 3.3*3.3, 4.4*4.4, 5.5*5.5))
+
+    def test_callback_large_struct(self):
+        class Check: pass
+
+        # This should mirror the structure in Modules/_ctypes/_ctypes_test.c
+        class X(Structure):
+            _fields_ = [
+                ('first', c_ulong),
+                ('second', c_ulong),
+                ('third', c_ulong),
+            ]
+
+        def callback(check, s):
+            check.first = s.first
+            check.second = s.second
+            check.third = s.third
+            # See issue #29565.
+            # The structure should be passed by value, so
+            # any changes to it should not be reflected in
+            # the value passed
+            s.first = s.second = s.third = 0x0badf00d
+
+        check = Check()
+        s = X()
+        s.first = 0xdeadbeef
+        s.second = 0xcafebabe
+        s.third = 0x0bad1dea
+
+        CALLBACK = CFUNCTYPE(None, X)
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_cbk_large_struct
+        func.argtypes = (X, CALLBACK)
+        func.restype = None
+        # the function just calls the callback with the passed structure
+        func(s, CALLBACK(functools.partial(callback, check)))
+        self.assertEqual(check.first, s.first)
+        self.assertEqual(check.second, s.second)
+        self.assertEqual(check.third, s.third)
+        self.assertEqual(check.first, 0xdeadbeef)
+        self.assertEqual(check.second, 0xcafebabe)
+        self.assertEqual(check.third, 0x0bad1dea)
+        # See issue #29565.
+        # Ensure that the original struct is unchanged.
+        self.assertEqual(s.first, check.first)
+        self.assertEqual(s.second, check.second)
+        self.assertEqual(s.third, check.third)
+
+    def test_callback_too_many_args(self):
+        def func(*args):
+            return len(args)
+
+        # valid call with nargs <= CTYPES_MAX_ARGCOUNT
+        proto = CFUNCTYPE(c_int, *(c_int,) * CTYPES_MAX_ARGCOUNT)
+        cb = proto(func)
+        args1 = (1,) * CTYPES_MAX_ARGCOUNT
+        self.assertEqual(cb(*args1), CTYPES_MAX_ARGCOUNT)
+
+        # invalid call with nargs > CTYPES_MAX_ARGCOUNT
+        args2 = (1,) * (CTYPES_MAX_ARGCOUNT + 1)
+        with self.assertRaises(ArgumentError):
+            cb(*args2)
+
+        # error when creating the type with too many arguments
+        with self.assertRaises(ArgumentError):
+            CFUNCTYPE(c_int, *(c_int,) * (CTYPES_MAX_ARGCOUNT + 1))
+
+    def test_convert_result_error(self):
+        def func():
+            return ("tuple",)
+
+        proto = CFUNCTYPE(c_int)
+        ctypes_func = proto(func)
+        with support.catch_unraisable_exception() as cm:
+            # don't test the result since it is an uninitialized value
+            result = ctypes_func()
+
+            self.assertIsInstance(cm.unraisable.exc_value, TypeError)
+            self.assertEqual(cm.unraisable.err_msg,
+                             "Exception ignored on converting result "
+                             "of ctypes callback function")
+            self.assertIs(cm.unraisable.object, func)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_cast.py

@@ -0,0 +1,99 @@
+from ctypes import *
+from ctypes.test import need_symbol
+import unittest
+import sys
+
+class Test(unittest.TestCase):
+
+    def test_array2pointer(self):
+        array = (c_int * 3)(42, 17, 2)
+
+        # casting an array to a pointer works.
+        ptr = cast(array, POINTER(c_int))
+        self.assertEqual([ptr[i] for i in range(3)], [42, 17, 2])
+
+        if 2*sizeof(c_short) == sizeof(c_int):
+            ptr = cast(array, POINTER(c_short))
+            if sys.byteorder == "little":
+                self.assertEqual([ptr[i] for i in range(6)],
+                                     [42, 0, 17, 0, 2, 0])
+            else:
+                self.assertEqual([ptr[i] for i in range(6)],
+                                     [0, 42, 0, 17, 0, 2])
+
+    def test_address2pointer(self):
+        array = (c_int * 3)(42, 17, 2)
+
+        address = addressof(array)
+        ptr = cast(c_void_p(address), POINTER(c_int))
+        self.assertEqual([ptr[i] for i in range(3)], [42, 17, 2])
+
+        ptr = cast(address, POINTER(c_int))
+        self.assertEqual([ptr[i] for i in range(3)], [42, 17, 2])
+
+    def test_p2a_objects(self):
+        array = (c_char_p * 5)()
+        self.assertEqual(array._objects, None)
+        array[0] = b"foo bar"
+        self.assertEqual(array._objects, {'0': b"foo bar"})
+
+        p = cast(array, POINTER(c_char_p))
+        # array and p share a common _objects attribute
+        self.assertIs(p._objects, array._objects)
+        self.assertEqual(array._objects, {'0': b"foo bar", id(array): array})
+        p[0] = b"spam spam"
+        self.assertEqual(p._objects, {'0': b"spam spam", id(array): array})
+        self.assertIs(array._objects, p._objects)
+        p[1] = b"foo bar"
+        self.assertEqual(p._objects, {'1': b'foo bar', '0': b"spam spam", id(array): array})
+        self.assertIs(array._objects, p._objects)
+
+    def test_other(self):
+        p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int))
+        self.assertEqual(p[:4], [1,2, 3, 4])
+        self.assertEqual(p[:4:], [1, 2, 3, 4])
+        self.assertEqual(p[3:-1:-1], [4, 3, 2, 1])
+        self.assertEqual(p[:4:3], [1, 4])
+        c_int()
+        self.assertEqual(p[:4], [1, 2, 3, 4])
+        self.assertEqual(p[:4:], [1, 2, 3, 4])
+        self.assertEqual(p[3:-1:-1], [4, 3, 2, 1])
+        self.assertEqual(p[:4:3], [1, 4])
+        p[2] = 96
+        self.assertEqual(p[:4], [1, 2, 96, 4])
+        self.assertEqual(p[:4:], [1, 2, 96, 4])
+        self.assertEqual(p[3:-1:-1], [4, 96, 2, 1])
+        self.assertEqual(p[:4:3], [1, 4])
+        c_int()
+        self.assertEqual(p[:4], [1, 2, 96, 4])
+        self.assertEqual(p[:4:], [1, 2, 96, 4])
+        self.assertEqual(p[3:-1:-1], [4, 96, 2, 1])
+        self.assertEqual(p[:4:3], [1, 4])
+
+    def test_char_p(self):
+        # This didn't work: bad argument to internal function
+        s = c_char_p(b"hiho")
+        self.assertEqual(cast(cast(s, c_void_p), c_char_p).value,
+                             b"hiho")
+
+    @need_symbol('c_wchar_p')
+    def test_wchar_p(self):
+        s = c_wchar_p("hiho")
+        self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value,
+                             "hiho")
+
+    def test_bad_type_arg(self):
+        # The type argument must be a ctypes pointer type.
+        array_type = c_byte * sizeof(c_int)
+        array = array_type()
+        self.assertRaises(TypeError, cast, array, None)
+        self.assertRaises(TypeError, cast, array, array_type)
+        class Struct(Structure):
+            _fields_ = [("a", c_int)]
+        self.assertRaises(TypeError, cast, array, Struct)
+        class MyUnion(Union):
+            _fields_ = [("a", c_int)]
+        self.assertRaises(TypeError, cast, array, MyUnion)
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_cfuncs.py

@@ -0,0 +1,218 @@
+# A lot of failures in these tests on Mac OS X.
+# Byte order related?
+
+import unittest
+from ctypes import *
+from ctypes.test import need_symbol
+
+import _ctypes_test
+
+class CFunctions(unittest.TestCase):
+    _dll = CDLL(_ctypes_test.__file__)
+
+    def S(self):
+        return c_longlong.in_dll(self._dll, "last_tf_arg_s").value
+    def U(self):
+        return c_ulonglong.in_dll(self._dll, "last_tf_arg_u").value
+
+    def test_byte(self):
+        self._dll.tf_b.restype = c_byte
+        self._dll.tf_b.argtypes = (c_byte,)
+        self.assertEqual(self._dll.tf_b(-126), -42)
+        self.assertEqual(self.S(), -126)
+
+    def test_byte_plus(self):
+        self._dll.tf_bb.restype = c_byte
+        self._dll.tf_bb.argtypes = (c_byte, c_byte)
+        self.assertEqual(self._dll.tf_bb(0, -126), -42)
+        self.assertEqual(self.S(), -126)
+
+    def test_ubyte(self):
+        self._dll.tf_B.restype = c_ubyte
+        self._dll.tf_B.argtypes = (c_ubyte,)
+        self.assertEqual(self._dll.tf_B(255), 85)
+        self.assertEqual(self.U(), 255)
+
+    def test_ubyte_plus(self):
+        self._dll.tf_bB.restype = c_ubyte
+        self._dll.tf_bB.argtypes = (c_byte, c_ubyte)
+        self.assertEqual(self._dll.tf_bB(0, 255), 85)
+        self.assertEqual(self.U(), 255)
+
+    def test_short(self):
+        self._dll.tf_h.restype = c_short
+        self._dll.tf_h.argtypes = (c_short,)
+        self.assertEqual(self._dll.tf_h(-32766), -10922)
+        self.assertEqual(self.S(), -32766)
+
+    def test_short_plus(self):
+        self._dll.tf_bh.restype = c_short
+        self._dll.tf_bh.argtypes = (c_byte, c_short)
+        self.assertEqual(self._dll.tf_bh(0, -32766), -10922)
+        self.assertEqual(self.S(), -32766)
+
+    def test_ushort(self):
+        self._dll.tf_H.restype = c_ushort
+        self._dll.tf_H.argtypes = (c_ushort,)
+        self.assertEqual(self._dll.tf_H(65535), 21845)
+        self.assertEqual(self.U(), 65535)
+
+    def test_ushort_plus(self):
+        self._dll.tf_bH.restype = c_ushort
+        self._dll.tf_bH.argtypes = (c_byte, c_ushort)
+        self.assertEqual(self._dll.tf_bH(0, 65535), 21845)
+        self.assertEqual(self.U(), 65535)
+
+    def test_int(self):
+        self._dll.tf_i.restype = c_int
+        self._dll.tf_i.argtypes = (c_int,)
+        self.assertEqual(self._dll.tf_i(-2147483646), -715827882)
+        self.assertEqual(self.S(), -2147483646)
+
+    def test_int_plus(self):
+        self._dll.tf_bi.restype = c_int
+        self._dll.tf_bi.argtypes = (c_byte, c_int)
+        self.assertEqual(self._dll.tf_bi(0, -2147483646), -715827882)
+        self.assertEqual(self.S(), -2147483646)
+
+    def test_uint(self):
+        self._dll.tf_I.restype = c_uint
+        self._dll.tf_I.argtypes = (c_uint,)
+        self.assertEqual(self._dll.tf_I(4294967295), 1431655765)
+        self.assertEqual(self.U(), 4294967295)
+
+    def test_uint_plus(self):
+        self._dll.tf_bI.restype = c_uint
+        self._dll.tf_bI.argtypes = (c_byte, c_uint)
+        self.assertEqual(self._dll.tf_bI(0, 4294967295), 1431655765)
+        self.assertEqual(self.U(), 4294967295)
+
+    def test_long(self):
+        self._dll.tf_l.restype = c_long
+        self._dll.tf_l.argtypes = (c_long,)
+        self.assertEqual(self._dll.tf_l(-2147483646), -715827882)
+        self.assertEqual(self.S(), -2147483646)
+
+    def test_long_plus(self):
+        self._dll.tf_bl.restype = c_long
+        self._dll.tf_bl.argtypes = (c_byte, c_long)
+        self.assertEqual(self._dll.tf_bl(0, -2147483646), -715827882)
+        self.assertEqual(self.S(), -2147483646)
+
+    def test_ulong(self):
+        self._dll.tf_L.restype = c_ulong
+        self._dll.tf_L.argtypes = (c_ulong,)
+        self.assertEqual(self._dll.tf_L(4294967295), 1431655765)
+        self.assertEqual(self.U(), 4294967295)
+
+    def test_ulong_plus(self):
+        self._dll.tf_bL.restype = c_ulong
+        self._dll.tf_bL.argtypes = (c_char, c_ulong)
+        self.assertEqual(self._dll.tf_bL(b' ', 4294967295), 1431655765)
+        self.assertEqual(self.U(), 4294967295)
+
+    @need_symbol('c_longlong')
+    def test_longlong(self):
+        self._dll.tf_q.restype = c_longlong
+        self._dll.tf_q.argtypes = (c_longlong, )
+        self.assertEqual(self._dll.tf_q(-9223372036854775806), -3074457345618258602)
+        self.assertEqual(self.S(), -9223372036854775806)
+
+    @need_symbol('c_longlong')
+    def test_longlong_plus(self):
+        self._dll.tf_bq.restype = c_longlong
+        self._dll.tf_bq.argtypes = (c_byte, c_longlong)
+        self.assertEqual(self._dll.tf_bq(0, -9223372036854775806), -3074457345618258602)
+        self.assertEqual(self.S(), -9223372036854775806)
+
+    @need_symbol('c_ulonglong')
+    def test_ulonglong(self):
+        self._dll.tf_Q.restype = c_ulonglong
+        self._dll.tf_Q.argtypes = (c_ulonglong, )
+        self.assertEqual(self._dll.tf_Q(18446744073709551615), 6148914691236517205)
+        self.assertEqual(self.U(), 18446744073709551615)
+
+    @need_symbol('c_ulonglong')
+    def test_ulonglong_plus(self):
+        self._dll.tf_bQ.restype = c_ulonglong
+        self._dll.tf_bQ.argtypes = (c_byte, c_ulonglong)
+        self.assertEqual(self._dll.tf_bQ(0, 18446744073709551615), 6148914691236517205)
+        self.assertEqual(self.U(), 18446744073709551615)
+
+    def test_float(self):
+        self._dll.tf_f.restype = c_float
+        self._dll.tf_f.argtypes = (c_float,)
+        self.assertEqual(self._dll.tf_f(-42.), -14.)
+        self.assertEqual(self.S(), -42)
+
+    def test_float_plus(self):
+        self._dll.tf_bf.restype = c_float
+        self._dll.tf_bf.argtypes = (c_byte, c_float)
+        self.assertEqual(self._dll.tf_bf(0, -42.), -14.)
+        self.assertEqual(self.S(), -42)
+
+    def test_double(self):
+        self._dll.tf_d.restype = c_double
+        self._dll.tf_d.argtypes = (c_double,)
+        self.assertEqual(self._dll.tf_d(42.), 14.)
+        self.assertEqual(self.S(), 42)
+
+    def test_double_plus(self):
+        self._dll.tf_bd.restype = c_double
+        self._dll.tf_bd.argtypes = (c_byte, c_double)
+        self.assertEqual(self._dll.tf_bd(0, 42.), 14.)
+        self.assertEqual(self.S(), 42)
+
+    @need_symbol('c_longdouble')
+    def test_longdouble(self):
+        self._dll.tf_D.restype = c_longdouble
+        self._dll.tf_D.argtypes = (c_longdouble,)
+        self.assertEqual(self._dll.tf_D(42.), 14.)
+        self.assertEqual(self.S(), 42)
+
+    @need_symbol('c_longdouble')
+    def test_longdouble_plus(self):
+        self._dll.tf_bD.restype = c_longdouble
+        self._dll.tf_bD.argtypes = (c_byte, c_longdouble)
+        self.assertEqual(self._dll.tf_bD(0, 42.), 14.)
+        self.assertEqual(self.S(), 42)
+
+    def test_callwithresult(self):
+        def process_result(result):
+            return result * 2
+        self._dll.tf_i.restype = process_result
+        self._dll.tf_i.argtypes = (c_int,)
+        self.assertEqual(self._dll.tf_i(42), 28)
+        self.assertEqual(self.S(), 42)
+        self.assertEqual(self._dll.tf_i(-42), -28)
+        self.assertEqual(self.S(), -42)
+
+    def test_void(self):
+        self._dll.tv_i.restype = None
+        self._dll.tv_i.argtypes = (c_int,)
+        self.assertEqual(self._dll.tv_i(42), None)
+        self.assertEqual(self.S(), 42)
+        self.assertEqual(self._dll.tv_i(-42), None)
+        self.assertEqual(self.S(), -42)
+
+# The following repeats the above tests with stdcall functions (where
+# they are available)
+try:
+    WinDLL
+except NameError:
+    def stdcall_dll(*_): pass
+else:
+    class stdcall_dll(WinDLL):
+        def __getattr__(self, name):
+            if name[:2] == '__' and name[-2:] == '__':
+                raise AttributeError(name)
+            func = self._FuncPtr(("s_" + name, self))
+            setattr(self, name, func)
+            return func
+
+@need_symbol('WinDLL')
+class stdcallCFunctions(CFunctions):
+    _dll = stdcall_dll(_ctypes_test.__file__)
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_checkretval.py

@@ -0,0 +1,36 @@
+import unittest
+
+from ctypes import *
+from ctypes.test import need_symbol
+
+class CHECKED(c_int):
+    def _check_retval_(value):
+        # Receives a CHECKED instance.
+        return str(value.value)
+    _check_retval_ = staticmethod(_check_retval_)
+
+class Test(unittest.TestCase):
+
+    def test_checkretval(self):
+
+        import _ctypes_test
+        dll = CDLL(_ctypes_test.__file__)
+        self.assertEqual(42, dll._testfunc_p_p(42))
+
+        dll._testfunc_p_p.restype = CHECKED
+        self.assertEqual("42", dll._testfunc_p_p(42))
+
+        dll._testfunc_p_p.restype = None
+        self.assertEqual(None, dll._testfunc_p_p(42))
+
+        del dll._testfunc_p_p.restype
+        self.assertEqual(42, dll._testfunc_p_p(42))
+
+    @need_symbol('oledll')
+    def test_oledll(self):
+        self.assertRaises(OSError,
+                              oledll.oleaut32.CreateTypeLib2,
+                              0, None, None)
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_delattr.py

@@ -0,0 +1,21 @@
+import unittest
+from ctypes import *
+
+class X(Structure):
+    _fields_ = [("foo", c_int)]
+
+class TestCase(unittest.TestCase):
+    def test_simple(self):
+        self.assertRaises(TypeError,
+                          delattr, c_int(42), "value")
+
+    def test_chararray(self):
+        self.assertRaises(TypeError,
+                          delattr, (c_char * 5)(), "value")
+
+    def test_struct(self):
+        self.assertRaises(TypeError,
+                          delattr, X(), "foo")
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_errno.py

@@ -0,0 +1,76 @@
+import unittest, os, errno
+import threading
+
+from ctypes import *
+from ctypes.util import find_library
+
+class Test(unittest.TestCase):
+    def test_open(self):
+        libc_name = find_library("c")
+        if libc_name is None:
+            raise unittest.SkipTest("Unable to find C library")
+        libc = CDLL(libc_name, use_errno=True)
+        if os.name == "nt":
+            libc_open = libc._open
+        else:
+            libc_open = libc.open
+
+        libc_open.argtypes = c_char_p, c_int
+
+        self.assertEqual(libc_open(b"", 0), -1)
+        self.assertEqual(get_errno(), errno.ENOENT)
+
+        self.assertEqual(set_errno(32), errno.ENOENT)
+        self.assertEqual(get_errno(), 32)
+
+        def _worker():
+            set_errno(0)
+
+            libc = CDLL(libc_name, use_errno=False)
+            if os.name == "nt":
+                libc_open = libc._open
+            else:
+                libc_open = libc.open
+            libc_open.argtypes = c_char_p, c_int
+            self.assertEqual(libc_open(b"", 0), -1)
+            self.assertEqual(get_errno(), 0)
+
+        t = threading.Thread(target=_worker)
+        t.start()
+        t.join()
+
+        self.assertEqual(get_errno(), 32)
+        set_errno(0)
+
+    @unittest.skipUnless(os.name == "nt", 'Test specific to Windows')
+    def test_GetLastError(self):
+        dll = WinDLL("kernel32", use_last_error=True)
+        GetModuleHandle = dll.GetModuleHandleA
+        GetModuleHandle.argtypes = [c_wchar_p]
+
+        self.assertEqual(0, GetModuleHandle("foo"))
+        self.assertEqual(get_last_error(), 126)
+
+        self.assertEqual(set_last_error(32), 126)
+        self.assertEqual(get_last_error(), 32)
+
+        def _worker():
+            set_last_error(0)
+
+            dll = WinDLL("kernel32", use_last_error=False)
+            GetModuleHandle = dll.GetModuleHandleW
+            GetModuleHandle.argtypes = [c_wchar_p]
+            GetModuleHandle("bar")
+
+            self.assertEqual(get_last_error(), 0)
+
+        t = threading.Thread(target=_worker)
+        t.start()
+        t.join()
+
+        self.assertEqual(get_last_error(), 32)
+
+        set_last_error(0)
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_find.py

@@ -0,0 +1,127 @@
+import unittest
+import unittest.mock
+import os.path
+import sys
+import test.support
+from test.support import os_helper
+from ctypes import *
+from ctypes.util import find_library
+
+# On some systems, loading the OpenGL libraries needs the RTLD_GLOBAL mode.
+class Test_OpenGL_libs(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        lib_gl = lib_glu = lib_gle = None
+        if sys.platform == "win32":
+            lib_gl = find_library("OpenGL32")
+            lib_glu = find_library("Glu32")
+        elif sys.platform == "darwin":
+            lib_gl = lib_glu = find_library("OpenGL")
+        else:
+            lib_gl = find_library("GL")
+            lib_glu = find_library("GLU")
+            lib_gle = find_library("gle")
+
+        ## print, for debugging
+        if test.support.verbose:
+            print("OpenGL libraries:")
+            for item in (("GL", lib_gl),
+                         ("GLU", lib_glu),
+                         ("gle", lib_gle)):
+                print("\t", item)
+
+        cls.gl = cls.glu = cls.gle = None
+        if lib_gl:
+            try:
+                cls.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            except OSError:
+                pass
+        if lib_glu:
+            try:
+                cls.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            except OSError:
+                pass
+        if lib_gle:
+            try:
+                cls.gle = CDLL(lib_gle)
+            except OSError:
+                pass
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.gl = cls.glu = cls.gle = None
+
+    def test_gl(self):
+        if self.gl is None:
+            self.skipTest('lib_gl not available')
+        self.gl.glClearIndex
+
+    def test_glu(self):
+        if self.glu is None:
+            self.skipTest('lib_glu not available')
+        self.glu.gluBeginCurve
+
+    def test_gle(self):
+        if self.gle is None:
+            self.skipTest('lib_gle not available')
+        self.gle.gleGetJoinStyle
+
+    def test_shell_injection(self):
+        result = find_library('; echo Hello shell > ' + os_helper.TESTFN)
+        self.assertFalse(os.path.lexists(os_helper.TESTFN))
+        self.assertIsNone(result)
+
+
+@unittest.skipUnless(sys.platform.startswith('linux'),
+                     'Test only valid for Linux')
+class FindLibraryLinux(unittest.TestCase):
+    def test_find_on_libpath(self):
+        import subprocess
+        import tempfile
+
+        try:
+            p = subprocess.Popen(['gcc', '--version'], stdout=subprocess.PIPE,
+                                 stderr=subprocess.DEVNULL)
+            out, _ = p.communicate()
+        except OSError:
+            raise unittest.SkipTest('gcc, needed for test, not available')
+        with tempfile.TemporaryDirectory() as d:
+            # create an empty temporary file
+            srcname = os.path.join(d, 'dummy.c')
+            libname = 'py_ctypes_test_dummy'
+            dstname = os.path.join(d, 'lib%s.so' % libname)
+            with open(srcname, 'wb') as f:
+                pass
+            self.assertTrue(os.path.exists(srcname))
+            # compile the file to a shared library
+            cmd = ['gcc', '-o', dstname, '--shared',
+                   '-Wl,-soname,lib%s.so' % libname, srcname]
+            out = subprocess.check_output(cmd)
+            self.assertTrue(os.path.exists(dstname))
+            # now check that the .so can't be found (since not in
+            # LD_LIBRARY_PATH)
+            self.assertIsNone(find_library(libname))
+            # now add the location to LD_LIBRARY_PATH
+            with os_helper.EnvironmentVarGuard() as env:
+                KEY = 'LD_LIBRARY_PATH'
+                if KEY not in env:
+                    v = d
+                else:
+                    v = '%s:%s' % (env[KEY], d)
+                env.set(KEY, v)
+                # now check that the .so can be found (since in
+                # LD_LIBRARY_PATH)
+                self.assertEqual(find_library(libname), 'lib%s.so' % libname)
+
+    def test_find_library_with_gcc(self):
+        with unittest.mock.patch("ctypes.util._findSoname_ldconfig", lambda *args: None):
+            self.assertNotEqual(find_library('c'), None)
+
+    def test_find_library_with_ld(self):
+        with unittest.mock.patch("ctypes.util._findSoname_ldconfig", lambda *args: None), \
+             unittest.mock.patch("ctypes.util._findLib_gcc", lambda *args: None):
+            self.assertNotEqual(find_library('c'), None)
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_frombuffer.py

@@ -0,0 +1,141 @@
+from ctypes import *
+import array
+import gc
+import unittest
+
+class X(Structure):
+    _fields_ = [("c_int", c_int)]
+    init_called = False
+    def __init__(self):
+        self._init_called = True
+
+class Test(unittest.TestCase):
+    def test_from_buffer(self):
+        a = array.array("i", range(16))
+        x = (c_int * 16).from_buffer(a)
+
+        y = X.from_buffer(a)
+        self.assertEqual(y.c_int, a[0])
+        self.assertFalse(y.init_called)
+
+        self.assertEqual(x[:], a.tolist())
+
+        a[0], a[-1] = 200, -200
+        self.assertEqual(x[:], a.tolist())
+
+        self.assertRaises(BufferError, a.append, 100)
+        self.assertRaises(BufferError, a.pop)
+
+        del x; del y; gc.collect(); gc.collect(); gc.collect()
+        a.append(100)
+        a.pop()
+        x = (c_int * 16).from_buffer(a)
+
+        self.assertIn(a, [obj.obj if isinstance(obj, memoryview) else obj
+                          for obj in x._objects.values()])
+
+        expected = x[:]
+        del a; gc.collect(); gc.collect(); gc.collect()
+        self.assertEqual(x[:], expected)
+
+        with self.assertRaisesRegex(TypeError, "not writable"):
+            (c_char * 16).from_buffer(b"a" * 16)
+        with self.assertRaisesRegex(TypeError, "not writable"):
+            (c_char * 16).from_buffer(memoryview(b"a" * 16))
+        with self.assertRaisesRegex(TypeError, "not C contiguous"):
+            (c_char * 16).from_buffer(memoryview(bytearray(b"a" * 16))[::-1])
+        msg = "bytes-like object is required"
+        with self.assertRaisesRegex(TypeError, msg):
+            (c_char * 16).from_buffer("a" * 16)
+
+    def test_fortran_contiguous(self):
+        try:
+            import _testbuffer
+        except ImportError as err:
+            self.skipTest(str(err))
+        flags = _testbuffer.ND_WRITABLE | _testbuffer.ND_FORTRAN
+        array = _testbuffer.ndarray(
+            [97] * 16, format="B", shape=[4, 4], flags=flags)
+        with self.assertRaisesRegex(TypeError, "not C contiguous"):
+            (c_char * 16).from_buffer(array)
+        array = memoryview(array)
+        self.assertTrue(array.f_contiguous)
+        self.assertFalse(array.c_contiguous)
+        with self.assertRaisesRegex(TypeError, "not C contiguous"):
+            (c_char * 16).from_buffer(array)
+
+    def test_from_buffer_with_offset(self):
+        a = array.array("i", range(16))
+        x = (c_int * 15).from_buffer(a, sizeof(c_int))
+
+        self.assertEqual(x[:], a.tolist()[1:])
+        with self.assertRaises(ValueError):
+            c_int.from_buffer(a, -1)
+        with self.assertRaises(ValueError):
+            (c_int * 16).from_buffer(a, sizeof(c_int))
+        with self.assertRaises(ValueError):
+            (c_int * 1).from_buffer(a, 16 * sizeof(c_int))
+
+    def test_from_buffer_memoryview(self):
+        a = [c_char.from_buffer(memoryview(bytearray(b'a')))]
+        a.append(a)
+        del a
+        gc.collect()  # Should not crash
+
+    def test_from_buffer_copy(self):
+        a = array.array("i", range(16))
+        x = (c_int * 16).from_buffer_copy(a)
+
+        y = X.from_buffer_copy(a)
+        self.assertEqual(y.c_int, a[0])
+        self.assertFalse(y.init_called)
+
+        self.assertEqual(x[:], list(range(16)))
+
+        a[0], a[-1] = 200, -200
+        self.assertEqual(x[:], list(range(16)))
+
+        a.append(100)
+        self.assertEqual(x[:], list(range(16)))
+
+        self.assertEqual(x._objects, None)
+
+        del a; gc.collect(); gc.collect(); gc.collect()
+        self.assertEqual(x[:], list(range(16)))
+
+        x = (c_char * 16).from_buffer_copy(b"a" * 16)
+        self.assertEqual(x[:], b"a" * 16)
+        with self.assertRaises(TypeError):
+            (c_char * 16).from_buffer_copy("a" * 16)
+
+    def test_from_buffer_copy_with_offset(self):
+        a = array.array("i", range(16))
+        x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
+
+        self.assertEqual(x[:], a.tolist()[1:])
+        with self.assertRaises(ValueError):
+            c_int.from_buffer_copy(a, -1)
+        with self.assertRaises(ValueError):
+            (c_int * 16).from_buffer_copy(a, sizeof(c_int))
+        with self.assertRaises(ValueError):
+            (c_int * 1).from_buffer_copy(a, 16 * sizeof(c_int))
+
+    def test_abstract(self):
+        from ctypes import _Pointer, _SimpleCData, _CFuncPtr
+
+        self.assertRaises(TypeError, Array.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, Structure.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, Union.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _CFuncPtr.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _Pointer.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, _SimpleCData.from_buffer, bytearray(10))
+
+        self.assertRaises(TypeError, Array.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, Structure.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, Union.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _CFuncPtr.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _Pointer.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, _SimpleCData.from_buffer_copy, b"123")
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_funcptr.py

@@ -0,0 +1,132 @@
+import unittest
+from ctypes import *
+
+try:
+    WINFUNCTYPE
+except NameError:
+    # fake to enable this test on Linux
+    WINFUNCTYPE = CFUNCTYPE
+
+import _ctypes_test
+lib = CDLL(_ctypes_test.__file__)
+
+class CFuncPtrTestCase(unittest.TestCase):
+    def test_basic(self):
+        X = WINFUNCTYPE(c_int, c_int, c_int)
+
+        def func(*args):
+            return len(args)
+
+        x = X(func)
+        self.assertEqual(x.restype, c_int)
+        self.assertEqual(x.argtypes, (c_int, c_int))
+        self.assertEqual(sizeof(x), sizeof(c_voidp))
+        self.assertEqual(sizeof(X), sizeof(c_voidp))
+
+    def test_first(self):
+        StdCallback = WINFUNCTYPE(c_int, c_int, c_int)
+        CdeclCallback = CFUNCTYPE(c_int, c_int, c_int)
+
+        def func(a, b):
+            return a + b
+
+        s = StdCallback(func)
+        c = CdeclCallback(func)
+
+        self.assertEqual(s(1, 2), 3)
+        self.assertEqual(c(1, 2), 3)
+        # The following no longer raises a TypeError - it is now
+        # possible, as in C, to call cdecl functions with more parameters.
+        #self.assertRaises(TypeError, c, 1, 2, 3)
+        self.assertEqual(c(1, 2, 3, 4, 5, 6), 3)
+        if not WINFUNCTYPE is CFUNCTYPE:
+            self.assertRaises(TypeError, s, 1, 2, 3)
+
+    def test_structures(self):
+        WNDPROC = WINFUNCTYPE(c_long, c_int, c_int, c_int, c_int)
+
+        def wndproc(hwnd, msg, wParam, lParam):
+            return hwnd + msg + wParam + lParam
+
+        HINSTANCE = c_int
+        HICON = c_int
+        HCURSOR = c_int
+        LPCTSTR = c_char_p
+
+        class WNDCLASS(Structure):
+            _fields_ = [("style", c_uint),
+                        ("lpfnWndProc", WNDPROC),
+                        ("cbClsExtra", c_int),
+                        ("cbWndExtra", c_int),
+                        ("hInstance", HINSTANCE),
+                        ("hIcon", HICON),
+                        ("hCursor", HCURSOR),
+                        ("lpszMenuName", LPCTSTR),
+                        ("lpszClassName", LPCTSTR)]
+
+        wndclass = WNDCLASS()
+        wndclass.lpfnWndProc = WNDPROC(wndproc)
+
+        WNDPROC_2 = WINFUNCTYPE(c_long, c_int, c_int, c_int, c_int)
+
+        # This is no longer true, now that WINFUNCTYPE caches created types internally.
+        ## # CFuncPtr subclasses are compared by identity, so this raises a TypeError:
+        ## self.assertRaises(TypeError, setattr, wndclass,
+        ##                  "lpfnWndProc", WNDPROC_2(wndproc))
+        # instead:
+
+        self.assertIs(WNDPROC, WNDPROC_2)
+        # 'wndclass.lpfnWndProc' leaks 94 references.  Why?
+        self.assertEqual(wndclass.lpfnWndProc(1, 2, 3, 4), 10)
+
+
+        f = wndclass.lpfnWndProc
+
+        del wndclass
+        del wndproc
+
+        self.assertEqual(f(10, 11, 12, 13), 46)
+
+    def test_dllfunctions(self):
+
+        def NoNullHandle(value):
+            if not value:
+                raise WinError()
+            return value
+
+        strchr = lib.my_strchr
+        strchr.restype = c_char_p
+        strchr.argtypes = (c_char_p, c_char)
+        self.assertEqual(strchr(b"abcdefghi", b"b"), b"bcdefghi")
+        self.assertEqual(strchr(b"abcdefghi", b"x"), None)
+
+
+        strtok = lib.my_strtok
+        strtok.restype = c_char_p
+        # Neither of this does work: strtok changes the buffer it is passed
+##        strtok.argtypes = (c_char_p, c_char_p)
+##        strtok.argtypes = (c_string, c_char_p)
+
+        def c_string(init):
+            size = len(init) + 1
+            return (c_char*size)(*init)
+
+        s = b"a\nb\nc"
+        b = c_string(s)
+
+##        b = (c_char * (len(s)+1))()
+##        b.value = s
+
+##        b = c_string(s)
+        self.assertEqual(strtok(b, b"\n"), b"a")
+        self.assertEqual(strtok(None, b"\n"), b"b")
+        self.assertEqual(strtok(None, b"\n"), b"c")
+        self.assertEqual(strtok(None, b"\n"), None)
+
+    def test_abstract(self):
+        from ctypes import _CFuncPtr
+
+        self.assertRaises(TypeError, _CFuncPtr, 13, "name", 42, "iid")
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_functions.py

@@ -0,0 +1,384 @@
+"""
+Here is probably the place to write the docs, since the test-cases
+show how the type behave.
+
+Later...
+"""
+
+from ctypes import *
+from ctypes.test import need_symbol
+import sys, unittest
+
+try:
+    WINFUNCTYPE
+except NameError:
+    # fake to enable this test on Linux
+    WINFUNCTYPE = CFUNCTYPE
+
+import _ctypes_test
+dll = CDLL(_ctypes_test.__file__)
+if sys.platform == "win32":
+    windll = WinDLL(_ctypes_test.__file__)
+
+class POINT(Structure):
+    _fields_ = [("x", c_int), ("y", c_int)]
+class RECT(Structure):
+    _fields_ = [("left", c_int), ("top", c_int),
+                ("right", c_int), ("bottom", c_int)]
+class FunctionTestCase(unittest.TestCase):
+
+    def test_mro(self):
+        # in Python 2.3, this raises TypeError: MRO conflict among bases classes,
+        # in Python 2.2 it works.
+        #
+        # But in early versions of _ctypes.c, the result of tp_new
+        # wasn't checked, and it even crashed Python.
+        # Found by Greg Chapman.
+
+        with self.assertRaises(TypeError):
+            class X(object, Array):
+                _length_ = 5
+                _type_ = "i"
+
+        from _ctypes import _Pointer
+        with self.assertRaises(TypeError):
+            class X(object, _Pointer):
+                pass
+
+        from _ctypes import _SimpleCData
+        with self.assertRaises(TypeError):
+            class X(object, _SimpleCData):
+                _type_ = "i"
+
+        with self.assertRaises(TypeError):
+            class X(object, Structure):
+                _fields_ = []
+
+    @need_symbol('c_wchar')
+    def test_wchar_parm(self):
+        f = dll._testfunc_i_bhilfd
+        f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
+        result = f(1, "x", 3, 4, 5.0, 6.0)
+        self.assertEqual(result, 139)
+        self.assertEqual(type(result), int)
+
+    @need_symbol('c_wchar')
+    def test_wchar_result(self):
+        f = dll._testfunc_i_bhilfd
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_wchar
+        result = f(0, 0, 0, 0, 0, 0)
+        self.assertEqual(result, '\x00')
+
+    def test_voidresult(self):
+        f = dll._testfunc_v
+        f.restype = None
+        f.argtypes = [c_int, c_int, POINTER(c_int)]
+        result = c_int()
+        self.assertEqual(None, f(1, 2, byref(result)))
+        self.assertEqual(result.value, 3)
+
+    def test_intresult(self):
+        f = dll._testfunc_i_bhilfd
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_int
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        self.assertEqual(result, 21)
+        self.assertEqual(type(result), int)
+
+        result = f(-1, -2, -3, -4, -5.0, -6.0)
+        self.assertEqual(result, -21)
+        self.assertEqual(type(result), int)
+
+        # If we declare the function to return a short,
+        # is the high part split off?
+        f.restype = c_short
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        self.assertEqual(result, 21)
+        self.assertEqual(type(result), int)
+
+        result = f(1, 2, 3, 0x10004, 5.0, 6.0)
+        self.assertEqual(result, 21)
+        self.assertEqual(type(result), int)
+
+        # You cannot assign character format codes as restype any longer
+        self.assertRaises(TypeError, setattr, f, "restype", "i")
+
+    def test_floatresult(self):
+        f = dll._testfunc_f_bhilfd
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_float
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        self.assertEqual(result, 21)
+        self.assertEqual(type(result), float)
+
+        result = f(-1, -2, -3, -4, -5.0, -6.0)
+        self.assertEqual(result, -21)
+        self.assertEqual(type(result), float)
+
+    def test_doubleresult(self):
+        f = dll._testfunc_d_bhilfd
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_double
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        self.assertEqual(result, 21)
+        self.assertEqual(type(result), float)
+
+        result = f(-1, -2, -3, -4, -5.0, -6.0)
+        self.assertEqual(result, -21)
+        self.assertEqual(type(result), float)
+
+    @need_symbol('c_longdouble')
+    def test_longdoubleresult(self):
+        f = dll._testfunc_D_bhilfD
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble]
+        f.restype = c_longdouble
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        self.assertEqual(result, 21)
+        self.assertEqual(type(result), float)
+
+        result = f(-1, -2, -3, -4, -5.0, -6.0)
+        self.assertEqual(result, -21)
+        self.assertEqual(type(result), float)
+
+    @need_symbol('c_longlong')
+    def test_longlongresult(self):
+        f = dll._testfunc_q_bhilfd
+        f.restype = c_longlong
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        result = f(1, 2, 3, 4, 5.0, 6.0)
+        self.assertEqual(result, 21)
+
+        f = dll._testfunc_q_bhilfdq
+        f.restype = c_longlong
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double, c_longlong]
+        result = f(1, 2, 3, 4, 5.0, 6.0, 21)
+        self.assertEqual(result, 42)
+
+    def test_stringresult(self):
+        f = dll._testfunc_p_p
+        f.argtypes = None
+        f.restype = c_char_p
+        result = f(b"123")
+        self.assertEqual(result, b"123")
+
+        result = f(None)
+        self.assertEqual(result, None)
+
+    def test_pointers(self):
+        f = dll._testfunc_p_p
+        f.restype = POINTER(c_int)
+        f.argtypes = [POINTER(c_int)]
+
+        # This only works if the value c_int(42) passed to the
+        # function is still alive while the pointer (the result) is
+        # used.
+
+        v = c_int(42)
+
+        self.assertEqual(pointer(v).contents.value, 42)
+        result = f(pointer(v))
+        self.assertEqual(type(result), POINTER(c_int))
+        self.assertEqual(result.contents.value, 42)
+
+        # This on works...
+        result = f(pointer(v))
+        self.assertEqual(result.contents.value, v.value)
+
+        p = pointer(c_int(99))
+        result = f(p)
+        self.assertEqual(result.contents.value, 99)
+
+        arg = byref(v)
+        result = f(arg)
+        self.assertNotEqual(result.contents, v.value)
+
+        self.assertRaises(ArgumentError, f, byref(c_short(22)))
+
+        # It is dangerous, however, because you don't control the lifetime
+        # of the pointer:
+        result = f(byref(c_int(99)))
+        self.assertNotEqual(result.contents, 99)
+
+    ################################################################
+    def test_shorts(self):
+        f = dll._testfunc_callback_i_if
+
+        args = []
+        expected = [262144, 131072, 65536, 32768, 16384, 8192, 4096, 2048,
+                    1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1]
+
+        def callback(v):
+            args.append(v)
+            return v
+
+        CallBack = CFUNCTYPE(c_int, c_int)
+
+        cb = CallBack(callback)
+        f(2**18, cb)
+        self.assertEqual(args, expected)
+
+    ################################################################
+
+
+    def test_callbacks(self):
+        f = dll._testfunc_callback_i_if
+        f.restype = c_int
+        f.argtypes = None
+
+        MyCallback = CFUNCTYPE(c_int, c_int)
+
+        def callback(value):
+            #print "called back with", value
+            return value
+
+        cb = MyCallback(callback)
+        result = f(-10, cb)
+        self.assertEqual(result, -18)
+
+        # test with prototype
+        f.argtypes = [c_int, MyCallback]
+        cb = MyCallback(callback)
+        result = f(-10, cb)
+        self.assertEqual(result, -18)
+
+        AnotherCallback = WINFUNCTYPE(c_int, c_int, c_int, c_int, c_int)
+
+        # check that the prototype works: we call f with wrong
+        # argument types
+        cb = AnotherCallback(callback)
+        self.assertRaises(ArgumentError, f, -10, cb)
+
+
+    def test_callbacks_2(self):
+        # Can also use simple datatypes as argument type specifiers
+        # for the callback function.
+        # In this case the call receives an instance of that type
+        f = dll._testfunc_callback_i_if
+        f.restype = c_int
+
+        MyCallback = CFUNCTYPE(c_int, c_int)
+
+        f.argtypes = [c_int, MyCallback]
+
+        def callback(value):
+            #print "called back with", value
+            self.assertEqual(type(value), int)
+            return value
+
+        cb = MyCallback(callback)
+        result = f(-10, cb)
+        self.assertEqual(result, -18)
+
+    @need_symbol('c_longlong')
+    def test_longlong_callbacks(self):
+
+        f = dll._testfunc_callback_q_qf
+        f.restype = c_longlong
+
+        MyCallback = CFUNCTYPE(c_longlong, c_longlong)
+
+        f.argtypes = [c_longlong, MyCallback]
+
+        def callback(value):
+            self.assertIsInstance(value, int)
+            return value & 0x7FFFFFFF
+
+        cb = MyCallback(callback)
+
+        self.assertEqual(13577625587, f(1000000000000, cb))
+
+    def test_errors(self):
+        self.assertRaises(AttributeError, getattr, dll, "_xxx_yyy")
+        self.assertRaises(ValueError, c_int.in_dll, dll, "_xxx_yyy")
+
+    def test_byval(self):
+
+        # without prototype
+        ptin = POINT(1, 2)
+        ptout = POINT()
+        # EXPORT int _testfunc_byval(point in, point *pout)
+        result = dll._testfunc_byval(ptin, byref(ptout))
+        got = result, ptout.x, ptout.y
+        expected = 3, 1, 2
+        self.assertEqual(got, expected)
+
+        # with prototype
+        ptin = POINT(101, 102)
+        ptout = POINT()
+        dll._testfunc_byval.argtypes = (POINT, POINTER(POINT))
+        dll._testfunc_byval.restype = c_int
+        result = dll._testfunc_byval(ptin, byref(ptout))
+        got = result, ptout.x, ptout.y
+        expected = 203, 101, 102
+        self.assertEqual(got, expected)
+
+    def test_struct_return_2H(self):
+        class S2H(Structure):
+            _fields_ = [("x", c_short),
+                        ("y", c_short)]
+        dll.ret_2h_func.restype = S2H
+        dll.ret_2h_func.argtypes = [S2H]
+        inp = S2H(99, 88)
+        s2h = dll.ret_2h_func(inp)
+        self.assertEqual((s2h.x, s2h.y), (99*2, 88*3))
+
+    @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
+    def test_struct_return_2H_stdcall(self):
+        class S2H(Structure):
+            _fields_ = [("x", c_short),
+                        ("y", c_short)]
+
+        windll.s_ret_2h_func.restype = S2H
+        windll.s_ret_2h_func.argtypes = [S2H]
+        s2h = windll.s_ret_2h_func(S2H(99, 88))
+        self.assertEqual((s2h.x, s2h.y), (99*2, 88*3))
+
+    def test_struct_return_8H(self):
+        class S8I(Structure):
+            _fields_ = [("a", c_int),
+                        ("b", c_int),
+                        ("c", c_int),
+                        ("d", c_int),
+                        ("e", c_int),
+                        ("f", c_int),
+                        ("g", c_int),
+                        ("h", c_int)]
+        dll.ret_8i_func.restype = S8I
+        dll.ret_8i_func.argtypes = [S8I]
+        inp = S8I(9, 8, 7, 6, 5, 4, 3, 2)
+        s8i = dll.ret_8i_func(inp)
+        self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
+                             (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
+
+    @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
+    def test_struct_return_8H_stdcall(self):
+        class S8I(Structure):
+            _fields_ = [("a", c_int),
+                        ("b", c_int),
+                        ("c", c_int),
+                        ("d", c_int),
+                        ("e", c_int),
+                        ("f", c_int),
+                        ("g", c_int),
+                        ("h", c_int)]
+        windll.s_ret_8i_func.restype = S8I
+        windll.s_ret_8i_func.argtypes = [S8I]
+        inp = S8I(9, 8, 7, 6, 5, 4, 3, 2)
+        s8i = windll.s_ret_8i_func(inp)
+        self.assertEqual(
+                (s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
+                (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
+
+    def test_sf1651235(self):
+        # see https://www.python.org/sf/1651235
+
+        proto = CFUNCTYPE(c_int, RECT, POINT)
+        def callback(*args):
+            return 0
+
+        callback = proto(callback)
+        self.assertRaises(ArgumentError, lambda: callback((1, 2, 3, 4), POINT()))
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_incomplete.py

@@ -0,0 +1,42 @@
+import unittest
+from ctypes import *
+
+################################################################
+#
+# The incomplete pointer example from the tutorial
+#
+
+class MyTestCase(unittest.TestCase):
+
+    def test_incomplete_example(self):
+        lpcell = POINTER("cell")
+        class cell(Structure):
+            _fields_ = [("name", c_char_p),
+                        ("next", lpcell)]
+
+        SetPointerType(lpcell, cell)
+
+        c1 = cell()
+        c1.name = b"foo"
+        c2 = cell()
+        c2.name = b"bar"
+
+        c1.next = pointer(c2)
+        c2.next = pointer(c1)
+
+        p = c1
+
+        result = []
+        for i in range(8):
+            result.append(p.name)
+            p = p.next[0]
+        self.assertEqual(result, [b"foo", b"bar"] * 4)
+
+        # to not leak references, we must clean _pointer_type_cache
+        from ctypes import _pointer_type_cache
+        del _pointer_type_cache[cell]
+
+################################################################
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_init.py

@@ -0,0 +1,40 @@
+from ctypes import *
+import unittest
+
+class X(Structure):
+    _fields_ = [("a", c_int),
+                ("b", c_int)]
+    new_was_called = False
+
+    def __new__(cls):
+        result = super().__new__(cls)
+        result.new_was_called = True
+        return result
+
+    def __init__(self):
+        self.a = 9
+        self.b = 12
+
+class Y(Structure):
+    _fields_ = [("x", X)]
+
+
+class InitTest(unittest.TestCase):
+    def test_get(self):
+        # make sure the only accessing a nested structure
+        # doesn't call the structure's __new__ and __init__
+        y = Y()
+        self.assertEqual((y.x.a, y.x.b), (0, 0))
+        self.assertEqual(y.x.new_was_called, False)
+
+        # But explicitly creating an X structure calls __new__ and __init__, of course.
+        x = X()
+        self.assertEqual((x.a, x.b), (9, 12))
+        self.assertEqual(x.new_was_called, True)
+
+        y.x = x
+        self.assertEqual((y.x.a, y.x.b), (9, 12))
+        self.assertEqual(y.x.new_was_called, False)
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_internals.py

@@ -0,0 +1,100 @@
+# This tests the internal _objects attribute
+import unittest
+from ctypes import *
+from sys import getrefcount as grc
+
+# XXX This test must be reviewed for correctness!!!
+
+# ctypes' types are container types.
+#
+# They have an internal memory block, which only consists of some bytes,
+# but it has to keep references to other objects as well. This is not
+# really needed for trivial C types like int or char, but it is important
+# for aggregate types like strings or pointers in particular.
+#
+# What about pointers?
+
+class ObjectsTestCase(unittest.TestCase):
+    def assertSame(self, a, b):
+        self.assertEqual(id(a), id(b))
+
+    def test_ints(self):
+        i = 42000123
+        refcnt = grc(i)
+        ci = c_int(i)
+        self.assertEqual(refcnt, grc(i))
+        self.assertEqual(ci._objects, None)
+
+    def test_c_char_p(self):
+        s = b"Hello, World"
+        refcnt = grc(s)
+        cs = c_char_p(s)
+        self.assertEqual(refcnt + 1, grc(s))
+        self.assertSame(cs._objects, s)
+
+    def test_simple_struct(self):
+        class X(Structure):
+            _fields_ = [("a", c_int), ("b", c_int)]
+
+        a = 421234
+        b = 421235
+        x = X()
+        self.assertEqual(x._objects, None)
+        x.a = a
+        x.b = b
+        self.assertEqual(x._objects, None)
+
+    def test_embedded_structs(self):
+        class X(Structure):
+            _fields_ = [("a", c_int), ("b", c_int)]
+
+        class Y(Structure):
+            _fields_ = [("x", X), ("y", X)]
+
+        y = Y()
+        self.assertEqual(y._objects, None)
+
+        x1, x2 = X(), X()
+        y.x, y.y = x1, x2
+        self.assertEqual(y._objects, {"0": {}, "1": {}})
+        x1.a, x2.b = 42, 93
+        self.assertEqual(y._objects, {"0": {}, "1": {}})
+
+    def test_xxx(self):
+        class X(Structure):
+            _fields_ = [("a", c_char_p), ("b", c_char_p)]
+
+        class Y(Structure):
+            _fields_ = [("x", X), ("y", X)]
+
+        s1 = b"Hello, World"
+        s2 = b"Hallo, Welt"
+
+        x = X()
+        x.a = s1
+        x.b = s2
+        self.assertEqual(x._objects, {"0": s1, "1": s2})
+
+        y = Y()
+        y.x = x
+        self.assertEqual(y._objects, {"0": {"0": s1, "1": s2}})
+##        x = y.x
+##        del y
+##        print x._b_base_._objects
+
+    def test_ptr_struct(self):
+        class X(Structure):
+            _fields_ = [("data", POINTER(c_int))]
+
+        A = c_int*4
+        a = A(11, 22, 33, 44)
+        self.assertEqual(a._objects, None)
+
+        x = X()
+        x.data = a
+##XXX        print x._objects
+##XXX        print x.data[0]
+##XXX        print x.data._objects
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_keeprefs.py

@@ -0,0 +1,153 @@
+from ctypes import *
+import unittest
+
+class SimpleTestCase(unittest.TestCase):
+    def test_cint(self):
+        x = c_int()
+        self.assertEqual(x._objects, None)
+        x.value = 42
+        self.assertEqual(x._objects, None)
+        x = c_int(99)
+        self.assertEqual(x._objects, None)
+
+    def test_ccharp(self):
+        x = c_char_p()
+        self.assertEqual(x._objects, None)
+        x.value = b"abc"
+        self.assertEqual(x._objects, b"abc")
+        x = c_char_p(b"spam")
+        self.assertEqual(x._objects, b"spam")
+
+class StructureTestCase(unittest.TestCase):
+    def test_cint_struct(self):
+        class X(Structure):
+            _fields_ = [("a", c_int),
+                        ("b", c_int)]
+
+        x = X()
+        self.assertEqual(x._objects, None)
+        x.a = 42
+        x.b = 99
+        self.assertEqual(x._objects, None)
+
+    def test_ccharp_struct(self):
+        class X(Structure):
+            _fields_ = [("a", c_char_p),
+                        ("b", c_char_p)]
+        x = X()
+        self.assertEqual(x._objects, None)
+
+        x.a = b"spam"
+        x.b = b"foo"
+        self.assertEqual(x._objects, {"0": b"spam", "1": b"foo"})
+
+    def test_struct_struct(self):
+        class POINT(Structure):
+            _fields_ = [("x", c_int), ("y", c_int)]
+        class RECT(Structure):
+            _fields_ = [("ul", POINT), ("lr", POINT)]
+
+        r = RECT()
+        r.ul.x = 0
+        r.ul.y = 1
+        r.lr.x = 2
+        r.lr.y = 3
+        self.assertEqual(r._objects, None)
+
+        r = RECT()
+        pt = POINT(1, 2)
+        r.ul = pt
+        self.assertEqual(r._objects, {'0': {}})
+        r.ul.x = 22
+        r.ul.y = 44
+        self.assertEqual(r._objects, {'0': {}})
+        r.lr = POINT()
+        self.assertEqual(r._objects, {'0': {}, '1': {}})
+
+class ArrayTestCase(unittest.TestCase):
+    def test_cint_array(self):
+        INTARR = c_int * 3
+
+        ia = INTARR()
+        self.assertEqual(ia._objects, None)
+        ia[0] = 1
+        ia[1] = 2
+        ia[2] = 3
+        self.assertEqual(ia._objects, None)
+
+        class X(Structure):
+            _fields_ = [("x", c_int),
+                        ("a", INTARR)]
+
+        x = X()
+        x.x = 1000
+        x.a[0] = 42
+        x.a[1] = 96
+        self.assertEqual(x._objects, None)
+        x.a = ia
+        self.assertEqual(x._objects, {'1': {}})
+
+class PointerTestCase(unittest.TestCase):
+    def test_p_cint(self):
+        i = c_int(42)
+        x = pointer(i)
+        self.assertEqual(x._objects, {'1': i})
+
+class DeletePointerTestCase(unittest.TestCase):
+    @unittest.skip('test disabled')
+    def test_X(self):
+        class X(Structure):
+            _fields_ = [("p", POINTER(c_char_p))]
+        x = X()
+        i = c_char_p("abc def")
+        from sys import getrefcount as grc
+        print("2?", grc(i))
+        x.p = pointer(i)
+        print("3?", grc(i))
+        for i in range(320):
+            c_int(99)
+            x.p[0]
+        print(x.p[0])
+##        del x
+##        print "2?", grc(i)
+##        del i
+        import gc
+        gc.collect()
+        for i in range(320):
+            c_int(99)
+            x.p[0]
+        print(x.p[0])
+        print(x.p.contents)
+##        print x._objects
+
+        x.p[0] = "spam spam"
+##        print x.p[0]
+        print("+" * 42)
+        print(x._objects)
+
+class PointerToStructure(unittest.TestCase):
+    def test(self):
+        class POINT(Structure):
+            _fields_ = [("x", c_int), ("y", c_int)]
+        class RECT(Structure):
+            _fields_ = [("a", POINTER(POINT)),
+                        ("b", POINTER(POINT))]
+        r = RECT()
+        p1 = POINT(1, 2)
+
+        r.a = pointer(p1)
+        r.b = pointer(p1)
+##        from pprint import pprint as pp
+##        pp(p1._objects)
+##        pp(r._objects)
+
+        r.a[0].x = 42
+        r.a[0].y = 99
+
+        # to avoid leaking when tests are run several times
+        # clean up the types left in the cache.
+        from ctypes import _pointer_type_cache
+        del _pointer_type_cache[POINT]
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_libc.py

@@ -0,0 +1,33 @@
+import unittest
+
+from ctypes import *
+import _ctypes_test
+
+lib = CDLL(_ctypes_test.__file__)
+
+def three_way_cmp(x, y):
+    """Return -1 if x < y, 0 if x == y and 1 if x > y"""
+    return (x > y) - (x < y)
+
+class LibTest(unittest.TestCase):
+    def test_sqrt(self):
+        lib.my_sqrt.argtypes = c_double,
+        lib.my_sqrt.restype = c_double
+        self.assertEqual(lib.my_sqrt(4.0), 2.0)
+        import math
+        self.assertEqual(lib.my_sqrt(2.0), math.sqrt(2.0))
+
+    def test_qsort(self):
+        comparefunc = CFUNCTYPE(c_int, POINTER(c_char), POINTER(c_char))
+        lib.my_qsort.argtypes = c_void_p, c_size_t, c_size_t, comparefunc
+        lib.my_qsort.restype = None
+
+        def sort(a, b):
+            return three_way_cmp(a[0], b[0])
+
+        chars = create_string_buffer(b"spam, spam, and spam")
+        lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort))
+        self.assertEqual(chars.raw, b"   ,,aaaadmmmnpppsss\x00")
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_loading.py

@@ -0,0 +1,182 @@
+from ctypes import *
+import os
+import shutil
+import subprocess
+import sys
+import unittest
+import test.support
+from test.support import import_helper
+from test.support import os_helper
+from ctypes.util import find_library
+
+libc_name = None
+
+def setUpModule():
+    global libc_name
+    if os.name == "nt":
+        libc_name = find_library("c")
+    elif sys.platform == "cygwin":
+        libc_name = "cygwin1.dll"
+    else:
+        libc_name = find_library("c")
+
+    if test.support.verbose:
+        print("libc_name is", libc_name)
+
+class LoaderTest(unittest.TestCase):
+
+    unknowndll = "xxrandomnamexx"
+
+    def test_load(self):
+        if libc_name is None:
+            self.skipTest('could not find libc')
+        CDLL(libc_name)
+        CDLL(os.path.basename(libc_name))
+        self.assertRaises(OSError, CDLL, self.unknowndll)
+
+    def test_load_version(self):
+        if libc_name is None:
+            self.skipTest('could not find libc')
+        if os.path.basename(libc_name) != 'libc.so.6':
+            self.skipTest('wrong libc path for test')
+        cdll.LoadLibrary("libc.so.6")
+        # linux uses version, libc 9 should not exist
+        self.assertRaises(OSError, cdll.LoadLibrary, "libc.so.9")
+        self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll)
+
+    def test_find(self):
+        for name in ("c", "m"):
+            lib = find_library(name)
+            if lib:
+                cdll.LoadLibrary(lib)
+                CDLL(lib)
+
+    @unittest.skipUnless(os.name == "nt",
+                         'test specific to Windows')
+    def test_load_library(self):
+        # CRT is no longer directly loadable. See issue23606 for the
+        # discussion about alternative approaches.
+        #self.assertIsNotNone(libc_name)
+        if test.support.verbose:
+            print(find_library("kernel32"))
+            print(find_library("user32"))
+
+        if os.name == "nt":
+            windll.kernel32.GetModuleHandleW
+            windll["kernel32"].GetModuleHandleW
+            windll.LoadLibrary("kernel32").GetModuleHandleW
+            WinDLL("kernel32").GetModuleHandleW
+            # embedded null character
+            self.assertRaises(ValueError, windll.LoadLibrary, "kernel32\0")
+
+    @unittest.skipUnless(os.name == "nt",
+                         'test specific to Windows')
+    def test_load_ordinal_functions(self):
+        import _ctypes_test
+        dll = WinDLL(_ctypes_test.__file__)
+        # We load the same function both via ordinal and name
+        func_ord = dll[2]
+        func_name = dll.GetString
+        # addressof gets the address where the function pointer is stored
+        a_ord = addressof(func_ord)
+        a_name = addressof(func_name)
+        f_ord_addr = c_void_p.from_address(a_ord).value
+        f_name_addr = c_void_p.from_address(a_name).value
+        self.assertEqual(hex(f_ord_addr), hex(f_name_addr))
+
+        self.assertRaises(AttributeError, dll.__getitem__, 1234)
+
+    @unittest.skipUnless(os.name == "nt", 'Windows-specific test')
+    def test_1703286_A(self):
+        from _ctypes import LoadLibrary, FreeLibrary
+        # On winXP 64-bit, advapi32 loads at an address that does
+        # NOT fit into a 32-bit integer.  FreeLibrary must be able
+        # to accept this address.
+
+        # These are tests for https://www.python.org/sf/1703286
+        handle = LoadLibrary("advapi32")
+        FreeLibrary(handle)
+
+    @unittest.skipUnless(os.name == "nt", 'Windows-specific test')
+    def test_1703286_B(self):
+        # Since on winXP 64-bit advapi32 loads like described
+        # above, the (arbitrarily selected) CloseEventLog function
+        # also has a high address.  'call_function' should accept
+        # addresses so large.
+        from _ctypes import call_function
+        advapi32 = windll.advapi32
+        # Calling CloseEventLog with a NULL argument should fail,
+        # but the call should not segfault or so.
+        self.assertEqual(0, advapi32.CloseEventLog(None))
+        windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p
+        windll.kernel32.GetProcAddress.restype = c_void_p
+        proc = windll.kernel32.GetProcAddress(advapi32._handle,
+                                              b"CloseEventLog")
+        self.assertTrue(proc)
+        # This is the real test: call the function via 'call_function'
+        self.assertEqual(0, call_function(proc, (None,)))
+
+    @unittest.skipUnless(os.name == "nt",
+                         'test specific to Windows')
+    def test_load_dll_with_flags(self):
+        _sqlite3 = import_helper.import_module("_sqlite3")
+        src = _sqlite3.__file__
+        if src.lower().endswith("_d.pyd"):
+            ext = "_d.dll"
+        else:
+            ext = ".dll"
+
+        with os_helper.temp_dir() as tmp:
+            # We copy two files and load _sqlite3.dll (formerly .pyd),
+            # which has a dependency on sqlite3.dll. Then we test
+            # loading it in subprocesses to avoid it starting in memory
+            # for each test.
+            target = os.path.join(tmp, "_sqlite3.dll")
+            shutil.copy(src, target)
+            shutil.copy(os.path.join(os.path.dirname(src), "sqlite3" + ext),
+                        os.path.join(tmp, "sqlite3" + ext))
+
+            def should_pass(command):
+                with self.subTest(command):
+                    subprocess.check_output(
+                        [sys.executable, "-c",
+                         "from ctypes import *; import nt;" + command],
+                        cwd=tmp
+                    )
+
+            def should_fail(command):
+                with self.subTest(command):
+                    with self.assertRaises(subprocess.CalledProcessError):
+                        subprocess.check_output(
+                            [sys.executable, "-c",
+                             "from ctypes import *; import nt;" + command],
+                            cwd=tmp, stderr=subprocess.STDOUT,
+                        )
+
+            # Default load should not find this in CWD
+            should_fail("WinDLL('_sqlite3.dll')")
+
+            # Relative path (but not just filename) should succeed
+            should_pass("WinDLL('./_sqlite3.dll')")
+
+            # Insecure load flags should succeed
+            # Clear the DLL directory to avoid safe search settings propagating
+            should_pass("windll.kernel32.SetDllDirectoryW(None); WinDLL('_sqlite3.dll', winmode=0)")
+
+            # Full path load without DLL_LOAD_DIR shouldn't find dependency
+            should_fail("WinDLL(nt._getfullpathname('_sqlite3.dll'), " +
+                        "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32)")
+
+            # Full path load with DLL_LOAD_DIR should succeed
+            should_pass("WinDLL(nt._getfullpathname('_sqlite3.dll'), " +
+                        "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32|" +
+                        "nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR)")
+
+            # User-specified directory should succeed
+            should_pass("import os; p = os.add_dll_directory(os.getcwd());" +
+                        "WinDLL('_sqlite3.dll'); p.close()")
+
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_macholib.py

@@ -0,0 +1,110 @@
+import os
+import sys
+import unittest
+
+# Bob Ippolito:
+#
+# Ok.. the code to find the filename for __getattr__ should look
+# something like:
+#
+# import os
+# from macholib.dyld import dyld_find
+#
+# def find_lib(name):
+#      possible = ['lib'+name+'.dylib', name+'.dylib',
+#      name+'.framework/'+name]
+#      for dylib in possible:
+#          try:
+#              return os.path.realpath(dyld_find(dylib))
+#          except ValueError:
+#              pass
+#      raise ValueError, "%s not found" % (name,)
+#
+# It'll have output like this:
+#
+#  >>> find_lib('pthread')
+# '/usr/lib/libSystem.B.dylib'
+#  >>> find_lib('z')
+# '/usr/lib/libz.1.dylib'
+#  >>> find_lib('IOKit')
+# '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit'
+#
+# -bob
+
+from ctypes.macholib.dyld import dyld_find
+from ctypes.macholib.dylib import dylib_info
+from ctypes.macholib.framework import framework_info
+
+def find_lib(name):
+    possible = ['lib'+name+'.dylib', name+'.dylib', name+'.framework/'+name]
+    for dylib in possible:
+        try:
+            return os.path.realpath(dyld_find(dylib))
+        except ValueError:
+            pass
+    raise ValueError("%s not found" % (name,))
+
+
+def d(location=None, name=None, shortname=None, version=None, suffix=None):
+    return {'location': location, 'name': name, 'shortname': shortname,
+            'version': version, 'suffix': suffix}
+
+
+class MachOTest(unittest.TestCase):
+    @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
+    def test_find(self):
+        self.assertEqual(dyld_find('libSystem.dylib'),
+                         '/usr/lib/libSystem.dylib')
+        self.assertEqual(dyld_find('System.framework/System'),
+                         '/System/Library/Frameworks/System.framework/System')
+
+        # On Mac OS 11, system dylibs are only present in the shared cache,
+        # so symlinks like libpthread.dylib -> libSystem.B.dylib will not
+        # be resolved by dyld_find
+        self.assertIn(find_lib('pthread'),
+                              ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib'))
+
+        result = find_lib('z')
+        # Issue #21093: dyld default search path includes $HOME/lib and
+        # /usr/local/lib before /usr/lib, which caused test failures if
+        # a local copy of libz exists in one of them. Now ignore the head
+        # of the path.
+        self.assertRegex(result, r".*/lib/libz.*\.dylib")
+
+        self.assertIn(find_lib('IOKit'),
+                              ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit',
+                              '/System/Library/Frameworks/IOKit.framework/IOKit'))
+
+    @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
+    def test_info(self):
+        self.assertIsNone(dylib_info('completely/invalid'))
+        self.assertIsNone(dylib_info('completely/invalide_debug'))
+        self.assertEqual(dylib_info('P/Foo.dylib'), d('P', 'Foo.dylib', 'Foo'))
+        self.assertEqual(dylib_info('P/Foo_debug.dylib'),
+                         d('P', 'Foo_debug.dylib', 'Foo', suffix='debug'))
+        self.assertEqual(dylib_info('P/Foo.A.dylib'),
+                         d('P', 'Foo.A.dylib', 'Foo', 'A'))
+        self.assertEqual(dylib_info('P/Foo_debug.A.dylib'),
+                         d('P', 'Foo_debug.A.dylib', 'Foo_debug', 'A'))
+        self.assertEqual(dylib_info('P/Foo.A_debug.dylib'),
+                         d('P', 'Foo.A_debug.dylib', 'Foo', 'A', 'debug'))
+
+    @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
+    def test_framework_info(self):
+        self.assertIsNone(framework_info('completely/invalid'))
+        self.assertIsNone(framework_info('completely/invalid/_debug'))
+        self.assertIsNone(framework_info('P/F.framework'))
+        self.assertIsNone(framework_info('P/F.framework/_debug'))
+        self.assertEqual(framework_info('P/F.framework/F'),
+                         d('P', 'F.framework/F', 'F'))
+        self.assertEqual(framework_info('P/F.framework/F_debug'),
+                         d('P', 'F.framework/F_debug', 'F', suffix='debug'))
+        self.assertIsNone(framework_info('P/F.framework/Versions'))
+        self.assertIsNone(framework_info('P/F.framework/Versions/A'))
+        self.assertEqual(framework_info('P/F.framework/Versions/A/F'),
+                         d('P', 'F.framework/Versions/A/F', 'F', 'A'))
+        self.assertEqual(framework_info('P/F.framework/Versions/A/F_debug'),
+                         d('P', 'F.framework/Versions/A/F_debug', 'F', 'A', 'debug'))
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_memfunctions.py

@@ -0,0 +1,79 @@
+import sys
+from test import support
+import unittest
+from ctypes import *
+from ctypes.test import need_symbol
+
+class MemFunctionsTest(unittest.TestCase):
+    @unittest.skip('test disabled')
+    def test_overflow(self):
+        # string_at and wstring_at must use the Python calling
+        # convention (which acquires the GIL and checks the Python
+        # error flag).  Provoke an error and catch it; see also issue
+        # #3554: <http://bugs.python.org/issue3554>
+        self.assertRaises((OverflowError, MemoryError, SystemError),
+                          lambda: wstring_at(u"foo", sys.maxint - 1))
+        self.assertRaises((OverflowError, MemoryError, SystemError),
+                          lambda: string_at("foo", sys.maxint - 1))
+
+    def test_memmove(self):
+        # large buffers apparently increase the chance that the memory
+        # is allocated in high address space.
+        a = create_string_buffer(1000000)
+        p = b"Hello, World"
+        result = memmove(a, p, len(p))
+        self.assertEqual(a.value, b"Hello, World")
+
+        self.assertEqual(string_at(result), b"Hello, World")
+        self.assertEqual(string_at(result, 5), b"Hello")
+        self.assertEqual(string_at(result, 16), b"Hello, World\0\0\0\0")
+        self.assertEqual(string_at(result, 0), b"")
+
+    def test_memset(self):
+        a = create_string_buffer(1000000)
+        result = memset(a, ord('x'), 16)
+        self.assertEqual(a.value, b"xxxxxxxxxxxxxxxx")
+
+        self.assertEqual(string_at(result), b"xxxxxxxxxxxxxxxx")
+        self.assertEqual(string_at(a), b"xxxxxxxxxxxxxxxx")
+        self.assertEqual(string_at(a, 20), b"xxxxxxxxxxxxxxxx\0\0\0\0")
+
+    def test_cast(self):
+        a = (c_ubyte * 32)(*map(ord, "abcdef"))
+        self.assertEqual(cast(a, c_char_p).value, b"abcdef")
+        self.assertEqual(cast(a, POINTER(c_byte))[:7],
+                             [97, 98, 99, 100, 101, 102, 0])
+        self.assertEqual(cast(a, POINTER(c_byte))[:7:],
+                             [97, 98, 99, 100, 101, 102, 0])
+        self.assertEqual(cast(a, POINTER(c_byte))[6:-1:-1],
+                             [0, 102, 101, 100, 99, 98, 97])
+        self.assertEqual(cast(a, POINTER(c_byte))[:7:2],
+                             [97, 99, 101, 0])
+        self.assertEqual(cast(a, POINTER(c_byte))[:7:7],
+                             [97])
+
+    @support.refcount_test
+    def test_string_at(self):
+        s = string_at(b"foo bar")
+        # XXX The following may be wrong, depending on how Python
+        # manages string instances
+        self.assertEqual(2, sys.getrefcount(s))
+        self.assertTrue(s, "foo bar")
+
+        self.assertEqual(string_at(b"foo bar", 7), b"foo bar")
+        self.assertEqual(string_at(b"foo bar", 3), b"foo")
+
+    @need_symbol('create_unicode_buffer')
+    def test_wstring_at(self):
+        p = create_unicode_buffer("Hello, World")
+        a = create_unicode_buffer(1000000)
+        result = memmove(a, p, len(p) * sizeof(c_wchar))
+        self.assertEqual(a.value, "Hello, World")
+
+        self.assertEqual(wstring_at(a), "Hello, World")
+        self.assertEqual(wstring_at(a, 5), "Hello")
+        self.assertEqual(wstring_at(a, 16), "Hello, World\0\0\0\0")
+        self.assertEqual(wstring_at(a, 0), "")
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_numbers.py

@@ -0,0 +1,295 @@
+from ctypes import *
+import unittest
+import struct
+
+def valid_ranges(*types):
+    # given a sequence of numeric types, collect their _type_
+    # attribute, which is a single format character compatible with
+    # the struct module, use the struct module to calculate the
+    # minimum and maximum value allowed for this format.
+    # Returns a list of (min, max) values.
+    result = []
+    for t in types:
+        fmt = t._type_
+        size = struct.calcsize(fmt)
+        a = struct.unpack(fmt, (b"\x00"*32)[:size])[0]
+        b = struct.unpack(fmt, (b"\xFF"*32)[:size])[0]
+        c = struct.unpack(fmt, (b"\x7F"+b"\x00"*32)[:size])[0]
+        d = struct.unpack(fmt, (b"\x80"+b"\xFF"*32)[:size])[0]
+        result.append((min(a, b, c, d), max(a, b, c, d)))
+    return result
+
+ArgType = type(byref(c_int(0)))
+
+unsigned_types = [c_ubyte, c_ushort, c_uint, c_ulong]
+signed_types = [c_byte, c_short, c_int, c_long, c_longlong]
+
+bool_types = []
+
+float_types = [c_double, c_float]
+
+try:
+    c_ulonglong
+    c_longlong
+except NameError:
+    pass
+else:
+    unsigned_types.append(c_ulonglong)
+    signed_types.append(c_longlong)
+
+try:
+    c_bool
+except NameError:
+    pass
+else:
+    bool_types.append(c_bool)
+
+unsigned_ranges = valid_ranges(*unsigned_types)
+signed_ranges = valid_ranges(*signed_types)
+bool_values = [True, False, 0, 1, -1, 5000, 'test', [], [1]]
+
+################################################################
+
+class NumberTestCase(unittest.TestCase):
+
+    def test_default_init(self):
+        # default values are set to zero
+        for t in signed_types + unsigned_types + float_types:
+            self.assertEqual(t().value, 0)
+
+    def test_unsigned_values(self):
+        # the value given to the constructor is available
+        # as the 'value' attribute
+        for t, (l, h) in zip(unsigned_types, unsigned_ranges):
+            self.assertEqual(t(l).value, l)
+            self.assertEqual(t(h).value, h)
+
+    def test_signed_values(self):
+        # see above
+        for t, (l, h) in zip(signed_types, signed_ranges):
+            self.assertEqual(t(l).value, l)
+            self.assertEqual(t(h).value, h)
+
+    def test_bool_values(self):
+        from operator import truth
+        for t, v in zip(bool_types, bool_values):
+            self.assertEqual(t(v).value, truth(v))
+
+    def test_typeerror(self):
+        # Only numbers are allowed in the constructor,
+        # otherwise TypeError is raised
+        for t in signed_types + unsigned_types + float_types:
+            self.assertRaises(TypeError, t, "")
+            self.assertRaises(TypeError, t, None)
+
+    @unittest.skip('test disabled')
+    def test_valid_ranges(self):
+        # invalid values of the correct type
+        # raise ValueError (not OverflowError)
+        for t, (l, h) in zip(unsigned_types, unsigned_ranges):
+            self.assertRaises(ValueError, t, l-1)
+            self.assertRaises(ValueError, t, h+1)
+
+    def test_from_param(self):
+        # the from_param class method attribute always
+        # returns PyCArgObject instances
+        for t in signed_types + unsigned_types + float_types:
+            self.assertEqual(ArgType, type(t.from_param(0)))
+
+    def test_byref(self):
+        # calling byref returns also a PyCArgObject instance
+        for t in signed_types + unsigned_types + float_types + bool_types:
+            parm = byref(t())
+            self.assertEqual(ArgType, type(parm))
+
+
+    def test_floats(self):
+        # c_float and c_double can be created from
+        # Python int and float
+        class FloatLike(object):
+            def __float__(self):
+                return 2.0
+        f = FloatLike()
+        for t in float_types:
+            self.assertEqual(t(2.0).value, 2.0)
+            self.assertEqual(t(2).value, 2.0)
+            self.assertEqual(t(2).value, 2.0)
+            self.assertEqual(t(f).value, 2.0)
+
+    def test_integers(self):
+        class FloatLike(object):
+            def __float__(self):
+                return 2.0
+        f = FloatLike()
+        class IntLike(object):
+            def __int__(self):
+                return 2
+        d = IntLike()
+        class IndexLike(object):
+            def __index__(self):
+                return 2
+        i = IndexLike()
+        # integers cannot be constructed from floats,
+        # but from integer-like objects
+        for t in signed_types + unsigned_types:
+            self.assertRaises(TypeError, t, 3.14)
+            self.assertRaises(TypeError, t, f)
+            self.assertRaises(TypeError, t, d)
+            self.assertEqual(t(i).value, 2)
+
+    def test_sizes(self):
+        for t in signed_types + unsigned_types + float_types + bool_types:
+            try:
+                size = struct.calcsize(t._type_)
+            except struct.error:
+                continue
+            # sizeof of the type...
+            self.assertEqual(sizeof(t), size)
+            # and sizeof of an instance
+            self.assertEqual(sizeof(t()), size)
+
+    def test_alignments(self):
+        for t in signed_types + unsigned_types + float_types:
+            code = t._type_ # the typecode
+            align = struct.calcsize("c%c" % code) - struct.calcsize(code)
+
+            # alignment of the type...
+            self.assertEqual((code, alignment(t)),
+                                 (code, align))
+            # and alignment of an instance
+            self.assertEqual((code, alignment(t())),
+                                 (code, align))
+
+    def test_int_from_address(self):
+        from array import array
+        for t in signed_types + unsigned_types:
+            # the array module doesn't support all format codes
+            # (no 'q' or 'Q')
+            try:
+                array(t._type_)
+            except ValueError:
+                continue
+            a = array(t._type_, [100])
+
+            # v now is an integer at an 'external' memory location
+            v = t.from_address(a.buffer_info()[0])
+            self.assertEqual(v.value, a[0])
+            self.assertEqual(type(v), t)
+
+            # changing the value at the memory location changes v's value also
+            a[0] = 42
+            self.assertEqual(v.value, a[0])
+
+
+    def test_float_from_address(self):
+        from array import array
+        for t in float_types:
+            a = array(t._type_, [3.14])
+            v = t.from_address(a.buffer_info()[0])
+            self.assertEqual(v.value, a[0])
+            self.assertIs(type(v), t)
+            a[0] = 2.3456e17
+            self.assertEqual(v.value, a[0])
+            self.assertIs(type(v), t)
+
+    def test_char_from_address(self):
+        from ctypes import c_char
+        from array import array
+
+        a = array('b', [0])
+        a[0] = ord('x')
+        v = c_char.from_address(a.buffer_info()[0])
+        self.assertEqual(v.value, b'x')
+        self.assertIs(type(v), c_char)
+
+        a[0] = ord('?')
+        self.assertEqual(v.value, b'?')
+
+    # array does not support c_bool / 't'
+    @unittest.skip('test disabled')
+    def test_bool_from_address(self):
+        from ctypes import c_bool
+        from array import array
+        a = array(c_bool._type_, [True])
+        v = t.from_address(a.buffer_info()[0])
+        self.assertEqual(v.value, a[0])
+        self.assertEqual(type(v) is t)
+        a[0] = False
+        self.assertEqual(v.value, a[0])
+        self.assertEqual(type(v) is t)
+
+    def test_init(self):
+        # c_int() can be initialized from Python's int, and c_int.
+        # Not from c_long or so, which seems strange, abc should
+        # probably be changed:
+        self.assertRaises(TypeError, c_int, c_long(42))
+
+    def test_float_overflow(self):
+        import sys
+        big_int = int(sys.float_info.max) * 2
+        for t in float_types + [c_longdouble]:
+            self.assertRaises(OverflowError, t, big_int)
+            if (hasattr(t, "__ctype_be__")):
+                self.assertRaises(OverflowError, t.__ctype_be__, big_int)
+            if (hasattr(t, "__ctype_le__")):
+                self.assertRaises(OverflowError, t.__ctype_le__, big_int)
+
+    @unittest.skip('test disabled')
+    def test_perf(self):
+        check_perf()
+
+from ctypes import _SimpleCData
+class c_int_S(_SimpleCData):
+    _type_ = "i"
+    __slots__ = []
+
+def run_test(rep, msg, func, arg=None):
+##    items = [None] * rep
+    items = range(rep)
+    from time import perf_counter as clock
+    if arg is not None:
+        start = clock()
+        for i in items:
+            func(arg); func(arg); func(arg); func(arg); func(arg)
+        stop = clock()
+    else:
+        start = clock()
+        for i in items:
+            func(); func(); func(); func(); func()
+        stop = clock()
+    print("%15s: %.2f us" % (msg, ((stop-start)*1e6/5/rep)))
+
+def check_perf():
+    # Construct 5 objects
+    from ctypes import c_int
+
+    REP = 200000
+
+    run_test(REP, "int()", int)
+    run_test(REP, "int(999)", int)
+    run_test(REP, "c_int()", c_int)
+    run_test(REP, "c_int(999)", c_int)
+    run_test(REP, "c_int_S()", c_int_S)
+    run_test(REP, "c_int_S(999)", c_int_S)
+
+# Python 2.3 -OO, win2k, P4 700 MHz:
+#
+#          int(): 0.87 us
+#       int(999): 0.87 us
+#        c_int(): 3.35 us
+#     c_int(999): 3.34 us
+#      c_int_S(): 3.23 us
+#   c_int_S(999): 3.24 us
+
+# Python 2.2 -OO, win2k, P4 700 MHz:
+#
+#          int(): 0.89 us
+#       int(999): 0.89 us
+#        c_int(): 9.99 us
+#     c_int(999): 10.02 us
+#      c_int_S(): 9.87 us
+#   c_int_S(999): 9.85 us
+
+if __name__ == '__main__':
+##    check_perf()
+    unittest.main()

Lib/ctypes/test/test_objects.py

@@ -0,0 +1,67 @@
+r'''
+This tests the '_objects' attribute of ctypes instances.  '_objects'
+holds references to objects that must be kept alive as long as the
+ctypes instance, to make sure that the memory buffer is valid.
+
+WARNING: The '_objects' attribute is exposed ONLY for debugging ctypes itself,
+it MUST NEVER BE MODIFIED!
+
+'_objects' is initialized to a dictionary on first use, before that it
+is None.
+
+Here is an array of string pointers:
+
+>>> from ctypes import *
+>>> array = (c_char_p * 5)()
+>>> print(array._objects)
+None
+>>>
+
+The memory block stores pointers to strings, and the strings itself
+assigned from Python must be kept.
+
+>>> array[4] = b'foo bar'
+>>> array._objects
+{'4': b'foo bar'}
+>>> array[4]
+b'foo bar'
+>>>
+
+It gets more complicated when the ctypes instance itself is contained
+in a 'base' object.
+
+>>> class X(Structure):
+...     _fields_ = [("x", c_int), ("y", c_int), ("array", c_char_p * 5)]
+...
+>>> x = X()
+>>> print(x._objects)
+None
+>>>
+
+The'array' attribute of the 'x' object shares part of the memory buffer
+of 'x' ('_b_base_' is either None, or the root object owning the memory block):
+
+>>> print(x.array._b_base_) # doctest: +ELLIPSIS
+<ctypes.test.test_objects.X object at 0x...>
+>>>
+
+>>> x.array[0] = b'spam spam spam'
+>>> x._objects
+{'0:2': b'spam spam spam'}
+>>> x.array._b_base_._objects
+{'0:2': b'spam spam spam'}
+>>>
+
+'''
+
+import unittest, doctest
+
+import ctypes.test.test_objects
+
+class TestCase(unittest.TestCase):
+    def test(self):
+        failures, tests = doctest.testmod(ctypes.test.test_objects)
+        self.assertFalse(failures, 'doctests failed, see output above')
+
+if __name__ == '__main__':
+    doctest.testmod(ctypes.test.test_objects)

Lib/ctypes/test/test_parameters.py

@@ -0,0 +1,250 @@
+import unittest
+from ctypes.test import need_symbol
+import test.support
+
+class SimpleTypesTestCase(unittest.TestCase):
+
+    def setUp(self):
+        import ctypes
+        try:
+            from _ctypes import set_conversion_mode
+        except ImportError:
+            pass
+        else:
+            self.prev_conv_mode = set_conversion_mode("ascii", "strict")
+
+    def tearDown(self):
+        try:
+            from _ctypes import set_conversion_mode
+        except ImportError:
+            pass
+        else:
+            set_conversion_mode(*self.prev_conv_mode)
+
+    def test_subclasses(self):
+        from ctypes import c_void_p, c_char_p
+        # ctypes 0.9.5 and before did overwrite from_param in SimpleType_new
+        class CVOIDP(c_void_p):
+            def from_param(cls, value):
+                return value * 2
+            from_param = classmethod(from_param)
+
+        class CCHARP(c_char_p):
+            def from_param(cls, value):
+                return value * 4
+            from_param = classmethod(from_param)
+
+        self.assertEqual(CVOIDP.from_param("abc"), "abcabc")
+        self.assertEqual(CCHARP.from_param("abc"), "abcabcabcabc")
+
+    @need_symbol('c_wchar_p')
+    def test_subclasses_c_wchar_p(self):
+        from ctypes import c_wchar_p
+
+        class CWCHARP(c_wchar_p):
+            def from_param(cls, value):
+                return value * 3
+            from_param = classmethod(from_param)
+
+        self.assertEqual(CWCHARP.from_param("abc"), "abcabcabc")
+
+    # XXX Replace by c_char_p tests
+    def test_cstrings(self):
+        from ctypes import c_char_p
+
+        # c_char_p.from_param on a Python String packs the string
+        # into a cparam object
+        s = b"123"
+        self.assertIs(c_char_p.from_param(s)._obj, s)
+
+        # new in 0.9.1: convert (encode) unicode to ascii
+        self.assertEqual(c_char_p.from_param(b"123")._obj, b"123")
+        self.assertRaises(TypeError, c_char_p.from_param, "123\377")
+        self.assertRaises(TypeError, c_char_p.from_param, 42)
+
+        # calling c_char_p.from_param with a c_char_p instance
+        # returns the argument itself:
+        a = c_char_p(b"123")
+        self.assertIs(c_char_p.from_param(a), a)
+
+    @need_symbol('c_wchar_p')
+    def test_cw_strings(self):
+        from ctypes import c_wchar_p
+
+        c_wchar_p.from_param("123")
+
+        self.assertRaises(TypeError, c_wchar_p.from_param, 42)
+        self.assertRaises(TypeError, c_wchar_p.from_param, b"123\377")
+
+        pa = c_wchar_p.from_param(c_wchar_p("123"))
+        self.assertEqual(type(pa), c_wchar_p)
+
+    def test_int_pointers(self):
+        from ctypes import c_short, c_uint, c_int, c_long, POINTER, pointer
+        LPINT = POINTER(c_int)
+
+##        p = pointer(c_int(42))
+##        x = LPINT.from_param(p)
+        x = LPINT.from_param(pointer(c_int(42)))
+        self.assertEqual(x.contents.value, 42)
+        self.assertEqual(LPINT(c_int(42)).contents.value, 42)
+
+        self.assertEqual(LPINT.from_param(None), None)
+
+        if c_int != c_long:
+            self.assertRaises(TypeError, LPINT.from_param, pointer(c_long(42)))
+        self.assertRaises(TypeError, LPINT.from_param, pointer(c_uint(42)))
+        self.assertRaises(TypeError, LPINT.from_param, pointer(c_short(42)))
+
+    def test_byref_pointer(self):
+        # The from_param class method of POINTER(typ) classes accepts what is
+        # returned by byref(obj), it type(obj) == typ
+        from ctypes import c_short, c_uint, c_int, c_long, POINTER, byref
+        LPINT = POINTER(c_int)
+
+        LPINT.from_param(byref(c_int(42)))
+
+        self.assertRaises(TypeError, LPINT.from_param, byref(c_short(22)))
+        if c_int != c_long:
+            self.assertRaises(TypeError, LPINT.from_param, byref(c_long(22)))
+        self.assertRaises(TypeError, LPINT.from_param, byref(c_uint(22)))
+
+    def test_byref_pointerpointer(self):
+        # See above
+        from ctypes import c_short, c_uint, c_int, c_long, pointer, POINTER, byref
+
+        LPLPINT = POINTER(POINTER(c_int))
+        LPLPINT.from_param(byref(pointer(c_int(42))))
+
+        self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_short(22))))
+        if c_int != c_long:
+            self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_long(22))))
+        self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_uint(22))))
+
+    def test_array_pointers(self):
+        from ctypes import c_short, c_uint, c_int, c_long, POINTER
+        INTARRAY = c_int * 3
+        ia = INTARRAY()
+        self.assertEqual(len(ia), 3)
+        self.assertEqual([ia[i] for i in range(3)], [0, 0, 0])
+
+        # Pointers are only compatible with arrays containing items of
+        # the same type!
+        LPINT = POINTER(c_int)
+        LPINT.from_param((c_int*3)())
+        self.assertRaises(TypeError, LPINT.from_param, c_short*3)
+        self.assertRaises(TypeError, LPINT.from_param, c_long*3)
+        self.assertRaises(TypeError, LPINT.from_param, c_uint*3)
+
+    def test_noctypes_argtype(self):
+        import _ctypes_test
+        from ctypes import CDLL, c_void_p, ArgumentError
+
+        func = CDLL(_ctypes_test.__file__)._testfunc_p_p
+        func.restype = c_void_p
+        # TypeError: has no from_param method
+        self.assertRaises(TypeError, setattr, func, "argtypes", (object,))
+
+        class Adapter(object):
+            def from_param(cls, obj):
+                return None
+
+        func.argtypes = (Adapter(),)
+        self.assertEqual(func(None), None)
+        self.assertEqual(func(object()), None)
+
+        class Adapter(object):
+            def from_param(cls, obj):
+                return obj
+
+        func.argtypes = (Adapter(),)
+        # don't know how to convert parameter 1
+        self.assertRaises(ArgumentError, func, object())
+        self.assertEqual(func(c_void_p(42)), 42)
+
+        class Adapter(object):
+            def from_param(cls, obj):
+                raise ValueError(obj)
+
+        func.argtypes = (Adapter(),)
+        # ArgumentError: argument 1: ValueError: 99
+        self.assertRaises(ArgumentError, func, 99)
+
+    def test_abstract(self):
+        from ctypes import (Array, Structure, Union, _Pointer,
+                            _SimpleCData, _CFuncPtr)
+
+        self.assertRaises(TypeError, Array.from_param, 42)
+        self.assertRaises(TypeError, Structure.from_param, 42)
+        self.assertRaises(TypeError, Union.from_param, 42)
+        self.assertRaises(TypeError, _CFuncPtr.from_param, 42)
+        self.assertRaises(TypeError, _Pointer.from_param, 42)
+        self.assertRaises(TypeError, _SimpleCData.from_param, 42)
+
+    @test.support.cpython_only
+    def test_issue31311(self):
+        # __setstate__ should neither raise a SystemError nor crash in case
+        # of a bad __dict__.
+        from ctypes import Structure
+
+        class BadStruct(Structure):
+            @property
+            def __dict__(self):
+                pass
+        with self.assertRaises(TypeError):
+            BadStruct().__setstate__({}, b'foo')
+
+        class WorseStruct(Structure):
+            @property
+            def __dict__(self):
+                1/0
+        with self.assertRaises(ZeroDivisionError):
+            WorseStruct().__setstate__({}, b'foo')
+
+    def test_parameter_repr(self):
+        from ctypes import (
+            c_bool,
+            c_char,
+            c_wchar,
+            c_byte,
+            c_ubyte,
+            c_short,
+            c_ushort,
+            c_int,
+            c_uint,
+            c_long,
+            c_ulong,
+            c_longlong,
+            c_ulonglong,
+            c_float,
+            c_double,
+            c_longdouble,
+            c_char_p,
+            c_wchar_p,
+            c_void_p,
+        )
+        self.assertRegex(repr(c_bool.from_param(True)), r"^<cparam '\?' at 0x[A-Fa-f0-9]+>$")
+        self.assertEqual(repr(c_char.from_param(97)), "<cparam 'c' ('a')>")
+        self.assertRegex(repr(c_wchar.from_param('a')), r"^<cparam 'u' at 0x[A-Fa-f0-9]+>$")
+        self.assertEqual(repr(c_byte.from_param(98)), "<cparam 'b' (98)>")
+        self.assertEqual(repr(c_ubyte.from_param(98)), "<cparam 'B' (98)>")
+        self.assertEqual(repr(c_short.from_param(511)), "<cparam 'h' (511)>")
+        self.assertEqual(repr(c_ushort.from_param(511)), "<cparam 'H' (511)>")
+        self.assertRegex(repr(c_int.from_param(20000)), r"^<cparam '[li]' \(20000\)>$")
+        self.assertRegex(repr(c_uint.from_param(20000)), r"^<cparam '[LI]' \(20000\)>$")
+        self.assertRegex(repr(c_long.from_param(20000)), r"^<cparam '[li]' \(20000\)>$")
+        self.assertRegex(repr(c_ulong.from_param(20000)), r"^<cparam '[LI]' \(20000\)>$")
+        self.assertRegex(repr(c_longlong.from_param(20000)), r"^<cparam '[liq]' \(20000\)>$")
+        self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^<cparam '[LIQ]' \(20000\)>$")
+        self.assertEqual(repr(c_float.from_param(1.5)), "<cparam 'f' (1.5)>")
+        self.assertEqual(repr(c_double.from_param(1.5)), "<cparam 'd' (1.5)>")
+        self.assertEqual(repr(c_double.from_param(1e300)), "<cparam 'd' (1e+300)>")
+        self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^<cparam ('d' \(1.5\)|'g' at 0x[A-Fa-f0-9]+)>$")
+        self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^<cparam 'z' \(0x[A-Fa-f0-9]+\)>$")
+        self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^<cparam 'Z' \(0x[A-Fa-f0-9]+\)>$")
+        self.assertRegex(repr(c_void_p.from_param(0x12)), r"^<cparam 'P' \(0x0*12\)>$")
+
+################################################################
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_pep3118.py

@@ -0,0 +1,235 @@
+import unittest
+from ctypes import *
+import re, sys
+
+if sys.byteorder == "little":
+    THIS_ENDIAN = "<"
+    OTHER_ENDIAN = ">"
+else:
+    THIS_ENDIAN = ">"
+    OTHER_ENDIAN = "<"
+
+def normalize(format):
+    # Remove current endian specifier and white space from a format
+    # string
+    if format is None:
+        return ""
+    format = format.replace(OTHER_ENDIAN, THIS_ENDIAN)
+    return re.sub(r"\s", "", format)
+
+class Test(unittest.TestCase):
+
+    def test_native_types(self):
+        for tp, fmt, shape, itemtp in native_types:
+            ob = tp()
+            v = memoryview(ob)
+            try:
+                self.assertEqual(normalize(v.format), normalize(fmt))
+                if shape:
+                    self.assertEqual(len(v), shape[0])
+                else:
+                    self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob))
+                self.assertEqual(v.itemsize, sizeof(itemtp))
+                self.assertEqual(v.shape, shape)
+                # XXX Issue #12851: PyCData_NewGetBuffer() must provide strides
+                #     if requested. memoryview currently reconstructs missing
+                #     stride information, so this assert will fail.
+                # self.assertEqual(v.strides, ())
+
+                # they are always read/write
+                self.assertFalse(v.readonly)
+
+                if v.shape:
+                    n = 1
+                    for dim in v.shape:
+                        n = n * dim
+                    self.assertEqual(n * v.itemsize, len(v.tobytes()))
+            except:
+                # so that we can see the failing type
+                print(tp)
+                raise
+
+    def test_endian_types(self):
+        for tp, fmt, shape, itemtp in endian_types:
+            ob = tp()
+            v = memoryview(ob)
+            try:
+                self.assertEqual(v.format, fmt)
+                if shape:
+                    self.assertEqual(len(v), shape[0])
+                else:
+                    self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob))
+                self.assertEqual(v.itemsize, sizeof(itemtp))
+                self.assertEqual(v.shape, shape)
+                # XXX Issue #12851
+                # self.assertEqual(v.strides, ())
+
+                # they are always read/write
+                self.assertFalse(v.readonly)
+
+                if v.shape:
+                    n = 1
+                    for dim in v.shape:
+                        n = n * dim
+                    self.assertEqual(n, len(v))
+            except:
+                # so that we can see the failing type
+                print(tp)
+                raise
+
+# define some structure classes
+
+class Point(Structure):
+    _fields_ = [("x", c_long), ("y", c_long)]
+
+class PackedPoint(Structure):
+    _pack_ = 2
+    _fields_ = [("x", c_long), ("y", c_long)]
+
+class Point2(Structure):
+    pass
+Point2._fields_ = [("x", c_long), ("y", c_long)]
+
+class EmptyStruct(Structure):
+    _fields_ = []
+
+class aUnion(Union):
+    _fields_ = [("a", c_int)]
+
+class StructWithArrays(Structure):
+    _fields_ = [("x", c_long * 3 * 2), ("y", Point * 4)]
+
+class Incomplete(Structure):
+    pass
+
+class Complete(Structure):
+    pass
+PComplete = POINTER(Complete)
+Complete._fields_ = [("a", c_long)]
+
+################################################################
+#
+# This table contains format strings as they look on little endian
+# machines.  The test replaces '<' with '>' on big endian machines.
+#
+
+# Platform-specific type codes
+s_bool = {1: '?', 2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_bool)]
+s_short = {2: 'h', 4: 'l', 8: 'q'}[sizeof(c_short)]
+s_ushort = {2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_ushort)]
+s_int = {2: 'h', 4: 'i', 8: 'q'}[sizeof(c_int)]
+s_uint = {2: 'H', 4: 'I', 8: 'Q'}[sizeof(c_uint)]
+s_long = {4: 'l', 8: 'q'}[sizeof(c_long)]
+s_ulong = {4: 'L', 8: 'Q'}[sizeof(c_ulong)]
+s_longlong = "q"
+s_ulonglong = "Q"
+s_float = "f"
+s_double = "d"
+s_longdouble = "g"
+
+# Alias definitions in ctypes/__init__.py
+if c_int is c_long:
+    s_int = s_long
+if c_uint is c_ulong:
+    s_uint = s_ulong
+if c_longlong is c_long:
+    s_longlong = s_long
+if c_ulonglong is c_ulong:
+    s_ulonglong = s_ulong
+if c_longdouble is c_double:
+    s_longdouble = s_double
+
+
+native_types = [
+    # type                      format                  shape           calc itemsize
+
+    ## simple types
+
+    (c_char,                    "<c",                   (),           c_char),
+    (c_byte,                    "<b",                   (),           c_byte),
+    (c_ubyte,                   "<B",                   (),           c_ubyte),
+    (c_short,                   "<" + s_short,          (),           c_short),
+    (c_ushort,                  "<" + s_ushort,         (),           c_ushort),
+
+    (c_int,                     "<" + s_int,            (),           c_int),
+    (c_uint,                    "<" + s_uint,           (),           c_uint),
+
+    (c_long,                    "<" + s_long,           (),           c_long),
+    (c_ulong,                   "<" + s_ulong,          (),           c_ulong),
+
+    (c_longlong,                "<" + s_longlong,       (),           c_longlong),
+    (c_ulonglong,               "<" + s_ulonglong,      (),           c_ulonglong),
+
+    (c_float,                   "<f",                   (),           c_float),
+    (c_double,                  "<d",                   (),           c_double),
+
+    (c_longdouble,              "<" + s_longdouble,     (),           c_longdouble),
+
+    (c_bool,                    "<" + s_bool,           (),           c_bool),
+    (py_object,                 "<O",                   (),           py_object),
+
+    ## pointers
+
+    (POINTER(c_byte),           "&<b",                  (),           POINTER(c_byte)),
+    (POINTER(POINTER(c_long)),  "&&<" + s_long,         (),           POINTER(POINTER(c_long))),
+
+    ## arrays and pointers
+
+    (c_double * 4,              "<d",                   (4,),           c_double),
+    (c_double * 0,              "<d",                   (0,),           c_double),
+    (c_float * 4 * 3 * 2,       "<f",                   (2,3,4),        c_float),
+    (c_float * 4 * 0 * 2,       "<f",                   (2,0,4),        c_float),
+    (POINTER(c_short) * 2,      "&<" + s_short,         (2,),           POINTER(c_short)),
+    (POINTER(c_short) * 2 * 3,  "&<" + s_short,         (3,2,),         POINTER(c_short)),
+    (POINTER(c_short * 2),      "&(2)<" + s_short,      (),             POINTER(c_short)),
+
+    ## structures and unions
+
+    (Point,                     "T{<l:x:<l:y:}".replace('l', s_long),  (),  Point),
+    # packed structures do not implement the pep
+    (PackedPoint,               "B",                                   (),  PackedPoint),
+    (Point2,                    "T{<l:x:<l:y:}".replace('l', s_long),  (),  Point2),
+    (EmptyStruct,               "T{}",                                 (),  EmptyStruct),
+    # the pep doesn't support unions
+    (aUnion,                    "B",                                   (),  aUnion),
+    # structure with sub-arrays
+    (StructWithArrays, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (), StructWithArrays),
+    (StructWithArrays * 3, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (3,), StructWithArrays),
+
+    ## pointer to incomplete structure
+    (Incomplete,                "B",                    (),           Incomplete),
+    (POINTER(Incomplete),       "&B",                   (),           POINTER(Incomplete)),
+
+    # 'Complete' is a structure that starts incomplete, but is completed after the
+    # pointer type to it has been created.
+    (Complete,                  "T{<l:a:}".replace('l', s_long), (), Complete),
+    # Unfortunately the pointer format string is not fixed...
+    (POINTER(Complete),         "&B",                   (),           POINTER(Complete)),
+
+    ## other
+
+    # function signatures are not implemented
+    (CFUNCTYPE(None),           "X{}",                  (),           CFUNCTYPE(None)),
+
+    ]
+
+class BEPoint(BigEndianStructure):
+    _fields_ = [("x", c_long), ("y", c_long)]
+
+class LEPoint(LittleEndianStructure):
+    _fields_ = [("x", c_long), ("y", c_long)]
+
+################################################################
+#
+# This table contains format strings as they really look, on both big
+# and little endian machines.
+#
+endian_types = [
+    (BEPoint, "T{>l:x:>l:y:}".replace('l', s_long), (), BEPoint),
+    (LEPoint, "T{<l:x:<l:y:}".replace('l', s_long), (), LEPoint),
+    (POINTER(BEPoint), "&T{>l:x:>l:y:}".replace('l', s_long), (), POINTER(BEPoint)),
+    (POINTER(LEPoint), "&T{<l:x:<l:y:}".replace('l', s_long), (), POINTER(LEPoint)),
+    ]
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_pickling.py

@@ -0,0 +1,81 @@
+import unittest
+import pickle
+from ctypes import *
+import _ctypes_test
+dll = CDLL(_ctypes_test.__file__)
+
+class X(Structure):
+    _fields_ = [("a", c_int), ("b", c_double)]
+    init_called = 0
+    def __init__(self, *args, **kw):
+        X.init_called += 1
+        self.x = 42
+
+class Y(X):
+    _fields_ = [("str", c_char_p)]
+
+class PickleTest:
+    def dumps(self, item):
+        return pickle.dumps(item, self.proto)
+
+    def loads(self, item):
+        return pickle.loads(item)
+
+    def test_simple(self):
+        for src in [
+            c_int(42),
+            c_double(3.14),
+            ]:
+            dst = self.loads(self.dumps(src))
+            self.assertEqual(src.__dict__, dst.__dict__)
+            self.assertEqual(memoryview(src).tobytes(),
+                                 memoryview(dst).tobytes())
+
+    def test_struct(self):
+        X.init_called = 0
+
+        x = X()
+        x.a = 42
+        self.assertEqual(X.init_called, 1)
+
+        y = self.loads(self.dumps(x))
+
+        # loads must NOT call __init__
+        self.assertEqual(X.init_called, 1)
+
+        # ctypes instances are identical when the instance __dict__
+        # and the memory buffer are identical
+        self.assertEqual(y.__dict__, x.__dict__)
+        self.assertEqual(memoryview(y).tobytes(),
+                             memoryview(x).tobytes())
+
+    def test_unpickable(self):
+        # ctypes objects that are pointers or contain pointers are
+        # unpickable.
+        self.assertRaises(ValueError, lambda: self.dumps(Y()))
+
+        prototype = CFUNCTYPE(c_int)
+
+        for item in [
+            c_char_p(),
+            c_wchar_p(),
+            c_void_p(),
+            pointer(c_int(42)),
+            dll._testfunc_p_p,
+            prototype(lambda: 42),
+            ]:
+            self.assertRaises(ValueError, lambda: self.dumps(item))
+
+    def test_wchar(self):
+        self.dumps(c_char(b"x"))
+        # Issue 5049
+        self.dumps(c_wchar("x"))
+
+for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+    name = 'PickleTest_%s' % proto
+    globals()[name] = type(name,
+                           (PickleTest, unittest.TestCase),
+                           {'proto': proto})
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_pointers.py

@@ -0,0 +1,223 @@
+import unittest, sys
+
+from ctypes import *
+import _ctypes_test
+
+ctype_types = [c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint,
+                 c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
+python_types = [int, int, int, int, int, int,
+                int, int, int, int, float, float]
+
+class PointersTestCase(unittest.TestCase):
+
+    def test_pointer_crash(self):
+
+        class A(POINTER(c_ulong)):
+            pass
+
+        POINTER(c_ulong)(c_ulong(22))
+        # Pointer can't set contents: has no _type_
+        self.assertRaises(TypeError, A, c_ulong(33))
+
+    def test_pass_pointers(self):
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_p_p
+        if sizeof(c_longlong) == sizeof(c_void_p):
+            func.restype = c_longlong
+        else:
+            func.restype = c_long
+
+        i = c_int(12345678)
+##        func.argtypes = (POINTER(c_int),)
+        address = func(byref(i))
+        self.assertEqual(c_int.from_address(address).value, 12345678)
+
+        func.restype = POINTER(c_int)
+        res = func(pointer(i))
+        self.assertEqual(res.contents.value, 12345678)
+        self.assertEqual(res[0], 12345678)
+
+    def test_change_pointers(self):
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_p_p
+
+        i = c_int(87654)
+        func.restype = POINTER(c_int)
+        func.argtypes = (POINTER(c_int),)
+
+        res = func(pointer(i))
+        self.assertEqual(res[0], 87654)
+        self.assertEqual(res.contents.value, 87654)
+
+        # C code: *res = 54345
+        res[0] = 54345
+        self.assertEqual(i.value, 54345)
+
+        # C code:
+        #   int x = 12321;
+        #   res = &x
+        x = c_int(12321)
+        res.contents = x
+        self.assertEqual(i.value, 54345)
+
+        x.value = -99
+        self.assertEqual(res.contents.value, -99)
+
+    def test_callbacks_with_pointers(self):
+        # a function type receiving a pointer
+        PROTOTYPE = CFUNCTYPE(c_int, POINTER(c_int))
+
+        self.result = []
+
+        def func(arg):
+            for i in range(10):
+##                print arg[i],
+                self.result.append(arg[i])
+##            print
+            return 0
+        callback = PROTOTYPE(func)
+
+        dll = CDLL(_ctypes_test.__file__)
+        # This function expects a function pointer,
+        # and calls this with an integer pointer as parameter.
+        # The int pointer points to a table containing the numbers 1..10
+        doit = dll._testfunc_callback_with_pointer
+
+##        i = c_int(42)
+##        callback(byref(i))
+##        self.assertEqual(i.value, 84)
+
+        doit(callback)
+##        print self.result
+        doit(callback)
+##        print self.result
+
+    def test_basics(self):
+        from operator import delitem
+        for ct, pt in zip(ctype_types, python_types):
+            i = ct(42)
+            p = pointer(i)
+##            print type(p.contents), ct
+            self.assertIs(type(p.contents), ct)
+            # p.contents is the same as p[0]
+##            print p.contents
+##            self.assertEqual(p.contents, 42)
+##            self.assertEqual(p[0], 42)
+
+            self.assertRaises(TypeError, delitem, p, 0)
+
+    def test_from_address(self):
+        from array import array
+        a = array('i', [100, 200, 300, 400, 500])
+        addr = a.buffer_info()[0]
+
+        p = POINTER(POINTER(c_int))
+##        print dir(p)
+##        print p.from_address
+##        print p.from_address(addr)[0][0]
+
+    def test_other(self):
+        class Table(Structure):
+            _fields_ = [("a", c_int),
+                        ("b", c_int),
+                        ("c", c_int)]
+
+        pt = pointer(Table(1, 2, 3))
+
+        self.assertEqual(pt.contents.a, 1)
+        self.assertEqual(pt.contents.b, 2)
+        self.assertEqual(pt.contents.c, 3)
+
+        pt.contents.c = 33
+
+        from ctypes import _pointer_type_cache
+        del _pointer_type_cache[Table]
+
+    def test_basic(self):
+        p = pointer(c_int(42))
+        # Although a pointer can be indexed, it has no length
+        self.assertRaises(TypeError, len, p)
+        self.assertEqual(p[0], 42)
+        self.assertEqual(p[0:1], [42])
+        self.assertEqual(p.contents.value, 42)
+
+    def test_charpp(self):
+        """Test that a character pointer-to-pointer is correctly passed"""
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_c_p_p
+        func.restype = c_char_p
+        argv = (c_char_p * 2)()
+        argc = c_int( 2 )
+        argv[0] = b'hello'
+        argv[1] = b'world'
+        result = func( byref(argc), argv )
+        self.assertEqual(result, b'world')
+
+    def test_bug_1467852(self):
+        # http://sourceforge.net/tracker/?func=detail&atid=532154&aid=1467852&group_id=71702
+        x = c_int(5)
+        dummy = []
+        for i in range(32000):
+            dummy.append(c_int(i))
+        y = c_int(6)
+        p = pointer(x)
+        pp = pointer(p)
+        q = pointer(y)
+        pp[0] = q         # <==
+        self.assertEqual(p[0], 6)
+    def test_c_void_p(self):
+        # http://sourceforge.net/tracker/?func=detail&aid=1518190&group_id=5470&atid=105470
+        if sizeof(c_void_p) == 4:
+            self.assertEqual(c_void_p(0xFFFFFFFF).value,
+                                 c_void_p(-1).value)
+            self.assertEqual(c_void_p(0xFFFFFFFFFFFFFFFF).value,
+                                 c_void_p(-1).value)
+        elif sizeof(c_void_p) == 8:
+            self.assertEqual(c_void_p(0xFFFFFFFF).value,
+                                 0xFFFFFFFF)
+            self.assertEqual(c_void_p(0xFFFFFFFFFFFFFFFF).value,
+                                 c_void_p(-1).value)
+            self.assertEqual(c_void_p(0xFFFFFFFFFFFFFFFFFFFFFFFF).value,
+                                 c_void_p(-1).value)
+
+        self.assertRaises(TypeError, c_void_p, 3.14) # make sure floats are NOT accepted
+        self.assertRaises(TypeError, c_void_p, object()) # nor other objects
+
+    def test_pointers_bool(self):
+        # NULL pointers have a boolean False value, non-NULL pointers True.
+        self.assertEqual(bool(POINTER(c_int)()), False)
+        self.assertEqual(bool(pointer(c_int())), True)
+
+        self.assertEqual(bool(CFUNCTYPE(None)(0)), False)
+        self.assertEqual(bool(CFUNCTYPE(None)(42)), True)
+
+        # COM methods are boolean True:
+        if sys.platform == "win32":
+            mth = WINFUNCTYPE(None)(42, "name", (), None)
+            self.assertEqual(bool(mth), True)
+
+    def test_pointer_type_name(self):
+        LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
+        self.assertTrue(POINTER(LargeNamedType))
+
+        # to not leak references, we must clean _pointer_type_cache
+        from ctypes import _pointer_type_cache
+        del _pointer_type_cache[LargeNamedType]
+
+    def test_pointer_type_str_name(self):
+        large_string = 'T' * 2 ** 25
+        P = POINTER(large_string)
+        self.assertTrue(P)
+
+        # to not leak references, we must clean _pointer_type_cache
+        from ctypes import _pointer_type_cache
+        del _pointer_type_cache[id(P)]
+
+    def test_abstract(self):
+        from ctypes import _Pointer
+
+        self.assertRaises(TypeError, _Pointer.set_type, 42)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_prototypes.py

@@ -0,0 +1,222 @@
+from ctypes import *
+from ctypes.test import need_symbol
+import unittest
+
+# IMPORTANT INFO:
+#
+# Consider this call:
+#    func.restype = c_char_p
+#    func(c_char_p("123"))
+# It returns
+#    "123"
+#
+# WHY IS THIS SO?
+#
+# argument tuple (c_char_p("123"), ) is destroyed after the function
+# func is called, but NOT before the result is actually built.
+#
+# If the arglist would be destroyed BEFORE the result has been built,
+# the c_char_p("123") object would already have a zero refcount,
+# and the pointer passed to (and returned by) the function would
+# probably point to deallocated space.
+#
+# In this case, there would have to be an additional reference to the argument...
+
+import _ctypes_test
+testdll = CDLL(_ctypes_test.__file__)
+
+# Return machine address `a` as a (possibly long) non-negative integer.
+# Starting with Python 2.5, id(anything) is always non-negative, and
+# the ctypes addressof() inherits that via PyLong_FromVoidPtr().
+def positive_address(a):
+    if a >= 0:
+        return a
+    # View the bits in `a` as unsigned instead.
+    import struct
+    num_bits = struct.calcsize("P") * 8 # num bits in native machine address
+    a += 1 << num_bits
+    assert a >= 0
+    return a
+
+def c_wbuffer(init):
+    n = len(init) + 1
+    return (c_wchar * n)(*init)
+
+class CharPointersTestCase(unittest.TestCase):
+
+    def setUp(self):
+        func = testdll._testfunc_p_p
+        func.restype = c_long
+        func.argtypes = None
+
+    def test_paramflags(self):
+        # function returns c_void_p result,
+        # and has a required parameter named 'input'
+        prototype = CFUNCTYPE(c_void_p, c_void_p)
+        func = prototype(("_testfunc_p_p", testdll),
+                         ((1, "input"),))
+
+        try:
+            func()
+        except TypeError as details:
+            self.assertEqual(str(details), "required argument 'input' missing")
+        else:
+            self.fail("TypeError not raised")
+
+        self.assertEqual(func(None), None)
+        self.assertEqual(func(input=None), None)
+
+
+    def test_int_pointer_arg(self):
+        func = testdll._testfunc_p_p
+        if sizeof(c_longlong) == sizeof(c_void_p):
+            func.restype = c_longlong
+        else:
+            func.restype = c_long
+        self.assertEqual(0, func(0))
+
+        ci = c_int(0)
+
+        func.argtypes = POINTER(c_int),
+        self.assertEqual(positive_address(addressof(ci)),
+                             positive_address(func(byref(ci))))
+
+        func.argtypes = c_char_p,
+        self.assertRaises(ArgumentError, func, byref(ci))
+
+        func.argtypes = POINTER(c_short),
+        self.assertRaises(ArgumentError, func, byref(ci))
+
+        func.argtypes = POINTER(c_double),
+        self.assertRaises(ArgumentError, func, byref(ci))
+
+    def test_POINTER_c_char_arg(self):
+        func = testdll._testfunc_p_p
+        func.restype = c_char_p
+        func.argtypes = POINTER(c_char),
+
+        self.assertEqual(None, func(None))
+        self.assertEqual(b"123", func(b"123"))
+        self.assertEqual(None, func(c_char_p(None)))
+        self.assertEqual(b"123", func(c_char_p(b"123")))
+
+        self.assertEqual(b"123", func(c_buffer(b"123")))
+        ca = c_char(b"a")
+        self.assertEqual(ord(b"a"), func(pointer(ca))[0])
+        self.assertEqual(ord(b"a"), func(byref(ca))[0])
+
+    def test_c_char_p_arg(self):
+        func = testdll._testfunc_p_p
+        func.restype = c_char_p
+        func.argtypes = c_char_p,
+
+        self.assertEqual(None, func(None))
+        self.assertEqual(b"123", func(b"123"))
+        self.assertEqual(None, func(c_char_p(None)))
+        self.assertEqual(b"123", func(c_char_p(b"123")))
+
+        self.assertEqual(b"123", func(c_buffer(b"123")))
+        ca = c_char(b"a")
+        self.assertEqual(ord(b"a"), func(pointer(ca))[0])
+        self.assertEqual(ord(b"a"), func(byref(ca))[0])
+
+    def test_c_void_p_arg(self):
+        func = testdll._testfunc_p_p
+        func.restype = c_char_p
+        func.argtypes = c_void_p,
+
+        self.assertEqual(None, func(None))
+        self.assertEqual(b"123", func(b"123"))
+        self.assertEqual(b"123", func(c_char_p(b"123")))
+        self.assertEqual(None, func(c_char_p(None)))
+
+        self.assertEqual(b"123", func(c_buffer(b"123")))
+        ca = c_char(b"a")
+        self.assertEqual(ord(b"a"), func(pointer(ca))[0])
+        self.assertEqual(ord(b"a"), func(byref(ca))[0])
+
+        func(byref(c_int()))
+        func(pointer(c_int()))
+        func((c_int * 3)())
+
+    @need_symbol('c_wchar_p')
+    def test_c_void_p_arg_with_c_wchar_p(self):
+        func = testdll._testfunc_p_p
+        func.restype = c_wchar_p
+        func.argtypes = c_void_p,
+
+        self.assertEqual(None, func(c_wchar_p(None)))
+        self.assertEqual("123", func(c_wchar_p("123")))
+
+    def test_instance(self):
+        func = testdll._testfunc_p_p
+        func.restype = c_void_p
+
+        class X:
+            _as_parameter_ = None
+
+        func.argtypes = c_void_p,
+        self.assertEqual(None, func(X()))
+
+        func.argtypes = None
+        self.assertEqual(None, func(X()))
+
+@need_symbol('c_wchar')
+class WCharPointersTestCase(unittest.TestCase):
+
+    def setUp(self):
+        func = testdll._testfunc_p_p
+        func.restype = c_int
+        func.argtypes = None
+
+
+    def test_POINTER_c_wchar_arg(self):
+        func = testdll._testfunc_p_p
+        func.restype = c_wchar_p
+        func.argtypes = POINTER(c_wchar),
+
+        self.assertEqual(None, func(None))
+        self.assertEqual("123", func("123"))
+        self.assertEqual(None, func(c_wchar_p(None)))
+        self.assertEqual("123", func(c_wchar_p("123")))
+
+        self.assertEqual("123", func(c_wbuffer("123")))
+        ca = c_wchar("a")
+        self.assertEqual("a", func(pointer(ca))[0])
+        self.assertEqual("a", func(byref(ca))[0])
+
+    def test_c_wchar_p_arg(self):
+        func = testdll._testfunc_p_p
+        func.restype = c_wchar_p
+        func.argtypes = c_wchar_p,
+
+        c_wchar_p.from_param("123")
+
+        self.assertEqual(None, func(None))
+        self.assertEqual("123", func("123"))
+        self.assertEqual(None, func(c_wchar_p(None)))
+        self.assertEqual("123", func(c_wchar_p("123")))
+
+        # XXX Currently, these raise TypeErrors, although they shouldn't:
+        self.assertEqual("123", func(c_wbuffer("123")))
+        ca = c_wchar("a")
+        self.assertEqual("a", func(pointer(ca))[0])
+        self.assertEqual("a", func(byref(ca))[0])
+
+class ArrayTest(unittest.TestCase):
+    def test(self):
+        func = testdll._testfunc_ai8
+        func.restype = POINTER(c_int)
+        func.argtypes = c_int * 8,
+
+        func((c_int * 8)(1, 2, 3, 4, 5, 6, 7, 8))
+
+        # This did crash before:
+
+        def func(): pass
+        CFUNCTYPE(None, c_int * 3)(func)
+
+################################################################
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_python_api.py

@@ -0,0 +1,85 @@
+from ctypes import *
+import unittest
+from test import support
+
+################################################################
+# This section should be moved into ctypes\__init__.py, when it's ready.
+
+from _ctypes import PyObj_FromPtr
+
+################################################################
+
+from sys import getrefcount as grc
+
+class PythonAPITestCase(unittest.TestCase):
+
+    def test_PyBytes_FromStringAndSize(self):
+        PyBytes_FromStringAndSize = pythonapi.PyBytes_FromStringAndSize
+
+        PyBytes_FromStringAndSize.restype = py_object
+        PyBytes_FromStringAndSize.argtypes = c_char_p, c_size_t
+
+        self.assertEqual(PyBytes_FromStringAndSize(b"abcdefghi", 3), b"abc")
+
+    @support.refcount_test
+    def test_PyString_FromString(self):
+        pythonapi.PyBytes_FromString.restype = py_object
+        pythonapi.PyBytes_FromString.argtypes = (c_char_p,)
+
+        s = b"abc"
+        refcnt = grc(s)
+        pyob = pythonapi.PyBytes_FromString(s)
+        self.assertEqual(grc(s), refcnt)
+        self.assertEqual(s, pyob)
+        del pyob
+        self.assertEqual(grc(s), refcnt)
+
+    @support.refcount_test
+    def test_PyLong_Long(self):
+        ref42 = grc(42)
+        pythonapi.PyLong_FromLong.restype = py_object
+        self.assertEqual(pythonapi.PyLong_FromLong(42), 42)
+
+        self.assertEqual(grc(42), ref42)
+
+        pythonapi.PyLong_AsLong.argtypes = (py_object,)
+        pythonapi.PyLong_AsLong.restype = c_long
+
+        res = pythonapi.PyLong_AsLong(42)
+        self.assertEqual(grc(res), ref42 + 1)
+        del res
+        self.assertEqual(grc(42), ref42)
+
+    @support.refcount_test
+    def test_PyObj_FromPtr(self):
+        s = "abc def ghi jkl"
+        ref = grc(s)
+        # id(python-object) is the address
+        pyobj = PyObj_FromPtr(id(s))
+        self.assertIs(s, pyobj)
+
+        self.assertEqual(grc(s), ref + 1)
+        del pyobj
+        self.assertEqual(grc(s), ref)
+
+    def test_PyOS_snprintf(self):
+        PyOS_snprintf = pythonapi.PyOS_snprintf
+        PyOS_snprintf.argtypes = POINTER(c_char), c_size_t, c_char_p
+
+        buf = c_buffer(256)
+        PyOS_snprintf(buf, sizeof(buf), b"Hello from %s", b"ctypes")
+        self.assertEqual(buf.value, b"Hello from ctypes")
+
+        PyOS_snprintf(buf, sizeof(buf), b"Hello from %s (%d, %d, %d)", b"ctypes", 1, 2, 3)
+        self.assertEqual(buf.value, b"Hello from ctypes (1, 2, 3)")
+
+        # not enough arguments
+        self.assertRaises(TypeError, PyOS_snprintf, buf)
+
+    def test_pyobject_repr(self):
+        self.assertEqual(repr(py_object()), "py_object(<NULL>)")
+        self.assertEqual(repr(py_object(42)), "py_object(42)")
+        self.assertEqual(repr(py_object(object)), "py_object(%r)" % object)
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_random_things.py

@@ -0,0 +1,77 @@
+from ctypes import *
+import contextlib
+from test import support
+import unittest
+import sys
+
+
+def callback_func(arg):
+    42 / arg
+    raise ValueError(arg)
+
+@unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
+class call_function_TestCase(unittest.TestCase):
+    # _ctypes.call_function is deprecated and private, but used by
+    # Gary Bishp's readline module.  If we have it, we must test it as well.
+
+    def test(self):
+        from _ctypes import call_function
+        windll.kernel32.LoadLibraryA.restype = c_void_p
+        windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p
+        windll.kernel32.GetProcAddress.restype = c_void_p
+
+        hdll = windll.kernel32.LoadLibraryA(b"kernel32")
+        funcaddr = windll.kernel32.GetProcAddress(hdll, b"GetModuleHandleA")
+
+        self.assertEqual(call_function(funcaddr, (None,)),
+                             windll.kernel32.GetModuleHandleA(None))
+
+class CallbackTracbackTestCase(unittest.TestCase):
+    # When an exception is raised in a ctypes callback function, the C
+    # code prints a traceback.
+    #
+    # This test makes sure the exception types *and* the exception
+    # value is printed correctly.
+    #
+    # Changed in 0.9.3: No longer is '(in callback)' prepended to the
+    # error message - instead an additional frame for the C code is
+    # created, then a full traceback printed.  When SystemExit is
+    # raised in a callback function, the interpreter exits.
+
+    @contextlib.contextmanager
+    def expect_unraisable(self, exc_type, exc_msg=None):
+        with support.catch_unraisable_exception() as cm:
+            yield
+
+            self.assertIsInstance(cm.unraisable.exc_value, exc_type)
+            if exc_msg is not None:
+                self.assertEqual(str(cm.unraisable.exc_value), exc_msg)
+            self.assertEqual(cm.unraisable.err_msg,
+                             "Exception ignored on calling ctypes "
+                             "callback function")
+            self.assertIs(cm.unraisable.object, callback_func)
+
+    def test_ValueError(self):
+        cb = CFUNCTYPE(c_int, c_int)(callback_func)
+        with self.expect_unraisable(ValueError, '42'):
+            cb(42)
+
+    def test_IntegerDivisionError(self):
+        cb = CFUNCTYPE(c_int, c_int)(callback_func)
+        with self.expect_unraisable(ZeroDivisionError):
+            cb(0)
+
+    def test_FloatDivisionError(self):
+        cb = CFUNCTYPE(c_int, c_double)(callback_func)
+        with self.expect_unraisable(ZeroDivisionError):
+            cb(0.0)
+
+    def test_TypeErrorDivisionError(self):
+        cb = CFUNCTYPE(c_int, c_char_p)(callback_func)
+        err_msg = "unsupported operand type(s) for /: 'int' and 'bytes'"
+        with self.expect_unraisable(TypeError, err_msg):
+            cb(b"spam")
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_refcounts.py

@@ -0,0 +1,116 @@
+import unittest
+from test import support
+import ctypes
+import gc
+
+MyCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)
+OtherCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_ulonglong)
+
+import _ctypes_test
+dll = ctypes.CDLL(_ctypes_test.__file__)
+
+class RefcountTestCase(unittest.TestCase):
+
+    @support.refcount_test
+    def test_1(self):
+        from sys import getrefcount as grc
+
+        f = dll._testfunc_callback_i_if
+        f.restype = ctypes.c_int
+        f.argtypes = [ctypes.c_int, MyCallback]
+
+        def callback(value):
+            #print "called back with", value
+            return value
+
+        self.assertEqual(grc(callback), 2)
+        cb = MyCallback(callback)
+
+        self.assertGreater(grc(callback), 2)
+        result = f(-10, cb)
+        self.assertEqual(result, -18)
+        cb = None
+
+        gc.collect()
+
+        self.assertEqual(grc(callback), 2)
+
+
+    @support.refcount_test
+    def test_refcount(self):
+        from sys import getrefcount as grc
+        def func(*args):
+            pass
+        # this is the standard refcount for func
+        self.assertEqual(grc(func), 2)
+
+        # the CFuncPtr instance holds at least one refcount on func:
+        f = OtherCallback(func)
+        self.assertGreater(grc(func), 2)
+
+        # and may release it again
+        del f
+        self.assertGreaterEqual(grc(func), 2)
+
+        # but now it must be gone
+        gc.collect()
+        self.assertEqual(grc(func), 2)
+
+        class X(ctypes.Structure):
+            _fields_ = [("a", OtherCallback)]
+        x = X()
+        x.a = OtherCallback(func)
+
+        # the CFuncPtr instance holds at least one refcount on func:
+        self.assertGreater(grc(func), 2)
+
+        # and may release it again
+        del x
+        self.assertGreaterEqual(grc(func), 2)
+
+        # and now it must be gone again
+        gc.collect()
+        self.assertEqual(grc(func), 2)
+
+        f = OtherCallback(func)
+
+        # the CFuncPtr instance holds at least one refcount on func:
+        self.assertGreater(grc(func), 2)
+
+        # create a cycle
+        f.cycle = f
+
+        del f
+        gc.collect()
+        self.assertEqual(grc(func), 2)
+
+class AnotherLeak(unittest.TestCase):
+    def test_callback(self):
+        import sys
+
+        proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int)
+        def func(a, b):
+            return a * b * 2
+        f = proto(func)
+
+        a = sys.getrefcount(ctypes.c_int)
+        f(1, 2)
+        self.assertEqual(sys.getrefcount(ctypes.c_int), a)
+
+    @support.refcount_test
+    def test_callback_py_object_none_return(self):
+        # bpo-36880: test that returning None from a py_object callback
+        # does not decrement the refcount of None.
+
+        for FUNCTYPE in (ctypes.CFUNCTYPE, ctypes.PYFUNCTYPE):
+            with self.subTest(FUNCTYPE=FUNCTYPE):
+                @FUNCTYPE(ctypes.py_object)
+                def func():
+                    return None
+
+                # Check that calling func does not affect None's refcount.
+                for _ in range(10000):
+                    func()
+
+if __name__ == '__main__':
+    unittest.main()

Lib/ctypes/test/test_repr.py

@@ -0,0 +1,29 @@
+from ctypes import *
+import unittest
+
+subclasses = []
+for base in [c_byte, c_short, c_int, c_long, c_longlong,
+        c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong,
+        c_float, c_double, c_longdouble, c_bool]:
+    class X(base):
+        pass
+    subclasses.append(X)
+
+class X(c_char):
+    pass
+
+# This test checks if the __repr__ is correct for subclasses of simple types
+
+class ReprTest(unittest.TestCase):
+    def test_numbers(self):
+        for typ in subclasses:
+            base = typ.__bases__[0]
+            self.assertTrue(repr(base(42)).startswith(base.__name__))
+            self.assertEqual("<X object at", repr(typ(42))[:12])
+
+    def test_char(self):
+        self.assertEqual("c_char(b'x')", repr(c_char(b'x')))
+        self.assertEqual("<X object at", repr(X(b'x'))[:12])
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_returnfuncptrs.py

@@ -0,0 +1,66 @@
+import unittest
+from ctypes import *
+
+import _ctypes_test
+
+class ReturnFuncPtrTestCase(unittest.TestCase):
+
+    def test_with_prototype(self):
+        # The _ctypes_test shared lib/dll exports quite some functions for testing.
+        # The get_strchr function returns a *pointer* to the C strchr function.
+        dll = CDLL(_ctypes_test.__file__)
+        get_strchr = dll.get_strchr
+        get_strchr.restype = CFUNCTYPE(c_char_p, c_char_p, c_char)
+        strchr = get_strchr()
+        self.assertEqual(strchr(b"abcdef", b"b"), b"bcdef")
+        self.assertEqual(strchr(b"abcdef", b"x"), None)
+        self.assertEqual(strchr(b"abcdef", 98), b"bcdef")
+        self.assertEqual(strchr(b"abcdef", 107), None)
+        self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0)
+        self.assertRaises(TypeError, strchr, b"abcdef")
+
+    def test_without_prototype(self):
+        dll = CDLL(_ctypes_test.__file__)
+        get_strchr = dll.get_strchr
+        # the default 'c_int' would not work on systems where sizeof(int) != sizeof(void *)
+        get_strchr.restype = c_void_p
+        addr = get_strchr()
+        # _CFuncPtr instances are now callable with an integer argument
+        # which denotes a function address:
+        strchr = CFUNCTYPE(c_char_p, c_char_p, c_char)(addr)
+        self.assertTrue(strchr(b"abcdef", b"b"), "bcdef")
+        self.assertEqual(strchr(b"abcdef", b"x"), None)
+        self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0)
+        self.assertRaises(TypeError, strchr, b"abcdef")
+
+    def test_from_dll(self):
+        dll = CDLL(_ctypes_test.__file__)
+        # _CFuncPtr instances are now callable with a tuple argument
+        # which denotes a function name and a dll:
+        strchr = CFUNCTYPE(c_char_p, c_char_p, c_char)(("my_strchr", dll))
+        self.assertTrue(strchr(b"abcdef", b"b"), "bcdef")
+        self.assertEqual(strchr(b"abcdef", b"x"), None)
+        self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0)
+        self.assertRaises(TypeError, strchr, b"abcdef")
+
+    # Issue 6083: Reference counting bug
+    def test_from_dll_refcount(self):
+        class BadSequence(tuple):
+            def __getitem__(self, key):
+                if key == 0:
+                    return "my_strchr"
+                if key == 1:
+                    return CDLL(_ctypes_test.__file__)
+                raise IndexError
+
+        # _CFuncPtr instances are now callable with a tuple argument
+        # which denotes a function name and a dll:
+        strchr = CFUNCTYPE(c_char_p, c_char_p, c_char)(
+                BadSequence(("my_strchr", CDLL(_ctypes_test.__file__))))
+        self.assertTrue(strchr(b"abcdef", b"b"), "bcdef")
+        self.assertEqual(strchr(b"abcdef", b"x"), None)
+        self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0)
+        self.assertRaises(TypeError, strchr, b"abcdef")
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_simplesubclasses.py

@@ -0,0 +1,55 @@
+import unittest
+from ctypes import *
+
+class MyInt(c_int):
+    def __eq__(self, other):
+        if type(other) != MyInt:
+            return NotImplementedError
+        return self.value == other.value
+
+class Test(unittest.TestCase):
+
+    def test_compare(self):
+        self.assertEqual(MyInt(3), MyInt(3))
+        self.assertNotEqual(MyInt(42), MyInt(43))
+
+    def test_ignore_retval(self):
+        # Test if the return value of a callback is ignored
+        # if restype is None
+        proto = CFUNCTYPE(None)
+        def func():
+            return (1, "abc", None)
+
+        cb = proto(func)
+        self.assertEqual(None, cb())
+
+
+    def test_int_callback(self):
+        args = []
+        def func(arg):
+            args.append(arg)
+            return arg
+
+        cb = CFUNCTYPE(None, MyInt)(func)
+
+        self.assertEqual(None, cb(42))
+        self.assertEqual(type(args[-1]), MyInt)
+
+        cb = CFUNCTYPE(c_int, c_int)(func)
+
+        self.assertEqual(42, cb(42))
+        self.assertEqual(type(args[-1]), int)
+
+    def test_int_struct(self):
+        class X(Structure):
+            _fields_ = [("x", MyInt)]
+
+        self.assertEqual(X().x, MyInt())
+
+        s = X()
+        s.x = MyInt(42)
+
+        self.assertEqual(s.x, MyInt(42))
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_sizes.py

@@ -0,0 +1,33 @@
+# Test specifically-sized containers.
+
+from ctypes import *
+
+import unittest
+
+
+class SizesTestCase(unittest.TestCase):
+    def test_8(self):
+        self.assertEqual(1, sizeof(c_int8))
+        self.assertEqual(1, sizeof(c_uint8))
+
+    def test_16(self):
+        self.assertEqual(2, sizeof(c_int16))
+        self.assertEqual(2, sizeof(c_uint16))
+
+    def test_32(self):
+        self.assertEqual(4, sizeof(c_int32))
+        self.assertEqual(4, sizeof(c_uint32))
+
+    def test_64(self):
+        self.assertEqual(8, sizeof(c_int64))
+        self.assertEqual(8, sizeof(c_uint64))
+
+    def test_size_t(self):
+        self.assertEqual(sizeof(c_void_p), sizeof(c_size_t))
+
+    def test_ssize_t(self):
+        self.assertEqual(sizeof(c_void_p), sizeof(c_ssize_t))
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/ctypes/test/test_slicing.py

@@ -0,0 +1,167 @@
+import unittest
+from ctypes import *
+from ctypes.test import need_symbol
+
+import _ctypes_test
+
+class SlicesTestCase(unittest.TestCase):
+    def test_getslice_cint(self):
+        a = (c_int * 100)(*range(1100, 1200))
+        b = list(range(1100, 1200))
+        self.assertEqual(a[0:2], b[0:2])
+        self.assertEqual(a[0:2:], b[0:2:])
+        self.assertEqual(len(a), len(b))
+        self.assertEqual(a[5:7], b[5:7])
+        self.assertEqual(a[5:7:], b[5:7:])
+        self.assertEqual(a[-1], b[-1])
+        self.assertEqual(a[:], b[:])
+        self.assertEqual(a[::], b[::])
+        self.assertEqual(a[10::-1], b[10::-1])
+        self.assertEqual(a[30:20:-1], b[30:20:-1])
+        self.assertEqual(a[:12:6], b[:12:6])
+        self.assertEqual(a[2:6:4], b[2:6:4])
+
+        a[0:5] = range(5, 10)
+        self.assertEqual(a[0:5], list(range(5, 10)))
+        self.assertEqual(a[0:5:], list(range(5, 10)))
+        self.assertEqual(a[4::-1], list(range(9, 4, -1)))
+
+    def test_setslice_cint(self):
+        a = (c_int * 100)(*range(1100, 1200))
+        b = list(range(1100, 1200))
+
+        a[32:47] = list(range(32, 47))
+        self.assertEqual(a[32:47], list(range(32, 47)))
+        a[32:47] = range(132, 147)
+        self.assertEqual(a[32:47:], list(range(132, 147)))
+        a[46:31:-1] = range(232, 247)
+        self.assertEqual(a[32:47:1], list(range(246, 231, -1)))
+
+        a[32:47] = range(1132, 1147)
+        self.assertEqual(a[:], b)
+        a[32:47:7] = range(3)
+        b[32:47:7] = range(3)
+        self.assertEqual(a[:], b)
+        a[33::-3] = range(12)
+        b[33::-3] = range(12)
+        self.assertEqual(a[:], b)
+
+        from operator import setitem
+
+        # TypeError: int expected instead of str instance
+        self.assertRaises(TypeError, setitem, a, slice(0, 5), "abcde")
+        # TypeError: int expected instead of str instance
+        self.assertRaises(TypeError, setitem, a, slice(0, 5),
+                          ["a", "b", "c", "d", "e"])
+        # TypeError: int expected instead of float instance
+        self.assertRaises(TypeError, setitem, a, slice(0, 5),
+                          [1, 2, 3, 4, 3.14])
+        # ValueError: Can only assign sequence of same size
+        self.assertRaises(ValueError, setitem, a, slice(0, 5), range(32))
+
+    def test_char_ptr(self):
+        s = b"abcdefghijklmnopqrstuvwxyz"
+
+        dll = CDLL(_ctypes_test.__file__)
+        dll.my_strdup.restype = POINTER(c_char)
+        dll.my_free.restype = None
+        res = dll.my_strdup(s)
+        self.assertEqual(res[:len(s)], s)
+        self.assertEqual(res[:3], s[:3])
+        self.assertEqual(res[:len(s):], s)
+        self.assertEqual(res[len(s)-1:-1:-1], s[::-1])
+        self.assertEqual(res[len(s)-1:5:-7], s[:5:-7])
+        self.assertEqual(res[0:-1:-1], s[0::-1])
+
+        import operator
+        self.assertRaises(ValueError, operator.getitem,
+                          res, slice(None, None, None))
+        self.assertRaises(ValueError, operator.getitem,
+                          res, slice(0, None, None))
+        self.assertRaises(ValueError, operator.getitem,
+                          res, slice(None, 5, -1))
+        self.assertRaises(ValueError, operator.getitem,
+                          res, slice(-5, None, None))
+
+        self.assertRaises(TypeError, operator.setitem,
+                          res, slice(0, 5), "abcde")
+        dll.my_free(res)
+
+        dll.my_strdup.restype = POINTER(c_byte)
+        res = dll.my_strdup(s)
+        self.assertEqual(res[:len(s)], list(range(ord("a"), ord("z")+1)))
+        self.assertEqual(res[:len(s):], list(range(ord("a"), ord("z")+1)))
+        dll.my_free(res)
+
+    def test_char_ptr_with_free(self):
+        dll = CDLL(_ctypes_test.__file__)
+        s = b"abcdefghijklmnopqrstuvwxyz"
+
+        class allocated_c_char_p(c_char_p):
+            pass
+
+        dll.my_free.restype = None
+        def errcheck(result, func, args):
+            retval = result.value
+            dll.my_free(result)
+            return retval
+
+        dll.my_strdup.restype = allocated_c_char_p
+        dll.my_strdup.errcheck = errcheck
+        try:
+            res = dll.my_strdup(s)
+            self.assertEqual(res, s)
+        finally:
+            del dll.my_strdup.errcheck
+
+
+    def test_char_array(self):
+        s = b"abcdefghijklmnopqrstuvwxyz\0"
+
+        p = (c_char * 27)(*s)
+        self.assertEqual(p[:], s)
+        self.assertEqual(p[::], s)
+        self.assertEqual(p[::-1], s[::-1])
+        self.assertEqual(p[5::-2], s[5::-2])
+        self.assertEqual(p[2:5:-3], s[2:5:-3])
+
+
+    @need_symbol('c_wchar')
+    def test_wchar_ptr(self):
+        s = "abcdefghijklmnopqrstuvwxyz\0"
+
+        dll = CDLL(_ctypes_test.__file__)
+        dll.my_wcsdup.restype = POINTER(c_wchar)
+        dll.my_wcsdup.argtypes = POINTER(c_wchar),
+        dll.my_free.restype = None
+        res = dll.my_wcsdup(s[:-1])
+        self.assertEqual(res[:len(s)], s)
+        self.assertEqual(res[:len(s):], s)
+        self.assertEqual(res[len(s)-1:-1:-1], s[::-1])
+        self.assertEqual(res[len(s)-1:5:-7], s[:5:-7])
+
+        import operator
+        self.assertRaises(TypeError, operator.setitem,
+                          res, slice(0, 5), "abcde")
+        dll.my_free(res)
+
+        if sizeof(c_wchar) == sizeof(c_short):
+            dll.my_wcsdup.restype = POINTER(c_short)
+        elif sizeof(c_wchar) == sizeof(c_int):
+            dll.my_wcsdup.restype = POINTER(c_int)
+        elif sizeof(c_wchar) == sizeof(c_long):
+            dll.my_wcsdup.restype = POINTER(c_long)
+        else:
+            self.skipTest('Pointers to c_wchar are not supported')
+        res = dll.my_wcsdup(s[:-1])
+        tmpl = list(range(ord("a"), ord("z")+1))
+        self.assertEqual(res[:len(s)-1], tmpl)
+        self.assertEqual(res[:len(s)-1:], tmpl)
+        self.assertEqual(res[len(s)-2:-1:-1], tmpl[::-1])
+        self.assertEqual(res[len(s)-2:5:-7], tmpl[:5:-7])
+        dll.my_free(res)
+
+################################################################
+
+if __name__ == "__main__":
+    unittest.main()