Update contextlib from 3.13.5, (#6056)

* Update `contextlib` from 3.13.5

* Add `test_contextlib_async.py`

.cspell.json

@@ -60,6 +60,7 @@
     "dedentations",
     "dedents",
     "deduped",
+    "downcastable",
     "downcasted",
     "dumpable",
     "emscripten",

.github/copilot-instructions.md

@@ -21,7 +21,7 @@ RustPython is a Python 3 interpreter written in Rust, implementing Python 3.13.0
   - `parser/` - Parser for converting Python source to AST
   - `core/` - Bytecode representation in Rust structures
   - `codegen/` - AST to bytecode compiler
-- `Lib/` - CPython's standard library in Python (copied from CPython)
+- `Lib/` - CPython's standard library in Python (copied from CPython). **IMPORTANT**: Do not edit this directory directly; The only allowed operation is copying files from CPython.
 - `derive/` - Rust macros for RustPython
 - `common/` - Common utilities
 - `extra_tests/` - Integration tests and snippets

.github/workflows/ci.yaml

@@ -113,6 +113,7 @@ jobs:
       RUST_BACKTRACE: full
     name: Run rust tests
     runs-on:  ${{ matrix.os }}
+    timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 30 }}
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
@@ -175,6 +176,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Ensure compilation on various targets
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -237,6 +239,7 @@ jobs:
       RUST_BACKTRACE: full
     name: Run snippets and cpython tests
     runs-on:  ${{ matrix.os }}
+    timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 30 }}
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
@@ -344,23 +347,31 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Run tests under miri
     runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      NIGHTLY_CHANNEL: nightly
     steps:
       - uses: actions/checkout@v4
+
       - uses: dtolnay/rust-toolchain@master
         with:
-            toolchain: nightly
+            toolchain: ${{ env.NIGHTLY_CHANNEL }}
             components: miri
 
       - uses: Swatinem/rust-cache@v2
+
       - name: Run tests under miri
+        run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test
+        env:
         # miri-ignore-leaks because the type-object circular reference means that there will always be
         # a memory leak, at least until we have proper cyclic gc
-        run: MIRIFLAGS='-Zmiri-ignore-leaks' cargo +nightly miri test -p rustpython-vm -- miri_test
+          MIRIFLAGS: '-Zmiri-ignore-leaks'
 
   wasm:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Check the WASM package and demo
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -421,6 +432,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Run snippets and cpython tests on wasm-wasi
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable

.github/workflows/comment-commands.yml

@@ -0,0 +1,21 @@
+name: Comment Commands
+
+on:
+  issue_comment:
+    types: created
+
+jobs:
+  issue_assign:
+    if: (!github.event.issue.pull_request) && github.event.comment.body == 'take'
+    runs-on: ubuntu-latest
+
+    concurrency:
+      group: ${{ github.actor }}-issue-assign
+
+    permissions:
+      issues: write
+
+    steps:
+      # Using REST API and not `gh issue edit`. https://github.com/cli/cli/issues/6235#issuecomment-1243487651
+      - run: |
+          curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees

Cargo.toml

@@ -82,7 +82,6 @@ opt-level = 3
 lto = "thin"
 
 [patch.crates-io]
-radium = { version = "1.1.0", git = "https://github.com/youknowone/ferrilab", branch = "fix-nightly" }
 # REDOX START, Uncomment when you want to compile/check with redoxer
 # REDOX END
 
@@ -118,8 +117,20 @@ template = "installer-config/installer.wxs"
 [workspace]
 resolver = "2"
 members = [
-    "compiler", "compiler/core", "compiler/codegen", "compiler/literal", "compiler/source",
-    ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", "wtf8",
+    "compiler",
+    "compiler/core",
+    "compiler/codegen",
+    "compiler/literal",
+    ".",
+    "common",
+    "derive",
+    "jit",
+    "vm",
+    "vm/sre_engine",
+    "pylib",
+    "stdlib",
+    "derive-impl",
+    "wtf8",
     "wasm/lib",
 ]
 
@@ -132,7 +143,6 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [workspace.dependencies]
-rustpython-compiler-source = { path = "compiler/source" }
 rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
 rustpython-compiler = { path = "compiler", version = "0.4.0" }
 rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }
@@ -155,7 +165,7 @@ ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.
 
 ahash = "0.8.11"
 ascii = "1.1"
-bitflags = "2.4.2"
+bitflags = "2.9.1"
 bstr = "1"
 cfg-if = "1.0"
 chrono = "0.4.39"
@@ -166,7 +176,7 @@ flame = "0.2.2"
 getrandom = { version = "0.3", features = ["std"] }
 glob = "0.3"
 hex = "0.4.3"
-indexmap = { version = "2.2.6", features = ["std"] }
+indexmap = { version = "2.10.0", features = ["std"] }
 insta = "1.42"
 itertools = "0.14.0"
 is-macro = "0.3.7"
@@ -190,7 +200,7 @@ paste = "1.0.15"
 proc-macro2 = "1.0.93"
 pymath = "0.0.2"
 quote = "1.0.38"
-radium = "1.1"
+radium = "1.1.1"
 rand = "0.9"
 rand_core = { version = "0.9", features = ["os_rng"] }
 rustix = { version = "1.0", features = ["event"] }
@@ -212,7 +222,7 @@ unic-ucd-category = "0.9.0"
 unic-ucd-ident = "0.9.0"
 unicode_names2 = "1.3.0"
 unicode-bidi-mirroring = "0.2"
-widestring = "1.1.0"
+widestring = "1.2.0"
 windows-sys = "0.59.0"
 wasm-bindgen = "0.2.100"
 

Lib/_colorize.py

@@ -1,20 +1,63 @@
+from __future__ import annotations
 import io
 import os
 import sys
 
 COLORIZE = True
 
+# types
+if False:
+    from typing import IO
+
 
 class ANSIColors:
+    RESET = "\x1b[0m"
+
+    BLACK = "\x1b[30m"
+    BLUE = "\x1b[34m"
+    CYAN = "\x1b[36m"
+    GREEN = "\x1b[32m"
+    MAGENTA = "\x1b[35m"
+    RED = "\x1b[31m"
+    WHITE = "\x1b[37m"  # more like LIGHT GRAY
+    YELLOW = "\x1b[33m"
+
+    BOLD_BLACK = "\x1b[1;30m"  # DARK GRAY
+    BOLD_BLUE = "\x1b[1;34m"
+    BOLD_CYAN = "\x1b[1;36m"
     BOLD_GREEN = "\x1b[1;32m"
     BOLD_MAGENTA = "\x1b[1;35m"
     BOLD_RED = "\x1b[1;31m"
-    GREEN = "\x1b[32m"
-    GREY = "\x1b[90m"
-    MAGENTA = "\x1b[35m"
-    RED = "\x1b[31m"
-    RESET = "\x1b[0m"
-    YELLOW = "\x1b[33m"
+    BOLD_WHITE = "\x1b[1;37m"  # actual WHITE
+    BOLD_YELLOW = "\x1b[1;33m"
+
+    # intense = like bold but without being bold
+    INTENSE_BLACK = "\x1b[90m"
+    INTENSE_BLUE = "\x1b[94m"
+    INTENSE_CYAN = "\x1b[96m"
+    INTENSE_GREEN = "\x1b[92m"
+    INTENSE_MAGENTA = "\x1b[95m"
+    INTENSE_RED = "\x1b[91m"
+    INTENSE_WHITE = "\x1b[97m"
+    INTENSE_YELLOW = "\x1b[93m"
+
+    BACKGROUND_BLACK = "\x1b[40m"
+    BACKGROUND_BLUE = "\x1b[44m"
+    BACKGROUND_CYAN = "\x1b[46m"
+    BACKGROUND_GREEN = "\x1b[42m"
+    BACKGROUND_MAGENTA = "\x1b[45m"
+    BACKGROUND_RED = "\x1b[41m"
+    BACKGROUND_WHITE = "\x1b[47m"
+    BACKGROUND_YELLOW = "\x1b[43m"
+
+    INTENSE_BACKGROUND_BLACK = "\x1b[100m"
+    INTENSE_BACKGROUND_BLUE = "\x1b[104m"
+    INTENSE_BACKGROUND_CYAN = "\x1b[106m"
+    INTENSE_BACKGROUND_GREEN = "\x1b[102m"
+    INTENSE_BACKGROUND_MAGENTA = "\x1b[105m"
+    INTENSE_BACKGROUND_RED = "\x1b[101m"
+    INTENSE_BACKGROUND_WHITE = "\x1b[107m"
+    INTENSE_BACKGROUND_YELLOW = "\x1b[103m"
 
 
 NoColors = ANSIColors()
@@ -24,14 +67,16 @@ for attr in dir(NoColors):
         setattr(NoColors, attr, "")
 
 
-def get_colors(colorize: bool = False, *, file=None) -> ANSIColors:
+def get_colors(
+    colorize: bool = False, *, file: IO[str] | IO[bytes] | None = None
+) -> ANSIColors:
     if colorize or can_colorize(file=file):
         return ANSIColors()
     else:
         return NoColors
 
 
-def can_colorize(*, file=None) -> bool:
+def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool:
     if file is None:
         file = sys.stdout
 
@@ -64,4 +109,4 @@ def can_colorize(*, file=None) -> bool:
     try:
         return os.isatty(file.fileno())
     except io.UnsupportedOperation:
-        return file.isatty()
+        return hasattr(file, "isatty") and file.isatty()

Lib/_pydecimal.py

@@ -13,104 +13,7 @@
 # bug) and will be backported.  At this point the spec is stabilizing
 # and the updates are becoming fewer, smaller, and less significant.
 
-"""
-This is an implementation of decimal floating point arithmetic based on
-the General Decimal Arithmetic Specification:
-
-    http://speleotrove.com/decimal/decarith.html
-
-and IEEE standard 854-1987:
-
-    http://en.wikipedia.org/wiki/IEEE_854-1987
-
-Decimal floating point has finite precision with arbitrarily large bounds.
-
-The purpose of this module is to support arithmetic using familiar
-"schoolhouse" rules and to avoid some of the tricky representation
-issues associated with binary floating point.  The package is especially
-useful for financial applications or for contexts where users have
-expectations that are at odds with binary floating point (for instance,
-in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead
-of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected
-Decimal('0.00')).
-
-Here are some examples of using the decimal module:
-
->>> from decimal import *
->>> setcontext(ExtendedContext)
->>> Decimal(0)
-Decimal('0')
->>> Decimal('1')
-Decimal('1')
->>> Decimal('-.0123')
-Decimal('-0.0123')
->>> Decimal(123456)
-Decimal('123456')
->>> Decimal('123.45e12345678')
-Decimal('1.2345E+12345680')
->>> Decimal('1.33') + Decimal('1.27')
-Decimal('2.60')
->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')
-Decimal('-2.20')
->>> dig = Decimal(1)
->>> print(dig / Decimal(3))
-0.333333333
->>> getcontext().prec = 18
->>> print(dig / Decimal(3))
-0.333333333333333333
->>> print(dig.sqrt())
-1
->>> print(Decimal(3).sqrt())
-1.73205080756887729
->>> print(Decimal(3) ** 123)
-4.85192780976896427E+58
->>> inf = Decimal(1) / Decimal(0)
->>> print(inf)
-Infinity
->>> neginf = Decimal(-1) / Decimal(0)
->>> print(neginf)
--Infinity
->>> print(neginf + inf)
-NaN
->>> print(neginf * inf)
--Infinity
->>> print(dig / 0)
-Infinity
->>> getcontext().traps[DivisionByZero] = 1
->>> print(dig / 0)
-Traceback (most recent call last):
-  ...
-  ...
-  ...
-decimal.DivisionByZero: x / 0
->>> c = Context()
->>> c.traps[InvalidOperation] = 0
->>> print(c.flags[InvalidOperation])
-0
->>> c.divide(Decimal(0), Decimal(0))
-Decimal('NaN')
->>> c.traps[InvalidOperation] = 1
->>> print(c.flags[InvalidOperation])
-1
->>> c.flags[InvalidOperation] = 0
->>> print(c.flags[InvalidOperation])
-0
->>> print(c.divide(Decimal(0), Decimal(0)))
-Traceback (most recent call last):
-  ...
-  ...
-  ...
-decimal.InvalidOperation: 0 / 0
->>> print(c.flags[InvalidOperation])
-1
->>> c.flags[InvalidOperation] = 0
->>> c.traps[InvalidOperation] = 0
->>> print(c.divide(Decimal(0), Decimal(0)))
-NaN
->>> print(c.flags[InvalidOperation])
-1
->>>
-"""
+"""Python decimal arithmetic module"""
 
 __all__ = [
     # Two major classes
@@ -140,8 +43,11 @@ __all__ = [
     # Limits for the C version for compatibility
     'MAX_PREC',  'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',
 
-    # C version: compile time choice that enables the thread local context
-    'HAVE_THREADS'
+    # C version: compile time choice that enables the thread local context (deprecated, now always true)
+    'HAVE_THREADS',
+
+    # C version: compile time choice that enables the coroutine local context
+    'HAVE_CONTEXTVAR'
 ]
 
 __xname__ = __name__    # sys.modules lookup (--without-threads)
@@ -156,7 +62,7 @@ import sys
 
 try:
     from collections import namedtuple as _namedtuple
-    DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent')
+    DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent', module='decimal')
 except ImportError:
     DecimalTuple = lambda *args: args
 
@@ -172,6 +78,7 @@ ROUND_05UP = 'ROUND_05UP'
 
 # Compatibility with the C version
 HAVE_THREADS = True
+HAVE_CONTEXTVAR = True
 if sys.maxsize == 2**63-1:
     MAX_PREC = 999999999999999999
     MAX_EMAX = 999999999999999999
@@ -190,7 +97,7 @@ class DecimalException(ArithmeticError):
 
     Used exceptions derive from this.
     If an exception derives from another exception besides this (such as
-    Underflow (Inexact, Rounded, Subnormal) that indicates that it is only
+    Underflow (Inexact, Rounded, Subnormal)) that indicates that it is only
     called if the others are present.  This isn't actually used for
     anything, though.
 
@@ -238,7 +145,7 @@ class InvalidOperation(DecimalException):
     x ** (+-)INF
     An operand is invalid
 
-    The result of the operation after these is a quiet positive NaN,
+    The result of the operation after this is a quiet positive NaN,
     except when the cause is a signaling NaN, in which case the result is
     also a quiet NaN, but with the original sign, and an optional
     diagnostic information.
@@ -431,82 +338,40 @@ _rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING,
 ##### Context Functions ##################################################
 
 # The getcontext() and setcontext() function manage access to a thread-local
-# current context.  Py2.4 offers direct support for thread locals.  If that
-# is not available, use threading.current_thread() which is slower but will
-# work for older Pythons.  If threads are not part of the build, create a
-# mock threading object with threading.local() returning the module namespace.
+# current context.
 
-try:
-    import threading
-except ImportError:
-    # Python was compiled without threads; create a mock object instead
-    class MockThreading(object):
-        def local(self, sys=sys):
-            return sys.modules[__xname__]
-    threading = MockThreading()
-    del MockThreading
+import contextvars
 
-try:
-    threading.local
+_current_context_var = contextvars.ContextVar('decimal_context')
 
-except AttributeError:
+_context_attributes = frozenset(
+    ['prec', 'Emin', 'Emax', 'capitals', 'clamp', 'rounding', 'flags', 'traps']
+)
 
-    # To fix reloading, force it to create a new context
-    # Old contexts have different exceptions in their dicts, making problems.
-    if hasattr(threading.current_thread(), '__decimal_context__'):
-        del threading.current_thread().__decimal_context__
+def getcontext():
+    """Returns this thread's context.
 
-    def setcontext(context):
-        """Set this thread's context to context."""
-        if context in (DefaultContext, BasicContext, ExtendedContext):
-            context = context.copy()
-            context.clear_flags()
-        threading.current_thread().__decimal_context__ = context
+    If this thread does not yet have a context, returns
+    a new context and sets this thread's context.
+    New contexts are copies of DefaultContext.
+    """
+    try:
+        return _current_context_var.get()
+    except LookupError:
+        context = Context()
+        _current_context_var.set(context)
+        return context
 
-    def getcontext():
-        """Returns this thread's context.
+def setcontext(context):
+    """Set this thread's context to context."""
+    if context in (DefaultContext, BasicContext, ExtendedContext):
+        context = context.copy()
+        context.clear_flags()
+    _current_context_var.set(context)
 
-        If this thread does not yet have a context, returns
-        a new context and sets this thread's context.
-        New contexts are copies of DefaultContext.
-        """
-        try:
-            return threading.current_thread().__decimal_context__
-        except AttributeError:
-            context = Context()
-            threading.current_thread().__decimal_context__ = context
-            return context
+del contextvars        # Don't contaminate the namespace
 
-else:
-
-    local = threading.local()
-    if hasattr(local, '__decimal_context__'):
-        del local.__decimal_context__
-
-    def getcontext(_local=local):
-        """Returns this thread's context.
-
-        If this thread does not yet have a context, returns
-        a new context and sets this thread's context.
-        New contexts are copies of DefaultContext.
-        """
-        try:
-            return _local.__decimal_context__
-        except AttributeError:
-            context = Context()
-            _local.__decimal_context__ = context
-            return context
-
-    def setcontext(context, _local=local):
-        """Set this thread's context to context."""
-        if context in (DefaultContext, BasicContext, ExtendedContext):
-            context = context.copy()
-            context.clear_flags()
-        _local.__decimal_context__ = context
-
-    del threading, local        # Don't contaminate the namespace
-
-def localcontext(ctx=None):
+def localcontext(ctx=None, **kwargs):
     """Return a context manager for a copy of the supplied context
 
     Uses a copy of the current context if no context is specified
@@ -542,8 +407,14 @@ def localcontext(ctx=None):
     >>> print(getcontext().prec)
     28
     """
-    if ctx is None: ctx = getcontext()
-    return _ContextManager(ctx)
+    if ctx is None:
+        ctx = getcontext()
+    ctx_manager = _ContextManager(ctx)
+    for key, value in kwargs.items():
+        if key not in _context_attributes:
+            raise TypeError(f"'{key}' is an invalid keyword argument for this function")
+        setattr(ctx_manager.new_context, key, value)
+    return ctx_manager
 
 
 ##### Decimal class #######################################################
@@ -553,7 +424,7 @@ def localcontext(ctx=None):
 # numbers.py for more detail.
 
 class Decimal(object):
-    """Floating point class for decimal arithmetic."""
+    """Floating-point class for decimal arithmetic."""
 
     __slots__ = ('_exp','_int','_sign', '_is_special')
     # Generally, the value of the Decimal instance is given by
@@ -993,7 +864,7 @@ class Decimal(object):
             if self.is_snan():
                 raise TypeError('Cannot hash a signaling NaN value.')
             elif self.is_nan():
-                return _PyHASH_NAN
+                return object.__hash__(self)
             else:
                 if self._sign:
                     return -_PyHASH_INF
@@ -1674,13 +1545,13 @@ class Decimal(object):
 
     __trunc__ = __int__
 
+    @property
     def real(self):
         return self
-    real = property(real)
 
+    @property
     def imag(self):
         return Decimal(0)
-    imag = property(imag)
 
     def conjugate(self):
         return self
@@ -2260,10 +2131,16 @@ class Decimal(object):
             else:
                 return None
 
-            if xc >= 10**p:
+            # An exact power of 10 is representable, but can convert to a
+            # string of any length. But an exact power of 10 shouldn't be
+            # possible at this point.
+            assert xc > 1, self
+            assert xc % 10 != 0, self
+            strxc = str(xc)
+            if len(strxc) > p:
                 return None
             xe = -e-xe
-            return _dec_from_triple(0, str(xc), xe)
+            return _dec_from_triple(0, strxc, xe)
 
         # now y is positive; find m and n such that y = m/n
         if ye >= 0:
@@ -2272,7 +2149,7 @@ class Decimal(object):
             if xe != 0 and len(str(abs(yc*xe))) <= -ye:
                 return None
             xc_bits = _nbits(xc)
-            if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye:
+            if len(str(abs(yc)*xc_bits)) <= -ye:
                 return None
             m, n = yc, 10**(-ye)
             while m % 2 == n % 2 == 0:
@@ -2285,7 +2162,7 @@ class Decimal(object):
         # compute nth root of xc*10**xe
         if n > 1:
             # if 1 < xc < 2**n then xc isn't an nth power
-            if xc != 1 and xc_bits <= n:
+            if xc_bits <= n:
                 return None
 
             xe, rem = divmod(xe, n)
@@ -2313,13 +2190,18 @@ class Decimal(object):
             return None
         xc = xc**m
         xe *= m
-        if xc > 10**p:
+        # An exact power of 10 is representable, but can convert to a string
+        # of any length. But an exact power of 10 shouldn't be possible at
+        # this point.
+        assert xc > 1, self
+        assert xc % 10 != 0, self
+        str_xc = str(xc)
+        if len(str_xc) > p:
             return None
 
         # by this point the result *is* exactly representable
         # adjust the exponent to get as close as possible to the ideal
         # exponent, if necessary
-        str_xc = str(xc)
         if other._isinteger() and other._sign == 0:
             ideal_exponent = self._exp*int(other)
             zeros = min(xe-ideal_exponent, p-len(str_xc))
@@ -3837,6 +3719,10 @@ class Decimal(object):
         # represented in fixed point; rescale them to 0e0.
         if not self and self._exp > 0 and spec['type'] in 'fF%':
             self = self._rescale(0, rounding)
+        if not self and spec['no_neg_0'] and self._sign:
+            adjusted_sign = 0
+        else:
+            adjusted_sign = self._sign
 
         # figure out placement of the decimal point
         leftdigits = self._exp + len(self._int)
@@ -3867,7 +3753,7 @@ class Decimal(object):
 
         # done with the decimal-specific stuff;  hand over the rest
         # of the formatting to the _format_number function
-        return _format_number(self._sign, intpart, fracpart, exp, spec)
+        return _format_number(adjusted_sign, intpart, fracpart, exp, spec)
 
 def _dec_from_triple(sign, coefficient, exponent, special=False):
     """Create a decimal instance directly, without any validation,
@@ -5677,8 +5563,6 @@ class _WorkRep(object):
     def __repr__(self):
         return "(%r, %r, %r)" % (self.sign, self.int, self.exp)
 
-    __str__ = __repr__
-
 
 
 def _normalize(op1, op2, prec = 0):
@@ -6187,7 +6071,7 @@ _exact_half = re.compile('50*$').match
 #
 # A format specifier for Decimal looks like:
 #
-#   [[fill]align][sign][#][0][minimumwidth][,][.precision][type]
+#   [[fill]align][sign][z][#][0][minimumwidth][,][.precision][type]
 
 _parse_format_specifier_regex = re.compile(r"""\A
 (?:
@@ -6195,6 +6079,7 @@ _parse_format_specifier_regex = re.compile(r"""\A
    (?P<align>[<>=^])
 )?
 (?P<sign>[-+ ])?
+(?P<no_neg_0>z)?
 (?P<alt>\#)?
 (?P<zeropad>0)?
 (?P<minimumwidth>(?!0)\d+)?

Lib/_weakrefset.py

@@ -80,8 +80,7 @@ class WeakSet:
         return wr in self.data
 
     def __reduce__(self):
-        return (self.__class__, (list(self),),
-                getattr(self, '__dict__', None))
+        return self.__class__, (list(self),), self.__getstate__()
 
     def add(self, item):
         if self._pending_removals:

Lib/ast.py

@@ -1,28 +1,24 @@
 """
-    ast
-    ~~~
+The `ast` module helps Python applications to process trees of the Python
+abstract syntax grammar.  The abstract syntax itself might change with
+each Python release; this module helps to find out programmatically what
+the current grammar looks like and allows modifications of it.
 
-    The `ast` module helps Python applications to process trees of the Python
-    abstract syntax grammar.  The abstract syntax itself might change with
-    each Python release; this module helps to find out programmatically what
-    the current grammar looks like and allows modifications of it.
+An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
+a flag to the `compile()` builtin function or by using the `parse()`
+function from this module.  The result will be a tree of objects whose
+classes all inherit from `ast.AST`.
 
-    An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
-    a flag to the `compile()` builtin function or by using the `parse()`
-    function from this module.  The result will be a tree of objects whose
-    classes all inherit from `ast.AST`.
+A modified abstract syntax tree can be compiled into a Python code object
+using the built-in `compile()` function.
 
-    A modified abstract syntax tree can be compiled into a Python code object
-    using the built-in `compile()` function.
+Additionally various helper functions are provided that make working with
+the trees simpler.  The main intention of the helper functions and this
+module in general is to provide an easy to use interface for libraries
+that work tightly with the python syntax (template engines for example).
 
-    Additionally various helper functions are provided that make working with
-    the trees simpler.  The main intention of the helper functions and this
-    module in general is to provide an easy to use interface for libraries
-    that work tightly with the python syntax (template engines for example).
-
-
-    :copyright: Copyright 2008 by Armin Ronacher.
-    :license: Python License.
+:copyright: Copyright 2008 by Armin Ronacher.
+:license: Python License.
 """
 import sys
 import re
@@ -32,13 +28,15 @@ from enum import IntEnum, auto, _simple_enum
 
 
 def parse(source, filename='<unknown>', mode='exec', *,
-          type_comments=False, feature_version=None):
+          type_comments=False, feature_version=None, optimize=-1):
     """
     Parse the source into an AST node.
     Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
     Pass type_comments=True to get back type comments where the syntax allows.
     """
     flags = PyCF_ONLY_AST
+    if optimize > 0:
+        flags |= PyCF_OPTIMIZED_AST
     if type_comments:
         flags |= PyCF_TYPE_COMMENTS
     if feature_version is None:
@@ -50,7 +48,7 @@ def parse(source, filename='<unknown>', mode='exec', *,
         feature_version = minor
     # Else it should be an int giving the minor version for 3.x.
     return compile(source, filename, mode, flags,
-                   _feature_version=feature_version)
+                   _feature_version=feature_version, optimize=optimize)
 
 
 def literal_eval(node_or_string):
@@ -112,7 +110,11 @@ def literal_eval(node_or_string):
     return _convert(node_or_string)
 
 
-def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
+def dump(
+    node, annotate_fields=True, include_attributes=False,
+    *,
+    indent=None, show_empty=False,
+):
     """
     Return a formatted dump of the tree in node.  This is mainly useful for
     debugging purposes.  If annotate_fields is true (by default),
@@ -123,6 +125,8 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
     include_attributes can be set to true.  If indent is a non-negative
     integer or string, then the tree will be pretty-printed with that indent
     level. None (the default) selects the single line representation.
+    If show_empty is False, then empty lists and fields that are None
+    will be omitted from the output for better readability.
     """
     def _format(node, level=0):
         if indent is not None:
@@ -135,6 +139,7 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
         if isinstance(node, AST):
             cls = type(node)
             args = []
+            args_buffer = []
             allsimple = True
             keywords = annotate_fields
             for name in node._fields:
@@ -146,6 +151,16 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
                 if value is None and getattr(cls, name, ...) is None:
                     keywords = True
                     continue
+                if not show_empty:
+                    if value == []:
+                        field_type = cls._field_types.get(name, object)
+                        if getattr(field_type, '__origin__', ...) is list:
+                            if not keywords:
+                                args_buffer.append(repr(value))
+                            continue
+                    if not keywords:
+                        args.extend(args_buffer)
+                        args_buffer = []
                 value, simple = _format(value, level)
                 allsimple = allsimple and simple
                 if keywords:
@@ -726,12 +741,11 @@ class _Unparser(NodeVisitor):
     output source code for the abstract syntax; original formatting
     is disregarded."""
 
-    def __init__(self, *, _avoid_backslashes=False):
+    def __init__(self):
         self._source = []
         self._precedences = {}
         self._type_ignores = {}
         self._indent = 0
-        self._avoid_backslashes = _avoid_backslashes
         self._in_try_star = False
 
     def interleave(self, inter, f, seq):
@@ -1104,12 +1118,21 @@ class _Unparser(NodeVisitor):
         if node.bound:
             self.write(": ")
             self.traverse(node.bound)
+        if node.default_value:
+            self.write(" = ")
+            self.traverse(node.default_value)
 
     def visit_TypeVarTuple(self, node):
         self.write("*" + node.name)
+        if node.default_value:
+            self.write(" = ")
+            self.traverse(node.default_value)
 
     def visit_ParamSpec(self, node):
         self.write("**" + node.name)
+        if node.default_value:
+            self.write(" = ")
+            self.traverse(node.default_value)
 
     def visit_TypeAlias(self, node):
         self.fill("type ")
@@ -1246,9 +1269,14 @@ class _Unparser(NodeVisitor):
                     fallback_to_repr = True
                     break
                 quote_types = new_quote_types
-            elif "\n" in value:
-                quote_types = [q for q in quote_types if q in _MULTI_QUOTES]
-                assert quote_types
+            else:
+                if "\n" in value:
+                    quote_types = [q for q in quote_types if q in _MULTI_QUOTES]
+                    assert quote_types
+
+                new_quote_types = [q for q in quote_types if q not in value]
+                if new_quote_types:
+                    quote_types = new_quote_types
             new_fstring_parts.append(value)
 
         if fallback_to_repr:
@@ -1268,13 +1296,19 @@ class _Unparser(NodeVisitor):
         quote_type = quote_types[0]
         self.write(f"{quote_type}{value}{quote_type}")
 
-    def _write_fstring_inner(self, node):
+    def _write_fstring_inner(self, node, is_format_spec=False):
         if isinstance(node, JoinedStr):
             # for both the f-string itself, and format_spec
             for value in node.values:
-                self._write_fstring_inner(value)
+                self._write_fstring_inner(value, is_format_spec=is_format_spec)
         elif isinstance(node, Constant) and isinstance(node.value, str):
             value = node.value.replace("{", "{{").replace("}", "}}")
+
+            if is_format_spec:
+                value = value.replace("\\", "\\\\")
+                value = value.replace("'", "\\'")
+                value = value.replace('"', '\\"')
+                value = value.replace("\n", "\\n")
             self.write(value)
         elif isinstance(node, FormattedValue):
             self.visit_FormattedValue(node)
@@ -1297,7 +1331,7 @@ class _Unparser(NodeVisitor):
                 self.write(f"!{chr(node.conversion)}")
             if node.format_spec:
                 self.write(":")
-                self._write_fstring_inner(node.format_spec)
+                self._write_fstring_inner(node.format_spec, is_format_spec=True)
 
     def visit_Name(self, node):
         self.write(node.id)
@@ -1317,8 +1351,6 @@ class _Unparser(NodeVisitor):
                 .replace("inf", _INFSTR)
                 .replace("nan", f"({_INFSTR}-{_INFSTR})")
             )
-        elif self._avoid_backslashes and isinstance(value, str):
-            self._write_str_avoiding_backslashes(value)
         else:
             self.write(repr(value))
 
@@ -1805,8 +1837,7 @@ def main():
     import argparse
 
     parser = argparse.ArgumentParser(prog='python -m ast')
-    parser.add_argument('infile', type=argparse.FileType(mode='rb'), nargs='?',
-                        default='-',
+    parser.add_argument('infile', nargs='?', default='-',
                         help='the file to parse; defaults to stdin')
     parser.add_argument('-m', '--mode', default='exec',
                         choices=('exec', 'single', 'eval', 'func_type'),
@@ -1820,9 +1851,14 @@ def main():
                         help='indentation of nodes (number of spaces)')
     args = parser.parse_args()
 
-    with args.infile as infile:
-        source = infile.read()
-    tree = parse(source, args.infile.name, args.mode, type_comments=args.no_type_comments)
+    if args.infile == '-':
+        name = '<stdin>'
+        source = sys.stdin.buffer.read()
+    else:
+        name = args.infile
+        with open(args.infile, 'rb') as infile:
+            source = infile.read()
+    tree = parse(source, name, args.mode, type_comments=args.no_type_comments)
     print(dump(tree, include_attributes=args.include_attributes, indent=args.indent))
 
 if __name__ == '__main__':

Lib/bz2.py

@@ -17,7 +17,7 @@ import _compression
 from _bz2 import BZ2Compressor, BZ2Decompressor
 
 
-_MODE_CLOSED   = 0
+# Value 0 no longer used
 _MODE_READ     = 1
 # Value 2 no longer used
 _MODE_WRITE    = 3
@@ -54,7 +54,7 @@ class BZ2File(_compression.BaseStream):
         """
         self._fp = None
         self._closefp = False
-        self._mode = _MODE_CLOSED
+        self._mode = None
 
         if not (1 <= compresslevel <= 9):
             raise ValueError("compresslevel must be between 1 and 9")
@@ -100,7 +100,7 @@ class BZ2File(_compression.BaseStream):
         May be called more than once without error. Once the file is
         closed, any other operation on it will raise a ValueError.
         """
-        if self._mode == _MODE_CLOSED:
+        if self.closed:
             return
         try:
             if self._mode == _MODE_READ:
@@ -115,13 +115,21 @@ class BZ2File(_compression.BaseStream):
             finally:
                 self._fp = None
                 self._closefp = False
-                self._mode = _MODE_CLOSED
                 self._buffer = None
 
     @property
     def closed(self):
         """True if this file is closed."""
-        return self._mode == _MODE_CLOSED
+        return self._fp is None
+
+    @property
+    def name(self):
+        self._check_not_closed()
+        return self._fp.name
+
+    @property
+    def mode(self):
+        return 'wb' if self._mode == _MODE_WRITE else 'rb'
 
     def fileno(self):
         """Return the file descriptor for the underlying file."""

Lib/cmd.py

@@ -42,7 +42,7 @@ listings of documented functions, miscellaneous topics, and undocumented
 functions respectively.
 """
 
-import string, sys
+import inspect, string, sys
 
 __all__ = ["Cmd"]
 
@@ -108,7 +108,15 @@ class Cmd:
                 import readline
                 self.old_completer = readline.get_completer()
                 readline.set_completer(self.complete)
-                readline.parse_and_bind(self.completekey+": complete")
+                if readline.backend == "editline":
+                    if self.completekey == 'tab':
+                        # libedit uses "^I" instead of "tab"
+                        command_string = "bind ^I rl_complete"
+                    else:
+                        command_string = f"bind {self.completekey} rl_complete"
+                else:
+                    command_string = f"{self.completekey}: complete"
+                readline.parse_and_bind(command_string)
             except ImportError:
                 pass
         try:
@@ -210,9 +218,8 @@ class Cmd:
         if cmd == '':
             return self.default(line)
         else:
-            try:
-                func = getattr(self, 'do_' + cmd)
-            except AttributeError:
+            func = getattr(self, 'do_' + cmd, None)
+            if func is None:
                 return self.default(line)
             return func(arg)
 
@@ -298,6 +305,7 @@ class Cmd:
             except AttributeError:
                 try:
                     doc=getattr(self, 'do_' + arg).__doc__
+                    doc = inspect.cleandoc(doc)
                     if doc:
                         self.stdout.write("%s\n"%str(doc))
                         return

Lib/codeop.py

@@ -44,6 +44,7 @@ __all__ = ["compile_command", "Compile", "CommandCompiler"]
 # Caveat emptor: These flags are undocumented on purpose and depending
 # on their effect outside the standard library is **unsupported**.
 PyCF_DONT_IMPLY_DEDENT = 0x200
+PyCF_ONLY_AST = 0x400
 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
 
 def _maybe_compile(compiler, source, filename, symbol):
@@ -73,15 +74,6 @@ def _maybe_compile(compiler, source, filename, symbol):
 
     return compiler(source, filename, symbol, incomplete_input=False)
 
-def _is_syntax_error(err1, err2):
-    rep1 = repr(err1)
-    rep2 = repr(err2)
-    if "was never closed" in rep1 and "was never closed" in rep2:
-        return False
-    if rep1 == rep2:
-        return True
-    return False
-
 def _compile(source, filename, symbol, incomplete_input=True):
     flags = 0
     if incomplete_input:
@@ -89,7 +81,6 @@ def _compile(source, filename, symbol, incomplete_input=True):
         flags |= PyCF_DONT_IMPLY_DEDENT
     return compile(source, filename, symbol, flags)
 
-
 def compile_command(source, filename="<input>", symbol="single"):
     r"""Compile a command and determine whether it is incomplete.
 
@@ -119,12 +110,14 @@ class Compile:
     def __init__(self):
         self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
 
-    def __call__(self, source, filename, symbol, **kwargs):
-        flags = self.flags
+    def __call__(self, source, filename, symbol, flags=0, **kwargs):
+        flags |= self.flags
         if kwargs.get('incomplete_input', True) is False:
             flags &= ~PyCF_DONT_IMPLY_DEDENT
             flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT
         codeob = compile(source, filename, symbol, flags, True)
+        if flags & PyCF_ONLY_AST:
+            return codeob  # this is an ast.Module in this case
         for feature in _features:
             if codeob.co_flags & feature.compiler_flag:
                 self.flags |= feature.compiler_flag

Lib/compileall.py

@@ -97,9 +97,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
     success = True
     if workers != 1 and ProcessPoolExecutor is not None:
+        import multiprocessing
+        if multiprocessing.get_start_method() == 'fork':
+            mp_context = multiprocessing.get_context('forkserver')
+        else:
+            mp_context = None
         # If workers == 0, let ProcessPoolExecutor choose
         workers = workers or None
-        with ProcessPoolExecutor(max_workers=workers) as executor:
+        with ProcessPoolExecutor(max_workers=workers,
+                                 mp_context=mp_context) as executor:
             results = executor.map(partial(compile_file,
                                            ddir=ddir, force=force,
                                            rx=rx, quiet=quiet,
@@ -110,7 +116,8 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
                                            prependdir=prependdir,
                                            limit_sl_dest=limit_sl_dest,
                                            hardlink_dupes=hardlink_dupes),
-                                   files)
+                                   files,
+                                   chunksize=4)
             success = min(results, default=True)
     else:
         for file in files:
@@ -166,13 +173,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
     if stripdir is not None:
         fullname_parts = fullname.split(os.path.sep)
         stripdir_parts = stripdir.split(os.path.sep)
-        ddir_parts = list(fullname_parts)
 
-        for spart, opart in zip(stripdir_parts, fullname_parts):
-            if spart == opart:
-                ddir_parts.remove(spart)
-
-        dfile = os.path.join(*ddir_parts)
+        if stripdir_parts != fullname_parts[:len(stripdir_parts)]:
+            if quiet < 2:
+                print("The stripdir path {!r} is not a valid prefix for "
+                      "source path {!r}; ignoring".format(stripdir, fullname))
+        else:
+            dfile = os.path.join(*fullname_parts[len(stripdir_parts):])
 
     if prependdir is not None:
         if dfile is None:

Lib/contextlib.py

@@ -20,6 +20,8 @@ class AbstractContextManager(abc.ABC):
 
     __class_getitem__ = classmethod(GenericAlias)
 
+    __slots__ = ()
+
     def __enter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -42,6 +44,8 @@ class AbstractAsyncContextManager(abc.ABC):
 
     __class_getitem__ = classmethod(GenericAlias)
 
+    __slots__ = ()
+
     async def __aenter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -565,11 +569,12 @@ class ExitStack(_BaseExitStack, AbstractContextManager):
         return self
 
     def __exit__(self, *exc_details):
-        received_exc = exc_details[0] is not None
+        exc = exc_details[1]
+        received_exc = exc is not None
 
         # We manipulate the exception state so it behaves as though
         # we were actually nesting multiple with statements
-        frame_exc = sys.exc_info()[1]
+        frame_exc = sys.exception()
         def _fix_exception_context(new_exc, old_exc):
             # Context may not be correct, so find the end of the chain
             while 1:
@@ -592,24 +597,28 @@ class ExitStack(_BaseExitStack, AbstractContextManager):
             is_sync, cb = self._exit_callbacks.pop()
             assert is_sync
             try:
+                if exc is None:
+                    exc_details = None, None, None
+                else:
+                    exc_details = type(exc), exc, exc.__traceback__
                 if cb(*exc_details):
                     suppressed_exc = True
                     pending_raise = False
-                    exc_details = (None, None, None)
-            except:
-                new_exc_details = sys.exc_info()
+                    exc = None
+            except BaseException as new_exc:
                 # simulate the stack of exceptions by setting the context
-                _fix_exception_context(new_exc_details[1], exc_details[1])
+                _fix_exception_context(new_exc, exc)
                 pending_raise = True
-                exc_details = new_exc_details
+                exc = new_exc
+
         if pending_raise:
             try:
-                # bare "raise exc_details[1]" replaces our carefully
+                # bare "raise exc" replaces our carefully
                 # set-up context
-                fixed_ctx = exc_details[1].__context__
-                raise exc_details[1]
+                fixed_ctx = exc.__context__
+                raise exc
             except BaseException:
-                exc_details[1].__context__ = fixed_ctx
+                exc.__context__ = fixed_ctx
                 raise
         return received_exc and suppressed_exc
 
@@ -705,11 +714,12 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         return self
 
     async def __aexit__(self, *exc_details):
-        received_exc = exc_details[0] is not None
+        exc = exc_details[1]
+        received_exc = exc is not None
 
         # We manipulate the exception state so it behaves as though
         # we were actually nesting multiple with statements
-        frame_exc = sys.exc_info()[1]
+        frame_exc = sys.exception()
         def _fix_exception_context(new_exc, old_exc):
             # Context may not be correct, so find the end of the chain
             while 1:
@@ -731,6 +741,10 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         while self._exit_callbacks:
             is_sync, cb = self._exit_callbacks.pop()
             try:
+                if exc is None:
+                    exc_details = None, None, None
+                else:
+                    exc_details = type(exc), exc, exc.__traceback__
                 if is_sync:
                     cb_suppress = cb(*exc_details)
                 else:
@@ -739,21 +753,21 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
                 if cb_suppress:
                     suppressed_exc = True
                     pending_raise = False
-                    exc_details = (None, None, None)
-            except:
-                new_exc_details = sys.exc_info()
+                    exc = None
+            except BaseException as new_exc:
                 # simulate the stack of exceptions by setting the context
-                _fix_exception_context(new_exc_details[1], exc_details[1])
+                _fix_exception_context(new_exc, exc)
                 pending_raise = True
-                exc_details = new_exc_details
+                exc = new_exc
+
         if pending_raise:
             try:
-                # bare "raise exc_details[1]" replaces our carefully
+                # bare "raise exc" replaces our carefully
                 # set-up context
-                fixed_ctx = exc_details[1].__context__
-                raise exc_details[1]
+                fixed_ctx = exc.__context__
+                raise exc
             except BaseException:
-                exc_details[1].__context__ = fixed_ctx
+                exc.__context__ = fixed_ctx
                 raise
         return received_exc and suppressed_exc
 

Lib/copy.py

@@ -4,8 +4,9 @@ Interface summary:
 
         import copy
 
-        x = copy.copy(y)        # make a shallow copy of y
-        x = copy.deepcopy(y)    # make a deep copy of y
+        x = copy.copy(y)                # make a shallow copy of y
+        x = copy.deepcopy(y)            # make a deep copy of y
+        x = copy.replace(y, a=1, b=2)   # new object with fields replaced, as defined by `__replace__`
 
 For module specific errors, copy.Error is raised.
 
@@ -56,7 +57,7 @@ class Error(Exception):
     pass
 error = Error   # backward compatibility
 
-__all__ = ["Error", "copy", "deepcopy"]
+__all__ = ["Error", "copy", "deepcopy", "replace"]
 
 def copy(x):
     """Shallow copy operation on arbitrary Python objects.
@@ -121,13 +122,13 @@ def deepcopy(x, memo=None, _nil=[]):
     See the module's __doc__ string for more info.
     """
 
+    d = id(x)
     if memo is None:
         memo = {}
-
-    d = id(x)
-    y = memo.get(d, _nil)
-    if y is not _nil:
-        return y
+    else:
+        y = memo.get(d, _nil)
+        if y is not _nil:
+            return y
 
     cls = type(x)
 
@@ -290,3 +291,16 @@ def _reconstruct(x, memo, func, args,
     return y
 
 del types, weakref
+
+
+def replace(obj, /, **changes):
+    """Return a new object replacing specified fields with new values.
+
+    This is especially useful for immutable objects, like named tuples or
+    frozen dataclasses.
+    """
+    cls = obj.__class__
+    func = getattr(cls, '__replace__', None)
+    if func is None:
+        raise TypeError(f"replace() does not support {cls.__name__} objects")
+    return func(obj, **changes)

Lib/csv.py

@@ -1,28 +1,90 @@
 
-"""
-csv.py - read/write/investigate CSV files
+r"""
+CSV parsing and writing.
+
+This module provides classes that assist in the reading and writing
+of Comma Separated Value (CSV) files, and implements the interface
+described by PEP 305.  Although many CSV files are simple to parse,
+the format is not formally defined by a stable specification and
+is subtle enough that parsing lines of a CSV file with something
+like line.split(",") is bound to fail.  The module supports three
+basic APIs: reading, writing, and registration of dialects.
+
+
+DIALECT REGISTRATION:
+
+Readers and writers support a dialect argument, which is a convenient
+handle on a group of settings.  When the dialect argument is a string,
+it identifies one of the dialects previously registered with the module.
+If it is a class or instance, the attributes of the argument are used as
+the settings for the reader or writer:
+
+    class excel:
+        delimiter = ','
+        quotechar = '"'
+        escapechar = None
+        doublequote = True
+        skipinitialspace = False
+        lineterminator = '\r\n'
+        quoting = QUOTE_MINIMAL
+
+SETTINGS:
+
+    * quotechar - specifies a one-character string to use as the
+        quoting character.  It defaults to '"'.
+    * delimiter - specifies a one-character string to use as the
+        field separator.  It defaults to ','.
+    * skipinitialspace - specifies how to interpret spaces which
+        immediately follow a delimiter.  It defaults to False, which
+        means that spaces immediately following a delimiter is part
+        of the following field.
+    * lineterminator - specifies the character sequence which should
+        terminate rows.
+    * quoting - controls when quotes should be generated by the writer.
+        It can take on any of the following module constants:
+
+        csv.QUOTE_MINIMAL means only when required, for example, when a
+            field contains either the quotechar or the delimiter
+        csv.QUOTE_ALL means that quotes are always placed around fields.
+        csv.QUOTE_NONNUMERIC means that quotes are always placed around
+            fields which do not parse as integers or floating-point
+            numbers.
+        csv.QUOTE_STRINGS means that quotes are always placed around
+            fields which are strings.  Note that the Python value None
+            is not a string.
+        csv.QUOTE_NOTNULL means that quotes are only placed around fields
+            that are not the Python value None.
+        csv.QUOTE_NONE means that quotes are never placed around fields.
+    * escapechar - specifies a one-character string used to escape
+        the delimiter when quoting is set to QUOTE_NONE.
+    * doublequote - controls the handling of quotes inside fields.  When
+        True, two consecutive quotes are interpreted as one during read,
+        and when writing, each quote character embedded in the data is
+        written as two quotes
 """
 
 import re
 import types
-from _csv import Error, __version__, writer, reader, register_dialect, \
+from _csv import Error, writer, reader, register_dialect, \
                  unregister_dialect, get_dialect, list_dialects, \
                  field_size_limit, \
                  QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \
-                 QUOTE_STRINGS, QUOTE_NOTNULL, \
-                 __doc__
+                 QUOTE_STRINGS, QUOTE_NOTNULL
 from _csv import Dialect as _Dialect
 
 from io import StringIO
 
 __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE",
            "QUOTE_STRINGS", "QUOTE_NOTNULL",
-           "Error", "Dialect", "__doc__", "excel", "excel_tab",
+           "Error", "Dialect", "excel", "excel_tab",
            "field_size_limit", "reader", "writer",
            "register_dialect", "get_dialect", "list_dialects", "Sniffer",
-           "unregister_dialect", "__version__", "DictReader", "DictWriter",
+           "unregister_dialect", "DictReader", "DictWriter",
            "unix_dialect"]
 
+__version__ = "1.0"
+
+
 class Dialect:
     """Describe a CSV dialect.
 
@@ -51,8 +113,8 @@ class Dialect:
         try:
             _Dialect(self)
         except TypeError as e:
-            # We do this for compatibility with py2.3
-            raise Error(str(e))
+            # Re-raise to get a traceback showing more user code.
+            raise Error(str(e)) from None
 
 class excel(Dialect):
     """Describe the usual properties of Excel-generated CSV files."""

Lib/decimal.py

@@ -1,11 +1,109 @@
+"""Decimal fixed-point and floating-point arithmetic.
+
+This is an implementation of decimal floating-point arithmetic based on
+the General Decimal Arithmetic Specification:
+
+    http://speleotrove.com/decimal/decarith.html
+
+and IEEE standard 854-1987:
+
+    http://en.wikipedia.org/wiki/IEEE_854-1987
+
+Decimal floating point has finite precision with arbitrarily large bounds.
+
+The purpose of this module is to support arithmetic using familiar
+"schoolhouse" rules and to avoid some of the tricky representation
+issues associated with binary floating point.  The package is especially
+useful for financial applications or for contexts where users have
+expectations that are at odds with binary floating point (for instance,
+in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead
+of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected
+Decimal('0.00')).
+
+Here are some examples of using the decimal module:
+
+>>> from decimal import *
+>>> setcontext(ExtendedContext)
+>>> Decimal(0)
+Decimal('0')
+>>> Decimal('1')
+Decimal('1')
+>>> Decimal('-.0123')
+Decimal('-0.0123')
+>>> Decimal(123456)
+Decimal('123456')
+>>> Decimal('123.45e12345678')
+Decimal('1.2345E+12345680')
+>>> Decimal('1.33') + Decimal('1.27')
+Decimal('2.60')
+>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')
+Decimal('-2.20')
+>>> dig = Decimal(1)
+>>> print(dig / Decimal(3))
+0.333333333
+>>> getcontext().prec = 18
+>>> print(dig / Decimal(3))
+0.333333333333333333
+>>> print(dig.sqrt())
+1
+>>> print(Decimal(3).sqrt())
+1.73205080756887729
+>>> print(Decimal(3) ** 123)
+4.85192780976896427E+58
+>>> inf = Decimal(1) / Decimal(0)
+>>> print(inf)
+Infinity
+>>> neginf = Decimal(-1) / Decimal(0)
+>>> print(neginf)
+-Infinity
+>>> print(neginf + inf)
+NaN
+>>> print(neginf * inf)
+-Infinity
+>>> print(dig / 0)
+Infinity
+>>> getcontext().traps[DivisionByZero] = 1
+>>> print(dig / 0)
+Traceback (most recent call last):
+  ...
+  ...
+  ...
+decimal.DivisionByZero: x / 0
+>>> c = Context()
+>>> c.traps[InvalidOperation] = 0
+>>> print(c.flags[InvalidOperation])
+0
+>>> c.divide(Decimal(0), Decimal(0))
+Decimal('NaN')
+>>> c.traps[InvalidOperation] = 1
+>>> print(c.flags[InvalidOperation])
+1
+>>> c.flags[InvalidOperation] = 0
+>>> print(c.flags[InvalidOperation])
+0
+>>> print(c.divide(Decimal(0), Decimal(0)))
+Traceback (most recent call last):
+  ...
+  ...
+  ...
+decimal.InvalidOperation: 0 / 0
+>>> print(c.flags[InvalidOperation])
+1
+>>> c.flags[InvalidOperation] = 0
+>>> c.traps[InvalidOperation] = 0
+>>> print(c.divide(Decimal(0), Decimal(0)))
+NaN
+>>> print(c.flags[InvalidOperation])
+1
+>>>
+"""
 
 try:
     from _decimal import *
-    from _decimal import __doc__
     from _decimal import __version__
     from _decimal import __libmpdec_version__
 except ImportError:
-    from _pydecimal import *
-    from _pydecimal import __doc__
-    from _pydecimal import __version__
-    from _pydecimal import __libmpdec_version__
+    import _pydecimal
+    import sys
+    _pydecimal.__doc__ = __doc__
+    sys.modules[__name__] = _pydecimal

Lib/gzip.py

@@ -5,11 +5,15 @@ but random access is not allowed."""
 
 # based on Andrew Kuchling's minigzip.py distributed with the zlib module
 
-import struct, sys, time, os
-import zlib
+import _compression
 import builtins
 import io
-import _compression
+import os
+import struct
+import sys
+import time
+import weakref
+import zlib
 
 __all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"]
 
@@ -125,10 +129,13 @@ class BadGzipFile(OSError):
 class _WriteBufferStream(io.RawIOBase):
     """Minimal object to pass WriteBuffer flushes into GzipFile"""
     def __init__(self, gzip_file):
-        self.gzip_file = gzip_file
+        self.gzip_file = weakref.ref(gzip_file)
 
     def write(self, data):
-        return self.gzip_file._write_raw(data)
+        gzip_file = self.gzip_file()
+        if gzip_file is None:
+            raise RuntimeError("lost gzip_file")
+        return gzip_file._write_raw(data)
 
     def seekable(self):
         return False
@@ -190,51 +197,58 @@ class GzipFile(_compression.BaseStream):
             raise ValueError("Invalid mode: {!r}".format(mode))
         if mode and 'b' not in mode:
             mode += 'b'
-        if fileobj is None:
-            fileobj = self.myfileobj = builtins.open(filename, mode or 'rb')
-        if filename is None:
-            filename = getattr(fileobj, 'name', '')
-            if not isinstance(filename, (str, bytes)):
-                filename = ''
-        else:
-            filename = os.fspath(filename)
-        origmode = mode
-        if mode is None:
-            mode = getattr(fileobj, 'mode', 'rb')
+
+        try:
+            if fileobj is None:
+                fileobj = self.myfileobj = builtins.open(filename, mode or 'rb')
+            if filename is None:
+                filename = getattr(fileobj, 'name', '')
+                if not isinstance(filename, (str, bytes)):
+                    filename = ''
+            else:
+                filename = os.fspath(filename)
+            origmode = mode
+            if mode is None:
+                mode = getattr(fileobj, 'mode', 'rb')
 
 
-        if mode.startswith('r'):
-            self.mode = READ
-            raw = _GzipReader(fileobj)
-            self._buffer = io.BufferedReader(raw)
-            self.name = filename
+            if mode.startswith('r'):
+                self.mode = READ
+                raw = _GzipReader(fileobj)
+                self._buffer = io.BufferedReader(raw)
+                self.name = filename
 
-        elif mode.startswith(('w', 'a', 'x')):
-            if origmode is None:
-                import warnings
-                warnings.warn(
-                    "GzipFile was opened for writing, but this will "
-                    "change in future Python releases.  "
-                    "Specify the mode argument for opening it for writing.",
-                    FutureWarning, 2)
-            self.mode = WRITE
-            self._init_write(filename)
-            self.compress = zlib.compressobj(compresslevel,
-                                             zlib.DEFLATED,
-                                             -zlib.MAX_WBITS,
-                                             zlib.DEF_MEM_LEVEL,
-                                             0)
-            self._write_mtime = mtime
-            self._buffer_size = _WRITE_BUFFER_SIZE
-            self._buffer = io.BufferedWriter(_WriteBufferStream(self),
-                                             buffer_size=self._buffer_size)
-        else:
-            raise ValueError("Invalid mode: {!r}".format(mode))
+            elif mode.startswith(('w', 'a', 'x')):
+                if origmode is None:
+                    import warnings
+                    warnings.warn(
+                        "GzipFile was opened for writing, but this will "
+                        "change in future Python releases.  "
+                        "Specify the mode argument for opening it for writing.",
+                        FutureWarning, 2)
+                self.mode = WRITE
+                self._init_write(filename)
+                self.compress = zlib.compressobj(compresslevel,
+                                                 zlib.DEFLATED,
+                                                 -zlib.MAX_WBITS,
+                                                 zlib.DEF_MEM_LEVEL,
+                                                 0)
+                self._write_mtime = mtime
+                self._buffer_size = _WRITE_BUFFER_SIZE
+                self._buffer = io.BufferedWriter(_WriteBufferStream(self),
+                                                 buffer_size=self._buffer_size)
+            else:
+                raise ValueError("Invalid mode: {!r}".format(mode))
 
-        self.fileobj = fileobj
+            self.fileobj = fileobj
 
-        if self.mode == WRITE:
-            self._write_gzip_header(compresslevel)
+            if self.mode == WRITE:
+                self._write_gzip_header(compresslevel)
+        except:
+            # Avoid a ResourceWarning if the write fails,
+            # eg read-only file or KeyboardInterrupt
+            self._close()
+            raise
 
     @property
     def mtime(self):
@@ -363,11 +377,14 @@ class GzipFile(_compression.BaseStream):
             elif self.mode == READ:
                 self._buffer.close()
         finally:
-            self.fileobj = None
-            myfileobj = self.myfileobj
-            if myfileobj:
-                self.myfileobj = None
-                myfileobj.close()
+            self._close()
+
+    def _close(self):
+        self.fileobj = None
+        myfileobj = self.myfileobj
+        if myfileobj is not None:
+            self.myfileobj = None
+            myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_not_closed()
@@ -580,12 +597,12 @@ class _GzipReader(_compression.DecompressReader):
         self._new_member = True
 
 
-def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=0):
+def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
     """Compress data in one shot and return the compressed string.
 
     compresslevel sets the compression level in range of 0-9.
-    mtime can be used to set the modification time.
-    The modification time is set to 0 by default, for reproducibility.
+    mtime can be used to set the modification time. The modification time is
+    set to the current time by default.
     """
     # Wbits=31 automatically includes a gzip header and trailer.
     gzip_data = zlib.compress(data, level=compresslevel, wbits=31)

Lib/html/__init__.py

@@ -25,7 +25,7 @@ def escape(s, quote=True):
     return s
 
 
-# see http://www.w3.org/TR/html5/syntax.html#tokenizing-character-references
+# see https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
 
 _invalid_charrefs = {
     0x00: '\ufffd',  # REPLACEMENT CHARACTER

Lib/html/entities.py

@@ -3,8 +3,7 @@
 __all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs']
 
 
-# maps the HTML entity name to the Unicode code point
-# from https://html.spec.whatwg.org/multipage/named-characters.html
+# maps HTML4 entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -261,7 +260,11 @@ name2codepoint = {
 }
 
 
-# maps the HTML5 named character references to the equivalent Unicode character(s)
+# HTML5 named character references
+# Generated by Tools/build/parse_html5_entities.py
+# from https://html.spec.whatwg.org/entities.json and
+# https://html.spec.whatwg.org/multipage/named-characters.html.
+# Map HTML5 named character references to the equivalent Unicode character(s).
 html5 = {
     'Aacute': '\xc1',
     'aacute': '\xe1',

Lib/html/parser.py

@@ -12,6 +12,7 @@ import re
 import _markupbase
 
 from html import unescape
+from html.entities import html5 as html5_entities
 
 
 __all__ = ['HTMLParser']
@@ -23,6 +24,7 @@ incomplete = re.compile('&[a-zA-Z#]')
 
 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
 charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
+attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?')
 
 starttagopen = re.compile('<[a-zA-Z]')
 piclose = re.compile('>')
@@ -57,6 +59,22 @@ endendtag = re.compile('>')
 # </ and the tag name, so maybe this should be fixed
 endtagfind = re.compile(r'</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
 
+# Character reference processing logic specific to attribute values
+# See: https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
+def _replace_attr_charref(match):
+    ref = match.group(0)
+    # Numeric / hex char refs must always be unescaped
+    if ref.startswith('&#'):
+        return unescape(ref)
+    # Named character / entity references must only be unescaped
+    # if they are an exact match, and they are not followed by an equals sign
+    if not ref.endswith('=') and ref[1:] in html5_entities:
+        return unescape(ref)
+    # Otherwise do not unescape
+    return ref
+
+def _unescape_attrvalue(s):
+    return attr_charref.sub(_replace_attr_charref, s)
 
 
 class HTMLParser(_markupbase.ParserBase):
@@ -89,6 +107,7 @@ class HTMLParser(_markupbase.ParserBase):
         If convert_charrefs is True (the default), all character references
         are automatically converted to the corresponding Unicode characters.
         """
+        super().__init__()
         self.convert_charrefs = convert_charrefs
         self.reset()
 
@@ -98,7 +117,7 @@ class HTMLParser(_markupbase.ParserBase):
         self.lasttag = '???'
         self.interesting = interesting_normal
         self.cdata_elem = None
-        _markupbase.ParserBase.reset(self)
+        super().reset()
 
     def feed(self, data):
         r"""Feed data to the parser.
@@ -241,7 +260,7 @@ class HTMLParser(_markupbase.ParserBase):
             else:
                 assert 0, "interesting.search() lied"
         # end while
-        if end and i < n and not self.cdata_elem:
+        if end and i < n:
             if self.convert_charrefs and not self.cdata_elem:
                 self.handle_data(unescape(rawdata[i:n]))
             else:
@@ -259,7 +278,7 @@ class HTMLParser(_markupbase.ParserBase):
         if rawdata[i:i+4] == '<!--':
             # this case is actually already handled in goahead()
             return self.parse_comment(i)
-        elif rawdata[i:i+3] == '<![':
+        elif rawdata[i:i+9] == '<![CDATA[':
             return self.parse_marked_section(i)
         elif rawdata[i:i+9].lower() == '<!doctype':
             # find the closing >
@@ -276,7 +295,7 @@ class HTMLParser(_markupbase.ParserBase):
     def parse_bogus_comment(self, i, report=1):
         rawdata = self.rawdata
         assert rawdata[i:i+2] in ('<!', '</'), ('unexpected call to '
-                                                'parse_comment()')
+                                                'parse_bogus_comment()')
         pos = rawdata.find('>', i+2)
         if pos == -1:
             return -1
@@ -322,7 +341,7 @@ class HTMLParser(_markupbase.ParserBase):
                  attrvalue[:1] == '"' == attrvalue[-1:]:
                 attrvalue = attrvalue[1:-1]
             if attrvalue:
-                attrvalue = unescape(attrvalue)
+                attrvalue = _unescape_attrvalue(attrvalue)
             attrs.append((attrname.lower(), attrvalue))
             k = m.end()
 

Lib/json/__init__.py

@@ -1,4 +1,4 @@
-r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of
+r"""JSON (JavaScript Object Notation) <https://json.org> is a subset of
 JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
 interchange format.
 

Lib/json/decoder.py

@@ -41,7 +41,6 @@ class JSONDecodeError(ValueError):
         pos += col
         return cls(msg, doc, pos)
 
-
     # Note that this exception is used from _json
     def __init__(self, msg, doc, pos):
         lineno = doc.count('\n', 0, pos) + 1
@@ -65,17 +64,18 @@ _CONSTANTS = {
 }
 
 
+HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS)
 STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
 BACKSLASH = {
     '"': '"', '\\': '\\', '/': '/',
     'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t',
 }
 
-def _decode_uXXXX(s, pos):
-    esc = s[pos + 1:pos + 5]
-    if len(esc) == 4 and esc[1] not in 'xX':
+def _decode_uXXXX(s, pos, _m=HEXDIGITS.match):
+    esc = _m(s, pos + 1)
+    if esc is not None:
         try:
-            return int(esc, 16)
+            return int(esc.group(), 16)
         except ValueError:
             pass
     msg = "Invalid \\uXXXX escape"
@@ -215,10 +215,13 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook,
             break
         elif nextchar != ',':
             raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
+        comma_idx = end - 1
         end = _w(s, end).end()
         nextchar = s[end:end + 1]
         end += 1
         if nextchar != '"':
+            if nextchar == '}':
+                raise JSONDecodeError("Illegal trailing comma before end of object", s, comma_idx)
             raise JSONDecodeError(
                 "Expecting property name enclosed in double quotes", s, end - 1)
     if object_pairs_hook is not None:
@@ -255,19 +258,23 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
             break
         elif nextchar != ',':
             raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
+        comma_idx = end - 1
         try:
             if s[end] in _ws:
                 end += 1
                 if s[end] in _ws:
                     end = _w(s, end + 1).end()
+            nextchar = s[end:end + 1]
         except IndexError:
             pass
+        if nextchar == ']':
+            raise JSONDecodeError("Illegal trailing comma before end of array", s, comma_idx)
 
     return values, end
 
 
 class JSONDecoder(object):
-    """Simple JSON <http://json.org> decoder
+    """Simple JSON <https://json.org> decoder
 
     Performs the following translations in decoding by default:
 

Lib/json/encoder.py

@@ -30,6 +30,7 @@ ESCAPE_DCT = {
 for i in range(0x20):
     ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
     #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+del i
 
 INFINITY = float('inf')
 
@@ -71,7 +72,7 @@ encode_basestring_ascii = (
     c_encode_basestring_ascii or py_encode_basestring_ascii)
 
 class JSONEncoder(object):
-    """Extensible JSON <http://json.org> encoder for Python data structures.
+    """Extensible JSON <https://json.org> encoder for Python data structures.
 
     Supports the following objects and types by default:
 
@@ -107,8 +108,8 @@ class JSONEncoder(object):
         """Constructor for JSONEncoder, with sensible defaults.
 
         If skipkeys is false, then it is a TypeError to attempt
-        encoding of keys that are not str, int, float or None.  If
-        skipkeys is True, such items are simply skipped.
+        encoding of keys that are not str, int, float, bool or None.
+        If skipkeys is True, such items are simply skipped.
 
         If ensure_ascii is true, the output is guaranteed to be str
         objects with all incoming non-ASCII characters escaped.  If
@@ -173,7 +174,7 @@ class JSONEncoder(object):
                 else:
                     return list(iterable)
                 # Let the base class default method raise the TypeError
-                return JSONEncoder.default(self, o)
+                return super().default(o)
 
         """
         raise TypeError(f'Object of type {o.__class__.__name__} '
@@ -243,15 +244,18 @@ class JSONEncoder(object):
             return text
 
 
-        if (_one_shot and c_make_encoder is not None
-                and self.indent is None):
+        if self.indent is None or isinstance(self.indent, str):
+            indent = self.indent
+        else:
+            indent = ' ' * self.indent
+        if _one_shot and c_make_encoder is not None:
             _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
+                markers, self.default, _encoder, indent,
                 self.key_separator, self.item_separator, self.sort_keys,
                 self.skipkeys, self.allow_nan)
         else:
             _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
+                markers, self.default, _encoder, indent, floatstr,
                 self.key_separator, self.item_separator, self.sort_keys,
                 self.skipkeys, _one_shot)
         return _iterencode(o, 0)
@@ -271,9 +275,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
         _intstr=int.__repr__,
     ):
 
-    if _indent is not None and not isinstance(_indent, str):
-        _indent = ' ' * _indent
-
     def _iterencode_list(lst, _current_indent_level):
         if not lst:
             yield '[]'
@@ -344,7 +345,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
             _current_indent_level += 1
             newline_indent = '\n' + _indent * _current_indent_level
             item_separator = _item_separator + newline_indent
-            yield newline_indent
         else:
             newline_indent = None
             item_separator = _item_separator
@@ -377,6 +377,8 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
                                 f'not {key.__class__.__name__}')
             if first:
                 first = False
+                if newline_indent is not None:
+                    yield newline_indent
             else:
                 yield item_separator
             yield _encoder(key)
@@ -403,7 +405,7 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
                 else:
                     chunks = _iterencode(value, _current_indent_level)
                 yield from chunks
-        if newline_indent is not None:
+        if not first and newline_indent is not None:
             _current_indent_level -= 1
             yield '\n' + _indent * _current_indent_level
         yield '}'

Lib/json/scanner.py

@@ -9,7 +9,7 @@ except ImportError:
 __all__ = ['make_scanner']
 
 NUMBER_RE = re.compile(
-    r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
+    r'(-?(?:0|[1-9][0-9]*))(\.[0-9]+)?([eE][-+]?[0-9]+)?',
     (re.VERBOSE | re.MULTILINE | re.DOTALL))
 
 def py_make_scanner(context):

Lib/json/tool.py

@@ -13,7 +13,6 @@ Usage::
 import argparse
 import json
 import sys
-from pathlib import Path
 
 
 def main():
@@ -22,11 +21,9 @@ def main():
                    'to validate and pretty-print JSON objects.')
     parser = argparse.ArgumentParser(prog=prog, description=description)
     parser.add_argument('infile', nargs='?',
-                        type=argparse.FileType(encoding="utf-8"),
                         help='a JSON file to be validated or pretty-printed',
-                        default=sys.stdin)
+                        default='-')
     parser.add_argument('outfile', nargs='?',
-                        type=Path,
                         help='write the output of infile to outfile',
                         default=None)
     parser.add_argument('--sort-keys', action='store_true', default=False,
@@ -59,23 +56,30 @@ def main():
         dump_args['indent'] = None
         dump_args['separators'] = ',', ':'
 
-    with options.infile as infile:
+    try:
+        if options.infile == '-':
+            infile = sys.stdin
+        else:
+            infile = open(options.infile, encoding='utf-8')
         try:
             if options.json_lines:
                 objs = (json.loads(line) for line in infile)
             else:
                 objs = (json.load(infile),)
+        finally:
+            if infile is not sys.stdin:
+                infile.close()
 
-            if options.outfile is None:
-                out = sys.stdout
-            else:
-                out = options.outfile.open('w', encoding='utf-8')
-            with out as outfile:
-                for obj in objs:
-                    json.dump(obj, outfile, **dump_args)
-                    outfile.write('\n')
-        except ValueError as e:
-            raise SystemExit(e)
+        if options.outfile is None:
+            outfile = sys.stdout
+        else:
+            outfile = open(options.outfile, 'w', encoding='utf-8')
+        with outfile:
+            for obj in objs:
+                json.dump(obj, outfile, **dump_args)
+                outfile.write('\n')
+    except ValueError as e:
+        raise SystemExit(e)
 
 
 if __name__ == '__main__':

Lib/logging/__init__.py

@@ -56,7 +56,7 @@ __date__    = "07 February 2010"
 #
 #_startTime is used as the base when calculating the relative time of events
 #
-_startTime = time.time()
+_startTime = time.time_ns()
 
 #
 #raiseExceptions is used to see if exceptions during handling should be
@@ -159,12 +159,9 @@ def addLevelName(level, levelName):
 
     This is used when converting levels to text during message formatting.
     """
-    _acquireLock()
-    try:    #unlikely to cause an exception, but you never know...
+    with _lock:
         _levelToName[level] = levelName
         _nameToLevel[levelName] = level
-    finally:
-        _releaseLock()
 
 if hasattr(sys, "_getframe"):
     currentframe = lambda: sys._getframe(1)
@@ -201,7 +198,7 @@ def _is_internal_frame(frame):
     """Signal whether the frame is a CPython or logging module internal."""
     filename = os.path.normcase(frame.f_code.co_filename)
     return filename == _srcfile or (
-            "importlib" in filename and "_bootstrap" in filename
+        "importlib" in filename and "_bootstrap" in filename
     )
 
 
@@ -231,21 +228,27 @@ def _checkLevel(level):
 #
 _lock = threading.RLock()
 
-def _acquireLock():
+def _prepareFork():
     """
-    Acquire the module-level lock for serializing access to shared data.
+    Prepare to fork a new child process by acquiring the module-level lock.
 
-    This should be released with _releaseLock().
+    This should be used in conjunction with _afterFork().
     """
-    if _lock:
+    # Wrap the lock acquisition in a try-except to prevent the lock from being
+    # abandoned in the event of an asynchronous exception. See gh-106238.
+    try:
         _lock.acquire()
-
-def _releaseLock():
-    """
-    Release the module-level lock acquired by calling _acquireLock().
-    """
-    if _lock:
+    except BaseException:
         _lock.release()
+        raise
+
+def _afterFork():
+    """
+    After a new child process has been forked, release the module-level lock.
+
+    This should be used in conjunction with _prepareFork().
+    """
+    _lock.release()
 
 
 # Prevent a held logging lock from blocking a child from logging.
@@ -260,23 +263,20 @@ else:
     _at_fork_reinit_lock_weakset = weakref.WeakSet()
 
     def _register_at_fork_reinit_lock(instance):
-        _acquireLock()
-        try:
+        with _lock:
             _at_fork_reinit_lock_weakset.add(instance)
-        finally:
-            _releaseLock()
 
     def _after_at_fork_child_reinit_locks():
         for handler in _at_fork_reinit_lock_weakset:
             handler._at_fork_reinit()
 
-        # _acquireLock() was called in the parent before forking.
+        # _prepareFork() was called in the parent before forking.
         # The lock is reinitialized to unlocked state.
         _lock._at_fork_reinit()
 
-    os.register_at_fork(before=_acquireLock,
+    os.register_at_fork(before=_prepareFork,
                         after_in_child=_after_at_fork_child_reinit_locks,
-                        after_in_parent=_releaseLock)
+                        after_in_parent=_afterFork)
 
 
 #---------------------------------------------------------------------------
@@ -300,7 +300,7 @@ class LogRecord(object):
         """
         Initialize a logging record with interesting information.
         """
-        ct = time.time()
+        ct = time.time_ns()
         self.name = name
         self.msg = msg
         #
@@ -322,7 +322,7 @@ class LogRecord(object):
         # Thus, while not removing the isinstance check, it does now look
         # for collections.abc.Mapping rather than, as before, dict.
         if (args and len(args) == 1 and isinstance(args[0], collections.abc.Mapping)
-                and args[0]):
+            and args[0]):
             args = args[0]
         self.args = args
         self.levelname = getLevelName(level)
@@ -339,9 +339,17 @@ class LogRecord(object):
         self.stack_info = sinfo
         self.lineno = lineno
         self.funcName = func
-        self.created = ct
-        self.msecs = int((ct - int(ct)) * 1000) + 0.0  # see gh-89047
-        self.relativeCreated = (self.created - _startTime) * 1000
+        self.created = ct / 1e9  # ns to float seconds
+        # Get the number of whole milliseconds (0-999) in the fractional part of seconds.
+        # Eg: 1_677_903_920_999_998_503 ns --> 999_998_503 ns--> 999 ms
+        # Convert to float by adding 0.0 for historical reasons. See gh-89047
+        self.msecs = (ct % 1_000_000_000) // 1_000_000 + 0.0
+        if self.msecs == 999.0 and int(self.created) != ct // 1_000_000_000:
+            # ns -> sec conversion can round up, e.g:
+            # 1_677_903_920_999_999_900 ns --> 1_677_903_921.0 sec
+            self.msecs = 0.0
+
+        self.relativeCreated = (ct - _startTime) / 1e6
         if logThreads:
             self.thread = threading.get_ident()
             self.threadName = threading.current_thread().name
@@ -378,7 +386,7 @@ class LogRecord(object):
 
     def __repr__(self):
         return '<LogRecord: %s, %s, %s, %s, "%s">'%(self.name, self.levelno,
-                                                    self.pathname, self.lineno, self.msg)
+            self.pathname, self.lineno, self.msg)
 
     def getMessage(self):
         """
@@ -572,7 +580,7 @@ class Formatter(object):
     %(lineno)d          Source line number where the logging call was issued
                         (if available)
     %(funcName)s        Function name
-    %(created)f         Time when the LogRecord was created (time.time()
+    %(created)f         Time when the LogRecord was created (time.time_ns() / 1e9
                         return value)
     %(asctime)s         Textual time when the LogRecord was created
     %(msecs)d           Millisecond portion of the creation time
@@ -583,6 +591,7 @@ class Formatter(object):
     %(threadName)s      Thread name (if available)
     %(taskName)s        Task name (if available)
     %(process)d         Process ID (if available)
+    %(processName)s     Process name (if available)
     %(message)s         The result of record.getMessage(), computed just as
                         the record is emitted
     """
@@ -608,7 +617,7 @@ class Formatter(object):
         """
         if style not in _STYLES:
             raise ValueError('Style must be one of: %s' % ','.join(
-                _STYLES.keys()))
+                             _STYLES.keys()))
         self._style = _STYLES[style][0](fmt, defaults=defaults)
         if validate:
             self._style.validate()
@@ -658,7 +667,7 @@ class Formatter(object):
         # See issues #9427, #1553375. Commented out for now.
         #if getattr(self, 'fullstack', False):
         #    traceback.print_stack(tb.tb_frame.f_back, file=sio)
-        traceback.print_exception(ei[0], ei[1], tb, None, sio)
+        traceback.print_exception(ei[0], ei[1], tb, limit=None, file=sio)
         s = sio.getvalue()
         sio.close()
         if s[-1:] == "\n":
@@ -879,25 +888,20 @@ def _removeHandlerRef(wr):
     # set to None. It can also be called from another thread. So we need to
     # pre-emptively grab the necessary globals and check if they're None,
     # to prevent race conditions and failures during interpreter shutdown.
-    acquire, release, handlers = _acquireLock, _releaseLock, _handlerList
-    if acquire and release and handlers:
-        acquire()
-        try:
-            handlers.remove(wr)
-        except ValueError:
-            pass
-        finally:
-            release()
+    handlers, lock = _handlerList, _lock
+    if lock and handlers:
+        with lock:
+            try:
+                handlers.remove(wr)
+            except ValueError:
+                pass
 
 def _addHandlerRef(handler):
     """
     Add a handler to the internal cleanup list using a weak reference.
     """
-    _acquireLock()
-    try:
+    with _lock:
         _handlerList.append(weakref.ref(handler, _removeHandlerRef))
-    finally:
-        _releaseLock()
 
 
 def getHandlerByName(name):
@@ -912,8 +916,7 @@ def getHandlerNames():
     """
     Return all known handler names as an immutable set.
     """
-    result = set(_handlers.keys())
-    return frozenset(result)
+    return frozenset(_handlers)
 
 
 class Handler(Filterer):
@@ -943,15 +946,12 @@ class Handler(Filterer):
         return self._name
 
     def set_name(self, name):
-        _acquireLock()
-        try:
+        with _lock:
             if self._name in _handlers:
                 del _handlers[self._name]
             self._name = name
             if name:
                 _handlers[name] = self
-        finally:
-            _releaseLock()
 
     name = property(get_name, set_name)
 
@@ -1023,11 +1023,8 @@ class Handler(Filterer):
         if isinstance(rv, LogRecord):
             record = rv
         if rv:
-            self.acquire()
-            try:
+            with self.lock:
                 self.emit(record)
-            finally:
-                self.release()
         return rv
 
     def setFormatter(self, fmt):
@@ -1055,13 +1052,10 @@ class Handler(Filterer):
         methods.
         """
         #get the module data lock, as we're updating a shared structure.
-        _acquireLock()
-        try:    #unlikely to raise an exception, but you never know...
+        with _lock:
             self._closed = True
             if self._name and self._name in _handlers:
                 del _handlers[self._name]
-        finally:
-            _releaseLock()
 
     def handleError(self, record):
         """
@@ -1076,14 +1070,14 @@ class Handler(Filterer):
         The record which was being processed is passed in to this method.
         """
         if raiseExceptions and sys.stderr:  # see issue 13807
-            t, v, tb = sys.exc_info()
+            exc = sys.exception()
             try:
                 sys.stderr.write('--- Logging error ---\n')
-                traceback.print_exception(t, v, tb, None, sys.stderr)
+                traceback.print_exception(exc, limit=None, file=sys.stderr)
                 sys.stderr.write('Call stack:\n')
                 # Walk the stack frame up until we're out of logging,
                 # so as to print the calling context.
-                frame = tb.tb_frame
+                frame = exc.__traceback__.tb_frame
                 while (frame and os.path.dirname(frame.f_code.co_filename) ==
                        __path__[0]):
                     frame = frame.f_back
@@ -1092,7 +1086,7 @@ class Handler(Filterer):
                 else:
                     # couldn't find the right stack frame, for some reason
                     sys.stderr.write('Logged from file %s, line %s\n' % (
-                        record.filename, record.lineno))
+                                     record.filename, record.lineno))
                 # Issue 18671: output logging message and arguments
                 try:
                     sys.stderr.write('Message: %r\n'
@@ -1104,11 +1098,11 @@ class Handler(Filterer):
                     sys.stderr.write('Unable to print the message and arguments'
                                      ' - possible formatting error.\nUse the'
                                      ' traceback above to help find the error.\n'
-                                     )
+                                    )
             except OSError: #pragma: no cover
                 pass    # see issue 5971
             finally:
-                del t, v, tb
+                del exc
 
     def __repr__(self):
         level = getLevelName(self.level)
@@ -1138,12 +1132,9 @@ class StreamHandler(Handler):
         """
         Flushes the stream.
         """
-        self.acquire()
-        try:
+        with self.lock:
             if self.stream and hasattr(self.stream, "flush"):
                 self.stream.flush()
-        finally:
-            self.release()
 
     def emit(self, record):
         """
@@ -1179,12 +1170,9 @@ class StreamHandler(Handler):
             result = None
         else:
             result = self.stream
-            self.acquire()
-            try:
+            with self.lock:
                 self.flush()
                 self.stream = stream
-            finally:
-                self.release()
         return result
 
     def __repr__(self):
@@ -1234,8 +1222,7 @@ class FileHandler(StreamHandler):
         """
         Closes the stream.
         """
-        self.acquire()
-        try:
+        with self.lock:
             try:
                 if self.stream:
                     try:
@@ -1251,8 +1238,6 @@ class FileHandler(StreamHandler):
                 # Also see Issue #42378: we also rely on
                 # self._closed being set to True there
                 StreamHandler.close(self)
-        finally:
-            self.release()
 
     def _open(self):
         """
@@ -1388,8 +1373,7 @@ class Manager(object):
         rv = None
         if not isinstance(name, str):
             raise TypeError('A logger name must be a string')
-        _acquireLock()
-        try:
+        with _lock:
             if name in self.loggerDict:
                 rv = self.loggerDict[name]
                 if isinstance(rv, PlaceHolder):
@@ -1404,8 +1388,6 @@ class Manager(object):
                 rv.manager = self
                 self.loggerDict[name] = rv
                 self._fixupParents(rv)
-        finally:
-            _releaseLock()
         return rv
 
     def setLoggerClass(self, klass):
@@ -1468,12 +1450,11 @@ class Manager(object):
         Called when level changes are made
         """
 
-        _acquireLock()
-        for logger in self.loggerDict.values():
-            if isinstance(logger, Logger):
-                logger._cache.clear()
-        self.root._cache.clear()
-        _releaseLock()
+        with _lock:
+            for logger in self.loggerDict.values():
+                if isinstance(logger, Logger):
+                    logger._cache.clear()
+            self.root._cache.clear()
 
 #---------------------------------------------------------------------------
 #   Logger classes and functions
@@ -1494,6 +1475,8 @@ class Logger(Filterer):
     level, and "input.csv", "input.xls" and "input.gnu" for the sub-levels.
     There is no arbitrary limit to the depth of nesting.
     """
+    _tls = threading.local()
+
     def __init__(self, name, level=NOTSET):
         """
         Initialize the logger with a name and an optional level.
@@ -1552,7 +1535,7 @@ class Logger(Filterer):
 
     def warn(self, msg, *args, **kwargs):
         warnings.warn("The 'warn' method is deprecated, "
-                      "use 'warning' instead", DeprecationWarning, 2)
+            "use 'warning' instead", DeprecationWarning, 2)
         self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
@@ -1649,7 +1632,7 @@ class Logger(Filterer):
         specialized LogRecords.
         """
         rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func,
-                               sinfo)
+                             sinfo)
         if extra is not None:
             for key in extra:
                 if (key in ["message", "asctime"]) or (key in rv.__dict__):
@@ -1690,36 +1673,35 @@ class Logger(Filterer):
         This method is used for unpickled records received from a socket, as
         well as those created locally. Logger-level filtering is applied.
         """
-        if self.disabled:
+        if self._is_disabled():
             return
-        maybe_record = self.filter(record)
-        if not maybe_record:
-            return
-        if isinstance(maybe_record, LogRecord):
-            record = maybe_record
-        self.callHandlers(record)
+
+        self._tls.in_progress = True
+        try:
+            maybe_record = self.filter(record)
+            if not maybe_record:
+                return
+            if isinstance(maybe_record, LogRecord):
+                record = maybe_record
+            self.callHandlers(record)
+        finally:
+            self._tls.in_progress = False
 
     def addHandler(self, hdlr):
         """
         Add the specified handler to this logger.
         """
-        _acquireLock()
-        try:
+        with _lock:
             if not (hdlr in self.handlers):
                 self.handlers.append(hdlr)
-        finally:
-            _releaseLock()
 
     def removeHandler(self, hdlr):
         """
         Remove the specified handler from this logger.
         """
-        _acquireLock()
-        try:
+        with _lock:
             if hdlr in self.handlers:
                 self.handlers.remove(hdlr)
-        finally:
-            _releaseLock()
 
     def hasHandlers(self):
         """
@@ -1791,22 +1773,19 @@ class Logger(Filterer):
         """
         Is this logger enabled for level 'level'?
         """
-        if self.disabled:
+        if self._is_disabled():
             return False
 
         try:
             return self._cache[level]
         except KeyError:
-            _acquireLock()
-            try:
+            with _lock:
                 if self.manager.disable >= level:
                     is_enabled = self._cache[level] = False
                 else:
                     is_enabled = self._cache[level] = (
-                            level >= self.getEffectiveLevel()
+                        level >= self.getEffectiveLevel()
                     )
-            finally:
-                _releaseLock()
             return is_enabled
 
     def getChild(self, suffix):
@@ -1836,16 +1815,18 @@ class Logger(Filterer):
             return 1 + logger.name.count('.')
 
         d = self.manager.loggerDict
-        _acquireLock()
-        try:
+        with _lock:
             # exclude PlaceHolders - the last check is to ensure that lower-level
             # descendants aren't returned - if there are placeholders, a logger's
             # parent field might point to a grandparent or ancestor thereof.
             return set(item for item in d.values()
                        if isinstance(item, Logger) and item.parent is self and
                        _hierlevel(item) == 1 + _hierlevel(item.parent))
-        finally:
-            _releaseLock()
+
+    def _is_disabled(self):
+        # We need to use getattr as it will only be set the first time a log
+        # message is recorded on any given thread
+        return self.disabled or getattr(self._tls, 'in_progress', False)
 
     def __repr__(self):
         level = getLevelName(self.getEffectiveLevel())
@@ -1881,7 +1862,7 @@ class LoggerAdapter(object):
     information in logging output.
     """
 
-    def __init__(self, logger, extra=None):
+    def __init__(self, logger, extra=None, merge_extra=False):
         """
         Initialize the adapter with a logger and a dict-like object which
         provides contextual information. This constructor signature allows
@@ -1891,9 +1872,20 @@ class LoggerAdapter(object):
         following example:
 
         adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2"))
+
+        By default, LoggerAdapter objects will drop the "extra" argument
+        passed on the individual log calls to use its own instead.
+
+        Initializing it with merge_extra=True will instead merge both
+        maps when logging, the individual call extra taking precedence
+        over the LoggerAdapter instance extra
+
+        .. versionchanged:: 3.13
+           The *merge_extra* argument was added.
         """
         self.logger = logger
         self.extra = extra
+        self.merge_extra = merge_extra
 
     def process(self, msg, kwargs):
         """
@@ -1905,7 +1897,10 @@ class LoggerAdapter(object):
         Normally, you'll only need to override this one method in a
         LoggerAdapter subclass for your specific needs.
         """
-        kwargs["extra"] = self.extra
+        if self.merge_extra and "extra" in kwargs:
+            kwargs["extra"] = {**self.extra, **kwargs["extra"]}
+        else:
+            kwargs["extra"] = self.extra
         return msg, kwargs
 
     #
@@ -1931,7 +1926,7 @@ class LoggerAdapter(object):
 
     def warn(self, msg, *args, **kwargs):
         warnings.warn("The 'warn' method is deprecated, "
-                      "use 'warning' instead", DeprecationWarning, 2)
+            "use 'warning' instead", DeprecationWarning, 2)
         self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
@@ -2088,8 +2083,7 @@ def basicConfig(**kwargs):
     """
     # Add thread safety in case someone mistakenly calls
     # basicConfig() from multiple threads
-    _acquireLock()
-    try:
+    with _lock:
         force = kwargs.pop('force', False)
         encoding = kwargs.pop('encoding', None)
         errors = kwargs.pop('errors', 'backslashreplace')
@@ -2125,7 +2119,7 @@ def basicConfig(**kwargs):
             style = kwargs.pop("style", '%')
             if style not in _STYLES:
                 raise ValueError('Style must be one of: %s' % ','.join(
-                    _STYLES.keys()))
+                                 _STYLES.keys()))
             fs = kwargs.pop("format", _STYLES[style][1])
             fmt = Formatter(fs, dfs, style)
             for h in handlers:
@@ -2138,8 +2132,6 @@ def basicConfig(**kwargs):
             if kwargs:
                 keys = ', '.join(kwargs.keys())
                 raise ValueError('Unrecognised argument(s): %s' % keys)
-    finally:
-        _releaseLock()
 
 #---------------------------------------------------------------------------
 # Utility functions at module level.
@@ -2202,7 +2194,7 @@ def warning(msg, *args, **kwargs):
 
 def warn(msg, *args, **kwargs):
     warnings.warn("The 'warn' function is deprecated, "
-                  "use 'warning' instead", DeprecationWarning, 2)
+        "use 'warning' instead", DeprecationWarning, 2)
     warning(msg, *args, **kwargs)
 
 def info(msg, *args, **kwargs):

Lib/logging/config.py

@@ -83,15 +83,12 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=Non
     formatters = _create_formatters(cp)
 
     # critical section
-    logging._acquireLock()
-    try:
+    with logging._lock:
         _clearExistingHandlers()
 
         # Handlers add themselves to logging._handlers
         handlers = _install_handlers(cp, formatters)
         _install_loggers(cp, handlers, disable_existing_loggers)
-    finally:
-        logging._releaseLock()
 
 
 def _resolve(name):
@@ -314,7 +311,7 @@ class ConvertingMixin(object):
             if replace:
                 self[key] = result
             if type(result) in (ConvertingDict, ConvertingList,
-                                ConvertingTuple):
+                               ConvertingTuple):
                 result.parent = self
                 result.key = key
         return result
@@ -323,7 +320,7 @@ class ConvertingMixin(object):
         result = self.configurator.convert(value)
         if value is not result:
             if type(result) in (ConvertingDict, ConvertingList,
-                                ConvertingTuple):
+                               ConvertingTuple):
                 result.parent = self
         return result
 
@@ -378,7 +375,7 @@ class BaseConfigurator(object):
 
     WORD_PATTERN = re.compile(r'^\s*(\w+)\s*')
     DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*')
-    INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*')
+    INDEX_PATTERN = re.compile(r'^\[([^\[\]]*)\]\s*')
     DIGIT_PATTERN = re.compile(r'^\d+$')
 
     value_converters = {
@@ -464,8 +461,8 @@ class BaseConfigurator(object):
         elif not isinstance(value, ConvertingList) and isinstance(value, list):
             value = ConvertingList(value)
             value.configurator = self
-        elif not isinstance(value, ConvertingTuple) and \
-                isinstance(value, tuple) and not hasattr(value, '_fields'):
+        elif not isinstance(value, ConvertingTuple) and\
+                 isinstance(value, tuple) and not hasattr(value, '_fields'):
             value = ConvertingTuple(value)
             value.configurator = self
         elif isinstance(value, str): # str for py3k
@@ -543,8 +540,7 @@ class DictConfigurator(BaseConfigurator):
             raise ValueError("Unsupported version: %s" % config['version'])
         incremental = config.pop('incremental', False)
         EMPTY_DICT = {}
-        logging._acquireLock()
-        try:
+        with logging._lock:
             if incremental:
                 handlers = config.get('handlers', EMPTY_DICT)
                 for name in handlers:
@@ -585,7 +581,7 @@ class DictConfigurator(BaseConfigurator):
                 for name in formatters:
                     try:
                         formatters[name] = self.configure_formatter(
-                            formatters[name])
+                                                            formatters[name])
                     except Exception as e:
                         raise ValueError('Unable to configure '
                                          'formatter %r' % name) from e
@@ -688,8 +684,6 @@ class DictConfigurator(BaseConfigurator):
                     except Exception as e:
                         raise ValueError('Unable to configure root '
                                          'logger') from e
-        finally:
-            logging._releaseLock()
 
     def configure_formatter(self, config):
         """Configure a formatter from a dictionary."""
@@ -700,10 +694,9 @@ class DictConfigurator(BaseConfigurator):
             except TypeError as te:
                 if "'format'" not in str(te):
                     raise
-                #Name of parameter changed from fmt to format.
-                #Retry with old name.
-                #This is so that code can be used with older Python versions
-                #(e.g. by Django)
+                # logging.Formatter and its subclasses expect the `fmt`
+                # parameter instead of `format`. Retry passing configuration
+                # with `fmt`.
                 config['fmt'] = config.pop('format')
                 config['()'] = factory
                 result = self.configure_custom(config)
@@ -812,7 +805,7 @@ class DictConfigurator(BaseConfigurator):
             elif issubclass(klass, logging.handlers.QueueHandler):
                 # Another special case for handler which refers to other handlers
                 # if 'handlers' not in config:
-                # raise ValueError('No handlers specified for a QueueHandler')
+                    # raise ValueError('No handlers specified for a QueueHandler')
                 if 'queue' in config:
                     qspec = config['queue']
 
@@ -836,8 +829,8 @@ class DictConfigurator(BaseConfigurator):
                     else:
                         if isinstance(lspec, str):
                             listener = self.resolve(lspec)
-                            if isinstance(listener, type) and \
-                                    not issubclass(listener, logging.handlers.QueueListener):
+                            if isinstance(listener, type) and\
+                                not issubclass(listener, logging.handlers.QueueListener):
                                 raise TypeError('Invalid listener specifier %r' % lspec)
                         elif isinstance(lspec, dict):
                             if '()' not in lspec:
@@ -861,11 +854,11 @@ class DictConfigurator(BaseConfigurator):
                     except Exception as e:
                         raise ValueError('Unable to set required handler %r' % hn) from e
                     config['handlers'] = hlist
-            elif issubclass(klass, logging.handlers.SMTPHandler) and \
-                    'mailhost' in config:
+            elif issubclass(klass, logging.handlers.SMTPHandler) and\
+                'mailhost' in config:
                 config['mailhost'] = self.as_tuple(config['mailhost'])
-            elif issubclass(klass, logging.handlers.SysLogHandler) and \
-                    'address' in config:
+            elif issubclass(klass, logging.handlers.SysLogHandler) and\
+                'address' in config:
                 config['address'] = self.as_tuple(config['address'])
             if issubclass(klass, logging.handlers.QueueHandler):
                 factory = functools.partial(self._configure_queue_handler, klass)
@@ -1018,9 +1011,8 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None):
         def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
                      handler=None, ready=None, verify=None):
             ThreadingTCPServer.__init__(self, (host, port), handler)
-            logging._acquireLock()
-            self.abort = 0
-            logging._releaseLock()
+            with logging._lock:
+                self.abort = 0
             self.timeout = 1
             self.ready = ready
             self.verify = verify
@@ -1034,9 +1026,8 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None):
                                            self.timeout)
                 if rd:
                     self.handle_request()
-                logging._acquireLock()
-                abort = self.abort
-                logging._releaseLock()
+                with logging._lock:
+                    abort = self.abort
             self.server_close()
 
     class Server(threading.Thread):
@@ -1057,9 +1048,8 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None):
                 self.port = server.server_address[1]
             self.ready.set()
             global _listener
-            logging._acquireLock()
-            _listener = server
-            logging._releaseLock()
+            with logging._lock:
+                _listener = server
             server.serve_until_stopped()
 
     return Server(ConfigSocketReceiver, ConfigStreamHandler, port, verify)
@@ -1069,10 +1059,7 @@ def stopListening():
     Stop the listening server which was created with a call to listen().
     """
     global _listener
-    logging._acquireLock()
-    try:
+    with logging._lock:
         if _listener:
             _listener.abort = 1
             _listener = None
-    finally:
-        logging._releaseLock()

Lib/logging/handlers.py

@@ -23,11 +23,17 @@ Copyright (C) 2001-2021 Vinay Sajip. All Rights Reserved.
 To use, simply 'import logging.handlers' and log away!
 """
 
-import io, logging, socket, os, pickle, struct, time, re
-from stat import ST_DEV, ST_INO, ST_MTIME
-import queue
-import threading
 import copy
+import io
+import logging
+import os
+import pickle
+import queue
+import re
+import socket
+import struct
+import threading
+import time
 
 #
 # Some constants...
@@ -272,7 +278,7 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         # path object (see Issue #27493), but self.baseFilename will be a string
         filename = self.baseFilename
         if os.path.exists(filename):
-            t = os.stat(filename)[ST_MTIME]
+            t = int(os.stat(filename).st_mtime)
         else:
             t = int(time.time())
         self.rolloverAt = self.computeRollover(t)
@@ -304,10 +310,10 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                 rotate_ts = _MIDNIGHT
             else:
                 rotate_ts = ((self.atTime.hour * 60 + self.atTime.minute)*60 +
-                             self.atTime.second)
+                    self.atTime.second)
 
             r = rotate_ts - ((currentHour * 60 + currentMinute) * 60 +
-                             currentSecond)
+                currentSecond)
             if r <= 0:
                 # Rotate time is before the current time (for example when
                 # self.rotateAt is 13:45 and it now 14:15), rotation is
@@ -465,8 +471,7 @@ class WatchedFileHandler(logging.FileHandler):
     This handler is not appropriate for use under Windows, because
     under Windows open files cannot be moved or renamed - logging
     opens the files with exclusive locks - and so there is no need
-    for such a handler. Furthermore, ST_INO is not supported under
-    Windows; stat always returns zero for this value.
+    for such a handler.
 
     This handler is based on a suggestion and patch by Chad J.
     Schroeder.
@@ -482,9 +487,11 @@ class WatchedFileHandler(logging.FileHandler):
         self._statstream()
 
     def _statstream(self):
-        if self.stream:
-            sres = os.fstat(self.stream.fileno())
-            self.dev, self.ino = sres[ST_DEV], sres[ST_INO]
+        if self.stream is None:
+            return
+        sres = os.fstat(self.stream.fileno())
+        self.dev = sres.st_dev
+        self.ino = sres.st_ino
 
     def reopenIfNeeded(self):
         """
@@ -494,6 +501,9 @@ class WatchedFileHandler(logging.FileHandler):
         has, close the old stream and reopen the file to get the
         current stream.
         """
+        if self.stream is None:
+            return
+
         # Reduce the chance of race conditions by stat'ing by path only
         # once and then fstat'ing our new fd if we opened a new log stream.
         # See issue #14632: Thanks to John Mulligan for the problem report
@@ -501,18 +511,23 @@ class WatchedFileHandler(logging.FileHandler):
         try:
             # stat the file by path, checking for existence
             sres = os.stat(self.baseFilename)
+
+            # compare file system stat with that of our stream file handle
+            reopen = (sres.st_dev != self.dev or sres.st_ino != self.ino)
         except FileNotFoundError:
-            sres = None
-        # compare file system stat with that of our stream file handle
-        if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino:
-            if self.stream is not None:
-                # we have an open file handle, clean it up
-                self.stream.flush()
-                self.stream.close()
-                self.stream = None  # See Issue #21742: _open () might fail.
-                # open a new file handle and get new stat info from that fd
-                self.stream = self._open()
-                self._statstream()
+            reopen = True
+
+        if not reopen:
+            return
+
+        # we have an open file handle, clean it up
+        self.stream.flush()
+        self.stream.close()
+        self.stream = None  # See Issue #21742: _open () might fail.
+
+        # open a new file handle and get new stat info from that fd
+        self.stream = self._open()
+        self._statstream()
 
     def emit(self, record):
         """
@@ -682,15 +697,12 @@ class SocketHandler(logging.Handler):
         """
         Closes the socket.
         """
-        self.acquire()
-        try:
+        with self.lock:
             sock = self.sock
             if sock:
                 self.sock = None
                 sock.close()
             logging.Handler.close(self)
-        finally:
-            self.release()
 
 class DatagramHandler(SocketHandler):
     """
@@ -803,7 +815,7 @@ class SysLogHandler(logging.Handler):
         "panic":    LOG_EMERG,      #  DEPRECATED
         "warn":     LOG_WARNING,    #  DEPRECATED
         "warning":  LOG_WARNING,
-    }
+        }
 
     facility_names = {
         "auth":         LOG_AUTH,
@@ -830,7 +842,7 @@ class SysLogHandler(logging.Handler):
         "local5":       LOG_LOCAL5,
         "local6":       LOG_LOCAL6,
         "local7":       LOG_LOCAL7,
-    }
+        }
 
     # Originally added to work around GH-43683. Unnecessary since GH-50043 but kept
     # for backwards compatibility.
@@ -950,15 +962,12 @@ class SysLogHandler(logging.Handler):
         """
         Closes the socket.
         """
-        self.acquire()
-        try:
+        with self.lock:
             sock = self.socket
             if sock:
                 self.socket = None
                 sock.close()
             logging.Handler.close(self)
-        finally:
-            self.release()
 
     def mapPriority(self, levelName):
         """
@@ -1031,7 +1040,8 @@ class SMTPHandler(logging.Handler):
         only be used when authentication credentials are supplied. The tuple
         will be either an empty tuple, or a single-value tuple with the name
         of a keyfile, or a 2-value tuple with the names of the keyfile and
-        certificate file. (This tuple is passed to the `starttls` method).
+        certificate file. (This tuple is passed to the
+        `ssl.SSLContext.load_cert_chain` method).
         A timeout in seconds can be specified for the SMTP connection (the
         default is one second).
         """
@@ -1084,8 +1094,23 @@ class SMTPHandler(logging.Handler):
             msg.set_content(self.format(record))
             if self.username:
                 if self.secure is not None:
+                    import ssl
+
+                    try:
+                        keyfile = self.secure[0]
+                    except IndexError:
+                        keyfile = None
+
+                    try:
+                        certfile = self.secure[1]
+                    except IndexError:
+                        certfile = None
+
+                    context = ssl._create_stdlib_context(
+                        certfile=certfile, keyfile=keyfile
+                    )
                     smtp.ehlo()
-                    smtp.starttls(*self.secure)
+                    smtp.starttls(context=context)
                     smtp.ehlo()
                 smtp.login(self.username, self.password)
             smtp.send_message(msg)
@@ -1132,10 +1157,10 @@ class NTEventLogHandler(logging.Handler):
                 logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE,
                 logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
                 logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
-            }
+         }
         except ImportError:
-            print("The Python Win32 extensions for NT (service, event " \
-                  "logging) appear not to be available.")
+            print("The Python Win32 extensions for NT (service, event "\
+                        "logging) appear not to be available.")
             self._welu = None
 
     def getMessageID(self, record):
@@ -1330,11 +1355,8 @@ class BufferingHandler(logging.Handler):
 
         This version just zaps the buffer to empty.
         """
-        self.acquire()
-        try:
+        with self.lock:
             self.buffer.clear()
-        finally:
-            self.release()
 
     def close(self):
         """
@@ -1378,17 +1400,14 @@ class MemoryHandler(BufferingHandler):
         Check for buffer full or a record at the flushLevel or higher.
         """
         return (len(self.buffer) >= self.capacity) or \
-            (record.levelno >= self.flushLevel)
+                (record.levelno >= self.flushLevel)
 
     def setTarget(self, target):
         """
         Set the target handler for this handler.
         """
-        self.acquire()
-        try:
+        with self.lock:
             self.target = target
-        finally:
-            self.release()
 
     def flush(self):
         """
@@ -1398,14 +1417,11 @@ class MemoryHandler(BufferingHandler):
 
         The record buffer is only cleared if a target has been set.
         """
-        self.acquire()
-        try:
+        with self.lock:
             if self.target:
                 for record in self.buffer:
                     self.target.handle(record)
                 self.buffer.clear()
-        finally:
-            self.release()
 
     def close(self):
         """
@@ -1416,12 +1432,9 @@ class MemoryHandler(BufferingHandler):
             if self.flushOnClose:
                 self.flush()
         finally:
-            self.acquire()
-            try:
+            with self.lock:
                 self.target = None
                 BufferingHandler.close(self)
-            finally:
-                self.release()
 
 
 class QueueHandler(logging.Handler):
@@ -1532,6 +1545,9 @@ class QueueListener(object):
         This starts up a background thread to monitor the queue for
         LogRecords to process.
         """
+        if self._thread is not None:
+            raise RuntimeError("Listener already started")
+
         self._thread = t = threading.Thread(target=self._monitor)
         t.daemon = True
         t.start()
@@ -1603,6 +1619,7 @@ class QueueListener(object):
         Note that if you don't call this before your application exits, there
         may be some records still left on the queue, which won't be processed.
         """
-        self.enqueue_sentinel()
-        self._thread.join()
-        self._thread = None
+        if self._thread:  # see gh-114706 - allow calling this more than once
+            self.enqueue_sentinel()
+            self._thread.join()
+            self._thread = None

Lib/lzma.py

@@ -128,7 +128,7 @@ class LZMAFile(_compression.BaseStream):
 
         if self._mode == _MODE_READ:
             raw = _compression.DecompressReader(self._fp, LZMADecompressor,
-                                                trailing_error=LZMAError, format=format, filters=filters)
+                trailing_error=LZMAError, format=format, filters=filters)
             self._buffer = io.BufferedReader(raw)
 
     def close(self):

Lib/multiprocessing/connection.py

@@ -19,7 +19,6 @@ import time
 import tempfile
 import itertools
 
-import _multiprocessing
 
 from . import util
 
@@ -28,6 +27,7 @@ from .context import reduction
 _ForkingPickler = reduction.ForkingPickler
 
 try:
+    import _multiprocessing
     import _winapi
     from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE
 except ImportError:
@@ -846,7 +846,7 @@ _MD5_DIGEST_LEN = 16
 _LEGACY_LENGTHS = (_MD5ONLY_MESSAGE_LENGTH, _MD5_DIGEST_LEN)
 
 
-def _get_digest_name_and_payload(message: bytes) -> (str, bytes):
+def _get_digest_name_and_payload(message):  # type: (bytes) -> tuple[str, bytes]
     """Returns a digest name and the payload for a response hash.
 
     If a legacy protocol is detected based on the message length
@@ -956,7 +956,7 @@ def answer_challenge(connection, authkey: bytes):
                 f'Protocol error, expected challenge: {message=}')
     message = message[len(_CHALLENGE):]
     if len(message) < _MD5ONLY_MESSAGE_LENGTH:
-        raise AuthenticationError('challenge too short: {len(message)} bytes')
+        raise AuthenticationError(f'challenge too short: {len(message)} bytes')
     digest = _create_response(authkey, message)
     connection.send_bytes(digest)
     response = connection.recv_bytes(256)        # reject large message
@@ -1012,8 +1012,20 @@ if sys.platform == 'win32':
         # returning the first signalled might create starvation issues.)
         L = list(handles)
         ready = []
+        # Windows limits WaitForMultipleObjects at 64 handles, and we use a
+        # few for synchronisation, so we switch to batched waits at 60.
+        if len(L) > 60:
+            try:
+                res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout)
+            except TimeoutError:
+                return []
+            ready.extend(L[i] for i in res)
+            if res:
+                L = [h for i, h in enumerate(L) if i > res[0] & i not in res]
+            timeout = 0
         while L:
-            res = _winapi.WaitForMultipleObjects(L, False, timeout)
+            short_L = L[:60] if len(L) > 60 else L
+            res = _winapi.WaitForMultipleObjects(short_L, False, timeout)
             if res == WAIT_TIMEOUT:
                 break
             elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):

Lib/multiprocessing/context.py

@@ -145,7 +145,7 @@ class BaseContext(object):
         '''Check whether this is a fake forked process in a frozen executable.
         If so then run code specified by commandline and exit.
         '''
-        if sys.platform == 'win32' and getattr(sys, 'frozen', False):
+        if self.get_start_method() == 'spawn' and getattr(sys, 'frozen', False):
             from .spawn import freeze_support
             freeze_support()
 

Lib/multiprocessing/forkserver.py

@@ -1,3 +1,4 @@
+import atexit
 import errno
 import os
 import selectors
@@ -167,6 +168,8 @@ class ForkServer(object):
 def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
     '''Run forkserver.'''
     if preload:
+        if sys_path is not None:
+            sys.path[:] = sys_path
         if '__main__' in preload and main_path is not None:
             process.current_process()._inheriting = True
             try:
@@ -271,6 +274,8 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
                                 selector.close()
                                 unused_fds = [alive_r, child_w, sig_r, sig_w]
                                 unused_fds.extend(pid_to_fd.values())
+                                atexit._clear()
+                                atexit.register(util._exit_function)
                                 code = _serve_one(child_r, fds,
                                                   unused_fds,
                                                   old_handlers)
@@ -278,6 +283,7 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
                                 sys.excepthook(*sys.exc_info())
                                 sys.stderr.flush()
                             finally:
+                                atexit._run_exitfuncs()
                                 os._exit(code)
                         else:
                             # Send pid to client process

Lib/multiprocessing/managers.py

@@ -90,7 +90,10 @@ def dispatch(c, id, methodname, args=(), kwds={}):
     kind, result = c.recv()
     if kind == '#RETURN':
         return result
-    raise convert_to_error(kind, result)
+    try:
+        raise convert_to_error(kind, result)
+    finally:
+        del result  # break reference cycle
 
 def convert_to_error(kind, result):
     if kind == '#ERROR':
@@ -755,22 +758,29 @@ class BaseProxy(object):
     _address_to_local = {}
     _mutex = util.ForkAwareThreadLock()
 
+    # Each instance gets a `_serial` number. Unlike `id(...)`, this number
+    # is never reused.
+    _next_serial = 1
+
     def __init__(self, token, serializer, manager=None,
                  authkey=None, exposed=None, incref=True, manager_owned=False):
         with BaseProxy._mutex:
-            tls_idset = BaseProxy._address_to_local.get(token.address, None)
-            if tls_idset is None:
-                tls_idset = util.ForkAwareLocal(), ProcessLocalSet()
-                BaseProxy._address_to_local[token.address] = tls_idset
+            tls_serials = BaseProxy._address_to_local.get(token.address, None)
+            if tls_serials is None:
+                tls_serials = util.ForkAwareLocal(), ProcessLocalSet()
+                BaseProxy._address_to_local[token.address] = tls_serials
+
+            self._serial = BaseProxy._next_serial
+            BaseProxy._next_serial += 1
 
         # self._tls is used to record the connection used by this
         # thread to communicate with the manager at token.address
-        self._tls = tls_idset[0]
+        self._tls = tls_serials[0]
 
-        # self._idset is used to record the identities of all shared
-        # objects for which the current process owns references and
+        # self._all_serials is a set used to record the identities of all
+        # shared objects for which the current process owns references and
         # which are in the manager at token.address
-        self._idset = tls_idset[1]
+        self._all_serials = tls_serials[1]
 
         self._token = token
         self._id = self._token.id
@@ -833,7 +843,10 @@ class BaseProxy(object):
             conn = self._Client(token.address, authkey=self._authkey)
             dispatch(conn, None, 'decref', (token.id,))
             return proxy
-        raise convert_to_error(kind, result)
+        try:
+            raise convert_to_error(kind, result)
+        finally:
+            del result   # break reference cycle
 
     def _getvalue(self):
         '''
@@ -850,20 +863,20 @@ class BaseProxy(object):
         dispatch(conn, None, 'incref', (self._id,))
         util.debug('INCREF %r', self._token.id)
 
-        self._idset.add(self._id)
+        self._all_serials.add(self._serial)
 
         state = self._manager and self._manager._state
 
         self._close = util.Finalize(
             self, BaseProxy._decref,
-            args=(self._token, self._authkey, state,
-                  self._tls, self._idset, self._Client),
+            args=(self._token, self._serial, self._authkey, state,
+                  self._tls, self._all_serials, self._Client),
             exitpriority=10
             )
 
     @staticmethod
-    def _decref(token, authkey, state, tls, idset, _Client):
-        idset.discard(token.id)
+    def _decref(token, serial, authkey, state, tls, idset, _Client):
+        idset.discard(serial)
 
         # check whether manager is still alive
         if state is None or state.value == State.STARTED:
@@ -1159,15 +1172,19 @@ class ListProxy(BaseListProxy):
         self._callmethod('__imul__', (value,))
         return self
 
+    __class_getitem__ = classmethod(types.GenericAlias)
 
-DictProxy = MakeProxyType('DictProxy', (
+
+_BaseDictProxy = MakeProxyType('DictProxy', (
     '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__',
     '__setitem__', 'clear', 'copy', 'get', 'items',
     'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'
     ))
-DictProxy._method_to_typeid_ = {
+_BaseDictProxy._method_to_typeid_ = {
     '__iter__': 'Iterator',
     }
+class DictProxy(_BaseDictProxy):
+    __class_getitem__ = classmethod(types.GenericAlias)
 
 
 ArrayProxy = MakeProxyType('ArrayProxy', (

Lib/multiprocessing/pool.py

@@ -200,7 +200,7 @@ class Pool(object):
         self._initargs = initargs
 
         if processes is None:
-            processes = os.cpu_count() or 1
+            processes = os.process_cpu_count() or 1
         if processes < 1:
             raise ValueError("Number of processes must be at least 1")
         if maxtasksperchild is not None:

Lib/multiprocessing/popen_fork.py

@@ -1,3 +1,4 @@
+import atexit
 import os
 import signal
 
@@ -66,10 +67,13 @@ class Popen(object):
         self.pid = os.fork()
         if self.pid == 0:
             try:
+                atexit._clear()
+                atexit.register(util._exit_function)
                 os.close(parent_r)
                 os.close(parent_w)
                 code = process_obj._bootstrap(parent_sentinel=child_r)
             finally:
+                atexit._run_exitfuncs()
                 os._exit(code)
         else:
             os.close(child_w)

Lib/multiprocessing/popen_spawn_win32.py

@@ -3,6 +3,7 @@ import msvcrt
 import signal
 import sys
 import _winapi
+from subprocess import STARTUPINFO, STARTF_FORCEOFFFEEDBACK
 
 from .context import reduction, get_spawning_popen, set_spawning_popen
 from . import spawn
@@ -74,7 +75,8 @@ class Popen(object):
             try:
                 hp, ht, pid, tid = _winapi.CreateProcess(
                     python_exe, cmd,
-                    None, None, False, 0, env, None, None)
+                    None, None, False, 0, env, None,
+                    STARTUPINFO(dwFlags=STARTF_FORCEOFFFEEDBACK))
                 _winapi.CloseHandle(ht)
             except:
                 _winapi.CloseHandle(rhandle)

Lib/multiprocessing/process.py

@@ -310,11 +310,8 @@ class BaseProcess(object):
                 # _run_after_forkers() is executed
                 del old_process
             util.info('child process calling self.run()')
-            try:
-                self.run()
-                exitcode = 0
-            finally:
-                util._exit_function()
+            self.run()
+            exitcode = 0
         except SystemExit as e:
             if e.code is None:
                 exitcode = 0

Lib/multiprocessing/queues.py

@@ -20,8 +20,6 @@ import errno
 
 from queue import Empty, Full
 
-import _multiprocessing
-
 from . import connection
 from . import context
 _ForkingPickler = context.reduction.ForkingPickler

Lib/multiprocessing/resource_tracker.py

@@ -29,8 +29,12 @@ __all__ = ['ensure_running', 'register', 'unregister']
 _HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask')
 _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM)
 
+def cleanup_noop(name):
+    raise RuntimeError('noop should never be registered or cleaned up')
+
 _CLEANUP_FUNCS = {
-    'noop': lambda: None,
+    'noop': cleanup_noop,
+    'dummy': lambda name: None,  # Dummy resource used in tests
 }
 
 if os.name == 'posix':
@@ -61,6 +65,7 @@ class ResourceTracker(object):
         self._lock = threading.RLock()
         self._fd = None
         self._pid = None
+        self._exitcode = None
 
     def _reentrant_call_error(self):
         # gh-109629: this happens if an explicit call to the ResourceTracker
@@ -70,22 +75,53 @@ class ResourceTracker(object):
         raise ReentrantCallError(
             "Reentrant call into the multiprocessing resource tracker")
 
-    def _stop(self):
-        with self._lock:
-            # This should not happen (_stop() isn't called by a finalizer)
-            # but we check for it anyway.
-            if self._lock._recursion_count() > 1:
-                return self._reentrant_call_error()
-            if self._fd is None:
-                # not running
-                return
+    def __del__(self):
+        # making sure child processess are cleaned before ResourceTracker
+        # gets destructed.
+        # see https://github.com/python/cpython/issues/88887
+        self._stop(use_blocking_lock=False)
 
-            # closing the "alive" file descriptor stops main()
-            os.close(self._fd)
-            self._fd = None
+    def _stop(self, use_blocking_lock=True):
+        if use_blocking_lock:
+            with self._lock:
+                self._stop_locked()
+        else:
+            acquired = self._lock.acquire(blocking=False)
+            try:
+                self._stop_locked()
+            finally:
+                if acquired:
+                    self._lock.release()
 
-            os.waitpid(self._pid, 0)
-            self._pid = None
+    def _stop_locked(
+        self,
+        close=os.close,
+        waitpid=os.waitpid,
+        waitstatus_to_exitcode=os.waitstatus_to_exitcode,
+    ):
+        # This shouldn't happen (it might when called by a finalizer)
+        # so we check for it anyway.
+        if self._lock._recursion_count() > 1:
+            return self._reentrant_call_error()
+        if self._fd is None:
+            # not running
+            return
+        if self._pid is None:
+            return
+
+        # closing the "alive" file descriptor stops main()
+        close(self._fd)
+        self._fd = None
+
+        _, status = waitpid(self._pid, 0)
+
+        self._pid = None
+
+        try:
+            self._exitcode = waitstatus_to_exitcode(status)
+        except ValueError:
+            # os.waitstatus_to_exitcode may raise an exception for invalid values
+            self._exitcode = None
 
     def getfd(self):
         self.ensure_running()
@@ -119,6 +155,7 @@ class ResourceTracker(object):
                     pass
                 self._fd = None
                 self._pid = None
+                self._exitcode = None
 
                 warnings.warn('resource_tracker: process died unexpectedly, '
                               'relaunching.  Some resources might leak.')
@@ -142,13 +179,14 @@ class ResourceTracker(object):
                 # that can make the child die before it registers signal handlers
                 # for SIGINT and SIGTERM. The mask is unregistered after spawning
                 # the child.
+                prev_sigmask = None
                 try:
                     if _HAVE_SIGMASK:
-                        signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS)
+                        prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS)
                     pid = util.spawnv_passfds(exe, args, fds_to_pass)
                 finally:
-                    if _HAVE_SIGMASK:
-                        signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS)
+                    if prev_sigmask is not None:
+                        signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask)
             except:
                 os.close(w)
                 raise
@@ -221,6 +259,8 @@ def main(fd):
             pass
 
     cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()}
+    exit_code = 0
+
     try:
         # keep track of registered/unregistered resources
         with open(fd, 'rb') as f:
@@ -242,6 +282,7 @@ def main(fd):
                     else:
                         raise RuntimeError('unrecognized command %r' % cmd)
                 except Exception:
+                    exit_code = 3
                     try:
                         sys.excepthook(*sys.exc_info())
                     except:
@@ -251,9 +292,17 @@ def main(fd):
         for rtype, rtype_cache in cache.items():
             if rtype_cache:
                 try:
-                    warnings.warn('resource_tracker: There appear to be %d '
-                                  'leaked %s objects to clean up at shutdown' %
-                                  (len(rtype_cache), rtype))
+                    exit_code = 1
+                    if rtype == 'dummy':
+                        # The test 'dummy' resource is expected to leak.
+                        # We skip the warning (and *only* the warning) for it.
+                        pass
+                    else:
+                        warnings.warn(
+                            f'resource_tracker: There appear to be '
+                            f'{len(rtype_cache)} leaked {rtype} objects to '
+                            f'clean up at shutdown: {rtype_cache}'
+                        )
                 except Exception:
                     pass
             for name in rtype_cache:
@@ -264,6 +313,9 @@ def main(fd):
                     try:
                         _CLEANUP_FUNCS[rtype](name)
                     except Exception as e:
+                        exit_code = 2
                         warnings.warn('resource_tracker: %r: %s' % (name, e))
                 finally:
                     pass
+
+        sys.exit(exit_code)

Lib/multiprocessing/shared_memory.py

@@ -71,8 +71,9 @@ class SharedMemory:
     _flags = os.O_RDWR
     _mode = 0o600
     _prepend_leading_slash = True if _USE_POSIX else False
+    _track = True
 
-    def __init__(self, name=None, create=False, size=0):
+    def __init__(self, name=None, create=False, size=0, *, track=True):
         if not size >= 0:
             raise ValueError("'size' must be a positive integer")
         if create:
@@ -82,6 +83,7 @@ class SharedMemory:
         if name is None and not self._flags & os.O_EXCL:
             raise ValueError("'name' can only be None if create=True")
 
+        self._track = track
         if _USE_POSIX:
 
             # POSIX Shared Memory
@@ -116,8 +118,8 @@ class SharedMemory:
             except OSError:
                 self.unlink()
                 raise
-
-            resource_tracker.register(self._name, "shared_memory")
+            if self._track:
+                resource_tracker.register(self._name, "shared_memory")
 
         else:
 
@@ -236,12 +238,20 @@ class SharedMemory:
     def unlink(self):
         """Requests that the underlying shared memory block be destroyed.
 
-        In order to ensure proper cleanup of resources, unlink should be
-        called once (and only once) across all processes which have access
-        to the shared memory block."""
+        Unlink should be called once (and only once) across all handles
+        which have access to the shared memory block, even if these
+        handles belong to different processes. Closing and unlinking may
+        happen in any order, but trying to access data inside a shared
+        memory block after unlinking may result in memory errors,
+        depending on platform.
+
+        This method has no effect on Windows, where the only way to
+        delete a shared memory block is to close all handles."""
+
         if _USE_POSIX and self._name:
             _posixshmem.shm_unlink(self._name)
-            resource_tracker.unregister(self._name, "shared_memory")
+            if self._track:
+                resource_tracker.unregister(self._name, "shared_memory")
 
 
 _encoding = "utf8"

Lib/multiprocessing/synchronize.py

@@ -174,7 +174,7 @@ class Lock(SemLock):
                 name = process.current_process().name
                 if threading.current_thread().name != 'MainThread':
                     name += '|' + threading.current_thread().name
-            elif self._semlock._get_value() == 1:
+            elif not self._semlock._is_zero():
                 name = 'None'
             elif self._semlock._count() > 0:
                 name = 'SomeOtherThread'
@@ -200,7 +200,7 @@ class RLock(SemLock):
                 if threading.current_thread().name != 'MainThread':
                     name += '|' + threading.current_thread().name
                 count = self._semlock._count()
-            elif self._semlock._get_value() == 1:
+            elif not self._semlock._is_zero():
                 name, count = 'None', 0
             elif self._semlock._count() > 0:
                 name, count = 'SomeOtherThread', 'nonzero'
@@ -360,7 +360,7 @@ class Event(object):
                 return True
             return False
 
-    def __repr__(self) -> str:
+    def __repr__(self):
         set_status = 'set' if self.is_set() else 'unset'
         return f"<{type(self).__qualname__} at {id(self):#x} {set_status}>"
 #

Lib/multiprocessing/util.py

@@ -64,8 +64,7 @@ def get_logger():
     global _logger
     import logging
 
-    logging._acquireLock()
-    try:
+    with logging._lock:
         if not _logger:
 
             _logger = logging.getLogger(LOGGER_NAME)
@@ -79,9 +78,6 @@ def get_logger():
                 atexit._exithandlers.remove((_exit_function, (), {}))
                 atexit._exithandlers.append((_exit_function, (), {}))
 
-    finally:
-        logging._releaseLock()
-
     return _logger
 
 def log_to_stderr(level=None):
@@ -106,11 +102,7 @@ def log_to_stderr(level=None):
 # Abstract socket support
 
 def _platform_supports_abstract_sockets():
-    if sys.platform == "linux":
-        return True
-    if hasattr(sys, 'getandroidapilevel'):
-        return True
-    return False
+    return sys.platform in ("linux", "android")
 
 
 def is_abstract_socket_namespace(address):
@@ -130,10 +122,7 @@ abstract_sockets_supported = _platform_supports_abstract_sockets()
 #
 
 def _remove_temp_dir(rmtree, tempdir):
-    def onerror(func, path, err_info):
-        if not issubclass(err_info[0], FileNotFoundError):
-            raise
-    rmtree(tempdir, onerror=onerror)
+    rmtree(tempdir)
 
     current_process = process.current_process()
     # current_process() can be None if the finalizer is called

Lib/operator.py

@@ -239,7 +239,7 @@ class attrgetter:
     """
     __slots__ = ('_attrs', '_call')
 
-    def __init__(self, attr, *attrs):
+    def __init__(self, attr, /, *attrs):
         if not attrs:
             if not isinstance(attr, str):
                 raise TypeError('attribute name must be a string')
@@ -257,7 +257,7 @@ class attrgetter:
                 return tuple(getter(obj) for getter in getters)
             self._call = func
 
-    def __call__(self, obj):
+    def __call__(self, obj, /):
         return self._call(obj)
 
     def __repr__(self):
@@ -276,7 +276,7 @@ class itemgetter:
     """
     __slots__ = ('_items', '_call')
 
-    def __init__(self, item, *items):
+    def __init__(self, item, /, *items):
         if not items:
             self._items = (item,)
             def func(obj):
@@ -288,7 +288,7 @@ class itemgetter:
                 return tuple(obj[i] for i in items)
             self._call = func
 
-    def __call__(self, obj):
+    def __call__(self, obj, /):
         return self._call(obj)
 
     def __repr__(self):
@@ -315,7 +315,7 @@ class methodcaller:
         self._args = args
         self._kwargs = kwargs
 
-    def __call__(self, obj):
+    def __call__(self, obj, /):
         return getattr(obj, self._name)(*self._args, **self._kwargs)
 
     def __repr__(self):

Lib/pyclbr.py

@@ -0,0 +1,314 @@
+"""Parse a Python module and describe its classes and functions.
+
+Parse enough of a Python file to recognize imports and class and
+function definitions, and to find out the superclasses of a class.
+
+The interface consists of a single function:
+    readmodule_ex(module, path=None)
+where module is the name of a Python module, and path is an optional
+list of directories where the module is to be searched.  If present,
+path is prepended to the system search path sys.path.  The return value
+is a dictionary.  The keys of the dictionary are the names of the
+classes and functions defined in the module (including classes that are
+defined via the from XXX import YYY construct).  The values are
+instances of classes Class and Function.  One special key/value pair is
+present for packages: the key '__path__' has a list as its value which
+contains the package search path.
+
+Classes and Functions have a common superclass: _Object.  Every instance
+has the following attributes:
+    module  -- name of the module;
+    name    -- name of the object;
+    file    -- file in which the object is defined;
+    lineno  -- line in the file where the object's definition starts;
+    end_lineno -- line in the file where the object's definition ends;
+    parent  -- parent of this object, if any;
+    children -- nested objects contained in this object.
+The 'children' attribute is a dictionary mapping names to objects.
+
+Instances of Function describe functions with the attributes from _Object,
+plus the following:
+    is_async -- if a function is defined with an 'async' prefix
+
+Instances of Class describe classes with the attributes from _Object,
+plus the following:
+    super   -- list of super classes (Class instances if possible);
+    methods -- mapping of method names to beginning line numbers.
+If the name of a super class is not recognized, the corresponding
+entry in the list of super classes is not a class instance but a
+string giving the name of the super class.  Since import statements
+are recognized and imported modules are scanned as well, this
+shouldn't happen often.
+"""
+
+import ast
+import sys
+import importlib.util
+
+__all__ = ["readmodule", "readmodule_ex", "Class", "Function"]
+
+_modules = {}  # Initialize cache of modules we've seen.
+
+
+class _Object:
+    "Information about Python class or function."
+    def __init__(self, module, name, file, lineno, end_lineno, parent):
+        self.module = module
+        self.name = name
+        self.file = file
+        self.lineno = lineno
+        self.end_lineno = end_lineno
+        self.parent = parent
+        self.children = {}
+        if parent is not None:
+            parent.children[name] = self
+
+
+# Odd Function and Class signatures are for back-compatibility.
+class Function(_Object):
+    "Information about a Python function, including methods."
+    def __init__(self, module, name, file, lineno,
+                 parent=None, is_async=False, *, end_lineno=None):
+        super().__init__(module, name, file, lineno, end_lineno, parent)
+        self.is_async = is_async
+        if isinstance(parent, Class):
+            parent.methods[name] = lineno
+
+
+class Class(_Object):
+    "Information about a Python class."
+    def __init__(self, module, name, super_, file, lineno,
+                 parent=None, *, end_lineno=None):
+        super().__init__(module, name, file, lineno, end_lineno, parent)
+        self.super = super_ or []
+        self.methods = {}
+
+
+# These 2 functions are used in these tests
+# Lib/test/test_pyclbr, Lib/idlelib/idle_test/test_browser.py
+def _nest_function(ob, func_name, lineno, end_lineno, is_async=False):
+    "Return a Function after nesting within ob."
+    return Function(ob.module, func_name, ob.file, lineno,
+                    parent=ob, is_async=is_async, end_lineno=end_lineno)
+
+def _nest_class(ob, class_name, lineno, end_lineno, super=None):
+    "Return a Class after nesting within ob."
+    return Class(ob.module, class_name, super, ob.file, lineno,
+                 parent=ob, end_lineno=end_lineno)
+
+
+def readmodule(module, path=None):
+    """Return Class objects for the top-level classes in module.
+
+    This is the original interface, before Functions were added.
+    """
+
+    res = {}
+    for key, value in _readmodule(module, path or []).items():
+        if isinstance(value, Class):
+            res[key] = value
+    return res
+
+def readmodule_ex(module, path=None):
+    """Return a dictionary with all functions and classes in module.
+
+    Search for module in PATH + sys.path.
+    If possible, include imported superclasses.
+    Do this by reading source, without importing (and executing) it.
+    """
+    return _readmodule(module, path or [])
+
+
+def _readmodule(module, path, inpackage=None):
+    """Do the hard work for readmodule[_ex].
+
+    If inpackage is given, it must be the dotted name of the package in
+    which we are searching for a submodule, and then PATH must be the
+    package search path; otherwise, we are searching for a top-level
+    module, and path is combined with sys.path.
+    """
+    # Compute the full module name (prepending inpackage if set).
+    if inpackage is not None:
+        fullmodule = "%s.%s" % (inpackage, module)
+    else:
+        fullmodule = module
+
+    # Check in the cache.
+    if fullmodule in _modules:
+        return _modules[fullmodule]
+
+    # Initialize the dict for this module's contents.
+    tree = {}
+
+    # Check if it is a built-in module; we don't do much for these.
+    if module in sys.builtin_module_names and inpackage is None:
+        _modules[module] = tree
+        return tree
+
+    # Check for a dotted module name.
+    i = module.rfind('.')
+    if i >= 0:
+        package = module[:i]
+        submodule = module[i+1:]
+        parent = _readmodule(package, path, inpackage)
+        if inpackage is not None:
+            package = "%s.%s" % (inpackage, package)
+        if not '__path__' in parent:
+            raise ImportError('No package named {}'.format(package))
+        return _readmodule(submodule, parent['__path__'], package)
+
+    # Search the path for the module.
+    f = None
+    if inpackage is not None:
+        search_path = path
+    else:
+        search_path = path + sys.path
+    spec = importlib.util._find_spec_from_path(fullmodule, search_path)
+    if spec is None:
+        raise ModuleNotFoundError(f"no module named {fullmodule!r}", name=fullmodule)
+    _modules[fullmodule] = tree
+    # Is module a package?
+    if spec.submodule_search_locations is not None:
+        tree['__path__'] = spec.submodule_search_locations
+    try:
+        source = spec.loader.get_source(fullmodule)
+    except (AttributeError, ImportError):
+        # If module is not Python source, we cannot do anything.
+        return tree
+    else:
+        if source is None:
+            return tree
+
+    fname = spec.loader.get_filename(fullmodule)
+    return _create_tree(fullmodule, path, fname, source, tree, inpackage)
+
+
+class _ModuleBrowser(ast.NodeVisitor):
+    def __init__(self, module, path, file, tree, inpackage):
+        self.path = path
+        self.tree = tree
+        self.file = file
+        self.module = module
+        self.inpackage = inpackage
+        self.stack = []
+
+    def visit_ClassDef(self, node):
+        bases = []
+        for base in node.bases:
+            name = ast.unparse(base)
+            if name in self.tree:
+                # We know this super class.
+                bases.append(self.tree[name])
+            elif len(names := name.split(".")) > 1:
+                # Super class form is module.class:
+                # look in module for class.
+                *_, module, class_ = names
+                if module in _modules:
+                    bases.append(_modules[module].get(class_, name))
+            else:
+                bases.append(name)
+
+        parent = self.stack[-1] if self.stack else None
+        class_ = Class(self.module, node.name, bases, self.file, node.lineno,
+                       parent=parent, end_lineno=node.end_lineno)
+        if parent is None:
+            self.tree[node.name] = class_
+        self.stack.append(class_)
+        self.generic_visit(node)
+        self.stack.pop()
+
+    def visit_FunctionDef(self, node, *, is_async=False):
+        parent = self.stack[-1] if self.stack else None
+        function = Function(self.module, node.name, self.file, node.lineno,
+                            parent, is_async, end_lineno=node.end_lineno)
+        if parent is None:
+            self.tree[node.name] = function
+        self.stack.append(function)
+        self.generic_visit(node)
+        self.stack.pop()
+
+    def visit_AsyncFunctionDef(self, node):
+        self.visit_FunctionDef(node, is_async=True)
+
+    def visit_Import(self, node):
+        if node.col_offset != 0:
+            return
+
+        for module in node.names:
+            try:
+                try:
+                    _readmodule(module.name, self.path, self.inpackage)
+                except ImportError:
+                    _readmodule(module.name, [])
+            except (ImportError, SyntaxError):
+                # If we can't find or parse the imported module,
+                # too bad -- don't die here.
+                continue
+
+    def visit_ImportFrom(self, node):
+        if node.col_offset != 0:
+            return
+        try:
+            module = "." * node.level
+            if node.module:
+                module += node.module
+            module = _readmodule(module, self.path, self.inpackage)
+        except (ImportError, SyntaxError):
+            return
+
+        for name in node.names:
+            if name.name in module:
+                self.tree[name.asname or name.name] = module[name.name]
+            elif name.name == "*":
+                for import_name, import_value in module.items():
+                    if import_name.startswith("_"):
+                        continue
+                    self.tree[import_name] = import_value
+
+
+def _create_tree(fullmodule, path, fname, source, tree, inpackage):
+    mbrowser = _ModuleBrowser(fullmodule, path, fname, tree, inpackage)
+    mbrowser.visit(ast.parse(source))
+    return mbrowser.tree
+
+
+def _main():
+    "Print module output (default this file) for quick visual check."
+    import os
+    try:
+        mod = sys.argv[1]
+    except:
+        mod = __file__
+    if os.path.exists(mod):
+        path = [os.path.dirname(mod)]
+        mod = os.path.basename(mod)
+        if mod.lower().endswith(".py"):
+            mod = mod[:-3]
+    else:
+        path = []
+    tree = readmodule_ex(mod, path)
+    lineno_key = lambda a: getattr(a, 'lineno', 0)
+    objs = sorted(tree.values(), key=lineno_key, reverse=True)
+    indent_level = 2
+    while objs:
+        obj = objs.pop()
+        if isinstance(obj, list):
+            # Value is a __path__ key.
+            continue
+        if not hasattr(obj, 'indent'):
+            obj.indent = 0
+
+        if isinstance(obj, _Object):
+            new_objs = sorted(obj.children.values(),
+                              key=lineno_key, reverse=True)
+            for ob in new_objs:
+                ob.indent = obj.indent + indent_level
+            objs.extend(new_objs)
+        if isinstance(obj, Class):
+            print("{}class {} {} {}"
+                  .format(' ' * obj.indent, obj.name, obj.super, obj.lineno))
+        elif isinstance(obj, Function):
+            print("{}def {} {}".format(' ' * obj.indent, obj.name, obj.lineno))
+
+if __name__ == "__main__":
+    _main()

Lib/quopri.py

@@ -67,10 +67,7 @@ def encode(input, output, quotetabs, header=False):
             output.write(s + lineEnd)
 
     prevline = None
-    while 1:
-        line = input.readline()
-        if not line:
-            break
+    while line := input.readline():
         outline = []
         # Strip off any readline induced trailing newline
         stripped = b''
@@ -126,9 +123,7 @@ def decode(input, output, header=False):
         return
 
     new = b''
-    while 1:
-        line = input.readline()
-        if not line: break
+    while line := input.readline():
         i, n = 0, len(line)
         if n > 0 and line[n-1:n] == b'\n':
             partial = 0; n = n-1

Lib/rlcompleter.py

@@ -31,7 +31,11 @@ Notes:
 
 import atexit
 import builtins
+import inspect
+import keyword
+import re
 import __main__
+import warnings
 
 __all__ = ["Completer"]
 
@@ -85,10 +89,11 @@ class Completer:
                 return None
 
         if state == 0:
-            if "." in text:
-                self.matches = self.attr_matches(text)
-            else:
-                self.matches = self.global_matches(text)
+            with warnings.catch_warnings(action="ignore"):
+                if "." in text:
+                    self.matches = self.attr_matches(text)
+                else:
+                    self.matches = self.global_matches(text)
         try:
             return self.matches[state]
         except IndexError:
@@ -96,7 +101,13 @@ class Completer:
 
     def _callable_postfix(self, val, word):
         if callable(val):
-            word = word + "("
+            word += "("
+            try:
+                if not inspect.signature(val).parameters:
+                    word += ")"
+            except ValueError:
+                pass
+
         return word
 
     def global_matches(self, text):
@@ -106,18 +117,17 @@ class Completer:
         defined in self.namespace that match.
 
         """
-        import keyword
         matches = []
         seen = {"__builtins__"}
         n = len(text)
-        for word in keyword.kwlist:
+        for word in keyword.kwlist + keyword.softkwlist:
             if word[:n] == text:
                 seen.add(word)
                 if word in {'finally', 'try'}:
                     word = word + ':'
                 elif word not in {'False', 'None', 'True',
                                   'break', 'continue', 'pass',
-                                  'else'}:
+                                  'else', '_'}:
                     word = word + ' '
                 matches.append(word)
         for nspace in [self.namespace, builtins.__dict__]:
@@ -139,7 +149,6 @@ class Completer:
         with a __getattr__ hook is evaluated.
 
         """
-        import re
         m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text)
         if not m:
             return []
@@ -169,13 +178,20 @@ class Completer:
                 if (word[:n] == attr and
                     not (noprefix and word[:n+1] == noprefix)):
                     match = "%s.%s" % (expr, word)
-                    try:
-                        val = getattr(thisobject, word)
-                    except Exception:
-                        pass  # Include even if attribute not set
+                    if isinstance(getattr(type(thisobject), word, None),
+                                  property):
+                        # bpo-44752: thisobject.word is a method decorated by
+                        # `@property`. What follows applies a postfix if
+                        # thisobject.word is callable, but know we know that
+                        # this is not callable (because it is a property).
+                        # Also, getattr(thisobject, word) will evaluate the
+                        # property method, which is not desirable.
+                        matches.append(match)
+                        continue
+                    if (value := getattr(thisobject, word, None)) is not None:
+                        matches.append(self._callable_postfix(value, match))
                     else:
-                        match = self._callable_postfix(val, match)
-                    matches.append(match)
+                        matches.append(match)
             if matches or not noprefix:
                 break
             if noprefix == '_':

Lib/secrets.py

@@ -2,7 +2,7 @@
 managing secrets such as account authentication, tokens, and similar.
 
 See PEP 506 for more information.
-https://www.python.org/dev/peps/pep-0506/
+https://peps.python.org/pep-0506/
 
 """
 
@@ -13,7 +13,6 @@ __all__ = ['choice', 'randbelow', 'randbits', 'SystemRandom',
 
 
 import base64
-import binascii
 
 from hmac import compare_digest
 from random import SystemRandom
@@ -56,7 +55,7 @@ def token_hex(nbytes=None):
     'f9bf78b9a18ce6d46a0cd2b0b86df9da'
 
     """
-    return binascii.hexlify(token_bytes(nbytes)).decode('ascii')
+    return token_bytes(nbytes).hex()
 
 def token_urlsafe(nbytes=None):
     """Return a random URL-safe text string, in Base64 encoding.

Lib/selectors.py

@@ -66,12 +66,16 @@ class _SelectorMapping(Mapping):
     def __len__(self):
         return len(self._selector._fd_to_key)
 
+    def get(self, fileobj, default=None):
+        fd = self._selector._fileobj_lookup(fileobj)
+        return self._selector._fd_to_key.get(fd, default)
+
     def __getitem__(self, fileobj):
-        try:
-            fd = self._selector._fileobj_lookup(fileobj)
-            return self._selector._fd_to_key[fd]
-        except KeyError:
-            raise KeyError("{!r} is not registered".format(fileobj)) from None
+        fd = self._selector._fileobj_lookup(fileobj)
+        key = self._selector._fd_to_key.get(fd)
+        if key is None:
+            raise KeyError("{!r} is not registered".format(fileobj))
+        return key
 
     def __iter__(self):
         return iter(self._selector._fd_to_key)
@@ -272,19 +276,6 @@ class _BaseSelectorImpl(BaseSelector):
     def get_map(self):
         return self._map
 
-    def _key_from_fd(self, fd):
-        """Return the key associated to a given file descriptor.
-
-        Parameters:
-        fd -- file descriptor
-
-        Returns:
-        corresponding key, or None if not found
-        """
-        try:
-            return self._fd_to_key[fd]
-        except KeyError:
-            return None
 
 
 class SelectSelector(_BaseSelectorImpl):
@@ -323,17 +314,15 @@ class SelectSelector(_BaseSelectorImpl):
             r, w, _ = self._select(self._readers, self._writers, [], timeout)
         except InterruptedError:
             return ready
-        r = set(r)
-        w = set(w)
-        for fd in r | w:
-            events = 0
-            if fd in r:
-                events |= EVENT_READ
-            if fd in w:
-                events |= EVENT_WRITE
-
-            key = self._key_from_fd(fd)
+        r = frozenset(r)
+        w = frozenset(w)
+        rw = r | w
+        fd_to_key_get = self._fd_to_key.get
+        for fd in rw:
+            key = fd_to_key_get(fd)
             if key:
+                events = ((fd in r and EVENT_READ)
+                          | (fd in w and EVENT_WRITE))
                 ready.append((key, events & key.events))
         return ready
 
@@ -350,11 +339,8 @@ class _PollLikeSelector(_BaseSelectorImpl):
 
     def register(self, fileobj, events, data=None):
         key = super().register(fileobj, events, data)
-        poller_events = 0
-        if events & EVENT_READ:
-            poller_events |= self._EVENT_READ
-        if events & EVENT_WRITE:
-            poller_events |= self._EVENT_WRITE
+        poller_events = ((events & EVENT_READ and self._EVENT_READ)
+                         | (events & EVENT_WRITE and self._EVENT_WRITE) )
         try:
             self._selector.register(key.fd, poller_events)
         except:
@@ -380,11 +366,8 @@ class _PollLikeSelector(_BaseSelectorImpl):
 
         changed = False
         if events != key.events:
-            selector_events = 0
-            if events & EVENT_READ:
-                selector_events |= self._EVENT_READ
-            if events & EVENT_WRITE:
-                selector_events |= self._EVENT_WRITE
+            selector_events = ((events & EVENT_READ and self._EVENT_READ)
+                               | (events & EVENT_WRITE and self._EVENT_WRITE))
             try:
                 self._selector.modify(key.fd, selector_events)
             except:
@@ -415,15 +398,13 @@ class _PollLikeSelector(_BaseSelectorImpl):
             fd_event_list = self._selector.poll(timeout)
         except InterruptedError:
             return ready
-        for fd, event in fd_event_list:
-            events = 0
-            if event & ~self._EVENT_READ:
-                events |= EVENT_WRITE
-            if event & ~self._EVENT_WRITE:
-                events |= EVENT_READ
 
-            key = self._key_from_fd(fd)
+        fd_to_key_get = self._fd_to_key.get
+        for fd, event in fd_event_list:
+            key = fd_to_key_get(fd)
             if key:
+                events = ((event & ~self._EVENT_READ and EVENT_WRITE)
+                           | (event & ~self._EVENT_WRITE and EVENT_READ))
                 ready.append((key, events & key.events))
         return ready
 
@@ -439,6 +420,9 @@ if hasattr(select, 'poll'):
 
 if hasattr(select, 'epoll'):
 
+    _NOT_EPOLLIN = ~select.EPOLLIN
+    _NOT_EPOLLOUT = ~select.EPOLLOUT
+
     class EpollSelector(_PollLikeSelector):
         """Epoll-based selector."""
         _selector_cls = select.epoll
@@ -461,22 +445,20 @@ if hasattr(select, 'epoll'):
             # epoll_wait() expects `maxevents` to be greater than zero;
             # we want to make sure that `select()` can be called when no
             # FD is registered.
-            max_ev = max(len(self._fd_to_key), 1)
+            max_ev = len(self._fd_to_key) or 1
 
             ready = []
             try:
                 fd_event_list = self._selector.poll(timeout, max_ev)
             except InterruptedError:
                 return ready
-            for fd, event in fd_event_list:
-                events = 0
-                if event & ~select.EPOLLIN:
-                    events |= EVENT_WRITE
-                if event & ~select.EPOLLOUT:
-                    events |= EVENT_READ
 
-                key = self._key_from_fd(fd)
+            fd_to_key = self._fd_to_key
+            for fd, event in fd_event_list:
+                key = fd_to_key.get(fd)
                 if key:
+                    events = ((event & _NOT_EPOLLIN and EVENT_WRITE)
+                              | (event & _NOT_EPOLLOUT and EVENT_READ))
                     ready.append((key, events & key.events))
             return ready
 
@@ -566,17 +548,15 @@ if hasattr(select, 'kqueue'):
                 kev_list = self._selector.control(None, max_ev, timeout)
             except InterruptedError:
                 return ready
+
+            fd_to_key_get = self._fd_to_key.get
             for kev in kev_list:
                 fd = kev.ident
                 flag = kev.filter
-                events = 0
-                if flag == select.KQ_FILTER_READ:
-                    events |= EVENT_READ
-                if flag == select.KQ_FILTER_WRITE:
-                    events |= EVENT_WRITE
-
-                key = self._key_from_fd(fd)
+                key = fd_to_key_get(fd)
                 if key:
+                    events = ((flag == select.KQ_FILTER_READ and EVENT_READ)
+                              | (flag == select.KQ_FILTER_WRITE and EVENT_WRITE))
                     ready.append((key, events & key.events))
             return ready
 

Lib/shlex.py

@@ -305,9 +305,7 @@ class shlex:
 def split(s, comments=False, posix=True):
     """Split the string *s* using shell-like syntax."""
     if s is None:
-        import warnings
-        warnings.warn("Passing None for 's' to shlex.split() is deprecated.",
-                      DeprecationWarning, stacklevel=2)
+        raise ValueError("s argument must not be None")
     lex = shlex(s, posix=posix)
     lex.whitespace_split = True
     if not comments:
@@ -335,10 +333,7 @@ def quote(s):
 
 
 def _print_tokens(lexer):
-    while 1:
-        tt = lexer.get_token()
-        if not tt:
-            break
+    while tt := lexer.get_token():
         print("Token: " + repr(tt))
 
 if __name__ == '__main__':

Lib/sqlite3/__main__.py

@@ -48,30 +48,18 @@ class SqliteInteractiveConsole(InteractiveConsole):
         Return True if more input is needed; buffering is done automatically.
         Return False is input is a complete statement ready for execution.
         """
-        if source == ".version":
-            print(f"{sqlite3.sqlite_version}")
-        elif source == ".help":
-            print("Enter SQL code and press enter.")
-        elif source == ".quit":
-            sys.exit(0)
-        elif not sqlite3.complete_statement(source):
-            return True
-        else:
-            execute(self._cur, source)
-            return False
-        # TODO: RUSTPYTHON match statement supporting
-        # match source:
-        #     case ".version":
-        #         print(f"{sqlite3.sqlite_version}")
-        #     case ".help":
-        #         print("Enter SQL code and press enter.")
-        #     case ".quit":
-        #         sys.exit(0)
-        #     case _:
-            # if not sqlite3.complete_statement(source):
-            #     return True
-            # execute(self._cur, source)
-        # return False
+        match source:
+            case ".version":
+                print(f"{sqlite3.sqlite_version}")
+            case ".help":
+                print("Enter SQL code and press enter.")
+            case ".quit":
+                sys.exit(0)
+            case _:
+                if not sqlite3.complete_statement(source):
+                    return True
+                execute(self._cur, source)
+        return False
 
 
 def main():

Lib/stat.py

@@ -110,22 +110,30 @@ S_IWOTH = 0o0002  # write by others
 S_IXOTH = 0o0001  # execute by others
 
 # Names for file flags
-
+UF_SETTABLE  = 0x0000ffff  # owner settable flags
 UF_NODUMP    = 0x00000001  # do not dump file
 UF_IMMUTABLE = 0x00000002  # file may not be changed
 UF_APPEND    = 0x00000004  # file may only be appended to
 UF_OPAQUE    = 0x00000008  # directory is opaque when viewed through a union stack
 UF_NOUNLINK  = 0x00000010  # file may not be renamed or deleted
-UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed
-UF_HIDDEN    = 0x00008000  # OS X: file should not be displayed
+UF_COMPRESSED = 0x00000020 # macOS: file is compressed
+UF_TRACKED   = 0x00000040  # macOS: used for handling document IDs
+UF_DATAVAULT = 0x00000080  # macOS: entitlement needed for I/O
+UF_HIDDEN    = 0x00008000  # macOS: file should not be displayed
+SF_SETTABLE  = 0xffff0000  # superuser settable flags
 SF_ARCHIVED  = 0x00010000  # file may be archived
 SF_IMMUTABLE = 0x00020000  # file may not be changed
 SF_APPEND    = 0x00040000  # file may only be appended to
+SF_RESTRICTED = 0x00080000 # macOS: entitlement needed for writing
 SF_NOUNLINK  = 0x00100000  # file may not be renamed or deleted
 SF_SNAPSHOT  = 0x00200000  # file is a snapshot file
+SF_FIRMLINK  = 0x00800000  # macOS: file is a firmlink
+SF_DATALESS  = 0x40000000  # macOS: file is a dataless object
 
 
 _filemode_table = (
+    # File type chars according to:
+    # http://en.wikibooks.org/wiki/C_Programming/POSIX_Reference/sys/stat.h
     ((S_IFLNK,         "l"),
      (S_IFSOCK,        "s"),  # Must appear before IFREG and IFDIR as IFSOCK == IFREG | IFDIR
      (S_IFREG,         "-"),
@@ -156,13 +164,17 @@ _filemode_table = (
 def filemode(mode):
     """Convert a file's mode to a string of the form '-rwxrwxrwx'."""
     perm = []
-    for table in _filemode_table:
+    for index, table in enumerate(_filemode_table):
         for bit, char in table:
             if mode & bit == bit:
                 perm.append(char)
                 break
         else:
-            perm.append("-")
+            if index == 0:
+                # Unknown filetype
+                perm.append("?")
+            else:
+                perm.append("-")
     return "".join(perm)
 
 

Lib/symtable.py

@@ -0,0 +1,414 @@
+"""Interface to the compiler's internal symbol tables"""
+
+import _symtable
+from _symtable import (USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM,
+     DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE,
+     LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL)
+
+import weakref
+from enum import StrEnum
+
+__all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"]
+
+def symtable(code, filename, compile_type):
+    """ Return the toplevel *SymbolTable* for the source code.
+
+    *filename* is the name of the file with the code
+    and *compile_type* is the *compile()* mode argument.
+    """
+    top = _symtable.symtable(code, filename, compile_type)
+    return _newSymbolTable(top, filename)
+
+class SymbolTableFactory:
+    def __init__(self):
+        self.__memo = weakref.WeakValueDictionary()
+
+    def new(self, table, filename):
+        if table.type == _symtable.TYPE_FUNCTION:
+            return Function(table, filename)
+        if table.type == _symtable.TYPE_CLASS:
+            return Class(table, filename)
+        return SymbolTable(table, filename)
+
+    def __call__(self, table, filename):
+        key = table, filename
+        obj = self.__memo.get(key, None)
+        if obj is None:
+            obj = self.__memo[key] = self.new(table, filename)
+        return obj
+
+_newSymbolTable = SymbolTableFactory()
+
+
+class SymbolTableType(StrEnum):
+    MODULE = "module"
+    FUNCTION = "function"
+    CLASS = "class"
+    ANNOTATION = "annotation"
+    TYPE_ALIAS = "type alias"
+    TYPE_PARAMETERS = "type parameters"
+    TYPE_VARIABLE = "type variable"
+
+
+class SymbolTable:
+
+    def __init__(self, raw_table, filename):
+        self._table = raw_table
+        self._filename = filename
+        self._symbols = {}
+
+    def __repr__(self):
+        if self.__class__ == SymbolTable:
+            kind = ""
+        else:
+            kind = "%s " % self.__class__.__name__
+
+        if self._table.name == "top":
+            return "<{0}SymbolTable for module {1}>".format(kind, self._filename)
+        else:
+            return "<{0}SymbolTable for {1} in {2}>".format(kind,
+                                                            self._table.name,
+                                                            self._filename)
+
+    def get_type(self):
+        """Return the type of the symbol table.
+
+        The value returned is one of the values in
+        the ``SymbolTableType`` enumeration.
+        """
+        if self._table.type == _symtable.TYPE_MODULE:
+            return SymbolTableType.MODULE
+        if self._table.type == _symtable.TYPE_FUNCTION:
+            return SymbolTableType.FUNCTION
+        if self._table.type == _symtable.TYPE_CLASS:
+            return SymbolTableType.CLASS
+        if self._table.type == _symtable.TYPE_ANNOTATION:
+            return SymbolTableType.ANNOTATION
+        if self._table.type == _symtable.TYPE_TYPE_ALIAS:
+            return SymbolTableType.TYPE_ALIAS
+        if self._table.type == _symtable.TYPE_TYPE_PARAMETERS:
+            return SymbolTableType.TYPE_PARAMETERS
+        if self._table.type == _symtable.TYPE_TYPE_VARIABLE:
+            return SymbolTableType.TYPE_VARIABLE
+        assert False, f"unexpected type: {self._table.type}"
+
+    def get_id(self):
+        """Return an identifier for the table.
+        """
+        return self._table.id
+
+    def get_name(self):
+        """Return the table's name.
+
+        This corresponds to the name of the class, function
+        or 'top' if the table is for a class, function or
+        global respectively.
+        """
+        return self._table.name
+
+    def get_lineno(self):
+        """Return the number of the first line in the
+        block for the table.
+        """
+        return self._table.lineno
+
+    def is_optimized(self):
+        """Return *True* if the locals in the table
+        are optimizable.
+        """
+        return bool(self._table.type == _symtable.TYPE_FUNCTION)
+
+    def is_nested(self):
+        """Return *True* if the block is a nested class
+        or function."""
+        return bool(self._table.nested)
+
+    def has_children(self):
+        """Return *True* if the block has nested namespaces.
+        """
+        return bool(self._table.children)
+
+    def get_identifiers(self):
+        """Return a view object containing the names of symbols in the table.
+        """
+        return self._table.symbols.keys()
+
+    def lookup(self, name):
+        """Lookup a *name* in the table.
+
+        Returns a *Symbol* instance.
+        """
+        sym = self._symbols.get(name)
+        if sym is None:
+            flags = self._table.symbols[name]
+            namespaces = self.__check_children(name)
+            module_scope = (self._table.name == "top")
+            sym = self._symbols[name] = Symbol(name, flags, namespaces,
+                                               module_scope=module_scope)
+        return sym
+
+    def get_symbols(self):
+        """Return a list of *Symbol* instances for
+        names in the table.
+        """
+        return [self.lookup(ident) for ident in self.get_identifiers()]
+
+    def __check_children(self, name):
+        return [_newSymbolTable(st, self._filename)
+                for st in self._table.children
+                if st.name == name]
+
+    def get_children(self):
+        """Return a list of the nested symbol tables.
+        """
+        return [_newSymbolTable(st, self._filename)
+                for st in self._table.children]
+
+
+class Function(SymbolTable):
+
+    # Default values for instance variables
+    __params = None
+    __locals = None
+    __frees = None
+    __globals = None
+    __nonlocals = None
+
+    def __idents_matching(self, test_func):
+        return tuple(ident for ident in self.get_identifiers()
+                     if test_func(self._table.symbols[ident]))
+
+    def get_parameters(self):
+        """Return a tuple of parameters to the function.
+        """
+        if self.__params is None:
+            self.__params = self.__idents_matching(lambda x:x & DEF_PARAM)
+        return self.__params
+
+    def get_locals(self):
+        """Return a tuple of locals in the function.
+        """
+        if self.__locals is None:
+            locs = (LOCAL, CELL)
+            test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs
+            self.__locals = self.__idents_matching(test)
+        return self.__locals
+
+    def get_globals(self):
+        """Return a tuple of globals in the function.
+        """
+        if self.__globals is None:
+            glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
+            test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob
+            self.__globals = self.__idents_matching(test)
+        return self.__globals
+
+    def get_nonlocals(self):
+        """Return a tuple of nonlocals in the function.
+        """
+        if self.__nonlocals is None:
+            self.__nonlocals = self.__idents_matching(lambda x:x & DEF_NONLOCAL)
+        return self.__nonlocals
+
+    def get_frees(self):
+        """Return a tuple of free variables in the function.
+        """
+        if self.__frees is None:
+            is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE
+            self.__frees = self.__idents_matching(is_free)
+        return self.__frees
+
+
+class Class(SymbolTable):
+
+    __methods = None
+
+    def get_methods(self):
+        """Return a tuple of methods declared in the class.
+        """
+        if self.__methods is None:
+            d = {}
+
+            def is_local_symbol(ident):
+                flags = self._table.symbols.get(ident, 0)
+                return ((flags >> SCOPE_OFF) & SCOPE_MASK) == LOCAL
+
+            for st in self._table.children:
+                # pick the function-like symbols that are local identifiers
+                if is_local_symbol(st.name):
+                    match st.type:
+                        case _symtable.TYPE_FUNCTION:
+                            # generators are of type TYPE_FUNCTION with a ".0"
+                            # parameter as a first parameter (which makes them
+                            # distinguishable from a function named 'genexpr')
+                            if st.name == 'genexpr' and '.0' in st.varnames:
+                                continue
+                            d[st.name] = 1
+                        case _symtable.TYPE_TYPE_PARAMETERS:
+                            # Get the function-def block in the annotation
+                            # scope 'st' with the same identifier, if any.
+                            scope_name = st.name
+                            for c in st.children:
+                                if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION:
+                                    # A generic generator of type TYPE_FUNCTION
+                                    # cannot be a direct child of 'st' (but it
+                                    # can be a descendant), e.g.:
+                                    #
+                                    # class A:
+                                    #   type genexpr[genexpr] = (x for x in [])
+                                    assert scope_name != 'genexpr' or '.0' not in c.varnames
+                                    d[scope_name] = 1
+                                    break
+            self.__methods = tuple(d)
+        return self.__methods
+
+
+class Symbol:
+
+    def __init__(self, name, flags, namespaces=None, *, module_scope=False):
+        self.__name = name
+        self.__flags = flags
+        self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope()
+        self.__namespaces = namespaces or ()
+        self.__module_scope = module_scope
+
+    def __repr__(self):
+        flags_str = '|'.join(self._flags_str())
+        return f'<symbol {self.__name!r}: {self._scope_str()}, {flags_str}>'
+
+    def _scope_str(self):
+        return _scopes_value_to_name.get(self.__scope) or str(self.__scope)
+
+    def _flags_str(self):
+        for flagname, flagvalue in _flags:
+            if self.__flags & flagvalue == flagvalue:
+                yield flagname
+
+    def get_name(self):
+        """Return a name of a symbol.
+        """
+        return self.__name
+
+    def is_referenced(self):
+        """Return *True* if the symbol is used in
+        its block.
+        """
+        return bool(self.__flags & _symtable.USE)
+
+    def is_parameter(self):
+        """Return *True* if the symbol is a parameter.
+        """
+        return bool(self.__flags & DEF_PARAM)
+
+    def is_global(self):
+        """Return *True* if the symbol is global.
+        """
+        return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
+                    or (self.__module_scope and self.__flags & DEF_BOUND))
+
+    def is_nonlocal(self):
+        """Return *True* if the symbol is nonlocal."""
+        return bool(self.__flags & DEF_NONLOCAL)
+
+    def is_declared_global(self):
+        """Return *True* if the symbol is declared global
+        with a global statement."""
+        return bool(self.__scope == GLOBAL_EXPLICIT)
+
+    def is_local(self):
+        """Return *True* if the symbol is local.
+        """
+        return bool(self.__scope in (LOCAL, CELL)
+                    or (self.__module_scope and self.__flags & DEF_BOUND))
+
+    def is_annotated(self):
+        """Return *True* if the symbol is annotated.
+        """
+        return bool(self.__flags & DEF_ANNOT)
+
+    def is_free(self):
+        """Return *True* if a referenced symbol is
+        not assigned to.
+        """
+        return bool(self.__scope == FREE)
+
+    def is_imported(self):
+        """Return *True* if the symbol is created from
+        an import statement.
+        """
+        return bool(self.__flags & DEF_IMPORT)
+
+    def is_assigned(self):
+        """Return *True* if a symbol is assigned to."""
+        return bool(self.__flags & DEF_LOCAL)
+
+    def is_namespace(self):
+        """Returns *True* if name binding introduces new namespace.
+
+        If the name is used as the target of a function or class
+        statement, this will be true.
+
+        Note that a single name can be bound to multiple objects.  If
+        is_namespace() is true, the name may also be bound to other
+        objects, like an int or list, that does not introduce a new
+        namespace.
+        """
+        return bool(self.__namespaces)
+
+    def get_namespaces(self):
+        """Return a list of namespaces bound to this name"""
+        return self.__namespaces
+
+    def get_namespace(self):
+        """Return the single namespace bound to this name.
+
+        Raises ValueError if the name is bound to multiple namespaces
+        or no namespace.
+        """
+        if len(self.__namespaces) == 0:
+            raise ValueError("name is not bound to any namespaces")
+        elif len(self.__namespaces) > 1:
+            raise ValueError("name is bound to multiple namespaces")
+        else:
+            return self.__namespaces[0]
+
+
+_flags = [('USE', USE)]
+_flags.extend(kv for kv in globals().items() if kv[0].startswith('DEF_'))
+_scopes_names = ('FREE', 'LOCAL', 'GLOBAL_IMPLICIT', 'GLOBAL_EXPLICIT', 'CELL')
+_scopes_value_to_name = {globals()[n]: n for n in _scopes_names}
+
+
+def main(args):
+    import sys
+    def print_symbols(table, level=0):
+        indent = '    ' * level
+        nested = "nested " if table.is_nested() else ""
+        if table.get_type() == 'module':
+            what = f'from file {table._filename!r}'
+        else:
+            what = f'{table.get_name()!r}'
+        print(f'{indent}symbol table for {nested}{table.get_type()} {what}:')
+        for ident in table.get_identifiers():
+            symbol = table.lookup(ident)
+            flags = ', '.join(symbol._flags_str()).lower()
+            print(f'    {indent}{symbol._scope_str().lower()} symbol {symbol.get_name()!r}: {flags}')
+        print()
+
+        for table2 in table.get_children():
+            print_symbols(table2, level + 1)
+
+    for filename in args or ['-']:
+        if filename == '-':
+            src = sys.stdin.read()
+            filename = '<stdin>'
+        else:
+            with open(filename, 'rb') as f:
+                src = f.read()
+        mod = symtable(src, filename, 'exec')
+        print_symbols(mod)
+
+
+if __name__ == "__main__":
+    import sys
+    main(sys.argv[1:])

Lib/tabnanny.py

@@ -23,8 +23,6 @@ __version__ = "6"
 import os
 import sys
 import tokenize
-if not hasattr(tokenize, 'NL'):
-    raise ValueError("tokenize.NL doesn't exist -- tokenize module too old")
 
 __all__ = ["check", "NannyNag", "process_tokens"]
 
@@ -37,6 +35,7 @@ def errprint(*args):
         sys.stderr.write(sep + str(arg))
         sep = " "
     sys.stderr.write("\n")
+    sys.exit(1)
 
 def main():
     import getopt
@@ -46,7 +45,6 @@ def main():
         opts, args = getopt.getopt(sys.argv[1:], "qv")
     except getopt.error as msg:
         errprint(msg)
-        return
     for o, a in opts:
         if o == '-q':
             filename_only = filename_only + 1
@@ -54,7 +52,6 @@ def main():
             verbose = verbose + 1
     if not args:
         errprint("Usage:", sys.argv[0], "[-v] file_or_directory ...")
-        return
     for arg in args:
         check(arg)
 
@@ -114,6 +111,10 @@ def check(file):
         errprint("%r: Indentation Error: %s" % (file, msg))
         return
 
+    except SyntaxError as msg:
+        errprint("%r: Syntax Error: %s" % (file, msg))
+        return
+
     except NannyNag as nag:
         badline = nag.get_lineno()
         line = nag.get_line()
@@ -275,6 +276,12 @@ def format_witnesses(w):
     return prefix + " " + ', '.join(firsts)
 
 def process_tokens(tokens):
+    try:
+        _process_tokens(tokens)
+    except TabError as e:
+        raise NannyNag(e.lineno, e.msg, e.text)
+
+def _process_tokens(tokens):
     INDENT = tokenize.INDENT
     DEDENT = tokenize.DEDENT
     NEWLINE = tokenize.NEWLINE

Lib/test/_test_multiprocessing.py

@@ -12,6 +12,7 @@ import itertools
 import sys
 import os
 import gc
+import importlib
 import errno
 import functools
 import signal
@@ -19,10 +20,11 @@ import array
 import socket
 import random
 import logging
+import shutil
 import subprocess
 import struct
+import tempfile
 import operator
-import pathlib
 import pickle
 import weakref
 import warnings
@@ -50,7 +52,7 @@ import multiprocessing.heap
 import multiprocessing.managers
 import multiprocessing.pool
 import multiprocessing.queues
-from multiprocessing.connection import wait, AuthenticationError
+from multiprocessing.connection import wait
 
 from multiprocessing import util
 
@@ -255,6 +257,9 @@ class TimingWrapper(object):
 class BaseTestCase(object):
 
     ALLOWED_TYPES = ('processes', 'manager', 'threads')
+    # If not empty, limit which start method suites run this class.
+    START_METHODS: set[str] = set()
+    start_method = None  # set by install_tests_in_module_dict()
 
     def assertTimingAlmostEqual(self, a, b):
         if CHECK_TIMINGS:
@@ -324,8 +329,9 @@ class _TestProcess(BaseTestCase):
             self.skipTest(f'test not appropriate for {self.TYPE}')
         paths = [
             sys.executable,               # str
-            sys.executable.encode(),      # bytes
-            pathlib.Path(sys.executable)  # os.PathLike
+            os.fsencode(sys.executable),  # bytes
+            os_helper.FakePath(sys.executable),  # os.PathLike
+            os_helper.FakePath(os.fsencode(sys.executable)),  # os.PathLike bytes
         ]
         for path in paths:
             self.set_executable(path)
@@ -505,6 +511,11 @@ class _TestProcess(BaseTestCase):
     def _sleep_some(cls):
         time.sleep(100)
 
+    @classmethod
+    def _sleep_some_event(cls, event):
+        event.set()
+        time.sleep(100)
+
     @classmethod
     def _test_sleep(cls, delay):
         time.sleep(delay)
@@ -513,7 +524,8 @@ class _TestProcess(BaseTestCase):
         if self.TYPE == 'threads':
             self.skipTest('test not appropriate for {}'.format(self.TYPE))
 
-        p = self.Process(target=self._sleep_some)
+        event = self.Event()
+        p = self.Process(target=self._sleep_some_event, args=(event,))
         p.daemon = True
         p.start()
 
@@ -531,8 +543,11 @@ class _TestProcess(BaseTestCase):
         self.assertTimingAlmostEqual(join.elapsed, 0.0)
         self.assertEqual(p.is_alive(), True)
 
-        # XXX maybe terminating too soon causes the problems on Gentoo...
-        time.sleep(1)
+        timeout = support.SHORT_TIMEOUT
+        if not event.wait(timeout):
+            p.terminate()
+            p.join()
+            self.fail(f"event not signaled in {timeout} seconds")
 
         meth(p)
 
@@ -582,12 +597,16 @@ class _TestProcess(BaseTestCase):
     def test_active_children(self):
         self.assertEqual(type(self.active_children()), list)
 
-        p = self.Process(target=time.sleep, args=(DELTA,))
+        event = self.Event()
+        p = self.Process(target=event.wait, args=())
         self.assertNotIn(p, self.active_children())
 
-        p.daemon = True
-        p.start()
-        self.assertIn(p, self.active_children())
+        try:
+            p.daemon = True
+            p.start()
+            self.assertIn(p, self.active_children())
+        finally:
+            event.set()
 
         p.join()
         self.assertNotIn(p, self.active_children())
@@ -1332,6 +1351,23 @@ class _TestQueue(BaseTestCase):
         self.assertTrue(not_serializable_obj.reduce_was_called)
         self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called)
 
+    def test_closed_queue_empty_exceptions(self):
+        # Assert that checking the emptiness of an unused closed queue
+        # does not raise an OSError. The rationale is that q.close() is
+        # a no-op upon construction and becomes effective once the queue
+        # has been used (e.g., by calling q.put()).
+        for q in multiprocessing.Queue(), multiprocessing.JoinableQueue():
+            q.close()  # this is a no-op since the feeder thread is None
+            q.join_thread()  # this is also a no-op
+            self.assertTrue(q.empty())
+
+        for q in multiprocessing.Queue(), multiprocessing.JoinableQueue():
+            q.put('foo')  # make sure that the queue is 'used'
+            q.close()  # close the feeder thread
+            q.join_thread()  # make sure to join the feeder thread
+            with self.assertRaisesRegex(OSError, 'is closed'):
+                q.empty()
+
     def test_closed_queue_put_get_exceptions(self):
         for q in multiprocessing.Queue(), multiprocessing.JoinableQueue():
             q.close()
@@ -1345,6 +1381,66 @@ class _TestQueue(BaseTestCase):
 
 class _TestLock(BaseTestCase):
 
+    @staticmethod
+    def _acquire(lock, l=None):
+        lock.acquire()
+        if l is not None:
+            l.append(repr(lock))
+
+    @staticmethod
+    def _acquire_event(lock, event):
+        lock.acquire()
+        event.set()
+        time.sleep(1.0)
+
+    def test_repr_lock(self):
+        if self.TYPE != 'processes':
+            self.skipTest('test not appropriate for {}'.format(self.TYPE))
+
+        lock = self.Lock()
+        self.assertEqual(f'<Lock(owner=None)>', repr(lock))
+
+        lock.acquire()
+        self.assertEqual(f'<Lock(owner=MainProcess)>', repr(lock))
+        lock.release()
+
+        tname = 'T1'
+        l = []
+        t = threading.Thread(target=self._acquire,
+                             args=(lock, l),
+                             name=tname)
+        t.start()
+        time.sleep(0.1)
+        self.assertEqual(f'<Lock(owner=MainProcess|{tname})>', l[0])
+        lock.release()
+
+        t = threading.Thread(target=self._acquire,
+                             args=(lock,),
+                             name=tname)
+        t.start()
+        time.sleep(0.1)
+        self.assertEqual('<Lock(owner=SomeOtherThread)>', repr(lock))
+        lock.release()
+
+        pname = 'P1'
+        l = multiprocessing.Manager().list()
+        p = self.Process(target=self._acquire,
+                         args=(lock, l),
+                         name=pname)
+        p.start()
+        p.join()
+        self.assertEqual(f'<Lock(owner={pname})>', l[0])
+
+        lock = self.Lock()
+        event = self.Event()
+        p = self.Process(target=self._acquire_event,
+                         args=(lock, event),
+                         name='P2')
+        p.start()
+        event.wait()
+        self.assertEqual(f'<Lock(owner=SomeOtherProcess)>', repr(lock))
+        p.terminate()
+
     def test_lock(self):
         lock = self.Lock()
         self.assertEqual(lock.acquire(), True)
@@ -1352,6 +1448,68 @@ class _TestLock(BaseTestCase):
         self.assertEqual(lock.release(), None)
         self.assertRaises((ValueError, threading.ThreadError), lock.release)
 
+    @staticmethod
+    def _acquire_release(lock, timeout, l=None, n=1):
+        for _ in range(n):
+            lock.acquire()
+        if l is not None:
+            l.append(repr(lock))
+        time.sleep(timeout)
+        for _ in range(n):
+            lock.release()
+
+    def test_repr_rlock(self):
+        if self.TYPE != 'processes':
+            self.skipTest('test not appropriate for {}'.format(self.TYPE))
+
+        lock = self.RLock()
+        self.assertEqual('<RLock(None, 0)>', repr(lock))
+
+        n = 3
+        for _ in range(n):
+            lock.acquire()
+        self.assertEqual(f'<RLock(MainProcess, {n})>', repr(lock))
+        for _ in range(n):
+            lock.release()
+
+        t, l = [], []
+        for i in range(n):
+            t.append(threading.Thread(target=self._acquire_release,
+                                      args=(lock, 0.1, l, i+1),
+                                      name=f'T{i+1}'))
+            t[-1].start()
+        for t_ in t:
+            t_.join()
+        for i in range(n):
+            self.assertIn(f'<RLock(MainProcess|T{i+1}, {i+1})>', l)
+
+
+        t = threading.Thread(target=self._acquire_release,
+                                 args=(lock, 0.2),
+                                 name=f'T1')
+        t.start()
+        time.sleep(0.1)
+        self.assertEqual('<RLock(SomeOtherThread, nonzero)>', repr(lock))
+        time.sleep(0.2)
+
+        pname = 'P1'
+        l = multiprocessing.Manager().list()
+        p = self.Process(target=self._acquire_release,
+                         args=(lock, 0.1, l),
+                         name=pname)
+        p.start()
+        p.join()
+        self.assertEqual(f'<RLock({pname}, 1)>', l[0])
+
+        event = self.Event()
+        lock = self.RLock()
+        p = self.Process(target=self._acquire_event,
+                         args=(lock, event))
+        p.start()
+        event.wait()
+        self.assertEqual('<RLock(SomeOtherProcess, nonzero)>', repr(lock))
+        p.join()
+
     def test_rlock(self):
         lock = self.RLock()
         self.assertEqual(lock.acquire(), True)
@@ -1432,14 +1590,13 @@ class _TestCondition(BaseTestCase):
         cond.release()
 
     def assertReachesEventually(self, func, value):
-        for i in range(10):
+        for _ in support.sleeping_retry(support.SHORT_TIMEOUT):
             try:
                 if func() == value:
                     break
             except NotImplementedError:
                 break
-            time.sleep(DELTA)
-        time.sleep(DELTA)
+
         self.assertReturnsIfImplemented(value, func)
 
     def check_invariant(self, cond):
@@ -1461,20 +1618,17 @@ class _TestCondition(BaseTestCase):
         p = self.Process(target=self.f, args=(cond, sleeping, woken))
         p.daemon = True
         p.start()
-        self.addCleanup(p.join)
 
-        p = threading.Thread(target=self.f, args=(cond, sleeping, woken))
-        p.daemon = True
-        p.start()
-        self.addCleanup(p.join)
+        t = threading.Thread(target=self.f, args=(cond, sleeping, woken))
+        t.daemon = True
+        t.start()
 
         # wait for both children to start sleeping
         sleeping.acquire()
         sleeping.acquire()
 
         # check no process/thread has woken up
-        time.sleep(DELTA)
-        self.assertReturnsIfImplemented(0, get_value, woken)
+        self.assertReachesEventually(lambda: get_value(woken), 0)
 
         # wake up one process/thread
         cond.acquire()
@@ -1482,8 +1636,7 @@ class _TestCondition(BaseTestCase):
         cond.release()
 
         # check one process/thread has woken up
-        time.sleep(DELTA)
-        self.assertReturnsIfImplemented(1, get_value, woken)
+        self.assertReachesEventually(lambda: get_value(woken), 1)
 
         # wake up another
         cond.acquire()
@@ -1491,12 +1644,13 @@ class _TestCondition(BaseTestCase):
         cond.release()
 
         # check other has woken up
-        time.sleep(DELTA)
-        self.assertReturnsIfImplemented(2, get_value, woken)
+        self.assertReachesEventually(lambda: get_value(woken), 2)
 
         # check state is not mucked up
         self.check_invariant(cond)
-        p.join()
+
+        threading_helper.join_thread(t)
+        join_process(p)
 
     def test_notify_all(self):
         cond = self.Condition()
@@ -1504,18 +1658,19 @@ class _TestCondition(BaseTestCase):
         woken = self.Semaphore(0)
 
         # start some threads/processes which will timeout
+        workers = []
         for i in range(3):
             p = self.Process(target=self.f,
                              args=(cond, sleeping, woken, TIMEOUT1))
             p.daemon = True
             p.start()
-            self.addCleanup(p.join)
+            workers.append(p)
 
             t = threading.Thread(target=self.f,
                                  args=(cond, sleeping, woken, TIMEOUT1))
             t.daemon = True
             t.start()
-            self.addCleanup(t.join)
+            workers.append(t)
 
         # wait for them all to sleep
         for i in range(6):
@@ -1534,12 +1689,12 @@ class _TestCondition(BaseTestCase):
             p = self.Process(target=self.f, args=(cond, sleeping, woken))
             p.daemon = True
             p.start()
-            self.addCleanup(p.join)
+            workers.append(p)
 
             t = threading.Thread(target=self.f, args=(cond, sleeping, woken))
             t.daemon = True
             t.start()
-            self.addCleanup(t.join)
+            workers.append(t)
 
         # wait for them to all sleep
         for i in range(6):
@@ -1555,27 +1710,34 @@ class _TestCondition(BaseTestCase):
         cond.release()
 
         # check they have all woken
-        self.assertReachesEventually(lambda: get_value(woken), 6)
+        for i in range(6):
+            woken.acquire()
+        self.assertReturnsIfImplemented(0, get_value, woken)
 
         # check state is not mucked up
         self.check_invariant(cond)
 
+        for w in workers:
+            # NOTE: join_process and join_thread are the same
+            threading_helper.join_thread(w)
+
     def test_notify_n(self):
         cond = self.Condition()
         sleeping = self.Semaphore(0)
         woken = self.Semaphore(0)
 
         # start some threads/processes
+        workers = []
         for i in range(3):
             p = self.Process(target=self.f, args=(cond, sleeping, woken))
             p.daemon = True
             p.start()
-            self.addCleanup(p.join)
+            workers.append(p)
 
             t = threading.Thread(target=self.f, args=(cond, sleeping, woken))
             t.daemon = True
             t.start()
-            self.addCleanup(t.join)
+            workers.append(t)
 
         # wait for them to all sleep
         for i in range(6):
@@ -1610,6 +1772,10 @@ class _TestCondition(BaseTestCase):
         # check state is not mucked up
         self.check_invariant(cond)
 
+        for w in workers:
+            # NOTE: join_process and join_thread are the same
+            threading_helper.join_thread(w)
+
     def test_timeout(self):
         cond = self.Condition()
         wait = TimingWrapper(cond.wait)
@@ -2812,8 +2978,8 @@ class _TestPool(BaseTestCase):
         self.pool.map(identity, objs)
 
         del objs
-        gc.collect()  # For PyPy or other GCs.
         time.sleep(DELTA)  # let threaded cleanup code run
+        support.gc_collect()  # For PyPy or other GCs.
         self.assertEqual(set(wr() for wr in refs), {None})
         # With a process pool, copies of the objects are returned, check
         # they were released too.
@@ -3174,6 +3340,44 @@ class _TestManagerRestart(BaseTestCase):
             if hasattr(manager, "shutdown"):
                 self.addCleanup(manager.shutdown)
 
+
+class FakeConnection:
+    def send(self, payload):
+        pass
+
+    def recv(self):
+        return '#ERROR', pyqueue.Empty()
+
+class TestManagerExceptions(unittest.TestCase):
+    # Issue 106558: Manager exceptions avoids creating cyclic references.
+    def setUp(self):
+        self.mgr = multiprocessing.Manager()
+
+    def tearDown(self):
+        self.mgr.shutdown()
+        self.mgr.join()
+
+    def test_queue_get(self):
+        queue = self.mgr.Queue()
+        if gc.isenabled():
+            gc.disable()
+            self.addCleanup(gc.enable)
+        try:
+            queue.get_nowait()
+        except pyqueue.Empty as e:
+            wr = weakref.ref(e)
+        self.assertEqual(wr(), None)
+
+    def test_dispatch(self):
+        if gc.isenabled():
+            gc.disable()
+            self.addCleanup(gc.enable)
+        try:
+            multiprocessing.managers.dispatch(FakeConnection(), None, None)
+        except pyqueue.Empty as e:
+            wr = weakref.ref(e)
+        self.assertEqual(wr(), None)
+
 #
 #
 #
@@ -4462,6 +4666,59 @@ class _TestSharedMemory(BaseTestCase):
                     "resource_tracker: There appear to be 1 leaked "
                     "shared_memory objects to clean up at shutdown", err)
 
+    @unittest.skipIf(os.name != "posix", "resource_tracker is posix only")
+    def test_shared_memory_untracking(self):
+        # gh-82300: When a separate Python process accesses shared memory
+        # with track=False, it must not cause the memory to be deleted
+        # when terminating.
+        cmd = '''if 1:
+            import sys
+            from multiprocessing.shared_memory import SharedMemory
+            mem = SharedMemory(create=False, name=sys.argv[1], track=False)
+            mem.close()
+        '''
+        mem = shared_memory.SharedMemory(create=True, size=10)
+        # The resource tracker shares pipes with the subprocess, and so
+        # err existing means that the tracker process has terminated now.
+        try:
+            rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name)
+            self.assertNotIn(b"resource_tracker", err)
+            self.assertEqual(rc, 0)
+            mem2 = shared_memory.SharedMemory(create=False, name=mem.name)
+            mem2.close()
+        finally:
+            try:
+                mem.unlink()
+            except OSError:
+                pass
+            mem.close()
+
+    @unittest.skipIf(os.name != "posix", "resource_tracker is posix only")
+    def test_shared_memory_tracking(self):
+        # gh-82300: When a separate Python process accesses shared memory
+        # with track=True, it must cause the memory to be deleted when
+        # terminating.
+        cmd = '''if 1:
+            import sys
+            from multiprocessing.shared_memory import SharedMemory
+            mem = SharedMemory(create=False, name=sys.argv[1], track=True)
+            mem.close()
+        '''
+        mem = shared_memory.SharedMemory(create=True, size=10)
+        try:
+            rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name)
+            self.assertEqual(rc, 0)
+            self.assertIn(
+                b"resource_tracker: There appear to be 1 leaked "
+                b"shared_memory objects to clean up at shutdown", err)
+        finally:
+            try:
+                mem.unlink()
+            except OSError:
+                pass
+            resource_tracker.unregister(mem._name, "shared_memory")
+            mem.close()
+
 #
 # Test to verify that `Finalize` works.
 #
@@ -4571,7 +4828,7 @@ class _TestFinalize(BaseTestCase):
         old_interval = sys.getswitchinterval()
         old_threshold = gc.get_threshold()
         try:
-            sys.setswitchinterval(1e-6)
+            support.setswitchinterval(1e-6)
             gc.set_threshold(5, 5, 5)
             threads = [threading.Thread(target=run_finalizers),
                        threading.Thread(target=make_finalizers)]
@@ -5557,8 +5814,9 @@ class TestResourceTracker(unittest.TestCase):
         '''
         for rtype in resource_tracker._CLEANUP_FUNCS:
             with self.subTest(rtype=rtype):
-                if rtype == "noop":
+                if rtype in ("noop", "dummy"):
                     # Artefact resource type used by the resource_tracker
+                    # or tests
                     continue
                 r, w = os.pipe()
                 p = subprocess.Popen([sys.executable,
@@ -5638,6 +5896,8 @@ class TestResourceTracker(unittest.TestCase):
         # Catchable signal (ignored by semaphore tracker)
         self.check_resource_tracker_death(signal.SIGTERM, False)
 
+    @unittest.skipIf(sys.platform.startswith("netbsd"),
+                     "gh-125620: Skip on NetBSD due to long wait for SIGKILL process termination.")
     def test_resource_tracker_sigkill(self):
         # Uncatchable signal.
         self.check_resource_tracker_death(signal.SIGKILL, True)
@@ -5678,6 +5938,59 @@ class TestResourceTracker(unittest.TestCase):
         with self.assertRaises(ValueError):
             resource_tracker.register(too_long_name_resource, rtype)
 
+    def _test_resource_tracker_leak_resources(self, cleanup):
+        # We use a separate instance for testing, since the main global
+        # _resource_tracker may be used to watch test infrastructure.
+        from multiprocessing.resource_tracker import ResourceTracker
+        tracker = ResourceTracker()
+        tracker.ensure_running()
+        self.assertTrue(tracker._check_alive())
+
+        self.assertIsNone(tracker._exitcode)
+        tracker.register('somename', 'dummy')
+        if cleanup:
+            tracker.unregister('somename', 'dummy')
+            expected_exit_code = 0
+        else:
+            expected_exit_code = 1
+
+        self.assertTrue(tracker._check_alive())
+        self.assertIsNone(tracker._exitcode)
+        tracker._stop()
+        self.assertEqual(tracker._exitcode, expected_exit_code)
+
+    def test_resource_tracker_exit_code(self):
+        """
+        Test the exit code of the resource tracker.
+
+        If no leaked resources were found, exit code should be 0, otherwise 1
+        """
+        for cleanup in [True, False]:
+            with self.subTest(cleanup=cleanup):
+                self._test_resource_tracker_leak_resources(
+                    cleanup=cleanup,
+                )
+
+    @unittest.skipUnless(hasattr(signal, "pthread_sigmask"), "pthread_sigmask is not available")
+    def test_resource_tracker_blocked_signals(self):
+        #
+        # gh-127586: Check that resource_tracker does not override blocked signals of caller.
+        #
+        from multiprocessing.resource_tracker import ResourceTracker
+        orig_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, set())
+        signals = {signal.SIGTERM, signal.SIGINT, signal.SIGUSR1}
+
+        try:
+            for sig in signals:
+                signal.pthread_sigmask(signal.SIG_SETMASK, {sig})
+                self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig})
+                tracker = ResourceTracker()
+                tracker.ensure_running()
+                self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig})
+                tracker._stop()
+        finally:
+            # restore sigmask to what it was before executing test
+            signal.pthread_sigmask(signal.SIG_SETMASK, orig_sigmask)
 
 class TestSimpleQueue(unittest.TestCase):
 
@@ -5691,6 +6004,15 @@ class TestSimpleQueue(unittest.TestCase):
         finally:
             parent_can_continue.set()
 
+    def test_empty_exceptions(self):
+        # Assert that checking emptiness of a closed queue raises
+        # an OSError, independently of whether the queue was used
+        # or not. This differs from Queue and JoinableQueue.
+        q = multiprocessing.SimpleQueue()
+        q.close()  # close the pipe
+        with self.assertRaisesRegex(OSError, 'is closed'):
+            q.empty()
+
     def test_empty(self):
         queue = multiprocessing.SimpleQueue()
         child_can_start = multiprocessing.Event()
@@ -6037,6 +6359,99 @@ class TestNamedResource(unittest.TestCase):
         self.assertFalse(err, msg=err.decode('utf-8'))
 
 
+class _TestAtExit(BaseTestCase):
+
+    ALLOWED_TYPES = ('processes',)
+
+    @classmethod
+    def _write_file_at_exit(self, output_path):
+        import atexit
+        def exit_handler():
+            with open(output_path, 'w') as f:
+                f.write("deadbeef")
+        atexit.register(exit_handler)
+
+    def test_atexit(self):
+        # gh-83856
+        with os_helper.temp_dir() as temp_dir:
+            output_path = os.path.join(temp_dir, 'output.txt')
+            p = self.Process(target=self._write_file_at_exit, args=(output_path,))
+            p.start()
+            p.join()
+            with open(output_path) as f:
+                self.assertEqual(f.read(), 'deadbeef')
+
+
+class _TestSpawnedSysPath(BaseTestCase):
+    """Test that sys.path is setup in forkserver and spawn processes."""
+
+    ALLOWED_TYPES = {'processes'}
+    # Not applicable to fork which inherits everything from the process as is.
+    START_METHODS = {"forkserver", "spawn"}
+
+    def setUp(self):
+        self._orig_sys_path = list(sys.path)
+        self._temp_dir = tempfile.mkdtemp(prefix="test_sys_path-")
+        self._mod_name = "unique_test_mod"
+        module_path = os.path.join(self._temp_dir, f"{self._mod_name}.py")
+        with open(module_path, "w", encoding="utf-8") as mod:
+            mod.write("# A simple test module\n")
+        sys.path[:] = [p for p in sys.path if p]  # remove any existing ""s
+        sys.path.insert(0, self._temp_dir)
+        sys.path.insert(0, "")  # Replaced with an abspath in child.
+        self.assertIn(self.start_method, self.START_METHODS)
+        self._ctx = multiprocessing.get_context(self.start_method)
+
+    def tearDown(self):
+        sys.path[:] = self._orig_sys_path
+        shutil.rmtree(self._temp_dir, ignore_errors=True)
+
+    @staticmethod
+    def enq_imported_module_names(queue):
+        queue.put(tuple(sys.modules))
+
+    def test_forkserver_preload_imports_sys_path(self):
+        if self._ctx.get_start_method() != "forkserver":
+            self.skipTest("forkserver specific test.")
+        self.assertNotIn(self._mod_name, sys.modules)
+        multiprocessing.forkserver._forkserver._stop()  # Must be fresh.
+        self._ctx.set_forkserver_preload(
+            ["test.test_multiprocessing_forkserver", self._mod_name])
+        q = self._ctx.Queue()
+        proc = self._ctx.Process(
+                target=self.enq_imported_module_names, args=(q,))
+        proc.start()
+        proc.join()
+        child_imported_modules = q.get()
+        q.close()
+        self.assertIn(self._mod_name, child_imported_modules)
+
+    @staticmethod
+    def enq_sys_path_and_import(queue, mod_name):
+        queue.put(sys.path)
+        try:
+            importlib.import_module(mod_name)
+        except ImportError as exc:
+            queue.put(exc)
+        else:
+            queue.put(None)
+
+    def test_child_sys_path(self):
+        q = self._ctx.Queue()
+        proc = self._ctx.Process(
+                target=self.enq_sys_path_and_import, args=(q, self._mod_name))
+        proc.start()
+        proc.join()
+        child_sys_path = q.get()
+        import_error = q.get()
+        q.close()
+        self.assertNotIn("", child_sys_path)  # replaced by an abspath
+        self.assertIn(self._temp_dir, child_sys_path)  # our addition
+        # ignore the first element, it is the absolute "" replacement
+        self.assertEqual(child_sys_path[1:], sys.path[1:])
+        self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}")
+
+
 class MiscTestCase(unittest.TestCase):
     def test__all__(self):
         # Just make sure names in not_exported are excluded
@@ -6061,6 +6476,46 @@ class MiscTestCase(unittest.TestCase):
         self.assertEqual(rc, 0)
         self.assertFalse(err, msg=err.decode('utf-8'))
 
+    def test_large_pool(self):
+        #
+        # gh-89240: Check that large pools are always okay
+        #
+        testfn = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, testfn)
+        with open(testfn, 'w', encoding='utf-8') as f:
+            f.write(textwrap.dedent('''\
+                import multiprocessing
+                def f(x): return x*x
+                if __name__ == '__main__':
+                    with multiprocessing.Pool(200) as p:
+                        print(sum(p.map(f, range(1000))))
+            '''))
+        rc, out, err = script_helper.assert_python_ok(testfn)
+        self.assertEqual("332833500", out.decode('utf-8').strip())
+        self.assertFalse(err, msg=err.decode('utf-8'))
+
+    def test_forked_thread_not_started(self):
+        # gh-134381: Ensure that a thread that has not been started yet in
+        # the parent process can be started within a forked child process.
+
+        if multiprocessing.get_start_method() != "fork":
+            self.skipTest("fork specific test")
+
+        q = multiprocessing.Queue()
+        t = threading.Thread(target=lambda: q.put("done"), daemon=True)
+
+        def child():
+            t.start()
+            t.join()
+
+        p = multiprocessing.Process(target=child)
+        p.start()
+        p.join(support.SHORT_TIMEOUT)
+
+        self.assertEqual(p.exitcode, 0)
+        self.assertEqual(q.get_nowait(), "done")
+        close_queue(q)
+
 
 #
 # Mixins
@@ -6213,6 +6668,8 @@ def install_tests_in_module_dict(remote_globs, start_method,
             if base is BaseTestCase:
                 continue
             assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES
+            if base.START_METHODS and start_method not in base.START_METHODS:
+                continue  # class not intended for this start method.
             for type_ in base.ALLOWED_TYPES:
                 if only_type and type_ != only_type:
                     continue
@@ -6226,6 +6683,7 @@ def install_tests_in_module_dict(remote_globs, start_method,
                     Temp = hashlib_helper.requires_hashdigest('sha256')(Temp)
                 Temp.__name__ = Temp.__qualname__ = newname
                 Temp.__module__ = __module__
+                Temp.start_method = start_method
                 remote_globs[newname] = Temp
         elif issubclass(base, unittest.TestCase):
             if only_type:

Lib/test/_test_venv_multiprocessing.py

@@ -0,0 +1,40 @@
+import multiprocessing
+import random
+import sys
+
+def fill_queue(queue, code):
+    queue.put(code)
+
+
+def drain_queue(queue, code):
+    if code != queue.get():
+        sys.exit(1)
+
+
+def test_func():
+    code = random.randrange(0, 1000)
+    queue = multiprocessing.Queue()
+    fill_pool = multiprocessing.Process(
+        target=fill_queue,
+        args=(queue, code)
+    )
+    drain_pool = multiprocessing.Process(
+        target=drain_queue,
+        args=(queue, code)
+    )
+    drain_pool.start()
+    fill_pool.start()
+    fill_pool.join()
+    drain_pool.join()
+
+
+def main():
+    multiprocessing.set_start_method('spawn')
+    test_pool = multiprocessing.Process(target=test_func)
+    test_pool.start()
+    test_pool.join()
+    sys.exit(test_pool.exitcode)
+
+
+if __name__ == "__main__":
+    main()

Lib/test/_typed_dict_helper.py

@@ -1,18 +0,0 @@
-"""Used to test `get_type_hints()` on a cross-module inherited `TypedDict` class
-
-This script uses future annotations to postpone a type that won't be available
-on the module inheriting from to `Foo`. The subclass in the other module should
-look something like this:
-
-    class Bar(_typed_dict_helper.Foo, total=False):
-        b: int
-"""
-
-from __future__ import annotations
-
-from typing import Optional, TypedDict
-
-OptionalIntType = Optional[int]
-
-class Foo(TypedDict):
-    a: OptionalIntType

Lib/test/ann_module.py

@@ -1,62 +0,0 @@
-
-
-"""
-The module for testing variable annotations.
-Empty lines above are for good reason (testing for correct line numbers)
-"""
-
-from typing import Optional
-from functools import wraps
-
-__annotations__[1] = 2
-
-class C:
-
-    x = 5; y: Optional['C'] = None
-
-from typing import Tuple
-x: int = 5; y: str = x; f: Tuple[int, int]
-
-class M(type):
-
-    __annotations__['123'] = 123
-    o: type = object
-
-(pars): bool = True
-
-class D(C):
-    j: str = 'hi'; k: str= 'bye'
-
-from types import new_class
-h_class = new_class('H', (C,))
-j_class = new_class('J')
-
-class F():
-    z: int = 5
-    def __init__(self, x):
-        pass
-
-class Y(F):
-    def __init__(self):
-        super(F, self).__init__(123)
-
-class Meta(type):
-    def __new__(meta, name, bases, namespace):
-        return super().__new__(meta, name, bases, namespace)
-
-class S(metaclass = Meta):
-    x: str = 'something'
-    y: str = 'something else'
-
-def foo(x: int = 10):
-    def bar(y: List[str]):
-        x: str = 'yes'
-    bar()
-
-def dec(func):
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-    return wrapper
-
-u: int | float

Lib/test/ann_module2.py

@@ -1,36 +0,0 @@
-"""
-Some correct syntax for variable annotation here.
-More examples are in test_grammar and test_parser.
-"""
-
-from typing import no_type_check, ClassVar
-
-i: int = 1
-j: int
-x: float = i/10
-
-def f():
-    class C: ...
-    return C()
-
-f().new_attr: object = object()
-
-class C:
-    def __init__(self, x: int) -> None:
-        self.x = x
-
-c = C(5)
-c.new_attr: int = 10
-
-__annotations__ = {}
-
-
-@no_type_check
-class NTC:
-    def meth(self, param: complex) -> None:
-        ...
-
-class CV:
-    var: ClassVar['CV']
-
-CV.var = CV()

Lib/test/ann_module3.py

@@ -1,18 +0,0 @@
-"""
-Correct syntax for variable annotation that should fail at runtime
-in a certain manner. More examples are in test_grammar and test_parser.
-"""
-
-def f_bad_ann():
-    __annotations__[1] = 2
-
-class C_OK:
-    def __init__(self, x: int) -> None:
-        self.x: no_such_name = x  # This one is OK as proposed by Guido
-
-class D_bad_ann:
-    def __init__(self, x: int) -> None:
-        sfel.y: int = 0
-
-def g_bad_ann():
-    no_such_name.attr: int = 0

Lib/test/ann_module4.py

@@ -1,5 +0,0 @@
-# This ann_module isn't for test_typing,
-# it's for test_module
-
-a:int=3
-b:str=4

Lib/test/ann_module5.py

@@ -1,10 +0,0 @@
-# Used by test_typing to verify that Final wrapped in ForwardRef works.
-
-from __future__ import annotations
-
-from typing import Final
-
-name: Final[str] = "final"
-
-class MyClass:
-    value: Final = 3000

Lib/test/ann_module6.py

@@ -1,7 +0,0 @@
-# Tests that top-level ClassVar is not allowed
-
-from __future__ import annotations
-
-from typing import ClassVar
-
-wrong: ClassVar[int] = 1

Lib/test/ann_module7.py

@@ -1,11 +0,0 @@
-# Tests class have ``__text_signature__``
-
-from __future__ import annotations
-
-DEFAULT_BUFFER_SIZE = 8192
-
-class BufferedReader(object):
-    """BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE)\n--\n\n
-    Create a new buffered reader using the given readable raw IO object.
-    """
-    pass

Lib/test/archivetestdata/README.md

@@ -0,0 +1,36 @@
+# Test data for `test_zipfile`, `test_tarfile` (and even some others)
+
+## `test_zipfile`
+
+The test executables in this directory are created manually from `header.sh` and
+the `testdata_module_inside_zip.py` file.  You must have Info-ZIP's zip utility
+installed (`apt install zip` on Debian).
+
+### Purpose of `exe_with_zip` and `exe_with_z64`
+
+These are used to test executable files with an appended zipfile, in a scenario
+where the executable is _not_ a Python interpreter itself so our automatic
+zipimport machinery (that'd look for `__main__.py`) is not being used.
+
+### Updating the test executables
+
+If you update header.sh or the testdata_module_inside_zip.py file, rerun the
+commands below.  These are expected to be rarely changed, if ever.
+
+#### Standard old format (2.0) zip file
+
+```
+zip -0 zip2.zip testdata_module_inside_zip.py
+cat header.sh zip2.zip >exe_with_zip
+rm zip2.zip
+```
+
+#### Modern format (4.5) zip64 file
+
+Redirecting from stdin forces Info-ZIP's zip tool to create a zip64.
+
+```
+zip -0 <testdata_module_inside_zip.py >zip64.zip
+cat header.sh zip64.zip >exe_with_z64
+rm zip64.zip
+```

Lib/test/archivetestdata/header.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+INTERPRETER_UNDER_TEST="$1"
+if [[ ! -x "${INTERPRETER_UNDER_TEST}" ]]; then
+    echo "Interpreter must be the command line argument."
+    exit 4
+fi
+EXECUTABLE="$0" exec "${INTERPRETER_UNDER_TEST}" -E - <<END_OF_PYTHON
+import os
+import zipfile
+
+namespace = {}
+
+filename = os.environ['EXECUTABLE']
+print(f'Opening {filename} as a zipfile.')
+with zipfile.ZipFile(filename, mode='r') as exe_zip:
+  for file_info in exe_zip.infolist():
+    data = exe_zip.read(file_info)
+    exec(data, namespace, namespace)
+    break  # Only use the first file in the archive.
+
+print('Favorite number in executable:', namespace["FAVORITE_NUMBER"])
+
+### Archive contents will be appended after this file. ###
+END_OF_PYTHON

Lib/test/archivetestdata/testdata_module_inside_zip.py

@@ -0,0 +1,2 @@
+# Test data file to be stored within a zip file.
+FAVORITE_NUMBER = 5

Lib/test/decimaltestdata/abs.decTest

@@ -20,7 +20,7 @@
 version: 2.59
 
 -- This set of tests primarily tests the existence of the operator.
--- Additon, subtraction, rounding, and more overflows are tested
+-- Addition, subtraction, rounding, and more overflows are tested
 -- elsewhere.
 
 precision:   9

Lib/test/decimaltestdata/ddFMA.decTest

@@ -1663,7 +1663,7 @@ ddfma375087 fma  1  12345678 1E-33                   -> 12345678.00000001 Inexac
 ddfma375088 fma  1  12345678 1E-34                   -> 12345678.00000001 Inexact Rounded
 ddfma375089 fma  1  12345678 1E-35                   -> 12345678.00000001 Inexact Rounded
 
--- desctructive subtraction (from remainder tests)
+-- destructive subtraction (from remainder tests)
 
 -- +++ some of these will be off-by-one remainder vs remainderNear
 

Lib/test/decimaltestdata/ddQuantize.decTest

@@ -462,7 +462,7 @@ ddqua520 quantize   1.234    1e359 -> 0E+359 Inexact Rounded
 ddqua521 quantize 123.456    1e359 -> 0E+359 Inexact Rounded
 ddqua522 quantize   1.234    1e359 -> 0E+359 Inexact Rounded
 ddqua523 quantize 123.456    1e359 -> 0E+359 Inexact Rounded
--- next four are "won't fit" overfl
+-- next four are "won't fit" overflow
 ddqua526 quantize   1.234   1e-299 -> NaN Invalid_operation
 ddqua527 quantize 123.456   1e-299 -> NaN Invalid_operation
 ddqua528 quantize   1.234   1e-299 -> NaN Invalid_operation

Lib/test/decimaltestdata/ddRemainder.decTest

@@ -422,7 +422,7 @@ ddrem757 remainder  1    sNaN   ->  NaN  Invalid_operation
 ddrem758 remainder  1000 sNaN   ->  NaN  Invalid_operation
 ddrem759 remainder  Inf -sNaN   -> -NaN  Invalid_operation
 
--- propaging NaNs
+-- propagating NaNs
 ddrem760 remainder  NaN1   NaN7   ->  NaN1
 ddrem761 remainder sNaN2   NaN8   ->  NaN2 Invalid_operation
 ddrem762 remainder  NaN3  sNaN9   ->  NaN9 Invalid_operation

Lib/test/decimaltestdata/ddRemainderNear.decTest

@@ -450,7 +450,7 @@ ddrmn757 remaindernear  1    sNaN   ->  NaN  Invalid_operation
 ddrmn758 remaindernear  1000 sNaN   ->  NaN  Invalid_operation
 ddrmn759 remaindernear  Inf -sNaN   -> -NaN  Invalid_operation
 
--- propaging NaNs
+-- propagating NaNs
 ddrmn760 remaindernear  NaN1   NaN7   ->  NaN1
 ddrmn761 remaindernear sNaN2   NaN8   ->  NaN2 Invalid_operation
 ddrmn762 remaindernear  NaN3  sNaN9   ->  NaN9 Invalid_operation

Lib/test/decimaltestdata/dqRemainder.decTest

@@ -418,7 +418,7 @@ dqrem757 remainder  1    sNaN   ->  NaN  Invalid_operation
 dqrem758 remainder  1000 sNaN   ->  NaN  Invalid_operation
 dqrem759 remainder  Inf -sNaN   -> -NaN  Invalid_operation
 
--- propaging NaNs
+-- propagating NaNs
 dqrem760 remainder  NaN1   NaN7   ->  NaN1
 dqrem761 remainder sNaN2   NaN8   ->  NaN2 Invalid_operation
 dqrem762 remainder  NaN3  sNaN9   ->  NaN9 Invalid_operation

Lib/test/decimaltestdata/dqRemainderNear.decTest

@@ -450,7 +450,7 @@ dqrmn757 remaindernear  1    sNaN   ->  NaN  Invalid_operation
 dqrmn758 remaindernear  1000 sNaN   ->  NaN  Invalid_operation
 dqrmn759 remaindernear  Inf -sNaN   -> -NaN  Invalid_operation
 
--- propaging NaNs
+-- propagating NaNs
 dqrmn760 remaindernear  NaN1   NaN7   ->  NaN1
 dqrmn761 remaindernear sNaN2   NaN8   ->  NaN2 Invalid_operation
 dqrmn762 remaindernear  NaN3  sNaN9   ->  NaN9 Invalid_operation

Lib/test/decimaltestdata/exp.decTest

@@ -28,7 +28,7 @@ rounding:    half_even
 maxExponent: 384
 minexponent: -383
 
--- basics (examples in specificiation, etc.)
+-- basics (examples in specification, etc.)
 expx001 exp  -Infinity     -> 0
 expx002 exp  -10           -> 0.0000453999298 Inexact Rounded
 expx003 exp  -1            -> 0.367879441 Inexact Rounded

Lib/test/decimaltestdata/extra.decTest

@@ -156,7 +156,7 @@ extr1302 fma -Inf 0E-456 sNaN148 -> NaN Invalid_operation
 
 -- max/min/max_mag/min_mag bug in 2.5.2/2.6/3.0: max(NaN, finite) gave
 -- incorrect answers when the finite number required rounding; similarly
--- for the other thre functions
+-- for the other three functions
 maxexponent: 999
 minexponent: -999
 precision: 6

Lib/test/decimaltestdata/remainder.decTest

@@ -435,7 +435,7 @@ remx757 remainder  1    sNaN   ->  NaN  Invalid_operation
 remx758 remainder  1000 sNaN   ->  NaN  Invalid_operation
 remx759 remainder  Inf -sNaN   -> -NaN  Invalid_operation
 
--- propaging NaNs
+-- propagating NaNs
 remx760 remainder  NaN1   NaN7   ->  NaN1
 remx761 remainder sNaN2   NaN8   ->  NaN2 Invalid_operation
 remx762 remainder  NaN3  sNaN9   ->  NaN9 Invalid_operation

Lib/test/decimaltestdata/remainderNear.decTest

@@ -498,7 +498,7 @@ rmnx758 remaindernear  1000 sNaN   ->  NaN  Invalid_operation
 rmnx759 remaindernear  Inf  sNaN   ->  NaN  Invalid_operation
 rmnx760 remaindernear  NaN  sNaN   ->  NaN  Invalid_operation
 
--- propaging NaNs
+-- propagating NaNs
 rmnx761 remaindernear  NaN1   NaN7   ->  NaN1
 rmnx762 remaindernear sNaN2   NaN8   ->  NaN2 Invalid_operation
 rmnx763 remaindernear  NaN3 -sNaN9   -> -NaN9 Invalid_operation

Lib/test/fork_wait.py

@@ -0,0 +1,80 @@
+"""This test case provides support for checking forking and wait behavior.
+
+To test different wait behavior, override the wait_impl method.
+
+We want fork1() semantics -- only the forking thread survives in the
+child after a fork().
+
+On some systems (e.g. Solaris without posix threads) we find that all
+active threads survive in the child after a fork(); this is an error.
+"""
+
+import os, time, unittest
+import threading
+from test import support
+from test.support import threading_helper
+import warnings
+
+
+LONGSLEEP = 2
+SHORTSLEEP = 0.5
+NUM_THREADS = 4
+
+class ForkWait(unittest.TestCase):
+
+    def setUp(self):
+        self._threading_key = threading_helper.threading_setup()
+        self.alive = {}
+        self.stop = 0
+        self.threads = []
+
+    def tearDown(self):
+        # Stop threads
+        self.stop = 1
+        for thread in self.threads:
+            thread.join()
+        thread = None
+        self.threads.clear()
+        threading_helper.threading_cleanup(*self._threading_key)
+
+    def f(self, id):
+        while not self.stop:
+            self.alive[id] = os.getpid()
+            try:
+                time.sleep(SHORTSLEEP)
+            except OSError:
+                pass
+
+    def wait_impl(self, cpid, *, exitcode):
+        support.wait_process(cpid, exitcode=exitcode)
+
+    def test_wait(self):
+        for i in range(NUM_THREADS):
+            thread = threading.Thread(target=self.f, args=(i,))
+            thread.start()
+            self.threads.append(thread)
+
+        # busy-loop to wait for threads
+        for _ in support.sleeping_retry(support.SHORT_TIMEOUT):
+            if len(self.alive) >= NUM_THREADS:
+                break
+
+        a = sorted(self.alive.keys())
+        self.assertEqual(a, list(range(NUM_THREADS)))
+
+        prefork_lives = self.alive.copy()
+
+        # Ignore the warning about fork with threads.
+        with warnings.catch_warnings(category=DeprecationWarning,
+                                     action="ignore"):
+            if (cpid := os.fork()) == 0:
+                # Child
+                time.sleep(LONGSLEEP)
+                n = 0
+                for key in self.alive:
+                    if self.alive[key] != prefork_lives[key]:
+                        n += 1
+                os._exit(n)
+            else:
+                # Parent
+                self.wait_impl(cpid, exitcode=0)

Lib/test/mod_generics_cache.py

@@ -1,53 +0,0 @@
-"""Module for testing the behavior of generics across different modules."""
-
-import sys
-from textwrap import dedent
-from typing import TypeVar, Generic, Optional
-
-
-if sys.version_info[:2] >= (3, 6):
-    exec(dedent("""
-    default_a: Optional['A'] = None
-    default_b: Optional['B'] = None
-
-    T = TypeVar('T')
-
-
-    class A(Generic[T]):
-        some_b: 'B'
-
-
-    class B(Generic[T]):
-        class A(Generic[T]):
-            pass
-
-        my_inner_a1: 'B.A'
-        my_inner_a2: A
-        my_outer_a: 'A'  # unless somebody calls get_type_hints with localns=B.__dict__
-    """))
-else:  # This should stay in sync with the syntax above.
-    __annotations__ = dict(
-        default_a=Optional['A'],
-        default_b=Optional['B'],
-    )
-    default_a = None
-    default_b = None
-
-    T = TypeVar('T')
-
-
-    class A(Generic[T]):
-        __annotations__ = dict(
-            some_b='B'
-        )
-
-
-    class B(Generic[T]):
-        class A(Generic[T]):
-            pass
-
-        __annotations__ = dict(
-            my_inner_a1='B.A',
-            my_inner_a2=A,
-            my_outer_a='A'  # unless somebody calls get_type_hints with localns=B.__dict__
-        )

Lib/test/pyclbr_input.py

@@ -0,0 +1,85 @@
+"""Test cases for test_pyclbr.py"""
+
+def f(): pass
+
+class Other(object):
+    @classmethod
+    def foo(c): pass
+
+    def om(self): pass
+
+class B (object):
+    def bm(self): pass
+
+class C (B):
+    d = 10
+
+    # This one is correctly considered by both test_pyclbr.py and pyclbr.py
+    # as a non-method of C.
+    foo = Other().foo
+
+    # This causes test_pyclbr.py to fail, but only because the
+    # introspection-based is_method() code in the test can't
+    # distinguish between this and a genuine method function like m().
+    #
+    # The pyclbr.py module gets this right as it parses the text.
+    om = Other.om
+    f = f
+
+    def m(self): pass
+
+    @staticmethod
+    def sm(self): pass
+
+    @classmethod
+    def cm(self): pass
+
+# Check that mangling is correctly handled
+
+class a:
+    def a(self): pass
+    def _(self): pass
+    def _a(self): pass
+    def __(self): pass
+    def ___(self): pass
+    def __a(self): pass
+
+class _:
+    def a(self): pass
+    def _(self): pass
+    def _a(self): pass
+    def __(self): pass
+    def ___(self): pass
+    def __a(self): pass
+
+class __:
+    def a(self): pass
+    def _(self): pass
+    def _a(self): pass
+    def __(self): pass
+    def ___(self): pass
+    def __a(self): pass
+
+class ___:
+    def a(self): pass
+    def _(self): pass
+    def _a(self): pass
+    def __(self): pass
+    def ___(self): pass
+    def __a(self): pass
+
+class _a:
+    def a(self): pass
+    def _(self): pass
+    def _a(self): pass
+    def __(self): pass
+    def ___(self): pass
+    def __a(self): pass
+
+class __a:
+    def a(self): pass
+    def _(self): pass
+    def _a(self): pass
+    def __(self): pass
+    def ___(self): pass
+    def __a(self): pass

Lib/test/string_tests.py

@@ -8,18 +8,12 @@ from test.support import import_helper
 from collections import UserList
 import random
 
+
 class Sequence:
     def __init__(self, seq='wxyz'): self.seq = seq
     def __len__(self): return len(self.seq)
     def __getitem__(self, i): return self.seq[i]
 
-class BadSeq1(Sequence):
-    def __init__(self): self.seq = [7, 'hello', 123]
-    def __str__(self): return '{0} {1} {2}'.format(*self.seq)
-
-class BadSeq2(Sequence):
-    def __init__(self): self.seq = ['a', 'b', 'c']
-    def __len__(self): return 8
 
 class BaseTest:
     # These tests are for buffers of values (bytes) and not
@@ -27,7 +21,7 @@ class BaseTest:
     # and various string implementations
 
     # The type to be tested
-    # Change in subclasses to change the behaviour of fixtesttype()
+    # Change in subclasses to change the behaviour of fixtype()
     type2test = None
 
     # Whether the "contained items" of the container are integers in
@@ -36,7 +30,7 @@ class BaseTest:
     contains_bytes = False
 
     # All tests pass their arguments to the testing methods
-    # as str objects. fixtesttype() can be used to propagate
+    # as str objects. fixtype() can be used to propagate
     # these arguments to the appropriate type
     def fixtype(self, obj):
         if isinstance(obj, str):
@@ -160,6 +154,12 @@ class BaseTest:
                     self.assertEqual(rem, 0, '%s != 0 for %s' % (rem, i))
                     self.assertEqual(r1, r2, '%s != %s for %s' % (r1, r2, i))
 
+    def test_count_keyword(self):
+        self.assertEqual('aa'.replace('a', 'b', 0), 'aa'.replace('a', 'b', count=0))
+        self.assertEqual('aa'.replace('a', 'b', 1), 'aa'.replace('a', 'b', count=1))
+        self.assertEqual('aa'.replace('a', 'b', 2), 'aa'.replace('a', 'b', count=2))
+        self.assertEqual('aa'.replace('a', 'b', 3), 'aa'.replace('a', 'b', count=3))
+
     def test_find(self):
         self.checkequal(0, 'abcdefghiabc', 'find', 'abc')
         self.checkequal(9, 'abcdefghiabc', 'find', 'abc', 1)
@@ -327,11 +327,12 @@ class BaseTest:
             for i in range(len(s)):
                 if s.startswith(p, i):
                     return i
+            if p == '' and s == '':
+                return 0
             return -1
 
-        rr = random.randrange
-        choices = random.choices
-        for _ in range(1000):
+        def check_pattern(rr):
+            choices = random.choices
             p0 = ''.join(choices('abcde', k=rr(10))) * rr(10, 20)
             p = p0[:len(p0) - rr(10)] # pop off some characters
             left = ''.join(choices('abcdef', k=rr(2000)))
@@ -341,6 +342,49 @@ class BaseTest:
                 self.checkequal(reference_find(p, text),
                                 text, 'find', p)
 
+        rr = random.randrange
+        for _ in range(1000):
+            check_pattern(rr)
+
+        # Test that empty string always work:
+        check_pattern(lambda *args: 0)
+
+    def test_find_many_lengths(self):
+        haystack_repeats = [a * 10**e for e in range(6) for a in (1,2,5)]
+        haystacks = [(n, self.fixtype("abcab"*n + "da")) for n in haystack_repeats]
+
+        needle_repeats = [a * 10**e for e in range(6) for a in (1, 3)]
+        needles = [(m, self.fixtype("abcab"*m + "da")) for m in needle_repeats]
+
+        for n, haystack1 in haystacks:
+            haystack2 = haystack1[:-1]
+            for m, needle in needles:
+                answer1 = 5 * (n - m) if m <= n else -1
+                self.assertEqual(haystack1.find(needle), answer1, msg=(n,m))
+                self.assertEqual(haystack2.find(needle), -1, msg=(n,m))
+
+    def test_adaptive_find(self):
+        # This would be very slow for the naive algorithm,
+        # but str.find() should be O(n + m).
+        for N in 1000, 10_000, 100_000, 1_000_000:
+            A, B = 'a' * N, 'b' * N
+            haystack = A + A + B + A + A
+            needle = A + B + B + A
+            self.checkequal(-1, haystack, 'find', needle)
+            self.checkequal(0, haystack, 'count', needle)
+            self.checkequal(len(haystack), haystack + needle, 'find', needle)
+            self.checkequal(1, haystack + needle, 'count', needle)
+
+    def test_find_with_memory(self):
+        # Test the "Skip with memory" path in the two-way algorithm.
+        for N in 1000, 3000, 10_000, 30_000:
+            needle = 'ab' * N
+            haystack = ('ab'*(N-1) + 'b') * 2
+            self.checkequal(-1, haystack, 'find', needle)
+            self.checkequal(0, haystack, 'count', needle)
+            self.checkequal(len(haystack), haystack + needle, 'find', needle)
+            self.checkequal(1, haystack + needle, 'count', needle)
+
     def test_find_shift_table_overflow(self):
         """When the table of 8-bit shifts overflows."""
         N = 2**8 + 100
@@ -724,6 +768,18 @@ class BaseTest:
         self.checkraises(TypeError, 'hello', 'replace', 42, 'h')
         self.checkraises(TypeError, 'hello', 'replace', 'h', 42)
 
+    def test_replace_uses_two_way_maxcount(self):
+        # Test that maxcount works in _two_way_count in fastsearch.h
+        A, B = "A"*1000, "B"*1000
+        AABAA = A + A + B + A + A
+        ABBA = A + B + B + A
+        self.checkequal(AABAA + ABBA,
+                        AABAA + ABBA, 'replace', ABBA, "ccc", 0)
+        self.checkequal(AABAA + "ccc",
+                        AABAA + ABBA, 'replace', ABBA, "ccc", 1)
+        self.checkequal(AABAA + "ccc",
+                        AABAA + ABBA, 'replace', ABBA, "ccc", 2)
+
     @unittest.skip("TODO: RUSTPYTHON, may only apply to 32-bit platforms")
     @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4,
                      'only applies to 32-bit platforms')
@@ -734,8 +790,6 @@ class BaseTest:
         self.checkraises(OverflowError, A2_16, "replace", "A", A2_16)
         self.checkraises(OverflowError, A2_16, "replace", "AA", A2_16+A2_16)
 
-
-    # Python 3.9
     def test_removeprefix(self):
         self.checkequal('am', 'spam', 'removeprefix', 'sp')
         self.checkequal('spamspam', 'spamspamspam', 'removeprefix', 'spam')
@@ -754,7 +808,6 @@ class BaseTest:
         self.checkraises(TypeError, 'hello', 'removeprefix', 'h', 42)
         self.checkraises(TypeError, 'hello', 'removeprefix', ("he", "l"))
 
-    # Python 3.9
     def test_removesuffix(self):
         self.checkequal('sp', 'spam', 'removesuffix', 'am')
         self.checkequal('spamspam', 'spamspamspam', 'removesuffix', 'spam')
@@ -1053,7 +1106,7 @@ class BaseTest:
         self.checkraises(TypeError, 'abc', 'splitlines', 42, 42)
 
 
-class CommonTest(BaseTest):
+class StringLikeTest(BaseTest):
     # This testcase contains tests that can be used in all
     # stringlike classes. Currently this is str and UserString.
 
@@ -1084,11 +1137,6 @@ class CommonTest(BaseTest):
         self.checkequal('\u019b\u1d00\u1d86\u0221\u1fb7',
                         '\u019b\u1d00\u1d86\u0221\u1fb7', 'capitalize')
 
-
-class MixinStrUnicodeUserStringTest:
-    # additional tests that only work for
-    # stringlike objects, i.e. str, UserString
-
     def test_startswith(self):
         self.checkequal(True, 'hello', 'startswith', 'he')
         self.checkequal(True, 'hello', 'startswith', 'hello')
@@ -1273,8 +1321,11 @@ class MixinStrUnicodeUserStringTest:
             self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join',
                  ('a' * i,) * i)
 
-        #self.checkequal(str(BadSeq1()), ' ', 'join', BadSeq1())
-        self.checkequal('a b c', ' ', 'join', BadSeq2())
+        class LiesAboutLengthSeq(Sequence):
+            def __init__(self): self.seq = ['a', 'b', 'c']
+            def __len__(self): return 8
+
+        self.checkequal('a b c', ' ', 'join', LiesAboutLengthSeq())
 
         self.checkraises(TypeError, ' ', 'join')
         self.checkraises(TypeError, ' ', 'join', None)
@@ -1459,19 +1510,19 @@ class MixinStrUnicodeUserStringTest:
         # issue 11828
         s = 'hello'
         x = 'x'
-        self.assertRaisesRegex(TypeError, r'^find\(', s.find,
+        self.assertRaisesRegex(TypeError, r'^find\b', s.find,
                                 x, None, None, None)
-        self.assertRaisesRegex(TypeError, r'^rfind\(', s.rfind,
+        self.assertRaisesRegex(TypeError, r'^rfind\b', s.rfind,
                                 x, None, None, None)
-        self.assertRaisesRegex(TypeError, r'^index\(', s.index,
+        self.assertRaisesRegex(TypeError, r'^index\b', s.index,
                                 x, None, None, None)
-        self.assertRaisesRegex(TypeError, r'^rindex\(', s.rindex,
+        self.assertRaisesRegex(TypeError, r'^rindex\b', s.rindex,
                                 x, None, None, None)
-        self.assertRaisesRegex(TypeError, r'^count\(', s.count,
+        self.assertRaisesRegex(TypeError, r'^count\b', s.count,
                                 x, None, None, None)
-        self.assertRaisesRegex(TypeError, r'^startswith\(', s.startswith,
+        self.assertRaisesRegex(TypeError, r'^startswith\b', s.startswith,
                                 x, None, None, None)
-        self.assertRaisesRegex(TypeError, r'^endswith\(', s.endswith,
+        self.assertRaisesRegex(TypeError, r'^endswith\b', s.endswith,
                                 x, None, None, None)
 
         # issue #15534

Lib/test/support/hypothesis_helper.py

@@ -5,13 +5,6 @@ try:
 except ImportError:
     from . import _hypothesis_stubs as hypothesis
 else:
-    # Regrtest changes to use a tempdir as the working directory, so we have
-    # to tell Hypothesis to use the original in order to persist the database.
-    from .os_helper import SAVEDCWD
-    from hypothesis.configuration import set_hypothesis_home_dir
-
-    set_hypothesis_home_dir(os.path.join(SAVEDCWD, ".hypothesis"))
-
     # When using the real Hypothesis, we'll configure it to ignore occasional
     # slow tests (avoiding flakiness from random VM slowness in CI).
     hypothesis.settings.register_profile(

Lib/test/support/interpreters/__init__.py

@@ -0,0 +1,258 @@
+"""Subinterpreters High Level Module."""
+
+import threading
+import weakref
+import _interpreters
+
+# aliases:
+from _interpreters import (
+    InterpreterError, InterpreterNotFoundError, NotShareableError,
+    is_shareable,
+)
+
+
+__all__ = [
+    'get_current', 'get_main', 'create', 'list_all', 'is_shareable',
+    'Interpreter',
+    'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed',
+    'NotShareableError',
+    'create_queue', 'Queue', 'QueueEmpty', 'QueueFull',
+]
+
+
+_queuemod = None
+
+def __getattr__(name):
+    if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'):
+        global create_queue, Queue, QueueEmpty, QueueFull
+        ns = globals()
+        from .queues import (
+            create as create_queue,
+            Queue, QueueEmpty, QueueFull,
+        )
+        return ns[name]
+    else:
+        raise AttributeError(name)
+
+
+_EXEC_FAILURE_STR = """
+{superstr}
+
+Uncaught in the interpreter:
+
+{formatted}
+""".strip()
+
+class ExecutionFailed(InterpreterError):
+    """An unhandled exception happened during execution.
+
+    This is raised from Interpreter.exec() and Interpreter.call().
+    """
+
+    def __init__(self, excinfo):
+        msg = excinfo.formatted
+        if not msg:
+            if excinfo.type and excinfo.msg:
+                msg = f'{excinfo.type.__name__}: {excinfo.msg}'
+            else:
+                msg = excinfo.type.__name__ or excinfo.msg
+        super().__init__(msg)
+        self.excinfo = excinfo
+
+    def __str__(self):
+        try:
+            formatted = self.excinfo.errdisplay
+        except Exception:
+            return super().__str__()
+        else:
+            return _EXEC_FAILURE_STR.format(
+                superstr=super().__str__(),
+                formatted=formatted,
+            )
+
+
+def create():
+    """Return a new (idle) Python interpreter."""
+    id = _interpreters.create(reqrefs=True)
+    return Interpreter(id, _ownsref=True)
+
+
+def list_all():
+    """Return all existing interpreters."""
+    return [Interpreter(id, _whence=whence)
+            for id, whence in _interpreters.list_all(require_ready=True)]
+
+
+def get_current():
+    """Return the currently running interpreter."""
+    id, whence = _interpreters.get_current()
+    return Interpreter(id, _whence=whence)
+
+
+def get_main():
+    """Return the main interpreter."""
+    id, whence = _interpreters.get_main()
+    assert whence == _interpreters.WHENCE_RUNTIME, repr(whence)
+    return Interpreter(id, _whence=whence)
+
+
+_known = weakref.WeakValueDictionary()
+
+class Interpreter:
+    """A single Python interpreter.
+
+    Attributes:
+
+    "id" - the unique process-global ID number for the interpreter
+    "whence" - indicates where the interpreter was created
+
+    If the interpreter wasn't created by this module
+    then any method that modifies the interpreter will fail,
+    i.e. .close(), .prepare_main(), .exec(), and .call()
+    """
+
+    _WHENCE_TO_STR = {
+       _interpreters.WHENCE_UNKNOWN: 'unknown',
+       _interpreters.WHENCE_RUNTIME: 'runtime init',
+       _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API',
+       _interpreters.WHENCE_CAPI: 'C-API',
+       _interpreters.WHENCE_XI: 'cross-interpreter C-API',
+       _interpreters.WHENCE_STDLIB: '_interpreters module',
+    }
+
+    def __new__(cls, id, /, _whence=None, _ownsref=None):
+        # There is only one instance for any given ID.
+        if not isinstance(id, int):
+            raise TypeError(f'id must be an int, got {id!r}')
+        id = int(id)
+        if _whence is None:
+            if _ownsref:
+                _whence = _interpreters.WHENCE_STDLIB
+            else:
+                _whence = _interpreters.whence(id)
+        assert _whence in cls._WHENCE_TO_STR, repr(_whence)
+        if _ownsref is None:
+            _ownsref = (_whence == _interpreters.WHENCE_STDLIB)
+        try:
+            self = _known[id]
+            assert hasattr(self, '_ownsref')
+        except KeyError:
+            self = super().__new__(cls)
+            _known[id] = self
+            self._id = id
+            self._whence = _whence
+            self._ownsref = _ownsref
+            if _ownsref:
+                # This may raise InterpreterNotFoundError:
+                _interpreters.incref(id)
+        return self
+
+    def __repr__(self):
+        return f'{type(self).__name__}({self.id})'
+
+    def __hash__(self):
+        return hash(self._id)
+
+    def __del__(self):
+        self._decref()
+
+    # for pickling:
+    def __getnewargs__(self):
+        return (self._id,)
+
+    # for pickling:
+    def __getstate__(self):
+        return None
+
+    def _decref(self):
+        if not self._ownsref:
+            return
+        self._ownsref = False
+        try:
+            _interpreters.decref(self._id)
+        except InterpreterNotFoundError:
+            pass
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def whence(self):
+        return self._WHENCE_TO_STR[self._whence]
+
+    def is_running(self):
+        """Return whether or not the identified interpreter is running."""
+        return _interpreters.is_running(self._id)
+
+    # Everything past here is available only to interpreters created by
+    # interpreters.create().
+
+    def close(self):
+        """Finalize and destroy the interpreter.
+
+        Attempting to destroy the current interpreter results
+        in an InterpreterError.
+        """
+        return _interpreters.destroy(self._id, restrict=True)
+
+    def prepare_main(self, ns=None, /, **kwargs):
+        """Bind the given values into the interpreter's __main__.
+
+        The values must be shareable.
+        """
+        ns = dict(ns, **kwargs) if ns is not None else kwargs
+        _interpreters.set___main___attrs(self._id, ns, restrict=True)
+
+    def exec(self, code, /):
+        """Run the given source code in the interpreter.
+
+        This is essentially the same as calling the builtin "exec"
+        with this interpreter, using the __dict__ of its __main__
+        module as both globals and locals.
+
+        There is no return value.
+
+        If the code raises an unhandled exception then an ExecutionFailed
+        exception is raised, which summarizes the unhandled exception.
+        The actual exception is discarded because objects cannot be
+        shared between interpreters.
+
+        This blocks the current Python thread until done.  During
+        that time, the previous interpreter is allowed to run
+        in other threads.
+        """
+        excinfo = _interpreters.exec(self._id, code, restrict=True)
+        if excinfo is not None:
+            raise ExecutionFailed(excinfo)
+
+    def call(self, callable, /):
+        """Call the object in the interpreter with given args/kwargs.
+
+        Only functions that take no arguments and have no closure
+        are supported.
+
+        The return value is discarded.
+
+        If the callable raises an exception then the error display
+        (including full traceback) is send back between the interpreters
+        and an ExecutionFailed exception is raised, much like what
+        happens with Interpreter.exec().
+        """
+        # XXX Support args and kwargs.
+        # XXX Support arbitrary callables.
+        # XXX Support returning the return value (e.g. via pickle).
+        excinfo = _interpreters.call(self._id, callable, restrict=True)
+        if excinfo is not None:
+            raise ExecutionFailed(excinfo)
+
+    def call_in_thread(self, callable, /):
+        """Return a new thread that calls the object in the interpreter.
+
+        The return value and any raised exception are discarded.
+        """
+        def task():
+            self.call(callable)
+        t = threading.Thread(target=task)
+        t.start()
+        return t

Lib/test/support/interpreters/_crossinterp.py

@@ -0,0 +1,102 @@
+"""Common code between queues and channels."""
+
+
+class ItemInterpreterDestroyed(Exception):
+    """Raised when trying to get an item whose interpreter was destroyed."""
+
+
+class classonly:
+    """A non-data descriptor that makes a value only visible on the class.
+
+    This is like the "classmethod" builtin, but does not show up on
+    instances of the class.  It may be used as a decorator.
+    """
+
+    def __init__(self, value):
+        self.value = value
+        self.getter = classmethod(value).__get__
+        self.name = None
+
+    def __set_name__(self, cls, name):
+        if self.name is not None:
+            raise TypeError('already used')
+        self.name = name
+
+    def __get__(self, obj, cls):
+        if obj is not None:
+            raise AttributeError(self.name)
+        # called on the class
+        return self.getter(None, cls)
+
+
+class UnboundItem:
+    """Represents a cross-interpreter item no longer bound to an interpreter.
+
+    An item is unbound when the interpreter that added it to the
+    cross-interpreter container is destroyed.
+    """
+
+    __slots__ = ()
+
+    @classonly
+    def singleton(cls, kind, module, name='UNBOUND'):
+        doc = cls.__doc__.replace('cross-interpreter container', kind)
+        doc = doc.replace('cross-interpreter', kind)
+        subclass = type(
+            f'Unbound{kind.capitalize()}Item',
+            (cls,),
+            dict(
+                _MODULE=module,
+                _NAME=name,
+                __doc__=doc,
+            ),
+        )
+        return object.__new__(subclass)
+
+    _MODULE = __name__
+    _NAME = 'UNBOUND'
+
+    def __new__(cls):
+        raise Exception(f'use {cls._MODULE}.{cls._NAME}')
+
+    def __repr__(self):
+        return f'{self._MODULE}.{self._NAME}'
+#        return f'interpreters.queues.UNBOUND'
+
+
+UNBOUND = object.__new__(UnboundItem)
+UNBOUND_ERROR = object()
+UNBOUND_REMOVE = object()
+
+_UNBOUND_CONSTANT_TO_FLAG = {
+    UNBOUND_REMOVE: 1,
+    UNBOUND_ERROR: 2,
+    UNBOUND: 3,
+}
+_UNBOUND_FLAG_TO_CONSTANT = {v: k
+                             for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()}
+
+
+def serialize_unbound(unbound):
+    op = unbound
+    try:
+        flag = _UNBOUND_CONSTANT_TO_FLAG[op]
+    except KeyError:
+        raise NotImplementedError(f'unsupported unbound replacement op {op!r}')
+    return flag,
+
+
+def resolve_unbound(flag, exctype_destroyed):
+    try:
+        op = _UNBOUND_FLAG_TO_CONSTANT[flag]
+    except KeyError:
+        raise NotImplementedError(f'unsupported unbound replacement op {flag!r}')
+    if op is UNBOUND_REMOVE:
+        # "remove" not possible here
+        raise NotImplementedError
+    elif op is UNBOUND_ERROR:
+        raise exctype_destroyed("item's original interpreter destroyed")
+    elif op is UNBOUND:
+        return UNBOUND
+    else:
+        raise NotImplementedError(repr(op))

Lib/test/support/interpreters/channels.py

@@ -0,0 +1,257 @@
+"""Cross-interpreter Channels High Level Module."""
+
+import time
+import _interpchannels as _channels
+from . import _crossinterp
+
+# aliases:
+from _interpchannels import (
+    ChannelError, ChannelNotFoundError, ChannelClosedError,
+    ChannelEmptyError, ChannelNotEmptyError,
+)
+from ._crossinterp import (
+    UNBOUND_ERROR, UNBOUND_REMOVE,
+)
+
+
+__all__ = [
+    'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE',
+    'create', 'list_all',
+    'SendChannel', 'RecvChannel',
+    'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError',
+    'ItemInterpreterDestroyed',
+]
+
+
+class ItemInterpreterDestroyed(ChannelError,
+                               _crossinterp.ItemInterpreterDestroyed):
+    """Raised from get() and get_nowait()."""
+
+
+UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__)
+
+
+def _serialize_unbound(unbound):
+    if unbound is UNBOUND:
+        unbound = _crossinterp.UNBOUND
+    return _crossinterp.serialize_unbound(unbound)
+
+
+def _resolve_unbound(flag):
+    resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed)
+    if resolved is _crossinterp.UNBOUND:
+        resolved = UNBOUND
+    return resolved
+
+
+def create(*, unbounditems=UNBOUND):
+    """Return (recv, send) for a new cross-interpreter channel.
+
+    The channel may be used to pass data safely between interpreters.
+
+    "unbounditems" sets the default for the send end of the channel.
+    See SendChannel.send() for supported values.  The default value
+    is UNBOUND, which replaces the unbound item when received.
+    """
+    unbound = _serialize_unbound(unbounditems)
+    unboundop, = unbound
+    cid = _channels.create(unboundop)
+    recv, send = RecvChannel(cid), SendChannel(cid, _unbound=unbound)
+    return recv, send
+
+
+def list_all():
+    """Return a list of (recv, send) for all open channels."""
+    return [(RecvChannel(cid), SendChannel(cid, _unbound=unbound))
+            for cid, unbound in _channels.list_all()]
+
+
+class _ChannelEnd:
+    """The base class for RecvChannel and SendChannel."""
+
+    _end = None
+
+    def __new__(cls, cid):
+        self = super().__new__(cls)
+        if self._end == 'send':
+            cid = _channels._channel_id(cid, send=True, force=True)
+        elif self._end == 'recv':
+            cid = _channels._channel_id(cid, recv=True, force=True)
+        else:
+            raise NotImplementedError(self._end)
+        self._id = cid
+        return self
+
+    def __repr__(self):
+        return f'{type(self).__name__}(id={int(self._id)})'
+
+    def __hash__(self):
+        return hash(self._id)
+
+    def __eq__(self, other):
+        if isinstance(self, RecvChannel):
+            if not isinstance(other, RecvChannel):
+                return NotImplemented
+        elif not isinstance(other, SendChannel):
+            return NotImplemented
+        return other._id == self._id
+
+    # for pickling:
+    def __getnewargs__(self):
+        return (int(self._id),)
+
+    # for pickling:
+    def __getstate__(self):
+        return None
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def _info(self):
+        return _channels.get_info(self._id)
+
+    @property
+    def is_closed(self):
+        return self._info.closed
+
+
+_NOT_SET = object()
+
+
+class RecvChannel(_ChannelEnd):
+    """The receiving end of a cross-interpreter channel."""
+
+    _end = 'recv'
+
+    def recv(self, timeout=None, *,
+             _sentinel=object(),
+             _delay=10 / 1000,  # 10 milliseconds
+             ):
+        """Return the next object from the channel.
+
+        This blocks until an object has been sent, if none have been
+        sent already.
+        """
+        if timeout is not None:
+            timeout = int(timeout)
+            if timeout < 0:
+                raise ValueError(f'timeout value must be non-negative')
+            end = time.time() + timeout
+        obj, unboundop = _channels.recv(self._id, _sentinel)
+        while obj is _sentinel:
+            time.sleep(_delay)
+            if timeout is not None and time.time() >= end:
+                raise TimeoutError
+            obj, unboundop = _channels.recv(self._id, _sentinel)
+        if unboundop is not None:
+            assert obj is None, repr(obj)
+            return _resolve_unbound(unboundop)
+        return obj
+
+    def recv_nowait(self, default=_NOT_SET):
+        """Return the next object from the channel.
+
+        If none have been sent then return the default if one
+        is provided or fail with ChannelEmptyError.  Otherwise this
+        is the same as recv().
+        """
+        if default is _NOT_SET:
+            obj, unboundop = _channels.recv(self._id)
+        else:
+            obj, unboundop = _channels.recv(self._id, default)
+        if unboundop is not None:
+            assert obj is None, repr(obj)
+            return _resolve_unbound(unboundop)
+        return obj
+
+    def close(self):
+        _channels.close(self._id, recv=True)
+
+
+class SendChannel(_ChannelEnd):
+    """The sending end of a cross-interpreter channel."""
+
+    _end = 'send'
+
+    def __new__(cls, cid, *, _unbound=None):
+        if _unbound is None:
+            try:
+                op = _channels.get_channel_defaults(cid)
+                _unbound = (op,)
+            except ChannelNotFoundError:
+                _unbound = _serialize_unbound(UNBOUND)
+        self = super().__new__(cls, cid)
+        self._unbound = _unbound
+        return self
+
+    @property
+    def is_closed(self):
+        info = self._info
+        return info.closed or info.closing
+
+    def send(self, obj, timeout=None, *,
+             unbound=None,
+             ):
+        """Send the object (i.e. its data) to the channel's receiving end.
+
+        This blocks until the object is received.
+        """
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True)
+
+    def send_nowait(self, obj, *,
+                    unbound=None,
+                    ):
+        """Send the object to the channel's receiving end.
+
+        If the object is immediately received then return True
+        (else False).  Otherwise this is the same as send().
+        """
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        # XXX Note that at the moment channel_send() only ever returns
+        # None.  This should be fixed when channel_send_wait() is added.
+        # See bpo-32604 and gh-19829.
+        return _channels.send(self._id, obj, unboundop, blocking=False)
+
+    def send_buffer(self, obj, timeout=None, *,
+                    unbound=None,
+                    ):
+        """Send the object's buffer to the channel's receiving end.
+
+        This blocks until the object is received.
+        """
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        _channels.send_buffer(self._id, obj, unboundop,
+                              timeout=timeout, blocking=True)
+
+    def send_buffer_nowait(self, obj, *,
+                           unbound=None,
+                           ):
+        """Send the object's buffer to the channel's receiving end.
+
+        If the object is immediately received then return True
+        (else False).  Otherwise this is the same as send().
+        """
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        return _channels.send_buffer(self._id, obj, unboundop, blocking=False)
+
+    def close(self):
+        _channels.close(self._id, send=True)
+
+
+# XXX This is causing leaks (gh-110318):
+_channels._register_end_types(SendChannel, RecvChannel)

Lib/test/support/interpreters/queues.py

@@ -0,0 +1,313 @@
+"""Cross-interpreter Queues High Level Module."""
+
+import pickle
+import queue
+import time
+import weakref
+import _interpqueues as _queues
+from . import _crossinterp
+
+# aliases:
+from _interpqueues import (
+    QueueError, QueueNotFoundError,
+)
+from ._crossinterp import (
+    UNBOUND_ERROR, UNBOUND_REMOVE,
+)
+
+__all__ = [
+    'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE',
+    'create', 'list_all',
+    'Queue',
+    'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull',
+    'ItemInterpreterDestroyed',
+]
+
+
+class QueueEmpty(QueueError, queue.Empty):
+    """Raised from get_nowait() when the queue is empty.
+
+    It is also raised from get() if it times out.
+    """
+
+
+class QueueFull(QueueError, queue.Full):
+    """Raised from put_nowait() when the queue is full.
+
+    It is also raised from put() if it times out.
+    """
+
+
+class ItemInterpreterDestroyed(QueueError,
+                               _crossinterp.ItemInterpreterDestroyed):
+    """Raised from get() and get_nowait()."""
+
+
+_SHARED_ONLY = 0
+_PICKLED = 1
+
+
+UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__)
+
+
+def _serialize_unbound(unbound):
+    if unbound is UNBOUND:
+        unbound = _crossinterp.UNBOUND
+    return _crossinterp.serialize_unbound(unbound)
+
+
+def _resolve_unbound(flag):
+    resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed)
+    if resolved is _crossinterp.UNBOUND:
+        resolved = UNBOUND
+    return resolved
+
+
+def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND):
+    """Return a new cross-interpreter queue.
+
+    The queue may be used to pass data safely between interpreters.
+
+    "syncobj" sets the default for Queue.put()
+    and Queue.put_nowait().
+
+    "unbounditems" likewise sets the default.  See Queue.put() for
+    supported values.  The default value is UNBOUND, which replaces
+    the unbound item.
+    """
+    fmt = _SHARED_ONLY if syncobj else _PICKLED
+    unbound = _serialize_unbound(unbounditems)
+    unboundop, = unbound
+    qid = _queues.create(maxsize, fmt, unboundop)
+    return Queue(qid, _fmt=fmt, _unbound=unbound)
+
+
+def list_all():
+    """Return a list of all open queues."""
+    return [Queue(qid, _fmt=fmt, _unbound=(unboundop,))
+            for qid, fmt, unboundop in _queues.list_all()]
+
+
+_known_queues = weakref.WeakValueDictionary()
+
+class Queue:
+    """A cross-interpreter queue."""
+
+    def __new__(cls, id, /, *, _fmt=None, _unbound=None):
+        # There is only one instance for any given ID.
+        if isinstance(id, int):
+            id = int(id)
+        else:
+            raise TypeError(f'id must be an int, got {id!r}')
+        if _fmt is None:
+            if _unbound is None:
+                _fmt, op = _queues.get_queue_defaults(id)
+                _unbound = (op,)
+            else:
+                _fmt, _ = _queues.get_queue_defaults(id)
+        elif _unbound is None:
+            _, op = _queues.get_queue_defaults(id)
+            _unbound = (op,)
+        try:
+            self = _known_queues[id]
+        except KeyError:
+            self = super().__new__(cls)
+            self._id = id
+            self._fmt = _fmt
+            self._unbound = _unbound
+            _known_queues[id] = self
+            _queues.bind(id)
+        return self
+
+    def __del__(self):
+        try:
+            _queues.release(self._id)
+        except QueueNotFoundError:
+            pass
+        try:
+            del _known_queues[self._id]
+        except KeyError:
+            pass
+
+    def __repr__(self):
+        return f'{type(self).__name__}({self.id})'
+
+    def __hash__(self):
+        return hash(self._id)
+
+    # for pickling:
+    def __getnewargs__(self):
+        return (self._id,)
+
+    # for pickling:
+    def __getstate__(self):
+        return None
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def maxsize(self):
+        try:
+            return self._maxsize
+        except AttributeError:
+            self._maxsize = _queues.get_maxsize(self._id)
+            return self._maxsize
+
+    def empty(self):
+        return self.qsize() == 0
+
+    def full(self):
+        return _queues.is_full(self._id)
+
+    def qsize(self):
+        return _queues.get_count(self._id)
+
+    def put(self, obj, timeout=None, *,
+            syncobj=None,
+            unbound=None,
+            _delay=10 / 1000,  # 10 milliseconds
+            ):
+        """Add the object to the queue.
+
+        This blocks while the queue is full.
+
+        If "syncobj" is None (the default) then it uses the
+        queue's default, set with create_queue().
+
+        If "syncobj" is false then all objects are supported,
+        at the expense of worse performance.
+
+        If "syncobj" is true then the object must be "shareable".
+        Examples of "shareable" objects include the builtin singletons,
+        str, and memoryview.  One benefit is that such objects are
+        passed through the queue efficiently.
+
+        The key difference, though, is conceptual: the corresponding
+        object returned from Queue.get() will be strictly equivalent
+        to the given obj.  In other words, the two objects will be
+        effectively indistinguishable from each other, even if the
+        object is mutable.  The received object may actually be the
+        same object, or a copy (immutable values only), or a proxy.
+        Regardless, the received object should be treated as though
+        the original has been shared directly, whether or not it
+        actually is.  That's a slightly different and stronger promise
+        than just (initial) equality, which is all "syncobj=False"
+        can promise.
+
+        "unbound" controls the behavior of Queue.get() for the given
+        object if the current interpreter (calling put()) is later
+        destroyed.
+
+        If "unbound" is None (the default) then it uses the
+        queue's default, set with create_queue(),
+        which is usually UNBOUND.
+
+        If "unbound" is UNBOUND_ERROR then get() will raise an
+        ItemInterpreterDestroyed exception if the original interpreter
+        has been destroyed.  This does not otherwise affect the queue;
+        the next call to put() will work like normal, returning the next
+        item in the queue.
+
+        If "unbound" is UNBOUND_REMOVE then the item will be removed
+        from the queue as soon as the original interpreter is destroyed.
+        Be aware that this will introduce an imbalance between put()
+        and get() calls.
+
+        If "unbound" is UNBOUND then it is returned by get() in place
+        of the unbound item.
+        """
+        if syncobj is None:
+            fmt = self._fmt
+        else:
+            fmt = _SHARED_ONLY if syncobj else _PICKLED
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        if timeout is not None:
+            timeout = int(timeout)
+            if timeout < 0:
+                raise ValueError(f'timeout value must be non-negative')
+            end = time.time() + timeout
+        if fmt is _PICKLED:
+            obj = pickle.dumps(obj)
+        while True:
+            try:
+                _queues.put(self._id, obj, fmt, unboundop)
+            except QueueFull as exc:
+                if timeout is not None and time.time() >= end:
+                    raise  # re-raise
+                time.sleep(_delay)
+            else:
+                break
+
+    def put_nowait(self, obj, *, syncobj=None, unbound=None):
+        if syncobj is None:
+            fmt = self._fmt
+        else:
+            fmt = _SHARED_ONLY if syncobj else _PICKLED
+        if unbound is None:
+            unboundop, = self._unbound
+        else:
+            unboundop, = _serialize_unbound(unbound)
+        if fmt is _PICKLED:
+            obj = pickle.dumps(obj)
+        _queues.put(self._id, obj, fmt, unboundop)
+
+    def get(self, timeout=None, *,
+            _delay=10 / 1000,  # 10 milliseconds
+            ):
+        """Return the next object from the queue.
+
+        This blocks while the queue is empty.
+
+        If the next item's original interpreter has been destroyed
+        then the "next object" is determined by the value of the
+        "unbound" argument to put().
+        """
+        if timeout is not None:
+            timeout = int(timeout)
+            if timeout < 0:
+                raise ValueError(f'timeout value must be non-negative')
+            end = time.time() + timeout
+        while True:
+            try:
+                obj, fmt, unboundop = _queues.get(self._id)
+            except QueueEmpty as exc:
+                if timeout is not None and time.time() >= end:
+                    raise  # re-raise
+                time.sleep(_delay)
+            else:
+                break
+        if unboundop is not None:
+            assert obj is None, repr(obj)
+            return _resolve_unbound(unboundop)
+        if fmt == _PICKLED:
+            obj = pickle.loads(obj)
+        else:
+            assert fmt == _SHARED_ONLY
+        return obj
+
+    def get_nowait(self):
+        """Return the next object from the channel.
+
+        If the queue is empty then raise QueueEmpty.  Otherwise this
+        is the same as get().
+        """
+        try:
+            obj, fmt, unboundop = _queues.get(self._id)
+        except QueueEmpty as exc:
+            raise  # re-raise
+        if unboundop is not None:
+            assert obj is None, repr(obj)
+            return _resolve_unbound(unboundop)
+        if fmt == _PICKLED:
+            obj = pickle.loads(obj)
+        else:
+            assert fmt == _SHARED_ONLY
+        return obj
+
+
+_queues._register_heap_types(Queue, QueueEmpty, QueueFull)

Lib/test/support/numbers.py

@@ -0,0 +1,80 @@
+# These are shared with test_tokenize and other test modules.
+#
+# Note: since several test cases filter out floats by looking for "e" and ".",
+# don't add hexadecimal literals that contain "e" or "E".
+VALID_UNDERSCORE_LITERALS = [
+    '0_0_0',
+    '4_2',
+    '1_0000_0000',
+    '0b1001_0100',
+    '0xffff_ffff',
+    '0o5_7_7',
+    '1_00_00.5',
+    '1_00_00.5e5',
+    '1_00_00e5_1',
+    '1e1_0',
+    '.1_4',
+    '.1_4e1',
+    '0b_0',
+    '0x_f',
+    '0o_5',
+    '1_00_00j',
+    '1_00_00.5j',
+    '1_00_00e5_1j',
+    '.1_4j',
+    '(1_2.5+3_3j)',
+    '(.5_6j)',
+]
+INVALID_UNDERSCORE_LITERALS = [
+    # Trailing underscores:
+    '0_',
+    '42_',
+    '1.4j_',
+    '0x_',
+    '0b1_',
+    '0xf_',
+    '0o5_',
+    '0 if 1_Else 1',
+    # Underscores in the base selector:
+    '0_b0',
+    '0_xf',
+    '0_o5',
+    # Old-style octal, still disallowed:
+    '0_7',
+    '09_99',
+    # Multiple consecutive underscores:
+    '4_______2',
+    '0.1__4',
+    '0.1__4j',
+    '0b1001__0100',
+    '0xffff__ffff',
+    '0x___',
+    '0o5__77',
+    '1e1__0',
+    '1e1__0j',
+    # Underscore right before a dot:
+    '1_.4',
+    '1_.4j',
+    # Underscore right after a dot:
+    '1._4',
+    '1._4j',
+    '._5',
+    '._5j',
+    # Underscore right after a sign:
+    '1.0e+_1',
+    '1.0e+_1j',
+    # Underscore right before j:
+    '1.4_j',
+    '1.4e5_j',
+    # Underscore right before e:
+    '1_e1',
+    '1.4_e1',
+    '1.4_e1j',
+    # Underscore right after e:
+    '1e_1',
+    '1.4e_1',
+    '1.4e_1j',
+    # Complex cases with parens:
+    '(1+1.5_j_)',
+    '(1+1.5_j)',
+]