Merge pull request #5789 from coolreader18/crt_fd-rework

Rework crt_fd to be more aligned with io-safety

.cspell.json

@@ -60,6 +60,7 @@
     "dedentations",
     "dedents",
     "deduped",
+    "downcastable",
     "downcasted",
     "dumpable",
     "emscripten",

.github/copilot-instructions.md

@@ -21,7 +21,7 @@ RustPython is a Python 3 interpreter written in Rust, implementing Python 3.13.0
   - `parser/` - Parser for converting Python source to AST
   - `core/` - Bytecode representation in Rust structures
   - `codegen/` - AST to bytecode compiler
-- `Lib/` - CPython's standard library in Python (copied from CPython)
+- `Lib/` - CPython's standard library in Python (copied from CPython). **IMPORTANT**: Do not edit this directory directly; The only allowed operation is copying files from CPython.
 - `derive/` - Rust macros for RustPython
 - `common/` - Common utilities
 - `extra_tests/` - Integration tests and snippets

.github/workflows/ci.yaml

@@ -113,6 +113,7 @@ jobs:
       RUST_BACKTRACE: full
     name: Run rust tests
     runs-on:  ${{ matrix.os }}
+    timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 30 }}
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
@@ -175,6 +176,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Ensure compilation on various targets
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -237,6 +239,7 @@ jobs:
       RUST_BACKTRACE: full
     name: Run snippets and cpython tests
     runs-on:  ${{ matrix.os }}
+    timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 30 }}
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
@@ -344,23 +347,31 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Run tests under miri
     runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      NIGHTLY_CHANNEL: nightly
     steps:
       - uses: actions/checkout@v4
+
       - uses: dtolnay/rust-toolchain@master
         with:
-            toolchain: nightly
+            toolchain: ${{ env.NIGHTLY_CHANNEL }}
             components: miri
 
       - uses: Swatinem/rust-cache@v2
+
       - name: Run tests under miri
+        run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test
+        env:
         # miri-ignore-leaks because the type-object circular reference means that there will always be
         # a memory leak, at least until we have proper cyclic gc
-        run: MIRIFLAGS='-Zmiri-ignore-leaks' cargo +nightly miri test -p rustpython-vm -- miri_test
+          MIRIFLAGS: '-Zmiri-ignore-leaks'
 
   wasm:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Check the WASM package and demo
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -421,6 +432,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Run snippets and cpython tests on wasm-wasi
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable

.github/workflows/comment-commands.yml

@@ -0,0 +1,21 @@
+name: Comment Commands
+
+on:
+  issue_comment:
+    types: created
+
+jobs:
+  issue_assign:
+    if: (!github.event.issue.pull_request) && github.event.comment.body == 'take'
+    runs-on: ubuntu-latest
+
+    concurrency:
+      group: ${{ github.actor }}-issue-assign
+
+    permissions:
+      issues: write
+
+    steps:
+      # Using REST API and not `gh issue edit`. https://github.com/cli/cli/issues/6235#issuecomment-1243487651
+      - run: |
+          curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees

.gitignore

@@ -21,3 +21,8 @@ flamescope.json
 
 extra_tests/snippets/resources
 extra_tests/not_impl.py
+
+Lib/site-packages/*
+!Lib/site-packages/README.txt
+Lib/test/data/*
+!Lib/test/data/README

Cargo.toml

@@ -82,7 +82,6 @@ opt-level = 3
 lto = "thin"
 
 [patch.crates-io]
-radium = { version = "1.1.0", git = "https://github.com/youknowone/ferrilab", branch = "fix-nightly" }
 # REDOX START, Uncomment when you want to compile/check with redoxer
 # REDOX END
 
@@ -118,8 +117,20 @@ template = "installer-config/installer.wxs"
 [workspace]
 resolver = "2"
 members = [
-    "compiler", "compiler/core", "compiler/codegen", "compiler/literal", "compiler/source",
-    ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", "wtf8",
+    "compiler",
+    "compiler/core",
+    "compiler/codegen",
+    "compiler/literal",
+    ".",
+    "common",
+    "derive",
+    "jit",
+    "vm",
+    "vm/sre_engine",
+    "pylib",
+    "stdlib",
+    "derive-impl",
+    "wtf8",
     "wasm/lib",
 ]
 
@@ -127,12 +138,11 @@ members = [
 version = "0.4.0"
 authors = ["RustPython Team"]
 edition = "2024"
-rust-version = "1.85.0"
+rust-version = "1.87.0"
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [workspace.dependencies]
-rustpython-compiler-source = { path = "compiler/source" }
 rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
 rustpython-compiler = { path = "compiler", version = "0.4.0" }
 rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }
@@ -155,7 +165,7 @@ ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.
 
 ahash = "0.8.11"
 ascii = "1.1"
-bitflags = "2.4.2"
+bitflags = "2.9.1"
 bstr = "1"
 cfg-if = "1.0"
 chrono = "0.4.39"
@@ -166,7 +176,7 @@ flame = "0.2.2"
 getrandom = { version = "0.3", features = ["std"] }
 glob = "0.3"
 hex = "0.4.3"
-indexmap = { version = "2.2.6", features = ["std"] }
+indexmap = { version = "2.10.0", features = ["std"] }
 insta = "1.42"
 itertools = "0.14.0"
 is-macro = "0.3.7"
@@ -190,11 +200,11 @@ paste = "1.0.15"
 proc-macro2 = "1.0.93"
 pymath = "0.0.2"
 quote = "1.0.38"
-radium = "1.1"
+radium = "1.1.1"
 rand = "0.9"
 rand_core = { version = "0.9", features = ["os_rng"] }
 rustix = { version = "1.0", features = ["event"] }
-rustyline = "15.0.0"
+rustyline = "17.0.0"
 serde = { version = "1.0.133", default-features = false }
 schannel = "0.1.27"
 static_assertions = "1.1"
@@ -212,7 +222,7 @@ unic-ucd-category = "0.9.0"
 unic-ucd-ident = "0.9.0"
 unicode_names2 = "1.3.0"
 unicode-bidi-mirroring = "0.2"
-widestring = "1.1.0"
+widestring = "1.2.0"
 windows-sys = "0.59.0"
 wasm-bindgen = "0.2.100"
 

Lib/_colorize.py

@@ -1,20 +1,63 @@
+from __future__ import annotations
 import io
 import os
 import sys
 
 COLORIZE = True
 
+# types
+if False:
+    from typing import IO
+
 
 class ANSIColors:
+    RESET = "\x1b[0m"
+
+    BLACK = "\x1b[30m"
+    BLUE = "\x1b[34m"
+    CYAN = "\x1b[36m"
+    GREEN = "\x1b[32m"
+    MAGENTA = "\x1b[35m"
+    RED = "\x1b[31m"
+    WHITE = "\x1b[37m"  # more like LIGHT GRAY
+    YELLOW = "\x1b[33m"
+
+    BOLD_BLACK = "\x1b[1;30m"  # DARK GRAY
+    BOLD_BLUE = "\x1b[1;34m"
+    BOLD_CYAN = "\x1b[1;36m"
     BOLD_GREEN = "\x1b[1;32m"
     BOLD_MAGENTA = "\x1b[1;35m"
     BOLD_RED = "\x1b[1;31m"
-    GREEN = "\x1b[32m"
-    GREY = "\x1b[90m"
-    MAGENTA = "\x1b[35m"
-    RED = "\x1b[31m"
-    RESET = "\x1b[0m"
-    YELLOW = "\x1b[33m"
+    BOLD_WHITE = "\x1b[1;37m"  # actual WHITE
+    BOLD_YELLOW = "\x1b[1;33m"
+
+    # intense = like bold but without being bold
+    INTENSE_BLACK = "\x1b[90m"
+    INTENSE_BLUE = "\x1b[94m"
+    INTENSE_CYAN = "\x1b[96m"
+    INTENSE_GREEN = "\x1b[92m"
+    INTENSE_MAGENTA = "\x1b[95m"
+    INTENSE_RED = "\x1b[91m"
+    INTENSE_WHITE = "\x1b[97m"
+    INTENSE_YELLOW = "\x1b[93m"
+
+    BACKGROUND_BLACK = "\x1b[40m"
+    BACKGROUND_BLUE = "\x1b[44m"
+    BACKGROUND_CYAN = "\x1b[46m"
+    BACKGROUND_GREEN = "\x1b[42m"
+    BACKGROUND_MAGENTA = "\x1b[45m"
+    BACKGROUND_RED = "\x1b[41m"
+    BACKGROUND_WHITE = "\x1b[47m"
+    BACKGROUND_YELLOW = "\x1b[43m"
+
+    INTENSE_BACKGROUND_BLACK = "\x1b[100m"
+    INTENSE_BACKGROUND_BLUE = "\x1b[104m"
+    INTENSE_BACKGROUND_CYAN = "\x1b[106m"
+    INTENSE_BACKGROUND_GREEN = "\x1b[102m"
+    INTENSE_BACKGROUND_MAGENTA = "\x1b[105m"
+    INTENSE_BACKGROUND_RED = "\x1b[101m"
+    INTENSE_BACKGROUND_WHITE = "\x1b[107m"
+    INTENSE_BACKGROUND_YELLOW = "\x1b[103m"
 
 
 NoColors = ANSIColors()
@@ -24,14 +67,16 @@ for attr in dir(NoColors):
         setattr(NoColors, attr, "")
 
 
-def get_colors(colorize: bool = False, *, file=None) -> ANSIColors:
+def get_colors(
+    colorize: bool = False, *, file: IO[str] | IO[bytes] | None = None
+) -> ANSIColors:
     if colorize or can_colorize(file=file):
         return ANSIColors()
     else:
         return NoColors
 
 
-def can_colorize(*, file=None) -> bool:
+def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool:
     if file is None:
         file = sys.stdout
 
@@ -64,4 +109,4 @@ def can_colorize(*, file=None) -> bool:
     try:
         return os.isatty(file.fileno())
     except io.UnsupportedOperation:
-        return file.isatty()
+        return hasattr(file, "isatty") and file.isatty()

Lib/_pydecimal.py

@@ -13,104 +13,7 @@
 # bug) and will be backported.  At this point the spec is stabilizing
 # and the updates are becoming fewer, smaller, and less significant.
 
-"""
-This is an implementation of decimal floating point arithmetic based on
-the General Decimal Arithmetic Specification:
-
-    http://speleotrove.com/decimal/decarith.html
-
-and IEEE standard 854-1987:
-
-    http://en.wikipedia.org/wiki/IEEE_854-1987
-
-Decimal floating point has finite precision with arbitrarily large bounds.
-
-The purpose of this module is to support arithmetic using familiar
-"schoolhouse" rules and to avoid some of the tricky representation
-issues associated with binary floating point.  The package is especially
-useful for financial applications or for contexts where users have
-expectations that are at odds with binary floating point (for instance,
-in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead
-of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected
-Decimal('0.00')).
-
-Here are some examples of using the decimal module:
-
->>> from decimal import *
->>> setcontext(ExtendedContext)
->>> Decimal(0)
-Decimal('0')
->>> Decimal('1')
-Decimal('1')
->>> Decimal('-.0123')
-Decimal('-0.0123')
->>> Decimal(123456)
-Decimal('123456')
->>> Decimal('123.45e12345678')
-Decimal('1.2345E+12345680')
->>> Decimal('1.33') + Decimal('1.27')
-Decimal('2.60')
->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')
-Decimal('-2.20')
->>> dig = Decimal(1)
->>> print(dig / Decimal(3))
-0.333333333
->>> getcontext().prec = 18
->>> print(dig / Decimal(3))
-0.333333333333333333
->>> print(dig.sqrt())
-1
->>> print(Decimal(3).sqrt())
-1.73205080756887729
->>> print(Decimal(3) ** 123)
-4.85192780976896427E+58
->>> inf = Decimal(1) / Decimal(0)
->>> print(inf)
-Infinity
->>> neginf = Decimal(-1) / Decimal(0)
->>> print(neginf)
--Infinity
->>> print(neginf + inf)
-NaN
->>> print(neginf * inf)
--Infinity
->>> print(dig / 0)
-Infinity
->>> getcontext().traps[DivisionByZero] = 1
->>> print(dig / 0)
-Traceback (most recent call last):
-  ...
-  ...
-  ...
-decimal.DivisionByZero: x / 0
->>> c = Context()
->>> c.traps[InvalidOperation] = 0
->>> print(c.flags[InvalidOperation])
-0
->>> c.divide(Decimal(0), Decimal(0))
-Decimal('NaN')
->>> c.traps[InvalidOperation] = 1
->>> print(c.flags[InvalidOperation])
-1
->>> c.flags[InvalidOperation] = 0
->>> print(c.flags[InvalidOperation])
-0
->>> print(c.divide(Decimal(0), Decimal(0)))
-Traceback (most recent call last):
-  ...
-  ...
-  ...
-decimal.InvalidOperation: 0 / 0
->>> print(c.flags[InvalidOperation])
-1
->>> c.flags[InvalidOperation] = 0
->>> c.traps[InvalidOperation] = 0
->>> print(c.divide(Decimal(0), Decimal(0)))
-NaN
->>> print(c.flags[InvalidOperation])
-1
->>>
-"""
+"""Python decimal arithmetic module"""
 
 __all__ = [
     # Two major classes
@@ -140,8 +43,11 @@ __all__ = [
     # Limits for the C version for compatibility
     'MAX_PREC',  'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',
 
-    # C version: compile time choice that enables the thread local context
-    'HAVE_THREADS'
+    # C version: compile time choice that enables the thread local context (deprecated, now always true)
+    'HAVE_THREADS',
+
+    # C version: compile time choice that enables the coroutine local context
+    'HAVE_CONTEXTVAR'
 ]
 
 __xname__ = __name__    # sys.modules lookup (--without-threads)
@@ -156,7 +62,7 @@ import sys
 
 try:
     from collections import namedtuple as _namedtuple
-    DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent')
+    DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent', module='decimal')
 except ImportError:
     DecimalTuple = lambda *args: args
 
@@ -172,6 +78,7 @@ ROUND_05UP = 'ROUND_05UP'
 
 # Compatibility with the C version
 HAVE_THREADS = True
+HAVE_CONTEXTVAR = True
 if sys.maxsize == 2**63-1:
     MAX_PREC = 999999999999999999
     MAX_EMAX = 999999999999999999
@@ -190,7 +97,7 @@ class DecimalException(ArithmeticError):
 
     Used exceptions derive from this.
     If an exception derives from another exception besides this (such as
-    Underflow (Inexact, Rounded, Subnormal) that indicates that it is only
+    Underflow (Inexact, Rounded, Subnormal)) that indicates that it is only
     called if the others are present.  This isn't actually used for
     anything, though.
 
@@ -238,7 +145,7 @@ class InvalidOperation(DecimalException):
     x ** (+-)INF
     An operand is invalid
 
-    The result of the operation after these is a quiet positive NaN,
+    The result of the operation after this is a quiet positive NaN,
     except when the cause is a signaling NaN, in which case the result is
     also a quiet NaN, but with the original sign, and an optional
     diagnostic information.
@@ -431,82 +338,40 @@ _rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING,
 ##### Context Functions ##################################################
 
 # The getcontext() and setcontext() function manage access to a thread-local
-# current context.  Py2.4 offers direct support for thread locals.  If that
-# is not available, use threading.current_thread() which is slower but will
-# work for older Pythons.  If threads are not part of the build, create a
-# mock threading object with threading.local() returning the module namespace.
+# current context.
 
-try:
-    import threading
-except ImportError:
-    # Python was compiled without threads; create a mock object instead
-    class MockThreading(object):
-        def local(self, sys=sys):
-            return sys.modules[__xname__]
-    threading = MockThreading()
-    del MockThreading
+import contextvars
 
-try:
-    threading.local
+_current_context_var = contextvars.ContextVar('decimal_context')
 
-except AttributeError:
+_context_attributes = frozenset(
+    ['prec', 'Emin', 'Emax', 'capitals', 'clamp', 'rounding', 'flags', 'traps']
+)
 
-    # To fix reloading, force it to create a new context
-    # Old contexts have different exceptions in their dicts, making problems.
-    if hasattr(threading.current_thread(), '__decimal_context__'):
-        del threading.current_thread().__decimal_context__
+def getcontext():
+    """Returns this thread's context.
 
-    def setcontext(context):
-        """Set this thread's context to context."""
-        if context in (DefaultContext, BasicContext, ExtendedContext):
-            context = context.copy()
-            context.clear_flags()
-        threading.current_thread().__decimal_context__ = context
+    If this thread does not yet have a context, returns
+    a new context and sets this thread's context.
+    New contexts are copies of DefaultContext.
+    """
+    try:
+        return _current_context_var.get()
+    except LookupError:
+        context = Context()
+        _current_context_var.set(context)
+        return context
 
-    def getcontext():
-        """Returns this thread's context.
+def setcontext(context):
+    """Set this thread's context to context."""
+    if context in (DefaultContext, BasicContext, ExtendedContext):
+        context = context.copy()
+        context.clear_flags()
+    _current_context_var.set(context)
 
-        If this thread does not yet have a context, returns
-        a new context and sets this thread's context.
-        New contexts are copies of DefaultContext.
-        """
-        try:
-            return threading.current_thread().__decimal_context__
-        except AttributeError:
-            context = Context()
-            threading.current_thread().__decimal_context__ = context
-            return context
+del contextvars        # Don't contaminate the namespace
 
-else:
-
-    local = threading.local()
-    if hasattr(local, '__decimal_context__'):
-        del local.__decimal_context__
-
-    def getcontext(_local=local):
-        """Returns this thread's context.
-
-        If this thread does not yet have a context, returns
-        a new context and sets this thread's context.
-        New contexts are copies of DefaultContext.
-        """
-        try:
-            return _local.__decimal_context__
-        except AttributeError:
-            context = Context()
-            _local.__decimal_context__ = context
-            return context
-
-    def setcontext(context, _local=local):
-        """Set this thread's context to context."""
-        if context in (DefaultContext, BasicContext, ExtendedContext):
-            context = context.copy()
-            context.clear_flags()
-        _local.__decimal_context__ = context
-
-    del threading, local        # Don't contaminate the namespace
-
-def localcontext(ctx=None):
+def localcontext(ctx=None, **kwargs):
     """Return a context manager for a copy of the supplied context
 
     Uses a copy of the current context if no context is specified
@@ -542,8 +407,14 @@ def localcontext(ctx=None):
     >>> print(getcontext().prec)
     28
     """
-    if ctx is None: ctx = getcontext()
-    return _ContextManager(ctx)
+    if ctx is None:
+        ctx = getcontext()
+    ctx_manager = _ContextManager(ctx)
+    for key, value in kwargs.items():
+        if key not in _context_attributes:
+            raise TypeError(f"'{key}' is an invalid keyword argument for this function")
+        setattr(ctx_manager.new_context, key, value)
+    return ctx_manager
 
 
 ##### Decimal class #######################################################
@@ -553,7 +424,7 @@ def localcontext(ctx=None):
 # numbers.py for more detail.
 
 class Decimal(object):
-    """Floating point class for decimal arithmetic."""
+    """Floating-point class for decimal arithmetic."""
 
     __slots__ = ('_exp','_int','_sign', '_is_special')
     # Generally, the value of the Decimal instance is given by
@@ -993,7 +864,7 @@ class Decimal(object):
             if self.is_snan():
                 raise TypeError('Cannot hash a signaling NaN value.')
             elif self.is_nan():
-                return _PyHASH_NAN
+                return object.__hash__(self)
             else:
                 if self._sign:
                     return -_PyHASH_INF
@@ -1674,13 +1545,13 @@ class Decimal(object):
 
     __trunc__ = __int__
 
+    @property
     def real(self):
         return self
-    real = property(real)
 
+    @property
     def imag(self):
         return Decimal(0)
-    imag = property(imag)
 
     def conjugate(self):
         return self
@@ -2260,10 +2131,16 @@ class Decimal(object):
             else:
                 return None
 
-            if xc >= 10**p:
+            # An exact power of 10 is representable, but can convert to a
+            # string of any length. But an exact power of 10 shouldn't be
+            # possible at this point.
+            assert xc > 1, self
+            assert xc % 10 != 0, self
+            strxc = str(xc)
+            if len(strxc) > p:
                 return None
             xe = -e-xe
-            return _dec_from_triple(0, str(xc), xe)
+            return _dec_from_triple(0, strxc, xe)
 
         # now y is positive; find m and n such that y = m/n
         if ye >= 0:
@@ -2272,7 +2149,7 @@ class Decimal(object):
             if xe != 0 and len(str(abs(yc*xe))) <= -ye:
                 return None
             xc_bits = _nbits(xc)
-            if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye:
+            if len(str(abs(yc)*xc_bits)) <= -ye:
                 return None
             m, n = yc, 10**(-ye)
             while m % 2 == n % 2 == 0:
@@ -2285,7 +2162,7 @@ class Decimal(object):
         # compute nth root of xc*10**xe
         if n > 1:
             # if 1 < xc < 2**n then xc isn't an nth power
-            if xc != 1 and xc_bits <= n:
+            if xc_bits <= n:
                 return None
 
             xe, rem = divmod(xe, n)
@@ -2313,13 +2190,18 @@ class Decimal(object):
             return None
         xc = xc**m
         xe *= m
-        if xc > 10**p:
+        # An exact power of 10 is representable, but can convert to a string
+        # of any length. But an exact power of 10 shouldn't be possible at
+        # this point.
+        assert xc > 1, self
+        assert xc % 10 != 0, self
+        str_xc = str(xc)
+        if len(str_xc) > p:
             return None
 
         # by this point the result *is* exactly representable
         # adjust the exponent to get as close as possible to the ideal
         # exponent, if necessary
-        str_xc = str(xc)
         if other._isinteger() and other._sign == 0:
             ideal_exponent = self._exp*int(other)
             zeros = min(xe-ideal_exponent, p-len(str_xc))
@@ -3837,6 +3719,10 @@ class Decimal(object):
         # represented in fixed point; rescale them to 0e0.
         if not self and self._exp > 0 and spec['type'] in 'fF%':
             self = self._rescale(0, rounding)
+        if not self and spec['no_neg_0'] and self._sign:
+            adjusted_sign = 0
+        else:
+            adjusted_sign = self._sign
 
         # figure out placement of the decimal point
         leftdigits = self._exp + len(self._int)
@@ -3867,7 +3753,7 @@ class Decimal(object):
 
         # done with the decimal-specific stuff;  hand over the rest
         # of the formatting to the _format_number function
-        return _format_number(self._sign, intpart, fracpart, exp, spec)
+        return _format_number(adjusted_sign, intpart, fracpart, exp, spec)
 
 def _dec_from_triple(sign, coefficient, exponent, special=False):
     """Create a decimal instance directly, without any validation,
@@ -5677,8 +5563,6 @@ class _WorkRep(object):
     def __repr__(self):
         return "(%r, %r, %r)" % (self.sign, self.int, self.exp)
 
-    __str__ = __repr__
-
 
 
 def _normalize(op1, op2, prec = 0):
@@ -6187,7 +6071,7 @@ _exact_half = re.compile('50*$').match
 #
 # A format specifier for Decimal looks like:
 #
-#   [[fill]align][sign][#][0][minimumwidth][,][.precision][type]
+#   [[fill]align][sign][z][#][0][minimumwidth][,][.precision][type]
 
 _parse_format_specifier_regex = re.compile(r"""\A
 (?:
@@ -6195,6 +6079,7 @@ _parse_format_specifier_regex = re.compile(r"""\A
    (?P<align>[<>=^])
 )?
 (?P<sign>[-+ ])?
+(?P<no_neg_0>z)?
 (?P<alt>\#)?
 (?P<zeropad>0)?
 (?P<minimumwidth>(?!0)\d+)?

Lib/_weakrefset.py

@@ -80,8 +80,7 @@ class WeakSet:
         return wr in self.data
 
     def __reduce__(self):
-        return (self.__class__, (list(self),),
-                getattr(self, '__dict__', None))
+        return self.__class__, (list(self),), self.__getstate__()
 
     def add(self, item):
         if self._pending_removals:

Lib/ast.py

@@ -1,28 +1,24 @@
 """
-    ast
-    ~~~
+The `ast` module helps Python applications to process trees of the Python
+abstract syntax grammar.  The abstract syntax itself might change with
+each Python release; this module helps to find out programmatically what
+the current grammar looks like and allows modifications of it.
 
-    The `ast` module helps Python applications to process trees of the Python
-    abstract syntax grammar.  The abstract syntax itself might change with
-    each Python release; this module helps to find out programmatically what
-    the current grammar looks like and allows modifications of it.
+An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
+a flag to the `compile()` builtin function or by using the `parse()`
+function from this module.  The result will be a tree of objects whose
+classes all inherit from `ast.AST`.
 
-    An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
-    a flag to the `compile()` builtin function or by using the `parse()`
-    function from this module.  The result will be a tree of objects whose
-    classes all inherit from `ast.AST`.
+A modified abstract syntax tree can be compiled into a Python code object
+using the built-in `compile()` function.
 
-    A modified abstract syntax tree can be compiled into a Python code object
-    using the built-in `compile()` function.
+Additionally various helper functions are provided that make working with
+the trees simpler.  The main intention of the helper functions and this
+module in general is to provide an easy to use interface for libraries
+that work tightly with the python syntax (template engines for example).
 
-    Additionally various helper functions are provided that make working with
-    the trees simpler.  The main intention of the helper functions and this
-    module in general is to provide an easy to use interface for libraries
-    that work tightly with the python syntax (template engines for example).
-
-
-    :copyright: Copyright 2008 by Armin Ronacher.
-    :license: Python License.
+:copyright: Copyright 2008 by Armin Ronacher.
+:license: Python License.
 """
 import sys
 import re
@@ -32,13 +28,15 @@ from enum import IntEnum, auto, _simple_enum
 
 
 def parse(source, filename='<unknown>', mode='exec', *,
-          type_comments=False, feature_version=None):
+          type_comments=False, feature_version=None, optimize=-1):
     """
     Parse the source into an AST node.
     Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
     Pass type_comments=True to get back type comments where the syntax allows.
     """
     flags = PyCF_ONLY_AST
+    if optimize > 0:
+        flags |= PyCF_OPTIMIZED_AST
     if type_comments:
         flags |= PyCF_TYPE_COMMENTS
     if feature_version is None:
@@ -50,7 +48,7 @@ def parse(source, filename='<unknown>', mode='exec', *,
         feature_version = minor
     # Else it should be an int giving the minor version for 3.x.
     return compile(source, filename, mode, flags,
-                   _feature_version=feature_version)
+                   _feature_version=feature_version, optimize=optimize)
 
 
 def literal_eval(node_or_string):
@@ -112,7 +110,11 @@ def literal_eval(node_or_string):
     return _convert(node_or_string)
 
 
-def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
+def dump(
+    node, annotate_fields=True, include_attributes=False,
+    *,
+    indent=None, show_empty=False,
+):
     """
     Return a formatted dump of the tree in node.  This is mainly useful for
     debugging purposes.  If annotate_fields is true (by default),
@@ -123,6 +125,8 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
     include_attributes can be set to true.  If indent is a non-negative
     integer or string, then the tree will be pretty-printed with that indent
     level. None (the default) selects the single line representation.
+    If show_empty is False, then empty lists and fields that are None
+    will be omitted from the output for better readability.
     """
     def _format(node, level=0):
         if indent is not None:
@@ -135,6 +139,7 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
         if isinstance(node, AST):
             cls = type(node)
             args = []
+            args_buffer = []
             allsimple = True
             keywords = annotate_fields
             for name in node._fields:
@@ -146,6 +151,16 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
                 if value is None and getattr(cls, name, ...) is None:
                     keywords = True
                     continue
+                if not show_empty:
+                    if value == []:
+                        field_type = cls._field_types.get(name, object)
+                        if getattr(field_type, '__origin__', ...) is list:
+                            if not keywords:
+                                args_buffer.append(repr(value))
+                            continue
+                    if not keywords:
+                        args.extend(args_buffer)
+                        args_buffer = []
                 value, simple = _format(value, level)
                 allsimple = allsimple and simple
                 if keywords:
@@ -726,12 +741,11 @@ class _Unparser(NodeVisitor):
     output source code for the abstract syntax; original formatting
     is disregarded."""
 
-    def __init__(self, *, _avoid_backslashes=False):
+    def __init__(self):
         self._source = []
         self._precedences = {}
         self._type_ignores = {}
         self._indent = 0
-        self._avoid_backslashes = _avoid_backslashes
         self._in_try_star = False
 
     def interleave(self, inter, f, seq):
@@ -1104,12 +1118,21 @@ class _Unparser(NodeVisitor):
         if node.bound:
             self.write(": ")
             self.traverse(node.bound)
+        if node.default_value:
+            self.write(" = ")
+            self.traverse(node.default_value)
 
     def visit_TypeVarTuple(self, node):
         self.write("*" + node.name)
+        if node.default_value:
+            self.write(" = ")
+            self.traverse(node.default_value)
 
     def visit_ParamSpec(self, node):
         self.write("**" + node.name)
+        if node.default_value:
+            self.write(" = ")
+            self.traverse(node.default_value)
 
     def visit_TypeAlias(self, node):
         self.fill("type ")
@@ -1246,9 +1269,14 @@ class _Unparser(NodeVisitor):
                     fallback_to_repr = True
                     break
                 quote_types = new_quote_types
-            elif "\n" in value:
-                quote_types = [q for q in quote_types if q in _MULTI_QUOTES]
-                assert quote_types
+            else:
+                if "\n" in value:
+                    quote_types = [q for q in quote_types if q in _MULTI_QUOTES]
+                    assert quote_types
+
+                new_quote_types = [q for q in quote_types if q not in value]
+                if new_quote_types:
+                    quote_types = new_quote_types
             new_fstring_parts.append(value)
 
         if fallback_to_repr:
@@ -1268,13 +1296,19 @@ class _Unparser(NodeVisitor):
         quote_type = quote_types[0]
         self.write(f"{quote_type}{value}{quote_type}")
 
-    def _write_fstring_inner(self, node):
+    def _write_fstring_inner(self, node, is_format_spec=False):
         if isinstance(node, JoinedStr):
             # for both the f-string itself, and format_spec
             for value in node.values:
-                self._write_fstring_inner(value)
+                self._write_fstring_inner(value, is_format_spec=is_format_spec)
         elif isinstance(node, Constant) and isinstance(node.value, str):
             value = node.value.replace("{", "{{").replace("}", "}}")
+
+            if is_format_spec:
+                value = value.replace("\\", "\\\\")
+                value = value.replace("'", "\\'")
+                value = value.replace('"', '\\"')
+                value = value.replace("\n", "\\n")
             self.write(value)
         elif isinstance(node, FormattedValue):
             self.visit_FormattedValue(node)
@@ -1297,7 +1331,7 @@ class _Unparser(NodeVisitor):
                 self.write(f"!{chr(node.conversion)}")
             if node.format_spec:
                 self.write(":")
-                self._write_fstring_inner(node.format_spec)
+                self._write_fstring_inner(node.format_spec, is_format_spec=True)
 
     def visit_Name(self, node):
         self.write(node.id)
@@ -1317,8 +1351,6 @@ class _Unparser(NodeVisitor):
                 .replace("inf", _INFSTR)
                 .replace("nan", f"({_INFSTR}-{_INFSTR})")
             )
-        elif self._avoid_backslashes and isinstance(value, str):
-            self._write_str_avoiding_backslashes(value)
         else:
             self.write(repr(value))
 
@@ -1805,8 +1837,7 @@ def main():
     import argparse
 
     parser = argparse.ArgumentParser(prog='python -m ast')
-    parser.add_argument('infile', type=argparse.FileType(mode='rb'), nargs='?',
-                        default='-',
+    parser.add_argument('infile', nargs='?', default='-',
                         help='the file to parse; defaults to stdin')
     parser.add_argument('-m', '--mode', default='exec',
                         choices=('exec', 'single', 'eval', 'func_type'),
@@ -1820,9 +1851,14 @@ def main():
                         help='indentation of nodes (number of spaces)')
     args = parser.parse_args()
 
-    with args.infile as infile:
-        source = infile.read()
-    tree = parse(source, args.infile.name, args.mode, type_comments=args.no_type_comments)
+    if args.infile == '-':
+        name = '<stdin>'
+        source = sys.stdin.buffer.read()
+    else:
+        name = args.infile
+        with open(args.infile, 'rb') as infile:
+            source = infile.read()
+    tree = parse(source, name, args.mode, type_comments=args.no_type_comments)
     print(dump(tree, include_attributes=args.include_attributes, indent=args.indent))
 
 if __name__ == '__main__':

Lib/bz2.py

@@ -17,7 +17,7 @@ import _compression
 from _bz2 import BZ2Compressor, BZ2Decompressor
 
 
-_MODE_CLOSED   = 0
+# Value 0 no longer used
 _MODE_READ     = 1
 # Value 2 no longer used
 _MODE_WRITE    = 3
@@ -54,7 +54,7 @@ class BZ2File(_compression.BaseStream):
         """
         self._fp = None
         self._closefp = False
-        self._mode = _MODE_CLOSED
+        self._mode = None
 
         if not (1 <= compresslevel <= 9):
             raise ValueError("compresslevel must be between 1 and 9")
@@ -100,7 +100,7 @@ class BZ2File(_compression.BaseStream):
         May be called more than once without error. Once the file is
         closed, any other operation on it will raise a ValueError.
         """
-        if self._mode == _MODE_CLOSED:
+        if self.closed:
             return
         try:
             if self._mode == _MODE_READ:
@@ -115,13 +115,21 @@ class BZ2File(_compression.BaseStream):
             finally:
                 self._fp = None
                 self._closefp = False
-                self._mode = _MODE_CLOSED
                 self._buffer = None
 
     @property
     def closed(self):
         """True if this file is closed."""
-        return self._mode == _MODE_CLOSED
+        return self._fp is None
+
+    @property
+    def name(self):
+        self._check_not_closed()
+        return self._fp.name
+
+    @property
+    def mode(self):
+        return 'wb' if self._mode == _MODE_WRITE else 'rb'
 
     def fileno(self):
         """Return the file descriptor for the underlying file."""

Lib/cmd.py

@@ -42,7 +42,7 @@ listings of documented functions, miscellaneous topics, and undocumented
 functions respectively.
 """
 
-import string, sys
+import inspect, string, sys
 
 __all__ = ["Cmd"]
 
@@ -108,7 +108,15 @@ class Cmd:
                 import readline
                 self.old_completer = readline.get_completer()
                 readline.set_completer(self.complete)
-                readline.parse_and_bind(self.completekey+": complete")
+                if readline.backend == "editline":
+                    if self.completekey == 'tab':
+                        # libedit uses "^I" instead of "tab"
+                        command_string = "bind ^I rl_complete"
+                    else:
+                        command_string = f"bind {self.completekey} rl_complete"
+                else:
+                    command_string = f"{self.completekey}: complete"
+                readline.parse_and_bind(command_string)
             except ImportError:
                 pass
         try:
@@ -210,9 +218,8 @@ class Cmd:
         if cmd == '':
             return self.default(line)
         else:
-            try:
-                func = getattr(self, 'do_' + cmd)
-            except AttributeError:
+            func = getattr(self, 'do_' + cmd, None)
+            if func is None:
                 return self.default(line)
             return func(arg)
 
@@ -298,6 +305,7 @@ class Cmd:
             except AttributeError:
                 try:
                     doc=getattr(self, 'do_' + arg).__doc__
+                    doc = inspect.cleandoc(doc)
                     if doc:
                         self.stdout.write("%s\n"%str(doc))
                         return

Lib/codeop.py

@@ -44,6 +44,7 @@ __all__ = ["compile_command", "Compile", "CommandCompiler"]
 # Caveat emptor: These flags are undocumented on purpose and depending
 # on their effect outside the standard library is **unsupported**.
 PyCF_DONT_IMPLY_DEDENT = 0x200
+PyCF_ONLY_AST = 0x400
 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
 
 def _maybe_compile(compiler, source, filename, symbol):
@@ -73,15 +74,6 @@ def _maybe_compile(compiler, source, filename, symbol):
 
     return compiler(source, filename, symbol, incomplete_input=False)
 
-def _is_syntax_error(err1, err2):
-    rep1 = repr(err1)
-    rep2 = repr(err2)
-    if "was never closed" in rep1 and "was never closed" in rep2:
-        return False
-    if rep1 == rep2:
-        return True
-    return False
-
 def _compile(source, filename, symbol, incomplete_input=True):
     flags = 0
     if incomplete_input:
@@ -89,7 +81,6 @@ def _compile(source, filename, symbol, incomplete_input=True):
         flags |= PyCF_DONT_IMPLY_DEDENT
     return compile(source, filename, symbol, flags)
 
-
 def compile_command(source, filename="<input>", symbol="single"):
     r"""Compile a command and determine whether it is incomplete.
 
@@ -119,12 +110,14 @@ class Compile:
     def __init__(self):
         self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
 
-    def __call__(self, source, filename, symbol, **kwargs):
-        flags = self.flags
+    def __call__(self, source, filename, symbol, flags=0, **kwargs):
+        flags |= self.flags
         if kwargs.get('incomplete_input', True) is False:
             flags &= ~PyCF_DONT_IMPLY_DEDENT
             flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT
         codeob = compile(source, filename, symbol, flags, True)
+        if flags & PyCF_ONLY_AST:
+            return codeob  # this is an ast.Module in this case
         for feature in _features:
             if codeob.co_flags & feature.compiler_flag:
                 self.flags |= feature.compiler_flag

Lib/compileall.py

@@ -97,9 +97,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
     success = True
     if workers != 1 and ProcessPoolExecutor is not None:
+        import multiprocessing
+        if multiprocessing.get_start_method() == 'fork':
+            mp_context = multiprocessing.get_context('forkserver')
+        else:
+            mp_context = None
         # If workers == 0, let ProcessPoolExecutor choose
         workers = workers or None
-        with ProcessPoolExecutor(max_workers=workers) as executor:
+        with ProcessPoolExecutor(max_workers=workers,
+                                 mp_context=mp_context) as executor:
             results = executor.map(partial(compile_file,
                                            ddir=ddir, force=force,
                                            rx=rx, quiet=quiet,
@@ -110,7 +116,8 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
                                            prependdir=prependdir,
                                            limit_sl_dest=limit_sl_dest,
                                            hardlink_dupes=hardlink_dupes),
-                                   files)
+                                   files,
+                                   chunksize=4)
             success = min(results, default=True)
     else:
         for file in files:
@@ -166,13 +173,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
     if stripdir is not None:
         fullname_parts = fullname.split(os.path.sep)
         stripdir_parts = stripdir.split(os.path.sep)
-        ddir_parts = list(fullname_parts)
 
-        for spart, opart in zip(stripdir_parts, fullname_parts):
-            if spart == opart:
-                ddir_parts.remove(spart)
-
-        dfile = os.path.join(*ddir_parts)
+        if stripdir_parts != fullname_parts[:len(stripdir_parts)]:
+            if quiet < 2:
+                print("The stripdir path {!r} is not a valid prefix for "
+                      "source path {!r}; ignoring".format(stripdir, fullname))
+        else:
+            dfile = os.path.join(*fullname_parts[len(stripdir_parts):])
 
     if prependdir is not None:
         if dfile is None:

Lib/configparser.py

@@ -18,8 +18,8 @@ ConfigParser -- responsible for parsing a list of
              delimiters=('=', ':'), comment_prefixes=('#', ';'),
              inline_comment_prefixes=None, strict=True,
              empty_lines_in_values=True, default_section='DEFAULT',
-             interpolation=<unset>, converters=<unset>):
-
+             interpolation=<unset>, converters=<unset>,
+             allow_unnamed_section=False):
         Create the parser. When `defaults` is given, it is initialized into the
         dictionary or intrinsic defaults. The keys must be strings, the values
         must be appropriate for %()s string interpolation.
@@ -68,6 +68,10 @@ ConfigParser -- responsible for parsing a list of
         converter gets its corresponding get*() method on the parser object and
         section proxies.
 
+        When `allow_unnamed_section` is True (default: False), options
+        without section are accepted: the section for these is
+        ``configparser.UNNAMED_SECTION``.
+
     sections()
         Return all the configuration section names, sans DEFAULT.
 
@@ -139,24 +143,28 @@ ConfigParser -- responsible for parsing a list of
         between keys and values are surrounded by spaces.
 """
 
-from collections.abc import MutableMapping
+# Do not import dataclasses; overhead is unacceptable (gh-117703)
+
+from collections.abc import Iterable, MutableMapping
 from collections import ChainMap as _ChainMap
+import contextlib
 import functools
 import io
 import itertools
 import os
 import re
 import sys
-import warnings
+import types
 
 __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError",
            "NoOptionError", "InterpolationError", "InterpolationDepthError",
            "InterpolationMissingOptionError", "InterpolationSyntaxError",
            "ParsingError", "MissingSectionHeaderError",
+           "MultilineContinuationError",
            "ConfigParser", "RawConfigParser",
            "Interpolation", "BasicInterpolation",  "ExtendedInterpolation",
-           "LegacyInterpolation", "SectionProxy", "ConverterMapping",
-           "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH")
+           "SectionProxy", "ConverterMapping",
+           "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION")
 
 _default_dict = dict
 DEFAULTSECT = "DEFAULT"
@@ -298,15 +306,33 @@ class InterpolationDepthError(InterpolationError):
 class ParsingError(Error):
     """Raised when a configuration file does not follow legal syntax."""
 
-    def __init__(self, source):
+    def __init__(self, source, *args):
         super().__init__(f'Source contains parsing errors: {source!r}')
         self.source = source
         self.errors = []
         self.args = (source, )
+        if args:
+            self.append(*args)
 
     def append(self, lineno, line):
         self.errors.append((lineno, line))
-        self.message += '\n\t[line %2d]: %s' % (lineno, line)
+        self.message += '\n\t[line %2d]: %s' % (lineno, repr(line))
+
+    def combine(self, others):
+        for other in others:
+            for error in other.errors:
+                self.append(*error)
+        return self
+
+    @staticmethod
+    def _raise_all(exceptions: Iterable['ParsingError']):
+        """
+        Combine any number of ParsingErrors into one and raise it.
+        """
+        exceptions = iter(exceptions)
+        with contextlib.suppress(StopIteration):
+            raise next(exceptions).combine(exceptions)
+
 
 
 class MissingSectionHeaderError(ParsingError):
@@ -323,6 +349,28 @@ class MissingSectionHeaderError(ParsingError):
         self.args = (filename, lineno, line)
 
 
+class MultilineContinuationError(ParsingError):
+    """Raised when a key without value is followed by continuation line"""
+    def __init__(self, filename, lineno, line):
+        Error.__init__(
+            self,
+            "Key without value continued with an indented line.\n"
+            "file: %r, line: %d\n%r"
+            %(filename, lineno, line))
+        self.source = filename
+        self.lineno = lineno
+        self.line = line
+        self.args = (filename, lineno, line)
+
+class _UnnamedSection:
+
+    def __repr__(self):
+        return "<UNNAMED_SECTION>"
+
+
+UNNAMED_SECTION = _UnnamedSection()
+
+
 # Used in parser getters to indicate the default behaviour when a specific
 # option is not found it to raise an exception. Created to enable `None` as
 # a valid fallback value.
@@ -478,6 +526,8 @@ class ExtendedInterpolation(Interpolation):
                 except (KeyError, NoSectionError, NoOptionError):
                     raise InterpolationMissingOptionError(
                         option, section, rawval, ":".join(path)) from None
+                if v is None:
+                    continue
                 if "$" in v:
                     self._interpolate_some(parser, opt, accum, v, sect,
                                            dict(parser.items(sect, raw=True)),
@@ -491,51 +541,50 @@ class ExtendedInterpolation(Interpolation):
                     "found: %r" % (rest,))
 
 
-class LegacyInterpolation(Interpolation):
-    """Deprecated interpolation used in old versions of ConfigParser.
-    Use BasicInterpolation or ExtendedInterpolation instead."""
+class _ReadState:
+    elements_added : set[str]
+    cursect : dict[str, str] | None = None
+    sectname : str | None = None
+    optname : str | None = None
+    lineno : int = 0
+    indent_level : int = 0
+    errors : list[ParsingError]
 
-    _KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
+    def __init__(self):
+        self.elements_added = set()
+        self.errors = list()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        warnings.warn(
-            "LegacyInterpolation has been deprecated since Python 3.2 "
-            "and will be removed from the configparser module in Python 3.13. "
-            "Use BasicInterpolation or ExtendedInterpolation instead.",
-            DeprecationWarning, stacklevel=2
+
+class _Line(str):
+
+    def __new__(cls, val, *args, **kwargs):
+        return super().__new__(cls, val)
+
+    def __init__(self, val, prefixes):
+        self.prefixes = prefixes
+
+    @functools.cached_property
+    def clean(self):
+        return self._strip_full() and self._strip_inline()
+
+    @property
+    def has_comments(self):
+        return self.strip() != self.clean
+
+    def _strip_inline(self):
+        """
+        Search for the earliest prefix at the beginning of the line or following a space.
+        """
+        matcher = re.compile(
+            '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline)
+            # match nothing if no prefixes
+            or '(?!)'
         )
+        match = matcher.search(self)
+        return self[:match.start() if match else None].strip()
 
-    def before_get(self, parser, section, option, value, vars):
-        rawval = value
-        depth = MAX_INTERPOLATION_DEPTH
-        while depth:                    # Loop through this until it's done
-            depth -= 1
-            if value and "%(" in value:
-                replace = functools.partial(self._interpolation_replace,
-                                            parser=parser)
-                value = self._KEYCRE.sub(replace, value)
-                try:
-                    value = value % vars
-                except KeyError as e:
-                    raise InterpolationMissingOptionError(
-                        option, section, rawval, e.args[0]) from None
-            else:
-                break
-        if value and "%(" in value:
-            raise InterpolationDepthError(option, section, rawval)
-        return value
-
-    def before_set(self, parser, section, option, value):
-        return value
-
-    @staticmethod
-    def _interpolation_replace(match, parser):
-        s = match.group(1)
-        if s is None:
-            return match.group()
-        else:
-            return "%%(%s)s" % parser.optionxform(s)
+    def _strip_full(self):
+        return '' if any(map(self.strip().startswith, self.prefixes.full)) else True
 
 
 class RawConfigParser(MutableMapping):
@@ -584,7 +633,8 @@ class RawConfigParser(MutableMapping):
                  comment_prefixes=('#', ';'), inline_comment_prefixes=None,
                  strict=True, empty_lines_in_values=True,
                  default_section=DEFAULTSECT,
-                 interpolation=_UNSET, converters=_UNSET):
+                 interpolation=_UNSET, converters=_UNSET,
+                 allow_unnamed_section=False,):
 
         self._dict = dict_type
         self._sections = self._dict()
@@ -603,8 +653,10 @@ class RawConfigParser(MutableMapping):
             else:
                 self._optcre = re.compile(self._OPT_TMPL.format(delim=d),
                                           re.VERBOSE)
-        self._comment_prefixes = tuple(comment_prefixes or ())
-        self._inline_comment_prefixes = tuple(inline_comment_prefixes or ())
+        self._prefixes = types.SimpleNamespace(
+            full=tuple(comment_prefixes or ()),
+            inline=tuple(inline_comment_prefixes or ()),
+        )
         self._strict = strict
         self._allow_no_value = allow_no_value
         self._empty_lines_in_values = empty_lines_in_values
@@ -623,6 +675,7 @@ class RawConfigParser(MutableMapping):
             self._converters.update(converters)
         if defaults:
             self._read_defaults(defaults)
+        self._allow_unnamed_section = allow_unnamed_section
 
     def defaults(self):
         return self._defaults
@@ -896,13 +949,19 @@ class RawConfigParser(MutableMapping):
         if self._defaults:
             self._write_section(fp, self.default_section,
                                     self._defaults.items(), d)
+        if UNNAMED_SECTION in self._sections:
+            self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True)
+
         for section in self._sections:
+            if section is UNNAMED_SECTION:
+                continue
             self._write_section(fp, section,
                                 self._sections[section].items(), d)
 
-    def _write_section(self, fp, section_name, section_items, delimiter):
-        """Write a single section to the specified `fp`."""
-        fp.write("[{}]\n".format(section_name))
+    def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False):
+        """Write a single section to the specified `fp'."""
+        if not unnamed:
+            fp.write("[{}]\n".format(section_name))
         for key, value in section_items:
             value = self._interpolation.before_write(self, section_name, key,
                                                      value)
@@ -988,110 +1047,113 @@ class RawConfigParser(MutableMapping):
         in an otherwise empty line or may be entered in lines holding values or
         section names. Please note that comments get stripped off when reading configuration files.
         """
-        elements_added = set()
-        cursect = None                        # None, or a dictionary
-        sectname = None
-        optname = None
-        lineno = 0
-        indent_level = 0
-        e = None                              # None, or an exception
-        for lineno, line in enumerate(fp, start=1):
-            comment_start = sys.maxsize
-            # strip inline comments
-            inline_prefixes = {p: -1 for p in self._inline_comment_prefixes}
-            while comment_start == sys.maxsize and inline_prefixes:
-                next_prefixes = {}
-                for prefix, index in inline_prefixes.items():
-                    index = line.find(prefix, index+1)
-                    if index == -1:
-                        continue
-                    next_prefixes[prefix] = index
-                    if index == 0 or (index > 0 and line[index-1].isspace()):
-                        comment_start = min(comment_start, index)
-                inline_prefixes = next_prefixes
-            # strip full line comments
-            for prefix in self._comment_prefixes:
-                if line.strip().startswith(prefix):
-                    comment_start = 0
-                    break
-            if comment_start == sys.maxsize:
-                comment_start = None
-            value = line[:comment_start].strip()
-            if not value:
+
+        try:
+            ParsingError._raise_all(self._read_inner(fp, fpname))
+        finally:
+            self._join_multiline_values()
+
+    def _read_inner(self, fp, fpname):
+        st = _ReadState()
+
+        Line = functools.partial(_Line, prefixes=self._prefixes)
+        for st.lineno, line in enumerate(map(Line, fp), start=1):
+            if not line.clean:
                 if self._empty_lines_in_values:
                     # add empty line to the value, but only if there was no
                     # comment on the line
-                    if (comment_start is None and
-                        cursect is not None and
-                        optname and
-                        cursect[optname] is not None):
-                        cursect[optname].append('') # newlines added at join
+                    if (not line.has_comments and
+                        st.cursect is not None and
+                        st.optname and
+                        st.cursect[st.optname] is not None):
+                        st.cursect[st.optname].append('') # newlines added at join
                 else:
                     # empty line marks end of value
-                    indent_level = sys.maxsize
+                    st.indent_level = sys.maxsize
                 continue
-            # continuation line?
+
             first_nonspace = self.NONSPACECRE.search(line)
-            cur_indent_level = first_nonspace.start() if first_nonspace else 0
-            if (cursect is not None and optname and
-                cur_indent_level > indent_level):
-                cursect[optname].append(value)
-            # a section header or option header?
-            else:
-                indent_level = cur_indent_level
-                # is it a section header?
-                mo = self.SECTCRE.match(value)
-                if mo:
-                    sectname = mo.group('header')
-                    if sectname in self._sections:
-                        if self._strict and sectname in elements_added:
-                            raise DuplicateSectionError(sectname, fpname,
-                                                        lineno)
-                        cursect = self._sections[sectname]
-                        elements_added.add(sectname)
-                    elif sectname == self.default_section:
-                        cursect = self._defaults
-                    else:
-                        cursect = self._dict()
-                        self._sections[sectname] = cursect
-                        self._proxies[sectname] = SectionProxy(self, sectname)
-                        elements_added.add(sectname)
-                    # So sections can't start with a continuation line
-                    optname = None
-                # no section header in the file?
-                elif cursect is None:
-                    raise MissingSectionHeaderError(fpname, lineno, line)
-                # an option line?
-                else:
-                    mo = self._optcre.match(value)
-                    if mo:
-                        optname, vi, optval = mo.group('option', 'vi', 'value')
-                        if not optname:
-                            e = self._handle_error(e, fpname, lineno, line)
-                        optname = self.optionxform(optname.rstrip())
-                        if (self._strict and
-                            (sectname, optname) in elements_added):
-                            raise DuplicateOptionError(sectname, optname,
-                                                       fpname, lineno)
-                        elements_added.add((sectname, optname))
-                        # This check is fine because the OPTCRE cannot
-                        # match if it would set optval to None
-                        if optval is not None:
-                            optval = optval.strip()
-                            cursect[optname] = [optval]
-                        else:
-                            # valueless option handling
-                            cursect[optname] = None
-                    else:
-                        # a non-fatal parsing error occurred. set up the
-                        # exception but keep going. the exception will be
-                        # raised at the end of the file and will contain a
-                        # list of all bogus lines
-                        e = self._handle_error(e, fpname, lineno, line)
-        self._join_multiline_values()
-        # if any parsing errors occurred, raise an exception
-        if e:
-            raise e
+            st.cur_indent_level = first_nonspace.start() if first_nonspace else 0
+
+            if self._handle_continuation_line(st, line, fpname):
+                continue
+
+            self._handle_rest(st, line, fpname)
+
+        return st.errors
+
+    def _handle_continuation_line(self, st, line, fpname):
+        # continuation line?
+        is_continue = (st.cursect is not None and st.optname and
+            st.cur_indent_level > st.indent_level)
+        if is_continue:
+            if st.cursect[st.optname] is None:
+                raise MultilineContinuationError(fpname, st.lineno, line)
+            st.cursect[st.optname].append(line.clean)
+        return is_continue
+
+    def _handle_rest(self, st, line, fpname):
+        # a section header or option header?
+        if self._allow_unnamed_section and st.cursect is None:
+            self._handle_header(st, UNNAMED_SECTION, fpname)
+
+        st.indent_level = st.cur_indent_level
+        # is it a section header?
+        mo = self.SECTCRE.match(line.clean)
+
+        if not mo and st.cursect is None:
+            raise MissingSectionHeaderError(fpname, st.lineno, line)
+
+        self._handle_header(st, mo.group('header'), fpname) if mo else self._handle_option(st, line, fpname)
+
+    def _handle_header(self, st, sectname, fpname):
+        st.sectname = sectname
+        if st.sectname in self._sections:
+            if self._strict and st.sectname in st.elements_added:
+                raise DuplicateSectionError(st.sectname, fpname,
+                                            st.lineno)
+            st.cursect = self._sections[st.sectname]
+            st.elements_added.add(st.sectname)
+        elif st.sectname == self.default_section:
+            st.cursect = self._defaults
+        else:
+            st.cursect = self._dict()
+            self._sections[st.sectname] = st.cursect
+            self._proxies[st.sectname] = SectionProxy(self, st.sectname)
+            st.elements_added.add(st.sectname)
+        # So sections can't start with a continuation line
+        st.optname = None
+
+    def _handle_option(self, st, line, fpname):
+        # an option line?
+        st.indent_level = st.cur_indent_level
+
+        mo = self._optcre.match(line.clean)
+        if not mo:
+            # a non-fatal parsing error occurred. set up the
+            # exception but keep going. the exception will be
+            # raised at the end of the file and will contain a
+            # list of all bogus lines
+            st.errors.append(ParsingError(fpname, st.lineno, line))
+            return
+
+        st.optname, vi, optval = mo.group('option', 'vi', 'value')
+        if not st.optname:
+            st.errors.append(ParsingError(fpname, st.lineno, line))
+        st.optname = self.optionxform(st.optname.rstrip())
+        if (self._strict and
+            (st.sectname, st.optname) in st.elements_added):
+            raise DuplicateOptionError(st.sectname, st.optname,
+                                    fpname, st.lineno)
+        st.elements_added.add((st.sectname, st.optname))
+        # This check is fine because the OPTCRE cannot
+        # match if it would set optval to None
+        if optval is not None:
+            optval = optval.strip()
+            st.cursect[st.optname] = [optval]
+        else:
+            # valueless option handling
+            st.cursect[st.optname] = None
 
     def _join_multiline_values(self):
         defaults = self.default_section, self._defaults
@@ -1111,12 +1173,6 @@ class RawConfigParser(MutableMapping):
         for key, value in defaults.items():
             self._defaults[self.optionxform(key)] = value
 
-    def _handle_error(self, exc, fpname, lineno, line):
-        if not exc:
-            exc = ParsingError(fpname)
-        exc.append(lineno, repr(line))
-        return exc
-
     def _unify_values(self, section, vars):
         """Create a sequence of lookups with 'vars' taking priority over
         the 'section' which takes priority over the DEFAULTSECT.

Lib/contextlib.py

@@ -20,6 +20,8 @@ class AbstractContextManager(abc.ABC):
 
     __class_getitem__ = classmethod(GenericAlias)
 
+    __slots__ = ()
+
     def __enter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -42,6 +44,8 @@ class AbstractAsyncContextManager(abc.ABC):
 
     __class_getitem__ = classmethod(GenericAlias)
 
+    __slots__ = ()
+
     async def __aenter__(self):
         """Return `self` upon entering the runtime context."""
         return self
@@ -565,11 +569,12 @@ class ExitStack(_BaseExitStack, AbstractContextManager):
         return self
 
     def __exit__(self, *exc_details):
-        received_exc = exc_details[0] is not None
+        exc = exc_details[1]
+        received_exc = exc is not None
 
         # We manipulate the exception state so it behaves as though
         # we were actually nesting multiple with statements
-        frame_exc = sys.exc_info()[1]
+        frame_exc = sys.exception()
         def _fix_exception_context(new_exc, old_exc):
             # Context may not be correct, so find the end of the chain
             while 1:
@@ -592,24 +597,28 @@ class ExitStack(_BaseExitStack, AbstractContextManager):
             is_sync, cb = self._exit_callbacks.pop()
             assert is_sync
             try:
+                if exc is None:
+                    exc_details = None, None, None
+                else:
+                    exc_details = type(exc), exc, exc.__traceback__
                 if cb(*exc_details):
                     suppressed_exc = True
                     pending_raise = False
-                    exc_details = (None, None, None)
-            except:
-                new_exc_details = sys.exc_info()
+                    exc = None
+            except BaseException as new_exc:
                 # simulate the stack of exceptions by setting the context
-                _fix_exception_context(new_exc_details[1], exc_details[1])
+                _fix_exception_context(new_exc, exc)
                 pending_raise = True
-                exc_details = new_exc_details
+                exc = new_exc
+
         if pending_raise:
             try:
-                # bare "raise exc_details[1]" replaces our carefully
+                # bare "raise exc" replaces our carefully
                 # set-up context
-                fixed_ctx = exc_details[1].__context__
-                raise exc_details[1]
+                fixed_ctx = exc.__context__
+                raise exc
             except BaseException:
-                exc_details[1].__context__ = fixed_ctx
+                exc.__context__ = fixed_ctx
                 raise
         return received_exc and suppressed_exc
 
@@ -705,11 +714,12 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         return self
 
     async def __aexit__(self, *exc_details):
-        received_exc = exc_details[0] is not None
+        exc = exc_details[1]
+        received_exc = exc is not None
 
         # We manipulate the exception state so it behaves as though
         # we were actually nesting multiple with statements
-        frame_exc = sys.exc_info()[1]
+        frame_exc = sys.exception()
         def _fix_exception_context(new_exc, old_exc):
             # Context may not be correct, so find the end of the chain
             while 1:
@@ -731,6 +741,10 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         while self._exit_callbacks:
             is_sync, cb = self._exit_callbacks.pop()
             try:
+                if exc is None:
+                    exc_details = None, None, None
+                else:
+                    exc_details = type(exc), exc, exc.__traceback__
                 if is_sync:
                     cb_suppress = cb(*exc_details)
                 else:
@@ -739,21 +753,21 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
                 if cb_suppress:
                     suppressed_exc = True
                     pending_raise = False
-                    exc_details = (None, None, None)
-            except:
-                new_exc_details = sys.exc_info()
+                    exc = None
+            except BaseException as new_exc:
                 # simulate the stack of exceptions by setting the context
-                _fix_exception_context(new_exc_details[1], exc_details[1])
+                _fix_exception_context(new_exc, exc)
                 pending_raise = True
-                exc_details = new_exc_details
+                exc = new_exc
+
         if pending_raise:
             try:
-                # bare "raise exc_details[1]" replaces our carefully
+                # bare "raise exc" replaces our carefully
                 # set-up context
-                fixed_ctx = exc_details[1].__context__
-                raise exc_details[1]
+                fixed_ctx = exc.__context__
+                raise exc
             except BaseException:
-                exc_details[1].__context__ = fixed_ctx
+                exc.__context__ = fixed_ctx
                 raise
         return received_exc and suppressed_exc
 

Lib/copy.py

@@ -4,8 +4,9 @@ Interface summary:
 
         import copy
 
-        x = copy.copy(y)        # make a shallow copy of y
-        x = copy.deepcopy(y)    # make a deep copy of y
+        x = copy.copy(y)                # make a shallow copy of y
+        x = copy.deepcopy(y)            # make a deep copy of y
+        x = copy.replace(y, a=1, b=2)   # new object with fields replaced, as defined by `__replace__`
 
 For module specific errors, copy.Error is raised.
 
@@ -56,7 +57,7 @@ class Error(Exception):
     pass
 error = Error   # backward compatibility
 
-__all__ = ["Error", "copy", "deepcopy"]
+__all__ = ["Error", "copy", "deepcopy", "replace"]
 
 def copy(x):
     """Shallow copy operation on arbitrary Python objects.
@@ -121,13 +122,13 @@ def deepcopy(x, memo=None, _nil=[]):
     See the module's __doc__ string for more info.
     """
 
+    d = id(x)
     if memo is None:
         memo = {}
-
-    d = id(x)
-    y = memo.get(d, _nil)
-    if y is not _nil:
-        return y
+    else:
+        y = memo.get(d, _nil)
+        if y is not _nil:
+            return y
 
     cls = type(x)
 
@@ -290,3 +291,16 @@ def _reconstruct(x, memo, func, args,
     return y
 
 del types, weakref
+
+
+def replace(obj, /, **changes):
+    """Return a new object replacing specified fields with new values.
+
+    This is especially useful for immutable objects, like named tuples or
+    frozen dataclasses.
+    """
+    cls = obj.__class__
+    func = getattr(cls, '__replace__', None)
+    if func is None:
+        raise TypeError(f"replace() does not support {cls.__name__} objects")
+    return func(obj, **changes)

Lib/csv.py

@@ -1,28 +1,90 @@
 
-"""
-csv.py - read/write/investigate CSV files
+r"""
+CSV parsing and writing.
+
+This module provides classes that assist in the reading and writing
+of Comma Separated Value (CSV) files, and implements the interface
+described by PEP 305.  Although many CSV files are simple to parse,
+the format is not formally defined by a stable specification and
+is subtle enough that parsing lines of a CSV file with something
+like line.split(",") is bound to fail.  The module supports three
+basic APIs: reading, writing, and registration of dialects.
+
+
+DIALECT REGISTRATION:
+
+Readers and writers support a dialect argument, which is a convenient
+handle on a group of settings.  When the dialect argument is a string,
+it identifies one of the dialects previously registered with the module.
+If it is a class or instance, the attributes of the argument are used as
+the settings for the reader or writer:
+
+    class excel:
+        delimiter = ','
+        quotechar = '"'
+        escapechar = None
+        doublequote = True
+        skipinitialspace = False
+        lineterminator = '\r\n'
+        quoting = QUOTE_MINIMAL
+
+SETTINGS:
+
+    * quotechar - specifies a one-character string to use as the
+        quoting character.  It defaults to '"'.
+    * delimiter - specifies a one-character string to use as the
+        field separator.  It defaults to ','.
+    * skipinitialspace - specifies how to interpret spaces which
+        immediately follow a delimiter.  It defaults to False, which
+        means that spaces immediately following a delimiter is part
+        of the following field.
+    * lineterminator - specifies the character sequence which should
+        terminate rows.
+    * quoting - controls when quotes should be generated by the writer.
+        It can take on any of the following module constants:
+
+        csv.QUOTE_MINIMAL means only when required, for example, when a
+            field contains either the quotechar or the delimiter
+        csv.QUOTE_ALL means that quotes are always placed around fields.
+        csv.QUOTE_NONNUMERIC means that quotes are always placed around
+            fields which do not parse as integers or floating-point
+            numbers.
+        csv.QUOTE_STRINGS means that quotes are always placed around
+            fields which are strings.  Note that the Python value None
+            is not a string.
+        csv.QUOTE_NOTNULL means that quotes are only placed around fields
+            that are not the Python value None.
+        csv.QUOTE_NONE means that quotes are never placed around fields.
+    * escapechar - specifies a one-character string used to escape
+        the delimiter when quoting is set to QUOTE_NONE.
+    * doublequote - controls the handling of quotes inside fields.  When
+        True, two consecutive quotes are interpreted as one during read,
+        and when writing, each quote character embedded in the data is
+        written as two quotes
 """
 
 import re
 import types
-from _csv import Error, __version__, writer, reader, register_dialect, \
+from _csv import Error, writer, reader, register_dialect, \
                  unregister_dialect, get_dialect, list_dialects, \
                  field_size_limit, \
                  QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \
-                 QUOTE_STRINGS, QUOTE_NOTNULL, \
-                 __doc__
+                 QUOTE_STRINGS, QUOTE_NOTNULL
 from _csv import Dialect as _Dialect
 
 from io import StringIO
 
 __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE",
            "QUOTE_STRINGS", "QUOTE_NOTNULL",
-           "Error", "Dialect", "__doc__", "excel", "excel_tab",
+           "Error", "Dialect", "excel", "excel_tab",
            "field_size_limit", "reader", "writer",
            "register_dialect", "get_dialect", "list_dialects", "Sniffer",
-           "unregister_dialect", "__version__", "DictReader", "DictWriter",
+           "unregister_dialect", "DictReader", "DictWriter",
            "unix_dialect"]
 
+__version__ = "1.0"
+
+
 class Dialect:
     """Describe a CSV dialect.
 
@@ -51,8 +113,8 @@ class Dialect:
         try:
             _Dialect(self)
         except TypeError as e:
-            # We do this for compatibility with py2.3
-            raise Error(str(e))
+            # Re-raise to get a traceback showing more user code.
+            raise Error(str(e)) from None
 
 class excel(Dialect):
     """Describe the usual properties of Excel-generated CSV files."""

Lib/decimal.py

@@ -1,11 +1,109 @@
+"""Decimal fixed-point and floating-point arithmetic.
+
+This is an implementation of decimal floating-point arithmetic based on
+the General Decimal Arithmetic Specification:
+
+    http://speleotrove.com/decimal/decarith.html
+
+and IEEE standard 854-1987:
+
+    http://en.wikipedia.org/wiki/IEEE_854-1987
+
+Decimal floating point has finite precision with arbitrarily large bounds.
+
+The purpose of this module is to support arithmetic using familiar
+"schoolhouse" rules and to avoid some of the tricky representation
+issues associated with binary floating point.  The package is especially
+useful for financial applications or for contexts where users have
+expectations that are at odds with binary floating point (for instance,
+in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead
+of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected
+Decimal('0.00')).
+
+Here are some examples of using the decimal module:
+
+>>> from decimal import *
+>>> setcontext(ExtendedContext)
+>>> Decimal(0)
+Decimal('0')
+>>> Decimal('1')
+Decimal('1')
+>>> Decimal('-.0123')
+Decimal('-0.0123')
+>>> Decimal(123456)
+Decimal('123456')
+>>> Decimal('123.45e12345678')
+Decimal('1.2345E+12345680')
+>>> Decimal('1.33') + Decimal('1.27')
+Decimal('2.60')
+>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')
+Decimal('-2.20')
+>>> dig = Decimal(1)
+>>> print(dig / Decimal(3))
+0.333333333
+>>> getcontext().prec = 18
+>>> print(dig / Decimal(3))
+0.333333333333333333
+>>> print(dig.sqrt())
+1
+>>> print(Decimal(3).sqrt())
+1.73205080756887729
+>>> print(Decimal(3) ** 123)
+4.85192780976896427E+58
+>>> inf = Decimal(1) / Decimal(0)
+>>> print(inf)
+Infinity
+>>> neginf = Decimal(-1) / Decimal(0)
+>>> print(neginf)
+-Infinity
+>>> print(neginf + inf)
+NaN
+>>> print(neginf * inf)
+-Infinity
+>>> print(dig / 0)
+Infinity
+>>> getcontext().traps[DivisionByZero] = 1
+>>> print(dig / 0)
+Traceback (most recent call last):
+  ...
+  ...
+  ...
+decimal.DivisionByZero: x / 0
+>>> c = Context()
+>>> c.traps[InvalidOperation] = 0
+>>> print(c.flags[InvalidOperation])
+0
+>>> c.divide(Decimal(0), Decimal(0))
+Decimal('NaN')
+>>> c.traps[InvalidOperation] = 1
+>>> print(c.flags[InvalidOperation])
+1
+>>> c.flags[InvalidOperation] = 0
+>>> print(c.flags[InvalidOperation])
+0
+>>> print(c.divide(Decimal(0), Decimal(0)))
+Traceback (most recent call last):
+  ...
+  ...
+  ...
+decimal.InvalidOperation: 0 / 0
+>>> print(c.flags[InvalidOperation])
+1
+>>> c.flags[InvalidOperation] = 0
+>>> c.traps[InvalidOperation] = 0
+>>> print(c.divide(Decimal(0), Decimal(0)))
+NaN
+>>> print(c.flags[InvalidOperation])
+1
+>>>
+"""
 
 try:
     from _decimal import *
-    from _decimal import __doc__
     from _decimal import __version__
     from _decimal import __libmpdec_version__
 except ImportError:
-    from _pydecimal import *
-    from _pydecimal import __doc__
-    from _pydecimal import __version__
-    from _pydecimal import __libmpdec_version__
+    import _pydecimal
+    import sys
+    _pydecimal.__doc__ = __doc__
+    sys.modules[__name__] = _pydecimal

Lib/difflib.py

@@ -1200,6 +1200,25 @@ def context_diff(a, b, fromfile='', tofile='',
     strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'.
     The modification times are normally expressed in the ISO 8601 format.
     If not specified, the strings default to blanks.
+
+    Example:
+
+    >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True),
+    ...       'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')),
+    ...       end="")
+    *** Original
+    --- Current
+    ***************
+    *** 1,4 ****
+      one
+    ! two
+    ! three
+      four
+    --- 1,4 ----
+    + zero
+      one
+    ! tree
+      four
     """
 
     _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm)
@@ -1609,7 +1628,7 @@ _file_template = """
 </html>"""
 
 _styles = """
-        table.diff {font-family:Courier; border:medium;}
+        table.diff {font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; border:medium}
         .diff_header {background-color:#e0e0e0}
         td.diff_header {text-align:right}
         .diff_next {background-color:#c0c0c0}

Lib/genericpath.py

@@ -7,8 +7,8 @@ import os
 import stat
 
 __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime',
-           'getsize', 'isdir', 'isfile', 'islink', 'samefile', 'sameopenfile',
-           'samestat']
+           'getsize', 'isdevdrive', 'isdir', 'isfile', 'isjunction', 'islink',
+           'lexists', 'samefile', 'sameopenfile', 'samestat', 'ALLOW_MISSING']
 
 
 # Does a path exist?
@@ -22,6 +22,15 @@ def exists(path):
     return True
 
 
+# Being true for dangling symbolic links is also useful.
+def lexists(path):
+    """Test whether a path exists.  Returns True for broken symbolic links"""
+    try:
+        os.lstat(path)
+    except (OSError, ValueError):
+        return False
+    return True
+
 # This follows symbolic links, so both islink() and isdir() can be true
 # for the same path on systems that support symlinks
 def isfile(path):
@@ -57,6 +66,21 @@ def islink(path):
     return stat.S_ISLNK(st.st_mode)
 
 
+# Is a path a junction?
+def isjunction(path):
+    """Test whether a path is a junction
+    Junctions are not supported on the current platform"""
+    os.fspath(path)
+    return False
+
+
+def isdevdrive(path):
+    """Determines whether the specified path is on a Windows Dev Drive.
+    Dev Drives are not supported on the current platform"""
+    os.fspath(path)
+    return False
+
+
 def getsize(filename):
     """Return the size of a file, reported by os.stat()."""
     return os.stat(filename).st_size
@@ -165,3 +189,12 @@ def _check_arg_types(funcname, *args):
                             f'os.PathLike object, not {s.__class__.__name__!r}') from None
     if hasstr and hasbytes:
         raise TypeError("Can't mix strings and bytes in path components") from None
+
+# A singleton with a true boolean value.
+@object.__new__
+class ALLOW_MISSING:
+    """Special value for use in realpath()."""
+    def __repr__(self):
+        return 'os.path.ALLOW_MISSING'
+    def __reduce__(self):
+        return self.__class__.__name__

Lib/gettext.py

@@ -46,6 +46,7 @@ internationalized, to the local language and cultural habits.
 #   find this format documented anywhere.
 
 
+import operator
 import os
 import re
 import sys
@@ -166,14 +167,28 @@ def _parse(tokens, priority=-1):
 
 def _as_int(n):
     try:
-        i = round(n)
+        round(n)
     except TypeError:
         raise TypeError('Plural value must be an integer, got %s' %
                         (n.__class__.__name__,)) from None
+    return _as_int2(n)
+
+def _as_int2(n):
+    try:
+        return operator.index(n)
+    except TypeError:
+        pass
+
     import warnings
+    frame = sys._getframe(1)
+    stacklevel = 2
+    while frame.f_back is not None and frame.f_globals.get('__name__') == __name__:
+        stacklevel += 1
+        frame = frame.f_back
     warnings.warn('Plural value must be an integer, got %s' %
                   (n.__class__.__name__,),
-                  DeprecationWarning, 4)
+                  DeprecationWarning,
+                  stacklevel)
     return n
 
 
@@ -200,7 +215,7 @@ def c2py(plural):
             elif c == ')':
                 depth -= 1
 
-        ns = {'_as_int': _as_int}
+        ns = {'_as_int': _as_int, '__name__': __name__}
         exec('''if True:
             def func(n):
                 if not isinstance(n, int):
@@ -280,6 +295,7 @@ class NullTranslations:
     def ngettext(self, msgid1, msgid2, n):
         if self._fallback:
             return self._fallback.ngettext(msgid1, msgid2, n)
+        n = _as_int2(n)
         if n == 1:
             return msgid1
         else:
@@ -293,6 +309,7 @@ class NullTranslations:
     def npgettext(self, context, msgid1, msgid2, n):
         if self._fallback:
             return self._fallback.npgettext(context, msgid1, msgid2, n)
+        n = _as_int2(n)
         if n == 1:
             return msgid1
         else:
@@ -579,6 +596,7 @@ def dngettext(domain, msgid1, msgid2, n):
     try:
         t = translation(domain, _localedirs.get(domain, None))
     except OSError:
+        n = _as_int2(n)
         if n == 1:
             return msgid1
         else:
@@ -598,6 +616,7 @@ def dnpgettext(domain, context, msgid1, msgid2, n):
     try:
         t = translation(domain, _localedirs.get(domain, None))
     except OSError:
+        n = _as_int2(n)
         if n == 1:
             return msgid1
         else:

Lib/gzip.py

@@ -5,11 +5,15 @@ but random access is not allowed."""
 
 # based on Andrew Kuchling's minigzip.py distributed with the zlib module
 
-import struct, sys, time, os
-import zlib
+import _compression
 import builtins
 import io
-import _compression
+import os
+import struct
+import sys
+import time
+import weakref
+import zlib
 
 __all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"]
 
@@ -125,10 +129,13 @@ class BadGzipFile(OSError):
 class _WriteBufferStream(io.RawIOBase):
     """Minimal object to pass WriteBuffer flushes into GzipFile"""
     def __init__(self, gzip_file):
-        self.gzip_file = gzip_file
+        self.gzip_file = weakref.ref(gzip_file)
 
     def write(self, data):
-        return self.gzip_file._write_raw(data)
+        gzip_file = self.gzip_file()
+        if gzip_file is None:
+            raise RuntimeError("lost gzip_file")
+        return gzip_file._write_raw(data)
 
     def seekable(self):
         return False
@@ -190,51 +197,58 @@ class GzipFile(_compression.BaseStream):
             raise ValueError("Invalid mode: {!r}".format(mode))
         if mode and 'b' not in mode:
             mode += 'b'
-        if fileobj is None:
-            fileobj = self.myfileobj = builtins.open(filename, mode or 'rb')
-        if filename is None:
-            filename = getattr(fileobj, 'name', '')
-            if not isinstance(filename, (str, bytes)):
-                filename = ''
-        else:
-            filename = os.fspath(filename)
-        origmode = mode
-        if mode is None:
-            mode = getattr(fileobj, 'mode', 'rb')
+
+        try:
+            if fileobj is None:
+                fileobj = self.myfileobj = builtins.open(filename, mode or 'rb')
+            if filename is None:
+                filename = getattr(fileobj, 'name', '')
+                if not isinstance(filename, (str, bytes)):
+                    filename = ''
+            else:
+                filename = os.fspath(filename)
+            origmode = mode
+            if mode is None:
+                mode = getattr(fileobj, 'mode', 'rb')
 
 
-        if mode.startswith('r'):
-            self.mode = READ
-            raw = _GzipReader(fileobj)
-            self._buffer = io.BufferedReader(raw)
-            self.name = filename
+            if mode.startswith('r'):
+                self.mode = READ
+                raw = _GzipReader(fileobj)
+                self._buffer = io.BufferedReader(raw)
+                self.name = filename
 
-        elif mode.startswith(('w', 'a', 'x')):
-            if origmode is None:
-                import warnings
-                warnings.warn(
-                    "GzipFile was opened for writing, but this will "
-                    "change in future Python releases.  "
-                    "Specify the mode argument for opening it for writing.",
-                    FutureWarning, 2)
-            self.mode = WRITE
-            self._init_write(filename)
-            self.compress = zlib.compressobj(compresslevel,
-                                             zlib.DEFLATED,
-                                             -zlib.MAX_WBITS,
-                                             zlib.DEF_MEM_LEVEL,
-                                             0)
-            self._write_mtime = mtime
-            self._buffer_size = _WRITE_BUFFER_SIZE
-            self._buffer = io.BufferedWriter(_WriteBufferStream(self),
-                                             buffer_size=self._buffer_size)
-        else:
-            raise ValueError("Invalid mode: {!r}".format(mode))
+            elif mode.startswith(('w', 'a', 'x')):
+                if origmode is None:
+                    import warnings
+                    warnings.warn(
+                        "GzipFile was opened for writing, but this will "
+                        "change in future Python releases.  "
+                        "Specify the mode argument for opening it for writing.",
+                        FutureWarning, 2)
+                self.mode = WRITE
+                self._init_write(filename)
+                self.compress = zlib.compressobj(compresslevel,
+                                                 zlib.DEFLATED,
+                                                 -zlib.MAX_WBITS,
+                                                 zlib.DEF_MEM_LEVEL,
+                                                 0)
+                self._write_mtime = mtime
+                self._buffer_size = _WRITE_BUFFER_SIZE
+                self._buffer = io.BufferedWriter(_WriteBufferStream(self),
+                                                 buffer_size=self._buffer_size)
+            else:
+                raise ValueError("Invalid mode: {!r}".format(mode))
 
-        self.fileobj = fileobj
+            self.fileobj = fileobj
 
-        if self.mode == WRITE:
-            self._write_gzip_header(compresslevel)
+            if self.mode == WRITE:
+                self._write_gzip_header(compresslevel)
+        except:
+            # Avoid a ResourceWarning if the write fails,
+            # eg read-only file or KeyboardInterrupt
+            self._close()
+            raise
 
     @property
     def mtime(self):
@@ -363,11 +377,14 @@ class GzipFile(_compression.BaseStream):
             elif self.mode == READ:
                 self._buffer.close()
         finally:
-            self.fileobj = None
-            myfileobj = self.myfileobj
-            if myfileobj:
-                self.myfileobj = None
-                myfileobj.close()
+            self._close()
+
+    def _close(self):
+        self.fileobj = None
+        myfileobj = self.myfileobj
+        if myfileobj is not None:
+            self.myfileobj = None
+            myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_not_closed()
@@ -580,12 +597,12 @@ class _GzipReader(_compression.DecompressReader):
         self._new_member = True
 
 
-def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=0):
+def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
     """Compress data in one shot and return the compressed string.
 
     compresslevel sets the compression level in range of 0-9.
-    mtime can be used to set the modification time.
-    The modification time is set to 0 by default, for reproducibility.
+    mtime can be used to set the modification time. The modification time is
+    set to the current time by default.
     """
     # Wbits=31 automatically includes a gzip header and trailer.
     gzip_data = zlib.compress(data, level=compresslevel, wbits=31)

Lib/html/__init__.py

@@ -25,7 +25,7 @@ def escape(s, quote=True):
     return s
 
 
-# see http://www.w3.org/TR/html5/syntax.html#tokenizing-character-references
+# see https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
 
 _invalid_charrefs = {
     0x00: '\ufffd',  # REPLACEMENT CHARACTER

Lib/html/entities.py

@@ -3,8 +3,7 @@
 __all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs']
 
 
-# maps the HTML entity name to the Unicode code point
-# from https://html.spec.whatwg.org/multipage/named-characters.html
+# maps HTML4 entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -261,7 +260,11 @@ name2codepoint = {
 }
 
 
-# maps the HTML5 named character references to the equivalent Unicode character(s)
+# HTML5 named character references
+# Generated by Tools/build/parse_html5_entities.py
+# from https://html.spec.whatwg.org/entities.json and
+# https://html.spec.whatwg.org/multipage/named-characters.html.
+# Map HTML5 named character references to the equivalent Unicode character(s).
 html5 = {
     'Aacute': '\xc1',
     'aacute': '\xe1',

Lib/html/parser.py

@@ -12,6 +12,7 @@ import re
 import _markupbase
 
 from html import unescape
+from html.entities import html5 as html5_entities
 
 
 __all__ = ['HTMLParser']
@@ -23,6 +24,7 @@ incomplete = re.compile('&[a-zA-Z#]')
 
 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
 charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
+attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?')
 
 starttagopen = re.compile('<[a-zA-Z]')
 piclose = re.compile('>')
@@ -57,6 +59,22 @@ endendtag = re.compile('>')
 # </ and the tag name, so maybe this should be fixed
 endtagfind = re.compile(r'</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
 
+# Character reference processing logic specific to attribute values
+# See: https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
+def _replace_attr_charref(match):
+    ref = match.group(0)
+    # Numeric / hex char refs must always be unescaped
+    if ref.startswith('&#'):
+        return unescape(ref)
+    # Named character / entity references must only be unescaped
+    # if they are an exact match, and they are not followed by an equals sign
+    if not ref.endswith('=') and ref[1:] in html5_entities:
+        return unescape(ref)
+    # Otherwise do not unescape
+    return ref
+
+def _unescape_attrvalue(s):
+    return attr_charref.sub(_replace_attr_charref, s)
 
 
 class HTMLParser(_markupbase.ParserBase):
@@ -89,6 +107,7 @@ class HTMLParser(_markupbase.ParserBase):
         If convert_charrefs is True (the default), all character references
         are automatically converted to the corresponding Unicode characters.
         """
+        super().__init__()
         self.convert_charrefs = convert_charrefs
         self.reset()
 
@@ -98,7 +117,7 @@ class HTMLParser(_markupbase.ParserBase):
         self.lasttag = '???'
         self.interesting = interesting_normal
         self.cdata_elem = None
-        _markupbase.ParserBase.reset(self)
+        super().reset()
 
     def feed(self, data):
         r"""Feed data to the parser.
@@ -241,7 +260,7 @@ class HTMLParser(_markupbase.ParserBase):
             else:
                 assert 0, "interesting.search() lied"
         # end while
-        if end and i < n and not self.cdata_elem:
+        if end and i < n:
             if self.convert_charrefs and not self.cdata_elem:
                 self.handle_data(unescape(rawdata[i:n]))
             else:
@@ -259,7 +278,7 @@ class HTMLParser(_markupbase.ParserBase):
         if rawdata[i:i+4] == '<!--':
             # this case is actually already handled in goahead()
             return self.parse_comment(i)
-        elif rawdata[i:i+3] == '<![':
+        elif rawdata[i:i+9] == '<![CDATA[':
             return self.parse_marked_section(i)
         elif rawdata[i:i+9].lower() == '<!doctype':
             # find the closing >
@@ -276,7 +295,7 @@ class HTMLParser(_markupbase.ParserBase):
     def parse_bogus_comment(self, i, report=1):
         rawdata = self.rawdata
         assert rawdata[i:i+2] in ('<!', '</'), ('unexpected call to '
-                                                'parse_comment()')
+                                                'parse_bogus_comment()')
         pos = rawdata.find('>', i+2)
         if pos == -1:
             return -1
@@ -322,7 +341,7 @@ class HTMLParser(_markupbase.ParserBase):
                  attrvalue[:1] == '"' == attrvalue[-1:]:
                 attrvalue = attrvalue[1:-1]
             if attrvalue:
-                attrvalue = unescape(attrvalue)
+                attrvalue = _unescape_attrvalue(attrvalue)
             attrs.append((attrname.lower(), attrvalue))
             k = m.end()
 

Lib/importlib/metadata/__init__.py

@@ -56,12 +56,6 @@ class PackageNotFoundError(ModuleNotFoundError):
         (name,) = self.args
         return name
 
-    # TODO: RUSTPYTHON; the entire setter is added to avoid errors
-    @name.setter
-    def name(self, value):
-        import sys
-        sys.stderr.write("set value to PackageNotFoundError ignored\n")
-
 
 class Sectioned:
     """

Lib/json/__init__.py

@@ -1,4 +1,4 @@
-r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of
+r"""JSON (JavaScript Object Notation) <https://json.org> is a subset of
 JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
 interchange format.
 

Lib/json/decoder.py

@@ -41,7 +41,6 @@ class JSONDecodeError(ValueError):
         pos += col
         return cls(msg, doc, pos)
 
-
     # Note that this exception is used from _json
     def __init__(self, msg, doc, pos):
         lineno = doc.count('\n', 0, pos) + 1
@@ -65,17 +64,18 @@ _CONSTANTS = {
 }
 
 
+HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS)
 STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
 BACKSLASH = {
     '"': '"', '\\': '\\', '/': '/',
     'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t',
 }
 
-def _decode_uXXXX(s, pos):
-    esc = s[pos + 1:pos + 5]
-    if len(esc) == 4 and esc[1] not in 'xX':
+def _decode_uXXXX(s, pos, _m=HEXDIGITS.match):
+    esc = _m(s, pos + 1)
+    if esc is not None:
         try:
-            return int(esc, 16)
+            return int(esc.group(), 16)
         except ValueError:
             pass
     msg = "Invalid \\uXXXX escape"
@@ -215,10 +215,13 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook,
             break
         elif nextchar != ',':
             raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
+        comma_idx = end - 1
         end = _w(s, end).end()
         nextchar = s[end:end + 1]
         end += 1
         if nextchar != '"':
+            if nextchar == '}':
+                raise JSONDecodeError("Illegal trailing comma before end of object", s, comma_idx)
             raise JSONDecodeError(
                 "Expecting property name enclosed in double quotes", s, end - 1)
     if object_pairs_hook is not None:
@@ -255,19 +258,23 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
             break
         elif nextchar != ',':
             raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
+        comma_idx = end - 1
         try:
             if s[end] in _ws:
                 end += 1
                 if s[end] in _ws:
                     end = _w(s, end + 1).end()
+            nextchar = s[end:end + 1]
         except IndexError:
             pass
+        if nextchar == ']':
+            raise JSONDecodeError("Illegal trailing comma before end of array", s, comma_idx)
 
     return values, end
 
 
 class JSONDecoder(object):
-    """Simple JSON <http://json.org> decoder
+    """Simple JSON <https://json.org> decoder
 
     Performs the following translations in decoding by default:
 

Lib/json/encoder.py

@@ -30,6 +30,7 @@ ESCAPE_DCT = {
 for i in range(0x20):
     ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
     #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+del i
 
 INFINITY = float('inf')
 
@@ -71,7 +72,7 @@ encode_basestring_ascii = (
     c_encode_basestring_ascii or py_encode_basestring_ascii)
 
 class JSONEncoder(object):
-    """Extensible JSON <http://json.org> encoder for Python data structures.
+    """Extensible JSON <https://json.org> encoder for Python data structures.
 
     Supports the following objects and types by default:
 
@@ -107,8 +108,8 @@ class JSONEncoder(object):
         """Constructor for JSONEncoder, with sensible defaults.
 
         If skipkeys is false, then it is a TypeError to attempt
-        encoding of keys that are not str, int, float or None.  If
-        skipkeys is True, such items are simply skipped.
+        encoding of keys that are not str, int, float, bool or None.
+        If skipkeys is True, such items are simply skipped.
 
         If ensure_ascii is true, the output is guaranteed to be str
         objects with all incoming non-ASCII characters escaped.  If
@@ -173,7 +174,7 @@ class JSONEncoder(object):
                 else:
                     return list(iterable)
                 # Let the base class default method raise the TypeError
-                return JSONEncoder.default(self, o)
+                return super().default(o)
 
         """
         raise TypeError(f'Object of type {o.__class__.__name__} '
@@ -243,15 +244,18 @@ class JSONEncoder(object):
             return text
 
 
-        if (_one_shot and c_make_encoder is not None
-                and self.indent is None):
+        if self.indent is None or isinstance(self.indent, str):
+            indent = self.indent
+        else:
+            indent = ' ' * self.indent
+        if _one_shot and c_make_encoder is not None:
             _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
+                markers, self.default, _encoder, indent,
                 self.key_separator, self.item_separator, self.sort_keys,
                 self.skipkeys, self.allow_nan)
         else:
             _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
+                markers, self.default, _encoder, indent, floatstr,
                 self.key_separator, self.item_separator, self.sort_keys,
                 self.skipkeys, _one_shot)
         return _iterencode(o, 0)
@@ -271,9 +275,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
         _intstr=int.__repr__,
     ):
 
-    if _indent is not None and not isinstance(_indent, str):
-        _indent = ' ' * _indent
-
     def _iterencode_list(lst, _current_indent_level):
         if not lst:
             yield '[]'
@@ -344,7 +345,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
             _current_indent_level += 1
             newline_indent = '\n' + _indent * _current_indent_level
             item_separator = _item_separator + newline_indent
-            yield newline_indent
         else:
             newline_indent = None
             item_separator = _item_separator
@@ -377,6 +377,8 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
                                 f'not {key.__class__.__name__}')
             if first:
                 first = False
+                if newline_indent is not None:
+                    yield newline_indent
             else:
                 yield item_separator
             yield _encoder(key)
@@ -403,7 +405,7 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
                 else:
                     chunks = _iterencode(value, _current_indent_level)
                 yield from chunks
-        if newline_indent is not None:
+        if not first and newline_indent is not None:
             _current_indent_level -= 1
             yield '\n' + _indent * _current_indent_level
         yield '}'

Lib/json/scanner.py

@@ -9,7 +9,7 @@ except ImportError:
 __all__ = ['make_scanner']
 
 NUMBER_RE = re.compile(
-    r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
+    r'(-?(?:0|[1-9][0-9]*))(\.[0-9]+)?([eE][-+]?[0-9]+)?',
     (re.VERBOSE | re.MULTILINE | re.DOTALL))
 
 def py_make_scanner(context):

Lib/json/tool.py

@@ -13,7 +13,6 @@ Usage::
 import argparse
 import json
 import sys
-from pathlib import Path
 
 
 def main():
@@ -22,11 +21,9 @@ def main():
                    'to validate and pretty-print JSON objects.')
     parser = argparse.ArgumentParser(prog=prog, description=description)
     parser.add_argument('infile', nargs='?',
-                        type=argparse.FileType(encoding="utf-8"),
                         help='a JSON file to be validated or pretty-printed',
-                        default=sys.stdin)
+                        default='-')
     parser.add_argument('outfile', nargs='?',
-                        type=Path,
                         help='write the output of infile to outfile',
                         default=None)
     parser.add_argument('--sort-keys', action='store_true', default=False,
@@ -59,23 +56,30 @@ def main():
         dump_args['indent'] = None
         dump_args['separators'] = ',', ':'
 
-    with options.infile as infile:
+    try:
+        if options.infile == '-':
+            infile = sys.stdin
+        else:
+            infile = open(options.infile, encoding='utf-8')
         try:
             if options.json_lines:
                 objs = (json.loads(line) for line in infile)
             else:
                 objs = (json.load(infile),)
+        finally:
+            if infile is not sys.stdin:
+                infile.close()
 
-            if options.outfile is None:
-                out = sys.stdout
-            else:
-                out = options.outfile.open('w', encoding='utf-8')
-            with out as outfile:
-                for obj in objs:
-                    json.dump(obj, outfile, **dump_args)
-                    outfile.write('\n')
-        except ValueError as e:
-            raise SystemExit(e)
+        if options.outfile is None:
+            outfile = sys.stdout
+        else:
+            outfile = open(options.outfile, 'w', encoding='utf-8')
+        with outfile:
+            for obj in objs:
+                json.dump(obj, outfile, **dump_args)
+                outfile.write('\n')
+    except ValueError as e:
+        raise SystemExit(e)
 
 
 if __name__ == '__main__':

Lib/locale.py

@@ -25,8 +25,8 @@ import functools
 # Yuck:  LC_MESSAGES is non-standard:  can't tell whether it exists before
 # trying the import.  So __all__ is also fiddled at the end of the file.
 __all__ = ["getlocale", "getdefaultlocale", "getpreferredencoding", "Error",
-           "setlocale", "resetlocale", "localeconv", "strcoll", "strxfrm",
-           "str", "atof", "atoi", "format", "format_string", "currency",
+           "setlocale", "localeconv", "strcoll", "strxfrm",
+           "str", "atof", "atoi", "format_string", "currency",
            "normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY",
            "LC_NUMERIC", "LC_ALL", "CHAR_MAX", "getencoding"]
 
@@ -247,21 +247,6 @@ def format_string(f, val, grouping=False, monetary=False):
 
     return new_f % val
 
-def format(percent, value, grouping=False, monetary=False, *additional):
-    """Deprecated, use format_string instead."""
-    import warnings
-    warnings.warn(
-        "This method will be removed in a future version of Python. "
-        "Use 'locale.format_string()' instead.",
-        DeprecationWarning, stacklevel=2
-    )
-
-    match = _percent_re.match(percent)
-    if not match or len(match.group())!= len(percent):
-        raise ValueError(("format() must be given exactly one %%char "
-                         "format specifier, %s not valid") % repr(percent))
-    return _format(percent, value, grouping, monetary, *additional)
-
 def currency(val, symbol=True, grouping=False, international=False):
     """Formats val according to the currency settings
     in the current locale."""
@@ -556,11 +541,15 @@ def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')):
     """
 
     import warnings
-    warnings.warn(
-        "Use setlocale(), getencoding() and getlocale() instead",
-        DeprecationWarning, stacklevel=2
-    )
+    warnings._deprecated(
+        "locale.getdefaultlocale",
+        "{name!r} is deprecated and slated for removal in Python {remove}. "
+        "Use setlocale(), getencoding() and getlocale() instead.",
+        remove=(3, 15))
+    return _getdefaultlocale(envvars)
 
+
+def _getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')):
     try:
         # check if it's supported by the _locale module
         import _locale
@@ -625,40 +614,15 @@ def setlocale(category, locale=None):
         locale = normalize(_build_localename(locale))
     return _setlocale(category, locale)
 
-def resetlocale(category=LC_ALL):
-
-    """ Sets the locale for category to the default setting.
-
-        The default setting is determined by calling
-        getdefaultlocale(). category defaults to LC_ALL.
-
-    """
-    import warnings
-    warnings.warn(
-        'Use locale.setlocale(locale.LC_ALL, "") instead',
-        DeprecationWarning, stacklevel=2
-    )
-
-    with warnings.catch_warnings():
-        warnings.simplefilter('ignore', category=DeprecationWarning)
-        loc = getdefaultlocale()
-
-    _setlocale(category, _build_localename(loc))
-
 
 try:
     from _locale import getencoding
 except ImportError:
+    # When _locale.getencoding() is missing, locale.getencoding() uses the
+    # Python filesystem encoding.
     def getencoding():
-        if hasattr(sys, 'getandroidapilevel'):
-            # On Android langinfo.h and CODESET are missing, and UTF-8 is
-            # always used in mbstowcs() and wcstombs().
-            return 'utf-8'
-        encoding = getdefaultlocale()[1]
-        if encoding is None:
-            # LANG not set, default to UTF-8
-            encoding = 'utf-8'
-        return encoding
+        return sys.getfilesystemencoding()
+
 
 try:
     CODESET
@@ -896,6 +860,28 @@ del k, v
 #    updated 'ca_es@valencia' -> 'ca_ES.ISO8859-15@valencia' to 'ca_ES.UTF-8@valencia'
 #    updated 'kk_kz' -> 'kk_KZ.RK1048' to 'kk_KZ.ptcp154'
 #    updated 'russian' -> 'ru_RU.ISO8859-5' to 'ru_RU.KOI8-R'
+#
+# SS 2025-02-04:
+# Updated alias mapping with glibc 2.41 supported locales and the latest
+# X lib alias mapping.
+#
+# These are the differences compared to the old mapping (Python 3.13.1
+# and older):
+#
+#    updated 'c.utf8' -> 'C.UTF-8' to 'en_US.UTF-8'
+#    updated 'de_it' -> 'de_IT.ISO8859-1' to 'de_IT.UTF-8'
+#    removed 'de_li.utf8'
+#    updated 'en_il' -> 'en_IL.UTF-8' to 'en_IL.ISO8859-1'
+#    removed 'english.iso88591'
+#    updated 'es_cu' -> 'es_CU.UTF-8' to 'es_CU.ISO8859-1'
+#    updated 'russian' -> 'ru_RU.KOI8-R' to 'ru_RU.ISO8859-5'
+#    updated 'sr@latn' -> 'sr_CS.UTF-8@latin' to 'sr_RS.UTF-8@latin'
+#    removed 'univ'
+#    removed 'universal'
+#
+# SS 2025-06-10:
+# Remove 'c.utf8' -> 'en_US.UTF-8' because 'en_US.UTF-8' does not exist
+# on all platforms.
 
 locale_alias = {
     'a3':                                   'az_AZ.KOI8-C',
@@ -975,7 +961,6 @@ locale_alias = {
     'c.ascii':                              'C',
     'c.en':                                 'C',
     'c.iso88591':                           'en_US.ISO8859-1',
-    'c.utf8':                               'en_US.UTF-8',
     'c_c':                                  'C',
     'c_c.c':                                'C',
     'ca':                                   'ca_ES.ISO8859-1',
@@ -992,6 +977,7 @@ locale_alias = {
     'chr_us':                               'chr_US.UTF-8',
     'ckb_iq':                               'ckb_IQ.UTF-8',
     'cmn_tw':                               'cmn_TW.UTF-8',
+    'crh_ru':                               'crh_RU.UTF-8',
     'crh_ua':                               'crh_UA.UTF-8',
     'croatian':                             'hr_HR.ISO8859-2',
     'cs':                                   'cs_CZ.ISO8859-2',
@@ -1013,11 +999,12 @@ locale_alias = {
     'de_be':                                'de_BE.ISO8859-1',
     'de_ch':                                'de_CH.ISO8859-1',
     'de_de':                                'de_DE.ISO8859-1',
-    'de_it':                                'de_IT.ISO8859-1',
-    'de_li.utf8':                           'de_LI.UTF-8',
+    'de_it':                                'de_IT.UTF-8',
+    'de_li':                                'de_LI.ISO8859-1',
     'de_lu':                                'de_LU.ISO8859-1',
     'deutsch':                              'de_DE.ISO8859-1',
     'doi_in':                               'doi_IN.UTF-8',
+    'dsb_de':                               'dsb_DE.UTF-8',
     'dutch':                                'nl_NL.ISO8859-1',
     'dutch.iso88591':                       'nl_BE.ISO8859-1',
     'dv_mv':                                'dv_MV.UTF-8',
@@ -1040,7 +1027,7 @@ locale_alias = {
     'en_gb':                                'en_GB.ISO8859-1',
     'en_hk':                                'en_HK.ISO8859-1',
     'en_ie':                                'en_IE.ISO8859-1',
-    'en_il':                                'en_IL.UTF-8',
+    'en_il':                                'en_IL.ISO8859-1',
     'en_in':                                'en_IN.ISO8859-1',
     'en_ng':                                'en_NG.UTF-8',
     'en_nz':                                'en_NZ.ISO8859-1',
@@ -1056,7 +1043,6 @@ locale_alias = {
     'en_zw.utf8':                           'en_ZS.UTF-8',
     'eng_gb':                               'en_GB.ISO8859-1',
     'english':                              'en_EN.ISO8859-1',
-    'english.iso88591':                     'en_US.ISO8859-1',
     'english_uk':                           'en_GB.ISO8859-1',
     'english_united-states':                'en_US.ISO8859-1',
     'english_united-states.437':            'C',
@@ -1072,7 +1058,7 @@ locale_alias = {
     'es_cl':                                'es_CL.ISO8859-1',
     'es_co':                                'es_CO.ISO8859-1',
     'es_cr':                                'es_CR.ISO8859-1',
-    'es_cu':                                'es_CU.UTF-8',
+    'es_cu':                                'es_CU.ISO8859-1',
     'es_do':                                'es_DO.ISO8859-1',
     'es_ec':                                'es_EC.ISO8859-1',
     'es_es':                                'es_ES.ISO8859-1',
@@ -1122,6 +1108,7 @@ locale_alias = {
     'ga_ie':                                'ga_IE.ISO8859-1',
     'galego':                               'gl_ES.ISO8859-1',
     'galician':                             'gl_ES.ISO8859-1',
+    'gbm_in':                               'gbm_IN.UTF-8',
     'gd':                                   'gd_GB.ISO8859-1',
     'gd_gb':                                'gd_GB.ISO8859-1',
     'ger_de':                               'de_DE.ISO8859-1',
@@ -1162,6 +1149,7 @@ locale_alias = {
     'icelandic':                            'is_IS.ISO8859-1',
     'id':                                   'id_ID.ISO8859-1',
     'id_id':                                'id_ID.ISO8859-1',
+    'ie':                                   'ie.UTF-8',
     'ig_ng':                                'ig_NG.UTF-8',
     'ik_ca':                                'ik_CA.UTF-8',
     'in':                                   'id_ID.ISO8859-1',
@@ -1216,6 +1204,7 @@ locale_alias = {
     'ks_in':                                'ks_IN.UTF-8',
     'ks_in@devanagari.utf8':                'ks_IN.UTF-8@devanagari',
     'ku_tr':                                'ku_TR.ISO8859-9',
+    'kv_ru':                                'kv_RU.UTF-8',
     'kw':                                   'kw_GB.ISO8859-1',
     'kw_gb':                                'kw_GB.ISO8859-1',
     'ky':                                   'ky_KG.UTF-8',
@@ -1234,6 +1223,7 @@ locale_alias = {
     'lo_la.mulelao1':                       'lo_LA.MULELAO-1',
     'lt':                                   'lt_LT.ISO8859-13',
     'lt_lt':                                'lt_LT.ISO8859-13',
+    'ltg_lv.utf8':                          'ltg_LV.UTF-8',
     'lv':                                   'lv_LV.ISO8859-13',
     'lv_lv':                                'lv_LV.ISO8859-13',
     'lzh_tw':                               'lzh_TW.UTF-8',
@@ -1241,6 +1231,7 @@ locale_alias = {
     'mai':                                  'mai_IN.UTF-8',
     'mai_in':                               'mai_IN.UTF-8',
     'mai_np':                               'mai_NP.UTF-8',
+    'mdf_ru':                               'mdf_RU.UTF-8',
     'mfe_mu':                               'mfe_MU.UTF-8',
     'mg_mg':                                'mg_MG.ISO8859-15',
     'mhr_ru':                               'mhr_RU.UTF-8',
@@ -1254,6 +1245,7 @@ locale_alias = {
     'ml_in':                                'ml_IN.UTF-8',
     'mn_mn':                                'mn_MN.UTF-8',
     'mni_in':                               'mni_IN.UTF-8',
+    'mnw_mm':                               'mnw_MM.UTF-8',
     'mr':                                   'mr_IN.UTF-8',
     'mr_in':                                'mr_IN.UTF-8',
     'ms':                                   'ms_MY.ISO8859-1',
@@ -1322,6 +1314,7 @@ locale_alias = {
     'pt_pt':                                'pt_PT.ISO8859-1',
     'quz_pe':                               'quz_PE.UTF-8',
     'raj_in':                               'raj_IN.UTF-8',
+    'rif_ma':                               'rif_MA.UTF-8',
     'ro':                                   'ro_RO.ISO8859-2',
     'ro_ro':                                'ro_RO.ISO8859-2',
     'romanian':                             'ro_RO.ISO8859-2',
@@ -1329,12 +1322,14 @@ locale_alias = {
     'ru_ru':                                'ru_RU.UTF-8',
     'ru_ua':                                'ru_UA.KOI8-U',
     'rumanian':                             'ro_RO.ISO8859-2',
-    'russian':                              'ru_RU.KOI8-R',
+    'russian':                              'ru_RU.ISO8859-5',
     'rw':                                   'rw_RW.ISO8859-1',
     'rw_rw':                                'rw_RW.ISO8859-1',
     'sa_in':                                'sa_IN.UTF-8',
+    'sah_ru':                               'sah_RU.UTF-8',
     'sat_in':                               'sat_IN.UTF-8',
     'sc_it':                                'sc_IT.UTF-8',
+    'scn_it':                               'scn_IT.UTF-8',
     'sd':                                   'sd_IN.UTF-8',
     'sd_in':                                'sd_IN.UTF-8',
     'sd_in@devanagari.utf8':                'sd_IN.UTF-8@devanagari',
@@ -1376,7 +1371,7 @@ locale_alias = {
     'sq_mk':                                'sq_MK.UTF-8',
     'sr':                                   'sr_RS.UTF-8',
     'sr@cyrillic':                          'sr_RS.UTF-8',
-    'sr@latn':                              'sr_CS.UTF-8@latin',
+    'sr@latn':                              'sr_RS.UTF-8@latin',
     'sr_cs':                                'sr_CS.UTF-8',
     'sr_cs.iso88592@latn':                  'sr_CS.ISO8859-2',
     'sr_cs@latn':                           'sr_CS.UTF-8@latin',
@@ -1395,14 +1390,17 @@ locale_alias = {
     'sr_yu@cyrillic':                       'sr_RS.UTF-8',
     'ss':                                   'ss_ZA.ISO8859-1',
     'ss_za':                                'ss_ZA.ISO8859-1',
+    'ssy_er':                               'ssy_ER.UTF-8',
     'st':                                   'st_ZA.ISO8859-1',
     'st_za':                                'st_ZA.ISO8859-1',
+    'su_id':                                'su_ID.UTF-8',
     'sv':                                   'sv_SE.ISO8859-1',
     'sv_fi':                                'sv_FI.ISO8859-1',
     'sv_se':                                'sv_SE.ISO8859-1',
     'sw_ke':                                'sw_KE.UTF-8',
     'sw_tz':                                'sw_TZ.UTF-8',
     'swedish':                              'sv_SE.ISO8859-1',
+    'syr':                                  'syr.UTF-8',
     'szl_pl':                               'szl_PL.UTF-8',
     'ta':                                   'ta_IN.TSCII-0',
     'ta_in':                                'ta_IN.TSCII-0',
@@ -1429,6 +1427,7 @@ locale_alias = {
     'tn':                                   'tn_ZA.ISO8859-15',
     'tn_za':                                'tn_ZA.ISO8859-15',
     'to_to':                                'to_TO.UTF-8',
+    'tok':                                  'tok.UTF-8',
     'tpi_pg':                               'tpi_PG.UTF-8',
     'tr':                                   'tr_TR.ISO8859-9',
     'tr_cy':                                'tr_CY.ISO8859-9',
@@ -1443,8 +1442,7 @@ locale_alias = {
     'ug_cn':                                'ug_CN.UTF-8',
     'uk':                                   'uk_UA.KOI8-U',
     'uk_ua':                                'uk_UA.KOI8-U',
-    'univ':                                 'en_US.utf',
-    'universal':                            'en_US.utf',
+    'univ.utf8':                            'en_US.UTF-8',
     'universal.utf8@ucs4':                  'en_US.UTF-8',
     'unm_us':                               'unm_US.UTF-8',
     'ur':                                   'ur_PK.CP1256',
@@ -1473,6 +1471,7 @@ locale_alias = {
     'yo_ng':                                'yo_NG.UTF-8',
     'yue_hk':                               'yue_HK.UTF-8',
     'yuw_pg':                               'yuw_PG.UTF-8',
+    'zgh_ma':                               'zgh_MA.UTF-8',
     'zh':                                   'zh_CN.eucCN',
     'zh_cn':                                'zh_CN.gb2312',
     'zh_cn.big5':                           'zh_TW.big5',
@@ -1496,7 +1495,8 @@ locale_alias = {
 # to include every locale up to Windows Vista.
 #
 # NOTE: this mapping is incomplete.  If your language is missing, please
-# submit a bug report to the Python bug tracker at http://bugs.python.org/
+# submit a bug report as detailed in the Python devguide at:
+#    https://devguide.python.org/triage/issue-tracker/
 # Make sure you include the missing language identifier and the suggested
 # locale code.
 #
@@ -1742,17 +1742,6 @@ def _print_locale():
         print('   Encoding: ', enc or '(undefined)')
         print()
 
-    print()
-    print('Locale settings after calling resetlocale():')
-    print('-'*72)
-    resetlocale()
-    for name,category in categories.items():
-        print(name, '...')
-        lang, enc = getlocale(category)
-        print('   Language: ', lang or '(undefined)')
-        print('   Encoding: ', enc or '(undefined)')
-        print()
-
     try:
         setlocale(LC_ALL, "")
     except:

Lib/logging/__init__.py

@@ -56,7 +56,7 @@ __date__    = "07 February 2010"
 #
 #_startTime is used as the base when calculating the relative time of events
 #
-_startTime = time.time()
+_startTime = time.time_ns()
 
 #
 #raiseExceptions is used to see if exceptions during handling should be
@@ -159,12 +159,9 @@ def addLevelName(level, levelName):
 
     This is used when converting levels to text during message formatting.
     """
-    _acquireLock()
-    try:    #unlikely to cause an exception, but you never know...
+    with _lock:
         _levelToName[level] = levelName
         _nameToLevel[levelName] = level
-    finally:
-        _releaseLock()
 
 if hasattr(sys, "_getframe"):
     currentframe = lambda: sys._getframe(1)
@@ -201,7 +198,7 @@ def _is_internal_frame(frame):
     """Signal whether the frame is a CPython or logging module internal."""
     filename = os.path.normcase(frame.f_code.co_filename)
     return filename == _srcfile or (
-            "importlib" in filename and "_bootstrap" in filename
+        "importlib" in filename and "_bootstrap" in filename
     )
 
 
@@ -231,21 +228,27 @@ def _checkLevel(level):
 #
 _lock = threading.RLock()
 
-def _acquireLock():
+def _prepareFork():
     """
-    Acquire the module-level lock for serializing access to shared data.
+    Prepare to fork a new child process by acquiring the module-level lock.
 
-    This should be released with _releaseLock().
+    This should be used in conjunction with _afterFork().
     """
-    if _lock:
+    # Wrap the lock acquisition in a try-except to prevent the lock from being
+    # abandoned in the event of an asynchronous exception. See gh-106238.
+    try:
         _lock.acquire()
-
-def _releaseLock():
-    """
-    Release the module-level lock acquired by calling _acquireLock().
-    """
-    if _lock:
+    except BaseException:
         _lock.release()
+        raise
+
+def _afterFork():
+    """
+    After a new child process has been forked, release the module-level lock.
+
+    This should be used in conjunction with _prepareFork().
+    """
+    _lock.release()
 
 
 # Prevent a held logging lock from blocking a child from logging.
@@ -260,23 +263,20 @@ else:
     _at_fork_reinit_lock_weakset = weakref.WeakSet()
 
     def _register_at_fork_reinit_lock(instance):
-        _acquireLock()
-        try:
+        with _lock:
             _at_fork_reinit_lock_weakset.add(instance)
-        finally:
-            _releaseLock()
 
     def _after_at_fork_child_reinit_locks():
         for handler in _at_fork_reinit_lock_weakset:
             handler._at_fork_reinit()
 
-        # _acquireLock() was called in the parent before forking.
+        # _prepareFork() was called in the parent before forking.
         # The lock is reinitialized to unlocked state.
         _lock._at_fork_reinit()
 
-    os.register_at_fork(before=_acquireLock,
+    os.register_at_fork(before=_prepareFork,
                         after_in_child=_after_at_fork_child_reinit_locks,
-                        after_in_parent=_releaseLock)
+                        after_in_parent=_afterFork)
 
 
 #---------------------------------------------------------------------------
@@ -300,7 +300,7 @@ class LogRecord(object):
         """
         Initialize a logging record with interesting information.
         """
-        ct = time.time()
+        ct = time.time_ns()
         self.name = name
         self.msg = msg
         #
@@ -322,7 +322,7 @@ class LogRecord(object):
         # Thus, while not removing the isinstance check, it does now look
         # for collections.abc.Mapping rather than, as before, dict.
         if (args and len(args) == 1 and isinstance(args[0], collections.abc.Mapping)
-                and args[0]):
+            and args[0]):
             args = args[0]
         self.args = args
         self.levelname = getLevelName(level)
@@ -339,9 +339,17 @@ class LogRecord(object):
         self.stack_info = sinfo
         self.lineno = lineno
         self.funcName = func
-        self.created = ct
-        self.msecs = int((ct - int(ct)) * 1000) + 0.0  # see gh-89047
-        self.relativeCreated = (self.created - _startTime) * 1000
+        self.created = ct / 1e9  # ns to float seconds
+        # Get the number of whole milliseconds (0-999) in the fractional part of seconds.
+        # Eg: 1_677_903_920_999_998_503 ns --> 999_998_503 ns--> 999 ms
+        # Convert to float by adding 0.0 for historical reasons. See gh-89047
+        self.msecs = (ct % 1_000_000_000) // 1_000_000 + 0.0
+        if self.msecs == 999.0 and int(self.created) != ct // 1_000_000_000:
+            # ns -> sec conversion can round up, e.g:
+            # 1_677_903_920_999_999_900 ns --> 1_677_903_921.0 sec
+            self.msecs = 0.0
+
+        self.relativeCreated = (ct - _startTime) / 1e6
         if logThreads:
             self.thread = threading.get_ident()
             self.threadName = threading.current_thread().name
@@ -378,7 +386,7 @@ class LogRecord(object):
 
     def __repr__(self):
         return '<LogRecord: %s, %s, %s, %s, "%s">'%(self.name, self.levelno,
-                                                    self.pathname, self.lineno, self.msg)
+            self.pathname, self.lineno, self.msg)
 
     def getMessage(self):
         """
@@ -572,7 +580,7 @@ class Formatter(object):
     %(lineno)d          Source line number where the logging call was issued
                         (if available)
     %(funcName)s        Function name
-    %(created)f         Time when the LogRecord was created (time.time()
+    %(created)f         Time when the LogRecord was created (time.time_ns() / 1e9
                         return value)
     %(asctime)s         Textual time when the LogRecord was created
     %(msecs)d           Millisecond portion of the creation time
@@ -583,6 +591,7 @@ class Formatter(object):
     %(threadName)s      Thread name (if available)
     %(taskName)s        Task name (if available)
     %(process)d         Process ID (if available)
+    %(processName)s     Process name (if available)
     %(message)s         The result of record.getMessage(), computed just as
                         the record is emitted
     """
@@ -608,7 +617,7 @@ class Formatter(object):
         """
         if style not in _STYLES:
             raise ValueError('Style must be one of: %s' % ','.join(
-                _STYLES.keys()))
+                             _STYLES.keys()))
         self._style = _STYLES[style][0](fmt, defaults=defaults)
         if validate:
             self._style.validate()
@@ -658,7 +667,7 @@ class Formatter(object):
         # See issues #9427, #1553375. Commented out for now.
         #if getattr(self, 'fullstack', False):
         #    traceback.print_stack(tb.tb_frame.f_back, file=sio)
-        traceback.print_exception(ei[0], ei[1], tb, None, sio)
+        traceback.print_exception(ei[0], ei[1], tb, limit=None, file=sio)
         s = sio.getvalue()
         sio.close()
         if s[-1:] == "\n":
@@ -879,25 +888,20 @@ def _removeHandlerRef(wr):
     # set to None. It can also be called from another thread. So we need to
     # pre-emptively grab the necessary globals and check if they're None,
     # to prevent race conditions and failures during interpreter shutdown.
-    acquire, release, handlers = _acquireLock, _releaseLock, _handlerList
-    if acquire and release and handlers:
-        acquire()
-        try:
-            handlers.remove(wr)
-        except ValueError:
-            pass
-        finally:
-            release()
+    handlers, lock = _handlerList, _lock
+    if lock and handlers:
+        with lock:
+            try:
+                handlers.remove(wr)
+            except ValueError:
+                pass
 
 def _addHandlerRef(handler):
     """
     Add a handler to the internal cleanup list using a weak reference.
     """
-    _acquireLock()
-    try:
+    with _lock:
         _handlerList.append(weakref.ref(handler, _removeHandlerRef))
-    finally:
-        _releaseLock()
 
 
 def getHandlerByName(name):
@@ -912,8 +916,7 @@ def getHandlerNames():
     """
     Return all known handler names as an immutable set.
     """
-    result = set(_handlers.keys())
-    return frozenset(result)
+    return frozenset(_handlers)
 
 
 class Handler(Filterer):
@@ -943,15 +946,12 @@ class Handler(Filterer):
         return self._name
 
     def set_name(self, name):
-        _acquireLock()
-        try:
+        with _lock:
             if self._name in _handlers:
                 del _handlers[self._name]
             self._name = name
             if name:
                 _handlers[name] = self
-        finally:
-            _releaseLock()
 
     name = property(get_name, set_name)
 
@@ -1023,11 +1023,8 @@ class Handler(Filterer):
         if isinstance(rv, LogRecord):
             record = rv
         if rv:
-            self.acquire()
-            try:
+            with self.lock:
                 self.emit(record)
-            finally:
-                self.release()
         return rv
 
     def setFormatter(self, fmt):
@@ -1055,13 +1052,10 @@ class Handler(Filterer):
         methods.
         """
         #get the module data lock, as we're updating a shared structure.
-        _acquireLock()
-        try:    #unlikely to raise an exception, but you never know...
+        with _lock:
             self._closed = True
             if self._name and self._name in _handlers:
                 del _handlers[self._name]
-        finally:
-            _releaseLock()
 
     def handleError(self, record):
         """
@@ -1076,14 +1070,14 @@ class Handler(Filterer):
         The record which was being processed is passed in to this method.
         """
         if raiseExceptions and sys.stderr:  # see issue 13807
-            t, v, tb = sys.exc_info()
+            exc = sys.exception()
             try:
                 sys.stderr.write('--- Logging error ---\n')
-                traceback.print_exception(t, v, tb, None, sys.stderr)
+                traceback.print_exception(exc, limit=None, file=sys.stderr)
                 sys.stderr.write('Call stack:\n')
                 # Walk the stack frame up until we're out of logging,
                 # so as to print the calling context.
-                frame = tb.tb_frame
+                frame = exc.__traceback__.tb_frame
                 while (frame and os.path.dirname(frame.f_code.co_filename) ==
                        __path__[0]):
                     frame = frame.f_back
@@ -1092,7 +1086,7 @@ class Handler(Filterer):
                 else:
                     # couldn't find the right stack frame, for some reason
                     sys.stderr.write('Logged from file %s, line %s\n' % (
-                        record.filename, record.lineno))
+                                     record.filename, record.lineno))
                 # Issue 18671: output logging message and arguments
                 try:
                     sys.stderr.write('Message: %r\n'
@@ -1104,11 +1098,11 @@ class Handler(Filterer):
                     sys.stderr.write('Unable to print the message and arguments'
                                      ' - possible formatting error.\nUse the'
                                      ' traceback above to help find the error.\n'
-                                     )
+                                    )
             except OSError: #pragma: no cover
                 pass    # see issue 5971
             finally:
-                del t, v, tb
+                del exc
 
     def __repr__(self):
         level = getLevelName(self.level)
@@ -1138,12 +1132,9 @@ class StreamHandler(Handler):
         """
         Flushes the stream.
         """
-        self.acquire()
-        try:
+        with self.lock:
             if self.stream and hasattr(self.stream, "flush"):
                 self.stream.flush()
-        finally:
-            self.release()
 
     def emit(self, record):
         """
@@ -1179,12 +1170,9 @@ class StreamHandler(Handler):
             result = None
         else:
             result = self.stream
-            self.acquire()
-            try:
+            with self.lock:
                 self.flush()
                 self.stream = stream
-            finally:
-                self.release()
         return result
 
     def __repr__(self):
@@ -1234,8 +1222,7 @@ class FileHandler(StreamHandler):
         """
         Closes the stream.
         """
-        self.acquire()
-        try:
+        with self.lock:
             try:
                 if self.stream:
                     try:
@@ -1251,8 +1238,6 @@ class FileHandler(StreamHandler):
                 # Also see Issue #42378: we also rely on
                 # self._closed being set to True there
                 StreamHandler.close(self)
-        finally:
-            self.release()
 
     def _open(self):
         """
@@ -1388,8 +1373,7 @@ class Manager(object):
         rv = None
         if not isinstance(name, str):
             raise TypeError('A logger name must be a string')
-        _acquireLock()
-        try:
+        with _lock:
             if name in self.loggerDict:
                 rv = self.loggerDict[name]
                 if isinstance(rv, PlaceHolder):
@@ -1404,8 +1388,6 @@ class Manager(object):
                 rv.manager = self
                 self.loggerDict[name] = rv
                 self._fixupParents(rv)
-        finally:
-            _releaseLock()
         return rv
 
     def setLoggerClass(self, klass):
@@ -1468,12 +1450,11 @@ class Manager(object):
         Called when level changes are made
         """
 
-        _acquireLock()
-        for logger in self.loggerDict.values():
-            if isinstance(logger, Logger):
-                logger._cache.clear()
-        self.root._cache.clear()
-        _releaseLock()
+        with _lock:
+            for logger in self.loggerDict.values():
+                if isinstance(logger, Logger):
+                    logger._cache.clear()
+            self.root._cache.clear()
 
 #---------------------------------------------------------------------------
 #   Logger classes and functions
@@ -1494,6 +1475,8 @@ class Logger(Filterer):
     level, and "input.csv", "input.xls" and "input.gnu" for the sub-levels.
     There is no arbitrary limit to the depth of nesting.
     """
+    _tls = threading.local()
+
     def __init__(self, name, level=NOTSET):
         """
         Initialize the logger with a name and an optional level.
@@ -1552,7 +1535,7 @@ class Logger(Filterer):
 
     def warn(self, msg, *args, **kwargs):
         warnings.warn("The 'warn' method is deprecated, "
-                      "use 'warning' instead", DeprecationWarning, 2)
+            "use 'warning' instead", DeprecationWarning, 2)
         self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
@@ -1649,7 +1632,7 @@ class Logger(Filterer):
         specialized LogRecords.
         """
         rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func,
-                               sinfo)
+                             sinfo)
         if extra is not None:
             for key in extra:
                 if (key in ["message", "asctime"]) or (key in rv.__dict__):
@@ -1690,36 +1673,35 @@ class Logger(Filterer):
         This method is used for unpickled records received from a socket, as
         well as those created locally. Logger-level filtering is applied.
         """
-        if self.disabled:
+        if self._is_disabled():
             return
-        maybe_record = self.filter(record)
-        if not maybe_record:
-            return
-        if isinstance(maybe_record, LogRecord):
-            record = maybe_record
-        self.callHandlers(record)
+
+        self._tls.in_progress = True
+        try:
+            maybe_record = self.filter(record)
+            if not maybe_record:
+                return
+            if isinstance(maybe_record, LogRecord):
+                record = maybe_record
+            self.callHandlers(record)
+        finally:
+            self._tls.in_progress = False
 
     def addHandler(self, hdlr):
         """
         Add the specified handler to this logger.
         """
-        _acquireLock()
-        try:
+        with _lock:
             if not (hdlr in self.handlers):
                 self.handlers.append(hdlr)
-        finally:
-            _releaseLock()
 
     def removeHandler(self, hdlr):
         """
         Remove the specified handler from this logger.
         """
-        _acquireLock()
-        try:
+        with _lock:
             if hdlr in self.handlers:
                 self.handlers.remove(hdlr)
-        finally:
-            _releaseLock()
 
     def hasHandlers(self):
         """
@@ -1791,22 +1773,19 @@ class Logger(Filterer):
         """
         Is this logger enabled for level 'level'?
         """
-        if self.disabled:
+        if self._is_disabled():
             return False
 
         try:
             return self._cache[level]
         except KeyError:
-            _acquireLock()
-            try:
+            with _lock:
                 if self.manager.disable >= level:
                     is_enabled = self._cache[level] = False
                 else:
                     is_enabled = self._cache[level] = (
-                            level >= self.getEffectiveLevel()
+                        level >= self.getEffectiveLevel()
                     )
-            finally:
-                _releaseLock()
             return is_enabled
 
     def getChild(self, suffix):
@@ -1836,16 +1815,18 @@ class Logger(Filterer):
             return 1 + logger.name.count('.')
 
         d = self.manager.loggerDict
-        _acquireLock()
-        try:
+        with _lock:
             # exclude PlaceHolders - the last check is to ensure that lower-level
             # descendants aren't returned - if there are placeholders, a logger's
             # parent field might point to a grandparent or ancestor thereof.
             return set(item for item in d.values()
                        if isinstance(item, Logger) and item.parent is self and
                        _hierlevel(item) == 1 + _hierlevel(item.parent))
-        finally:
-            _releaseLock()
+
+    def _is_disabled(self):
+        # We need to use getattr as it will only be set the first time a log
+        # message is recorded on any given thread
+        return self.disabled or getattr(self._tls, 'in_progress', False)
 
     def __repr__(self):
         level = getLevelName(self.getEffectiveLevel())
@@ -1881,7 +1862,7 @@ class LoggerAdapter(object):
     information in logging output.
     """
 
-    def __init__(self, logger, extra=None):
+    def __init__(self, logger, extra=None, merge_extra=False):
         """
         Initialize the adapter with a logger and a dict-like object which
         provides contextual information. This constructor signature allows
@@ -1891,9 +1872,20 @@ class LoggerAdapter(object):
         following example:
 
         adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2"))
+
+        By default, LoggerAdapter objects will drop the "extra" argument
+        passed on the individual log calls to use its own instead.
+
+        Initializing it with merge_extra=True will instead merge both
+        maps when logging, the individual call extra taking precedence
+        over the LoggerAdapter instance extra
+
+        .. versionchanged:: 3.13
+           The *merge_extra* argument was added.
         """
         self.logger = logger
         self.extra = extra
+        self.merge_extra = merge_extra
 
     def process(self, msg, kwargs):
         """
@@ -1905,7 +1897,10 @@ class LoggerAdapter(object):
         Normally, you'll only need to override this one method in a
         LoggerAdapter subclass for your specific needs.
         """
-        kwargs["extra"] = self.extra
+        if self.merge_extra and "extra" in kwargs:
+            kwargs["extra"] = {**self.extra, **kwargs["extra"]}
+        else:
+            kwargs["extra"] = self.extra
         return msg, kwargs
 
     #
@@ -1931,7 +1926,7 @@ class LoggerAdapter(object):
 
     def warn(self, msg, *args, **kwargs):
         warnings.warn("The 'warn' method is deprecated, "
-                      "use 'warning' instead", DeprecationWarning, 2)
+            "use 'warning' instead", DeprecationWarning, 2)
         self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
@@ -2088,8 +2083,7 @@ def basicConfig(**kwargs):
     """
     # Add thread safety in case someone mistakenly calls
     # basicConfig() from multiple threads
-    _acquireLock()
-    try:
+    with _lock:
         force = kwargs.pop('force', False)
         encoding = kwargs.pop('encoding', None)
         errors = kwargs.pop('errors', 'backslashreplace')
@@ -2125,7 +2119,7 @@ def basicConfig(**kwargs):
             style = kwargs.pop("style", '%')
             if style not in _STYLES:
                 raise ValueError('Style must be one of: %s' % ','.join(
-                    _STYLES.keys()))
+                                 _STYLES.keys()))
             fs = kwargs.pop("format", _STYLES[style][1])
             fmt = Formatter(fs, dfs, style)
             for h in handlers:
@@ -2138,8 +2132,6 @@ def basicConfig(**kwargs):
             if kwargs:
                 keys = ', '.join(kwargs.keys())
                 raise ValueError('Unrecognised argument(s): %s' % keys)
-    finally:
-        _releaseLock()
 
 #---------------------------------------------------------------------------
 # Utility functions at module level.
@@ -2202,7 +2194,7 @@ def warning(msg, *args, **kwargs):
 
 def warn(msg, *args, **kwargs):
     warnings.warn("The 'warn' function is deprecated, "
-                  "use 'warning' instead", DeprecationWarning, 2)
+        "use 'warning' instead", DeprecationWarning, 2)
     warning(msg, *args, **kwargs)
 
 def info(msg, *args, **kwargs):

Lib/logging/config.py

@@ -83,15 +83,12 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=Non
     formatters = _create_formatters(cp)
 
     # critical section
-    logging._acquireLock()
-    try:
+    with logging._lock:
         _clearExistingHandlers()
 
         # Handlers add themselves to logging._handlers
         handlers = _install_handlers(cp, formatters)
         _install_loggers(cp, handlers, disable_existing_loggers)
-    finally:
-        logging._releaseLock()
 
 
 def _resolve(name):
@@ -314,7 +311,7 @@ class ConvertingMixin(object):
             if replace:
                 self[key] = result
             if type(result) in (ConvertingDict, ConvertingList,
-                                ConvertingTuple):
+                               ConvertingTuple):
                 result.parent = self
                 result.key = key
         return result
@@ -323,7 +320,7 @@ class ConvertingMixin(object):
         result = self.configurator.convert(value)
         if value is not result:
             if type(result) in (ConvertingDict, ConvertingList,
-                                ConvertingTuple):
+                               ConvertingTuple):
                 result.parent = self
         return result
 
@@ -378,7 +375,7 @@ class BaseConfigurator(object):
 
     WORD_PATTERN = re.compile(r'^\s*(\w+)\s*')
     DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*')
-    INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*')
+    INDEX_PATTERN = re.compile(r'^\[([^\[\]]*)\]\s*')
     DIGIT_PATTERN = re.compile(r'^\d+$')
 
     value_converters = {
@@ -464,8 +461,8 @@ class BaseConfigurator(object):
         elif not isinstance(value, ConvertingList) and isinstance(value, list):
             value = ConvertingList(value)
             value.configurator = self
-        elif not isinstance(value, ConvertingTuple) and \
-                isinstance(value, tuple) and not hasattr(value, '_fields'):
+        elif not isinstance(value, ConvertingTuple) and\
+                 isinstance(value, tuple) and not hasattr(value, '_fields'):
             value = ConvertingTuple(value)
             value.configurator = self
         elif isinstance(value, str): # str for py3k
@@ -543,8 +540,7 @@ class DictConfigurator(BaseConfigurator):
             raise ValueError("Unsupported version: %s" % config['version'])
         incremental = config.pop('incremental', False)
         EMPTY_DICT = {}
-        logging._acquireLock()
-        try:
+        with logging._lock:
             if incremental:
                 handlers = config.get('handlers', EMPTY_DICT)
                 for name in handlers:
@@ -585,7 +581,7 @@ class DictConfigurator(BaseConfigurator):
                 for name in formatters:
                     try:
                         formatters[name] = self.configure_formatter(
-                            formatters[name])
+                                                            formatters[name])
                     except Exception as e:
                         raise ValueError('Unable to configure '
                                          'formatter %r' % name) from e
@@ -688,8 +684,6 @@ class DictConfigurator(BaseConfigurator):
                     except Exception as e:
                         raise ValueError('Unable to configure root '
                                          'logger') from e
-        finally:
-            logging._releaseLock()
 
     def configure_formatter(self, config):
         """Configure a formatter from a dictionary."""
@@ -700,10 +694,9 @@ class DictConfigurator(BaseConfigurator):
             except TypeError as te:
                 if "'format'" not in str(te):
                     raise
-                #Name of parameter changed from fmt to format.
-                #Retry with old name.
-                #This is so that code can be used with older Python versions
-                #(e.g. by Django)
+                # logging.Formatter and its subclasses expect the `fmt`
+                # parameter instead of `format`. Retry passing configuration
+                # with `fmt`.
                 config['fmt'] = config.pop('format')
                 config['()'] = factory
                 result = self.configure_custom(config)
@@ -812,7 +805,7 @@ class DictConfigurator(BaseConfigurator):
             elif issubclass(klass, logging.handlers.QueueHandler):
                 # Another special case for handler which refers to other handlers
                 # if 'handlers' not in config:
-                # raise ValueError('No handlers specified for a QueueHandler')
+                    # raise ValueError('No handlers specified for a QueueHandler')
                 if 'queue' in config:
                     qspec = config['queue']
 
@@ -836,8 +829,8 @@ class DictConfigurator(BaseConfigurator):
                     else:
                         if isinstance(lspec, str):
                             listener = self.resolve(lspec)
-                            if isinstance(listener, type) and \
-                                    not issubclass(listener, logging.handlers.QueueListener):
+                            if isinstance(listener, type) and\
+                                not issubclass(listener, logging.handlers.QueueListener):
                                 raise TypeError('Invalid listener specifier %r' % lspec)
                         elif isinstance(lspec, dict):
                             if '()' not in lspec:
@@ -861,11 +854,11 @@ class DictConfigurator(BaseConfigurator):
                     except Exception as e:
                         raise ValueError('Unable to set required handler %r' % hn) from e
                     config['handlers'] = hlist
-            elif issubclass(klass, logging.handlers.SMTPHandler) and \
-                    'mailhost' in config:
+            elif issubclass(klass, logging.handlers.SMTPHandler) and\
+                'mailhost' in config:
                 config['mailhost'] = self.as_tuple(config['mailhost'])
-            elif issubclass(klass, logging.handlers.SysLogHandler) and \
-                    'address' in config:
+            elif issubclass(klass, logging.handlers.SysLogHandler) and\
+                'address' in config:
                 config['address'] = self.as_tuple(config['address'])
             if issubclass(klass, logging.handlers.QueueHandler):
                 factory = functools.partial(self._configure_queue_handler, klass)
@@ -1018,9 +1011,8 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None):
         def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
                      handler=None, ready=None, verify=None):
             ThreadingTCPServer.__init__(self, (host, port), handler)
-            logging._acquireLock()
-            self.abort = 0
-            logging._releaseLock()
+            with logging._lock:
+                self.abort = 0
             self.timeout = 1
             self.ready = ready
             self.verify = verify
@@ -1034,9 +1026,8 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None):
                                            self.timeout)
                 if rd:
                     self.handle_request()
-                logging._acquireLock()
-                abort = self.abort
-                logging._releaseLock()
+                with logging._lock:
+                    abort = self.abort
             self.server_close()
 
     class Server(threading.Thread):
@@ -1057,9 +1048,8 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None):
                 self.port = server.server_address[1]
             self.ready.set()
             global _listener
-            logging._acquireLock()
-            _listener = server
-            logging._releaseLock()
+            with logging._lock:
+                _listener = server
             server.serve_until_stopped()
 
     return Server(ConfigSocketReceiver, ConfigStreamHandler, port, verify)
@@ -1069,10 +1059,7 @@ def stopListening():
     Stop the listening server which was created with a call to listen().
     """
     global _listener
-    logging._acquireLock()
-    try:
+    with logging._lock:
         if _listener:
             _listener.abort = 1
             _listener = None
-    finally:
-        logging._releaseLock()

Lib/logging/handlers.py

@@ -23,11 +23,17 @@ Copyright (C) 2001-2021 Vinay Sajip. All Rights Reserved.
 To use, simply 'import logging.handlers' and log away!
 """
 
-import io, logging, socket, os, pickle, struct, time, re
-from stat import ST_DEV, ST_INO, ST_MTIME
-import queue
-import threading
 import copy
+import io
+import logging
+import os
+import pickle
+import queue
+import re
+import socket
+import struct
+import threading
+import time
 
 #
 # Some constants...
@@ -272,7 +278,7 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         # path object (see Issue #27493), but self.baseFilename will be a string
         filename = self.baseFilename
         if os.path.exists(filename):
-            t = os.stat(filename)[ST_MTIME]
+            t = int(os.stat(filename).st_mtime)
         else:
             t = int(time.time())
         self.rolloverAt = self.computeRollover(t)
@@ -304,10 +310,10 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                 rotate_ts = _MIDNIGHT
             else:
                 rotate_ts = ((self.atTime.hour * 60 + self.atTime.minute)*60 +
-                             self.atTime.second)
+                    self.atTime.second)
 
             r = rotate_ts - ((currentHour * 60 + currentMinute) * 60 +
-                             currentSecond)
+                currentSecond)
             if r <= 0:
                 # Rotate time is before the current time (for example when
                 # self.rotateAt is 13:45 and it now 14:15), rotation is
@@ -465,8 +471,7 @@ class WatchedFileHandler(logging.FileHandler):
     This handler is not appropriate for use under Windows, because
     under Windows open files cannot be moved or renamed - logging
     opens the files with exclusive locks - and so there is no need
-    for such a handler. Furthermore, ST_INO is not supported under
-    Windows; stat always returns zero for this value.
+    for such a handler.
 
     This handler is based on a suggestion and patch by Chad J.
     Schroeder.
@@ -482,9 +487,11 @@ class WatchedFileHandler(logging.FileHandler):
         self._statstream()
 
     def _statstream(self):
-        if self.stream:
-            sres = os.fstat(self.stream.fileno())
-            self.dev, self.ino = sres[ST_DEV], sres[ST_INO]
+        if self.stream is None:
+            return
+        sres = os.fstat(self.stream.fileno())
+        self.dev = sres.st_dev
+        self.ino = sres.st_ino
 
     def reopenIfNeeded(self):
         """
@@ -494,6 +501,9 @@ class WatchedFileHandler(logging.FileHandler):
         has, close the old stream and reopen the file to get the
         current stream.
         """
+        if self.stream is None:
+            return
+
         # Reduce the chance of race conditions by stat'ing by path only
         # once and then fstat'ing our new fd if we opened a new log stream.
         # See issue #14632: Thanks to John Mulligan for the problem report
@@ -501,18 +511,23 @@ class WatchedFileHandler(logging.FileHandler):
         try:
             # stat the file by path, checking for existence
             sres = os.stat(self.baseFilename)
+
+            # compare file system stat with that of our stream file handle
+            reopen = (sres.st_dev != self.dev or sres.st_ino != self.ino)
         except FileNotFoundError:
-            sres = None
-        # compare file system stat with that of our stream file handle
-        if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino:
-            if self.stream is not None:
-                # we have an open file handle, clean it up
-                self.stream.flush()
-                self.stream.close()
-                self.stream = None  # See Issue #21742: _open () might fail.
-                # open a new file handle and get new stat info from that fd
-                self.stream = self._open()
-                self._statstream()
+            reopen = True
+
+        if not reopen:
+            return
+
+        # we have an open file handle, clean it up
+        self.stream.flush()
+        self.stream.close()
+        self.stream = None  # See Issue #21742: _open () might fail.
+
+        # open a new file handle and get new stat info from that fd
+        self.stream = self._open()
+        self._statstream()
 
     def emit(self, record):
         """
@@ -682,15 +697,12 @@ class SocketHandler(logging.Handler):
         """
         Closes the socket.
         """
-        self.acquire()
-        try:
+        with self.lock:
             sock = self.sock
             if sock:
                 self.sock = None
                 sock.close()
             logging.Handler.close(self)
-        finally:
-            self.release()
 
 class DatagramHandler(SocketHandler):
     """
@@ -803,7 +815,7 @@ class SysLogHandler(logging.Handler):
         "panic":    LOG_EMERG,      #  DEPRECATED
         "warn":     LOG_WARNING,    #  DEPRECATED
         "warning":  LOG_WARNING,
-    }
+        }
 
     facility_names = {
         "auth":         LOG_AUTH,
@@ -830,7 +842,7 @@ class SysLogHandler(logging.Handler):
         "local5":       LOG_LOCAL5,
         "local6":       LOG_LOCAL6,
         "local7":       LOG_LOCAL7,
-    }
+        }
 
     # Originally added to work around GH-43683. Unnecessary since GH-50043 but kept
     # for backwards compatibility.
@@ -950,15 +962,12 @@ class SysLogHandler(logging.Handler):
         """
         Closes the socket.
         """
-        self.acquire()
-        try:
+        with self.lock:
             sock = self.socket
             if sock:
                 self.socket = None
                 sock.close()
             logging.Handler.close(self)
-        finally:
-            self.release()
 
     def mapPriority(self, levelName):
         """
@@ -1031,7 +1040,8 @@ class SMTPHandler(logging.Handler):
         only be used when authentication credentials are supplied. The tuple
         will be either an empty tuple, or a single-value tuple with the name
         of a keyfile, or a 2-value tuple with the names of the keyfile and
-        certificate file. (This tuple is passed to the `starttls` method).
+        certificate file. (This tuple is passed to the
+        `ssl.SSLContext.load_cert_chain` method).
         A timeout in seconds can be specified for the SMTP connection (the
         default is one second).
         """
@@ -1084,8 +1094,23 @@ class SMTPHandler(logging.Handler):
             msg.set_content(self.format(record))
             if self.username:
                 if self.secure is not None:
+                    import ssl
+
+                    try:
+                        keyfile = self.secure[0]
+                    except IndexError:
+                        keyfile = None
+
+                    try:
+                        certfile = self.secure[1]
+                    except IndexError:
+                        certfile = None
+
+                    context = ssl._create_stdlib_context(
+                        certfile=certfile, keyfile=keyfile
+                    )
                     smtp.ehlo()
-                    smtp.starttls(*self.secure)
+                    smtp.starttls(context=context)
                     smtp.ehlo()
                 smtp.login(self.username, self.password)
             smtp.send_message(msg)
@@ -1132,10 +1157,10 @@ class NTEventLogHandler(logging.Handler):
                 logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE,
                 logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
                 logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
-            }
+         }
         except ImportError:
-            print("The Python Win32 extensions for NT (service, event " \
-                  "logging) appear not to be available.")
+            print("The Python Win32 extensions for NT (service, event "\
+                        "logging) appear not to be available.")
             self._welu = None
 
     def getMessageID(self, record):
@@ -1330,11 +1355,8 @@ class BufferingHandler(logging.Handler):
 
         This version just zaps the buffer to empty.
         """
-        self.acquire()
-        try:
+        with self.lock:
             self.buffer.clear()
-        finally:
-            self.release()
 
     def close(self):
         """
@@ -1378,17 +1400,14 @@ class MemoryHandler(BufferingHandler):
         Check for buffer full or a record at the flushLevel or higher.
         """
         return (len(self.buffer) >= self.capacity) or \
-            (record.levelno >= self.flushLevel)
+                (record.levelno >= self.flushLevel)
 
     def setTarget(self, target):
         """
         Set the target handler for this handler.
         """
-        self.acquire()
-        try:
+        with self.lock:
             self.target = target
-        finally:
-            self.release()
 
     def flush(self):
         """
@@ -1398,14 +1417,11 @@ class MemoryHandler(BufferingHandler):
 
         The record buffer is only cleared if a target has been set.
         """
-        self.acquire()
-        try:
+        with self.lock:
             if self.target:
                 for record in self.buffer:
                     self.target.handle(record)
                 self.buffer.clear()
-        finally:
-            self.release()
 
     def close(self):
         """
@@ -1416,12 +1432,9 @@ class MemoryHandler(BufferingHandler):
             if self.flushOnClose:
                 self.flush()
         finally:
-            self.acquire()
-            try:
+            with self.lock:
                 self.target = None
                 BufferingHandler.close(self)
-            finally:
-                self.release()
 
 
 class QueueHandler(logging.Handler):
@@ -1532,6 +1545,9 @@ class QueueListener(object):
         This starts up a background thread to monitor the queue for
         LogRecords to process.
         """
+        if self._thread is not None:
+            raise RuntimeError("Listener already started")
+
         self._thread = t = threading.Thread(target=self._monitor)
         t.daemon = True
         t.start()
@@ -1603,6 +1619,7 @@ class QueueListener(object):
         Note that if you don't call this before your application exits, there
         may be some records still left on the queue, which won't be processed.
         """
-        self.enqueue_sentinel()
-        self._thread.join()
-        self._thread = None
+        if self._thread:  # see gh-114706 - allow calling this more than once
+            self.enqueue_sentinel()
+            self._thread.join()
+            self._thread = None

Lib/lzma.py

@@ -128,7 +128,7 @@ class LZMAFile(_compression.BaseStream):
 
         if self._mode == _MODE_READ:
             raw = _compression.DecompressReader(self._fp, LZMADecompressor,
-                                                trailing_error=LZMAError, format=format, filters=filters)
+                trailing_error=LZMAError, format=format, filters=filters)
             self._buffer = io.BufferedReader(raw)
 
     def close(self):

Lib/multiprocessing/connection.py

@@ -19,7 +19,6 @@ import time
 import tempfile
 import itertools
 
-import _multiprocessing
 
 from . import util
 
@@ -28,6 +27,7 @@ from .context import reduction
 _ForkingPickler = reduction.ForkingPickler
 
 try:
+    import _multiprocessing
     import _winapi
     from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE
 except ImportError:
@@ -846,7 +846,7 @@ _MD5_DIGEST_LEN = 16
 _LEGACY_LENGTHS = (_MD5ONLY_MESSAGE_LENGTH, _MD5_DIGEST_LEN)
 
 
-def _get_digest_name_and_payload(message: bytes) -> (str, bytes):
+def _get_digest_name_and_payload(message):  # type: (bytes) -> tuple[str, bytes]
     """Returns a digest name and the payload for a response hash.
 
     If a legacy protocol is detected based on the message length
@@ -956,7 +956,7 @@ def answer_challenge(connection, authkey: bytes):
                 f'Protocol error, expected challenge: {message=}')
     message = message[len(_CHALLENGE):]
     if len(message) < _MD5ONLY_MESSAGE_LENGTH:
-        raise AuthenticationError('challenge too short: {len(message)} bytes')
+        raise AuthenticationError(f'challenge too short: {len(message)} bytes')
     digest = _create_response(authkey, message)
     connection.send_bytes(digest)
     response = connection.recv_bytes(256)        # reject large message
@@ -1012,8 +1012,20 @@ if sys.platform == 'win32':
         # returning the first signalled might create starvation issues.)
         L = list(handles)
         ready = []
+        # Windows limits WaitForMultipleObjects at 64 handles, and we use a
+        # few for synchronisation, so we switch to batched waits at 60.
+        if len(L) > 60:
+            try:
+                res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout)
+            except TimeoutError:
+                return []
+            ready.extend(L[i] for i in res)
+            if res:
+                L = [h for i, h in enumerate(L) if i > res[0] & i not in res]
+            timeout = 0
         while L:
-            res = _winapi.WaitForMultipleObjects(L, False, timeout)
+            short_L = L[:60] if len(L) > 60 else L
+            res = _winapi.WaitForMultipleObjects(short_L, False, timeout)
             if res == WAIT_TIMEOUT:
                 break
             elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):

Lib/multiprocessing/context.py

@@ -145,7 +145,7 @@ class BaseContext(object):
         '''Check whether this is a fake forked process in a frozen executable.
         If so then run code specified by commandline and exit.
         '''
-        if sys.platform == 'win32' and getattr(sys, 'frozen', False):
+        if self.get_start_method() == 'spawn' and getattr(sys, 'frozen', False):
             from .spawn import freeze_support
             freeze_support()
 

Lib/multiprocessing/forkserver.py

@@ -1,3 +1,4 @@
+import atexit
 import errno
 import os
 import selectors
@@ -167,6 +168,8 @@ class ForkServer(object):
 def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
     '''Run forkserver.'''
     if preload:
+        if sys_path is not None:
+            sys.path[:] = sys_path
         if '__main__' in preload and main_path is not None:
             process.current_process()._inheriting = True
             try:
@@ -271,6 +274,8 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
                                 selector.close()
                                 unused_fds = [alive_r, child_w, sig_r, sig_w]
                                 unused_fds.extend(pid_to_fd.values())
+                                atexit._clear()
+                                atexit.register(util._exit_function)
                                 code = _serve_one(child_r, fds,
                                                   unused_fds,
                                                   old_handlers)
@@ -278,6 +283,7 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
                                 sys.excepthook(*sys.exc_info())
                                 sys.stderr.flush()
                             finally:
+                                atexit._run_exitfuncs()
                                 os._exit(code)
                         else:
                             # Send pid to client process

Lib/multiprocessing/managers.py

@@ -90,7 +90,10 @@ def dispatch(c, id, methodname, args=(), kwds={}):
     kind, result = c.recv()
     if kind == '#RETURN':
         return result
-    raise convert_to_error(kind, result)
+    try:
+        raise convert_to_error(kind, result)
+    finally:
+        del result  # break reference cycle
 
 def convert_to_error(kind, result):
     if kind == '#ERROR':
@@ -755,22 +758,29 @@ class BaseProxy(object):
     _address_to_local = {}
     _mutex = util.ForkAwareThreadLock()
 
+    # Each instance gets a `_serial` number. Unlike `id(...)`, this number
+    # is never reused.
+    _next_serial = 1
+
     def __init__(self, token, serializer, manager=None,
                  authkey=None, exposed=None, incref=True, manager_owned=False):
         with BaseProxy._mutex:
-            tls_idset = BaseProxy._address_to_local.get(token.address, None)
-            if tls_idset is None:
-                tls_idset = util.ForkAwareLocal(), ProcessLocalSet()
-                BaseProxy._address_to_local[token.address] = tls_idset
+            tls_serials = BaseProxy._address_to_local.get(token.address, None)
+            if tls_serials is None:
+                tls_serials = util.ForkAwareLocal(), ProcessLocalSet()
+                BaseProxy._address_to_local[token.address] = tls_serials
+
+            self._serial = BaseProxy._next_serial
+            BaseProxy._next_serial += 1
 
         # self._tls is used to record the connection used by this
         # thread to communicate with the manager at token.address
-        self._tls = tls_idset[0]
+        self._tls = tls_serials[0]
 
-        # self._idset is used to record the identities of all shared
-        # objects for which the current process owns references and
+        # self._all_serials is a set used to record the identities of all
+        # shared objects for which the current process owns references and
         # which are in the manager at token.address
-        self._idset = tls_idset[1]
+        self._all_serials = tls_serials[1]
 
         self._token = token
         self._id = self._token.id
@@ -833,7 +843,10 @@ class BaseProxy(object):
             conn = self._Client(token.address, authkey=self._authkey)
             dispatch(conn, None, 'decref', (token.id,))
             return proxy
-        raise convert_to_error(kind, result)
+        try:
+            raise convert_to_error(kind, result)
+        finally:
+            del result   # break reference cycle
 
     def _getvalue(self):
         '''
@@ -850,20 +863,20 @@ class BaseProxy(object):
         dispatch(conn, None, 'incref', (self._id,))
         util.debug('INCREF %r', self._token.id)
 
-        self._idset.add(self._id)
+        self._all_serials.add(self._serial)
 
         state = self._manager and self._manager._state
 
         self._close = util.Finalize(
             self, BaseProxy._decref,
-            args=(self._token, self._authkey, state,
-                  self._tls, self._idset, self._Client),
+            args=(self._token, self._serial, self._authkey, state,
+                  self._tls, self._all_serials, self._Client),
             exitpriority=10
             )
 
     @staticmethod
-    def _decref(token, authkey, state, tls, idset, _Client):
-        idset.discard(token.id)
+    def _decref(token, serial, authkey, state, tls, idset, _Client):
+        idset.discard(serial)
 
         # check whether manager is still alive
         if state is None or state.value == State.STARTED:
@@ -1159,15 +1172,19 @@ class ListProxy(BaseListProxy):
         self._callmethod('__imul__', (value,))
         return self
 
+    __class_getitem__ = classmethod(types.GenericAlias)
 
-DictProxy = MakeProxyType('DictProxy', (
+
+_BaseDictProxy = MakeProxyType('DictProxy', (
     '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__',
     '__setitem__', 'clear', 'copy', 'get', 'items',
     'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'
     ))
-DictProxy._method_to_typeid_ = {
+_BaseDictProxy._method_to_typeid_ = {
     '__iter__': 'Iterator',
     }
+class DictProxy(_BaseDictProxy):
+    __class_getitem__ = classmethod(types.GenericAlias)
 
 
 ArrayProxy = MakeProxyType('ArrayProxy', (

Lib/multiprocessing/pool.py

@@ -200,7 +200,7 @@ class Pool(object):
         self._initargs = initargs
 
         if processes is None:
-            processes = os.cpu_count() or 1
+            processes = os.process_cpu_count() or 1
         if processes < 1:
             raise ValueError("Number of processes must be at least 1")
         if maxtasksperchild is not None:

Lib/multiprocessing/popen_fork.py

@@ -1,3 +1,4 @@
+import atexit
 import os
 import signal
 
@@ -66,10 +67,13 @@ class Popen(object):
         self.pid = os.fork()
         if self.pid == 0:
             try:
+                atexit._clear()
+                atexit.register(util._exit_function)
                 os.close(parent_r)
                 os.close(parent_w)
                 code = process_obj._bootstrap(parent_sentinel=child_r)
             finally:
+                atexit._run_exitfuncs()
                 os._exit(code)
         else:
             os.close(child_w)

Lib/multiprocessing/popen_spawn_win32.py

@@ -3,6 +3,7 @@ import msvcrt
 import signal
 import sys
 import _winapi
+from subprocess import STARTUPINFO, STARTF_FORCEOFFFEEDBACK
 
 from .context import reduction, get_spawning_popen, set_spawning_popen
 from . import spawn
@@ -74,7 +75,8 @@ class Popen(object):
             try:
                 hp, ht, pid, tid = _winapi.CreateProcess(
                     python_exe, cmd,
-                    None, None, False, 0, env, None, None)
+                    None, None, False, 0, env, None,
+                    STARTUPINFO(dwFlags=STARTF_FORCEOFFFEEDBACK))
                 _winapi.CloseHandle(ht)
             except:
                 _winapi.CloseHandle(rhandle)

Lib/multiprocessing/process.py

@@ -310,11 +310,8 @@ class BaseProcess(object):
                 # _run_after_forkers() is executed
                 del old_process
             util.info('child process calling self.run()')
-            try:
-                self.run()
-                exitcode = 0
-            finally:
-                util._exit_function()
+            self.run()
+            exitcode = 0
         except SystemExit as e:
             if e.code is None:
                 exitcode = 0

Lib/multiprocessing/queues.py

@@ -20,8 +20,6 @@ import errno
 
 from queue import Empty, Full
 
-import _multiprocessing
-
 from . import connection
 from . import context
 _ForkingPickler = context.reduction.ForkingPickler

Lib/multiprocessing/resource_tracker.py

@@ -29,8 +29,12 @@ __all__ = ['ensure_running', 'register', 'unregister']
 _HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask')
 _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM)
 
+def cleanup_noop(name):
+    raise RuntimeError('noop should never be registered or cleaned up')
+
 _CLEANUP_FUNCS = {
-    'noop': lambda: None,
+    'noop': cleanup_noop,
+    'dummy': lambda name: None,  # Dummy resource used in tests
 }
 
 if os.name == 'posix':
@@ -61,6 +65,7 @@ class ResourceTracker(object):
         self._lock = threading.RLock()
         self._fd = None
         self._pid = None
+        self._exitcode = None
 
     def _reentrant_call_error(self):
         # gh-109629: this happens if an explicit call to the ResourceTracker
@@ -70,22 +75,53 @@ class ResourceTracker(object):
         raise ReentrantCallError(
             "Reentrant call into the multiprocessing resource tracker")
 
-    def _stop(self):
-        with self._lock:
-            # This should not happen (_stop() isn't called by a finalizer)
-            # but we check for it anyway.
-            if self._lock._recursion_count() > 1:
-                return self._reentrant_call_error()
-            if self._fd is None:
-                # not running
-                return
+    def __del__(self):
+        # making sure child processess are cleaned before ResourceTracker
+        # gets destructed.
+        # see https://github.com/python/cpython/issues/88887
+        self._stop(use_blocking_lock=False)
 
-            # closing the "alive" file descriptor stops main()
-            os.close(self._fd)
-            self._fd = None
+    def _stop(self, use_blocking_lock=True):
+        if use_blocking_lock:
+            with self._lock:
+                self._stop_locked()
+        else:
+            acquired = self._lock.acquire(blocking=False)
+            try:
+                self._stop_locked()
+            finally:
+                if acquired:
+                    self._lock.release()
 
-            os.waitpid(self._pid, 0)
-            self._pid = None
+    def _stop_locked(
+        self,
+        close=os.close,
+        waitpid=os.waitpid,
+        waitstatus_to_exitcode=os.waitstatus_to_exitcode,
+    ):
+        # This shouldn't happen (it might when called by a finalizer)
+        # so we check for it anyway.
+        if self._lock._recursion_count() > 1:
+            return self._reentrant_call_error()
+        if self._fd is None:
+            # not running
+            return
+        if self._pid is None:
+            return
+
+        # closing the "alive" file descriptor stops main()
+        close(self._fd)
+        self._fd = None
+
+        _, status = waitpid(self._pid, 0)
+
+        self._pid = None
+
+        try:
+            self._exitcode = waitstatus_to_exitcode(status)
+        except ValueError:
+            # os.waitstatus_to_exitcode may raise an exception for invalid values
+            self._exitcode = None
 
     def getfd(self):
         self.ensure_running()
@@ -119,6 +155,7 @@ class ResourceTracker(object):
                     pass
                 self._fd = None
                 self._pid = None
+                self._exitcode = None
 
                 warnings.warn('resource_tracker: process died unexpectedly, '
                               'relaunching.  Some resources might leak.')
@@ -142,13 +179,14 @@ class ResourceTracker(object):
                 # that can make the child die before it registers signal handlers
                 # for SIGINT and SIGTERM. The mask is unregistered after spawning
                 # the child.
+                prev_sigmask = None
                 try:
                     if _HAVE_SIGMASK:
-                        signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS)
+                        prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS)
                     pid = util.spawnv_passfds(exe, args, fds_to_pass)
                 finally:
-                    if _HAVE_SIGMASK:
-                        signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS)
+                    if prev_sigmask is not None:
+                        signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask)
             except:
                 os.close(w)
                 raise
@@ -221,6 +259,8 @@ def main(fd):
             pass
 
     cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()}
+    exit_code = 0
+
     try:
         # keep track of registered/unregistered resources
         with open(fd, 'rb') as f:
@@ -242,6 +282,7 @@ def main(fd):
                     else:
                         raise RuntimeError('unrecognized command %r' % cmd)
                 except Exception:
+                    exit_code = 3
                     try:
                         sys.excepthook(*sys.exc_info())
                     except:
@@ -251,9 +292,17 @@ def main(fd):
         for rtype, rtype_cache in cache.items():
             if rtype_cache:
                 try:
-                    warnings.warn('resource_tracker: There appear to be %d '
-                                  'leaked %s objects to clean up at shutdown' %
-                                  (len(rtype_cache), rtype))
+                    exit_code = 1
+                    if rtype == 'dummy':
+                        # The test 'dummy' resource is expected to leak.
+                        # We skip the warning (and *only* the warning) for it.
+                        pass
+                    else:
+                        warnings.warn(
+                            f'resource_tracker: There appear to be '
+                            f'{len(rtype_cache)} leaked {rtype} objects to '
+                            f'clean up at shutdown: {rtype_cache}'
+                        )
                 except Exception:
                     pass
             for name in rtype_cache:
@@ -264,6 +313,9 @@ def main(fd):
                     try:
                         _CLEANUP_FUNCS[rtype](name)
                     except Exception as e:
+                        exit_code = 2
                         warnings.warn('resource_tracker: %r: %s' % (name, e))
                 finally:
                     pass
+
+        sys.exit(exit_code)

Lib/multiprocessing/shared_memory.py

@@ -71,8 +71,9 @@ class SharedMemory:
     _flags = os.O_RDWR
     _mode = 0o600
     _prepend_leading_slash = True if _USE_POSIX else False
+    _track = True
 
-    def __init__(self, name=None, create=False, size=0):
+    def __init__(self, name=None, create=False, size=0, *, track=True):
         if not size >= 0:
             raise ValueError("'size' must be a positive integer")
         if create:
@@ -82,6 +83,7 @@ class SharedMemory:
         if name is None and not self._flags & os.O_EXCL:
             raise ValueError("'name' can only be None if create=True")
 
+        self._track = track
         if _USE_POSIX:
 
             # POSIX Shared Memory
@@ -116,8 +118,8 @@ class SharedMemory:
             except OSError:
                 self.unlink()
                 raise
-
-            resource_tracker.register(self._name, "shared_memory")
+            if self._track:
+                resource_tracker.register(self._name, "shared_memory")
 
         else:
 
@@ -236,12 +238,20 @@ class SharedMemory:
     def unlink(self):
         """Requests that the underlying shared memory block be destroyed.
 
-        In order to ensure proper cleanup of resources, unlink should be
-        called once (and only once) across all processes which have access
-        to the shared memory block."""
+        Unlink should be called once (and only once) across all handles
+        which have access to the shared memory block, even if these
+        handles belong to different processes. Closing and unlinking may
+        happen in any order, but trying to access data inside a shared
+        memory block after unlinking may result in memory errors,
+        depending on platform.
+
+        This method has no effect on Windows, where the only way to
+        delete a shared memory block is to close all handles."""
+
         if _USE_POSIX and self._name:
             _posixshmem.shm_unlink(self._name)
-            resource_tracker.unregister(self._name, "shared_memory")
+            if self._track:
+                resource_tracker.unregister(self._name, "shared_memory")
 
 
 _encoding = "utf8"

Lib/multiprocessing/synchronize.py

@@ -174,7 +174,7 @@ class Lock(SemLock):
                 name = process.current_process().name
                 if threading.current_thread().name != 'MainThread':
                     name += '|' + threading.current_thread().name
-            elif self._semlock._get_value() == 1:
+            elif not self._semlock._is_zero():
                 name = 'None'
             elif self._semlock._count() > 0:
                 name = 'SomeOtherThread'
@@ -200,7 +200,7 @@ class RLock(SemLock):
                 if threading.current_thread().name != 'MainThread':
                     name += '|' + threading.current_thread().name
                 count = self._semlock._count()
-            elif self._semlock._get_value() == 1:
+            elif not self._semlock._is_zero():
                 name, count = 'None', 0
             elif self._semlock._count() > 0:
                 name, count = 'SomeOtherThread', 'nonzero'
@@ -360,7 +360,7 @@ class Event(object):
                 return True
             return False
 
-    def __repr__(self) -> str:
+    def __repr__(self):
         set_status = 'set' if self.is_set() else 'unset'
         return f"<{type(self).__qualname__} at {id(self):#x} {set_status}>"
 #

Lib/multiprocessing/util.py

@@ -64,8 +64,7 @@ def get_logger():
     global _logger
     import logging
 
-    logging._acquireLock()
-    try:
+    with logging._lock:
         if not _logger:
 
             _logger = logging.getLogger(LOGGER_NAME)
@@ -79,9 +78,6 @@ def get_logger():
                 atexit._exithandlers.remove((_exit_function, (), {}))
                 atexit._exithandlers.append((_exit_function, (), {}))
 
-    finally:
-        logging._releaseLock()
-
     return _logger
 
 def log_to_stderr(level=None):
@@ -106,11 +102,7 @@ def log_to_stderr(level=None):
 # Abstract socket support
 
 def _platform_supports_abstract_sockets():
-    if sys.platform == "linux":
-        return True
-    if hasattr(sys, 'getandroidapilevel'):
-        return True
-    return False
+    return sys.platform in ("linux", "android")
 
 
 def is_abstract_socket_namespace(address):
@@ -130,10 +122,7 @@ abstract_sockets_supported = _platform_supports_abstract_sockets()
 #
 
 def _remove_temp_dir(rmtree, tempdir):
-    def onerror(func, path, err_info):
-        if not issubclass(err_info[0], FileNotFoundError):
-            raise
-    rmtree(tempdir, onerror=onerror)
+    rmtree(tempdir)
 
     current_process = process.current_process()
     # current_process() can be None if the finalizer is called

Lib/netrc.py

@@ -2,11 +2,24 @@
 
 # Module and documentation by Eric S. Raymond, 21 Dec 1998
 
-import os, shlex, stat
+import os, stat
 
 __all__ = ["netrc", "NetrcParseError"]
 
 
+def _can_security_check():
+    # On WASI, getuid() is indicated as a stub but it may also be missing.
+    return os.name == 'posix' and hasattr(os, 'getuid')
+
+
+def _getpwuid(uid):
+    try:
+        import pwd
+        return pwd.getpwuid(uid)[0]
+    except (ImportError, LookupError):
+        return f'uid {uid}'
+
+
 class NetrcParseError(Exception):
     """Exception raised on syntax errors in the .netrc file."""
     def __init__(self, msg, filename=None, lineno=None):
@@ -142,18 +155,12 @@ class netrc:
             self._security_check(fp, default_netrc, self.hosts[entryname][0])
 
     def _security_check(self, fp, default_netrc, login):
-        if os.name == 'posix' and default_netrc and login != "anonymous":
+        if _can_security_check() and default_netrc and login != "anonymous":
             prop = os.fstat(fp.fileno())
-            if prop.st_uid != os.getuid():
-                import pwd
-                try:
-                    fowner = pwd.getpwuid(prop.st_uid)[0]
-                except KeyError:
-                    fowner = 'uid %s' % prop.st_uid
-                try:
-                    user = pwd.getpwuid(os.getuid())[0]
-                except KeyError:
-                    user = 'uid %s' % os.getuid()
+            current_user_id = os.getuid()
+            if prop.st_uid != current_user_id:
+                fowner = _getpwuid(prop.st_uid)
+                user = _getpwuid(current_user_id)
                 raise NetrcParseError(
                     (f"~/.netrc file owner ({fowner}, {user}) does not match"
                      " current user"))

Lib/ntpath.py

@@ -19,18 +19,17 @@ devnull = 'nul'
 
 import os
 import sys
-import stat
 import genericpath
 from genericpath import *
 
-
 __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext",
            "basename","dirname","commonprefix","getsize","getmtime",
            "getatime","getctime", "islink","exists","lexists","isdir","isfile",
-           "ismount", "expanduser","expandvars","normpath","abspath",
-           "curdir","pardir","sep","pathsep","defpath","altsep",
+           "ismount","isreserved","expanduser","expandvars","normpath",
+           "abspath","curdir","pardir","sep","pathsep","defpath","altsep",
            "extsep","devnull","realpath","supports_unicode_filenames","relpath",
-           "samefile", "sameopenfile", "samestat", "commonpath", "isjunction"]
+           "samefile", "sameopenfile", "samestat", "commonpath", "isjunction",
+           "isdevdrive", "ALLOW_MISSING"]
 
 def _get_bothseps(path):
     if isinstance(path, bytes):
@@ -78,12 +77,6 @@ except ImportError:
         return s.replace('/', '\\').lower()
 
 
-# Return whether a path is absolute.
-# Trivial in Posix, harder on Windows.
-# For Windows it is absolute if it starts with a slash or backslash (current
-# volume), or if a pathname after the volume-letter-and-colon or UNC-resource
-# starts with a slash or backslash.
-
 def isabs(s):
     """Test whether a path is absolute"""
     s = os.fspath(s)
@@ -91,16 +84,15 @@ def isabs(s):
         sep = b'\\'
         altsep = b'/'
         colon_sep = b':\\'
+        double_sep = b'\\\\'
     else:
         sep = '\\'
         altsep = '/'
         colon_sep = ':\\'
+        double_sep = '\\\\'
     s = s[:3].replace(altsep, sep)
     # Absolute: UNC, device, and paths with a drive and root.
-    # LEGACY BUG: isabs("/x") should be false since the path has no drive.
-    if s.startswith(sep) or s.startswith(colon_sep, 1):
-        return True
-    return False
+    return s.startswith(colon_sep, 1) or s.startswith(double_sep)
 
 
 # Join two (or more) paths.
@@ -109,16 +101,14 @@ def join(path, *paths):
     if isinstance(path, bytes):
         sep = b'\\'
         seps = b'\\/'
-        colon = b':'
+        colon_seps = b':\\/'
     else:
         sep = '\\'
         seps = '\\/'
-        colon = ':'
+        colon_seps = ':\\/'
     try:
-        if not paths:
-            path[:0] + sep  #23780: Ensure compatible data type even if p is null.
         result_drive, result_root, result_path = splitroot(path)
-        for p in map(os.fspath, paths):
+        for p in paths:
             p_drive, p_root, p_path = splitroot(p)
             if p_root:
                 # Second path is absolute
@@ -142,7 +132,7 @@ def join(path, *paths):
             result_path = result_path + p_path
         ## add separator between UNC and non-absolute path
         if (result_path and not result_root and
-            result_drive and result_drive[-1:] not in colon + seps):
+            result_drive and result_drive[-1] not in colon_seps):
             return result_drive + sep + result_path
         return result_drive + result_root + result_path
     except (TypeError, AttributeError, BytesWarning):
@@ -176,56 +166,52 @@ def splitdrive(p):
     return drive, root + tail
 
 
-def splitroot(p):
-    """Split a pathname into drive, root and tail. The drive is defined
-    exactly as in splitdrive(). On Windows, the root may be a single path
-    separator or an empty string. The tail contains anything after the root.
-    For example:
+try:
+    from nt import _path_splitroot_ex as splitroot
+except ImportError:
+    def splitroot(p):
+        """Split a pathname into drive, root and tail.
 
-        splitroot('//server/share/') == ('//server/share', '/', '')
-        splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney')
-        splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham')
-        splitroot('Windows/notepad') == ('', '', 'Windows/notepad')
-    """
-    p = os.fspath(p)
-    if isinstance(p, bytes):
-        sep = b'\\'
-        altsep = b'/'
-        colon = b':'
-        unc_prefix = b'\\\\?\\UNC\\'
-        empty = b''
-    else:
-        sep = '\\'
-        altsep = '/'
-        colon = ':'
-        unc_prefix = '\\\\?\\UNC\\'
-        empty = ''
-    normp = p.replace(altsep, sep)
-    if normp[:1] == sep:
-        if normp[1:2] == sep:
-            # UNC drives, e.g. \\server\share or \\?\UNC\server\share
-            # Device drives, e.g. \\.\device or \\?\device
-            start = 8 if normp[:8].upper() == unc_prefix else 2
-            index = normp.find(sep, start)
-            if index == -1:
-                return p, empty, empty
-            index2 = normp.find(sep, index + 1)
-            if index2 == -1:
-                return p, empty, empty
-            return p[:index2], p[index2:index2 + 1], p[index2 + 1:]
+        The tail contains anything after the root."""
+        p = os.fspath(p)
+        if isinstance(p, bytes):
+            sep = b'\\'
+            altsep = b'/'
+            colon = b':'
+            unc_prefix = b'\\\\?\\UNC\\'
+            empty = b''
         else:
-            # Relative path with root, e.g. \Windows
-            return empty, p[:1], p[1:]
-    elif normp[1:2] == colon:
-        if normp[2:3] == sep:
-            # Absolute drive-letter path, e.g. X:\Windows
-            return p[:2], p[2:3], p[3:]
+            sep = '\\'
+            altsep = '/'
+            colon = ':'
+            unc_prefix = '\\\\?\\UNC\\'
+            empty = ''
+        normp = p.replace(altsep, sep)
+        if normp[:1] == sep:
+            if normp[1:2] == sep:
+                # UNC drives, e.g. \\server\share or \\?\UNC\server\share
+                # Device drives, e.g. \\.\device or \\?\device
+                start = 8 if normp[:8].upper() == unc_prefix else 2
+                index = normp.find(sep, start)
+                if index == -1:
+                    return p, empty, empty
+                index2 = normp.find(sep, index + 1)
+                if index2 == -1:
+                    return p, empty, empty
+                return p[:index2], p[index2:index2 + 1], p[index2 + 1:]
+            else:
+                # Relative path with root, e.g. \Windows
+                return empty, p[:1], p[1:]
+        elif normp[1:2] == colon:
+            if normp[2:3] == sep:
+                # Absolute drive-letter path, e.g. X:\Windows
+                return p[:2], p[2:3], p[3:]
+            else:
+                # Relative path with drive, e.g. X:Windows
+                return p[:2], empty, p[2:]
         else:
-            # Relative path with drive, e.g. X:Windows
-            return p[:2], empty, p[2:]
-    else:
-        # Relative path, e.g. Windows
-        return empty, empty, p
+            # Relative path, e.g. Windows
+            return empty, empty, p
 
 
 # Split a path in head (everything up to the last '/') and tail (the
@@ -277,33 +263,6 @@ def dirname(p):
     return split(p)[0]
 
 
-# Is a path a junction?
-
-if hasattr(os.stat_result, 'st_reparse_tag'):
-    def isjunction(path):
-        """Test whether a path is a junction"""
-        try:
-            st = os.lstat(path)
-        except (OSError, ValueError, AttributeError):
-            return False
-        return bool(st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)
-else:
-    def isjunction(path):
-        """Test whether a path is a junction"""
-        os.fspath(path)
-        return False
-
-
-# Being true for dangling symbolic links is also useful.
-
-def lexists(path):
-    """Test whether a path exists.  Returns True for broken symbolic links"""
-    try:
-        st = os.lstat(path)
-    except (OSError, ValueError):
-        return False
-    return True
-
 # Is a path a mount point?
 # Any drive letter root (eg c:\)
 # Any share UNC (eg \\server\share)
@@ -338,6 +297,40 @@ def ismount(path):
         return False
 
 
+_reserved_chars = frozenset(
+    {chr(i) for i in range(32)} |
+    {'"', '*', ':', '<', '>', '?', '|', '/', '\\'}
+)
+
+_reserved_names = frozenset(
+    {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} |
+    {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} |
+    {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'}
+)
+
+def isreserved(path):
+    """Return true if the pathname is reserved by the system."""
+    # Refer to "Naming Files, Paths, and Namespaces":
+    # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    path = os.fsdecode(splitroot(path)[2]).replace(altsep, sep)
+    return any(_isreservedname(name) for name in reversed(path.split(sep)))
+
+def _isreservedname(name):
+    """Return true if the filename is reserved by the system."""
+    # Trailing dots and spaces are reserved.
+    if name[-1:] in ('.', ' '):
+        return name not in ('.', '..')
+    # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved.
+    # ASCII control characters (0-31) are reserved.
+    # Colon is reserved for file streams (e.g. "name:stream[:type]").
+    if _reserved_chars.intersection(name):
+        return True
+    # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules
+    # are complex and vary across Windows versions. On the side of
+    # caution, return True for names that may not be reserved.
+    return name.partition('.')[0].rstrip(' ').upper() in _reserved_names
+
+
 # Expand paths beginning with '~' or '~user'.
 # '~' means $HOME; '~user' means that user's home directory.
 # If the path doesn't begin with '~', or if the user or $HOME is unknown,
@@ -353,24 +346,23 @@ def expanduser(path):
     If user or $HOME is unknown, do nothing."""
     path = os.fspath(path)
     if isinstance(path, bytes):
+        seps = b'\\/'
         tilde = b'~'
     else:
+        seps = '\\/'
         tilde = '~'
     if not path.startswith(tilde):
         return path
     i, n = 1, len(path)
-    while i < n and path[i] not in _get_bothseps(path):
+    while i < n and path[i] not in seps:
         i += 1
 
     if 'USERPROFILE' in os.environ:
         userhome = os.environ['USERPROFILE']
-    elif not 'HOMEPATH' in os.environ:
+    elif 'HOMEPATH' not in os.environ:
         return path
     else:
-        try:
-            drive = os.environ['HOMEDRIVE']
-        except KeyError:
-            drive = ''
+        drive = os.environ.get('HOMEDRIVE', '')
         userhome = join(drive, os.environ['HOMEPATH'])
 
     if i != 1: #~user
@@ -521,7 +513,7 @@ def expandvars(path):
 # Previously, this function also truncated pathnames to 8+3 format,
 # but as this module is called "ntpath", that's obviously wrong!
 try:
-    from nt import _path_normpath
+    from nt import _path_normpath as normpath
 
 except ImportError:
     def normpath(path):
@@ -560,37 +552,22 @@ except ImportError:
             comps.append(curdir)
         return prefix + sep.join(comps)
 
-else:
-    def normpath(path):
-        """Normalize path, eliminating double slashes, etc."""
-        path = os.fspath(path)
-        if isinstance(path, bytes):
-            return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
-        return _path_normpath(path) or "."
-
-
-def _abspath_fallback(path):
-    """Return the absolute version of a path as a fallback function in case
-    `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for
-    more.
-
-    """
-
-    path = os.fspath(path)
-    if not isabs(path):
-        if isinstance(path, bytes):
-            cwd = os.getcwdb()
-        else:
-            cwd = os.getcwd()
-        path = join(cwd, path)
-    return normpath(path)
 
 # Return an absolute path.
 try:
     from nt import _getfullpathname
 
 except ImportError: # not running on Windows - mock up something sensible
-    abspath = _abspath_fallback
+    def abspath(path):
+        """Return the absolute version of a path."""
+        path = os.fspath(path)
+        if not isabs(path):
+            if isinstance(path, bytes):
+                cwd = os.getcwdb()
+            else:
+                cwd = os.getcwd()
+            path = join(cwd, path)
+        return normpath(path)
 
 else:  # use native Windows method on Windows
     def abspath(path):
@@ -598,15 +575,36 @@ else:  # use native Windows method on Windows
         try:
             return _getfullpathname(normpath(path))
         except (OSError, ValueError):
-            return _abspath_fallback(path)
+            # See gh-75230, handle outside for cleaner traceback
+            pass
+        path = os.fspath(path)
+        if not isabs(path):
+            if isinstance(path, bytes):
+                sep = b'\\'
+                getcwd = os.getcwdb
+            else:
+                sep = '\\'
+                getcwd = os.getcwd
+            drive, root, path = splitroot(path)
+            # Either drive or root can be nonempty, but not both.
+            if drive or root:
+                try:
+                    path = join(_getfullpathname(drive + root), path)
+                except (OSError, ValueError):
+                    # Drive "\0:" cannot exist; use the root directory.
+                    path = drive + sep + path
+            else:
+                path = join(getcwd(), path)
+        return normpath(path)
 
 try:
-    from nt import _getfinalpathname, readlink as _nt_readlink
+    from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink
 except ImportError:
     # realpath is a no-op on systems without _getfinalpathname support.
-    realpath = abspath
+    def realpath(path, *, strict=False):
+        return abspath(path)
 else:
-    def _readlink_deep(path):
+    def _readlink_deep(path, ignored_error=OSError):
         # These error codes indicate that we should stop reading links and
         # return the path we currently have.
         # 1: ERROR_INVALID_FUNCTION
@@ -639,7 +637,7 @@ else:
                         path = old_path
                         break
                     path = normpath(join(dirname(old_path), path))
-            except OSError as ex:
+            except ignored_error as ex:
                 if ex.winerror in allowed_winerror:
                     break
                 raise
@@ -648,7 +646,7 @@ else:
                 break
         return path
 
-    def _getfinalpathname_nonstrict(path):
+    def _getfinalpathname_nonstrict(path, ignored_error=OSError):
         # These error codes indicate that we should stop resolving the path
         # and return the value we currently have.
         # 1: ERROR_INVALID_FUNCTION
@@ -664,9 +662,10 @@ else:
         # 87: ERROR_INVALID_PARAMETER
         # 123: ERROR_INVALID_NAME
         # 161: ERROR_BAD_PATHNAME
+        # 1005: ERROR_UNRECOGNIZED_VOLUME
         # 1920: ERROR_CANT_ACCESS_FILE
         # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink)
-        allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921
+        allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1005, 1920, 1921
 
         # Non-strict algorithm is to find as much of the target directory
         # as we can and join the rest.
@@ -675,23 +674,29 @@ else:
             try:
                 path = _getfinalpathname(path)
                 return join(path, tail) if tail else path
-            except OSError as ex:
+            except ignored_error as ex:
                 if ex.winerror not in allowed_winerror:
                     raise
                 try:
                     # The OS could not resolve this path fully, so we attempt
                     # to follow the link ourselves. If we succeed, join the tail
                     # and return.
-                    new_path = _readlink_deep(path)
+                    new_path = _readlink_deep(path,
+                                              ignored_error=ignored_error)
                     if new_path != path:
                         return join(new_path, tail) if tail else new_path
-                except OSError:
+                except ignored_error:
                     # If we fail to readlink(), let's keep traversing
                     pass
-                path, name = split(path)
-                # TODO (bpo-38186): Request the real file name from the directory
-                # entry using FindFirstFileW. For now, we will return the path
-                # as best we have it
+                # If we get these errors, try to get the real name of the file without accessing it.
+                if ex.winerror in (1, 5, 32, 50, 87, 1920, 1921):
+                    try:
+                        name = _findfirstfile(path)
+                        path, _ = split(path)
+                    except ignored_error:
+                        path, name = split(path)
+                else:
+                    path, name = split(path)
                 if path and not name:
                     return path + tail
                 tail = join(name, tail) if tail else name
@@ -705,7 +710,8 @@ else:
             new_unc_prefix = b'\\\\'
             cwd = os.getcwdb()
             # bpo-38081: Special case for realpath(b'nul')
-            if normcase(path) == normcase(os.fsencode(devnull)):
+            devnull = b'nul'
+            if normcase(path) == devnull:
                 return b'\\\\.\\NUL'
         else:
             prefix = '\\\\?\\'
@@ -713,9 +719,19 @@ else:
             new_unc_prefix = '\\\\'
             cwd = os.getcwd()
             # bpo-38081: Special case for realpath('nul')
-            if normcase(path) == normcase(devnull):
+            devnull = 'nul'
+            if normcase(path) == devnull:
                 return '\\\\.\\NUL'
         had_prefix = path.startswith(prefix)
+
+        if strict is ALLOW_MISSING:
+            ignored_error = FileNotFoundError
+            strict = True
+        elif strict:
+            ignored_error = ()
+        else:
+            ignored_error = OSError
+
         if not had_prefix and not isabs(path):
             path = join(cwd, path)
         try:
@@ -723,17 +739,16 @@ else:
             initial_winerror = 0
         except ValueError as ex:
             # gh-106242: Raised for embedded null characters
-            # In strict mode, we convert into an OSError.
+            # In strict modes, we convert into an OSError.
             # Non-strict mode returns the path as-is, since we've already
             # made it absolute.
             if strict:
                 raise OSError(str(ex)) from None
             path = normpath(path)
-        except OSError as ex:
-            if strict:
-                raise
+        except ignored_error as ex:
             initial_winerror = ex.winerror
-            path = _getfinalpathname_nonstrict(path)
+            path = _getfinalpathname_nonstrict(path,
+                                               ignored_error=ignored_error)
         # The path returned by _getfinalpathname will always start with \\?\ -
         # strip off that prefix unless it was already provided on the original
         # path.
@@ -766,6 +781,9 @@ supports_unicode_filenames = True
 def relpath(path, start=None):
     """Return a relative version of a path"""
     path = os.fspath(path)
+    if not path:
+        raise ValueError("no path specified")
+
     if isinstance(path, bytes):
         sep = b'\\'
         curdir = b'.'
@@ -777,22 +795,20 @@ def relpath(path, start=None):
 
     if start is None:
         start = curdir
+    else:
+        start = os.fspath(start)
 
-    if not path:
-        raise ValueError("no path specified")
-
-    start = os.fspath(start)
     try:
-        start_abs = abspath(normpath(start))
-        path_abs = abspath(normpath(path))
+        start_abs = abspath(start)
+        path_abs = abspath(path)
         start_drive, _, start_rest = splitroot(start_abs)
         path_drive, _, path_rest = splitroot(path_abs)
         if normcase(start_drive) != normcase(path_drive):
             raise ValueError("path is on mount %r, start on mount %r" % (
                 path_drive, start_drive))
 
-        start_list = [x for x in start_rest.split(sep) if x]
-        path_list = [x for x in path_rest.split(sep) if x]
+        start_list = start_rest.split(sep) if start_rest else []
+        path_list = path_rest.split(sep) if path_rest else []
         # Work out how much of the filepath is shared by start and path.
         i = 0
         for e1, e2 in zip(start_list, path_list):
@@ -803,29 +819,28 @@ def relpath(path, start=None):
         rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
         if not rel_list:
             return curdir
-        return join(*rel_list)
+        return sep.join(rel_list)
     except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning):
         genericpath._check_arg_types('relpath', path, start)
         raise
 
 
-# Return the longest common sub-path of the sequence of paths given as input.
+# Return the longest common sub-path of the iterable of paths given as input.
 # The function is case-insensitive and 'separator-insensitive', i.e. if the
 # only difference between two paths is the use of '\' versus '/' as separator,
 # they are deemed to be equal.
 #
 # However, the returned path will have the standard '\' separator (even if the
 # given paths had the alternative '/' separator) and will have the case of the
-# first path given in the sequence. Additionally, any trailing separator is
+# first path given in the iterable. Additionally, any trailing separator is
 # stripped from the returned path.
 
 def commonpath(paths):
-    """Given a sequence of path names, returns the longest common sub-path."""
-
-    if not paths:
-        raise ValueError('commonpath() arg is an empty sequence')
-
+    """Given an iterable of path names, returns the longest common sub-path."""
     paths = tuple(map(os.fspath, paths))
+    if not paths:
+        raise ValueError('commonpath() arg is an empty iterable')
+
     if isinstance(paths[0], bytes):
         sep = b'\\'
         altsep = b'/'
@@ -839,9 +854,6 @@ def commonpath(paths):
         drivesplits = [splitroot(p.replace(altsep, sep).lower()) for p in paths]
         split_paths = [p.split(sep) for d, r, p in drivesplits]
 
-        if len({r for d, r, p in drivesplits}) != 1:
-            raise ValueError("Can't mix absolute and relative paths")
-
         # Check that all drive letters or UNC paths match. The check is made only
         # now otherwise type errors for mixing strings and bytes would not be
         # caught.
@@ -849,6 +861,12 @@ def commonpath(paths):
             raise ValueError("Paths don't have the same drive")
 
         drive, root, path = splitroot(paths[0].replace(altsep, sep))
+        if len({r for d, r, p in drivesplits}) != 1:
+            if drive:
+                raise ValueError("Can't mix absolute and relative paths")
+            else:
+                raise ValueError("Can't mix rooted and not-rooted paths")
+
         common = path.split(sep)
         common = [c for c in common if c and c != curdir]
 
@@ -869,13 +887,15 @@ def commonpath(paths):
 
 
 try:
-    # The isdir(), isfile(), islink() and exists() implementations in
-    # genericpath use os.stat(). This is overkill on Windows. Use simpler
+    # The isdir(), isfile(), islink(), exists() and lexists() implementations
+    # in genericpath use os.stat(). This is overkill on Windows. Use simpler
     # builtin functions if they are available.
     from nt import _path_isdir as isdir
     from nt import _path_isfile as isfile
     from nt import _path_islink as islink
+    from nt import _path_isjunction as isjunction
     from nt import _path_exists as exists
+    from nt import _path_lexists as lexists
 except ImportError:
     # Use genericpath.* as imported above
     pass
@@ -883,15 +903,12 @@ except ImportError:
 
 try:
     from nt import _path_isdevdrive
-except ImportError:
-    def isdevdrive(path):
-        """Determines whether the specified path is on a Windows Dev Drive."""
-        # Never a Dev Drive
-        return False
-else:
     def isdevdrive(path):
         """Determines whether the specified path is on a Windows Dev Drive."""
         try:
             return _path_isdevdrive(abspath(path))
         except OSError:
             return False
+except ImportError:
+    # Use genericpath.isdevdrive as imported above
+    pass

Lib/operator.py

@@ -239,7 +239,7 @@ class attrgetter:
     """
     __slots__ = ('_attrs', '_call')
 
-    def __init__(self, attr, *attrs):
+    def __init__(self, attr, /, *attrs):
         if not attrs:
             if not isinstance(attr, str):
                 raise TypeError('attribute name must be a string')
@@ -257,7 +257,7 @@ class attrgetter:
                 return tuple(getter(obj) for getter in getters)
             self._call = func
 
-    def __call__(self, obj):
+    def __call__(self, obj, /):
         return self._call(obj)
 
     def __repr__(self):
@@ -276,7 +276,7 @@ class itemgetter:
     """
     __slots__ = ('_items', '_call')
 
-    def __init__(self, item, *items):
+    def __init__(self, item, /, *items):
         if not items:
             self._items = (item,)
             def func(obj):
@@ -288,7 +288,7 @@ class itemgetter:
                 return tuple(obj[i] for i in items)
             self._call = func
 
-    def __call__(self, obj):
+    def __call__(self, obj, /):
         return self._call(obj)
 
     def __repr__(self):
@@ -315,7 +315,7 @@ class methodcaller:
         self._args = args
         self._kwargs = kwargs
 
-    def __call__(self, obj):
+    def __call__(self, obj, /):
         return getattr(obj, self._name)(*self._args, **self._kwargs)
 
     def __repr__(self):

Lib/os.py

@@ -110,6 +110,7 @@ if _exists("_have_functions"):
     _add("HAVE_FCHMODAT",   "chmod")
     _add("HAVE_FCHOWNAT",   "chown")
     _add("HAVE_FSTATAT",    "stat")
+    _add("HAVE_LSTAT",      "lstat")
     _add("HAVE_FUTIMESAT",  "utime")
     _add("HAVE_LINKAT",     "link")
     _add("HAVE_MKDIRAT",    "mkdir")
@@ -131,6 +132,7 @@ if _exists("_have_functions"):
     _set = set()
     _add("HAVE_FCHDIR",     "chdir")
     _add("HAVE_FCHMOD",     "chmod")
+    _add("MS_WINDOWS",      "chmod")
     _add("HAVE_FCHOWN",     "chown")
     _add("HAVE_FDOPENDIR",  "listdir")
     _add("HAVE_FDOPENDIR",  "scandir")
@@ -171,6 +173,7 @@ if _exists("_have_functions"):
     _add("HAVE_FSTATAT",    "stat")
     _add("HAVE_LCHFLAGS",   "chflags")
     _add("HAVE_LCHMOD",     "chmod")
+    _add("MS_WINDOWS",      "chmod")
     if _exists("lchown"): # mac os x10.3
         _add("HAVE_LCHOWN", "chown")
     _add("HAVE_LINKAT",     "link")
@@ -279,6 +282,10 @@ def renames(old, new):
 
 __all__.extend(["makedirs", "removedirs", "renames"])
 
+# Private sentinel that makes walk() classify all symlinks and junctions as
+# regular files.
+_walk_symlinks_as_files = object()
+
 def walk(top, topdown=True, onerror=None, followlinks=False):
     """Directory tree generator.
 
@@ -331,12 +338,12 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
 
     import os
     from os.path import join, getsize
-    for root, dirs, files in os.walk('python/Lib/email'):
+    for root, dirs, files in os.walk('python/Lib/xml'):
         print(root, "consumes ")
         print(sum(getsize(join(root, name)) for name in files), end=" ")
         print("bytes in", len(files), "non-directory files")
-        if 'CVS' in dirs:
-            dirs.remove('CVS')  # don't visit CVS directories
+        if '__pycache__' in dirs:
+            dirs.remove('__pycache__')  # don't visit __pycache__ directories
 
     """
     sys.audit("os.walk", top, topdown, onerror, followlinks)
@@ -380,7 +387,10 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
                     break
 
                 try:
-                    is_dir = entry.is_dir()
+                    if followlinks is _walk_symlinks_as_files:
+                        is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction()
+                    else:
+                        is_dir = entry.is_dir()
                 except OSError:
                     # If is_dir() raises an OSError, consider the entry not to
                     # be a directory, same behaviour as os.path.isdir().
@@ -459,34 +469,69 @@ if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd:
         Example:
 
         import os
-        for root, dirs, files, rootfd in os.fwalk('python/Lib/email'):
+        for root, dirs, files, rootfd in os.fwalk('python/Lib/xml'):
             print(root, "consumes", end="")
             print(sum(os.stat(name, dir_fd=rootfd).st_size for name in files),
                   end="")
             print("bytes in", len(files), "non-directory files")
-            if 'CVS' in dirs:
-                dirs.remove('CVS')  # don't visit CVS directories
+            if '__pycache__' in dirs:
+                dirs.remove('__pycache__')  # don't visit __pycache__ directories
         """
         sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd)
         top = fspath(top)
-        # Note: To guard against symlink races, we use the standard
-        # lstat()/open()/fstat() trick.
-        if not follow_symlinks:
-            orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd)
-        topfd = open(top, O_RDONLY | O_NONBLOCK, dir_fd=dir_fd)
+        stack = [(_fwalk_walk, (True, dir_fd, top, top, None))]
+        isbytes = isinstance(top, bytes)
         try:
-            if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and
-                                    path.samestat(orig_st, stat(topfd)))):
-                yield from _fwalk(topfd, top, isinstance(top, bytes),
-                                  topdown, onerror, follow_symlinks)
+            while stack:
+                yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks)
         finally:
-            close(topfd)
+            # Close any file descriptors still on the stack.
+            while stack:
+                action, value = stack.pop()
+                if action == _fwalk_close:
+                    close(value)
 
-    def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks):
+    # Each item in the _fwalk() stack is a pair (action, args).
+    _fwalk_walk = 0  # args: (isroot, dirfd, toppath, topname, entry)
+    _fwalk_yield = 1  # args: (toppath, dirnames, filenames, topfd)
+    _fwalk_close = 2  # args: dirfd
+
+    def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks):
         # Note: This uses O(depth of the directory tree) file descriptors: if
         # necessary, it can be adapted to only require O(1) FDs, see issue
         # #13734.
 
+        action, value = stack.pop()
+        if action == _fwalk_close:
+            close(value)
+            return
+        elif action == _fwalk_yield:
+            yield value
+            return
+        assert action == _fwalk_walk
+        isroot, dirfd, toppath, topname, entry = value
+        try:
+            if not follow_symlinks:
+                # Note: To guard against symlink races, we use the standard
+                # lstat()/open()/fstat() trick.
+                if entry is None:
+                    orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd)
+                else:
+                    orig_st = entry.stat(follow_symlinks=False)
+            topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd)
+        except OSError as err:
+            if isroot:
+                raise
+            if onerror is not None:
+                onerror(err)
+            return
+        stack.append((_fwalk_close, topfd))
+        if not follow_symlinks:
+            if isroot and not st.S_ISDIR(orig_st.st_mode):
+                return
+            if not path.samestat(orig_st, stat(topfd)):
+                return
+
         scandir_it = scandir(topfd)
         dirs = []
         nondirs = []
@@ -512,31 +557,18 @@ if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd:
 
         if topdown:
             yield toppath, dirs, nondirs, topfd
+        else:
+            stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd)))
 
-        for name in dirs if entries is None else zip(dirs, entries):
-            try:
-                if not follow_symlinks:
-                    if topdown:
-                        orig_st = stat(name, dir_fd=topfd, follow_symlinks=False)
-                    else:
-                        assert entries is not None
-                        name, entry = name
-                        orig_st = entry.stat(follow_symlinks=False)
-                dirfd = open(name, O_RDONLY | O_NONBLOCK, dir_fd=topfd)
-            except OSError as err:
-                if onerror is not None:
-                    onerror(err)
-                continue
-            try:
-                if follow_symlinks or path.samestat(orig_st, stat(dirfd)):
-                    dirpath = path.join(toppath, name)
-                    yield from _fwalk(dirfd, dirpath, isbytes,
-                                      topdown, onerror, follow_symlinks)
-            finally:
-                close(dirfd)
-
-        if not topdown:
-            yield toppath, dirs, nondirs, topfd
+        toppath = path.join(toppath, toppath[:0])  # Add trailing slash.
+        if entries is None:
+            stack.extend(
+                (_fwalk_walk, (False, topfd, toppath + name, name, None))
+                for name in dirs[::-1])
+        else:
+            stack.extend(
+                (_fwalk_walk, (False, topfd, toppath + name, name, entry))
+                for name, entry in zip(dirs[::-1], entries[::-1]))
 
     __all__.append("fwalk")
 
@@ -1061,6 +1093,12 @@ def _fspath(path):
         else:
             raise TypeError("expected str, bytes or os.PathLike object, "
                             "not " + path_type.__name__)
+    except TypeError:
+        if path_type.__fspath__ is None:
+            raise TypeError("expected str, bytes or os.PathLike object, "
+                            "not " + path_type.__name__) from None
+        else:
+            raise
     if isinstance(path_repr, (str, bytes)):
         return path_repr
     else:
@@ -1079,6 +1117,8 @@ class PathLike(abc.ABC):
 
     """Abstract base class for implementing the file system path protocol."""
 
+    __slots__ = ()
+
     @abc.abstractmethod
     def __fspath__(self):
         """Return the file system path representation of the object."""
@@ -1128,3 +1168,17 @@ if name == 'nt':
             cookie,
             nt._remove_dll_directory
         )
+
+
+if _exists('sched_getaffinity') and sys._get_cpu_count_config() < 0:
+    def process_cpu_count():
+        """
+        Get the number of CPUs of the current process.
+
+        Return the number of logical CPUs usable by the calling thread of the
+        current process. Return None if indeterminable.
+        """
+        return len(sched_getaffinity(0))
+else:
+    # Just an alias to cpu_count() (same docstring)
+    process_cpu_count = cpu_count

Lib/pickle.py

@@ -314,16 +314,17 @@ class _Unframer:
 # Tools used for pickling.
 
 def _getattribute(obj, name):
+    top = obj
     for subpath in name.split('.'):
         if subpath == '<locals>':
             raise AttributeError("Can't get local attribute {!r} on {!r}"
-                                 .format(name, obj))
+                                 .format(name, top))
         try:
             parent = obj
             obj = getattr(obj, subpath)
         except AttributeError:
             raise AttributeError("Can't get attribute {!r} on {!r}"
-                                 .format(name, obj)) from None
+                                 .format(name, top)) from None
     return obj, parent
 
 def whichmodule(obj, name):
@@ -396,6 +397,8 @@ def decode_long(data):
     return int.from_bytes(data, byteorder='little', signed=True)
 
 
+_NoValue = object()
+
 # Pickling machinery
 
 class _Pickler:
@@ -530,10 +533,11 @@ class _Pickler:
         self.framer.commit_frame()
 
         # Check for persistent id (defined by a subclass)
-        pid = self.persistent_id(obj)
-        if pid is not None and save_persistent_id:
-            self.save_pers(pid)
-            return
+        if save_persistent_id:
+            pid = self.persistent_id(obj)
+            if pid is not None:
+                self.save_pers(pid)
+                return
 
         # Check the memo
         x = self.memo.get(id(obj))
@@ -542,8 +546,8 @@ class _Pickler:
             return
 
         rv = NotImplemented
-        reduce = getattr(self, "reducer_override", None)
-        if reduce is not None:
+        reduce = getattr(self, "reducer_override", _NoValue)
+        if reduce is not _NoValue:
             rv = reduce(obj)
 
         if rv is NotImplemented:
@@ -556,8 +560,8 @@ class _Pickler:
 
             # Check private dispatch table if any, or else
             # copyreg.dispatch_table
-            reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
-            if reduce is not None:
+            reduce = getattr(self, 'dispatch_table', dispatch_table).get(t, _NoValue)
+            if reduce is not _NoValue:
                 rv = reduce(obj)
             else:
                 # Check for a class with a custom metaclass; treat as regular
@@ -567,12 +571,12 @@ class _Pickler:
                     return
 
                 # Check for a __reduce_ex__ method, fall back to __reduce__
-                reduce = getattr(obj, "__reduce_ex__", None)
-                if reduce is not None:
+                reduce = getattr(obj, "__reduce_ex__", _NoValue)
+                if reduce is not _NoValue:
                     rv = reduce(self.proto)
                 else:
-                    reduce = getattr(obj, "__reduce__", None)
-                    if reduce is not None:
+                    reduce = getattr(obj, "__reduce__", _NoValue)
+                    if reduce is not _NoValue:
                         rv = reduce()
                     else:
                         raise PicklingError("Can't pickle %r object: %r" %
@@ -780,14 +784,10 @@ class _Pickler:
             self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
     dispatch[float] = save_float
 
-    def save_bytes(self, obj):
-        if self.proto < 3:
-            if not obj: # bytes object is empty
-                self.save_reduce(bytes, (), obj=obj)
-            else:
-                self.save_reduce(codecs.encode,
-                                 (str(obj, 'latin1'), 'latin1'), obj=obj)
-            return
+    def _save_bytes_no_memo(self, obj):
+        # helper for writing bytes objects for protocol >= 3
+        # without memoizing them
+        assert self.proto >= 3
         n = len(obj)
         if n <= 0xff:
             self.write(SHORT_BINBYTES + pack("<B", n) + obj)
@@ -797,9 +797,29 @@ class _Pickler:
             self._write_large_bytes(BINBYTES + pack("<I", n), obj)
         else:
             self.write(BINBYTES + pack("<I", n) + obj)
+
+    def save_bytes(self, obj):
+        if self.proto < 3:
+            if not obj: # bytes object is empty
+                self.save_reduce(bytes, (), obj=obj)
+            else:
+                self.save_reduce(codecs.encode,
+                                 (str(obj, 'latin1'), 'latin1'), obj=obj)
+            return
+        self._save_bytes_no_memo(obj)
         self.memoize(obj)
     dispatch[bytes] = save_bytes
 
+    def _save_bytearray_no_memo(self, obj):
+        # helper for writing bytearray objects for protocol >= 5
+        # without memoizing them
+        assert self.proto >= 5
+        n = len(obj)
+        if n >= self.framer._FRAME_SIZE_TARGET:
+            self._write_large_bytes(BYTEARRAY8 + pack("<Q", n), obj)
+        else:
+            self.write(BYTEARRAY8 + pack("<Q", n) + obj)
+
     def save_bytearray(self, obj):
         if self.proto < 5:
             if not obj:  # bytearray is empty
@@ -807,18 +827,14 @@ class _Pickler:
             else:
                 self.save_reduce(bytearray, (bytes(obj),), obj=obj)
             return
-        n = len(obj)
-        if n >= self.framer._FRAME_SIZE_TARGET:
-            self._write_large_bytes(BYTEARRAY8 + pack("<Q", n), obj)
-        else:
-            self.write(BYTEARRAY8 + pack("<Q", n) + obj)
+        self._save_bytearray_no_memo(obj)
         self.memoize(obj)
     dispatch[bytearray] = save_bytearray
 
     if _HAVE_PICKLE_BUFFER:
         def save_picklebuffer(self, obj):
             if self.proto < 5:
-                raise PicklingError("PickleBuffer can only pickled with "
+                raise PicklingError("PickleBuffer can only be pickled with "
                                     "protocol >= 5")
             with obj.raw() as m:
                 if not m.contiguous:
@@ -830,10 +846,18 @@ class _Pickler:
                 if in_band:
                     # Write data in-band
                     # XXX The C implementation avoids a copy here
+                    buf = m.tobytes()
+                    in_memo = id(buf) in self.memo
                     if m.readonly:
-                        self.save_bytes(m.tobytes())
+                        if in_memo:
+                            self._save_bytes_no_memo(buf)
+                        else:
+                            self.save_bytes(buf)
                     else:
-                        self.save_bytearray(m.tobytes())
+                        if in_memo:
+                            self._save_bytearray_no_memo(buf)
+                        else:
+                            self.save_bytearray(buf)
                 else:
                     # Write data out-of-band
                     self.write(NEXT_BUFFER)
@@ -1070,11 +1094,16 @@ class _Pickler:
                     (obj, module_name, name))
 
         if self.proto >= 2:
-            code = _extension_registry.get((module_name, name))
-            if code:
-                assert code > 0
+            code = _extension_registry.get((module_name, name), _NoValue)
+            if code is not _NoValue:
                 if code <= 0xff:
-                    write(EXT1 + pack("<B", code))
+                    data = pack("<B", code)
+                    if data == b'\0':
+                        # Should never happen in normal circumstances,
+                        # since the type and the value of the code are
+                        # checked in copyreg.add_extension().
+                        raise RuntimeError("extension code 0 is out of range")
+                    write(EXT1 + data)
                 elif code <= 0xffff:
                     write(EXT2 + pack("<H", code))
                 else:
@@ -1088,11 +1117,35 @@ class _Pickler:
             self.save(module_name)
             self.save(name)
             write(STACK_GLOBAL)
-        elif parent is not module:
-            self.save_reduce(getattr, (parent, lastname))
-        elif self.proto >= 3:
-            write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
-                  bytes(name, "utf-8") + b'\n')
+        elif '.' in name:
+            # In protocol < 4, objects with multi-part __qualname__
+            # are represented as
+            # getattr(getattr(..., attrname1), attrname2).
+            dotted_path = name.split('.')
+            name = dotted_path.pop(0)
+            save = self.save
+            for attrname in dotted_path:
+                save(getattr)
+                if self.proto < 2:
+                    write(MARK)
+            self._save_toplevel_by_name(module_name, name)
+            for attrname in dotted_path:
+                save(attrname)
+                if self.proto < 2:
+                    write(TUPLE)
+                else:
+                    write(TUPLE2)
+                write(REDUCE)
+        else:
+            self._save_toplevel_by_name(module_name, name)
+
+        self.memoize(obj)
+
+    def _save_toplevel_by_name(self, module_name, name):
+        if self.proto >= 3:
+            # Non-ASCII identifiers are supported only with protocols >= 3.
+            self.write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
+                       bytes(name, "utf-8") + b'\n')
         else:
             if self.fix_imports:
                 r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
@@ -1102,14 +1155,12 @@ class _Pickler:
                 elif module_name in r_import_mapping:
                     module_name = r_import_mapping[module_name]
             try:
-                write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
-                      bytes(name, "ascii") + b'\n')
+                self.write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
+                           bytes(name, "ascii") + b'\n')
             except UnicodeEncodeError:
                 raise PicklingError(
                     "can't pickle global identifier '%s.%s' using "
-                    "pickle protocol %i" % (module, name, self.proto)) from None
-
-        self.memoize(obj)
+                    "pickle protocol %i" % (module_name, name, self.proto)) from None
 
     def save_type(self, obj):
         if obj is type(None):
@@ -1546,9 +1597,8 @@ class _Unpickler:
     dispatch[EXT4[0]] = load_ext4
 
     def get_extension(self, code):
-        nil = []
-        obj = _extension_cache.get(code, nil)
-        if obj is not nil:
+        obj = _extension_cache.get(code, _NoValue)
+        if obj is not _NoValue:
             self.append(obj)
             return
         key = _inverted_registry.get(code)
@@ -1705,8 +1755,8 @@ class _Unpickler:
         stack = self.stack
         state = stack.pop()
         inst = stack[-1]
-        setstate = getattr(inst, "__setstate__", None)
-        if setstate is not None:
+        setstate = getattr(inst, "__setstate__", _NoValue)
+        if setstate is not _NoValue:
             setstate(state)
             return
         slotstate = None

Lib/pickletools.py

@@ -312,7 +312,7 @@ uint8 = ArgumentDescriptor(
             doc="Eight-byte unsigned integer, little-endian.")
 
 
-def read_stringnl(f, decode=True, stripquotes=True):
+def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'):
     r"""
     >>> import io
     >>> read_stringnl(io.BytesIO(b"'abcd'\nefg\n"))
@@ -356,7 +356,7 @@ def read_stringnl(f, decode=True, stripquotes=True):
             raise ValueError("no string quotes around %r" % data)
 
     if decode:
-        data = codecs.escape_decode(data)[0].decode("ascii")
+        data = codecs.escape_decode(data)[0].decode(encoding)
     return data
 
 stringnl = ArgumentDescriptor(
@@ -370,7 +370,7 @@ stringnl = ArgumentDescriptor(
                    """)
 
 def read_stringnl_noescape(f):
-    return read_stringnl(f, stripquotes=False)
+    return read_stringnl(f, stripquotes=False, encoding='utf-8')
 
 stringnl_noescape = ArgumentDescriptor(
                         name='stringnl_noescape',
@@ -2513,7 +2513,10 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0):
             # make a mild effort to align arguments
             line += ' ' * (10 - len(opcode.name))
             if arg is not None:
-                line += ' ' + repr(arg)
+                if opcode.name in ("STRING", "BINSTRING", "SHORT_BINSTRING"):
+                    line += ' ' + ascii(arg)
+                else:
+                    line += ' ' + repr(arg)
             if markmsg:
                 line += ' ' + markmsg
         if annotate:

Lib/posixpath.py

@@ -22,6 +22,7 @@ defpath = '/bin:/usr/bin'
 altsep = None
 devnull = '/dev/null'
 
+import errno
 import os
 import sys
 import stat
@@ -35,7 +36,7 @@ __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext"
            "samefile","sameopenfile","samestat",
            "curdir","pardir","sep","pathsep","defpath","altsep","extsep",
            "devnull","realpath","supports_unicode_filenames","relpath",
-           "commonpath", "isjunction"]
+           "commonpath", "isjunction","isdevdrive","ALLOW_MISSING"]
 
 
 def _get_sep(path):
@@ -77,12 +78,11 @@ def join(a, *p):
     sep = _get_sep(a)
     path = a
     try:
-        if not p:
-            path[:0] + sep  #23780: Ensure compatible data type even if p is null.
-        for b in map(os.fspath, p):
-            if b.startswith(sep):
+        for b in p:
+            b = os.fspath(b)
+            if b.startswith(sep) or not path:
                 path = b
-            elif not path or path.endswith(sep):
+            elif path.endswith(sep):
                 path += b
             else:
                 path += sep + b
@@ -135,33 +135,30 @@ def splitdrive(p):
     return p[:0], p
 
 
-def splitroot(p):
-    """Split a pathname into drive, root and tail. On Posix, drive is always
-    empty; the root may be empty, a single slash, or two slashes. The tail
-    contains anything after the root. For example:
+try:
+    from posix import _path_splitroot_ex as splitroot
+except ImportError:
+    def splitroot(p):
+        """Split a pathname into drive, root and tail.
 
-        splitroot('foo/bar') == ('', '', 'foo/bar')
-        splitroot('/foo/bar') == ('', '/', 'foo/bar')
-        splitroot('//foo/bar') == ('', '//', 'foo/bar')
-        splitroot('///foo/bar') == ('', '/', '//foo/bar')
-    """
-    p = os.fspath(p)
-    if isinstance(p, bytes):
-        sep = b'/'
-        empty = b''
-    else:
-        sep = '/'
-        empty = ''
-    if p[:1] != sep:
-        # Relative path, e.g.: 'foo'
-        return empty, empty, p
-    elif p[1:2] != sep or p[2:3] == sep:
-        # Absolute path, e.g.: '/foo', '///foo', '////foo', etc.
-        return empty, sep, p[1:]
-    else:
-        # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see
-        # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13
-        return empty, p[:2], p[2:]
+        The tail contains anything after the root."""
+        p = os.fspath(p)
+        if isinstance(p, bytes):
+            sep = b'/'
+            empty = b''
+        else:
+            sep = '/'
+            empty = ''
+        if p[:1] != sep:
+            # Relative path, e.g.: 'foo'
+            return empty, empty, p
+        elif p[1:2] != sep or p[2:3] == sep:
+            # Absolute path, e.g.: '/foo', '///foo', '////foo', etc.
+            return empty, sep, p[1:]
+        else:
+            # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see
+            # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13
+            return empty, p[:2], p[2:]
 
 
 # Return the tail (basename) part of a path, same as split(path)[1].
@@ -187,26 +184,6 @@ def dirname(p):
     return head
 
 
-# Is a path a junction?
-
-def isjunction(path):
-    """Test whether a path is a junction
-    Junctions are not a part of posix semantics"""
-    os.fspath(path)
-    return False
-
-
-# Being true for dangling symbolic links is also useful.
-
-def lexists(path):
-    """Test whether a path exists.  Returns True for broken symbolic links"""
-    try:
-        os.lstat(path)
-    except (OSError, ValueError):
-        return False
-    return True
-
-
 # Is a path a mount point?
 # (Does this work for all UNIXes?  Is it even guaranteed to work by Posix?)
 
@@ -227,21 +204,17 @@ def ismount(path):
         parent = join(path, b'..')
     else:
         parent = join(path, '..')
-    parent = realpath(parent)
     try:
         s2 = os.lstat(parent)
-    except (OSError, ValueError):
-        return False
+    except OSError:
+        parent = realpath(parent)
+        try:
+            s2 = os.lstat(parent)
+        except OSError:
+            return False
 
-    dev1 = s1.st_dev
-    dev2 = s2.st_dev
-    if dev1 != dev2:
-        return True     # path/.. on a different device as path
-    ino1 = s1.st_ino
-    ino2 = s2.st_ino
-    if ino1 == ino2:
-        return True     # path/.. is the same i-node as path
-    return False
+    # path/.. on a different device as path or the same i-node as path
+    return s1.st_dev != s2.st_dev or s1.st_ino == s2.st_ino
 
 
 # Expand paths beginning with '~' or '~user'.
@@ -290,7 +263,7 @@ def expanduser(path):
             return path
         name = path[1:i]
         if isinstance(name, bytes):
-            name = str(name, 'ASCII')
+            name = os.fsdecode(name)
         try:
             pwent = pwd.getpwnam(name)
         except KeyError:
@@ -303,11 +276,8 @@ def expanduser(path):
         return path
     if isinstance(path, bytes):
         userhome = os.fsencode(userhome)
-        root = b'/'
-    else:
-        root = '/'
-    userhome = userhome.rstrip(root)
-    return (userhome + path[i:]) or root
+    userhome = userhome.rstrip(sep)
+    return (userhome + path[i:]) or sep
 
 
 # Expand paths containing shell variable substitutions.
@@ -371,7 +341,7 @@ def expandvars(path):
 # if it contains symbolic links!
 
 try:
-    from posix import _path_normpath
+    from posix import _path_normpath as normpath
 
 except ImportError:
     def normpath(path):
@@ -379,21 +349,19 @@ except ImportError:
         path = os.fspath(path)
         if isinstance(path, bytes):
             sep = b'/'
-            empty = b''
             dot = b'.'
             dotdot = b'..'
         else:
             sep = '/'
-            empty = ''
             dot = '.'
             dotdot = '..'
-        if path == empty:
+        if not path:
             return dot
         _, initial_slashes, path = splitroot(path)
         comps = path.split(sep)
         new_comps = []
         for comp in comps:
-            if comp in (empty, dot):
+            if not comp or comp == dot:
                 continue
             if (comp != dotdot or (not initial_slashes and not new_comps) or
                  (new_comps and new_comps[-1] == dotdot)):
@@ -404,24 +372,16 @@ except ImportError:
         path = initial_slashes + sep.join(comps)
         return path or dot
 
-else:
-    def normpath(path):
-        """Normalize path, eliminating double slashes, etc."""
-        path = os.fspath(path)
-        if isinstance(path, bytes):
-            return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
-        return _path_normpath(path) or "."
-
 
 def abspath(path):
     """Return an absolute path."""
     path = os.fspath(path)
-    if not isabs(path):
-        if isinstance(path, bytes):
-            cwd = os.getcwdb()
-        else:
-            cwd = os.getcwd()
-        path = join(cwd, path)
+    if isinstance(path, bytes):
+        if not path.startswith(b'/'):
+            path = join(os.getcwdb(), path)
+    else:
+        if not path.startswith('/'):
+            path = join(os.getcwd(), path)
     return normpath(path)
 
 
@@ -432,72 +392,109 @@ def realpath(filename, *, strict=False):
     """Return the canonical path of the specified filename, eliminating any
 symbolic links encountered in the path."""
     filename = os.fspath(filename)
-    path, ok = _joinrealpath(filename[:0], filename, strict, {})
-    return abspath(path)
-
-# Join two paths, normalizing and eliminating any symbolic links
-# encountered in the second path.
-def _joinrealpath(path, rest, strict, seen):
-    if isinstance(path, bytes):
+    if isinstance(filename, bytes):
         sep = b'/'
         curdir = b'.'
         pardir = b'..'
+        getcwd = os.getcwdb
     else:
         sep = '/'
         curdir = '.'
         pardir = '..'
+        getcwd = os.getcwd
+    if strict is ALLOW_MISSING:
+        ignored_error = FileNotFoundError
+        strict = True
+    elif strict:
+        ignored_error = ()
+    else:
+        ignored_error = OSError
 
-    if isabs(rest):
-        rest = rest[1:]
-        path = sep
+    maxlinks = None
 
-    while rest:
-        name, _, rest = rest.partition(sep)
+    # The stack of unresolved path parts. When popped, a special value of None
+    # indicates that a symlink target has been resolved, and that the original
+    # symlink path can be retrieved by popping again. The [::-1] slice is a
+    # very fast way of spelling list(reversed(...)).
+    rest = filename.split(sep)[::-1]
+
+    # Number of unprocessed parts in 'rest'. This can differ from len(rest)
+    # later, because 'rest' might contain markers for unresolved symlinks.
+    part_count = len(rest)
+
+    # The resolved path, which is absolute throughout this function.
+    # Note: getcwd() returns a normalized and symlink-free path.
+    path = sep if filename.startswith(sep) else getcwd()
+
+    # Mapping from symlink paths to *fully resolved* symlink targets. If a
+    # symlink is encountered but not yet resolved, the value is None. This is
+    # used both to detect symlink loops and to speed up repeated traversals of
+    # the same links.
+    seen = {}
+
+    while part_count:
+        name = rest.pop()
+        if name is None:
+            # resolved symlink target
+            seen[rest.pop()] = path
+            continue
+        part_count -= 1
         if not name or name == curdir:
             # current dir
             continue
         if name == pardir:
             # parent dir
-            if path:
-                path, name = split(path)
-                if name == pardir:
-                    path = join(path, pardir, pardir)
-            else:
-                path = pardir
+            path = path[:path.rindex(sep)] or sep
             continue
-        newpath = join(path, name)
-        try:
-            st = os.lstat(newpath)
-        except OSError:
-            if strict:
-                raise
-            is_link = False
+        if path == sep:
+            newpath = path + name
         else:
-            is_link = stat.S_ISLNK(st.st_mode)
-        if not is_link:
-            path = newpath
-            continue
-        # Resolve the symbolic link
-        if newpath in seen:
-            # Already seen this path
-            path = seen[newpath]
-            if path is not None:
-                # use cached value
+            newpath = path + sep + name
+        try:
+            st_mode = os.lstat(newpath).st_mode
+            if not stat.S_ISLNK(st_mode):
+                if strict and part_count and not stat.S_ISDIR(st_mode):
+                    raise OSError(errno.ENOTDIR, os.strerror(errno.ENOTDIR),
+                                  newpath)
+                path = newpath
                 continue
-            # The symlink is not resolved, so we must have a symlink loop.
-            if strict:
-                # Raise OSError(errno.ELOOP)
-                os.stat(newpath)
-            else:
-                # Return already resolved part + rest of the path unchanged.
-                return join(newpath, rest), False
-        seen[newpath] = None # not resolved symlink
-        path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen)
-        if not ok:
-            return join(path, rest), False
-        seen[newpath] = path # resolved symlink
+            if newpath in seen:
+                # Already seen this path
+                path = seen[newpath]
+                if path is not None:
+                    # use cached value
+                    continue
+                # The symlink is not resolved, so we must have a symlink loop.
+                if strict:
+                    # Raise OSError(errno.ELOOP)
+                    os.stat(newpath)
+                path = newpath
+                continue
+            target = os.readlink(newpath)
+        except ignored_error:
+            pass
+        else:
+            # Resolve the symbolic link
+            if target.startswith(sep):
+                # Symlink target is absolute; reset resolved path.
+                path = sep
+            if maxlinks is None:
+                # Mark this symlink as seen but not fully resolved.
+                seen[newpath] = None
+                # Push the symlink path onto the stack, and signal its specialness
+                # by also pushing None. When these entries are popped, we'll
+                # record the fully-resolved symlink target in the 'seen' mapping.
+                rest.append(newpath)
+                rest.append(None)
+            # Push the unresolved symlink target parts onto the stack.
+            target_parts = target.split(sep)[::-1]
+            rest.extend(target_parts)
+            part_count += len(target_parts)
+            continue
+        # An error occurred and was ignored.
+        path = newpath
 
-    return path, True
+    return path
 
 
 supports_unicode_filenames = (sys.platform == 'darwin')
@@ -505,10 +502,10 @@ supports_unicode_filenames = (sys.platform == 'darwin')
 def relpath(path, start=None):
     """Return a relative version of a path"""
 
+    path = os.fspath(path)
     if not path:
         raise ValueError("no path specified")
 
-    path = os.fspath(path)
     if isinstance(path, bytes):
         curdir = b'.'
         sep = b'/'
@@ -524,15 +521,17 @@ def relpath(path, start=None):
         start = os.fspath(start)
 
     try:
-        start_list = [x for x in abspath(start).split(sep) if x]
-        path_list = [x for x in abspath(path).split(sep) if x]
+        start_tail = abspath(start).lstrip(sep)
+        path_tail = abspath(path).lstrip(sep)
+        start_list = start_tail.split(sep) if start_tail else []
+        path_list = path_tail.split(sep) if path_tail else []
         # Work out how much of the filepath is shared by start and path.
         i = len(commonprefix([start_list, path_list]))
 
         rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
         if not rel_list:
             return curdir
-        return join(*rel_list)
+        return sep.join(rel_list)
     except (TypeError, AttributeError, BytesWarning, DeprecationWarning):
         genericpath._check_arg_types('relpath', path, start)
         raise
@@ -546,10 +545,11 @@ def relpath(path, start=None):
 def commonpath(paths):
     """Given a sequence of path names, returns the longest common sub-path."""
 
+    paths = tuple(map(os.fspath, paths))
+
     if not paths:
         raise ValueError('commonpath() arg is an empty sequence')
 
-    paths = tuple(map(os.fspath, paths))
     if isinstance(paths[0], bytes):
         sep = b'/'
         curdir = b'.'
@@ -561,7 +561,7 @@ def commonpath(paths):
         split_paths = [path.split(sep) for path in paths]
 
         try:
-            isabs, = set(p[:1] == sep for p in paths)
+            isabs, = {p.startswith(sep) for p in paths}
         except ValueError:
             raise ValueError("Can't mix absolute and relative paths") from None
 

Lib/pyclbr.py

@@ -0,0 +1,314 @@
+"""Parse a Python module and describe its classes and functions.
+
+Parse enough of a Python file to recognize imports and class and
+function definitions, and to find out the superclasses of a class.
+
+The interface consists of a single function:
+    readmodule_ex(module, path=None)
+where module is the name of a Python module, and path is an optional
+list of directories where the module is to be searched.  If present,
+path is prepended to the system search path sys.path.  The return value
+is a dictionary.  The keys of the dictionary are the names of the
+classes and functions defined in the module (including classes that are
+defined via the from XXX import YYY construct).  The values are
+instances of classes Class and Function.  One special key/value pair is
+present for packages: the key '__path__' has a list as its value which
+contains the package search path.
+
+Classes and Functions have a common superclass: _Object.  Every instance
+has the following attributes:
+    module  -- name of the module;
+    name    -- name of the object;
+    file    -- file in which the object is defined;
+    lineno  -- line in the file where the object's definition starts;
+    end_lineno -- line in the file where the object's definition ends;
+    parent  -- parent of this object, if any;
+    children -- nested objects contained in this object.
+The 'children' attribute is a dictionary mapping names to objects.
+
+Instances of Function describe functions with the attributes from _Object,
+plus the following:
+    is_async -- if a function is defined with an 'async' prefix
+
+Instances of Class describe classes with the attributes from _Object,
+plus the following:
+    super   -- list of super classes (Class instances if possible);
+    methods -- mapping of method names to beginning line numbers.
+If the name of a super class is not recognized, the corresponding
+entry in the list of super classes is not a class instance but a
+string giving the name of the super class.  Since import statements
+are recognized and imported modules are scanned as well, this
+shouldn't happen often.
+"""
+
+import ast
+import sys
+import importlib.util
+
+__all__ = ["readmodule", "readmodule_ex", "Class", "Function"]
+
+_modules = {}  # Initialize cache of modules we've seen.
+
+
+class _Object:
+    "Information about Python class or function."
+    def __init__(self, module, name, file, lineno, end_lineno, parent):
+        self.module = module
+        self.name = name
+        self.file = file
+        self.lineno = lineno
+        self.end_lineno = end_lineno
+        self.parent = parent
+        self.children = {}
+        if parent is not None:
+            parent.children[name] = self
+
+
+# Odd Function and Class signatures are for back-compatibility.
+class Function(_Object):
+    "Information about a Python function, including methods."
+    def __init__(self, module, name, file, lineno,
+                 parent=None, is_async=False, *, end_lineno=None):
+        super().__init__(module, name, file, lineno, end_lineno, parent)
+        self.is_async = is_async
+        if isinstance(parent, Class):
+            parent.methods[name] = lineno
+
+
+class Class(_Object):
+    "Information about a Python class."
+    def __init__(self, module, name, super_, file, lineno,
+                 parent=None, *, end_lineno=None):
+        super().__init__(module, name, file, lineno, end_lineno, parent)
+        self.super = super_ or []
+        self.methods = {}
+
+
+# These 2 functions are used in these tests
+# Lib/test/test_pyclbr, Lib/idlelib/idle_test/test_browser.py
+def _nest_function(ob, func_name, lineno, end_lineno, is_async=False):
+    "Return a Function after nesting within ob."
+    return Function(ob.module, func_name, ob.file, lineno,
+                    parent=ob, is_async=is_async, end_lineno=end_lineno)
+
+def _nest_class(ob, class_name, lineno, end_lineno, super=None):
+    "Return a Class after nesting within ob."
+    return Class(ob.module, class_name, super, ob.file, lineno,
+                 parent=ob, end_lineno=end_lineno)
+
+
+def readmodule(module, path=None):
+    """Return Class objects for the top-level classes in module.
+
+    This is the original interface, before Functions were added.
+    """
+
+    res = {}
+    for key, value in _readmodule(module, path or []).items():
+        if isinstance(value, Class):
+            res[key] = value
+    return res
+
+def readmodule_ex(module, path=None):
+    """Return a dictionary with all functions and classes in module.
+
+    Search for module in PATH + sys.path.
+    If possible, include imported superclasses.
+    Do this by reading source, without importing (and executing) it.
+    """
+    return _readmodule(module, path or [])
+
+
+def _readmodule(module, path, inpackage=None):
+    """Do the hard work for readmodule[_ex].
+
+    If inpackage is given, it must be the dotted name of the package in
+    which we are searching for a submodule, and then PATH must be the
+    package search path; otherwise, we are searching for a top-level
+    module, and path is combined with sys.path.
+    """
+    # Compute the full module name (prepending inpackage if set).
+    if inpackage is not None:
+        fullmodule = "%s.%s" % (inpackage, module)
+    else:
+        fullmodule = module
+
+    # Check in the cache.
+    if fullmodule in _modules:
+        return _modules[fullmodule]
+
+    # Initialize the dict for this module's contents.
+    tree = {}
+
+    # Check if it is a built-in module; we don't do much for these.
+    if module in sys.builtin_module_names and inpackage is None:
+        _modules[module] = tree
+        return tree
+
+    # Check for a dotted module name.
+    i = module.rfind('.')
+    if i >= 0:
+        package = module[:i]
+        submodule = module[i+1:]
+        parent = _readmodule(package, path, inpackage)
+        if inpackage is not None:
+            package = "%s.%s" % (inpackage, package)
+        if not '__path__' in parent:
+            raise ImportError('No package named {}'.format(package))
+        return _readmodule(submodule, parent['__path__'], package)
+
+    # Search the path for the module.
+    f = None
+    if inpackage is not None:
+        search_path = path
+    else:
+        search_path = path + sys.path
+    spec = importlib.util._find_spec_from_path(fullmodule, search_path)
+    if spec is None:
+        raise ModuleNotFoundError(f"no module named {fullmodule!r}", name=fullmodule)
+    _modules[fullmodule] = tree
+    # Is module a package?
+    if spec.submodule_search_locations is not None:
+        tree['__path__'] = spec.submodule_search_locations
+    try:
+        source = spec.loader.get_source(fullmodule)
+    except (AttributeError, ImportError):
+        # If module is not Python source, we cannot do anything.
+        return tree
+    else:
+        if source is None:
+            return tree
+
+    fname = spec.loader.get_filename(fullmodule)
+    return _create_tree(fullmodule, path, fname, source, tree, inpackage)
+
+
+class _ModuleBrowser(ast.NodeVisitor):
+    def __init__(self, module, path, file, tree, inpackage):
+        self.path = path
+        self.tree = tree
+        self.file = file
+        self.module = module
+        self.inpackage = inpackage
+        self.stack = []
+
+    def visit_ClassDef(self, node):
+        bases = []
+        for base in node.bases:
+            name = ast.unparse(base)
+            if name in self.tree:
+                # We know this super class.
+                bases.append(self.tree[name])
+            elif len(names := name.split(".")) > 1:
+                # Super class form is module.class:
+                # look in module for class.
+                *_, module, class_ = names
+                if module in _modules:
+                    bases.append(_modules[module].get(class_, name))
+            else:
+                bases.append(name)
+
+        parent = self.stack[-1] if self.stack else None
+        class_ = Class(self.module, node.name, bases, self.file, node.lineno,
+                       parent=parent, end_lineno=node.end_lineno)
+        if parent is None:
+            self.tree[node.name] = class_
+        self.stack.append(class_)
+        self.generic_visit(node)
+        self.stack.pop()
+
+    def visit_FunctionDef(self, node, *, is_async=False):
+        parent = self.stack[-1] if self.stack else None
+        function = Function(self.module, node.name, self.file, node.lineno,
+                            parent, is_async, end_lineno=node.end_lineno)
+        if parent is None:
+            self.tree[node.name] = function
+        self.stack.append(function)
+        self.generic_visit(node)
+        self.stack.pop()
+
+    def visit_AsyncFunctionDef(self, node):
+        self.visit_FunctionDef(node, is_async=True)
+
+    def visit_Import(self, node):
+        if node.col_offset != 0:
+            return
+
+        for module in node.names:
+            try:
+                try:
+                    _readmodule(module.name, self.path, self.inpackage)
+                except ImportError:
+                    _readmodule(module.name, [])
+            except (ImportError, SyntaxError):
+                # If we can't find or parse the imported module,
+                # too bad -- don't die here.
+                continue
+
+    def visit_ImportFrom(self, node):
+        if node.col_offset != 0:
+            return
+        try:
+            module = "." * node.level
+            if node.module:
+                module += node.module
+            module = _readmodule(module, self.path, self.inpackage)
+        except (ImportError, SyntaxError):
+            return
+
+        for name in node.names:
+            if name.name in module:
+                self.tree[name.asname or name.name] = module[name.name]
+            elif name.name == "*":
+                for import_name, import_value in module.items():
+                    if import_name.startswith("_"):
+                        continue
+                    self.tree[import_name] = import_value
+
+
+def _create_tree(fullmodule, path, fname, source, tree, inpackage):
+    mbrowser = _ModuleBrowser(fullmodule, path, fname, tree, inpackage)
+    mbrowser.visit(ast.parse(source))
+    return mbrowser.tree
+
+
+def _main():
+    "Print module output (default this file) for quick visual check."
+    import os
+    try:
+        mod = sys.argv[1]
+    except:
+        mod = __file__
+    if os.path.exists(mod):
+        path = [os.path.dirname(mod)]
+        mod = os.path.basename(mod)
+        if mod.lower().endswith(".py"):
+            mod = mod[:-3]
+    else:
+        path = []
+    tree = readmodule_ex(mod, path)
+    lineno_key = lambda a: getattr(a, 'lineno', 0)
+    objs = sorted(tree.values(), key=lineno_key, reverse=True)
+    indent_level = 2
+    while objs:
+        obj = objs.pop()
+        if isinstance(obj, list):
+            # Value is a __path__ key.
+            continue
+        if not hasattr(obj, 'indent'):
+            obj.indent = 0
+
+        if isinstance(obj, _Object):
+            new_objs = sorted(obj.children.values(),
+                              key=lineno_key, reverse=True)
+            for ob in new_objs:
+                ob.indent = obj.indent + indent_level
+            objs.extend(new_objs)
+        if isinstance(obj, Class):
+            print("{}class {} {} {}"
+                  .format(' ' * obj.indent, obj.name, obj.super, obj.lineno))
+        elif isinstance(obj, Function):
+            print("{}def {} {}".format(' ' * obj.indent, obj.name, obj.lineno))
+
+if __name__ == "__main__":
+    _main()

Lib/quopri.py

@@ -67,10 +67,7 @@ def encode(input, output, quotetabs, header=False):
             output.write(s + lineEnd)
 
     prevline = None
-    while 1:
-        line = input.readline()
-        if not line:
-            break
+    while line := input.readline():
         outline = []
         # Strip off any readline induced trailing newline
         stripped = b''
@@ -126,9 +123,7 @@ def decode(input, output, header=False):
         return
 
     new = b''
-    while 1:
-        line = input.readline()
-        if not line: break
+    while line := input.readline():
         i, n = 0, len(line)
         if n > 0 and line[n-1:n] == b'\n':
             partial = 0; n = n-1

Lib/rlcompleter.py

@@ -31,7 +31,11 @@ Notes:
 
 import atexit
 import builtins
+import inspect
+import keyword
+import re
 import __main__
+import warnings
 
 __all__ = ["Completer"]
 
@@ -85,10 +89,11 @@ class Completer:
                 return None
 
         if state == 0:
-            if "." in text:
-                self.matches = self.attr_matches(text)
-            else:
-                self.matches = self.global_matches(text)
+            with warnings.catch_warnings(action="ignore"):
+                if "." in text:
+                    self.matches = self.attr_matches(text)
+                else:
+                    self.matches = self.global_matches(text)
         try:
             return self.matches[state]
         except IndexError:
@@ -96,7 +101,13 @@ class Completer:
 
     def _callable_postfix(self, val, word):
         if callable(val):
-            word = word + "("
+            word += "("
+            try:
+                if not inspect.signature(val).parameters:
+                    word += ")"
+            except ValueError:
+                pass
+
         return word
 
     def global_matches(self, text):
@@ -106,18 +117,17 @@ class Completer:
         defined in self.namespace that match.
 
         """
-        import keyword
         matches = []
         seen = {"__builtins__"}
         n = len(text)
-        for word in keyword.kwlist:
+        for word in keyword.kwlist + keyword.softkwlist:
             if word[:n] == text:
                 seen.add(word)
                 if word in {'finally', 'try'}:
                     word = word + ':'
                 elif word not in {'False', 'None', 'True',
                                   'break', 'continue', 'pass',
-                                  'else'}:
+                                  'else', '_'}:
                     word = word + ' '
                 matches.append(word)
         for nspace in [self.namespace, builtins.__dict__]:
@@ -139,7 +149,6 @@ class Completer:
         with a __getattr__ hook is evaluated.
 
         """
-        import re
         m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text)
         if not m:
             return []
@@ -169,13 +178,20 @@ class Completer:
                 if (word[:n] == attr and
                     not (noprefix and word[:n+1] == noprefix)):
                     match = "%s.%s" % (expr, word)
-                    try:
-                        val = getattr(thisobject, word)
-                    except Exception:
-                        pass  # Include even if attribute not set
+                    if isinstance(getattr(type(thisobject), word, None),
+                                  property):
+                        # bpo-44752: thisobject.word is a method decorated by
+                        # `@property`. What follows applies a postfix if
+                        # thisobject.word is callable, but know we know that
+                        # this is not callable (because it is a property).
+                        # Also, getattr(thisobject, word) will evaluate the
+                        # property method, which is not desirable.
+                        matches.append(match)
+                        continue
+                    if (value := getattr(thisobject, word, None)) is not None:
+                        matches.append(self._callable_postfix(value, match))
                     else:
-                        match = self._callable_postfix(val, match)
-                    matches.append(match)
+                        matches.append(match)
             if matches or not noprefix:
                 break
             if noprefix == '_':

Lib/secrets.py

@@ -2,7 +2,7 @@
 managing secrets such as account authentication, tokens, and similar.
 
 See PEP 506 for more information.
-https://www.python.org/dev/peps/pep-0506/
+https://peps.python.org/pep-0506/
 
 """
 
@@ -13,7 +13,6 @@ __all__ = ['choice', 'randbelow', 'randbits', 'SystemRandom',
 
 
 import base64
-import binascii
 
 from hmac import compare_digest
 from random import SystemRandom
@@ -56,7 +55,7 @@ def token_hex(nbytes=None):
     'f9bf78b9a18ce6d46a0cd2b0b86df9da'
 
     """
-    return binascii.hexlify(token_bytes(nbytes)).decode('ascii')
+    return token_bytes(nbytes).hex()
 
 def token_urlsafe(nbytes=None):
     """Return a random URL-safe text string, in Base64 encoding.

Lib/selectors.py

@@ -66,12 +66,16 @@ class _SelectorMapping(Mapping):
     def __len__(self):
         return len(self._selector._fd_to_key)
 
+    def get(self, fileobj, default=None):
+        fd = self._selector._fileobj_lookup(fileobj)
+        return self._selector._fd_to_key.get(fd, default)
+
     def __getitem__(self, fileobj):
-        try:
-            fd = self._selector._fileobj_lookup(fileobj)
-            return self._selector._fd_to_key[fd]
-        except KeyError:
-            raise KeyError("{!r} is not registered".format(fileobj)) from None
+        fd = self._selector._fileobj_lookup(fileobj)
+        key = self._selector._fd_to_key.get(fd)
+        if key is None:
+            raise KeyError("{!r} is not registered".format(fileobj))
+        return key
 
     def __iter__(self):
         return iter(self._selector._fd_to_key)
@@ -272,19 +276,6 @@ class _BaseSelectorImpl(BaseSelector):
     def get_map(self):
         return self._map
 
-    def _key_from_fd(self, fd):
-        """Return the key associated to a given file descriptor.
-
-        Parameters:
-        fd -- file descriptor
-
-        Returns:
-        corresponding key, or None if not found
-        """
-        try:
-            return self._fd_to_key[fd]
-        except KeyError:
-            return None
 
 
 class SelectSelector(_BaseSelectorImpl):
@@ -323,17 +314,15 @@ class SelectSelector(_BaseSelectorImpl):
             r, w, _ = self._select(self._readers, self._writers, [], timeout)
         except InterruptedError:
             return ready
-        r = set(r)
-        w = set(w)
-        for fd in r | w:
-            events = 0
-            if fd in r:
-                events |= EVENT_READ
-            if fd in w:
-                events |= EVENT_WRITE
-
-            key = self._key_from_fd(fd)
+        r = frozenset(r)
+        w = frozenset(w)
+        rw = r | w
+        fd_to_key_get = self._fd_to_key.get
+        for fd in rw:
+            key = fd_to_key_get(fd)
             if key:
+                events = ((fd in r and EVENT_READ)
+                          | (fd in w and EVENT_WRITE))
                 ready.append((key, events & key.events))
         return ready
 
@@ -350,11 +339,8 @@ class _PollLikeSelector(_BaseSelectorImpl):
 
     def register(self, fileobj, events, data=None):
         key = super().register(fileobj, events, data)
-        poller_events = 0
-        if events & EVENT_READ:
-            poller_events |= self._EVENT_READ
-        if events & EVENT_WRITE:
-            poller_events |= self._EVENT_WRITE
+        poller_events = ((events & EVENT_READ and self._EVENT_READ)
+                         | (events & EVENT_WRITE and self._EVENT_WRITE) )
         try:
             self._selector.register(key.fd, poller_events)
         except:
@@ -380,11 +366,8 @@ class _PollLikeSelector(_BaseSelectorImpl):
 
         changed = False
         if events != key.events:
-            selector_events = 0
-            if events & EVENT_READ:
-                selector_events |= self._EVENT_READ
-            if events & EVENT_WRITE:
-                selector_events |= self._EVENT_WRITE
+            selector_events = ((events & EVENT_READ and self._EVENT_READ)
+                               | (events & EVENT_WRITE and self._EVENT_WRITE))
             try:
                 self._selector.modify(key.fd, selector_events)
             except:
@@ -415,15 +398,13 @@ class _PollLikeSelector(_BaseSelectorImpl):
             fd_event_list = self._selector.poll(timeout)
         except InterruptedError:
             return ready
-        for fd, event in fd_event_list:
-            events = 0
-            if event & ~self._EVENT_READ:
-                events |= EVENT_WRITE
-            if event & ~self._EVENT_WRITE:
-                events |= EVENT_READ
 
-            key = self._key_from_fd(fd)
+        fd_to_key_get = self._fd_to_key.get
+        for fd, event in fd_event_list:
+            key = fd_to_key_get(fd)
             if key:
+                events = ((event & ~self._EVENT_READ and EVENT_WRITE)
+                           | (event & ~self._EVENT_WRITE and EVENT_READ))
                 ready.append((key, events & key.events))
         return ready
 
@@ -439,6 +420,9 @@ if hasattr(select, 'poll'):
 
 if hasattr(select, 'epoll'):
 
+    _NOT_EPOLLIN = ~select.EPOLLIN
+    _NOT_EPOLLOUT = ~select.EPOLLOUT
+
     class EpollSelector(_PollLikeSelector):
         """Epoll-based selector."""
         _selector_cls = select.epoll
@@ -461,22 +445,20 @@ if hasattr(select, 'epoll'):
             # epoll_wait() expects `maxevents` to be greater than zero;
             # we want to make sure that `select()` can be called when no
             # FD is registered.
-            max_ev = max(len(self._fd_to_key), 1)
+            max_ev = len(self._fd_to_key) or 1
 
             ready = []
             try:
                 fd_event_list = self._selector.poll(timeout, max_ev)
             except InterruptedError:
                 return ready
-            for fd, event in fd_event_list:
-                events = 0
-                if event & ~select.EPOLLIN:
-                    events |= EVENT_WRITE
-                if event & ~select.EPOLLOUT:
-                    events |= EVENT_READ
 
-                key = self._key_from_fd(fd)
+            fd_to_key = self._fd_to_key
+            for fd, event in fd_event_list:
+                key = fd_to_key.get(fd)
                 if key:
+                    events = ((event & _NOT_EPOLLIN and EVENT_WRITE)
+                              | (event & _NOT_EPOLLOUT and EVENT_READ))
                     ready.append((key, events & key.events))
             return ready
 
@@ -566,17 +548,15 @@ if hasattr(select, 'kqueue'):
                 kev_list = self._selector.control(None, max_ev, timeout)
             except InterruptedError:
                 return ready
+
+            fd_to_key_get = self._fd_to_key.get
             for kev in kev_list:
                 fd = kev.ident
                 flag = kev.filter
-                events = 0
-                if flag == select.KQ_FILTER_READ:
-                    events |= EVENT_READ
-                if flag == select.KQ_FILTER_WRITE:
-                    events |= EVENT_WRITE
-
-                key = self._key_from_fd(fd)
+                key = fd_to_key_get(fd)
                 if key:
+                    events = ((flag == select.KQ_FILTER_READ and EVENT_READ)
+                              | (flag == select.KQ_FILTER_WRITE and EVENT_WRITE))
                     ready.append((key, events & key.events))
             return ready
 

Lib/shlex.py

@@ -305,9 +305,7 @@ class shlex:
 def split(s, comments=False, posix=True):
     """Split the string *s* using shell-like syntax."""
     if s is None:
-        import warnings
-        warnings.warn("Passing None for 's' to shlex.split() is deprecated.",
-                      DeprecationWarning, stacklevel=2)
+        raise ValueError("s argument must not be None")
     lex = shlex(s, posix=posix)
     lex.whitespace_split = True
     if not comments:
@@ -335,10 +333,7 @@ def quote(s):
 
 
 def _print_tokens(lexer):
-    while 1:
-        tt = lexer.get_token()
-        if not tt:
-            break
+    while tt := lexer.get_token():
         print("Token: " + repr(tt))
 
 if __name__ == '__main__':

Lib/sqlite3/__main__.py

@@ -48,30 +48,18 @@ class SqliteInteractiveConsole(InteractiveConsole):
         Return True if more input is needed; buffering is done automatically.
         Return False is input is a complete statement ready for execution.
         """
-        if source == ".version":
-            print(f"{sqlite3.sqlite_version}")
-        elif source == ".help":
-            print("Enter SQL code and press enter.")
-        elif source == ".quit":
-            sys.exit(0)
-        elif not sqlite3.complete_statement(source):
-            return True
-        else:
-            execute(self._cur, source)
-            return False
-        # TODO: RUSTPYTHON match statement supporting
-        # match source:
-        #     case ".version":
-        #         print(f"{sqlite3.sqlite_version}")
-        #     case ".help":
-        #         print("Enter SQL code and press enter.")
-        #     case ".quit":
-        #         sys.exit(0)
-        #     case _:
-            # if not sqlite3.complete_statement(source):
-            #     return True
-            # execute(self._cur, source)
-        # return False
+        match source:
+            case ".version":
+                print(f"{sqlite3.sqlite_version}")
+            case ".help":
+                print("Enter SQL code and press enter.")
+            case ".quit":
+                sys.exit(0)
+            case _:
+                if not sqlite3.complete_statement(source):
+                    return True
+                execute(self._cur, source)
+        return False
 
 
 def main():

Lib/stat.py

@@ -110,22 +110,30 @@ S_IWOTH = 0o0002  # write by others
 S_IXOTH = 0o0001  # execute by others
 
 # Names for file flags
-
+UF_SETTABLE  = 0x0000ffff  # owner settable flags
 UF_NODUMP    = 0x00000001  # do not dump file
 UF_IMMUTABLE = 0x00000002  # file may not be changed
 UF_APPEND    = 0x00000004  # file may only be appended to
 UF_OPAQUE    = 0x00000008  # directory is opaque when viewed through a union stack
 UF_NOUNLINK  = 0x00000010  # file may not be renamed or deleted
-UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed
-UF_HIDDEN    = 0x00008000  # OS X: file should not be displayed
+UF_COMPRESSED = 0x00000020 # macOS: file is compressed
+UF_TRACKED   = 0x00000040  # macOS: used for handling document IDs
+UF_DATAVAULT = 0x00000080  # macOS: entitlement needed for I/O
+UF_HIDDEN    = 0x00008000  # macOS: file should not be displayed
+SF_SETTABLE  = 0xffff0000  # superuser settable flags
 SF_ARCHIVED  = 0x00010000  # file may be archived
 SF_IMMUTABLE = 0x00020000  # file may not be changed
 SF_APPEND    = 0x00040000  # file may only be appended to
+SF_RESTRICTED = 0x00080000 # macOS: entitlement needed for writing
 SF_NOUNLINK  = 0x00100000  # file may not be renamed or deleted
 SF_SNAPSHOT  = 0x00200000  # file is a snapshot file
+SF_FIRMLINK  = 0x00800000  # macOS: file is a firmlink
+SF_DATALESS  = 0x40000000  # macOS: file is a dataless object
 
 
 _filemode_table = (
+    # File type chars according to:
+    # http://en.wikibooks.org/wiki/C_Programming/POSIX_Reference/sys/stat.h
     ((S_IFLNK,         "l"),
      (S_IFSOCK,        "s"),  # Must appear before IFREG and IFDIR as IFSOCK == IFREG | IFDIR
      (S_IFREG,         "-"),
@@ -156,13 +164,17 @@ _filemode_table = (
 def filemode(mode):
     """Convert a file's mode to a string of the form '-rwxrwxrwx'."""
     perm = []
-    for table in _filemode_table:
+    for index, table in enumerate(_filemode_table):
         for bit, char in table:
             if mode & bit == bit:
                 perm.append(char)
                 break
         else:
-            perm.append("-")
+            if index == 0:
+                # Unknown filetype
+                perm.append("?")
+            else:
+                perm.append("-")
     return "".join(perm)
 
 

Lib/symtable.py

@@ -0,0 +1,414 @@
+"""Interface to the compiler's internal symbol tables"""
+
+import _symtable
+from _symtable import (USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM,
+     DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE,
+     LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL)
+
+import weakref
+from enum import StrEnum
+
+__all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"]
+
+def symtable(code, filename, compile_type):
+    """ Return the toplevel *SymbolTable* for the source code.
+
+    *filename* is the name of the file with the code
+    and *compile_type* is the *compile()* mode argument.
+    """
+    top = _symtable.symtable(code, filename, compile_type)
+    return _newSymbolTable(top, filename)
+
+class SymbolTableFactory:
+    def __init__(self):
+        self.__memo = weakref.WeakValueDictionary()
+
+    def new(self, table, filename):
+        if table.type == _symtable.TYPE_FUNCTION:
+            return Function(table, filename)
+        if table.type == _symtable.TYPE_CLASS:
+            return Class(table, filename)
+        return SymbolTable(table, filename)
+
+    def __call__(self, table, filename):
+        key = table, filename
+        obj = self.__memo.get(key, None)
+        if obj is None:
+            obj = self.__memo[key] = self.new(table, filename)
+        return obj
+
+_newSymbolTable = SymbolTableFactory()
+
+
+class SymbolTableType(StrEnum):
+    MODULE = "module"
+    FUNCTION = "function"
+    CLASS = "class"
+    ANNOTATION = "annotation"
+    TYPE_ALIAS = "type alias"
+    TYPE_PARAMETERS = "type parameters"
+    TYPE_VARIABLE = "type variable"
+
+
+class SymbolTable:
+
+    def __init__(self, raw_table, filename):
+        self._table = raw_table
+        self._filename = filename
+        self._symbols = {}
+
+    def __repr__(self):
+        if self.__class__ == SymbolTable:
+            kind = ""
+        else:
+            kind = "%s " % self.__class__.__name__
+
+        if self._table.name == "top":
+            return "<{0}SymbolTable for module {1}>".format(kind, self._filename)
+        else:
+            return "<{0}SymbolTable for {1} in {2}>".format(kind,
+                                                            self._table.name,
+                                                            self._filename)
+
+    def get_type(self):
+        """Return the type of the symbol table.
+
+        The value returned is one of the values in
+        the ``SymbolTableType`` enumeration.
+        """
+        if self._table.type == _symtable.TYPE_MODULE:
+            return SymbolTableType.MODULE
+        if self._table.type == _symtable.TYPE_FUNCTION:
+            return SymbolTableType.FUNCTION
+        if self._table.type == _symtable.TYPE_CLASS:
+            return SymbolTableType.CLASS
+        if self._table.type == _symtable.TYPE_ANNOTATION:
+            return SymbolTableType.ANNOTATION
+        if self._table.type == _symtable.TYPE_TYPE_ALIAS:
+            return SymbolTableType.TYPE_ALIAS
+        if self._table.type == _symtable.TYPE_TYPE_PARAMETERS:
+            return SymbolTableType.TYPE_PARAMETERS
+        if self._table.type == _symtable.TYPE_TYPE_VARIABLE:
+            return SymbolTableType.TYPE_VARIABLE
+        assert False, f"unexpected type: {self._table.type}"
+
+    def get_id(self):
+        """Return an identifier for the table.
+        """
+        return self._table.id
+
+    def get_name(self):
+        """Return the table's name.
+
+        This corresponds to the name of the class, function
+        or 'top' if the table is for a class, function or
+        global respectively.
+        """
+        return self._table.name
+
+    def get_lineno(self):
+        """Return the number of the first line in the
+        block for the table.
+        """
+        return self._table.lineno
+
+    def is_optimized(self):
+        """Return *True* if the locals in the table
+        are optimizable.
+        """
+        return bool(self._table.type == _symtable.TYPE_FUNCTION)
+
+    def is_nested(self):
+        """Return *True* if the block is a nested class
+        or function."""
+        return bool(self._table.nested)
+
+    def has_children(self):
+        """Return *True* if the block has nested namespaces.
+        """
+        return bool(self._table.children)
+
+    def get_identifiers(self):
+        """Return a view object containing the names of symbols in the table.
+        """
+        return self._table.symbols.keys()
+
+    def lookup(self, name):
+        """Lookup a *name* in the table.
+
+        Returns a *Symbol* instance.
+        """
+        sym = self._symbols.get(name)
+        if sym is None:
+            flags = self._table.symbols[name]
+            namespaces = self.__check_children(name)
+            module_scope = (self._table.name == "top")
+            sym = self._symbols[name] = Symbol(name, flags, namespaces,
+                                               module_scope=module_scope)
+        return sym
+
+    def get_symbols(self):
+        """Return a list of *Symbol* instances for
+        names in the table.
+        """
+        return [self.lookup(ident) for ident in self.get_identifiers()]
+
+    def __check_children(self, name):
+        return [_newSymbolTable(st, self._filename)
+                for st in self._table.children
+                if st.name == name]
+
+    def get_children(self):
+        """Return a list of the nested symbol tables.
+        """
+        return [_newSymbolTable(st, self._filename)
+                for st in self._table.children]
+
+
+class Function(SymbolTable):
+
+    # Default values for instance variables
+    __params = None
+    __locals = None
+    __frees = None
+    __globals = None
+    __nonlocals = None
+
+    def __idents_matching(self, test_func):
+        return tuple(ident for ident in self.get_identifiers()
+                     if test_func(self._table.symbols[ident]))
+
+    def get_parameters(self):
+        """Return a tuple of parameters to the function.
+        """
+        if self.__params is None:
+            self.__params = self.__idents_matching(lambda x:x & DEF_PARAM)
+        return self.__params
+
+    def get_locals(self):
+        """Return a tuple of locals in the function.
+        """
+        if self.__locals is None:
+            locs = (LOCAL, CELL)
+            test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs
+            self.__locals = self.__idents_matching(test)
+        return self.__locals
+
+    def get_globals(self):
+        """Return a tuple of globals in the function.
+        """
+        if self.__globals is None:
+            glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
+            test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob
+            self.__globals = self.__idents_matching(test)
+        return self.__globals
+
+    def get_nonlocals(self):
+        """Return a tuple of nonlocals in the function.
+        """
+        if self.__nonlocals is None:
+            self.__nonlocals = self.__idents_matching(lambda x:x & DEF_NONLOCAL)
+        return self.__nonlocals
+
+    def get_frees(self):
+        """Return a tuple of free variables in the function.
+        """
+        if self.__frees is None:
+            is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE
+            self.__frees = self.__idents_matching(is_free)
+        return self.__frees
+
+
+class Class(SymbolTable):
+
+    __methods = None
+
+    def get_methods(self):
+        """Return a tuple of methods declared in the class.
+        """
+        if self.__methods is None:
+            d = {}
+
+            def is_local_symbol(ident):
+                flags = self._table.symbols.get(ident, 0)
+                return ((flags >> SCOPE_OFF) & SCOPE_MASK) == LOCAL
+
+            for st in self._table.children:
+                # pick the function-like symbols that are local identifiers
+                if is_local_symbol(st.name):
+                    match st.type:
+                        case _symtable.TYPE_FUNCTION:
+                            # generators are of type TYPE_FUNCTION with a ".0"
+                            # parameter as a first parameter (which makes them
+                            # distinguishable from a function named 'genexpr')
+                            if st.name == 'genexpr' and '.0' in st.varnames:
+                                continue
+                            d[st.name] = 1
+                        case _symtable.TYPE_TYPE_PARAMETERS:
+                            # Get the function-def block in the annotation
+                            # scope 'st' with the same identifier, if any.
+                            scope_name = st.name
+                            for c in st.children:
+                                if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION:
+                                    # A generic generator of type TYPE_FUNCTION
+                                    # cannot be a direct child of 'st' (but it
+                                    # can be a descendant), e.g.:
+                                    #
+                                    # class A:
+                                    #   type genexpr[genexpr] = (x for x in [])
+                                    assert scope_name != 'genexpr' or '.0' not in c.varnames
+                                    d[scope_name] = 1
+                                    break
+            self.__methods = tuple(d)
+        return self.__methods
+
+
+class Symbol:
+
+    def __init__(self, name, flags, namespaces=None, *, module_scope=False):
+        self.__name = name
+        self.__flags = flags
+        self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope()
+        self.__namespaces = namespaces or ()
+        self.__module_scope = module_scope
+
+    def __repr__(self):
+        flags_str = '|'.join(self._flags_str())
+        return f'<symbol {self.__name!r}: {self._scope_str()}, {flags_str}>'
+
+    def _scope_str(self):
+        return _scopes_value_to_name.get(self.__scope) or str(self.__scope)
+
+    def _flags_str(self):
+        for flagname, flagvalue in _flags:
+            if self.__flags & flagvalue == flagvalue:
+                yield flagname
+
+    def get_name(self):
+        """Return a name of a symbol.
+        """
+        return self.__name
+
+    def is_referenced(self):
+        """Return *True* if the symbol is used in
+        its block.
+        """
+        return bool(self.__flags & _symtable.USE)
+
+    def is_parameter(self):
+        """Return *True* if the symbol is a parameter.
+        """
+        return bool(self.__flags & DEF_PARAM)
+
+    def is_global(self):
+        """Return *True* if the symbol is global.
+        """
+        return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
+                    or (self.__module_scope and self.__flags & DEF_BOUND))
+
+    def is_nonlocal(self):
+        """Return *True* if the symbol is nonlocal."""
+        return bool(self.__flags & DEF_NONLOCAL)
+
+    def is_declared_global(self):
+        """Return *True* if the symbol is declared global
+        with a global statement."""
+        return bool(self.__scope == GLOBAL_EXPLICIT)
+
+    def is_local(self):
+        """Return *True* if the symbol is local.
+        """
+        return bool(self.__scope in (LOCAL, CELL)
+                    or (self.__module_scope and self.__flags & DEF_BOUND))
+
+    def is_annotated(self):
+        """Return *True* if the symbol is annotated.
+        """
+        return bool(self.__flags & DEF_ANNOT)
+
+    def is_free(self):
+        """Return *True* if a referenced symbol is
+        not assigned to.
+        """
+        return bool(self.__scope == FREE)
+
+    def is_imported(self):
+        """Return *True* if the symbol is created from
+        an import statement.
+        """
+        return bool(self.__flags & DEF_IMPORT)
+
+    def is_assigned(self):
+        """Return *True* if a symbol is assigned to."""
+        return bool(self.__flags & DEF_LOCAL)
+
+    def is_namespace(self):
+        """Returns *True* if name binding introduces new namespace.
+
+        If the name is used as the target of a function or class
+        statement, this will be true.
+
+        Note that a single name can be bound to multiple objects.  If
+        is_namespace() is true, the name may also be bound to other
+        objects, like an int or list, that does not introduce a new
+        namespace.
+        """
+        return bool(self.__namespaces)
+
+    def get_namespaces(self):
+        """Return a list of namespaces bound to this name"""
+        return self.__namespaces
+
+    def get_namespace(self):
+        """Return the single namespace bound to this name.
+
+        Raises ValueError if the name is bound to multiple namespaces
+        or no namespace.
+        """
+        if len(self.__namespaces) == 0:
+            raise ValueError("name is not bound to any namespaces")
+        elif len(self.__namespaces) > 1:
+            raise ValueError("name is bound to multiple namespaces")
+        else:
+            return self.__namespaces[0]
+
+
+_flags = [('USE', USE)]
+_flags.extend(kv for kv in globals().items() if kv[0].startswith('DEF_'))
+_scopes_names = ('FREE', 'LOCAL', 'GLOBAL_IMPLICIT', 'GLOBAL_EXPLICIT', 'CELL')
+_scopes_value_to_name = {globals()[n]: n for n in _scopes_names}
+
+
+def main(args):
+    import sys
+    def print_symbols(table, level=0):
+        indent = '    ' * level
+        nested = "nested " if table.is_nested() else ""
+        if table.get_type() == 'module':
+            what = f'from file {table._filename!r}'
+        else:
+            what = f'{table.get_name()!r}'
+        print(f'{indent}symbol table for {nested}{table.get_type()} {what}:')
+        for ident in table.get_identifiers():
+            symbol = table.lookup(ident)
+            flags = ', '.join(symbol._flags_str()).lower()
+            print(f'    {indent}{symbol._scope_str().lower()} symbol {symbol.get_name()!r}: {flags}')
+        print()
+
+        for table2 in table.get_children():
+            print_symbols(table2, level + 1)
+
+    for filename in args or ['-']:
+        if filename == '-':
+            src = sys.stdin.read()
+            filename = '<stdin>'
+        else:
+            with open(filename, 'rb') as f:
+                src = f.read()
+        mod = symtable(src, filename, 'exec')
+        print_symbols(mod)
+
+
+if __name__ == "__main__":
+    import sys
+    main(sys.argv[1:])

Lib/tabnanny.py

@@ -23,8 +23,6 @@ __version__ = "6"
 import os
 import sys
 import tokenize
-if not hasattr(tokenize, 'NL'):
-    raise ValueError("tokenize.NL doesn't exist -- tokenize module too old")
 
 __all__ = ["check", "NannyNag", "process_tokens"]
 
@@ -37,6 +35,7 @@ def errprint(*args):
         sys.stderr.write(sep + str(arg))
         sep = " "
     sys.stderr.write("\n")
+    sys.exit(1)
 
 def main():
     import getopt
@@ -46,7 +45,6 @@ def main():
         opts, args = getopt.getopt(sys.argv[1:], "qv")
     except getopt.error as msg:
         errprint(msg)
-        return
     for o, a in opts:
         if o == '-q':
             filename_only = filename_only + 1
@@ -54,7 +52,6 @@ def main():
             verbose = verbose + 1
     if not args:
         errprint("Usage:", sys.argv[0], "[-v] file_or_directory ...")
-        return
     for arg in args:
         check(arg)
 
@@ -114,6 +111,10 @@ def check(file):
         errprint("%r: Indentation Error: %s" % (file, msg))
         return
 
+    except SyntaxError as msg:
+        errprint("%r: Syntax Error: %s" % (file, msg))
+        return
+
     except NannyNag as nag:
         badline = nag.get_lineno()
         line = nag.get_line()
@@ -275,6 +276,12 @@ def format_witnesses(w):
     return prefix + " " + ', '.join(firsts)
 
 def process_tokens(tokens):
+    try:
+        _process_tokens(tokens)
+    except TabError as e:
+        raise NannyNag(e.lineno, e.msg, e.text)
+
+def _process_tokens(tokens):
     INDENT = tokenize.INDENT
     DEDENT = tokenize.DEDENT
     NEWLINE = tokenize.NEWLINE

Lib/test/_test_multiprocessing.py

@@ -12,6 +12,7 @@ import itertools
 import sys
 import os
 import gc
+import importlib
 import errno
 import functools
 import signal
@@ -19,10 +20,11 @@ import array
 import socket
 import random
 import logging
+import shutil
 import subprocess
 import struct
+import tempfile
 import operator
-import pathlib
 import pickle
 import weakref
 import warnings
@@ -50,7 +52,7 @@ import multiprocessing.heap
 import multiprocessing.managers
 import multiprocessing.pool
 import multiprocessing.queues
-from multiprocessing.connection import wait, AuthenticationError
+from multiprocessing.connection import wait
 
 from multiprocessing import util
 
@@ -255,6 +257,9 @@ class TimingWrapper(object):
 class BaseTestCase(object):
 
     ALLOWED_TYPES = ('processes', 'manager', 'threads')
+    # If not empty, limit which start method suites run this class.
+    START_METHODS: set[str] = set()
+    start_method = None  # set by install_tests_in_module_dict()
 
     def assertTimingAlmostEqual(self, a, b):
         if CHECK_TIMINGS:
@@ -324,8 +329,9 @@ class _TestProcess(BaseTestCase):
             self.skipTest(f'test not appropriate for {self.TYPE}')
         paths = [
             sys.executable,               # str
-            sys.executable.encode(),      # bytes
-            pathlib.Path(sys.executable)  # os.PathLike
+            os.fsencode(sys.executable),  # bytes
+            os_helper.FakePath(sys.executable),  # os.PathLike
+            os_helper.FakePath(os.fsencode(sys.executable)),  # os.PathLike bytes
         ]
         for path in paths:
             self.set_executable(path)
@@ -505,6 +511,11 @@ class _TestProcess(BaseTestCase):
     def _sleep_some(cls):
         time.sleep(100)
 
+    @classmethod
+    def _sleep_some_event(cls, event):
+        event.set()
+        time.sleep(100)
+
     @classmethod
     def _test_sleep(cls, delay):
         time.sleep(delay)
@@ -513,7 +524,8 @@ class _TestProcess(BaseTestCase):
         if self.TYPE == 'threads':
             self.skipTest('test not appropriate for {}'.format(self.TYPE))
 
-        p = self.Process(target=self._sleep_some)
+        event = self.Event()
+        p = self.Process(target=self._sleep_some_event, args=(event,))
         p.daemon = True
         p.start()
 
@@ -531,8 +543,11 @@ class _TestProcess(BaseTestCase):
         self.assertTimingAlmostEqual(join.elapsed, 0.0)
         self.assertEqual(p.is_alive(), True)
 
-        # XXX maybe terminating too soon causes the problems on Gentoo...
-        time.sleep(1)
+        timeout = support.SHORT_TIMEOUT
+        if not event.wait(timeout):
+            p.terminate()
+            p.join()
+            self.fail(f"event not signaled in {timeout} seconds")
 
         meth(p)
 
@@ -582,12 +597,16 @@ class _TestProcess(BaseTestCase):
     def test_active_children(self):
         self.assertEqual(type(self.active_children()), list)
 
-        p = self.Process(target=time.sleep, args=(DELTA,))
+        event = self.Event()
+        p = self.Process(target=event.wait, args=())
         self.assertNotIn(p, self.active_children())
 
-        p.daemon = True
-        p.start()
-        self.assertIn(p, self.active_children())
+        try:
+            p.daemon = True
+            p.start()
+            self.assertIn(p, self.active_children())
+        finally:
+            event.set()
 
         p.join()
         self.assertNotIn(p, self.active_children())
@@ -1332,6 +1351,23 @@ class _TestQueue(BaseTestCase):
         self.assertTrue(not_serializable_obj.reduce_was_called)
         self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called)
 
+    def test_closed_queue_empty_exceptions(self):
+        # Assert that checking the emptiness of an unused closed queue
+        # does not raise an OSError. The rationale is that q.close() is
+        # a no-op upon construction and becomes effective once the queue
+        # has been used (e.g., by calling q.put()).
+        for q in multiprocessing.Queue(), multiprocessing.JoinableQueue():
+            q.close()  # this is a no-op since the feeder thread is None
+            q.join_thread()  # this is also a no-op
+            self.assertTrue(q.empty())
+
+        for q in multiprocessing.Queue(), multiprocessing.JoinableQueue():
+            q.put('foo')  # make sure that the queue is 'used'
+            q.close()  # close the feeder thread
+            q.join_thread()  # make sure to join the feeder thread
+            with self.assertRaisesRegex(OSError, 'is closed'):
+                q.empty()
+
     def test_closed_queue_put_get_exceptions(self):
         for q in multiprocessing.Queue(), multiprocessing.JoinableQueue():
             q.close()
@@ -1345,6 +1381,66 @@ class _TestQueue(BaseTestCase):
 
 class _TestLock(BaseTestCase):
 
+    @staticmethod
+    def _acquire(lock, l=None):
+        lock.acquire()
+        if l is not None:
+            l.append(repr(lock))
+
+    @staticmethod
+    def _acquire_event(lock, event):
+        lock.acquire()
+        event.set()
+        time.sleep(1.0)
+
+    def test_repr_lock(self):
+        if self.TYPE != 'processes':
+            self.skipTest('test not appropriate for {}'.format(self.TYPE))
+
+        lock = self.Lock()
+        self.assertEqual(f'<Lock(owner=None)>', repr(lock))
+
+        lock.acquire()
+        self.assertEqual(f'<Lock(owner=MainProcess)>', repr(lock))
+        lock.release()
+
+        tname = 'T1'
+        l = []
+        t = threading.Thread(target=self._acquire,
+                             args=(lock, l),
+                             name=tname)
+        t.start()
+        time.sleep(0.1)
+        self.assertEqual(f'<Lock(owner=MainProcess|{tname})>', l[0])
+        lock.release()
+
+        t = threading.Thread(target=self._acquire,
+                             args=(lock,),
+                             name=tname)
+        t.start()
+        time.sleep(0.1)
+        self.assertEqual('<Lock(owner=SomeOtherThread)>', repr(lock))
+        lock.release()
+
+        pname = 'P1'
+        l = multiprocessing.Manager().list()
+        p = self.Process(target=self._acquire,
+                         args=(lock, l),
+                         name=pname)
+        p.start()
+        p.join()
+        self.assertEqual(f'<Lock(owner={pname})>', l[0])
+
+        lock = self.Lock()
+        event = self.Event()
+        p = self.Process(target=self._acquire_event,
+                         args=(lock, event),
+                         name='P2')
+        p.start()
+        event.wait()
+        self.assertEqual(f'<Lock(owner=SomeOtherProcess)>', repr(lock))
+        p.terminate()
+
     def test_lock(self):
         lock = self.Lock()
         self.assertEqual(lock.acquire(), True)
@@ -1352,6 +1448,68 @@ class _TestLock(BaseTestCase):
         self.assertEqual(lock.release(), None)
         self.assertRaises((ValueError, threading.ThreadError), lock.release)
 
+    @staticmethod
+    def _acquire_release(lock, timeout, l=None, n=1):
+        for _ in range(n):
+            lock.acquire()
+        if l is not None:
+            l.append(repr(lock))
+        time.sleep(timeout)
+        for _ in range(n):
+            lock.release()
+
+    def test_repr_rlock(self):
+        if self.TYPE != 'processes':
+            self.skipTest('test not appropriate for {}'.format(self.TYPE))
+
+        lock = self.RLock()
+        self.assertEqual('<RLock(None, 0)>', repr(lock))
+
+        n = 3
+        for _ in range(n):
+            lock.acquire()
+        self.assertEqual(f'<RLock(MainProcess, {n})>', repr(lock))
+        for _ in range(n):
+            lock.release()
+
+        t, l = [], []
+        for i in range(n):
+            t.append(threading.Thread(target=self._acquire_release,
+                                      args=(lock, 0.1, l, i+1),
+                                      name=f'T{i+1}'))
+            t[-1].start()
+        for t_ in t:
+            t_.join()
+        for i in range(n):
+            self.assertIn(f'<RLock(MainProcess|T{i+1}, {i+1})>', l)
+
+
+        t = threading.Thread(target=self._acquire_release,
+                                 args=(lock, 0.2),
+                                 name=f'T1')
+        t.start()
+        time.sleep(0.1)
+        self.assertEqual('<RLock(SomeOtherThread, nonzero)>', repr(lock))
+        time.sleep(0.2)
+
+        pname = 'P1'
+        l = multiprocessing.Manager().list()
+        p = self.Process(target=self._acquire_release,
+                         args=(lock, 0.1, l),
+                         name=pname)
+        p.start()
+        p.join()
+        self.assertEqual(f'<RLock({pname}, 1)>', l[0])
+
+        event = self.Event()
+        lock = self.RLock()
+        p = self.Process(target=self._acquire_event,
+                         args=(lock, event))
+        p.start()
+        event.wait()
+        self.assertEqual('<RLock(SomeOtherProcess, nonzero)>', repr(lock))
+        p.join()
+
     def test_rlock(self):
         lock = self.RLock()
         self.assertEqual(lock.acquire(), True)
@@ -1432,14 +1590,13 @@ class _TestCondition(BaseTestCase):
         cond.release()
 
     def assertReachesEventually(self, func, value):
-        for i in range(10):
+        for _ in support.sleeping_retry(support.SHORT_TIMEOUT):
             try:
                 if func() == value:
                     break
             except NotImplementedError:
                 break
-            time.sleep(DELTA)
-        time.sleep(DELTA)
+
         self.assertReturnsIfImplemented(value, func)
 
     def check_invariant(self, cond):
@@ -1461,20 +1618,17 @@ class _TestCondition(BaseTestCase):
         p = self.Process(target=self.f, args=(cond, sleeping, woken))
         p.daemon = True
         p.start()
-        self.addCleanup(p.join)
 
-        p = threading.Thread(target=self.f, args=(cond, sleeping, woken))
-        p.daemon = True
-        p.start()
-        self.addCleanup(p.join)
+        t = threading.Thread(target=self.f, args=(cond, sleeping, woken))
+        t.daemon = True
+        t.start()
 
         # wait for both children to start sleeping
         sleeping.acquire()
         sleeping.acquire()
 
         # check no process/thread has woken up
-        time.sleep(DELTA)
-        self.assertReturnsIfImplemented(0, get_value, woken)
+        self.assertReachesEventually(lambda: get_value(woken), 0)
 
         # wake up one process/thread
         cond.acquire()
@@ -1482,8 +1636,7 @@ class _TestCondition(BaseTestCase):
         cond.release()
 
         # check one process/thread has woken up
-        time.sleep(DELTA)
-        self.assertReturnsIfImplemented(1, get_value, woken)
+        self.assertReachesEventually(lambda: get_value(woken), 1)
 
         # wake up another
         cond.acquire()
@@ -1491,12 +1644,13 @@ class _TestCondition(BaseTestCase):
         cond.release()
 
         # check other has woken up
-        time.sleep(DELTA)
-        self.assertReturnsIfImplemented(2, get_value, woken)
+        self.assertReachesEventually(lambda: get_value(woken), 2)
 
         # check state is not mucked up
         self.check_invariant(cond)
-        p.join()
+
+        threading_helper.join_thread(t)
+        join_process(p)
 
     def test_notify_all(self):
         cond = self.Condition()
@@ -1504,18 +1658,19 @@ class _TestCondition(BaseTestCase):
         woken = self.Semaphore(0)
 
         # start some threads/processes which will timeout
+        workers = []
         for i in range(3):
             p = self.Process(target=self.f,
                              args=(cond, sleeping, woken, TIMEOUT1))
             p.daemon = True
             p.start()
-            self.addCleanup(p.join)
+            workers.append(p)
 
             t = threading.Thread(target=self.f,
                                  args=(cond, sleeping, woken, TIMEOUT1))
             t.daemon = True
             t.start()
-            self.addCleanup(t.join)
+            workers.append(t)
 
         # wait for them all to sleep
         for i in range(6):
@@ -1534,12 +1689,12 @@ class _TestCondition(BaseTestCase):
             p = self.Process(target=self.f, args=(cond, sleeping, woken))
             p.daemon = True
             p.start()
-            self.addCleanup(p.join)
+            workers.append(p)
 
             t = threading.Thread(target=self.f, args=(cond, sleeping, woken))
             t.daemon = True
             t.start()
-            self.addCleanup(t.join)
+            workers.append(t)
 
         # wait for them to all sleep
         for i in range(6):
@@ -1555,27 +1710,34 @@ class _TestCondition(BaseTestCase):
         cond.release()
 
         # check they have all woken
-        self.assertReachesEventually(lambda: get_value(woken), 6)
+        for i in range(6):
+            woken.acquire()
+        self.assertReturnsIfImplemented(0, get_value, woken)
 
         # check state is not mucked up
         self.check_invariant(cond)
 
+        for w in workers:
+            # NOTE: join_process and join_thread are the same
+            threading_helper.join_thread(w)
+
     def test_notify_n(self):
         cond = self.Condition()
         sleeping = self.Semaphore(0)
         woken = self.Semaphore(0)
 
         # start some threads/processes
+        workers = []
         for i in range(3):
             p = self.Process(target=self.f, args=(cond, sleeping, woken))
             p.daemon = True
             p.start()
-            self.addCleanup(p.join)
+            workers.append(p)
 
             t = threading.Thread(target=self.f, args=(cond, sleeping, woken))
             t.daemon = True
             t.start()
-            self.addCleanup(t.join)
+            workers.append(t)
 
         # wait for them to all sleep
         for i in range(6):
@@ -1610,6 +1772,10 @@ class _TestCondition(BaseTestCase):
         # check state is not mucked up
         self.check_invariant(cond)
 
+        for w in workers:
+            # NOTE: join_process and join_thread are the same
+            threading_helper.join_thread(w)
+
     def test_timeout(self):
         cond = self.Condition()
         wait = TimingWrapper(cond.wait)
@@ -2812,8 +2978,8 @@ class _TestPool(BaseTestCase):
         self.pool.map(identity, objs)
 
         del objs
-        gc.collect()  # For PyPy or other GCs.
         time.sleep(DELTA)  # let threaded cleanup code run
+        support.gc_collect()  # For PyPy or other GCs.
         self.assertEqual(set(wr() for wr in refs), {None})
         # With a process pool, copies of the objects are returned, check
         # they were released too.
@@ -3174,6 +3340,44 @@ class _TestManagerRestart(BaseTestCase):
             if hasattr(manager, "shutdown"):
                 self.addCleanup(manager.shutdown)
 
+
+class FakeConnection:
+    def send(self, payload):
+        pass
+
+    def recv(self):
+        return '#ERROR', pyqueue.Empty()
+
+class TestManagerExceptions(unittest.TestCase):
+    # Issue 106558: Manager exceptions avoids creating cyclic references.
+    def setUp(self):
+        self.mgr = multiprocessing.Manager()
+
+    def tearDown(self):
+        self.mgr.shutdown()
+        self.mgr.join()
+
+    def test_queue_get(self):
+        queue = self.mgr.Queue()
+        if gc.isenabled():
+            gc.disable()
+            self.addCleanup(gc.enable)
+        try:
+            queue.get_nowait()
+        except pyqueue.Empty as e:
+            wr = weakref.ref(e)
+        self.assertEqual(wr(), None)
+
+    def test_dispatch(self):
+        if gc.isenabled():
+            gc.disable()
+            self.addCleanup(gc.enable)
+        try:
+            multiprocessing.managers.dispatch(FakeConnection(), None, None)
+        except pyqueue.Empty as e:
+            wr = weakref.ref(e)
+        self.assertEqual(wr(), None)
+
 #
 #
 #
@@ -4462,6 +4666,59 @@ class _TestSharedMemory(BaseTestCase):
                     "resource_tracker: There appear to be 1 leaked "
                     "shared_memory objects to clean up at shutdown", err)
 
+    @unittest.skipIf(os.name != "posix", "resource_tracker is posix only")
+    def test_shared_memory_untracking(self):
+        # gh-82300: When a separate Python process accesses shared memory
+        # with track=False, it must not cause the memory to be deleted
+        # when terminating.
+        cmd = '''if 1:
+            import sys
+            from multiprocessing.shared_memory import SharedMemory
+            mem = SharedMemory(create=False, name=sys.argv[1], track=False)
+            mem.close()
+        '''
+        mem = shared_memory.SharedMemory(create=True, size=10)
+        # The resource tracker shares pipes with the subprocess, and so
+        # err existing means that the tracker process has terminated now.
+        try:
+            rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name)
+            self.assertNotIn(b"resource_tracker", err)
+            self.assertEqual(rc, 0)
+            mem2 = shared_memory.SharedMemory(create=False, name=mem.name)
+            mem2.close()
+        finally:
+            try:
+                mem.unlink()
+            except OSError:
+                pass
+            mem.close()
+
+    @unittest.skipIf(os.name != "posix", "resource_tracker is posix only")
+    def test_shared_memory_tracking(self):
+        # gh-82300: When a separate Python process accesses shared memory
+        # with track=True, it must cause the memory to be deleted when
+        # terminating.
+        cmd = '''if 1:
+            import sys
+            from multiprocessing.shared_memory import SharedMemory
+            mem = SharedMemory(create=False, name=sys.argv[1], track=True)
+            mem.close()
+        '''
+        mem = shared_memory.SharedMemory(create=True, size=10)
+        try:
+            rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name)
+            self.assertEqual(rc, 0)
+            self.assertIn(
+                b"resource_tracker: There appear to be 1 leaked "
+                b"shared_memory objects to clean up at shutdown", err)
+        finally:
+            try:
+                mem.unlink()
+            except OSError:
+                pass
+            resource_tracker.unregister(mem._name, "shared_memory")
+            mem.close()
+
 #
 # Test to verify that `Finalize` works.
 #
@@ -4571,7 +4828,7 @@ class _TestFinalize(BaseTestCase):
         old_interval = sys.getswitchinterval()
         old_threshold = gc.get_threshold()
         try:
-            sys.setswitchinterval(1e-6)
+            support.setswitchinterval(1e-6)
             gc.set_threshold(5, 5, 5)
             threads = [threading.Thread(target=run_finalizers),
                        threading.Thread(target=make_finalizers)]
@@ -5557,8 +5814,9 @@ class TestResourceTracker(unittest.TestCase):
         '''
         for rtype in resource_tracker._CLEANUP_FUNCS:
             with self.subTest(rtype=rtype):
-                if rtype == "noop":
+                if rtype in ("noop", "dummy"):
                     # Artefact resource type used by the resource_tracker
+                    # or tests
                     continue
                 r, w = os.pipe()
                 p = subprocess.Popen([sys.executable,
@@ -5638,6 +5896,8 @@ class TestResourceTracker(unittest.TestCase):
         # Catchable signal (ignored by semaphore tracker)
         self.check_resource_tracker_death(signal.SIGTERM, False)
 
+    @unittest.skipIf(sys.platform.startswith("netbsd"),
+                     "gh-125620: Skip on NetBSD due to long wait for SIGKILL process termination.")
     def test_resource_tracker_sigkill(self):
         # Uncatchable signal.
         self.check_resource_tracker_death(signal.SIGKILL, True)
@@ -5678,6 +5938,59 @@ class TestResourceTracker(unittest.TestCase):
         with self.assertRaises(ValueError):
             resource_tracker.register(too_long_name_resource, rtype)
 
+    def _test_resource_tracker_leak_resources(self, cleanup):
+        # We use a separate instance for testing, since the main global
+        # _resource_tracker may be used to watch test infrastructure.
+        from multiprocessing.resource_tracker import ResourceTracker
+        tracker = ResourceTracker()
+        tracker.ensure_running()
+        self.assertTrue(tracker._check_alive())
+
+        self.assertIsNone(tracker._exitcode)
+        tracker.register('somename', 'dummy')
+        if cleanup:
+            tracker.unregister('somename', 'dummy')
+            expected_exit_code = 0
+        else:
+            expected_exit_code = 1
+
+        self.assertTrue(tracker._check_alive())
+        self.assertIsNone(tracker._exitcode)
+        tracker._stop()
+        self.assertEqual(tracker._exitcode, expected_exit_code)
+
+    def test_resource_tracker_exit_code(self):
+        """
+        Test the exit code of the resource tracker.
+
+        If no leaked resources were found, exit code should be 0, otherwise 1
+        """
+        for cleanup in [True, False]:
+            with self.subTest(cleanup=cleanup):
+                self._test_resource_tracker_leak_resources(
+                    cleanup=cleanup,
+                )
+
+    @unittest.skipUnless(hasattr(signal, "pthread_sigmask"), "pthread_sigmask is not available")
+    def test_resource_tracker_blocked_signals(self):
+        #
+        # gh-127586: Check that resource_tracker does not override blocked signals of caller.
+        #
+        from multiprocessing.resource_tracker import ResourceTracker
+        orig_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, set())
+        signals = {signal.SIGTERM, signal.SIGINT, signal.SIGUSR1}
+
+        try:
+            for sig in signals:
+                signal.pthread_sigmask(signal.SIG_SETMASK, {sig})
+                self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig})
+                tracker = ResourceTracker()
+                tracker.ensure_running()
+                self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig})
+                tracker._stop()
+        finally:
+            # restore sigmask to what it was before executing test
+            signal.pthread_sigmask(signal.SIG_SETMASK, orig_sigmask)
 
 class TestSimpleQueue(unittest.TestCase):
 
@@ -5691,6 +6004,15 @@ class TestSimpleQueue(unittest.TestCase):
         finally:
             parent_can_continue.set()
 
+    def test_empty_exceptions(self):
+        # Assert that checking emptiness of a closed queue raises
+        # an OSError, independently of whether the queue was used
+        # or not. This differs from Queue and JoinableQueue.
+        q = multiprocessing.SimpleQueue()
+        q.close()  # close the pipe
+        with self.assertRaisesRegex(OSError, 'is closed'):
+            q.empty()
+
     def test_empty(self):
         queue = multiprocessing.SimpleQueue()
         child_can_start = multiprocessing.Event()
@@ -6037,6 +6359,99 @@ class TestNamedResource(unittest.TestCase):
         self.assertFalse(err, msg=err.decode('utf-8'))
 
 
+class _TestAtExit(BaseTestCase):
+
+    ALLOWED_TYPES = ('processes',)
+
+    @classmethod
+    def _write_file_at_exit(self, output_path):
+        import atexit
+        def exit_handler():
+            with open(output_path, 'w') as f:
+                f.write("deadbeef")
+        atexit.register(exit_handler)
+
+    def test_atexit(self):
+        # gh-83856
+        with os_helper.temp_dir() as temp_dir:
+            output_path = os.path.join(temp_dir, 'output.txt')
+            p = self.Process(target=self._write_file_at_exit, args=(output_path,))
+            p.start()
+            p.join()
+            with open(output_path) as f:
+                self.assertEqual(f.read(), 'deadbeef')
+
+
+class _TestSpawnedSysPath(BaseTestCase):
+    """Test that sys.path is setup in forkserver and spawn processes."""
+
+    ALLOWED_TYPES = {'processes'}
+    # Not applicable to fork which inherits everything from the process as is.
+    START_METHODS = {"forkserver", "spawn"}
+
+    def setUp(self):
+        self._orig_sys_path = list(sys.path)
+        self._temp_dir = tempfile.mkdtemp(prefix="test_sys_path-")
+        self._mod_name = "unique_test_mod"
+        module_path = os.path.join(self._temp_dir, f"{self._mod_name}.py")
+        with open(module_path, "w", encoding="utf-8") as mod:
+            mod.write("# A simple test module\n")
+        sys.path[:] = [p for p in sys.path if p]  # remove any existing ""s
+        sys.path.insert(0, self._temp_dir)
+        sys.path.insert(0, "")  # Replaced with an abspath in child.
+        self.assertIn(self.start_method, self.START_METHODS)
+        self._ctx = multiprocessing.get_context(self.start_method)
+
+    def tearDown(self):
+        sys.path[:] = self._orig_sys_path
+        shutil.rmtree(self._temp_dir, ignore_errors=True)
+
+    @staticmethod
+    def enq_imported_module_names(queue):
+        queue.put(tuple(sys.modules))
+
+    def test_forkserver_preload_imports_sys_path(self):
+        if self._ctx.get_start_method() != "forkserver":
+            self.skipTest("forkserver specific test.")
+        self.assertNotIn(self._mod_name, sys.modules)
+        multiprocessing.forkserver._forkserver._stop()  # Must be fresh.
+        self._ctx.set_forkserver_preload(
+            ["test.test_multiprocessing_forkserver", self._mod_name])
+        q = self._ctx.Queue()
+        proc = self._ctx.Process(
+                target=self.enq_imported_module_names, args=(q,))
+        proc.start()
+        proc.join()
+        child_imported_modules = q.get()
+        q.close()
+        self.assertIn(self._mod_name, child_imported_modules)
+
+    @staticmethod
+    def enq_sys_path_and_import(queue, mod_name):
+        queue.put(sys.path)
+        try:
+            importlib.import_module(mod_name)
+        except ImportError as exc:
+            queue.put(exc)
+        else:
+            queue.put(None)
+
+    def test_child_sys_path(self):
+        q = self._ctx.Queue()
+        proc = self._ctx.Process(
+                target=self.enq_sys_path_and_import, args=(q, self._mod_name))
+        proc.start()
+        proc.join()
+        child_sys_path = q.get()
+        import_error = q.get()
+        q.close()
+        self.assertNotIn("", child_sys_path)  # replaced by an abspath
+        self.assertIn(self._temp_dir, child_sys_path)  # our addition
+        # ignore the first element, it is the absolute "" replacement
+        self.assertEqual(child_sys_path[1:], sys.path[1:])
+        self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}")
+
+
 class MiscTestCase(unittest.TestCase):
     def test__all__(self):
         # Just make sure names in not_exported are excluded
@@ -6061,6 +6476,46 @@ class MiscTestCase(unittest.TestCase):
         self.assertEqual(rc, 0)
         self.assertFalse(err, msg=err.decode('utf-8'))
 
+    def test_large_pool(self):
+        #
+        # gh-89240: Check that large pools are always okay
+        #
+        testfn = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, testfn)
+        with open(testfn, 'w', encoding='utf-8') as f:
+            f.write(textwrap.dedent('''\
+                import multiprocessing
+                def f(x): return x*x
+                if __name__ == '__main__':
+                    with multiprocessing.Pool(200) as p:
+                        print(sum(p.map(f, range(1000))))
+            '''))
+        rc, out, err = script_helper.assert_python_ok(testfn)
+        self.assertEqual("332833500", out.decode('utf-8').strip())
+        self.assertFalse(err, msg=err.decode('utf-8'))
+
+    def test_forked_thread_not_started(self):
+        # gh-134381: Ensure that a thread that has not been started yet in
+        # the parent process can be started within a forked child process.
+
+        if multiprocessing.get_start_method() != "fork":
+            self.skipTest("fork specific test")
+
+        q = multiprocessing.Queue()
+        t = threading.Thread(target=lambda: q.put("done"), daemon=True)
+
+        def child():
+            t.start()
+            t.join()
+
+        p = multiprocessing.Process(target=child)
+        p.start()
+        p.join(support.SHORT_TIMEOUT)
+
+        self.assertEqual(p.exitcode, 0)
+        self.assertEqual(q.get_nowait(), "done")
+        close_queue(q)
+
 
 #
 # Mixins
@@ -6213,6 +6668,8 @@ def install_tests_in_module_dict(remote_globs, start_method,
             if base is BaseTestCase:
                 continue
             assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES
+            if base.START_METHODS and start_method not in base.START_METHODS:
+                continue  # class not intended for this start method.
             for type_ in base.ALLOWED_TYPES:
                 if only_type and type_ != only_type:
                     continue
@@ -6226,6 +6683,7 @@ def install_tests_in_module_dict(remote_globs, start_method,
                     Temp = hashlib_helper.requires_hashdigest('sha256')(Temp)
                 Temp.__name__ = Temp.__qualname__ = newname
                 Temp.__module__ = __module__
+                Temp.start_method = start_method
                 remote_globs[newname] = Temp
         elif issubclass(base, unittest.TestCase):
             if only_type:

Lib/test/_test_venv_multiprocessing.py

@@ -0,0 +1,40 @@
+import multiprocessing
+import random
+import sys
+
+def fill_queue(queue, code):
+    queue.put(code)
+
+
+def drain_queue(queue, code):
+    if code != queue.get():
+        sys.exit(1)
+
+
+def test_func():
+    code = random.randrange(0, 1000)
+    queue = multiprocessing.Queue()
+    fill_pool = multiprocessing.Process(
+        target=fill_queue,
+        args=(queue, code)
+    )
+    drain_pool = multiprocessing.Process(
+        target=drain_queue,
+        args=(queue, code)
+    )
+    drain_pool.start()
+    fill_pool.start()
+    fill_pool.join()
+    drain_pool.join()
+
+
+def main():
+    multiprocessing.set_start_method('spawn')
+    test_pool = multiprocessing.Process(target=test_func)
+    test_pool.start()
+    test_pool.join()
+    sys.exit(test_pool.exitcode)
+
+
+if __name__ == "__main__":
+    main()

Lib/test/_typed_dict_helper.py

@@ -1,18 +0,0 @@
-"""Used to test `get_type_hints()` on a cross-module inherited `TypedDict` class
-
-This script uses future annotations to postpone a type that won't be available
-on the module inheriting from to `Foo`. The subclass in the other module should
-look something like this:
-
-    class Bar(_typed_dict_helper.Foo, total=False):
-        b: int
-"""
-
-from __future__ import annotations
-
-from typing import Optional, TypedDict
-
-OptionalIntType = Optional[int]
-
-class Foo(TypedDict):
-    a: OptionalIntType

Lib/test/ann_module.py

@@ -1,62 +0,0 @@
-
-
-"""
-The module for testing variable annotations.
-Empty lines above are for good reason (testing for correct line numbers)
-"""
-
-from typing import Optional
-from functools import wraps
-
-__annotations__[1] = 2
-
-class C:
-
-    x = 5; y: Optional['C'] = None
-
-from typing import Tuple
-x: int = 5; y: str = x; f: Tuple[int, int]
-
-class M(type):
-
-    __annotations__['123'] = 123
-    o: type = object
-
-(pars): bool = True
-
-class D(C):
-    j: str = 'hi'; k: str= 'bye'
-
-from types import new_class
-h_class = new_class('H', (C,))
-j_class = new_class('J')
-
-class F():
-    z: int = 5
-    def __init__(self, x):
-        pass
-
-class Y(F):
-    def __init__(self):
-        super(F, self).__init__(123)
-
-class Meta(type):
-    def __new__(meta, name, bases, namespace):
-        return super().__new__(meta, name, bases, namespace)
-
-class S(metaclass = Meta):
-    x: str = 'something'
-    y: str = 'something else'
-
-def foo(x: int = 10):
-    def bar(y: List[str]):
-        x: str = 'yes'
-    bar()
-
-def dec(func):
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-    return wrapper
-
-u: int | float

Lib/test/ann_module2.py

@@ -1,36 +0,0 @@
-"""
-Some correct syntax for variable annotation here.
-More examples are in test_grammar and test_parser.
-"""
-
-from typing import no_type_check, ClassVar
-
-i: int = 1
-j: int
-x: float = i/10
-
-def f():
-    class C: ...
-    return C()
-
-f().new_attr: object = object()
-
-class C:
-    def __init__(self, x: int) -> None:
-        self.x = x
-
-c = C(5)
-c.new_attr: int = 10
-
-__annotations__ = {}
-
-
-@no_type_check
-class NTC:
-    def meth(self, param: complex) -> None:
-        ...
-
-class CV:
-    var: ClassVar['CV']
-
-CV.var = CV()

Lib/test/ann_module3.py

@@ -1,18 +0,0 @@
-"""
-Correct syntax for variable annotation that should fail at runtime
-in a certain manner. More examples are in test_grammar and test_parser.
-"""
-
-def f_bad_ann():
-    __annotations__[1] = 2
-
-class C_OK:
-    def __init__(self, x: int) -> None:
-        self.x: no_such_name = x  # This one is OK as proposed by Guido
-
-class D_bad_ann:
-    def __init__(self, x: int) -> None:
-        sfel.y: int = 0
-
-def g_bad_ann():
-    no_such_name.attr: int = 0

Lib/test/ann_module4.py

@@ -1,5 +0,0 @@
-# This ann_module isn't for test_typing,
-# it's for test_module
-
-a:int=3
-b:str=4

Lib/test/ann_module5.py

@@ -1,10 +0,0 @@
-# Used by test_typing to verify that Final wrapped in ForwardRef works.
-
-from __future__ import annotations
-
-from typing import Final
-
-name: Final[str] = "final"
-
-class MyClass:
-    value: Final = 3000

Lib/test/ann_module6.py

@@ -1,7 +0,0 @@
-# Tests that top-level ClassVar is not allowed
-
-from __future__ import annotations
-
-from typing import ClassVar
-
-wrong: ClassVar[int] = 1

Lib/test/ann_module7.py

@@ -1,11 +0,0 @@
-# Tests class have ``__text_signature__``
-
-from __future__ import annotations
-
-DEFAULT_BUFFER_SIZE = 8192
-
-class BufferedReader(object):
-    """BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE)\n--\n\n
-    Create a new buffered reader using the given readable raw IO object.
-    """
-    pass

Lib/test/archivetestdata/README.md

@@ -0,0 +1,36 @@
+# Test data for `test_zipfile`, `test_tarfile` (and even some others)
+
+## `test_zipfile`
+
+The test executables in this directory are created manually from `header.sh` and
+the `testdata_module_inside_zip.py` file.  You must have Info-ZIP's zip utility
+installed (`apt install zip` on Debian).
+
+### Purpose of `exe_with_zip` and `exe_with_z64`
+
+These are used to test executable files with an appended zipfile, in a scenario
+where the executable is _not_ a Python interpreter itself so our automatic
+zipimport machinery (that'd look for `__main__.py`) is not being used.
+
+### Updating the test executables
+
+If you update header.sh or the testdata_module_inside_zip.py file, rerun the
+commands below.  These are expected to be rarely changed, if ever.
+
+#### Standard old format (2.0) zip file
+
+```
+zip -0 zip2.zip testdata_module_inside_zip.py
+cat header.sh zip2.zip >exe_with_zip
+rm zip2.zip
+```
+
+#### Modern format (4.5) zip64 file
+
+Redirecting from stdin forces Info-ZIP's zip tool to create a zip64.
+
+```
+zip -0 <testdata_module_inside_zip.py >zip64.zip
+cat header.sh zip64.zip >exe_with_z64
+rm zip64.zip
+```

Lib/test/archivetestdata/header.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+INTERPRETER_UNDER_TEST="$1"
+if [[ ! -x "${INTERPRETER_UNDER_TEST}" ]]; then
+    echo "Interpreter must be the command line argument."
+    exit 4
+fi
+EXECUTABLE="$0" exec "${INTERPRETER_UNDER_TEST}" -E - <<END_OF_PYTHON
+import os
+import zipfile
+
+namespace = {}
+
+filename = os.environ['EXECUTABLE']
+print(f'Opening {filename} as a zipfile.')
+with zipfile.ZipFile(filename, mode='r') as exe_zip:
+  for file_info in exe_zip.infolist():
+    data = exe_zip.read(file_info)
+    exec(data, namespace, namespace)
+    break  # Only use the first file in the archive.
+
+print('Favorite number in executable:', namespace["FAVORITE_NUMBER"])
+
+### Archive contents will be appended after this file. ###
+END_OF_PYTHON

Lib/test/archivetestdata/testdata_module_inside_zip.py

@@ -0,0 +1,2 @@
+# Test data file to be stored within a zip file.
+FAVORITE_NUMBER = 5

Lib/test/data/README

@@ -0,0 +1,2 @@
+This empty directory serves as destination for temporary files
+created by some tests, in particular, the test_codecmaps_* tests.

Lib/test/decimaltestdata/abs.decTest

@@ -20,7 +20,7 @@
 version: 2.59
 
 -- This set of tests primarily tests the existence of the operator.
--- Additon, subtraction, rounding, and more overflows are tested
+-- Addition, subtraction, rounding, and more overflows are tested
 -- elsewhere.
 
 precision:   9

Lib/test/decimaltestdata/ddFMA.decTest

@@ -1663,7 +1663,7 @@ ddfma375087 fma  1  12345678 1E-33                   -> 12345678.00000001 Inexac
 ddfma375088 fma  1  12345678 1E-34                   -> 12345678.00000001 Inexact Rounded
 ddfma375089 fma  1  12345678 1E-35                   -> 12345678.00000001 Inexact Rounded
 
--- desctructive subtraction (from remainder tests)
+-- destructive subtraction (from remainder tests)
 
 -- +++ some of these will be off-by-one remainder vs remainderNear
 

Lib/test/decimaltestdata/ddQuantize.decTest

@@ -462,7 +462,7 @@ ddqua520 quantize   1.234    1e359 -> 0E+359 Inexact Rounded
 ddqua521 quantize 123.456    1e359 -> 0E+359 Inexact Rounded
 ddqua522 quantize   1.234    1e359 -> 0E+359 Inexact Rounded
 ddqua523 quantize 123.456    1e359 -> 0E+359 Inexact Rounded
--- next four are "won't fit" overfl
+-- next four are "won't fit" overflow
 ddqua526 quantize   1.234   1e-299 -> NaN Invalid_operation
 ddqua527 quantize 123.456   1e-299 -> NaN Invalid_operation
 ddqua528 quantize   1.234   1e-299 -> NaN Invalid_operation

Lib/test/decimaltestdata/ddRemainder.decTest

@@ -422,7 +422,7 @@ ddrem757 remainder  1    sNaN   ->  NaN  Invalid_operation
 ddrem758 remainder  1000 sNaN   ->  NaN  Invalid_operation
 ddrem759 remainder  Inf -sNaN   -> -NaN  Invalid_operation
 
--- propaging NaNs
+-- propagating NaNs
 ddrem760 remainder  NaN1   NaN7   ->  NaN1
 ddrem761 remainder sNaN2   NaN8   ->  NaN2 Invalid_operation
 ddrem762 remainder  NaN3  sNaN9   ->  NaN9 Invalid_operation

Lib/test/decimaltestdata/ddRemainderNear.decTest

@@ -450,7 +450,7 @@ ddrmn757 remaindernear  1    sNaN   ->  NaN  Invalid_operation
 ddrmn758 remaindernear  1000 sNaN   ->  NaN  Invalid_operation
 ddrmn759 remaindernear  Inf -sNaN   -> -NaN  Invalid_operation
 
--- propaging NaNs
+-- propagating NaNs
 ddrmn760 remaindernear  NaN1   NaN7   ->  NaN1
 ddrmn761 remaindernear sNaN2   NaN8   ->  NaN2 Invalid_operation
 ddrmn762 remaindernear  NaN3  sNaN9   ->  NaN9 Invalid_operation