Enable test_mtestfile

* Fix usize not using the same hash as PyInt when used as key into a dictionary

* Fix test that unexpectedly succeed!

* Update extra_tests/snippets/builtin_object.py

.cspell.dict/rust-more.txt

@@ -51,7 +51,6 @@ powf
 powi
 prepended
 punct
-puruspe
 replacen
 rmatch
 rposition

.github/copilot-instructions.md

@@ -0,0 +1,186 @@
+# GitHub Copilot Instructions for RustPython
+
+This document provides guidelines for working with GitHub Copilot when contributing to the RustPython project.
+
+## Project Overview
+
+RustPython is a Python 3 interpreter written in Rust, implementing Python 3.13.0+ compatibility. The project aims to provide:
+
+- A complete Python-3 environment entirely in Rust (not CPython bindings)
+- A clean implementation without compatibility hacks
+- Cross-platform support, including WebAssembly compilation
+- The ability to embed Python scripting in Rust applications
+
+## Repository Structure
+
+- `src/` - Top-level code for the RustPython binary
+- `vm/` - The Python virtual machine implementation
+  - `builtins/` - Python built-in types and functions
+  - `stdlib/` - Essential standard library modules implemented in Rust, required to run the Python core
+- `compiler/` - Python compiler components
+  - `parser/` - Parser for converting Python source to AST
+  - `core/` - Bytecode representation in Rust structures
+  - `codegen/` - AST to bytecode compiler
+- `Lib/` - CPython's standard library in Python (copied from CPython)
+- `derive/` - Rust macros for RustPython
+- `common/` - Common utilities
+- `extra_tests/` - Integration tests and snippets
+- `stdlib/` - Non-essential Python standard library modules implemented in Rust (useful but not required for core functionality)
+- `wasm/` - WebAssembly support
+- `jit/` - Experimental JIT compiler implementation
+- `pylib/` - Python standard library packaging (do not modify this directory directly - its contents are generated automatically)
+
+## Important Development Notes
+
+### Running Python Code
+
+When testing Python code, always use RustPython instead of the standard `python` command:
+
+```bash
+# Use this instead of python script.py
+cargo run -- script.py
+
+# For interactive REPL
+cargo run
+
+# With specific features
+cargo run --features ssl
+
+# Release mode (recommended for better performance)
+cargo run --release -- script.py
+```
+
+### Comparing with CPython
+
+When you need to compare behavior with CPython or run test suites:
+
+```bash
+# Use python command to explicitly run CPython
+python my_test_script.py
+
+# Run RustPython
+cargo run -- my_test_script.py
+```
+
+### Working with the Lib Directory
+
+The `Lib/` directory contains Python standard library files copied from the CPython repository. Important notes:
+
+- These files should be edited very conservatively
+- Modifications should be minimal and only to work around RustPython limitations
+- Tests in `Lib/test` often use one of the following markers:
+  - Add a `# TODO: RUSTPYTHON` comment when modifications are made
+  - `unittest.skip("TODO: RustPython <reason>")`
+  - `unittest.expectedFailure` with `# TODO: RUSTPYTHON <reason>` comment
+
+### Testing
+
+```bash
+# Run Rust unit tests
+cargo test --workspace --exclude rustpython_wasm
+
+# Run Python snippets tests
+cd extra_tests
+pytest -v
+
+# Run the Python test module
+cargo run --release -- -m test
+```
+
+### Determining What to Implement
+
+Run `./whats_left.py` to get a list of unimplemented methods, which is helpful when looking for contribution opportunities.
+
+## Coding Guidelines
+
+### Rust Code
+
+- Follow the default rustfmt code style (`cargo fmt` to format)
+- Use clippy to lint code (`cargo clippy`)
+- Follow Rust best practices for error handling and memory management
+- Use the macro system (`pyclass`, `pymodule`, `pyfunction`, etc.) when implementing Python functionality in Rust
+
+### Python Code
+
+- Follow PEP 8 style for custom Python code
+- Use ruff for linting Python code
+- Minimize modifications to CPython standard library files
+
+## Integration Between Rust and Python
+
+The project provides several mechanisms for integration:
+
+- `pymodule` macro for creating Python modules in Rust
+- `pyclass` macro for implementing Python classes in Rust
+- `pyfunction` macro for exposing Rust functions to Python
+- `PyObjectRef` and other types for working with Python objects in Rust
+
+## Common Patterns
+
+### Implementing a Python Module in Rust
+
+```rust
+#[pymodule]
+mod mymodule {
+    use rustpython_vm::prelude::*;
+
+    #[pyfunction]
+    fn my_function(value: i32) -> i32 {
+        value * 2
+    }
+
+    #[pyattr]
+    #[pyclass(name = "MyClass")]
+    #[derive(Debug, PyPayload)]
+    struct MyClass {
+        value: usize,
+    }
+
+    #[pyclass]
+    impl MyClass {
+        #[pymethod]
+        fn get_value(&self) -> usize {
+            self.value
+        }
+    }
+}
+```
+
+### Adding a Python Module to the Interpreter
+
+```rust
+vm.add_native_module(
+    "my_module_name".to_owned(),
+    Box::new(my_module::make_module),
+);
+```
+
+## Building for Different Targets
+
+### WebAssembly
+
+```bash
+# Build for WASM
+cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release
+```
+
+### JIT Support
+
+```bash
+# Enable JIT support
+cargo run --features jit
+```
+
+### SSL Support
+
+```bash
+# Enable SSL support
+cargo run --features ssl
+```
+
+## Documentation
+
+- Check the [architecture document](architecture/architecture.md) for a high-level overview
+- Read the [development guide](DEVELOPMENT.md) for detailed setup instructions
+- Generate documentation with `cargo doc --no-deps --all`
+- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/)

Cargo.lock

@@ -1240,16 +1240,6 @@ dependencies = [
  "cpufeatures",
 ]
 
-[[package]]
-name = "lambert_w"
-version = "1.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd4d9b9fa6582f5d77f954729c91c32a7c85834332e470b014d12e1678fd1793"
-dependencies = [
- "num-complex",
- "num-traits",
-]
-
 [[package]]
 name = "lazy_static"
 version = "0.2.11"
@@ -1306,9 +1296,9 @@ checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
 
 [[package]]
 name = "libffi"
-version = "3.2.0"
+version = "4.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce826c243048e3d5cec441799724de52e2d42f820468431fc3fceee2341871e2"
+checksum = "4a9434b6fc77375fb624698d5f8c49d7e80b10d59eb1219afda27d1f824d4074"
 dependencies = [
  "libc",
  "libffi-sys",
@@ -1316,9 +1306,9 @@ dependencies = [
 
 [[package]]
 name = "libffi-sys"
-version = "2.3.0"
+version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f36115160c57e8529781b4183c2bb51fdc1f6d6d1ed345591d84be7703befb3c"
+checksum = "ead36a2496acfc8edd6cc32352110e9478ac5b9b5f5b9856ebd3d28019addb84"
 dependencies = [
  "cc",
 ]
@@ -1889,13 +1879,12 @@ dependencies = [
 ]
 
 [[package]]
-name = "puruspe"
-version = "0.4.0"
+name = "pymath"
+version = "0.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d76c522e44709f541a403db419a7e34d6fbbc8e6b208589ae29a030cddeefd96"
+checksum = "5b66ab66a8610ce209d8b36cd0fecc3a15c494f715e0cb26f0586057f293abc9"
 dependencies = [
- "lambert_w",
- "num-complex",
+ "libc",
 ]
 
 [[package]]
@@ -2495,7 +2484,7 @@ dependencies = [
  "page_size",
  "parking_lot",
  "paste",
- "puruspe",
+ "pymath",
  "rand_core 0.9.3",
  "rustix",
  "rustpython-common",
@@ -2768,6 +2757,14 @@ dependencies = [
  "keccak",
 ]
 
+[[package]]
+name = "shared-build"
+version = "0.2.0"
+source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.2.0#198fc35b1f18f4eda401f97a641908f321b1403a"
+dependencies = [
+ "bindgen",
+]
+
 [[package]]
 name = "shlex"
 version = "1.3.0"
@@ -2901,11 +2898,11 @@ checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a"
 
 [[package]]
 name = "tcl-sys"
-version = "0.1.0"
-source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.1.0#09a4f62e894df64692b34e6c7f81af1e1ae376dd"
+version = "0.2.0"
+source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.2.0#198fc35b1f18f4eda401f97a641908f321b1403a"
 dependencies = [
- "bindgen",
  "pkg-config",
+ "shared-build",
 ]
 
 [[package]]
@@ -3017,11 +3014,11 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tk-sys"
-version = "0.1.0"
-source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.1.0#09a4f62e894df64692b34e6c7f81af1e1ae376dd"
+version = "0.2.0"
+source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.2.0#198fc35b1f18f4eda401f97a641908f321b1403a"
 dependencies = [
- "bindgen",
  "pkg-config",
+ "shared-build",
 ]
 
 [[package]]

Cargo.toml

@@ -168,7 +168,7 @@ itertools = "0.14.0"
 is-macro = "0.3.7"
 junction = "1.2.0"
 libc = "0.2.169"
-libffi = "3.2"
+libffi = "4.0"
 log = "0.4.27"
 nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
 malachite-bigint = "0.6"
@@ -184,6 +184,7 @@ once_cell = "1.20.3"
 parking_lot = "0.12.3"
 paste = "1.0.15"
 proc-macro2 = "1.0.93"
+pymath = "0.0.2"
 quote = "1.0.38"
 rand = "0.9"
 rand_core = { version = "0.9", features = ["os_rng"] }

Lib/codeop.py

@@ -66,7 +66,12 @@ def _maybe_compile(compiler, source, filename, symbol):
                 compiler(source + "\n", filename, symbol)
                 return None
             except SyntaxError as e:
-                if "incomplete input" in str(e):
+                # XXX: RustPython; support multiline definitions in REPL
+                # See also: https://github.com/RustPython/RustPython/pull/5743
+                strerr = str(e)
+                if source.endswith(":") and "expected an indented block" in strerr:
+                    return None
+                elif "incomplete input" in str(e):
                     return None
                 # fallthrough
 

Lib/fileinput.py

@@ -53,7 +53,7 @@ __getitem__() method which implements the sequence behavior.  The
 sequence must be accessed in strictly sequential order; sequence
 access and readline() cannot be mixed.
 
-Optional in-place filtering: if the keyword argument inplace=1 is
+Optional in-place filtering: if the keyword argument inplace=True is
 passed to input() or to the FileInput constructor, the file is moved
 to a backup file and standard output is directed to the input file.
 This makes it possible to write a filter that rewrites its input file
@@ -399,7 +399,7 @@ class FileInput:
 
 
 def hook_compressed(filename, mode, *, encoding=None, errors=None):
-    if encoding is None:  # EncodingWarning is emitted in FileInput() already.
+    if encoding is None and "b" not in mode:  # EncodingWarning is emitted in FileInput() already.
         encoding = "locale"
     ext = os.path.splitext(filename)[1]
     if ext == '.gz':

Lib/getpass.py

@@ -18,7 +18,6 @@ import contextlib
 import io
 import os
 import sys
-import warnings
 
 __all__ = ["getpass","getuser","GetPassWarning"]
 
@@ -118,6 +117,7 @@ def win_getpass(prompt='Password: ', stream=None):
 
 
 def fallback_getpass(prompt='Password: ', stream=None):
+    import warnings
     warnings.warn("Can not control echo on the terminal.", GetPassWarning,
                   stacklevel=2)
     if not stream:
@@ -156,7 +156,11 @@ def getuser():
 
     First try various environment variables, then the password
     database.  This works on Windows as long as USERNAME is set.
+    Any failure to find a username raises OSError.
 
+    .. versionchanged:: 3.13
+        Previously, various exceptions beyond just :exc:`OSError`
+        were raised.
     """
 
     for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'):
@@ -164,9 +168,12 @@ def getuser():
         if user:
             return user
 
-    # If this fails, the exception will "explain" why
-    import pwd
-    return pwd.getpwuid(os.getuid())[0]
+    try:
+        import pwd
+        return pwd.getpwuid(os.getuid())[0]
+    except (ImportError, KeyError) as e:
+        raise OSError('No username set in the environment') from e
+
 
 # Bind the name getpass to the appropriate function
 try:

Lib/reprlib.py

@@ -29,49 +29,100 @@ def recursive_repr(fillvalue='...'):
         wrapper.__name__ = getattr(user_function, '__name__')
         wrapper.__qualname__ = getattr(user_function, '__qualname__')
         wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
+        wrapper.__type_params__ = getattr(user_function, '__type_params__', ())
+        wrapper.__wrapped__ = user_function
         return wrapper
 
     return decorating_function
 
 class Repr:
+    _lookup = {
+        'tuple': 'builtins',
+        'list': 'builtins',
+        'array': 'array',
+        'set': 'builtins',
+        'frozenset': 'builtins',
+        'deque': 'collections',
+        'dict': 'builtins',
+        'str': 'builtins',
+        'int': 'builtins'
+    }
 
-    def __init__(self):
-        self.maxlevel = 6
-        self.maxtuple = 6
-        self.maxlist = 6
-        self.maxarray = 5
-        self.maxdict = 4
-        self.maxset = 6
-        self.maxfrozenset = 6
-        self.maxdeque = 6
-        self.maxstring = 30
-        self.maxlong = 40
-        self.maxother = 30
+    def __init__(
+        self, *, maxlevel=6, maxtuple=6, maxlist=6, maxarray=5, maxdict=4,
+        maxset=6, maxfrozenset=6, maxdeque=6, maxstring=30, maxlong=40,
+        maxother=30, fillvalue='...', indent=None,
+    ):
+        self.maxlevel = maxlevel
+        self.maxtuple = maxtuple
+        self.maxlist = maxlist
+        self.maxarray = maxarray
+        self.maxdict = maxdict
+        self.maxset = maxset
+        self.maxfrozenset = maxfrozenset
+        self.maxdeque = maxdeque
+        self.maxstring = maxstring
+        self.maxlong = maxlong
+        self.maxother = maxother
+        self.fillvalue = fillvalue
+        self.indent = indent
 
     def repr(self, x):
         return self.repr1(x, self.maxlevel)
 
     def repr1(self, x, level):
-        typename = type(x).__name__
+        cls = type(x)
+        typename = cls.__name__
+
         if ' ' in typename:
             parts = typename.split()
             typename = '_'.join(parts)
-        if hasattr(self, 'repr_' + typename):
-            return getattr(self, 'repr_' + typename)(x, level)
-        else:
-            return self.repr_instance(x, level)
+
+        method = getattr(self, 'repr_' + typename, None)
+        if method:
+            # not defined in this class
+            if typename not in self._lookup:
+                return method(x, level)
+            module = getattr(cls, '__module__', None)
+            # defined in this class and is the module intended
+            if module == self._lookup[typename]:
+                return method(x, level)
+
+        return self.repr_instance(x, level)
+
+    def _join(self, pieces, level):
+        if self.indent is None:
+            return ', '.join(pieces)
+        if not pieces:
+            return ''
+        indent = self.indent
+        if isinstance(indent, int):
+            if indent < 0:
+                raise ValueError(
+                    f'Repr.indent cannot be negative int (was {indent!r})'
+                )
+            indent *= ' '
+        try:
+            sep = ',\n' + (self.maxlevel - level + 1) * indent
+        except TypeError as error:
+            raise TypeError(
+                f'Repr.indent must be a str, int or None, not {type(indent)}'
+            ) from error
+        return sep.join(('', *pieces, ''))[1:-len(indent) or None]
 
     def _repr_iterable(self, x, level, left, right, maxiter, trail=''):
         n = len(x)
         if level <= 0 and n:
-            s = '...'
+            s = self.fillvalue
         else:
             newlevel = level - 1
             repr1 = self.repr1
             pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)]
-            if n > maxiter:  pieces.append('...')
-            s = ', '.join(pieces)
-            if n == 1 and trail:  right = trail + right
+            if n > maxiter:
+                pieces.append(self.fillvalue)
+            s = self._join(pieces, level)
+            if n == 1 and trail and self.indent is None:
+                right = trail + right
         return '%s%s%s' % (left, s, right)
 
     def repr_tuple(self, x, level):
@@ -104,8 +155,10 @@ class Repr:
 
     def repr_dict(self, x, level):
         n = len(x)
-        if n == 0: return '{}'
-        if level <= 0: return '{...}'
+        if n == 0:
+            return '{}'
+        if level <= 0:
+            return '{' + self.fillvalue + '}'
         newlevel = level - 1
         repr1 = self.repr1
         pieces = []
@@ -113,8 +166,9 @@ class Repr:
             keyrepr = repr1(key, newlevel)
             valrepr = repr1(x[key], newlevel)
             pieces.append('%s: %s' % (keyrepr, valrepr))
-        if n > self.maxdict: pieces.append('...')
-        s = ', '.join(pieces)
+        if n > self.maxdict:
+            pieces.append(self.fillvalue)
+        s = self._join(pieces, level)
         return '{%s}' % (s,)
 
     def repr_str(self, x, level):
@@ -123,7 +177,7 @@ class Repr:
             i = max(0, (self.maxstring-3)//2)
             j = max(0, self.maxstring-3-i)
             s = builtins.repr(x[:i] + x[len(x)-j:])
-            s = s[:i] + '...' + s[len(s)-j:]
+            s = s[:i] + self.fillvalue + s[len(s)-j:]
         return s
 
     def repr_int(self, x, level):
@@ -131,7 +185,7 @@ class Repr:
         if len(s) > self.maxlong:
             i = max(0, (self.maxlong-3)//2)
             j = max(0, self.maxlong-3-i)
-            s = s[:i] + '...' + s[len(s)-j:]
+            s = s[:i] + self.fillvalue + s[len(s)-j:]
         return s
 
     def repr_instance(self, x, level):
@@ -144,7 +198,7 @@ class Repr:
         if len(s) > self.maxother:
             i = max(0, (self.maxother-3)//2)
             j = max(0, self.maxother-3-i)
-            s = s[:i] + '...' + s[len(s)-j:]
+            s = s[:i] + self.fillvalue + s[len(s)-j:]
         return s
 
 

Lib/test/support/i18n_helper.py

@@ -0,0 +1,63 @@
+import re
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+from test.support import REPO_ROOT, TEST_HOME_DIR, requires_subprocess
+from test.test_tools import skip_if_missing
+
+
+pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py'
+
+msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)',
+                           re.DOTALL)
+msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"')
+
+
+def _generate_po_file(path, *, stdout_only=True):
+    res = subprocess.run([sys.executable, pygettext,
+                          '--no-location', '-o', '-', path],
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                         text=True)
+    if stdout_only:
+        return res.stdout
+    return res
+
+
+def _extract_msgids(po):
+    msgids = []
+    for msgid in msgid_pattern.findall(po):
+        msgid_string = ''.join(msgid_string_pattern.findall(msgid))
+        msgid_string = msgid_string.replace(r'\"', '"')
+        if msgid_string:
+            msgids.append(msgid_string)
+    return sorted(msgids)
+
+
+def _get_snapshot_path(module_name):
+    return Path(TEST_HOME_DIR) / 'translationdata' / module_name / 'msgids.txt'
+
+
+@requires_subprocess()
+class TestTranslationsBase(unittest.TestCase):
+
+    def assertMsgidsEqual(self, module):
+        '''Assert that msgids extracted from a given module match a
+        snapshot.
+
+        '''
+        skip_if_missing('i18n')
+        res = _generate_po_file(module.__file__, stdout_only=False)
+        self.assertEqual(res.returncode, 0)
+        self.assertEqual(res.stderr, '')
+        msgids = _extract_msgids(res.stdout)
+        snapshot_path = _get_snapshot_path(module.__name__)
+        snapshot = snapshot_path.read_text().splitlines()
+        self.assertListEqual(msgids, snapshot)
+
+
+def update_translation_snapshots(module):
+    contents = _generate_po_file(module.__file__)
+    msgids = _extract_msgids(contents)
+    snapshot_path = _get_snapshot_path(module.__name__)
+    snapshot_path.write_text('\n'.join(msgids))

Lib/test/test_fileinput.py

@@ -23,10 +23,9 @@ except ImportError:
 
 from io import BytesIO, StringIO
 from fileinput import FileInput, hook_encoded
-from pathlib import Path
 
 from test.support import verbose
-from test.support.os_helper import TESTFN
+from test.support.os_helper import TESTFN, FakePath
 from test.support.os_helper import unlink as safe_unlink
 from test.support import os_helper
 from test import support
@@ -151,7 +150,7 @@ class BufferSizesTests(BaseTests, unittest.TestCase):
             print('6. Inplace')
         savestdout = sys.stdout
         try:
-            fi = FileInput(files=(t1, t2, t3, t4), inplace=1, encoding="utf-8")
+            fi = FileInput(files=(t1, t2, t3, t4), inplace=True, encoding="utf-8")
             for line in fi:
                 line = line[:-1].upper()
                 print(line)
@@ -256,7 +255,7 @@ class FileInputTests(BaseTests, unittest.TestCase):
     def test_file_opening_hook(self):
         try:
             # cannot use openhook and inplace mode
-            fi = FileInput(inplace=1, openhook=lambda f, m: None)
+            fi = FileInput(inplace=True, openhook=lambda f, m: None)
             self.fail("FileInput should raise if both inplace "
                              "and openhook arguments are given")
         except ValueError:
@@ -478,23 +477,23 @@ class FileInputTests(BaseTests, unittest.TestCase):
             self.assertRaises(StopIteration, next, fi)
             self.assertEqual(src.linesread, [])
 
-    def test_pathlib_file(self):
-        t1 = Path(self.writeTmp("Pathlib file."))
+    def test_pathlike_file(self):
+        t1 = FakePath(self.writeTmp("Path-like file."))
         with FileInput(t1, encoding="utf-8") as fi:
             line = fi.readline()
-            self.assertEqual(line, 'Pathlib file.')
+            self.assertEqual(line, 'Path-like file.')
             self.assertEqual(fi.lineno(), 1)
             self.assertEqual(fi.filelineno(), 1)
             self.assertEqual(fi.filename(), os.fspath(t1))
 
-    def test_pathlib_file_inplace(self):
-        t1 = Path(self.writeTmp('Pathlib file.'))
+    def test_pathlike_file_inplace(self):
+        t1 = FakePath(self.writeTmp('Path-like file.'))
         with FileInput(t1, inplace=True, encoding="utf-8") as fi:
             line = fi.readline()
-            self.assertEqual(line, 'Pathlib file.')
+            self.assertEqual(line, 'Path-like file.')
             print('Modified %s' % line)
         with open(t1, encoding="utf-8") as f:
-            self.assertEqual(f.read(), 'Modified Pathlib file.\n')
+            self.assertEqual(f.read(), 'Modified Path-like file.\n')
 
 
 class MockFileInput:
@@ -855,29 +854,29 @@ class Test_hook_compressed(unittest.TestCase):
         self.fake_open = InvocationRecorder()
 
     def test_empty_string(self):
-        self.do_test_use_builtin_open("", 1)
+        self.do_test_use_builtin_open_text("", "r")
 
     def test_no_ext(self):
-        self.do_test_use_builtin_open("abcd", 2)
+        self.do_test_use_builtin_open_text("abcd", "r")
 
     @unittest.skipUnless(gzip, "Requires gzip and zlib")
     def test_gz_ext_fake(self):
         original_open = gzip.open
         gzip.open = self.fake_open
         try:
-            result = fileinput.hook_compressed("test.gz", "3")
+            result = fileinput.hook_compressed("test.gz", "r")
         finally:
             gzip.open = original_open
 
         self.assertEqual(self.fake_open.invocation_count, 1)
-        self.assertEqual(self.fake_open.last_invocation, (("test.gz", "3"), {}))
+        self.assertEqual(self.fake_open.last_invocation, (("test.gz", "r"), {}))
 
     @unittest.skipUnless(gzip, "Requires gzip and zlib")
     def test_gz_with_encoding_fake(self):
         original_open = gzip.open
         gzip.open = lambda filename, mode: io.BytesIO(b'Ex-binary string')
         try:
-            result = fileinput.hook_compressed("test.gz", "3", encoding="utf-8")
+            result = fileinput.hook_compressed("test.gz", "r", encoding="utf-8")
         finally:
             gzip.open = original_open
         self.assertEqual(list(result), ['Ex-binary string'])
@@ -887,23 +886,40 @@ class Test_hook_compressed(unittest.TestCase):
         original_open = bz2.BZ2File
         bz2.BZ2File = self.fake_open
         try:
-            result = fileinput.hook_compressed("test.bz2", "4")
+            result = fileinput.hook_compressed("test.bz2", "r")
         finally:
             bz2.BZ2File = original_open
 
         self.assertEqual(self.fake_open.invocation_count, 1)
-        self.assertEqual(self.fake_open.last_invocation, (("test.bz2", "4"), {}))
+        self.assertEqual(self.fake_open.last_invocation, (("test.bz2", "r"), {}))
 
     def test_blah_ext(self):
-        self.do_test_use_builtin_open("abcd.blah", "5")
+        self.do_test_use_builtin_open_binary("abcd.blah", "rb")
 
     def test_gz_ext_builtin(self):
-        self.do_test_use_builtin_open("abcd.Gz", "6")
+        self.do_test_use_builtin_open_binary("abcd.Gz", "rb")
 
     def test_bz2_ext_builtin(self):
-        self.do_test_use_builtin_open("abcd.Bz2", "7")
+        self.do_test_use_builtin_open_binary("abcd.Bz2", "rb")
 
-    def do_test_use_builtin_open(self, filename, mode):
+    def test_binary_mode_encoding(self):
+        self.do_test_use_builtin_open_binary("abcd", "rb")
+
+    def test_text_mode_encoding(self):
+        self.do_test_use_builtin_open_text("abcd", "r")
+
+    def do_test_use_builtin_open_binary(self, filename, mode):
+        original_open = self.replace_builtin_open(self.fake_open)
+        try:
+            result = fileinput.hook_compressed(filename, mode)
+        finally:
+            self.replace_builtin_open(original_open)
+
+        self.assertEqual(self.fake_open.invocation_count, 1)
+        self.assertEqual(self.fake_open.last_invocation,
+                         ((filename, mode), {'encoding': None, 'errors': None}))
+
+    def do_test_use_builtin_open_text(self, filename, mode):
         original_open = self.replace_builtin_open(self.fake_open)
         try:
             result = fileinput.hook_compressed(filename, mode)

Lib/test/test_fstring.py

@@ -1631,8 +1631,6 @@ x = (
         self.assertEqual(f"{x!s:}", "test")
         self.assertEqual(f"{x!r:}", "'test'")
 
-    # TODO: RUSTPYTHON d[0] error
-    @unittest.expectedFailure
     def test_str_format_differences(self):
         d = {
             "a": "string",

Lib/test/test_getopt.py

@@ -1,19 +1,19 @@
 # test_getopt.py
 # David Goodger <dgoodger@bigfoot.com> 2000-08-19
 
-from test.support.os_helper import EnvironmentVarGuard
 import doctest
-import unittest
-
 import getopt
+import sys
+import unittest
+from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots
+from test.support.os_helper import EnvironmentVarGuard
 
 sentinel = object()
 
 class GetoptTests(unittest.TestCase):
     def setUp(self):
         self.env = self.enterContext(EnvironmentVarGuard())
-        if "POSIXLY_CORRECT" in self.env:
-            del self.env["POSIXLY_CORRECT"]
+        del self.env["POSIXLY_CORRECT"]
 
     def assertError(self, *args, **kwargs):
         self.assertRaises(getopt.GetoptError, *args, **kwargs)
@@ -173,10 +173,20 @@ def test_libref_examples():
     ['a1', 'a2']
     """
 
+
+class TestTranslations(TestTranslationsBase):
+    def test_translations(self):
+        self.assertMsgidsEqual(getopt)
+
+
 def load_tests(loader, tests, pattern):
     tests.addTest(doctest.DocTestSuite())
     return tests
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
+    # To regenerate translation snapshots
+    if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
+        update_translation_snapshots(getopt)
+        sys.exit(0)
     unittest.main()

Lib/test/test_getpass.py

@@ -26,7 +26,10 @@ class GetpassGetuserTest(unittest.TestCase):
         environ.get.return_value = None
         try:
             getpass.getuser()
-        except ImportError: # in case there's no pwd module
+        except OSError:  # in case there's no pwd module
+            pass
+        except KeyError:
+            # current user has no pwd entry
             pass
         self.assertEqual(
             environ.get.call_args_list,
@@ -44,7 +47,7 @@ class GetpassGetuserTest(unittest.TestCase):
                                  getpass.getuser())
                 getpw.assert_called_once_with(42)
         else:
-            self.assertRaises(ImportError, getpass.getuser)
+            self.assertRaises(OSError, getpass.getuser)
 
 
 class GetpassRawinputTest(unittest.TestCase):

Lib/test/test_math.py

@@ -1374,6 +1374,7 @@ class MathTests(unittest.TestCase):
         self.assertEqual(sumprod([True, False] * 10, [0.1] * 20), 1.0)
         self.assertEqual(sumprod([1.0, 10E100, 1.0, -10E100], [1.0]*4), 2.0)
 
+    @unittest.skip("TODO: RUSTPYTHON, Taking a few minutes.")
     @support.requires_resource('cpu')
     def test_sumprod_stress(self):
         sumprod = math.sumprod
@@ -2079,7 +2080,6 @@ class MathTests(unittest.TestCase):
             self.fail('Failures in test_testfile:\n  ' +
                       '\n  '.join(failures))
 
-    @unittest.skip("TODO: RUSTPYTHON, Currently hangs. Function never finishes.")
     @requires_IEEE_754
     def test_mtestfile(self):
         fail_fmt = "{}: {}({!r}): {}"

Lib/test/test_reprlib.py

@@ -9,6 +9,7 @@ import shutil
 import importlib
 import importlib.util
 import unittest
+import textwrap
 
 from test.support import verbose
 from test.support.os_helper import create_empty_file
@@ -25,6 +26,29 @@ def nestedTuple(nesting):
 
 class ReprTests(unittest.TestCase):
 
+    def test_init_kwargs(self):
+        example_kwargs = {
+            "maxlevel": 101,
+            "maxtuple": 102,
+            "maxlist": 103,
+            "maxarray": 104,
+            "maxdict": 105,
+            "maxset": 106,
+            "maxfrozenset": 107,
+            "maxdeque": 108,
+            "maxstring": 109,
+            "maxlong": 110,
+            "maxother": 111,
+            "fillvalue": "x" * 112,
+            "indent": "x" * 113,
+        }
+        r1 = Repr()
+        for attr, val in example_kwargs.items():
+            setattr(r1, attr, val)
+        r2 = Repr(**example_kwargs)
+        for attr in example_kwargs:
+            self.assertEqual(getattr(r1, attr), getattr(r2, attr), msg=attr)
+
     def test_string(self):
         eq = self.assertEqual
         eq(r("abc"), "'abc'")
@@ -51,6 +75,15 @@ class ReprTests(unittest.TestCase):
         expected = repr(t3)[:-2] + "...)"
         eq(r2.repr(t3), expected)
 
+        # modified fillvalue:
+        r3 = Repr()
+        r3.fillvalue = '+++'
+        r3.maxtuple = 2
+        expected = repr(t3)[:-2] + "+++)"
+        eq(r3.repr(t3), expected)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_container(self):
         from array import array
         from collections import deque
@@ -223,6 +256,382 @@ class ReprTests(unittest.TestCase):
         r(y)
         r(z)
 
+    def test_valid_indent(self):
+        test_cases = [
+            {
+                'object': (),
+                'tests': (
+                    (dict(indent=None), '()'),
+                    (dict(indent=False), '()'),
+                    (dict(indent=True), '()'),
+                    (dict(indent=0), '()'),
+                    (dict(indent=1), '()'),
+                    (dict(indent=4), '()'),
+                    (dict(indent=4, maxlevel=2), '()'),
+                    (dict(indent=''), '()'),
+                    (dict(indent='-->'), '()'),
+                    (dict(indent='....'), '()'),
+                ),
+            },
+            {
+                'object': '',
+                'tests': (
+                    (dict(indent=None), "''"),
+                    (dict(indent=False), "''"),
+                    (dict(indent=True), "''"),
+                    (dict(indent=0), "''"),
+                    (dict(indent=1), "''"),
+                    (dict(indent=4), "''"),
+                    (dict(indent=4, maxlevel=2), "''"),
+                    (dict(indent=''), "''"),
+                    (dict(indent='-->'), "''"),
+                    (dict(indent='....'), "''"),
+                ),
+            },
+            {
+                'object': [1, 'spam', {'eggs': True, 'ham': []}],
+                'tests': (
+                    (dict(indent=None), '''\
+                        [1, 'spam', {'eggs': True, 'ham': []}]'''),
+                    (dict(indent=False), '''\
+                        [
+                        1,
+                        'spam',
+                        {
+                        'eggs': True,
+                        'ham': [],
+                        },
+                        ]'''),
+                    (dict(indent=True), '''\
+                        [
+                         1,
+                         'spam',
+                         {
+                          'eggs': True,
+                          'ham': [],
+                         },
+                        ]'''),
+                    (dict(indent=0), '''\
+                        [
+                        1,
+                        'spam',
+                        {
+                        'eggs': True,
+                        'ham': [],
+                        },
+                        ]'''),
+                    (dict(indent=1), '''\
+                        [
+                         1,
+                         'spam',
+                         {
+                          'eggs': True,
+                          'ham': [],
+                         },
+                        ]'''),
+                    (dict(indent=4), '''\
+                        [
+                            1,
+                            'spam',
+                            {
+                                'eggs': True,
+                                'ham': [],
+                            },
+                        ]'''),
+                    (dict(indent=4, maxlevel=2), '''\
+                        [
+                            1,
+                            'spam',
+                            {
+                                'eggs': True,
+                                'ham': [],
+                            },
+                        ]'''),
+                    (dict(indent=''), '''\
+                        [
+                        1,
+                        'spam',
+                        {
+                        'eggs': True,
+                        'ham': [],
+                        },
+                        ]'''),
+                    (dict(indent='-->'), '''\
+                        [
+                        -->1,
+                        -->'spam',
+                        -->{
+                        -->-->'eggs': True,
+                        -->-->'ham': [],
+                        -->},
+                        ]'''),
+                    (dict(indent='....'), '''\
+                        [
+                        ....1,
+                        ....'spam',
+                        ....{
+                        ........'eggs': True,
+                        ........'ham': [],
+                        ....},
+                        ]'''),
+                ),
+            },
+            {
+                'object': {
+                    1: 'two',
+                    b'three': [
+                        (4.5, 6.7),
+                        [set((8, 9)), frozenset((10, 11))],
+                    ],
+                },
+                'tests': (
+                    (dict(indent=None), '''\
+                        {1: 'two', b'three': [(4.5, 6.7), [{8, 9}, frozenset({10, 11})]]}'''),
+                    (dict(indent=False), '''\
+                        {
+                        1: 'two',
+                        b'three': [
+                        (
+                        4.5,
+                        6.7,
+                        ),
+                        [
+                        {
+                        8,
+                        9,
+                        },
+                        frozenset({
+                        10,
+                        11,
+                        }),
+                        ],
+                        ],
+                        }'''),
+                    (dict(indent=True), '''\
+                        {
+                         1: 'two',
+                         b'three': [
+                          (
+                           4.5,
+                           6.7,
+                          ),
+                          [
+                           {
+                            8,
+                            9,
+                           },
+                           frozenset({
+                            10,
+                            11,
+                           }),
+                          ],
+                         ],
+                        }'''),
+                    (dict(indent=0), '''\
+                        {
+                        1: 'two',
+                        b'three': [
+                        (
+                        4.5,
+                        6.7,
+                        ),
+                        [
+                        {
+                        8,
+                        9,
+                        },
+                        frozenset({
+                        10,
+                        11,
+                        }),
+                        ],
+                        ],
+                        }'''),
+                    (dict(indent=1), '''\
+                        {
+                         1: 'two',
+                         b'three': [
+                          (
+                           4.5,
+                           6.7,
+                          ),
+                          [
+                           {
+                            8,
+                            9,
+                           },
+                           frozenset({
+                            10,
+                            11,
+                           }),
+                          ],
+                         ],
+                        }'''),
+                    (dict(indent=4), '''\
+                        {
+                            1: 'two',
+                            b'three': [
+                                (
+                                    4.5,
+                                    6.7,
+                                ),
+                                [
+                                    {
+                                        8,
+                                        9,
+                                    },
+                                    frozenset({
+                                        10,
+                                        11,
+                                    }),
+                                ],
+                            ],
+                        }'''),
+                    (dict(indent=4, maxlevel=2), '''\
+                        {
+                            1: 'two',
+                            b'three': [
+                                (...),
+                                [...],
+                            ],
+                        }'''),
+                    (dict(indent=''), '''\
+                        {
+                        1: 'two',
+                        b'three': [
+                        (
+                        4.5,
+                        6.7,
+                        ),
+                        [
+                        {
+                        8,
+                        9,
+                        },
+                        frozenset({
+                        10,
+                        11,
+                        }),
+                        ],
+                        ],
+                        }'''),
+                    (dict(indent='-->'), '''\
+                        {
+                        -->1: 'two',
+                        -->b'three': [
+                        -->-->(
+                        -->-->-->4.5,
+                        -->-->-->6.7,
+                        -->-->),
+                        -->-->[
+                        -->-->-->{
+                        -->-->-->-->8,
+                        -->-->-->-->9,
+                        -->-->-->},
+                        -->-->-->frozenset({
+                        -->-->-->-->10,
+                        -->-->-->-->11,
+                        -->-->-->}),
+                        -->-->],
+                        -->],
+                        }'''),
+                    (dict(indent='....'), '''\
+                        {
+                        ....1: 'two',
+                        ....b'three': [
+                        ........(
+                        ............4.5,
+                        ............6.7,
+                        ........),
+                        ........[
+                        ............{
+                        ................8,
+                        ................9,
+                        ............},
+                        ............frozenset({
+                        ................10,
+                        ................11,
+                        ............}),
+                        ........],
+                        ....],
+                        }'''),
+                ),
+            },
+        ]
+        for test_case in test_cases:
+            with self.subTest(test_object=test_case['object']):
+                for repr_settings, expected_repr in test_case['tests']:
+                    with self.subTest(repr_settings=repr_settings):
+                        r = Repr()
+                        for attribute, value in repr_settings.items():
+                            setattr(r, attribute, value)
+                        resulting_repr = r.repr(test_case['object'])
+                        expected_repr = textwrap.dedent(expected_repr)
+                        self.assertEqual(resulting_repr, expected_repr)
+
+    def test_invalid_indent(self):
+        test_object = [1, 'spam', {'eggs': True, 'ham': []}]
+        test_cases = [
+            (-1, (ValueError, '[Nn]egative|[Pp]ositive')),
+            (-4, (ValueError, '[Nn]egative|[Pp]ositive')),
+            ((), (TypeError, None)),
+            ([], (TypeError, None)),
+            ((4,), (TypeError, None)),
+            ([4,], (TypeError, None)),
+            (object(), (TypeError, None)),
+        ]
+        for indent, (expected_error, expected_msg) in test_cases:
+            with self.subTest(indent=indent):
+                r = Repr()
+                r.indent = indent
+                expected_msg = expected_msg or f'{type(indent)}'
+                with self.assertRaisesRegex(expected_error, expected_msg):
+                    r.repr(test_object)
+
+    def test_shadowed_stdlib_array(self):
+        # Issue #113570: repr() should not be fooled by an array
+        class array:
+            def __repr__(self):
+                return "not array.array"
+
+        self.assertEqual(r(array()), "not array.array")
+
+    def test_shadowed_builtin(self):
+        # Issue #113570: repr() should not be fooled
+        # by a shadowed builtin function
+        class list:
+            def __repr__(self):
+                return "not builtins.list"
+
+        self.assertEqual(r(list()), "not builtins.list")
+
+    def test_custom_repr(self):
+        class MyRepr(Repr):
+
+            def repr_TextIOWrapper(self, obj, level):
+                if obj.name in {'<stdin>', '<stdout>', '<stderr>'}:
+                    return obj.name
+                return repr(obj)
+
+        aRepr = MyRepr()
+        self.assertEqual(aRepr.repr(sys.stdin), "<stdin>")
+
+    def test_custom_repr_class_with_spaces(self):
+        class TypeWithSpaces:
+            pass
+
+        t = TypeWithSpaces()
+        type(t).__name__ = "type with spaces"
+        self.assertEqual(type(t).__name__, "type with spaces")
+
+        class MyRepr(Repr):
+            def repr_type_with_spaces(self, obj, level):
+                return "Type With Spaces"
+
+
+        aRepr = MyRepr()
+        self.assertEqual(aRepr.repr(t), "Type With Spaces")
+
 def write_file(path, text):
     with open(path, 'w', encoding='ASCII') as fp:
         fp.write(text)
@@ -408,5 +817,27 @@ class TestRecursiveRepr(unittest.TestCase):
         for name in assigned:
             self.assertIs(getattr(wrapper, name), getattr(wrapped, name))
 
+    def test__wrapped__(self):
+        class X:
+            def __repr__(self):
+                return 'X()'
+            f = __repr__ # save reference to check it later
+            __repr__ = recursive_repr()(__repr__)
+
+        self.assertIs(X.f, X.__repr__.__wrapped__)
+
+    # TODO: RUSTPYTHON: AttributeError: 'TypeVar' object has no attribute '__name__'
+    @unittest.expectedFailure
+    def test__type_params__(self):
+        class My:
+            @recursive_repr()
+            def __repr__[T: str](self, default: T = '') -> str:
+                return default
+
+        type_params = My().__repr__.__type_params__
+        self.assertEqual(len(type_params), 1)
+        self.assertEqual(type_params[0].__name__, 'T')
+        self.assertEqual(type_params[0].__bound__, str)
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_tools/__init__.py

@@ -0,0 +1,43 @@
+"""Support functions for testing scripts in the Tools directory."""
+import contextlib
+import importlib
+import os.path
+import unittest
+from test import support
+from test.support import import_helper
+
+
+if not support.has_subprocess_support:
+    raise unittest.SkipTest("test module requires subprocess")
+
+
+basepath = os.path.normpath(
+        os.path.dirname(                 # <src/install dir>
+            os.path.dirname(                # Lib
+                os.path.dirname(                # test
+                    os.path.dirname(__file__)))))    # test_tools
+
+toolsdir = os.path.join(basepath, 'Tools')
+scriptsdir = os.path.join(toolsdir, 'scripts')
+
+def skip_if_missing(tool=None):
+    if tool:
+        tooldir = os.path.join(toolsdir, tool)
+    else:
+        tool = 'scripts'
+        tooldir = scriptsdir
+    if not os.path.isdir(tooldir):
+        raise unittest.SkipTest(f'{tool} directory could not be found')
+
+@contextlib.contextmanager
+def imports_under_tool(name, *subdirs):
+    tooldir = os.path.join(toolsdir, name, *subdirs)
+    with import_helper.DirsOnSysPath(tooldir) as cm:
+        yield cm
+
+def import_tool(toolname):
+    with import_helper.DirsOnSysPath(scriptsdir):
+        return importlib.import_module(toolname)
+
+def load_tests(*args):
+    return support.load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_tools/__main__.py

@@ -0,0 +1,4 @@
+from test.test_tools import load_tests
+import unittest
+
+unittest.main()

Lib/test/test_tools/i18n_data/ascii-escapes.pot

@@ -0,0 +1,45 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: escapes.py:5
+msgid ""
+"\"\t\n"
+"\r\\"
+msgstr ""
+
+#: escapes.py:8
+msgid ""
+"\000\001\002\003\004\005\006\007\010\t\n"
+"\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037"
+msgstr ""
+
+#: escapes.py:13
+msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
+msgstr ""
+
+#: escapes.py:17
+msgid "\177"
+msgstr ""
+
+#: escapes.py:20
+msgid "€   ÿ"
+msgstr ""
+
+#: escapes.py:23
+msgid "α ㄱ 𓂀"
+msgstr ""
+

Lib/test/test_tools/i18n_data/docstrings.pot

@@ -0,0 +1,40 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: docstrings.py:7
+#, docstring
+msgid ""
+msgstr ""
+
+#: docstrings.py:18
+#, docstring
+msgid ""
+"multiline\n"
+"    docstring\n"
+"    "
+msgstr ""
+
+#: docstrings.py:25
+#, docstring
+msgid "docstring1"
+msgstr ""
+
+#: docstrings.py:30
+#, docstring
+msgid "Hello, {}!"
+msgstr ""
+

Lib/test/test_tools/i18n_data/docstrings.py

@@ -0,0 +1,41 @@
+# Test docstring extraction
+from gettext import gettext as _
+
+
+# Empty docstring
+def test(x):
+    """"""
+
+
+# Leading empty line
+def test2(x):
+
+    """docstring"""  # XXX This should be extracted but isn't.
+
+
+# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`.
+def test3(x):
+    """multiline
+    docstring
+    """
+
+
+# Multiple docstrings - only the first should be extracted
+def test4(x):
+    """docstring1"""
+    """docstring2"""
+
+
+def test5(x):
+    """Hello, {}!""".format("world!")  # XXX This should not be extracted.
+
+
+# Nested docstrings
+def test6(x):
+    def inner(y):
+        """nested docstring"""  # XXX This should be extracted but isn't.
+
+
+class Outer:
+    class Inner:
+        "nested class docstring"  # XXX This should be extracted but isn't.

Lib/test/test_tools/i18n_data/escapes.pot

@@ -0,0 +1,45 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: escapes.py:5
+msgid ""
+"\"\t\n"
+"\r\\"
+msgstr ""
+
+#: escapes.py:8
+msgid ""
+"\000\001\002\003\004\005\006\007\010\t\n"
+"\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037"
+msgstr ""
+
+#: escapes.py:13
+msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
+msgstr ""
+
+#: escapes.py:17
+msgid "\177"
+msgstr ""
+
+#: escapes.py:20
+msgid "\302\200 \302\240 \303\277"
+msgstr ""
+
+#: escapes.py:23
+msgid "\316\261 \343\204\261 \360\223\202\200"
+msgstr ""
+

Lib/test/test_tools/i18n_data/escapes.py

@@ -0,0 +1,23 @@
+import gettext as _
+
+
+# Special characters that are always escaped in the POT file
+_('"\t\n\r\\')
+
+# All ascii characters 0-31
+_('\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n'
+  '\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15'
+  '\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f')
+
+# All ascii characters 32-126
+_(' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+  '[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
+
+# ascii char 127
+_('\x7f')
+
+# some characters in the 128-255 range
+_('\x80 \xa0 ÿ')
+
+# some characters >= 256 encoded as 2, 3 and 4 bytes, respectively
+_('α ㄱ 𓂀')

Lib/test/test_tools/i18n_data/fileloc.pot

@@ -0,0 +1,35 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: fileloc.py:5 fileloc.py:6
+msgid "foo"
+msgstr ""
+
+#: fileloc.py:9
+msgid "bar"
+msgstr ""
+
+#: fileloc.py:14 fileloc.py:18
+#, docstring
+msgid "docstring"
+msgstr ""
+
+#: fileloc.py:22 fileloc.py:26
+#, docstring
+msgid "baz"
+msgstr ""
+

Lib/test/test_tools/i18n_data/fileloc.py

@@ -0,0 +1,26 @@
+# Test file locations
+from gettext import gettext as _
+
+# Duplicate strings
+_('foo')
+_('foo')
+
+# Duplicate strings on the same line should only add one location to the output
+_('bar'), _('bar')
+
+
+# Duplicate docstrings
+class A:
+    """docstring"""
+
+
+def f():
+    """docstring"""
+
+
+# Duplicate message and docstring
+_('baz')
+
+
+def g():
+    """baz"""

Lib/test/test_tools/i18n_data/messages.pot

@@ -0,0 +1,67 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: messages.py:5
+msgid ""
+msgstr ""
+
+#: messages.py:8 messages.py:9
+msgid "parentheses"
+msgstr ""
+
+#: messages.py:12
+msgid "Hello, world!"
+msgstr ""
+
+#: messages.py:15
+msgid ""
+"Hello,\n"
+"    multiline!\n"
+msgstr ""
+
+#: messages.py:29
+msgid "Hello, {}!"
+msgstr ""
+
+#: messages.py:33
+msgid "1"
+msgstr ""
+
+#: messages.py:33
+msgid "2"
+msgstr ""
+
+#: messages.py:34 messages.py:35
+msgid "A"
+msgstr ""
+
+#: messages.py:34 messages.py:35
+msgid "B"
+msgstr ""
+
+#: messages.py:36
+msgid "set"
+msgstr ""
+
+#: messages.py:42
+msgid "nested string"
+msgstr ""
+
+#: messages.py:47
+msgid "baz"
+msgstr ""
+

Lib/test/test_tools/i18n_data/messages.py

@@ -0,0 +1,64 @@
+# Test message extraction
+from gettext import gettext as _
+
+# Empty string
+_("")
+
+# Extra parentheses
+(_("parentheses"))
+((_("parentheses")))
+
+# Multiline strings
+_("Hello, "
+  "world!")
+
+_("""Hello,
+    multiline!
+""")
+
+# Invalid arguments
+_()
+_(None)
+_(1)
+_(False)
+_(x="kwargs are not allowed")
+_("foo", "bar")
+_("something", x="something else")
+
+# .format()
+_("Hello, {}!").format("world")  # valid
+_("Hello, {}!".format("world"))  # invalid
+
+# Nested structures
+_("1"), _("2")
+arr = [_("A"), _("B")]
+obj = {'a': _("A"), 'b': _("B")}
+{{{_('set')}}}
+
+
+# Nested functions and classes
+def test():
+    _("nested string")  # XXX This should be extracted but isn't.
+    [_("nested string")]
+
+
+class Foo:
+    def bar(self):
+        return _("baz")
+
+
+def bar(x=_('default value')):  # XXX This should be extracted but isn't.
+    pass
+
+
+def baz(x=[_('default value')]):  # XXX This should be extracted but isn't.
+    pass
+
+
+# Shadowing _()
+def _(x):
+    pass
+
+
+def _(x="don't extract me"):
+    pass

Lib/test/test_tools/msgfmt_data/fuzzy.json

@@ -0,0 +1 @@
+[]

Lib/test/test_tools/msgfmt_data/fuzzy.po

@@ -0,0 +1,23 @@
+# Fuzzy translations are not written to the .mo file.
+#, fuzzy
+msgid "foo"
+msgstr "bar"
+
+# comment
+#, fuzzy
+msgctxt "abc"
+msgid "foo"
+msgstr "bar"
+
+#, fuzzy
+# comment
+msgctxt "xyz"
+msgid "foo"
+msgstr "bar"
+
+#, fuzzy
+msgctxt "abc"
+msgid "One email sent."
+msgid_plural "%d emails sent."
+msgstr[0] "One email sent."
+msgstr[1] "%d emails sent."

Lib/test/test_tools/msgfmt_data/general.json

@@ -0,0 +1,58 @@
+[
+    [
+        "",
+        "Project-Id-Version: PACKAGE VERSION\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME <EMAIL@ADDRESS>\nLanguage-Team: LANGUAGE <LL@li.org>\nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\n"
+    ],
+    [
+        "\n newlines \n",
+        "\n translated \n"
+    ],
+    [
+        "\"escapes\"",
+        "\"translated\""
+    ],
+    [
+        "Multilinestring",
+        "Multilinetranslation"
+    ],
+    [
+        "abc\u0004foo",
+        "bar"
+    ],
+    [
+        "bar",
+        "baz"
+    ],
+    [
+        "xyz\u0004foo",
+        "bar"
+    ],
+    [
+        [
+            "One email sent.",
+            0
+        ],
+        "One email sent."
+    ],
+    [
+        [
+            "One email sent.",
+            1
+        ],
+        "%d emails sent."
+    ],
+    [
+        [
+            "abc\u0004One email sent.",
+            0
+        ],
+        "One email sent."
+    ],
+    [
+        [
+            "abc\u0004One email sent.",
+            1
+        ],
+        "%d emails sent."
+    ]
+]

Lib/test/test_tools/msgfmt_data/general.po

@@ -0,0 +1,47 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2024-10-26 18:06+0200\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+msgid "foo"
+msgstr ""
+
+msgid "bar"
+msgstr "baz"
+
+msgctxt "abc"
+msgid "foo"
+msgstr "bar"
+
+# comment
+msgctxt "xyz"
+msgid "foo"
+msgstr "bar"
+
+msgid "Multiline"
+"string"
+msgstr "Multiline"
+"translation"
+
+msgid "\"escapes\""
+msgstr "\"translated\""
+
+msgid "\n newlines \n"
+msgstr "\n translated \n"
+
+msgid "One email sent."
+msgid_plural "%d emails sent."
+msgstr[0] "One email sent."
+msgstr[1] "%d emails sent."
+
+msgctxt "abc"
+msgid "One email sent."
+msgid_plural "%d emails sent."
+msgstr[0] "One email sent."
+msgstr[1] "%d emails sent."

Lib/test/test_tools/test_freeze.py

@@ -0,0 +1,37 @@
+"""Sanity-check tests for the "freeze" tool."""
+
+import sys
+import textwrap
+import unittest
+
+from test import support
+from test.support import os_helper
+from test.test_tools import imports_under_tool, skip_if_missing
+
+skip_if_missing('freeze')
+with imports_under_tool('freeze', 'test'):
+    import freeze as helper
+
+@support.requires_zlib()
+@unittest.skipIf(sys.platform.startswith('win'), 'not supported on Windows')
+@unittest.skipIf(sys.platform == 'darwin' and sys._framework,
+        'not supported for frameworks builds on macOS')
+@support.skip_if_buildbot('not all buildbots have enough space')
+# gh-103053: Skip test if Python is built with Profile Guided Optimization
+# (PGO), since the test is just too slow in this case.
+@unittest.skipIf(support.check_cflags_pgo(),
+                 'test is too slow with PGO')
+class TestFreeze(unittest.TestCase):
+
+    @support.requires_resource('cpu') # Building Python is slow
+    def test_freeze_simple_script(self):
+        script = textwrap.dedent("""
+            import sys
+            print('running...')
+            sys.exit(0)
+            """)
+        with os_helper.temp_dir() as outdir:
+            outdir, scriptfile, python = helper.prepare(script, outdir)
+            executable = helper.freeze(python, scriptfile, outdir)
+            text = helper.run(executable)
+        self.assertEqual(text, 'running...')

Lib/test/test_tools/test_i18n.py

@@ -0,0 +1,444 @@
+"""Tests to cover the Tools/i18n package"""
+
+import os
+import re
+import sys
+import unittest
+from textwrap import dedent
+from pathlib import Path
+
+from test.support.script_helper import assert_python_ok
+from test.test_tools import skip_if_missing, toolsdir
+from test.support.os_helper import temp_cwd, temp_dir
+
+
+skip_if_missing()
+
+DATA_DIR = Path(__file__).resolve().parent / 'i18n_data'
+
+
+def normalize_POT_file(pot):
+    """Normalize the POT creation timestamp, charset and
+    file locations to make the POT file easier to compare.
+
+    """
+    # Normalize the creation date.
+    date_pattern = re.compile(r'"POT-Creation-Date: .+?\\n"')
+    header = r'"POT-Creation-Date: 2000-01-01 00:00+0000\\n"'
+    pot = re.sub(date_pattern, header, pot)
+
+    # Normalize charset to UTF-8 (currently there's no way to specify the output charset).
+    charset_pattern = re.compile(r'"Content-Type: text/plain; charset=.+?\\n"')
+    charset = r'"Content-Type: text/plain; charset=UTF-8\\n"'
+    pot = re.sub(charset_pattern, charset, pot)
+
+    # Normalize file location path separators in case this test is
+    # running on Windows (which uses '\').
+    fileloc_pattern = re.compile(r'#:.+')
+
+    def replace(match):
+        return match[0].replace(os.sep, "/")
+    pot = re.sub(fileloc_pattern, replace, pot)
+    return pot
+
+
+class Test_pygettext(unittest.TestCase):
+    """Tests for the pygettext.py tool"""
+
+    script = Path(toolsdir, 'i18n', 'pygettext.py')
+
+    def get_header(self, data):
+        """ utility: return the header of a .po file as a dictionary """
+        headers = {}
+        for line in data.split('\n'):
+            if not line or line.startswith(('#', 'msgid', 'msgstr')):
+                continue
+            line = line.strip('"')
+            key, val = line.split(':', 1)
+            headers[key] = val.strip()
+        return headers
+
+    def get_msgids(self, data):
+        """ utility: return all msgids in .po file as a list of strings """
+        msgids = []
+        reading_msgid = False
+        cur_msgid = []
+        for line in data.split('\n'):
+            if reading_msgid:
+                if line.startswith('"'):
+                    cur_msgid.append(line.strip('"'))
+                else:
+                    msgids.append('\n'.join(cur_msgid))
+                    cur_msgid = []
+                    reading_msgid = False
+                    continue
+            if line.startswith('msgid '):
+                line = line[len('msgid '):]
+                cur_msgid.append(line.strip('"'))
+                reading_msgid = True
+        else:
+            if reading_msgid:
+                msgids.append('\n'.join(cur_msgid))
+
+        return msgids
+
+    def assert_POT_equal(self, expected, actual):
+        """Check if two POT files are equal"""
+        self.maxDiff = None
+        self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual))
+
+    def extract_from_str(self, module_content, *, args=(), strict=True):
+        """Return all msgids extracted from module_content."""
+        filename = 'test.py'
+        with temp_cwd(None):
+            with open(filename, 'w', encoding='utf-8') as fp:
+                fp.write(module_content)
+            res = assert_python_ok('-Xutf8', self.script, *args, filename)
+            if strict:
+                self.assertEqual(res.err, b'')
+            with open('messages.pot', encoding='utf-8') as fp:
+                data = fp.read()
+        return self.get_msgids(data)
+
+    def extract_docstrings_from_str(self, module_content):
+        """Return all docstrings extracted from module_content."""
+        return self.extract_from_str(module_content, args=('--docstrings',), strict=False)
+
+    def test_header(self):
+        """Make sure the required fields are in the header, according to:
+           http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry
+        """
+        with temp_cwd(None) as cwd:
+            assert_python_ok('-Xutf8', self.script)
+            with open('messages.pot', encoding='utf-8') as fp:
+                data = fp.read()
+            header = self.get_header(data)
+
+            self.assertIn("Project-Id-Version", header)
+            self.assertIn("POT-Creation-Date", header)
+            self.assertIn("PO-Revision-Date", header)
+            self.assertIn("Last-Translator", header)
+            self.assertIn("Language-Team", header)
+            self.assertIn("MIME-Version", header)
+            self.assertIn("Content-Type", header)
+            self.assertIn("Content-Transfer-Encoding", header)
+            self.assertIn("Generated-By", header)
+
+            # not clear if these should be required in POT (template) files
+            #self.assertIn("Report-Msgid-Bugs-To", header)
+            #self.assertIn("Language", header)
+
+            #"Plural-Forms" is optional
+
+    @unittest.skipIf(sys.platform.startswith('aix'),
+                     'bpo-29972: broken test on AIX')
+    def test_POT_Creation_Date(self):
+        """ Match the date format from xgettext for POT-Creation-Date """
+        from datetime import datetime
+        with temp_cwd(None) as cwd:
+            assert_python_ok('-Xutf8', self.script)
+            with open('messages.pot', encoding='utf-8') as fp:
+                data = fp.read()
+            header = self.get_header(data)
+            creationDate = header['POT-Creation-Date']
+
+            # peel off the escaped newline at the end of string
+            if creationDate.endswith('\\n'):
+                creationDate = creationDate[:-len('\\n')]
+
+            # This will raise if the date format does not exactly match.
+            datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z')
+
+    def test_funcdocstring(self):
+        for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
+            with self.subTest(doc):
+                msgids = self.extract_docstrings_from_str(dedent('''\
+                def foo(bar):
+                    %s
+                ''' % doc))
+                self.assertIn('doc', msgids)
+
+    def test_funcdocstring_bytes(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        def foo(bar):
+            b"""doc"""
+        '''))
+        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
+
+    def test_funcdocstring_fstring(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        def foo(bar):
+            f"""doc"""
+        '''))
+        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
+
+    def test_classdocstring(self):
+        for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
+            with self.subTest(doc):
+                msgids = self.extract_docstrings_from_str(dedent('''\
+                class C:
+                    %s
+                ''' % doc))
+                self.assertIn('doc', msgids)
+
+    def test_classdocstring_bytes(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        class C:
+            b"""doc"""
+        '''))
+        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
+
+    def test_classdocstring_fstring(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        class C:
+            f"""doc"""
+        '''))
+        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
+
+    def test_moduledocstring(self):
+        for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
+            with self.subTest(doc):
+                msgids = self.extract_docstrings_from_str(dedent('''\
+                %s
+                ''' % doc))
+                self.assertIn('doc', msgids)
+
+    def test_moduledocstring_bytes(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        b"""doc"""
+        '''))
+        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
+
+    def test_moduledocstring_fstring(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"""doc"""
+        '''))
+        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
+
+    def test_msgid(self):
+        msgids = self.extract_docstrings_from_str(
+                '''_("""doc""" r'str' u"ing")''')
+        self.assertIn('docstring', msgids)
+
+    def test_msgid_bytes(self):
+        msgids = self.extract_docstrings_from_str('_(b"""doc""")')
+        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
+
+    def test_msgid_fstring(self):
+        msgids = self.extract_docstrings_from_str('_(f"""doc""")')
+        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
+
+    def test_funcdocstring_annotated_args(self):
+        """ Test docstrings for functions with annotated args """
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        def foo(bar: str):
+            """doc"""
+        '''))
+        self.assertIn('doc', msgids)
+
+    def test_funcdocstring_annotated_return(self):
+        """ Test docstrings for functions with annotated return type """
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        def foo(bar) -> str:
+            """doc"""
+        '''))
+        self.assertIn('doc', msgids)
+
+    def test_funcdocstring_defvalue_args(self):
+        """ Test docstring for functions with default arg values """
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        def foo(bar=()):
+            """doc"""
+        '''))
+        self.assertIn('doc', msgids)
+
+    def test_funcdocstring_multiple_funcs(self):
+        """ Test docstring extraction for multiple functions combining
+        annotated args, annotated return types and default arg values
+        """
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        def foo1(bar: tuple=()) -> str:
+            """doc1"""
+
+        def foo2(bar: List[1:2]) -> (lambda x: x):
+            """doc2"""
+
+        def foo3(bar: 'func'=lambda x: x) -> {1: 2}:
+            """doc3"""
+        '''))
+        self.assertIn('doc1', msgids)
+        self.assertIn('doc2', msgids)
+        self.assertIn('doc3', msgids)
+
+    def test_classdocstring_early_colon(self):
+        """ Test docstring extraction for a class with colons occurring within
+        the parentheses.
+        """
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        class D(L[1:2], F({1: 2}), metaclass=M(lambda x: x)):
+            """doc"""
+        '''))
+        self.assertIn('doc', msgids)
+
+    def test_calls_in_fstrings(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo bar')}"
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_raw(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        rf"{_('foo bar')}"
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_nested(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"""{f'{_("foo bar")}'}"""
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_attribute(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{obj._('foo bar')}"
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_with_call_on_call(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{type(str)('foo bar')}"
+        '''))
+        self.assertNotIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_with_format(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo {bar}').format(bar='baz')}"
+        '''))
+        self.assertIn('foo {bar}', msgids)
+
+    def test_calls_in_fstrings_with_wrong_input_1(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_(f'foo {bar}')}"
+        '''))
+        self.assertFalse([msgid for msgid in msgids if 'foo {bar}' in msgid])
+
+    def test_calls_in_fstrings_with_wrong_input_2(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_(1)}"
+        '''))
+        self.assertNotIn(1, msgids)
+
+    def test_calls_in_fstring_with_multiple_args(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo', 'bar')}"
+        '''))
+        self.assertNotIn('foo', msgids)
+        self.assertNotIn('bar', msgids)
+
+    def test_calls_in_fstring_with_keyword_args(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo', bar='baz')}"
+        '''))
+        self.assertNotIn('foo', msgids)
+        self.assertNotIn('bar', msgids)
+        self.assertNotIn('baz', msgids)
+
+    def test_calls_in_fstring_with_partially_wrong_expression(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_(f'foo') + _('bar')}"
+        '''))
+        self.assertNotIn('foo', msgids)
+        self.assertIn('bar', msgids)
+
+    def test_function_and_class_names(self):
+        """Test that function and class names are not mistakenly extracted."""
+        msgids = self.extract_from_str(dedent('''\
+        def _(x):
+            pass
+
+        def _(x="foo"):
+            pass
+
+        async def _(x):
+            pass
+
+        class _(object):
+            pass
+        '''))
+        self.assertEqual(msgids, [''])
+
+    def test_pygettext_output(self):
+        """Test that the pygettext output exactly matches snapshots."""
+        for input_file, output_file, output in extract_from_snapshots():
+            with self.subTest(input_file=input_file):
+                expected = output_file.read_text(encoding='utf-8')
+                self.assert_POT_equal(expected, output)
+
+    def test_files_list(self):
+        """Make sure the directories are inspected for source files
+           bpo-31920
+        """
+        text1 = 'Text to translate1'
+        text2 = 'Text to translate2'
+        text3 = 'Text to ignore'
+        with temp_cwd(None), temp_dir(None) as sdir:
+            pymod = Path(sdir, 'pypkg', 'pymod.py')
+            pymod.parent.mkdir()
+            pymod.write_text(f'_({text1!r})', encoding='utf-8')
+
+            pymod2 = Path(sdir, 'pkg.py', 'pymod2.py')
+            pymod2.parent.mkdir()
+            pymod2.write_text(f'_({text2!r})', encoding='utf-8')
+
+            pymod3 = Path(sdir, 'CVS', 'pymod3.py')
+            pymod3.parent.mkdir()
+            pymod3.write_text(f'_({text3!r})', encoding='utf-8')
+
+            assert_python_ok('-Xutf8', self.script, sdir)
+            data = Path('messages.pot').read_text(encoding='utf-8')
+            self.assertIn(f'msgid "{text1}"', data)
+            self.assertIn(f'msgid "{text2}"', data)
+            self.assertNotIn(text3, data)
+
+
+def extract_from_snapshots():
+    snapshots = {
+        'messages.py': ('--docstrings',),
+        'fileloc.py': ('--docstrings',),
+        'docstrings.py': ('--docstrings',),
+        # == Test character escaping
+        # Escape ascii and unicode:
+        'escapes.py': ('--escape',),
+        # Escape only ascii and let unicode pass through:
+        ('escapes.py', 'ascii-escapes.pot'): (),
+    }
+
+    for filename, args in snapshots.items():
+        if isinstance(filename, tuple):
+            filename, output_file = filename
+            output_file = DATA_DIR / output_file
+            input_file = DATA_DIR / filename
+        else:
+            input_file = DATA_DIR / filename
+            output_file = input_file.with_suffix('.pot')
+        contents = input_file.read_bytes()
+        with temp_cwd(None):
+            Path(input_file.name).write_bytes(contents)
+            assert_python_ok('-Xutf8', Test_pygettext.script, *args,
+                             input_file.name)
+            yield (input_file, output_file,
+                   Path('messages.pot').read_text(encoding='utf-8'))
+
+
+def update_POT_snapshots():
+    for _, output_file, output in extract_from_snapshots():
+        output = normalize_POT_file(output)
+        output_file.write_text(output, encoding='utf-8')
+
+
+if __name__ == '__main__':
+    # To regenerate POT files
+    if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
+        update_POT_snapshots()
+        sys.exit(0)
+    unittest.main()

Lib/test/test_tools/test_makefile.py

@@ -0,0 +1,81 @@
+"""
+Tests for `Makefile`.
+"""
+
+import os
+import unittest
+from test import support
+import sysconfig
+
+MAKEFILE = sysconfig.get_makefile_filename()
+
+if not support.check_impl_detail(cpython=True):
+    raise unittest.SkipTest('cpython only')
+if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE):
+    raise unittest.SkipTest('Makefile could not be found')
+
+
+class TestMakefile(unittest.TestCase):
+    def list_test_dirs(self):
+        result = []
+        found_testsubdirs = False
+        with open(MAKEFILE, 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.startswith('TESTSUBDIRS='):
+                    found_testsubdirs = True
+                    result.append(
+                        line.removeprefix('TESTSUBDIRS=').replace(
+                            '\\', '',
+                        ).strip(),
+                    )
+                    continue
+                if found_testsubdirs:
+                    if '\t' not in line:
+                        break
+                    result.append(line.replace('\\', '').strip())
+        return result
+
+    @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules")
+    def test_makefile_test_folders(self):
+        test_dirs = self.list_test_dirs()
+        idle_test = 'idlelib/idle_test'
+        self.assertIn(idle_test, test_dirs)
+
+        used = set([idle_test])
+        for dirpath, dirs, files in os.walk(support.TEST_HOME_DIR):
+            dirname = os.path.basename(dirpath)
+            # Skip temporary dirs:
+            if dirname == '__pycache__' or dirname.startswith('.'):
+                dirs.clear()  # do not process subfolders
+                continue
+            # Skip empty dirs:
+            if not dirs and not files:
+                continue
+            # Skip dirs with hidden-only files:
+            if files and all(
+                filename.startswith('.') or filename == '__pycache__'
+                for filename in files
+            ):
+                continue
+
+            relpath = os.path.relpath(dirpath, support.STDLIB_DIR)
+            with self.subTest(relpath=relpath):
+                self.assertIn(
+                    relpath,
+                    test_dirs,
+                    msg=(
+                        f"{relpath!r} is not included in the Makefile's list "
+                        "of test directories to install"
+                    )
+                )
+                used.add(relpath)
+
+        # Don't check the wheel dir when Python is built --with-wheel-pkg-dir
+        if sysconfig.get_config_var('WHEEL_PKG_DIR'):
+            test_dirs.remove('test/wheeldata')
+            used.discard('test/wheeldata')
+
+        # Check that there are no extra entries:
+        unique_test_dirs = set(test_dirs)
+        self.assertSetEqual(unique_test_dirs, used)
+        self.assertEqual(len(test_dirs), len(unique_test_dirs))

Lib/test/test_tools/test_makeunicodedata.py

@@ -0,0 +1,122 @@
+import unittest
+from test.test_tools import skip_if_missing, imports_under_tool
+from test import support
+from test.support.hypothesis_helper import hypothesis
+
+st = hypothesis.strategies
+given = hypothesis.given
+example = hypothesis.example
+
+
+skip_if_missing("unicode")
+with imports_under_tool("unicode"):
+    from dawg import Dawg, build_compression_dawg, lookup, inverse_lookup
+
+
+@st.composite
+def char_name_db(draw, min_length=1, max_length=30):
+    m = draw(st.integers(min_value=min_length, max_value=max_length))
+    names = draw(
+        st.sets(st.text("abcd", min_size=1, max_size=10), min_size=m, max_size=m)
+    )
+    characters = draw(st.sets(st.characters(), min_size=m, max_size=m))
+    return list(zip(names, characters))
+
+
+class TestDawg(unittest.TestCase):
+    """Tests for the directed acyclic word graph data structure that is used
+    to store the unicode character names in unicodedata. Tests ported from PyPy
+    """
+
+    def test_dawg_direct_simple(self):
+        dawg = Dawg()
+        dawg.insert("a", -4)
+        dawg.insert("c", -2)
+        dawg.insert("cat", -1)
+        dawg.insert("catarr", 0)
+        dawg.insert("catnip", 1)
+        dawg.insert("zcatnip", 5)
+        packed, data, inverse = dawg.finish()
+
+        self.assertEqual(lookup(packed, data, b"a"), -4)
+        self.assertEqual(lookup(packed, data, b"c"), -2)
+        self.assertEqual(lookup(packed, data, b"cat"), -1)
+        self.assertEqual(lookup(packed, data, b"catarr"), 0)
+        self.assertEqual(lookup(packed, data, b"catnip"), 1)
+        self.assertEqual(lookup(packed, data, b"zcatnip"), 5)
+        self.assertRaises(KeyError, lookup, packed, data, b"b")
+        self.assertRaises(KeyError, lookup, packed, data, b"catni")
+        self.assertRaises(KeyError, lookup, packed, data, b"catnipp")
+
+        self.assertEqual(inverse_lookup(packed, inverse, -4), b"a")
+        self.assertEqual(inverse_lookup(packed, inverse, -2), b"c")
+        self.assertEqual(inverse_lookup(packed, inverse, -1), b"cat")
+        self.assertEqual(inverse_lookup(packed, inverse, 0), b"catarr")
+        self.assertEqual(inverse_lookup(packed, inverse, 1), b"catnip")
+        self.assertEqual(inverse_lookup(packed, inverse, 5), b"zcatnip")
+        self.assertRaises(KeyError, inverse_lookup, packed, inverse, 12)
+
+    def test_forbid_empty_dawg(self):
+        dawg = Dawg()
+        self.assertRaises(ValueError, dawg.finish)
+
+    @given(char_name_db())
+    @example([("abc", "a"), ("abd", "b")])
+    @example(
+        [
+            ("bab", "1"),
+            ("a", ":"),
+            ("ad", "@"),
+            ("b", "<"),
+            ("aacc", "?"),
+            ("dab", "D"),
+            ("aa", "0"),
+            ("ab", "F"),
+            ("aaa", "7"),
+            ("cbd", "="),
+            ("abad", ";"),
+            ("ac", "B"),
+            ("abb", "4"),
+            ("bb", "2"),
+            ("aab", "9"),
+            ("caaaaba", "E"),
+            ("ca", ">"),
+            ("bbaaa", "5"),
+            ("d", "3"),
+            ("baac", "8"),
+            ("c", "6"),
+            ("ba", "A"),
+        ]
+    )
+    @example(
+        [
+            ("bcdac", "9"),
+            ("acc", "g"),
+            ("d", "d"),
+            ("daabdda", "0"),
+            ("aba", ";"),
+            ("c", "6"),
+            ("aa", "7"),
+            ("abbd", "c"),
+            ("badbd", "?"),
+            ("bbd", "f"),
+            ("cc", "@"),
+            ("bb", "8"),
+            ("daca", ">"),
+            ("ba", ":"),
+            ("baac", "3"),
+            ("dbdddac", "a"),
+            ("a", "2"),
+            ("cabd", "b"),
+            ("b", "="),
+            ("abd", "4"),
+            ("adcbd", "5"),
+            ("abc", "e"),
+            ("ab", "1"),
+        ]
+    )
+    def test_dawg(self, data):
+        # suppress debug prints
+        with support.captured_stdout() as output:
+            # it's enough to build it, building will also check the result
+            build_compression_dawg(data)

Lib/test/test_tools/test_msgfmt.py

@@ -0,0 +1,159 @@
+"""Tests for the Tools/i18n/msgfmt.py tool."""
+
+import json
+import sys
+import unittest
+from gettext import GNUTranslations
+from pathlib import Path
+
+from test.support.os_helper import temp_cwd
+from test.support.script_helper import assert_python_failure, assert_python_ok
+from test.test_tools import skip_if_missing, toolsdir
+
+
+skip_if_missing('i18n')
+
+data_dir = (Path(__file__).parent / 'msgfmt_data').resolve()
+script_dir = Path(toolsdir) / 'i18n'
+msgfmt = script_dir / 'msgfmt.py'
+
+
+def compile_messages(po_file, mo_file):
+    assert_python_ok(msgfmt, '-o', mo_file, po_file)
+
+
+class CompilationTest(unittest.TestCase):
+
+    def test_compilation(self):
+        self.maxDiff = None
+        with temp_cwd():
+            for po_file in data_dir.glob('*.po'):
+                with self.subTest(po_file=po_file):
+                    mo_file = po_file.with_suffix('.mo')
+                    with open(mo_file, 'rb') as f:
+                        expected = GNUTranslations(f)
+
+                    tmp_mo_file = mo_file.name
+                    compile_messages(po_file, tmp_mo_file)
+                    with open(tmp_mo_file, 'rb') as f:
+                        actual = GNUTranslations(f)
+
+                    self.assertDictEqual(actual._catalog, expected._catalog)
+
+    def test_translations(self):
+        with open(data_dir / 'general.mo', 'rb') as f:
+            t = GNUTranslations(f)
+
+        self.assertEqual(t.gettext('foo'), 'foo')
+        self.assertEqual(t.gettext('bar'), 'baz')
+        self.assertEqual(t.pgettext('abc', 'foo'), 'bar')
+        self.assertEqual(t.pgettext('xyz', 'foo'), 'bar')
+        self.assertEqual(t.gettext('Multilinestring'), 'Multilinetranslation')
+        self.assertEqual(t.gettext('"escapes"'), '"translated"')
+        self.assertEqual(t.gettext('\n newlines \n'), '\n translated \n')
+        self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 1),
+                         'One email sent.')
+        self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 2),
+                         '%d emails sent.')
+        self.assertEqual(t.npgettext('abc', 'One email sent.',
+                                     '%d emails sent.', 1),
+                         'One email sent.')
+        self.assertEqual(t.npgettext('abc', 'One email sent.',
+                                     '%d emails sent.', 2),
+                         '%d emails sent.')
+
+    def test_invalid_msgid_plural(self):
+        with temp_cwd():
+            Path('invalid.po').write_text('''\
+msgid_plural "plural"
+msgstr[0] "singular"
+''')
+
+            res = assert_python_failure(msgfmt, 'invalid.po')
+            err = res.err.decode('utf-8')
+            self.assertIn('msgid_plural not preceded by msgid', err)
+
+    def test_plural_without_msgid_plural(self):
+        with temp_cwd():
+            Path('invalid.po').write_text('''\
+msgid "foo"
+msgstr[0] "bar"
+''')
+
+            res = assert_python_failure(msgfmt, 'invalid.po')
+            err = res.err.decode('utf-8')
+            self.assertIn('plural without msgid_plural', err)
+
+    def test_indexed_msgstr_without_msgid_plural(self):
+        with temp_cwd():
+            Path('invalid.po').write_text('''\
+msgid "foo"
+msgid_plural "foos"
+msgstr "bar"
+''')
+
+            res = assert_python_failure(msgfmt, 'invalid.po')
+            err = res.err.decode('utf-8')
+            self.assertIn('indexed msgstr required for plural', err)
+
+    def test_generic_syntax_error(self):
+        with temp_cwd():
+            Path('invalid.po').write_text('''\
+"foo"
+''')
+
+            res = assert_python_failure(msgfmt, 'invalid.po')
+            err = res.err.decode('utf-8')
+            self.assertIn('Syntax error', err)
+
+class CLITest(unittest.TestCase):
+
+    def test_help(self):
+        for option in ('--help', '-h'):
+            res = assert_python_ok(msgfmt, option)
+            err = res.err.decode('utf-8')
+            self.assertIn('Generate binary message catalog from textual translation description.', err)
+
+    def test_version(self):
+        for option in ('--version', '-V'):
+            res = assert_python_ok(msgfmt, option)
+            out = res.out.decode('utf-8').strip()
+            self.assertEqual('msgfmt.py 1.2', out)
+
+    def test_invalid_option(self):
+        res = assert_python_failure(msgfmt, '--invalid-option')
+        err = res.err.decode('utf-8')
+        self.assertIn('Generate binary message catalog from textual translation description.', err)
+        self.assertIn('option --invalid-option not recognized', err)
+
+    def test_no_input_file(self):
+        res = assert_python_ok(msgfmt)
+        err = res.err.decode('utf-8').replace('\r\n', '\n')
+        self.assertIn('No input file given\n'
+                      "Try `msgfmt --help' for more information.", err)
+
+    def test_nonexistent_file(self):
+        assert_python_failure(msgfmt, 'nonexistent.po')
+
+
+def update_catalog_snapshots():
+    for po_file in data_dir.glob('*.po'):
+        mo_file = po_file.with_suffix('.mo')
+        compile_messages(po_file, mo_file)
+        # Create a human-readable JSON file which is
+        # easier to review than the binary .mo file.
+        with open(mo_file, 'rb') as f:
+            translations = GNUTranslations(f)
+        catalog_file = po_file.with_suffix('.json')
+        with open(catalog_file, 'w') as f:
+            data = translations._catalog.items()
+            data = sorted(data, key=lambda x: (isinstance(x[0], tuple), x[0]))
+            json.dump(data, f, indent=4)
+            f.write('\n')
+
+
+if __name__ == '__main__':
+    if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
+        update_catalog_snapshots()
+        sys.exit(0)
+    unittest.main()

Lib/test/test_tools/test_reindent.py

@@ -0,0 +1,35 @@
+"""Tests for scripts in the Tools directory.
+
+This file contains regression tests for some of the scripts found in the
+Tools directory of a Python checkout or tarball, such as reindent.py.
+"""
+
+import os
+import unittest
+from test.support.script_helper import assert_python_ok
+from test.support import findfile
+
+from test.test_tools import toolsdir, skip_if_missing
+
+skip_if_missing()
+
+class ReindentTests(unittest.TestCase):
+    script = os.path.join(toolsdir, 'patchcheck', 'reindent.py')
+
+    def test_noargs(self):
+        assert_python_ok(self.script)
+
+    def test_help(self):
+        rc, out, err = assert_python_ok(self.script, '-h')
+        self.assertEqual(out, b'')
+        self.assertGreater(err, b'')
+
+    def test_reindent_file_with_bad_encoding(self):
+        bad_coding_path = findfile('bad_coding.py', subdir='tokenizedata')
+        rc, out, err = assert_python_ok(self.script, '-r', bad_coding_path)
+        self.assertEqual(out, b'')
+        self.assertNotEqual(err, b'')
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_tools/test_sundry.py

@@ -0,0 +1,30 @@
+"""Tests for scripts in the Tools/scripts directory.
+
+This file contains extremely basic regression tests for the scripts found in
+the Tools directory of a Python checkout or tarball which don't have separate
+tests of their own.
+"""
+
+import os
+import unittest
+from test.support import import_helper
+
+from test.test_tools import scriptsdir, import_tool, skip_if_missing
+
+skip_if_missing()
+
+class TestSundryScripts(unittest.TestCase):
+    # import logging registers "atfork" functions which keep indirectly the
+    # logging module dictionary alive. Mock the function to be able to unload
+    # cleanly the logging module.
+    @import_helper.mock_register_at_fork
+    def test_sundry(self, mock_os):
+        for fn in os.listdir(scriptsdir):
+            if not fn.endswith('.py'):
+                continue
+            name = fn[:-3]
+            import_tool(name)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_weakref.py

@@ -1997,6 +1997,7 @@ class MappingTestCase(TestBase):
         # copying should not result in a crash.
         self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True)
 
+    @unittest.skip("TODO: RUSTPYTHON; occasionally crash (Exit code -6)")
     def test_threaded_weak_value_dict_copy(self):
         # Issue #35615: Weakref keys or values getting GC'ed during dict
         # copying should not result in a crash.

Lib/timeit.py

@@ -50,9 +50,9 @@ Functions:
 """
 
 import gc
+import itertools
 import sys
 import time
-import itertools
 
 __all__ = ["Timer", "timeit", "repeat", "default_timer"]
 
@@ -77,9 +77,11 @@ def inner(_it, _timer{init}):
     return _t1 - _t0
 """
 
+
 def reindent(src, indent):
     """Helper to reindent a multi-line statement."""
-    return src.replace("\n", "\n" + " "*indent)
+    return src.replace("\n", "\n" + " " * indent)
+
 
 class Timer:
     """Class for timing execution speed of small code snippets.
@@ -166,7 +168,7 @@ class Timer:
 
         To be precise, this executes the setup statement once, and
         then returns the time it takes to execute the main statement
-        a number of times, as a float measured in seconds.  The
+        a number of times, as float seconds if using the default timer.   The
         argument is the number of times through the loop, defaulting
         to one million.  The main statement, the setup statement and
         the timer function to be used are passed to the constructor.
@@ -230,16 +232,19 @@ class Timer:
                     return (number, time_taken)
             i *= 10
 
+
 def timeit(stmt="pass", setup="pass", timer=default_timer,
            number=default_number, globals=None):
     """Convenience function to create Timer object and call timeit method."""
     return Timer(stmt, setup, timer, globals).timeit(number)
 
+
 def repeat(stmt="pass", setup="pass", timer=default_timer,
            repeat=default_repeat, number=default_number, globals=None):
     """Convenience function to create Timer object and call repeat method."""
     return Timer(stmt, setup, timer, globals).repeat(repeat, number)
 
+
 def main(args=None, *, _wrap_timer=None):
     """Main program, used when run as a script.
 
@@ -261,10 +266,9 @@ def main(args=None, *, _wrap_timer=None):
         args = sys.argv[1:]
     import getopt
     try:
-        opts, args = getopt.getopt(args, "n:u:s:r:tcpvh",
+        opts, args = getopt.getopt(args, "n:u:s:r:pvh",
                                    ["number=", "setup=", "repeat=",
-                                    "time", "clock", "process",
-                                    "verbose", "unit=", "help"])
+                                    "process", "verbose", "unit=", "help"])
     except getopt.error as err:
         print(err)
         print("use -h/--help for command line help")
@@ -272,7 +276,7 @@ def main(args=None, *, _wrap_timer=None):
 
     timer = default_timer
     stmt = "\n".join(args) or "pass"
-    number = 0 # auto-determine
+    number = 0  # auto-determine
     setup = []
     repeat = default_repeat
     verbose = 0
@@ -289,7 +293,7 @@ def main(args=None, *, _wrap_timer=None):
                 time_unit = a
             else:
                 print("Unrecognized unit. Please select nsec, usec, msec, or sec.",
-                    file=sys.stderr)
+                      file=sys.stderr)
                 return 2
         if o in ("-r", "--repeat"):
             repeat = int(a)
@@ -323,7 +327,7 @@ def main(args=None, *, _wrap_timer=None):
                 msg = "{num} loop{s} -> {secs:.{prec}g} secs"
                 plural = (number != 1)
                 print(msg.format(num=number, s='s' if plural else '',
-                                  secs=time_taken, prec=precision))
+                                 secs=time_taken, prec=precision))
         try:
             number, _ = t.autorange(callback)
         except:
@@ -374,5 +378,6 @@ def main(args=None, *, _wrap_timer=None):
                                UserWarning, '', 0)
     return None
 
+
 if __name__ == "__main__":
     sys.exit(main())

common/src/hash.rs

@@ -139,6 +139,11 @@ pub fn hash_bigint(value: &BigInt) -> PyHash {
     fix_sentinel(ret)
 }
 
+#[inline]
+pub fn hash_usize(data: usize) -> PyHash {
+    fix_sentinel(mod_int(data as i64))
+}
+
 #[inline(always)]
 pub fn fix_sentinel(x: PyHash) -> PyHash {
     if x == SENTINEL { -2 } else { x }

compiler/codegen/src/compile.rs

@@ -41,8 +41,10 @@ use rustpython_compiler_core::{
 };
 use rustpython_compiler_source::SourceCode;
 // use rustpython_parser_core::source_code::{LineNumber, SourceLocation};
+use crate::error::InternalError;
 use std::borrow::Cow;
 
+pub(crate) type InternalResult<T> = Result<T, InternalError>;
 type CompileResult<T> = Result<T, CodegenError>;
 
 #[derive(PartialEq, Eq, Clone, Copy)]
@@ -210,6 +212,54 @@ macro_rules! emit {
     };
 }
 
+fn eprint_location(zelf: &Compiler<'_>) {
+    let start = zelf
+        .source_code
+        .source_location(zelf.current_source_range.start());
+    let end = zelf
+        .source_code
+        .source_location(zelf.current_source_range.end());
+    eprintln!(
+        "LOCATION: {} from {}:{} to {}:{}",
+        zelf.source_code.path.to_owned(),
+        start.row,
+        start.column,
+        end.row,
+        end.column
+    );
+}
+
+/// Better traceback for internal error
+fn unwrap_internal<T>(zelf: &Compiler<'_>, r: InternalResult<T>) -> T {
+    if let Err(ref r_err) = r {
+        eprintln!("=== CODEGEN PANIC INFO ===");
+        eprintln!("This IS an internal error: {}", r_err);
+        eprint_location(zelf);
+        eprintln!("=== END PANIC INFO ===");
+    }
+    r.unwrap()
+}
+
+fn compiler_unwrap_option<T>(zelf: &Compiler<'_>, o: Option<T>) -> T {
+    if o.is_none() {
+        eprintln!("=== CODEGEN PANIC INFO ===");
+        eprintln!("This IS an internal error, an option was unwrapped during codegen");
+        eprint_location(zelf);
+        eprintln!("=== END PANIC INFO ===");
+    }
+    o.unwrap()
+}
+
+// fn compiler_result_unwrap<T, E: std::fmt::Debug>(zelf: &Compiler<'_>, result: Result<T, E>) -> T {
+//     if result.is_err() {
+//         eprintln!("=== CODEGEN PANIC INFO ===");
+//         eprintln!("This IS an internal error, an result was unwrapped during codegen");
+//         eprint_location(zelf);
+//         eprintln!("=== END PANIC INFO ===");
+//     }
+//     result.unwrap()
+// }
+
 /// The pattern context holds information about captured names and jump targets.
 #[derive(Clone)]
 pub struct PatternContext {
@@ -372,10 +422,9 @@ impl Compiler<'_> {
     fn pop_code_object(&mut self) -> CodeObject {
         let table = self.pop_symbol_table();
         assert!(table.sub_tables.is_empty());
-        self.code_stack
-            .pop()
-            .unwrap()
-            .finalize_code(self.opts.optimize)
+        let pop = self.code_stack.pop();
+        let stack_top = compiler_unwrap_option(self, pop);
+        unwrap_internal(self, stack_top.finalize_code(self.opts.optimize))
     }
 
     // could take impl Into<Cow<str>>, but everything is borrowed from ast structs; we never
@@ -482,7 +531,8 @@ impl Compiler<'_> {
                     self.current_block().instructions.pop(); // pop Instruction::Pop
                 }
                 Stmt::FunctionDef(_) | Stmt::ClassDef(_) => {
-                    let store_inst = self.current_block().instructions.pop().unwrap(); // pop Instruction::Store
+                    let pop_instructions = self.current_block().instructions.pop();
+                    let store_inst = compiler_unwrap_option(self, pop_instructions); // pop Instruction::Store
                     emit!(self, Instruction::Duplicate);
                     self.current_block().instructions.push(store_inst);
                 }
@@ -540,8 +590,11 @@ impl Compiler<'_> {
         self.check_forbidden_name(&name, usage)?;
 
         let symbol_table = self.symbol_table_stack.last().unwrap();
-        let symbol = symbol_table.lookup(name.as_ref()).unwrap_or_else(||
-            unreachable!("the symbol '{name}' should be present in the symbol table, even when it is undefined in python."),
+        let symbol = unwrap_internal(
+            self,
+            symbol_table
+                .lookup(name.as_ref())
+                .ok_or_else(|| InternalError::MissingSymbol(name.to_string())),
         );
         let info = self.code_stack.last_mut().unwrap();
         let mut cache = &mut info.name_cache;
@@ -1476,12 +1529,12 @@ impl Compiler<'_> {
         }
         for var in &*code.freevars {
             let table = self.symbol_table_stack.last().unwrap();
-            let symbol = table.lookup(var).unwrap_or_else(|| {
-                panic!(
-                    "couldn't look up var {} in {} in {}",
-                    var, code.obj_name, self.source_code.path
-                )
-            });
+            let symbol = unwrap_internal(
+                self,
+                table
+                    .lookup(var)
+                    .ok_or_else(|| InternalError::MissingSymbol(var.to_owned())),
+            );
             let parent_code = self.code_stack.last().unwrap();
             let vars = match symbol.scope {
                 SymbolScope::Free => &parent_code.freevar_cache,
@@ -1564,8 +1617,11 @@ impl Compiler<'_> {
 
         // Check if the class is declared global
         let symbol_table = self.symbol_table_stack.last().unwrap();
-        let symbol = symbol_table.lookup(name.as_ref()).expect(
-            "The symbol must be present in the symbol table, even when it is undefined in python.",
+        let symbol = unwrap_internal(
+            self,
+            symbol_table
+                .lookup(name.as_ref())
+                .ok_or_else(|| InternalError::MissingSymbol(name.to_owned())),
         );
         let mut global_path_prefix = Vec::new();
         if symbol.scope == SymbolScope::GlobalExplicit {

compiler/codegen/src/error.rs

@@ -34,6 +34,27 @@ impl fmt::Display for CodegenError {
     }
 }
 
+#[derive(Debug)]
+#[non_exhaustive]
+pub enum InternalError {
+    StackOverflow,
+    StackUnderflow,
+    MissingSymbol(String),
+}
+
+impl Display for InternalError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::StackOverflow => write!(f, "stack overflow"),
+            Self::StackUnderflow => write!(f, "stack underflow"),
+            Self::MissingSymbol(s) => write!(
+                f,
+                "The symbol '{s}' must be present in the symbol table, even when it is undefined in python."
+            ),
+        }
+    }
+}
+
 #[derive(Debug)]
 #[non_exhaustive]
 pub enum CodegenErrorType {

compiler/codegen/src/ir.rs

@@ -1,6 +1,7 @@
 use std::ops;
 
 use crate::IndexSet;
+use crate::error::InternalError;
 use ruff_source_file::{OneIndexed, SourceLocation};
 use rustpython_compiler_core::bytecode::{
     CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label, OpArg,
@@ -82,12 +83,12 @@ pub struct CodeInfo {
     pub freevar_cache: IndexSet<String>,
 }
 impl CodeInfo {
-    pub fn finalize_code(mut self, optimize: u8) -> CodeObject {
+    pub fn finalize_code(mut self, optimize: u8) -> crate::InternalResult<CodeObject> {
         if optimize > 0 {
             self.dce();
         }
 
-        let max_stackdepth = self.max_stackdepth();
+        let max_stackdepth = self.max_stackdepth()?;
         let cell2arg = self.cell2arg();
 
         let CodeInfo {
@@ -154,7 +155,7 @@ impl CodeInfo {
             locations.clear()
         }
 
-        CodeObject {
+        Ok(CodeObject {
             flags,
             posonlyarg_count,
             arg_count,
@@ -172,7 +173,7 @@ impl CodeInfo {
             cellvars: cellvar_cache.into_iter().collect(),
             freevars: freevar_cache.into_iter().collect(),
             cell2arg,
-        }
+        })
     }
 
     fn cell2arg(&self) -> Option<Box<[i32]>> {
@@ -219,7 +220,7 @@ impl CodeInfo {
         }
     }
 
-    fn max_stackdepth(&self) -> u32 {
+    fn max_stackdepth(&self) -> crate::InternalResult<u32> {
         let mut maxdepth = 0u32;
         let mut stack = Vec::with_capacity(self.blocks.len());
         let mut start_depths = vec![u32::MAX; self.blocks.len()];
@@ -244,10 +245,13 @@ impl CodeInfo {
                     let instr_display = instr.display(display_arg, self);
                     eprint!("{instr_display}: {depth} {effect:+} => ");
                 }
-                if effect < 0 && depth < effect.unsigned_abs() {
-                    panic!("The stack will underflow at {depth} with {effect} effect on {instr:?}");
-                }
-                let new_depth = depth.checked_add_signed(effect).unwrap();
+                let new_depth = depth.checked_add_signed(effect).ok_or({
+                    if effect < 0 {
+                        InternalError::StackUnderflow
+                    } else {
+                        InternalError::StackOverflow
+                    }
+                })?;
                 if DEBUG {
                     eprintln!("{new_depth}");
                 }
@@ -264,7 +268,13 @@ impl CodeInfo {
                     )
                 {
                     let effect = instr.stack_effect(ins.arg, true);
-                    let target_depth = depth.checked_add_signed(effect).unwrap();
+                    let target_depth = depth.checked_add_signed(effect).ok_or({
+                        if effect < 0 {
+                            InternalError::StackUnderflow
+                        } else {
+                            InternalError::StackOverflow
+                        }
+                    })?;
                     if target_depth > maxdepth {
                         maxdepth = target_depth
                     }
@@ -280,7 +290,7 @@ impl CodeInfo {
         if DEBUG {
             eprintln!("DONE: {maxdepth}");
         }
-        maxdepth
+        Ok(maxdepth)
     }
 }
 

compiler/codegen/src/lib.rs

@@ -18,6 +18,8 @@ mod unparse;
 pub use compile::CompileOpts;
 use ruff_python_ast::Expr;
 
+pub(crate) use compile::InternalResult;
+
 pub trait ToPythonName {
     /// Returns a short name for the node suitable for use in error messages.
     fn python_name(&self) -> &'static str;

extra_tests/snippets/builtin_object.py

@@ -24,3 +24,10 @@ assert obj.__ne__(obj) is False
 assert not hasattr(obj, 'a')
 obj.__dict__ = {'a': 1}
 assert obj.a == 1
+
+# Value inside the formatter goes through a different path of resolution.
+# Check that it still works all the same
+d = {
+    0: "ab",
+}
+assert "ab ab" == "{k[0]} {vv}".format(k=d, vv=d[0])

src/shell.rs

@@ -49,13 +49,16 @@ fn shell_exec(
 
             let bad_error = match err {
                 CompileError::Parse(ref p) => {
-                    if matches!(
-                        p.error,
-                        ParseErrorType::Lexical(LexicalErrorType::IndentationError)
-                    ) {
-                        continuing // && p.location.is_some()
-                    } else {
-                        true // !matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _))
+                    match &p.error {
+                        ParseErrorType::Lexical(LexicalErrorType::IndentationError) => continuing, // && p.location.is_some()
+                        ParseErrorType::OtherError(msg) => {
+                            if msg.starts_with("Expected an indented block") {
+                                continuing
+                            } else {
+                                true
+                            }
+                        }
+                        _ => true, // !matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _))
                     }
                 }
                 _ => true, // It is a bad error for everything else

stdlib/Cargo.toml

@@ -45,7 +45,7 @@ memchr = { workspace = true }
 base64 = "0.22"
 csv-core = "0.1.11"
 dyn-clone = "1.0.10"
-puruspe = "0.4.0"
+pymath = { workspace = true }
 xml-rs = "0.8.14"
 
 # random
@@ -82,8 +82,8 @@ libz-sys = { package = "libz-rs-sys", version = "0.5" }
 bzip2 = { version = "0.5", features = ["libbz2-rs-sys"] }
 
 # tkinter
-tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.1.0", optional = true }
-tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.1.0", optional = true }
+tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true }
+tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true }
 
 # uuid
 [target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32", target_os = "redox")))'.dependencies]

stdlib/src/math.rs

@@ -1,5 +1,7 @@
 pub(crate) use math::make_module;
 
+use crate::{builtins::PyBaseExceptionRef, vm::VirtualMachine};
+
 #[pymodule]
 mod math {
     use crate::vm::{
@@ -17,6 +19,8 @@ mod math {
     // Constants
     #[pyattr]
     use std::f64::consts::{E as e, PI as pi, TAU as tau};
+
+    use super::pymath_error_to_exception;
     #[pyattr(name = "inf")]
     const INF: f64 = f64::INFINITY;
     #[pyattr(name = "nan")]
@@ -143,7 +147,7 @@ mod math {
     fn log1p(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> {
         let x = *x;
         if x.is_nan() || x > -1.0_f64 {
-            Ok((x + 1.0_f64).ln())
+            Ok(x.ln_1p())
         } else {
             Err(vm.new_value_error("math domain error".to_owned()))
         }
@@ -475,38 +479,22 @@ mod math {
     // Special functions:
     #[pyfunction]
     fn erf(x: ArgIntoFloat) -> f64 {
-        let x = *x;
-        if x.is_nan() { x } else { puruspe::erf(x) }
+        pymath::erf(*x)
     }
 
     #[pyfunction]
     fn erfc(x: ArgIntoFloat) -> f64 {
-        let x = *x;
-        if x.is_nan() { x } else { puruspe::erfc(x) }
+        pymath::erfc(*x)
     }
 
     #[pyfunction]
-    fn gamma(x: ArgIntoFloat) -> f64 {
-        let x = *x;
-        if x.is_finite() {
-            puruspe::gamma(x)
-        } else if x.is_nan() || x.is_sign_positive() {
-            x
-        } else {
-            f64::NAN
-        }
+    fn gamma(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> {
+        pymath::gamma(*x).map_err(|err| pymath_error_to_exception(err, vm))
     }
 
     #[pyfunction]
-    fn lgamma(x: ArgIntoFloat) -> f64 {
-        let x = *x;
-        if x.is_finite() {
-            puruspe::ln_gamma(x)
-        } else if x.is_nan() {
-            x
-        } else {
-            f64::INFINITY
-        }
+    fn lgamma(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> {
+        pymath::lgamma(*x).map_err(|err| pymath_error_to_exception(err, vm))
     }
 
     fn try_magic_method(
@@ -1000,3 +988,10 @@ mod math {
         Ok(result)
     }
 }
+
+fn pymath_error_to_exception(err: pymath::Error, vm: &VirtualMachine) -> PyBaseExceptionRef {
+    match err {
+        pymath::Error::EDOM => vm.new_value_error("math domain error".to_owned()),
+        pymath::Error::ERANGE => vm.new_overflow_error("math range error".to_owned()),
+    }
+}

vm/src/dict_inner.rs

@@ -993,8 +993,8 @@ impl DictKey for usize {
         *self
     }
 
-    fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> {
-        Ok(vm.state.hash_secret.hash_value(self))
+    fn key_hash(&self, _vm: &VirtualMachine) -> PyResult<HashValue> {
+        Ok(hash::hash_usize(*self))
     }
 
     fn key_is(&self, _other: &PyObject) -> bool {