diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index f96537ec0..b469c8ab2 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,5 +1,9 @@ +import contextlib import dis +import io +import itertools import math +import opcode import os import unittest import sys @@ -8,10 +12,17 @@ import _ast import tempfile import types import textwrap -from test import support -from test.support import script_helper, requires_debug_ranges -from test.support.os_helper import FakePath +import warnings +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None +from test import support +from test.support import (script_helper, requires_debug_ranges, run_code, + requires_specialization) +from test.support.bytecode_helper import instructions_with_positions +from test.support.os_helper import FakePath class TestSpecifics(unittest.TestCase): @@ -108,29 +119,30 @@ class TestSpecifics(unittest.TestCase): exec('z = a', g, d) self.assertEqual(d['z'], 12) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_extended_arg(self): - # default: 1000 * 2.5 = 2500 repetitions - repeat = int(sys.getrecursionlimit() * 2.5) + repeat = 100 longexpr = 'x = x or ' + '-x' * repeat g = {} - code = ''' -def f(x): - %s - %s - %s - %s - %s - %s - %s - %s - %s - %s - # the expressions above have no effect, x == argument - while x: - x -= 1 - # EXTENDED_ARG/JUMP_ABSOLUTE here - return x -''' % ((longexpr,)*10) + code = textwrap.dedent(''' + def f(x): + %s + %s + %s + %s + %s + %s + %s + %s + %s + %s + # the expressions above have no effect, x == argument + while x: + x -= 1 + # EXTENDED_ARG/JUMP_ABSOLUTE here + return x + ''' % ((longexpr,)*10)) exec(code, g) self.assertEqual(g['f'](5), 0) @@ -146,10 +158,11 @@ def f(x): def test_indentation(self): # testing compile() of indented block w/o trailing newline" - s = """ -if 1: - if 2: - pass""" + s = textwrap.dedent(""" + if 1: + if 2: + pass + """) compile(s, "", "exec") # This test is probably specific to CPython and may not generalize @@ -160,9 +173,8 @@ if 1: s256 = "".join(["\n"] * 256 + ["spam"]) co = compile(s256, 'fn', 'exec') self.assertEqual(co.co_firstlineno, 1) - lines = list(co.co_lines()) - self.assertEqual(lines[0][2], 0) - self.assertEqual(lines[1][2], 257) + lines = [line for _, _, line in co.co_lines()] + self.assertEqual(lines, [0, 257]) def test_literals_with_leading_zeroes(self): for arg in ["077787", "0xj", "0x.", "0e", "090000000000000", @@ -197,8 +209,7 @@ if 1: self.assertEqual(eval("0o777"), 511) self.assertEqual(eval("-0o0000010"), -8) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised def test_int_literals_too_long(self): n = 3000 source = f"a = 1\nb = 2\nc = {'3'*n}\nd = 4" @@ -272,8 +283,7 @@ if 1: self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by compile def test_import(self): succeed = [ 'import sys', @@ -334,8 +344,11 @@ if 1: l = lambda: "foo" self.assertIsNone(l.__doc__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_lambda_consts(self): + l = lambda: "this is the only const" + self.assertEqual(l.__code__.co_consts, ("this is the only const",)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by compile def test_encoding(self): code = b'# -*- coding: badencoding -*-\npass\n' self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') @@ -440,16 +453,134 @@ if 1: def f(): __mangled = 1 __not_mangled__ = 2 - import __mangled_mod - import __package__.module + import __mangled_mod # noqa: F401 + import __package__.module # noqa: F401 self.assertIn("_A__mangled", A.f.__code__.co_varnames) self.assertIn("__not_mangled__", A.f.__code__.co_varnames) self.assertIn("_A__mangled_mod", A.f.__code__.co_varnames) self.assertIn("__package__", A.f.__code__.co_varnames) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_condition_expression_with_dead_blocks_compiles(self): + # See gh-113054 + compile('if (5 if 5 else T): 0', '', 'exec') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_condition_expression_with_redundant_comparisons_compiles(self): + # See gh-113054, gh-114083 + exprs = [ + 'if 9<9<9and 9or 9:9', + 'if 9<9<9and 9or 9or 9:9', + 'if 9<9<9and 9or 9or 9or 9:9', + 'if 9<9<9and 9or 9or 9or 9or 9:9', + ] + for expr in exprs: + with self.subTest(expr=expr): + with self.assertWarns(SyntaxWarning): + compile(expr, '', 'exec') + + def test_dead_code_with_except_handler_compiles(self): + compile(textwrap.dedent(""" + if None: + with CM: + x = 1 + else: + x = 2 + """), '', 'exec') + + def test_try_except_in_while_with_chained_condition_compiles(self): + # see gh-124871 + compile(textwrap.dedent(""" + name_1, name_2, name_3 = 1, 2, 3 + while name_3 <= name_2 > name_1: + try: + raise + except: + pass + finally: + pass + """), '', 'exec') + + def test_compile_invalid_namedexpr(self): + # gh-109351 + m = ast.Module( + body=[ + ast.Expr( + value=ast.ListComp( + elt=ast.NamedExpr( + target=ast.Constant(value=1), + value=ast.Constant(value=3), + ), + generators=[ + ast.comprehension( + target=ast.Name(id="x", ctx=ast.Store()), + iter=ast.Name(id="y", ctx=ast.Load()), + ifs=[], + is_async=0, + ) + ], + ) + ) + ], + type_ignores=[], + ) + + with self.assertRaisesRegex(TypeError, "NamedExpr target must be a Name"): + compile(ast.fix_missing_locations(m), "", "exec") + + def test_compile_redundant_jumps_and_nops_after_moving_cold_blocks(self): + # See gh-120367 + code=textwrap.dedent(""" + try: + pass + except: + pass + else: + match name_2: + case b'': + pass + finally: + something + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + + def test_compile_redundant_jump_after_convert_pseudo_ops(self): + # See gh-120367 + code=textwrap.dedent(""" + if name_2: + pass + else: + try: + pass + except: + pass + ~name_5 + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: at 0xb77555080 file "1", line 1> != at 0xb77554f00 file "3", line 1> def test_compile_ast(self): fname = __file__ if fname.lower().endswith('pyc'): @@ -478,13 +609,33 @@ if 1: self.assertRaises(TypeError, compile, co1, '', 'eval') # raise exception when node type is no start node - self.assertRaises(TypeError, compile, _ast.If(), '', 'exec') + self.assertRaises(TypeError, compile, _ast.If(test=_ast.Name(id='x', ctx=_ast.Load())), '', 'exec') # raise exception when node has invalid children ast = _ast.Module() - ast.body = [_ast.BoolOp()] + ast.body = [_ast.BoolOp(op=_ast.Or())] self.assertRaises(TypeError, compile, ast, '', 'exec') + def test_compile_invalid_typealias(self): + # gh-109341 + m = ast.Module( + body=[ + ast.TypeAlias( + name=ast.Subscript( + value=ast.Name(id="foo", ctx=ast.Load()), + slice=ast.Constant(value="x"), + ctx=ast.Store(), + ), + type_params=[], + value=ast.Name(id="Callable", ctx=ast.Load()), + ) + ], + type_ignores=[], + ) + + with self.assertRaisesRegex(TypeError, "TypeAlias with non-Name name"): + compile(ast.fix_missing_locations(m), "", "exec") + def test_dict_evaluation_order(self): i = 0 @@ -496,18 +647,32 @@ if 1: d = {f(): f(), f(): f()} self.assertEqual(d, {1: 2, 3: 4}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_compile_filename(self): for filename in 'file.py', b'file.py': code = compile('pass', filename, 'exec') self.assertEqual(code.co_filename, 'file.py') for filename in bytearray(b'file.py'), memoryview(b'file.py'): - with self.assertWarns(DeprecationWarning): - code = compile('pass', filename, 'exec') - self.assertEqual(code.co_filename, 'file.py') + with self.assertRaises(TypeError): + compile('pass', filename, 'exec') self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec') + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type bool, not EvilBool + def test_compile_filename_refleak(self): + # Regression tests for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'mode' AC conversion failed + self.assertRaises(TypeError, compile, b'', mortal_str, mode=1234) + # check error path when 'optimize' AC conversion failed + self.assertRaises(OverflowError, compile, b'', mortal_str, + 'exec', optimize=1 << 1000) + # check error path when 'dont_inherit' AC conversion failed + class EvilBool: + def __bool__(self): raise ValueError + self.assertRaises(ValueError, compile, b'', mortal_str, + 'exec', dont_inherit=EvilBool()) + @support.cpython_only def test_same_filename_used(self): s = """def f(): pass\ndef g(): pass""" @@ -533,8 +698,7 @@ if 1: self.compile_single("class T:\n pass") self.compile_single("c = '''\na=1\nb=2\nc=3\n'''") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by compile_single def test_bad_single_statement(self): self.assertInvalidSingle('1\n2') self.assertInvalidSingle('def f(): pass') @@ -546,8 +710,7 @@ if 1: self.assertInvalidSingle('x = 5 # comment\nx = 6\n') self.assertInvalidSingle("c = '''\nd=1\n'''\na = 1\n\nb = 2\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'source code cannot contain null bytes' not found in b'OSError: stream did not contain valid UTF-8\n' def test_particularly_evil_undecodable(self): # Issue 24022 src = b'0000\x00\n00000000000\n\x00\n\x9e\n' @@ -556,10 +719,9 @@ if 1: with open(fn, "wb") as fp: fp.write(src) res = script_helper.run_python_until_end(fn)[0] - self.assertIn(b"Non-UTF-8", res.err) + self.assertIn(b"source code cannot contain null bytes", res.err) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'source code cannot contain null bytes' not found in b'OSError: stream did not contain valid UTF-8\n' def test_yet_more_evil_still_undecodable(self): # Issue #25388 src = b"#\x00\n#\xfd\n" @@ -568,30 +730,25 @@ if 1: with open(fn, "wb") as fp: fp.write(src) res = script_helper.run_python_until_end(fn)[0] - self.assertIn(b"Non-UTF-8", res.err) + self.assertIn(b"source code cannot contain null bytes", res.err) @support.cpython_only + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_compiler_recursion_limit(self): - # Expected limit is sys.getrecursionlimit() * the scaling factor - # in symtable.c (currently 3) - # We expect to fail *at* that limit, because we use up some of - # the stack depth limit in the test suite code - # So we check the expected limit and 75% of that - # XXX (ncoghlan): duplicating the scaling factor here is a little - # ugly. Perhaps it should be exposed somewhere... - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + # Compiler frames are small + limit = 100 + # Android test devices have less memory. + crash_depth = limit * (1000 if sys.platform == "android" else 5000) + success_depth = limit def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth compile(expect_ok, '', mode) - for depth in (fail_depth, crash_depth): - broken = prefix + repeated * depth - details = "Compiling ({!r} + {!r} * {})".format( - prefix, repeated, depth) - with self.assertRaises(RecursionError, msg=details): - compile(broken, '', mode) + broken = prefix + repeated * crash_depth + details = f"Compiling ({prefix!r} + {repeated!r} * {crash_depth})" + with self.assertRaises(RecursionError, msg=details): + compile(broken, '', mode) check_limit("a", "()") check_limit("a", ".b") @@ -601,14 +758,13 @@ if 1: # check_limit("a", " if a else a") # check_limit("if a: pass", "\nelif a: pass", mode="exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "cannot contain null" does not match "invalid syntax (, line 1)" def test_null_terminated(self): # The source code is null-terminated internally, but bytes-like # objects are accepted, which could be not terminated. - with self.assertRaisesRegex(ValueError, "cannot contain null"): + with self.assertRaisesRegex(SyntaxError, "cannot contain null"): compile("123\x00", "", "eval") - with self.assertRaisesRegex(ValueError, "cannot contain null"): + with self.assertRaisesRegex(SyntaxError, "cannot contain null"): compile(memoryview(b"123\x00"), "", "eval") code = compile(memoryview(b"123\x00")[1:-1], "", "eval") self.assertEqual(eval(code), 23) @@ -649,7 +805,6 @@ if 1: self.assertEqual(repr(f1()), repr(const)) check_same_constant(None) - check_same_constant(0) check_same_constant(0.0) check_same_constant(b'abc') check_same_constant('abc') @@ -664,7 +819,7 @@ if 1: # Merge constants in tuple or frozenset f1, f2 = lambda: "not a name", lambda: ("not a name",) f3 = lambda x: x in {("not a name",)} - self.assertIs(f1.__code__.co_consts[1], + self.assertIs(f1.__code__.co_consts[0], f2.__code__.co_consts[1][0]) self.assertIs(next(iter(f3.__code__.co_consts[1])), f2.__code__.co_consts[1]) @@ -686,16 +841,62 @@ if 1: self.assertIs(f1.__code__.co_linetable, f2.__code__.co_linetable) + @support.cpython_only + def test_remove_unused_consts(self): + def f(): + "docstring" + if True: + return "used" + else: + return "unused" + + self.assertEqual(f.__code__.co_consts, + (f.__doc__, "used")) + + @support.cpython_only + def test_remove_unused_consts_no_docstring(self): + # the first item (None for no docstring in this case) is + # always retained. + def f(): + if True: + return "used" + else: + return "unused" + + self.assertEqual(f.__code__.co_consts, + (True, "used")) + + @support.cpython_only + def test_remove_unused_consts_extended_args(self): + N = 1000 + code = ["def f():\n"] + code.append("\ts = ''\n") + code.append("\tfor i in range(1):\n") + for i in range(N): + code.append(f"\t\tif True: s += 't{i}'\n") + code.append(f"\t\tif False: s += 'f{i}'\n") + code.append("\treturn s\n") + + code = "".join(code) + g = {} + eval(compile(code, "file.py", "exec"), g) + exec(code, g) + f = g['f'] + expected = tuple([''] + [f't{i}' for i in range(N)]) + self.assertEqual(f.__code__.co_consts, expected) + expected = "".join(expected[1:]) + self.assertEqual(expected, f()) + # Stripping unused constants is not a strict requirement for the # Python semantics, it's a more an implementation detail. @support.cpython_only - def test_strip_unused_consts(self): + def test_strip_unused_None(self): # Python 3.10rc1 appended None to co_consts when None is not used # at all. See bpo-45056. def f1(): "docstring" return 42 - self.assertEqual(f1.__code__.co_consts, ("docstring", 42)) + self.assertEqual(f1.__code__.co_consts, (f1.__doc__,)) # This is a regression test for a CPython specific peephole optimizer # implementation bug present in a few releases. It's assertion verifies @@ -715,8 +916,90 @@ if 1: 'RETURN_VALUE', list(dis.get_instructions(unused_code_at_end))[-1].opname) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only + def test_docstring(self): + src = textwrap.dedent(""" + def with_docstring(): + "docstring" + + def two_strings(): + "docstring" + "not docstring" + + def with_fstring(): + f"not docstring" + + def with_const_expression(): + "also" + " not docstring" + + def multiple_const_strings(): + "not docstring " * 3 + """) + + for opt in [0, 1, 2]: + with self.subTest(opt=opt): + code = compile(src, "", "exec", optimize=opt) + ns = {} + exec(code, ns) + + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + self.assertEqual(ns['two_strings'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + self.assertIsNone(ns['two_strings'].__doc__) + self.assertIsNone(ns['with_fstring'].__doc__) + self.assertIsNone(ns['with_const_expression'].__doc__) + self.assertIsNone(ns['multiple_const_strings'].__doc__) + + @support.cpython_only + def test_docstring_interactive_mode(self): + srcs = [ + """def with_docstring(): + "docstring" + """, + """class with_docstring: + "docstring" + """, + ] + + for opt in [0, 1, 2]: + for src in srcs: + with self.subTest(opt=opt, src=src): + code = compile(textwrap.dedent(src), "", "single", optimize=opt) + ns = {} + exec(code, ns) + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + + @support.cpython_only + def test_docstring_omitted(self): + # See gh-115347 + src = textwrap.dedent(""" + def f(): + "docstring1" + def h(): + "docstring2" + return 42 + + class C: + "docstring3" + pass + + return h + """) + for opt in [-1, 0, 1, 2]: + for mode in ["exec", "single"]: + with self.subTest(opt=opt, mode=mode): + code = compile(src, "", mode, optimize=opt) + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.dis(code) + self.assertNotIn('NOP', output.getvalue()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: unable to find constant -0.0 in (0.0,) def test_dont_merge_constants(self): # Issue #25843: compile() must not merge constants which are equal # but have a different type. @@ -733,7 +1016,6 @@ if 1: self.assertEqual(repr(f1()), repr(const1)) self.assertEqual(repr(f2()), repr(const2)) - check_different_constants(0, 0.0) check_different_constants(+0.0, -0.0) check_different_constants((0,), (0.0,)) check_different_constants('a', b'a') @@ -761,10 +1043,13 @@ if 1: # An implicit test for PyUnicode_FSDecoder(). compile("42", FakePath("test_compile_pathlike"), "single") + # bpo-31113: Stack overflow when compile a long sequence of + # complex statements. + @support.requires_resource('cpu') def test_stack_overflow(self): - # bpo-31113: Stack overflow when compile a long sequence of - # complex statements. - compile("if a: b\n" * 200000, "", "exec") + # Android test devices have less memory. + size = 100_000 if sys.platform == "android" else 200_000 + compile("if a: b\n" * size, "", "exec") # Multiple users rely on the fact that CPython does not generate # bytecode for dead code blocks. See bpo-37500 for more context. @@ -796,12 +1081,10 @@ if 1: for func in funcs: opcodes = list(dis.get_instructions(func)) self.assertLessEqual(len(opcodes), 4) - self.assertEqual('LOAD_CONST', opcodes[-2].opname) - self.assertEqual(None, opcodes[-2].argval) self.assertEqual('RETURN_VALUE', opcodes[-1].opname) + self.assertEqual(None, opcodes[-1].argval) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 3 != 8 def test_false_while_loop(self): def break_in_while(): while False: @@ -817,12 +1100,10 @@ if 1: for func in funcs: opcodes = list(dis.get_instructions(func)) self.assertEqual(3, len(opcodes)) - self.assertEqual('LOAD_CONST', opcodes[1].opname) + self.assertEqual('RETURN_VALUE', opcodes[-1].opname) self.assertEqual(None, opcodes[1].argval) - self.assertEqual('RETURN_VALUE', opcodes[2].opname) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_consts_in_conditionals(self): def and_true(x): return True and x @@ -846,8 +1127,6 @@ if 1: self.assertIn('LOAD_', opcodes[-2].opname) self.assertEqual('RETURN_VALUE', opcodes[-1].opname) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_imported_load_method(self): sources = [ """\ @@ -880,7 +1159,33 @@ if 1: instructions = [opcode.opname for opcode in opcodes] self.assertNotIn('LOAD_METHOD', instructions) self.assertIn('LOAD_ATTR', instructions) - self.assertIn('PRECALL', instructions) + self.assertIn('CALL', instructions) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'LOAD_SMALL_INT' not found in ['RESUME', 'LOAD_CONST', 'RETURN_VALUE'] + def test_folding_type_param(self): + get_code_fn_cls = lambda x: x.co_consts[0].co_consts[2] + get_code_type_alias = lambda x: x.co_consts[0].co_consts[3] + snippets = [ + ("def foo[T = 40 + 5](): pass", get_code_fn_cls), + ("def foo[**P = 40 + 5](): pass", get_code_fn_cls), + ("def foo[*Ts = 40 + 5](): pass", get_code_fn_cls), + ("class foo[T = 40 + 5]: pass", get_code_fn_cls), + ("class foo[**P = 40 + 5]: pass", get_code_fn_cls), + ("class foo[*Ts = 40 + 5]: pass", get_code_fn_cls), + ("type foo[T = 40 + 5] = 1", get_code_type_alias), + ("type foo[**P = 40 + 5] = 1", get_code_type_alias), + ("type foo[*Ts = 40 + 5] = 1", get_code_type_alias), + ] + for snippet, get_code in snippets: + c = compile(snippet, "", "exec") + code = get_code(c) + opcodes = list(dis.get_instructions(code)) + instructions = [opcode.opname for opcode in opcodes] + args = [opcode.oparg for opcode in opcodes] + self.assertNotIn(40, args) + self.assertNotIn(5, args) + self.assertIn('LOAD_SMALL_INT', instructions) + self.assertIn(45, args) def test_lineno_procedure_call(self): def call(): @@ -890,8 +1195,7 @@ if 1: line1 = call.__code__.co_firstlineno + 1 assert line1 not in [line for (_, _, line) in call.__code__.co_lines()] - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_after_implicit_return(self): TRUE = True # Don't use constant True or False, as compiler will remove test @@ -935,10 +1239,12 @@ if 1: for func in (no_code1, no_code2): with self.subTest(func=func): + if func is no_code1 and no_code1.__doc__ is None: + continue code = func.__code__ - lines = list(code.co_lines()) - start, end, line = lines[0] + [(start, end, line)] = code.co_lines() self.assertEqual(start, 0) + self.assertEqual(end, len(code.co_code)) self.assertEqual(line, code.co_firstlineno) def get_code_lines(self, code): @@ -950,8 +1256,7 @@ if 1: last_line = line return res - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_attribute(self): def load_attr(): return ( @@ -996,8 +1301,7 @@ if 1: code_lines = self.get_code_lines(func.__code__) self.assertEqual(lines, code_lines) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + [0] def test_line_number_genexp(self): def return_genexp(): @@ -1006,9 +1310,9 @@ if 1: x in y) - genexp_lines = [0, 2, 0] + genexp_lines = [0, 4, 2, 0, 4] - genexp_code = return_genexp.__code__.co_consts[1] + genexp_code = return_genexp.__code__.co_consts[0] code_lines = self.get_code_lines(genexp_code) self.assertEqual(genexp_lines, code_lines) @@ -1022,6 +1326,73 @@ if 1: code_lines = self.get_code_lines(test.__code__) self.assertEqual(expected_lines, code_lines) + def check_line_numbers(self, code, opnames=None): + # Check that all instructions whose op matches opnames + # have a line number. opnames can be a single name, or + # a sequence of names. If it is None, match all ops. + + if isinstance(opnames, str): + opnames = (opnames, ) + for inst in dis.Bytecode(code): + if opnames and inst.opname in opnames: + self.assertIsNotNone(inst.positions.lineno) + + def test_line_number_synthetic_jump_multiple_predecessors(self): + def f(): + for x in it: + try: + if C1: + yield 2 + except OSError: + pass + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + X = 4 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_more_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + if C4: + X = 4 + except OSError: + try: + if C3: + if C4: + X = 5 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_lineno_of_backward_jump_conditional_in_loop(self): + # Issue gh-107901 + def f(): + for i in x: + if y: + pass + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + def test_big_dict_literal(self): # The compiler has a flushing point in "compiler_dict" that calls compiles # a portion of the dictionary literal when the loop that iterates over the items @@ -1071,14 +1442,109 @@ if 1: for instr in dis.Bytecode(while_not_chained): self.assertNotEqual(instr.opname, "EXTENDED_ARG") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only + def test_uses_slice_instructions(self): + + def check_op_count(func, op, expected): + actual = 0 + for instr in dis.Bytecode(func): + if instr.opname == op: + actual += 1 + self.assertEqual(actual, expected) + + def check_consts(func, typ, expected): + expected = set([repr(x) for x in expected]) + all_consts = set() + consts = func.__code__.co_consts + for instr in dis.Bytecode(func): + if instr.opname == "LOAD_CONST" and isinstance(consts[instr.oparg], typ): + all_consts.add(repr(consts[instr.oparg])) + self.assertEqual(all_consts, expected) + + def load(): + return x[a:b] + x [a:] + x[:b] + x[:] + + check_op_count(load, "BINARY_SLICE", 3) + check_op_count(load, "BUILD_SLICE", 0) + check_consts(load, slice, [slice(None, None, None)]) + check_op_count(load, "BINARY_OP", 4) + + def store(): + x[a:b] = y + x [a:] = y + x[:b] = y + x[:] = y + + check_op_count(store, "STORE_SLICE", 3) + check_op_count(store, "BUILD_SLICE", 0) + check_op_count(store, "STORE_SUBSCR", 1) + check_consts(store, slice, [slice(None, None, None)]) + + def long_slice(): + return x[a:b:c] + + check_op_count(long_slice, "BUILD_SLICE", 1) + check_op_count(long_slice, "BINARY_SLICE", 0) + check_consts(long_slice, slice, []) + check_op_count(long_slice, "BINARY_OP", 1) + + def aug(): + x[a:b] += y + + check_op_count(aug, "BINARY_SLICE", 1) + check_op_count(aug, "STORE_SLICE", 1) + check_op_count(aug, "BUILD_SLICE", 0) + check_op_count(aug, "BINARY_OP", 1) + check_op_count(aug, "STORE_SUBSCR", 0) + check_consts(aug, slice, []) + + def aug_const(): + x[1:2] += y + + check_op_count(aug_const, "BINARY_SLICE", 0) + check_op_count(aug_const, "STORE_SLICE", 0) + check_op_count(aug_const, "BINARY_OP", 2) + check_op_count(aug_const, "STORE_SUBSCR", 1) + check_consts(aug_const, slice, [slice(1, 2)]) + + def compound_const_slice(): + x[1:2:3, 4:5:6] = y + + check_op_count(compound_const_slice, "BINARY_SLICE", 0) + check_op_count(compound_const_slice, "BUILD_SLICE", 0) + check_op_count(compound_const_slice, "STORE_SLICE", 0) + check_op_count(compound_const_slice, "STORE_SUBSCR", 1) + check_consts(compound_const_slice, slice, []) + check_consts(compound_const_slice, tuple, [(slice(1, 2, 3), slice(4, 5, 6))]) + + def mutable_slice(): + x[[]:] = y + + check_consts(mutable_slice, slice, {}) + + def different_but_equal(): + x[:0] = y + x[:0.0] = y + x[:False] = y + x[:None] = y + + check_consts( + different_but_equal, + slice, + [ + slice(None, 0, None), + slice(None, 0.0, None), + slice(None, False, None), + slice(None, None, None) + ] + ) + def test_compare_positions(self): - for opname, op in [ - ("COMPARE_OP", "<"), - ("COMPARE_OP", "<="), - ("COMPARE_OP", ">"), - ("COMPARE_OP", ">="), + for opname_prefix, op in [ + ("COMPARE_", "<"), + ("COMPARE_", "<="), + ("COMPARE_", ">"), + ("COMPARE_", ">="), ("CONTAINS_OP", "in"), ("CONTAINS_OP", "not in"), ("IS_OP", "is"), @@ -1093,11 +1559,313 @@ if 1: actual_positions = [ instruction.positions for instruction in dis.get_instructions(code) - if instruction.opname == opname + if instruction.opname.startswith(opname_prefix) ] with self.subTest(source): self.assertEqual(actual_positions, expected_positions) + def test_if_expression_expression_empty_block(self): + # See regression in gh-99708 + exprs = [ + "assert (False if 1 else True)", + "def f():\n\tif not (False if 1 else True): raise AssertionError", + "def f():\n\tif not (False if 1 else True): return 12", + ] + for expr in exprs: + with self.subTest(expr=expr): + compile(expr, "", "exec") + + def test_multi_line_lambda_as_argument(self): + # See gh-101928 + code = textwrap.dedent(""" + def foo(param, lambda_exp): + pass + + foo(param=0, + lambda_exp=lambda: + 1) + """) + compile(code, "", "exec") + + def test_apply_static_swaps(self): + def f(x, y): + a, a = x, y + return a + self.assertEqual(f("x", "y"), "y") + + def test_apply_static_swaps_2(self): + def f(x, y, z): + a, b, a = x, y, z + return a + self.assertEqual(f("x", "y", "z"), "z") + + def test_apply_static_swaps_3(self): + def f(x, y, z): + a, a, b = x, y, z + return a + self.assertEqual(f("x", "y", "z"), "y") + + def test_variable_dependent(self): + # gh-104635: Since the value of b is dependent on the value of a + # the first STORE_FAST for a should not be skipped. (e.g POP_TOP). + # This test case is added to prevent potential regression from aggressive optimization. + def f(): + a = 42; b = a + 54; a = 54 + return a, b + self.assertEqual(f(), (54, 96)) + + def test_duplicated_small_exit_block(self): + # See gh-109627 + def f(): + while element and something: + try: + return something + except: + pass + + def test_cold_block_moved_to_end(self): + # See gh-109719 + def f(): + while name: + try: + break + except: + pass + else: + 1 if 1 else 1 + + def test_remove_empty_basic_block_with_jump_target_label(self): + # See gh-109823 + def f(x): + while x: + 0 if 1 else 0 + + def test_remove_redundant_nop_edge_case(self): + # See gh-109889 + def f(): + a if (1 if b else c) else d + + def test_global_declaration_in_except_used_in_else(self): + # See gh-111123 + code = textwrap.dedent("""\ + def f(): + try: + pass + %s Exception: + global a + else: + print(a) + """) + + g, l = {'a': 5}, {} + for kw in ("except", "except*"): + exec(code % kw, g, l); + + def test_regression_gh_120225(self): + async def name_4(): + match b'': + case True: + pass + case name_5 if f'e': + {name_3: name_4 async for name_2 in name_5} + case []: + pass + [[]] + + def test_globals_dict_subclass(self): + # gh-132386 + class WeirdDict(dict): + pass + + ns = {} + exec('def foo(): return a', WeirdDict(), ns) + + self.assertRaises(NameError, ns['foo']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + [3, 5, 3, 5] + def test_compile_warnings(self): + # Each invocation of compile() emits compiler warnings, even if they + # have the same message and line number. + source = textwrap.dedent(r""" + # tokenizer + 1or 0 # line 3 + # code generator + 1 is 1 # line 5 + """) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("default") + for i in range(2): + # Even if compile() is at the same line. + compile(source, '', 'exec') + + self.assertEqual([wm.lineno for wm in caught], [3, 5] * 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + [5, 9] + def test_compile_warning_in_finally(self): + # Ensure that warnings inside finally blocks are + # only emitted once despite the block being + # compiled twice (for normal execution and for + # exception handling). + source = textwrap.dedent(""" + try: + pass + finally: + 1 is 1 # line 5 + try: + pass + finally: # nested + 1 is 1 # line 9 + """) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + compile(source, '', 'exec') + + self.assertEqual(sorted(wm.lineno for wm in caught), [5, 9]) + for wm in caught: + self.assertEqual(wm.category, SyntaxWarning) + self.assertIn("\"is\" with 'int' literal", str(wm.message)) + + # Other code path is used for "try" with "except*". + source = textwrap.dedent(""" + try: + pass + except *Exception: + pass + finally: + 1 is 1 # line 7 + try: + pass + except *Exception: + pass + finally: # nested + 1 is 1 # line 13 + """) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + compile(source, '', 'exec') + + self.assertEqual(sorted(wm.lineno for wm in caught), [7, 13]) + for wm in caught: + self.assertEqual(wm.category, SyntaxWarning) + self.assertIn("\"is\" with 'int' literal", str(wm.message)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.subTests('src', [ + textwrap.dedent(""" + def f(): + try: + pass + finally: + return 42 + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + break + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + continue + """), + ]) + def test_pep_765_warnings(self, src): + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + compile(src, '', 'exec') + with warnings.catch_warnings(): + warnings.simplefilter("error") + tree = ast.parse(src) + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + compile(tree, '', 'exec') + + @support.subTests('src', [ + textwrap.dedent(""" + try: + pass + finally: + def f(): + return 42 + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + break + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + continue + """), + ]) + def test_pep_765_no_warnings(self, src): + with warnings.catch_warnings(): + warnings.simplefilter("error") + compile(src, '', 'exec') + + +class TestBooleanExpression(unittest.TestCase): + class Value: + def __init__(self): + self.called = 0 + + def __bool__(self): + self.called += 1 + return self.value + + class Yes(Value): + value = True + + class No(Value): + value = False + + def test_short_circuit_and(self): + v = [self.Yes(), self.No(), self.Yes()] + res = v[0] and v[1] and v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_short_circuit_or(self): + v = [self.No(), self.Yes(), self.No()] + res = v[0] or v[1] or v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_compound(self): + # See gh-124285 + v = [self.No(), self.Yes(), self.Yes(), self.Yes()] + res = v[0] and v[1] or v[2] or v[3] + self.assertIs(res, v[2]) + self.assertEqual([e.called for e in v], [1, 0, 1, 0]) + + v = [self.No(), self.No(), self.Yes(), self.Yes(), self.No()] + res = v[0] or v[1] and v[2] or v[3] or v[4] + self.assertIs(res, v[3]) + self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) + + def test_exception(self): + # See gh-137288 + class Foo: + def __bool__(self): + raise NotImplementedError() + + a = Foo() + b = Foo() + + with self.assertRaises(NotImplementedError): + bool(a) + + with self.assertRaises(NotImplementedError): + c = a or b @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): @@ -1116,7 +1884,7 @@ class TestSourcePositions(unittest.TestCase): class SourceOffsetVisitor(ast.NodeVisitor): def generic_visit(self, node): super().generic_visit(node) - if not isinstance(node, ast.expr) and not isinstance(node, ast.stmt): + if not isinstance(node, (ast.expr, ast.stmt, ast.pattern)): return lines.add(node.lineno) end_lines.add(node.end_lineno) @@ -1145,8 +1913,8 @@ class TestSourcePositions(unittest.TestCase): def assertOpcodeSourcePositionIs(self, code, opcode, line, end_line, column, end_column, occurrence=1): - for instr, position in zip( - dis.Bytecode(code, show_caches=True), code.co_positions(), strict=True + for instr, position in instructions_with_positions( + dis.Bytecode(code), code.co_positions() ): if instr.opname == opcode: occurrence -= 1 @@ -1184,15 +1952,303 @@ class TestSourcePositions(unittest.TestCase): column=2, end_column=9, occurrence=2) def test_multiline_expression(self): - snippet = """\ -f( - 1, 2, 3, 4 -) -""" + snippet = textwrap.dedent("""\ + f( + 1, 2, 3, 4 + ) + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', line=1, end_line=3, column=0, end_column=1) + @requires_specialization + def test_multiline_boolean_expression(self): + snippet = textwrap.dedent("""\ + if (a or + (b and not c) or + not ( + d > 0)): + x = 42 + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + # jump if a is true: + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', + line=1, end_line=1, column=4, end_column=5, occurrence=1) + # jump if b is false: + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_FALSE', + line=2, end_line=2, column=5, end_column=6, occurrence=1) + # jump if c is false: + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_FALSE', + line=2, end_line=2, column=15, end_column=16, occurrence=2) + # compare d and 0 + self.assertOpcodeSourcePositionIs(compiled_code, 'COMPARE_OP', + line=4, end_line=4, column=8, end_column=13, occurrence=1) + # jump if comparison it True + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', + line=4, end_line=4, column=8, end_column=13, occurrence=2) + + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") + def test_multiline_assert(self): + snippet = textwrap.dedent("""\ + assert (a > 0 and + bb > 0 and + ccc == 1000000), "error msg" + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_COMMON_CONSTANT', + line=1, end_line=3, column=0, end_column=36, occurrence=1) + # The "error msg": + self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_CONST', + line=3, end_line=3, column=25, end_column=36, occurrence=2) + self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', + line=1, end_line=3, column=0, end_column=36, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RAISE_VARARGS', + line=1, end_line=3, column=8, end_column=22, occurrence=1) + + def test_multiline_generator_expression(self): + snippet = textwrap.dedent("""\ + ((x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)) + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=4, end_line=4, column=7, end_column=14, occurrence=1) + + def test_multiline_async_generator_expression(self): + snippet = textwrap.dedent("""\ + ((x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)) + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE', + line=1, end_line=2, column=1, end_column=8, occurrence=2) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=1, end_line=6, column=0, end_column=32, occurrence=1) + + def test_multiline_list_comprehension(self): + snippet = textwrap.dedent("""\ + [(x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)] + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + + def test_multiline_async_list_comprehension(self): + snippet = textwrap.dedent("""\ + async def f(): + [(x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)] + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + g = {} + eval(compiled_code, g) + compiled_code = g['f'].__code__ + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=2, end_line=7, column=4, end_column=36, occurrence=1) + + def test_multiline_set_comprehension(self): + snippet = textwrap.dedent("""\ + {(x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + + def test_multiline_async_set_comprehension(self): + snippet = textwrap.dedent("""\ + async def f(): + {(x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + g = {} + eval(compiled_code, g) + compiled_code = g['f'].__code__ + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=2, end_line=7, column=4, end_column=36, occurrence=1) + + def test_multiline_dict_comprehension(self): + snippet = textwrap.dedent("""\ + {x: + 2*x + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', + line=1, end_line=2, column=1, end_column=7, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=7, occurrence=1) + + def test_multiline_async_dict_comprehension(self): + snippet = textwrap.dedent("""\ + async def f(): + {x: + 2*x + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + g = {} + eval(compiled_code, g) + compiled_code = g['f'].__code__ + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', + line=2, end_line=3, column=5, end_column=11, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=2, end_line=3, column=5, end_column=11, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=2, end_line=7, column=4, end_column=36, occurrence=1) + + def test_matchcase_sequence(self): + snippet = textwrap.dedent("""\ + match x: + case a, b: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_SEQUENCE', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=13, occurrence=2) + + def test_matchcase_sequence_wildcard(self): + snippet = textwrap.dedent("""\ + match x: + case a, *b, c: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', + line=2, end_line=2, column=9, end_column=17, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_EX', + line=2, end_line=2, column=9, end_column=17, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=17, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=17, occurrence=2) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=17, occurrence=3) + + def test_matchcase_mapping(self): + snippet = textwrap.dedent("""\ + match x: + case {"a" : a, "b": b}: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', + line=2, end_line=2, column=9, end_column=26, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_KEYS', + line=2, end_line=2, column=9, end_column=26, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=26, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=26, occurrence=2) + + def test_matchcase_mapping_wildcard(self): + snippet = textwrap.dedent("""\ + match x: + case {"a" : a, "b": b, **c}: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', + line=2, end_line=2, column=9, end_column=31, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_KEYS', + line=2, end_line=2, column=9, end_column=31, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=31, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=31, occurrence=2) + + def test_matchcase_class(self): + snippet = textwrap.dedent("""\ + match x: + case C(a, b): + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', + line=2, end_line=2, column=9, end_column=16, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_SEQUENCE', + line=2, end_line=2, column=9, end_column=16, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=16, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=16, occurrence=2) + + def test_matchcase_or(self): + snippet = textwrap.dedent("""\ + match x: + case C(1) | C(2): + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', + line=2, end_line=2, column=16, end_column=20, occurrence=2) + def test_very_long_line_end_offset(self): # Make sure we get the correct column offset for offsets # too large to store in a byte. @@ -1207,16 +2263,16 @@ f( snippet = "a - b @ (c * x['key'] + 23)" compiled_code, _ = self.check_positions_against_ast(snippet) - self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_SUBSCR', + self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', line=1, end_line=1, column=13, end_column=21) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=9, end_column=21, occurrence=1) + line=1, end_line=1, column=9, end_column=21, occurrence=2) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=9, end_column=26, occurrence=2) + line=1, end_line=1, column=9, end_column=26, occurrence=3) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=4, end_column=27, occurrence=3) + line=1, end_line=1, column=4, end_column=27, occurrence=4) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=0, end_column=27, occurrence=4) + line=1, end_line=1, column=0, end_column=27, occurrence=5) def test_multiline_assert_rewritten_as_method_call(self): # GH-94694: Don't crash if pytest rewrites a multiline assert as a @@ -1266,7 +2322,9 @@ f( with self.subTest(body): namespace = {} source = textwrap.dedent(source_template.format(body)) - exec(source, namespace) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', SyntaxWarning) + exec(source, namespace) code = namespace["f"].__code__ self.assertOpcodeSourcePositionIs( code, @@ -1312,7 +2370,7 @@ f( source = "(\n lhs \n . \n rhs \n )()" code = compile(source, "", "exec") self.assertOpcodeSourcePositionIs( - code, "LOAD_METHOD", line=4, end_line=4, column=5, end_column=8 + code, "LOAD_ATTR", line=4, end_line=4, column=5, end_column=8 ) self.assertOpcodeSourcePositionIs( code, "CALL", line=4, end_line=5, column=5, end_column=10 @@ -1343,9 +2401,6 @@ f( for source in [ "lambda: a", "(a for b in c)", - "[a for b in c]", - "{a for b in c}", - "{a: b for c in d}", ]: with self.subTest(source): code = compile(f"{source}, {source}", "", "eval") @@ -1358,6 +2413,141 @@ f( list(code.co_consts[1].co_positions()), ) + def test_load_super_attr(self): + source = "class C:\n def __init__(self):\n super().__init__()" + for const in compile(source, "", "exec").co_consts[0].co_consts: + if isinstance(const, types.CodeType): + code = const + break + self.assertOpcodeSourcePositionIs( + code, "LOAD_GLOBAL", line=3, end_line=3, column=4, end_column=9 + ) + + def test_lambda_return_position(self): + snippets = [ + "f = lambda: x", + "f = lambda: 42", + "f = lambda: 1 + 2", + "f = lambda: a + b", + ] + for snippet in snippets: + with self.subTest(snippet=snippet): + lamb = run_code(snippet)["f"] + positions = lamb.__code__.co_positions() + # assert that all positions are within the lambda + for i, pos in enumerate(positions): + with self.subTest(i=i, pos=pos): + start_line, end_line, start_col, end_col = pos + if i == 0 and start_col == end_col == 0: + # ignore the RESUME in the beginning + continue + self.assertEqual(start_line, 1) + self.assertEqual(end_line, 1) + code_start = snippet.find(":") + 2 + code_end = len(snippet) + self.assertGreaterEqual(start_col, code_start) + self.assertLessEqual(end_col, code_end) + self.assertGreaterEqual(end_col, start_col) + self.assertLessEqual(end_col, code_end) + + def test_return_in_with_positions(self): + # See gh-98442 + def f(): + with xyz: + 1 + 2 + 3 + 4 + return R + + # All instructions should have locations on a single line + for instr in dis.get_instructions(f): + start_line, end_line, _, _ = instr.positions + self.assertEqual(start_line, end_line) + + # Expect four `LOAD_CONST None` instructions: + # three for the no-exception __exit__ call, and one for the return. + # They should all have the locations of the context manager ('xyz'). + + load_none = [instr for instr in dis.get_instructions(f) if + instr.opname == 'LOAD_CONST' and instr.argval is None] + return_value = [instr for instr in dis.get_instructions(f) if + instr.opname == 'RETURN_VALUE'] + + self.assertEqual(len(load_none), 4) + self.assertEqual(len(return_value), 2) + for instr in load_none + return_value: + start_line, end_line, start_col, end_col = instr.positions + self.assertEqual(start_line, f.__code__.co_firstlineno + 1) + self.assertEqual(end_line, f.__code__.co_firstlineno + 1) + self.assertEqual(start_col, 17) + self.assertEqual(end_col, 20) + + +class TestStaticAttributes(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__' + def test_basic(self): + class C: + def f(self): + self.a = self.b = 42 + # read fields are not included + self.f() + self.arr[3] + + self.assertIsInstance(C.__static_attributes__, tuple) + self.assertEqual(sorted(C.__static_attributes__), ['a', 'b']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__' + def test_nested_function(self): + class C: + def f(self): + self.x = 1 + self.y = 2 + self.x = 3 # check deduplication + + def g(self, obj): + self.y = 4 + self.z = 5 + + def h(self, a): + self.u = 6 + self.v = 7 + + obj.self = 8 + + self.assertEqual(sorted(C.__static_attributes__), ['u', 'v', 'x', 'y', 'z']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__' + def test_nested_class(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D: + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(C.D.__static_attributes__), ['y', 'z']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__' + def test_subclass(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D(C): + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(D.__static_attributes__), ['y', 'z']) + class TestExpressionStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object @@ -1391,28 +2581,23 @@ class TestExpressionStackSize(unittest.TestCase): def test_binop(self): self.check_stack_size("x + " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 not less than or equal to 6 def test_list(self): self.check_stack_size("[" + "x, " * self.N + "x]") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 not less than or equal to 6 def test_tuple(self): self.check_stack_size("(" + "x, " * self.N + "x)") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 not less than or equal to 6 def test_set(self): self.check_stack_size("{" + "x, " * self.N + "x}") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 202 not less than or equal to 7 def test_dict(self): self.check_stack_size("{" + "x:x, " * self.N + "x:x}") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 102 not less than or equal to 6 def test_func_args(self): self.check_stack_size("f(" + "x, " * self.N + ")") @@ -1420,8 +2605,7 @@ class TestExpressionStackSize(unittest.TestCase): kwargs = (f'a{i}=x' for i in range(self.N)) self.check_stack_size("f(" + ", ".join(kwargs) + ")") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 102 not less than or equal to 6 def test_meth_args(self): self.check_stack_size("o.m(" + "x, " * self.N + ")") @@ -1458,7 +2642,9 @@ class TestStackSizeStability(unittest.TestCase): script = """def func():\n""" + i * snippet if async_: script = "async " + script - code = compile(script, "