diff --git a/Lib/codeop.py b/Lib/codeop.py
index e29c0b38c..2213b69f2 100644
--- a/Lib/codeop.py
+++ b/Lib/codeop.py
@@ -10,30 +10,6 @@ and:
syntax error (OverflowError and ValueError can be produced by
malformed literals).
-Approach:
-
-First, check if the source consists entirely of blank lines and
-comments; if so, replace it with 'pass', because the built-in
-parser doesn't always do the right thing for these.
-
-Compile three times: as is, with \n, and with \n\n appended. If it
-compiles as is, it's complete. If it compiles with one \n appended,
-we expect more. If it doesn't compile either way, we compare the
-error we get when compiling with \n or \n\n appended. If the errors
-are the same, the code is broken. But if the errors are different, we
-expect more. Not intuitive; not even guaranteed to hold in future
-releases; but this matches the compiler's behavior from Python 1.4
-through 2.2, at least.
-
-Caveat:
-
-It is possible (but not likely) that the parser stops parsing with a
-successful outcome before reaching the end of the source; in this
-case, trailing symbols may be ignored instead of causing an error.
-For example, a backslash followed by two newlines may be followed by
-arbitrary garbage. This will be fixed once the API for the parser is
-better.
-
The two interfaces are:
compile_command(source, filename, symbol):
@@ -64,54 +40,50 @@ _features = [getattr(__future__, fname)
__all__ = ["compile_command", "Compile", "CommandCompiler"]
-PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h
-
+# The following flags match the values from Include/cpython/compile.h
+# Caveat emptor: These flags are undocumented on purpose and depending
+# on their effect outside the standard library is **unsupported**.
+PyCF_DONT_IMPLY_DEDENT = 0x200
+PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
def _maybe_compile(compiler, source, filename, symbol):
- # Check for source consisting of only blank lines and comments
+ # Check for source consisting of only blank lines and comments.
for line in source.split("\n"):
line = line.strip()
if line and line[0] != '#':
- break # Leave it alone
+ break # Leave it alone.
else:
if symbol != "eval":
source = "pass" # Replace it with a 'pass' statement
- err = err1 = err2 = None
- code = code1 = code2 = None
-
- try:
- code = compiler(source, filename, symbol)
- except SyntaxError:
- pass
-
- # Catch syntax warnings after the first compile
- # to emit warnings (SyntaxWarning, DeprecationWarning) at most once.
+ # Disable compiler warnings when checking for incomplete input.
with warnings.catch_warnings():
- warnings.simplefilter("error")
-
+ warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
try:
- code1 = compiler(source + "\n", filename, symbol)
- except SyntaxError as e:
- err1 = e
+ compiler(source, filename, symbol)
+ except SyntaxError: # Let other compile() errors propagate.
+ try:
+ compiler(source + "\n", filename, symbol)
+ return None
+ except SyntaxError as e:
+ if "incomplete input" in str(e):
+ return None
+ # fallthrough
- try:
- code2 = compiler(source + "\n\n", filename, symbol)
- except SyntaxError as e:
- err2 = e
+ return compiler(source, filename, symbol)
- try:
- if code:
- return code
- if not code1 and repr(err1) == repr(err2):
- raise err1
- finally:
- err1 = err2 = None
+def _is_syntax_error(err1, err2):
+ rep1 = repr(err1)
+ rep2 = repr(err2)
+ if "was never closed" in rep1 and "was never closed" in rep2:
+ return False
+ if rep1 == rep2:
+ return True
+ return False
def _compile(source, filename, symbol):
- return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT)
-
+ return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT)
def compile_command(source, filename="", symbol="single"):
r"""Compile a command and determine whether it is incomplete.
@@ -134,15 +106,13 @@ def compile_command(source, filename="", symbol="single"):
"""
return _maybe_compile(_compile, source, filename, symbol)
-
class Compile:
"""Instances of this class behave much like the built-in compile
function, but if one is used to compile text containing a future
statement, it "remembers" and compiles all subsequent program texts
with the statement in force."""
-
def __init__(self):
- self.flags = PyCF_DONT_IMPLY_DEDENT
+ self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
def __call__(self, source, filename, symbol):
codeob = compile(source, filename, symbol, self.flags, True)
@@ -151,7 +121,6 @@ class Compile:
self.flags |= feature.compiler_flag
return codeob
-
class CommandCompiler:
"""Instances of this class have __call__ methods identical in
signature to compile_command; the difference is that if the
diff --git a/Lib/test/test_opcode.py b/Lib/test/test_opcode.py
new file mode 100644
index 000000000..170eb1cb1
--- /dev/null
+++ b/Lib/test/test_opcode.py
@@ -0,0 +1,356 @@
+"""
+ Test cases for codeop.py
+ Nick Mathewson
+"""
+import sys
+import unittest
+import warnings
+from test import support
+from test.support import warnings_helper
+
+from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
+import io
+
+if support.is_jython:
+
+ def unify_callables(d):
+ for n,v in d.items():
+ if hasattr(v, '__call__'):
+ d[n] = True
+ return d
+
+class CodeopTests(unittest.TestCase):
+
+ def assertValid(self, str, symbol='single'):
+ '''succeed iff str is a valid piece of code'''
+ if support.is_jython:
+ code = compile_command(str, "", symbol)
+ self.assertTrue(code)
+ if symbol == "single":
+ d,r = {},{}
+ saved_stdout = sys.stdout
+ sys.stdout = io.StringIO()
+ try:
+ exec(code, d)
+ exec(compile(str,"","single"), r)
+ finally:
+ sys.stdout = saved_stdout
+ elif symbol == 'eval':
+ ctx = {'a': 2}
+ d = { 'value': eval(code,ctx) }
+ r = { 'value': eval(str,ctx) }
+ self.assertEqual(unify_callables(r),unify_callables(d))
+ else:
+ expected = compile(str, "", symbol, PyCF_DONT_IMPLY_DEDENT)
+ self.assertEqual(compile_command(str, "", symbol), expected)
+
+ def assertIncomplete(self, str, symbol='single'):
+ '''succeed iff str is the start of a valid piece of code'''
+ self.assertEqual(compile_command(str, symbol=symbol), None)
+
+ def assertInvalid(self, str, symbol='single', is_syntax=1):
+ '''succeed iff str is the start of an invalid piece of code'''
+ try:
+ compile_command(str,symbol=symbol)
+ self.fail("No exception raised for invalid code")
+ except SyntaxError:
+ self.assertTrue(is_syntax)
+ except OverflowError:
+ self.assertTrue(not is_syntax)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_valid(self):
+ av = self.assertValid
+
+ # special case
+ if not support.is_jython:
+ self.assertEqual(compile_command(""),
+ compile("pass", "", 'single',
+ PyCF_DONT_IMPLY_DEDENT))
+ self.assertEqual(compile_command("\n"),
+ compile("pass", "", 'single',
+ PyCF_DONT_IMPLY_DEDENT))
+ else:
+ av("")
+ av("\n")
+
+ av("a = 1")
+ av("\na = 1")
+ av("a = 1\n")
+ av("a = 1\n\n")
+ av("\n\na = 1\n\n")
+
+ av("def x():\n pass\n")
+ av("if 1:\n pass\n")
+
+ av("\n\nif 1: pass\n")
+ av("\n\nif 1: pass\n\n")
+
+ av("def x():\n\n pass\n")
+ av("def x():\n pass\n \n")
+ av("def x():\n pass\n \n")
+
+ av("pass\n")
+ av("3**3\n")
+
+ av("if 9==3:\n pass\nelse:\n pass\n")
+ av("if 1:\n pass\n if 1:\n pass\n else:\n pass\n")
+
+ av("#a\n#b\na = 3\n")
+ av("#a\n\n \na=3\n")
+ av("a=3\n\n")
+ av("a = 9+ \\\n3")
+
+ av("3**3","eval")
+ av("(lambda z: \n z**3)","eval")
+
+ av("9+ \\\n3","eval")
+ av("9+ \\\n3\n","eval")
+
+ av("\n\na**3","eval")
+ av("\n \na**3","eval")
+ av("#a\n#b\na**3","eval")
+
+ av("\n\na = 1\n\n")
+ av("\n\nif 1: a=1\n\n")
+
+ av("if 1:\n pass\n if 1:\n pass\n else:\n pass\n")
+ av("#a\n\n \na=3\n\n")
+
+ av("\n\na**3","eval")
+ av("\n \na**3","eval")
+ av("#a\n#b\na**3","eval")
+
+ av("def f():\n try: pass\n finally: [x for x in (1,2)]\n")
+ av("def f():\n pass\n#foo\n")
+ av("@a.b.c\ndef f():\n pass\n")
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_incomplete(self):
+ ai = self.assertIncomplete
+
+ ai("(a **")
+ ai("(a,b,")
+ ai("(a,b,(")
+ ai("(a,b,(")
+ ai("a = (")
+ ai("a = {")
+ ai("b + {")
+
+ ai("print([1,\n2,")
+ ai("print({1:1,\n2:3,")
+ ai("print((1,\n2,")
+
+ ai("if 9==3:\n pass\nelse:")
+ ai("if 9==3:\n pass\nelse:\n")
+ ai("if 9==3:\n pass\nelse:\n pass")
+ ai("if 1:")
+ ai("if 1:\n")
+ ai("if 1:\n pass\n if 1:\n pass\n else:")
+ ai("if 1:\n pass\n if 1:\n pass\n else:\n")
+ ai("if 1:\n pass\n if 1:\n pass\n else:\n pass")
+
+ ai("def x():")
+ ai("def x():\n")
+ ai("def x():\n\n")
+
+ ai("def x():\n pass")
+ ai("def x():\n pass\n ")
+ ai("def x():\n pass\n ")
+ ai("\n\ndef x():\n pass")
+
+ ai("a = 9+ \\")
+ ai("a = 'a\\")
+ ai("a = '''xy")
+
+ ai("","eval")
+ ai("\n","eval")
+ ai("(","eval")
+ ai("(9+","eval")
+ ai("9+ \\","eval")
+ ai("lambda z: \\","eval")
+
+ ai("if True:\n if True:\n if True: \n")
+
+ ai("@a(")
+ ai("@a(b")
+ ai("@a(b,")
+ ai("@a(b,c")
+ ai("@a(b,c,")
+
+ ai("from a import (")
+ ai("from a import (b")
+ ai("from a import (b,")
+ ai("from a import (b,c")
+ ai("from a import (b,c,")
+
+ ai("[")
+ ai("[a")
+ ai("[a,")
+ ai("[a,b")
+ ai("[a,b,")
+
+ ai("{")
+ ai("{a")
+ ai("{a:")
+ ai("{a:b")
+ ai("{a:b,")
+ ai("{a:b,c")
+ ai("{a:b,c:")
+ ai("{a:b,c:d")
+ ai("{a:b,c:d,")
+
+ ai("a(")
+ ai("a(b")
+ ai("a(b,")
+ ai("a(b,c")
+ ai("a(b,c,")
+
+ ai("a[")
+ ai("a[b")
+ ai("a[b,")
+ ai("a[b:")
+ ai("a[b:c")
+ ai("a[b:c:")
+ ai("a[b:c:d")
+
+ ai("def a(")
+ ai("def a(b")
+ ai("def a(b,")
+ ai("def a(b,c")
+ ai("def a(b,c,")
+
+ ai("(")
+ ai("(a")
+ ai("(a,")
+ ai("(a,b")
+ ai("(a,b,")
+
+ ai("if a:\n pass\nelif b:")
+ ai("if a:\n pass\nelif b:\n pass\nelse:")
+
+ ai("while a:")
+ ai("while a:\n pass\nelse:")
+
+ ai("for a in b:")
+ ai("for a in b:\n pass\nelse:")
+
+ ai("try:")
+ ai("try:\n pass\nexcept:")
+ ai("try:\n pass\nfinally:")
+ ai("try:\n pass\nexcept:\n pass\nfinally:")
+
+ ai("with a:")
+ ai("with a as b:")
+
+ ai("class a:")
+ ai("class a(")
+ ai("class a(b")
+ ai("class a(b,")
+ ai("class a():")
+
+ ai("[x for")
+ ai("[x for x in")
+ ai("[x for x in (")
+
+ ai("(x for")
+ ai("(x for x in")
+ ai("(x for x in (")
+
+ def test_invalid(self):
+ ai = self.assertInvalid
+ ai("a b")
+
+ ai("a @")
+ ai("a b @")
+ ai("a ** @")
+
+ ai("a = ")
+ ai("a = 9 +")
+
+ ai("def x():\n\npass\n")
+
+ ai("\n\n if 1: pass\n\npass")
+
+ ai("a = 9+ \\\n")
+ ai("a = 'a\\ ")
+ ai("a = 'a\\\n")
+
+ ai("a = 1","eval")
+ ai("]","eval")
+ ai("())","eval")
+ ai("[}","eval")
+ ai("9+","eval")
+ ai("lambda z:","eval")
+ ai("a b","eval")
+
+ ai("return 2.3")
+ ai("if (a == 1 and b = 2): pass")
+
+ ai("del 1")
+ ai("del (1,)")
+ ai("del [1]")
+ ai("del '1'")
+
+ ai("[i for i in range(10)] = (1, 2, 3)")
+
+ def test_invalid_exec(self):
+ ai = self.assertInvalid
+ ai("raise = 4", symbol="exec")
+ ai('def a-b', symbol='exec')
+ ai('await?', symbol='exec')
+ ai('=!=', symbol='exec')
+ ai('a await raise b', symbol='exec')
+ ai('a await raise b?+1', symbol='exec')
+
+ def test_filename(self):
+ self.assertEqual(compile_command("a = 1\n", "abc").co_filename,
+ compile("a = 1\n", "abc", 'single').co_filename)
+ self.assertNotEqual(compile_command("a = 1\n", "abc").co_filename,
+ compile("a = 1\n", "def", 'single').co_filename)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_warning(self):
+ # Test that the warning is only returned once.
+ with warnings_helper.check_warnings(
+ (".*literal", SyntaxWarning),
+ (".*invalid", DeprecationWarning),
+ ) as w:
+ compile_command(r"'\e' is 0")
+ self.assertEqual(len(w.warnings), 2)
+
+ # bpo-41520: check SyntaxWarning treated as an SyntaxError
+ with warnings.catch_warnings(), self.assertRaises(SyntaxError):
+ warnings.simplefilter('error', SyntaxWarning)
+ compile_command('1 is 1', symbol='exec')
+
+ # Check DeprecationWarning treated as an SyntaxError
+ with warnings.catch_warnings(), self.assertRaises(SyntaxError):
+ warnings.simplefilter('error', DeprecationWarning)
+ compile_command(r"'\e'", symbol='exec')
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_incomplete_warning(self):
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter('always')
+ self.assertIncomplete("'\\e' + (")
+ self.assertEqual(w, [])
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_invalid_warning(self):
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter('always')
+ self.assertInvalid("'\\e' 1")
+ self.assertEqual(len(w), 1)
+ self.assertEqual(w[0].category, DeprecationWarning)
+ self.assertRegex(str(w[0].message), 'invalid escape sequence')
+ self.assertEqual(w[0].filename, '')
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py
deleted file mode 100644
index e880c3f1a..000000000
--- a/Lib/test/test_opcodes.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# Python test set -- part 2, opcodes
-
-import unittest
-from test import ann_module, support
-
-class OpcodeTest(unittest.TestCase):
-
- def test_try_inside_for_loop(self):
- n = 0
- for i in range(10):
- n = n+i
- try: 1/0
- except NameError: pass
- except ZeroDivisionError: pass
- except TypeError: pass
- try: pass
- except: pass
- try: pass
- finally: pass
- n = n+i
- if n != 90:
- self.fail('try inside for')
-
- def test_setup_annotations_line(self):
- # check that SETUP_ANNOTATIONS does not create spurious line numbers
- try:
- with open(ann_module.__file__, encoding="utf-8") as f:
- txt = f.read()
- co = compile(txt, ann_module.__file__, 'exec')
- self.assertEqual(co.co_firstlineno, 1)
- except OSError:
- pass
-
- def test_default_annotations_exist(self):
- class C: pass
- self.assertEqual(C.__annotations__, {})
-
- def test_use_existing_annotations(self):
- ns = {'__annotations__': {1: 2}}
- exec('x: int', ns)
- self.assertEqual(ns['__annotations__'], {'x': int, 1: 2})
-
- def test_do_not_recreate_annotations(self):
- # Don't rely on the existence of the '__annotations__' global.
- with support.swap_item(globals(), '__annotations__', {}):
- del globals()['__annotations__']
- class C:
- del __annotations__
- with self.assertRaises(NameError):
- x: int
-
- def test_raise_class_exceptions(self):
-
- class AClass(Exception): pass
- class BClass(AClass): pass
- class CClass(Exception): pass
- class DClass(AClass):
- def __init__(self, ignore):
- pass
-
- try: raise AClass()
- except: pass
-
- try: raise AClass()
- except AClass: pass
-
- try: raise BClass()
- except AClass: pass
-
- try: raise BClass()
- except CClass: self.fail()
- except: pass
-
- a = AClass()
- b = BClass()
-
- try:
- raise b
- except AClass as v:
- self.assertEqual(v, b)
- else:
- self.fail("no exception")
-
- # not enough arguments
- ##try: raise BClass, a
- ##except TypeError: pass
- ##else: self.fail("no exception")
-
- try: raise DClass(a)
- except DClass as v:
- self.assertIsInstance(v, DClass)
- else:
- self.fail("no exception")
-
- def test_compare_function_objects(self):
-
- f = eval('lambda: None')
- g = eval('lambda: None')
- self.assertNotEqual(f, g)
-
- f = eval('lambda a: a')
- g = eval('lambda a: a')
- self.assertNotEqual(f, g)
-
- f = eval('lambda a=1: a')
- g = eval('lambda a=1: a')
- self.assertNotEqual(f, g)
-
- f = eval('lambda: 0')
- g = eval('lambda: 1')
- self.assertNotEqual(f, g)
-
- f = eval('lambda: None')
- g = eval('lambda a: None')
- self.assertNotEqual(f, g)
-
- f = eval('lambda a: None')
- g = eval('lambda b: None')
- self.assertNotEqual(f, g)
-
- f = eval('lambda a: None')
- g = eval('lambda a=None: None')
- self.assertNotEqual(f, g)
-
- f = eval('lambda a=0: None')
- g = eval('lambda a=1: None')
- self.assertNotEqual(f, g)
-
- def test_modulo_of_string_subclasses(self):
- class MyString(str):
- def __mod__(self, value):
- return 42
- self.assertEqual(MyString() % 3, 42)
-
-
-if __name__ == '__main__':
- unittest.main()