Revert "Use ruff for Expr unparsing (#6124)"

This reverts commit 0fb7d0fae2.

Cargo.lock

@@ -2282,29 +2282,6 @@ dependencies = [
  "thiserror 2.0.17",
 ]
 
-[[package]]
-name = "ruff_python_codegen"
-version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8"
-dependencies = [
- "ruff_python_ast",
- "ruff_python_literal",
- "ruff_python_parser",
- "ruff_source_file",
- "ruff_text_size",
-]
-
-[[package]]
-name = "ruff_python_literal"
-version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?tag=0.14.1#2bffef59665ce7d2630dfd72ee99846663660db8"
-dependencies = [
- "bitflags 2.9.4",
- "itertools 0.14.0",
- "ruff_python_ast",
- "unic-ucd-category",
-]
-
 [[package]]
 name = "ruff_python_parser"
 version = "0.0.0"
@@ -2410,9 +2387,7 @@ dependencies = [
  "num-complex",
  "num-traits",
  "ruff_python_ast",
- "ruff_python_codegen",
  "ruff_python_parser",
- "ruff_source_file",
  "ruff_text_size",
  "rustpython-compiler-core",
  "rustpython-literal",

Cargo.toml

@@ -159,7 +159,6 @@ rustpython-wtf8 = { path = "wtf8", version = "0.4.0" }
 rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
 
 ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" }
-ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" }
 ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" }
 ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" }
 ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" }

Lib/test/test_future_stmt/badsyntax_future3.py

@@ -0,0 +1,10 @@
+"""This is a test"""
+from __future__ import nested_scopes
+from __future__ import rested_snopes
+
+def f(x):
+    def g(y):
+        return x + y
+    return g
+
+result = f(2)(4)

Lib/test/test_future_stmt/badsyntax_future4.py

@@ -0,0 +1,10 @@
+"""This is a test"""
+import __future__
+from __future__ import nested_scopes
+
+def f(x):
+    def g(y):
+        return x + y
+    return g
+
+result = f(2)(4)

Lib/test/test_future_stmt/badsyntax_future5.py

@@ -0,0 +1,12 @@
+"""This is a test"""
+from __future__ import nested_scopes
+import foo
+from __future__ import nested_scopes
+
+
+def f(x):
+    def g(y):
+        return x + y
+    return g
+
+result = f(2)(4)

Lib/test/test_future_stmt/badsyntax_future6.py

@@ -0,0 +1,10 @@
+"""This is a test"""
+"this isn't a doc string"
+from __future__ import nested_scopes
+
+def f(x):
+    def g(y):
+        return x + y
+    return g
+
+result = f(2)(4)

Lib/test/test_future_stmt/badsyntax_future7.py

@@ -0,0 +1,11 @@
+"""This is a test"""
+
+from __future__ import nested_scopes; import string; from __future__ import \
+     nested_scopes
+
+def f(x):
+    def g(y):
+        return x + y
+    return g
+
+result = f(2)(4)

Lib/test/test_future_stmt/badsyntax_future8.py

@@ -0,0 +1,10 @@
+"""This is a test"""
+
+from __future__ import *
+
+def f(x):
+    def g(y):
+        return x + y
+    return g
+
+print(f(2)(4))

Lib/test/test_future_stmt/badsyntax_future9.py

@@ -0,0 +1,10 @@
+"""This is a test"""
+
+from __future__ import nested_scopes, braces
+
+def f(x):
+    def g(y):
+        return x + y
+    return g
+
+print(f(2)(4))

Lib/test/test_future_stmt/test_future.py

@@ -10,8 +10,6 @@ import os
 import re
 import sys
 
-TOP_LEVEL_MSG = 'from __future__ imports must occur at the beginning of the file'
-
 rx = re.compile(r'\((\S+).py, line (\d+)')
 
 def get_error_location(msg):
@@ -20,48 +18,21 @@ def get_error_location(msg):
 
 class FutureTest(unittest.TestCase):
 
-    def check_syntax_error(self, err, basename,
-                           *,
-                           lineno,
-                           message=TOP_LEVEL_MSG, offset=1):
-        if basename != '<string>':
-            basename += '.py'
-
-        self.assertEqual(f'{message} ({basename}, line {lineno})', str(err))
-        self.assertEqual(os.path.basename(err.filename), basename)
+    def check_syntax_error(self, err, basename, lineno, offset=1):
+        self.assertIn('%s.py, line %d' % (basename, lineno), str(err))
+        self.assertEqual(os.path.basename(err.filename), basename + '.py')
         self.assertEqual(err.lineno, lineno)
         self.assertEqual(err.offset, offset)
 
-    def assertSyntaxError(self, code,
-                          *,
-                          lineno=1,
-                          message=TOP_LEVEL_MSG, offset=1,
-                          parametrize_docstring=True):
-        code = dedent(code.lstrip('\n'))
-        for add_docstring in ([False, True] if parametrize_docstring else [False]):
-            with self.subTest(code=code, add_docstring=add_docstring):
-                if add_docstring:
-                    code = '"""Docstring"""\n' + code
-                    lineno += 1
-                with self.assertRaises(SyntaxError) as cm:
-                    exec(code)
-                self.check_syntax_error(cm.exception, "<string>",
-                                        lineno=lineno,
-                                        message=message,
-                                        offset=offset)
+    def test_future1(self):
+        with import_helper.CleanImport('test.test_future_stmt.future_test1'):
+            from test.test_future_stmt import future_test1
+            self.assertEqual(future_test1.result, 6)
 
-    def test_import_nested_scope_twice(self):
-        # Import the name nested_scopes twice to trigger SF bug #407394
-        with import_helper.CleanImport(
-            'test.test_future_stmt.import_nested_scope_twice',
-        ):
-            from test.test_future_stmt import import_nested_scope_twice
-        self.assertEqual(import_nested_scope_twice.result, 6)
-
-    def test_nested_scope(self):
-        with import_helper.CleanImport('test.test_future_stmt.nested_scope'):
-            from test.test_future_stmt import nested_scope
-        self.assertEqual(nested_scope.result, 6)
+    def test_future2(self):
+        with import_helper.CleanImport('test.test_future_stmt.future_test2'):
+            from test.test_future_stmt import future_test2
+            self.assertEqual(future_test2.result, 6)
 
     def test_future_single_import(self):
         with import_helper.CleanImport(
@@ -81,87 +52,47 @@ class FutureTest(unittest.TestCase):
         ):
             from test.test_future_stmt import test_future_multiple_features
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message
-    def test_unknown_future_flag(self):
-        code = """
-            from __future__ import nested_scopes
-            from __future__ import rested_snopes  # typo error here: nested => rested
-        """
-        self.assertSyntaxError(
-            code, lineno=2,
-            message='future feature rested_snopes is not defined', offset=24,
-        )
-
-    @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message
-    def test_future_import_not_on_top(self):
-        code = """
-            import some_module
-            from __future__ import annotations
-        """
-        self.assertSyntaxError(code, lineno=2)
-
-        code = """
-            import __future__
-            from __future__ import annotations
-        """
-        self.assertSyntaxError(code, lineno=2)
-
-        code = """
-            from __future__ import absolute_import
-            "spam, bar, blah"
-            from __future__ import print_function
-        """
-        self.assertSyntaxError(code, lineno=3)
-
-    @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message
-    def test_future_import_with_extra_string(self):
-        code = """
-            '''Docstring'''
-            "this isn't a doc string"
-            from __future__ import nested_scopes
-        """
-        self.assertSyntaxError(code, lineno=3, parametrize_docstring=False)
-
-    @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message
-    def test_multiple_import_statements_on_same_line(self):
-        # With `\`:
-        code = """
-            from __future__ import nested_scopes; import string; from __future__ import \
-        nested_scopes
-        """
-        self.assertSyntaxError(code, offset=54)
-
-        # Without `\`:
-        code = """
-            from __future__ import nested_scopes; import string; from __future__ import  nested_scopes
-        """
-        self.assertSyntaxError(code, offset=54)
-
-    @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message
-    def test_future_import_star(self):
-        code = """
-            from __future__ import *
-        """
-        self.assertSyntaxError(code, message='future feature * is not defined', offset=24)
-
-    @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message
-    def test_future_import_braces(self):
-        code = """
-            from __future__ import braces
-        """
-        # Congrats, you found an easter egg!
-        self.assertSyntaxError(code, message='not a chance', offset=24)
-
-        code = """
-            from __future__ import nested_scopes, braces
-        """
-        self.assertSyntaxError(code, message='not a chance', offset=39)
-
-    @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message
-    def test_module_with_future_import_not_on_top(self):
+    def test_badfuture3(self):
         with self.assertRaises(SyntaxError) as cm:
-            from test.test_future_stmt import badsyntax_future
-        self.check_syntax_error(cm.exception, "badsyntax_future", lineno=3)
+            from test.test_future_stmt import badsyntax_future3
+        self.check_syntax_error(cm.exception, "badsyntax_future3", 3)
+
+    def test_badfuture4(self):
+        with self.assertRaises(SyntaxError) as cm:
+            from test.test_future_stmt import badsyntax_future4
+        self.check_syntax_error(cm.exception, "badsyntax_future4", 3)
+
+    def test_badfuture5(self):
+        with self.assertRaises(SyntaxError) as cm:
+            from test.test_future_stmt import badsyntax_future5
+        self.check_syntax_error(cm.exception, "badsyntax_future5", 4)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_badfuture6(self):
+        with self.assertRaises(SyntaxError) as cm:
+            from test.test_future_stmt import badsyntax_future6
+        self.check_syntax_error(cm.exception, "badsyntax_future6", 3)
+
+    def test_badfuture7(self):
+        with self.assertRaises(SyntaxError) as cm:
+            from test.test_future_stmt import badsyntax_future7
+        self.check_syntax_error(cm.exception, "badsyntax_future7", 3, 54)
+
+    def test_badfuture8(self):
+        with self.assertRaises(SyntaxError) as cm:
+            from test.test_future_stmt import badsyntax_future8
+        self.check_syntax_error(cm.exception, "badsyntax_future8", 3)
+
+    def test_badfuture9(self):
+        with self.assertRaises(SyntaxError) as cm:
+            from test.test_future_stmt import badsyntax_future9
+        self.check_syntax_error(cm.exception, "badsyntax_future9", 3)
+
+    def test_badfuture10(self):
+        with self.assertRaises(SyntaxError) as cm:
+            from test.test_future_stmt import badsyntax_future10
+        self.check_syntax_error(cm.exception, "badsyntax_future10", 3)
 
     def test_ensure_flags_dont_clash(self):
         # bpo-39562: test that future flags and compiler flags doesn't clash
@@ -178,6 +109,26 @@ class FutureTest(unittest.TestCase):
         }
         self.assertCountEqual(set(flags.values()), flags.values())
 
+    def test_parserhack(self):
+        # test that the parser.c::future_hack function works as expected
+        # Note: although this test must pass, it's not testing the original
+        #       bug as of 2.6 since the with statement is not optional and
+        #       the parser hack disabled. If a new keyword is introduced in
+        #       2.6, change this to refer to the new future import.
+        try:
+            exec("from __future__ import print_function; print 0")
+        except SyntaxError:
+            pass
+        else:
+            self.fail("syntax error didn't occur")
+
+        try:
+            exec("from __future__ import (print_function); print 0")
+        except SyntaxError:
+            pass
+        else:
+            self.fail("syntax error didn't occur")
+
     def test_unicode_literals_exec(self):
         scope = {}
         exec("from __future__ import unicode_literals; x = ''", {}, scope)
@@ -190,26 +141,6 @@ class FutureTest(unittest.TestCase):
         out = kill_python(p)
         self.assertNotIn(b'SyntaxError: invalid syntax', out)
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: future feature spam is not defined
-    def test_future_dotted_import(self):
-        with self.assertRaises(ImportError):
-            exec("from .__future__ import spam")
-
-        code = dedent(
-            """
-            from __future__ import print_function
-            from ...__future__ import ham
-            """
-        )
-        with self.assertRaises(ImportError):
-            exec(code)
-
-        code = """
-            from .__future__ import nested_scopes
-            from __future__ import barry_as_FLUFL
-        """
-        self.assertSyntaxError(code, lineno=2)
-
 class AnnotationsFutureTestCase(unittest.TestCase):
     template = dedent(
         """
@@ -267,7 +198,6 @@ class AnnotationsFutureTestCase(unittest.TestCase):
         )
         return scope
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; 'a,' != '(a,)'
     def test_annotations(self):
         eq = self.assertAnnotationEqual
         eq('...')
@@ -432,7 +362,6 @@ class AnnotationsFutureTestCase(unittest.TestCase):
         eq('(((a, b)))', '(a, b)')
         eq("1 + 2 + 3")
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; "f'{x=!r}'" != "f'x={x!r}'"
     def test_fstring_debug_annotations(self):
         # f-strings with '=' don't round trip very well, so set the expected
         # result explicitly.
@@ -443,7 +372,6 @@ class AnnotationsFutureTestCase(unittest.TestCase):
         self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'")
         self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'")
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; '1e309, 1e309j' != '(1e309, 1e309j)'
     def test_infinity_numbers(self):
         inf = "1e" + repr(sys.float_info.max_10_exp + 1)
         infj = f"{inf}j"
@@ -456,7 +384,8 @@ class AnnotationsFutureTestCase(unittest.TestCase):
         self.assertAnnotationEqual("('inf', 1e1000, 'infxxx', 1e1000j)", expected=f"('inf', {inf}, 'infxxx', {infj})")
         self.assertAnnotationEqual("(1e1000, (1e1000j,))", expected=f"({inf}, ({infj},))")
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_annotation_with_complex_target(self):
         with self.assertRaises(SyntaxError):
             exec(
@@ -480,7 +409,8 @@ class AnnotationsFutureTestCase(unittest.TestCase):
         self.assertEqual(foo.__code__.co_cellvars, ())
         self.assertEqual(foo().__code__.co_freevars, ())
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; "f'{x=!r}'" != "f'x={x!r}'"
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_annotations_forbidden(self):
         with self.assertRaises(SyntaxError):
             self._exec_future("test: (yield)")

compiler/codegen/Cargo.toml

@@ -13,9 +13,6 @@ rustpython-compiler-core = { workspace = true }
 rustpython-literal = {workspace = true }
 rustpython-wtf8 = { workspace = true }
 ruff_python_ast = { workspace = true }
-ruff_python_parser = { workspace = true }
-ruff_python_codegen = { workspace = true }
-ruff_source_file = { workspace = true }
 ruff_text_size = { workspace = true }
 
 ahash = { workspace = true }

compiler/codegen/src/compile.rs

@@ -14,6 +14,7 @@ use crate::{
     error::{CodegenError, CodegenErrorType, InternalError, PatternUnreachableReason},
     ir::{self, BlockIdx},
     symboltable::{self, CompilerScope, SymbolFlags, SymbolScope, SymbolTable},
+    unparse::UnparseExpr,
 };
 use itertools::Itertools;
 use malachite_bigint::BigInt;
@@ -30,7 +31,6 @@ use ruff_python_ast::{
     PatternMatchStar, PatternMatchValue, Singleton, Stmt, StmtExpr, TypeParam, TypeParamParamSpec,
     TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem,
 };
-use ruff_source_file::LineEnding;
 use ruff_text_size::{Ranged, TextRange};
 use rustpython_compiler_core::{
     Mode, OneIndexed, PositionEncoding, SourceFile, SourceLocation,
@@ -147,15 +147,6 @@ enum ComprehensionType {
     Dict,
 }
 
-fn unparse_expr(expr: &Expr) -> String {
-    use ruff_python_ast::str::Quote;
-    use ruff_python_codegen::{Generator, Indentation};
-
-    Generator::new(&Indentation::default(), LineEnding::default())
-        .with_preferred_quote(Some(Quote::Single))
-        .expr(expr)
-}
-
 fn validate_duplicate_params(params: &Parameters) -> Result<(), CodegenErrorType> {
     let mut seen_params = HashSet::new();
     for param in params {
@@ -3618,7 +3609,7 @@ impl Compiler {
                         | Expr::NoneLiteral(_)
                 );
                 let key_repr = if is_literal {
-                    unparse_expr(key)
+                    UnparseExpr::new(key, &self.source_file).to_string()
                 } else if is_attribute {
                     String::new()
                 } else {
@@ -4172,7 +4163,9 @@ impl Compiler {
     fn compile_annotation(&mut self, annotation: &Expr) -> CompileResult<()> {
         if self.future_annotations {
             self.emit_load_const(ConstantData::Str {
-                value: unparse_expr(annotation).into(),
+                value: UnparseExpr::new(annotation, &self.source_file)
+                    .to_string()
+                    .into(),
             });
         } else {
             let was_in_annotation = self.in_annotation;

compiler/codegen/src/lib.rs

@@ -13,6 +13,7 @@ pub mod error;
 pub mod ir;
 mod string_parser;
 pub mod symboltable;
+mod unparse;
 
 pub use compile::CompileOpts;
 use ruff_python_ast::Expr;