Update to ruff_python_parser 0.11

2026-06-02 19:39:49 +09:00 · 2025-03-19 21:37:06 -05:00
parent a6b4ef7f5d
commit 950a8d5694
13 changed files with 83 additions and 74 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -214,6 +214,15 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"

+[[package]]
+name = "castaway"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "cc"
 version = "1.2.14"
@@ -269,6 +278,20 @@ dependencies = [
 "error-code",
 ]

+[[package]]
+name = "compact_str"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32"
+dependencies = [
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "rustversion",
+ "ryu",
+ "static_assertions",
+]
+
 [[package]]
 name = "console"
 version = "0.15.10"
@@ -360,7 +383,7 @@ dependencies = [
 "hashbrown 0.14.5",
 "log",
 "regalloc2",
- "rustc-hash 2.1.1",
+ "rustc-hash",
 "serde",
 "smallvec",
 "target-lexicon 0.13.2",
@@ -1007,15 +1030,6 @@ dependencies = [
 "either",
 ]

-[[package]]
-name = "itertools"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itertools"
 version = "0.14.0"
@@ -1895,7 +1909,7 @@ dependencies = [
 "bumpalo",
 "hashbrown 0.15.2",
 "log",
- "rustc-hash 2.1.1",
+ "rustc-hash",
 "smallvec",
 ]

@@ -1964,25 +1978,25 @@ dependencies = [
 [[package]]
 name = "ruff_python_ast"
 version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
+source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a"
 dependencies = [
 "aho-corasick",
 "bitflags 2.8.0",
+ "compact_str",
 "is-macro",
- "itertools 0.13.0",
- "once_cell",
+ "itertools 0.14.0",
+ "memchr",
 "ruff_python_trivia",
 "ruff_source_file",
 "ruff_text_size",
- "rustc-hash 1.1.0",
+ "rustc-hash",
 ]

 [[package]]
 name = "ruff_python_codegen"
 version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
+source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a"
 dependencies = [
- "once_cell",
 "ruff_python_ast",
 "ruff_python_literal",
 "ruff_python_parser",
@@ -1993,10 +2007,10 @@ dependencies = [
 [[package]]
 name = "ruff_python_literal"
 version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
+source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a"
 dependencies = [
 "bitflags 2.8.0",
- "itertools 0.13.0",
+ "itertools 0.14.0",
 "ruff_python_ast",
 "unic-ucd-category",
 ]
@@ -2004,15 +2018,16 @@ dependencies = [
 [[package]]
 name = "ruff_python_parser"
 version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
+source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a"
 dependencies = [
 "bitflags 2.8.0",
 "bstr",
+ "compact_str",
 "memchr",
 "ruff_python_ast",
 "ruff_python_trivia",
 "ruff_text_size",
- "rustc-hash 1.1.0",
+ "rustc-hash",
 "static_assertions",
 "unicode-ident",
 "unicode-normalization",
@@ -2022,9 +2037,9 @@ dependencies = [
 [[package]]
 name = "ruff_python_trivia"
 version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
+source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a"
 dependencies = [
- "itertools 0.13.0",
+ "itertools 0.14.0",
 "ruff_source_file",
 "ruff_text_size",
 "unicode-ident",
@@ -2033,23 +2048,16 @@ dependencies = [
 [[package]]
 name = "ruff_source_file"
 version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
+source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a"
 dependencies = [
 "memchr",
- "once_cell",
 "ruff_text_size",
 ]

 [[package]]
 name = "ruff_text_size"
 version = "0.0.0"
-source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
-
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a"

 [[package]]
 name = "rustc-hash"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -122,11 +122,11 @@ rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.
 rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
 rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }

-ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
-ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
-ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
-ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
-ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
+ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
+ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
 # rustpython-literal = { version = "0.4.0" }
 # rustpython-parser-core = { version = "0.4.0" }
 # rustpython-parser = { version = "0.4.0" }
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -1619,8 +1619,6 @@ class GrammarTests(unittest.TestCase):
        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
                             ('Macdonalds', 'Cheeseburger')])

-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
    def test_genexps(self):
        # generator expression tests
        g = ([x for x in range(10)] for x in range(1))
--- a/compiler/codegen/src/compile.rs
+++ b/compiler/codegen/src/compile.rs
@@ -2029,8 +2029,7 @@ impl Compiler<'_> {
        if self.future_annotations {
            // FIXME: codegen?
            let ident = Default::default();
-            let codegen =
-                ruff_python_codegen::Generator::new(&ident, Default::default(), Default::default());
+            let codegen = ruff_python_codegen::Generator::new(&ident, Default::default());
            self.emit_load_const(ConstantData::Str {
                value: codegen.expr(annotation),
            });
@@ -3595,18 +3594,19 @@ impl ToU32 for usize {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use ruff_python_ast::name::Name;
    use ruff_python_ast::*;

    /// Test if the compiler can correctly identify fstrings containing an `await` expression.
    #[test]
    fn test_fstring_contains_await() {
        let range = TextRange::default();
-        let flags = FStringFlags::default();
+        let flags = FStringFlags::empty();

        // f'{x}'
        let expr_x = Expr::Name(ExprName {
            range,
-            id: "x".to_owned(),
+            id: Name::new("x"),
            ctx: ExprContext::Load,
        });
        let not_present = &Expr::FString(ExprFString {
@@ -3631,7 +3631,7 @@ mod tests {
            range,
            value: Box::new(Expr::Name(ExprName {
                range,
-                id: "x".to_owned(),
+                id: Name::new("x"),
                ctx: ExprContext::Load,
            })),
        });
@@ -3655,14 +3655,14 @@ mod tests {
        // f'{x:{await y}}'
        let expr_x = Expr::Name(ExprName {
            range,
-            id: "x".to_owned(),
+            id: Name::new("x"),
            ctx: ExprContext::Load,
        });
        let expr_await_y = Expr::Await(ExprAwait {
            range,
            value: Box::new(Expr::Name(ExprName {
                range,
-                id: "y".to_owned(),
+                id: Name::new("y"),
                ctx: ExprContext::Load,
            })),
        });
--- a/compiler/src/lib.rs
+++ b/compiler/src/lib.rs
@@ -119,7 +119,7 @@ fn _compile(
    mode: Mode,
    opts: CompileOpts,
 ) -> Result<CodeObject, CompileError> {
-    let parsed = parser::parse(source_code.text, mode.into())
+    let parsed = parser::parse(source_code.text, parser::Mode::from(mode).into())
        .map_err(|err| CompileError::from_ruff_parse_error(err, &source_code))?;
    let ast = parsed.into_syntax();
    compile::compile_top(ast, source_code, mode, opts).map_err(|e| e.into())
@@ -145,9 +145,8 @@ pub fn _compile_symtable(
            symboltable::SymbolTable::scan_program(&ast.into_syntax(), source_code.clone())
        }
        Mode::Eval => {
-            let ast =
-                ruff_python_parser::parse(source_code.text, ruff_python_parser::Mode::Expression)
-                    .map_err(|e| CompileError::from_ruff_parse_error(e, &source_code))?;
+            let ast = ruff_python_parser::parse(source_code.text, parser::Mode::Expression.into())
+                .map_err(|e| CompileError::from_ruff_parse_error(e, &source_code))?;
            symboltable::SymbolTable::scan_expr(
                &ast.into_syntax().expect_expression(),
                source_code.clone(),
--- a/src/shell.rs
+++ b/src/shell.rs
@@ -1,7 +1,7 @@
 mod helper;

 use rustpython_compiler::{
-    CompileError, ParseError, parser::ParseErrorType, parser::lexer::LexicalErrorType,
+    CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType,
 };
 use rustpython_vm::{
    AsObject, PyResult, VirtualMachine,
--- a/vm/src/stdlib/ast.rs
+++ b/vm/src/stdlib/ast.rs
@@ -242,7 +242,7 @@ pub(crate) fn parse(
    mode: parser::Mode,
 ) -> Result<PyObjectRef, CompileError> {
    let source_code = SourceCodeOwned::new("".to_owned(), source.to_owned());
-    let top = parser::parse(source, mode)
+    let top = parser::parse(source, mode.into())
        .map_err(|parse_error| ParseError {
            error: parse_error.error,
            location: text_range_to_source_range(&source_code, parse_error.location)
--- a/vm/src/stdlib/ast/constant.rs
+++ b/vm/src/stdlib/ast/constant.rs
@@ -255,7 +255,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr {
                value: ruff::StringLiteralValue::single(ruff::StringLiteral {
                    range,
                    value,
-                    flags: ruff::StringLiteralFlags::default().with_prefix(prefix),
+                    flags: ruff::StringLiteralFlags::empty().with_prefix(prefix),
                }),
            })
        }
@@ -265,7 +265,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr {
                value: ruff::BytesLiteralValue::single(ruff::BytesLiteral {
                    range,
                    value,
-                    flags: Default::default(), // TODO
+                    flags: ruff::BytesLiteralFlags::empty(), // TODO
                }),
            })
        }
@@ -293,7 +293,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr {
            // idk lol
            func: Box::new(ruff::Expr::Name(ruff::ExprName {
                range: TextRange::default(),
-                id: "frozenset".to_owned(),
+                id: ruff::name::Name::new_static("frozenset"),
                ctx: ruff::ExprContext::Load,
            })),
            arguments: ruff::Arguments {
--- a/vm/src/stdlib/ast/expression.rs
+++ b/vm/src/stdlib/ast/expression.rs
@@ -977,7 +977,7 @@ impl Node for ruff::ExprName {
        object: PyObjectRef,
    ) -> PyResult<Self> {
        Ok(Self {
-            id: get_node_field(vm, &object, "id", "Name")?.try_into_value(vm)?,
+            id: Node::ast_from_object(vm, source_code, get_node_field(vm, &object, "id", "Name")?)?,
            ctx: Node::ast_from_object(
                vm,
                source_code,
--- a/vm/src/stdlib/ast/other.rs
+++ b/vm/src/stdlib/ast/other.rs
@@ -19,18 +19,22 @@ impl Node for ruff::ConversionFlag {
 }

 // /// This is just a string, not strictly an AST node. But it makes AST conversions easier.
-// impl Node for ruff::name::Name {
-//     fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
-//         vm.ctx.new_str(self.as_str()).to_pyobject(vm)
-//     }
+impl Node for ruff::name::Name {
+    fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
+        vm.ctx.new_str(self.as_str()).to_pyobject(vm)
+    }

-//     fn ast_from_object(vm: &VirtualMachine, source_code: &SourceCodeOwned, object: PyObjectRef) -> PyResult<Self> {
-//         match object.downcast::<PyStr>() {
-//             Ok(name) => Ok(Self::new(name)),
-//             Err(_) => Err(vm.new_value_error("expected str for name".to_owned())),
-//         }
-//     }
-// }
+    fn ast_from_object(
+        vm: &VirtualMachine,
+        _source_code: &SourceCodeOwned,
+        object: PyObjectRef,
+    ) -> PyResult<Self> {
+        match object.downcast::<PyStr>() {
+            Ok(name) => Ok(Self::new(name)),
+            Err(_) => Err(vm.new_value_error("expected str for name".to_owned())),
+        }
+    }
+}

 impl Node for ruff::Decorator {
    fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
--- a/vm/src/stdlib/ast/string.rs
+++ b/vm/src/stdlib/ast/string.rs
@@ -7,7 +7,7 @@ fn ruff_fstring_value_into_iter(
    let default = ruff::FStringPart::FString(ruff::FString {
        range: Default::default(),
        elements: Default::default(),
-        flags: Default::default(),
+        flags: ruff::FStringFlags::empty(),
    });
    (0..fstring_value.as_slice().len()).map(move |i| {
        let fstring_value = &mut fstring_value;
@@ -100,7 +100,7 @@ fn ruff_fstring_element_to_ruff_fstring_part(element: ruff::FStringElement) -> r
            ruff::FStringPart::Literal(ruff::StringLiteral {
                range,
                value,
-                flags: Default::default(),
+                flags: ruff::StringLiteralFlags::empty(),
            })
        }
        ruff::FStringElement::Expression(value) => {
@@ -123,7 +123,7 @@ fn ruff_fstring_element_to_ruff_fstring_part(element: ruff::FStringElement) -> r
                    },
                )]
                .into(),
-                flags: Default::default(),
+                flags: ruff::FStringFlags::empty(),
            })
        }
    }
@@ -164,7 +164,7 @@ impl JoinedStr {
                0 => ruff::FStringValue::single(ruff::FString {
                    range,
                    elements: vec![].into(),
-                    flags: Default::default(),
+                    flags: ruff::FStringFlags::empty(),
                }),
                1 => ruff::FStringValue::single(
                    Box::<[_]>::into_iter(values)
@@ -172,7 +172,7 @@ impl JoinedStr {
                        .map(|element| ruff::FString {
                            range,
                            elements: vec![element].into(),
-                            flags: Default::default(),
+                            flags: ruff::FStringFlags::empty(),
                        })
                        .next()
                        .expect("FString has exactly one part"),
--- a/vm/src/vm/vm_new.rs
+++ b/vm/src/vm/vm_new.rs
@@ -270,7 +270,7 @@ impl VirtualMachine {
            crate::compiler::CompileError::Parse(rustpython_compiler::ParseError {
                error:
                    ruff_python_parser::ParseErrorType::Lexical(
-                        ruff_python_parser::lexer::LexicalErrorType::IndentationError,
+                        ruff_python_parser::LexicalErrorType::IndentationError,
                    ),
                ..
            }) => self.ctx.exceptions.tab_error,
--- a/wasm/lib/src/convert.rs
+++ b/wasm/lib/src/convert.rs
@@ -6,7 +6,7 @@ use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Arr
 use rustpython_vm::{
    AsObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine,
    builtins::PyBaseExceptionRef,
-    compiler::{CompileError, ParseError, parser::ParseErrorType, parser::lexer::LexicalErrorType},
+    compiler::{CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType},
    exceptions,
    function::{ArgBytesLike, FuncArgs},
    py_serde,