diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py index 687d564e9..8f3dcd72e 100755 --- a/ast/asdl_rs.py +++ b/ast/asdl_rs.py @@ -23,6 +23,7 @@ builtin_type_mapping = { } assert builtin_type_mapping.keys() == asdl.builtin_types + def get_rust_type(name): """Return a string for the C name of the type. @@ -33,6 +34,7 @@ def get_rust_type(name): else: return "".join(part.capitalize() for part in name.split("_")) + def is_simple(sum): """Return True if a sum is a simple. @@ -44,6 +46,7 @@ def is_simple(sum): return False return True + def asdl_of(name, obj): if isinstance(obj, asdl.Product) or isinstance(obj, asdl.Constructor): fields = ", ".join(map(str, obj.fields)) @@ -55,11 +58,10 @@ def asdl_of(name, obj): types = " | ".join(type.name for type in obj.types) else: sep = "\n{}| ".format(" " * (len(name) + 1)) - types = sep.join( - asdl_of(type.name, type) for type in obj.types - ) + types = sep.join(asdl_of(type.name, type) for type in obj.types) return "{} = {}".format(name, types) + class EmitVisitor(asdl.VisitorBase): """Visit that emits lines""" @@ -80,6 +82,7 @@ class EmitVisitor(asdl.VisitorBase): line = (" " * TABSIZE * depth) + line self.file.write(line + "\n") + class TypeInfo: def __init__(self, name): self.name = name @@ -105,6 +108,7 @@ class TypeInfo: stack.remove(self.name) return self.has_userdata + class FindUserdataTypesVisitor(asdl.VisitorBase): def __init__(self, typeinfo): self.typeinfo = typeinfo @@ -146,12 +150,14 @@ class FindUserdataTypesVisitor(asdl.VisitorBase): def add_children(self, name, fields): self.typeinfo[name].children.update((field.type, field.seq) for field in fields) + def rust_field(field_name): - if field_name == 'type': - return 'type_' + if field_name == "type": + return "type_" else: return field_name + class TypeInfoEmitVisitor(EmitVisitor): def __init__(self, file, typeinfo): self.typeinfo = typeinfo @@ -166,6 +172,7 @@ class TypeInfoEmitVisitor(EmitVisitor): else: return ["" for g in generics] + class StructVisitor(TypeInfoEmitVisitor): """Visitor to generate typedefs for AST.""" @@ -208,7 +215,10 @@ class StructVisitor(TypeInfoEmitVisitor): self.visit(t, typeinfo, depth + 1) self.emit("}", depth) if sum.attributes: - self.emit(f"pub type {rustname} = Located<{enumname}{generics_applied}, U>;", depth) + self.emit( + f"pub type {rustname} = Located<{enumname}{generics_applied}, U>;", + depth, + ) self.emit("", depth) def visitConstructor(self, cons, parent, depth): @@ -257,7 +267,10 @@ class FoldTraitDefVisitor(TypeInfoEmitVisitor): self.emit("pub trait Fold {", depth) self.emit("type TargetU;", depth + 1) self.emit("type Error;", depth + 1) - self.emit("fn map_user(&mut self, user: U) -> Result;", depth + 2) + self.emit( + "fn map_user(&mut self, user: U) -> Result;", + depth + 2, + ) for dfn in mod.dfns: self.visit(dfn, depth + 2) self.emit("}", depth) @@ -266,15 +279,24 @@ class FoldTraitDefVisitor(TypeInfoEmitVisitor): name = type.name apply_u, apply_target_u = self.get_generics(name, "U", "Self::TargetU") enumname = get_rust_type(name) - self.emit(f"fn fold_{name}(&mut self, node: {enumname}{apply_u}) -> Result<{enumname}{apply_target_u}, Self::Error> {{", depth) + self.emit( + f"fn fold_{name}(&mut self, node: {enumname}{apply_u}) -> Result<{enumname}{apply_target_u}, Self::Error> {{", + depth, + ) self.emit(f"fold_{name}(self, node)", depth + 1) self.emit("}", depth) class FoldImplVisitor(TypeInfoEmitVisitor): def visitModule(self, mod, depth): - self.emit("fn fold_located + ?Sized, T, MT>(folder: &mut F, node: Located, f: impl FnOnce(&mut F, T) -> Result) -> Result, F::Error> {", depth) - self.emit("Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, node: f(folder, node.node)? })", depth + 1) + self.emit( + "fn fold_located + ?Sized, T, MT>(folder: &mut F, node: Located, f: impl FnOnce(&mut F, T) -> Result) -> Result, F::Error> {", + depth, + ) + self.emit( + "Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, node: f(folder, node.node)? })", + depth + 1, + ) self.emit("}", depth) for dfn in mod.dfns: self.visit(dfn, depth) @@ -283,25 +305,35 @@ class FoldImplVisitor(TypeInfoEmitVisitor): self.visit(type.value, type.name, depth) def visitSum(self, sum, name, depth): - apply_t, apply_u, apply_target_u = self.get_generics(name, "T", "U", "F::TargetU") + apply_t, apply_u, apply_target_u = self.get_generics( + name, "T", "U", "F::TargetU" + ) enumname = get_rust_type(name) is_located = bool(sum.attributes) self.emit(f"impl Foldable for {enumname}{apply_t} {{", depth) self.emit(f"type Mapped = {enumname}{apply_u};", depth + 1) - self.emit("fn fold + ?Sized>(self, folder: &mut F) -> Result {", depth + 1) + self.emit( + "fn fold + ?Sized>(self, folder: &mut F) -> Result {", + depth + 1, + ) self.emit(f"folder.fold_{name}(self)", depth + 2) self.emit("}", depth + 1) self.emit("}", depth) - self.emit(f"pub fn fold_{name} + ?Sized>(#[allow(unused)] folder: &mut F, node: {enumname}{apply_u}) -> Result<{enumname}{apply_target_u}, F::Error> {{", depth) + self.emit( + f"pub fn fold_{name} + ?Sized>(#[allow(unused)] folder: &mut F, node: {enumname}{apply_u}) -> Result<{enumname}{apply_target_u}, F::Error> {{", + depth, + ) if is_located: self.emit("fold_located(folder, node, |folder, node| {", depth) enumname += "Kind" self.emit("match node {", depth + 1) for cons in sum.types: fields_pattern = self.make_pattern(cons.fields) - self.emit(f"{enumname}::{cons.name} {{ {fields_pattern} }} => {{", depth + 2) + self.emit( + f"{enumname}::{cons.name} {{ {fields_pattern} }} => {{", depth + 2 + ) self.gen_construction(f"{enumname}::{cons.name}", cons.fields, depth + 3) self.emit("}", depth + 2) self.emit("}", depth + 1) @@ -309,20 +341,27 @@ class FoldImplVisitor(TypeInfoEmitVisitor): self.emit("})", depth) self.emit("}", depth) - def visitProduct(self, product, name, depth): - apply_t, apply_u, apply_target_u = self.get_generics(name, "T", "U", "F::TargetU") + apply_t, apply_u, apply_target_u = self.get_generics( + name, "T", "U", "F::TargetU" + ) structname = get_rust_type(name) is_located = bool(product.attributes) self.emit(f"impl Foldable for {structname}{apply_t} {{", depth) self.emit(f"type Mapped = {structname}{apply_u};", depth + 1) - self.emit("fn fold + ?Sized>(self, folder: &mut F) -> Result {", depth + 1) + self.emit( + "fn fold + ?Sized>(self, folder: &mut F) -> Result {", + depth + 1, + ) self.emit(f"folder.fold_{name}(self)", depth + 2) self.emit("}", depth + 1) self.emit("}", depth) - self.emit(f"pub fn fold_{name} + ?Sized>(#[allow(unused)] folder: &mut F, node: {structname}{apply_u}) -> Result<{structname}{apply_target_u}, F::Error> {{", depth) + self.emit( + f"pub fn fold_{name} + ?Sized>(#[allow(unused)] folder: &mut F, node: {structname}{apply_u}) -> Result<{structname}{apply_target_u}, F::Error> {{", + depth, + ) if is_located: self.emit("fold_located(folder, node, |folder, node| {", depth) structname += "Data" @@ -357,7 +396,6 @@ class FoldModuleVisitor(TypeInfoEmitVisitor): class ClassDefVisitor(EmitVisitor): - def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) @@ -367,10 +405,13 @@ class ClassDefVisitor(EmitVisitor): def visitSum(self, sum, name, depth): structname = "NodeKind" + get_rust_type(name) - self.emit(f'#[pyclass(module = "_ast", name = {json.dumps(name)}, base = "AstNode")]', depth) - self.emit(f'struct {structname};', depth) - self.emit( '#[pyclass(flags(HAS_DICT, BASETYPE))]', depth) - self.emit(f'impl {structname} {{}}', depth) + self.emit( + f'#[pyclass(module = "_ast", name = {json.dumps(name)}, base = "AstNode")]', + depth, + ) + self.emit(f"struct {structname};", depth) + self.emit("#[pyclass(flags(HAS_DICT, BASETYPE))]", depth) + self.emit(f"impl {structname} {{}}", depth) for cons in sum.types: self.visit(cons, sum.attributes, structname, depth) @@ -387,19 +428,35 @@ class ClassDefVisitor(EmitVisitor): self.emit("#[pyclass(flags(HAS_DICT, BASETYPE))]", depth) self.emit(f"impl {structname} {{", depth) self.emit(f"#[extend_class]", depth + 1) - self.emit("fn extend_class_with_fields(ctx: &Context, class: &'static Py) {", depth + 1) - fields = ",".join(f"ctx.new_str(ascii!({json.dumps(f.name)})).into()" for f in fields) - self.emit(f'class.set_attr(identifier!(ctx, _fields), ctx.new_list(vec![{fields}]).into());', depth + 2) - attrs = ",".join(f"ctx.new_str(ascii!({json.dumps(attr.name)})).into()" for attr in attrs) - self.emit(f'class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![{attrs}]).into());', depth + 2) + self.emit( + "fn extend_class_with_fields(ctx: &Context, class: &'static Py) {", + depth + 1, + ) + fields = ",".join( + f"ctx.new_str(ascii!({json.dumps(f.name)})).into()" for f in fields + ) + self.emit( + f"class.set_attr(identifier!(ctx, _fields), ctx.new_list(vec![{fields}]).into());", + depth + 2, + ) + attrs = ",".join( + f"ctx.new_str(ascii!({json.dumps(attr.name)})).into()" for attr in attrs + ) + self.emit( + f"class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![{attrs}]).into());", + depth + 2, + ) self.emit("}", depth + 1) self.emit("}", depth) -class ExtendModuleVisitor(EmitVisitor): +class ExtendModuleVisitor(EmitVisitor): def visitModule(self, mod): depth = 0 - self.emit("pub fn extend_module_nodes(vm: &VirtualMachine, module: &PyObject) {", depth) + self.emit( + "pub fn extend_module_nodes(vm: &VirtualMachine, module: &PyObject) {", + depth, + ) self.emit("extend_module!(vm, module, {", depth + 1) for dfn in mod.dfns: self.visit(dfn, depth + 2) @@ -425,7 +482,6 @@ class ExtendModuleVisitor(EmitVisitor): class TraitImplVisitor(EmitVisitor): - def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) @@ -443,13 +499,18 @@ class TraitImplVisitor(EmitVisitor): self.emit(f"const NAME: &'static str = {json.dumps(name)};", depth + 1) self.emit("}", depth) self.emit(f"impl Node for ast::{enumname} {{", depth) - self.emit("fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef {", depth + 1) + self.emit( + "fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef {", depth + 1 + ) self.emit("match self {", depth + 2) for variant in sum.types: self.constructor_to_object(variant, enumname, depth + 3) self.emit("}", depth + 2) self.emit("}", depth + 1) - self.emit("fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult {", depth + 1) + self.emit( + "fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult {", + depth + 1, + ) self.gen_sum_fromobj(sum, name, enumname, depth + 2) self.emit("}", depth + 1) self.emit("}", depth) @@ -469,12 +530,17 @@ class TraitImplVisitor(EmitVisitor): self.emit(f"const NAME: &'static str = {json.dumps(name)};", depth + 1) self.emit("}", depth) self.emit(f"impl Node for ast::{structname} {{", depth) - self.emit("fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef {", depth + 1) + self.emit( + "fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef {", depth + 1 + ) fields_pattern = self.make_pattern(product.fields) self.emit(f"let ast::{structname} {{ {fields_pattern} }} = self;", depth + 2) self.make_node(name, product.fields, depth + 2) self.emit("}", depth + 1) - self.emit("fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult {", depth + 1) + self.emit( + "fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult {", + depth + 1, + ) self.gen_product_fromobj(product, name, structname, depth + 2) self.emit("}", depth + 1) self.emit("}", depth) @@ -485,7 +551,10 @@ class TraitImplVisitor(EmitVisitor): if fields: self.emit("let _dict = _node.as_object().dict().unwrap();", depth) for f in fields: - self.emit(f"_dict.set_item({json.dumps(f.name)}, {rust_field(f.name)}.ast_to_object(_vm), _vm).unwrap();", depth) + self.emit( + f"_dict.set_item({json.dumps(f.name)}, {rust_field(f.name)}.ast_to_object(_vm), _vm).unwrap();", + depth, + ) self.emit("_node.into()", depth) def make_pattern(self, fields): @@ -518,12 +587,15 @@ class TraitImplVisitor(EmitVisitor): def gen_construction(self, cons_path, cons, name, depth): self.emit(f"ast::{cons_path} {{", depth) for field in cons.fields: - self.emit(f"{rust_field(field.name)}: {self.decode_field(field, name)},", depth + 1) + self.emit( + f"{rust_field(field.name)}: {self.decode_field(field, name)},", + depth + 1, + ) self.emit("}", depth) def extract_location(self, typename, depth): - row = self.decode_field(asdl.Field('int', 'lineno'), typename) - column = self.decode_field(asdl.Field('int', 'col_offset'), typename) + row = self.decode_field(asdl.Field("int", "lineno"), typename) + column = self.decode_field(asdl.Field("int", "col_offset"), typename) self.emit(f"let _location = ast::Location::new({row}, {column});", depth) def wrap_located_node(self, depth): @@ -536,6 +608,7 @@ class TraitImplVisitor(EmitVisitor): else: return f"Node::ast_from_object(_vm, get_node_field(_vm, &_object, {name}, {json.dumps(typename)})?)?" + class ChainOfVisitors: def __init__(self, *visitors): self.visitors = visitors @@ -547,49 +620,65 @@ class ChainOfVisitors: def write_ast_def(mod, typeinfo, f): - f.write('pub use crate::constant::*;\n') - f.write('pub use crate::location::Location;\n') - f.write('\n') - f.write('type Ident = String;\n') - f.write('\n') - StructVisitor(f, typeinfo).emit_attrs(0) - f.write('pub struct Located {\n') - f.write(' pub location: Location,\n') - f.write(' pub custom: U,\n') - f.write(' pub node: T,\n') - f.write('}\n') - f.write('\n') - f.write('impl Located {\n') - f.write(' pub fn new(location: Location, node: T) -> Self {\n') - f.write(' Self { location, custom: (), node }\n') - f.write(' }\n') - f.write('}\n') - f.write('\n') + f.write( + textwrap.dedent( + """ + #![allow(clippy::derive_partial_eq_without_eq)] + + pub use crate::constant::*; + pub use crate::location::Location; - c = ChainOfVisitors(StructVisitor(f, typeinfo), - FoldModuleVisitor(f, typeinfo)) + type Ident = String; + \n + """ + ) + ) + StructVisitor(f, typeinfo).emit_attrs(0) + f.write( + textwrap.dedent( + """ + pub struct Located { + pub location: Location, + pub custom: U, + pub node: T, + } + + impl Located { + pub fn new(location: Location, node: T) -> Self { + Self { location, custom: (), node } + } + } + \n + """.lstrip() + ) + ) + + c = ChainOfVisitors(StructVisitor(f, typeinfo), FoldModuleVisitor(f, typeinfo)) c.visit(mod) def write_ast_mod(mod, f): - f.write(textwrap.dedent(""" + f.write( + textwrap.dedent( + """ #![allow(clippy::all)] use super::*; use crate::common::ascii; - """)) + """ + ) + ) - c = ChainOfVisitors(ClassDefVisitor(f), - TraitImplVisitor(f), - ExtendModuleVisitor(f)) + c = ChainOfVisitors(ClassDefVisitor(f), TraitImplVisitor(f), ExtendModuleVisitor(f)) c.visit(mod) + def main(input_filename, ast_mod_filename, ast_def_filename, dump_module=False): auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:])) mod = asdl.parse(input_filename) if dump_module: - print('Parsed Module:') + print("Parsed Module:") print(mod) if not asdl.check(mod): sys.exit(1) @@ -597,8 +686,7 @@ def main(input_filename, ast_mod_filename, ast_def_filename, dump_module=False): typeinfo = {} FindUserdataTypesVisitor(typeinfo).visit(mod) - with ast_def_filename.open("w") as def_file, \ - ast_mod_filename.open("w") as mod_file: + with ast_def_filename.open("w") as def_file, ast_mod_filename.open("w") as mod_file: def_file.write(auto_gen_msg) write_ast_def(mod, typeinfo, def_file) @@ -607,6 +695,7 @@ def main(input_filename, ast_mod_filename, ast_def_filename, dump_module=False): print(f"{ast_def_filename}, {ast_mod_filename} regenerated.") + if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("input_file", type=Path) diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs index 0e40dc4a1..03f7c3129 100644 --- a/ast/src/ast_gen.rs +++ b/ast/src/ast_gen.rs @@ -1,4 +1,7 @@ // File automatically generated by ast/asdl_rs.py. + +#![allow(clippy::derive_partial_eq_without_eq)] + pub use crate::constant::*; pub use crate::location::Location; diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index 87ac684d5..35a4e268a 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -566,10 +566,16 @@ mod decl { let trailing_garbage_error = || Err(vm.new_value_error("Trailing garbage".to_string())); for chunk in b.get(1..).unwrap_or_default().chunks(4) { - let char_a = chunk.get(0).map_or(Ok(0), |x| uu_a2b_read(x, vm))?; - let char_b = chunk.get(1).map_or(Ok(0), |x| uu_a2b_read(x, vm))?; - let char_c = chunk.get(2).map_or(Ok(0), |x| uu_a2b_read(x, vm))?; - let char_d = chunk.get(3).map_or(Ok(0), |x| uu_a2b_read(x, vm))?; + let (char_a, char_b, char_c, char_d) = { + let mut chunk = chunk + .iter() + .map(|x| uu_a2b_read(x, vm)) + .collect::, _>>()?; + while chunk.len() < 4 { + chunk.push(0); + } + (chunk[0], chunk[1], chunk[2], chunk[3]) + }; if res.len() < length { res.push(char_a << 2 | char_b >> 4); @@ -628,7 +634,7 @@ mod decl { res.push(uu_b2a(length as u8, backtick)); for chunk in b.chunks(3) { - let char_a = *chunk.get(0).unwrap_or(&0); + let char_a = *chunk.first().unwrap(); let char_b = *chunk.get(1).unwrap_or(&0); let char_c = *chunk.get(2).unwrap_or(&0); diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index d28488fdc..a9e1c9534 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -354,7 +354,7 @@ impl CFormatSpec { _ => unreachable!(), }; - let formatted = if self.flags.contains(CConversionFlags::ZERO_PAD) { + if self.flags.contains(CConversionFlags::ZERO_PAD) { let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { '0' } else { @@ -377,9 +377,7 @@ impl CFormatSpec { None, false, ) - }; - - formatted + } } fn bytes_format(&self, vm: &VirtualMachine, obj: PyObjectRef) -> PyResult> { diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 6dd7e3dd3..a77847ff6 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -99,7 +99,7 @@ struct DictEntry { } static_assertions::assert_eq_size!(DictEntry, Option>); -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq)] pub struct DictSize { indices_size: usize, pub entries_size: usize, diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs index 21d575e3b..11ecc6f55 100644 --- a/vm/src/stdlib/marshal.rs +++ b/vm/src/stdlib/marshal.rs @@ -179,7 +179,7 @@ mod decl { buf.push(Type::Bytes as u8); let data = bytes.as_bytes(); write_size(buf, data.len(), vm)?; - buf.extend(&*data); + buf.extend(data); } bytes @ PyByteArray => { buf.push(Type::Bytes as u8); diff --git a/vm/src/vm/context.rs b/vm/src/vm/context.rs index c5a64dad2..70df3070e 100644 --- a/vm/src/vm/context.rs +++ b/vm/src/vm/context.rs @@ -327,7 +327,7 @@ impl Context { #[inline] pub fn new_int + ToPrimitive>(&self, i: T) -> PyIntRef { if let Some(i) = i.to_i32() { - if i >= Self::INT_CACHE_POOL_MIN && i <= Self::INT_CACHE_POOL_MAX { + if (Self::INT_CACHE_POOL_MIN..=Self::INT_CACHE_POOL_MAX).contains(&i) { let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; return self.int_cache_pool[inner_idx].clone(); } @@ -338,7 +338,7 @@ impl Context { #[inline] pub fn new_bigint(&self, i: &BigInt) -> PyIntRef { if let Some(i) = i.to_i32() { - if i >= Self::INT_CACHE_POOL_MIN && i <= Self::INT_CACHE_POOL_MAX { + if (Self::INT_CACHE_POOL_MIN..=Self::INT_CACHE_POOL_MAX).contains(&i) { let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; return self.int_cache_pool[inner_idx].clone(); }