From b8aabb7bb7015aa659e100573e657b7f85ed4744 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sat, 28 Nov 2020 14:57:00 -0600 Subject: [PATCH] Implement asdl in the ast and parser --- Cargo.lock | 26 +- ast/Python.asdl | 12 +- ast/asdl.py | 8 +- ast/asdl_rs.py | 427 ++++----- ast/src/ast.rs | 556 ----------- ast/src/ast_gen.rs | 389 ++++++++ ast/src/constant.rs | 41 + ast/src/impls.rs | 53 ++ ast/src/lib.rs | 8 +- compiler/Cargo.toml | 2 +- parser/Cargo.toml | 3 + parser/src/fstring.rs | 256 +++-- parser/src/function.rs | 44 +- parser/src/lexer.rs | 14 +- parser/src/mode.rs | 21 +- parser/src/parser.rs | 472 ++-------- parser/src/python.lalrpop | 875 +++++++++--------- ...s__fstring_parse_selfdocumenting_base.snap | 63 ++ ...tring_parse_selfdocumenting_base_more.snap | 137 +++ ..._fstring_parse_selfdocumenting_format.snap | 77 ++ ...__fstring__tests__parse_empty_fstring.snap | 17 + ...parser__fstring__tests__parse_fstring.snap | 72 ++ ..._fstring__tests__parse_fstring_equals.snap | 69 ++ ...ing__tests__parse_fstring_nested_spec.snap | 49 + ...ring__tests__parse_fstring_not_equals.snap | 69 ++ ..._tests__parse_fstring_not_nested_spec.snap | 40 + ...sts__parse_fstring_selfdoc_prec_space.snap | 63 ++ ..._parse_fstring_selfdoc_trailing_space.snap | 63 ++ ...ring__tests__parse_fstring_yield_expr.snap | 25 + ...on_parser__parser__tests__parse_class.snap | 160 ++++ ...rser__tests__parse_dict_comprehension.snap | 63 ++ ...ests__parse_double_list_comprehension.snap | 192 ++++ ...on_parser__parser__tests__parse_empty.snap | 5 + ...er__parser__tests__parse_if_elif_else.snap | 159 ++++ ...n_parser__parser__tests__parse_kwargs.snap | 84 ++ ...n_parser__parser__tests__parse_lambda.snap | 90 ++ ...rser__tests__parse_list_comprehension.snap | 52 ++ ..._parser__parser__tests__parse_print_2.snap | 71 ++ ...ser__parser__tests__parse_print_hello.snap | 51 + ...n_parser__parser__tests__parse_tuples.snap | 104 +++ parser/src/token.rs | 8 +- 41 files changed, 3135 insertions(+), 1855 deletions(-) delete mode 100644 ast/src/ast.rs create mode 100644 ast/src/ast_gen.rs create mode 100644 ast/src/constant.rs create mode 100644 ast/src/impls.rs create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap diff --git a/Cargo.lock b/Cargo.lock index 7e6c4dae1..7342d3b11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -292,15 +292,14 @@ dependencies = [ [[package]] name = "console" -version = "0.12.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" +checksum = "7cc80946b3480f421c2f17ed1cb841753a371c7c5104f51d507e13f532c856aa" dependencies = [ "encode_unicode", "lazy_static 1.4.0", "libc", "terminal_size", - "termios", "winapi", ] @@ -919,9 +918,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.3.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "863bf97e7130bf788f29a99bc4073735af6b8ecc3da6a39c23b3a688d2d3109a" +checksum = "bca6f2bcc5e2ce13f3652ecd05a643b986d035add3f0c38fbabd78f723b5f7e9" dependencies = [ "console", "difference", @@ -929,6 +928,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "uuid", ] [[package]] @@ -1923,6 +1923,7 @@ name = "rustpython-parser" version = "0.1.2" dependencies = [ "ahash", + "insta", "lalrpop", "lalrpop-util", "log", @@ -2329,15 +2330,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "termios" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b" -dependencies = [ - "libc", -] - [[package]] name = "textwrap" version = "0.11.0" @@ -2642,6 +2634,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "uuid" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" + [[package]] name = "vcpkg" version = "0.2.10" diff --git a/ast/Python.asdl b/ast/Python.asdl index 889712b4b..b3abe1628 100644 --- a/ast/Python.asdl +++ b/ast/Python.asdl @@ -26,7 +26,7 @@ module Python | Assign(expr* targets, expr value, string? type_comment) | AugAssign(expr target, operator op, expr value) -- 'simple' indicates that we annotate simple name without parens - | AnnAssign(expr target, expr annotation, expr? value, int simple) + | AnnAssign(expr target, expr annotation, expr? value, bool simple) -- use 'orelse' because else is a keyword in target languages | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) @@ -41,7 +41,7 @@ module Python | Assert(expr test, expr? msg) | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) + | ImportFrom(identifier? module, alias* names, int level) | Global(identifier* names) | Nonlocal(identifier* names) @@ -58,7 +58,7 @@ module Python | UnaryOp(unaryop op, expr operand) | Lambda(arguments args, expr body) | IfExp(expr test, expr body, expr orelse) - | Dict(expr* keys, expr* values) + | Dict(expr?* keys, expr* values) | Set(expr* elts) | ListComp(expr elt, comprehension* generators) | SetComp(expr elt, comprehension* generators) @@ -72,7 +72,7 @@ module Python -- x < 4 < 3 and (x < 4) < 3 | Compare(expr left, cmpop* ops, expr* comparators) | Call(expr func, expr* args, keyword* keywords) - | FormattedValue(expr value, int? conversion, expr? format_spec) + | FormattedValue(expr value, conversion_flag? conversion, expr? format_spec) | JoinedStr(expr* values) | Constant(constant value, string? kind) @@ -101,13 +101,13 @@ module Python cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn - comprehension = (expr target, expr iter, expr* ifs, int is_async) + comprehension = (expr target, expr iter, expr* ifs, bool is_async) excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) arguments = (arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, - expr* kw_defaults, arg? kwarg, expr* defaults) + expr?* kw_defaults, arg? kwarg, expr* defaults) arg = (identifier arg, expr? annotation, string? type_comment) attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) diff --git a/ast/asdl.py b/ast/asdl.py index 7f509488b..74f84c898 100644 --- a/ast/asdl.py +++ b/ast/asdl.py @@ -33,7 +33,7 @@ __all__ = [ # See the EBNF at the top of the file to understand the logical connection # between the various node types. -builtin_types = {'identifier', 'string', 'int', 'constant'} +builtin_types = {'identifier', 'string', 'int', 'constant', 'bool', 'conversion_flag'} class AST: def __repr__(self): @@ -340,12 +340,12 @@ class ASDLParser: def _parse_optional_field_quantifier(self): is_seq, is_opt = False, False + if self.cur_token.kind == TokenKind.Question: + is_opt = True + self._advance() if self.cur_token.kind == TokenKind.Asterisk: is_seq = True self._advance() - elif self.cur_token.kind == TokenKind.Question: - is_opt = True - self._advance() return is_seq, is_opt def _advance(self): diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py index 34bd7010c..737d1e44b 100755 --- a/ast/asdl_rs.py +++ b/ast/asdl_rs.py @@ -1,5 +1,5 @@ #! /usr/bin/env python -"""Generate C code from an ASDL description.""" +"""Generate Rust code from an ASDL description.""" import os import sys @@ -11,63 +11,27 @@ from pathlib import Path import asdl TABSIZE = 4 -MAX_COL = 80 AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n" -def get_c_type(name): +builtin_type_mapping = { + 'identifier': 'Ident', + 'string': 'String', + 'int': 'usize', + 'constant': 'Constant', + 'bool': 'bool', + 'conversion_flag': 'ConversionFlag', +} +assert builtin_type_mapping.keys() == asdl.builtin_types + +def get_rust_type(name): """Return a string for the C name of the type. This function special cases the default types provided by asdl. """ if name in asdl.builtin_types: - return name + return builtin_type_mapping[name] else: - return "%s_ty" % name - -def reflow_lines(s, depth): - """Reflow the line s indented depth tabs. - - Return a sequence of lines where no line extends beyond MAX_COL - when properly indented. The first line is properly indented based - exclusively on depth * TABSIZE. All following lines -- these are - the reflowed lines generated by this function -- start at the same - column as the first character beyond the opening { in the first - line. - """ - size = MAX_COL - depth * TABSIZE - if len(s) < size: - return [s] - - lines = [] - cur = s - padding = "" - while len(cur) > size: - i = cur.rfind(' ', 0, size) - # XXX this should be fixed for real - if i == -1 and 'GeneratorExp' in cur: - i = size + 3 - assert i != -1, "Impossible line %d to reflow: %r" % (size, s) - lines.append(padding + cur[:i]) - if len(lines) == 1: - # find new size based on brace - j = cur.find('{', 0, i) - if j >= 0: - j += 2 # account for the brace and the space after it - size -= j - padding = " " * j - else: - j = cur.find('(', 0, i) - if j >= 0: - j += 1 # account for the paren (no space after it) - size -= j - padding = " " * j - cur = cur[i+1:] - else: - lines.append(padding + cur) - return lines - -def reflow_c_string(s, depth): - return '"%s"' % s.replace('\n', '\\n"\n%s"' % (' ' * depth * TABSIZE)) + return "".join(part.capitalize() for part in name.split("_")) def is_simple(sum): """Return True if a sum is a simple. @@ -102,32 +66,93 @@ class EmitVisitor(asdl.VisitorBase): def __init__(self, file): self.file = file self.identifiers = set() - self.singletons = set() - self.types = set() super(EmitVisitor, self).__init__() def emit_identifier(self, name): - self.identifiers.add(str(name)) + name = str(name) + if name in self.identifiers: + return + self.emit("_Py_IDENTIFIER(%s);" % name, 0) + self.identifiers.add(name) - def emit_singleton(self, name): - self.singletons.add(str(name)) + def emit(self, line, depth): + if line: + line = (" " * TABSIZE * depth) + line + self.file.write(line + "\n") - def emit_type(self, name): - self.types.add(str(name)) +class TypeInfo: + def __init__(self, name): + self.name = name + self.has_userdata = None + self.children = set() + self.boxed = False - def emit(self, s, depth, reflow=True): - # XXX reflow long lines? - if reflow: - lines = reflow_lines(s, depth) + def __repr__(self): + return f"" + + def determine_userdata(self, typeinfo, stack): + if self.name in stack: + return None + stack.add(self.name) + for child, child_seq in self.children: + if child in asdl.builtin_types: + continue + childinfo = typeinfo[child] + child_has_userdata = childinfo.determine_userdata(typeinfo, stack) + if self.has_userdata is None and child_has_userdata is True: + self.has_userdata = True + + stack.remove(self.name) + return self.has_userdata + +class FindUserdataTypesVisitor(asdl.VisitorBase): + def __init__(self, typeinfo): + self.typeinfo = typeinfo + super().__init__() + + def visitModule(self, mod): + for dfn in mod.dfns: + self.visit(dfn) + stack = set() + for info in self.typeinfo.values(): + info.determine_userdata(self.typeinfo, stack) + + def visitType(self, type): + self.typeinfo[type.name] = TypeInfo(type.name) + self.visit(type.value, type.name) + + def visitSum(self, sum, name): + info = self.typeinfo[name] + if is_simple(sum): + info.has_userdata = False else: - lines = [s] - for line in lines: - if line: - line = (" " * TABSIZE * depth) + line - self.file.write(line + "\n") + if len(sum.types) > 1: + info.boxed = True + if sum.attributes: + # attributes means Located, which has the `custom: U` field + info.has_userdata = True + for variant in sum.types: + self.add_children(name, variant.fields) + def visitProduct(self, product, name): + info = self.typeinfo[name] + if product.attributes: + # attributes means Located, which has the `custom: U` field + info.has_userdata = True + if len(product.fields) > 2: + info.boxed = True + self.add_children(name, product.fields) + + def add_children(self, name, fields): + self.typeinfo[name].children.update((field.type, field.seq) for field in fields) + +class StructVisitor(EmitVisitor): + """Visitor to generate typedefs for AST.""" + + def __init__(self, file, typeinfo): + self.typeinfo = typeinfo + super().__init__(file) -class TypeDefVisitor(EmitVisitor): def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) @@ -141,128 +166,82 @@ class TypeDefVisitor(EmitVisitor): else: self.sum_with_constructors(sum, name, depth) + def emit_attrs(self, depth): + self.emit("#[derive(Debug, PartialEq)]", depth) + def simple_sum(self, sum, name, depth): - enum = [] - for i in range(len(sum.types)): - type = sum.types[i] - enum.append("%s=%d" % (type.name, i + 1)) - enums = ", ".join(enum) - ctype = get_c_type(name) - s = "typedef enum _%s { %s } %s;" % (name, enums, ctype) - self.emit(s, depth) + rustname = get_rust_type(name) + self.emit_attrs(depth) + self.emit(f"pub enum {rustname} {{", depth) + for variant in sum.types: + self.emit(f"{variant.name},", depth + 1) + self.emit("}", depth) self.emit("", depth) def sum_with_constructors(self, sum, name, depth): - ctype = get_c_type(name) - s = "typedef struct _%(name)s *%(ctype)s;" % locals() - self.emit(s, depth) - self.emit("", depth) - - def visitProduct(self, product, name, depth): - ctype = get_c_type(name) - s = "typedef struct _%(name)s *%(ctype)s;" % locals() - self.emit(s, depth) - self.emit("", depth) - -class SequenceDefVisitor(EmitVisitor): - def visitModule(self, mod): - for dfn in mod.dfns: - self.visit(dfn) - - def visitType(self, type, depth=0): - self.visit(type.value, type.name, depth) - - def visitSum(self, sum, name, depth): - if is_simple(sum): - return - self.emit_sequence_constructor(name, depth) - - def emit_sequence_constructor(self, name,depth): - ctype = get_c_type(name) - self.emit("""\ -typedef struct { - _ASDL_SEQ_HEAD - %(ctype)s typed_elements[1]; -} asdl_%(name)s_seq;""" % locals(), reflow=False, depth=depth) - self.emit("", depth) - self.emit("asdl_%(name)s_seq *_Py_asdl_%(name)s_seq_new(Py_ssize_t size, PyArena *arena);" % locals(), depth) - self.emit("", depth) - - def visitProduct(self, product, name, depth): - self.emit_sequence_constructor(name, depth) - -class StructVisitor(EmitVisitor): - """Visitor to generate typedefs for AST.""" - - def visitModule(self, mod): - for dfn in mod.dfns: - self.visit(dfn) - - def visitType(self, type, depth=0): - self.visit(type.value, type.name, depth) - - def visitSum(self, sum, name, depth): - if not is_simple(sum): - self.sum_with_constructors(sum, name, depth) - - def sum_with_constructors(self, sum, name, depth): - def emit(s, depth=depth): - self.emit(s % sys._getframe(1).f_locals, depth) - enum = [] - for i in range(len(sum.types)): - type = sum.types[i] - enum.append("%s_kind=%d" % (type.name, i + 1)) - - emit("enum _%(name)s_kind {" + ", ".join(enum) + "};") - - emit("struct _%(name)s {") - emit("enum _%(name)s_kind kind;", depth + 1) - emit("union {", depth + 1) + typeinfo = self.typeinfo[name] + generics, generics_applied = self.get_generics(name) + enumname = rustname = get_rust_type(name) + # all the attributes right now are for location, so if it has attrs we + # can just wrap it in Located<> + if sum.attributes: + enumname = rustname + "Kind" + self.emit_attrs(depth) + self.emit(f"pub enum {enumname}{generics} {{", depth) for t in sum.types: - self.visit(t, depth + 2) - emit("} v;", depth + 1) - for field in sum.attributes: - # rudimentary attribute handling - type = str(field.type) - assert type in asdl.builtin_types, type - emit("%s %s;" % (type, field.name), depth + 1); - emit("};") - emit("") - - def visitConstructor(self, cons, depth): - if cons.fields: - self.emit("struct {", depth) - for f in cons.fields: - self.visit(f, depth + 1) - self.emit("} %s;" % cons.name, depth) - self.emit("", depth) - - def visitField(self, field, depth): - # XXX need to lookup field.type, because it might be something - # like a builtin... - ctype = get_c_type(field.type) - name = field.name - if field.seq: - if field.type == 'cmpop': - self.emit("asdl_int_seq *%(name)s;" % locals(), depth) - else: - _type = field.type - self.emit("asdl_%(_type)s_seq *%(name)s;" % locals(), depth) - else: - self.emit("%(ctype)s %(name)s;" % locals(), depth) - - def visitProduct(self, product, name, depth): - self.emit("struct _%(name)s {" % locals(), depth) - for f in product.fields: - self.visit(f, depth + 1) - for field in product.attributes: - # rudimentary attribute handling - type = str(field.type) - assert type in asdl.builtin_types, type - self.emit("%s %s;" % (type, field.name), depth + 1); - self.emit("};", depth) + self.visit(t, typeinfo, depth + 1) + self.emit("}", depth) + if sum.attributes: + self.emit(f"pub type {rustname} = Located<{enumname}{generics_applied}, U>;", depth) self.emit("", depth) + def visitConstructor(self, cons, parent, depth): + if cons.fields: + self.emit(f"{cons.name} {{", depth) + for f in cons.fields: + self.visit(f, parent, "", depth + 1) + self.emit("},", depth) + else: + self.emit(f"{cons.name},", depth) + + def visitField(self, field, parent, vis, depth): + typ = get_rust_type(field.type) + fieldtype = self.typeinfo.get(field.type) + if fieldtype and fieldtype.has_userdata: + typ = f"{typ}" + # don't box if we're doing Vec, but do box if we're doing Vec>> + if fieldtype and fieldtype.boxed and (not field.seq or field.opt): + typ = f"Box<{typ}>" + if field.opt: + typ = f"Option<{typ}>" + if field.seq: + typ = f"Vec<{typ}>" + name = field.name + if name == 'type': + name = 'type_' + self.emit(f"{vis}{name}: {typ},", depth) + + def visitProduct(self, product, name, depth): + typeinfo = self.typeinfo[name] + generics, generics_applied = self.get_generics(name) + dataname = rustname = get_rust_type(name) + if product.attributes: + dataname = rustname + "Data" + self.emit_attrs(depth) + self.emit(f"pub struct {dataname}{generics} {{", depth) + for f in product.fields: + self.visit(f, typeinfo, "pub ", depth + 1) + self.emit("}", depth) + if product.attributes: + # attributes should just be location info + self.emit(f"pub type {rustname} = Located<{dataname}{generics_applied}, U>;", depth); + self.emit("", depth) + + def get_generics(self, typ): + if self.typeinfo[typ].has_userdata: + return "", "" + else: + return "", "" class PrototypeVisitor(EmitVisitor): """Generate function prototypes for the .h file""" @@ -689,6 +668,8 @@ class PyTypesVisitor(PickleVisitor): def visitModule(self, mod): self.emit(""" +_Py_IDENTIFIER(_fields); +_Py_IDENTIFIER(_attributes); typedef struct { PyObject_HEAD @@ -1181,16 +1162,16 @@ class StaticVisitor(PickleVisitor): class ObjVisitor(PickleVisitor): def func_begin(self, name): - ctype = get_c_type(name) + ctype = get_rust_type(name) self.emit("PyObject*", 0) - self.emit("ast2obj_%s(struct ast_state *state, void* _o)" % (name), 0) + self.emit("ast2obj_%s(void* _o)" % (name), 0) self.emit("{", 0) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("PyObject *result = NULL, *value = NULL;", 1) - self.emit("PyTypeObject *tp;", 1) self.emit('if (!o) {', 1) self.emit("Py_RETURN_NONE;", 2) self.emit("}", 1) + self.emit('', 0) def func_end(self): self.emit("return result;", 1) @@ -1299,6 +1280,9 @@ PyObject* PyAST_mod2obj(mod_ty t) return NULL; } return ast2obj_mod(state, t); + if (!init_types()) + return NULL; + return ast2obj_mod(t); } /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ @@ -1487,34 +1471,30 @@ get_ast_state(void) f.write(' return 1;\n') f.write('};\n\n') -def write_header(mod, f): - f.write('#ifndef Py_PYTHON_AST_H\n') - f.write('#define Py_PYTHON_AST_H\n') - f.write('#ifdef __cplusplus\n') - f.write('extern "C" {\n') - f.write('#endif\n') +def write_ast_def(mod, f): + f.write('pub use crate::location::Location;\n') + f.write('pub use crate::constant::*;\n') f.write('\n') - f.write('#ifndef Py_LIMITED_API\n') - f.write('#include "asdl.h"\n') + f.write('type Ident = String;\n') f.write('\n') - f.write('#undef Yield /* undefine macro conflicting with */\n') - f.write('\n') - c = ChainOfVisitors(TypeDefVisitor(f), - SequenceDefVisitor(f), - StructVisitor(f)) - c.visit(mod) - f.write("// Note: these macros affect function definitions, not only call sites.\n") - PrototypeVisitor(f).visit(mod) - f.write("\n") - f.write("PyObject* PyAST_mod2obj(mod_ty t);\n") - f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n") - f.write("int PyAST_Check(PyObject* obj);\n") - f.write("#endif /* !Py_LIMITED_API */\n") - f.write('\n') - f.write('#ifdef __cplusplus\n') + StructVisitor(f, {}).emit_attrs(0) + f.write('pub struct Located {\n') + f.write(' pub location: Location,\n') + f.write(' pub custom: U,\n') + f.write(' pub node: T,\n') f.write('}\n') - f.write('#endif\n') - f.write('#endif /* !Py_PYTHON_AST_H */\n') + f.write('\n') + f.write('impl Located {\n') + f.write(' pub fn new(location: Location, node: T) -> Self {\n') + f.write(' Self { location, custom: (), node }\n') + f.write(' }\n') + f.write('}\n') + f.write('\n') + + typeinfo = {} + FindUserdataTypesVisitor(typeinfo).visit(mod) + + StructVisitor(f, typeinfo).visit(mod) def write_internal_h_header(mod, f): @@ -1563,7 +1543,7 @@ def write_source(mod, f, internal_h_file): ) v.visit(mod) -def main(input_filename, c_filename, h_filename, internal_h_filename, dump_module=False): +def main(input_filename, ast_mod_filename, ast_def_filename, dump_module=False): auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:])) mod = asdl.parse(input_filename) if dump_module: @@ -1572,28 +1552,21 @@ def main(input_filename, c_filename, h_filename, internal_h_filename, dump_modul if not asdl.check(mod): sys.exit(1) - with c_filename.open("w") as c_file, \ - h_filename.open("w") as h_file, \ - internal_h_filename.open("w") as internal_h_file: - c_file.write(auto_gen_msg) - h_file.write(auto_gen_msg) - internal_h_file.write(auto_gen_msg) + with ast_def_filename.open("w") as def_file, \ + ast_mod_filename.open("w") as mod_file: + def_file.write(auto_gen_msg) + mod_file.write(auto_gen_msg) - write_internal_h_header(mod, internal_h_file) - write_source(mod, c_file, internal_h_file) - write_header(mod, h_file) - write_internal_h_footer(mod, internal_h_file) + write_ast_def(mod, def_file) - print(f"{c_filename}, {h_filename}, {internal_h_filename} regenerated.") + print(f"{ast_def_filename}, {ast_mod_filename} regenerated.") if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("input_file", type=Path) - parser.add_argument("-C", "--c-file", type=Path, required=True) - parser.add_argument("-H", "--h-file", type=Path, required=True) - parser.add_argument("-I", "--internal-h-file", type=Path, required=True) + parser.add_argument("-M", "--mod-file", type=Path, required=True) + parser.add_argument("-D", "--def-file", type=Path, required=True) parser.add_argument("-d", "--dump-module", action="store_true") args = parser.parse_args() - main(args.input_file, args.c_file, args.h_file, - args.internal_h_file, args.dump_module) + main(args.input_file, args.mod_file, args.def_file, args.dump_module) diff --git a/ast/src/ast.rs b/ast/src/ast.rs deleted file mode 100644 index 43b04ff44..000000000 --- a/ast/src/ast.rs +++ /dev/null @@ -1,556 +0,0 @@ -//! Implement abstract syntax tree (AST) nodes for the python language. -//! -//! Roughly equivalent to [the python AST](https://docs.python.org/3/library/ast.html) -//! Many AST nodes have a location attribute, to determine the sourcecode -//! location of the node. - -pub use crate::location::Location; -use num_bigint::BigInt; - -#[allow(clippy::large_enum_variant)] -#[derive(Debug, PartialEq)] -pub enum Top { - Program(Program), - Statement(Vec>), - Expression(Expression), -} - -#[derive(Debug, PartialEq)] -/// A full python program, it's a sequence of statements. -pub struct Program { - pub statements: Suite, -} - -#[derive(Debug, PartialEq)] -pub struct ImportSymbol { - pub symbol: String, - pub alias: Option, -} - -#[derive(Debug, PartialEq)] -pub struct Located { - pub location: Location, - pub node: T, - pub custom: U, -} - -pub type Statement = Located, U>; -pub type Suite = Vec>; - -/// Abstract syntax tree nodes for python statements. -#[derive(Debug, PartialEq)] -pub enum StatementType { - /// A [`break`](https://docs.python.org/3/reference/simple_stmts.html#the-break-statement) statement. - Break, - - /// A [`continue`](https://docs.python.org/3/reference/simple_stmts.html#the-continue-statement) statement. - Continue, - - /// A [`return`](https://docs.python.org/3/reference/simple_stmts.html#the-return-statement) statement. - /// This is used to return from a function. - Return { value: Option> }, - - /// An [`import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement) statement. - Import { names: Vec }, - - /// An [`import` `from`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement) statement. - ImportFrom { - level: usize, - module: Option, - names: Vec, - }, - - /// A [`pass`](https://docs.python.org/3/reference/simple_stmts.html#pass) statement. - Pass, - - /// An [`assert`](https://docs.python.org/3/reference/simple_stmts.html#the-assert-statement) statement. - Assert { - test: Expression, - msg: Option>, - }, - - /// A `del` statement, to delete some variables. - Delete { targets: Vec> }, - - /// Variable assignment. Note that we can assign to multiple targets. - Assign { - targets: Vec>, - value: Expression, - }, - - /// Augmented assignment. - AugAssign { - target: Box>, - op: Operator, - value: Box>, - }, - - /// A type annotated assignment. - AnnAssign { - target: Box>, - annotation: Box>, - value: Option>, - }, - - /// An expression used as a statement. - Expression { expression: Expression }, - - /// The [`global`](https://docs.python.org/3/reference/simple_stmts.html#the-global-statement) statement, - /// to declare names as global variables. - Global { names: Vec }, - - /// A [`nonlocal`](https://docs.python.org/3/reference/simple_stmts.html#the-nonlocal-statement) statement, - /// to declare names a non-local variables. - Nonlocal { names: Vec }, - - /// An [`if`](https://docs.python.org/3/reference/compound_stmts.html#the-if-statement) statement. - If { - test: Expression, - body: Suite, - orelse: Option>, - }, - - /// A [`while`](https://docs.python.org/3/reference/compound_stmts.html#the-while-statement) statement. - While { - test: Expression, - body: Suite, - orelse: Option>, - }, - - /// The [`with`](https://docs.python.org/3/reference/compound_stmts.html#the-with-statement) statement. - With { - is_async: bool, - items: Vec>, - body: Suite, - }, - - /// A [`for`](https://docs.python.org/3/reference/compound_stmts.html#the-for-statement) statement. - /// Contains the body of the loop, and the `else` clause. - For { - is_async: bool, - target: Box>, - iter: Box>, - body: Suite, - orelse: Option>, - }, - - /// A `raise` statement. - Raise { - exception: Option>, - cause: Option>, - }, - - /// A [`try`](https://docs.python.org/3/reference/compound_stmts.html#the-try-statement) statement. - Try { - body: Suite, - handlers: Vec>, - orelse: Option>, - finalbody: Option>, - }, - - /// A [class definition](https://docs.python.org/3/reference/compound_stmts.html#class-definitions). - ClassDef { - name: String, - body: Suite, - bases: Vec>, - keywords: Vec>, - decorator_list: Vec>, - }, - - /// A [function definition](https://docs.python.org/3/reference/compound_stmts.html#function-definitions). - /// Contains the name of the function, it's body - /// some decorators and formal parameters to the function. - FunctionDef { - is_async: bool, - name: String, - args: Box>, - body: Suite, - decorator_list: Vec>, - returns: Option>, - }, -} - -#[derive(Debug, PartialEq)] -pub struct WithItem { - pub context_expr: Expression, - pub optional_vars: Option>, -} - -/// An expression at a given location in the sourcecode. -pub type Expression = Located, U>; - -/// A certain type of expression. -#[derive(Debug, PartialEq)] -pub enum ExpressionType { - BoolOp { - op: BooleanOperator, - values: Vec>, - }, - - /// A binary operation on two operands. - Binop { - a: Box>, - op: Operator, - b: Box>, - }, - - /// Subscript operation. - Subscript { - a: Box>, - b: Box>, - }, - - /// An unary operation. - Unop { - op: UnaryOperator, - a: Box>, - }, - - /// An await expression. - Await { - value: Box>, - }, - - /// A yield expression. - Yield { - value: Option>>, - }, - - // A yield from expression. - YieldFrom { - value: Box>, - }, - - /// A chained comparison. Note that in python you can use - /// `1 < a < 10` for example. - Compare { - vals: Vec>, - ops: Vec, - }, - - /// Attribute access in the form of `value.name`. - Attribute { - value: Box>, - name: String, - }, - - /// A call expression. - Call { - function: Box>, - args: Vec>, - keywords: Vec>, - }, - - /// A numeric literal. - Number { - value: Number, - }, - - /// A `list` literal value. - List { - elements: Vec>, - }, - - /// A `tuple` literal value. - Tuple { - elements: Vec>, - }, - - /// A `dict` literal value. - /// For example: `{2: 'two', 3: 'three'}` - Dict { - elements: Vec<(Option>, Expression)>, - }, - - /// A `set` literal. - Set { - elements: Vec>, - }, - - Comprehension { - kind: Box>, - generators: Vec>, - }, - - /// A starred expression. - Starred { - value: Box>, - }, - - /// A slice expression. - Slice { - elements: Vec>, - }, - - /// A string literal. - String { - value: StringGroup, - }, - - /// A bytes literal. - Bytes { - value: Vec, - }, - - /// An identifier, designating a certain variable or type. - Identifier { - name: String, - }, - - /// A `lambda` function expression. - Lambda { - args: Box>, - body: Box>, - }, - - /// An if-expression. - IfExpression { - test: Box>, - body: Box>, - orelse: Box>, - }, - - // A named expression - NamedExpression { - left: Box>, - right: Box>, - }, - - /// The literal 'True'. - True, - - /// The literal 'False'. - False, - - // The literal `None`. - None, - - /// The ellipsis literal `...`. - Ellipsis, -} - -impl Expression { - /// Returns a short name for the node suitable for use in error messages. - pub fn name(&self) -> &'static str { - use self::ExpressionType::*; - use self::StringGroup::*; - - match &self.node { - BoolOp { .. } | Binop { .. } | Unop { .. } => "operator", - Subscript { .. } => "subscript", - Await { .. } => "await expression", - Yield { .. } | YieldFrom { .. } => "yield expression", - Compare { .. } => "comparison", - Attribute { .. } => "attribute", - Call { .. } => "function call", - Number { .. } - | String { - value: Constant { .. }, - } - | Bytes { .. } => "literal", - List { .. } => "list", - Tuple { .. } => "tuple", - Dict { .. } => "dict display", - Set { .. } => "set display", - Comprehension { kind, .. } => match **kind { - ComprehensionKind::List { .. } => "list comprehension", - ComprehensionKind::Dict { .. } => "dict comprehension", - ComprehensionKind::Set { .. } => "set comprehension", - ComprehensionKind::GeneratorExpression { .. } => "generator expression", - }, - Starred { .. } => "starred", - Slice { .. } => "slice", - String { - value: Joined { .. }, - } - | String { - value: FormattedValue { .. }, - } => "f-string expression", - Identifier { .. } => "named expression", - Lambda { .. } => "lambda", - IfExpression { .. } => "conditional expression", - True | False | None => "keyword", - Ellipsis => "ellipsis", - NamedExpression { .. } => "named expression", - } - } -} - -/// Formal parameters to a function. -/// -/// In cpython this is called arguments, but we choose parameters to -/// distinguish between function parameters and actual call arguments. -#[derive(Debug, PartialEq, Default)] -pub struct Parameters { - pub posonlyargs_count: usize, - pub args: Vec>, - pub kwonlyargs: Vec>, - pub vararg: Varargs, // Optionally we handle optionally named '*args' or '*' - pub kwarg: Varargs, - pub defaults: Vec>, - pub kw_defaults: Vec>>, -} - -/// A single formal parameter to a function. -#[derive(Debug, PartialEq, Default)] -pub struct Parameter { - pub location: Location, - pub arg: String, - pub annotation: Option>>, -} - -#[allow(clippy::large_enum_variant)] -#[derive(Debug, PartialEq)] -pub enum ComprehensionKind { - GeneratorExpression { - element: Expression, - }, - List { - element: Expression, - }, - Set { - element: Expression, - }, - Dict { - key: Expression, - value: Expression, - }, -} - -/// A list/set/dict/generator compression. -#[derive(Debug, PartialEq)] -pub struct Comprehension { - pub location: Location, - pub target: Expression, - pub iter: Expression, - pub ifs: Vec>, - pub is_async: bool, -} - -#[derive(Debug, PartialEq)] -pub struct ArgumentList { - pub args: Vec>, - pub keywords: Vec>, -} - -#[derive(Debug, PartialEq)] -pub struct Keyword { - pub name: Option, - pub value: Expression, -} - -#[derive(Debug, PartialEq)] -pub struct ExceptHandler { - pub location: Location, - pub typ: Option>, - pub name: Option, - pub body: Suite, -} - -/// An operator for a binary operation (an operation with two operands). -#[derive(Debug, PartialEq)] -pub enum Operator { - Add, - Sub, - Mult, - MatMult, - Div, - Mod, - Pow, - LShift, - RShift, - BitOr, - BitXor, - BitAnd, - FloorDiv, -} - -/// A boolean operation. -#[derive(Debug, PartialEq)] -pub enum BooleanOperator { - And, - Or, -} - -/// An unary operator. This is an operation with only a single operand. -#[derive(Debug, PartialEq)] -pub enum UnaryOperator { - Pos, - Neg, - Not, - Inv, -} - -/// A comparison operation. -#[derive(Debug, PartialEq)] -pub enum Comparison { - Equal, - NotEqual, - Less, - LessOrEqual, - Greater, - GreaterOrEqual, - In, - NotIn, - Is, - IsNot, -} - -/// A numeric literal. -#[derive(Debug, PartialEq)] -pub enum Number { - Integer { value: BigInt }, - Float { value: f64 }, - Complex { real: f64, imag: f64 }, -} - -/// Transforms a value prior to formatting it. -#[derive(Copy, Clone, Debug, PartialEq)] -pub enum ConversionFlag { - /// Converts by calling `str()`. - Str, - /// Converts by calling `ascii()`. - Ascii, - /// Converts by calling `repr()`. - Repr, -} - -#[derive(Debug, PartialEq)] -pub enum StringGroup { - Constant { - value: String, - }, - FormattedValue { - value: Box>, - conversion: Option, - spec: Option>>, - }, - Joined { - values: Vec>, - }, -} - -#[derive(Debug, PartialEq)] -pub enum Varargs { - None, - Unnamed, - Named(Parameter), -} - -impl Default for Varargs { - fn default() -> Varargs { - Varargs::None - } -} - -impl From>>> for Varargs { - fn from(opt: Option>>) -> Varargs { - match opt { - Some(inner_opt) => match inner_opt { - Some(param) => Varargs::Named(param), - None => Varargs::Unnamed, - }, - None => Varargs::None, - } - } -} diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs new file mode 100644 index 000000000..6dd97bae3 --- /dev/null +++ b/ast/src/ast_gen.rs @@ -0,0 +1,389 @@ +// File automatically generated by ast/asdl_rs.py. + +pub use crate::constant::*; +pub use crate::location::Location; + +type Ident = String; + +#[derive(Debug, PartialEq)] +pub struct Located { + pub location: Location, + pub custom: U, + pub node: T, +} + +impl Located { + pub fn new(location: Location, node: T) -> Self { + Self { + location, + custom: (), + node, + } + } +} + +#[derive(Debug, PartialEq)] +pub enum Mod { + Module { + body: Vec>, + type_ignores: Vec, + }, + Interactive { + body: Vec>, + }, + Expression { + body: Box>, + }, + FunctionType { + argtypes: Vec>, + returns: Box>, + }, +} + +#[derive(Debug, PartialEq)] +pub enum StmtKind { + FunctionDef { + name: Ident, + args: Box>, + body: Vec>, + decorator_list: Vec>, + returns: Option>>, + type_comment: Option, + }, + AsyncFunctionDef { + name: Ident, + args: Box>, + body: Vec>, + decorator_list: Vec>, + returns: Option>>, + type_comment: Option, + }, + ClassDef { + name: Ident, + bases: Vec>, + keywords: Vec>, + body: Vec>, + decorator_list: Vec>, + }, + Return { + value: Option>>, + }, + Delete { + targets: Vec>, + }, + Assign { + targets: Vec>, + value: Box>, + type_comment: Option, + }, + AugAssign { + target: Box>, + op: Operator, + value: Box>, + }, + AnnAssign { + target: Box>, + annotation: Box>, + value: Option>>, + simple: bool, + }, + For { + target: Box>, + iter: Box>, + body: Vec>, + orelse: Vec>, + type_comment: Option, + }, + AsyncFor { + target: Box>, + iter: Box>, + body: Vec>, + orelse: Vec>, + type_comment: Option, + }, + While { + test: Box>, + body: Vec>, + orelse: Vec>, + }, + If { + test: Box>, + body: Vec>, + orelse: Vec>, + }, + With { + items: Vec>, + body: Vec>, + type_comment: Option, + }, + AsyncWith { + items: Vec>, + body: Vec>, + type_comment: Option, + }, + Raise { + exc: Option>>, + cause: Option>>, + }, + Try { + body: Vec>, + handlers: Vec>, + orelse: Vec>, + finalbody: Vec>, + }, + Assert { + test: Box>, + msg: Option>>, + }, + Import { + names: Vec, + }, + ImportFrom { + module: Option, + names: Vec, + level: usize, + }, + Global { + names: Vec, + }, + Nonlocal { + names: Vec, + }, + Expr { + value: Box>, + }, + Pass, + Break, + Continue, +} +pub type Stmt = Located, U>; + +#[derive(Debug, PartialEq)] +pub enum ExprKind { + BoolOp { + op: Boolop, + values: Vec>, + }, + NamedExpr { + target: Box>, + value: Box>, + }, + BinOp { + left: Box>, + op: Operator, + right: Box>, + }, + UnaryOp { + op: Unaryop, + operand: Box>, + }, + Lambda { + args: Box>, + body: Box>, + }, + IfExp { + test: Box>, + body: Box>, + orelse: Box>, + }, + Dict { + keys: Vec>>>, + values: Vec>, + }, + Set { + elts: Vec>, + }, + ListComp { + elt: Box>, + generators: Vec>, + }, + SetComp { + elt: Box>, + generators: Vec>, + }, + DictComp { + key: Box>, + value: Box>, + generators: Vec>, + }, + GeneratorExp { + elt: Box>, + generators: Vec>, + }, + Await { + value: Box>, + }, + Yield { + value: Option>>, + }, + YieldFrom { + value: Box>, + }, + Compare { + left: Box>, + ops: Vec, + comparators: Vec>, + }, + Call { + func: Box>, + args: Vec>, + keywords: Vec>, + }, + FormattedValue { + value: Box>, + conversion: Option, + format_spec: Option>>, + }, + JoinedStr { + values: Vec>, + }, + Constant { + value: Constant, + kind: Option, + }, + Attribute { + value: Box>, + attr: Ident, + ctx: ExprContext, + }, + Subscript { + value: Box>, + slice: Box>, + ctx: ExprContext, + }, + Starred { + value: Box>, + ctx: ExprContext, + }, + Name { + id: Ident, + ctx: ExprContext, + }, + List { + elts: Vec>, + ctx: ExprContext, + }, + Tuple { + elts: Vec>, + ctx: ExprContext, + }, + Slice { + lower: Option>>, + upper: Option>>, + step: Option>>, + }, +} +pub type Expr = Located, U>; + +#[derive(Debug, PartialEq)] +pub enum ExprContext { + Load, + Store, + Del, +} + +#[derive(Debug, PartialEq)] +pub enum Boolop { + And, + Or, +} + +#[derive(Debug, PartialEq)] +pub enum Operator { + Add, + Sub, + Mult, + MatMult, + Div, + Mod, + Pow, + LShift, + RShift, + BitOr, + BitXor, + BitAnd, + FloorDiv, +} + +#[derive(Debug, PartialEq)] +pub enum Unaryop { + Invert, + Not, + UAdd, + USub, +} + +#[derive(Debug, PartialEq)] +pub enum Cmpop { + Eq, + NotEq, + Lt, + LtE, + Gt, + GtE, + Is, + IsNot, + In, + NotIn, +} + +#[derive(Debug, PartialEq)] +pub struct Comprehension { + pub target: Box>, + pub iter: Box>, + pub ifs: Vec>, + pub is_async: bool, +} + +#[derive(Debug, PartialEq)] +pub enum ExcepthandlerKind { + ExceptHandler { + type_: Option>>, + name: Option, + body: Vec>, + }, +} +pub type Excepthandler = Located, U>; + +#[derive(Debug, PartialEq)] +pub struct Arguments { + pub posonlyargs: Vec>, + pub args: Vec>, + pub vararg: Option>>, + pub kwonlyargs: Vec>, + pub kw_defaults: Vec>>>, + pub kwarg: Option>>, + pub defaults: Vec>, +} + +#[derive(Debug, PartialEq)] +pub struct ArgData { + pub arg: Ident, + pub annotation: Option>>, + pub type_comment: Option, +} +pub type Arg = Located, U>; + +#[derive(Debug, PartialEq)] +pub struct KeywordData { + pub arg: Option, + pub value: Box>, +} +pub type Keyword = Located, U>; + +#[derive(Debug, PartialEq)] +pub struct Alias { + pub name: Ident, + pub asname: Option, +} + +#[derive(Debug, PartialEq)] +pub struct Withitem { + pub context_expr: Box>, + pub optional_vars: Option>>, +} + +#[derive(Debug, PartialEq)] +pub enum TypeIgnore { + TypeIgnore { lineno: usize, tag: String }, +} diff --git a/ast/src/constant.rs b/ast/src/constant.rs new file mode 100644 index 000000000..f6c523c4c --- /dev/null +++ b/ast/src/constant.rs @@ -0,0 +1,41 @@ +use num_bigint::BigInt; + +#[derive(Debug, PartialEq)] +pub enum Constant { + None, + Bool(bool), + Str(String), + Bytes(Vec), + Int(BigInt), + Tuple(Vec), + Float(f64), + Complex { real: f64, imag: f64 }, + Ellipsis, +} + +impl From for Constant { + fn from(s: String) -> Constant { + Self::Str(s) + } +} +impl From> for Constant { + fn from(b: Vec) -> Constant { + Self::Bytes(b) + } +} +impl From for Constant { + fn from(b: bool) -> Constant { + Self::Bool(b) + } +} + +/// Transforms a value prior to formatting it. +#[derive(Copy, Clone, Debug, PartialEq)] +pub enum ConversionFlag { + /// Converts by calling `str()`. + Str, + /// Converts by calling `ascii()`. + Ascii, + /// Converts by calling `repr()`. + Repr, +} diff --git a/ast/src/impls.rs b/ast/src/impls.rs new file mode 100644 index 000000000..666acd1fd --- /dev/null +++ b/ast/src/impls.rs @@ -0,0 +1,53 @@ +use crate::{Constant, ExprKind}; + +impl ExprKind { + /// Returns a short name for the node suitable for use in error messages. + pub fn name(&self) -> &'static str { + match self { + ExprKind::BoolOp { .. } | ExprKind::BinOp { .. } | ExprKind::UnaryOp { .. } => { + "operator" + } + ExprKind::Subscript { .. } => "subscript", + ExprKind::Await { .. } => "await expression", + ExprKind::Yield { .. } | ExprKind::YieldFrom { .. } => "yield expression", + ExprKind::Compare { .. } => "comparison", + ExprKind::Attribute { .. } => "attribute", + ExprKind::Call { .. } => "function call", + ExprKind::Constant { value, .. } => match value { + Constant::Str(_) + | Constant::Int(_) + | Constant::Float(_) + | Constant::Complex { .. } + | Constant::Bytes(_) => "literal", + Constant::Tuple(_) => "tuple", + Constant::Bool(_) | Constant::None => "keyword", + Constant::Ellipsis => "ellipsis", + }, + ExprKind::List { .. } => "list", + ExprKind::Tuple { .. } => "tuple", + ExprKind::Dict { .. } => "dict display", + ExprKind::Set { .. } => "set display", + ExprKind::ListComp { .. } => "list comprehension", + ExprKind::DictComp { .. } => "dict comprehension", + ExprKind::SetComp { .. } => "set comprehension", + ExprKind::GeneratorExp { .. } => "generator expression", + ExprKind::Starred { .. } => "starred", + ExprKind::Slice { .. } => "slice", + ExprKind::JoinedStr { values } => { + if values + .iter() + .any(|e| matches!(e.node, ExprKind::JoinedStr { .. })) + { + "f-string expression" + } else { + "literal" + } + } + ExprKind::FormattedValue { .. } => "f-string expression", + ExprKind::Name { .. } => "name", + ExprKind::Lambda { .. } => "lambda", + ExprKind::IfExp { .. } => "conditional expression", + ExprKind::NamedExpr { .. } => "named expression", + } + } +} diff --git a/ast/src/lib.rs b/ast/src/lib.rs index acc0e3621..4bbb1e024 100644 --- a/ast/src/lib.rs +++ b/ast/src/lib.rs @@ -1,5 +1,9 @@ -mod ast; +mod ast_gen; +mod constant; +mod impls; mod location; -pub use ast::*; +pub use ast_gen::*; pub use location::Location; + +pub type Suite = Vec>; diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index 9baa8ff4d..d302365e2 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -19,4 +19,4 @@ ahash = "0.6" [dev-dependencies] rustpython-parser = { path = "../parser" } -insta = "1.1" +insta = "1.5" diff --git a/parser/Cargo.toml b/parser/Cargo.toml index 2550555fd..4282a587a 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -22,3 +22,6 @@ unic-ucd-ident = "0.9" unicode_names2 = "0.4" phf = { version = "0.8", features = ["macros"] } ahash = "0.6" + +[dev-dependencies] +insta = "1.5" diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index b67f9e462..34928d3a7 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -2,25 +2,31 @@ use std::iter; use std::mem; use std::str; -use crate::ast::{ConversionFlag, Expression, Location, StringGroup}; +use crate::ast::{Constant, ConversionFlag, Expr, ExprKind, Location}; use crate::error::{FStringError, FStringErrorType, ParseError}; use crate::parser::parse_expression; use self::FStringErrorType::*; -use self::StringGroup::*; struct FStringParser<'a> { chars: iter::Peekable>, + str_location: Location, } impl<'a> FStringParser<'a> { - fn new(source: &'a str) -> Self { + fn new(source: &'a str, str_location: Location) -> Self { Self { chars: source.chars().peekable(), + str_location, } } - fn parse_formatted_value(&mut self) -> Result { + #[inline] + fn expr(&self, node: ExprKind) -> Expr { + Expr::new(self.str_location, node) + } + + fn parse_formatted_value(&mut self) -> Result, FStringErrorType> { let mut expression = String::new(); let mut spec = None; let mut delims = Vec::new(); @@ -114,20 +120,23 @@ impl<'a> FStringParser<'a> { if in_nested { return Err(UnclosedLbrace); } - if nested { - spec = Some(Box::new(FormattedValue { - value: Box::new( - parse_fstring_expr(&spec_expression) - .map_err(|e| InvalidExpression(Box::new(e.error)))?, - ), - conversion: None, - spec: None, - })) + spec = Some(if nested { + Box::new( + self.expr(ExprKind::FormattedValue { + value: Box::new( + parse_fstring_expr(&spec_expression) + .map_err(|e| InvalidExpression(Box::new(e.error)))?, + ), + conversion: None, + format_spec: None, + }), + ) } else { - spec = Some(Box::new(Constant { - value: spec_expression.to_owned(), + Box::new(self.expr(ExprKind::Constant { + value: spec_expression.to_owned().into(), + kind: None, })) - } + }) } '(' | '{' | '[' => { expression.push(ch); @@ -155,35 +164,36 @@ impl<'a> FStringParser<'a> { if expression.is_empty() { return Err(EmptyExpression); } - if pred_expression_text.is_empty() { - return Ok(FormattedValue { + let ret = if pred_expression_text.is_empty() { + vec![self.expr(ExprKind::FormattedValue { value: Box::new( parse_fstring_expr(&expression) .map_err(|e| InvalidExpression(Box::new(e.error)))?, ), conversion, - spec, - }); + format_spec: spec, + })] } else { - return Ok(Joined { - values: vec![ - Constant { - value: pred_expression_text + "=", - }, - Constant { - value: trailing_seq, - }, - FormattedValue { - value: Box::new( - parse_fstring_expr(&expression) - .map_err(|e| InvalidExpression(Box::new(e.error)))?, - ), - conversion, - spec, - }, - ], - }); - } + vec![ + self.expr(ExprKind::Constant { + value: Constant::Str(pred_expression_text + "="), + kind: None, + }), + self.expr(ExprKind::Constant { + value: trailing_seq.into(), + kind: None, + }), + self.expr(ExprKind::FormattedValue { + value: Box::new( + parse_fstring_expr(&expression) + .map_err(|e| InvalidExpression(Box::new(e.error)))?, + ), + conversion, + format_spec: spec, + }), + ] + }; + return Ok(ret); } '"' | '\'' => { expression.push(ch); @@ -205,7 +215,7 @@ impl<'a> FStringParser<'a> { Err(UnclosedLbrace) } - fn parse(mut self) -> Result { + fn parse(mut self) -> Result { let mut content = String::new(); let mut values = vec![]; @@ -217,12 +227,13 @@ impl<'a> FStringParser<'a> { content.push('{'); } else { if !content.is_empty() { - values.push(Constant { - value: mem::replace(&mut content, String::new()), - }); + values.push(self.expr(ExprKind::Constant { + value: mem::take(&mut content).into(), + kind: None, + })); } - values.push(self.parse_formatted_value()?); + values.extend(self.parse_formatted_value()?); } } '}' => { @@ -240,151 +251,96 @@ impl<'a> FStringParser<'a> { } if !content.is_empty() { - values.push(Constant { value: content }) + values.push(self.expr(ExprKind::Constant { + value: content.into(), + kind: None, + })) } - Ok(match values.len() { - 0 => Constant { - value: String::new(), - }, + let s = match values.len() { + 0 => self.expr(ExprKind::Constant { + value: String::new().into(), + kind: None, + }), 1 => values.into_iter().next().unwrap(), - _ => Joined { values }, - }) + _ => self.expr(ExprKind::JoinedStr { values }), + }; + Ok(s) } } -fn parse_fstring_expr(source: &str) -> Result { +fn parse_fstring_expr(source: &str) -> Result { let fstring_body = format!("({})", source); - let mut expression = parse_expression(&fstring_body)?; - expression.location.go_left(); - Ok(expression) -} - -/// Parse an f-string into a string group. -fn parse_fstring(source: &str) -> Result { - FStringParser::new(source).parse() + parse_expression(&fstring_body) } /// Parse an fstring from a string, located at a certain position in the sourcecode. /// In case of errors, we will get the location and the error returned. -pub fn parse_located_fstring( - source: &str, - location: Location, -) -> Result { - parse_fstring(source).map_err(|error| FStringError { error, location }) +pub fn parse_located_fstring(source: &str, location: Location) -> Result { + FStringParser::new(source, location) + .parse() + .map_err(|error| FStringError { error, location }) } #[cfg(test)] mod tests { - use crate::ast; - use super::*; - fn mk_ident(name: &str, row: usize, col: usize) -> ast::Expression { - ast::Expression { - location: ast::Location::new(row, col), - custom: (), - node: ast::ExpressionType::Identifier { - name: name.to_owned(), - }, - } + fn parse_fstring(source: &str) -> Result { + FStringParser::new(source, Location::default()).parse() } #[test] fn test_parse_fstring() { - let source = String::from("{a}{ b }{{foo}}"); + let source = "{a}{ b }{{foo}}"; let parse_ast = parse_fstring(&source).unwrap(); - assert_eq!( - parse_ast, - Joined { - values: vec![ - FormattedValue { - value: Box::new(mk_ident("a", 1, 1)), - conversion: None, - spec: None, - }, - FormattedValue { - value: Box::new(mk_ident("b", 1, 2)), - conversion: None, - spec: None, - }, - Constant { - value: "{foo}".to_owned() - } - ] - } - ); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_fstring_nested_spec() { - let source = String::from("{foo:{spec}}"); + let source = "{foo:{spec}}"; let parse_ast = parse_fstring(&source).unwrap(); - assert_eq!( - parse_ast, - FormattedValue { - value: Box::new(mk_ident("foo", 1, 1)), - conversion: None, - spec: Some(Box::new(FormattedValue { - value: Box::new(mk_ident("spec", 1, 1)), - conversion: None, - spec: None, - })), - } - ); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_fstring_not_nested_spec() { - let source = String::from("{foo:spec}"); + let source = "{foo:spec}"; let parse_ast = parse_fstring(&source).unwrap(); - assert_eq!( - parse_ast, - FormattedValue { - value: Box::new(mk_ident("foo", 1, 1)), - conversion: None, - spec: Some(Box::new(Constant { - value: "spec".to_owned(), - })), - } - ); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_empty_fstring() { - assert_eq!( - parse_fstring(""), - Ok(Constant { - value: String::new(), - }), - ); + insta::assert_debug_snapshot!(parse_fstring("").unwrap()); } #[test] fn test_fstring_parse_selfdocumenting_base() { - let src = String::from("{user=}"); - let parse_ast = parse_fstring(&src); + let src = "{user=}"; + let parse_ast = parse_fstring(&src).unwrap(); - assert!(parse_ast.is_ok()); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_fstring_parse_selfdocumenting_base_more() { - let src = String::from("mix {user=} with text and {second=}"); - let parse_ast = parse_fstring(&src); + let src = "mix {user=} with text and {second=}"; + let parse_ast = parse_fstring(&src).unwrap(); - assert!(parse_ast.is_ok()); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_fstring_parse_selfdocumenting_format() { - let src = String::from("{user=:>10}"); - let parse_ast = parse_fstring(&src); + let src = "{user=:>10}"; + let parse_ast = parse_fstring(&src).unwrap(); - assert!(parse_ast.is_ok()); + insta::assert_debug_snapshot!(parse_ast); } #[test] @@ -414,36 +370,36 @@ mod tests { #[test] fn test_parse_fstring_not_equals() { - let source = String::from("{1 != 2}"); - let parse_ast = parse_fstring(&source); - assert!(parse_ast.is_ok()); + let source = "{1 != 2}"; + let parse_ast = parse_fstring(&source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_fstring_equals() { - let source = String::from("{42 == 42}"); - let parse_ast = parse_fstring(&source); - assert!(parse_ast.is_ok()); + let source = "{42 == 42}"; + let parse_ast = parse_fstring(&source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_fstring_selfdoc_prec_space() { - let source = String::from("{x =}"); - let parse_ast = parse_fstring(&source); - assert!(parse_ast.is_ok()); + let source = "{x =}"; + let parse_ast = parse_fstring(&source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_fstring_selfdoc_trailing_space() { - let source = String::from("{x= }"); - let parse_ast = parse_fstring(&source); - assert!(parse_ast.is_ok()); + let source = "{x= }"; + let parse_ast = parse_fstring(&source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_fstring_yield_expr() { - let source = String::from("{yield}"); - let parse_ast = parse_fstring(&source); - assert!(parse_ast.is_ok()); + let source = "{yield}"; + let parse_ast = parse_fstring(&source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); } } diff --git a/parser/src/function.rs b/parser/src/function.rs index 2ff18b44f..68d890f20 100644 --- a/parser/src/function.rs +++ b/parser/src/function.rs @@ -4,16 +4,22 @@ use std::collections::HashSet; use crate::ast; use crate::error::{LexicalError, LexicalErrorType}; -type ParameterDefs = (Vec, Vec); -type ParameterDef = (ast::Parameter, Option); +pub struct ArgumentList { + pub args: Vec, + pub keywords: Vec, +} + +type ParameterDefs = (Vec, Vec, Vec); +type ParameterDef = (ast::Arg, Option); pub fn parse_params( params: (Vec, Vec), ) -> Result { - let mut names = vec![]; + let mut posonly = Vec::with_capacity(params.0.len()); + let mut names = Vec::with_capacity(params.1.len()); let mut defaults = vec![]; - let mut try_default = |name: &ast::Parameter, default| { + let mut try_default = |name: &ast::Arg, default| { if let Some(default) = default { defaults.push(default); } else if !defaults.is_empty() { @@ -29,7 +35,7 @@ pub fn parse_params( for (name, default) in params.0 { try_default(&name, default)?; - names.push(name); + posonly.push(name); } for (name, default) in params.1 { @@ -37,31 +43,37 @@ pub fn parse_params( names.push(name); } - Ok((names, defaults)) + Ok((posonly, names, defaults)) } -type FunctionArgument = (Option>, ast::Expression); +type FunctionArgument = (Option<(ast::Location, Option)>, ast::Expr); -pub fn parse_args(func_args: Vec) -> Result { +pub fn parse_args(func_args: Vec) -> Result { let mut args = vec![]; let mut keywords = vec![]; let mut keyword_names = HashSet::with_capacity_and_hasher(func_args.len(), RandomState::new()); for (name, value) in func_args { match name { - Some(n) => { - if let Some(keyword_name) = n.clone() { - if keyword_names.contains(&keyword_name) { + Some((location, name)) => { + if let Some(keyword_name) = &name { + if keyword_names.contains(keyword_name) { return Err(LexicalError { error: LexicalErrorType::DuplicateKeywordArgumentError, - location: value.location, + location, }); } keyword_names.insert(keyword_name.clone()); } - keywords.push(ast::Keyword { name: n, value }); + keywords.push(ast::Keyword::new( + location, + ast::KeywordData { + arg: name, + value: Box::new(value), + }, + )); } None => { // Allow starred args after keyword arguments. @@ -76,9 +88,9 @@ pub fn parse_args(func_args: Vec) -> Result bool { - matches!(exp.node, ast::ExpressionType::Starred { .. }) +fn is_starred(exp: &ast::Expr) -> bool { + matches!(exp.node, ast::ExprKind::Starred { .. }) } diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 268be3863..9885fd5fd 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -111,9 +111,17 @@ pub static KEYWORDS: phf::Map<&'static str, Tok> = phf::phf_map! { pub type Spanned = (Location, Tok, Location); pub type LexResult = Result; +#[inline] pub fn make_tokenizer(source: &str) -> impl Iterator + '_ { + make_tokenizer_located(source, Location::new(0, 0)) +} + +pub fn make_tokenizer_located( + source: &str, + start_location: Location, +) -> impl Iterator + '_ { let nlh = NewlineHandler::new(source.chars()); - Lexer::new(nlh) + Lexer::new(nlh, start_location) } // The newline handler is an iterator which collapses different newline @@ -177,7 +185,7 @@ impl Lexer where T: Iterator, { - pub fn new(input: T) -> Self { + pub fn new(input: T, start: Location) -> Self { let mut lxr = Lexer { chars: input, at_begin_of_line: true, @@ -185,7 +193,7 @@ where indentation_stack: vec![Default::default()], pending: Vec::new(), chr0: None, - location: Location::new(0, 0), + location: start, chr1: None, chr2: None, }; diff --git a/parser/src/mode.rs b/parser/src/mode.rs index 7614aa82f..c03a538eb 100644 --- a/parser/src/mode.rs +++ b/parser/src/mode.rs @@ -1,15 +1,28 @@ +use crate::token::Tok; + #[derive(Clone, Copy)] pub enum Mode { - Program, - Statement, + Module, + Interactive, + Expression, +} + +impl Mode { + pub(crate) fn to_marker(self) -> Tok { + match self { + Self::Module => Tok::StartModule, + Self::Interactive => Tok::StartInteractive, + Self::Expression => Tok::StartExpression, + } + } } impl std::str::FromStr for Mode { type Err = ModeParseError; fn from_str(s: &str) -> Result { match s { - "exec" | "single" => Ok(Mode::Program), - "eval" => Ok(Mode::Statement), + "exec" | "single" => Ok(Mode::Module), + "eval" => Ok(Mode::Expression), _ => Err(ModeParseError { _priv: () }), } } diff --git a/parser/src/parser.rs b/parser/src/parser.rs index a2659c428..02845f775 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -12,7 +12,6 @@ use crate::error::ParseError; use crate::lexer; pub use crate::mode::Mode; use crate::python; -use crate::token; /* * Parse python code. @@ -20,33 +19,12 @@ use crate::token; * https://github.com/antlr/grammars-v4/tree/master/python3 */ -macro_rules! do_lalr_parsing { - ($input: expr, $pat: ident, $tok: ident) => {{ - let lxr = lexer::make_tokenizer($input); - let marker_token = (Default::default(), token::Tok::$tok, Default::default()); - let tokenizer = iter::once(Ok(marker_token)).chain(lxr); - - match python::TopParser::new().parse(tokenizer) { - Err(err) => Err(ParseError::from(err)), - Ok(top) => { - if let ast::Top::$pat(x) = top { - Ok(x) - } else { - unreachable!() - } - } - } - }}; -} - /// Parse a full python program, containing usually multiple lines. -pub fn parse_program(source: &str) -> Result { - do_lalr_parsing!(source, Program, StartProgram) -} - -/// Parse a single statement. -pub fn parse_statement(source: &str) -> Result, ParseError> { - do_lalr_parsing!(source, Statement, StartStatement) +pub fn parse_program(source: &str) -> Result { + parse(source, Mode::Module).map(|top| match top { + ast::Mod::Module { body, .. } => body, + _ => unreachable!(), + }) } /// Parses a python expression @@ -54,478 +32,136 @@ pub fn parse_statement(source: &str) -> Result, ParseError> /// # Example /// ``` /// extern crate num_bigint; -/// use num_bigint::BigInt; /// use rustpython_parser::{parser, ast}; /// let expr = parser::parse_expression("1 + 2").unwrap(); /// -/// assert_eq!(ast::Expression { +/// assert_eq!( +/// expr, +/// ast::Expr { /// location: ast::Location::new(1, 3), /// custom: (), -/// node: ast::ExpressionType::Binop { -/// a: Box::new(ast::Expression { +/// node: ast::ExprKind::BinOp { +/// left: Box::new(ast::Expr { /// location: ast::Location::new(1, 1), /// custom: (), -/// node: ast::ExpressionType::Number { -/// value: ast::Number::Integer { value: BigInt::from(1) } +/// node: ast::ExprKind::Constant { +/// value: ast::Constant::Int(1.into()), +/// kind: None, /// } /// }), /// op: ast::Operator::Add, -/// b: Box::new(ast::Expression { +/// right: Box::new(ast::Expr { /// location: ast::Location::new(1, 5), /// custom: (), -/// node: ast::ExpressionType::Number { -/// value: ast::Number::Integer { value: BigInt::from(2) } +/// node: ast::ExprKind::Constant { +/// value: ast::Constant::Int(2.into()), +/// kind: None, /// } /// }) /// } /// }, -/// expr); +/// ); /// /// ``` -pub fn parse_expression(source: &str) -> Result { - do_lalr_parsing!(source, Expression, StartExpression) +pub fn parse_expression(source: &str) -> Result { + parse(source, Mode::Expression).map(|top| match top { + ast::Mod::Expression { body } => *body, + _ => unreachable!(), + }) } // Parse a given source code -pub fn parse(source: &str, mode: Mode) -> Result { - Ok(match mode { - Mode::Program => { - let ast = parse_program(source)?; - ast::Top::Program(ast) - } - Mode::Statement => { - let statement = parse_statement(source)?; - ast::Top::Statement(statement) - } - }) +pub fn parse(source: &str, mode: Mode) -> Result { + let lxr = lexer::make_tokenizer(source); + let marker_token = (Default::default(), mode.to_marker(), Default::default()); + let tokenizer = iter::once(Ok(marker_token)).chain(lxr); + + python::TopParser::new() + .parse(tokenizer) + .map_err(ParseError::from) } #[cfg(test)] mod tests { - use super::ast; - use super::parse_expression; - use super::parse_program; - use super::parse_statement; - use num_bigint::BigInt; - - fn mk_ident(name: &str, row: usize, col: usize) -> ast::Expression { - ast::Expression { - location: ast::Location::new(row, col), - custom: (), - node: ast::ExpressionType::Identifier { - name: name.to_owned(), - }, - } - } - - fn make_int(value: i32, row: usize, col: usize) -> ast::Expression { - ast::Expression { - location: ast::Location::new(row, col), - custom: (), - node: ast::ExpressionType::Number { - value: ast::Number::Integer { - value: BigInt::from(value), - }, - }, - } - } - - fn make_string(value: &str, row: usize, col: usize) -> ast::Expression { - ast::Expression { - location: ast::Location::new(row, col), - custom: (), - node: ast::ExpressionType::String { - value: ast::StringGroup::Constant { - value: String::from(value), - }, - }, - } - } - - fn as_statement(expr: ast::Expression) -> ast::Statement { - ast::Statement { - location: expr.location, - custom: (), - node: ast::StatementType::Expression { expression: expr }, - } - } + use super::*; #[test] fn test_parse_empty() { - let parse_ast = parse_program(""); - assert_eq!(parse_ast, Ok(ast::Program { statements: vec![] })) + let parse_ast = parse_program("").unwrap(); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_print_hello() { let source = String::from("print('Hello world')"); let parse_ast = parse_program(&source).unwrap(); - assert_eq!( - parse_ast, - ast::Program { - statements: vec![ast::Statement { - location: ast::Location::new(1, 1), - custom: (), - node: ast::StatementType::Expression { - expression: ast::Expression { - custom: (), - location: ast::Location::new(1, 6), - node: ast::ExpressionType::Call { - function: Box::new(mk_ident("print", 1, 1)), - args: vec![make_string("Hello world", 1, 8)], - keywords: vec![], - } - }, - }, - },], - } - ); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_print_2() { let source = String::from("print('Hello world', 2)"); let parse_ast = parse_program(&source).unwrap(); - assert_eq!( - parse_ast, - ast::Program { - statements: vec![ast::Statement { - location: ast::Location::new(1, 1), - custom: (), - node: ast::StatementType::Expression { - expression: ast::Expression { - location: ast::Location::new(1, 6), - custom: (), - node: ast::ExpressionType::Call { - function: Box::new(mk_ident("print", 1, 1)), - args: vec![make_string("Hello world", 1, 8), make_int(2, 1, 22),], - keywords: vec![], - }, - }, - }, - },], - } - ); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_kwargs() { let source = String::from("my_func('positional', keyword=2)"); let parse_ast = parse_program(&source).unwrap(); - assert_eq!( - parse_ast, - ast::Program { - statements: vec![ast::Statement { - location: ast::Location::new(1, 1), - custom: (), - node: ast::StatementType::Expression { - expression: ast::Expression { - location: ast::Location::new(1, 8), - custom: (), - node: ast::ExpressionType::Call { - function: Box::new(mk_ident("my_func", 1, 1)), - args: vec![make_string("positional", 1, 10)], - keywords: vec![ast::Keyword { - name: Some("keyword".to_owned()), - value: make_int(2, 1, 31), - }], - } - }, - }, - },], - } - ); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_if_elif_else() { let source = String::from("if 1: 10\nelif 2: 20\nelse: 30"); - let parse_ast = parse_statement(&source).unwrap(); - assert_eq!( - parse_ast, - vec![ast::Statement { - location: ast::Location::new(1, 1), - custom: (), - node: ast::StatementType::If { - test: make_int(1, 1, 4), - body: vec![as_statement(make_int(10, 1, 7))], - orelse: Some(vec![ast::Statement { - location: ast::Location::new(2, 1), - custom: (), - node: ast::StatementType::If { - test: make_int(2, 2, 6), - body: vec![as_statement(make_int(20, 2, 9))], - orelse: Some(vec![as_statement(make_int(30, 3, 7))]), - } - },]), - } - }] - ); + let parse_ast = parse_program(&source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_lambda() { - let source = String::from("lambda x, y: x * y"); // lambda(x, y): x * y"); - let parse_ast = parse_statement(&source); - assert_eq!( - parse_ast, - Ok(vec![as_statement(ast::Expression { - location: ast::Location::new(1, 1), - custom: (), - node: ast::ExpressionType::Lambda { - args: Box::new(ast::Parameters { - posonlyargs_count: 0, - args: vec![ - ast::Parameter { - location: ast::Location::new(1, 8), - arg: String::from("x"), - annotation: None, - }, - ast::Parameter { - location: ast::Location::new(1, 11), - arg: String::from("y"), - annotation: None, - } - ], - kwonlyargs: vec![], - vararg: ast::Varargs::None, - kwarg: ast::Varargs::None, - defaults: vec![], - kw_defaults: vec![], - }), - body: Box::new(ast::Expression { - location: ast::Location::new(1, 16), - custom: (), - node: ast::ExpressionType::Binop { - a: Box::new(mk_ident("x", 1, 14)), - op: ast::Operator::Mult, - b: Box::new(mk_ident("y", 1, 18)) - } - }) - } - })]) - ) + let source = "lambda x, y: x * y"; // lambda(x, y): x * y"; + let parse_ast = parse_program(&source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_tuples() { - let source = String::from("a, b = 4, 5"); + let source = "a, b = 4, 5"; - assert_eq!( - parse_statement(&source), - Ok(vec![ast::Statement { - location: ast::Location::new(1, 1), - custom: (), - node: ast::StatementType::Assign { - targets: vec![ast::Expression { - custom: (), - location: ast::Location::new(1, 1), - node: ast::ExpressionType::Tuple { - elements: vec![mk_ident("a", 1, 1), mk_ident("b", 1, 4),] - } - }], - value: ast::Expression { - location: ast::Location::new(1, 8), - custom: (), - node: ast::ExpressionType::Tuple { - elements: vec![make_int(4, 1, 8), make_int(5, 1, 11),] - } - } - } - }]) - ) + insta::assert_debug_snapshot!(parse_program(&source).unwrap()); } #[test] fn test_parse_class() { - let source = String::from( - "class Foo(A, B):\n def __init__(self):\n pass\n def method_with_default(self, arg='default'):\n pass", - ); - assert_eq!( - parse_statement(&source), - Ok(vec![ast::Statement { - location: ast::Location::new(1, 1), - custom: (), - node: ast::StatementType::ClassDef { - name: String::from("Foo"), - bases: vec![mk_ident("A", 1, 11), mk_ident("B", 1, 14)], - keywords: vec![], - body: vec![ - ast::Statement { - location: ast::Location::new(2, 2), - custom: (), - node: ast::StatementType::FunctionDef { - is_async: false, - name: String::from("__init__"), - args: Box::new(ast::Parameters { - posonlyargs_count: 0, - args: vec![ast::Parameter { - location: ast::Location::new(2, 15), - arg: String::from("self"), - annotation: None, - }], - kwonlyargs: vec![], - vararg: ast::Varargs::None, - kwarg: ast::Varargs::None, - defaults: vec![], - kw_defaults: vec![], - }), - body: vec![ast::Statement { - location: ast::Location::new(3, 3), - custom: (), - node: ast::StatementType::Pass, - }], - decorator_list: vec![], - returns: None, - } - }, - ast::Statement { - location: ast::Location::new(4, 2), - custom: (), - node: ast::StatementType::FunctionDef { - is_async: false, - name: String::from("method_with_default"), - args: Box::new(ast::Parameters { - posonlyargs_count: 0, - args: vec![ - ast::Parameter { - location: ast::Location::new(4, 26), - arg: String::from("self"), - annotation: None, - }, - ast::Parameter { - location: ast::Location::new(4, 32), - arg: String::from("arg"), - annotation: None, - } - ], - kwonlyargs: vec![], - vararg: ast::Varargs::None, - kwarg: ast::Varargs::None, - defaults: vec![make_string("default", 4, 37)], - kw_defaults: vec![], - }), - body: vec![ast::Statement { - location: ast::Location::new(5, 3), - custom: (), - node: ast::StatementType::Pass, - }], - decorator_list: vec![], - returns: None, - } - } - ], - decorator_list: vec![], - } - }]) - ) + let source = "\ +class Foo(A, B): + def __init__(self): + pass + def method_with_default(self, arg='default'): + pass"; + insta::assert_debug_snapshot!(parse_program(&source).unwrap()); } #[test] fn test_parse_dict_comprehension() { let source = String::from("{x1: x2 for y in z}"); let parse_ast = parse_expression(&source).unwrap(); - assert_eq!( - parse_ast, - ast::Expression { - location: ast::Location::new(1, 1), - custom: (), - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::Dict { - key: mk_ident("x1", 1, 2), - value: mk_ident("x2", 1, 6), - }), - generators: vec![ast::Comprehension { - location: ast::Location::new(1, 9), - target: mk_ident("y", 1, 13), - iter: mk_ident("z", 1, 18), - ifs: vec![], - is_async: false, - }], - } - } - ); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_list_comprehension() { let source = String::from("[x for y in z]"); let parse_ast = parse_expression(&source).unwrap(); - assert_eq!( - parse_ast, - ast::Expression { - custom: (), - location: ast::Location::new(1, 1), - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::List { - element: mk_ident("x", 1, 2), - }), - generators: vec![ast::Comprehension { - location: ast::Location::new(1, 4), - target: mk_ident("y", 1, 8), - iter: mk_ident("z", 1, 13), - ifs: vec![], - is_async: false, - }], - } - } - ); + insta::assert_debug_snapshot!(parse_ast); } #[test] fn test_parse_double_list_comprehension() { let source = String::from("[x for y, y2 in z for a in b if a < 5 if a > 10]"); let parse_ast = parse_expression(&source).unwrap(); - assert_eq!( - parse_ast, - ast::Expression { - custom: (), - location: ast::Location::new(1, 1), - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::List { - element: mk_ident("x", 1, 2) - }), - generators: vec![ - ast::Comprehension { - location: ast::Location::new(1, 4), - target: ast::Expression { - custom: (), - location: ast::Location::new(1, 8), - node: ast::ExpressionType::Tuple { - elements: vec![mk_ident("y", 1, 8), mk_ident("y2", 1, 11),], - } - }, - iter: mk_ident("z", 1, 17), - ifs: vec![], - is_async: false, - }, - ast::Comprehension { - location: ast::Location::new(1, 19), - target: mk_ident("a", 1, 23), - iter: mk_ident("b", 1, 28), - ifs: vec![ - ast::Expression { - custom: (), - location: ast::Location::new(1, 35), - node: ast::ExpressionType::Compare { - vals: vec![mk_ident("a", 1, 33), make_int(5, 1, 37),], - ops: vec![ast::Comparison::Less], - } - }, - ast::Expression { - custom: (), - location: ast::Location::new(1, 44), - node: ast::ExpressionType::Compare { - vals: vec![mk_ident("a", 1, 42), make_int(10, 1, 46),], - ops: vec![ast::Comparison::Greater], - }, - }, - ], - is_async: false, - } - ], - } - } - ); + insta::assert_debug_snapshot!(parse_ast); } } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 4fdfdc2f3..db0152e90 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -7,7 +7,7 @@ use std::iter::FromIterator; use crate::ast; use crate::fstring::parse_located_fstring; -use crate::function::{parse_args, parse_params}; +use crate::function::{ArgumentList, parse_args, parse_params}; use crate::error::LexicalError; use crate::lexer; @@ -18,15 +18,15 @@ grammar; // This is a hack to reduce the amount of lalrpop tables generated: // For each public entry point, a full parse table is generated. // By having only a single pub function, we reduce this to one. -pub Top: ast::Top = { - StartProgram => ast::Top::Program(p), - StartStatement => ast::Top::Statement(s), - StartExpression ("\n")* => ast::Top::Expression(e), +pub Top: ast::Mod = { + StartModule => ast::Mod::Module { body, type_ignores: vec![] }, + StartInteractive => ast::Mod::Interactive { body }, + StartExpression ("\n")* => ast::Mod::Expression { body: Box::new(body) }, }; -Program: ast::Program = { - => ast::Program { - statements: Vec::from_iter(lines.into_iter().flatten()) +Program: ast::Suite = { + => { + lines.into_iter().flatten().collect() }, }; @@ -54,7 +54,7 @@ SimpleStatement: ast::Suite = { } }; -SmallStatement: ast::Statement = { +SmallStatement: ast::Stmt = { ExpressionStatement, PassStatement, DelStatement, @@ -65,34 +65,34 @@ SmallStatement: ast::Statement = { AssertStatement, }; -PassStatement: ast::Statement = { +PassStatement: ast::Stmt = { "pass" => { - ast::Statement { + ast::Stmt { location, custom: (), - node: ast::StatementType::Pass, + node: ast::StmtKind::Pass, } }, }; -DelStatement: ast::Statement = { +DelStatement: ast::Stmt = { "del" => { - ast::Statement { + ast::Stmt { location, custom: (), - node: ast::StatementType::Delete { targets }, + node: ast::StmtKind::Delete { targets }, } }, }; -ExpressionStatement: ast::Statement = { +ExpressionStatement: ast::Stmt = { => { // Just an expression, no assignment: if suffix.is_empty() { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Expression { expression } + node: ast::StmtKind::Expr { value: Box::new(expression) } } } else { let mut targets = vec![expression]; @@ -102,20 +102,20 @@ ExpressionStatement: ast::Statement = { targets.push(values.remove(0)); } - let value = values.into_iter().next().unwrap(); + let value = Box::new(values.into_iter().next().unwrap()); - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Assign { targets, value }, + node: ast::StmtKind::Assign { targets, value, type_comment: None }, } } }, => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::AugAssign { + node: ast::StmtKind::AugAssign { target: Box::new(target), op, value: Box::new(rhs) @@ -123,61 +123,45 @@ ExpressionStatement: ast::Statement = { } }, ":" => { - ast::Statement { + let simple = matches!(target.node, ast::ExprKind::Name { .. }); + ast::Stmt { custom: (), location, - node: ast::StatementType::AnnAssign { + node: ast::StmtKind::AnnAssign { target: Box::new(target), annotation: Box::new(annotation), - value: rhs + value: rhs.map(Box::new), + simple, }, } }, }; -AssignSuffix: ast::Expression = { +AssignSuffix: ast::Expr = { "=" => e }; -TestListOrYieldExpr: ast::Expression = { +TestListOrYieldExpr: ast::Expr = { TestList, YieldExpr } -TestOrStarExprList: ast::Expression = { - > => { - if elements.len() == 1 && comma.is_none() { - elements.into_iter().next().unwrap() - } else { - ast::Expression { - location, - custom: (), - node: ast::ExpressionType::Tuple { elements } - } - } - } +#[inline] +TestOrStarExprList: ast::Expr = { + // as far as I can tell, these were the same + TestList }; -TestOrStarNamedExprList: ast::Expression = { - > => { - if elements.len() == 1 && comma.is_none() { - elements.into_iter().next().unwrap() - } else { - ast::Expression { - location, - custom: (), - node: ast::ExpressionType::Tuple { elements } - } - } - } +TestOrStarNamedExprList: ast::Expr = { + GenericList }; -TestOrStarExpr: ast::Expression = { +TestOrStarExpr: ast::Expr = { Test, StarExpr, }; -TestOrStarNamedExpr: ast::Expression = { +TestOrStarNamedExpr: ast::Expr = { NamedExpressionTest, StarExpr, }; @@ -198,69 +182,69 @@ AugAssign: ast::Operator = { "//=" => ast::Operator::FloorDiv, }; -FlowStatement: ast::Statement = { +FlowStatement: ast::Stmt = { "break" => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Break, + node: ast::StmtKind::Break, } }, "continue" => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Continue, + node: ast::StmtKind::Continue, } }, "return" => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Return { value }, + node: ast::StmtKind::Return { value: value.map(Box::new) }, } }, => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Expression { expression }, + node: ast::StmtKind::Expr { value: Box::new(expression) }, } }, RaiseStatement, }; -RaiseStatement: ast::Statement = { +RaiseStatement: ast::Stmt = { "raise" => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Raise { exception: None, cause: None }, + node: ast::StmtKind::Raise { exc: None, cause: None }, } }, "raise" => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Raise { exception: Some(t), cause: c.map(|x| x.1) }, + node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) }, } }, }; -ImportStatement: ast::Statement = { +ImportStatement: ast::Stmt = { "import" >> => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Import { names }, + node: ast::StmtKind::Import { names }, } }, "from" "import" => { let (level, module) = source; - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::ImportFrom { + node: ast::StmtKind::ImportFrom { level, module, names @@ -283,19 +267,19 @@ ImportDots: usize = { "." => 1, }; -ImportAsNames: Vec = { +ImportAsNames: Vec = { >> => i, "(" >> ","? ")" => i, "*" => { // Star import all - vec![ast::ImportSymbol { symbol: "*".to_string(), alias: None }] + vec![ast::Alias { name: "*".to_string(), asname: None }] }, }; #[inline] -ImportAsAlias: ast::ImportSymbol = { - => ast::ImportSymbol { symbol, alias: a.map(|a| a.1) }, +ImportAsAlias: ast::Alias = { + => ast::Alias { name, asname: a.map(|a| a.1) }, }; // A name like abc or abc.def.ghi @@ -311,39 +295,40 @@ DottedName: String = { }, }; -GlobalStatement: ast::Statement = { +GlobalStatement: ast::Stmt = { "global" > => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Global { names } + node: ast::StmtKind::Global { names } } }, }; -NonlocalStatement: ast::Statement = { +NonlocalStatement: ast::Stmt = { "nonlocal" > => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Nonlocal { names } + node: ast::StmtKind::Nonlocal { names } } }, }; -AssertStatement: ast::Statement = { +AssertStatement: ast::Stmt = { "assert" => { - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::Assert { - test, msg: msg.map(|e| e.1) + node: ast::StmtKind::Assert { + test: Box::new(test), + msg: msg.map(|e| Box::new(e.1)) } } }, }; -CompoundStatement: ast::Statement = { +CompoundStatement: ast::Stmt = { IfStatement, WhileStatement, ForStatement, @@ -353,37 +338,37 @@ CompoundStatement: ast::Statement = { ClassDef, }; -IfStatement: ast::Statement = { +IfStatement: ast::Stmt = { "if" ":" => { // Determine last else: - let mut last = s3.map(|s| s.2); + let mut last = s3.map(|s| s.2).unwrap_or_default(); // handle elif: for i in s2.into_iter().rev() { - let x = ast::Statement { + let x = ast::Stmt { custom: (), location: i.0, - node: ast::StatementType::If { test: i.2, body: i.4, orelse: last }, + node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last }, }; - last = Some(vec![x]); + last = vec![x]; } - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::If { test, body, orelse: last } + node: ast::StmtKind::If { test: Box::new(test), body, orelse: last } } }, }; -WhileStatement: ast::Statement = { +WhileStatement: ast::Stmt = { "while" ":" => { - let orelse = s2.map(|s| s.2); - ast::Statement { + let orelse = s2.map(|s| s.2).unwrap_or_default(); + ast::Stmt { custom: (), location, - node: ast::StatementType::While { - test, + node: ast::StmtKind::While { + test: Box::new(test), body, orelse }, @@ -391,32 +376,29 @@ WhileStatement: ast::Statement = { }, }; -ForStatement: ast::Statement = { +ForStatement: ast::Stmt = { "for" "in" ":" => { - let is_async = is_async.is_some(); - let orelse = s2.map(|s| s.2); - ast::Statement { - custom: (), - location, - node: ast::StatementType::For { - is_async, - target: Box::new(target), - iter: Box::new(iter), - body, - orelse - }, - } + let orelse = s2.map(|s| s.2).unwrap_or_default(); + let target = Box::new(target); + let iter = Box::new(iter); + let type_comment = None; + let node = if is_async.is_some() { + ast::StmtKind::AsyncFor { target, iter, body, orelse, type_comment } + } else { + ast::StmtKind::For { target, iter, body, orelse, type_comment } + }; + ast::Stmt::new(location, node) }, }; -TryStatement: ast::Statement = { +TryStatement: ast::Stmt = { "try" ":" => { - let orelse = else_suite.map(|s| s.2); - let finalbody = finally.map(|s| s.2); - ast::Statement { + let orelse = else_suite.map(|s| s.2).unwrap_or_default(); + let finalbody = finally.map(|s| s.2).unwrap_or_default(); + ast::Stmt { custom: (), location, - node: ast::StatementType::Try { + node: ast::StmtKind::Try { body, handlers, orelse, @@ -426,12 +408,12 @@ TryStatement: ast::Statement = { }, "try" ":" => { let handlers = vec![]; - let orelse = None; - let finalbody = Some(finally.2); - ast::Statement { + let orelse = vec![]; + let finalbody = finally.2; + ast::Stmt { custom: (), location, - node: ast::StatementType::Try { + node: ast::StmtKind::Try { body, handlers, orelse, @@ -441,124 +423,134 @@ TryStatement: ast::Statement = { }, }; -ExceptClause: ast::ExceptHandler = { +ExceptClause: ast::Excepthandler = { "except" ":" => { - ast::ExceptHandler { + ast::Excepthandler::new( location, - typ, - name: None, - body, - } + ast::ExcepthandlerKind::ExceptHandler { + type_: typ.map(Box::new), + name: None, + body, + }, + ) }, "except" ":" => { - ast::ExceptHandler { + ast::Excepthandler::new( location, - typ: Some(x.0), - name: Some(x.2), - body, - } - }, -}; - -WithStatement: ast::Statement = { - "with" > ":" => { - let is_async = is_async.is_some(); - ast::Statement { - custom: (), - location, - node: ast::StatementType::With { is_async, items, body }, - } - }, -}; - -WithItem: ast::WithItem = { - => { - let optional_vars = n.map(|val| val.1); - ast::WithItem { context_expr, optional_vars } - }, -}; - -FuncDef: ast::Statement = { - "def" " Test)?> ":" => { - let is_async = is_async.is_some(); - ast::Statement { - custom: (), - location, - node: ast::StatementType::FunctionDef { - is_async, - name, - args: Box::new(args), + ast::ExcepthandlerKind::ExceptHandler { + type_: Some(Box::new(x.0)), + name: Some(x.2), body, - decorator_list, - returns: r.map(|x| x.1), - } - } + }, + ) }, }; -Parameters: ast::Parameters = { - "(" )?> ")" => a.unwrap_or_default(), +WithStatement: ast::Stmt = { + "with" > ":" => { + let type_comment = None; + let node = if is_async.is_some() { + ast::StmtKind::AsyncWith { items, body, type_comment } + } else { + ast::StmtKind::With { items, body, type_comment } + }; + ast::Stmt::new(location, node) + }, +}; + +WithItem: ast::Withitem = { + => { + let optional_vars = n.map(|val| Box::new(val.1)); + let context_expr = Box::new(context_expr); + ast::Withitem { context_expr, optional_vars } + }, +}; + +FuncDef: ast::Stmt = { + "def" " Test)?> ":" => { + let args = Box::new(args); + let returns = r.map(|x| Box::new(x.1)); + let type_comment = None; + let node = if is_async.is_some() { + ast::StmtKind::AsyncFunctionDef { name, args, body, decorator_list, returns, type_comment } + } else { + ast::StmtKind::FunctionDef { name, args, body, decorator_list, returns, type_comment } + }; + ast::Stmt::new(location, node) + }, +}; + +Parameters: ast::Arguments = { + "(" )?> ")" => { + a.unwrap_or_else(|| ast::Arguments { + posonlyargs: vec![], + args: vec![], + vararg: None, + kwonlyargs: vec![], + kw_defaults: vec![], + kwarg: None, + defaults: vec![] + }) + } }; // Note that this is a macro which is used once for function defs, and // once for lambda defs. -ParameterList: ast::Parameters = { +ParameterList: ast::Arguments = { > )?> ","? =>? { - let posonlyargs_count = param1.0.len(); - let (names, defaults) = parse_params(param1)?; + let (posonlyargs, args, defaults) = parse_params(param1)?; // Now gather rest of parameters: let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1); - Ok(ast::Parameters { - posonlyargs_count, - args: names, + Ok(ast::Arguments { + posonlyargs, + args, kwonlyargs, - vararg: vararg.into(), - kwarg: kwarg.into(), + vararg, + kwarg, defaults, kw_defaults, }) }, > )> ","? =>? { - let posonlyargs_count = param1.0.len(); - let (names, defaults) = parse_params(param1)?; + let (posonlyargs, args, defaults) = parse_params(param1)?; // Now gather rest of parameters: let vararg = None; let kwonlyargs = vec![]; let kw_defaults = vec![]; - let kwarg = Some(kw.1); + let kwarg = kw.1; - Ok(ast::Parameters { - posonlyargs_count, - args: names, + Ok(ast::Arguments { + posonlyargs, + args, kwonlyargs, - vararg: vararg.into(), - kwarg: kwarg.into(), + vararg, + kwarg, defaults, kw_defaults, }) }, > ","? => { let (vararg, kwonlyargs, kw_defaults, kwarg) = params; - ast::Parameters { - posonlyargs_count: 0, + ast::Arguments { + posonlyargs: vec![], args: vec![], kwonlyargs, - vararg: vararg.into(), - kwarg: kwarg.into(), + vararg, + kwarg, defaults: vec![], kw_defaults, } }, - > ","? => { - ast::Parameters { - posonlyargs_count: 0, + > ","? => { + ast::Arguments { + posonlyargs: vec![], args: vec![], kwonlyargs: vec![], - vararg: ast::Varargs::None, - kwarg: Some(kw).into(), + vararg: None, + kwarg, defaults: vec![], kw_defaults: vec![], } @@ -567,7 +559,7 @@ ParameterList: ast::Parameters = { // Use inline here to make sure the "," is not creating an ambiguity. #[inline] -ParameterDefs: (Vec<(ast::Parameter, Option)>, Vec<(ast::Parameter, Option)>) = { +ParameterDefs: (Vec<(ast::Arg, Option)>, Vec<(ast::Arg, Option)>) = { >> => { (vec![], args) }, @@ -576,57 +568,61 @@ ParameterDefs: (Vec<(ast::Parameter, Option)>, Vec<(as }, }; -ParameterDef: (ast::Parameter, Option) = { +ParameterDef: (ast::Arg, Option) = { => (i, None), "=" => (i, Some(e)), }; -UntypedParameter: ast::Parameter = { - => ast::Parameter { location, arg, annotation: None }, +UntypedParameter: ast::Arg = { + => ast::Arg::new( + location, + ast::ArgData { arg, annotation: None, type_comment: None }, + ), }; -TypedParameter: ast::Parameter = { +TypedParameter: ast::Arg = { => { let annotation = a.map(|x| Box::new(x.1)); - ast::Parameter { location, arg, annotation } + ast::Arg::new(location, ast::ArgData { arg, annotation, type_comment: None }) }, }; // Use inline here to make sure the "," is not creating an ambiguity. // TODO: figure out another grammar that makes this inline no longer required. #[inline] -ParameterListStarArgs: (Option>, Vec, Vec>, Option>) = { +ParameterListStarArgs: (Option>, Vec, Vec>>, Option>) = { "*" )*> )?> => { // Extract keyword arguments: let mut kwonlyargs = vec![]; let mut kw_defaults = vec![]; for (name, value) in kw.into_iter().map(|x| x.1) { kwonlyargs.push(name); - kw_defaults.push(value); + kw_defaults.push(value.map(Box::new)); } - let kwarg = kwarg.map(|n| n.1); + let kwarg = kwarg.map(|n| n.1).flatten(); + let va = va.map(Box::new); - (Some(va), kwonlyargs, kw_defaults, kwarg) + (va, kwonlyargs, kw_defaults, kwarg) } }; -KwargParameter: Option = { +KwargParameter: Option> = { "**" => { - kwarg + kwarg.map(Box::new) } }; -ClassDef: ast::Statement = { +ClassDef: ast::Stmt = { "class" ":" => { let (bases, keywords) = match a { Some((_, arg, _)) => (arg.args, arg.keywords), None => (vec![], vec![]), }; - ast::Statement { + ast::Stmt { custom: (), location, - node: ast::StatementType::ClassDef { + node: ast::StmtKind::ClassDef { name, bases, keywords, @@ -637,51 +633,33 @@ ClassDef: ast::Statement = { }, }; -Path: ast::Expression = { - => ast::Expression { - location, - custom: (), - node: ast::ExpressionType::Identifier { name: n } - }, - "." => { - ast::Expression { - location, - custom: (), - node: ast::ExpressionType::Attribute { - value: Box::new(p), - name: n, - } - } - }, -}; - // Decorators: -Decorator: ast::Expression = { +Decorator: ast::Expr = { "@" "\n" => { p }, }; -YieldExpr: ast::Expression = { - "yield" => ast::Expression { +YieldExpr: ast::Expr = { + "yield" => ast::Expr { location, custom: (), - node: ast::ExpressionType::Yield { value: value.map(Box::new) } + node: ast::ExprKind::Yield { value: value.map(Box::new) } }, - "yield" "from" => ast::Expression { + "yield" "from" => ast::Expr { location, custom: (), - node: ast::ExpressionType::YieldFrom { value: Box::new(e) } + node: ast::ExprKind::YieldFrom { value: Box::new(e) } }, }; -Test: ast::Expression = { +Test: ast::Expr = { => { if let Some(c) = condition { - ast::Expression { + ast::Expr { location: c.0, custom: (), - node: ast::ExpressionType::IfExpression { + node: ast::ExprKind::IfExp { test: Box::new(c.2), body: Box::new(expr), orelse: Box::new(c.4), @@ -694,20 +672,18 @@ Test: ast::Expression = { LambdaDef, }; -NamedExpressionTest: ast::Expression = { +NamedExpressionTest: ast::Expr = { ?> ":" => - ast::Expression { +LambdaDef: ast::Expr = { + "lambda" ?> ":" => { + let p = p.unwrap_or_else(|| { + ast::Arguments { + posonlyargs: vec![], + args: vec![], + vararg: None, + kwonlyargs: vec![], + kw_defaults: vec![], + kwarg: None, + defaults: vec![] + } + }); + ast::Expr { location, custom: (), - node: ast::ExpressionType::Lambda { - args: Box::new(p.unwrap_or_default()), + node: ast::ExprKind::Lambda { + args: Box::new(p), body: Box::new(body) } - } + } + } } -OrTest: ast::Expression = { +OrTest: ast::Expr = { => { if e2.is_empty() { e1 } else { let mut values = vec![e1]; values.extend(e2.into_iter().map(|e| e.1)); - ast::Expression { + ast::Expr { location, custom: (), - node: ast::ExpressionType::BoolOp { op: ast::BooleanOperator::Or, values } + node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values } } } }, }; -AndTest: ast::Expression = { +AndTest: ast::Expr = { => { if e2.is_empty() { e1 } else { let mut values = vec![e1]; values.extend(e2.into_iter().map(|e| e.1)); - ast::Expression { + ast::Expr { location, custom: (), - node: ast::ExpressionType::BoolOp { op: ast::BooleanOperator::And, values } + node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values } } } }, }; -NotTest: ast::Expression = { - "not" => ast::Expression { +NotTest: ast::Expr = { + "not" => ast::Expr { location, custom: (), - node: ast::ExpressionType::Unop { a: Box::new(e), op: ast::UnaryOperator::Not } + node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not } }, Comparison, }; -Comparison: ast::Expression = { - => { - let mut vals = vec![e]; - let mut ops = vec![]; - for x in comparisons { - ops.push(x.0); - vals.push(x.1); - } - ast::Expression { +Comparison: ast::Expr = { + => { + let (ops, comparators) = comparisons.into_iter().unzip(); + ast::Expr { location, custom: (), - node: ast::ExpressionType::Compare { vals, ops } + node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators } } }, Expression, }; -CompOp: ast::Comparison = { - "==" => ast::Comparison::Equal, - "!=" => ast::Comparison::NotEqual, - "<" => ast::Comparison::Less, - "<=" => ast::Comparison::LessOrEqual, - ">" => ast::Comparison::Greater, - ">=" => ast::Comparison::GreaterOrEqual, - "in" => ast::Comparison::In, - "not" "in" => ast::Comparison::NotIn, - "is" => ast::Comparison::Is, - "is" "not" => ast::Comparison::IsNot, +CompOp: ast::Cmpop = { + "==" => ast::Cmpop::Eq, + "!=" => ast::Cmpop::NotEq, + "<" => ast::Cmpop::Lt, + "<=" => ast::Cmpop::LtE, + ">" => ast::Cmpop::Gt, + ">=" => ast::Cmpop::GtE, + "in" => ast::Cmpop::In, + "not" "in" => ast::Cmpop::NotIn, + "is" => ast::Cmpop::Is, + "is" "not" => ast::Cmpop::IsNot, }; -Expression: ast::Expression = { - "|" => ast::Expression { +Expression: ast::Expr = { + "|" => ast::Expr { location, custom: (), - node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) } + node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) } }, XorExpression, }; -XorExpression: ast::Expression = { - "^" => ast::Expression { +XorExpression: ast::Expr = { + "^" => ast::Expr { location, custom: (), - node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitXor, b: Box::new(e2) } + node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) } }, AndExpression, }; -AndExpression: ast::Expression = { - "&" => ast::Expression { +AndExpression: ast::Expr = { + "&" => ast::Expr { location, custom: (), - node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitAnd, b: Box::new(e2) } + node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) } }, ShiftExpression, }; -ShiftExpression: ast::Expression = { - => ast::Expression { +ShiftExpression: ast::Expr = { + => ast::Expr { location, custom: (), - node: ast::ExpressionType::Binop { a: Box::new(e1), op, b: Box::new(e2) } + node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) } }, ArithmaticExpression, }; @@ -840,11 +823,11 @@ ShiftOp: ast::Operator = { ">>" => ast::Operator::RShift, }; -ArithmaticExpression: ast::Expression = { - => ast::Expression { +ArithmaticExpression: ast::Expr = { + => ast::Expr { location, custom: (), - node: ast::ExpressionType::Binop { a: Box::new(a), op, b: Box::new(b) } + node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) } }, Term, }; @@ -854,11 +837,11 @@ AddOp: ast::Operator = { "-" => ast::Operator::Sub, }; -Term: ast::Expression = { - => ast::Expression { +Term: ast::Expr = { + => ast::Expr { location, custom: (), - node: ast::ExpressionType::Binop { a: Box::new(a), op, b: Box::new(b) } + node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) } }, Factor, }; @@ -871,41 +854,41 @@ MulOp: ast::Operator = { "@" => ast::Operator::MatMult, }; -Factor: ast::Expression = { - => ast::Expression { +Factor: ast::Expr = { + => ast::Expr { location, custom: (), - node: ast::ExpressionType::Unop { a: Box::new(e), op } + node: ast::ExprKind::UnaryOp { operand: Box::new(e), op } }, Power, }; -UnOp: ast::UnaryOperator = { - "+" => ast::UnaryOperator::Pos, - "-" => ast::UnaryOperator::Neg, - "~" => ast::UnaryOperator::Inv, +UnaryOp: ast::Unaryop = { + "+" => ast::Unaryop::UAdd, + "-" => ast::Unaryop::USub, + "~" => ast::Unaryop::Invert, }; -Power: ast::Expression = { +Power: ast::Expr = { => { match e2 { None => e, - Some((location, _, b)) => ast::Expression { + Some((location, _, b)) => ast::Expr { location, custom: (), - node: ast::ExpressionType::Binop { a: Box::new(e), op: ast::Operator::Pow, b: Box::new(b) } + node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) } }, } } }; -AtomExpr: ast::Expression = { +AtomExpr: ast::Expr = { => { if is_await.is_some() { - ast::Expression { + ast::Expr { location, custom: (), - node: ast::ExpressionType::Await { value: Box::new(atom) } + node: ast::ExprKind::Await { value: Box::new(atom) } } } else { atom @@ -913,28 +896,28 @@ AtomExpr: ast::Expression = { } } -AtomExpr2: ast::Expression = { +AtomExpr2: ast::Expr = { Atom, "(" ")" => { - ast::Expression { + ast::Expr { location, custom: (), - node: ast::ExpressionType::Call { function: Box::new(f), args: a.args, keywords: a.keywords } + node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords } } }, - "[" "]" => ast::Expression { + "[" "]" => ast::Expr { location, custom: (), - node: ast::ExpressionType::Subscript { a: Box::new(e), b: Box::new(s) } + node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load } }, - "." => ast::Expression { + "." => ast::Expr { location, custom: (), - node: ast::ExpressionType::Attribute { value: Box::new(e), name } + node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load } }, }; -SubscriptList: ast::Expression = { +SubscriptList: ast::Expr = { ","? => { if s2.is_empty() { s1 @@ -944,144 +927,148 @@ SubscriptList: ast::Expression = { dims.push(x.1) } - ast::Expression { + ast::Expr { location, custom: (), - node: ast::ExpressionType::Tuple { elements: dims }, + node: ast::ExprKind::Tuple { elts: dims, ctx: ast::ExprContext::Load }, } } } }; -Subscript: ast::Expression = { +Subscript: ast::Expr = { Test, - ":" => { - let s1 = e1.unwrap_or(ast::Expression { location, custom: (), node: ast::ExpressionType::None }); - let s2 = e2.unwrap_or(ast::Expression { location, custom: (), node: ast::ExpressionType::None }); - let s3 = e3.unwrap_or(ast::Expression { location, custom: (), node: ast::ExpressionType::None }); - ast::Expression { + ":" => { + let lower = e1.map(Box::new); + let upper = e2.map(Box::new); + let step = e3.flatten().map(Box::new); + ast::Expr { location, custom: (), - node: ast::ExpressionType::Slice { elements: vec![s1, s2, s3] } + node: ast::ExprKind::Slice { lower, upper, step } } } }; -SliceOp: ast::Expression = { - ":" => e.unwrap_or(ast::Expression {location, custom: (), node: ast::ExpressionType::None}) +SliceOp: Option = { + ":" => e, } -Atom: ast::Expression = { - => ast::Expression { - location, - custom: (), - node: ast::ExpressionType::String { value } +Atom: ast::Expr = { + =>? { + let values = s.into_iter().map(|(loc, (value, is_fstring))| { + if is_fstring { + parse_located_fstring(&value, loc) + } else { + Ok(ast::Expr::new( + loc, + ast::ExprKind::Constant { value: value.into(), kind: None }, + )) + } + }); + let values = values.collect::, _>>()?; + + Ok(if values.len() > 1 { + ast::Expr::new(location, ast::ExprKind::JoinedStr { values }) + } else { + values.into_iter().next().unwrap() + }) }, - => ast::Expression { + => ast::Expr { location, custom: (), - node: ast::ExpressionType::Bytes { value } + node: ast::ExprKind::Constant { value, kind: None } }, - => ast::Expression { + => ast::Expr { location, custom: (), - node: ast::ExpressionType::Number { value } - }, - => ast::Expression { - location, - custom: (), - node: ast::ExpressionType::Identifier { name } + node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load } }, "[" "]" => { - let elements = e.unwrap_or_default(); - ast::Expression { + let elts = e.unwrap_or_default(); + ast::Expr { location, custom: (), - node: ast::ExpressionType::List { elements } + node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load } } }, - "[" "]" => { - ast::Expression { + "[" "]" => { + ast::Expr { location, custom: (), - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::List { element }), - generators, - } + node: ast::ExprKind::ListComp { elt: Box::new(elt), generators } } }, "(" ")" => { - elements.unwrap_or(ast::Expression { + elements.unwrap_or(ast::Expr { location, custom: (), - node: ast::ExpressionType::Tuple { elements: Vec::new() } + node: ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load } }) }, "(" ")" => e, - "(" ")" => { - ast::Expression { + "(" ")" => { + ast::Expr { location, custom: (), - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element }), - generators, - } + node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators } } }, - "{" "}" => ast::Expression { - location, - custom: (), - node: ast::ExpressionType::Dict { elements: e.unwrap_or_default() } + "{" "}" => { + let (keys, values) = e.unwrap_or_default(); + ast::Expr { + location, + custom: (), + node: ast::ExprKind::Dict { keys, values } + } }, "{" "}" => { - ast::Expression { + ast::Expr { location, custom: (), - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::Dict { key: e1.0, value: e1.1 }), + node: ast::ExprKind::DictComp { + key: Box::new(e1.0), + value: Box::new(e1.1), generators, } } }, - "{" "}" => ast::Expression { + "{" "}" => ast::Expr { location, custom: (), - node: ast::ExpressionType::Set { elements } + node: ast::ExprKind::Set { elts } }, - "{" "}" => { - ast::Expression { + "{" "}" => { + ast::Expr { location, custom: (), - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::Set { element }), - generators, - } + node: ast::ExprKind::SetComp { elt: Box::new(elt), generators } } }, - "True" => ast::Expression { location, custom: (), node: ast::ExpressionType::True }, - "False" => ast::Expression { location, custom: (), node: ast::ExpressionType::False }, - "None" => ast::Expression { location, custom: (), node: ast::ExpressionType::None }, - "..." => ast::Expression { location, custom: (), node: ast::ExpressionType::Ellipsis }, + "True" => ast::Expr::new(location, ast::ExprKind::Constant { value: true.into(), kind: None }), + "False" => ast::Expr::new(location, ast::ExprKind::Constant { value: false.into(), kind: None }), + "None" => ast::Expr::new(location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }), + "..." => ast::Expr::new(location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }), }; -ListLiteralValues: Vec = { +ListLiteralValues: Vec = { > ","? => e, }; -DictLiteralValues: Vec<(Option, ast::Expression)> = { - > ","? => elements, +DictLiteralValues: (Vec>>, Vec) = { + > ","? => elements.into_iter().unzip(), }; -DictEntry: (ast::Expression, ast::Expression) = { +DictEntry: (ast::Expr, ast::Expr) = { ":" => (e1, e2), }; -DictElement: (Option, ast::Expression) = { - => (Some(e.0), e.1), +DictElement: (Option>, ast::Expr) = { + => (Some(Box::new(e.0)), e.1), "**" => (None, e), }; -SetLiteralValues: Vec = { +SetLiteralValues: Vec = { > ","? => e1 }; @@ -1090,21 +1077,11 @@ ExpressionOrStarExpression = { StarExpr }; -ExpressionList: ast::Expression = { - > => { - if elements.len() == 1 && trailing_comma.is_none() { - elements.into_iter().next().unwrap() - } else { - ast::Expression { - location, - custom: (), - node: ast::ExpressionType::Tuple { elements }, - } - } - }, +ExpressionList: ast::Expr = { + GenericList }; -ExpressionList2: Vec = { +ExpressionList2: Vec = { > ","? => elements, }; @@ -1112,26 +1089,31 @@ ExpressionList2: Vec = { // - a list of expressions // - a single expression // - a single expression followed by a trailing comma -TestList: ast::Expression = { - > => { - if elements.len() == 1 && trailing_comma.is_none() { - elements.into_iter().next().unwrap() +#[inline] +TestList: ast::Expr = { + GenericList +}; + +GenericList: ast::Expr = { + > => { + if elts.len() == 1 && trailing_comma.is_none() { + elts.into_iter().next().unwrap() } else { - ast::Expression { + ast::Expr { location, custom: (), - node: ast::ExpressionType::Tuple { elements }, + node: ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load } } } } -}; +} // Test -StarExpr: ast::Expression = { - "*" => ast::Expression { +StarExpr: ast::Expr = { + "*" => ast::Expr { location, custom: (), - node: ast::ExpressionType::Starred { value: Box::new(e) }, + node: ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load }, } }; @@ -1141,28 +1123,33 @@ CompFor: Vec = => c; SingleForComprehension: ast::Comprehension = { "for" "in" => { let is_async = is_async.is_some(); - ast::Comprehension { location, target, iter, ifs, is_async } + ast::Comprehension { + target: Box::new(target), + iter: Box::new(iter), + ifs, + is_async + } } }; -ExpressionNoCond: ast::Expression = OrTest; -ComprehensionIf: ast::Expression = "if" => c; +ExpressionNoCond: ast::Expr = OrTest; +ComprehensionIf: ast::Expr = "if" => c; -ArgumentList: ast::ArgumentList = { +ArgumentList: ArgumentList = { > =>? { let arg_list = parse_args(e)?; Ok(arg_list) } }; -FunctionArgument: (Option>, ast::Expression) = { +FunctionArgument: (Option<(ast::Location, Option)>, ast::Expr) = { => { let expr = match c { - Some(c) => ast::Expression { + Some(c) => ast::Expr { location: e.location, custom: (), - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element: e }), + node: ast::ExprKind::GeneratorExp { + elt: Box::new(e), generators: c, } }, @@ -1170,9 +1157,15 @@ FunctionArgument: (Option>, ast::Expression) = { }; (None, expr) }, - "=" => (Some(Some(i)), e), - "*" => (None, ast::Expression { location, custom: (), node: ast::ExpressionType::Starred { value: Box::new(e) } }), - "**" => (Some(None), e), + "=" => (Some((location, Some(i))), e), + "*" => { + let expr = ast::Expr::new( + location, + ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load }, + ); + (None, expr) + }, + "**" => (Some((location, None)), e), }; #[inline] @@ -1193,29 +1186,11 @@ OneOrMore: Vec = { } }; -Number: ast::Number = { - => { ast::Number::Integer { value } }, - => { ast::Number::Float { value } }, - => { ast::Number::Complex { real: s.0, imag: s.1 } }, -}; - -StringGroup: ast::StringGroup = { - =>? { - let mut values = vec![]; - for (value, is_fstring) in s { - values.push(if is_fstring { - parse_located_fstring(&value, loc)? - } else { - ast::StringGroup::Constant { value } - }) - } - - Ok(if values.len() > 1 { - ast::StringGroup::Joined { values } - } else { - values.into_iter().next().unwrap() - }) - }, +Constant: ast::Constant = { + => ast::Constant::Bytes(b.into_iter().flatten().collect()), + => ast::Constant::Int(value), + => ast::Constant::Float(value), + => ast::Constant::Complex { real: s.0, imag: s.1 }, }; Bytes: Vec = { @@ -1234,8 +1209,8 @@ extern { enum lexer::Tok { Indent => lexer::Tok::Indent, Dedent => lexer::Tok::Dedent, - StartProgram => lexer::Tok::StartProgram, - StartStatement => lexer::Tok::StartStatement, + StartModule => lexer::Tok::StartModule, + StartInteractive => lexer::Tok::StartInteractive, StartExpression => lexer::Tok::StartExpression, "+" => lexer::Tok::Plus, "-" => lexer::Tok::Minus, diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap new file mode 100644 index 000000000..33156007c --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap @@ -0,0 +1,63 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: JoinedStr { + values: [ + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "user=", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "user", + ctx: Load, + }, + }, + conversion: None, + format_spec: None, + }, + }, + ], + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap new file mode 100644 index 000000000..6cc933de4 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap @@ -0,0 +1,137 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: JoinedStr { + values: [ + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "mix ", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "user=", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "user", + ctx: Load, + }, + }, + conversion: None, + format_spec: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + " with text and ", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "second=", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "second", + ctx: Load, + }, + }, + conversion: None, + format_spec: None, + }, + }, + ], + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap new file mode 100644 index 000000000..e0713f5ed --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap @@ -0,0 +1,77 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: JoinedStr { + values: [ + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "user=", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "user", + ctx: Load, + }, + }, + conversion: None, + format_spec: Some( + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + ">10", + ), + kind: None, + }, + }, + ), + }, + }, + ], + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap new file mode 100644 index 000000000..850638674 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap @@ -0,0 +1,17 @@ +--- +source: parser/src/fstring.rs +expression: "parse_fstring(\"\").unwrap()" +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "", + ), + kind: None, + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap new file mode 100644 index 000000000..457853a5b --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap @@ -0,0 +1,72 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: JoinedStr { + values: [ + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + conversion: None, + format_spec: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 3, + }, + custom: (), + node: Name { + id: "b", + ctx: Load, + }, + }, + conversion: None, + format_spec: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "{foo}", + ), + kind: None, + }, + }, + ], + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap new file mode 100644 index 000000000..40f1a4dae --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap @@ -0,0 +1,69 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 5, + }, + custom: (), + node: Compare { + left: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 42, + ], + }, + }, + ), + kind: None, + }, + }, + ops: [ + Eq, + ], + comparators: [ + Located { + location: Location { + row: 1, + column: 8, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 42, + ], + }, + }, + ), + kind: None, + }, + }, + ], + }, + }, + conversion: None, + format_spec: None, + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap new file mode 100644 index 000000000..8ecd704bb --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap @@ -0,0 +1,49 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "foo", + ctx: Load, + }, + }, + conversion: None, + format_spec: Some( + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "spec", + ctx: Load, + }, + }, + conversion: None, + format_spec: None, + }, + }, + ), + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap new file mode 100644 index 000000000..d9e84936d --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap @@ -0,0 +1,69 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 4, + }, + custom: (), + node: Compare { + left: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 1, + ], + }, + }, + ), + kind: None, + }, + }, + ops: [ + NotEq, + ], + comparators: [ + Located { + location: Location { + row: 1, + column: 7, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 2, + ], + }, + }, + ), + kind: None, + }, + }, + ], + }, + }, + conversion: None, + format_spec: None, + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap new file mode 100644 index 000000000..12147ebb7 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap @@ -0,0 +1,40 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "foo", + ctx: Load, + }, + }, + conversion: None, + format_spec: Some( + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "spec", + ), + kind: None, + }, + }, + ), + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap new file mode 100644 index 000000000..4b9b44582 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap @@ -0,0 +1,63 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: JoinedStr { + values: [ + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "x =", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "x", + ctx: Load, + }, + }, + conversion: None, + format_spec: None, + }, + }, + ], + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap new file mode 100644 index 000000000..09c0f32d2 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap @@ -0,0 +1,63 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: JoinedStr { + values: [ + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "x=", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + " ", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "x", + ctx: Load, + }, + }, + conversion: None, + format_spec: None, + }, + }, + ], + }, +} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap new file mode 100644 index 000000000..2330059af --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap @@ -0,0 +1,25 @@ +--- +source: parser/src/fstring.rs +expression: parse_ast +--- +Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Yield { + value: None, + }, + }, + conversion: None, + format_spec: None, + }, +} diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap new file mode 100644 index 000000000..869669e9a --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap @@ -0,0 +1,160 @@ +--- +source: parser/src/parser.rs +expression: parse_program(&source).unwrap() +--- +[ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: ClassDef { + name: "Foo", + bases: [ + Located { + location: Location { + row: 1, + column: 11, + }, + custom: (), + node: Name { + id: "A", + ctx: Load, + }, + }, + Located { + location: Location { + row: 1, + column: 14, + }, + custom: (), + node: Name { + id: "B", + ctx: Load, + }, + }, + ], + keywords: [], + body: [ + Located { + location: Location { + row: 2, + column: 2, + }, + custom: (), + node: FunctionDef { + name: "__init__", + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 2, + column: 15, + }, + custom: (), + node: ArgData { + arg: "self", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: [ + Located { + location: Location { + row: 3, + column: 3, + }, + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 4, + column: 2, + }, + custom: (), + node: FunctionDef { + name: "method_with_default", + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 4, + column: 26, + }, + custom: (), + node: ArgData { + arg: "self", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 4, + column: 32, + }, + custom: (), + node: ArgData { + arg: "arg", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: None, + defaults: [ + Located { + location: Location { + row: 4, + column: 37, + }, + custom: (), + node: Constant { + value: Str( + "default", + ), + kind: None, + }, + }, + ], + }, + body: [ + Located { + location: Location { + row: 5, + column: 3, + }, + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + ], + decorator_list: [], + }, + }, +] diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap new file mode 100644 index 000000000..65369dc31 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap @@ -0,0 +1,63 @@ +--- +source: parser/src/parser.rs +expression: parse_ast +--- +Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: DictComp { + key: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "x1", + ctx: Load, + }, + }, + value: Located { + location: Location { + row: 1, + column: 6, + }, + custom: (), + node: Name { + id: "x2", + ctx: Load, + }, + }, + generators: [ + Comprehension { + target: Located { + location: Location { + row: 1, + column: 13, + }, + custom: (), + node: Name { + id: "y", + ctx: Load, + }, + }, + iter: Located { + location: Location { + row: 1, + column: 18, + }, + custom: (), + node: Name { + id: "z", + ctx: Load, + }, + }, + ifs: [], + is_async: false, + }, + ], + }, +} diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap new file mode 100644 index 000000000..24343f3c9 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap @@ -0,0 +1,192 @@ +--- +source: parser/src/parser.rs +expression: parse_ast +--- +Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: ListComp { + elt: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "x", + ctx: Load, + }, + }, + generators: [ + Comprehension { + target: Located { + location: Location { + row: 1, + column: 8, + }, + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 1, + column: 8, + }, + custom: (), + node: Name { + id: "y", + ctx: Load, + }, + }, + Located { + location: Location { + row: 1, + column: 11, + }, + custom: (), + node: Name { + id: "y2", + ctx: Load, + }, + }, + ], + ctx: Load, + }, + }, + iter: Located { + location: Location { + row: 1, + column: 17, + }, + custom: (), + node: Name { + id: "z", + ctx: Load, + }, + }, + ifs: [], + is_async: false, + }, + Comprehension { + target: Located { + location: Location { + row: 1, + column: 23, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + iter: Located { + location: Location { + row: 1, + column: 28, + }, + custom: (), + node: Name { + id: "b", + ctx: Load, + }, + }, + ifs: [ + Located { + location: Location { + row: 1, + column: 35, + }, + custom: (), + node: Compare { + left: Located { + location: Location { + row: 1, + column: 33, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + ops: [ + Lt, + ], + comparators: [ + Located { + location: Location { + row: 1, + column: 37, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 5, + ], + }, + }, + ), + kind: None, + }, + }, + ], + }, + }, + Located { + location: Location { + row: 1, + column: 44, + }, + custom: (), + node: Compare { + left: Located { + location: Location { + row: 1, + column: 42, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + ops: [ + Gt, + ], + comparators: [ + Located { + location: Location { + row: 1, + column: 46, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 10, + ], + }, + }, + ), + kind: None, + }, + }, + ], + }, + }, + ], + is_async: false, + }, + ], + }, +} diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap new file mode 100644 index 000000000..64d89c15f --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap @@ -0,0 +1,5 @@ +--- +source: parser/src/parser.rs +expression: parse_ast +--- +[] diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap new file mode 100644 index 000000000..2a6c911c6 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap @@ -0,0 +1,159 @@ +--- +source: parser/src/parser.rs +expression: parse_ast +--- +[ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: If { + test: Located { + location: Location { + row: 1, + column: 4, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 1, + ], + }, + }, + ), + kind: None, + }, + }, + body: [ + Located { + location: Location { + row: 1, + column: 7, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 7, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 10, + ], + }, + }, + ), + kind: None, + }, + }, + }, + }, + ], + orelse: [ + Located { + location: Location { + row: 2, + column: 1, + }, + custom: (), + node: If { + test: Located { + location: Location { + row: 2, + column: 6, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 2, + ], + }, + }, + ), + kind: None, + }, + }, + body: [ + Located { + location: Location { + row: 2, + column: 9, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 2, + column: 9, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 20, + ], + }, + }, + ), + kind: None, + }, + }, + }, + }, + ], + orelse: [ + Located { + location: Location { + row: 3, + column: 7, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 3, + column: 7, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 30, + ], + }, + }, + ), + kind: None, + }, + }, + }, + }, + ], + }, + }, + ], + }, + }, +] diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap new file mode 100644 index 000000000..43d2d98e6 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap @@ -0,0 +1,84 @@ +--- +source: parser/src/parser.rs +expression: parse_ast +--- +[ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 8, + }, + custom: (), + node: Call { + func: Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Name { + id: "my_func", + ctx: Load, + }, + }, + args: [ + Located { + location: Location { + row: 1, + column: 10, + }, + custom: (), + node: Constant { + value: Str( + "positional", + ), + kind: None, + }, + }, + ], + keywords: [ + Located { + location: Location { + row: 1, + column: 23, + }, + custom: (), + node: KeywordData { + arg: Some( + "keyword", + ), + value: Located { + location: Location { + row: 1, + column: 31, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 2, + ], + }, + }, + ), + kind: None, + }, + }, + }, + }, + ], + }, + }, + }, + }, +] diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap new file mode 100644 index 000000000..2f70292d6 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap @@ -0,0 +1,90 @@ +--- +source: parser/src/parser.rs +expression: parse_ast +--- +[ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Lambda { + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 1, + column: 8, + }, + custom: (), + node: ArgData { + arg: "x", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 11, + }, + custom: (), + node: ArgData { + arg: "y", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: Located { + location: Location { + row: 1, + column: 16, + }, + custom: (), + node: BinOp { + left: Located { + location: Location { + row: 1, + column: 14, + }, + custom: (), + node: Name { + id: "x", + ctx: Load, + }, + }, + op: Mult, + right: Located { + location: Location { + row: 1, + column: 18, + }, + custom: (), + node: Name { + id: "y", + ctx: Load, + }, + }, + }, + }, + }, + }, + }, + }, +] diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap new file mode 100644 index 000000000..d67e166da --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap @@ -0,0 +1,52 @@ +--- +source: parser/src/parser.rs +expression: parse_ast +--- +Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: ListComp { + elt: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "x", + ctx: Load, + }, + }, + generators: [ + Comprehension { + target: Located { + location: Location { + row: 1, + column: 8, + }, + custom: (), + node: Name { + id: "y", + ctx: Load, + }, + }, + iter: Located { + location: Location { + row: 1, + column: 13, + }, + custom: (), + node: Name { + id: "z", + ctx: Load, + }, + }, + ifs: [], + is_async: false, + }, + ], + }, +} diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap new file mode 100644 index 000000000..e4e0a3c90 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap @@ -0,0 +1,71 @@ +--- +source: parser/src/parser.rs +expression: parse_ast +--- +[ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 6, + }, + custom: (), + node: Call { + func: Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Name { + id: "print", + ctx: Load, + }, + }, + args: [ + Located { + location: Location { + row: 1, + column: 8, + }, + custom: (), + node: Constant { + value: Str( + "Hello world", + ), + kind: None, + }, + }, + Located { + location: Location { + row: 1, + column: 22, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 2, + ], + }, + }, + ), + kind: None, + }, + }, + ], + keywords: [], + }, + }, + }, + }, +] diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap new file mode 100644 index 000000000..b4e78d8d0 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap @@ -0,0 +1,51 @@ +--- +source: parser/src/parser.rs +expression: parse_ast +--- +[ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 6, + }, + custom: (), + node: Call { + func: Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Name { + id: "print", + ctx: Load, + }, + }, + args: [ + Located { + location: Location { + row: 1, + column: 8, + }, + custom: (), + node: Constant { + value: Str( + "Hello world", + ), + kind: None, + }, + }, + ], + keywords: [], + }, + }, + }, + }, +] diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap new file mode 100644 index 000000000..5b0e15339 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap @@ -0,0 +1,104 @@ +--- +source: parser/src/parser.rs +expression: parse_program(&source).unwrap() +--- +[ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Assign { + targets: [ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 1, + column: 1, + }, + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + Located { + location: Location { + row: 1, + column: 4, + }, + custom: (), + node: Name { + id: "b", + ctx: Load, + }, + }, + ], + ctx: Load, + }, + }, + ], + value: Located { + location: Location { + row: 1, + column: 8, + }, + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 1, + column: 8, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 4, + ], + }, + }, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 1, + column: 11, + }, + custom: (), + node: Constant { + value: Int( + BigInt { + sign: Plus, + data: BigUint { + data: [ + 5, + ], + }, + }, + ), + kind: None, + }, + }, + ], + ctx: Load, + }, + }, + type_comment: None, + }, + }, +] diff --git a/parser/src/token.rs b/parser/src/token.rs index 316c34e48..1cedc66a6 100644 --- a/parser/src/token.rs +++ b/parser/src/token.rs @@ -15,8 +15,8 @@ pub enum Tok { Newline, Indent, Dedent, - StartProgram, - StartStatement, + StartModule, + StartInteractive, StartExpression, EndOfFile, Lpar, @@ -136,8 +136,8 @@ impl fmt::Display for Tok { Newline => f.write_str("Newline"), Indent => f.write_str("Indent"), Dedent => f.write_str("Dedent"), - StartProgram => f.write_str("StartProgram"), - StartStatement => f.write_str("StartStatement"), + StartModule => f.write_str("StartProgram"), + StartInteractive => f.write_str("StartInteractive"), StartExpression => f.write_str("StartExpression"), EndOfFile => f.write_str("EOF"), Lpar => f.write_str("'('"),