From 09e2c027cd8ae533e7a07cd99cd9ff5489d90300 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 24 Oct 2018 16:02:59 +0200 Subject: [PATCH] Extend ast module with more nodes. Test send method on generators. --- parser/src/ast.rs | 4 +- parser/src/python.lalrpop | 250 +++++++++++++------------- tests/snippets/comprehensions.py | 4 + tests/snippets/generators.py | 11 ++ vm/src/compile.rs | 8 +- vm/src/obj/objgenerator.rs | 2 +- vm/src/stdlib/ast.rs | 298 +++++++++++++++++++++++++++++-- 7 files changed, 439 insertions(+), 138 deletions(-) diff --git a/parser/src/ast.rs b/parser/src/ast.rs index a8232c677..9a62ff2bb 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -143,10 +143,10 @@ pub enum Expression { a: Box, }, Yield { - expression: Option>, + value: Option>, }, YieldFrom { - expression: Box, + value: Box, }, Compare { a: Box, diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index b763530ec..18dd0995c 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -21,7 +21,9 @@ pub Top: ast::Top = { }; Program: ast::Program = { - => ast::Program { statements: Vec::from_iter(lines.into_iter().filter_map(|e| e)) }, + => ast::Program { + statements: Vec::from_iter(lines.into_iter().filter_map(|e| e)) + }, }; // A file line either has a declaration, or an empty newline: @@ -31,8 +33,8 @@ FileLine: Option = { }; Suite: Vec = { - => vec![s], - "\n" indent dedent => s, + => vec![s], + "\n" indent dedent => s, }; Statement: ast::LocatedStatement = { @@ -89,19 +91,10 @@ ExpressionStatement: ast::LocatedStatement = { } } else { let mut targets = vec![expr]; - let mut values : Vec = suffix - .into_iter() - .map(|test_list| if test_list.len() > 1 { - ast::Expression::Tuple { - elements: test_list - } - } else { - test_list.into_iter().next().unwrap() - }) - .collect(); + let mut values = suffix; while values.len() > 1 { - targets.push(values.remove(0)); + targets.push(values.remove(0)); } let value = values.into_iter().next().unwrap(); @@ -122,14 +115,23 @@ ExpressionStatement: ast::LocatedStatement = { // TODO: this works in most cases: let rhs = e2.into_iter().next().unwrap(); ast::LocatedStatement { - location: loc, - node: ast::Statement::AugAssign { target: expr, op: op, value: rhs }, + location: loc, + node: ast::Statement::AugAssign { target: expr, op: op, value: rhs }, } }, }; -AssignSuffix: Vec = { - "=" => e, +AssignSuffix: ast::Expression = { + "=" => { + if e.len() > 1 { + ast::Expression::Tuple { + elements: e + } + } else { + e.into_iter().next().unwrap() + } + }, + "=" => e, }; TestOrStarExprList: Vec = { @@ -146,52 +148,52 @@ TestOrStarExpr: ast::Expression = { }; AugAssign: ast::Operator = { - "+=" => ast::Operator::Add, - "-=" => ast::Operator::Sub, - "*=" => ast::Operator::Mult, - "@=" => ast::Operator::MatMult, - "/=" => ast::Operator::Div, - "%=" => ast::Operator::Mod, - "&=" => ast::Operator::BitAnd, - "|=" => ast::Operator::BitOr, - "^=" => ast::Operator::BitXor, - "<<=" => ast::Operator::LShift, - ">>=" => ast::Operator::RShift, - "**=" => ast::Operator::Pow, - "//=" => ast::Operator::FloorDiv, + "+=" => ast::Operator::Add, + "-=" => ast::Operator::Sub, + "*=" => ast::Operator::Mult, + "@=" => ast::Operator::MatMult, + "/=" => ast::Operator::Div, + "%=" => ast::Operator::Mod, + "&=" => ast::Operator::BitAnd, + "|=" => ast::Operator::BitOr, + "^=" => ast::Operator::BitXor, + "<<=" => ast::Operator::LShift, + ">>=" => ast::Operator::RShift, + "**=" => ast::Operator::Pow, + "//=" => ast::Operator::FloorDiv, }; FlowStatement: ast::LocatedStatement = { - "break" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Break, - } - }, - "continue" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Continue, - } - }, - "return" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Return { value: t }, - } - }, - "raise" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Raise { expression: t }, - } - }, - => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Expression { expression: y }, - } - }, + "break" => { + ast::LocatedStatement { + location: loc, + node: ast::Statement::Break, + } + }, + "continue" => { + ast::LocatedStatement { + location: loc, + node: ast::Statement::Continue, + } + }, + "return" => { + ast::LocatedStatement { + location: loc, + node: ast::Statement::Return { value: t }, + } + }, + "raise" => { + ast::LocatedStatement { + location: loc, + node: ast::Statement::Raise { expression: t }, + } + }, + => { + ast::LocatedStatement { + location: loc, + node: ast::Statement::Expression { expression: y }, + } + }, }; ImportStatement: ast::LocatedStatement = { @@ -264,30 +266,30 @@ ImportPart: (String, Option) = { // A name like abc or abc.def.ghi DottedName: String = { - => n, - => { - let mut r = n.to_string(); - for x in n2 { - r.push_str("."); - r.push_str(&x.1); - } - r - }, + => n, + => { + let mut r = n.to_string(); + for x in n2 { + r.push_str("."); + r.push_str(&x.1); + } + r + }, }; AssertStatement: ast::LocatedStatement = { - "assert" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Assert { - test: t, - msg: match m { - Some(e) => Some(e.1), - None => None, + "assert" => { + ast::LocatedStatement { + location: loc, + node: ast::Statement::Assert { + test: t, + msg: match m { + Some(e) => Some(e.1), + None => None, + } + } } - } - } - }, + }, }; CompoundStatement: ast::LocatedStatement = { @@ -376,33 +378,33 @@ TryStatement: ast::LocatedStatement = { }; ExceptClause: ast::ExceptHandler = { - "except" ":" => { - ast::ExceptHandler { - typ: typ, - name: None, - body: body, - } - }, - "except" ":" => { - ast::ExceptHandler { - typ: Some(x.0), - name: Some(x.2), - body: body, - } - }, + "except" ":" => { + ast::ExceptHandler { + typ: typ, + name: None, + body: body, + } + }, + "except" ":" => { + ast::ExceptHandler { + typ: Some(x.0), + name: Some(x.2), + body: body, + } + }, }; WithStatement: ast::LocatedStatement = { - "with" ":" => { - let mut items = vec![i1]; - for item in i2 { - items.push(item.1); - } - ast::LocatedStatement { - location: loc, - node: ast::Statement::With { items: items, body: s }, - } - }, + "with" ":" => { + let mut items = vec![i1]; + for item in i2 { + items.push(item.1); + } + ast::LocatedStatement { + location: loc, + node: ast::Statement::With { items: items, body: s }, + } + }, }; WithItem: ast::WithItem = { @@ -569,7 +571,7 @@ Decorator: ast::Expression = { YieldExpr: ast::Expression = { "yield" => { ast::Expression::Yield { - expression: ex.map(|expr| Box::new( + value: ex.map(|expr| Box::new( if expr.len() > 1 { ast::Expression::Tuple { elements: expr } } else { @@ -580,7 +582,7 @@ YieldExpr: ast::Expression = { }, "yield" "from" => { ast::Expression::YieldFrom { - expression: Box::new(e), + value: Box::new(e), } }, }; @@ -630,21 +632,21 @@ Comparison: ast::Expression = { }; CompOp: ast::Comparison = { - "==" => ast::Comparison::Equal, - "!=" => ast::Comparison::NotEqual, - "<" => ast::Comparison::Less, - "<=" => ast::Comparison::LessOrEqual, - ">" => ast::Comparison::Greater, - ">=" => ast::Comparison::GreaterOrEqual, - "in" => ast::Comparison::In, - "not" "in" => ast::Comparison::NotIn, - "is" => ast::Comparison::Is, - "is" "not" => ast::Comparison::IsNot, + "==" => ast::Comparison::Equal, + "!=" => ast::Comparison::NotEqual, + "<" => ast::Comparison::Less, + "<=" => ast::Comparison::LessOrEqual, + ">" => ast::Comparison::Greater, + ">=" => ast::Comparison::GreaterOrEqual, + "in" => ast::Comparison::In, + "not" "in" => ast::Comparison::NotIn, + "is" => ast::Comparison::Is, + "is" "not" => ast::Comparison::IsNot, }; Expression: ast::Expression = { - "|" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) }, - => e, + "|" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) }, + => e, }; XorExpression: ast::Expression = { @@ -854,7 +856,7 @@ StarExpr: ast::Expression = { // Comprehensions: CompIter: (Option>, Option) = { - // CompIf, + // => (None, Some(e)), => (Some(c), None), }; @@ -891,7 +893,13 @@ ExpressionNoCond: ast::Expression = { //CompIf: ast::Expression = { // "if" => { -// c +// match c2 { +// None => { +// vec![] +// }, +// Some() => { +// }, +// } // } //}; diff --git a/tests/snippets/comprehensions.py b/tests/snippets/comprehensions.py index 13733832b..4a4563dce 100644 --- a/tests/snippets/comprehensions.py +++ b/tests/snippets/comprehensions.py @@ -18,3 +18,7 @@ v = {b * 2 for b in x} # TODO: #u = {str(b): b-2 for b in x} +# TODO: With if filtering: +#y = [a+2 for a in x if a % 2] +#print(y) +#assert y == [3, 5] diff --git a/tests/snippets/generators.py b/tests/snippets/generators.py index 8e4d5986d..6431ec767 100644 --- a/tests/snippets/generators.py +++ b/tests/snippets/generators.py @@ -14,3 +14,14 @@ assert r == [1, 2, 42, 3] r = list(x for x in [1, 2, 3]) assert r == [1, 2, 3] + +def g2(x): + x = yield x + yield x + 5 + yield x + 7 + +i = g2(23) +assert 23 == next(i) +assert 15 == i.send(10) +assert 17 == i.send(10) + diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 309f5166c..96f734f5b 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -895,9 +895,9 @@ impl Compiler { } self.emit(Instruction::BuildSlice { size: size }); } - ast::Expression::Yield { expression } => { + ast::Expression::Yield { value } => { self.mark_generator(); - match expression { + match value { Some(expression) => self.compile_expression(expression)?, None => self.emit(Instruction::LoadConst { value: bytecode::Constant::None, @@ -905,9 +905,9 @@ impl Compiler { }; self.emit(Instruction::YieldValue); } - ast::Expression::YieldFrom { expression } => { + ast::Expression::YieldFrom { value } => { self.mark_generator(); - self.compile_expression(expression)?; + self.compile_expression(value)?; self.emit(Instruction::YieldValue); unimplemented!("yield from todo"); } diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs index e3800aed6..d76e7b894 100644 --- a/vm/src/obj/objgenerator.rs +++ b/vm/src/obj/objgenerator.rs @@ -14,7 +14,7 @@ pub fn init(context: &PyContext) { let ref generator_type = context.generator_type; generator_type.set_attr("__iter__", context.new_rustfunc(generator_iter)); generator_type.set_attr("__next__", context.new_rustfunc(generator_next)); - generator_type.set_attr("__send__", context.new_rustfunc(generator_send)); + generator_type.set_attr("send", context.new_rustfunc(generator_send)); } pub fn new_generator(vm: &mut VirtualMachine, frame: Frame) -> PyResult { diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index f87154660..1f1767db5 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -13,6 +13,7 @@ use super::super::pyobject::{ AttributeProtocol, DictProtocol, PyContext, PyFuncArgs, PyObjectRef, PyResult, TypeProtocol, }; use super::super::VirtualMachine; +use std::ops::Deref; /* * Idea: maybe we can create a sort of struct with some helper functions? @@ -83,7 +84,7 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj } ast::Statement::FunctionDef { name, - args: _, + args, body, decorator_list, } => { @@ -92,6 +93,8 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj // Set name: node.set_attr("name", ctx.new_str(name.to_string())); + node.set_attr("args", parameters_to_ast(ctx, args)); + // Set body: let py_body = statements_to_ast(ctx, body); node.set_attr("body", py_body); @@ -112,6 +115,19 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj let node = create_node(ctx, "Pass"); node } + ast::Statement::Assert { test, msg } => { + let node = create_node(ctx, "Pass"); + + node.set_attr("test", expression_to_ast(ctx, test)); + + let py_msg = match msg { + Some(msg) => expression_to_ast(ctx, msg), + None => ctx.none(), + }; + node.set_attr("msg", py_msg); + + node + } ast::Statement::Delete { targets } => { let node = create_node(ctx, "Delete"); @@ -161,20 +177,18 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj node } ast::Statement::For { - target: _, - iter: _, + target, + iter, body, orelse, } => { let node = create_node(ctx, "For"); - /* let py_target = expression_to_ast(ctx, target); node.set_attr("target", py_target); - - let py_iter = expression_to_ast(ctx, iter); + + let py_iter = expressions_to_ast(ctx, iter); node.set_attr("iter", py_iter); - */ let py_body = statements_to_ast(ctx, body); node.set_attr("body", py_body); @@ -280,6 +294,65 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR node.set_attr("right", py_b); node } + ast::Expression::Unop { op, a } => { + let node = create_node(ctx, "UnaryOp"); + + let str_op = match op { + ast::UnaryOperator::Not => "Not", + ast::UnaryOperator::Neg => "USub", + }; + let py_op = ctx.new_str(str_op.to_string()); + node.set_attr("op", py_op); + + let py_a = expression_to_ast(ctx, a); + node.set_attr("operand", py_a); + + node + } + ast::Expression::BoolOp { a, op, b } => { + let node = create_node(ctx, "BoolOp"); + + // Attach values: + let py_a = expression_to_ast(ctx, a); + let py_b = expression_to_ast(ctx, b); + let py_values = ctx.new_tuple(vec![py_a, py_b]); + node.set_attr("values", py_values); + + let str_op = match op { + ast::BooleanOperator::And => "And", + ast::BooleanOperator::Or => "Or", + }; + let py_op = ctx.new_str(str_op.to_string()); + node.set_attr("op", py_op); + + node + } + ast::Expression::Compare { a, op, b } => { + let node = create_node(ctx, "Compare"); + + let py_a = expression_to_ast(ctx, a); + node.set_attr("left", py_a); + + // Operator: + let str_op = match op { + ast::Comparison::Equal => "Eq", + ast::Comparison::NotEqual => "NotEq", + ast::Comparison::Less => "Lt", + ast::Comparison::LessOrEqual => "LtE", + ast::Comparison::Greater => "Gt", + ast::Comparison::GreaterOrEqual => "GtE", + ast::Comparison::In => "In", + ast::Comparison::NotIn => "NotIn", + ast::Comparison::Is => "Is", + ast::Comparison::IsNot => "IsNot", + }; + let py_ops = ctx.new_list(vec![ctx.new_str(str_op.to_string())]); + node.set_attr("ops", py_ops); + + let py_b = ctx.new_list(vec![expression_to_ast(ctx, b)]); + node.set_attr("comparators", py_b); + node + } ast::Expression::Identifier { name } => { let node = create_node(ctx, "Identifier"); @@ -288,14 +361,188 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR node.set_attr("id", py_name); node } + ast::Expression::Lambda { args, body } => { + let node = create_node(ctx, "Lambda"); + + node.set_attr("args", parameters_to_ast(ctx, args)); + + let py_body = expression_to_ast(ctx, body); + node.set_attr("body", py_body); + + node + } + ast::Expression::IfExpression { test, body, orelse } => { + let node = create_node(ctx, "IfExp"); + + let py_test = expression_to_ast(ctx, test); + node.set_attr("test", py_test); + + let py_body = expression_to_ast(ctx, body); + node.set_attr("body", py_body); + + let py_orelse = expression_to_ast(ctx, orelse); + node.set_attr("orelse", py_orelse); + + node + } + ast::Expression::Number { value } => { + let node = create_node(ctx, "Num"); + + let py_n = match value { + ast::Number::Integer { value } => ctx.new_int(*value), + ast::Number::Float { value } => ctx.new_float(*value), + }; + node.set_attr("n", py_n); + + node + } + ast::Expression::True => { + let node = create_node(ctx, "NameConstant"); + + node.set_attr("value", ctx.new_bool(true)); + + node + } + ast::Expression::False => { + let node = create_node(ctx, "NameConstant"); + + node.set_attr("value", ctx.new_bool(false)); + + node + } + ast::Expression::None => { + let node = create_node(ctx, "NameConstant"); + + node.set_attr("value", ctx.none()); + + node + } + ast::Expression::List { elements } => { + let node = create_node(ctx, "List"); + + let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect(); + let py_elts = ctx.new_list(elts); + node.set_attr("elts", py_elts); + + node + } + ast::Expression::Tuple { elements } => { + let node = create_node(ctx, "Tuple"); + + let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect(); + let py_elts = ctx.new_list(elts); + node.set_attr("elts", py_elts); + + node + } + ast::Expression::Set { elements } => { + let node = create_node(ctx, "Set"); + + let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect(); + let py_elts = ctx.new_list(elts); + node.set_attr("elts", py_elts); + + node + } + ast::Expression::Dict { elements } => { + let node = create_node(ctx, "Dict"); + + let mut keys = Vec::new(); + let mut values = Vec::new(); + for (k, v) in elements { + keys.push(expression_to_ast(ctx, k)); + values.push(expression_to_ast(ctx, v)); + } + + let py_keys = ctx.new_list(keys); + node.set_attr("keys", py_keys); + + let py_values = ctx.new_list(values); + node.set_attr("values", py_values); + + node + } + ast::Expression::Comprehension { kind, generators } => { + let node = match kind.deref() { + ast::ComprehensionKind::GeneratorExpression { .. } => { + create_node(ctx, "GeneratorExp") + } + ast::ComprehensionKind::List { .. } => create_node(ctx, "ListComp"), + ast::ComprehensionKind::Set { .. } => create_node(ctx, "SetComp"), + ast::ComprehensionKind::Dict { .. } => create_node(ctx, "DictComp"), + }; + + let g = generators + .iter() + .map(|g| comprehension_to_ast(ctx, g)) + .collect(); + let py_generators = ctx.new_list(g); + node.set_attr("generators", py_generators); + + node + } + ast::Expression::Yield { value } => { + let node = create_node(ctx, "Yield"); + + let py_value = match value { + Some(value) => expression_to_ast(ctx, value), + None => ctx.none(), + }; + node.set_attr("value", py_value); + + node + } + ast::Expression::YieldFrom { value } => { + let node = create_node(ctx, "YieldFrom"); + + let py_value = expression_to_ast(ctx, value); + node.set_attr("value", py_value); + + node + } + ast::Expression::Subscript { a, b } => { + let node = create_node(ctx, "Subscript"); + + let py_value = expression_to_ast(ctx, a); + node.set_attr("value", py_value); + + let py_slice = expression_to_ast(ctx, b); + node.set_attr("slice", py_slice); + + node + } + ast::Expression::Attribute { value, name } => { + let node = create_node(ctx, "Attribute"); + + let py_value = expression_to_ast(ctx, value); + node.set_attr("value", py_value); + + let py_attr = ctx.new_str(name.to_string()); + node.set_attr("attr", py_attr); + + node + } + ast::Expression::Starred { value } => { + let node = create_node(ctx, "Starred"); + + let py_value = expression_to_ast(ctx, value); + node.set_attr("value", py_value); + + node + } + ast::Expression::Slice { elements } => { + let node = create_node(ctx, "Slice"); + + let py_value = expressions_to_ast(ctx, elements); + node.set_attr("bounds", py_value); + + node + } ast::Expression::String { value } => { let node = create_node(ctx, "Str"); node.set_attr("s", ctx.new_str(value.clone())); node } - n => { - unimplemented!("{:?}", n); - } }; // TODO: retrieve correct lineno: @@ -305,6 +552,37 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR node } +fn parameters_to_ast(ctx: &PyContext, args: &ast::Parameters) -> PyObjectRef { + let node = create_node(ctx, "arguments"); + + node.set_attr( + "args", + ctx.new_list( + args.args + .iter() + .map(|a| ctx.new_str(a.to_string())) + .collect(), + ), + ); + + node +} + +fn comprehension_to_ast(ctx: &PyContext, comprehension: &ast::Comprehension) -> PyObjectRef { + let node = create_node(ctx, "comprehension"); + + let py_target = expression_to_ast(ctx, &comprehension.target); + node.set_attr("target", py_target); + + let py_iter = expression_to_ast(ctx, &comprehension.iter); + node.set_attr("iter", py_iter); + + let py_ifs = expressions_to_ast(ctx, &comprehension.ifs); + node.set_attr("ifs", py_ifs); + + node +} + fn ast_parse(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(source, Some(vm.ctx.str_type()))]);