Extend ast module with more nodes. Test send method on generators.

This commit is contained in:
Windel Bouwman
2018-10-24 16:02:59 +02:00
parent babb8ed2f0
commit 09e2c027cd
7 changed files with 439 additions and 138 deletions

View File

@@ -143,10 +143,10 @@ pub enum Expression {
a: Box<Expression>,
},
Yield {
expression: Option<Box<Expression>>,
value: Option<Box<Expression>>,
},
YieldFrom {
expression: Box<Expression>,
value: Box<Expression>,
},
Compare {
a: Box<Expression>,

View File

@@ -21,7 +21,9 @@ pub Top: ast::Top = {
};
Program: ast::Program = {
<lines:FileLine*> => ast::Program { statements: Vec::from_iter(lines.into_iter().filter_map(|e| e)) },
<lines:FileLine*> => ast::Program {
statements: Vec::from_iter(lines.into_iter().filter_map(|e| e))
},
};
// A file line either has a declaration, or an empty newline:
@@ -31,8 +33,8 @@ FileLine: Option<ast::LocatedStatement> = {
};
Suite: Vec<ast::LocatedStatement> = {
<s:SimpleStatement> => vec![s],
"\n" indent <s:Statement+> dedent => s,
<s:SimpleStatement> => vec![s],
"\n" indent <s:Statement+> dedent => s,
};
Statement: ast::LocatedStatement = {
@@ -89,19 +91,10 @@ ExpressionStatement: ast::LocatedStatement = {
}
} else {
let mut targets = vec![expr];
let mut values : Vec<ast::Expression> = suffix
.into_iter()
.map(|test_list| if test_list.len() > 1 {
ast::Expression::Tuple {
elements: test_list
}
} else {
test_list.into_iter().next().unwrap()
})
.collect();
let mut values = suffix;
while values.len() > 1 {
targets.push(values.remove(0));
targets.push(values.remove(0));
}
let value = values.into_iter().next().unwrap();
@@ -122,14 +115,23 @@ ExpressionStatement: ast::LocatedStatement = {
// TODO: this works in most cases:
let rhs = e2.into_iter().next().unwrap();
ast::LocatedStatement {
location: loc,
node: ast::Statement::AugAssign { target: expr, op: op, value: rhs },
location: loc,
node: ast::Statement::AugAssign { target: expr, op: op, value: rhs },
}
},
};
AssignSuffix: Vec<ast::Expression> = {
"=" <e:TestList> => e,
AssignSuffix: ast::Expression = {
"=" <e:TestList> => {
if e.len() > 1 {
ast::Expression::Tuple {
elements: e
}
} else {
e.into_iter().next().unwrap()
}
},
"=" <e:YieldExpr> => e,
};
TestOrStarExprList: Vec<ast::Expression> = {
@@ -146,52 +148,52 @@ TestOrStarExpr: ast::Expression = {
};
AugAssign: ast::Operator = {
"+=" => ast::Operator::Add,
"-=" => ast::Operator::Sub,
"*=" => ast::Operator::Mult,
"@=" => ast::Operator::MatMult,
"/=" => ast::Operator::Div,
"%=" => ast::Operator::Mod,
"&=" => ast::Operator::BitAnd,
"|=" => ast::Operator::BitOr,
"^=" => ast::Operator::BitXor,
"<<=" => ast::Operator::LShift,
">>=" => ast::Operator::RShift,
"**=" => ast::Operator::Pow,
"//=" => ast::Operator::FloorDiv,
"+=" => ast::Operator::Add,
"-=" => ast::Operator::Sub,
"*=" => ast::Operator::Mult,
"@=" => ast::Operator::MatMult,
"/=" => ast::Operator::Div,
"%=" => ast::Operator::Mod,
"&=" => ast::Operator::BitAnd,
"|=" => ast::Operator::BitOr,
"^=" => ast::Operator::BitXor,
"<<=" => ast::Operator::LShift,
">>=" => ast::Operator::RShift,
"**=" => ast::Operator::Pow,
"//=" => ast::Operator::FloorDiv,
};
FlowStatement: ast::LocatedStatement = {
<loc:@L> "break" => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Break,
}
},
<loc:@L> "continue" => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Continue,
}
},
<loc:@L> "return" <t:TestList?> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Return { value: t },
}
},
<loc:@L> "raise" <t:Test?> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Raise { expression: t },
}
},
<loc:@L> <y:YieldExpr> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Expression { expression: y },
}
},
<loc:@L> "break" => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Break,
}
},
<loc:@L> "continue" => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Continue,
}
},
<loc:@L> "return" <t:TestList?> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Return { value: t },
}
},
<loc:@L> "raise" <t:Test?> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Raise { expression: t },
}
},
<loc:@L> <y:YieldExpr> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Expression { expression: y },
}
},
};
ImportStatement: ast::LocatedStatement = {
@@ -264,30 +266,30 @@ ImportPart<I>: (String, Option<String>) = {
// A name like abc or abc.def.ghi
DottedName: String = {
<n:name> => n,
<n:name> <n2: ("." Identifier)+> => {
let mut r = n.to_string();
for x in n2 {
r.push_str(".");
r.push_str(&x.1);
}
r
},
<n:name> => n,
<n:name> <n2: ("." Identifier)+> => {
let mut r = n.to_string();
for x in n2 {
r.push_str(".");
r.push_str(&x.1);
}
r
},
};
AssertStatement: ast::LocatedStatement = {
<loc:@L> "assert" <t:Test> <m: ("," Test)?> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Assert {
test: t,
msg: match m {
Some(e) => Some(e.1),
None => None,
<loc:@L> "assert" <t:Test> <m: ("," Test)?> => {
ast::LocatedStatement {
location: loc,
node: ast::Statement::Assert {
test: t,
msg: match m {
Some(e) => Some(e.1),
None => None,
}
}
}
}
}
},
},
};
CompoundStatement: ast::LocatedStatement = {
@@ -376,33 +378,33 @@ TryStatement: ast::LocatedStatement = {
};
ExceptClause: ast::ExceptHandler = {
"except" <typ:Test?> ":" <body:Suite> => {
ast::ExceptHandler {
typ: typ,
name: None,
body: body,
}
},
"except" <x:(Test "as" Identifier)> ":" <body:Suite> => {
ast::ExceptHandler {
typ: Some(x.0),
name: Some(x.2),
body: body,
}
},
"except" <typ:Test?> ":" <body:Suite> => {
ast::ExceptHandler {
typ: typ,
name: None,
body: body,
}
},
"except" <x:(Test "as" Identifier)> ":" <body:Suite> => {
ast::ExceptHandler {
typ: Some(x.0),
name: Some(x.2),
body: body,
}
},
};
WithStatement: ast::LocatedStatement = {
<loc:@L> "with" <i1:WithItem> <i2:("," WithItem)*> ":" <s:Suite> => {
let mut items = vec![i1];
for item in i2 {
items.push(item.1);
}
ast::LocatedStatement {
location: loc,
node: ast::Statement::With { items: items, body: s },
}
},
<loc:@L> "with" <i1:WithItem> <i2:("," WithItem)*> ":" <s:Suite> => {
let mut items = vec![i1];
for item in i2 {
items.push(item.1);
}
ast::LocatedStatement {
location: loc,
node: ast::Statement::With { items: items, body: s },
}
},
};
WithItem: ast::WithItem = {
@@ -569,7 +571,7 @@ Decorator: ast::Expression = {
YieldExpr: ast::Expression = {
"yield" <ex:TestList?> => {
ast::Expression::Yield {
expression: ex.map(|expr| Box::new(
value: ex.map(|expr| Box::new(
if expr.len() > 1 {
ast::Expression::Tuple { elements: expr }
} else {
@@ -580,7 +582,7 @@ YieldExpr: ast::Expression = {
},
"yield" "from" <e:Test> => {
ast::Expression::YieldFrom {
expression: Box::new(e),
value: Box::new(e),
}
},
};
@@ -630,21 +632,21 @@ Comparison: ast::Expression = {
};
CompOp: ast::Comparison = {
"==" => ast::Comparison::Equal,
"!=" => ast::Comparison::NotEqual,
"<" => ast::Comparison::Less,
"<=" => ast::Comparison::LessOrEqual,
">" => ast::Comparison::Greater,
">=" => ast::Comparison::GreaterOrEqual,
"in" => ast::Comparison::In,
"not" "in" => ast::Comparison::NotIn,
"is" => ast::Comparison::Is,
"is" "not" => ast::Comparison::IsNot,
"==" => ast::Comparison::Equal,
"!=" => ast::Comparison::NotEqual,
"<" => ast::Comparison::Less,
"<=" => ast::Comparison::LessOrEqual,
">" => ast::Comparison::Greater,
">=" => ast::Comparison::GreaterOrEqual,
"in" => ast::Comparison::In,
"not" "in" => ast::Comparison::NotIn,
"is" => ast::Comparison::Is,
"is" "not" => ast::Comparison::IsNot,
};
Expression: ast::Expression = {
<e1:Expression> "|" <e2:XorExpression> => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) },
<e:XorExpression> => e,
<e1:Expression> "|" <e2:XorExpression> => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) },
<e:XorExpression> => e,
};
XorExpression: ast::Expression = {
@@ -854,7 +856,7 @@ StarExpr: ast::Expression = {
// Comprehensions:
CompIter: (Option<Vec<ast::Comprehension>>, Option<ast::Expression>) = {
// CompIf,
// <e:CompIf> => (None, Some(e)),
<c:CompFor> => (Some(c), None),
};
@@ -891,7 +893,13 @@ ExpressionNoCond: ast::Expression = {
//CompIf: ast::Expression = {
// "if" <c:ExpressionNoCond> <c2:CompIter?> => {
// c
// match c2 {
// None => {
// vec![]
// },
// Some() => {
// },
// }
// }
//};

View File

@@ -18,3 +18,7 @@ v = {b * 2 for b in x}
# TODO:
#u = {str(b): b-2 for b in x}
# TODO: With if filtering:
#y = [a+2 for a in x if a % 2]
#print(y)
#assert y == [3, 5]

View File

@@ -14,3 +14,14 @@ assert r == [1, 2, 42, 3]
r = list(x for x in [1, 2, 3])
assert r == [1, 2, 3]
def g2(x):
x = yield x
yield x + 5
yield x + 7
i = g2(23)
assert 23 == next(i)
assert 15 == i.send(10)
assert 17 == i.send(10)

View File

@@ -895,9 +895,9 @@ impl Compiler {
}
self.emit(Instruction::BuildSlice { size: size });
}
ast::Expression::Yield { expression } => {
ast::Expression::Yield { value } => {
self.mark_generator();
match expression {
match value {
Some(expression) => self.compile_expression(expression)?,
None => self.emit(Instruction::LoadConst {
value: bytecode::Constant::None,
@@ -905,9 +905,9 @@ impl Compiler {
};
self.emit(Instruction::YieldValue);
}
ast::Expression::YieldFrom { expression } => {
ast::Expression::YieldFrom { value } => {
self.mark_generator();
self.compile_expression(expression)?;
self.compile_expression(value)?;
self.emit(Instruction::YieldValue);
unimplemented!("yield from todo");
}

View File

@@ -14,7 +14,7 @@ pub fn init(context: &PyContext) {
let ref generator_type = context.generator_type;
generator_type.set_attr("__iter__", context.new_rustfunc(generator_iter));
generator_type.set_attr("__next__", context.new_rustfunc(generator_next));
generator_type.set_attr("__send__", context.new_rustfunc(generator_send));
generator_type.set_attr("send", context.new_rustfunc(generator_send));
}
pub fn new_generator(vm: &mut VirtualMachine, frame: Frame) -> PyResult {

View File

@@ -13,6 +13,7 @@ use super::super::pyobject::{
AttributeProtocol, DictProtocol, PyContext, PyFuncArgs, PyObjectRef, PyResult, TypeProtocol,
};
use super::super::VirtualMachine;
use std::ops::Deref;
/*
* Idea: maybe we can create a sort of struct with some helper functions?
@@ -83,7 +84,7 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj
}
ast::Statement::FunctionDef {
name,
args: _,
args,
body,
decorator_list,
} => {
@@ -92,6 +93,8 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj
// Set name:
node.set_attr("name", ctx.new_str(name.to_string()));
node.set_attr("args", parameters_to_ast(ctx, args));
// Set body:
let py_body = statements_to_ast(ctx, body);
node.set_attr("body", py_body);
@@ -112,6 +115,19 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj
let node = create_node(ctx, "Pass");
node
}
ast::Statement::Assert { test, msg } => {
let node = create_node(ctx, "Pass");
node.set_attr("test", expression_to_ast(ctx, test));
let py_msg = match msg {
Some(msg) => expression_to_ast(ctx, msg),
None => ctx.none(),
};
node.set_attr("msg", py_msg);
node
}
ast::Statement::Delete { targets } => {
let node = create_node(ctx, "Delete");
@@ -161,20 +177,18 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj
node
}
ast::Statement::For {
target: _,
iter: _,
target,
iter,
body,
orelse,
} => {
let node = create_node(ctx, "For");
/*
let py_target = expression_to_ast(ctx, target);
node.set_attr("target", py_target);
let py_iter = expression_to_ast(ctx, iter);
let py_iter = expressions_to_ast(ctx, iter);
node.set_attr("iter", py_iter);
*/
let py_body = statements_to_ast(ctx, body);
node.set_attr("body", py_body);
@@ -280,6 +294,65 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR
node.set_attr("right", py_b);
node
}
ast::Expression::Unop { op, a } => {
let node = create_node(ctx, "UnaryOp");
let str_op = match op {
ast::UnaryOperator::Not => "Not",
ast::UnaryOperator::Neg => "USub",
};
let py_op = ctx.new_str(str_op.to_string());
node.set_attr("op", py_op);
let py_a = expression_to_ast(ctx, a);
node.set_attr("operand", py_a);
node
}
ast::Expression::BoolOp { a, op, b } => {
let node = create_node(ctx, "BoolOp");
// Attach values:
let py_a = expression_to_ast(ctx, a);
let py_b = expression_to_ast(ctx, b);
let py_values = ctx.new_tuple(vec![py_a, py_b]);
node.set_attr("values", py_values);
let str_op = match op {
ast::BooleanOperator::And => "And",
ast::BooleanOperator::Or => "Or",
};
let py_op = ctx.new_str(str_op.to_string());
node.set_attr("op", py_op);
node
}
ast::Expression::Compare { a, op, b } => {
let node = create_node(ctx, "Compare");
let py_a = expression_to_ast(ctx, a);
node.set_attr("left", py_a);
// Operator:
let str_op = match op {
ast::Comparison::Equal => "Eq",
ast::Comparison::NotEqual => "NotEq",
ast::Comparison::Less => "Lt",
ast::Comparison::LessOrEqual => "LtE",
ast::Comparison::Greater => "Gt",
ast::Comparison::GreaterOrEqual => "GtE",
ast::Comparison::In => "In",
ast::Comparison::NotIn => "NotIn",
ast::Comparison::Is => "Is",
ast::Comparison::IsNot => "IsNot",
};
let py_ops = ctx.new_list(vec![ctx.new_str(str_op.to_string())]);
node.set_attr("ops", py_ops);
let py_b = ctx.new_list(vec![expression_to_ast(ctx, b)]);
node.set_attr("comparators", py_b);
node
}
ast::Expression::Identifier { name } => {
let node = create_node(ctx, "Identifier");
@@ -288,14 +361,188 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR
node.set_attr("id", py_name);
node
}
ast::Expression::Lambda { args, body } => {
let node = create_node(ctx, "Lambda");
node.set_attr("args", parameters_to_ast(ctx, args));
let py_body = expression_to_ast(ctx, body);
node.set_attr("body", py_body);
node
}
ast::Expression::IfExpression { test, body, orelse } => {
let node = create_node(ctx, "IfExp");
let py_test = expression_to_ast(ctx, test);
node.set_attr("test", py_test);
let py_body = expression_to_ast(ctx, body);
node.set_attr("body", py_body);
let py_orelse = expression_to_ast(ctx, orelse);
node.set_attr("orelse", py_orelse);
node
}
ast::Expression::Number { value } => {
let node = create_node(ctx, "Num");
let py_n = match value {
ast::Number::Integer { value } => ctx.new_int(*value),
ast::Number::Float { value } => ctx.new_float(*value),
};
node.set_attr("n", py_n);
node
}
ast::Expression::True => {
let node = create_node(ctx, "NameConstant");
node.set_attr("value", ctx.new_bool(true));
node
}
ast::Expression::False => {
let node = create_node(ctx, "NameConstant");
node.set_attr("value", ctx.new_bool(false));
node
}
ast::Expression::None => {
let node = create_node(ctx, "NameConstant");
node.set_attr("value", ctx.none());
node
}
ast::Expression::List { elements } => {
let node = create_node(ctx, "List");
let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect();
let py_elts = ctx.new_list(elts);
node.set_attr("elts", py_elts);
node
}
ast::Expression::Tuple { elements } => {
let node = create_node(ctx, "Tuple");
let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect();
let py_elts = ctx.new_list(elts);
node.set_attr("elts", py_elts);
node
}
ast::Expression::Set { elements } => {
let node = create_node(ctx, "Set");
let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect();
let py_elts = ctx.new_list(elts);
node.set_attr("elts", py_elts);
node
}
ast::Expression::Dict { elements } => {
let node = create_node(ctx, "Dict");
let mut keys = Vec::new();
let mut values = Vec::new();
for (k, v) in elements {
keys.push(expression_to_ast(ctx, k));
values.push(expression_to_ast(ctx, v));
}
let py_keys = ctx.new_list(keys);
node.set_attr("keys", py_keys);
let py_values = ctx.new_list(values);
node.set_attr("values", py_values);
node
}
ast::Expression::Comprehension { kind, generators } => {
let node = match kind.deref() {
ast::ComprehensionKind::GeneratorExpression { .. } => {
create_node(ctx, "GeneratorExp")
}
ast::ComprehensionKind::List { .. } => create_node(ctx, "ListComp"),
ast::ComprehensionKind::Set { .. } => create_node(ctx, "SetComp"),
ast::ComprehensionKind::Dict { .. } => create_node(ctx, "DictComp"),
};
let g = generators
.iter()
.map(|g| comprehension_to_ast(ctx, g))
.collect();
let py_generators = ctx.new_list(g);
node.set_attr("generators", py_generators);
node
}
ast::Expression::Yield { value } => {
let node = create_node(ctx, "Yield");
let py_value = match value {
Some(value) => expression_to_ast(ctx, value),
None => ctx.none(),
};
node.set_attr("value", py_value);
node
}
ast::Expression::YieldFrom { value } => {
let node = create_node(ctx, "YieldFrom");
let py_value = expression_to_ast(ctx, value);
node.set_attr("value", py_value);
node
}
ast::Expression::Subscript { a, b } => {
let node = create_node(ctx, "Subscript");
let py_value = expression_to_ast(ctx, a);
node.set_attr("value", py_value);
let py_slice = expression_to_ast(ctx, b);
node.set_attr("slice", py_slice);
node
}
ast::Expression::Attribute { value, name } => {
let node = create_node(ctx, "Attribute");
let py_value = expression_to_ast(ctx, value);
node.set_attr("value", py_value);
let py_attr = ctx.new_str(name.to_string());
node.set_attr("attr", py_attr);
node
}
ast::Expression::Starred { value } => {
let node = create_node(ctx, "Starred");
let py_value = expression_to_ast(ctx, value);
node.set_attr("value", py_value);
node
}
ast::Expression::Slice { elements } => {
let node = create_node(ctx, "Slice");
let py_value = expressions_to_ast(ctx, elements);
node.set_attr("bounds", py_value);
node
}
ast::Expression::String { value } => {
let node = create_node(ctx, "Str");
node.set_attr("s", ctx.new_str(value.clone()));
node
}
n => {
unimplemented!("{:?}", n);
}
};
// TODO: retrieve correct lineno:
@@ -305,6 +552,37 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR
node
}
fn parameters_to_ast(ctx: &PyContext, args: &ast::Parameters) -> PyObjectRef {
let node = create_node(ctx, "arguments");
node.set_attr(
"args",
ctx.new_list(
args.args
.iter()
.map(|a| ctx.new_str(a.to_string()))
.collect(),
),
);
node
}
fn comprehension_to_ast(ctx: &PyContext, comprehension: &ast::Comprehension) -> PyObjectRef {
let node = create_node(ctx, "comprehension");
let py_target = expression_to_ast(ctx, &comprehension.target);
node.set_attr("target", py_target);
let py_iter = expression_to_ast(ctx, &comprehension.iter);
node.set_attr("iter", py_iter);
let py_ifs = expressions_to_ast(ctx, &comprehension.ifs);
node.set_attr("ifs", py_ifs);
node
}
fn ast_parse(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
arg_check!(vm, args, required = [(source, Some(vm.ctx.str_type()))]);