RustPython/parser/src/python.lalrpop

// See also: file:///usr/share/doc/python/html/reference/grammar.html?highlight=grammar
// See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4
// See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions
// See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword

use std::iter::FromIterator;

use crate::ast;
use crate::fstring::parse_located_fstring;
use crate::function::{parse_args, parse_params};
use crate::error::LexicalError;
use crate::lexer;
use crate::location;

use num_bigint::BigInt;

grammar;

// This is a hack to reduce the amount of lalrpop tables generated:
// For each public entry point, a full parse table is generated.
// By having only a single pub function, we reduce this to one.
pub Top: ast::Top = {
    StartProgram <p:Program> => ast::Top::Program(p),
    StartStatement <s:Statement> => ast::Top::Statement(s),
    StartExpression <e:Test> ("\n")* => ast::Top::Expression(e),
};

Program: ast::Program = {
    <lines:FileLine*> => ast::Program {
        statements: Vec::from_iter(lines.into_iter().flatten())
    },
};

// A file line either has a declaration, or an empty newline:
FileLine: ast::Suite = {
    Statement,
    "\n" => vec![],
};

Suite: ast::Suite = {
    SimpleStatement,
    "\n" Indent <s:Statement+> Dedent => s.into_iter().flatten().collect(),
};

Statement: ast::Suite = {
    SimpleStatement,
    <s:CompoundStatement> => vec![s],
};

SimpleStatement: ast::Suite = {
    <s1:SmallStatement> <s2:(";" SmallStatement)*> ";"? "\n" => {
        let mut statements = vec![s1];
        statements.extend(s2.into_iter().map(|e| e.1));
        statements
    }
};

SmallStatement: ast::Statement = {
    ExpressionStatement,
    PassStatement,
    DelStatement,
    FlowStatement,
    ImportStatement,
    GlobalStatement,
    NonlocalStatement,
    AssertStatement,
};

PassStatement: ast::Statement = {
    <location:@L> "pass" => {
        ast::Statement {
            location,
            node: ast::StatementType::Pass,
        }
    },
};

DelStatement: ast::Statement = {
    <location:@L> "del" <targets:ExpressionList2> => {
        ast::Statement {
            location,
            node: ast::StatementType::Delete { targets },
        }
    },
};

ExpressionStatement: ast::Statement = {
    <location:@L> <expression:TestOrStarExprList> <suffix:AssignSuffix*> => {
        // Just an expression, no assignment:
        if suffix.is_empty() {
            ast::Statement {
                location,
                node: ast::StatementType::Expression { expression }
            }
        } else {
            let mut targets = vec![expression];
            let mut values = suffix;

            while values.len() > 1 {
                targets.push(values.remove(0));
            }

            let value = values.into_iter().next().unwrap();

            ast::Statement {
                location,
                node: ast::StatementType::Assign { targets, value },
            }
        }
    },
    <location:@L> <target:TestOrStarExprList> <op:AugAssign> <rhs:TestList> => {
        ast::Statement {
            location,
            node: ast::StatementType::AugAssign {
                target: Box::new(target),
                op,
                value: Box::new(rhs)
            },
        }
    },
    <location:@L> <target:Test> ":" <annotation:Test> <rhs:("=" Test)?> => {
        ast::Statement {
            location,
            node: ast::StatementType::AnnAssign {
                target: Box::new(target),
                annotation: Box::new(annotation),
                value: rhs.map(|e| e.1)
            },
        }
    },
};

AssignSuffix: ast::Expression = {
    "=" <e:TestList> => e,
    "=" <e:YieldExpr> => e,
};

TestOrStarExprList: ast::Expression = {
    <location:@L> <elements:OneOrMore<TestOrStarExpr>> <comma:","?> => {
        if elements.len() == 1 && comma.is_none() {
            elements.into_iter().next().unwrap()
        } else {
            ast::Expression {
                location,
                node: ast::ExpressionType::Tuple { elements }
            }
        }
    }
};

TestOrStarExpr: ast::Expression = {
    Test,
    StarExpr,
};

AugAssign: ast::Operator = {
    "+=" => ast::Operator::Add,
    "-=" => ast::Operator::Sub,
    "*=" => ast::Operator::Mult,
    "@=" => ast::Operator::MatMult,
    "/=" => ast::Operator::Div,
    "%=" => ast::Operator::Mod,
    "&=" => ast::Operator::BitAnd,
    "|=" => ast::Operator::BitOr,
    "^=" => ast::Operator::BitXor,
    "<<=" => ast::Operator::LShift,
    ">>=" => ast::Operator::RShift,
    "**=" => ast::Operator::Pow,
    "//=" => ast::Operator::FloorDiv,
};

FlowStatement: ast::Statement = {
    <location:@L> "break" => {
        ast::Statement {
            location,
            node: ast::StatementType::Break,
        }
    },
    <location:@L> "continue" => {
        ast::Statement {
            location,
            node: ast::StatementType::Continue,
        }
    },
    <location:@L> "return" <value:TestList?> => {
        ast::Statement {
            location,
            node: ast::StatementType::Return { value },
        }
    },
    <location:@L> <expression:YieldExpr> => {
        ast::Statement {
            location,
            node: ast::StatementType::Expression { expression },
        }
    },
    RaiseStatement,
};

RaiseStatement: ast::Statement = {
    <location:@L> "raise" => {
        ast::Statement {
            location,
            node: ast::StatementType::Raise { exception: None, cause: None },
        }
    },
    <location:@L> "raise" <t:Test> <c:("from" Test)?> => {
        ast::Statement {
            location,
            node: ast::StatementType::Raise { exception: Some(t), cause: c.map(|x| x.1) },
        }
    },
};

ImportStatement: ast::Statement = {
    <location:@L> "import" <names: OneOrMore<ImportAsAlias<DottedName>>> => {
        ast::Statement {
            location,
            node: ast::StatementType::Import { names },
        }
    },
    <location:@L> "from" <source:ImportFromLocation> "import" <names: ImportAsNames> => {
        let (level, module) = source;
        ast::Statement {
            location,
            node: ast::StatementType::ImportFrom {
                level,
                module,
                names
            },
        }
    },
};

ImportFromLocation: (usize, Option<String>) = {
    <dots: ImportDots*> <name:DottedName> => {
        (dots.iter().sum(), Some(name))
    },
    <dots: ImportDots+> => {
        (dots.iter().sum(), None)
    },
};

ImportDots: usize = {
    "..." => 3,
    "." => 1,
};

ImportAsNames: Vec<ast::ImportSymbol> = {
    <i:OneOrMore<ImportAsAlias<Identifier>>> => i,
    "(" <i:OneOrMore<ImportAsAlias<Identifier>>> ","? ")" => i,
    "*" => {
        // Star import all
        vec![ast::ImportSymbol { symbol: "*".to_string(), alias: None }]
    },
};


#[inline]
ImportAsAlias<I>: ast::ImportSymbol = {
    <symbol:I> <a: ("as" Identifier)?> => ast::ImportSymbol { symbol, alias: a.map(|a| a.1) },
};

// A name like abc or abc.def.ghi
DottedName: String = {
    <n:name> => n,
    <n:name> <n2: ("." Identifier)+>  => {
        let mut r = n.to_string();
        for x in n2 {
            r.push_str(".");
            r.push_str(&x.1);
        }
        r
    },
};

GlobalStatement: ast::Statement = {
    <location:@L> "global" <names:OneOrMore<Identifier>> => {
        ast::Statement {
            location,
            node: ast::StatementType::Global { names }
        }
    },
};

NonlocalStatement: ast::Statement = {
    <location:@L> "nonlocal" <names:OneOrMore<Identifier>> => {
        ast::Statement {
            location,
            node: ast::StatementType::Nonlocal { names }
        }
    },
};

AssertStatement: ast::Statement = {
    <location:@L> "assert" <test:Test> <msg: ("," Test)?> => {
        ast::Statement {
            location,
            node: ast::StatementType::Assert {
                test, msg: msg.map(|e| e.1)
            }
        }
    },
};

CompoundStatement: ast::Statement = {
    IfStatement,
    WhileStatement,
    ForStatement,
    TryStatement,
    WithStatement,
    FuncDef,
    ClassDef,
};

IfStatement: ast::Statement = {
    <location:@L> "if" <test:Test> ":" <body:Suite> <s2:(@L "elif" Test ":" Suite)*> <s3:("else" ":" Suite)?> => {
        // Determine last else:
        let mut last = s3.map(|s| s.2);

        // handle elif:
        for i in s2.into_iter().rev() {
            let x = ast::Statement {
                location: i.0,
                node: ast::StatementType::If { test: i.2, body: i.4, orelse: last },
            };
            last = Some(vec![x]);
        }

        ast::Statement {
            location,
            node: ast::StatementType::If { test, body, orelse: last }
        }
    },
};

WhileStatement: ast::Statement = {
    <location:@L> "while" <test:Test> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
        let orelse = s2.map(|s| s.2);
        ast::Statement {
            location,
            node: ast::StatementType::While {
                test,
                body,
                orelse
            },
        }
    },
};

ForStatement: ast::Statement = {
    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
        let is_async = is_async.is_some();
        let orelse = s2.map(|s| s.2);
        ast::Statement {
            location,
            node: ast::StatementType::For {
                is_async,
                target: Box::new(target),
                iter: Box::new(iter),
                body,
                orelse
            },
        }
    },
};

TryStatement: ast::Statement = {
    <location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> => {
        let orelse = else_suite.map(|s| s.2);
        let finalbody = finally.map(|s| s.2);
        ast::Statement {
            location,
            node: ast::StatementType::Try {
                body,
                handlers,
                orelse,
                finalbody,
            },
        }
    },
    <location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => {
        let handlers = vec![];
        let orelse = None;
        let finalbody = Some(finally.2);
        ast::Statement {
            location,
            node: ast::StatementType::Try {
                body,
                handlers,
                orelse,
                finalbody,
            },
        }
    },
};

ExceptClause: ast::ExceptHandler = {
    <location:@L> "except" <typ:Test?> ":" <body:Suite> => {
        ast::ExceptHandler {
            location,
            typ,
            name: None,
            body,
        }
    },
    <location:@L> "except" <x:(Test "as" Identifier)> ":" <body:Suite> => {
        ast::ExceptHandler {
            location,
            typ: Some(x.0),
            name: Some(x.2),
            body,
        }
    },
};

WithStatement: ast::Statement = {
    <location:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <body:Suite> => {
        let is_async = is_async.is_some();
        ast::Statement {
            location,
            node: ast::StatementType::With { is_async, items, body },
        }
    },
};

WithItem: ast::WithItem = {
    <context_expr:Test> <n:("as" Expression)?> => {
        let optional_vars = n.map(|val| val.1);
        ast::WithItem { context_expr, optional_vars }
    },
};

FuncDef: ast::Statement = {
    <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test)?> ":" <body:Suite> => {
        let is_async = is_async.is_some();
        ast::Statement {
            location,
            node: ast::StatementType::FunctionDef {
                is_async,
                name,
                args: Box::new(args),
                body,
                decorator_list,
                returns: r.map(|x| x.1),
            }
        }
    },
};

Parameters: ast::Parameters = {
    "(" <a: (ParameterList<TypedParameter>)?> ")" => a.unwrap_or_default(),
};

// Note that this is a macro which is used once for function defs, and
// once for lambda defs.
ParameterList<ArgType>: ast::Parameters = {
    <param1:ParameterDefs<ArgType>> <args2:("," ParameterListStarArgs<ArgType>)?> ","? =>? {
        let (names, default_elements) = parse_params(param1)?;

        // Now gather rest of parameters:
        let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1);

        Ok(ast::Parameters {
            args: names,
            kwonlyargs,
            vararg: vararg.into(),
            kwarg: kwarg.into(),
            defaults: default_elements,
            kw_defaults: kw_defaults,
        })
    },
    <param1:ParameterDefs<ArgType>> <kw:("," KwargParameter<ArgType>)> ","? =>? {
        let (names, default_elements) = parse_params(param1)?;

        // Now gather rest of parameters:
        let vararg = None;
        let kwonlyargs = vec![];
        let kw_defaults = vec![];
        let kwarg = Some(kw.1);

        Ok(ast::Parameters {
            args: names,
            kwonlyargs,
            vararg: vararg.into(),
            kwarg: kwarg.into(),
            defaults: default_elements,
            kw_defaults: kw_defaults,
        })
    },
    <params:ParameterListStarArgs<ArgType>> ","? => {
        let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
        ast::Parameters {
            args: vec![],
            kwonlyargs,
            vararg: vararg.into(),
            kwarg: kwarg.into(),
            defaults: vec![],
            kw_defaults: kw_defaults,
        }
    },
    <kw:KwargParameter<ArgType>> ","? => {
        ast::Parameters {
            args: vec![],
            kwonlyargs: vec![],
            vararg: ast::Varargs::None,
            kwarg: Some(kw).into(),
            defaults: vec![],
            kw_defaults: vec![],
        }
    },
};

// Use inline here to make sure the "," is not creating an ambiguity.
#[inline]
ParameterDefs<ArgType>: Vec<(ast::Parameter, Option<ast::Expression>)> = {
    <args:OneOrMore<ParameterDef<ArgType>>> => {
        args
    }
};

ParameterDef<ArgType>: (ast::Parameter, Option<ast::Expression>) = {
    <i:ArgType> => (i, None),
    <i:ArgType> "=" <e:Test> => (i, Some(e)),
};

UntypedParameter: ast::Parameter = {
    <location:@L> <arg:Identifier> => ast::Parameter { location, arg, annotation: None },
};

TypedParameter: ast::Parameter = {
    <location:@L> <arg:Identifier> <a:(":" Test)?>=> {
        let annotation = a.map(|x| Box::new(x.1));
        ast::Parameter { location, arg, annotation }
    },
};

// Use inline here to make sure the "," is not creating an ambiguity.
// TODO: figure out another grammar that makes this inline no longer required.
#[inline]
ParameterListStarArgs<ArgType>: (Option<Option<ast::Parameter>>, Vec<ast::Parameter>, Vec<Option<ast::Expression>>, Option<Option<ast::Parameter>>) = {
    "*" <va:ArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> => {
        // Extract keyword arguments:
        let mut kwonlyargs = vec![];
        let mut kw_defaults = vec![];
        for (name, value) in kw.into_iter().map(|x| x.1) {
            kwonlyargs.push(name);
            kw_defaults.push(value);
        }

        let kwarg = kwarg.map(|n| n.1);

        (Some(va), kwonlyargs, kw_defaults, kwarg)
    }
};

KwargParameter<ArgType>: Option<ast::Parameter> = {
    "**" <kwarg:ArgType?> => {
        kwarg
    }
};

ClassDef: ast::Statement = {
    <decorator_list:Decorator*> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> => {
        let (bases, keywords) = match a {
            Some((_, arg, _)) => (arg.args, arg.keywords),
            None => (vec![], vec![]),
        };
        ast::Statement {
            location,
            node: ast::StatementType::ClassDef {
                name,
                bases,
                keywords,
                body,
                decorator_list,
            },
        }
    },
};

Path: ast::Expression = {
    <location:@L> <n:Identifier> => ast::Expression {
        location,
        node: ast::ExpressionType::Identifier { name: n }
    },
    <p:Path> <location:@L> "." <n:name> => {
        ast::Expression {
            location,
            node: ast::ExpressionType::Attribute {
                value: Box::new(p),
                name: n,
            }
        }
    },
};

// Decorators:
Decorator: ast::Expression = {
    "@" <p:Path> <a: (@L "(" ArgumentList ")")?> "\n" => {
        match a {
            Some((location, _, arg, _)) => {
                ast::Expression {
                    location,
                    node: ast::ExpressionType::Call {
                        function: Box::new(p),
                        args: arg.args,
                        keywords: arg.keywords,
                    }
                }
            },
            None => p,
        }
    },
};

YieldExpr: ast::Expression = {
    <location:@L> "yield" <value:TestList?> => ast::Expression {
        location,
        node: ast::ExpressionType::Yield { value: value.map(Box::new) }
    },
    <location:@L> "yield" "from" <e:Test> => ast::Expression {
        location,
        node: ast::ExpressionType::YieldFrom { value: Box::new(e) }
    },
};

Test: ast::Expression = {
    <expr:OrTest> <condition: (@L "if" OrTest "else" Test)?> => {
        if let Some(c) = condition {
            ast::Expression {
                location: c.0,
                node: ast::ExpressionType::IfExpression {
                    test: Box::new(c.2),
                    body: Box::new(expr),
                    orelse: Box::new(c.4),
                }
            }
        } else {
            expr
        }
    },
    LambdaDef,
};

LambdaDef: ast::Expression = {
    <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test> =>
        ast::Expression {
            location,
            node: ast::ExpressionType::Lambda {
                args: Box::new(p.unwrap_or_default()),
                body: Box::new(body)
            }
       }
}

OrTest: ast::Expression = {
    <e1:AndTest> <location:@L> <e2:("or" AndTest)*> => {
        if e2.is_empty() {
            e1
        } else {
            let mut values = vec![e1];
            values.extend(e2.into_iter().map(|e| e.1));
            ast::Expression {
                location,
                node: ast::ExpressionType::BoolOp { op: ast::BooleanOperator::Or, values }
            }
        }
    },
};

AndTest: ast::Expression = {
    <e1:NotTest> <location:@L> <e2:("and" NotTest)*> => {
        if e2.is_empty() {
            e1
        } else {
            let mut values = vec![e1];
            values.extend(e2.into_iter().map(|e| e.1));
            ast::Expression {
                location,
                node: ast::ExpressionType::BoolOp { op: ast::BooleanOperator::And, values }
            }
        }
    },
};

NotTest: ast::Expression = {
    <location:@L> "not" <e:NotTest> => ast::Expression {
        location,
        node: ast::ExpressionType::Unop { a: Box::new(e), op: ast::UnaryOperator::Not }
    },
    Comparison,
};

Comparison: ast::Expression = {
    <e:Expression> <location:@L> <comparisons:(CompOp Expression)+> => {
        let mut vals = vec![e];
        let mut ops = vec![];
        for x in comparisons {
            ops.push(x.0);
            vals.push(x.1);
        }
        ast::Expression {
            location,
            node: ast::ExpressionType::Compare { vals, ops }
        }
    },
    Expression,
};

CompOp: ast::Comparison = {
    "==" => ast::Comparison::Equal,
    "!=" => ast::Comparison::NotEqual,
    "<" => ast::Comparison::Less,
    "<=" => ast::Comparison::LessOrEqual,
    ">" => ast::Comparison::Greater,
    ">=" => ast::Comparison::GreaterOrEqual,
    "in" => ast::Comparison::In,
    "not" "in" => ast::Comparison::NotIn,
    "is" => ast::Comparison::Is,
    "is" "not" => ast::Comparison::IsNot,
};

Expression: ast::Expression = {
    <e1:Expression> <location:@L> "|" <e2:XorExpression> => ast::Expression {
        location,
        node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) }
    },
    XorExpression,
};

XorExpression: ast::Expression = {
    <e1:XorExpression> <location:@L> "^" <e2:AndExpression> => ast::Expression {
        location,
        node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitXor, b: Box::new(e2) }
    },
    AndExpression,
};

AndExpression: ast::Expression = {
    <e1:AndExpression> <location:@L> "&" <e2:ShiftExpression> => ast::Expression {
        location,
        node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitAnd, b: Box::new(e2) }
    },
    ShiftExpression,
};

ShiftExpression: ast::Expression = {
    <e1:ShiftExpression> <location:@L> <op:ShiftOp> <e2:ArithmaticExpression> => ast::Expression {
        location,
        node: ast::ExpressionType::Binop { a: Box::new(e1), op, b: Box::new(e2) }
    },
    ArithmaticExpression,
};

ShiftOp: ast::Operator = {
    "<<" => ast::Operator::LShift,
    ">>" => ast::Operator::RShift,
};

ArithmaticExpression: ast::Expression = {
    <a:ArithmaticExpression> <location:@L> <op:AddOp> <b:Term> => ast::Expression {
        location,
        node: ast::ExpressionType::Binop { a: Box::new(a), op, b: Box::new(b) }
    },
    Term,
};

AddOp: ast::Operator = {
    "+" => ast::Operator::Add,
    "-" => ast::Operator::Sub,
};

Term: ast::Expression = {
    <a:Term> <location:@L> <op:MulOp> <b:Factor> => ast::Expression {
        location,
        node: ast::ExpressionType::Binop { a: Box::new(a), op, b: Box::new(b) }
    },
    Factor,
};

MulOp: ast::Operator = {
    "*" => ast::Operator::Mult,
    "/" => ast::Operator::Div,
    "//" => ast::Operator::FloorDiv,
    "%" => ast::Operator::Mod,
    "@" => ast::Operator::MatMult,
};

Factor: ast::Expression = {
    <location:@L> <op:UnOp> <e:Factor> => ast::Expression {
        location,
        node: ast::ExpressionType::Unop { a: Box::new(e), op }
    },
    Power,
};

UnOp: ast::UnaryOperator = {
    "+" => ast::UnaryOperator::Pos,
    "-" => ast::UnaryOperator::Neg,
    "~" => ast::UnaryOperator::Inv,
};

Power: ast::Expression = {
    <e:AtomExpr> <e2:(@L "**" Factor)?> => {
        match e2 {
            None => e,
            Some((location, _, b)) => ast::Expression {
                location,
                node: ast::ExpressionType::Binop { a: Box::new(e), op: ast::Operator::Pow, b: Box::new(b) }
            },
        }
    }
};

AtomExpr: ast::Expression = {
    <location:@L> <is_await:"await"?> <atom:AtomExpr2> => {
        if is_await.is_some() {
            ast::Expression {
                location,
                node: ast::ExpressionType::Await { value: Box::new(atom) }
            }
        } else {
            atom
        }
    }
}

AtomExpr2: ast::Expression = {
    Atom,
    <f:AtomExpr2> <location:@L> "(" <a:ArgumentList> ")" => {
        ast::Expression {
            location,
            node: ast::ExpressionType::Call { function: Box::new(f), args: a.args, keywords: a.keywords }
        }
    },
    <e:AtomExpr2> <location:@L> "[" <s:SubscriptList> "]" => ast::Expression {
        location,
        node: ast::ExpressionType::Subscript { a: Box::new(e), b: Box::new(s) }
    },
    <e:AtomExpr2> <location:@L> "." <name:Identifier> => ast::Expression {
        location,
        node: ast::ExpressionType::Attribute { value: Box::new(e), name }
    },
};

SubscriptList: ast::Expression = {
    <location:@L> <s1:Subscript> <s2:("," Subscript)*> ","? => {
        if s2.is_empty() {
            s1
        } else {
            let mut dims = vec![s1];
            for x in s2 {
                dims.push(x.1)
            }

            ast::Expression {
                location,
                node: ast::ExpressionType::Tuple { elements: dims },
            }
        }
    }
};

Subscript: ast::Expression = {
    Test,
    <e1:Test?> <location:@L> ":" <e2:Test?> <e3:SliceOp?>  => {
        let s1 = e1.unwrap_or(ast::Expression { location: location.clone(), node: ast::ExpressionType::None });
        let s2 = e2.unwrap_or(ast::Expression { location: location.clone(), node: ast::ExpressionType::None });
        let s3 = e3.unwrap_or(ast::Expression { location: location.clone(), node: ast::ExpressionType::None });
        ast::Expression {
            location,
            node: ast::ExpressionType::Slice { elements: vec![s1, s2, s3] }
        }
    }
};

SliceOp: ast::Expression = {
    <location:@L> ":" <e:Test?> => e.unwrap_or(ast::Expression {location, node: ast::ExpressionType::None})
}

Atom: ast::Expression = {
    <location:@L> <value:StringGroup> => ast::Expression {
        location,
        node: ast::ExpressionType::String { value }
    },
    <location:@L> <value:Bytes> => ast::Expression {
        location,
        node: ast::ExpressionType::Bytes { value }
    },
    <location:@L> <value:Number> => ast::Expression {
        location,
        node: ast::ExpressionType::Number { value }
    },
    <location:@L> <name:Identifier> => ast::Expression {
        location,
        node: ast::ExpressionType::Identifier { name }
    },
    <location:@L> "[" <e:ListLiteralValues?> "]" => {
        let elements = e.unwrap_or_default();
        ast::Expression {
            location,
            node: ast::ExpressionType::List { elements }
        }
    },
    <location:@L> "[" <element:TestOrStarExpr> <generators:CompFor> "]" => {
        ast::Expression {
            location,
            node: ast::ExpressionType::Comprehension {
                kind: Box::new(ast::ComprehensionKind::List { element }),
                generators,
            }
        }
    },
    <location:@L> "(" <elements:TestOrStarExprList?> ")" => {
        elements.unwrap_or(ast::Expression {
             location,
             node: ast::ExpressionType::Tuple { elements: Vec::new() }
        })
    },
    "(" <e:YieldExpr> ")" => e,
    <location:@L> "(" <element:Test> <generators:CompFor> ")" => {
        ast::Expression {
            location,
            node: ast::ExpressionType::Comprehension {
                kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element }),
                generators,
            }
        }
    },
    <location:@L> "{" <e:DictLiteralValues?> "}" => ast::Expression {
        location,
        node: ast::ExpressionType::Dict { elements: e.unwrap_or_default() }
    },
    <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" => {
        ast::Expression {
            location,
            node: ast::ExpressionType::Comprehension {
                kind: Box::new(ast::ComprehensionKind::Dict { key: e1.0, value: e1.1 }),
                generators,
            }
        }
    },
    <location:@L> "{" <elements:SetLiteralValues> "}" => ast::Expression {
        location,
        node: ast::ExpressionType::Set { elements }
    },
    <location:@L> "{" <element:Test> <generators:CompFor> "}" => {
        ast::Expression {
            location,
            node: ast::ExpressionType::Comprehension {
                kind: Box::new(ast::ComprehensionKind::Set { element }),
                generators,
            }
        }
    },
    <location:@L> "True" => ast::Expression { location, node: ast::ExpressionType::True },
    <location:@L> "False" => ast::Expression { location, node: ast::ExpressionType::False },
    <location:@L> "None" => ast::Expression { location, node: ast::ExpressionType::None },
    <location:@L> "..." => ast::Expression { location, node: ast::ExpressionType::Ellipsis },
};

ListLiteralValues: Vec<ast::Expression> = {
    <e:OneOrMore<TestOrStarExpr>> ","? => e,
};

DictLiteralValues: Vec<(Option<ast::Expression>, ast::Expression)> = {
    <elements:OneOrMore<DictElement>> ","? => elements,
};

DictEntry: (ast::Expression, ast::Expression) = {
    <e1: Test> ":" <e2: Test> => (e1, e2),
};

DictElement: (Option<ast::Expression>, ast::Expression) = {
    <e:DictEntry> => (Some(e.0), e.1),
    "**" <e:Expression> => (None, e),
};

SetLiteralValues: Vec<ast::Expression> = {
    <e1:OneOrMore<TestOrStarExpr>> ","? => e1
};

ExpressionOrStarExpression = {
    Expression,
    StarExpr
};

ExpressionList: ast::Expression = {
    <location:@L> <elements: OneOrMore<ExpressionOrStarExpression>> <trailing_comma:","?> => {
        if elements.len() == 1 && trailing_comma.is_none() {
            elements.into_iter().next().unwrap()
        } else {
            ast::Expression {
                location,
                node: ast::ExpressionType::Tuple { elements },
            }
        }
    },
};

ExpressionList2: Vec<ast::Expression> = {
    <elements:OneOrMore<Expression>> ","? => elements,
};

// A test list is one of:
// - a list of expressions
// - a single expression
// - a single expression followed by a trailing comma
TestList: ast::Expression = {
    <location:@L> <elements:OneOrMore<Test>> <trailing_comma: ","?> => {
        if elements.len() == 1 && trailing_comma.is_none() {
            elements.into_iter().next().unwrap()
        } else {
            ast::Expression {
                location,
                node: ast::ExpressionType::Tuple { elements },
            }
        }
    }
};

// Test
StarExpr: ast::Expression = {
    <location:@L> "*" <e:Expression> => ast::Expression {
        location,
        node: ast::ExpressionType::Starred { value: Box::new(e) },
    }
};

// Comprehensions:
CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c;

SingleForComprehension: ast::Comprehension = {
    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> => {
        let is_async = is_async.is_some();
        ast::Comprehension { location, target, iter, ifs, is_async }
    }
};

ExpressionNoCond: ast::Expression = OrTest;
ComprehensionIf: ast::Expression = "if" <c:ExpressionNoCond> => c;

ArgumentList: ast::ArgumentList = {
    <e: Comma<FunctionArgument>> =>? {
        let arg_list = parse_args(e)?;
        Ok(arg_list)
    }
};

FunctionArgument: (Option<Option<String>>, ast::Expression) = {
    <e:Test> <c:CompFor?> => {
        let expr = match c {
            Some(c) => ast::Expression {
                location: e.location.clone(),
                node: ast::ExpressionType::Comprehension {
                    kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element: e }),
                    generators: c,
                }
            },
            None => e,
        };
        (None, expr)
    },
    <i:Identifier> "=" <e:Test> => (Some(Some(i.clone())), e),
    <location:@L> "*" <e:Test> => (None, ast::Expression { location, node: ast::ExpressionType::Starred { value: Box::new(e) } }),
    "**" <e:Test> => (Some(None), e),
};

#[inline]
Comma<T>: Vec<T> = {
    <items: (<T> ",")*> <last: T?> => {
        let mut items = items;
        items.extend(last);
        items
    }
};

#[inline]
OneOrMore<T>: Vec<T> = {
    <i1: T> <i2:("," T)*> => {
        let mut items = vec![i1];
        items.extend(i2.into_iter().map(|e| e.1));
        items
    }
};

Number: ast::Number = {
    <value:int> => { ast::Number::Integer { value } },
    <value:float> => { ast::Number::Float { value } },
    <s:complex> => { ast::Number::Complex { real: s.0, imag: s.1 } },
};

StringGroup: ast::StringGroup = {
    <loc:@L> <s:string+> =>? {
        let mut values = vec![];
        for (value, is_fstring) in s {
            values.push(if is_fstring {
                parse_located_fstring(&value, loc.clone())?
            } else {
                ast::StringGroup::Constant { value }
            })
        }

        Ok(if values.len() > 1 {
            ast::StringGroup::Joined { values }
        } else {
            values.into_iter().next().unwrap()
        })
    },
};

Bytes: Vec<u8> = {
    <s:bytes+> => {
        s.into_iter().flatten().collect::<Vec<u8>>()
    },
};

Identifier: String = <s:name> => s;

// Hook external lexer:
extern {
    type Location = location::Location;
    type Error = LexicalError;

    enum lexer::Tok {
        Indent => lexer::Tok::Indent,
        Dedent => lexer::Tok::Dedent,
        StartProgram => lexer::Tok::StartProgram,
        StartStatement => lexer::Tok::StartStatement,
        StartExpression => lexer::Tok::StartExpression,
        "+" => lexer::Tok::Plus,
        "-" => lexer::Tok::Minus,
        "~" => lexer::Tok::Tilde,
        ":" => lexer::Tok::Colon,
        "." => lexer::Tok::Dot,
        "..." => lexer::Tok::Ellipsis,
        "," => lexer::Tok::Comma,
        "*" => lexer::Tok::Star,
        "**" => lexer::Tok::DoubleStar,
        "&" => lexer::Tok::Amper,
        "@" => lexer::Tok::At,
        "%" => lexer::Tok::Percent,
        "//" => lexer::Tok::DoubleSlash,
        "^" => lexer::Tok::CircumFlex,
        "|" => lexer::Tok::Vbar,
        "<<" => lexer::Tok::LeftShift,
        ">>" => lexer::Tok::RightShift,
        "/" => lexer::Tok::Slash,
        "(" => lexer::Tok::Lpar,
        ")" => lexer::Tok::Rpar,
        "[" => lexer::Tok::Lsqb,
        "]" => lexer::Tok::Rsqb,
        "{" => lexer::Tok::Lbrace,
        "}" => lexer::Tok::Rbrace,
        "=" => lexer::Tok::Equal,
        "+=" => lexer::Tok::PlusEqual,
        "-=" => lexer::Tok::MinusEqual,
        "*=" => lexer::Tok::StarEqual,
        "@=" => lexer::Tok::AtEqual,
        "/=" => lexer::Tok::SlashEqual,
        "%=" => lexer::Tok::PercentEqual,
        "&=" => lexer::Tok::AmperEqual,
        "|=" => lexer::Tok::VbarEqual,
        "^=" => lexer::Tok::CircumflexEqual,
        "<<=" => lexer::Tok::LeftShiftEqual,
        ">>=" => lexer::Tok::RightShiftEqual,
        "**=" => lexer::Tok::DoubleStarEqual,
        "//=" => lexer::Tok::DoubleSlashEqual,
        "==" => lexer::Tok::EqEqual,
        "!=" => lexer::Tok::NotEqual,
        "<" => lexer::Tok::Less,
        "<=" => lexer::Tok::LessEqual,
        ">" => lexer::Tok::Greater,
        ">=" => lexer::Tok::GreaterEqual,
        "->" => lexer::Tok::Rarrow,
        "and" => lexer::Tok::And,
        "as" => lexer::Tok::As,
        "assert" => lexer::Tok::Assert,
        "async" => lexer::Tok::Async,
        "await" => lexer::Tok::Await,
        "break" => lexer::Tok::Break,
        "class" => lexer::Tok::Class,
        "continue" => lexer::Tok::Continue,
        "def" => lexer::Tok::Def,
        "del" => lexer::Tok::Del,
        "elif" => lexer::Tok::Elif,
        "else" => lexer::Tok::Else,
        "except" => lexer::Tok::Except,
        "finally" => lexer::Tok::Finally,
        "for" => lexer::Tok::For,
        "from" => lexer::Tok::From,
        "global" => lexer::Tok::Global,
        "if" => lexer::Tok::If,
        "in" => lexer::Tok::In,
        "is" => lexer::Tok::Is,
        "import" => lexer::Tok::Import,
        "from" => lexer::Tok::From,
        "lambda" => lexer::Tok::Lambda,
        "nonlocal" => lexer::Tok::Nonlocal,
        "not" => lexer::Tok::Not,
        "or" => lexer::Tok::Or,
        "pass" => lexer::Tok::Pass,
        "raise" => lexer::Tok::Raise,
        "return" => lexer::Tok::Return,
        "try" => lexer::Tok::Try,
        "while" => lexer::Tok::While,
        "with" => lexer::Tok::With,
        "yield" => lexer::Tok::Yield,
        "True" => lexer::Tok::True,
        "False" => lexer::Tok::False,
        "None" => lexer::Tok::None,
        int => lexer::Tok::Int { value: <BigInt> },
        float => lexer::Tok::Float { value: <f64> },
        complex => lexer::Tok::Complex { real: <f64>, imag: <f64> },
        string => lexer::Tok::String { value: <String>, is_fstring: <bool> },
        bytes => lexer::Tok::Bytes { value: <Vec<u8>> },
        name => lexer::Tok::Name { name: <String> },
        "\n" => lexer::Tok::Newline,
        ";" => lexer::Tok::Semi,
    }
}