Rust-related/RustPython
1
0
Fork 1
mirror of https://github.com/RustPython/RustPython.git synced 2026-06-09 22:49:57 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1177 from RustPython/example1

Browse Source 
Add an example usecase of the parser.
This commit is contained in:
Windel Bouwman
2019-07-26 21:22:20 +02:00
committed by GitHub
parent 95561f4fc1 1fddce461f
commit a02e131bd5
8 changed files with 199 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
compiler/src/compile.rs
View File

@@ -385,35 +385,47 @@ impl Compiler {
}
self.set_label(end_label);
}
With { items, body } => {
let end_label = self.new_label();
for item in items {
self.compile_expression(&item.context_expr)?;
self.emit(Instruction::SetupWith { end: end_label });
match &item.optional_vars {
Some(var) => {
self.compile_store(var)?;
}
None => {
self.emit(Instruction::Pop);
With {
is_async,
items,
body,
} => {
if *is_async {
unimplemented!("async with");
} else {
let end_label = self.new_label();
for item in items {
self.compile_expression(&item.context_expr)?;
self.emit(Instruction::SetupWith { end: end_label });
match &item.optional_vars {
Some(var) => {
self.compile_store(var)?;
}
None => {
self.emit(Instruction::Pop);
}
}
}
}
self.compile_statements(body)?;
for _ in 0..items.len() {
self.emit(Instruction::CleanupWith { end: end_label });
self.compile_statements(body)?;
for _ in 0..items.len() {
self.emit(Instruction::CleanupWith { end: end_label });
}
self.set_label(end_label);
}
self.set_label(end_label);
}
For {
is_async,
target,
iter,
body,
orelse,
} => self.compile_for(target, iter, body, orelse)?,
AsyncFor { .. } => {
unimplemented!("async for");
} => {
if *is_async {
unimplemented!("async for");
} else {
self.compile_for(target, iter, body, orelse)?
}
}
Raise { exception, cause } => match exception {
Some(value) => {
@@ -439,14 +451,18 @@ impl Compiler {
finalbody,
} => self.compile_try_statement(body, handlers, orelse, finalbody)?,
FunctionDef {
is_async,
name,
args,
body,
decorator_list,
returns,
} => self.compile_function_def(name, args, body, decorator_list, returns)?,
AsyncFunctionDef { .. } => {
unimplemented!("async def");
} => {
if *is_async {
unimplemented!("async def");
} else {
self.compile_function_def(name, args, body, decorator_list, returns)?
}
}
ClassDef {
name,

17
compiler/src/symboltable.rs
View File

@@ -240,13 +240,7 @@ impl SymbolTableBuilder {
args,
decorator_list,
returns,
}
| AsyncFunctionDef {
name,
body,
args,
decorator_list,
returns,
..
} => {
self.scan_expressions(decorator_list)?;
self.register_name(name, SymbolRole::Assigned)?;
@@ -289,12 +283,7 @@ impl SymbolTableBuilder {
iter,
body,
orelse,
}
| AsyncFor {
target,
iter,
body,
orelse,
..
} => {
self.scan_expression(target)?;
self.scan_expression(iter)?;
@@ -346,7 +335,7 @@ impl SymbolTableBuilder {
self.scan_expression(target)?;
self.scan_expression(value)?;
}
With { items, body } => {
With { items, body, .. } => {
for item in items {
self.scan_expression(&item.context_expr)?;
if let Some(expression) = &item.optional_vars {

70
examples/parse_folder.rs Normal file
View File

@@ -0,0 +1,70 @@
/// This an example usage of the rustpython_parser crate.
/// This program crawls over a directory of python files and
/// tries to parse them into an abstract syntax tree (AST)
///
/// example usage:
/// $ RUST_LOG=info cargo run --release parse_folder /usr/lib/python3.7
#[macro_use]
extern crate clap;
extern crate env_logger;
#[macro_use]
extern crate log;
use clap::{App, Arg};
use rustpython_parser::{ast, parser};
use std::path::Path;
fn main() {
env_logger::init();
let app = App::new("parse_folders")
.version(crate_version!())
.author(crate_authors!())
.about("Walks over all .py files in a folder, and parses them.")
.arg(
Arg::with_name("folder")
.help("Folder to scan")
.required(true),
);
let matches = app.get_matches();
let folder = Path::new(matches.value_of("folder").unwrap());
if folder.exists() && folder.is_dir() {
println!("Parsing folder of python code: {:?}", folder);
let res = parse_folder(&folder).unwrap();
println!("Processed {:?} files", res.len());
} else {
println!("{:?} is not a folder.", folder);
}
}
fn parse_folder(path: &Path) -> std::io::Result<Vec<ast::Program>> {
let mut res = vec![];
info!("Parsing folder of python code: {:?}", path);
for entry in path.read_dir()? {
debug!("Entry: {:?}", entry);
let entry = entry?;
let metadata = entry.metadata()?;
let path = entry.path();
if metadata.is_dir() {
let x = parse_folder(&path)?;
res.extend(x);
}
if metadata.is_file() && path.extension().and_then(|s| s.to_str()) == Some("py") {
match parse_python_file(&path) {
Ok(x) => res.push(x),
Err(y) => error!("Erreur in file {:?} {:?}", path, y),
}
}
}
Ok(res)
}
fn parse_python_file(filename: &Path) -> Result<ast::Program, String> {
info!("Parsing file {:?}", filename);
let source = std::fs::read_to_string(filename).map_err(|e| e.to_string())?;
parser::parse_program(&source).map_err(|e| e.to_string())
}

16
parser/src/ast.rs
View File

@@ -94,16 +94,12 @@ pub enum StatementType {
orelse: Option<Vec<Statement>>,
},
With {
is_async: bool,
items: Vec<WithItem>,
body: Vec<Statement>,
},
For {
target: Expression,
iter: Expression,
body: Vec<Statement>,
orelse: Option<Vec<Statement>>,
},
AsyncFor {
is_async: bool,
target: Expression,
iter: Expression,
body: Vec<Statement>,
@@ -127,13 +123,7 @@ pub enum StatementType {
decorator_list: Vec<Expression>,
},
FunctionDef {
name: String,
args: Parameters,
body: Vec<Statement>,
decorator_list: Vec<Expression>,
returns: Option<Expression>,
},
AsyncFunctionDef {
is_async: bool,
name: String,
args: Parameters,
body: Vec<Statement>,

13
parser/src/lexer.rs
View File

@@ -694,6 +694,13 @@ where
spaces = 0;
tabs = 0;
}
Some('\x0C') => {
// Form feed character!
// Reset indentation for the Emacs user.
self.next_char();
spaces = 0;
tabs = 0;
}
Some('\n') => {
// Empty line!
self.next_char();
@@ -1157,9 +1164,13 @@ where
self.emit((tok_start, Tok::Newline, tok_end));
}
}
' ' => {
' ' | '\t' | '\x0C' => {
// Skip whitespaces
self.next_char();
while self.chr0 == Some(' ') || self.chr0 == Some('\t') || self.chr0 == Some('\x0C')
{
self.next_char();
}
}
_ => {
let c = self.next_char();

2
parser/src/parser.rs
View File

@@ -306,6 +306,7 @@ mod tests {
ast::Statement {
location: ast::Location::new(2, 2),
node: ast::StatementType::FunctionDef {
is_async: false,
name: String::from("__init__"),
args: ast::Parameters {
args: vec![ast::Parameter {
@@ -329,6 +330,7 @@ mod tests {
ast::Statement {
location: ast::Location::new(4, 2),
node: ast::StatementType::FunctionDef {
is_async: false,
name: String::from("method_with_default"),
args: ast::Parameters {
args: vec![

40
parser/src/python.lalrpop
View File

@@ -334,14 +334,11 @@ WhileStatement: ast::Statement = {
ForStatement: ast::Statement = {
<location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
let is_async = is_async.is_some();
let orelse = s2.map(|s| s.2);
ast::Statement {
location,
node: if is_async.is_some() {
ast::StatementType::AsyncFor { target, iter, body, orelse }
} else {
ast::StatementType::For { target, iter, body, orelse }
},
node: ast::StatementType::For { is_async, target, iter, body, orelse },
}
},
};
@@ -380,10 +377,11 @@ ExceptClause: ast::ExceptHandler = {
};
WithStatement: ast::Statement = {
<location:@L> "with" <items:OneOrMore<WithItem>> ":" <s:Suite> => {
<location:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <body:Suite> => {
let is_async = is_async.is_some();
ast::Statement {
location,
node: ast::StatementType::With { items: items, body: s },
node: ast::StatementType::With { is_async, items, body },
}
},
};
@@ -396,26 +394,18 @@ WithItem: ast::WithItem = {
};
FuncDef: ast::Statement = {
<d:Decorator*> <location:@L> <is_async:"async"?> "def" <i:Identifier> <a:Parameters> <r:("->" Test)?> ":" <s:Suite> => {
<d:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <a:Parameters> <r:("->" Test)?> ":" <body:Suite> => {
let is_async = is_async.is_some();
ast::Statement {
location,
node: if is_async.is_some() {
ast::StatementType::AsyncFunctionDef {
name: i,
args: a,
body: s,
decorator_list: d,
returns: r.map(|x| x.1),
}
} else {
ast::StatementType::FunctionDef {
name: i,
args: a,
body: s,
decorator_list: d,
returns: r.map(|x| x.1),
}
}
node: ast::StatementType::FunctionDef {
is_async,
name,
args: a,
body,
decorator_list: d,
returns: r.map(|x| x.1),
}
}
},
};

97
vm/src/stdlib/ast.rs
View File

@@ -82,31 +82,31 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::Statement) -> PyResult
decorator_list => expressions_to_ast(vm, decorator_list)?,
}),
FunctionDef {
is_async,
name,
args,
body,
decorator_list,
returns,
} => node!(vm, FunctionDef, {
name => vm.ctx.new_str(name.to_string()),
args => parameters_to_ast(vm, args)?,
body => statements_to_ast(vm, body)?,
decorator_list => expressions_to_ast(vm, decorator_list)?,
returns => optional_expression_to_ast(vm, returns)?
}),
AsyncFunctionDef {
name,
args,
body,
decorator_list,
returns,
} => node!(vm, AsyncFunctionDef, {
name => vm.ctx.new_str(name.to_string()),
args => parameters_to_ast(vm, args)?,
body => statements_to_ast(vm, body)?,
decorator_list => expressions_to_ast(vm, decorator_list)?,
returns => optional_expression_to_ast(vm, returns)?
}),
} => {
if *is_async {
node!(vm, AsyncFunctionDef, {
name => vm.ctx.new_str(name.to_string()),
args => parameters_to_ast(vm, args)?,
body => statements_to_ast(vm, body)?,
decorator_list => expressions_to_ast(vm, decorator_list)?,
returns => optional_expression_to_ast(vm, returns)?
})
} else {
node!(vm, FunctionDef, {
name => vm.ctx.new_str(name.to_string()),
args => parameters_to_ast(vm, args)?,
body => statements_to_ast(vm, body)?,
decorator_list => expressions_to_ast(vm, decorator_list)?,
returns => optional_expression_to_ast(vm, returns)?
})
}
}
Continue => node!(vm, Continue),
Break => node!(vm, Break),
Pass => node!(vm, Pass),
@@ -131,36 +131,50 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::Statement) -> PyResult
orelse => optional_statements_to_ast(vm, orelse)?
}),
For {
is_async,
target,
iter,
body,
orelse,
} => node!(vm, For, {
target => expression_to_ast(vm, target)?,
iter => expression_to_ast(vm, iter)?,
body => statements_to_ast(vm, body)?,
orelse => optional_statements_to_ast(vm, orelse)?
}),
AsyncFor {
target,
iter,
body,
orelse,
} => node!(vm, AsyncFor, {
target => expression_to_ast(vm, target)?,
iter => expression_to_ast(vm, iter)?,
body => statements_to_ast(vm, body)?,
orelse => optional_statements_to_ast(vm, orelse)?
}),
} => {
if *is_async {
node!(vm, AsyncFor, {
target => expression_to_ast(vm, target)?,
iter => expression_to_ast(vm, iter)?,
body => statements_to_ast(vm, body)?,
orelse => optional_statements_to_ast(vm, orelse)?
})
} else {
node!(vm, For, {
target => expression_to_ast(vm, target)?,
iter => expression_to_ast(vm, iter)?,
body => statements_to_ast(vm, body)?,
orelse => optional_statements_to_ast(vm, orelse)?
})
}
}
While { test, body, orelse } => node!(vm, While, {
test => expression_to_ast(vm, test)?,
body => statements_to_ast(vm, body)?,
orelse => optional_statements_to_ast(vm, orelse)?
}),
With { items, body } => node!(vm, With, {
items => map_ast(with_item_to_ast, vm, items)?,
body => statements_to_ast(vm, body)?
}),
With {
is_async,
items,
body,
} => {
if *is_async {
node!(vm, AsyncWith, {
items => map_ast(with_item_to_ast, vm, items)?,
body => statements_to_ast(vm, body)?
})
} else {
node!(vm, With, {
items => map_ast(with_item_to_ast, vm, items)?,
body => statements_to_ast(vm, body)?
})
}
}
Try {
body,
handlers,
@@ -597,6 +611,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
"AugAssign" => py_class!(ctx, "AugAssign", ast_base.clone(), {}),
"AsyncFor" => py_class!(ctx, "AsyncFor", ast_base.clone(), {}),
"AsyncFunctionDef" => py_class!(ctx, "AsyncFunctionDef", ast_base.clone(), {}),
"AsyncWith" => py_class!(ctx, "AsyncWith", ast_base.clone(), {}),
"Assert" => py_class!(ctx, "Assert", ast_base.clone(), {}),
"Attribute" => py_class!(ctx, "Attribute", ast_base.clone(), {}),
"Await" => py_class!(ctx, "Await", ast_base.clone(), {}),