Rust-related/RustPython
1
0
Fork 1
mirror of https://github.com/RustPython/RustPython.git synced 2026-06-02 19:39:49 +09:00
Code Issues Packages Projects Releases Wiki Activity

Minor tweak on lexer to fix the REPL.

Browse Source
This commit is contained in:
Windel Bouwman
2019-07-21 22:11:37 +02:00
parent c7ffa2e8b0
commit 255b744217
1 changed files with 21 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
parser/src/lexer.rs
View File

@@ -636,6 +636,22 @@ where
}
}
/// This is the main entry point. Call this function to retrieve the next token.
/// This function is used by the iterator implementation.
fn inner_next(&mut self) -> LexResult {
// top loop, keep on processing, until we have something pending.
while self.pending.is_empty() {
// Detect indentation levels
if self.at_begin_of_line {
self.handle_indentations()?;
}
self.consume_normal()?;
}
Ok(self.pending.remove(0))
}
/// Given we are at the start of a line, count the number of spaces and/or tabs until the first character.
fn eat_indentation(&mut self) -> Result<IndentationLevel, LexicalError> {
// Determine indentation:
@@ -684,7 +700,11 @@ where
spaces = 0;
tabs = 0;
}
None => {
break;
}
_ => {
self.at_begin_of_line = false;
break;
}
}
@@ -693,23 +713,6 @@ where
Ok(IndentationLevel { spaces, tabs })
}
/// This is the main entry point. Call this function to retrieve the next token.
/// This function is used by the iterator implementation.
fn inner_next(&mut self) -> LexResult {
// top loop, keep on processing, until we have something pending.
while self.pending.is_empty() {
// Detect indentation levels
if self.at_begin_of_line {
self.at_begin_of_line = false;
self.handle_indentations()?;
}
self.consume_normal()?;
}
Ok(self.pending.remove(0))
}
fn handle_indentations(&mut self) -> Result<(), LexicalError> {
let indentation_level = self.eat_indentation()?;
@@ -756,7 +759,7 @@ where
location: self.get_pos(),
});
}
};
}
}
}
}
@@ -1456,7 +1459,6 @@ mod tests {
Tok::Int { value: BigInt::from(99) },
Tok::Newline,
Tok::Dedent,
Tok::Newline,
]
);
}
@@ -1501,7 +1503,6 @@ mod tests {
Tok::Newline,
Tok::Dedent,
Tok::Dedent,
Tok::Newline,
]
);
}
@@ -1540,7 +1541,6 @@ mod tests {
Tok::Newline,
Tok::Dedent,
Tok::Dedent,
Tok::Newline,
]
);
}
@@ -1580,7 +1580,6 @@ mod tests {
Tok::Int { value: BigInt::from(2) },
Tok::Rsqb,
Tok::Newline,
Tok::Newline,
]
);
}