From 71de630336c03ab45c660b21b352c61641abb433 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Mon, 17 Oct 2022 09:35:03 -0400
Subject: [PATCH 1/2] Expose a method to parse AST from tokens directly

---
 compiler/parser/src/parser.rs | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/compiler/parser/src/parser.rs b/compiler/parser/src/parser.rs
index 49d2e53f6..9531eb3a0 100644
--- a/compiler/parser/src/parser.rs
+++ b/compiler/parser/src/parser.rs
@@ -5,9 +5,11 @@
 //! parse a whole program, a single statement, or a single
 //! expression.
 
+use std::iter;
+
+use crate::lexer::LexResult;
 pub use crate::mode::Mode;
 use crate::{ast, error::ParseError, lexer, python};
-use std::iter;
 
 /*
  * Parse python code.
@@ -23,6 +25,17 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
     })
 }
 
+/// Parse the token stream for a full python program.
+pub fn parse_program_tokens(
+    lxr: impl IntoIterator<Item=LexResult>,
+    source_path: &str,
+) -> Result<ast::Suite, ParseError> {
+    parse_tokens(lxr, Mode::Module, source_path).map(|top| match top {
+        ast::Mod::Module { body, .. } => body,
+        _ => unreachable!(),
+    })
+}
+
 /// Parses a python expression
 ///
 /// # Example
@@ -80,6 +93,20 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
         .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
 }
 
+// Parse a given token iterator.
+pub fn parse_tokens(
+    lxr: impl IntoIterator<Item=LexResult>,
+    mode: Mode,
+    source_path: &str,
+) -> Result<ast::Mod, ParseError> {
+    let marker_token = (Default::default(), mode.to_marker(), Default::default());
+    let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
+
+    python::TopParser::new()
+        .parse(tokenizer)
+        .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

From 9ced976cde6bf427b83fa805933dfd5dbb4491e0 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Mon, 17 Oct 2022 11:59:22 -0400
Subject: [PATCH 2/2] Remove parse_program_tokens

---
 compiler/parser/src/parser.rs | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/compiler/parser/src/parser.rs b/compiler/parser/src/parser.rs
index 9531eb3a0..90903cf24 100644
--- a/compiler/parser/src/parser.rs
+++ b/compiler/parser/src/parser.rs
@@ -5,11 +5,10 @@
 //! parse a whole program, a single statement, or a single
 //! expression.
 
-use std::iter;
-
 use crate::lexer::LexResult;
 pub use crate::mode::Mode;
 use crate::{ast, error::ParseError, lexer, python};
+use std::iter;
 
 /*
  * Parse python code.
@@ -25,17 +24,6 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
     })
 }
 
-/// Parse the token stream for a full python program.
-pub fn parse_program_tokens(
-    lxr: impl IntoIterator<Item=LexResult>,
-    source_path: &str,
-) -> Result<ast::Suite, ParseError> {
-    parse_tokens(lxr, Mode::Module, source_path).map(|top| match top {
-        ast::Mod::Module { body, .. } => body,
-        _ => unreachable!(),
-    })
-}
-
 /// Parses a python expression
 ///
 /// # Example
@@ -95,7 +83,7 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
 
 // Parse a given token iterator.
 pub fn parse_tokens(
-    lxr: impl IntoIterator<Item=LexResult>,
+    lxr: impl IntoIterator<Item = LexResult>,
     mode: Mode,
     source_path: &str,
 ) -> Result<ast::Mod, ParseError> {