Merge pull request #1469 from RustPython/coolreader18/tabcomplete

Add tab autocompletion to the REPL

src/main.rs

@@ -5,23 +5,24 @@ extern crate env_logger;
 extern crate log;
 
 use clap::{App, AppSettings, Arg, ArgMatches};
-use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType};
-use rustpython_parser::error::ParseErrorType;
+use rustpython_compiler::compile;
 use rustpython_vm::{
     import, match_class,
     obj::{objint::PyInt, objtuple::PyTuple, objtype},
     print_exception,
-    pyobject::{ItemProtocol, PyObjectRef, PyResult},
+    pyobject::{ItemProtocol, PyResult},
     scope::Scope,
     util, PySettings, VirtualMachine,
 };
-use std::convert::TryInto;
 
+use std::convert::TryInto;
 use std::env;
 use std::path::PathBuf;
 use std::process;
 use std::str::FromStr;
 
+mod shell;
+
 fn main() {
     #[cfg(feature = "flame-it")]
     let main_guard = flame::start_guard("RustPython main");
@@ -365,7 +366,7 @@ fn run_rustpython(vm: &VirtualMachine, matches: &ArgMatches) -> PyResult<()> {
     } else if let Some(filename) = matches.value_of("script") {
         run_script(&vm, scope, filename)?
     } else {
-        run_shell(&vm, scope)?;
+        shell::run_shell(&vm, scope)?;
     }
 
     Ok(())
@@ -457,220 +458,3 @@ fn test_run_script() {
     let r = run_script(&vm, vm.new_scope_with_builtins(), "tests/snippets/dir_main");
     assert!(r.is_ok());
 }
-
-enum ShellExecResult {
-    Ok,
-    PyErr(PyObjectRef),
-    Continue,
-}
-
-fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> ShellExecResult {
-    match vm.compile(source, compile::Mode::Single, "<stdin>".to_string()) {
-        Ok(code) => {
-            match vm.run_code_obj(code, scope.clone()) {
-                Ok(value) => {
-                    // Save non-None values as "_"
-                    if !vm.is_none(&value) {
-                        let key = "_";
-                        scope.globals.set_item(key, value, vm).unwrap();
-                    }
-                    ShellExecResult::Ok
-                }
-                Err(err) => ShellExecResult::PyErr(err),
-            }
-        }
-        Err(CompileError {
-            error: CompileErrorType::Parse(ParseErrorType::EOF),
-            ..
-        }) => ShellExecResult::Continue,
-        Err(err) => ShellExecResult::PyErr(vm.new_syntax_error(&err)),
-    }
-}
-
-#[cfg(not(target_os = "wasi"))]
-fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
-    use rustyline::{error::ReadlineError, Editor};
-
-    println!(
-        "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}",
-        crate_version!()
-    );
-
-    // Read a single line:
-    let mut input = String::new();
-    let mut repl = Editor::<()>::new();
-
-    // Retrieve a `history_path_str` dependent on the OS
-    let repl_history_path = match dirs::config_dir() {
-        Some(mut path) => {
-            path.push("rustpython");
-            path.push("repl_history.txt");
-            path
-        }
-        None => ".repl_history.txt".into(),
-    };
-
-    if !repl_history_path.exists() {
-        if let Some(parent) = repl_history_path.parent() {
-            std::fs::create_dir_all(parent).unwrap();
-        }
-    }
-
-    if repl.load_history(&repl_history_path).is_err() {
-        println!("No previous history.");
-    }
-
-    let mut continuing = false;
-
-    loop {
-        let prompt_name = if continuing { "ps2" } else { "ps1" };
-        let prompt = vm
-            .get_attribute(vm.sys_module.clone(), prompt_name)
-            .and_then(|prompt| vm.to_str(&prompt));
-        let prompt = match prompt {
-            Ok(ref s) => s.as_str(),
-            Err(_) => "",
-        };
-        let result = match repl.readline(prompt) {
-            Ok(line) => {
-                debug!("You entered {:?}", line);
-
-                repl.add_history_entry(line.trim_end());
-
-                let stop_continuing = line.is_empty();
-
-                if input.is_empty() {
-                    input = line;
-                } else {
-                    input.push_str(&line);
-                }
-                input.push_str("\n");
-
-                if continuing {
-                    if stop_continuing {
-                        continuing = false;
-                    } else {
-                        continue;
-                    }
-                }
-
-                match shell_exec(vm, &input, scope.clone()) {
-                    ShellExecResult::Ok => {
-                        input.clear();
-                        Ok(())
-                    }
-                    ShellExecResult::Continue => {
-                        continuing = true;
-                        Ok(())
-                    }
-                    ShellExecResult::PyErr(err) => {
-                        input.clear();
-                        Err(err)
-                    }
-                }
-            }
-            Err(ReadlineError::Interrupted) => {
-                continuing = false;
-                input.clear();
-                let keyboard_interrupt = vm
-                    .new_empty_exception(vm.ctx.exceptions.keyboard_interrupt.clone())
-                    .unwrap();
-                Err(keyboard_interrupt)
-            }
-            Err(ReadlineError::Eof) => {
-                break;
-            }
-            Err(err) => {
-                eprintln!("Readline error: {:?}", err);
-                break;
-            }
-        };
-
-        if let Err(exc) = result {
-            if objtype::isinstance(&exc, &vm.ctx.exceptions.system_exit) {
-                repl.save_history(&repl_history_path).unwrap();
-                return Err(exc);
-            }
-            print_exception(vm, &exc);
-        }
-    }
-    repl.save_history(&repl_history_path).unwrap();
-
-    Ok(())
-}
-
-#[cfg(target_os = "wasi")]
-fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
-    use std::io::prelude::*;
-    use std::io::{self, BufRead};
-
-    println!(
-        "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}",
-        crate_version!()
-    );
-
-    // Read a single line:
-    let mut input = String::new();
-    let mut continuing = false;
-
-    loop {
-        let prompt_name = if continuing { "ps2" } else { "ps1" };
-        let prompt = vm
-            .get_attribute(vm.sys_module.clone(), prompt_name)
-            .and_then(|prompt| vm.to_str(&prompt));
-        let prompt = match prompt {
-            Ok(ref s) => s.as_str(),
-            Err(_) => "",
-        };
-        print!("{}", prompt);
-        io::stdout().flush().ok().expect("Could not flush stdout");
-
-        let stdin = io::stdin();
-
-        let result = match stdin.lock().lines().next().unwrap() {
-            Ok(line) => {
-                debug!("You entered {:?}", line);
-                let stop_continuing = line.is_empty();
-
-                if input.is_empty() {
-                    input = line;
-                } else {
-                    input.push_str(&line);
-                }
-                input.push_str("\n");
-
-                if continuing {
-                    if stop_continuing {
-                        continuing = false;
-                    } else {
-                        continue;
-                    }
-                }
-
-                match shell_exec(vm, &input, scope.clone()) {
-                    ShellExecResult::Ok => {
-                        input.clear();
-                        Ok(())
-                    }
-                    ShellExecResult::Continue => {
-                        continuing = true;
-                        Ok(())
-                    }
-                    ShellExecResult::PyErr(err) => {
-                        input.clear();
-                        Err(err)
-                    }
-                }
-            }
-            Err(err) => {
-                eprintln!("Readline error: {:?}", err);
-                break;
-            }
-        };
-
-        if let Err(exc) = result {
-            print_exception(vm, &exc);
-        }
-    }
-    Ok(())
-}

src/shell.rs

@@ -0,0 +1,260 @@
+#[cfg(not(target_os = "wasi"))]
+mod rustyline_helper;
+
+use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType};
+use rustpython_parser::error::ParseErrorType;
+use rustpython_vm::{
+    obj::objtype,
+    print_exception,
+    pyobject::{ItemProtocol, PyObjectRef, PyResult},
+    scope::Scope,
+    VirtualMachine,
+};
+#[cfg(not(target_os = "wasi"))]
+use rustyline_helper::ShellHelper;
+
+use std::io;
+use std::path::Path;
+
+enum ShellExecResult {
+    Ok,
+    PyErr(PyObjectRef),
+    Continue,
+}
+
+fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> ShellExecResult {
+    match vm.compile(source, compile::Mode::Single, "<stdin>".to_string()) {
+        Ok(code) => {
+            match vm.run_code_obj(code, scope.clone()) {
+                Ok(value) => {
+                    // Save non-None values as "_"
+                    if !vm.is_none(&value) {
+                        let key = "_";
+                        scope.globals.set_item(key, value, vm).unwrap();
+                    }
+                    ShellExecResult::Ok
+                }
+                Err(err) => ShellExecResult::PyErr(err),
+            }
+        }
+        Err(CompileError {
+            error: CompileErrorType::Parse(ParseErrorType::EOF),
+            ..
+        }) => ShellExecResult::Continue,
+        Err(err) => ShellExecResult::PyErr(vm.new_syntax_error(&err)),
+    }
+}
+
+enum ReadlineResult {
+    Line(String),
+    EOF,
+    Interrupt,
+    IO(std::io::Error),
+    EncodingError,
+    Other(Box<dyn std::error::Error>),
+}
+
+#[allow(unused)]
+struct BasicReadline;
+
+#[allow(unused)]
+impl BasicReadline {
+    fn new(_vm: &VirtualMachine, _scope: Scope) -> Self {
+        BasicReadline
+    }
+
+    fn load_history(&mut self, _path: &Path) -> io::Result<()> {
+        Ok(())
+    }
+
+    fn save_history(&mut self, _path: &Path) -> io::Result<()> {
+        Ok(())
+    }
+
+    fn add_history_entry(&mut self, _entry: &str) {}
+
+    fn readline(&mut self, prompt: &str) -> ReadlineResult {
+        use std::io::prelude::*;
+        print!("{}", prompt);
+        if let Err(e) = io::stdout().flush() {
+            return ReadlineResult::IO(e);
+        }
+
+        match io::stdin().lock().lines().next() {
+            Some(Ok(line)) => ReadlineResult::Line(line),
+            None => ReadlineResult::EOF,
+            Some(Err(e)) => match e.kind() {
+                io::ErrorKind::Interrupted => ReadlineResult::Interrupt,
+                io::ErrorKind::InvalidData => ReadlineResult::EncodingError,
+                _ => ReadlineResult::IO(e),
+            },
+        }
+    }
+}
+
+#[cfg(target_os = "wasi")]
+type Readline = BasicReadline;
+
+#[cfg(not(target_os = "wasi"))]
+struct RustylineReadline<'vm> {
+    repl: rustyline::Editor<ShellHelper<'vm>>,
+}
+#[cfg(not(target_os = "wasi"))]
+impl<'vm> RustylineReadline<'vm> {
+    fn new(vm: &'vm VirtualMachine, scope: Scope) -> Self {
+        use rustyline::{CompletionType, Config, Editor};
+        let mut repl = Editor::with_config(
+            Config::builder()
+                .completion_type(CompletionType::List)
+                .tab_stop(4)
+                .build(),
+        );
+        repl.set_helper(Some(ShellHelper::new(vm, scope)));
+        RustylineReadline { repl }
+    }
+
+    fn load_history(&mut self, path: &Path) -> rustyline::Result<()> {
+        self.repl.load_history(path)
+    }
+
+    fn save_history(&mut self, path: &Path) -> rustyline::Result<()> {
+        if !path.exists() {
+            if let Some(parent) = path.parent() {
+                std::fs::create_dir_all(parent)?;
+            }
+        }
+        self.repl.save_history(path)
+    }
+
+    fn add_history_entry(&mut self, entry: &str) {
+        self.repl.add_history_entry(entry);
+    }
+
+    fn readline(&mut self, prompt: &str) -> ReadlineResult {
+        use rustyline::error::ReadlineError;
+        match self.repl.readline(prompt) {
+            Ok(line) => ReadlineResult::Line(line),
+            Err(ReadlineError::Interrupted) => ReadlineResult::Interrupt,
+            Err(ReadlineError::Eof) => ReadlineResult::EOF,
+            Err(ReadlineError::Io(e)) => ReadlineResult::IO(e),
+            #[cfg(unix)]
+            Err(ReadlineError::Utf8Error) => ReadlineResult::EncodingError,
+            #[cfg(windows)]
+            Err(ReadlineError::Decode(_)) => ReadlineResult::EncodingError,
+            Err(e) => ReadlineResult::Other(e.into()),
+        }
+    }
+}
+
+#[cfg(not(target_os = "wasi"))]
+type Readline<'a> = RustylineReadline<'a>;
+
+pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
+    println!(
+        "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}",
+        crate_version!()
+    );
+
+    let mut repl = Readline::new(vm, scope.clone());
+    let mut full_input = String::new();
+
+    // Retrieve a `history_path_str` dependent on the OS
+    let repl_history_path = match dirs::config_dir() {
+        Some(mut path) => {
+            path.push("rustpython");
+            path.push("repl_history.txt");
+            path
+        }
+        None => ".repl_history.txt".into(),
+    };
+
+    if repl.load_history(&repl_history_path).is_err() {
+        println!("No previous history.");
+    }
+
+    let mut continuing = false;
+
+    loop {
+        let prompt_name = if continuing { "ps2" } else { "ps1" };
+        let prompt = vm
+            .get_attribute(vm.sys_module.clone(), prompt_name)
+            .and_then(|prompt| vm.to_str(&prompt));
+        let prompt = match prompt {
+            Ok(ref s) => s.as_str(),
+            Err(_) => "",
+        };
+        let result = match repl.readline(prompt) {
+            ReadlineResult::Line(line) => {
+                debug!("You entered {:?}", line);
+
+                repl.add_history_entry(line.trim_end());
+
+                let stop_continuing = line.is_empty();
+
+                if full_input.is_empty() {
+                    full_input = line;
+                } else {
+                    full_input.push_str(&line);
+                }
+                full_input.push_str("\n");
+
+                if continuing {
+                    if stop_continuing {
+                        continuing = false;
+                    } else {
+                        continue;
+                    }
+                }
+
+                match shell_exec(vm, &full_input, scope.clone()) {
+                    ShellExecResult::Ok => {
+                        full_input.clear();
+                        Ok(())
+                    }
+                    ShellExecResult::Continue => {
+                        continuing = true;
+                        Ok(())
+                    }
+                    ShellExecResult::PyErr(err) => {
+                        full_input.clear();
+                        Err(err)
+                    }
+                }
+            }
+            ReadlineResult::Interrupt => {
+                continuing = false;
+                full_input.clear();
+                let keyboard_interrupt = vm
+                    .new_empty_exception(vm.ctx.exceptions.keyboard_interrupt.clone())
+                    .unwrap();
+                Err(keyboard_interrupt)
+            }
+            ReadlineResult::EOF => {
+                break;
+            }
+            ReadlineResult::EncodingError => {
+                eprintln!("Invalid UTF-8 entered");
+                Ok(())
+            }
+            ReadlineResult::Other(err) => {
+                eprintln!("Readline error: {:?}", err);
+                break;
+            }
+            ReadlineResult::IO(err) => {
+                eprintln!("IO error: {:?}", err);
+                break;
+            }
+        };
+
+        if let Err(exc) = result {
+            if objtype::isinstance(&exc, &vm.ctx.exceptions.system_exit) {
+                repl.save_history(&repl_history_path).unwrap();
+                return Err(exc);
+            }
+            print_exception(vm, &exc);
+        }
+    }
+    repl.save_history(&repl_history_path).unwrap();
+
+    Ok(())
+}

src/shell/rustyline_helper.rs

@@ -0,0 +1,168 @@
+use rustpython_vm::obj::objstr::PyStringRef;
+use rustpython_vm::pyobject::{PyIterable, PyResult, TryFromObject};
+use rustpython_vm::scope::{NameProtocol, Scope};
+use rustpython_vm::VirtualMachine;
+use rustyline::{completion::Completer, highlight::Highlighter, hint::Hinter, Context, Helper};
+
+pub struct ShellHelper<'vm> {
+    vm: &'vm VirtualMachine,
+    scope: Scope,
+}
+
+fn reverse_string(s: &mut String) {
+    let rev = s.chars().rev().collect();
+    *s = rev;
+}
+
+fn split_idents_on_dot(line: &str) -> Option<(usize, Vec<String>)> {
+    let mut words = vec![String::new()];
+    let mut startpos = 0;
+    for (i, c) in line.chars().rev().enumerate() {
+        match c {
+            '.' => {
+                // check for a double dot
+                if i != 0 && words.last().map_or(false, |s| s.is_empty()) {
+                    return None;
+                }
+                reverse_string(words.last_mut().unwrap());
+                if words.len() == 1 {
+                    startpos = line.len() - i;
+                }
+                words.push(String::new());
+            }
+            c if c.is_alphanumeric() || c == '_' => words.last_mut().unwrap().push(c),
+            _ => {
+                if words.len() == 1 {
+                    if words.last().unwrap().is_empty() {
+                        return None;
+                    }
+                    startpos = line.len() - i;
+                }
+                break;
+            }
+        }
+    }
+    if words == [String::new()] {
+        return None;
+    }
+    reverse_string(words.last_mut().unwrap());
+    words.reverse();
+
+    Some((startpos, words))
+}
+
+impl<'vm> ShellHelper<'vm> {
+    pub fn new(vm: &'vm VirtualMachine, scope: Scope) -> Self {
+        ShellHelper { vm, scope }
+    }
+
+    #[allow(clippy::type_complexity)]
+    fn get_available_completions<'w>(
+        &self,
+        words: &'w [String],
+    ) -> Option<(
+        &'w str,
+        Box<dyn Iterator<Item = PyResult<PyStringRef>> + 'vm>,
+    )> {
+        // the very first word and then all the ones after the dot
+        let (first, rest) = words.split_first().unwrap();
+
+        let str_iter_method = |obj, name| {
+            let iter = self.vm.call_method(obj, name, vec![])?;
+            PyIterable::<PyStringRef>::try_from_object(self.vm, iter)?.iter(self.vm)
+        };
+
+        if let Some((last, parents)) = rest.split_last() {
+            // we need to get an attribute based off of the dir() of an object
+
+            // last: the last word, could be empty if it ends with a dot
+            // parents: the words before the dot
+
+            let mut current = self.scope.load_global(self.vm, first)?;
+
+            for attr in parents {
+                current = self.vm.get_attribute(current.clone(), attr.as_str()).ok()?;
+            }
+
+            let current_iter = str_iter_method(&current, "__dir__").ok()?;
+
+            Some((&last, Box::new(current_iter) as _))
+        } else {
+            // we need to get a variable based off of globals/builtins
+
+            let globals = str_iter_method(self.scope.globals.as_object(), "keys").ok()?;
+            let builtins = str_iter_method(&self.vm.builtins, "__dir__").ok()?;
+            Some((&first, Box::new(Iterator::chain(globals, builtins)) as _))
+        }
+    }
+
+    fn complete_opt(&self, line: &str) -> Option<(usize, Vec<String>)> {
+        let (startpos, words) = split_idents_on_dot(line)?;
+
+        let (word_start, iter) = self.get_available_completions(&words)?;
+
+        let all_completions = iter
+            .filter(|res| {
+                res.as_ref()
+                    .ok()
+                    .map_or(true, |s| s.as_str().starts_with(word_start))
+            })
+            .collect::<Result<Vec<_>, _>>()
+            .ok()?;
+        let mut completions = if word_start.starts_with('_') {
+            // if they're already looking for something starting with a '_', just give
+            // them all the completions
+            all_completions
+        } else {
+            // only the completions that don't start with a '_'
+            let no_underscore = all_completions
+                .iter()
+                .cloned()
+                .filter(|s| !s.as_str().starts_with('_'))
+                .collect::<Vec<_>>();
+
+            // if there are only completions that start with a '_', give them all of the
+            // completions, otherwise only the ones that don't start with '_'
+            if no_underscore.is_empty() {
+                all_completions
+            } else {
+                no_underscore
+            }
+        };
+
+        // sort the completions alphabetically
+        completions.sort_by(|a, b| std::cmp::Ord::cmp(a.as_str(), b.as_str()));
+
+        Some((
+            startpos,
+            completions
+                .into_iter()
+                .map(|s| s.as_str().to_owned())
+                .collect(),
+        ))
+    }
+}
+
+impl Completer for ShellHelper<'_> {
+    type Candidate = String;
+
+    fn complete(
+        &self,
+        line: &str,
+        pos: usize,
+        _ctx: &Context,
+    ) -> rustyline::Result<(usize, Vec<String>)> {
+        if pos != line.len() {
+            return Ok((0, vec![]));
+        }
+        Ok(self
+            .complete_opt(line)
+            // as far as I can tell, there's no better way to do both completion
+            // and indentation (or even just indentation)
+            .unwrap_or_else(|| (line.len(), vec!["    ".to_string()])))
+    }
+}
+
+impl Hinter for ShellHelper<'_> {}
+impl Highlighter for ShellHelper<'_> {}
+impl Helper for ShellHelper<'_> {}

wapm.toml

@@ -1,20 +1,17 @@
 [package]
-name="rustpython"
-version="0.0.4"
-description="A Python-3 (CPython >= 3.5.0) Interpreter written in Rust 🐍 😱 🤘"
-license-file="LICENSE"
+name = "rustpython"
+version = "0.1.1"
+description = "A Python-3 (CPython >= 3.5.0) Interpreter written in Rust 🐍 😱 🤘"
+license-file = "LICENSE"
 readme = "README.md"
 repository = "https://github.com/RustPython/RustPython"
 
 [[module]]
-name="rustpython"
-source="target/wasm32-wasi/release/rustpython.wasm"
+name = "rustpython"
+source = "target/wasm32-wasi/release/rustpython.wasm"
 abi = "wasi"
-interfaces = {wasi= "0.0.0-unstable"}
+interfaces = { wasi = "0.0.0-unstable" }
 
 [[command]]
-name="rustpython"
-module="rustpython"
-
-# [fs]
-# "Lib"="Lib"
+name = "rustpython"
+module = "rustpython"