Merge branch 'master' of github.com:RustPython/RustPython into update-docs

.travis.yml

@@ -1,17 +1,18 @@
-language: rust
-rust: stable
-cache: cargo
 
 matrix:
   fast_finish: true
   include:
-    - name: rust unittests and doctests
+    - name: Run rust tests
       language: rust
       rust: stable
       cache: cargo
       script:
         - cargo build --verbose --all
         - cargo test --verbose --all
+      env:
+        # Prevention of cache corruption.
+        # See: https://docs.travis-ci.com/user/caching/#caches-and-build-matrices
+        - JOBCACHE=1
 
     # To test the snippets, we use Travis' Python environment (because
     # installing rust ourselves is a lot easier than installing Python)
@@ -19,29 +20,25 @@ matrix:
       language: python
       python: 3.6
       cache:
-        pip: true
-        # Because we're using the Python Travis environment, we can't use
-        # the built-in cargo cacher
-        directories:
-          - /home/travis/.cargo
-          - target
+        - pip
+        - cargo
       env:
+        - JOBCACHE=2
         - TRAVIS_RUST_VERSION=stable
         - CODE_COVERAGE=false
       script: tests/.travis-runner.sh
-    - name: rustfmt
+
+    - name: Check rust code style with rustfmt
       language: rust
       rust: stable
       cache: cargo
       before_script:
-        - rustup component add rustfmt-preview
+        - rustup component add rustfmt
       script:
-        # Code references the generated python.rs, so put something in
-        # place to make `cargo fmt` happy.  (We use `echo` rather than
-        # `touch` because rustfmt complains about the empty file touch
-        # creates.)
-        - echo > parser/src/python.rs
         - cargo fmt --all -- --check
+      env:
+        - JOBCACHE=3
+
     - name: publish documentation
       language: rust
       rust: stable
@@ -50,7 +47,17 @@ matrix:
         - cargo doc --no-deps --all
       if: branch = release
       env:
-        - DEPLOY_DOC=true
+        - JOBCACHE=4
+      deploy:
+        - provider: pages
+          repo: RustPython/website
+          target-branch: master
+          local-dir: target/doc
+          skip-cleanup: true
+          # Set in the settings page of your repository, as a secure variable
+          github-token: $WEBSITE_GITHUB_TOKEN
+          keep-history: true
+
     - name: WASM online demo
       language: rust
       rust: stable
@@ -65,42 +72,38 @@ matrix:
         - npm run dist
       if: branch = release
       env:
-        - DEPLOY_DEMO=true
-    - name: cargo-clippy
-      language: rust
-      rust: stable
-      cache: cargo
-      before_script:
-        - rustup component add clippy
-      script:
-        - cargo clippy
+        - JOBCACHE=5
+      deploy:
+        - provider: pages
+          repo: RustPython/demo
+          target-branch: master
+          local-dir: wasm/demo/dist
+          skip-cleanup: true
+          # Set in the settings page of your repository, as a secure variable
+          github-token: $WEBSITE_GITHUB_TOKEN
+          keep-history: true
+
     - name: Code Coverage
       language: python
       python: 3.6
       cache:
-        pip: true
-        # Because we're using the Python Travis environment, we can't use
-        # the built-in cargo cacher
-        directories:
-          - /home/travis/.cargo
-          - target
+        - pip
+        - cargo
       script:
         - tests/.travis-runner.sh
       # Only do code coverage on master via a cron job.
       if: branch = master AND type = cron
       env:
+        - JOBCACHE=6
         - TRAVIS_RUST_VERSION=nightly
         - CODE_COVERAGE=true
+
     - name: test WASM
       language: python
       python: 3.6
       cache:
-        pip: true
-        # Because we're using the Python Travis environment, we can't use
-        # the built-in cargo cacher
-        directories:
-          - /home/travis/.cargo
-          - target
+        - pip
+        - cargo
       addons:
         firefox: latest
       install:
@@ -109,30 +112,5 @@ matrix:
       script:
         - wasm/tests/.travis-runner.sh
       env:
+        - JOBCACHE=7
         - TRAVIS_RUST_VERSION=stable
-  allow_failures:
-    - name: cargo-clippy
-
-deploy:
-  - provider: pages
-    repo: RustPython/website
-    target-branch: master
-    local-dir: target/doc
-    skip-cleanup: true
-    # Set in the settings page of your repository, as a secure variable
-    github-token: $WEBSITE_GITHUB_TOKEN
-    keep-history: true
-    on:
-      branch: release
-      condition: $DEPLOY_DOC = true
-  - provider: pages
-    repo: RustPython/demo
-    target-branch: master
-    local-dir: wasm/demo/dist
-    skip-cleanup: true
-    # Set in the settings page of your repository, as a secure variable
-    github-token: $WEBSITE_GITHUB_TOKEN
-    keep-history: true
-    on:
-      branch: release
-      condition: $DEPLOY_DEMO = true

azure-pipelines.yml

@@ -8,8 +8,8 @@ jobs:
     vmImage: 'vs2017-win2016'
   strategy:
     matrix:
-      Python37:
-        python.version: '3.7'
+      Python36:
+        python.version: '3.6'
     maxParallel: 10
 
   steps:

compiler/src/error.rs

@@ -1,4 +1,4 @@
-use rustpython_parser::error::ParseError;
+use rustpython_parser::error::{ParseError, ParseErrorType};
 use rustpython_parser::lexer::Location;
 
 use std::error::Error;
@@ -13,8 +13,8 @@ pub struct CompileError {
 impl From<ParseError> for CompileError {
     fn from(error: ParseError) -> Self {
         CompileError {
-            error: CompileErrorType::Parse(error),
-            location: Default::default(), // TODO: extract location from parse error!
+            error: CompileErrorType::Parse(error.error),
+            location: error.location,
         }
     }
 }
@@ -28,7 +28,7 @@ pub enum CompileErrorType {
     /// Expected an expression got a statement
     ExpectExpr,
     /// Parser error
-    Parse(ParseError),
+    Parse(ParseErrorType),
     SyntaxError(String),
     /// Multiple `*` detected
     StarArgs,
@@ -56,7 +56,7 @@ impl fmt::Display for CompileError {
         }?;
 
         // Print line number:
-        write!(f, " at line {:?}", self.location.row())
+        write!(f, " at {}", self.location)
     }
 }
 

parser/src/error.rs

@@ -3,28 +3,31 @@
 extern crate lalrpop_util;
 use self::lalrpop_util::ParseError as InnerError;
 
-use crate::lexer::{LexicalError, Location};
+use crate::lexer::{LexicalError, LexicalErrorType, Location};
 use crate::token::Tok;
 
 use std::error::Error;
 use std::fmt;
 
-// A token of type `Tok` was observed, with a span given by the two Location values
-type TokSpan = (Location, Tok, Location);
-
 /// Represents an error during parsing
 #[derive(Debug, PartialEq)]
-pub enum ParseError {
+pub struct ParseError {
+    pub error: ParseErrorType,
+    pub location: Location,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum ParseErrorType {
     /// Parser encountered an unexpected end of input
-    EOF(Option<Location>),
+    EOF,
     /// Parser encountered an extra token
-    ExtraToken(TokSpan),
+    ExtraToken(Tok),
     /// Parser encountered an invalid token
-    InvalidToken(Location),
+    InvalidToken,
     /// Parser encountered an unexpected token
-    UnrecognizedToken(TokSpan, Vec<String>),
+    UnrecognizedToken(Tok, Vec<String>),
     /// Maps to `User` type from `lalrpop-util`
-    Other,
+    Lexical(LexicalErrorType),
 }
 
 /// Convert `lalrpop_util::ParseError` to our internal type
@@ -32,15 +35,29 @@ impl From<InnerError<Location, Tok, LexicalError>> for ParseError {
     fn from(err: InnerError<Location, Tok, LexicalError>) -> Self {
         match err {
             // TODO: Are there cases where this isn't an EOF?
-            InnerError::InvalidToken { location } => ParseError::EOF(Some(location)),
-            InnerError::ExtraToken { token } => ParseError::ExtraToken(token),
-            // Inner field is a unit-like enum `LexicalError::StringError` with no useful info
-            InnerError::User { .. } => ParseError::Other,
+            InnerError::InvalidToken { location } => ParseError {
+                error: ParseErrorType::EOF,
+                location,
+            },
+            InnerError::ExtraToken { token } => ParseError {
+                error: ParseErrorType::ExtraToken(token.1),
+                location: token.0,
+            },
+            InnerError::User { error } => ParseError {
+                error: ParseErrorType::Lexical(error.error),
+                location: error.location,
+            },
             InnerError::UnrecognizedToken { token, expected } => {
                 match token {
-                    Some(tok) => ParseError::UnrecognizedToken(tok, expected),
+                    Some(tok) => ParseError {
+                        error: ParseErrorType::UnrecognizedToken(tok.1, expected),
+                        location: tok.0,
+                    },
                     // EOF was observed when it was unexpected
-                    None => ParseError::EOF(None),
+                    None => ParseError {
+                        error: ParseErrorType::EOF,
+                        location: Default::default(),
+                    },
                 }
             }
         }
@@ -48,26 +65,21 @@ impl From<InnerError<Location, Tok, LexicalError>> for ParseError {
 }
 
 impl fmt::Display for ParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{} at {}", self.error, self.location)
+    }
+}
+
+impl fmt::Display for ParseErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match *self {
-            ParseError::EOF(ref location) => {
-                if let Some(l) = location {
-                    write!(f, "Got unexpected EOF at: {:?}", l)
-                } else {
-                    write!(f, "Got unexpected EOF")
-                }
+            ParseErrorType::EOF => write!(f, "Got unexpected EOF"),
+            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {:?}", tok),
+            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
+            ParseErrorType::UnrecognizedToken(ref tok, _) => {
+                write!(f, "Got unexpected token: {:?}", tok)
             }
-            ParseError::ExtraToken(ref t_span) => {
-                write!(f, "Got extraneous token: {:?} at: {:?}", t_span.1, t_span.0)
-            }
-            ParseError::InvalidToken(ref location) => {
-                write!(f, "Got invalid token at: {:?}", location)
-            }
-            ParseError::UnrecognizedToken(ref t_span, _) => {
-                write!(f, "Got unexpected token: {:?} at {:?}", t_span.1, t_span.0)
-            }
-            // This is user defined, it probably means a more useful error should have been given upstream.
-            ParseError::Other => write!(f, "Got unsupported token(s)"),
+            ParseErrorType::Lexical(ref error) => write!(f, "{}", error),
         }
     }
 }

parser/src/lexer.rs

@@ -10,6 +10,7 @@ use num_traits::Num;
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 use std::collections::HashMap;
+use std::fmt;
 use std::str::FromStr;
 use unic_emoji_char::is_emoji_presentation;
 use unicode_xid::UnicodeXID;
@@ -59,13 +60,13 @@ pub struct Lexer<T: Iterator<Item = char>> {
     keywords: HashMap<String, Tok>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub struct LexicalError {
     pub error: LexicalErrorType,
     pub location: Location,
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub enum LexicalErrorType {
     StringError,
     UnicodeError,
@@ -74,12 +75,32 @@ pub enum LexicalErrorType {
     OtherError(String),
 }
 
+impl fmt::Display for LexicalErrorType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            LexicalErrorType::StringError => write!(f, "Got unexpected string"),
+            LexicalErrorType::UnicodeError => write!(f, "Got unexpected unicode"),
+            LexicalErrorType::NestingError => write!(f, "Got unexpected nesting"),
+            LexicalErrorType::UnrecognizedToken { tok } => {
+                write!(f, "Got unexpected token {}", tok)
+            }
+            LexicalErrorType::OtherError(ref msg) => write!(f, "{}", msg),
+        }
+    }
+}
+
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct Location {
     row: usize,
     column: usize,
 }
 
+impl fmt::Display for Location {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "line {} column {}", self.row, self.column)
+    }
+}
+
 impl Location {
     pub fn new(row: usize, column: usize) -> Self {
         Location { row, column }

src/main.rs

@@ -7,7 +7,7 @@ extern crate rustyline;
 
 use clap::{App, Arg};
 use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType};
-use rustpython_parser::error::ParseError;
+use rustpython_parser::error::ParseErrorType;
 use rustpython_vm::{
     frame::Scope,
     import,
@@ -181,7 +181,7 @@ fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), Com
         // Don't inject syntax errors for line continuation
         Err(
             err @ CompileError {
-                error: CompileErrorType::Parse(ParseError::EOF(_)),
+                error: CompileErrorType::Parse(ParseErrorType::EOF),
                 ..
             },
         ) => Err(err),
@@ -258,7 +258,7 @@ fn run_shell(vm: &VirtualMachine) -> PyResult {
 
                 match shell_exec(vm, &input, vars.clone()) {
                     Err(CompileError {
-                        error: CompileErrorType::Parse(ParseError::EOF(_)),
+                        error: CompileErrorType::Parse(ParseErrorType::EOF),
                         ..
                     }) => {
                         continuing = true;

wasm/lib/src/vm_class.rs

@@ -5,7 +5,7 @@ use std::rc::{Rc, Weak};
 use js_sys::{Object, Reflect, SyntaxError, TypeError};
 use wasm_bindgen::prelude::*;
 
-use rustpython_compiler::{compile, error::CompileErrorType};
+use rustpython_compiler::compile;
 use rustpython_vm::frame::{NameProtocol, Scope};
 use rustpython_vm::function::PyFuncArgs;
 use rustpython_vm::import;
@@ -44,7 +44,7 @@ impl StoredVirtualMachine {
             setup_browser_module(&vm);
         }
 
-        import::init_importlib(&vm, false);
+        import::init_importlib(&vm, false).unwrap();
 
         StoredVirtualMachine {
             vm,
@@ -281,30 +281,13 @@ impl WASMVirtualMachine {
                 let code = vm.compile(&source, &mode, "<wasm>".to_string());
                 let code = code.map_err(|err| {
                     let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err));
-                    if let CompileErrorType::Parse(ref parse_error) = err.error {
-                        use rustpython_parser::error::ParseError;
-                        if let ParseError::EOF(Some(ref loc))
-                        | ParseError::ExtraToken((ref loc, ..))
-                        | ParseError::InvalidToken(ref loc)
-                        | ParseError::UnrecognizedToken((ref loc, ..), _) = parse_error
-                        {
-                            let _ =
-                                Reflect::set(&js_err, &"row".into(), &(loc.row() as u32).into());
-                            let _ =
-                                Reflect::set(&js_err, &"col".into(), &(loc.column() as u32).into());
-                        }
-                        if let ParseError::ExtraToken((_, _, ref loc))
-                        | ParseError::UnrecognizedToken((_, _, ref loc), _) = parse_error
-                        {
-                            let _ =
-                                Reflect::set(&js_err, &"endrow".into(), &(loc.row() as u32).into());
-                            let _ = Reflect::set(
-                                &js_err,
-                                &"endcol".into(),
-                                &(loc.column() as u32).into(),
-                            );
-                        }
-                    }
+                    let _ =
+                        Reflect::set(&js_err, &"row".into(), &(err.location.row() as u32).into());
+                    let _ = Reflect::set(
+                        &js_err,
+                        &"col".into(),
+                        &(err.location.column() as u32).into(),
+                    );
                     js_err
                 })?;
                 let result = vm.run_code_obj(code, scope.borrow().clone());

wasm/lib/src/wasm_builtins.rs

@@ -7,12 +7,9 @@
 use js_sys::{self, Array};
 use web_sys::{self, console};
 
-use rustpython_vm::function::{Args, KwArgs, PyFuncArgs};
-use rustpython_vm::obj::{
-    objstr::{self, PyStringRef},
-    objtype,
-};
-use rustpython_vm::pyobject::{IdProtocol, ItemProtocol, PyObjectRef, PyResult, TypeProtocol};
+use rustpython_vm::function::PyFuncArgs;
+use rustpython_vm::obj::{objstr, objtype};
+use rustpython_vm::pyobject::{IdProtocol, PyObjectRef, PyResult, TypeProtocol};
 use rustpython_vm::VirtualMachine;
 
 pub(crate) fn window() -> web_sys::Window {