Move attributes dictionary to PyObject.

Add type annotations to parser.

.dockerignore

@@ -0,0 +1,2 @@
+target
+**/node_modules

.gitignore

@@ -1,8 +1,10 @@
 /target
-wasm/target
 **/*.rs.bk
 **/*.bytecode
 __pycache__
 **/*.pytest_cache
 .*sw*
 .repl_history.txt
+.vscode
+wasm-pack.log
+.idea/

.travis.yml

@@ -1,4 +1,3 @@
-
 language: rust
 
 rust:
@@ -7,60 +6,140 @@ rust:
   - nightly
 
 script:
-    - cargo build --verbose --all
+  - cargo build --verbose --all
-    - cargo test --verbose --all
+  - cargo test --verbose --all
 
 env:
-    # This is used to only capture the regular nightly test in allow_failures
+  # This is used to only capture the regular nightly test in allow_failures
-    - REGULAR_TEST=true
+  - REGULAR_TEST=true
 
 cache: cargo
 
 matrix:
-    include:
+  fast_finish: true
-        # To test the snippets, we use Travis' Python environment (because
+  include:
-        # installing rust ourselves is a lot easier than installing Python)
+    # To test the snippets, we use Travis' Python environment (because
-        - language: python
+    # installing rust ourselves is a lot easier than installing Python)
-          python: 3.6
+    - language: python
-          cache:
+      python: 3.6
-              pip: true
+      cache:
-              # Because we're using the Python Travis environment, we can't use
+        pip: true
-              # the built-in cargo cacher
+        # Because we're using the Python Travis environment, we can't use
-              directories:
+        # the built-in cargo cacher
-                  - /home/travis/.cargo
+        directories:
-                  - target
+          - /home/travis/.cargo
-          env:
+          - target
-              - TRAVIS_RUST_VERSION=stable
+      env:
-              - REGULAR_TEST=false
+        - TRAVIS_RUST_VERSION=stable
-          script: tests/.travis-runner.sh
+        - REGULAR_TEST=false
-        - language: python
+        - CODE_COVERAGE=false
-          python: 3.6
+      script: tests/.travis-runner.sh
-          cache:
+    - language: python
-              pip: true
+      python: 3.6
-              # Because we're using the Python Travis environment, we can't use
+      cache:
-              # the built-in cargo cacher
+        pip: true
-              directories:
+        # Because we're using the Python Travis environment, we can't use
-                  - /home/travis/.cargo
+        # the built-in cargo cacher
-                  - target
+        directories:
-          env:
+          - /home/travis/.cargo
-              - TRAVIS_RUST_VERSION=beta
+          - target
-              - REGULAR_TEST=false
+      env:
-          script: tests/.travis-runner.sh
+        - TRAVIS_RUST_VERSION=beta
-        - name: rustfmt
+        - REGULAR_TEST=false
-          language: rust
+        - CODE_COVERAGE=false
-          rust: nightly
+      script: tests/.travis-runner.sh
-          cache: cargo
+    - name: rustfmt
-          before_script:
+      language: rust
-              - rustup component add rustfmt-preview
+      rust: stable
-          script:
+      cache: cargo
-              # Code references the generated python.rs, so put something in
+      before_script:
-              # place to make `cargo fmt` happy.  (We use `echo` rather than
+        - rustup component add rustfmt-preview
-              # `touch` because rustfmt complains about the empty file touch
+      script:
-              # creates.)
+        # Code references the generated python.rs, so put something in
-              - echo > parser/src/python.rs
+        # place to make `cargo fmt` happy.  (We use `echo` rather than
-              - cargo fmt --all -- --check
+        # `touch` because rustfmt complains about the empty file touch
-          env:
+        # creates.)
-              - REGULAR_TEST=false
+        - echo > parser/src/python.rs
-    allow_failures:
+        - cargo fmt --all -- --check
-        - rust: nightly
+      env:
-          env: REGULAR_TEST=true
+        - REGULAR_TEST=false
+    - name: publish documentation
+      language: rust
+      rust: stable
+      cache: cargo
+      script:
+        - cargo doc --no-deps --all
+      if: branch = release
+      env:
+        - REGULAR_TEST=false
+        - DEPLOY_DOC=true
+    - name: WASM online demo
+      language: rust
+      rust: nightly
+      cache: cargo
+      install:
+        - nvm install node
+        # install wasm-pack
+        - curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+      script:
+        - cd wasm/demo
+        - npm install
+        - npm run dist
+      if: branch = release
+      env:
+        - REGULAR_TEST=false
+        - DEPLOY_DEMO=true
+    - name: cargo-clippy
+      language: rust
+      rust: stable
+      cache: cargo
+      before_script:
+        - rustup component add clippy
+      script:
+        - cargo clippy
+      env:
+        - REGULAR_TEST=true
+    - name: Code Coverage
+      language: python
+      python: 3.6
+      cache:
+        pip: true
+        # Because we're using the Python Travis environment, we can't use
+        # the built-in cargo cacher
+        directories:
+          - /home/travis/.cargo
+          - target
+      script:
+        - tests/.travis-runner.sh
+      env:
+        - TRAVIS_RUST_VERSION=nightly
+        - REGULAR_TEST=false
+        - CODE_COVERAGE=true
+  allow_failures:
+    - rust: nightly
+      env: REGULAR_TEST=true
+    - name: cargo-clippy
+
+deploy:
+  - provider: pages
+    repo: RustPython/website
+    target-branch: master
+    local-dir: target/doc
+    skip-cleanup: true
+    # Set in the settings page of your repository, as a secure variable
+    github-token: $WEBSITE_GITHUB_TOKEN
+    keep-history: true
+    on:
+      branch: release
+      condition: $DEPLOY_DOC = true
+  - provider: pages
+    repo: RustPython/demo
+    target-branch: master
+    local-dir: wasm/demo/dist
+    skip-cleanup: true
+    # Set in the settings page of your repository, as a secure variable
+    github-token: $WEBSITE_GITHUB_TOKEN
+    keep-history: true
+    on:
+      branch: release
+      condition: $DEPLOY_DEMO = true

Cargo.lock

@@ -1,3 +1,5 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
 [[package]]
 name = "aho-corasick"
 version = "0.6.4"
@@ -131,6 +133,15 @@ name = "byteorder"
 version = "1.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "caseless"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.25"
@@ -155,6 +166,14 @@ dependencies = [
  "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "cloudabi"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "constant_time_eq"
 version = "0.1.3"
@@ -188,7 +207,7 @@ name = "docopt"
 version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -213,10 +232,22 @@ dependencies = [
  "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "env_logger"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "failure"
 version = "0.1.2"
@@ -261,6 +292,11 @@ name = "fuchsia-zircon-sys"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "futures"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "generic-array"
 version = "0.9.0"
@@ -269,6 +305,14 @@ dependencies = [
  "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "heck"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "humantime"
 version = "1.1.1"
@@ -290,6 +334,14 @@ name = "itoa"
 version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "js-sys"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "kernel32-sys"
 version = "0.2.2"
@@ -352,9 +404,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "lazy_static"
-version = "1.0.1"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "lexical"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lexical-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "lexical-core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "libc"
 version = "0.2.42"
@@ -409,6 +481,14 @@ name = "nodrop"
 version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "nom"
+version = "4.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "num-bigint"
 version = "0.2.1"
@@ -434,6 +514,16 @@ dependencies = [
  "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "num-rational"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.6"
@@ -522,6 +612,31 @@ dependencies = [
  "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rand"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "redox_syscall"
 version = "0.1.40"
@@ -554,19 +669,19 @@ dependencies = [
  "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "regex"
-version = "1.0.0"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -596,9 +711,26 @@ name = "rustc-demangle"
 version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "rustc_version"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rustc_version_runtime"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rustpython"
-version = "0.0.1"
+version = "0.0.1-pre-alpha.1"
 dependencies = [
  "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -606,6 +738,7 @@ dependencies = [
  "rustpython_parser 0.0.1",
  "rustpython_vm 0.1.0",
  "rustyline 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -626,15 +759,39 @@ version = "0.1.0"
 dependencies = [
  "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lexical 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustpython_parser 0.0.1",
  "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)",
+ "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rustpython_wasm"
+version = "0.1.0"
+dependencies = [
+ "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
+ "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustpython_parser 0.0.1",
+ "rustpython_vm 0.1.0",
+ "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-futures 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "web-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -655,7 +812,7 @@ dependencies = [
 
 [[package]]
 name = "ryu"
-version = "0.2.5"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
@@ -663,6 +820,19 @@ name = "scoped_threadpool"
 version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "semver"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "serde"
 version = "1.0.66"
@@ -684,7 +854,7 @@ version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "ryu 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -704,12 +874,38 @@ name = "siphasher"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "sourcefile"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "stackvector"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "static_assertions"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "statrs"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "string_cache"
 version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)",
  "precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -755,6 +951,16 @@ dependencies = [
  "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "syn"
+version = "0.15.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "synstructure"
 version = "0.9.0"
@@ -783,6 +989,14 @@ dependencies = [
  "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "termcolor"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "termion"
 version = "1.5.1"
@@ -803,11 +1017,10 @@ dependencies = [
 
 [[package]]
 name = "thread_local"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -820,6 +1033,11 @@ name = "ucd-util"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "unicode-normalization"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "unicode-segmentation"
 version = "1.2.1"
@@ -863,6 +1081,99 @@ name = "void"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "wasm-bindgen-macro 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 0.15.23 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
+ "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-macro-support 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 0.15.23 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "wasm-bindgen-webidl"
+version = "0.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 0.15.23 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "weedle 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "web-sys"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-webidl 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "weedle"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "nom 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "winapi"
 version = "0.2.8"
@@ -887,6 +1198,14 @@ name = "winapi-i686-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
@@ -900,6 +1219,20 @@ dependencies = [
  "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "wincolor"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "xdg"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [metadata]
 "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"
 "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
@@ -918,9 +1251,11 @@ dependencies = [
 "checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab"
 "checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40"
 "checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781"
+"checksum caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f"
 "checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16"
 "checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18"
 "checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536"
+"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
 "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e"
 "checksum diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "3c2b69f912779fbb121ceb775d74d51e915af17aaebc38d28a592843a2dd0a3a"
 "checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603"
@@ -929,21 +1264,27 @@ dependencies = [
 "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0"
 "checksum ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cabe5a5078ac8c506d3e4430763b1ba9b609b1286913e7d08e581d1c2de9b7e5"
 "checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a"
+"checksum env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "afb070faf94c85d17d50ca44f6ad076bce18ae92f0037d350947240a36e9d42e"
 "checksum failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7efb22686e4a466b1ec1a15c2898f91fa9cb340452496dca654032de20ff95b9"
 "checksum failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "946d0e98a50d9831f5d589038d2ca7f8f455b1c21028c0db0e84116a12696426"
 "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
 "checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33"
 "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
 "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
+"checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b"
 "checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d"
+"checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82"
 "checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e"
 "checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450"
 "checksum itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5adb58558dcd1d786b5f0bd15f3226ee23486e24b7b58304b60f64dc68e62606"
+"checksum js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58cfec35fd4a94f3cf357d5cb7da71c71cd52720c2f2a7320090a8db5f06f655"
 "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
 "checksum lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ba451f7bd819b7afc99d4cf4bdcd5a4861e64955ba9680ac70df3a50625ad6cf"
 "checksum lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60013fd6be14317d43f47658b1440956a9ca48a9ed0257e0e0a59aac13e43a1f"
 "checksum lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60c6c48ba857cd700673ce88907cadcdd7e2cd7783ed02378537c5ffd4f6460c"
-"checksum lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e6412c5e2ad9584b0b8e979393122026cdd6d2a80b933f890dcd694ddbe73739"
+"checksum lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1"
+"checksum lexical 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e4fac65df7e751b57bb3a334c346239cb4ce2601907d698726ceeb82a54ba4ef"
+"checksum lexical-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "025babf624c0c2b4bed1373efd684d5d0b2eecd61138d26ec3eec77bf0f2e33d"
 "checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1"
 "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
 "checksum log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6fddaa003a65722a7fb9e26b0ce95921fe4ba590542ced664d8ce2fa26f9f3ac"
@@ -951,9 +1292,11 @@ dependencies = [
 "checksum new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0cdc457076c78ab54d5e0d6fa7c47981757f1e34dc39ff92787f217dede586c4"
 "checksum nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d37e713a259ff641624b6cb20e3b12b2952313ba36b6823c0f16e6cfd9e5de17"
 "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2"
+"checksum nom 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9c349f68f25f596b9f44cf0e7c69752a5c633b0550c3ff849518bfba0233774a"
 "checksum num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "10b8423ea72ec64751198856a853e07b37087cfc9b53a87ecb19bff67b6d1320"
 "checksum num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "107b9be86cd2481930688277b675b0114578227f034674726605b8a482d8baf8"
 "checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea"
+"checksum num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4e96f040177bb3da242b5b1ecf3f54b5d5af3efbbfb18608977a5d2767b22f10"
 "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1"
 "checksum ordermap 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063"
 "checksum petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f"
@@ -966,37 +1309,51 @@ dependencies = [
 "checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8"
 "checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035"
 "checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5"
+"checksum rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e464cd887e869cddcae8792a4ee31d23c7edd516700695608f5b98c67ee0131c"
+"checksum rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1961a422c4d189dfb50ffa9320bf1f2a9bd54ecb92792fb9477f99a1045f3372"
+"checksum rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db"
 "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1"
 "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
 "checksum redox_users 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "214a97e49be64fd2c86f568dd0cb2c757d2cc53de95b273b6ad0a1c908482f26"
 "checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384"
-"checksum regex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75ecf88252dce580404a22444fc7d626c01815debba56a7f4f536772a5ff19d3"
+"checksum regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3d8c9f33201f46669484bacc312b00e7541bed6aaf296dffe2bb4e0ac6b8ce2a"
 "checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e"
 "checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7"
 "checksum regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f1ac0f60d675cc6cf13a20ec076568254472551051ad5dd050364d70671bf6b"
 "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395"
+"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
+"checksum rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6de8ecd7fad7731f306f69b6e10ec5a3178c61e464dcc06979427aa4cc891145"
 "checksum rustyline 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6010155119d53aac4f5b987cb8f6ea913d0d64d9b237da36f8f96a90cb3f5385"
-"checksum ryu 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e7c066b8e2923f05d4718a06d2622f189ff362bc642bfade6c6629f0440f3827"
+"checksum ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7"
 "checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8"
+"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
+"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
 "checksum serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a2d9a9ac5120e0f768801ca2b58ad6eec929dc9d1d616c162f208869c2ce95"
 "checksum serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "0a90213fa7e0f5eac3f7afe2d5ff6b088af515052cc7303bd68c7e3b91a3fb79"
 "checksum serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "44dd2cfde475037451fa99b7e5df77aa3cfd1536575fa8e7a538ab36dcde49ae"
 "checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0"
 "checksum siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0df90a788073e8d0235a67e50441d47db7c8ad9debd91cbf43736a2a92d36537"
+"checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3"
+"checksum stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c049c77bf85fbc036484c97b008276d539d9ebff9dfbde37b632ebcd5b8746b6"
+"checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5"
+"checksum statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "10102ac8d55e35db2b3fafc26f81ba8647da2e15879ab686a67e6d19af2685e8"
 "checksum string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25d70109977172b127fe834e5449e5ab1740b9ba49fa18a2020f509174f25423"
 "checksum string_cache_codegen 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "35293b05cf1494e8ddd042a7df6756bf18d07f42d234f32e71dce8a7aabb0191"
 "checksum string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc"
 "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"
 "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
 "checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741"
+"checksum syn 0.15.23 (registry+https://github.com/rust-lang/crates.io-index)" = "9545a6a093a3f0bd59adb472700acc08cad3776f860f16a897dfce8c88721cbc"
 "checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7"
 "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1"
 "checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83"
+"checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f"
 "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
 "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693"
-"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963"
+"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
 "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169"
 "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d"
+"checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25"
 "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1"
 "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526"
 "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
@@ -1005,9 +1362,21 @@ dependencies = [
 "checksum utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d"
 "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
 "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
+"checksum wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "91f95b8f30407b9ca0c2de157281d3828bbed1fc1f55bea6eb54f40c52ec75ec"
+"checksum wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "ab7c242ebcb45bae45340986c48d1853eb2c1c52ff551f7724951b62a2c51429"
+"checksum wasm-bindgen-futures 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d1784e7401a90119b2a4e8ec9c8d37c3594c3e3bb9ba24533ee1969eebaf0485"
+"checksum wasm-bindgen-macro 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "6e353f83716dec9a3597b5719ef88cb6c9e461ec16528f38aa023d3224b4e569"
+"checksum wasm-bindgen-macro-support 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "3cc90b65fe69c3dd5a09684517dc79f42b847baa2d479c234d125e0a629d9b0a"
+"checksum wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "a71a37df4f5845025f96f279d20bbe5b19cbcb77f5410a3a90c6c544d889a162"
+"checksum wasm-bindgen-webidl 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)" = "b064b8b2336b5a6bf5f31bc95fc1310842395df29877d910cb6f8f791070f319"
+"checksum web-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d7c588c2e5a091bc4532c5a87032955f9133b644e868b54d08ead0185dcc5b9"
+"checksum weedle 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "26a4c67f132386d965390b8a734d5d10adbcd30eb5cc74bd9229af8b83f10044"
 "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
 "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+"checksum winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "afc5508759c5bf4285e61feb862b6083c8480aec864fa17a81fdec6f69b461ab"
 "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 "checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767"
+"checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba"
+"checksum xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57"

Cargo.toml

@@ -1,9 +1,11 @@
 [package]
 name = "rustpython"
-version = "0.0.1"
+version = "0.0.1-pre-alpha.1"
 authors = ["Windel Bouwman", "Shing Lyu <shing.lyu@gmail.com>"]
+edition = "2018"
 
 [workspace]
+members = [".", "vm", "wasm/lib", "parser"]
 
 [dependencies]
 log="0.4.1"
@@ -12,3 +14,7 @@ clap = "2.31.2"
 rustpython_parser = {path = "parser"}
 rustpython_vm = {path = "vm"}
 rustyline = "2.1.0"
+xdg = "2.2.0"
+
+[profile.release]
+opt-level = "s"

Dockerfile.bin

@@ -0,0 +1,9 @@
+FROM rust:1.31-slim
+
+WORKDIR /rustpython
+
+COPY . .
+
+RUN cargo build --release
+
+CMD [ "/rustpython/target/release/rustpython" ]

Dockerfile.wasm

@@ -0,0 +1,19 @@
+FROM rust:1.31-slim
+
+RUN apt-get update && apt-get install curl gnupg -y && \
+    curl -o- https://deb.nodesource.com/setup_10.x | bash && \
+    apt-get install nodejs -y && \
+    curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && \
+    npm i -g serve
+
+WORKDIR /rustpython
+
+COPY . .
+
+RUN cd ./wasm/lib/ && \
+    cargo build --release && \
+    cd ../demo && \
+    npm install && \
+    npm run dist
+
+CMD [ "serve", "/rustpython/wasm/demo/dist" ]

README.md

@@ -1,13 +1,18 @@
 # RustPython
-A Python-3  (CPython >= 3.5.0) Interpreter written in Rust :snake: :scream: :metal:.
+
+A Python-3 (CPython >= 3.5.0) Interpreter written in Rust :snake: :scream: :metal:.
 
 [![Build Status](https://travis-ci.org/RustPython/RustPython.svg?branch=master)](https://travis-ci.org/RustPython/RustPython)
+[![Build Status](https://dev.azure.com/ryan0463/ryan/_apis/build/status/RustPython.RustPython?branchName=master)](https://dev.azure.com/ryan0463/ryan/_build/latest?definitionId=1&branchName=master)
+[![codecov](https://codecov.io/gh/RustPython/RustPython/branch/master/graph/badge.svg)](https://codecov.io/gh/RustPython/RustPython)
 [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
 [![Contributors](https://img.shields.io/github/contributors/RustPython/RustPython.svg)](https://github.com/RustPython/RustPython/graphs/contributors)
 [![Gitter](https://badges.gitter.im/RustPython/Lobby.svg)](https://gitter.im/rustpython/Lobby)
 
 # Usage
 
+### Check out our [online demo](https://rustpython.github.io/demo/) running on WebAssembly.
+
 To test RustPython, do the following:
 
     $ git clone https://github.com/RustPython/RustPython
@@ -22,6 +27,13 @@ Or use the interactive shell:
     >>>>> 2+2
     4
 
+# Disclaimer
+
+  RustPython is in a development phase and should not be used in production or a fault intolerant setting.
+
+  Our current build supports only a subset of Python syntax.
+
+  Contribution is also more than welcome! See our contribution section for more information on this. 
 
 # Goals
 
@@ -30,35 +42,48 @@ Or use the interactive shell:
 
 # Documentation
 
-Currently the project is in an early phase, and so is the documentation.
+Currently along with other areas of the project, documentation is still in an early phase.
 
-You can generate documentation by running:
+You can read the [online documentation](https://rustpython.github.io/website/rustpython/index.html) for the latest code on master.
+
+You can also generate documentation locally by running:
 
 ```shell
-$ cargo doc
+$ cargo doc # Including documentation for all dependencies
+$ cargo doc --no-deps --all # Excluding all dependencies
 ```
 
 Documentation HTML files can then be found in the `target/doc` directory.
 
+If you wish to update the online documentation, push directly to the `release` branch (or ask a maintainer to do so). This will trigger a Travis build that updates the documentation and WebAssembly demo page.
+
 # Code organization
 
-- `parser`: python lexing, parsing and ast
+- `parser/src`: python lexing, parsing and ast
-- `vm`: python virtual machine
+- `vm/src`: python virtual machine
+  - `builtins.rs`: Builtin functions
+  - `compile.rs`: the python compiler from ast to bytecode
+  - `obj`: python builtin types
 - `src`: using the other subcrates to bring rustpython to life.
 - `docs`: documentation (work in progress)
-- `py_code_object`: CPython bytecode to rustpython bytecode convertor (work in progress)
+- `py_code_object`: CPython bytecode to rustpython bytecode converter (work in progress)
-- `wasm`: Binary crate and resources for WebAssembly build 
+- `wasm`: Binary crate and resources for WebAssembly build
 - `tests`: integration test snippets
 
 # Contributing
 
-To start contributing, there are a lot of things that need to be done.
+Contributions are more than welcome, and in many cases we are happy to guide contributors through PRs or on gitter.
+
+With that in mind, please note this project is maintained by volunteers, some of the best ways to get started are below:
+
 Most tasks are listed in the [issue tracker](https://github.com/RustPython/RustPython/issues).
-Another approach is to checkout the sourcecode: builtin functions and object methods are often the simplest
+Check issues labeled with `good first issue` if you wish to start coding.
-and easiest way to contribute. 
+
+Another approach is to checkout the source code: builtin functions and object methods are often the simplest
+and easiest way to contribute.
 
 You can also simply run
-`cargo run tests/snippets/todo.py` to assist in finding any
+`cargo run tests/snippets/whats_left_to_implement.py` to assist in finding any
 unimplemented method.
 
 # Testing
@@ -68,11 +93,11 @@ To test rustpython, there is a collection of python snippets located in the
 
 ```shell
 $ cd tests
-$ pipenv shell
+$ pipenv install
-$ pytest -v
+$ pipenv run pytest -v
 ```
 
-There also are some unittests, you can run those will cargo:
+There also are some unit tests, you can run those with cargo:
 
 ```shell
 $ cargo test --all
@@ -102,8 +127,7 @@ At this stage RustPython only has preliminary support for web assembly. The inst
 
 ## Setup
 
-To get started, install [wasm-bingden](https://rustwasm.github.io/wasm-bindgen/whirlwind-tour/basic-usage.html)
+To get started, install [wasm-pack](https://rustwasm.github.io/wasm-pack/installer/) and `npm`. ([wasm-bindgen](https://rustwasm.github.io/wasm-bindgen/whirlwind-tour/basic-usage.html) should be installed by `wasm-pack`. if not, install it yourself)
-and [wasm-webpack](https://rustwasm.github.io/wasm-pack/installer/). You will also need to have `npm` installed.
 
 <!-- Using `rustup` add the compile target `wasm32-unknown-emscripten`. To do so you will need to have [rustup](https://rustup.rs/) installed.
 
@@ -121,55 +145,53 @@ cd emsdk-portable/
 ./emsdk activate sdk-incoming-64bit
 ``` -->
 
-
-
 ## Build
 
-Move into the `wasm` directory. This contains a custom library crate optimized for wasm build of RustPython.   
+Move into the `wasm` directory. This directory contains a library crate for interop
+with python to rust to js and back in `wasm/lib`, the demo website found at
+https://rustpython.github.io/demo in `wasm/demo`, and an example of how to use
+the crate as a library in one's own JS app in `wasm/example`.
 
-```bash
+```sh
 cd wasm
 ```
 
-From here run the build. This can take several minutes depending on the machine.
+Go to the demo directory. This is the best way of seeing the changes made to either
+the library or the JS demo, as the `rustpython_wasm` module is set to the global
+JS variable `rp` on the website.
 
-```
+```sh
-wasm-pack build
+cd demo
 ```
 
-Upon successful build, cd in the the `/pkg` directory and run:
+Now, start the webpack development server. It'll compile the crate and then
+the demo app. This will likely take a long time, both the wasm-pack portion and
+the webpack portion (from after it says "Your crate has been correctly compiled"),
+so be patient.
 
-```
+```sh
-npm link
+npm run dev
 ```
 
-Now move back out into the `/app` directory. The files here have been adapted from [wasm-pack-template](https://github.com/rustwasm/wasm-pack-template).
+You can now open the webpage on https://localhost:8080 and Python code in either
-
+the text box or browser devtools with:
-Finally, run:
-
-```
-npm install
-```
-
-and you will be able to run the files with:
-
-```
-webpack-dev-server
-```
-
-Open a browser console and see the output of rustpython_wasm. To verify this, modify the line in `app/index.js`
 
 ```js
-rp.run_code("print('Hello Python!')\n");
+rp.pyEval(
+  `
+print(js_vars['a'] * 9)
+`,
+  {
+    vars: {
+      a: 9
+    }
+  }
+);
 ```
 
-To the following:
+Alternatively, you can run `npm run build` to build the app once, without watching
-
+for changes, or `npm run dist` to build the app in release mode, both for the
-```js
+crate and webpack.
-rp.run_code("assert(False)\n");
-```
-
-and you should observe: `Execution failed` in your console output, indicating that the execution of RustPython has failed.
 
 # Code style
 
@@ -192,4 +214,3 @@ These are some useful links to related projects:
 - https://github.com/ProgVal/pythonvm-rust
 - https://github.com/shinglyu/RustPython
 - https://github.com/windelbouwman/rspython
-

azure-pipelines.yml

@@ -0,0 +1,56 @@
+trigger:
+- master
+
+jobs:
+
+- job: 'Test'
+  pool:
+    vmImage: 'vs2017-win2016'
+  strategy:
+    matrix:
+      Python37:
+        python.version: '3.7'
+    maxParallel: 10
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(python.version)'
+      architecture: 'x64'
+
+  - script: |
+      "C:\Program Files\Git\mingw64\bin\curl.exe" -sSf -o rustup-init.exe https://win.rustup.rs/
+      .\rustup-init.exe -y
+      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+      rustc -V
+      cargo -V
+    displayName: 'Installing Rust'
+
+  - script: |
+      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+      cargo build --verbose --all
+    displayName: 'Build'
+
+  - script: |
+      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+      cargo test --verbose --all
+    displayName: 'Run tests'
+
+  - script: |
+      pip install pipenv
+      pushd tests
+      pipenv install
+      popd
+    displayName: 'Install pipenv and python packages'
+
+  - script: |
+      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+      cargo build --verbose --release
+    displayName: 'Build release'
+
+  - script: |
+      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+      pushd tests
+      pipenv run pytest
+      popd
+    displayName: 'Run snippet tests'

docs/builtins.md

@@ -1,6 +1,6 @@
 Byterun
 
-* Builtins are exposted to frame.f_builtins
+* Builtins are exposed to frame.f_builtins
 * f_builtins is assigned during frame creation,
     self.f_builtins = f_locals['__builtins__']
     if hasattr(self.f_builtins, '__dict__'):
@@ -21,10 +21,10 @@ TODO:
 * Implement a new type NativeFunction
 * Wrap a function pointer in NativeFunction
 * Refactor the CALL_FUNCTION case so it can call both python function and native function
-* During frame creation, force push a nativefunction `print` into the namespace
+* During frame creation, force push a native function `print` into the namespace
 * Modify LOAD_* so they can search for names in builtins
 
 * Create a module type
 * In VM initialization, load the builtins module into locals
-* During frame creation, create a field that conatins the builtins dict
+* During frame creation, create a field that contains the builtins dict
 

parser/Cargo.toml

@@ -3,6 +3,7 @@ name = "rustpython_parser"
 version = "0.0.1"
 authors = [ "Shing Lyu", "Windel Bouwman" ]
 build = "build.rs"
+edition = "2018"
 
 [build-dependencies]
 lalrpop="0.15.1"

parser/build.rs

@@ -1,4 +1,4 @@
-extern crate lalrpop;
+use lalrpop;
 
 fn main() {
     lalrpop::process_root().unwrap();

parser/src/ast.rs

@@ -65,9 +65,9 @@ pub enum Statement {
         value: Expression,
     },
     AugAssign {
-        target: Expression,
+        target: Box<Expression>,
         op: Operator,
-        value: Expression,
+        value: Box<Expression>,
     },
     Expression {
         expression: Expression,
@@ -122,6 +122,7 @@ pub enum Statement {
         // docstring: String,
         body: Vec<LocatedStatement>,
         decorator_list: Vec<Expression>,
+        returns: Option<Expression>,
     },
 }
 
@@ -197,7 +198,7 @@ pub enum Expression {
         elements: Vec<Expression>,
     },
     String {
-        value: String,
+        value: StringGroup,
     },
     Bytes {
         value: Vec<u8>,
@@ -217,22 +218,74 @@ pub enum Expression {
     True,
     False,
     None,
+    Ellipsis,
+}
+
+impl Expression {
+    /// Returns a short name for the node suitable for use in error messages.
+    pub fn name(&self) -> &'static str {
+        use self::Expression::*;
+        use self::StringGroup::*;
+
+        match self {
+            BoolOp { .. } | Binop { .. } | Unop { .. } => "operator",
+            Subscript { .. } => "subscript",
+            Yield { .. } | YieldFrom { .. } => "yield expression",
+            Compare { .. } => "comparison",
+            Attribute { .. } => "attribute",
+            Call { .. } => "function call",
+            Number { .. }
+            | String {
+                value: Constant { .. },
+            }
+            | Bytes { .. } => "literal",
+            List { .. } => "list",
+            Tuple { .. } => "tuple",
+            Dict { .. } => "dict display",
+            Set { .. } => "set display",
+            Comprehension { kind, .. } => match **kind {
+                ComprehensionKind::List { .. } => "list comprehension",
+                ComprehensionKind::Dict { .. } => "dict comprehension",
+                ComprehensionKind::Set { .. } => "set comprehension",
+                ComprehensionKind::GeneratorExpression { .. } => "generator expression",
+            },
+            Starred { .. } => "starred",
+            Slice { .. } => "slice",
+            String {
+                value: Joined { .. },
+            }
+            | String {
+                value: FormattedValue { .. },
+            } => "f-string expression",
+            Identifier { .. } => "named expression",
+            Lambda { .. } => "lambda",
+            IfExpression { .. } => "conditional expression",
+            True | False | None => "keyword",
+            Ellipsis => "ellipsis",
+        }
+    }
 }
 
 /*
  * In cpython this is called arguments, but we choose parameters to
- * distuingish between function parameters and actual call arguments.
+ * distinguish between function parameters and actual call arguments.
  */
 #[derive(Debug, PartialEq, Default)]
 pub struct Parameters {
-    pub args: Vec<String>,
+    pub args: Vec<Parameter>,
-    pub kwonlyargs: Vec<String>,
+    pub kwonlyargs: Vec<Parameter>,
-    pub vararg: Option<Option<String>>, // Optionally we handle optionally named '*args' or '*'
+    pub vararg: Option<Option<Parameter>>, // Optionally we handle optionally named '*args' or '*'
-    pub kwarg: Option<Option<String>>,
+    pub kwarg: Option<Option<Parameter>>,
     pub defaults: Vec<Expression>,
     pub kw_defaults: Vec<Option<Expression>>,
 }
 
+#[derive(Debug, PartialEq, Default)]
+pub struct Parameter {
+    pub arg: String,
+    pub annotation: Option<Box<Expression>>,
+}
+
 #[derive(Debug, PartialEq)]
 pub enum ComprehensionKind {
     GeneratorExpression { element: Expression },
@@ -312,3 +365,29 @@ pub enum Number {
     Float { value: f64 },
     Complex { real: f64, imag: f64 },
 }
+
+/// Transforms a value prior to formatting it.
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub enum ConversionFlag {
+    /// Converts by calling `str(<value>)`.
+    Str,
+    /// Converts by calling `ascii(<value>)`.
+    Ascii,
+    /// Converts by calling `repr(<value>)`.
+    Repr,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum StringGroup {
+    Constant {
+        value: String,
+    },
+    FormattedValue {
+        value: Box<Expression>,
+        conversion: Option<ConversionFlag>,
+        spec: String,
+    },
+    Joined {
+        values: Vec<StringGroup>,
+    },
+}

parser/src/error.rs

@@ -0,0 +1,79 @@
+//! Define internal parse error types
+//! The goal is to provide a matching and a safe error API, maksing errors from LALR
+extern crate lalrpop_util;
+use self::lalrpop_util::ParseError as InnerError;
+
+use crate::lexer::{LexicalError, Location};
+use crate::token::Tok;
+
+use std::error::Error;
+use std::fmt;
+
+// A token of type `Tok` was observed, with a span given by the two Location values
+type TokSpan = (Location, Tok, Location);
+
+/// Represents an error during parsing
+#[derive(Debug, PartialEq)]
+pub enum ParseError {
+    /// Parser encountered an unexpected end of input
+    EOF(Option<Location>),
+    /// Parser encountered an extra token
+    ExtraToken(TokSpan),
+    /// Parser encountered an invalid token
+    InvalidToken(Location),
+    /// Parser encountered an unexpected token
+    UnrecognizedToken(TokSpan, Vec<String>),
+    /// Maps to `User` type from `lalrpop-util`
+    Other,
+}
+
+/// Convert `lalrpop_util::ParseError` to our internal type
+impl From<InnerError<Location, Tok, LexicalError>> for ParseError {
+    fn from(err: InnerError<Location, Tok, LexicalError>) -> Self {
+        match err {
+            // TODO: Are there cases where this isn't an EOF?
+            InnerError::InvalidToken { location } => ParseError::EOF(Some(location)),
+            InnerError::ExtraToken { token } => ParseError::ExtraToken(token),
+            // Inner field is a unit-like enum `LexicalError::StringError` with no useful info
+            InnerError::User { .. } => ParseError::Other,
+            InnerError::UnrecognizedToken { token, expected } => {
+                match token {
+                    Some(tok) => ParseError::UnrecognizedToken(tok, expected),
+                    // EOF was observed when it was unexpected
+                    None => ParseError::EOF(None),
+                }
+            }
+        }
+    }
+}
+
+impl fmt::Display for ParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            ParseError::EOF(ref location) => {
+                if let Some(l) = location {
+                    write!(f, "Got unexpected EOF at: {:?}", l)
+                } else {
+                    write!(f, "Got unexpected EOF")
+                }
+            }
+            ParseError::ExtraToken(ref t_span) => {
+                write!(f, "Got extraneous token: {:?} at: {:?}", t_span.1, t_span.0)
+            }
+            ParseError::InvalidToken(ref location) => {
+                write!(f, "Got invalid token at: {:?}", location)
+            }
+            ParseError::UnrecognizedToken(ref t_span, _) => {
+                write!(f, "Got unexpected token: {:?} at {:?}", t_span.1, t_span.0)
+            }
+            // This is user defined, it probably means a more useful error should have been given upstream.
+            ParseError::Other => write!(f, "Got unsupported token(s)"),
+        }
+    }
+}
+
+impl Error for ParseError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        None
+    }
+}

parser/src/fstring.rs

@@ -0,0 +1,234 @@
+use std::iter;
+use std::mem;
+use std::str;
+
+use lalrpop_util::ParseError as LalrpopError;
+
+use crate::ast::{ConversionFlag, StringGroup};
+use crate::lexer::{LexicalError, Location, Tok};
+use crate::parser::parse_expression;
+
+use self::FStringError::*;
+use self::StringGroup::*;
+
+// TODO: consolidate these with ParseError
+#[derive(Debug, PartialEq)]
+pub enum FStringError {
+    UnclosedLbrace,
+    UnopenedRbrace,
+    InvalidExpression,
+    InvalidConversionFlag,
+    EmptyExpression,
+    MismatchedDelimiter,
+}
+
+impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
+    fn from(_err: FStringError) -> Self {
+        lalrpop_util::ParseError::User {
+            error: LexicalError::StringError,
+        }
+    }
+}
+
+struct FStringParser<'a> {
+    chars: iter::Peekable<str::Chars<'a>>,
+}
+
+impl<'a> FStringParser<'a> {
+    fn new(source: &'a str) -> Self {
+        Self {
+            chars: source.chars().peekable(),
+        }
+    }
+
+    fn parse_formatted_value(&mut self) -> Result<StringGroup, FStringError> {
+        let mut expression = String::new();
+        let mut spec = String::new();
+        let mut delims = Vec::new();
+        let mut conversion = None;
+
+        while let Some(ch) = self.chars.next() {
+            match ch {
+                '!' if delims.is_empty() => {
+                    conversion = Some(match self.chars.next() {
+                        Some('s') => ConversionFlag::Str,
+                        Some('a') => ConversionFlag::Ascii,
+                        Some('r') => ConversionFlag::Repr,
+                        Some(_) => {
+                            return Err(InvalidConversionFlag);
+                        }
+                        None => {
+                            break;
+                        }
+                    })
+                }
+                ':' if delims.is_empty() => {
+                    while let Some(&next) = self.chars.peek() {
+                        if next != '}' {
+                            spec.push(next);
+                            self.chars.next();
+                        } else {
+                            break;
+                        }
+                    }
+                }
+                '(' | '{' | '[' => {
+                    expression.push(ch);
+                    delims.push(ch);
+                }
+                ')' => {
+                    if delims.pop() != Some('(') {
+                        return Err(MismatchedDelimiter);
+                    }
+                    expression.push(ch);
+                }
+                ']' => {
+                    if delims.pop() != Some('[') {
+                        return Err(MismatchedDelimiter);
+                    }
+                    expression.push(ch);
+                }
+                '}' if !delims.is_empty() => {
+                    if delims.pop() != Some('{') {
+                        return Err(MismatchedDelimiter);
+                    }
+                    expression.push(ch);
+                }
+                '}' => {
+                    if expression.is_empty() {
+                        return Err(EmptyExpression);
+                    }
+                    return Ok(FormattedValue {
+                        value: Box::new(
+                            parse_expression(expression.trim()).map_err(|_| InvalidExpression)?,
+                        ),
+                        conversion,
+                        spec,
+                    });
+                }
+                '"' | '\'' => {
+                    expression.push(ch);
+                    while let Some(next) = self.chars.next() {
+                        expression.push(next);
+                        if next == ch {
+                            break;
+                        }
+                    }
+                }
+                _ => {
+                    expression.push(ch);
+                }
+            }
+        }
+
+        Err(UnclosedLbrace)
+    }
+
+    fn parse(mut self) -> Result<StringGroup, FStringError> {
+        let mut content = String::new();
+        let mut values = vec![];
+
+        while let Some(ch) = self.chars.next() {
+            match ch {
+                '{' => {
+                    if let Some('{') = self.chars.peek() {
+                        self.chars.next();
+                        content.push('{');
+                    } else {
+                        if !content.is_empty() {
+                            values.push(Constant {
+                                value: mem::replace(&mut content, String::new()),
+                            });
+                        }
+
+                        values.push(self.parse_formatted_value()?);
+                    }
+                }
+                '}' => {
+                    if let Some('}') = self.chars.peek() {
+                        self.chars.next();
+                        content.push('}');
+                    } else {
+                        return Err(UnopenedRbrace);
+                    }
+                }
+                _ => {
+                    content.push(ch);
+                }
+            }
+        }
+
+        if !content.is_empty() {
+            values.push(Constant { value: content })
+        }
+
+        Ok(match values.len() {
+            0 => Constant {
+                value: String::new(),
+            },
+            1 => values.into_iter().next().unwrap(),
+            _ => Joined { values },
+        })
+    }
+}
+
+pub fn parse_fstring(source: &str) -> Result<StringGroup, FStringError> {
+    FStringParser::new(source).parse()
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::ast;
+
+    use super::*;
+
+    fn mk_ident(name: &str) -> ast::Expression {
+        ast::Expression::Identifier {
+            name: name.to_owned(),
+        }
+    }
+
+    #[test]
+    fn test_parse_fstring() {
+        let source = String::from("{a}{ b }{{foo}}");
+        let parse_ast = parse_fstring(&source).unwrap();
+
+        assert_eq!(
+            parse_ast,
+            Joined {
+                values: vec![
+                    FormattedValue {
+                        value: Box::new(mk_ident("a")),
+                        conversion: None,
+                        spec: String::new(),
+                    },
+                    FormattedValue {
+                        value: Box::new(mk_ident("b")),
+                        conversion: None,
+                        spec: String::new(),
+                    },
+                    Constant {
+                        value: "{foo}".to_owned()
+                    }
+                ]
+            }
+        );
+    }
+
+    #[test]
+    fn test_parse_empty_fstring() {
+        assert_eq!(
+            parse_fstring(""),
+            Ok(Constant {
+                value: String::new(),
+            }),
+        );
+    }
+
+    #[test]
+    fn test_parse_invalid_fstring() {
+        assert_eq!(parse_fstring("{"), Err(UnclosedLbrace));
+        assert_eq!(parse_fstring("}"), Err(UnopenedRbrace));
+        assert_eq!(parse_fstring("{class}"), Err(InvalidExpression));
+    }
+}

parser/src/lexer.rs

@@ -1,17 +1,50 @@
 //! This module takes care of lexing python source text. This means source
-//! code is translated into seperate tokens.
+//! code is translated into separate tokens.
 
 pub use super::token::Tok;
 use num_bigint::BigInt;
 use num_traits::Num;
+use std::cmp::Ordering;
 use std::collections::HashMap;
 use std::str::FromStr;
 
+#[derive(Clone, Copy, PartialEq, Debug)]
+struct IndentationLevel {
+    tabs: usize,
+    spaces: usize,
+}
+
+impl IndentationLevel {
+    fn new() -> IndentationLevel {
+        IndentationLevel { tabs: 0, spaces: 0 }
+    }
+    fn compare_strict(&self, other: &IndentationLevel) -> Option<Ordering> {
+        // We only know for sure that we're smaller or bigger if tabs
+        // and spaces both differ in the same direction. Otherwise we're
+        // dependent on the size of tabs.
+        if self.tabs < other.tabs {
+            if self.spaces <= other.spaces {
+                Some(Ordering::Less)
+            } else {
+                None
+            }
+        } else if self.tabs > other.tabs {
+            if self.spaces >= other.spaces {
+                Some(Ordering::Greater)
+            } else {
+                None
+            }
+        } else {
+            Some(self.spaces.cmp(&other.spaces))
+        }
+    }
+}
+
 pub struct Lexer<T: Iterator<Item = char>> {
     chars: T,
     at_begin_of_line: bool,
     nesting: usize, // Amount of parenthesis
-    indentation_stack: Vec<usize>,
+    indentation_stack: Vec<IndentationLevel>,
     pending: Vec<Spanned<Tok>>,
     chr0: Option<char>,
     chr1: Option<char>,
@@ -21,6 +54,8 @@ pub struct Lexer<T: Iterator<Item = char>> {
 #[derive(Debug)]
 pub enum LexicalError {
     StringError,
+    NestingError,
+    UnrecognizedToken { tok: char },
 }
 
 #[derive(Clone, Debug, Default, PartialEq)]
@@ -31,10 +66,7 @@ pub struct Location {
 
 impl Location {
     pub fn new(row: usize, column: usize) -> Self {
-        Location {
+        Location { row, column }
-            row: row,
-            column: column,
-        }
     }
 
     pub fn get_row(&self) -> usize {
@@ -93,8 +125,7 @@ pub type Spanned<Tok> = Result<(Location, Tok, Location), LexicalError>;
 pub fn make_tokenizer<'a>(source: &'a str) -> impl Iterator<Item = Spanned<Tok>> + 'a {
     let nlh = NewlineHandler::new(source.chars());
     let lch = LineContinationHandler::new(nlh);
-    let lexer = Lexer::new(lch);
+    Lexer::new(lch)
-    lexer
 }
 
 // The newline handler is an iterator which collapses different newline
@@ -111,7 +142,7 @@ where
 {
     pub fn new(source: T) -> Self {
         let mut nlh = NewlineHandler {
-            source: source,
+            source,
             chr0: None,
             chr1: None,
         };
@@ -167,7 +198,7 @@ where
 {
     pub fn new(source: T) -> Self {
         let mut nlh = LineContinationHandler {
-            source: source,
+            source,
             chr0: None,
             chr1: None,
         };
@@ -218,7 +249,7 @@ where
             chars: input,
             at_begin_of_line: true,
             nesting: 0,
-            indentation_stack: vec![0],
+            indentation_stack: vec![IndentationLevel::new()],
             pending: Vec::new(),
             chr0: None,
             location: Location::new(0, 0),
@@ -244,7 +275,6 @@ where
         let mut saw_f = false;
         loop {
             // Detect r"", f"", b"" and u""
-            // TODO: handle f-strings
             if !(saw_b || saw_u || saw_f) && (self.chr0 == Some('b') || self.chr0 == Some('B')) {
                 saw_b = true;
             } else if !(saw_b || saw_r || saw_u || saw_f)
@@ -280,7 +310,7 @@ where
         if keywords.contains_key(&name) {
             Ok((start_pos, keywords.remove(&name).unwrap(), end_pos))
         } else {
-            Ok((start_pos, Tok::Name { name: name }, end_pos))
+            Ok((start_pos, Tok::Name { name }, end_pos))
         }
     }
 
@@ -325,7 +355,7 @@ where
 
         let end_pos = self.get_pos();
         let value = BigInt::from_str_radix(&value_text, radix).unwrap();
-        Ok((start_pos, Tok::Int { value: value }, end_pos))
+        Ok((start_pos, Tok::Int { value }, end_pos))
     }
 
     fn lex_normal_number(&mut self) -> Spanned<Tok> {
@@ -377,7 +407,7 @@ where
                 ))
             } else {
                 let end_pos = self.get_pos();
-                Ok((start_pos, Tok::Float { value: value }, end_pos))
+                Ok((start_pos, Tok::Float { value }, end_pos))
             }
         } else {
             // Parse trailing 'j':
@@ -385,18 +415,11 @@ where
                 self.next_char();
                 let end_pos = self.get_pos();
                 let imag = f64::from_str(&value_text).unwrap();
-                Ok((
+                Ok((start_pos, Tok::Complex { real: 0.0, imag }, end_pos))
-                    start_pos,
-                    Tok::Complex {
-                        real: 0.0,
-                        imag: imag,
-                    },
-                    end_pos,
-                ))
             } else {
                 let end_pos = self.get_pos();
                 let value = value_text.parse::<BigInt>().unwrap();
-                Ok((start_pos, Tok::Int { value: value }, end_pos))
+                Ok((start_pos, Tok::Int { value }, end_pos))
             }
         }
     }
@@ -406,9 +429,7 @@ where
         self.next_char();
         loop {
             match self.chr0 {
-                Some('\n') => {
+                Some('\n') => return,
-                    return;
-                }
                 Some(_) => {}
                 None => return,
             }
@@ -421,7 +442,7 @@ where
         is_bytes: bool,
         is_raw: bool,
         _is_unicode: bool,
-        _is_fstring: bool,
+        is_fstring: bool,
     ) -> Spanned<Tok> {
         let quote_char = self.next_char().unwrap();
         let mut string_content = String::new();
@@ -512,36 +533,37 @@ where
         } else {
             Tok::String {
                 value: string_content,
+                is_fstring,
             }
         };
 
-        return Ok((start_pos, tok, end_pos));
+        Ok((start_pos, tok, end_pos))
     }
 
     fn is_char(&self) -> bool {
         match self.chr0 {
-            Some('a'...'z') | Some('A'...'Z') | Some('_') | Some('0'...'9') => return true,
+            Some('a'..='z') | Some('A'..='Z') | Some('_') | Some('0'..='9') => true,
-            _ => return false,
+            _ => false,
         }
     }
 
     fn is_number(&self, radix: u32) -> bool {
         match radix {
             2 => match self.chr0 {
-                Some('0'...'1') => return true,
+                Some('0'..='1') => true,
-                _ => return false,
+                _ => false,
             },
             8 => match self.chr0 {
-                Some('0'...'7') => return true,
+                Some('0'..='7') => true,
-                _ => return false,
+                _ => false,
             },
             10 => match self.chr0 {
-                Some('0'...'9') => return true,
+                Some('0'..='9') => true,
-                _ => return false,
+                _ => false,
             },
             16 => match self.chr0 {
-                Some('0'...'9') | Some('a'...'f') | Some('A'...'F') => return true,
+                Some('0'..='9') | Some('a'..='f') | Some('A'..='F') => true,
-                _ => return false,
+                _ => false,
             },
             x => unimplemented!("Radix not implemented: {}", x),
         }
@@ -576,12 +598,23 @@ where
                 self.at_begin_of_line = false;
 
                 // Determine indentation:
-                let mut col: usize = 0;
+                let mut spaces: usize = 0;
+                let mut tabs: usize = 0;
                 loop {
                     match self.chr0 {
                         Some(' ') => {
                             self.next_char();
-                            col += 1;
+                            spaces += 1;
+                        }
+                        Some('\t') => {
+                            if spaces != 0 {
+                                // Don't allow tabs after spaces as part of indentation.
+                                // This is technically stricter than python3 but spaces before
+                                // tabs is even more insane than mixing spaces and tabs.
+                                panic!("Tabs not allowed as part of indentation after spaces");
+                            }
+                            self.next_char();
+                            tabs += 1;
                         }
                         Some('#') => {
                             self.lex_comment();
@@ -601,41 +634,63 @@ where
                     }
                 }
 
+                let indentation_level = IndentationLevel { spaces, tabs };
+
                 if self.nesting == 0 {
                     // Determine indent or dedent:
                     let current_indentation = *self.indentation_stack.last().unwrap();
-                    if col == current_indentation {
+                    let ordering = indentation_level.compare_strict(&current_indentation);
-                        // Same same
+                    match ordering {
-                    } else if col > current_indentation {
+                        Some(Ordering::Equal) => {
-                        // New indentation level:
+                            // Same same
-                        self.indentation_stack.push(col);
+                        }
-                        let tok_start = self.get_pos();
+                        Some(Ordering::Greater) => {
-                        let tok_end = tok_start.clone();
+                            // New indentation level:
-                        return Some(Ok((tok_start, Tok::Indent, tok_end)));
+                            self.indentation_stack.push(indentation_level);
-                    } else if col < current_indentation {
-                        // One or more dedentations
-                        // Pop off other levels until col is found:
-
-                        while col < *self.indentation_stack.last().unwrap() {
-                            self.indentation_stack.pop().unwrap();
                             let tok_start = self.get_pos();
                             let tok_end = tok_start.clone();
-                            self.pending.push(Ok((tok_start, Tok::Dedent, tok_end)));
+                            return Some(Ok((tok_start, Tok::Indent, tok_end)));
                         }
+                        Some(Ordering::Less) => {
+                            // One or more dedentations
+                            // Pop off other levels until col is found:
 
-                        if col != *self.indentation_stack.last().unwrap() {
+                            loop {
-                            // TODO: handle wrong indentations
+                                let ordering = indentation_level
-                            panic!("Non matching indentation levels!");
+                                    .compare_strict(self.indentation_stack.last().unwrap());
+                                match ordering {
+                                    Some(Ordering::Less) => {
+                                        self.indentation_stack.pop();
+                                        let tok_start = self.get_pos();
+                                        let tok_end = tok_start.clone();
+                                        self.pending.push(Ok((tok_start, Tok::Dedent, tok_end)));
+                                    }
+                                    None => {
+                                        panic!("inconsistent use of tabs and spaces in indentation")
+                                    }
+                                    _ => {
+                                        break;
+                                    }
+                                };
+                            }
+
+                            if indentation_level != *self.indentation_stack.last().unwrap() {
+                                // TODO: handle wrong indentations
+                                panic!("Non matching indentation levels!");
+                            }
+
+                            return Some(self.pending.remove(0));
                         }
-
+                        None => panic!("inconsistent use of tabs and spaces in indentation"),
-                        return Some(self.pending.remove(0));
                     }
                 }
             }
 
             match self.chr0 {
-                Some('0'...'9') => return Some(self.lex_number()),
+                Some('0'..='9') => return Some(self.lex_number()),
-                Some('_') | Some('a'...'z') | Some('A'...'Z') => return Some(self.lex_identifier()),
+                Some('_') | Some('a'..='z') | Some('A'..='Z') => {
+                    return Some(self.lex_identifier());
+                }
                 Some('#') => {
                     self.lex_comment();
                     continue;
@@ -664,16 +719,13 @@ where
                 Some('+') => {
                     let tok_start = self.get_pos();
                     self.next_char();
-                    match self.chr0 {
+                    if let Some('=') = self.chr0 {
-                        Some('=') => {
+                        self.next_char();
-                            self.next_char();
+                        let tok_end = self.get_pos();
-                            let tok_end = self.get_pos();
+                        return Some(Ok((tok_start, Tok::PlusEqual, tok_end)));
-                            return Some(Ok((tok_start, Tok::PlusEqual, tok_end)));
+                    } else {
-                        }
+                        let tok_end = self.get_pos();
-                        _ => {
+                        return Some(Ok((tok_start, Tok::Plus, tok_end)));
-                            let tok_end = self.get_pos();
-                            return Some(Ok((tok_start, Tok::Plus, tok_end)));
-                        }
                     }
                 }
                 Some('*') => {
@@ -737,61 +789,49 @@ where
                 Some('%') => {
                     let tok_start = self.get_pos();
                     self.next_char();
-                    match self.chr0 {
+                    if let Some('=') = self.chr0 {
-                        Some('=') => {
+                        self.next_char();
-                            self.next_char();
+                        let tok_end = self.get_pos();
-                            let tok_end = self.get_pos();
+                        return Some(Ok((tok_start, Tok::PercentEqual, tok_end)));
-                            return Some(Ok((tok_start, Tok::PercentEqual, tok_end)));
+                    } else {
-                        }
+                        let tok_end = self.get_pos();
-                        _ => {
+                        return Some(Ok((tok_start, Tok::Percent, tok_end)));
-                            let tok_end = self.get_pos();
-                            return Some(Ok((tok_start, Tok::Percent, tok_end)));
-                        }
                     }
                 }
                 Some('|') => {
                     let tok_start = self.get_pos();
                     self.next_char();
-                    match self.chr0 {
+                    if let Some('=') = self.chr0 {
-                        Some('=') => {
+                        self.next_char();
-                            self.next_char();
+                        let tok_end = self.get_pos();
-                            let tok_end = self.get_pos();
+                        return Some(Ok((tok_start, Tok::VbarEqual, tok_end)));
-                            return Some(Ok((tok_start, Tok::VbarEqual, tok_end)));
+                    } else {
-                        }
+                        let tok_end = self.get_pos();
-                        _ => {
+                        return Some(Ok((tok_start, Tok::Vbar, tok_end)));
-                            let tok_end = self.get_pos();
-                            return Some(Ok((tok_start, Tok::Vbar, tok_end)));
-                        }
                     }
                 }
                 Some('^') => {
                     let tok_start = self.get_pos();
                     self.next_char();
-                    match self.chr0 {
+                    if let Some('=') = self.chr0 {
-                        Some('=') => {
+                        self.next_char();
-                            self.next_char();
+                        let tok_end = self.get_pos();
-                            let tok_end = self.get_pos();
+                        return Some(Ok((tok_start, Tok::CircumflexEqual, tok_end)));
-                            return Some(Ok((tok_start, Tok::CircumflexEqual, tok_end)));
+                    } else {
-                        }
+                        let tok_end = self.get_pos();
-                        _ => {
+                        return Some(Ok((tok_start, Tok::CircumFlex, tok_end)));
-                            let tok_end = self.get_pos();
-                            return Some(Ok((tok_start, Tok::CircumFlex, tok_end)));
-                        }
                     }
                 }
                 Some('&') => {
                     let tok_start = self.get_pos();
                     self.next_char();
-                    match self.chr0 {
+                    if let Some('=') = self.chr0 {
-                        Some('=') => {
+                        self.next_char();
-                            self.next_char();
+                        let tok_end = self.get_pos();
-                            let tok_end = self.get_pos();
+                        return Some(Ok((tok_start, Tok::AmperEqual, tok_end)));
-                            return Some(Ok((tok_start, Tok::AmperEqual, tok_end)));
+                    } else {
-                        }
+                        let tok_end = self.get_pos();
-                        _ => {
+                        return Some(Ok((tok_start, Tok::Amper, tok_end)));
-                            let tok_end = self.get_pos();
-                            return Some(Ok((tok_start, Tok::Amper, tok_end)));
-                        }
                     }
                 }
                 Some('-') => {
@@ -817,16 +857,13 @@ where
                 Some('@') => {
                     let tok_start = self.get_pos();
                     self.next_char();
-                    match self.chr0 {
+                    if let Some('=') = self.chr0 {
-                        Some('=') => {
+                        self.next_char();
-                            self.next_char();
+                        let tok_end = self.get_pos();
-                            let tok_end = self.get_pos();
+                        return Some(Ok((tok_start, Tok::AtEqual, tok_end)));
-                            return Some(Ok((tok_start, Tok::AtEqual, tok_end)));
+                    } else {
-                        }
+                        let tok_end = self.get_pos();
-                        _ => {
+                        return Some(Ok((tok_start, Tok::At, tok_end)));
-                            let tok_end = self.get_pos();
-                            return Some(Ok((tok_start, Tok::At, tok_end)));
-                        }
                     }
                 }
                 Some('!') => {
@@ -851,6 +888,9 @@ where
                 }
                 Some(')') => {
                     let result = self.eat_single_char(Tok::Rpar);
+                    if self.nesting == 0 {
+                        return Some(Err(LexicalError::NestingError));
+                    }
                     self.nesting -= 1;
                     return Some(result);
                 }
@@ -861,6 +901,9 @@ where
                 }
                 Some(']') => {
                     let result = self.eat_single_char(Tok::Rsqb);
+                    if self.nesting == 0 {
+                        return Some(Err(LexicalError::NestingError));
+                    }
                     self.nesting -= 1;
                     return Some(result);
                 }
@@ -871,6 +914,9 @@ where
                 }
                 Some('}') => {
                     let result = self.eat_single_char(Tok::Rbrace);
+                    if self.nesting == 0 {
+                        return Some(Err(LexicalError::NestingError));
+                    }
                     self.nesting -= 1;
                     return Some(result);
                 }
@@ -947,8 +993,15 @@ where
                 Some('.') => {
                     let tok_start = self.get_pos();
                     self.next_char();
-                    let tok_end = self.get_pos();
+                    if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) {
-                    return Some(Ok((tok_start, Tok::Dot, tok_end)));
+                        self.next_char();
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        return Some(Ok((tok_start, Tok::Ellipsis, tok_end)));
+                    } else {
+                        let tok_end = self.get_pos();
+                        return Some(Ok((tok_start, Tok::Dot, tok_end)));
+                    }
                 }
                 Some('\n') => {
                     let tok_start = self.get_pos();
@@ -972,7 +1025,7 @@ where
                 None => return None,
                 _ => {
                     let c = self.next_char();
-                    panic!("Not impl {:?}", c)
+                    return Some(Err(LexicalError::UnrecognizedToken { tok: c.unwrap() }));
                 } // Ignore all the rest..
             }
         }
@@ -1047,9 +1100,11 @@ mod tests {
             vec![
                 Tok::String {
                     value: "\\\\".to_string(),
+                    is_fstring: false,
                 },
                 Tok::String {
                     value: "\\".to_string(),
+                    is_fstring: false,
                 }
             ]
         );
@@ -1233,12 +1288,56 @@ mod tests {
         }
     }
 
+    macro_rules! test_double_dedent_with_tabs {
+        ($($name:ident: $eol:expr,)*) => {
+        $(
+            #[test]
+            fn $name() {
+                let source = String::from(format!("def foo():{}\tif x:{}{}\t return 99{}{}", $eol, $eol, $eol, $eol, $eol));
+                let tokens = lex_source(&source);
+                assert_eq!(
+                    tokens,
+                    vec![
+                        Tok::Def,
+                        Tok::Name {
+                            name: String::from("foo"),
+                        },
+                        Tok::Lpar,
+                        Tok::Rpar,
+                        Tok::Colon,
+                        Tok::Newline,
+                        Tok::Indent,
+                        Tok::If,
+                        Tok::Name {
+                            name: String::from("x"),
+                        },
+                        Tok::Colon,
+                        Tok::Newline,
+                        Tok::Indent,
+                        Tok::Return,
+                        Tok::Int { value: BigInt::from(99) },
+                        Tok::Newline,
+                        Tok::Dedent,
+                        Tok::Dedent,
+                    ]
+                );
+            }
+        )*
+        }
+    }
+
     test_double_dedent_with_eol! {
         test_double_dedent_windows_eol: WINDOWS_EOL,
         test_double_dedent_mac_eol: MAC_EOL,
         test_double_dedent_unix_eol: UNIX_EOL,
     }
 
+    test_double_dedent_with_tabs! {
+        test_double_dedent_tabs_windows_eol: WINDOWS_EOL,
+        test_double_dedent_tabs_mac_eol: MAC_EOL,
+        test_double_dedent_tabs_unix_eol: UNIX_EOL,
+    }
+
     macro_rules! test_newline_in_brackets {
         ($($name:ident: $eol:expr,)*) => {
         $(
@@ -1297,21 +1396,27 @@ mod tests {
             vec![
                 Tok::String {
                     value: String::from("double"),
+                    is_fstring: false,
                 },
                 Tok::String {
                     value: String::from("single"),
+                    is_fstring: false,
                 },
                 Tok::String {
                     value: String::from("can't"),
+                    is_fstring: false,
                 },
                 Tok::String {
                     value: String::from("\\\""),
+                    is_fstring: false,
                 },
                 Tok::String {
                     value: String::from("\t\r\n"),
+                    is_fstring: false,
                 },
                 Tok::String {
                     value: String::from("\\g"),
+                    is_fstring: false,
                 },
             ]
         );
@@ -1329,6 +1434,7 @@ mod tests {
                     vec![
                         Tok::String {
                             value: String::from("abcdef"),
+                            is_fstring: false,
                         },
                     ]
                 )

parser/src/lib.rs

@@ -1,13 +1,11 @@
 #[macro_use]
 extern crate log;
 
-extern crate num_bigint;
-extern crate num_traits;
-
 pub mod ast;
+pub mod error;
+mod fstring;
 pub mod lexer;
 pub mod parser;
+#[cfg_attr(rustfmt, rustfmt_skip)]
 mod python;
 pub mod token;
-
-pub use self::parser::parse;

parser/src/parser.rs

@@ -1,30 +1,10 @@
-extern crate lalrpop_util;
-
-use std::error::Error;
-use std::fs::File;
-use std::io::Read;
 use std::iter;
-use std::path::Path;
 
-use super::ast;
+use crate::ast;
-use super::lexer;
+use crate::error::ParseError;
-use super::python;
+use crate::lexer;
-use super::token;
+use crate::python;
-
+use crate::token;
-pub fn read_file(filename: &Path) -> Result<String, String> {
-    info!("Loading file {:?}", filename);
-    match File::open(&filename) {
-        Ok(mut file) => {
-            let mut s = String::new();
-
-            match file.read_to_string(&mut s) {
-                Err(why) => Err(String::from("Reading file failed: ") + why.description()),
-                Ok(_) => Ok(s),
-            }
-        }
-        Err(why) => Err(String::from("Opening file failed: ") + why.description()),
-    }
-}
 
 /*
  * Parse python code.
@@ -32,13 +12,6 @@ pub fn read_file(filename: &Path) -> Result<String, String> {
  * https://github.com/antlr/grammars-v4/tree/master/python3
  */
 
-pub fn parse(filename: &Path) -> Result<ast::Program, String> {
-    info!("Parsing: {}", filename.display());
-    let txt = read_file(filename)?;
-    debug!("Read contents of file: {}", txt);
-    parse_program(&txt)
-}
-
 macro_rules! do_lalr_parsing {
     ($input: expr, $pat: ident, $tok: ident) => {{
         let lxr = lexer::make_tokenizer($input);
@@ -46,7 +19,7 @@ macro_rules! do_lalr_parsing {
         let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
 
         match python::TopParser::new().parse(tokenizer) {
-            Err(why) => Err(format!("{:?}", why)),
+            Err(err) => Err(ParseError::from(err)),
             Ok(top) => {
                 if let ast::Top::$pat(x) = top {
                     Ok(x)
@@ -58,11 +31,11 @@ macro_rules! do_lalr_parsing {
     }};
 }
 
-pub fn parse_program(source: &str) -> Result<ast::Program, String> {
+pub fn parse_program(source: &str) -> Result<ast::Program, ParseError> {
     do_lalr_parsing!(source, Program, StartProgram)
 }
 
-pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, String> {
+pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, ParseError> {
     do_lalr_parsing!(source, Statement, StartStatement)
 }
 
@@ -88,7 +61,7 @@ pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, String> {
 ///     expr);
 ///
 /// ```
-pub fn parse_expression(source: &str) -> Result<ast::Expression, String> {
+pub fn parse_expression(source: &str) -> Result<ast::Expression, ParseError> {
     do_lalr_parsing!(source, Expression, StartExpression)
 }
 
@@ -103,7 +76,6 @@ mod tests {
     #[test]
     fn test_parse_empty() {
         let parse_ast = parse_program(&String::from("\n"));
-
         assert_eq!(parse_ast, Ok(ast::Program { statements: vec![] }))
     }
 
@@ -122,7 +94,9 @@ mod tests {
                                 name: String::from("print"),
                             }),
                             args: vec![ast::Expression::String {
-                                value: String::from("Hello world"),
+                                value: ast::StringGroup::Constant {
+                                    value: String::from("Hello world")
+                                }
                             }],
                             keywords: vec![],
                         },
@@ -148,7 +122,9 @@ mod tests {
                             }),
                             args: vec![
                                 ast::Expression::String {
-                                    value: String::from("Hello world"),
+                                    value: ast::StringGroup::Constant {
+                                        value: String::from("Hello world"),
+                                    }
                                 },
                                 ast::Expression::Number {
                                     value: ast::Number::Integer {
@@ -179,7 +155,9 @@ mod tests {
                                 name: String::from("my_func"),
                             }),
                             args: vec![ast::Expression::String {
-                                value: String::from("positional"),
+                                value: ast::StringGroup::Constant {
+                                    value: String::from("positional"),
+                                }
                             }],
                             keywords: vec![ast::Keyword {
                                 name: Some("keyword".to_string()),
@@ -266,7 +244,16 @@ mod tests {
                 node: ast::Statement::Expression {
                     expression: ast::Expression::Lambda {
                         args: ast::Parameters {
-                            args: vec![String::from("x"), String::from("y")],
+                            args: vec![
+                                ast::Parameter {
+                                    arg: String::from("x"),
+                                    annotation: None,
+                                },
+                                ast::Parameter {
+                                    arg: String::from("y"),
+                                    annotation: None,
+                                }
+                            ],
                             kwonlyargs: vec![],
                             vararg: None,
                             kwarg: None,
@@ -328,7 +315,9 @@ mod tests {
 
     #[test]
     fn test_parse_class() {
-        let source = String::from("class Foo(A, B):\n def __init__(self):\n  pass\n def method_with_default(self, arg='default'):\n  pass\n");
+        let source = String::from(
+            "class Foo(A, B):\n def __init__(self):\n  pass\n def method_with_default(self, arg='default'):\n  pass\n",
+        );
         assert_eq!(
             parse_statement(&source),
             Ok(ast::LocatedStatement {
@@ -350,7 +339,10 @@ mod tests {
                             node: ast::Statement::FunctionDef {
                                 name: String::from("__init__"),
                                 args: ast::Parameters {
-                                    args: vec![String::from("self")],
+                                    args: vec![ast::Parameter {
+                                        arg: String::from("self"),
+                                        annotation: None,
+                                    }],
                                     kwonlyargs: vec![],
                                     vararg: None,
                                     kwarg: None,
@@ -362,6 +354,7 @@ mod tests {
                                     node: ast::Statement::Pass,
                                 }],
                                 decorator_list: vec![],
+                                returns: None,
                             }
                         },
                         ast::LocatedStatement {
@@ -369,12 +362,23 @@ mod tests {
                             node: ast::Statement::FunctionDef {
                                 name: String::from("method_with_default"),
                                 args: ast::Parameters {
-                                    args: vec![String::from("self"), String::from("arg"),],
+                                    args: vec![
+                                        ast::Parameter {
+                                            arg: String::from("self"),
+                                            annotation: None,
+                                        },
+                                        ast::Parameter {
+                                            arg: String::from("arg"),
+                                            annotation: None,
+                                        }
+                                    ],
                                     kwonlyargs: vec![],
                                     vararg: None,
                                     kwarg: None,
                                     defaults: vec![ast::Expression::String {
-                                        value: "default".to_string()
+                                        value: ast::StringGroup::Constant {
+                                            value: "default".to_string()
+                                        }
                                     }],
                                     kw_defaults: vec![],
                                 },
@@ -383,6 +387,7 @@ mod tests {
                                     node: ast::Statement::Pass,
                                 }],
                                 decorator_list: vec![],
+                                returns: None,
                             }
                         }
                     ],

parser/src/python.lalrpop

@@ -4,9 +4,12 @@
 // See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword
 #![allow(unknown_lints,clippy)]
 
-use super::ast;
-use super::lexer;
 use std::iter::FromIterator;
+
+use crate::ast;
+use crate::fstring::parse_fstring;
+use crate::lexer;
+
 use num_bigint::BigInt;
 
 grammar;
@@ -105,7 +108,11 @@ ExpressionStatement: ast::LocatedStatement = {
         let rhs = e2.into_iter().next().unwrap();
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::AugAssign { target: expr, op: op, value: rhs },
+            node: ast::Statement::AugAssign {
+                target: Box::new(expr),
+                op,
+                value: Box::new(rhs)
+            },
         }
     },
 };
@@ -439,7 +446,7 @@ WithItem: ast::WithItem = {
 };
 
 FuncDef: ast::LocatedStatement = {
-    <d:Decorator*> <loc:@L> "def" <i:Identifier> <a:Parameters> ":" <s:Suite> => {
+    <d:Decorator*> <loc:@L> "def" <i:Identifier> <a:Parameters> <r:("->" Test)?> ":" <s:Suite> => {
         ast::LocatedStatement {
             location: loc,
             node: ast::Statement::FunctionDef {
@@ -447,13 +454,14 @@ FuncDef: ast::LocatedStatement = {
                 args: a,
                 body: s,
                 decorator_list: d,
+                returns: r.map(|x| x.1),
             }
         }
     },
 };
 
 Parameters: ast::Parameters = {
-    "(" <a: (TypedArgsList)?> ")" => {
+    "(" <a: (TypedArgsList<TypedParameter>)?> ")" => {
         match a {
             Some(a) => a,
             None => Default::default(),
@@ -463,8 +471,10 @@ Parameters: ast::Parameters = {
 
 // parameters are (String, None), kwargs are (String, Some(Test)) where Test is
 // the default
-TypedArgsList: ast::Parameters = {
+// Note that this is a macro which is used once for function defs, and
-    <param1:TypedParameters> <args2:("," ParameterListStarArgs)?> => {
+// once for lambda defs.
+TypedArgsList<ArgType>: ast::Parameters = {
+    <param1:TypedParameters<ArgType>> <args2:("," ParameterListStarArgs<ArgType>)?> => {
         let (names, default_elements) = param1;
 
         // Now gather rest of parameters:
@@ -482,7 +492,7 @@ TypedArgsList: ast::Parameters = {
             kw_defaults: kw_defaults,
         }
     },
-    <param1:TypedParameters> <kw:("," KwargParameter)> => {
+    <param1:TypedParameters<ArgType>> <kw:("," KwargParameter<ArgType>)> => {
         let (names, default_elements) = param1;
 
         // Now gather rest of parameters:
@@ -500,7 +510,7 @@ TypedArgsList: ast::Parameters = {
             kw_defaults: kw_defaults,
         }
     },
-    <params:ParameterListStarArgs> => {
+    <params:ParameterListStarArgs<ArgType>> => {
         let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
         ast::Parameters {
             args: vec![],
@@ -511,7 +521,7 @@ TypedArgsList: ast::Parameters = {
             kw_defaults: kw_defaults,
         }
     },
-    <kw:KwargParameter> => {
+    <kw:KwargParameter<ArgType>> => {
         ast::Parameters {
             args: vec![],
             kwonlyargs: vec![],
@@ -525,8 +535,8 @@ TypedArgsList: ast::Parameters = {
 
 // Use inline here to make sure the "," is not creating an ambiguity.
 #[inline]
-TypedParameters: (Vec<String>, Vec<ast::Expression>) = {
+TypedParameters<ArgType>: (Vec<ast::Parameter>, Vec<ast::Expression>) = {
-    <param1:TypedParameterDef> <param2:("," TypedParameterDef)*> => {
+    <param1:TypedParameterDef<ArgType>> <param2:("," TypedParameterDef<ArgType>)*> => {
         // Combine first parameters:
         let mut args = vec![param1];
         args.extend(param2.into_iter().map(|x| x.1));
@@ -535,7 +545,6 @@ TypedParameters: (Vec<String>, Vec<ast::Expression>) = {
         let mut default_elements = vec![];
 
         for (name, default) in args.into_iter() {
-            names.push(name.clone());
             if let Some(default) = default {
                 default_elements.push(default);
             } else {
@@ -544,28 +553,35 @@ TypedParameters: (Vec<String>, Vec<ast::Expression>) = {
                     // have defaults
                     panic!(
                         "non-default argument follows default argument: {}",
-                        name
+                        &name.arg
                     );
                 }
             }
+            names.push(name);
         }
 
         (names, default_elements)
     }
 };
 
-TypedParameterDef: (String, Option<ast::Expression>) = {
+TypedParameterDef<ArgType>: (ast::Parameter, Option<ast::Expression>) = {
-    <i:TypedParameter> => (i, None),
+    <i:ArgType> => (i, None),
-    <i:TypedParameter> "=" <e:Test> => (i, Some(e)),
+    <i:ArgType> "=" <e:Test> => (i, Some(e)),
 };
 
-// TODO: add type annotations here:
+UntypedParameter: ast::Parameter = {
-TypedParameter: String = {
+    <i:Identifier> => ast::Parameter { arg: i, annotation: None },
-    Identifier,
 };
 
-ParameterListStarArgs: (Option<Option<String>>, Vec<String>, Vec<Option<ast::Expression>>, Option<Option<String>>) = {
+TypedParameter: ast::Parameter = {
-    "*" <va:Identifier?> <kw:("," TypedParameterDef)*> <kwarg:("," KwargParameter)?> => {
+    <arg:Identifier> <a:(":" Test)?>=> {
+        let annotation = a.map(|x| Box::new(x.1));
+        ast::Parameter { arg, annotation }
+    },
+};
+
+ParameterListStarArgs<ArgType>: (Option<Option<ast::Parameter>>, Vec<ast::Parameter>, Vec<Option<ast::Expression>>, Option<Option<ast::Parameter>>) = {
+    "*" <va:ArgType?> <kw:("," TypedParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> => {
         // Extract keyword arguments:
         let mut kwonlyargs = vec![];
         let mut kw_defaults = vec![];
@@ -583,8 +599,8 @@ ParameterListStarArgs: (Option<Option<String>>, Vec<String>, Vec<Option<ast::Exp
     }
 };
 
-KwargParameter: Option<String> = {
+KwargParameter<ArgType>: Option<ast::Parameter> = {
-    "**" <kwarg:Identifier?> => {
+    "**" <kwarg:ArgType?> => {
         kwarg
     }
 };
@@ -608,17 +624,26 @@ ClassDef: ast::LocatedStatement = {
     },
 };
 
+Path: ast::Expression = {
+    <n:Identifier> => ast::Expression::Identifier { name: n },
+    <p:Path> "." <n:name> => {
+        ast::Expression::Attribute {
+            value: Box::new(p),
+            name: n,
+        }
+    },
+};
+
 // Decorators:
 Decorator: ast::Expression = {
-    "@" <n:DottedName> <a: ("(" ArgumentList ")")?> "\n" => {
+    "@" <p:Path> <a: ("(" ArgumentList ")")?> "\n" => {
-        let name = ast::Expression::Identifier { name: n };
         match a {
             Some((_, args, _)) => ast::Expression::Call {
-                function: Box::new(name),
+                function: Box::new(p),
                 args: args.0,
                 keywords: args.1,
             },
-            None => name,
+            None => p,
         }
     },
 };
@@ -659,7 +684,7 @@ Test: ast::Expression = {
 };
 
 LambdaDef: ast::Expression = {
-    "lambda" <p:TypedArgsList?> ":" <b:Expression> =>
+    "lambda" <p:TypedArgsList<UntypedParameter>?> ":" <b:Test> =>
         ast::Expression::Lambda {
             args: p.unwrap_or(Default::default()),
             body:Box::new(b)
@@ -785,7 +810,8 @@ SliceOp: ast::Expression = {
 }
 
 Atom: ast::Expression = {
-    StringConstant,
+    <s:StringGroup> => ast::Expression::String { value: s },
+    <b:Bytes> => ast::Expression::Bytes { value: b },
     <n:Number> => ast::Expression::Number { value: n },
     <i:Identifier> => ast::Expression::Identifier { name: i },
     "[" <e:TestListComp?> "]" => {
@@ -822,6 +848,7 @@ Atom: ast::Expression = {
     "True" => ast::Expression::True,
     "False" => ast::Expression::False,
     "None" => ast::Expression::None,
+    "..." => ast::Expression::Ellipsis,
 };
 
 TestListComp: Vec<ast::Expression> = {
@@ -988,14 +1015,28 @@ Number: ast::Number = {
     <s:complex> => { ast::Number::Complex { real: s.0, imag: s.1 } },
 };
 
-StringConstant: ast::Expression = {
+StringGroup: ast::StringGroup = {
-    <s:string+> => {
+    <s:string+> =>? {
-        let glued = s.join("");
+        let mut values = vec![];
-        ast::Expression::String { value: glued }
+        for (value, is_fstring) in s {
+            values.push(if is_fstring {
+                parse_fstring(&value)?
+            } else {
+                ast::StringGroup::Constant { value }
+            })
+        }
+
+        Ok(if values.len() > 1 {
+            ast::StringGroup::Joined { values }
+        } else {
+            values.into_iter().next().unwrap()
+        })
     },
+};
+
+Bytes: Vec<u8> = {
     <s:bytes+> => {
-        let glued = s.into_iter().flatten().collect::<Vec<u8>>();
+        s.into_iter().flatten().collect::<Vec<u8>>()
-        ast::Expression::Bytes { value: glued }
     },
 };
 
@@ -1017,6 +1058,7 @@ extern {
         "~" => lexer::Tok::Tilde,
         ":" => lexer::Tok::Colon,
         "." => lexer::Tok::Dot,
+        "..." => lexer::Tok::Ellipsis,
         "," => lexer::Tok::Comma,
         "*" => lexer::Tok::Star,
         "**" => lexer::Tok::DoubleStar,
@@ -1055,6 +1097,7 @@ extern {
         "<=" => lexer::Tok::LessEqual,
         ">" => lexer::Tok::Greater,
         ">=" => lexer::Tok::GreaterEqual,
+        "->" => lexer::Tok::Rarrow,
         "and" => lexer::Tok::And,
         "as" => lexer::Tok::As,
         "assert" => lexer::Tok::Assert,
@@ -1092,7 +1135,7 @@ extern {
         int => lexer::Tok::Int { value: <BigInt> },
         float => lexer::Tok::Float { value: <f64> },
         complex => lexer::Tok::Complex { real: <f64>, imag: <f64> },
-        string => lexer::Tok::String { value: <String> },
+        string => lexer::Tok::String { value: <String>, is_fstring: <bool> },
         bytes => lexer::Tok::Bytes { value: <Vec<u8>> },
         name => lexer::Tok::Name { name: <String> },
         "\n" => lexer::Tok::Newline,

parser/src/token.rs

@@ -9,7 +9,7 @@ pub enum Tok {
     Int { value: BigInt },
     Float { value: f64 },
     Complex { real: f64, imag: f64 },
-    String { value: String },
+    String { value: String, is_fstring: bool },
     Bytes { value: Vec<u8> },
     Newline,
     Indent,

py_code_object/Cargo.toml

@@ -2,6 +2,7 @@
 name = "py_code_object"
 version = "0.1.0"
 authors = ["Shing Lyu <shing.lyu@gmail.com>"]
+edition = "2018"
 
 [dependencies]
 log = "0.3"

py_code_object/python_compiler/Cargo.toml

@@ -2,6 +2,7 @@
 name = "python_compiler"
 version = "0.1.0"
 authors = ["Shing Lyu <shing.lyu@gmail.com>"]
+edition = "2018"
 
 [dependencies]
 cpython = { git = "https://github.com/dgrunwald/rust-cpython.git" }

py_code_object/src/vm_old.rs

@@ -56,7 +56,7 @@ impl VirtualMachine {
         }
     }
 
-    // Can we get rid of the code paramter?
+    // Can we get rid of the code parameter?
 
     fn make_frame(&self, code: PyCodeObject, callargs: HashMap<String, Rc<NativeType>>, globals: Option<HashMap<String, Rc<NativeType>>>) -> Frame {
         //populate the globals and locals
@@ -345,7 +345,7 @@ impl VirtualMachine {
                 let exception = match argc {
                     1 => curr_frame.stack.pop().unwrap(),
                     0 | 2 | 3 => panic!("Not implemented!"),
-                    _ => panic!("Invalid paramter for RAISE_VARARGS, must be between 0 to 3")
+                    _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3")
                 };
                 panic!("{:?}", exception);
             }

rustfmt.toml

@@ -0,0 +1 @@
+edition = "2018"

src/main.rs

@@ -1,4 +1,3 @@
-//extern crate rustpython_parser;
 #[macro_use]
 extern crate clap;
 extern crate env_logger;
@@ -9,16 +8,19 @@ extern crate rustpython_vm;
 extern crate rustyline;
 
 use clap::{App, Arg};
-use rustpython_parser::parser;
+use rustpython_parser::error::ParseError;
-use rustpython_vm::obj::objstr;
+use rustpython_vm::{
-use rustpython_vm::print_exception;
+    compile,
-use rustpython_vm::pyobject::{AttributeProtocol, PyObjectRef, PyResult};
+    error::CompileError,
-use rustpython_vm::VirtualMachine;
+    frame::ScopeRef,
-use rustpython_vm::{compile, import};
+    import,
-use rustyline::error::ReadlineError;
+    obj::objstr,
-use rustyline::Editor;
+    print_exception,
-use std::path::Path;
+    pyobject::{AttributeProtocol, PyResult},
-use std::path::PathBuf;
+    util, VirtualMachine,
+};
+use rustyline::{error::ReadlineError, Editor};
+use std::path::{Path, PathBuf};
 
 fn main() {
     env_logger::init();
@@ -60,7 +62,7 @@ fn main() {
         // Figure out if a script was passed:
         match matches.value_of("script") {
             None => run_shell(&mut vm),
-            Some(filename) => run_script(&mut vm, &filename.to_string()),
+            Some(filename) => run_script(&mut vm, filename),
         }
     };
 
@@ -68,8 +70,17 @@ fn main() {
     handle_exception(&mut vm, result);
 }
 
-fn _run_string(vm: &mut VirtualMachine, source: &str, source_path: Option<String>) -> PyResult {
+fn _run_string(vm: &mut VirtualMachine, source: &str, source_path: String) -> PyResult {
-    let code_obj = compile::compile(vm, &source.to_string(), compile::Mode::Exec, source_path)?;
+    let code_obj = compile::compile(
+        source,
+        &compile::Mode::Exec,
+        source_path,
+        vm.ctx.code_type(),
+    )
+    .map_err(|err| {
+        let syntax_error = vm.context().exceptions.syntax_error.clone();
+        vm.new_exception(syntax_error, err.to_string())
+    })?;
     // trace!("Code object: {:?}", code_obj.borrow());
     let builtins = vm.get_builtin_scope();
     let vars = vm.context().new_scope(Some(builtins)); // Keep track of local variables
@@ -77,12 +88,9 @@ fn _run_string(vm: &mut VirtualMachine, source: &str, source_path: Option<String
 }
 
 fn handle_exception(vm: &mut VirtualMachine, result: PyResult) {
-    match result {
+    if let Err(err) = result {
-        Ok(_value) => {}
+        print_exception(vm, &err);
-        Err(err) => {
+        std::process::exit(1);
-            print_exception(vm, &err);
-            std::process::exit(1);
-        }
     }
 }
 
@@ -90,8 +98,8 @@ fn run_command(vm: &mut VirtualMachine, mut source: String) -> PyResult {
     debug!("Running command {}", source);
 
     // This works around https://github.com/RustPython/RustPython/issues/17
-    source.push_str("\n");
+    source.push('\n');
-    _run_string(vm, &source, None)
+    _run_string(vm, &source, "<stdin>".to_string())
 }
 
 fn run_module(vm: &mut VirtualMachine, module: &str) -> PyResult {
@@ -103,43 +111,53 @@ fn run_module(vm: &mut VirtualMachine, module: &str) -> PyResult {
 fn run_script(vm: &mut VirtualMachine, script_file: &str) -> PyResult {
     debug!("Running file {}", script_file);
     // Parse an ast from it:
-    let filepath = Path::new(script_file);
+    let file_path = Path::new(script_file);
-    match parser::read_file(filepath) {
+    match util::read_file(file_path) {
-        Ok(source) => _run_string(vm, &source, Some(filepath.to_str().unwrap().to_string())),
+        Ok(source) => _run_string(vm, &source, file_path.to_str().unwrap().to_string()),
-        Err(msg) => {
+        Err(err) => {
-            error!("Parsing went horribly wrong: {}", msg);
+            error!("Failed reading file: {:?}", err.kind());
             std::process::exit(1);
         }
     }
 }
 
-fn shell_exec(vm: &mut VirtualMachine, source: &str, scope: PyObjectRef) -> bool {
+fn shell_exec(vm: &mut VirtualMachine, source: &str, scope: ScopeRef) -> Result<(), CompileError> {
-    match compile::compile(vm, &source.to_string(), compile::Mode::Single, None) {
+    match compile::compile(
+        source,
+        &compile::Mode::Single,
+        "<stdin>".to_string(),
+        vm.ctx.code_type(),
+    ) {
         Ok(code) => {
-            match vm.run_code_obj(code, scope) {
+            if let Err(err) = vm.run_code_obj(code, scope) {
-                Ok(_value) => {
-                    // Printed already.
-                }
-                Err(err) => {
-                    print_exception(vm, &err);
-                }
-            }
-        }
-        Err(err) => {
-            // Enum rather than special string here.
-            let name = vm.new_str("msg".to_string());
-            let msg = match vm.get_attribute(err.clone(), name) {
-                Ok(value) => objstr::get_value(&value),
-                Err(_) => panic!("Expected msg attribute on exception object!"),
-            };
-            if msg == "Unexpected end of input." {
-                return false;
-            } else {
                 print_exception(vm, &err);
             }
+            Ok(())
         }
-    };
+        // Don't inject syntax errors for line continuation
-    true
+        Err(err @ CompileError::Parse(ParseError::EOF(_))) => Err(err),
+        Err(err) => {
+            let syntax_error = vm.context().exceptions.syntax_error.clone();
+            let exc = vm.new_exception(syntax_error, format!("{}", err));
+            print_exception(vm, &exc);
+            Err(err)
+        }
+    }
+}
+
+#[cfg(not(unix))]
+fn get_history_path() -> PathBuf {
+    PathBuf::from(".repl_history.txt")
+}
+
+#[cfg(unix)]
+fn get_history_path() -> PathBuf {
+    //work around for windows dependent builds. The xdg crate is unix specific
+    //so access to the BaseDirectories struct breaks builds on python.
+    extern crate xdg;
+
+    let xdg_dirs = xdg::BaseDirectories::with_prefix("rustpython").unwrap();
+    xdg_dirs.place_cache_file("repl_history.txt").unwrap()
 }
 
 fn run_shell(vm: &mut VirtualMachine) -> PyResult {
@@ -152,57 +170,34 @@ fn run_shell(vm: &mut VirtualMachine) -> PyResult {
 
     // Read a single line:
     let mut input = String::new();
-    let mut rl = Editor::<()>::new();
+    let mut repl = Editor::<()>::new();
 
-    // TODO: Store the history in a proper XDG directory
+    // Retrieve a `history_path_str` dependent on the OS
-    let repl_history_path = ".repl_history.txt";
+    let repl_history_path_str = &get_history_path();
-    if rl.load_history(repl_history_path).is_err() {
+    if repl.load_history(repl_history_path_str).is_err() {
         println!("No previous history.");
     }
 
+    let ps1 = &objstr::get_value(&vm.sys_module.get_attr("ps1").unwrap());
+    let ps2 = &objstr::get_value(&vm.sys_module.get_attr("ps2").unwrap());
+    let mut prompt = ps1;
+
     loop {
-        // TODO: modules dont support getattr / setattr yet
+        match repl.readline(prompt) {
-        //let prompt = match vm.get_attribute(vm.sys_module.clone(), "ps1") {
-        //        Ok(value) => objstr::get_value(&value),
-        //        Err(_) => ">>>>> ".to_string(),
-        //};
-
-        // We can customize the prompt:
-        let ps1 = objstr::get_value(&vm.sys_module.get_attr("ps1").unwrap());
-        let ps2 = objstr::get_value(&vm.sys_module.get_attr("ps2").unwrap());
-
-        match rl.readline(&ps1) {
             Ok(line) => {
+                debug!("You entered {:?}", line);
                 input.push_str(&line);
                 input.push_str("\n");
+                repl.add_history_entry(line.trim_end().as_ref());
 
-                debug!("You entered {:?}", input);
+                match shell_exec(vm, &input, vars.clone()) {
-                if shell_exec(vm, &input, vars.clone()) {
+                    Err(CompileError::Parse(ParseError::EOF(_))) => {
-                    // Line was complete.
+                        prompt = ps2;
-                    rl.add_history_entry(input.trim_right().as_ref());
+                        continue;
-                    input = String::new();
+                    }
-                } else {
+                    _ => {
-                    loop {
+                        prompt = ps1;
-                        // until an empty line is pressed AND the code is complete
+                        input = String::new();
-                        //let prompt = match vm.get_attribute(vm.sys_module.clone(), "ps2") {
-                        //        Ok(value) => objstr::get_value(&value),
-                        //        Err(_) => "..... ".to_string(),
-                        //};
-                        match rl.readline(&ps2) {
-                            Ok(line) => {
-                                if line.len() == 0 {
-                                    if shell_exec(vm, &input, vars.clone()) {
-                                        rl.add_history_entry(input.trim_right().as_ref());
-                                        input = String::new();
-                                        break;
-                                    }
-                                } else {
-                                    input.push_str(&line);
-                                    input.push_str("\n");
-                                }
-                            }
-                            Err(msg) => panic!("Error: {:?}", msg),
-                        }
                     }
                 }
             }
@@ -220,7 +215,7 @@ fn run_shell(vm: &mut VirtualMachine) -> PyResult {
             }
         };
     }
-    rl.save_history(repl_history_path).unwrap();
+    repl.save_history(repl_history_path_str).unwrap();
 
     Ok(vm.get_none())
 }

tests/.travis-runner.sh

@@ -10,7 +10,32 @@ pip install pipenv
 (cd tests; pipenv install)
 
 # Build outside of the test runner
-cargo build --verbose --release
+if [ $CODE_COVERAGE = "true" ]
+then
+    find . -name '*.gcda' -delete
+
+    export CARGO_INCREMENTAL=0
+    export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads"
+
+    cargo build --verbose
+else
+    cargo build --verbose --release
+fi
 
 # Run the tests
 (cd tests; pipenv run pytest)
+
+if [ $CODE_COVERAGE = "true" ]
+then
+    cargo test --verbose --all
+    zip -0 ccov.zip `find . \( -name "rustpython*.gc*" \) -print`
+
+    # Install grcov
+    curl -L https://github.com/mozilla/grcov/releases/download/v0.4.1/grcov-linux-x86_64.tar.bz2 | tar jxf -
+
+    ./grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir "/*" -p "x" > lcov.info
+
+    # Install codecov.io reporter
+    curl -s https://codecov.io/bash -o codecov.sh
+    bash codecov.sh -f lcov.info
+fi

tests/snippets/basic_types.py

@@ -44,7 +44,12 @@ assert int() == 0
 a = complex(2, 4)
 assert type(a) is complex
 assert type(a + a) is complex
+assert repr(a) == '(2+4j)'
+a = 10j
+assert repr(a) == '10j'
 
+a = 1
+assert a.conjugate() == a
 
 a = 12345
 

tests/snippets/bools.py

@@ -16,6 +16,9 @@ assert bool() == False
 assert bool(1) == True
 assert bool({}) == False
 
+assert bool(NotImplemented) == True
+assert bool(...) == True
+
 if not 1:
     raise BaseException
 
@@ -46,3 +49,5 @@ assert True + True == 2
 assert False * 7 == 0
 assert True > 0
 assert int(True) == 1
+assert True.conjugate() == 1
+assert isinstance(True.conjugate(), int)

tests/snippets/builtin_callable.py

@@ -0,0 +1,26 @@
+assert not callable(1)
+def f(): pass
+# TODO uncomment when callable types get unified __call__ (or equivalent)
+#assert callable(f)
+#assert callable(len)
+#assert callable(lambda: 1)
+assert callable(int)
+
+class C:
+    def __init__(self):
+        # must be defined on class
+        self.__call__ = lambda self: 1
+    def f(self): pass
+assert callable(C)
+assert not callable(C())
+#assert callable(C().f)
+
+class C:
+    def __call__(self): pass
+assert callable(C())
+class C1(C): pass
+assert callable(C1())
+class C:
+    __call__ = 1
+# CPython returns true here, but fails when actually calling it
+assert callable(C())

tests/snippets/builtin_complex.py

@@ -0,0 +1,46 @@
+# __abs__
+
+assert abs(complex(3, 4)) == 5
+assert abs(complex(3, -4)) == 5
+assert abs(complex(1.5, 2.5)) == 2.9154759474226504
+
+# __eq__
+
+assert complex(1, -1) == complex(1, -1)
+assert complex(1, 0) == 1
+assert 1 == complex(1, 0)
+assert complex(1, 1) != 1
+assert 1 != complex(1, 1)
+assert complex(1, 0) == 1.0
+assert 1.0 == complex(1, 0)
+assert complex(1, 1) != 1.0
+assert 1.0 != complex(1, 1)
+assert complex(1, 0) != 1.5
+assert not 1.0 != complex(1, 0)
+assert bool(complex(1, 0))
+assert complex(1, 2) != complex(1, 1)
+assert complex(1, 2) != 'foo'
+assert complex(1, 2).__eq__('foo') == NotImplemented
+
+# __neg__
+
+assert -complex(1, -1) == complex(-1, 1)
+assert -complex(0, 0) == complex(0, 0)
+
+# real
+
+a = complex(3, 4)
+b = 4j
+assert a.real == 3
+assert b.real == 0
+
+# imag
+
+assert a.imag == 4
+assert b.imag == 4
+
+# int and complex addition
+assert 1 + 1j == complex(1, 1)
+assert 1j + 1 == complex(1, 1)
+assert (1j + 1) + 3 == complex(4, 1)
+assert 3 + (1j + 1) == complex(4, 1)

tests/snippets/builtin_dict.py

@@ -4,3 +4,7 @@ assert len({}) == 0
 assert len({"a": "b"}) == 1
 assert len({"a": "b", "b": 1}) == 2
 assert len({"a": "b", "b": 1, "a" + "b": 2*2}) == 3
+
+d = {}
+d['a'] = d
+assert repr(d) == "{'a': {...}}"

tests/snippets/builtin_dir.py

@@ -0,0 +1,12 @@
+
+class A:
+	def test():
+		pass
+
+a = A()
+
+assert "test" in dir(a)
+
+import socket
+
+assert "AF_INET" in dir(socket)

tests/snippets/builtin_divmod.py

@@ -1,3 +1,8 @@
+from testutils import assert_raises
+
 assert divmod(11, 3) == (3, 2)
 assert divmod(8,11) == (0, 8)
 assert divmod(0.873, 0.252) == (3.0, 0.11699999999999999)
+
+assert_raises(ZeroDivisionError, lambda: divmod(5, 0), 'divmod by zero')
+assert_raises(ZeroDivisionError, lambda: divmod(5.0, 0.0), 'divmod by zero')

tests/snippets/builtin_enumerate.py

@@ -0,0 +1,22 @@
+assert list(enumerate(['a', 'b', 'c'])) == [(0, 'a'), (1, 'b'), (2, 'c')]
+
+assert type(enumerate([])) == enumerate
+
+assert list(enumerate(['a', 'b', 'c'], -100)) == [(-100, 'a'), (-99, 'b'), (-98, 'c')]
+assert list(enumerate(['a', 'b', 'c'], 2**200)) == [(2**200, 'a'), (2**200 + 1, 'b'), (2**200 + 2, 'c')]
+
+# test infinite iterator
+class Counter(object):
+    counter = 0
+
+    def __next__(self):
+        self.counter += 1
+        return self.counter
+
+    def __iter__(self):
+        return self
+
+
+it = enumerate(Counter())
+assert next(it) == (0, 1)
+assert next(it) == (1, 2)

tests/snippets/builtin_filter.py

@@ -0,0 +1,32 @@
+assert list(filter(lambda x: ((x % 2) == 0), [0, 1, 2])) == [0, 2]
+
+# None implies identity
+assert list(filter(None, [0, 1, 2])) == [1, 2]
+
+assert type(filter(None, [])) == filter
+
+
+# test infinite iterator
+class Counter(object):
+    counter = 0
+
+    def __next__(self):
+        self.counter += 1
+        return self.counter
+
+    def __iter__(self):
+        return self
+
+
+it = filter(lambda x: ((x % 2) == 0), Counter())
+assert next(it) == 2
+assert next(it) == 4
+
+
+def predicate(x):
+    if x == 0:
+        raise StopIteration()
+    return True
+
+
+assert list(filter(predicate, [1, 2, 0, 4, 5])) == [1, 2]

tests/snippets/builtin_format.py

@@ -0,0 +1,9 @@
+from testutils import assert_raises
+
+assert format(5, "b") == "101"
+
+assert_raises(TypeError, lambda: format(2, 3), 'format called with number')
+
+assert format({}) == "{}"
+
+assert_raises(TypeError, lambda: format({}, 'b'), 'format_spec not empty for dict')

tests/snippets/builtin_hex.py

@@ -1,9 +1,6 @@
+from testutils import assert_raises
+
 assert hex(16) == '0x10'
 assert hex(-16) == '-0x10'
 
-try:
+assert_raises(TypeError, lambda: hex({}), 'ord() called with dict')
-    hex({})
-except TypeError:
-    pass
-else:
-    assert False, "TypeError not raised when ord() is called with a dict"

tests/snippets/builtin_locals.py

@@ -0,0 +1,19 @@
+
+a = 5
+b = 6
+
+loc = locals()
+
+assert loc['a'] == 5
+assert loc['b'] == 6
+
+def f():
+	c = 4
+	a = 7
+
+	loc = locals()
+
+	assert loc['a'] == 4
+	assert loc['c'] == 7
+	assert not 'b' in loc
+

tests/snippets/builtin_map.py

@@ -0,0 +1,34 @@
+a = list(map(str, [1, 2, 3]))
+assert a == ['1', '2', '3']
+
+
+b = list(map(lambda x, y: x + y, [1, 2, 4], [3, 5]))
+assert b == [4, 7]
+
+assert type(map(lambda x: x, [])) == map
+
+
+# test infinite iterator
+class Counter(object):
+    counter = 0
+
+    def __next__(self):
+        self.counter += 1
+        return self.counter
+
+    def __iter__(self):
+        return self
+
+
+it = map(lambda x: x+1, Counter())
+assert next(it) == 2
+assert next(it) == 3
+
+
+def mapping(x):
+    if x == 0:
+        raise StopIteration()
+    return x
+
+
+assert list(map(mapping, [1, 2, 0, 4, 5])) == [1, 2]

tests/snippets/builtin_max.py

@@ -1,3 +1,5 @@
+from testutils import assert_raises
+
 # simple values
 assert max(0, 0) == 0
 assert max(1, 0) == 1
@@ -14,32 +16,17 @@ assert max({
 }) == "b"
 assert max([1, 2], default=0) == 2
 assert max([], default=0) == 0
-try:
+assert_raises(ValueError, lambda: max([]))
-    max([])
-except ValueError:
-    pass
-else:
-    assert False, "ValueError was not raised"
 
 # key parameter
 assert max(1, 2, -3, key=abs) == -3
 assert max([1, 2, -3], key=abs) == -3
 
 # no argument
-try:
+assert_raises(TypeError, lambda: max())
-    max()
-except TypeError:
-    pass
-else:
-    assert False, "TypeError was not raised"
 
 # one non-iterable argument
-try:
+assert_raises(TypeError, lambda: max(1))
-    max(1)
-except TypeError:
-    pass
-else:
-    assert False, "TypeError was not raised"
 
 
 # custom class
@@ -64,9 +51,4 @@ class MyNotComparable():
     pass
 
 
-try:
+assert_raises(TypeError, lambda: max(MyNotComparable(), MyNotComparable()))
-    max(MyNotComparable(), MyNotComparable())
-except TypeError:
-    pass
-else:
-    assert False, "TypeError was not raised"

tests/snippets/builtin_min.py

@@ -1,3 +1,5 @@
+from testutils import assert_raises
+
 # simple values
 assert min(0, 0) == 0
 assert min(1, 0) == 0
@@ -14,32 +16,18 @@ assert min({
 }) == "a"
 assert min([1, 2], default=0) == 1
 assert min([], default=0) == 0
-try:
+
-    min([])
+assert_raises(ValueError, lambda: min([]))
-except ValueError:
-    pass
-else:
-    assert False, "ValueError was not raised"
 
 # key parameter
 assert min(1, 2, -3, key=abs) == 1
 assert min([1, 2, -3], key=abs) == 1
 
 # no argument
-try:
+assert_raises(TypeError, lambda: min())
-    min()
-except TypeError:
-    pass
-else:
-    assert False, "TypeError was not raised"
 
 # one non-iterable argument
-try:
+assert_raises(TypeError, lambda: min(1))
-    min(1)
-except TypeError:
-    pass
-else:
-    assert False, "TypeError was not raised"
 
 
 # custom class
@@ -64,9 +52,4 @@ class MyNotComparable():
     pass
 
 
-try:
+assert_raises(TypeError, lambda: min(MyNotComparable(), MyNotComparable()))
-    min(MyNotComparable(), MyNotComparable())
-except TypeError:
-    pass
-else:
-    assert False, "TypeError was not raised"

tests/snippets/builtin_open.py

@@ -0,0 +1,6 @@
+from testutils import assert_raises
+
+fd = open('README.md')
+assert 'RustPython' in fd.read()
+
+assert_raises(FileNotFoundError, lambda: open('DoesNotExist'))

tests/snippets/builtin_ord.py

@@ -1,23 +1,9 @@
+from testutils import assert_raises
+
 assert ord("a") == 97
 assert ord("é") == 233
 assert ord("🤡") == 129313
-try:
-    ord()
-except TypeError:
-    pass
-else:
-    assert False, "TypeError not raised when ord() is called with no argument"
 
-try:
+assert_raises(TypeError, lambda: ord(), "ord() is called with no argument")
-    ord("")
+assert_raises(TypeError, lambda: ord(""), "ord() is called with an empty string")
-except TypeError:
+assert_raises(TypeError, lambda: ord("ab"), "ord() is called with more than one character")
-    pass
-else:
-    assert False, "TypeError not raised when ord() is called with an empty string"
-
-try:
-    ord("ab")
-except TypeError:
-    pass
-else:
-    assert False, "TypeError not raised when ord() is called with more than one character"

tests/snippets/builtin_range.py

@@ -0,0 +1,52 @@
+from testutils import assert_raises
+
+assert range(2**63+1)[2**63] == 9223372036854775808
+
+# len tests
+assert len(range(10, 5)) == 0, 'Range with no elements should have length = 0'
+assert len(range(10, 5, -2)) == 3, 'Expected length 3, for elements: 10, 8, 6'
+assert len(range(5, 10, 2)) == 3, 'Expected length 3, for elements: 5, 7, 9'
+
+# index tests
+assert range(10).index(6) == 6
+assert range(4, 10).index(6) == 2
+assert range(4, 10, 2).index(6) == 1
+assert range(10, 4, -2).index(8) == 1
+
+assert_raises(ValueError, lambda: range(10).index(-1), 'out of bounds')
+assert_raises(ValueError, lambda: range(10).index(10), 'out of bounds')
+assert_raises(ValueError, lambda: range(4, 10, 2).index(5), 'out of step')
+assert_raises(ValueError, lambda: range(10).index('foo'), 'not an int')
+
+# count tests
+assert range(10).count(2) == 1
+assert range(10).count(11) == 0
+assert range(10).count(-1) == 0
+assert range(9, 12).count(10) == 1
+assert range(4, 10, 2).count(4) == 1
+assert range(4, 10, 2).count(7) == 0
+assert range(10).count("foo") == 0
+
+# __bool__
+assert bool(range(1))
+assert bool(range(1, 2))
+
+assert not bool(range(0))
+assert not bool(range(1, 1))
+
+# __contains__
+assert 6 in range(10)
+assert 6 in range(4, 10)
+assert 6 in range(4, 10, 2)
+assert 10 in range(10, 4, -2)
+assert 8 in range(10, 4, -2)
+
+assert -1 not in range(10)
+assert 9 not in range(10, 4, -2)
+assert 4 not in range(10, 4, -2)
+assert 'foo' not in range(10)
+
+# __reversed__
+assert list(reversed(range(5))) == [4, 3, 2, 1, 0]
+assert list(reversed(range(5, 0, -1))) == [1, 2, 3, 4, 5]
+assert list(reversed(range(1,10,5))) == [6, 1]

tests/snippets/builtin_reversed.py

@@ -0,0 +1 @@
+assert list(reversed(range(5))) == [4, 3, 2, 1, 0]

tests/snippets/builtin_slice.py

@@ -0,0 +1,73 @@
+from testutils import assert_raises
+
+a = []
+assert a[:] == []
+assert a[:2**100] == []
+assert a[-2**100:] == []
+assert a[::2**100] == []
+assert a[10:20] == []
+assert a[-20:-10] == []
+
+b = [1, 2]
+
+assert b[:] == [1, 2]
+assert b[:2**100] == [1, 2]
+assert b[-2**100:] == [1, 2]
+assert b[2**100:] == []
+assert b[::2**100] == [1]
+assert b[-10:1] == [1]
+assert b[0:0] == []
+assert b[1:0] == []
+
+assert_raises(ValueError, lambda: b[::0], 'zero step slice')
+
+assert b[::-1] == [2, 1]
+assert b[1::-1] == [2, 1]
+assert b[0::-1] == [1]
+assert b[0:-5:-1] == [1]
+assert b[:0:-1] == [2]
+assert b[5:0:-1] == [2]
+
+c = list(range(10))
+
+assert c[9:6:-3] == [9]
+assert c[9::-3] == [9, 6, 3, 0]
+assert c[9::-4] == [9, 5, 1]
+assert c[8::-2**100] == [8]
+
+assert c[7:7:-2] == []
+assert c[7:8:-2] == []
+
+d = "123456"
+
+assert d[3::-1] == "4321"
+assert d[4::-3] == "52"
+
+
+slice_a = slice(5)
+assert slice_a.start is None
+assert slice_a.stop == 5
+assert slice_a.step is None
+
+slice_b = slice(1, 5)
+assert slice_b.start == 1
+assert slice_b.stop == 5
+assert slice_b.step is None
+
+slice_c = slice(1, 5, 2)
+assert slice_c.start == 1
+assert slice_c.stop == 5
+assert slice_c.step == 2
+
+
+class SubScript(object):
+    def __getitem__(self, item):
+        assert type(item) == slice
+
+    def __setitem__(self, key, value):
+        assert type(key) == slice
+
+
+ss = SubScript()
+_ = ss[:]
+ss[:1] = 1

tests/snippets/builtin_zip.py

@@ -0,0 +1,24 @@
+assert list(zip(['a', 'b', 'c'], range(3), [9, 8, 7, 99])) == [('a', 0, 9), ('b', 1, 8), ('c', 2, 7)]
+
+assert list(zip(['a', 'b', 'c'])) == [('a',), ('b',), ('c',)]
+assert list(zip()) == []
+
+assert list(zip(*zip(['a', 'b', 'c'], range(1, 4)))) == [('a', 'b', 'c'), (1, 2, 3)]
+
+
+# test infinite iterator
+class Counter(object):
+    def __init__(self, counter=0):
+        self.counter = counter
+
+    def __next__(self):
+        self.counter += 1
+        return self.counter
+
+    def __iter__(self):
+        return self
+
+
+it = zip(Counter(), Counter(3))
+assert next(it) == (1, 4)
+assert next(it) == (2, 5)

tests/snippets/builtins.py

@@ -1,22 +1,12 @@
-
+x = sum(map(int, ['1', '2', '3']))
-a = list(map(str, [1, 2, 3]))
-assert a == ['1', '2', '3']
-
-x = sum(map(int, a))
 assert x == 6
 
 assert callable(type)
 # TODO:
 # assert callable(callable)
 
-assert list(enumerate(['a', 'b', 'c'])) == [(0, 'a'), (1, 'b'), (2, 'c')]
-
 assert type(frozenset) is type
 
-assert list(zip(['a', 'b', 'c'], range(3), [9, 8, 7, 99])) == [('a', 0, 9), ('b', 1, 8), ('c', 2, 7)]
-
-assert list(filter(lambda x: ((x % 2) == 0), [0, 1, 2])) == [0, 2]
-
 assert 3 == eval('1+2')
 
 code = compile('5+3', 'x.py', 'eval')

tests/snippets/bytearray.py

@@ -0,0 +1,67 @@
+#__getitem__ not implemented yet
+#a = bytearray(b'abc')
+#assert a[0] == b'a'
+#assert a[1] == b'b'
+
+assert len(bytearray([1,2,3])) == 3
+
+assert bytearray(b'1a23').isalnum()
+assert not bytearray(b'1%a23').isalnum()
+
+assert bytearray(b'abc').isalpha()
+assert not bytearray(b'abc1').isalpha()
+
+# travis doesn't like this
+#assert bytearray(b'xyz').isascii()
+#assert not bytearray([128, 157, 32]).isascii()
+
+assert bytearray(b'1234567890').isdigit()
+assert not bytearray(b'12ab').isdigit()
+
+l = bytearray(b'lower')
+assert l.islower()
+assert not l.isupper()
+assert l.upper().isupper()
+assert not bytearray(b'Super Friends').islower()
+
+assert bytearray(b' \n\t').isspace()
+assert not bytearray(b'\td\n').isspace()
+
+b = bytearray(b'UPPER')
+assert b.isupper()
+assert not b.islower()
+assert b.lower().islower()
+assert not bytearray(b'tuPpEr').isupper()
+
+assert bytearray(b'Is Title Case').istitle()
+assert not bytearray(b'is Not title casE').istitle()
+
+a = bytearray(b'abcd')
+a.clear()
+assert len(a) == 0
+
+try:
+    bytearray([400])
+except ValueError:
+      pass
+else:
+    assert False
+
+b = bytearray(b'test')
+assert len(b) == 4
+b.pop()
+assert len(b) == 3
+
+c = bytearray([123, 255, 111])
+assert len(c) == 3
+c.pop()
+assert len(c) == 2
+c.pop()
+c.pop()
+
+try:
+    c.pop()
+except IndexError:
+    pass
+else:
+    assert False

tests/snippets/code.py

@@ -0,0 +1,31 @@
+c1 = compile("1 + 1", "", 'eval')
+
+code_class = type(c1)
+
+def f(x, y, *args, power=1, **kwargs):
+    print("Constant String", 2, None, (2, 4))
+    assert code_class == type(c1)
+    z = x * y
+    return z ** power
+
+c2 = f.__code__
+# print(c2)
+assert type(c2) == code_class
+# print(dir(c2))
+assert c2.co_argcount == 2
+# assert c2.co_cellvars == ()
+# assert isinstance(c2.co_code, bytes)
+assert "Constant String" in c2.co_consts, c2.co_consts
+print(c2.co_consts)
+assert 2 in c2.co_consts, c2.co_consts
+assert "code.py" in c2.co_filename
+assert c2.co_firstlineno == 5, str(c2.co_firstlineno)
+# assert isinstance(c2.co_flags, int) # 'OPTIMIZED, NEWLOCALS, NOFREE'
+# assert c2.co_freevars == (), str(c2.co_freevars)
+assert c2.co_kwonlyargcount == 1, (c2.co_kwonlyargcount)
+# assert c2.co_lnotab == 0, c2.co_lnotab  # b'\x00\x01' # Line number table
+assert c2.co_name == 'f', c2.co_name
+# assert c2.co_names == ('code_class', 'type', 'c1', 'AssertionError'), c2.co_names # , c2.co_names
+# assert c2.co_nlocals == 4, c2.co_nlocals #
+# assert c2.co_stacksize == 2, 'co_stacksize',
+# assert c2.co_varnames == ('x', 'y', 'power', 'z'), c2.co_varnames

tests/snippets/control_flow.py

@@ -0,0 +1,33 @@
+# break from a nested for loop
+
+def foo():
+    sum = 0
+    for i in range(10):
+        sum += i
+        for j in range(10):
+            sum += j
+            break
+    return sum
+
+assert foo() == 45
+
+
+# continue statement
+
+def primes(limit):
+    """Finds all the primes from 2 up to a given number using the Sieve of Eratosthenes."""
+    sieve = [False] * (limit + 1)
+    for i in range(2, limit + 1):
+        if sieve[i]:
+            continue
+        yield i
+
+        for j in range(2 * i, limit + 1, i):
+            sieve[j] = True
+
+
+assert list(primes(1)) == []
+assert list(primes(2)) == [2]
+assert list(primes(10)) == [2, 3, 5, 7]
+assert list(primes(13)) == [2, 3, 5, 7, 11, 13]
+

tests/snippets/decorators.py

@@ -14,3 +14,15 @@ c = add(10, 3)
 
 assert c == 14
 
+
+def f(func): return lambda: 42
+class A: pass
+a = A()
+a.a = A()
+a.a.x = f
+
+@a.a.x
+def func():
+	pass
+
+assert func() == 42

tests/snippets/dict.py

@@ -0,0 +1,19 @@
+def dict_eq(d1, d2):
+    return (all(k in d2 and d1[k] == d2[k] for k in d1)
+            and all(k in d1 and d1[k] == d2[k] for k in d2))
+
+
+assert dict_eq(dict(a=2, b=3), {'a': 2, 'b': 3})
+assert dict_eq(dict({'a': 2, 'b': 3}, b=4), {'a': 2, 'b': 4})
+assert dict_eq(dict([('a', 2), ('b', 3)]), {'a': 2, 'b': 3})
+
+a = {'g': 5}
+b = {'a': a, 'd': 9}
+c = dict(b)
+c['d'] = 3
+c['a']['g'] = 2
+assert dict_eq(a, {'g': 2})
+assert dict_eq(b, {'a': a, 'd': 9})
+
+a.clear()
+assert len(a) == 0

tests/snippets/dismod.py

@@ -0,0 +1,34 @@
+import dis
+
+dis.dis(compile("5 + x + 5 or 2", "", "eval"))
+print("\n")
+dis.dis(compile("def f(x):\n   return 1", "", "exec"))
+print("\n")
+dis.dis(compile("if a:\n 1 or 2\nelif x == 'hello':\n 3\nelse:\n 4", "", "exec"))
+print("\n")
+dis.dis(compile("f(x=1, y=2)", "", "eval"))
+print("\n")
+
+def f():
+    with g():
+        try:
+            for a in {1: 4, 2: 5}:
+                yield [True and False or True, []]
+        except Exception:
+            raise not ValueError({1 for i in [1,2,3]})
+
+dis.dis(f)
+
+class A(object):
+    def f():
+        x += 1
+        pass
+    def g():
+        for i in range(5):
+            if i:
+                continue
+            else:
+                break
+
+print("A.f\n")
+dis.dis(A.f)

tests/snippets/division_by_zero.py

@@ -0,0 +1,11 @@
+from testutils import assert_raises
+
+assert_raises(ZeroDivisionError, lambda: 5 / 0)
+assert_raises(ZeroDivisionError, lambda: 5 / -0.0)
+assert_raises(ZeroDivisionError, lambda: 5 / (2-2))
+assert_raises(ZeroDivisionError, lambda: 5 % 0)
+assert_raises(ZeroDivisionError, lambda: 5 // 0)
+assert_raises(ZeroDivisionError, lambda: 5.3 // (-0.0))
+assert_raises(ZeroDivisionError, lambda: divmod(5, 0))
+
+assert issubclass(ZeroDivisionError, ArithmeticError)

tests/snippets/division_of_big_ints.py

@@ -0,0 +1,19 @@
+from testutils import assert_raises
+
+# 2.456984346552728
+res = 10**500 / (4 * 10**499 + 7 * 10**497 + 3 * 10**494)
+assert 2.456984 <= res <= 2.456985
+
+# 95.23809523809524
+res = 10**3000 / (10**2998 + 5 * 10**2996)
+assert 95.238095 <= res <= 95.238096
+
+assert 10**500 / (2*10**(500-308)) == 5e307
+assert 10**500 / (10**(500-308)) == 1e308
+assert_raises(OverflowError, lambda: 10**500 / (10**(500-309)), 'too big result')
+
+# a bit more than f64::MAX = 1.7976931348623157e+308_f64
+assert (2 * 10**308) / 2 == 1e308
+
+# when dividing too big int by a float, the operation should fail
+assert_raises(OverflowError, lambda: (2 * 10**308) / 2.0, 'division of big int by float')

tests/snippets/ellipsis.py

@@ -0,0 +1,8 @@
+
+
+a = ...
+b = ...
+c = type(a)()  # Test singleton behavior
+
+assert a is b
+assert b is c

tests/snippets/floats.py

@@ -1,3 +1,7 @@
+import math
+
+from testutils import assert_raises
+
 1 + 1.1
 
 a = 1.2
@@ -15,3 +19,90 @@ assert b >= a
 assert c >= a
 assert not a >= b
 
+assert a + b == 2.5
+assert a - c == 0
+assert a / c == 1
+
+assert a < 5
+assert a <= 5
+try:
+    assert a < 'a'
+except TypeError:
+    pass
+try:
+    assert a <= 'a'
+except TypeError:
+    pass
+assert a > 1
+assert a >= 1
+try:
+    assert a > 'a'
+except TypeError:
+    pass
+try:
+    assert a >= 'a'
+except TypeError:
+    pass
+
+assert math.isnan(float('nan'))
+assert math.isnan(float('NaN'))
+assert math.isnan(float('+NaN'))
+assert math.isnan(float('-NaN'))
+
+assert math.isinf(float('inf'))
+assert math.isinf(float('Inf'))
+assert math.isinf(float('+Inf'))
+assert math.isinf(float('-Inf'))
+
+assert float('+Inf') > 0
+assert float('-Inf') < 0
+
+assert float('3.14') == 3.14
+assert float('2.99e-23') == 2.99e-23
+
+assert float(b'3.14') == 3.14
+assert float(b'2.99e-23') == 2.99e-23
+
+assert_raises(ValueError, lambda: float('foo'))
+assert_raises(OverflowError, lambda: float(2**10000))
+
+# check that magic methods are implemented for ints and floats
+
+assert 1.0.__add__(1.0) == 2.0
+assert 1.0.__radd__(1.0) == 2.0
+assert 2.0.__sub__(1.0) == 1.0
+assert 2.0.__rmul__(1.0) == 2.0
+assert 1.0.__truediv__(2.0) == 0.5
+assert 1.0.__rtruediv__(2.0) == 2.0
+
+assert 1.0.__add__(1) == 2.0
+assert 1.0.__radd__(1) == 2.0
+assert 2.0.__sub__(1) == 1.0
+assert 2.0.__rmul__(1) == 2.0
+assert 1.0.__truediv__(2) == 0.5
+assert 1.0.__rtruediv__(2) == 2.0
+assert 2.0.__mul__(1) == 2.0
+assert 2.0.__rsub__(1) == -1.0
+
+assert (1.7).real == 1.7
+assert (1.3).is_integer() == False
+assert (1.0).is_integer()    == True
+
+assert (0.875).as_integer_ratio() == (7, 8)
+assert (-0.875).as_integer_ratio() == (-7, 8)
+assert (0.0).as_integer_ratio() == (0, 1)
+assert (11.5).as_integer_ratio() == (23, 2)
+assert (0.0).as_integer_ratio() == (0, 1)
+assert (2.5).as_integer_ratio() == (5, 2)
+assert (0.5).as_integer_ratio() == (1, 2)
+assert (2.1).as_integer_ratio() == (4728779608739021, 2251799813685248)
+assert (-2.1).as_integer_ratio() == (-4728779608739021, 2251799813685248)
+assert (-2100.0).as_integer_ratio() == (-2100, 1)
+assert (2.220446049250313e-16).as_integer_ratio() == (1, 4503599627370496)
+assert (1.7976931348623157e+308).as_integer_ratio() == (179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368, 1)
+assert (2.2250738585072014e-308).as_integer_ratio() == (1, 44942328371557897693232629769725618340449424473557664318357520289433168951375240783177119330601884005280028469967848339414697442203604155623211857659868531094441973356216371319075554900311523529863270738021251442209537670585615720368478277635206809290837627671146574559986811484619929076208839082406056034304)
+
+assert_raises(OverflowError, float('inf').as_integer_ratio)
+assert_raises(OverflowError, float('-inf').as_integer_ratio)
+assert_raises(ValueError, float('nan').as_integer_ratio)
+

tests/snippets/fstrings.py

@@ -0,0 +1,41 @@
+foo = 'bar'
+
+assert f"{''}" == ''
+assert f"{f'{foo}'}" == 'bar'
+assert f"foo{foo}" == 'foobar'
+assert f"{foo}foo" == 'barfoo'
+assert f"foo{foo}foo" == 'foobarfoo'
+assert f"{{foo}}" == '{foo}'
+assert f"{ {foo} }" == "{'bar'}"
+assert f"{f'{{}}'}" == '{}' # don't include escaped braces in nested f-strings
+assert f'{f"{{"}' == '{'
+assert f'{f"}}"}' == '}'
+assert f'{foo}' f"{foo}" 'foo' == 'barbarfoo'
+assert f'{"!:"}' == '!:'
+assert fr'x={4*10}\n' == 'x=40\\n'
+assert f'{16:0>+#10x}' == '00000+0x10'
+assert f"{{{(lambda x: f'hello, {x}')('world}')}" == '{hello, world}'
+
+# Normally `!` cannot appear outside of delimiters in the expression but
+# cpython makes an exception for `!=`, so we should too.
+
+# assert f'{1 != 2}' == 'True'
+
+
+# conversion flags
+
+class Value:
+    def __format__(self, spec):
+        return "foo"
+
+    def __repr__(self):
+        return "bar"
+
+    def __str__(self):
+        return "baz"
+
+v = Value()
+
+assert f'{v}' == 'foo'
+assert f'{v!r}' == 'bar'
+assert f'{v!s}' == 'baz'

tests/snippets/func_defaults.py

@@ -1,21 +1,12 @@
+from testutils import assert_raises
+
 def no_args():
     pass
 
 no_args()
 
-try:
+assert_raises(TypeError, lambda: no_args('one_arg'), '1 arg to no_args')
-    no_args('one_arg')
+assert_raises(TypeError, lambda: no_args(kw='should fail'), 'kwarg to no_args')
-except TypeError:
-    pass
-else:
-    assert False, 'no TypeError raised: 1 arg to no_args'
-
-try:
-    no_args(kw='should fail')
-except TypeError:
-    pass
-else:
-    assert False, 'no TypeError raised: kwarg to no_args'
 
 
 def one_arg(arg):
@@ -24,40 +15,20 @@ def one_arg(arg):
 one_arg('one_arg')
 assert "arg" == one_arg(arg="arg")
 
-try:
+assert_raises(TypeError, lambda: one_arg(), 'no args to one_arg')
-    one_arg()
+assert_raises(TypeError,
-except TypeError:
+              lambda: one_arg(wrong_arg='wont work'),
-    pass
+              'incorrect kwarg to one_arg')
-else:
+assert_raises(TypeError,
-    assert False, 'no TypeError raised: no args to one_arg'
+              lambda: one_arg('one_arg', 'two_arg'),
+              'two args to one_arg')
+assert_raises(TypeError,
+              lambda: one_arg('one_arg', extra_arg='wont work'),
+              'no TypeError raised: extra kwarg to one_arg')
 
-try:
+assert_raises(TypeError,
-    one_arg(wrong_arg='wont work')
+              lambda: one_arg('one_arg', arg='duplicate'),
-except TypeError:
+              'same pos and kwarg to one_arg')
-    pass
-else:
-    assert False, 'no TypeError raised: incorrect kwarg to one_arg'
-
-try:
-    one_arg('one_arg', 'two_arg')
-except TypeError:
-    pass
-else:
-    assert False, 'no TypeError raised: two args to one_arg'
-
-try:
-    one_arg('one_arg', extra_arg='wont work')
-except TypeError:
-    pass
-else:
-    assert False, 'no TypeError raised: extra kwarg to one_arg'
-
-try:
-    one_arg('one_arg', arg='duplicate')
-except TypeError:
-    pass
-else:
-    assert False, 'no TypeError raised: same pos and kwarg to one_arg'
 
 
 def one_default_arg(arg="default"):
@@ -67,12 +38,9 @@ assert 'default' == one_default_arg()
 assert 'arg' == one_default_arg('arg')
 assert 'kwarg' == one_default_arg(arg='kwarg')
 
-try:
+assert_raises(TypeError,
-    one_default_arg('one_arg', 'two_arg')
+              lambda: one_default_arg('one_arg', 'two_arg'),
-except TypeError:
+              'two args to one_default_arg')
-    pass
-else:
-    assert False, 'no TypeError raised: two args to one_default_arg'
 
 
 def one_normal_one_default_arg(pos, arg="default"):
@@ -81,19 +49,13 @@ def one_normal_one_default_arg(pos, arg="default"):
 assert ('arg', 'default') == one_normal_one_default_arg('arg')
 assert ('arg', 'arg2') == one_normal_one_default_arg('arg', 'arg2')
 
-try:
+assert_raises(TypeError,
-    one_normal_one_default_arg()
+              lambda: one_normal_one_default_arg(),
-except TypeError:
+              'no args to one_normal_one_default_arg')
-    pass
-else:
-    assert False, 'no TypeError raised: no args to one_normal_one_default_arg'
 
-try:
+assert_raises(TypeError,
-    one_normal_one_default_arg('one', 'two', 'three')
+              lambda: one_normal_one_default_arg('one', 'two', 'three'),
-except TypeError:
+              'three args to one_normal_one_default_arg')
-    pass
-else:
-    assert False, 'no TypeError raised: three args to one_normal_one_default_arg'
 
 
 def two_pos(a, b):

tests/snippets/funky_syntax.py

@@ -5,4 +5,10 @@ assert b == 6
 c = 2 + 4 if a > 5 else 'boe'
 assert c == 'boe'
 
+d = lambda x, y: x > y
+assert d(5, 4)
+
+e = lambda x: 1 if x else 0
+assert e(True) == 1
+assert e(False) == 0
 

tests/snippets/getframe.py

@@ -10,6 +10,7 @@ def test_function():
     x = 17
     assert sys._getframe().f_locals is not locals_dict
     assert sys._getframe().f_locals['x'] == 17
+    assert sys._getframe(1).f_locals['foo'] == 'bar'
 
 test_function()
 

tests/snippets/import.py

@@ -1,6 +1,7 @@
 import import_target, import_target as aliased
 from import_target import func, other_func
 from import_target import func as aliased_func, other_func as aliased_other_func
+from import_star import *
 
 assert import_target.X == import_target.func()
 assert import_target.X == func()
@@ -13,6 +14,14 @@ assert import_target.Y == aliased.Y
 assert import_target.X == aliased_func()
 assert import_target.Y == aliased_other_func()
 
+assert STAR_IMPORT == '123'
+
+try:
+    from import_target import func, unknown_name
+    raise AssertionError('`unknown_name` does not cause an exception')
+except ImportError:
+    pass
+
 # TODO: Once we can determine current directory, use that to construct this
 # path:
 #import sys

tests/snippets/import_star.py

@@ -0,0 +1,3 @@
+# This is used by import.py; the two should be modified in concert
+
+STAR_IMPORT = '123'

tests/snippets/indentation.py

@@ -0,0 +1,11 @@
+# WARNING! This file contains mixed tabs and spaces
+# (because that's what it is testing)
+
+def weird_indentation():
+	    return_value = "hi"
+	    if False:
+		    return return_value
+	    return "hi"
+
+assert weird_indentation() == "hi"
+   

tests/snippets/index_overflow.py

@@ -0,0 +1,26 @@
+import sys
+
+
+def expect_cannot_fit_index_error(s, index):
+    try:
+        s[index]
+    except IndexError:
+        pass
+    # TODO: Replace current except block with commented
+    # after solving https://github.com/RustPython/RustPython/issues/322
+    # except IndexError as error:
+    #     assert str(error) == "cannot fit 'int' into an index-sized integer"
+    else:
+        assert False
+
+
+MAX_INDEX = sys.maxsize + 1
+MIN_INDEX = -(MAX_INDEX + 1)
+
+test_str = "test"
+expect_cannot_fit_index_error(test_str, MIN_INDEX)
+expect_cannot_fit_index_error(test_str, MAX_INDEX)
+
+test_list = [0, 1, 2, 3]
+expect_cannot_fit_index_error(test_list, MIN_INDEX)
+expect_cannot_fit_index_error(test_list, MAX_INDEX)

tests/snippets/inplace_ops.py

@@ -0,0 +1,109 @@
+class InPlace:
+    def __init__(self, val):
+        self.val = val
+
+    def __ipow__(self, other):
+        self.val **= other
+        return self
+
+    def __imul__(self, other):
+        self.val *= other
+        return self
+
+    def __imatmul__(self, other):
+        # I guess you could think of an int as a 1x1 matrix
+        self.val *= other
+        return self
+
+    def __itruediv__(self, other):
+        self.val /= other
+        return self
+
+    def __ifloordiv__(self, other):
+        self.val //= other
+        return self
+
+    def __imod__(self, other):
+        self.val %= other
+        return self
+
+    def __iadd__(self, other):
+        self.val += other
+        return self
+
+    def __isub__(self, other):
+        self.val -= other
+        return self
+
+    def __ilshift__(self, other):
+        self.val <<= other
+        return self
+
+    def __irshift__(self, other):
+        self.val >>= other
+        return self
+
+    def __iand__(self, other):
+        self.val &= other
+        return self
+
+    def __ixor__(self, other):
+        self.val ^= other
+        return self
+
+    def __ior__(self, other):
+        self.val |= other
+        return self
+
+
+i = InPlace(2)
+i **= 3
+assert i.val == 8
+
+i = InPlace(2)
+i *= 2
+assert i.val == 4
+
+i = InPlace(2)
+i @= 2
+assert i.val == 4
+
+i = InPlace(1)
+i /= 2
+assert i.val == 0.5
+
+i = InPlace(1)
+i //= 2
+assert i.val == 0
+
+i = InPlace(10)
+i %= 3
+assert i.val == 1
+
+i = InPlace(1)
+i += 1
+assert i.val == 2
+
+i = InPlace(2)
+i -= 1
+assert i.val == 1
+
+i = InPlace(2)
+i <<= 3
+assert i.val == 16
+
+i = InPlace(16)
+i >>= 3
+assert i.val == 2
+
+i = InPlace(0b010101)
+i &= 0b111000
+assert i.val == 0b010000
+
+i = InPlace(0b010101)
+i ^= 0b111000
+assert i.val == 0b101101
+
+i = InPlace(0b010101)
+i |= 0b111000
+assert i.val == 0b111101

tests/snippets/int_float_equality.py

@@ -0,0 +1,15 @@
+# 10**308 cannot be represented exactly in f64, thus it is not equal to 1e308 float
+assert not (10**308 == 1e308)
+# but the 1e308 float can be converted to big int and then it still should be equal to itself
+assert int(1e308) == 1e308
+
+# and the equalities should be the same when operands switch sides
+assert not (1e308 == 10**308)
+assert 1e308 == int(1e308)
+
+# floats that cannot be converted to big ints shouldn’t crash the vm
+import math
+assert not (10**500 == math.inf)
+assert not (math.inf == 10**500)
+assert not (10**500 == math.nan)
+assert not (math.nan == 10**500)

tests/snippets/ints.py

@@ -0,0 +1,60 @@
+from testutils import assert_raises
+
+# int to int comparisons
+
+assert 1 == 1
+assert not 1 != 1
+
+assert (1).__eq__(1)
+assert not (1).__ne__(1)
+
+# int to float comparisons
+
+assert 1 == 1.0
+assert not 1 != 1.0
+assert not 1 > 1.0
+assert not 1 < 1.0
+assert 1 >= 1.0
+assert 1 <= 1.0
+
+# check for argument handling
+
+assert int("101", base=2) == 5
+
+# magic methods should only be implemented for other ints
+
+assert (1).__eq__(1) == True
+assert (1).__ne__(1) == False
+assert (1).__gt__(1) == False
+assert (1).__ge__(1) == True
+assert (1).__lt__(1) == False
+assert (1).__le__(1) == True
+assert (1).__add__(1) == 2
+assert (1).__radd__(1) == 2
+assert (2).__sub__(1) == 1
+assert (2).__rsub__(1) == -1
+assert (2).__mul__(1) == 2
+assert (2).__rmul__(1) == 2
+assert (2).__truediv__(1) == 2.0
+assert (2).__rtruediv__(1) == 0.5
+
+# real/imag attributes
+assert (1).real == 1
+assert (1).imag == 0
+
+assert_raises(OverflowError, lambda: 1 << 10 ** 100000)
+
+assert (1).__eq__(1.0) == NotImplemented
+assert (1).__ne__(1.0) == NotImplemented
+assert (1).__gt__(1.0) == NotImplemented
+assert (1).__ge__(1.0) == NotImplemented
+assert (1).__lt__(1.0) == NotImplemented
+assert (1).__le__(1.0) == NotImplemented
+assert (1).__add__(1.0) == NotImplemented
+assert (2).__sub__(1.0) == NotImplemented
+assert (1).__radd__(1.0) == NotImplemented
+assert (2).__rsub__(1.0) == NotImplemented
+assert (2).__mul__(1.0) == NotImplemented
+assert (2).__rmul__(1.0) == NotImplemented
+assert (2).__truediv__(1.0) == NotImplemented
+assert (2).__rtruediv__(1.0) == NotImplemented

tests/snippets/isinstance.py

@@ -0,0 +1,54 @@
+
+class Regular:
+    pass
+
+
+assert isinstance(Regular(), Regular)
+
+
+class MCNotInstanceOf(type):
+    def __instancecheck__(self, instance):
+        return False
+
+
+class NotInstanceOf(metaclass=MCNotInstanceOf):
+    pass
+
+
+class InheritedNotInstanceOf(NotInstanceOf):
+    pass
+
+
+assert not isinstance(Regular(), NotInstanceOf)
+assert not isinstance(1, NotInstanceOf)
+
+# weird cpython behaviour if exact match then isinstance return true
+assert isinstance(NotInstanceOf(), NotInstanceOf)
+assert not NotInstanceOf.__instancecheck__(NotInstanceOf())
+assert not isinstance(InheritedNotInstanceOf(), NotInstanceOf)
+
+
+class MCAlwaysInstanceOf(type):
+    def __instancecheck__(self, instance):
+        return True
+
+
+class AlwaysInstanceOf(metaclass=MCAlwaysInstanceOf):
+    pass
+
+
+assert isinstance(AlwaysInstanceOf(), AlwaysInstanceOf)
+assert isinstance(Regular(), AlwaysInstanceOf)
+assert isinstance(1, AlwaysInstanceOf)
+
+
+class MCReturnInt(type):
+    def __instancecheck__(self, instance):
+        return 3
+
+
+class ReturnInt(metaclass=MCReturnInt):
+    pass
+
+
+assert isinstance("a", ReturnInt) is True

tests/snippets/issubclass.py

@@ -0,0 +1,63 @@
+
+class A:
+    pass
+
+
+class B(A):
+    pass
+
+
+assert issubclass(A, A)
+assert issubclass(B, A)
+assert not issubclass(A, B)
+
+
+class MCNotSubClass(type):
+    def __subclasscheck__(self, subclass):
+        return False
+
+
+class NotSubClass(metaclass=MCNotSubClass):
+    pass
+
+
+class InheritedNotSubClass(NotSubClass):
+    pass
+
+
+assert not issubclass(A, NotSubClass)
+assert not issubclass(NotSubClass, NotSubClass)
+assert not issubclass(InheritedNotSubClass, NotSubClass)
+assert not issubclass(NotSubClass, InheritedNotSubClass)
+
+
+class MCAlwaysSubClass(type):
+    def __subclasscheck__(self, subclass):
+        return True
+
+
+class AlwaysSubClass(metaclass=MCAlwaysSubClass):
+    pass
+
+
+class InheritedAlwaysSubClass(AlwaysSubClass):
+    pass
+
+
+assert issubclass(A, AlwaysSubClass)
+assert issubclass(AlwaysSubClass, AlwaysSubClass)
+assert issubclass(InheritedAlwaysSubClass, AlwaysSubClass)
+assert issubclass(AlwaysSubClass, InheritedAlwaysSubClass)
+
+
+class MCAVirtualSubClass(type):
+    def __subclasscheck__(self, subclass):
+        return subclass is A
+
+
+class AVirtualSubClass(metaclass=MCAVirtualSubClass):
+    pass
+
+
+assert issubclass(A, AVirtualSubClass)
+assert not isinstance(B, AVirtualSubClass)

tests/snippets/json_snippet.py

@@ -31,8 +31,7 @@ assert '{}' == json.dumps({})
 assert 1 == json.loads("1")
 assert -1 == json.loads("-1")
 assert 1.0 == json.loads("1.0")
-# TODO: uncomment once negative floats are implemented
+assert -1.0 == json.loads("-1.0")
-# assert -1.0 == json.loads("-1.0")
 assert "str" == json.loads('"str"')
 assert True is json.loads('true')
 assert False is json.loads('false')

tests/snippets/list.py

@@ -1,3 +1,5 @@
+from testutils import assert_raises
+
 x = [1, 2, 3]
 assert x[0] == 1
 assert x[1] == 2
@@ -9,3 +11,148 @@ assert y == [2, 1, 2, 3]
 y.extend(x)
 assert y == [2, 1, 2, 3, 1, 2, 3]
 
+assert x * 0 == [], "list __mul__ by 0 failed"
+assert x * -1 == [], "list __mul__ by -1 failed"
+assert x * 2 == [1, 2, 3, 1, 2, 3], "list __mul__ by 2 failed"
+
+# index()
+assert ['a', 'b', 'c'].index('b') == 1
+assert [5, 6, 7].index(7) == 2
+assert_raises(ValueError, lambda: ['a', 'b', 'c'].index('z'))
+
+x = [[1,0,-3], 'a', 1]
+y = [[3,2,1], 'z', 2]
+assert x < y, "list __lt__ failed"
+
+x = [5, 13, 31]
+y = [1, 10, 29]
+assert x > y, "list __gt__ failed"
+
+
+assert [1,2,'a'].pop() == 'a', "list pop failed"
+assert_raises(IndexError, lambda: [].pop())
+
+recursive = []
+recursive.append(recursive)
+assert repr(recursive) == "[[...]]"
+
+# insert()
+x = ['a', 'b', 'c']
+x.insert(0, 'z') # insert is in-place, no return value
+assert x == ['z', 'a', 'b', 'c']
+
+x = ['a', 'b', 'c']
+x.insert(100, 'z')
+assert x == ['a', 'b', 'c', 'z']
+
+x = ['a', 'b', 'c']
+x.insert(-1, 'z')
+assert x == ['a', 'b', 'z', 'c']
+
+x = ['a', 'b', 'c']
+x.insert(-100, 'z')
+assert x == ['z', 'a', 'b', 'c']
+
+assert_raises(OverflowError, lambda: x.insert(100000000000000000000, 'z'))
+
+x = [[], 2, {}]
+y = x.copy()
+assert x is not y
+assert x == y
+assert all(a is b for a, b in zip(x, y))
+y.append(4)
+assert x != y
+
+a = [1, 2, 3]
+assert len(a) == 3
+a.remove(1)
+assert len(a) == 2
+assert not 1 in a
+
+assert_raises(ValueError, lambda: a.remove(10), 'Remove not exist element')
+
+foo = bar = [1]
+foo += [2]
+assert (foo, bar) == ([1, 2], [1, 2])
+
+
+x = [1]
+x.append(x)
+assert x in x
+assert x.index(x) == 1
+assert x.count(x) == 1
+x.remove(x)
+assert x not in x
+
+class Foo(object):
+    def __eq__(self, x):
+        return False
+
+foo = Foo()
+foo1 = Foo()
+x = [1, foo, 2, foo, []]
+assert x == x
+assert foo in x
+assert 2 in x
+assert x.index(foo) == 1
+assert x.count(foo) == 2
+assert x.index(2) == 2
+assert [] in x
+assert x.index([]) == 4
+assert foo1 not in x
+x.remove(foo)
+assert x.index(foo) == 2
+assert x.count(foo) == 1
+
+x = []
+x.append(x)
+assert x == x
+
+a = [1, 2, 3]
+b = [1, 2, 3]
+c = [a, b]
+a.append(c)
+b.append(c)
+
+assert a == b
+
+assert [foo] == [foo]
+
+for size in [1, 2, 3, 4, 5, 8, 10, 100, 1000]:
+   lst = list(range(size))
+   orig = lst[:]
+   lst.sort()
+   assert lst == orig
+   assert sorted(lst) == orig
+   assert_raises(ZeroDivisionError, lambda: sorted(lst, key=lambda x: 1/x))
+   lst.reverse()
+   assert sorted(lst) == orig
+   assert sorted(lst, reverse=True) == lst
+   assert sorted(lst, key=lambda x: -x) == lst
+   assert sorted(lst, key=lambda x: -x, reverse=True) == orig
+
+assert sorted([(1, 2, 3), (0, 3, 6)]) == [(0, 3, 6), (1, 2, 3)]
+assert sorted([(1, 2, 3), (0, 3, 6)], key=lambda x: x[0]) == [(0, 3, 6), (1, 2, 3)]
+assert sorted([(1, 2, 3), (0, 3, 6)], key=lambda x: x[1]) == [(1, 2, 3), (0, 3, 6)]
+assert sorted([(1, 2), (), (5,)], key=len) == [(), (5,), (1, 2)]
+
+lst = [3, 1, 5, 2, 4]
+class C:
+  def __init__(self, x): self.x = x
+  def __lt__(self, other): return self.x < other.x
+lst.sort(key=C)
+assert lst == [1, 2, 3, 4, 5]
+
+lst = [3, 1, 5, 2, 4]
+class C:
+  def __init__(self, x): self.x = x
+  def __gt__(self, other): return self.x > other.x
+lst.sort(key=C)
+assert lst == [1, 2, 3, 4, 5]
+
+lst = [5, 1, 2, 3, 4]
+def f(x):
+    lst.append(1)
+    return x
+assert_raises(ValueError, lambda: lst.sort(key=f)) # "list modified during sort"
+assert lst == [1, 2, 3, 4, 5]

tests/snippets/math_basics.py

@@ -18,4 +18,12 @@ assert -a == -4
 assert +a == 4
 
 # import math
-# print(math.cos(1.2))
+# assert(math.exp(2) == math.exp(2.0))
+# assert(math.exp(True) == math.exp(1.0))
+#
+# class Conversible():
+#     def __float__(self):
+#         print("Converting to float now!")
+#         return 1.1111
+#
+# assert math.log(1.1111) == math.log(Conversible())

tests/snippets/membership.py

@@ -1,3 +1,5 @@
+from testutils import assert_raises
+
 # test lists
 assert 3 in [1, 2, 3]
 assert 3 not in [1, 2]
@@ -13,6 +15,13 @@ assert "whatever" not in "foobar"
 # TODO: uncomment this when bytes are implemented
 # assert b"foo" in b"foobar"
 # assert b"whatever" not in b"foobar"
+assert b"1" < b"2"
+assert b"1" <= b"2"
+assert b"5" <= b"5"
+assert b"4" > b"2"
+assert not b"1" >= b"2"
+assert b"10" >= b"10"
+assert_raises(TypeError, lambda: bytes() > 2)
 
 # test tuple
 assert 1 in (1, 2)
@@ -41,12 +50,7 @@ class MyNotContainingClass():
     pass
 
 
-try:
+assert_raises(TypeError, lambda: 1 in MyNotContainingClass())
-    1 in MyNotContainingClass()
-except TypeError:
-    pass
-else:
-    assert False, "TypeError not raised"
 
 
 class MyContainingClass():

tests/snippets/none.py

@@ -14,3 +14,7 @@ assert none() is none()
 assert none() is x
 
 assert none() is none2()
+
+assert str(None) == 'None'
+assert repr(None) == 'None'
+assert type(None)() is None

tests/snippets/numbers.py

@@ -2,9 +2,43 @@ x = 5
 x.__init__(6)
 assert x == 5
 
+
 class A(int):
     pass
 
+
 x = A(7)
 assert x == 7
 assert type(x) is A
+
+assert int(2).__index__() == 2
+assert int(2).__trunc__() == 2
+assert int(2).__ceil__() == 2
+assert int(2).__floor__() == 2
+assert int(2).__round__() == 2
+assert int(2).__round__(3) == 2
+assert int(-2).__index__() == -2
+assert int(-2).__trunc__() == -2
+assert int(-2).__ceil__() == -2
+assert int(-2).__floor__() == -2
+assert int(-2).__round__() == -2
+assert int(-2).__round__(3) == -2
+
+assert round(10) == 10
+assert round(10, 2) == 10
+assert round(10, -1) == 10
+
+assert int(2).__bool__() == True
+assert int(0.5).__bool__() == False
+assert int(-1).__bool__() == True
+
+assert int(0).__invert__() == -1
+assert int(-3).__invert__() == 2
+assert int(4).__invert__() == -5
+
+assert int(0).__rxor__(0) == 0
+assert int(1).__rxor__(0) == 1
+assert int(0).__rxor__(1) == 1
+assert int(1).__rxor__(1) == 0
+assert int(3).__rxor__(-3) == -2
+assert int(3).__rxor__(4) == 7

tests/snippets/object.py

@@ -0,0 +1,15 @@
+class MyObject:
+    pass
+
+assert not MyObject() == MyObject()
+assert MyObject() != MyObject()
+myobj = MyObject()
+assert myobj == myobj
+assert not myobj != myobj
+
+assert MyObject().__eq__(MyObject()) == NotImplemented
+assert MyObject().__ne__(MyObject()) == NotImplemented
+assert MyObject().__lt__(MyObject()) == NotImplemented
+assert MyObject().__le__(MyObject()) == NotImplemented
+assert MyObject().__gt__(MyObject()) == NotImplemented
+assert MyObject().__ge__(MyObject()) == NotImplemented

tests/snippets/os_info.py

@@ -0,0 +1,3 @@
+import os 
+
+assert os.name == 'posix' or os.name == 'nt'

tests/snippets/print_const.py

@@ -1 +0,0 @@
-print(2 + 3)

tests/snippets/printing.py

@@ -0,0 +1,11 @@
+from testutils import assert_raises
+
+print(2 + 3)
+
+assert_raises(TypeError, lambda: print('test', end=4), 'wrong type passed to end')
+assert_raises(TypeError, lambda: print('test', sep=['a']), 'wrong type passed to sep')
+
+try:
+    print('test', end=None, sep=None, flush=None)
+except:
+    assert False, 'Expected None passed to end, sep, and flush to not raise errors'

tests/snippets/set.py

@@ -0,0 +1,146 @@
+from testutils import assert_raises, assertRaises
+
+assert set([1,2]) == set([1,2])
+assert not set([1,2,3]) == set([1,2])
+
+assert set([1,2,3]) >= set([1,2])
+assert set([1,2]) >= set([1,2])
+assert not set([1,3]) >= set([1,2])
+
+assert set([1,2,3]).issuperset(set([1,2]))
+assert set([1,2]).issuperset(set([1,2]))
+assert not set([1,3]).issuperset(set([1,2]))
+
+assert set([1,2,3]) > set([1,2])
+assert not set([1,2]) > set([1,2])
+assert not set([1,3]) > set([1,2])
+
+assert set([1,2]) <= set([1,2,3])
+assert set([1,2]) <= set([1,2])
+assert not set([1,3]) <= set([1,2])
+
+assert set([1,2]).issubset(set([1,2,3]))
+assert set([1,2]).issubset(set([1,2]))
+assert not set([1,3]).issubset(set([1,2]))
+
+assert set([1,2]) < set([1,2,3])
+assert not set([1,2]) < set([1,2])
+assert not set([1,3]) < set([1,2])
+
+class Hashable(object):
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __repr__(self):
+        return repr(self.obj)
+
+    def __hash__(self):
+        return id(self)
+
+
+recursive = set()
+recursive.add(Hashable(recursive))
+assert repr(recursive) == "{set(...)}"
+
+a = set([1, 2, 3])
+assert len(a) == 3
+a.clear()
+assert len(a) == 0
+
+assert set([1,2,3]).union(set([4,5])) == set([1,2,3,4,5])
+assert set([1,2,3]).union(set([1,2,3,4,5])) == set([1,2,3,4,5])
+
+assert set([1,2,3]) | set([4,5]) == set([1,2,3,4,5])
+assert set([1,2,3]) | set([1,2,3,4,5]) == set([1,2,3,4,5])
+
+assert set([1,2,3]).intersection(set([1,2])) == set([1,2])
+assert set([1,2,3]).intersection(set([5,6])) == set([])
+
+assert set([1,2,3]) & set([4,5]) == set([])
+assert set([1,2,3]) & set([1,2,3,4,5]) == set([1,2,3])
+
+assert set([1,2,3]).difference(set([1,2])) == set([3])
+assert set([1,2,3]).difference(set([5,6])) == set([1,2,3])
+
+assert set([1,2,3]) - set([4,5]) == set([1,2,3])
+assert set([1,2,3]) - set([1,2,3,4,5]) == set([])
+
+assert set([1,2,3]).symmetric_difference(set([1,2])) == set([3])
+assert set([1,2,3]).symmetric_difference(set([5,6])) == set([1,2,3,5,6])
+
+assert set([1,2,3]) ^ set([4,5]) == set([1,2,3,4,5])
+assert set([1,2,3]) ^ set([1,2,3,4,5]) == set([4,5])
+
+assert_raises(TypeError, lambda: set([[]]))
+assert_raises(TypeError, lambda: set().add([]))
+
+a = set([1, 2, 3])
+assert a.discard(1) is None
+assert not 1 in a
+assert a.discard(42) is None
+
+a = set([1,2,3])
+b = a.copy()
+assert len(a) == 3
+assert len(b) == 3
+b.clear()
+assert len(a) == 3
+assert len(b) == 0
+
+a = set([1,2])
+b = a.pop()
+assert b in [1,2]
+c = a.pop()
+assert (c in [1,2] and c != b) 
+assert_raises(KeyError, lambda: a.pop())
+
+a = set([1,2,3])
+a.update([3,4,5])
+assert a == set([1,2,3,4,5])
+assert_raises(TypeError, lambda: a.update(1))
+
+a = set([1,2,3])
+b = set()
+for e in a:
+	assert e == 1 or e == 2 or e == 3
+	b.add(e)
+assert a == b
+
+a = set([1,2,3])
+a |= set([3,4,5])
+assert a == set([1,2,3,4,5])
+with assertRaises(TypeError):
+	a |= 1
+
+a = set([1,2,3])
+a.intersection_update([2,3,4,5])
+assert a == set([2,3])
+assert_raises(TypeError, lambda: a.intersection_update(1))
+
+a = set([1,2,3])
+a &= set([2,3,4,5])
+assert a == set([2,3])
+with assertRaises(TypeError):
+	a &= 1
+
+a = set([1,2,3])
+a.difference_update([3,4,5])
+assert a == set([1,2])
+assert_raises(TypeError, lambda: a.difference_update(1))
+
+a = set([1,2,3])
+a -= set([3,4,5])
+assert a == set([1,2])
+with assertRaises(TypeError):
+	a -= 1
+
+a = set([1,2,3])
+a.symmetric_difference_update([3,4,5])
+assert a == set([1,2,4,5])
+assert_raises(TypeError, lambda: a.difference_update(1))
+
+a = set([1,2,3])
+a ^= set([3,4,5])
+assert a == set([1,2,4,5])
+with assertRaises(TypeError):
+	a ^= 1

tests/snippets/stdlib_io.py

@@ -0,0 +1,10 @@
+from io import BufferedReader, FileIO
+
+fi = FileIO('README.md')
+bb = BufferedReader(fi)
+
+result = bb.read()
+
+assert len(result) <= 8*1024
+assert len(result) >= 0
+assert isinstance(result, bytes)

tests/snippets/stdlib_os.py

@@ -0,0 +1,13 @@
+import os 
+
+from testutils import assert_raises
+
+assert os.open('README.md', 0) > 0
+
+
+assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0))
+
+
+assert os.O_RDONLY == 0
+assert os.O_WRONLY == 1
+assert os.O_RDWR == 2

tests/snippets/stdlib_socket.py

@@ -0,0 +1,75 @@
+import socket
+from testutils import assertRaises
+
+MESSAGE_A = b'aaaa'
+MESSAGE_B= b'bbbbb'
+
+# TCP
+
+listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+listener.bind(("127.0.0.1", 0))
+listener.listen(1)
+
+connector = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+connector.connect(("127.0.0.1", listener.getsockname()[1]))
+(connection, addr) = listener.accept()
+assert addr == connector.getsockname()
+
+connector.send(MESSAGE_A)
+connection.send(MESSAGE_B)
+recv_a = connection.recv(len(MESSAGE_A))
+recv_b = connector.recv(len(MESSAGE_B))
+assert recv_a == MESSAGE_A
+assert recv_b == MESSAGE_B
+connection.close()
+connector.close()
+listener.close()
+
+s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+with assertRaises(TypeError):
+	s.connect(("127.0.0.1", 8888, 8888))
+
+with assertRaises(TypeError):
+	s.bind(("127.0.0.1", 8888, 8888))
+
+with assertRaises(TypeError):
+	s.bind((888, 8888))
+
+s.close()
+
+# UDP
+sock1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+sock1.bind(("127.0.0.1", 0))
+
+sock2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+
+sock2.sendto(MESSAGE_A, sock1.getsockname())
+(recv_a, addr1) = sock1.recvfrom(len(MESSAGE_A))
+assert recv_a == MESSAGE_A
+
+sock2.sendto(MESSAGE_B, sock1.getsockname())
+(recv_b, addr2) = sock1.recvfrom(len(MESSAGE_B))
+assert recv_b == MESSAGE_B
+assert addr1[0] == addr2[0]
+assert addr1[1] == addr2[1]
+
+sock2.close()
+
+sock3 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+sock3.bind(("127.0.0.1", 0))
+sock3.sendto(MESSAGE_A, sock1.getsockname())
+(recv_a, addr) = sock1.recvfrom(len(MESSAGE_A))
+assert recv_a == MESSAGE_A
+assert addr == sock3.getsockname()
+
+sock1.connect(("127.0.0.1", sock3.getsockname()[1]))
+sock3.connect(("127.0.0.1", sock1.getsockname()[1]))
+
+sock1.send(MESSAGE_A)
+sock3.send(MESSAGE_B)
+recv_a = sock3.recv(len(MESSAGE_A))
+recv_b = sock1.recv(len(MESSAGE_B))
+assert recv_a == MESSAGE_A
+assert recv_b == MESSAGE_B
+sock1.close()
+sock3.close()

tests/snippets/strings.py

@@ -8,6 +8,9 @@ assert "\n" == """
 """
 
 assert len(""" " \" """) == 5
+assert len("é") == 1
+assert len("é") == 2
+assert len("あ") == 1
 
 assert type("") is str
 assert type(b"") is bytes
@@ -32,6 +35,15 @@ assert a.startswith('H')
 assert not a.startswith('f')
 assert a.endswith('llo')
 assert not a.endswith('on')
+assert a.zfill(8) == '000Hallo'
+assert a.isalnum()
+assert not a.isdigit()
+assert not a.isdecimal()
+assert not a.isnumeric()
+assert a.istitle()
+assert a.isalpha()
+
+
 
 b = '  hallo  '
 assert b.strip() == 'hallo'
@@ -40,3 +52,36 @@ assert b.rstrip() == '  hallo'
 
 c = 'hallo'
 assert c.capitalize() == 'Hallo'
+assert c.center(11, '-') == '---hallo---'
+# assert c.isascii()
+assert c.index('a') == 1
+assert c.rindex('l') == 3
+assert c.find('h') == 0
+assert c.rfind('x') == -1
+assert c.islower()
+assert c.title() == 'Hallo'
+assert c.count('l') == 2
+
+assert '   '.isspace()
+assert 'hello\nhallo\nHallo'.splitlines() == ['hello', 'hallo', 'Hallo']
+assert 'abc\t12345\txyz'.expandtabs() == 'abc     12345   xyz'
+assert '-'.join(['1', '2', '3']) == '1-2-3'
+assert 'HALLO'.isupper()
+assert "hello, my name is".partition("my ") == ('hello, ', 'my ', 'name is')
+assert "hello, my name is".rpartition("is") == ('hello, my name ', 'is', '')
+assert not ''.isdecimal()
+assert '123'.isdecimal()
+assert not '\u00B2'.isdecimal()
+
+# String Formatting
+assert "{} {}".format(1,2) == "1 2"
+assert "{0} {1}".format(2,3) == "2 3"
+assert "--{:s>4}--".format(1) == "--sss1--"
+assert "{keyword} {0}".format(1, keyword=2) == "2 1"
+
+assert 'a' < 'b'
+assert 'a' <= 'b'
+assert 'a' <= 'a'
+assert 'z' > 'b'
+assert 'z' >= 'b'
+assert 'a' >= 'a'

tests/snippets/test_bitwise.py

@@ -0,0 +1,21 @@
+from testutils import assert_raises
+
+#
+# Tests
+#
+assert 8 >> 3 == 1
+assert 8 << 3 == 64
+
+# Left shift raises type error
+assert_raises(TypeError, lambda: 1 << 0.1)
+assert_raises(TypeError, lambda: 1 << "abc")
+
+# Right shift raises type error
+assert_raises(TypeError, lambda: 1 >> 0.1)
+assert_raises(TypeError, lambda: 1 >> "abc")
+
+# Left shift raises value error on negative
+assert_raises(ValueError, lambda: 1 << -1)
+
+# Right shift raises value error on negative
+assert_raises(ValueError, lambda: 1 >> -1)

tests/snippets/test_exec.py

@@ -0,0 +1,42 @@
+exec("def square(x):\n return x * x\n")
+assert 16 == square(4)
+
+d = {}
+exec("def square(x):\n return x * x\n", {}, d)
+assert 16 == d['square'](4)
+
+exec("assert 2 == x", {}, {'x': 2})
+exec("assert 2 == x", {'x': 2}, {})
+exec("assert 4 == x", {'x': 2}, {'x': 4})
+
+exec("assert max(1, 2) == 2", {}, {})
+
+exec("assert max(1, 5, square(5)) == 25", None)
+
+#
+# These doesn't work yet:
+#
+# Local environment shouldn't replace global environment:
+#
+# exec("assert max(1, 5, square(5)) == 25", None, {})
+#
+# Closures aren't available if local scope is replaced:
+#
+# def g():
+#     seven = "seven"
+#     def f():
+#         try:
+#             exec("seven", None, {})
+#         except NameError:
+#             pass
+#         else:
+#             raise NameError("seven shouldn't be in scope")
+#     f()
+# g()
+
+try:
+    exec("", 1)
+except TypeError:
+    pass
+else:
+    raise TypeError("exec should fail unless globals is a dict or None")

tests/snippets/test_sets.py

@@ -0,0 +1,20 @@
+
+empty_set = set()
+non_empty_set = set([1,2,3])
+set_from_literal = {1,2,3}
+
+assert 1 in non_empty_set
+assert 4 not in non_empty_set
+
+assert 1 in set_from_literal
+assert 4 not in set_from_literal
+
+# TODO: Assert that empty aruguments raises exception.
+non_empty_set.add('a')
+assert 'a' in non_empty_set
+
+# TODO: Assert that empty arguments, or item not in set raises exception.
+non_empty_set.remove(1)
+assert 1 not in non_empty_set
+
+# TODO: Assert that adding the same thing to a set once it's already there doesn't do anything.

tests/snippets/test_string.py

@@ -0,0 +1,13 @@
+import string
+
+
+assert string.ascii_letters == 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+assert string.ascii_lowercase == 'abcdefghijklmnopqrstuvwxyz'
+assert string.ascii_uppercase == 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+assert string.digits == '0123456789'
+assert string.hexdigits == '0123456789abcdefABCDEF'
+assert string.octdigits == '01234567'
+assert string.punctuation == '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'
+# FIXME
+#assert string.whitespace == ' \t\n\r\x0b\x0c', string.whitespace
+#assert string.printable == '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c'

tests/snippets/testutils.py

@@ -0,0 +1,36 @@
+def assert_raises(exc_type, expr, msg=None):
+    """
+    Helper function to assert `expr` raises an exception of type `exc_type`.
+    Args:
+        expr: Callable
+        exec_type: Exception
+    Returns:
+        None
+    Raises:
+        Assertion error on failure
+    """
+    try:
+        expr()
+    except exc_type:
+        pass
+    else:
+        failmsg = '{!s} was not raised'.format(exc_type.__name__)
+        if msg is not None:
+            failmsg += ': {!s}'.format(msg)
+        assert False, failmsg
+
+
+class assertRaises:
+    def __init__(self, expected):
+        self.expected = expected
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is None:
+            failmsg = '{!s} was not raised'.format(self.expected.__name_)
+            assert False, failmsg            
+        if not issubclass(exc_type, self.expected):
+            return False
+        return True

tests/snippets/try_exceptions.py

@@ -8,6 +8,19 @@ except BaseException as ex:
     # print(ex.__traceback__)
     # print(type(ex.__traceback__))
 
+try:
+    raise ZeroDivisionError
+except ZeroDivisionError as ex:
+    pass
+
+class E(Exception):
+    def __init__(self):
+        asdf
+
+try:
+    raise E
+except NameError as ex:
+    pass
 
 l = []
 try:

tests/snippets/tuple.py

@@ -5,3 +5,32 @@ assert x[0] == 1
 
 y = (1,)
 assert y[0] == 1
+
+assert x + y == (1, 2, 1)
+
+assert x * 3 == (1, 2, 1, 2, 1, 2)
+# assert 3 * x == (1, 2, 1, 2, 1, 2)
+assert x * 0 == ()
+assert x * -1 == ()  # integers less than zero treated as 0
+
+assert y < x, "tuple __lt__ failed"
+assert x > y, "tuple __gt__ failed"
+
+
+b = (1,2,3)
+assert b.index(2) == 1
+
+recursive_list = []
+recursive = (recursive_list,)
+recursive_list.append(recursive)
+assert repr(recursive) == "([(...)],)"
+
+assert (None, "", 1).index(1) == 2
+assert 1 in (None, "", 1)
+
+class Foo(object):
+    def __eq__(self, x):
+        return False
+
+foo = Foo()
+assert (foo,) == (foo,)

tests/snippets/type_hints.py

@@ -0,0 +1,7 @@
+
+# See also: https://github.com/RustPython/RustPython/issues/587
+
+def curry(foo: int) -> float:
+    return foo * 3.1415926 * 2
+
+assert curry(2) > 10

tests/snippets/unicode_slicing.py

@@ -0,0 +1,33 @@
+def test_slice_bounds(s):
+    # End out of range
+    assert s[0:100] == s
+    assert s[0:-100] == '' 
+    # Start out of range
+    assert s[100:1] == ''
+    # Out of range both sides
+    # This is the behaviour in cpython
+    # assert s[-100:100] == s
+
+def expect_index_error(s, index):
+    try:
+        s[index]
+    except IndexError:
+        pass
+    else:
+        assert False
+
+unicode_str = "∀∂"
+assert unicode_str[0] == "∀"
+assert unicode_str[1] == "∂"
+assert unicode_str[-1] == "∂"
+
+test_slice_bounds(unicode_str)
+expect_index_error(unicode_str, 100)
+expect_index_error(unicode_str, -100)
+
+ascii_str = "hello world"
+test_slice_bounds(ascii_str)
+assert ascii_str[0] == "h"
+assert ascii_str[1] == "e"
+assert ascii_str[-1] == "d"
+

tests/snippets/weakrefs.py

@@ -8,5 +8,6 @@ class X:
 a = X()
 b = ref(a)
 
+assert callable(b)
 assert b() is a
 

tests/snippets/whats_left_to_implement.py

@@ -1,4 +1,7 @@
-bool_expected_methods = [
+expected_methods = []
+
+# TODO: using tuple could have been better
+expected_methods.append({'name': 'bool', 'methods': [
     '__abs__',
     '__add__',
     '__and__',
@@ -69,9 +72,8 @@ bool_expected_methods = [
     'numerator',
     'real',
     'to_bytes',
-]
+], 'type': bool})
-
+expected_methods.append({'name': 'bytearray', 'type': bytearray, 'methods': [
-bytearray_expected_methods = [
     '__add__',
     '__alloc__',
     '__class__',
@@ -157,9 +159,8 @@ bytearray_expected_methods = [
     'translate',
     'upper',
     'zfill',
-]
+]})
-
+expected_methods.append({'name': 'bytes', 'type': bytes, 'methods': [
-bytes_expected_methods = [
     '__add__',
     '__class__',
     '__contains__',
@@ -233,9 +234,8 @@ bytes_expected_methods = [
     'translate',
     'upper',
     'zfill',
-]
+]})
-
+expected_methods.append({'name': 'complex', 'type': complex, 'methods': [
-complex_expected_methods = [
     '__abs__',
     '__add__',
     '__bool__',
@@ -285,9 +285,8 @@ complex_expected_methods = [
     'conjugate',
     'imag',
     'real',
-]
+]})
-
+expected_methods.append({'name': 'dict', 'type': dict, 'methods': [
-dict_expected_methods = [
     '__class__',
     '__contains__',
     '__delattr__',
@@ -328,9 +327,8 @@ dict_expected_methods = [
     'setdefault',
     'update',
     'values',
-]
+]})
-
+expected_methods.append({'name': 'float','type':float,'methods':[
-float_expected_methods = [
     '__abs__',
     '__add__',
     '__bool__',
@@ -388,9 +386,8 @@ float_expected_methods = [
     'imag',
     'is_integer',
     'real',
-]
+]})
-
+expected_methods.append({'name': 'frozenset','type':frozenset, 'methods': [
-frozenset_expected_methods = [
     '__and__',
     '__class__',
     '__contains__',
@@ -433,9 +430,8 @@ frozenset_expected_methods = [
     'issuperset',
     'symmetric_difference',
     'union',
-]
+]})
-
+expected_methods.append({'name': 'int', 'type':int, 'methods': [
-int_expected_methods = [
     '__abs__',
     '__add__',
     '__and__',
@@ -506,9 +502,8 @@ int_expected_methods = [
     'numerator',
     'real',
     'to_bytes',
-]
+]})
-
+expected_methods.append({'name': 'iter','type':iter,'methods':[
-iter_expected_methods = [
     '__class__',
     '__delattr__',
     '__dir__',
@@ -536,9 +531,8 @@ iter_expected_methods = [
     '__sizeof__',
     '__str__',
     '__subclasshook__',
-]
+]})
-
+expected_methods.append({'name': 'list','type':list,'methods':[
-list_expected_methods = [
     '__add__',
     '__class__',
     '__contains__',
@@ -585,9 +579,8 @@ list_expected_methods = [
     'remove',
     'reverse',
     'sort',
-]
+]})
-
+expected_methods.append({'name': 'memoryview','type':memoryview,'methods':[
-memoryview_expected_methods = [
     '__class__',
     '__delattr__',
     '__delitem__',
@@ -634,9 +627,8 @@ memoryview_expected_methods = [
     'suboffsets',
     'tobytes',
     'tolist',
-]
+]})
-
+expected_methods.append({'name': 'range','type':range,'methods':[
-range_expected_methods = [
     '__bool__',
     '__class__',
     '__contains__',
@@ -671,9 +663,8 @@ range_expected_methods = [
     'start',
     'step',
     'stop',
-]
+]})
-
+expected_methods.append({'name': 'set','type':set,'methods':[
-set_expected_methods = [
     '__and__',
     '__class__',
     '__contains__',
@@ -729,9 +720,8 @@ set_expected_methods = [
     'symmetric_difference_update',
     'union',
     'update',
-]
+]})
-
+expected_methods.append({'name': 'string','type':str,'methods':[
-string_expected_methods = [
     '__add__',
     '__class__',
     '__contains__',
@@ -810,9 +800,8 @@ string_expected_methods = [
     'translate',
     'upper',
     'zfill'
-]
+]})
-
+expected_methods.append({'name': 'tuple','type':tuple, 'methods': [
-tuple_expected_methods = [
     '__add__',
     '__class__',
     '__contains__',
@@ -846,116 +835,42 @@ tuple_expected_methods = [
     '__subclasshook__',
     'count',
     'index',
-]
+]})
+expected_methods.append({'name': 'object', 'type':object, 'methods':[
+    '__repr__',
+    '__hash__',
+    '__str__',
+    '__getattribute__',
+    '__setattr__',
+    '__delattr__',
+    '__lt__',
+    '__le__',
+    '__eq__',
+    '__ne__',
+    '__gt__',
+    '__ge__',
+    '__init__',
+    '__new__',
+    '__reduce_ex__',
+    '__reduce__',
+    '__subclasshook__',
+    '__init_subclass__',
+    '__format__',
+    '__sizeof__',
+    '__dir__',
+    '__class__',
+    '__doc__'
+]})
 
 not_implemented = []
 
-for method in bool_expected_methods:
+for item in expected_methods:
-    try:
+    for method in item['methods']:
-        if not hasattr(bool, method):
+        try:
-            not_implemented.append(("bool", method))
+            if not hasattr(item['type'], method):
-    except NameError:
+                not_implemented.append((item['name'], method))
-        not_implemented.append(("bool", method))
+        except NameError:
-
+            not_implemented.append((item['name'], method))
-for method in bytearray_expected_methods:
-    try:
-        if not hasattr(bytearray(), method):
-            not_implemented.append(("bytearray", method))
-    except NameError:
-        not_implemented.append(("bytearray", method))
-
-for method in bytes_expected_methods:
-    try:
-        if not hasattr(bytes, method):
-            not_implemented.append(("bytes", method))
-    except NameError:
-        not_implemented.append(("bytes", method))
-
-for method in complex_expected_methods:
-    try:
-        if not hasattr(complex, method):
-            not_implemented.append(("complex", method))
-    except NameError:
-        not_implemented.append(("complex", method))
-
-for method in dict_expected_methods:
-    try:
-        if not hasattr(dict, method):
-            not_implemented.append(("dict", method))
-    except NameError:
-        not_implemented.append(("dict", method))
-
-for method in float_expected_methods:
-    try:
-        if not hasattr(float, method):
-            not_implemented.append(("float", method))
-    except NameError:
-        not_implemented.append(("float", method))
-
-for method in frozenset_expected_methods:
-# TODO: uncomment this when frozenset is implemented
-#    try:
-#        if not hasattr(frozenset, method):
-#            not_implemented.append(("frozenset", method))
-#    except NameError:
-    not_implemented.append(("frozenset", method))
-
-for method in int_expected_methods:
-    try:
-        if not hasattr(int, method):
-            not_implemented.append(("int", method))
-    except NameError:
-        not_implemented.append(("int", method))
-
-for method in iter_expected_methods:
-    try:
-        if not hasattr(iter([]), method):
-            not_implemented.append(("iter", method))
-    except NameError:
-        not_implemented.append(("iter", method))
-
-for method in list_expected_methods:
-    try:
-        if not hasattr(list, method):
-            not_implemented.append(("list", method))
-    except NameError:
-        not_implemented.append(("list", method))
-
-for method in memoryview_expected_methods:
-# TODO: uncomment this when memoryview is implemented
-#    try:
-#        if not hasattr(memoryview, method):
-#            not_implemented.append(("memoryview", method))
-#    except NameError:
-    not_implemented.append(("memoryview", method))
-
-for method in range_expected_methods:
-    try:
-        if not hasattr(range, method):
-            not_implemented.append(("range", method))
-    except NameError:
-        not_implemented.append(("range", method))
-
-for method in set_expected_methods:
-    try:
-        if not hasattr(set, method):
-            not_implemented.append(("set", method))
-    except NameError:
-        not_implemented.append(("set", method))
-
-for method in string_expected_methods:
-    try:
-        if not hasattr(str, method):
-            not_implemented.append(("string", method))
-    except NameError:
-        not_implemented.append(("string", method))
-
-for method in tuple_expected_methods:
-    try:
-        if not hasattr(tuple, method):
-            not_implemented.append(("tuple", method))
-    except NameError:
-        not_implemented.append(("tuple", method))
 
 for r in not_implemented:
     print(r[0], ".", r[1])

tests/test_snippets.py

@@ -77,8 +77,12 @@ def run_via_rustpython(filename, test_type):
     log_level = 'info' if test_type == _TestType.benchmark else 'trace'
     env['RUST_LOG'] = '{},cargo=error,jobserver=error'.format(log_level)
     env['RUST_BACKTRACE'] = '1'
-    subprocess.check_call(
+    if env.get('CODE_COVERAGE', 'false') == 'true':
-        ['cargo', 'run', '--release', filename], env=env)
+        subprocess.check_call(
+            ['cargo', 'run', filename], env=env)
+    else:
+        subprocess.check_call(
+            ['cargo', 'run', '--release', filename], env=env)
 
 
 def create_test_function(cls, filename, method, test_type):

vm/Cargo.toml

@@ -2,12 +2,16 @@
 name = "rustpython_vm"
 version = "0.1.0"
 authors = ["Shing Lyu <shing.lyu@gmail.com>"]
+edition = "2018"
 
 [dependencies]
 bitflags = "1.0.4"
 num-complex = "0.2"
 num-bigint = "0.2.1"
 num-traits = "0.2"
+num-integer = "0.1.39"
+num-rational = "0.2.1"
+rand = "0.5"
 log = "0.3"
 rustpython_parser = {path = "../parser"}
 serde = "1.0.66"
@@ -15,3 +19,9 @@ serde_derive = "1.0.66"
 serde_json = "1.0.26"
 byteorder = "1.2.6"
 regex = "1"
+rustc_version_runtime = "0.1.*"
+statrs = "0.10.0"
+caseless = "0.2.1"
+unicode-segmentation = "1.2.1"
+lazy_static = "^1.0.1"
+lexical = "2.0.0"