Set all versions to 0.1.0

Update documentation folder and readme.

.github/ISSUE_TEMPLATE/report-incompatibility.md

@@ -0,0 +1,16 @@
+---
+name: Report incompatibility
+about: Report an incompatibility between RustPython and CPython
+title: ''
+labels: feat
+assignees: ''
+
+---
+
+## Feature
+
+<!-- What Python feature is missing from RustPython? Give a short description of the feature and how you ran into its absence. -->
+
+## Python Documentation
+
+<!-- Give a link to the feature in the CPython documentation (https://docs.python.org/3/) in order to assist in its implementation. -->

.github/ISSUE_TEMPLATE/rfc.md

@@ -0,0 +1,24 @@
+---
+name: RFC
+about: Make a suggestion in a Request for Comments format to RustPython
+title: "[RFC] "
+labels: RFC
+assignees: ''
+
+---
+
+## Summary
+
+<!-- A quick overview of your suggestion -->
+
+## Detailed Explanation
+
+<!-- Elaborate on your suggestion in all its details -->
+
+## Drawbacks, Rationale, and Alternatives
+
+<!-- What drawbacks might this solution have? Why do you feel it is necessary? What other options might there be to solving this problem? -->
+
+## Unresolved Questions
+
+<!-- What would you like feedback on for fleshing out your suggestion? -->

.gitignore

@@ -1,4 +1,5 @@
 /target
+/*/target
 **/*.rs.bk
 **/*.bytecode
 __pycache__
@@ -8,3 +9,4 @@ __pycache__
 .vscode
 wasm-pack.log
 .idea/
+tests/snippets/resources

.travis.yml

@@ -1,68 +1,44 @@
-language: rust
-
-rust:
-  - stable
-  - beta
-  - nightly
-
-script:
-  - cargo build --verbose --all
-  - cargo test --verbose --all
-
-env:
-  # This is used to only capture the regular nightly test in allow_failures
-  - REGULAR_TEST=true
-
-cache: cargo
 
 matrix:
   fast_finish: true
   include:
+    - name: Run rust tests
+      language: rust
+      rust: stable
+      cache: cargo
+      script:
+        - cargo build --verbose --all
+        - cargo test --verbose --all
+      env:
+        # Prevention of cache corruption.
+        # See: https://docs.travis-ci.com/user/caching/#caches-and-build-matrices
+        - JOBCACHE=1
+
     # To test the snippets, we use Travis' Python environment (because
     # installing rust ourselves is a lot easier than installing Python)
-    - language: python
+    - name: python test snippets
+      language: python
       python: 3.6
       cache:
-        pip: true
-        # Because we're using the Python Travis environment, we can't use
-        # the built-in cargo cacher
-        directories:
-          - /home/travis/.cargo
-          - target
+        - pip
+        - cargo
       env:
+        - JOBCACHE=2
         - TRAVIS_RUST_VERSION=stable
-        - REGULAR_TEST=false
         - CODE_COVERAGE=false
       script: tests/.travis-runner.sh
-    - language: python
-      python: 3.6
-      cache:
-        pip: true
-        # Because we're using the Python Travis environment, we can't use
-        # the built-in cargo cacher
-        directories:
-          - /home/travis/.cargo
-          - target
-      env:
-        - TRAVIS_RUST_VERSION=beta
-        - REGULAR_TEST=false
-        - CODE_COVERAGE=false
-      script: tests/.travis-runner.sh
-    - name: rustfmt
+
+    - name: Check rust code style with rustfmt
       language: rust
       rust: stable
       cache: cargo
       before_script:
-        - rustup component add rustfmt-preview
+        - rustup component add rustfmt
       script:
-        # Code references the generated python.rs, so put something in
-        # place to make `cargo fmt` happy.  (We use `echo` rather than
-        # `touch` because rustfmt complains about the empty file touch
-        # creates.)
-        - echo > parser/src/python.rs
         - cargo fmt --all -- --check
       env:
-        - REGULAR_TEST=false
+        - JOBCACHE=3
+
     - name: publish documentation
       language: rust
       rust: stable
@@ -71,11 +47,20 @@ matrix:
         - cargo doc --no-deps --all
       if: branch = release
       env:
-        - REGULAR_TEST=false
-        - DEPLOY_DOC=true
+        - JOBCACHE=4
+      deploy:
+        - provider: pages
+          repo: RustPython/website
+          target-branch: master
+          local-dir: target/doc
+          skip-cleanup: true
+          # Set in the settings page of your repository, as a secure variable
+          github-token: $WEBSITE_GITHUB_TOKEN
+          keep-history: true
+
     - name: WASM online demo
       language: rust
-      rust: nightly
+      rust: stable
       cache: cargo
       install:
         - nvm install node
@@ -87,59 +72,45 @@ matrix:
         - npm run dist
       if: branch = release
       env:
-        - REGULAR_TEST=false
-        - DEPLOY_DEMO=true
-    - name: cargo-clippy
-      language: rust
-      rust: stable
-      cache: cargo
-      before_script:
-        - rustup component add clippy
-      script:
-        - cargo clippy
-      env:
-        - REGULAR_TEST=true
+        - JOBCACHE=5
+      deploy:
+        - provider: pages
+          repo: RustPython/demo
+          target-branch: master
+          local-dir: wasm/demo/dist
+          skip-cleanup: true
+          # Set in the settings page of your repository, as a secure variable
+          github-token: $WEBSITE_GITHUB_TOKEN
+          keep-history: true
+
     - name: Code Coverage
       language: python
       python: 3.6
       cache:
-        pip: true
-        # Because we're using the Python Travis environment, we can't use
-        # the built-in cargo cacher
-        directories:
-          - /home/travis/.cargo
-          - target
+        - pip
+        - cargo
       script:
         - tests/.travis-runner.sh
+      # Only do code coverage on master via a cron job.
+      if: branch = master AND type = cron
       env:
+        - JOBCACHE=6
         - TRAVIS_RUST_VERSION=nightly
-        - REGULAR_TEST=false
         - CODE_COVERAGE=true
-  allow_failures:
-    - rust: nightly
-      env: REGULAR_TEST=true
-    - name: cargo-clippy
 
-deploy:
-  - provider: pages
-    repo: RustPython/website
-    target-branch: master
-    local-dir: target/doc
-    skip-cleanup: true
-    # Set in the settings page of your repository, as a secure variable
-    github-token: $WEBSITE_GITHUB_TOKEN
-    keep-history: true
-    on:
-      branch: release
-      condition: $DEPLOY_DOC = true
-  - provider: pages
-    repo: RustPython/demo
-    target-branch: master
-    local-dir: wasm/demo/dist
-    skip-cleanup: true
-    # Set in the settings page of your repository, as a secure variable
-    github-token: $WEBSITE_GITHUB_TOKEN
-    keep-history: true
-    on:
-      branch: release
-      condition: $DEPLOY_DEMO = true
+    - name: test WASM
+      language: python
+      python: 3.6
+      cache:
+        - pip
+        - cargo
+      addons:
+        firefox: latest
+      install:
+        - nvm install node
+        - pip install pipenv
+      script:
+        - wasm/tests/.travis-runner.sh
+      env:
+        - JOBCACHE=7
+        - TRAVIS_RUST_VERSION=stable

Cargo.toml

@@ -1,20 +1,29 @@
 [package]
 name = "rustpython"
-version = "0.0.1-pre-alpha.1"
-authors = ["Windel Bouwman", "Shing Lyu <shing.lyu@gmail.com>"]
+version = "0.1.0"
+authors = ["RustPython Team"]
 edition = "2018"
+description = "A python interpreter written in rust."
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
 
 [workspace]
-members = [".", "vm", "wasm/lib", "parser"]
+members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode"]
+
+[[bench]]
+name = "bench"
+path = "./benchmarks/bench.rs"
+
 
 [dependencies]
 log="0.4.1"
 env_logger="0.5.10"
 clap = "2.31.2"
-rustpython_parser = {path = "parser"}
-rustpython_vm = {path = "vm"}
-rustyline = "2.1.0"
+rustpython-compiler = {path = "compiler", version = "0.1.0"}
+rustpython-parser = {path = "parser", version = "0.1.0"}
+rustpython-vm = {path = "vm", version = "0.1.0"}
+rustyline = "4.1.0"
 xdg = "2.2.0"
 
-[profile.release]
-opt-level = "s"
+[dev-dependencies.cpython]
+version = "0.2"

Lib/PSF-LICENSE

@@ -0,0 +1,254 @@
+A. HISTORY OF THE SOFTWARE
+==========================
+
+Python was created in the early 1990s by Guido van Rossum at Stichting
+Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
+as a successor of a language called ABC.  Guido remains Python's
+principal author, although it includes many contributions from others.
+
+In 1995, Guido continued his work on Python at the Corporation for
+National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
+in Reston, Virginia where he released several versions of the
+software.
+
+In May 2000, Guido and the Python core development team moved to
+BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
+year, the PythonLabs team moved to Digital Creations, which became
+Zope Corporation.  In 2001, the Python Software Foundation (PSF, see
+https://www.python.org/psf/) was formed, a non-profit organization
+created specifically to own Python-related Intellectual Property.
+Zope Corporation was a sponsoring member of the PSF.
+
+All Python releases are Open Source (see http://www.opensource.org for
+the Open Source Definition).  Historically, most, but not all, Python
+releases have also been GPL-compatible; the table below summarizes
+the various releases.
+
+    Release         Derived     Year        Owner       GPL-
+                    from                                compatible? (1)
+
+    0.9.0 thru 1.2              1991-1995   CWI         yes
+    1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
+    1.6             1.5.2       2000        CNRI        no
+    2.0             1.6         2000        BeOpen.com  no
+    1.6.1           1.6         2001        CNRI        yes (2)
+    2.1             2.0+1.6.1   2001        PSF         no
+    2.0.1           2.0+1.6.1   2001        PSF         yes
+    2.1.1           2.1+2.0.1   2001        PSF         yes
+    2.1.2           2.1.1       2002        PSF         yes
+    2.1.3           2.1.2       2002        PSF         yes
+    2.2 and above   2.1.1       2001-now    PSF         yes
+
+Footnotes:
+
+(1) GPL-compatible doesn't mean that we're distributing Python under
+    the GPL.  All Python licenses, unlike the GPL, let you distribute
+    a modified version without making your changes open source.  The
+    GPL-compatible licenses make it possible to combine Python with
+    other software that is released under the GPL; the others don't.
+
+(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
+    because its license has a choice of law clause.  According to
+    CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
+    is "not incompatible" with the GPL.
+
+Thanks to the many outside volunteers who have worked under Guido's
+direction to make these releases possible.
+
+
+B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
+===============================================================
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All
+Rights Reserved" are retained in Python alone or in any derivative version
+prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
+-------------------------------------------
+
+BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
+
+1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
+office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
+Individual or Organization ("Licensee") accessing and otherwise using
+this software in source or binary form and its associated
+documentation ("the Software").
+
+2. Subject to the terms and conditions of this BeOpen Python License
+Agreement, BeOpen hereby grants Licensee a non-exclusive,
+royalty-free, world-wide license to reproduce, analyze, test, perform
+and/or display publicly, prepare derivative works, distribute, and
+otherwise use the Software alone or in any derivative version,
+provided, however, that the BeOpen Python License is retained in the
+Software, alone or in any derivative version prepared by Licensee.
+
+3. BeOpen is making the Software available to Licensee on an "AS IS"
+basis.  BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
+SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
+AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
+DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+5. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+6. This License Agreement shall be governed by and interpreted in all
+respects by the law of the State of California, excluding conflict of
+law provisions.  Nothing in this License Agreement shall be deemed to
+create any relationship of agency, partnership, or joint venture
+between BeOpen and Licensee.  This License Agreement does not grant
+permission to use BeOpen trademarks or trade names in a trademark
+sense to endorse or promote products or services of Licensee, or any
+third party.  As an exception, the "BeOpen Python" logos available at
+http://www.pythonlabs.com/logos.html may be used according to the
+permissions granted on that web page.
+
+7. By copying, installing or otherwise using the software, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
+---------------------------------------
+
+1. This LICENSE AGREEMENT is between the Corporation for National
+Research Initiatives, having an office at 1895 Preston White Drive,
+Reston, VA 20191 ("CNRI"), and the Individual or Organization
+("Licensee") accessing and otherwise using Python 1.6.1 software in
+source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, CNRI
+hereby grants Licensee a nonexclusive, royalty-free, world-wide
+license to reproduce, analyze, test, perform and/or display publicly,
+prepare derivative works, distribute, and otherwise use Python 1.6.1
+alone or in any derivative version, provided, however, that CNRI's
+License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
+1995-2001 Corporation for National Research Initiatives; All Rights
+Reserved" are retained in Python 1.6.1 alone or in any derivative
+version prepared by Licensee.  Alternately, in lieu of CNRI's License
+Agreement, Licensee may substitute the following text (omitting the
+quotes): "Python 1.6.1 is made available subject to the terms and
+conditions in CNRI's License Agreement.  This Agreement together with
+Python 1.6.1 may be located on the Internet using the following
+unique, persistent identifier (known as a handle): 1895.22/1013.  This
+Agreement may also be obtained from a proxy server on the Internet
+using the following URL: http://hdl.handle.net/1895.22/1013".
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python 1.6.1 or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python 1.6.1.
+
+4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
+basis.  CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. This License Agreement shall be governed by the federal
+intellectual property law of the United States, including without
+limitation the federal copyright law, and, to the extent such
+U.S. federal law does not apply, by the law of the Commonwealth of
+Virginia, excluding Virginia's conflict of law provisions.
+Notwithstanding the foregoing, with regard to derivative works based
+on Python 1.6.1 that incorporate non-separable material that was
+previously distributed under the GNU General Public License (GPL), the
+law of the Commonwealth of Virginia shall govern this License
+Agreement only as to issues arising under or with respect to
+Paragraphs 4, 5, and 7 of this License Agreement.  Nothing in this
+License Agreement shall be deemed to create any relationship of
+agency, partnership, or joint venture between CNRI and Licensee.  This
+License Agreement does not grant permission to use CNRI trademarks or
+trade name in a trademark sense to endorse or promote products or
+services of Licensee, or any third party.
+
+8. By clicking on the "ACCEPT" button where indicated, or by copying,
+installing or otherwise using Python 1.6.1, Licensee agrees to be
+bound by the terms and conditions of this License Agreement.
+
+        ACCEPT
+
+
+CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
+--------------------------------------------------
+
+Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
+The Netherlands.  All rights reserved.
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and that
+both that copyright notice and this permission notice appear in
+supporting documentation, and that the name of Stichting Mathematisch
+Centrum or CWI not be used in advertising or publicity pertaining to
+distribution of the software without specific, written prior
+permission.
+
+STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
+THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
+FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

Lib/README.md

@@ -0,0 +1,10 @@
+Standard Library for RustPython
+===============================
+
+This directory contains all of the Python files that make up the standard library for RustPython.
+
+Most of these files are copied over from the CPython repository(the 3.7 branch), with slight modifications to allow them 
+to work under RustPython. The current goal is to complete the standard library with as few modifications as possible. 
+Current modifications are just temporary workarounds for bugs/missing feature within the RustPython implementation.
+
+The first target is to run the ``unittest`` module, so we can leverage the CPython test suite.

Lib/_py_abc.py

@@ -0,0 +1,147 @@
+from _weakrefset import WeakSet
+
+
+def get_cache_token():
+    """Returns the current ABC cache token.
+
+    The token is an opaque object (supporting equality testing) identifying the
+    current version of the ABC cache for virtual subclasses. The token changes
+    with every call to ``register()`` on any ABC.
+    """
+    return ABCMeta._abc_invalidation_counter
+
+
+class ABCMeta(type):
+    """Metaclass for defining Abstract Base Classes (ABCs).
+
+    Use this metaclass to create an ABC.  An ABC can be subclassed
+    directly, and then acts as a mix-in class.  You can also register
+    unrelated concrete classes (even built-in classes) and unrelated
+    ABCs as 'virtual subclasses' -- these and their descendants will
+    be considered subclasses of the registering ABC by the built-in
+    issubclass() function, but the registering ABC won't show up in
+    their MRO (Method Resolution Order) nor will method
+    implementations defined by the registering ABC be callable (not
+    even via super()).
+    """
+
+    # A global counter that is incremented each time a class is
+    # registered as a virtual subclass of anything.  It forces the
+    # negative cache to be cleared before its next use.
+    # Note: this counter is private. Use `abc.get_cache_token()` for
+    #       external code.
+    _abc_invalidation_counter = 0
+
+    def __new__(mcls, name, bases, namespace, **kwargs):
+        cls = type.__new__(mcls, name, bases, namespace, **kwargs)
+        # Compute set of abstract method names
+        abstracts = {name
+                     for name, value in namespace.items()
+                     if getattr(value, "__isabstractmethod__", False)}
+        for base in bases:
+            for name in getattr(base, "__abstractmethods__", set()):
+                value = getattr(cls, name, None)
+                if getattr(value, "__isabstractmethod__", False):
+                    abstracts.add(name)
+        cls.__abstractmethods__ = set(abstracts)
+        # Set up inheritance registry
+        cls._abc_registry = WeakSet()
+        cls._abc_cache = WeakSet()
+        cls._abc_negative_cache = WeakSet()
+        cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
+        return cls
+
+    def register(cls, subclass):
+        """Register a virtual subclass of an ABC.
+
+        Returns the subclass, to allow usage as a class decorator.
+        """
+        if not isinstance(subclass, type):
+            raise TypeError("Can only register classes")
+        if issubclass(subclass, cls):
+            return subclass  # Already a subclass
+        # Subtle: test for cycles *after* testing for "already a subclass";
+        # this means we allow X.register(X) and interpret it as a no-op.
+        if issubclass(cls, subclass):
+            # This would create a cycle, which is bad for the algorithm below
+            raise RuntimeError("Refusing to create an inheritance cycle")
+        cls._abc_registry.add(subclass)
+        ABCMeta._abc_invalidation_counter += 1  # Invalidate negative cache
+        return subclass
+
+    def _dump_registry(cls, file=None):
+        """Debug helper to print the ABC registry."""
+        print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file)
+        print(f"Inv. counter: {get_cache_token()}", file=file)
+        for name in cls.__dict__:
+            if name.startswith("_abc_"):
+                value = getattr(cls, name)
+                if isinstance(value, WeakSet):
+                    value = set(value)
+                print(f"{name}: {value!r}", file=file)
+
+    def _abc_registry_clear(cls):
+        """Clear the registry (for debugging or testing)."""
+        cls._abc_registry.clear()
+
+    def _abc_caches_clear(cls):
+        """Clear the caches (for debugging or testing)."""
+        cls._abc_cache.clear()
+        cls._abc_negative_cache.clear()
+
+    def __instancecheck__(cls, instance):
+        """Override for isinstance(instance, cls)."""
+        # Inline the cache checking
+        subclass = instance.__class__
+        if subclass in cls._abc_cache:
+            return True
+        subtype = type(instance)
+        if subtype is subclass:
+            if (cls._abc_negative_cache_version ==
+                ABCMeta._abc_invalidation_counter and
+                subclass in cls._abc_negative_cache):
+                return False
+            # Fall back to the subclass check.
+            return cls.__subclasscheck__(subclass)
+        return any(cls.__subclasscheck__(c) for c in (subclass, subtype))
+
+    def __subclasscheck__(cls, subclass):
+        """Override for issubclass(subclass, cls)."""
+        if not isinstance(subclass, type):
+            raise TypeError('issubclass() arg 1 must be a class')
+        # Check cache
+        if subclass in cls._abc_cache:
+            return True
+        # Check negative cache; may have to invalidate
+        if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter:
+            # Invalidate the negative cache
+            cls._abc_negative_cache = WeakSet()
+            cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
+        elif subclass in cls._abc_negative_cache:
+            return False
+        # Check the subclass hook
+        ok = cls.__subclasshook__(subclass)
+        if ok is not NotImplemented:
+            assert isinstance(ok, bool)
+            if ok:
+                cls._abc_cache.add(subclass)
+            else:
+                cls._abc_negative_cache.add(subclass)
+            return ok
+        # Check if it's a direct subclass
+        if cls in getattr(subclass, '__mro__', ()):
+            cls._abc_cache.add(subclass)
+            return True
+        # Check if it's a subclass of a registered class (recursive)
+        for rcls in cls._abc_registry:
+            if issubclass(subclass, rcls):
+                cls._abc_cache.add(subclass)
+                return True
+        # Check if it's a subclass of a subclass (recursive)
+        for scls in cls.__subclasses__():
+            if issubclass(subclass, scls):
+                cls._abc_cache.add(subclass)
+                return True
+        # No dice; update negative cache
+        cls._abc_negative_cache.add(subclass)
+        return False

Lib/_weakrefset.py

@@ -0,0 +1,196 @@
+# Access WeakSet through the weakref module.
+# This code is separated-out because it is needed
+# by abc.py to load everything else at startup.
+
+from _weakref import ref
+
+__all__ = ['WeakSet']
+
+
+class _IterationGuard:
+    # This context manager registers itself in the current iterators of the
+    # weak container, such as to delay all removals until the context manager
+    # exits.
+    # This technique should be relatively thread-safe (since sets are).
+
+    def __init__(self, weakcontainer):
+        # Don't create cycles
+        self.weakcontainer = ref(weakcontainer)
+
+    def __enter__(self):
+        w = self.weakcontainer()
+        if w is not None:
+            w._iterating.add(self)
+        return self
+
+    def __exit__(self, e, t, b):
+        w = self.weakcontainer()
+        if w is not None:
+            s = w._iterating
+            s.remove(self)
+            if not s:
+                w._commit_removals()
+
+
+class WeakSet:
+    def __init__(self, data=None):
+        self.data = set()
+        def _remove(item, selfref=ref(self)):
+            self = selfref()
+            if self is not None:
+                if self._iterating:
+                    self._pending_removals.append(item)
+                else:
+                    self.data.discard(item)
+        self._remove = _remove
+        # A list of keys to be removed
+        self._pending_removals = []
+        self._iterating = set()
+        if data is not None:
+            self.update(data)
+
+    def _commit_removals(self):
+        l = self._pending_removals
+        discard = self.data.discard
+        while l:
+            discard(l.pop())
+
+    def __iter__(self):
+        with _IterationGuard(self):
+            for itemref in self.data:
+                item = itemref()
+                if item is not None:
+                    # Caveat: the iterator will keep a strong reference to
+                    # `item` until it is resumed or closed.
+                    yield item
+
+    def __len__(self):
+        return len(self.data) - len(self._pending_removals)
+
+    def __contains__(self, item):
+        try:
+            wr = ref(item)
+        except TypeError:
+            return False
+        return wr in self.data
+
+    def __reduce__(self):
+        return (self.__class__, (list(self),),
+                getattr(self, '__dict__', None))
+
+    def add(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.add(ref(item, self._remove))
+
+    def clear(self):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.clear()
+
+    def copy(self):
+        return self.__class__(self)
+
+    def pop(self):
+        if self._pending_removals:
+            self._commit_removals()
+        while True:
+            try:
+                itemref = self.data.pop()
+            except KeyError:
+                raise KeyError('pop from empty WeakSet') from None
+            item = itemref()
+            if item is not None:
+                return item
+
+    def remove(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.remove(ref(item))
+
+    def discard(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.discard(ref(item))
+
+    def update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        for element in other:
+            self.add(element)
+
+    def __ior__(self, other):
+        self.update(other)
+        return self
+
+    def difference(self, other):
+        newset = self.copy()
+        newset.difference_update(other)
+        return newset
+    __sub__ = difference
+
+    def difference_update(self, other):
+        self.__isub__(other)
+    def __isub__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.difference_update(ref(item) for item in other)
+        return self
+
+    def intersection(self, other):
+        return self.__class__(item for item in other if item in self)
+    __and__ = intersection
+
+    def intersection_update(self, other):
+        self.__iand__(other)
+    def __iand__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.intersection_update(ref(item) for item in other)
+        return self
+
+    def issubset(self, other):
+        return self.data.issubset(ref(item) for item in other)
+    __le__ = issubset
+
+    def __lt__(self, other):
+        return self.data < set(map(ref, other))
+
+    def issuperset(self, other):
+        return self.data.issuperset(ref(item) for item in other)
+    __ge__ = issuperset
+
+    def __gt__(self, other):
+        return self.data > set(map(ref, other))
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return self.data == set(map(ref, other))
+
+    def symmetric_difference(self, other):
+        newset = self.copy()
+        newset.symmetric_difference_update(other)
+        return newset
+    __xor__ = symmetric_difference
+
+    def symmetric_difference_update(self, other):
+        self.__ixor__(other)
+    def __ixor__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.symmetric_difference_update(ref(item, self._remove) for item in other)
+        return self
+
+    def union(self, other):
+        return self.__class__(e for s in (self, other) for e in s)
+    __or__ = union
+
+    def isdisjoint(self, other):
+        return len(self.intersection(other)) == 0

Lib/abc.py

@@ -0,0 +1,170 @@
+# Copyright 2007 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Abstract Base Classes (ABCs) according to PEP 3119."""
+
+
+def abstractmethod(funcobj):
+    """A decorator indicating abstract methods.
+
+    Requires that the metaclass is ABCMeta or derived from it.  A
+    class that has a metaclass derived from ABCMeta cannot be
+    instantiated unless all of its abstract methods are overridden.
+    The abstract methods can be called using any of the normal
+    'super' call mechanisms.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractmethod
+            def my_abstract_method(self, ...):
+                ...
+    """
+    funcobj.__isabstractmethod__ = True
+    return funcobj
+
+
+class abstractclassmethod(classmethod):
+    """A decorator indicating abstract classmethods.
+
+    Similar to abstractmethod.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractclassmethod
+            def my_abstract_classmethod(cls, ...):
+                ...
+
+    'abstractclassmethod' is deprecated. Use 'classmethod' with
+    'abstractmethod' instead.
+    """
+
+    __isabstractmethod__ = True
+
+    def __init__(self, callable):
+        callable.__isabstractmethod__ = True
+        super().__init__(callable)
+
+
+class abstractstaticmethod(staticmethod):
+    """A decorator indicating abstract staticmethods.
+
+    Similar to abstractmethod.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractstaticmethod
+            def my_abstract_staticmethod(...):
+                ...
+
+    'abstractstaticmethod' is deprecated. Use 'staticmethod' with
+    'abstractmethod' instead.
+    """
+
+    __isabstractmethod__ = True
+
+    def __init__(self, callable):
+        callable.__isabstractmethod__ = True
+        super().__init__(callable)
+
+
+class abstractproperty(property):
+    """A decorator indicating abstract properties.
+
+    Requires that the metaclass is ABCMeta or derived from it.  A
+    class that has a metaclass derived from ABCMeta cannot be
+    instantiated unless all of its abstract properties are overridden.
+    The abstract properties can be called using any of the normal
+    'super' call mechanisms.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractproperty
+            def my_abstract_property(self):
+                ...
+
+    This defines a read-only property; you can also define a read-write
+    abstract property using the 'long' form of property declaration:
+
+        class C(metaclass=ABCMeta):
+            def getx(self): ...
+            def setx(self, value): ...
+            x = abstractproperty(getx, setx)
+
+    'abstractproperty' is deprecated. Use 'property' with 'abstractmethod'
+    instead.
+    """
+
+    __isabstractmethod__ = True
+
+
+try:
+    from _abc import (get_cache_token, _abc_init, _abc_register,
+                      _abc_instancecheck, _abc_subclasscheck, _get_dump,
+                      _reset_registry, _reset_caches)
+except ImportError:
+    from _py_abc import ABCMeta, get_cache_token
+    ABCMeta.__module__ = 'abc'
+else:
+    class ABCMeta(type):
+        """Metaclass for defining Abstract Base Classes (ABCs).
+
+        Use this metaclass to create an ABC.  An ABC can be subclassed
+        directly, and then acts as a mix-in class.  You can also register
+        unrelated concrete classes (even built-in classes) and unrelated
+        ABCs as 'virtual subclasses' -- these and their descendants will
+        be considered subclasses of the registering ABC by the built-in
+        issubclass() function, but the registering ABC won't show up in
+        their MRO (Method Resolution Order) nor will method
+        implementations defined by the registering ABC be callable (not
+        even via super()).
+        """
+        def __new__(mcls, name, bases, namespace, **kwargs):
+            cls = super().__new__(mcls, name, bases, namespace, **kwargs)
+            _abc_init(cls)
+            return cls
+
+        def register(cls, subclass):
+            """Register a virtual subclass of an ABC.
+
+            Returns the subclass, to allow usage as a class decorator.
+            """
+            return _abc_register(cls, subclass)
+
+        def __instancecheck__(cls, instance):
+            """Override for isinstance(instance, cls)."""
+            return _abc_instancecheck(cls, instance)
+
+        def __subclasscheck__(cls, subclass):
+            """Override for issubclass(subclass, cls)."""
+            return _abc_subclasscheck(cls, subclass)
+
+        def _dump_registry(cls, file=None):
+            """Debug helper to print the ABC registry."""
+            print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file)
+            print(f"Inv. counter: {get_cache_token()}", file=file)
+            (_abc_registry, _abc_cache, _abc_negative_cache,
+             _abc_negative_cache_version) = _get_dump(cls)
+            print(f"_abc_registry: {_abc_registry!r}", file=file)
+            print(f"_abc_cache: {_abc_cache!r}", file=file)
+            print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file)
+            print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}",
+                  file=file)
+
+        def _abc_registry_clear(cls):
+            """Clear the registry (for debugging or testing)."""
+            _reset_registry(cls)
+
+        def _abc_caches_clear(cls):
+            """Clear the caches (for debugging or testing)."""
+            _reset_caches(cls)
+
+
+class ABC(metaclass=ABCMeta):
+    """Helper class that provides a standard way to create an ABC using
+    inheritance.
+    """
+    __slots__ = ()

Lib/collections/abc.py

@@ -0,0 +1,2 @@
+from _collections_abc import *
+from _collections_abc import __all__

Lib/functools.py

@@ -0,0 +1,828 @@
+"""functools.py - Tools for working with functions and callable objects
+"""
+# Python module wrapper for _functools C module
+# to allow utilities written in Python to be added
+# to the functools module.
+# Written by Nick Coghlan <ncoghlan at gmail.com>,
+# Raymond Hettinger <python at rcn.com>,
+# and Łukasz Langa <lukasz at langa.pl>.
+#   Copyright (C) 2006-2013 Python Software Foundation.
+# See C source code for _functools credits/copyright
+
+__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
+           'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial',
+           'partialmethod', 'singledispatch']
+
+try:
+    from _functools import reduce
+except ImportError:
+    pass
+from abc import get_cache_token
+from collections import namedtuple
+# import types, weakref  # Deferred to single_dispatch()
+from reprlib import recursive_repr
+from _thread import RLock
+
+
+################################################################################
+### update_wrapper() and wraps() decorator
+################################################################################
+
+# update_wrapper() and wraps() are tools to help write
+# wrapper functions that can handle naive introspection
+
+WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__',
+                       '__annotations__')
+WRAPPER_UPDATES = ('__dict__',)
+def update_wrapper(wrapper,
+                   wrapped,
+                   assigned = WRAPPER_ASSIGNMENTS,
+                   updated = WRAPPER_UPDATES):
+    """Update a wrapper function to look like the wrapped function
+
+       wrapper is the function to be updated
+       wrapped is the original function
+       assigned is a tuple naming the attributes assigned directly
+       from the wrapped function to the wrapper function (defaults to
+       functools.WRAPPER_ASSIGNMENTS)
+       updated is a tuple naming the attributes of the wrapper that
+       are updated with the corresponding attribute from the wrapped
+       function (defaults to functools.WRAPPER_UPDATES)
+    """
+    for attr in assigned:
+        try:
+            value = getattr(wrapped, attr)
+        except AttributeError:
+            pass
+        else:
+            setattr(wrapper, attr, value)
+    for attr in updated:
+        getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
+    # Issue #17482: set __wrapped__ last so we don't inadvertently copy it
+    # from the wrapped function when updating __dict__
+    wrapper.__wrapped__ = wrapped
+    # Return the wrapper so this can be used as a decorator via partial()
+    return wrapper
+
+def wraps(wrapped,
+          assigned = WRAPPER_ASSIGNMENTS,
+          updated = WRAPPER_UPDATES):
+    """Decorator factory to apply update_wrapper() to a wrapper function
+
+       Returns a decorator that invokes update_wrapper() with the decorated
+       function as the wrapper argument and the arguments to wraps() as the
+       remaining arguments. Default arguments are as for update_wrapper().
+       This is a convenience function to simplify applying partial() to
+       update_wrapper().
+    """
+    return partial(update_wrapper, wrapped=wrapped,
+                   assigned=assigned, updated=updated)
+
+
+################################################################################
+### total_ordering class decorator
+################################################################################
+
+# The total ordering functions all invoke the root magic method directly
+# rather than using the corresponding operator.  This avoids possible
+# infinite recursion that could occur when the operator dispatch logic
+# detects a NotImplemented result and then calls a reflected method.
+
+def _gt_from_lt(self, other, NotImplemented=NotImplemented):
+    'Return a > b.  Computed by @total_ordering from (not a < b) and (a != b).'
+    op_result = self.__lt__(other)
+    if op_result is NotImplemented:
+        return op_result
+    return not op_result and self != other
+
+def _le_from_lt(self, other, NotImplemented=NotImplemented):
+    'Return a <= b.  Computed by @total_ordering from (a < b) or (a == b).'
+    op_result = self.__lt__(other)
+    return op_result or self == other
+
+def _ge_from_lt(self, other, NotImplemented=NotImplemented):
+    'Return a >= b.  Computed by @total_ordering from (not a < b).'
+    op_result = self.__lt__(other)
+    if op_result is NotImplemented:
+        return op_result
+    return not op_result
+
+def _ge_from_le(self, other, NotImplemented=NotImplemented):
+    'Return a >= b.  Computed by @total_ordering from (not a <= b) or (a == b).'
+    op_result = self.__le__(other)
+    if op_result is NotImplemented:
+        return op_result
+    return not op_result or self == other
+
+def _lt_from_le(self, other, NotImplemented=NotImplemented):
+    'Return a < b.  Computed by @total_ordering from (a <= b) and (a != b).'
+    op_result = self.__le__(other)
+    if op_result is NotImplemented:
+        return op_result
+    return op_result and self != other
+
+def _gt_from_le(self, other, NotImplemented=NotImplemented):
+    'Return a > b.  Computed by @total_ordering from (not a <= b).'
+    op_result = self.__le__(other)
+    if op_result is NotImplemented:
+        return op_result
+    return not op_result
+
+def _lt_from_gt(self, other, NotImplemented=NotImplemented):
+    'Return a < b.  Computed by @total_ordering from (not a > b) and (a != b).'
+    op_result = self.__gt__(other)
+    if op_result is NotImplemented:
+        return op_result
+    return not op_result and self != other
+
+def _ge_from_gt(self, other, NotImplemented=NotImplemented):
+    'Return a >= b.  Computed by @total_ordering from (a > b) or (a == b).'
+    op_result = self.__gt__(other)
+    return op_result or self == other
+
+def _le_from_gt(self, other, NotImplemented=NotImplemented):
+    'Return a <= b.  Computed by @total_ordering from (not a > b).'
+    op_result = self.__gt__(other)
+    if op_result is NotImplemented:
+        return op_result
+    return not op_result
+
+def _le_from_ge(self, other, NotImplemented=NotImplemented):
+    'Return a <= b.  Computed by @total_ordering from (not a >= b) or (a == b).'
+    op_result = self.__ge__(other)
+    if op_result is NotImplemented:
+        return op_result
+    return not op_result or self == other
+
+def _gt_from_ge(self, other, NotImplemented=NotImplemented):
+    'Return a > b.  Computed by @total_ordering from (a >= b) and (a != b).'
+    op_result = self.__ge__(other)
+    if op_result is NotImplemented:
+        return op_result
+    return op_result and self != other
+
+def _lt_from_ge(self, other, NotImplemented=NotImplemented):
+    'Return a < b.  Computed by @total_ordering from (not a >= b).'
+    op_result = self.__ge__(other)
+    if op_result is NotImplemented:
+        return op_result
+    return not op_result
+
+_convert = {
+    '__lt__': [('__gt__', _gt_from_lt),
+               ('__le__', _le_from_lt),
+               ('__ge__', _ge_from_lt)],
+    '__le__': [('__ge__', _ge_from_le),
+               ('__lt__', _lt_from_le),
+               ('__gt__', _gt_from_le)],
+    '__gt__': [('__lt__', _lt_from_gt),
+               ('__ge__', _ge_from_gt),
+               ('__le__', _le_from_gt)],
+    '__ge__': [('__le__', _le_from_ge),
+               ('__gt__', _gt_from_ge),
+               ('__lt__', _lt_from_ge)]
+}
+
+def total_ordering(cls):
+    """Class decorator that fills in missing ordering methods"""
+    # Find user-defined comparisons (not those inherited from object).
+    roots = {op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)}
+    if not roots:
+        raise ValueError('must define at least one ordering operation: < > <= >=')
+    root = max(roots)       # prefer __lt__ to __le__ to __gt__ to __ge__
+    for opname, opfunc in _convert[root]:
+        if opname not in roots:
+            opfunc.__name__ = opname
+            setattr(cls, opname, opfunc)
+    return cls
+
+
+################################################################################
+### cmp_to_key() function converter
+################################################################################
+
+def cmp_to_key(mycmp):
+    """Convert a cmp= function into a key= function"""
+    class K(object):
+        __slots__ = ['obj']
+        def __init__(self, obj):
+            self.obj = obj
+        def __lt__(self, other):
+            return mycmp(self.obj, other.obj) < 0
+        def __gt__(self, other):
+            return mycmp(self.obj, other.obj) > 0
+        def __eq__(self, other):
+            return mycmp(self.obj, other.obj) == 0
+        def __le__(self, other):
+            return mycmp(self.obj, other.obj) <= 0
+        def __ge__(self, other):
+            return mycmp(self.obj, other.obj) >= 0
+        __hash__ = None
+    return K
+
+try:
+    from _functools import cmp_to_key
+except ImportError:
+    pass
+
+
+################################################################################
+### partial() argument application
+################################################################################
+
+# Purely functional, no descriptor behaviour
+class partial:
+    """New function with partial application of the given arguments
+    and keywords.
+    """
+
+    __slots__ = "func", "args", "keywords", "__dict__", "__weakref__"
+
+    def __new__(*args, **keywords):
+        if not args:
+            raise TypeError("descriptor '__new__' of partial needs an argument")
+        if len(args) < 2:
+            raise TypeError("type 'partial' takes at least one argument")
+        cls, func, *args = args
+        if not callable(func):
+            raise TypeError("the first argument must be callable")
+        args = tuple(args)
+
+        if hasattr(func, "func"):
+            args = func.args + args
+            tmpkw = func.keywords.copy()
+            tmpkw.update(keywords)
+            keywords = tmpkw
+            del tmpkw
+            func = func.func
+
+        self = super(partial, cls).__new__(cls)
+
+        self.func = func
+        self.args = args
+        self.keywords = keywords
+        return self
+
+    def __call__(*args, **keywords):
+        if not args:
+            raise TypeError("descriptor '__call__' of partial needs an argument")
+        self, *args = args
+        newkeywords = self.keywords.copy()
+        newkeywords.update(keywords)
+        return self.func(*self.args, *args, **newkeywords)
+
+    @recursive_repr()
+    def __repr__(self):
+        qualname = type(self).__qualname__
+        args = [repr(self.func)]
+        args.extend(repr(x) for x in self.args)
+        args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items())
+        if type(self).__module__ == "functools":
+            return f"functools.{qualname}({', '.join(args)})"
+        return f"{qualname}({', '.join(args)})"
+
+    def __reduce__(self):
+        return type(self), (self.func,), (self.func, self.args,
+               self.keywords or None, self.__dict__ or None)
+
+    def __setstate__(self, state):
+        if not isinstance(state, tuple):
+            raise TypeError("argument to __setstate__ must be a tuple")
+        if len(state) != 4:
+            raise TypeError(f"expected 4 items in state, got {len(state)}")
+        func, args, kwds, namespace = state
+        if (not callable(func) or not isinstance(args, tuple) or
+           (kwds is not None and not isinstance(kwds, dict)) or
+           (namespace is not None and not isinstance(namespace, dict))):
+            raise TypeError("invalid partial state")
+
+        args = tuple(args) # just in case it's a subclass
+        if kwds is None:
+            kwds = {}
+        elif type(kwds) is not dict: # XXX does it need to be *exactly* dict?
+            kwds = dict(kwds)
+        if namespace is None:
+            namespace = {}
+
+        self.__dict__ = namespace
+        self.func = func
+        self.args = args
+        self.keywords = kwds
+
+try:
+    from _functools import partial
+except ImportError:
+    pass
+
+# Descriptor version
+class partialmethod(object):
+    """Method descriptor with partial application of the given arguments
+    and keywords.
+
+    Supports wrapping existing descriptors and handles non-descriptor
+    callables as instance methods.
+    """
+
+    def __init__(self, func, *args, **keywords):
+        if not callable(func) and not hasattr(func, "__get__"):
+            raise TypeError("{!r} is not callable or a descriptor"
+                                 .format(func))
+
+        # func could be a descriptor like classmethod which isn't callable,
+        # so we can't inherit from partial (it verifies func is callable)
+        if isinstance(func, partialmethod):
+            # flattening is mandatory in order to place cls/self before all
+            # other arguments
+            # it's also more efficient since only one function will be called
+            self.func = func.func
+            self.args = func.args + args
+            self.keywords = func.keywords.copy()
+            self.keywords.update(keywords)
+        else:
+            self.func = func
+            self.args = args
+            self.keywords = keywords
+
+    def __repr__(self):
+        args = ", ".join(map(repr, self.args))
+        keywords = ", ".join("{}={!r}".format(k, v)
+                                 for k, v in self.keywords.items())
+        format_string = "{module}.{cls}({func}, {args}, {keywords})"
+        return format_string.format(module=self.__class__.__module__,
+                                    cls=self.__class__.__qualname__,
+                                    func=self.func,
+                                    args=args,
+                                    keywords=keywords)
+
+    def _make_unbound_method(self):
+        def _method(*args, **keywords):
+            call_keywords = self.keywords.copy()
+            call_keywords.update(keywords)
+            cls_or_self, *rest = args
+            call_args = (cls_or_self,) + self.args + tuple(rest)
+            return self.func(*call_args, **call_keywords)
+        _method.__isabstractmethod__ = self.__isabstractmethod__
+        _method._partialmethod = self
+        return _method
+
+    def __get__(self, obj, cls):
+        get = getattr(self.func, "__get__", None)
+        result = None
+        if get is not None:
+            new_func = get(obj, cls)
+            if new_func is not self.func:
+                # Assume __get__ returning something new indicates the
+                # creation of an appropriate callable
+                result = partial(new_func, *self.args, **self.keywords)
+                try:
+                    result.__self__ = new_func.__self__
+                except AttributeError:
+                    pass
+        if result is None:
+            # If the underlying descriptor didn't do anything, treat this
+            # like an instance method
+            result = self._make_unbound_method().__get__(obj, cls)
+        return result
+
+    @property
+    def __isabstractmethod__(self):
+        return getattr(self.func, "__isabstractmethod__", False)
+
+
+################################################################################
+### LRU Cache function decorator
+################################################################################
+
+_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
+
+class _HashedSeq(list):
+    """ This class guarantees that hash() will be called no more than once
+        per element.  This is important because the lru_cache() will hash
+        the key multiple times on a cache miss.
+
+    """
+
+    __slots__ = 'hashvalue'
+
+    def __init__(self, tup, hash=hash):
+        self[:] = tup
+        self.hashvalue = hash(tup)
+
+    def __hash__(self):
+        return self.hashvalue
+
+def _make_key(args, kwds, typed,
+             kwd_mark = (object(),),
+             fasttypes = {int, str, frozenset, type(None)},
+             tuple=tuple, type=type, len=len):
+    """Make a cache key from optionally typed positional and keyword arguments
+
+    The key is constructed in a way that is flat as possible rather than
+    as a nested structure that would take more memory.
+
+    If there is only a single argument and its data type is known to cache
+    its hash value, then that argument is returned without a wrapper.  This
+    saves space and improves lookup speed.
+
+    """
+    # All of code below relies on kwds preserving the order input by the user.
+    # Formerly, we sorted() the kwds before looping.  The new way is *much*
+    # faster; however, it means that f(x=1, y=2) will now be treated as a
+    # distinct call from f(y=2, x=1) which will be cached separately.
+    key = args
+    if kwds:
+        key += kwd_mark
+        for item in kwds.items():
+            key += item
+    if typed:
+        key += tuple(type(v) for v in args)
+        if kwds:
+            key += tuple(type(v) for v in kwds.values())
+    elif len(key) == 1 and type(key[0]) in fasttypes:
+        return key[0]
+    return _HashedSeq(key)
+
+def lru_cache(maxsize=128, typed=False):
+    """Least-recently-used cache decorator.
+
+    If *maxsize* is set to None, the LRU features are disabled and the cache
+    can grow without bound.
+
+    If *typed* is True, arguments of different types will be cached separately.
+    For example, f(3.0) and f(3) will be treated as distinct calls with
+    distinct results.
+
+    Arguments to the cached function must be hashable.
+
+    View the cache statistics named tuple (hits, misses, maxsize, currsize)
+    with f.cache_info().  Clear the cache and statistics with f.cache_clear().
+    Access the underlying function with f.__wrapped__.
+
+    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
+
+    """
+
+    # Users should only access the lru_cache through its public API:
+    #       cache_info, cache_clear, and f.__wrapped__
+    # The internals of the lru_cache are encapsulated for thread safety and
+    # to allow the implementation to change (including a possible C version).
+
+    # Early detection of an erroneous call to @lru_cache without any arguments
+    # resulting in the inner function being passed to maxsize instead of an
+    # integer or None.
+    if maxsize is not None and not isinstance(maxsize, int):
+        raise TypeError('Expected maxsize to be an integer or None')
+
+    def decorating_function(user_function):
+        wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo)
+        return update_wrapper(wrapper, user_function)
+
+    return decorating_function
+
+def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo):
+    # Constants shared by all lru cache instances:
+    sentinel = object()          # unique object used to signal cache misses
+    make_key = _make_key         # build a key from the function arguments
+    PREV, NEXT, KEY, RESULT = 0, 1, 2, 3   # names for the link fields
+
+    cache = {}
+    hits = misses = 0
+    full = False
+    cache_get = cache.get    # bound method to lookup a key or return None
+    cache_len = cache.__len__  # get cache size without calling len()
+    lock = RLock()           # because linkedlist updates aren't threadsafe
+    root = []                # root of the circular doubly linked list
+    root[:] = [root, root, None, None]     # initialize by pointing to self
+
+    if maxsize == 0:
+
+        def wrapper(*args, **kwds):
+            # No caching -- just a statistics update after a successful call
+            nonlocal misses
+            result = user_function(*args, **kwds)
+            misses += 1
+            return result
+
+    elif maxsize is None:
+
+        def wrapper(*args, **kwds):
+            # Simple caching without ordering or size limit
+            nonlocal hits, misses
+            key = make_key(args, kwds, typed)
+            result = cache_get(key, sentinel)
+            if result is not sentinel:
+                hits += 1
+                return result
+            result = user_function(*args, **kwds)
+            cache[key] = result
+            misses += 1
+            return result
+
+    else:
+
+        def wrapper(*args, **kwds):
+            # Size limited caching that tracks accesses by recency
+            nonlocal root, hits, misses, full
+            key = make_key(args, kwds, typed)
+            with lock:
+                link = cache_get(key)
+                if link is not None:
+                    # Move the link to the front of the circular queue
+                    link_prev, link_next, _key, result = link
+                    link_prev[NEXT] = link_next
+                    link_next[PREV] = link_prev
+                    last = root[PREV]
+                    last[NEXT] = root[PREV] = link
+                    link[PREV] = last
+                    link[NEXT] = root
+                    hits += 1
+                    return result
+            result = user_function(*args, **kwds)
+            with lock:
+                if key in cache:
+                    # Getting here means that this same key was added to the
+                    # cache while the lock was released.  Since the link
+                    # update is already done, we need only return the
+                    # computed result and update the count of misses.
+                    pass
+                elif full:
+                    # Use the old root to store the new key and result.
+                    oldroot = root
+                    oldroot[KEY] = key
+                    oldroot[RESULT] = result
+                    # Empty the oldest link and make it the new root.
+                    # Keep a reference to the old key and old result to
+                    # prevent their ref counts from going to zero during the
+                    # update. That will prevent potentially arbitrary object
+                    # clean-up code (i.e. __del__) from running while we're
+                    # still adjusting the links.
+                    root = oldroot[NEXT]
+                    oldkey = root[KEY]
+                    oldresult = root[RESULT]
+                    root[KEY] = root[RESULT] = None
+                    # Now update the cache dictionary.
+                    del cache[oldkey]
+                    # Save the potentially reentrant cache[key] assignment
+                    # for last, after the root and links have been put in
+                    # a consistent state.
+                    cache[key] = oldroot
+                else:
+                    # Put result in a new link at the front of the queue.
+                    last = root[PREV]
+                    link = [last, root, key, result]
+                    last[NEXT] = root[PREV] = cache[key] = link
+                    # Use the cache_len bound method instead of the len() function
+                    # which could potentially be wrapped in an lru_cache itself.
+                    full = (cache_len() >= maxsize)
+                misses += 1
+            return result
+
+    def cache_info():
+        """Report cache statistics"""
+        with lock:
+            return _CacheInfo(hits, misses, maxsize, cache_len())
+
+    def cache_clear():
+        """Clear the cache and cache statistics"""
+        nonlocal hits, misses, full
+        with lock:
+            cache.clear()
+            root[:] = [root, root, None, None]
+            hits = misses = 0
+            full = False
+
+    wrapper.cache_info = cache_info
+    wrapper.cache_clear = cache_clear
+    return wrapper
+
+try:
+    from _functools import _lru_cache_wrapper
+except ImportError:
+    pass
+
+
+################################################################################
+### singledispatch() - single-dispatch generic function decorator
+################################################################################
+
+def _c3_merge(sequences):
+    """Merges MROs in *sequences* to a single MRO using the C3 algorithm.
+
+    Adapted from http://www.python.org/download/releases/2.3/mro/.
+
+    """
+    result = []
+    while True:
+        sequences = [s for s in sequences if s]   # purge empty sequences
+        if not sequences:
+            return result
+        for s1 in sequences:   # find merge candidates among seq heads
+            candidate = s1[0]
+            for s2 in sequences:
+                if candidate in s2[1:]:
+                    candidate = None
+                    break      # reject the current head, it appears later
+            else:
+                break
+        if candidate is None:
+            raise RuntimeError("Inconsistent hierarchy")
+        result.append(candidate)
+        # remove the chosen candidate
+        for seq in sequences:
+            if seq[0] == candidate:
+                del seq[0]
+
+def _c3_mro(cls, abcs=None):
+    """Computes the method resolution order using extended C3 linearization.
+
+    If no *abcs* are given, the algorithm works exactly like the built-in C3
+    linearization used for method resolution.
+
+    If given, *abcs* is a list of abstract base classes that should be inserted
+    into the resulting MRO. Unrelated ABCs are ignored and don't end up in the
+    result. The algorithm inserts ABCs where their functionality is introduced,
+    i.e. issubclass(cls, abc) returns True for the class itself but returns
+    False for all its direct base classes. Implicit ABCs for a given class
+    (either registered or inferred from the presence of a special method like
+    __len__) are inserted directly after the last ABC explicitly listed in the
+    MRO of said class. If two implicit ABCs end up next to each other in the
+    resulting MRO, their ordering depends on the order of types in *abcs*.
+
+    """
+    for i, base in enumerate(reversed(cls.__bases__)):
+        if hasattr(base, '__abstractmethods__'):
+            boundary = len(cls.__bases__) - i
+            break   # Bases up to the last explicit ABC are considered first.
+    else:
+        boundary = 0
+    abcs = list(abcs) if abcs else []
+    explicit_bases = list(cls.__bases__[:boundary])
+    abstract_bases = []
+    other_bases = list(cls.__bases__[boundary:])
+    for base in abcs:
+        if issubclass(cls, base) and not any(
+                issubclass(b, base) for b in cls.__bases__
+            ):
+            # If *cls* is the class that introduces behaviour described by
+            # an ABC *base*, insert said ABC to its MRO.
+            abstract_bases.append(base)
+    for base in abstract_bases:
+        abcs.remove(base)
+    explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases]
+    abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases]
+    other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases]
+    return _c3_merge(
+        [[cls]] +
+        explicit_c3_mros + abstract_c3_mros + other_c3_mros +
+        [explicit_bases] + [abstract_bases] + [other_bases]
+    )
+
+def _compose_mro(cls, types):
+    """Calculates the method resolution order for a given class *cls*.
+
+    Includes relevant abstract base classes (with their respective bases) from
+    the *types* iterable. Uses a modified C3 linearization algorithm.
+
+    """
+    bases = set(cls.__mro__)
+    # Remove entries which are already present in the __mro__ or unrelated.
+    def is_related(typ):
+        return (typ not in bases and hasattr(typ, '__mro__')
+                                 and issubclass(cls, typ))
+    types = [n for n in types if is_related(n)]
+    # Remove entries which are strict bases of other entries (they will end up
+    # in the MRO anyway.
+    def is_strict_base(typ):
+        for other in types:
+            if typ != other and typ in other.__mro__:
+                return True
+        return False
+    types = [n for n in types if not is_strict_base(n)]
+    # Subclasses of the ABCs in *types* which are also implemented by
+    # *cls* can be used to stabilize ABC ordering.
+    type_set = set(types)
+    mro = []
+    for typ in types:
+        found = []
+        for sub in typ.__subclasses__():
+            if sub not in bases and issubclass(cls, sub):
+                found.append([s for s in sub.__mro__ if s in type_set])
+        if not found:
+            mro.append(typ)
+            continue
+        # Favor subclasses with the biggest number of useful bases
+        found.sort(key=len, reverse=True)
+        for sub in found:
+            for subcls in sub:
+                if subcls not in mro:
+                    mro.append(subcls)
+    return _c3_mro(cls, abcs=mro)
+
+def _find_impl(cls, registry):
+    """Returns the best matching implementation from *registry* for type *cls*.
+
+    Where there is no registered implementation for a specific type, its method
+    resolution order is used to find a more generic implementation.
+
+    Note: if *registry* does not contain an implementation for the base
+    *object* type, this function may return None.
+
+    """
+    mro = _compose_mro(cls, registry.keys())
+    match = None
+    for t in mro:
+        if match is not None:
+            # If *match* is an implicit ABC but there is another unrelated,
+            # equally matching implicit ABC, refuse the temptation to guess.
+            if (t in registry and t not in cls.__mro__
+                              and match not in cls.__mro__
+                              and not issubclass(match, t)):
+                raise RuntimeError("Ambiguous dispatch: {} or {}".format(
+                    match, t))
+            break
+        if t in registry:
+            match = t
+    return registry.get(match)
+
+def singledispatch(func):
+    """Single-dispatch generic function decorator.
+
+    Transforms a function into a generic function, which can have different
+    behaviours depending upon the type of its first argument. The decorated
+    function acts as the default implementation, and additional
+    implementations can be registered using the register() attribute of the
+    generic function.
+    """
+    # There are many programs that use functools without singledispatch, so we
+    # trade-off making singledispatch marginally slower for the benefit of
+    # making start-up of such applications slightly faster.
+    import types, weakref
+
+    registry = {}
+    dispatch_cache = weakref.WeakKeyDictionary()
+    cache_token = None
+
+    def dispatch(cls):
+        """generic_func.dispatch(cls) -> <function implementation>
+
+        Runs the dispatch algorithm to return the best available implementation
+        for the given *cls* registered on *generic_func*.
+
+        """
+        nonlocal cache_token
+        if cache_token is not None:
+            current_token = get_cache_token()
+            if cache_token != current_token:
+                dispatch_cache.clear()
+                cache_token = current_token
+        try:
+            impl = dispatch_cache[cls]
+        except KeyError:
+            try:
+                impl = registry[cls]
+            except KeyError:
+                impl = _find_impl(cls, registry)
+            dispatch_cache[cls] = impl
+        return impl
+
+    def register(cls, func=None):
+        """generic_func.register(cls, func) -> func
+
+        Registers a new implementation for the given *cls* on a *generic_func*.
+
+        """
+        nonlocal cache_token
+        if func is None:
+            if isinstance(cls, type):
+                return lambda f: register(cls, f)
+            ann = getattr(cls, '__annotations__', {})
+            if not ann:
+                raise TypeError(
+                    f"Invalid first argument to `register()`: {cls!r}. "
+                    f"Use either `@register(some_class)` or plain `@register` "
+                    f"on an annotated function."
+                )
+            func = cls
+
+            # only import typing if annotation parsing is necessary
+            from typing import get_type_hints
+            argname, cls = next(iter(get_type_hints(func).items()))
+            assert isinstance(cls, type), (
+                f"Invalid annotation for {argname!r}. {cls!r} is not a class."
+            )
+        registry[cls] = func
+        if cache_token is None and hasattr(cls, '__abstractmethods__'):
+            cache_token = get_cache_token()
+        dispatch_cache.clear()
+        return func
+
+    def wrapper(*args, **kw):
+        return dispatch(args[0].__class__)(*args, **kw)
+
+    registry[object] = func
+    wrapper.register = register
+    wrapper.dispatch = dispatch
+    wrapper.registry = types.MappingProxyType(registry)
+    wrapper._clear_cache = dispatch_cache.clear
+    update_wrapper(wrapper, func)
+    return wrapper

Lib/genericpath.py

@@ -0,0 +1,151 @@
+"""
+Path operations common to more than one OS
+Do not use directly.  The OS specific modules import the appropriate
+functions from this module themselves.
+"""
+import os
+import stat
+
+__all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime',
+           'getsize', 'isdir', 'isfile', 'samefile', 'sameopenfile',
+           'samestat']
+
+
+# Does a path exist?
+# This is false for dangling symbolic links on systems that support them.
+def exists(path):
+    """Test whether a path exists.  Returns False for broken symbolic links"""
+    try:
+        os.stat(path)
+    except (OSError, ValueError):
+        return False
+    return True
+
+
+# This follows symbolic links, so both islink() and isdir() can be true
+# for the same path on systems that support symlinks
+def isfile(path):
+    """Test whether a path is a regular file"""
+    try:
+        st = os.stat(path)
+    except (OSError, ValueError):
+        return False
+    return stat.S_ISREG(st.st_mode)
+
+
+# Is a path a directory?
+# This follows symbolic links, so both islink() and isdir()
+# can be true for the same path on systems that support symlinks
+def isdir(s):
+    """Return true if the pathname refers to an existing directory."""
+    try:
+        st = os.stat(s)
+    except (OSError, ValueError):
+        return False
+    return stat.S_ISDIR(st.st_mode)
+
+
+def getsize(filename):
+    """Return the size of a file, reported by os.stat()."""
+    return os.stat(filename).st_size
+
+
+def getmtime(filename):
+    """Return the last modification time of a file, reported by os.stat()."""
+    return os.stat(filename).st_mtime
+
+
+def getatime(filename):
+    """Return the last access time of a file, reported by os.stat()."""
+    return os.stat(filename).st_atime
+
+
+def getctime(filename):
+    """Return the metadata change time of a file, reported by os.stat()."""
+    return os.stat(filename).st_ctime
+
+
+# Return the longest prefix of all list elements.
+def commonprefix(m):
+    "Given a list of pathnames, returns the longest common leading component"
+    if not m: return ''
+    # Some people pass in a list of pathname parts to operate in an OS-agnostic
+    # fashion; don't try to translate in that case as that's an abuse of the
+    # API and they are already doing what they need to be OS-agnostic and so
+    # they most likely won't be using an os.PathLike object in the sublists.
+    if not isinstance(m[0], (list, tuple)):
+        m = tuple(map(os.fspath, m))
+    s1 = min(m)
+    s2 = max(m)
+    for i, c in enumerate(s1):
+        if c != s2[i]:
+            return s1[:i]
+    return s1
+
+# Are two stat buffers (obtained from stat, fstat or lstat)
+# describing the same file?
+def samestat(s1, s2):
+    """Test whether two stat buffers reference the same file"""
+    return (s1.st_ino == s2.st_ino and
+            s1.st_dev == s2.st_dev)
+
+
+# Are two filenames really pointing to the same file?
+def samefile(f1, f2):
+    """Test whether two pathnames reference the same actual file"""
+    s1 = os.stat(f1)
+    s2 = os.stat(f2)
+    return samestat(s1, s2)
+
+
+# Are two open files really referencing the same file?
+# (Not necessarily the same file descriptor!)
+def sameopenfile(fp1, fp2):
+    """Test whether two open file objects reference the same file"""
+    s1 = os.fstat(fp1)
+    s2 = os.fstat(fp2)
+    return samestat(s1, s2)
+
+
+# Split a path in root and extension.
+# The extension is everything starting at the last dot in the last
+# pathname component; the root is everything before that.
+# It is always true that root + ext == p.
+
+# Generic implementation of splitext, to be parametrized with
+# the separators
+def _splitext(p, sep, altsep, extsep):
+    """Split the extension from a pathname.
+
+    Extension is everything from the last dot to the end, ignoring
+    leading dots.  Returns "(root, ext)"; ext may be empty."""
+    # NOTE: This code must work for text and bytes strings.
+
+    sepIndex = p.rfind(sep)
+    if altsep:
+        altsepIndex = p.rfind(altsep)
+        sepIndex = max(sepIndex, altsepIndex)
+
+    dotIndex = p.rfind(extsep)
+    if dotIndex > sepIndex:
+        # skip all leading dots
+        filenameIndex = sepIndex + 1
+        while filenameIndex < dotIndex:
+            if p[filenameIndex:filenameIndex+1] != extsep:
+                return p[:dotIndex], p[dotIndex:]
+            filenameIndex += 1
+
+    return p, p[:0]
+
+def _check_arg_types(funcname, *args):
+    hasstr = hasbytes = False
+    for s in args:
+        if isinstance(s, str):
+            hasstr = True
+        elif isinstance(s, bytes):
+            hasbytes = True
+        else:
+            raise TypeError('%s() argument must be str or bytes, not %r' %
+                            (funcname, s.__class__.__name__)) from None
+    if hasstr and hasbytes:
+        raise TypeError("Can't mix strings and bytes in path components") from None

Lib/heapq.py

@@ -0,0 +1,601 @@
+"""Heap queue algorithm (a.k.a. priority queue).
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+Usage:
+
+heap = []            # creates an empty heap
+heappush(heap, item) # pushes a new item on the heap
+item = heappop(heap) # pops the smallest item from the heap
+item = heap[0]       # smallest item on the heap without popping it
+heapify(x)           # transforms list into a heap, in-place, in linear time
+item = heapreplace(heap, item) # pops and returns smallest item, and adds
+                               # new item; the heap size is unchanged
+
+Our API differs from textbook heap algorithms as follows:
+
+- We use 0-based indexing.  This makes the relationship between the
+  index for a node and the indexes for its children slightly less
+  obvious, but is more suitable since Python uses 0-based indexing.
+
+- Our heappop() method returns the smallest item, not the largest.
+
+These two make it possible to view the heap as a regular Python list
+without surprises: heap[0] is the smallest item, and heap.sort()
+maintains the heap invariant!
+"""
+
+# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
+
+__about__ = """Heap queues
+
+[explanation by François Pinard]
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+The strange invariant above is meant to be an efficient memory
+representation for a tournament.  The numbers below are `k', not a[k]:
+
+                                   0
+
+                  1                                 2
+
+          3               4                5               6
+
+      7       8       9       10      11      12      13      14
+
+    15 16   17 18   19 20   21 22   23 24   25 26   27 28   29 30
+
+
+In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'.  In
+a usual binary tournament we see in sports, each cell is the winner
+over the two cells it tops, and we can trace the winner down the tree
+to see all opponents s/he had.  However, in many computer applications
+of such tournaments, we do not need to trace the history of a winner.
+To be more memory efficient, when a winner is promoted, we try to
+replace it by something else at a lower level, and the rule becomes
+that a cell and the two cells it tops contain three different items,
+but the top cell "wins" over the two topped cells.
+
+If this heap invariant is protected at all time, index 0 is clearly
+the overall winner.  The simplest algorithmic way to remove it and
+find the "next" winner is to move some loser (let's say cell 30 in the
+diagram above) into the 0 position, and then percolate this new 0 down
+the tree, exchanging values, until the invariant is re-established.
+This is clearly logarithmic on the total number of items in the tree.
+By iterating over all items, you get an O(n ln n) sort.
+
+A nice feature of this sort is that you can efficiently insert new
+items while the sort is going on, provided that the inserted items are
+not "better" than the last 0'th element you extracted.  This is
+especially useful in simulation contexts, where the tree holds all
+incoming events, and the "win" condition means the smallest scheduled
+time.  When an event schedule other events for execution, they are
+scheduled into the future, so they can easily go into the heap.  So, a
+heap is a good structure for implementing schedulers (this is what I
+used for my MIDI sequencer :-).
+
+Various structures for implementing schedulers have been extensively
+studied, and heaps are good for this, as they are reasonably speedy,
+the speed is almost constant, and the worst case is not much different
+than the average case.  However, there are other representations which
+are more efficient overall, yet the worst cases might be terrible.
+
+Heaps are also very useful in big disk sorts.  You most probably all
+know that a big sort implies producing "runs" (which are pre-sorted
+sequences, which size is usually related to the amount of CPU memory),
+followed by a merging passes for these runs, which merging is often
+very cleverly organised[1].  It is very important that the initial
+sort produces the longest runs possible.  Tournaments are a good way
+to that.  If, using all the memory available to hold a tournament, you
+replace and percolate items that happen to fit the current run, you'll
+produce runs which are twice the size of the memory for random input,
+and much better for input fuzzily ordered.
+
+Moreover, if you output the 0'th item on disk and get an input which
+may not fit in the current tournament (because the value "wins" over
+the last output value), it cannot fit in the heap, so the size of the
+heap decreases.  The freed memory could be cleverly reused immediately
+for progressively building a second heap, which grows at exactly the
+same rate the first heap is melting.  When the first heap completely
+vanishes, you switch heaps and start a new run.  Clever and quite
+effective!
+
+In a word, heaps are useful memory structures to know.  I use them in
+a few applications, and I think it is good to keep a `heap' module
+around. :-)
+
+--------------------
+[1] The disk balancing algorithms which are current, nowadays, are
+more annoying than clever, and this is a consequence of the seeking
+capabilities of the disks.  On devices which cannot seek, like big
+tape drives, the story was quite different, and one had to be very
+clever to ensure (far in advance) that each tape movement will be the
+most effective possible (that is, will best participate at
+"progressing" the merge).  Some tapes were even able to read
+backwards, and this was also used to avoid the rewinding time.
+Believe me, real good tape sorts were quite spectacular to watch!
+From all times, sorting has always been a Great Art! :-)
+"""
+
+__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
+           'nlargest', 'nsmallest', 'heappushpop']
+
+def heappush(heap, item):
+    """Push item onto heap, maintaining the heap invariant."""
+    heap.append(item)
+    _siftdown(heap, 0, len(heap)-1)
+
+def heappop(heap):
+    """Pop the smallest item off the heap, maintaining the heap invariant."""
+    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
+    if heap:
+        returnitem = heap[0]
+        heap[0] = lastelt
+        _siftup(heap, 0)
+        return returnitem
+    return lastelt
+
+def heapreplace(heap, item):
+    """Pop and return the current smallest value, and add the new item.
+
+    This is more efficient than heappop() followed by heappush(), and can be
+    more appropriate when using a fixed-size heap.  Note that the value
+    returned may be larger than item!  That constrains reasonable uses of
+    this routine unless written as part of a conditional replacement:
+
+        if item > heap[0]:
+            item = heapreplace(heap, item)
+    """
+    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
+    heap[0] = item
+    _siftup(heap, 0)
+    return returnitem
+
+def heappushpop(heap, item):
+    """Fast version of a heappush followed by a heappop."""
+    if heap and heap[0] < item:
+        item, heap[0] = heap[0], item
+        _siftup(heap, 0)
+    return item
+
+def heapify(x):
+    """Transform list into a heap, in-place, in O(len(x)) time."""
+    n = len(x)
+    # Transform bottom-up.  The largest index there's any point to looking at
+    # is the largest with a child index in-range, so must have 2*i + 1 < n,
+    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
+    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is
+    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
+    for i in reversed(range(n//2)):
+        _siftup(x, i)
+
+def _heappop_max(heap):
+    """Maxheap version of a heappop."""
+    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
+    if heap:
+        returnitem = heap[0]
+        heap[0] = lastelt
+        _siftup_max(heap, 0)
+        return returnitem
+    return lastelt
+
+def _heapreplace_max(heap, item):
+    """Maxheap version of a heappop followed by a heappush."""
+    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
+    heap[0] = item
+    _siftup_max(heap, 0)
+    return returnitem
+
+def _heapify_max(x):
+    """Transform list into a maxheap, in-place, in O(len(x)) time."""
+    n = len(x)
+    for i in reversed(range(n//2)):
+        _siftup_max(x, i)
+
+# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
+# is the index of a leaf with a possibly out-of-order value.  Restore the
+# heap invariant.
+def _siftdown(heap, startpos, pos):
+    newitem = heap[pos]
+    # Follow the path to the root, moving parents down until finding a place
+    # newitem fits.
+    while pos > startpos:
+        parentpos = (pos - 1) >> 1
+        parent = heap[parentpos]
+        if newitem < parent:
+            heap[pos] = parent
+            pos = parentpos
+            continue
+        break
+    heap[pos] = newitem
+
+# The child indices of heap index pos are already heaps, and we want to make
+# a heap at index pos too.  We do this by bubbling the smaller child of
+# pos up (and so on with that child's children, etc) until hitting a leaf,
+# then using _siftdown to move the oddball originally at index pos into place.
+#
+# We *could* break out of the loop as soon as we find a pos where newitem <=
+# both its children, but turns out that's not a good idea, and despite that
+# many books write the algorithm that way.  During a heap pop, the last array
+# element is sifted in, and that tends to be large, so that comparing it
+# against values starting from the root usually doesn't pay (= usually doesn't
+# get us out of the loop early).  See Knuth, Volume 3, where this is
+# explained and quantified in an exercise.
+#
+# Cutting the # of comparisons is important, since these routines have no
+# way to extract "the priority" from an array element, so that intelligence
+# is likely to be hiding in custom comparison methods, or in array elements
+# storing (priority, record) tuples.  Comparisons are thus potentially
+# expensive.
+#
+# On random arrays of length 1000, making this change cut the number of
+# comparisons made by heapify() a little, and those made by exhaustive
+# heappop() a lot, in accord with theory.  Here are typical results from 3
+# runs (3 just to demonstrate how small the variance is):
+#
+# Compares needed by heapify     Compares needed by 1000 heappops
+# --------------------------     --------------------------------
+# 1837 cut to 1663               14996 cut to 8680
+# 1855 cut to 1659               14966 cut to 8678
+# 1847 cut to 1660               15024 cut to 8703
+#
+# Building the heap by using heappush() 1000 times instead required
+# 2198, 2148, and 2219 compares:  heapify() is more efficient, when
+# you can use it.
+#
+# The total compares needed by list.sort() on the same lists were 8627,
+# 8627, and 8632 (this should be compared to the sum of heapify() and
+# heappop() compares):  list.sort() is (unsurprisingly!) more efficient
+# for sorting.
+
+def _siftup(heap, pos):
+    endpos = len(heap)
+    startpos = pos
+    newitem = heap[pos]
+    # Bubble up the smaller child until hitting a leaf.
+    childpos = 2*pos + 1    # leftmost child position
+    while childpos < endpos:
+        # Set childpos to index of smaller child.
+        rightpos = childpos + 1
+        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
+            childpos = rightpos
+        # Move the smaller child up.
+        heap[pos] = heap[childpos]
+        pos = childpos
+        childpos = 2*pos + 1
+    # The leaf at pos is empty now.  Put newitem there, and bubble it up
+    # to its final resting place (by sifting its parents down).
+    heap[pos] = newitem
+    _siftdown(heap, startpos, pos)
+
+def _siftdown_max(heap, startpos, pos):
+    'Maxheap variant of _siftdown'
+    newitem = heap[pos]
+    # Follow the path to the root, moving parents down until finding a place
+    # newitem fits.
+    while pos > startpos:
+        parentpos = (pos - 1) >> 1
+        parent = heap[parentpos]
+        if parent < newitem:
+            heap[pos] = parent
+            pos = parentpos
+            continue
+        break
+    heap[pos] = newitem
+
+def _siftup_max(heap, pos):
+    'Maxheap variant of _siftup'
+    endpos = len(heap)
+    startpos = pos
+    newitem = heap[pos]
+    # Bubble up the larger child until hitting a leaf.
+    childpos = 2*pos + 1    # leftmost child position
+    while childpos < endpos:
+        # Set childpos to index of larger child.
+        rightpos = childpos + 1
+        if rightpos < endpos and not heap[rightpos] < heap[childpos]:
+            childpos = rightpos
+        # Move the larger child up.
+        heap[pos] = heap[childpos]
+        pos = childpos
+        childpos = 2*pos + 1
+    # The leaf at pos is empty now.  Put newitem there, and bubble it up
+    # to its final resting place (by sifting its parents down).
+    heap[pos] = newitem
+    _siftdown_max(heap, startpos, pos)
+
+def merge(*iterables, key=None, reverse=False):
+    '''Merge multiple sorted inputs into a single sorted output.
+
+    Similar to sorted(itertools.chain(*iterables)) but returns a generator,
+    does not pull the data into memory all at once, and assumes that each of
+    the input streams is already sorted (smallest to largest).
+
+    >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
+    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+
+    If *key* is not None, applies a key function to each element to determine
+    its sort order.
+
+    >>> list(merge(['dog', 'horse'], ['cat', 'fish', 'kangaroo'], key=len))
+    ['dog', 'cat', 'fish', 'horse', 'kangaroo']
+
+    '''
+
+    h = []
+    h_append = h.append
+
+    if reverse:
+        _heapify = _heapify_max
+        _heappop = _heappop_max
+        _heapreplace = _heapreplace_max
+        direction = -1
+    else:
+        _heapify = heapify
+        _heappop = heappop
+        _heapreplace = heapreplace
+        direction = 1
+
+    if key is None:
+        for order, it in enumerate(map(iter, iterables)):
+            try:
+                next = it.__next__
+                h_append([next(), order * direction, next])
+            except StopIteration:
+                pass
+        _heapify(h)
+        while len(h) > 1:
+            try:
+                while True:
+                    value, order, next = s = h[0]
+                    yield value
+                    s[0] = next()           # raises StopIteration when exhausted
+                    _heapreplace(h, s)      # restore heap condition
+            except StopIteration:
+                _heappop(h)                 # remove empty iterator
+        if h:
+            # fast case when only a single iterator remains
+            value, order, next = h[0]
+            yield value
+            yield from next.__self__
+        return
+
+    for order, it in enumerate(map(iter, iterables)):
+        try:
+            next = it.__next__
+            value = next()
+            h_append([key(value), order * direction, value, next])
+        except StopIteration:
+            pass
+    _heapify(h)
+    while len(h) > 1:
+        try:
+            while True:
+                key_value, order, value, next = s = h[0]
+                yield value
+                value = next()
+                s[0] = key(value)
+                s[2] = value
+                _heapreplace(h, s)
+        except StopIteration:
+            _heappop(h)
+    if h:
+        key_value, order, value, next = h[0]
+        yield value
+        yield from next.__self__
+
+
+# Algorithm notes for nlargest() and nsmallest()
+# ==============================================
+#
+# Make a single pass over the data while keeping the k most extreme values
+# in a heap.  Memory consumption is limited to keeping k values in a list.
+#
+# Measured performance for random inputs:
+#
+#                                   number of comparisons
+#    n inputs     k-extreme values  (average of 5 trials)   % more than min()
+# -------------   ----------------  ---------------------   -----------------
+#      1,000           100                  3,317               231.7%
+#     10,000           100                 14,046                40.5%
+#    100,000           100                105,749                 5.7%
+#  1,000,000           100              1,007,751                 0.8%
+# 10,000,000           100             10,009,401                 0.1%
+#
+# Theoretical number of comparisons for k smallest of n random inputs:
+#
+# Step   Comparisons                  Action
+# ----   --------------------------   ---------------------------
+#  1     1.66 * k                     heapify the first k-inputs
+#  2     n - k                        compare remaining elements to top of heap
+#  3     k * (1 + lg2(k)) * ln(n/k)   replace the topmost value on the heap
+#  4     k * lg2(k) - (k/2)           final sort of the k most extreme values
+#
+# Combining and simplifying for a rough estimate gives:
+#
+#        comparisons = n + k * (log(k, 2) * log(n/k) + log(k, 2) + log(n/k))
+#
+# Computing the number of comparisons for step 3:
+# -----------------------------------------------
+# * For the i-th new value from the iterable, the probability of being in the
+#   k most extreme values is k/i.  For example, the probability of the 101st
+#   value seen being in the 100 most extreme values is 100/101.
+# * If the value is a new extreme value, the cost of inserting it into the
+#   heap is 1 + log(k, 2).
+# * The probability times the cost gives:
+#            (k/i) * (1 + log(k, 2))
+# * Summing across the remaining n-k elements gives:
+#            sum((k/i) * (1 + log(k, 2)) for i in range(k+1, n+1))
+# * This reduces to:
+#            (H(n) - H(k)) * k * (1 + log(k, 2))
+# * Where H(n) is the n-th harmonic number estimated by:
+#            gamma = 0.5772156649
+#            H(n) = log(n, e) + gamma + 1 / (2 * n)
+#   http://en.wikipedia.org/wiki/Harmonic_series_(mathematics)#Rate_of_divergence
+# * Substituting the H(n) formula:
+#            comparisons = k * (1 + log(k, 2)) * (log(n/k, e) + (1/n - 1/k) / 2)
+#
+# Worst-case for step 3:
+# ----------------------
+# In the worst case, the input data is reversed sorted so that every new element
+# must be inserted in the heap:
+#
+#             comparisons = 1.66 * k + log(k, 2) * (n - k)
+#
+# Alternative Algorithms
+# ----------------------
+# Other algorithms were not used because they:
+# 1) Took much more auxiliary memory,
+# 2) Made multiple passes over the data.
+# 3) Made more comparisons in common cases (small k, large n, semi-random input).
+# See the more detailed comparison of approach at:
+# http://code.activestate.com/recipes/577573-compare-algorithms-for-heapqsmallest
+
+def nsmallest(n, iterable, key=None):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key)[:n]
+    """
+
+    # Short-cut for n==1 is to use min()
+    if n == 1:
+        it = iter(iterable)
+        sentinel = object()
+        result = min(it, default=sentinel, key=key)
+        return [] if result is sentinel else [result]
+
+    # When n>=size, it's faster to use sorted()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = iter(iterable)
+        # put the range(n) first so that zip() doesn't
+        # consume one too many elements from the iterator
+        result = [(elem, i) for i, elem in zip(range(n), it)]
+        if not result:
+            return result
+        _heapify_max(result)
+        top = result[0][0]
+        order = n
+        _heapreplace = _heapreplace_max
+        for elem in it:
+            if elem < top:
+                _heapreplace(result, (elem, order))
+                top, _order = result[0]
+                order += 1
+        result.sort()
+        return [elem for (elem, order) in result]
+
+    # General case, slowest method
+    it = iter(iterable)
+    result = [(key(elem), i, elem) for i, elem in zip(range(n), it)]
+    if not result:
+        return result
+    _heapify_max(result)
+    top = result[0][0]
+    order = n
+    _heapreplace = _heapreplace_max
+    for elem in it:
+        k = key(elem)
+        if k < top:
+            _heapreplace(result, (k, order, elem))
+            top, _order, _elem = result[0]
+            order += 1
+    result.sort()
+    return [elem for (k, order, elem) in result]
+
+def nlargest(n, iterable, key=None):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
+    """
+
+    # Short-cut for n==1 is to use max()
+    if n == 1:
+        it = iter(iterable)
+        sentinel = object()
+        result = max(it, default=sentinel, key=key)
+        return [] if result is sentinel else [result]
+
+    # When n>=size, it's faster to use sorted()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key, reverse=True)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = iter(iterable)
+        result = [(elem, i) for i, elem in zip(range(0, -n, -1), it)]
+        if not result:
+            return result
+        heapify(result)
+        top = result[0][0]
+        order = -n
+        _heapreplace = heapreplace
+        for elem in it:
+            if top < elem:
+                _heapreplace(result, (elem, order))
+                top, _order = result[0]
+                order -= 1
+        result.sort(reverse=True)
+        return [elem for (elem, order) in result]
+
+    # General case, slowest method
+    it = iter(iterable)
+    result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)]
+    if not result:
+        return result
+    heapify(result)
+    top = result[0][0]
+    order = -n
+    _heapreplace = heapreplace
+    for elem in it:
+        k = key(elem)
+        if top < k:
+            _heapreplace(result, (k, order, elem))
+            top, _order, _elem = result[0]
+            order -= 1
+    result.sort(reverse=True)
+    return [elem for (k, order, elem) in result]
+
+# If available, use C implementation
+try:
+    from _heapq import *
+except ImportError:
+    pass
+try:
+    from _heapq import _heapreplace_max
+except ImportError:
+    pass
+try:
+    from _heapq import _heapify_max
+except ImportError:
+    pass
+try:
+    from _heapq import _heappop_max
+except ImportError:
+    pass
+
+
+if __name__ == "__main__":
+
+    import doctest
+    print(doctest.testmod())

Lib/io.py

@@ -0,0 +1 @@
+from _io import *

Lib/linecache.py

@@ -0,0 +1,177 @@
+"""Cache lines from Python source files.
+
+This is intended to read lines from modules imported -- hence if a filename
+is not found, it will look down the module search path for a file by
+that name.
+"""
+
+import functools
+import sys
+import os
+import tokenize
+
+__all__ = ["getline", "clearcache", "checkcache"]
+
+def getline(filename, lineno, module_globals=None):
+    lines = getlines(filename, module_globals)
+    if 1 <= lineno <= len(lines):
+        return lines[lineno-1]
+    else:
+        return ''
+
+
+# The cache
+
+# The cache. Maps filenames to either a thunk which will provide source code,
+# or a tuple (size, mtime, lines, fullname) once loaded.
+cache = {}
+
+
+def clearcache():
+    """Clear the cache entirely."""
+
+    global cache
+    cache = {}
+
+
+def getlines(filename, module_globals=None):
+    """Get the lines for a Python source file from the cache.
+    Update the cache if it doesn't contain an entry for this file already."""
+
+    if filename in cache:
+        entry = cache[filename]
+        if len(entry) != 1:
+            return cache[filename][2]
+
+    try:
+        return updatecache(filename, module_globals)
+    except MemoryError:
+        clearcache()
+        return []
+
+
+def checkcache(filename=None):
+    """Discard cache entries that are out of date.
+    (This is not checked upon each call!)"""
+
+    if filename is None:
+        filenames = list(cache.keys())
+    else:
+        if filename in cache:
+            filenames = [filename]
+        else:
+            return
+
+    for filename in filenames:
+        entry = cache[filename]
+        if len(entry) == 1:
+            # lazy cache entry, leave it lazy.
+            continue
+        size, mtime, lines, fullname = entry
+        if mtime is None:
+            continue   # no-op for files loaded via a __loader__
+        try:
+            stat = os.stat(fullname)
+        except OSError:
+            del cache[filename]
+            continue
+        if size != stat.st_size or mtime != stat.st_mtime:
+            del cache[filename]
+
+
+def updatecache(filename, module_globals=None):
+    """Update a cache entry and return its list of lines.
+    If something's wrong, print a message, discard the cache entry,
+    and return an empty list."""
+
+    if filename in cache:
+        if len(cache[filename]) != 1:
+            del cache[filename]
+    if not filename or (filename.startswith('<') and filename.endswith('>')):
+        return []
+
+    fullname = filename
+    try:
+        stat = os.stat(fullname)
+    except OSError:
+        basename = filename
+
+        # Realise a lazy loader based lookup if there is one
+        # otherwise try to lookup right now.
+        if lazycache(filename, module_globals):
+            try:
+                data = cache[filename][0]()
+            except (ImportError, OSError):
+                pass
+            else:
+                if data is None:
+                    # No luck, the PEP302 loader cannot find the source
+                    # for this module.
+                    return []
+                cache[filename] = (
+                    len(data), None,
+                    [line+'\n' for line in data.splitlines()], fullname
+                )
+                return cache[filename][2]
+
+        # Try looking through the module search path, which is only useful
+        # when handling a relative filename.
+        if os.path.isabs(filename):
+            return []
+
+        for dirname in sys.path:
+            try:
+                fullname = os.path.join(dirname, basename)
+            except (TypeError, AttributeError):
+                # Not sufficiently string-like to do anything useful with.
+                continue
+            try:
+                stat = os.stat(fullname)
+                break
+            except OSError:
+                pass
+        else:
+            return []
+    try:
+        with tokenize.open(fullname) as fp:
+            lines = fp.readlines()
+    except OSError:
+        return []
+    if lines and not lines[-1].endswith('\n'):
+        lines[-1] += '\n'
+    size, mtime = stat.st_size, stat.st_mtime
+    cache[filename] = size, mtime, lines, fullname
+    return lines
+
+
+def lazycache(filename, module_globals):
+    """Seed the cache for filename with module_globals.
+
+    The module loader will be asked for the source only when getlines is
+    called, not immediately.
+
+    If there is an entry in the cache already, it is not altered.
+
+    :return: True if a lazy load is registered in the cache,
+        otherwise False. To register such a load a module loader with a
+        get_source method must be found, the filename must be a cachable
+        filename, and the filename must not be already cached.
+    """
+    if filename in cache:
+        if len(cache[filename]) == 1:
+            return True
+        else:
+            return False
+    if not filename or (filename.startswith('<') and filename.endswith('>')):
+        return False
+    # Try for a __loader__, if available
+    if module_globals and '__loader__' in module_globals:
+        name = module_globals.get('__name__')
+        loader = module_globals['__loader__']
+        get_source = getattr(loader, 'get_source', None)
+
+        if name and get_source:
+            get_lines = functools.partial(get_source, name)
+            cache[filename] = (get_lines,)
+            return True
+    return False

Lib/ntpath.py

@@ -0,0 +1,661 @@
+# Module 'ntpath' -- common operations on WinNT/Win95 pathnames
+"""Common pathname manipulations, WindowsNT/95 version.
+
+Instead of importing this module directly, import os and refer to this
+module as os.path.
+"""
+
+# strings representing various path-related bits and pieces
+# These are primarily for export; internally, they are hardcoded.
+# Should be set before imports for resolving cyclic dependency.
+curdir = '.'
+pardir = '..'
+extsep = '.'
+sep = '\\'
+pathsep = ';'
+altsep = '/'
+defpath = '.;C:\\bin'
+devnull = 'nul'
+
+import os
+import sys
+import stat
+import genericpath
+from genericpath import *
+
+__all__ = ["normcase","isabs","join","splitdrive","split","splitext",
+           "basename","dirname","commonprefix","getsize","getmtime",
+           "getatime","getctime", "islink","exists","lexists","isdir","isfile",
+           "ismount", "expanduser","expandvars","normpath","abspath",
+           "curdir","pardir","sep","pathsep","defpath","altsep",
+           "extsep","devnull","realpath","supports_unicode_filenames","relpath",
+           "samefile", "sameopenfile", "samestat", "commonpath"]
+
+def _get_bothseps(path):
+    if isinstance(path, bytes):
+        return b'\\/'
+    else:
+        return '\\/'
+
+# Normalize the case of a pathname and map slashes to backslashes.
+# Other normalizations (such as optimizing '../' away) are not done
+# (this is done by normpath).
+
+def normcase(s):
+    """Normalize case of pathname.
+
+    Makes all characters lowercase and all slashes into backslashes."""
+    s = os.fspath(s)
+    if isinstance(s, bytes):
+        return s.replace(b'/', b'\\').lower()
+    else:
+        return s.replace('/', '\\').lower()
+
+
+# Return whether a path is absolute.
+# Trivial in Posix, harder on Windows.
+# For Windows it is absolute if it starts with a slash or backslash (current
+# volume), or if a pathname after the volume-letter-and-colon or UNC-resource
+# starts with a slash or backslash.
+
+def isabs(s):
+    """Test whether a path is absolute"""
+    s = os.fspath(s)
+    s = splitdrive(s)[1]
+    return len(s) > 0 and s[0] in _get_bothseps(s)
+
+
+# Join two (or more) paths.
+def join(path, *paths):
+    path = os.fspath(path)
+    if isinstance(path, bytes):
+        sep = b'\\'
+        seps = b'\\/'
+        colon = b':'
+    else:
+        sep = '\\'
+        seps = '\\/'
+        colon = ':'
+    try:
+        if not paths:
+            path[:0] + sep  #23780: Ensure compatible data type even if p is null.
+        result_drive, result_path = splitdrive(path)
+        for p in map(os.fspath, paths):
+            p_drive, p_path = splitdrive(p)
+            if p_path and p_path[0] in seps:
+                # Second path is absolute
+                if p_drive or not result_drive:
+                    result_drive = p_drive
+                result_path = p_path
+                continue
+            elif p_drive and p_drive != result_drive:
+                if p_drive.lower() != result_drive.lower():
+                    # Different drives => ignore the first path entirely
+                    result_drive = p_drive
+                    result_path = p_path
+                    continue
+                # Same drive in different case
+                result_drive = p_drive
+            # Second path is relative to the first
+            if result_path and result_path[-1] not in seps:
+                result_path = result_path + sep
+            result_path = result_path + p_path
+        ## add separator between UNC and non-absolute path
+        if (result_path and result_path[0] not in seps and
+            result_drive and result_drive[-1:] != colon):
+            return result_drive + sep + result_path
+        return result_drive + result_path
+    except (TypeError, AttributeError, BytesWarning):
+        genericpath._check_arg_types('join', path, *paths)
+        raise
+
+
+# Split a path in a drive specification (a drive letter followed by a
+# colon) and the path specification.
+# It is always true that drivespec + pathspec == p
+def splitdrive(p):
+    """Split a pathname into drive/UNC sharepoint and relative path specifiers.
+    Returns a 2-tuple (drive_or_unc, path); either part may be empty.
+
+    If you assign
+        result = splitdrive(p)
+    It is always true that:
+        result[0] + result[1] == p
+
+    If the path contained a drive letter, drive_or_unc will contain everything
+    up to and including the colon.  e.g. splitdrive("c:/dir") returns ("c:", "/dir")
+
+    If the path contained a UNC path, the drive_or_unc will contain the host name
+    and share up to but not including the fourth directory separator character.
+    e.g. splitdrive("//host/computer/dir") returns ("//host/computer", "/dir")
+
+    Paths cannot contain both a drive letter and a UNC path.
+
+    """
+    p = os.fspath(p)
+    if len(p) >= 2:
+        if isinstance(p, bytes):
+            sep = b'\\'
+            altsep = b'/'
+            colon = b':'
+        else:
+            sep = '\\'
+            altsep = '/'
+            colon = ':'
+        normp = p.replace(altsep, sep)
+        if (normp[0:2] == sep*2) and (normp[2:3] != sep):
+            # is a UNC path:
+            # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path
+            # \\machine\mountpoint\directory\etc\...
+            #           directory ^^^^^^^^^^^^^^^
+            index = normp.find(sep, 2)
+            if index == -1:
+                return p[:0], p
+            index2 = normp.find(sep, index + 1)
+            # a UNC path can't have two slashes in a row
+            # (after the initial two)
+            if index2 == index + 1:
+                return p[:0], p
+            if index2 == -1:
+                index2 = len(p)
+            return p[:index2], p[index2:]
+        if normp[1:2] == colon:
+            return p[:2], p[2:]
+    return p[:0], p
+
+
+# Split a path in head (everything up to the last '/') and tail (the
+# rest).  After the trailing '/' is stripped, the invariant
+# join(head, tail) == p holds.
+# The resulting head won't end in '/' unless it is the root.
+
+def split(p):
+    """Split a pathname.
+
+    Return tuple (head, tail) where tail is everything after the final slash.
+    Either part may be empty."""
+    p = os.fspath(p)
+    seps = _get_bothseps(p)
+    d, p = splitdrive(p)
+    # set i to index beyond p's last slash
+    i = len(p)
+    while i and p[i-1] not in seps:
+        i -= 1
+    head, tail = p[:i], p[i:]  # now tail has no slashes
+    # remove trailing slashes from head, unless it's all slashes
+    head = head.rstrip(seps) or head
+    return d + head, tail
+
+
+# Split a path in root and extension.
+# The extension is everything starting at the last dot in the last
+# pathname component; the root is everything before that.
+# It is always true that root + ext == p.
+
+def splitext(p):
+    p = os.fspath(p)
+    if isinstance(p, bytes):
+        return genericpath._splitext(p, b'\\', b'/', b'.')
+    else:
+        return genericpath._splitext(p, '\\', '/', '.')
+splitext.__doc__ = genericpath._splitext.__doc__
+
+
+# Return the tail (basename) part of a path.
+
+def basename(p):
+    """Returns the final component of a pathname"""
+    return split(p)[1]
+
+
+# Return the head (dirname) part of a path.
+
+def dirname(p):
+    """Returns the directory component of a pathname"""
+    return split(p)[0]
+
+# Is a path a symbolic link?
+# This will always return false on systems where os.lstat doesn't exist.
+
+def islink(path):
+    """Test whether a path is a symbolic link.
+    This will always return false for Windows prior to 6.0.
+    """
+    try:
+        st = os.lstat(path)
+    except (OSError, ValueError, AttributeError):
+        return False
+    return stat.S_ISLNK(st.st_mode)
+
+# Being true for dangling symbolic links is also useful.
+
+def lexists(path):
+    """Test whether a path exists.  Returns True for broken symbolic links"""
+    try:
+        st = os.lstat(path)
+    except (OSError, ValueError):
+        return False
+    return True
+
+# Is a path a mount point?
+# Any drive letter root (eg c:\)
+# Any share UNC (eg \\server\share)
+# Any volume mounted on a filesystem folder
+#
+# No one method detects all three situations. Historically we've lexically
+# detected drive letter roots and share UNCs. The canonical approach to
+# detecting mounted volumes (querying the reparse tag) fails for the most
+# common case: drive letter roots. The alternative which uses GetVolumePathName
+# fails if the drive letter is the result of a SUBST.
+try:
+    from nt import _getvolumepathname
+except ImportError:
+    _getvolumepathname = None
+def ismount(path):
+    """Test whether a path is a mount point (a drive root, the root of a
+    share, or a mounted volume)"""
+    path = os.fspath(path)
+    seps = _get_bothseps(path)
+    path = abspath(path)
+    root, rest = splitdrive(path)
+    if root and root[0] in seps:
+        return (not rest) or (rest in seps)
+    if rest in seps:
+        return True
+
+    if _getvolumepathname:
+        return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps)
+    else:
+        return False
+
+
+# Expand paths beginning with '~' or '~user'.
+# '~' means $HOME; '~user' means that user's home directory.
+# If the path doesn't begin with '~', or if the user or $HOME is unknown,
+# the path is returned unchanged (leaving error reporting to whatever
+# function is called with the expanded path as argument).
+# See also module 'glob' for expansion of *, ? and [...] in pathnames.
+# (A function should also be defined to do full *sh-style environment
+# variable expansion.)
+
+def expanduser(path):
+    """Expand ~ and ~user constructs.
+
+    If user or $HOME is unknown, do nothing."""
+    path = os.fspath(path)
+    if isinstance(path, bytes):
+        tilde = b'~'
+    else:
+        tilde = '~'
+    if not path.startswith(tilde):
+        return path
+    i, n = 1, len(path)
+    while i < n and path[i] not in _get_bothseps(path):
+        i += 1
+
+    if 'USERPROFILE' in os.environ:
+        userhome = os.environ['USERPROFILE']
+    elif not 'HOMEPATH' in os.environ:
+        return path
+    else:
+        try:
+            drive = os.environ['HOMEDRIVE']
+        except KeyError:
+            drive = ''
+        userhome = join(drive, os.environ['HOMEPATH'])
+
+    if isinstance(path, bytes):
+        userhome = os.fsencode(userhome)
+
+    if i != 1: #~user
+        userhome = join(dirname(userhome), path[1:i])
+
+    return userhome + path[i:]
+
+
+# Expand paths containing shell variable substitutions.
+# The following rules apply:
+#       - no expansion within single quotes
+#       - '$$' is translated into '$'
+#       - '%%' is translated into '%' if '%%' are not seen in %var1%%var2%
+#       - ${varname} is accepted.
+#       - $varname is accepted.
+#       - %varname% is accepted.
+#       - varnames can be made out of letters, digits and the characters '_-'
+#         (though is not verified in the ${varname} and %varname% cases)
+# XXX With COMMAND.COM you can use any characters in a variable name,
+# XXX except '^|<>='.
+
+def expandvars(path):
+    """Expand shell variables of the forms $var, ${var} and %var%.
+
+    Unknown variables are left unchanged."""
+    path = os.fspath(path)
+    if isinstance(path, bytes):
+        if b'$' not in path and b'%' not in path:
+            return path
+        import string
+        varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii')
+        quote = b'\''
+        percent = b'%'
+        brace = b'{'
+        rbrace = b'}'
+        dollar = b'$'
+        environ = getattr(os, 'environb', None)
+    else:
+        if '$' not in path and '%' not in path:
+            return path
+        import string
+        varchars = string.ascii_letters + string.digits + '_-'
+        quote = '\''
+        percent = '%'
+        brace = '{'
+        rbrace = '}'
+        dollar = '$'
+        environ = os.environ
+    res = path[:0]
+    index = 0
+    pathlen = len(path)
+    while index < pathlen:
+        c = path[index:index+1]
+        if c == quote:   # no expansion within single quotes
+            path = path[index + 1:]
+            pathlen = len(path)
+            try:
+                index = path.index(c)
+                res += c + path[:index + 1]
+            except ValueError:
+                res += c + path
+                index = pathlen - 1
+        elif c == percent:  # variable or '%'
+            if path[index + 1:index + 2] == percent:
+                res += c
+                index += 1
+            else:
+                path = path[index+1:]
+                pathlen = len(path)
+                try:
+                    index = path.index(percent)
+                except ValueError:
+                    res += percent + path
+                    index = pathlen - 1
+                else:
+                    var = path[:index]
+                    try:
+                        if environ is None:
+                            value = os.fsencode(os.environ[os.fsdecode(var)])
+                        else:
+                            value = environ[var]
+                    except KeyError:
+                        value = percent + var + percent
+                    res += value
+        elif c == dollar:  # variable or '$$'
+            if path[index + 1:index + 2] == dollar:
+                res += c
+                index += 1
+            elif path[index + 1:index + 2] == brace:
+                path = path[index+2:]
+                pathlen = len(path)
+                try:
+                    index = path.index(rbrace)
+                except ValueError:
+                    res += dollar + brace + path
+                    index = pathlen - 1
+                else:
+                    var = path[:index]
+                    try:
+                        if environ is None:
+                            value = os.fsencode(os.environ[os.fsdecode(var)])
+                        else:
+                            value = environ[var]
+                    except KeyError:
+                        value = dollar + brace + var + rbrace
+                    res += value
+            else:
+                var = path[:0]
+                index += 1
+                c = path[index:index + 1]
+                while c and c in varchars:
+                    var += c
+                    index += 1
+                    c = path[index:index + 1]
+                try:
+                    if environ is None:
+                        value = os.fsencode(os.environ[os.fsdecode(var)])
+                    else:
+                        value = environ[var]
+                except KeyError:
+                    value = dollar + var
+                res += value
+                if c:
+                    index -= 1
+        else:
+            res += c
+        index += 1
+    return res
+
+
+# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B.
+# Previously, this function also truncated pathnames to 8+3 format,
+# but as this module is called "ntpath", that's obviously wrong!
+
+def normpath(path):
+    """Normalize path, eliminating double slashes, etc."""
+    path = os.fspath(path)
+    if isinstance(path, bytes):
+        sep = b'\\'
+        altsep = b'/'
+        curdir = b'.'
+        pardir = b'..'
+        special_prefixes = (b'\\\\.\\', b'\\\\?\\')
+    else:
+        sep = '\\'
+        altsep = '/'
+        curdir = '.'
+        pardir = '..'
+        special_prefixes = ('\\\\.\\', '\\\\?\\')
+    if path.startswith(special_prefixes):
+        # in the case of paths with these prefixes:
+        # \\.\ -> device names
+        # \\?\ -> literal paths
+        # do not do any normalization, but return the path unchanged
+        return path
+    path = path.replace(altsep, sep)
+    prefix, path = splitdrive(path)
+
+    # collapse initial backslashes
+    if path.startswith(sep):
+        prefix += sep
+        path = path.lstrip(sep)
+
+    comps = path.split(sep)
+    i = 0
+    while i < len(comps):
+        if not comps[i] or comps[i] == curdir:
+            del comps[i]
+        elif comps[i] == pardir:
+            if i > 0 and comps[i-1] != pardir:
+                del comps[i-1:i+1]
+                i -= 1
+            elif i == 0 and prefix.endswith(sep):
+                del comps[i]
+            else:
+                i += 1
+        else:
+            i += 1
+    # If the path is now empty, substitute '.'
+    if not prefix and not comps:
+        comps.append(curdir)
+    return prefix + sep.join(comps)
+
+def _abspath_fallback(path):
+    """Return the absolute version of a path as a fallback function in case
+    `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for
+    more.
+
+    """
+
+    path = os.fspath(path)
+    if not isabs(path):
+        if isinstance(path, bytes):
+            cwd = os.getcwdb()
+        else:
+            cwd = os.getcwd()
+        path = join(cwd, path)
+    return normpath(path)
+
+# Return an absolute path.
+try:
+    from nt import _getfullpathname
+
+except ImportError: # not running on Windows - mock up something sensible
+    abspath = _abspath_fallback
+
+else:  # use native Windows method on Windows
+    def abspath(path):
+        """Return the absolute version of a path."""
+        try:
+            return normpath(_getfullpathname(path))
+        except (OSError, ValueError):
+            return _abspath_fallback(path)
+
+# realpath is a no-op on systems without islink support
+realpath = abspath
+# Win9x family and earlier have no Unicode filename support.
+supports_unicode_filenames = (hasattr(sys, "getwindowsversion") and
+                              sys.getwindowsversion()[3] >= 2)
+
+def relpath(path, start=None):
+    """Return a relative version of a path"""
+    path = os.fspath(path)
+    if isinstance(path, bytes):
+        sep = b'\\'
+        curdir = b'.'
+        pardir = b'..'
+    else:
+        sep = '\\'
+        curdir = '.'
+        pardir = '..'
+
+    if start is None:
+        start = curdir
+
+    if not path:
+        raise ValueError("no path specified")
+
+    start = os.fspath(start)
+    try:
+        start_abs = abspath(normpath(start))
+        path_abs = abspath(normpath(path))
+        start_drive, start_rest = splitdrive(start_abs)
+        path_drive, path_rest = splitdrive(path_abs)
+        if normcase(start_drive) != normcase(path_drive):
+            raise ValueError("path is on mount %r, start on mount %r" % (
+                path_drive, start_drive))
+
+        start_list = [x for x in start_rest.split(sep) if x]
+        path_list = [x for x in path_rest.split(sep) if x]
+        # Work out how much of the filepath is shared by start and path.
+        i = 0
+        for e1, e2 in zip(start_list, path_list):
+            if normcase(e1) != normcase(e2):
+                break
+            i += 1
+
+        rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
+        if not rel_list:
+            return curdir
+        return join(*rel_list)
+    except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning):
+        genericpath._check_arg_types('relpath', path, start)
+        raise
+
+
+# Return the longest common sub-path of the sequence of paths given as input.
+# The function is case-insensitive and 'separator-insensitive', i.e. if the
+# only difference between two paths is the use of '\' versus '/' as separator,
+# they are deemed to be equal.
+#
+# However, the returned path will have the standard '\' separator (even if the
+# given paths had the alternative '/' separator) and will have the case of the
+# first path given in the sequence. Additionally, any trailing separator is
+# stripped from the returned path.
+
+def commonpath(paths):
+    """Given a sequence of path names, returns the longest common sub-path."""
+
+    if not paths:
+        raise ValueError('commonpath() arg is an empty sequence')
+
+    paths = tuple(map(os.fspath, paths))
+    if isinstance(paths[0], bytes):
+        sep = b'\\'
+        altsep = b'/'
+        curdir = b'.'
+    else:
+        sep = '\\'
+        altsep = '/'
+        curdir = '.'
+
+    try:
+        drivesplits = [splitdrive(p.replace(altsep, sep).lower()) for p in paths]
+        split_paths = [p.split(sep) for d, p in drivesplits]
+
+        try:
+            isabs, = set(p[:1] == sep for d, p in drivesplits)
+        except ValueError:
+            raise ValueError("Can't mix absolute and relative paths") from None
+
+        # Check that all drive letters or UNC paths match. The check is made only
+        # now otherwise type errors for mixing strings and bytes would not be
+        # caught.
+        if len(set(d for d, p in drivesplits)) != 1:
+            raise ValueError("Paths don't have the same drive")
+
+        drive, path = splitdrive(paths[0].replace(altsep, sep))
+        common = path.split(sep)
+        common = [c for c in common if c and c != curdir]
+
+        split_paths = [[c for c in s if c and c != curdir] for s in split_paths]
+        s1 = min(split_paths)
+        s2 = max(split_paths)
+        for i, c in enumerate(s1):
+            if c != s2[i]:
+                common = common[:i]
+                break
+        else:
+            common = common[:len(s1)]
+
+        prefix = drive + sep if isabs else drive
+        return prefix + sep.join(common)
+    except (TypeError, AttributeError):
+        genericpath._check_arg_types('commonpath', *paths)
+        raise
+
+
+# determine if two files are in fact the same file
+try:
+    # GetFinalPathNameByHandle is available starting with Windows 6.0.
+    # Windows XP and non-Windows OS'es will mock _getfinalpathname.
+    if sys.getwindowsversion()[:2] >= (6, 0):
+        from nt import _getfinalpathname
+    else:
+        raise ImportError
+except (AttributeError, ImportError):
+    # On Windows XP and earlier, two files are the same if their absolute
+    # pathnames are the same.
+    # Non-Windows operating systems fake this method with an XP
+    # approximation.
+    def _getfinalpathname(f):
+        return normcase(abspath(f))
+
+
+try:
+    # The genericpath.isdir implementation uses os.stat and checks the mode
+    # attribute to tell whether or not the path is a directory.
+    # This is overkill on Windows - just pass the path to GetFileAttributes
+    # and check the attribute from there.
+    from nt import _isdir as isdir
+except ImportError:
+    # Use genericpath.isdir as imported above.
+    pass

Lib/operator.py

@@ -1,15 +1,464 @@
+"""
+Operator Interface
+
+This module exports a set of functions corresponding to the intrinsic
+operators of Python.  For example, operator.add(x, y) is equivalent
+to the expression x+y.  The function names are those used for special
+methods; variants without leading and trailing '__' are also provided
+for convenience.
+
+This is the pure Python implementation of the module.
+"""
+
+__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf',
+           'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand',
+           'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul',
+           'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift',
+           'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le',
+           'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod',
+           'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift',
+           'setitem', 'sub', 'truediv', 'truth', 'xor']
+
+from builtins import abs as _abs
 
 
-# Comparison operations:
+# Comparison Operations *******************************************************#
+
+def lt(a, b):
+    "Same as a < b."
+    return a < b
+
+def le(a, b):
+    "Same as a <= b."
+    return a <= b
 
 def eq(a, b):
+    "Same as a == b."
     return a == b
 
-
 def ne(a, b):
+    "Same as a != b."
     return a != b
 
-
 def ge(a, b):
+    "Same as a >= b."
     return a >= b
 
+def gt(a, b):
+    "Same as a > b."
+    return a > b
+
+# Logical Operations **********************************************************#
+
+def not_(a):
+    "Same as not a."
+    return not a
+
+def truth(a):
+    "Return True if a is true, False otherwise."
+    return True if a else False
+
+def is_(a, b):
+    "Same as a is b."
+    return a is b
+
+def is_not(a, b):
+    "Same as a is not b."
+    return a is not b
+
+# Mathematical/Bitwise Operations *********************************************#
+
+def abs(a):
+    "Same as abs(a)."
+    return _abs(a)
+
+def add(a, b):
+    "Same as a + b."
+    return a + b
+
+def and_(a, b):
+    "Same as a & b."
+    return a & b
+
+def floordiv(a, b):
+    "Same as a // b."
+    return a // b
+
+def index(a):
+    "Same as a.__index__()."
+    return a.__index__()
+
+def inv(a):
+    "Same as ~a."
+    return ~a
+invert = inv
+
+def lshift(a, b):
+    "Same as a << b."
+    return a << b
+
+def mod(a, b):
+    "Same as a % b."
+    return a % b
+
+def mul(a, b):
+    "Same as a * b."
+    return a * b
+
+def matmul(a, b):
+    "Same as a @ b."
+    return a @ b
+
+def neg(a):
+    "Same as -a."
+    return -a
+
+def or_(a, b):
+    "Same as a | b."
+    return a | b
+
+def pos(a):
+    "Same as +a."
+    return +a
+
+def pow(a, b):
+    "Same as a ** b."
+    return a ** b
+
+def rshift(a, b):
+    "Same as a >> b."
+    return a >> b
+
+def sub(a, b):
+    "Same as a - b."
+    return a - b
+
+def truediv(a, b):
+    "Same as a / b."
+    return a / b
+
+def xor(a, b):
+    "Same as a ^ b."
+    return a ^ b
+
+# Sequence Operations *********************************************************#
+
+def concat(a, b):
+    "Same as a + b, for a and b sequences."
+    if not hasattr(a, '__getitem__'):
+        msg = "'%s' object can't be concatenated" % type(a).__name__
+        raise TypeError(msg)
+    return a + b
+
+def contains(a, b):
+    "Same as b in a (note reversed operands)."
+    return b in a
+
+def countOf(a, b):
+    "Return the number of times b occurs in a."
+    count = 0
+    for i in a:
+        if i == b:
+            count += 1
+    return count
+
+def delitem(a, b):
+    "Same as del a[b]."
+    del a[b]
+
+def getitem(a, b):
+    "Same as a[b]."
+    return a[b]
+
+def indexOf(a, b):
+    "Return the first index of b in a."
+    for i, j in enumerate(a):
+        if j == b:
+            return i
+    else:
+        raise ValueError('sequence.index(x): x not in sequence')
+
+def setitem(a, b, c):
+    "Same as a[b] = c."
+    a[b] = c
+
+def length_hint(obj, default=0):
+    """
+    Return an estimate of the number of items in obj.
+    This is useful for presizing containers when building from an iterable.
+
+    If the object supports len(), the result will be exact. Otherwise, it may
+    over- or under-estimate by an arbitrary amount. The result will be an
+    integer >= 0.
+    """
+    if not isinstance(default, int):
+        msg = ("'%s' object cannot be interpreted as an integer" %
+               type(default).__name__)
+        raise TypeError(msg)
+
+    try:
+        return len(obj)
+    except TypeError:
+        pass
+
+    try:
+        hint = type(obj).__length_hint__
+    except AttributeError:
+        return default
+
+    try:
+        val = hint(obj)
+    except TypeError:
+        return default
+    if val is NotImplemented:
+        return default
+    if not isinstance(val, int):
+        msg = ('__length_hint__ must be integer, not %s' %
+               type(val).__name__)
+        raise TypeError(msg)
+    if val < 0:
+        msg = '__length_hint__() should return >= 0'
+        raise ValueError(msg)
+    return val
+
+# Generalized Lookup Objects **************************************************#
+
+class attrgetter:
+    """
+    Return a callable object that fetches the given attribute(s) from its operand.
+    After f = attrgetter('name'), the call f(r) returns r.name.
+    After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date).
+    After h = attrgetter('name.first', 'name.last'), the call h(r) returns
+    (r.name.first, r.name.last).
+    """
+    __slots__ = ('_attrs', '_call')
+
+    def __init__(self, attr, *attrs):
+        if not attrs:
+            if not isinstance(attr, str):
+                raise TypeError('attribute name must be a string')
+            self._attrs = (attr,)
+            names = attr.split('.')
+            def func(obj):
+                for name in names:
+                    obj = getattr(obj, name)
+                return obj
+            self._call = func
+        else:
+            self._attrs = (attr,) + attrs
+            getters = tuple(map(attrgetter, self._attrs))
+            def func(obj):
+                return tuple(getter(obj) for getter in getters)
+            self._call = func
+
+    def __call__(self, obj):
+        return self._call(obj)
+
+    def __repr__(self):
+        return '%s.%s(%s)' % (self.__class__.__module__,
+                              self.__class__.__qualname__,
+                              ', '.join(map(repr, self._attrs)))
+
+    def __reduce__(self):
+        return self.__class__, self._attrs
+
+class itemgetter:
+    """
+    Return a callable object that fetches the given item(s) from its operand.
+    After f = itemgetter(2), the call f(r) returns r[2].
+    After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3])
+    """
+    __slots__ = ('_items', '_call')
+
+    def __init__(self, item, *items):
+        if not items:
+            self._items = (item,)
+            def func(obj):
+                return obj[item]
+            self._call = func
+        else:
+            self._items = items = (item,) + items
+            def func(obj):
+                return tuple(obj[i] for i in items)
+            self._call = func
+
+    def __call__(self, obj):
+        return self._call(obj)
+
+    def __repr__(self):
+        return '%s.%s(%s)' % (self.__class__.__module__,
+                              self.__class__.__name__,
+                              ', '.join(map(repr, self._items)))
+
+    def __reduce__(self):
+        return self.__class__, self._items
+
+class methodcaller:
+    """
+    Return a callable object that calls the given method on its operand.
+    After f = methodcaller('name'), the call f(r) returns r.name().
+    After g = methodcaller('name', 'date', foo=1), the call g(r) returns
+    r.name('date', foo=1).
+    """
+    __slots__ = ('_name', '_args', '_kwargs')
+
+    def __init__(*args, **kwargs):
+        if len(args) < 2:
+            msg = "methodcaller needs at least one argument, the method name"
+            raise TypeError(msg)
+        self = args[0]
+        self._name = args[1]
+        if not isinstance(self._name, str):
+            raise TypeError('method name must be a string')
+        self._args = args[2:]
+        self._kwargs = kwargs
+
+    def __call__(self, obj):
+        return getattr(obj, self._name)(*self._args, **self._kwargs)
+
+    def __repr__(self):
+        args = [repr(self._name)]
+        args.extend(map(repr, self._args))
+        args.extend('%s=%r' % (k, v) for k, v in self._kwargs.items())
+        return '%s.%s(%s)' % (self.__class__.__module__,
+                              self.__class__.__name__,
+                              ', '.join(args))
+
+    def __reduce__(self):
+        if not self._kwargs:
+            return self.__class__, (self._name,) + self._args
+        else:
+            from functools import partial
+            return partial(self.__class__, self._name, **self._kwargs), self._args
+
+
+# In-place Operations *********************************************************#
+
+def iadd(a, b):
+    "Same as a += b."
+    a += b
+    return a
+
+def iand(a, b):
+    "Same as a &= b."
+    a &= b
+    return a
+
+def iconcat(a, b):
+    "Same as a += b, for a and b sequences."
+    if not hasattr(a, '__getitem__'):
+        msg = "'%s' object can't be concatenated" % type(a).__name__
+        raise TypeError(msg)
+    a += b
+    return a
+
+def ifloordiv(a, b):
+    "Same as a //= b."
+    a //= b
+    return a
+
+def ilshift(a, b):
+    "Same as a <<= b."
+    a <<= b
+    return a
+
+def imod(a, b):
+    "Same as a %= b."
+    a %= b
+    return a
+
+def imul(a, b):
+    "Same as a *= b."
+    a *= b
+    return a
+
+def imatmul(a, b):
+    "Same as a @= b."
+    a @= b
+    return a
+
+def ior(a, b):
+    "Same as a |= b."
+    a |= b
+    return a
+
+def ipow(a, b):
+    "Same as a **= b."
+    a **=b
+    return a
+
+def irshift(a, b):
+    "Same as a >>= b."
+    a >>= b
+    return a
+
+def isub(a, b):
+    "Same as a -= b."
+    a -= b
+    return a
+
+def itruediv(a, b):
+    "Same as a /= b."
+    a /= b
+    return a
+
+def ixor(a, b):
+    "Same as a ^= b."
+    a ^= b
+    return a
+
+
+try:
+    from _operator import *
+except ImportError:
+    pass
+else:
+    from _operator import __doc__
+
+# All of these "__func__ = func" assignments have to happen after importing
+# from _operator to make sure they're set to the right function
+__lt__ = lt
+__le__ = le
+__eq__ = eq
+__ne__ = ne
+__ge__ = ge
+__gt__ = gt
+__not__ = not_
+__abs__ = abs
+__add__ = add
+__and__ = and_
+__floordiv__ = floordiv
+__index__ = index
+__inv__ = inv
+__invert__ = invert
+__lshift__ = lshift
+__mod__ = mod
+__mul__ = mul
+__matmul__ = matmul
+__neg__ = neg
+__or__ = or_
+__pos__ = pos
+__pow__ = pow
+__rshift__ = rshift
+__sub__ = sub
+__truediv__ = truediv
+__xor__ = xor
+__concat__ = concat
+__contains__ = contains
+__delitem__ = delitem
+__getitem__ = getitem
+__setitem__ = setitem
+__iadd__ = iadd
+__iand__ = iand
+__iconcat__ = iconcat
+__ifloordiv__ = ifloordiv
+__ilshift__ = ilshift
+__imod__ = imod
+__imul__ = imul
+__imatmul__ = imatmul
+__ior__ = ior
+__ipow__ = ipow
+__irshift__ = irshift
+__isub__ = isub
+__itruediv__ = itruediv
+__ixor__ = ixor

Lib/os.py

@@ -0,0 +1,135 @@
+import sys
+
+from _os import *
+
+
+if name == 'nt':
+    linesep = '\r\n'
+    import ntpath as path
+else:
+    linesep = '\n'
+    import posixpath as path
+
+
+sys.modules['os.path'] = path
+from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, altsep,
+    devnull)
+
+# Change environ to automatically call putenv(), unsetenv if they exist.
+from _collections_abc import MutableMapping
+
+class _Environ(MutableMapping):
+    def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv):
+        self.encodekey = encodekey
+        self.decodekey = decodekey
+        self.encodevalue = encodevalue
+        self.decodevalue = decodevalue
+        self.putenv = putenv
+        self.unsetenv = unsetenv
+        self._data = data
+
+    def __getitem__(self, key):
+        try:
+            value = self._data[self.encodekey(key)]
+        except KeyError:
+            # raise KeyError with the original key value
+            raise KeyError(key) from None
+
+        return self.decodevalue(value)
+
+    def __setitem__(self, key, value):
+        key = self.encodekey(key)
+        value = self.encodevalue(value)
+        self.putenv(key, value)
+        self._data[key] = value
+
+    def __delitem__(self, key):
+        encodedkey = self.encodekey(key)
+        self.unsetenv(encodedkey)
+        try:
+            del self._data[encodedkey]
+        except KeyError:
+            # raise KeyError with the original key value
+            raise KeyError(key) from None
+
+    def __iter__(self):
+        # list() from dict object is an atomic operation
+        keys = list(self._data)
+        for key in keys:
+            yield self.decodekey(key)
+
+    def __len__(self):
+        return len(self._data)
+
+    def __repr__(self):
+        return 'environ({{{}}})'.format(', '.join(
+            ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value))
+            for key, value in self._data.items())))
+
+    def copy(self):
+        return dict(self)
+
+    def setdefault(self, key, value):
+        if key not in self:
+            self[key] = value
+        return self[key]
+
+try:
+    _putenv = putenv
+except NameError:
+    _putenv = lambda key, value: None
+# else:
+#     if "putenv" not in __all__:
+#         __all__.append("putenv")
+
+try:
+    _unsetenv = unsetenv
+except NameError:
+    _unsetenv = lambda key: _putenv(key, "")
+# else:
+#     if "unsetenv" not in __all__:
+#         __all__.append("unsetenv")
+
+def _createenviron():
+    # if name == 'nt':
+    #     # Where Env Var Names Must Be UPPERCASE
+    #     def check_str(value):
+    #         if not isinstance(value, str):
+    #             raise TypeError("str expected, not %s" % type(value).__name__)
+    #         return value
+    #     encode = check_str
+    #     decode = str
+    #     def encodekey(key):
+    #         return encode(key).upper()
+    #     data = {}
+    #     for key, value in environ.items():
+    #         data[encodekey(key)] = value
+    # else:
+    #     # Where Env Var Names Can Be Mixed Case
+    #     encoding = sys.getfilesystemencoding()
+    #     def encode(value):
+    #         if not isinstance(value, str):
+    #             raise TypeError("str expected, not %s" % type(value).__name__)
+    #         return value.encode(encoding, 'surrogateescape')
+    #     def decode(value):
+    #         return value.decode(encoding, 'surrogateescape')
+    #     encodekey = encode
+    decode = str
+    encode = str
+    encodekey = encode
+    data = environ
+    return _Environ(data,
+        encodekey, decode,
+        encode, decode,
+        _putenv, _unsetenv)
+
+# unicode environ
+environ = _createenviron()
+del _createenviron
+
+
+def getenv(key, default=None):
+    """Get an environment variable, return None if it doesn't exist.
+    The optional second argument can specify an alternate default.
+    key, default and the result are str."""
+    return environ.get(key, default)

Lib/posixpath.py

@@ -0,0 +1,525 @@
+"""Common operations on Posix pathnames.
+
+Instead of importing this module directly, import os and refer to
+this module as os.path.  The "os.path" name is an alias for this
+module on Posix systems; on other systems (e.g. Windows),
+os.path provides the same operations in a manner specific to that
+platform, and is an alias to another module (e.g. ntpath).
+
+Some of this can actually be useful on non-Posix systems too, e.g.
+for manipulation of the pathname component of URLs.
+"""
+
+# Strings representing various path-related bits and pieces.
+# These are primarily for export; internally, they are hardcoded.
+# Should be set before imports for resolving cyclic dependency.
+curdir = '.'
+pardir = '..'
+extsep = '.'
+sep = '/'
+pathsep = ':'
+defpath = '/bin:/usr/bin'
+altsep = None
+devnull = '/dev/null'
+
+import os
+import sys
+import stat
+import genericpath
+from genericpath import *
+
+__all__ = ["normcase","isabs","join","splitdrive","split","splitext",
+           "basename","dirname","commonprefix","getsize","getmtime",
+           "getatime","getctime","islink","exists","lexists","isdir","isfile",
+           "ismount", "expanduser","expandvars","normpath","abspath",
+           "samefile","sameopenfile","samestat",
+           "curdir","pardir","sep","pathsep","defpath","altsep","extsep",
+           "devnull","realpath","supports_unicode_filenames","relpath",
+           "commonpath"]
+
+
+def _get_sep(path):
+    if isinstance(path, bytes):
+        return b'/'
+    else:
+        return '/'
+
+# Normalize the case of a pathname.  Trivial in Posix, string.lower on Mac.
+# On MS-DOS this may also turn slashes into backslashes; however, other
+# normalizations (such as optimizing '../' away) are not allowed
+# (another function should be defined to do that).
+
+def normcase(s):
+    """Normalize case of pathname.  Has no effect under Posix"""
+    return os.fspath(s)
+
+
+# Return whether a path is absolute.
+# Trivial in Posix, harder on the Mac or MS-DOS.
+
+def isabs(s):
+    """Test whether a path is absolute"""
+    s = os.fspath(s)
+    sep = _get_sep(s)
+    return s.startswith(sep)
+
+
+# Join pathnames.
+# Ignore the previous parts if a part is absolute.
+# Insert a '/' unless the first part is empty or already ends in '/'.
+
+def join(a, *p):
+    """Join two or more pathname components, inserting '/' as needed.
+    If any component is an absolute path, all previous path components
+    will be discarded.  An empty last part will result in a path that
+    ends with a separator."""
+    a = os.fspath(a)
+    sep = _get_sep(a)
+    path = a
+    try:
+        if not p:
+            path[:0] + sep  #23780: Ensure compatible data type even if p is null.
+        for b in map(os.fspath, p):
+            if b.startswith(sep):
+                path = b
+            elif not path or path.endswith(sep):
+                path += b
+            else:
+                path += sep + b
+    except (TypeError, AttributeError, BytesWarning):
+        genericpath._check_arg_types('join', a, *p)
+        raise
+    return path
+
+
+# Split a path in head (everything up to the last '/') and tail (the
+# rest).  If the path ends in '/', tail will be empty.  If there is no
+# '/' in the path, head  will be empty.
+# Trailing '/'es are stripped from head unless it is the root.
+
+def split(p):
+    """Split a pathname.  Returns tuple "(head, tail)" where "tail" is
+    everything after the final slash.  Either part may be empty."""
+    p = os.fspath(p)
+    sep = _get_sep(p)
+    i = p.rfind(sep) + 1
+    head, tail = p[:i], p[i:]
+    if head and head != sep*len(head):
+        head = head.rstrip(sep)
+    return head, tail
+
+
+# Split a path in root and extension.
+# The extension is everything starting at the last dot in the last
+# pathname component; the root is everything before that.
+# It is always true that root + ext == p.
+
+def splitext(p):
+    p = os.fspath(p)
+    if isinstance(p, bytes):
+        sep = b'/'
+        extsep = b'.'
+    else:
+        sep = '/'
+        extsep = '.'
+    return genericpath._splitext(p, sep, None, extsep)
+splitext.__doc__ = genericpath._splitext.__doc__
+
+# Split a pathname into a drive specification and the rest of the
+# path.  Useful on DOS/Windows/NT; on Unix, the drive is always empty.
+
+def splitdrive(p):
+    """Split a pathname into drive and path. On Posix, drive is always
+    empty."""
+    p = os.fspath(p)
+    return p[:0], p
+
+
+# Return the tail (basename) part of a path, same as split(path)[1].
+
+def basename(p):
+    """Returns the final component of a pathname"""
+    p = os.fspath(p)
+    sep = _get_sep(p)
+    i = p.rfind(sep) + 1
+    return p[i:]
+
+
+# Return the head (dirname) part of a path, same as split(path)[0].
+
+def dirname(p):
+    """Returns the directory component of a pathname"""
+    p = os.fspath(p)
+    sep = _get_sep(p)
+    i = p.rfind(sep) + 1
+    head = p[:i]
+    if head and head != sep*len(head):
+        head = head.rstrip(sep)
+    return head
+
+
+# Is a path a symbolic link?
+# This will always return false on systems where os.lstat doesn't exist.
+
+def islink(path):
+    """Test whether a path is a symbolic link"""
+    try:
+        st = os.lstat(path)
+    except (OSError, ValueError, AttributeError):
+        return False
+    return stat.S_ISLNK(st.st_mode)
+
+# Being true for dangling symbolic links is also useful.
+
+def lexists(path):
+    """Test whether a path exists.  Returns True for broken symbolic links"""
+    try:
+        os.lstat(path)
+    except (OSError, ValueError):
+        return False
+    return True
+
+
+# Is a path a mount point?
+# (Does this work for all UNIXes?  Is it even guaranteed to work by Posix?)
+
+def ismount(path):
+    """Test whether a path is a mount point"""
+    try:
+        s1 = os.lstat(path)
+    except (OSError, ValueError):
+        # It doesn't exist -- so not a mount point. :-)
+        return False
+    else:
+        # A symlink can never be a mount point
+        if stat.S_ISLNK(s1.st_mode):
+            return False
+
+    if isinstance(path, bytes):
+        parent = join(path, b'..')
+    else:
+        parent = join(path, '..')
+    parent = realpath(parent)
+    try:
+        s2 = os.lstat(parent)
+    except (OSError, ValueError):
+        return False
+
+    dev1 = s1.st_dev
+    dev2 = s2.st_dev
+    if dev1 != dev2:
+        return True     # path/.. on a different device as path
+    ino1 = s1.st_ino
+    ino2 = s2.st_ino
+    if ino1 == ino2:
+        return True     # path/.. is the same i-node as path
+    return False
+
+
+# Expand paths beginning with '~' or '~user'.
+# '~' means $HOME; '~user' means that user's home directory.
+# If the path doesn't begin with '~', or if the user or $HOME is unknown,
+# the path is returned unchanged (leaving error reporting to whatever
+# function is called with the expanded path as argument).
+# See also module 'glob' for expansion of *, ? and [...] in pathnames.
+# (A function should also be defined to do full *sh-style environment
+# variable expansion.)
+
+def expanduser(path):
+    """Expand ~ and ~user constructions.  If user or $HOME is unknown,
+    do nothing."""
+    path = os.fspath(path)
+    if isinstance(path, bytes):
+        tilde = b'~'
+    else:
+        tilde = '~'
+    if not path.startswith(tilde):
+        return path
+    sep = _get_sep(path)
+    i = path.find(sep, 1)
+    if i < 0:
+        i = len(path)
+    if i == 1:
+        if 'HOME' not in os.environ:
+            import pwd
+            try:
+                userhome = pwd.getpwuid(os.getuid()).pw_dir
+            except KeyError:
+                # bpo-10496: if the current user identifier doesn't exist in the
+                # password database, return the path unchanged
+                return path
+        else:
+            userhome = os.environ['HOME']
+    else:
+        import pwd
+        name = path[1:i]
+        if isinstance(name, bytes):
+            name = str(name, 'ASCII')
+        try:
+            pwent = pwd.getpwnam(name)
+        except KeyError:
+            # bpo-10496: if the user name from the path doesn't exist in the
+            # password database, return the path unchanged
+            return path
+        userhome = pwent.pw_dir
+    if isinstance(path, bytes):
+        userhome = os.fsencode(userhome)
+        root = b'/'
+    else:
+        root = '/'
+    userhome = userhome.rstrip(root)
+    return (userhome + path[i:]) or root
+
+
+# Expand paths containing shell variable substitutions.
+# This expands the forms $variable and ${variable} only.
+# Non-existent variables are left unchanged.
+
+_varprog = None
+_varprogb = None
+
+def expandvars(path):
+    """Expand shell variables of form $var and ${var}.  Unknown variables
+    are left unchanged."""
+    path = os.fspath(path)
+    global _varprog, _varprogb
+    if isinstance(path, bytes):
+        if b'$' not in path:
+            return path
+        if not _varprogb:
+            import re
+            _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII)
+        search = _varprogb.search
+        start = b'{'
+        end = b'}'
+        environ = getattr(os, 'environb', None)
+    else:
+        if '$' not in path:
+            return path
+        if not _varprog:
+            import re
+            _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII)
+        search = _varprog.search
+        start = '{'
+        end = '}'
+        environ = os.environ
+    i = 0
+    while True:
+        m = search(path, i)
+        if not m:
+            break
+        i, j = m.span(0)
+        name = m.group(1)
+        if name.startswith(start) and name.endswith(end):
+            name = name[1:-1]
+        try:
+            if environ is None:
+                value = os.fsencode(os.environ[os.fsdecode(name)])
+            else:
+                value = environ[name]
+        except KeyError:
+            i = j
+        else:
+            tail = path[j:]
+            path = path[:i] + value
+            i = len(path)
+            path += tail
+    return path
+
+
+# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B.
+# It should be understood that this may change the meaning of the path
+# if it contains symbolic links!
+
+def normpath(path):
+    """Normalize path, eliminating double slashes, etc."""
+    path = os.fspath(path)
+    if isinstance(path, bytes):
+        sep = b'/'
+        empty = b''
+        dot = b'.'
+        dotdot = b'..'
+    else:
+        sep = '/'
+        empty = ''
+        dot = '.'
+        dotdot = '..'
+    if path == empty:
+        return dot
+    initial_slashes = path.startswith(sep)
+    # POSIX allows one or two initial slashes, but treats three or more
+    # as single slash.
+    if (initial_slashes and
+        path.startswith(sep*2) and not path.startswith(sep*3)):
+        initial_slashes = 2
+    comps = path.split(sep)
+    new_comps = []
+    for comp in comps:
+        if comp in (empty, dot):
+            continue
+        if (comp != dotdot or (not initial_slashes and not new_comps) or
+             (new_comps and new_comps[-1] == dotdot)):
+            new_comps.append(comp)
+        elif new_comps:
+            new_comps.pop()
+    comps = new_comps
+    path = sep.join(comps)
+    if initial_slashes:
+        path = sep*initial_slashes + path
+    return path or dot
+
+
+def abspath(path):
+    """Return an absolute path."""
+    path = os.fspath(path)
+    if not isabs(path):
+        if isinstance(path, bytes):
+            cwd = os.getcwdb()
+        else:
+            cwd = os.getcwd()
+        path = join(cwd, path)
+    return normpath(path)
+
+
+# Return a canonical path (i.e. the absolute location of a file on the
+# filesystem).
+
+def realpath(filename):
+    """Return the canonical path of the specified filename, eliminating any
+symbolic links encountered in the path."""
+    filename = os.fspath(filename)
+    path, ok = _joinrealpath(filename[:0], filename, {})
+    return abspath(path)
+
+# Join two paths, normalizing and eliminating any symbolic links
+# encountered in the second path.
+def _joinrealpath(path, rest, seen):
+    if isinstance(path, bytes):
+        sep = b'/'
+        curdir = b'.'
+        pardir = b'..'
+    else:
+        sep = '/'
+        curdir = '.'
+        pardir = '..'
+
+    if isabs(rest):
+        rest = rest[1:]
+        path = sep
+
+    while rest:
+        name, _, rest = rest.partition(sep)
+        if not name or name == curdir:
+            # current dir
+            continue
+        if name == pardir:
+            # parent dir
+            if path:
+                path, name = split(path)
+                if name == pardir:
+                    path = join(path, pardir, pardir)
+            else:
+                path = pardir
+            continue
+        newpath = join(path, name)
+        if not islink(newpath):
+            path = newpath
+            continue
+        # Resolve the symbolic link
+        if newpath in seen:
+            # Already seen this path
+            path = seen[newpath]
+            if path is not None:
+                # use cached value
+                continue
+            # The symlink is not resolved, so we must have a symlink loop.
+            # Return already resolved part + rest of the path unchanged.
+            return join(newpath, rest), False
+        seen[newpath] = None # not resolved symlink
+        path, ok = _joinrealpath(path, os.readlink(newpath), seen)
+        if not ok:
+            return join(path, rest), False
+        seen[newpath] = path # resolved symlink
+
+    return path, True
+
+
+supports_unicode_filenames = (sys.platform == 'darwin')
+
+def relpath(path, start=None):
+    """Return a relative version of a path"""
+
+    if not path:
+        raise ValueError("no path specified")
+
+    path = os.fspath(path)
+    if isinstance(path, bytes):
+        curdir = b'.'
+        sep = b'/'
+        pardir = b'..'
+    else:
+        curdir = '.'
+        sep = '/'
+        pardir = '..'
+
+    if start is None:
+        start = curdir
+    else:
+        start = os.fspath(start)
+
+    try:
+        start_list = [x for x in abspath(start).split(sep) if x]
+        path_list = [x for x in abspath(path).split(sep) if x]
+        # Work out how much of the filepath is shared by start and path.
+        i = len(commonprefix([start_list, path_list]))
+
+        rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
+        if not rel_list:
+            return curdir
+        return join(*rel_list)
+    except (TypeError, AttributeError, BytesWarning, DeprecationWarning):
+        genericpath._check_arg_types('relpath', path, start)
+        raise
+
+
+# Return the longest common sub-path of the sequence of paths given as input.
+# The paths are not normalized before comparing them (this is the
+# responsibility of the caller). Any trailing separator is stripped from the
+# returned path.
+
+def commonpath(paths):
+    """Given a sequence of path names, returns the longest common sub-path."""
+
+    if not paths:
+        raise ValueError('commonpath() arg is an empty sequence')
+
+    paths = tuple(map(os.fspath, paths))
+    if isinstance(paths[0], bytes):
+        sep = b'/'
+        curdir = b'.'
+    else:
+        sep = '/'
+        curdir = '.'
+
+    try:
+        split_paths = [path.split(sep) for path in paths]
+
+        try:
+            isabs, = set(p[:1] == sep for p in paths)
+        except ValueError:
+            raise ValueError("Can't mix absolute and relative paths") from None
+
+        split_paths = [[c for c in s if c and c != curdir] for s in split_paths]
+        s1 = min(split_paths)
+        s2 = max(split_paths)
+        common = s1
+        for i, c in enumerate(s1):
+            if c != s2[i]:
+                common = s1[:i]
+                break
+
+        prefix = sep if isabs else sep[:0]
+        return prefix + sep.join(common)
+    except (TypeError, AttributeError):
+        genericpath._check_arg_types('commonpath', *paths)
+        raise

Lib/re.py

@@ -1,11 +0,0 @@
-
-""" Regular expressions """
-
-
-def match(pattern, string, flags=0):
-    return _compile(pattern, flags).match(string)
-
-
-def _compile(pattern, flags):
-    p = sre_compile.compile(pattern, flags)
-    return p

Lib/reprlib.py

@@ -0,0 +1,161 @@
+"""Redo the builtin repr() (representation) but with limits on most sizes."""
+
+__all__ = ["Repr", "repr", "recursive_repr"]
+
+import builtins
+from itertools import islice
+from _thread import get_ident
+
+def recursive_repr(fillvalue='...'):
+    'Decorator to make a repr function return fillvalue for a recursive call'
+
+    def decorating_function(user_function):
+        repr_running = set()
+
+        def wrapper(self):
+            key = id(self), get_ident()
+            if key in repr_running:
+                return fillvalue
+            repr_running.add(key)
+            try:
+                result = user_function(self)
+            finally:
+                repr_running.discard(key)
+            return result
+
+        # Can't use functools.wraps() here because of bootstrap issues
+        wrapper.__module__ = getattr(user_function, '__module__')
+        wrapper.__doc__ = getattr(user_function, '__doc__')
+        wrapper.__name__ = getattr(user_function, '__name__')
+        wrapper.__qualname__ = getattr(user_function, '__qualname__')
+        wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
+        return wrapper
+
+    return decorating_function
+
+class Repr:
+
+    def __init__(self):
+        self.maxlevel = 6
+        self.maxtuple = 6
+        self.maxlist = 6
+        self.maxarray = 5
+        self.maxdict = 4
+        self.maxset = 6
+        self.maxfrozenset = 6
+        self.maxdeque = 6
+        self.maxstring = 30
+        self.maxlong = 40
+        self.maxother = 30
+
+    def repr(self, x):
+        return self.repr1(x, self.maxlevel)
+
+    def repr1(self, x, level):
+        typename = type(x).__name__
+        if ' ' in typename:
+            parts = typename.split()
+            typename = '_'.join(parts)
+        if hasattr(self, 'repr_' + typename):
+            return getattr(self, 'repr_' + typename)(x, level)
+        else:
+            return self.repr_instance(x, level)
+
+    def _repr_iterable(self, x, level, left, right, maxiter, trail=''):
+        n = len(x)
+        if level <= 0 and n:
+            s = '...'
+        else:
+            newlevel = level - 1
+            repr1 = self.repr1
+            pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)]
+            if n > maxiter:  pieces.append('...')
+            s = ', '.join(pieces)
+            if n == 1 and trail:  right = trail + right
+        return '%s%s%s' % (left, s, right)
+
+    def repr_tuple(self, x, level):
+        return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',')
+
+    def repr_list(self, x, level):
+        return self._repr_iterable(x, level, '[', ']', self.maxlist)
+
+    def repr_array(self, x, level):
+        if not x:
+            return "array('%s')" % x.typecode
+        header = "array('%s', [" % x.typecode
+        return self._repr_iterable(x, level, header, '])', self.maxarray)
+
+    def repr_set(self, x, level):
+        if not x:
+            return 'set()'
+        x = _possibly_sorted(x)
+        return self._repr_iterable(x, level, '{', '}', self.maxset)
+
+    def repr_frozenset(self, x, level):
+        if not x:
+            return 'frozenset()'
+        x = _possibly_sorted(x)
+        return self._repr_iterable(x, level, 'frozenset({', '})',
+                                   self.maxfrozenset)
+
+    def repr_deque(self, x, level):
+        return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque)
+
+    def repr_dict(self, x, level):
+        n = len(x)
+        if n == 0: return '{}'
+        if level <= 0: return '{...}'
+        newlevel = level - 1
+        repr1 = self.repr1
+        pieces = []
+        for key in islice(_possibly_sorted(x), self.maxdict):
+            keyrepr = repr1(key, newlevel)
+            valrepr = repr1(x[key], newlevel)
+            pieces.append('%s: %s' % (keyrepr, valrepr))
+        if n > self.maxdict: pieces.append('...')
+        s = ', '.join(pieces)
+        return '{%s}' % (s,)
+
+    def repr_str(self, x, level):
+        s = builtins.repr(x[:self.maxstring])
+        if len(s) > self.maxstring:
+            i = max(0, (self.maxstring-3)//2)
+            j = max(0, self.maxstring-3-i)
+            s = builtins.repr(x[:i] + x[len(x)-j:])
+            s = s[:i] + '...' + s[len(s)-j:]
+        return s
+
+    def repr_int(self, x, level):
+        s = builtins.repr(x) # XXX Hope this isn't too slow...
+        if len(s) > self.maxlong:
+            i = max(0, (self.maxlong-3)//2)
+            j = max(0, self.maxlong-3-i)
+            s = s[:i] + '...' + s[len(s)-j:]
+        return s
+
+    def repr_instance(self, x, level):
+        try:
+            s = builtins.repr(x)
+            # Bugs in x.__repr__() can cause arbitrary
+            # exceptions -- then make up something
+        except Exception:
+            return '<%s instance at %#x>' % (x.__class__.__name__, id(x))
+        if len(s) > self.maxother:
+            i = max(0, (self.maxother-3)//2)
+            j = max(0, self.maxother-3-i)
+            s = s[:i] + '...' + s[len(s)-j:]
+        return s
+
+
+def _possibly_sorted(x):
+    # Since not all sequences of items can be sorted and comparison
+    # functions may raise arbitrary exceptions, return an unsorted
+    # sequence in that case.
+    try:
+        return sorted(x)
+    except Exception:
+        return list(x)
+
+aRepr = Repr()
+repr = aRepr.repr

Lib/stat.py

@@ -0,0 +1,179 @@
+"""Constants/functions for interpreting results of os.stat() and os.lstat().
+
+Suggested usage: from stat import *
+"""
+
+# Indices for stat struct members in the tuple returned by os.stat()
+
+ST_MODE  = 0
+ST_INO   = 1
+ST_DEV   = 2
+ST_NLINK = 3
+ST_UID   = 4
+ST_GID   = 5
+ST_SIZE  = 6
+ST_ATIME = 7
+ST_MTIME = 8
+ST_CTIME = 9
+
+# Extract bits from the mode
+
+def S_IMODE(mode):
+    """Return the portion of the file's mode that can be set by
+    os.chmod().
+    """
+    return mode & 0o7777
+
+def S_IFMT(mode):
+    """Return the portion of the file's mode that describes the
+    file type.
+    """
+    return mode & 0o170000
+
+# Constants used as S_IFMT() for various file types
+# (not all are implemented on all systems)
+
+S_IFDIR  = 0o040000  # directory
+S_IFCHR  = 0o020000  # character device
+S_IFBLK  = 0o060000  # block device
+S_IFREG  = 0o100000  # regular file
+S_IFIFO  = 0o010000  # fifo (named pipe)
+S_IFLNK  = 0o120000  # symbolic link
+S_IFSOCK = 0o140000  # socket file
+
+# Functions to test for each file type
+
+def S_ISDIR(mode):
+    """Return True if mode is from a directory."""
+    return S_IFMT(mode) == S_IFDIR
+
+def S_ISCHR(mode):
+    """Return True if mode is from a character special device file."""
+    return S_IFMT(mode) == S_IFCHR
+
+def S_ISBLK(mode):
+    """Return True if mode is from a block special device file."""
+    return S_IFMT(mode) == S_IFBLK
+
+def S_ISREG(mode):
+    """Return True if mode is from a regular file."""
+    return S_IFMT(mode) == S_IFREG
+
+def S_ISFIFO(mode):
+    """Return True if mode is from a FIFO (named pipe)."""
+    return S_IFMT(mode) == S_IFIFO
+
+def S_ISLNK(mode):
+    """Return True if mode is from a symbolic link."""
+    return S_IFMT(mode) == S_IFLNK
+
+def S_ISSOCK(mode):
+    """Return True if mode is from a socket."""
+    return S_IFMT(mode) == S_IFSOCK
+
+# Names for permission bits
+
+S_ISUID = 0o4000  # set UID bit
+S_ISGID = 0o2000  # set GID bit
+S_ENFMT = S_ISGID # file locking enforcement
+S_ISVTX = 0o1000  # sticky bit
+S_IREAD = 0o0400  # Unix V7 synonym for S_IRUSR
+S_IWRITE = 0o0200 # Unix V7 synonym for S_IWUSR
+S_IEXEC = 0o0100  # Unix V7 synonym for S_IXUSR
+S_IRWXU = 0o0700  # mask for owner permissions
+S_IRUSR = 0o0400  # read by owner
+S_IWUSR = 0o0200  # write by owner
+S_IXUSR = 0o0100  # execute by owner
+S_IRWXG = 0o0070  # mask for group permissions
+S_IRGRP = 0o0040  # read by group
+S_IWGRP = 0o0020  # write by group
+S_IXGRP = 0o0010  # execute by group
+S_IRWXO = 0o0007  # mask for others (not in group) permissions
+S_IROTH = 0o0004  # read by others
+S_IWOTH = 0o0002  # write by others
+S_IXOTH = 0o0001  # execute by others
+
+# Names for file flags
+
+UF_NODUMP    = 0x00000001  # do not dump file
+UF_IMMUTABLE = 0x00000002  # file may not be changed
+UF_APPEND    = 0x00000004  # file may only be appended to
+UF_OPAQUE    = 0x00000008  # directory is opaque when viewed through a union stack
+UF_NOUNLINK  = 0x00000010  # file may not be renamed or deleted
+UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed
+UF_HIDDEN    = 0x00008000  # OS X: file should not be displayed
+SF_ARCHIVED  = 0x00010000  # file may be archived
+SF_IMMUTABLE = 0x00020000  # file may not be changed
+SF_APPEND    = 0x00040000  # file may only be appended to
+SF_NOUNLINK  = 0x00100000  # file may not be renamed or deleted
+SF_SNAPSHOT  = 0x00200000  # file is a snapshot file
+
+
+_filemode_table = (
+    ((S_IFLNK,         "l"),
+     (S_IFSOCK,        "s"),  # Must appear before IFREG and IFDIR as IFSOCK == IFREG | IFDIR
+     (S_IFREG,         "-"),
+     (S_IFBLK,         "b"),
+     (S_IFDIR,         "d"),
+     (S_IFCHR,         "c"),
+     (S_IFIFO,         "p")),
+
+    ((S_IRUSR,         "r"),),
+    ((S_IWUSR,         "w"),),
+    ((S_IXUSR|S_ISUID, "s"),
+     (S_ISUID,         "S"),
+     (S_IXUSR,         "x")),
+
+    ((S_IRGRP,         "r"),),
+    ((S_IWGRP,         "w"),),
+    ((S_IXGRP|S_ISGID, "s"),
+     (S_ISGID,         "S"),
+     (S_IXGRP,         "x")),
+
+    ((S_IROTH,         "r"),),
+    ((S_IWOTH,         "w"),),
+    ((S_IXOTH|S_ISVTX, "t"),
+     (S_ISVTX,         "T"),
+     (S_IXOTH,         "x"))
+)
+
+def filemode(mode):
+    """Convert a file's mode to a string of the form '-rwxrwxrwx'."""
+    perm = []
+    for table in _filemode_table:
+        for bit, char in table:
+            if mode & bit == bit:
+                perm.append(char)
+                break
+        else:
+            perm.append("-")
+    return "".join(perm)
+
+
+# Windows FILE_ATTRIBUTE constants for interpreting os.stat()'s
+# "st_file_attributes" member
+
+FILE_ATTRIBUTE_ARCHIVE = 32
+FILE_ATTRIBUTE_COMPRESSED = 2048
+FILE_ATTRIBUTE_DEVICE = 64
+FILE_ATTRIBUTE_DIRECTORY = 16
+FILE_ATTRIBUTE_ENCRYPTED = 16384
+FILE_ATTRIBUTE_HIDDEN = 2
+FILE_ATTRIBUTE_INTEGRITY_STREAM = 32768
+FILE_ATTRIBUTE_NORMAL = 128
+FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 8192
+FILE_ATTRIBUTE_NO_SCRUB_DATA = 131072
+FILE_ATTRIBUTE_OFFLINE = 4096
+FILE_ATTRIBUTE_READONLY = 1
+FILE_ATTRIBUTE_REPARSE_POINT = 1024
+FILE_ATTRIBUTE_SPARSE_FILE = 512
+FILE_ATTRIBUTE_SYSTEM = 4
+FILE_ATTRIBUTE_TEMPORARY = 256
+FILE_ATTRIBUTE_VIRTUAL = 65536
+
+
+# If available, use C implementation
+try:
+    from _stat import *
+except ImportError:
+    pass

Lib/this.py

@@ -0,0 +1,28 @@
+s = """Gur Mra bs Clguba, ol Gvz Crgref
+
+Ornhgvshy vf orggre guna htyl.
+Rkcyvpvg vf orggre guna vzcyvpvg.
+Fvzcyr vf orggre guna pbzcyrk.
+Pbzcyrk vf orggre guna pbzcyvpngrq.
+Syng vf orggre guna arfgrq.
+Fcnefr vf orggre guna qrafr.
+Ernqnovyvgl pbhagf.
+Fcrpvny pnfrf nera'g fcrpvny rabhtu gb oernx gur ehyrf.
+Nygubhtu cenpgvpnyvgl orngf chevgl.
+Reebef fubhyq arire cnff fvyragyl.
+Hayrff rkcyvpvgyl fvyraprq.
+Va gur snpr bs nzovthvgl, ershfr gur grzcgngvba gb thrff.
+Gurer fubhyq or bar-- naq cersrenoyl bayl bar --boivbhf jnl gb qb vg.
+Nygubhtu gung jnl znl abg or boivbhf ng svefg hayrff lbh'er Qhgpu.
+Abj vf orggre guna arire.
+Nygubhtu arire vf bsgra orggre guna *evtug* abj.
+Vs gur vzcyrzragngvba vf uneq gb rkcynva, vg'f n onq vqrn.
+Vs gur vzcyrzragngvba vf rnfl gb rkcynva, vg znl or n tbbq vqrn.
+Anzrfcnprf ner bar ubaxvat terng vqrn -- yrg'f qb zber bs gubfr!"""
+
+d = {}
+for c in (65, 97):
+    for i in range(26):
+        d[chr(i+c)] = chr((i+13) % 26 + c)
+
+print("".join([d.get(c, c) for c in s]))

Lib/types.py

@@ -0,0 +1,295 @@
+"""
+Define names for built-in types that aren't directly accessible as a builtin.
+"""
+import sys
+
+# Iterators in Python aren't a matter of type but of protocol.  A large
+# and changing number of builtin types implement *some* flavor of
+# iterator.  Don't check the type!  Use hasattr to check for both
+# "__iter__" and "__next__" attributes instead.
+
+def _f(): pass
+FunctionType = type(_f)
+LambdaType = type(lambda: None)         # Same as FunctionType
+CodeType = type(_f.__code__)
+MappingProxyType = type(type.__dict__)
+SimpleNamespace = type(sys.implementation)
+
+def _g():
+    yield 1
+GeneratorType = type(_g())
+
+# async def _c(): pass
+# _c = _c()
+# CoroutineType = type(_c)
+# _c.close()  # Prevent ResourceWarning
+
+# async def _ag():
+#     yield
+# _ag = _ag()
+# AsyncGeneratorType = type(_ag)
+
+class _C:
+    def _m(self): pass
+MethodType = type(_C()._m)
+
+BuiltinFunctionType = type(len)
+BuiltinMethodType = type([].append)     # Same as BuiltinFunctionType
+
+WrapperDescriptorType = type(object.__init__)
+MethodWrapperType = type(object().__str__)
+MethodDescriptorType = type(str.join)
+ClassMethodDescriptorType = type(dict.__dict__['fromkeys'])
+
+ModuleType = type(sys)
+
+# try:
+#     raise TypeError
+# except TypeError:
+#     tb = sys.exc_info()[2]
+#     TracebackType = type(tb)
+#     FrameType = type(tb.tb_frame)
+#     tb = None; del tb
+
+# For Jython, the following two types are identical
+GetSetDescriptorType = type(FunctionType.__code__)
+# MemberDescriptorType = type(FunctionType.__globals__)
+
+del sys, _f, _g, _C  # Not for export
+
+
+# Provide a PEP 3115 compliant mechanism for class creation
+def new_class(name, bases=(), kwds=None, exec_body=None):
+    """Create a class object dynamically using the appropriate metaclass."""
+    resolved_bases = resolve_bases(bases)
+    meta, ns, kwds = prepare_class(name, resolved_bases, kwds)
+    if exec_body is not None:
+        exec_body(ns)
+    if resolved_bases is not bases:
+        ns['__orig_bases__'] = bases
+    return meta(name, resolved_bases, ns, **kwds)
+
+def resolve_bases(bases):
+    """Resolve MRO entries dynamically as specified by PEP 560."""
+    new_bases = list(bases)
+    updated = False
+    shift = 0
+    for i, base in enumerate(bases):
+        if isinstance(base, type):
+            continue
+        if not hasattr(base, "__mro_entries__"):
+            continue
+        new_base = base.__mro_entries__(bases)
+        updated = True
+        if not isinstance(new_base, tuple):
+            raise TypeError("__mro_entries__ must return a tuple")
+        else:
+            new_bases[i+shift:i+shift+1] = new_base
+            shift += len(new_base) - 1
+    if not updated:
+        return bases
+    return tuple(new_bases)
+
+def prepare_class(name, bases=(), kwds=None):
+    """Call the __prepare__ method of the appropriate metaclass.
+
+    Returns (metaclass, namespace, kwds) as a 3-tuple
+
+    *metaclass* is the appropriate metaclass
+    *namespace* is the prepared class namespace
+    *kwds* is an updated copy of the passed in kwds argument with any
+    'metaclass' entry removed. If no kwds argument is passed in, this will
+    be an empty dict.
+    """
+    if kwds is None:
+        kwds = {}
+    else:
+        kwds = dict(kwds) # Don't alter the provided mapping
+    if 'metaclass' in kwds:
+        meta = kwds.pop('metaclass')
+    else:
+        if bases:
+            meta = type(bases[0])
+        else:
+            meta = type
+    if isinstance(meta, type):
+        # when meta is a type, we first determine the most-derived metaclass
+        # instead of invoking the initial candidate directly
+        meta = _calculate_meta(meta, bases)
+    if hasattr(meta, '__prepare__'):
+        ns = meta.__prepare__(name, bases, **kwds)
+    else:
+        ns = {}
+    return meta, ns, kwds
+
+def _calculate_meta(meta, bases):
+    """Calculate the most derived metaclass."""
+    winner = meta
+    for base in bases:
+        base_meta = type(base)
+        if issubclass(winner, base_meta):
+            continue
+        if issubclass(base_meta, winner):
+            winner = base_meta
+            continue
+        # else:
+        raise TypeError("metaclass conflict: "
+                        "the metaclass of a derived class "
+                        "must be a (non-strict) subclass "
+                        "of the metaclasses of all its bases")
+    return winner
+
+class DynamicClassAttribute:
+    """Route attribute access on a class to __getattr__.
+
+    This is a descriptor, used to define attributes that act differently when
+    accessed through an instance and through a class.  Instance access remains
+    normal, but access to an attribute through a class will be routed to the
+    class's __getattr__ method; this is done by raising AttributeError.
+
+    This allows one to have properties active on an instance, and have virtual
+    attributes on the class with the same name (see Enum for an example).
+
+    """
+    def __init__(self, fget=None, fset=None, fdel=None, doc=None):
+        self.fget = fget
+        self.fset = fset
+        self.fdel = fdel
+        # next two lines make DynamicClassAttribute act the same as property
+        self.__doc__ = doc or fget.__doc__
+        self.overwrite_doc = doc is None
+        # support for abstract methods
+        self.__isabstractmethod__ = bool(getattr(fget, '__isabstractmethod__', False))
+
+    def __get__(self, instance, ownerclass=None):
+        if instance is None:
+            if self.__isabstractmethod__:
+                return self
+            raise AttributeError()
+        elif self.fget is None:
+            raise AttributeError("unreadable attribute")
+        return self.fget(instance)
+
+    def __set__(self, instance, value):
+        if self.fset is None:
+            raise AttributeError("can't set attribute")
+        self.fset(instance, value)
+
+    def __delete__(self, instance):
+        if self.fdel is None:
+            raise AttributeError("can't delete attribute")
+        self.fdel(instance)
+
+    def getter(self, fget):
+        fdoc = fget.__doc__ if self.overwrite_doc else None
+        result = type(self)(fget, self.fset, self.fdel, fdoc or self.__doc__)
+        result.overwrite_doc = self.overwrite_doc
+        return result
+
+    def setter(self, fset):
+        result = type(self)(self.fget, fset, self.fdel, self.__doc__)
+        result.overwrite_doc = self.overwrite_doc
+        return result
+
+    def deleter(self, fdel):
+        result = type(self)(self.fget, self.fset, fdel, self.__doc__)
+        result.overwrite_doc = self.overwrite_doc
+        return result
+
+
+class _GeneratorWrapper:
+    # TODO: Implement this in C.
+    def __init__(self, gen):
+        self.__wrapped = gen
+        self.__isgen = gen.__class__ is GeneratorType
+        self.__name__ = getattr(gen, '__name__', None)
+        self.__qualname__ = getattr(gen, '__qualname__', None)
+    def send(self, val):
+        return self.__wrapped.send(val)
+    def throw(self, tp, *rest):
+        return self.__wrapped.throw(tp, *rest)
+    def close(self):
+        return self.__wrapped.close()
+    @property
+    def gi_code(self):
+        return self.__wrapped.gi_code
+    @property
+    def gi_frame(self):
+        return self.__wrapped.gi_frame
+    @property
+    def gi_running(self):
+        return self.__wrapped.gi_running
+    @property
+    def gi_yieldfrom(self):
+        return self.__wrapped.gi_yieldfrom
+    cr_code = gi_code
+    cr_frame = gi_frame
+    cr_running = gi_running
+    cr_await = gi_yieldfrom
+    def __next__(self):
+        return next(self.__wrapped)
+    def __iter__(self):
+        if self.__isgen:
+            return self.__wrapped
+        return self
+    __await__ = __iter__
+
+def coroutine(func):
+    """Convert regular generator function to a coroutine."""
+
+    if not callable(func):
+        raise TypeError('types.coroutine() expects a callable')
+
+    if (func.__class__ is FunctionType and
+        getattr(func, '__code__', None).__class__ is CodeType):
+
+        co_flags = func.__code__.co_flags
+
+        # Check if 'func' is a coroutine function.
+        # (0x180 == CO_COROUTINE | CO_ITERABLE_COROUTINE)
+        if co_flags & 0x180:
+            return func
+
+        # Check if 'func' is a generator function.
+        # (0x20 == CO_GENERATOR)
+        if co_flags & 0x20:
+            # TODO: Implement this in C.
+            co = func.__code__
+            func.__code__ = CodeType(
+                co.co_argcount, co.co_kwonlyargcount, co.co_nlocals,
+                co.co_stacksize,
+                co.co_flags | 0x100,  # 0x100 == CO_ITERABLE_COROUTINE
+                co.co_code,
+                co.co_consts, co.co_names, co.co_varnames, co.co_filename,
+                co.co_name, co.co_firstlineno, co.co_lnotab, co.co_freevars,
+                co.co_cellvars)
+            return func
+
+    # The following code is primarily to support functions that
+    # return generator-like objects (for instance generators
+    # compiled with Cython).
+
+    # Delay functools and _collections_abc import for speeding up types import.
+    import functools
+    import _collections_abc
+    @functools.wraps(func)
+    def wrapped(*args, **kwargs):
+        coro = func(*args, **kwargs)
+        if (coro.__class__ is CoroutineType or
+            coro.__class__ is GeneratorType and coro.gi_code.co_flags & 0x100):
+            # 'coro' is a native coroutine object or an iterable coroutine
+            return coro
+        if (isinstance(coro, _collections_abc.Generator) and
+            not isinstance(coro, _collections_abc.Coroutine)):
+            # 'coro' is either a pure Python generator iterator, or it
+            # implements collections.abc.Generator (and does not implement
+            # collections.abc.Coroutine).
+            return _GeneratorWrapper(coro)
+        # 'coro' is either an instance of collections.abc.Coroutine or
+        # some other object -- pass it through.
+        return coro
+
+    return wrapped
+
+
+__all__ = [n for n in globals() if n[:1] != '_']

Lib/warnings.py

@@ -0,0 +1,554 @@
+"""Python part of the warnings subsystem."""
+
+import sys
+
+
+__all__ = ["warn", "warn_explicit", "showwarning",
+           "formatwarning", "filterwarnings", "simplefilter",
+           "resetwarnings", "catch_warnings"]
+
+def showwarning(message, category, filename, lineno, file=None, line=None):
+    """Hook to write a warning to a file; replace if you like."""
+    msg = WarningMessage(message, category, filename, lineno, file, line)
+    _showwarnmsg_impl(msg)
+
+def formatwarning(message, category, filename, lineno, line=None):
+    """Function to format a warning the standard way."""
+    msg = WarningMessage(message, category, filename, lineno, None, line)
+    return _formatwarnmsg_impl(msg)
+
+def _showwarnmsg_impl(msg):
+    file = msg.file
+    if file is None:
+        file = sys.stderr
+        if file is None:
+            # sys.stderr is None when run with pythonw.exe:
+            # warnings get lost
+            return
+    text = _formatwarnmsg(msg)
+    try:
+        file.write(text)
+    except OSError:
+        # the file (probably stderr) is invalid - this warning gets lost.
+        pass
+
+def _formatwarnmsg_impl(msg):
+    s =  ("%s:%s: %s: %s\n"
+          % (msg.filename, msg.lineno, msg.category.__name__,
+             msg.message))
+
+    if msg.line is None:
+        try:
+            import linecache
+            line = linecache.getline(msg.filename, msg.lineno)
+        except Exception:
+            # When a warning is logged during Python shutdown, linecache
+            # and the import machinery don't work anymore
+            line = None
+            linecache = None
+    else:
+        line = msg.line
+    if line:
+        line = line.strip()
+        s += "  %s\n" % line
+
+    if msg.source is not None:
+        try:
+            import tracemalloc
+            tb = tracemalloc.get_object_traceback(msg.source)
+        except Exception:
+            # When a warning is logged during Python shutdown, tracemalloc
+            # and the import machinery don't work anymore
+            tb = None
+
+        if tb is not None:
+            s += 'Object allocated at (most recent call last):\n'
+            for frame in tb:
+                s += ('  File "%s", lineno %s\n'
+                      % (frame.filename, frame.lineno))
+
+                try:
+                    if linecache is not None:
+                        line = linecache.getline(frame.filename, frame.lineno)
+                    else:
+                        line = None
+                except Exception:
+                    line = None
+                if line:
+                    line = line.strip()
+                    s += '    %s\n' % line
+    return s
+
+# Keep a reference to check if the function was replaced
+_showwarning_orig = showwarning
+
+def _showwarnmsg(msg):
+    """Hook to write a warning to a file; replace if you like."""
+    try:
+        sw = showwarning
+    except NameError:
+        pass
+    else:
+        if sw is not _showwarning_orig:
+            # warnings.showwarning() was replaced
+            if not callable(sw):
+                raise TypeError("warnings.showwarning() must be set to a "
+                                "function or method")
+
+            sw(msg.message, msg.category, msg.filename, msg.lineno,
+               msg.file, msg.line)
+            return
+    _showwarnmsg_impl(msg)
+
+# Keep a reference to check if the function was replaced
+_formatwarning_orig = formatwarning
+
+def _formatwarnmsg(msg):
+    """Function to format a warning the standard way."""
+    try:
+        fw = formatwarning
+    except NameError:
+        pass
+    else:
+        if fw is not _formatwarning_orig:
+            # warnings.formatwarning() was replaced
+            return fw(msg.message, msg.category,
+                      msg.filename, msg.lineno, line=msg.line)
+    return _formatwarnmsg_impl(msg)
+
+def filterwarnings(action, message="", category=Warning, module="", lineno=0,
+                   append=False):
+    """Insert an entry into the list of warnings filters (at the front).
+
+    'action' -- one of "error", "ignore", "always", "default", "module",
+                or "once"
+    'message' -- a regex that the warning message must match
+    'category' -- a class that the warning must be a subclass of
+    'module' -- a regex that the module name must match
+    'lineno' -- an integer line number, 0 matches all warnings
+    'append' -- if true, append to the list of filters
+    """
+    assert action in ("error", "ignore", "always", "default", "module",
+                      "once"), "invalid action: %r" % (action,)
+    assert isinstance(message, str), "message must be a string"
+    assert isinstance(category, type), "category must be a class"
+    assert issubclass(category, Warning), "category must be a Warning subclass"
+    assert isinstance(module, str), "module must be a string"
+    assert isinstance(lineno, int) and lineno >= 0, \
+           "lineno must be an int >= 0"
+
+    if message or module:
+        import re
+
+    if message:
+        message = re.compile(message, re.I)
+    else:
+        message = None
+    if module:
+        module = re.compile(module)
+    else:
+        module = None
+
+    _add_filter(action, message, category, module, lineno, append=append)
+
+def simplefilter(action, category=Warning, lineno=0, append=False):
+    """Insert a simple entry into the list of warnings filters (at the front).
+
+    A simple filter matches all modules and messages.
+    'action' -- one of "error", "ignore", "always", "default", "module",
+                or "once"
+    'category' -- a class that the warning must be a subclass of
+    'lineno' -- an integer line number, 0 matches all warnings
+    'append' -- if true, append to the list of filters
+    """
+    assert action in ("error", "ignore", "always", "default", "module",
+                      "once"), "invalid action: %r" % (action,)
+    assert isinstance(lineno, int) and lineno >= 0, \
+           "lineno must be an int >= 0"
+    _add_filter(action, None, category, None, lineno, append=append)
+
+def _add_filter(*item, append):
+    # Remove possible duplicate filters, so new one will be placed
+    # in correct place. If append=True and duplicate exists, do nothing.
+    if not append:
+        try:
+            filters.remove(item)
+        except ValueError:
+            pass
+        filters.insert(0, item)
+    else:
+        if item not in filters:
+            filters.append(item)
+    _filters_mutated()
+
+def resetwarnings():
+    """Clear the list of warning filters, so that no filters are active."""
+    filters[:] = []
+    _filters_mutated()
+
+class _OptionError(Exception):
+    """Exception used by option processing helpers."""
+    pass
+
+# Helper to process -W options passed via sys.warnoptions
+def _processoptions(args):
+    for arg in args:
+        try:
+            _setoption(arg)
+        except _OptionError as msg:
+            print("Invalid -W option ignored:", msg, file=sys.stderr)
+
+# Helper for _processoptions()
+def _setoption(arg):
+    import re
+    parts = arg.split(':')
+    if len(parts) > 5:
+        raise _OptionError("too many fields (max 5): %r" % (arg,))
+    while len(parts) < 5:
+        parts.append('')
+    action, message, category, module, lineno = [s.strip()
+                                                 for s in parts]
+    action = _getaction(action)
+    message = re.escape(message)
+    category = _getcategory(category)
+    module = re.escape(module)
+    if module:
+        module = module + '$'
+    if lineno:
+        try:
+            lineno = int(lineno)
+            if lineno < 0:
+                raise ValueError
+        except (ValueError, OverflowError):
+            raise _OptionError("invalid lineno %r" % (lineno,)) from None
+    else:
+        lineno = 0
+    filterwarnings(action, message, category, module, lineno)
+
+# Helper for _setoption()
+def _getaction(action):
+    if not action:
+        return "default"
+    if action == "all": return "always" # Alias
+    for a in ('default', 'always', 'ignore', 'module', 'once', 'error'):
+        if a.startswith(action):
+            return a
+    raise _OptionError("invalid action: %r" % (action,))
+
+# Helper for _setoption()
+def _getcategory(category):
+    import re
+    if not category:
+        return Warning
+    if re.match("^[a-zA-Z0-9_]+$", category):
+        try:
+            cat = eval(category)
+        except NameError:
+            raise _OptionError("unknown warning category: %r" % (category,)) from None
+    else:
+        i = category.rfind(".")
+        module = category[:i]
+        klass = category[i+1:]
+        try:
+            m = __import__(module, None, None, [klass])
+        except ImportError:
+            raise _OptionError("invalid module name: %r" % (module,)) from None
+        try:
+            cat = getattr(m, klass)
+        except AttributeError:
+            raise _OptionError("unknown warning category: %r" % (category,)) from None
+    if not issubclass(cat, Warning):
+        raise _OptionError("invalid warning category: %r" % (category,))
+    return cat
+
+
+def _is_internal_frame(frame):
+    """Signal whether the frame is an internal CPython implementation detail."""
+    filename = frame.f_code.co_filename
+    return 'importlib' in filename and '_bootstrap' in filename
+
+
+def _next_external_frame(frame):
+    """Find the next frame that doesn't involve CPython internals."""
+    frame = frame.f_back
+    while frame is not None and _is_internal_frame(frame):
+        frame = frame.f_back
+    return frame
+
+
+# Code typically replaced by _warnings
+def warn(message, category=None, stacklevel=1, source=None):
+    """Issue a warning, or maybe ignore it or raise an exception."""
+    # Check if message is already a Warning object
+    if isinstance(message, Warning):
+        category = message.__class__
+    # Check category argument
+    if category is None:
+        category = UserWarning
+    if not (isinstance(category, type) and issubclass(category, Warning)):
+        raise TypeError("category must be a Warning subclass, "
+                        "not '{:s}'".format(type(category).__name__))
+    # Get context information
+    try:
+        if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)):
+            # If frame is too small to care or if the warning originated in
+            # internal code, then do not try to hide any frames.
+            frame = sys._getframe(stacklevel)
+        else:
+            frame = sys._getframe(1)
+            # Look for one frame less since the above line starts us off.
+            for x in range(stacklevel-1):
+                frame = _next_external_frame(frame)
+                if frame is None:
+                    raise ValueError
+    except ValueError:
+        globals = sys.__dict__
+        lineno = 1
+    else:
+        globals = frame.f_globals
+        lineno = frame.f_lineno
+    if '__name__' in globals:
+        module = globals['__name__']
+    else:
+        module = "<string>"
+    filename = globals.get('__file__')
+    if filename:
+        fnl = filename.lower()
+        if fnl.endswith(".pyc"):
+            filename = filename[:-1]
+    else:
+        if module == "__main__":
+            try:
+                filename = sys.argv[0]
+            except AttributeError:
+                # embedded interpreters don't have sys.argv, see bug #839151
+                filename = '__main__'
+        if not filename:
+            filename = module
+    registry = globals.setdefault("__warningregistry__", {})
+    warn_explicit(message, category, filename, lineno, module, registry,
+                  globals, source)
+
+def warn_explicit(message, category, filename, lineno,
+                  module=None, registry=None, module_globals=None,
+                  source=None):
+    lineno = int(lineno)
+    if module is None:
+        module = filename or "<unknown>"
+        if module[-3:].lower() == ".py":
+            module = module[:-3] # XXX What about leading pathname?
+    if registry is None:
+        registry = {}
+    if registry.get('version', 0) != _filters_version:
+        registry.clear()
+        registry['version'] = _filters_version
+    if isinstance(message, Warning):
+        text = str(message)
+        category = message.__class__
+    else:
+        text = message
+        message = category(message)
+    key = (text, category, lineno)
+    # Quick test for common case
+    if registry.get(key):
+        return
+    # Search the filters
+    for item in filters:
+        action, msg, cat, mod, ln = item
+        if ((msg is None or msg.match(text)) and
+            issubclass(category, cat) and
+            (mod is None or mod.match(module)) and
+            (ln == 0 or lineno == ln)):
+            break
+    else:
+        action = defaultaction
+    # Early exit actions
+    if action == "ignore":
+        return
+
+    # Prime the linecache for formatting, in case the
+    # "file" is actually in a zipfile or something.
+    import linecache
+    linecache.getlines(filename, module_globals)
+
+    if action == "error":
+        raise message
+    # Other actions
+    if action == "once":
+        registry[key] = 1
+        oncekey = (text, category)
+        if onceregistry.get(oncekey):
+            return
+        onceregistry[oncekey] = 1
+    elif action == "always":
+        pass
+    elif action == "module":
+        registry[key] = 1
+        altkey = (text, category, 0)
+        if registry.get(altkey):
+            return
+        registry[altkey] = 1
+    elif action == "default":
+        registry[key] = 1
+    else:
+        # Unrecognized actions are errors
+        raise RuntimeError(
+              "Unrecognized action (%r) in warnings.filters:\n %s" %
+              (action, item))
+    # Print message and context
+    msg = WarningMessage(message, category, filename, lineno, source)
+    _showwarnmsg(msg)
+
+
+class WarningMessage(object):
+
+    _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
+                        "line", "source")
+
+    def __init__(self, message, category, filename, lineno, file=None,
+                 line=None, source=None):
+        self.message = message
+        self.category = category
+        self.filename = filename
+        self.lineno = lineno
+        self.file = file
+        self.line = line
+        self.source = source
+        self._category_name = category.__name__ if category else None
+
+    def __str__(self):
+        return ("{message : %r, category : %r, filename : %r, lineno : %s, "
+                    "line : %r}" % (self.message, self._category_name,
+                                    self.filename, self.lineno, self.line))
+
+
+class catch_warnings(object):
+
+    """A context manager that copies and restores the warnings filter upon
+    exiting the context.
+
+    The 'record' argument specifies whether warnings should be captured by a
+    custom implementation of warnings.showwarning() and be appended to a list
+    returned by the context manager. Otherwise None is returned by the context
+    manager. The objects appended to the list are arguments whose attributes
+    mirror the arguments to showwarning().
+
+    The 'module' argument is to specify an alternative module to the module
+    named 'warnings' and imported under that name. This argument is only useful
+    when testing the warnings module itself.
+
+    """
+
+    def __init__(self, *, record=False, module=None):
+        """Specify whether to record warnings and if an alternative module
+        should be used other than sys.modules['warnings'].
+
+        For compatibility with Python 3.0, please consider all arguments to be
+        keyword-only.
+
+        """
+        self._record = record
+        self._module = sys.modules['warnings'] if module is None else module
+        self._entered = False
+
+    def __repr__(self):
+        args = []
+        if self._record:
+            args.append("record=True")
+        if self._module is not sys.modules['warnings']:
+            args.append("module=%r" % self._module)
+        name = type(self).__name__
+        return "%s(%s)" % (name, ", ".join(args))
+
+    def __enter__(self):
+        if self._entered:
+            raise RuntimeError("Cannot enter %r twice" % self)
+        self._entered = True
+        self._filters = self._module.filters
+        self._module.filters = self._filters[:]
+        self._module._filters_mutated()
+        self._showwarning = self._module.showwarning
+        self._showwarnmsg_impl = self._module._showwarnmsg_impl
+        if self._record:
+            log = []
+            self._module._showwarnmsg_impl = log.append
+            # Reset showwarning() to the default implementation to make sure
+            # that _showwarnmsg() calls _showwarnmsg_impl()
+            self._module.showwarning = self._module._showwarning_orig
+            return log
+        else:
+            return None
+
+    def __exit__(self, *exc_info):
+        if not self._entered:
+            raise RuntimeError("Cannot exit %r without entering first" % self)
+        self._module.filters = self._filters
+        self._module._filters_mutated()
+        self._module.showwarning = self._showwarning
+        self._module._showwarnmsg_impl = self._showwarnmsg_impl
+
+
+# Private utility function called by _PyErr_WarnUnawaitedCoroutine
+def _warn_unawaited_coroutine(coro):
+    msg_lines = [
+        f"coroutine '{coro.__qualname__}' was never awaited\n"
+    ]
+    if coro.cr_origin is not None:
+        import linecache, traceback
+        def extract():
+            for filename, lineno, funcname in reversed(coro.cr_origin):
+                line = linecache.getline(filename, lineno)
+                yield (filename, lineno, funcname, line)
+        msg_lines.append("Coroutine created at (most recent call last)\n")
+        msg_lines += traceback.format_list(list(extract()))
+    msg = "".join(msg_lines).rstrip("\n")
+    # Passing source= here means that if the user happens to have tracemalloc
+    # enabled and tracking where the coroutine was created, the warning will
+    # contain that traceback. This does mean that if they have *both*
+    # coroutine origin tracking *and* tracemalloc enabled, they'll get two
+    # partially-redundant tracebacks. If we wanted to be clever we could
+    # probably detect this case and avoid it, but for now we don't bother.
+    warn(msg, category=RuntimeWarning, stacklevel=2, source=coro)
+
+
+# filters contains a sequence of filter 5-tuples
+# The components of the 5-tuple are:
+# - an action: error, ignore, always, default, module, or once
+# - a compiled regex that must match the warning message
+# - a class representing the warning category
+# - a compiled regex that must match the module that is being warned
+# - a line number for the line being warning, or 0 to mean any line
+# If either if the compiled regexs are None, match anything.
+try:
+    from _warnings import (filters, _defaultaction, _onceregistry,
+                           warn, warn_explicit, _filters_mutated)
+    defaultaction = _defaultaction
+    onceregistry = _onceregistry
+    _warnings_defaults = True
+except ImportError:
+    filters = []
+    defaultaction = "default"
+    onceregistry = {}
+
+    _filters_version = 1
+
+    def _filters_mutated():
+        global _filters_version
+        _filters_version += 1
+
+    _warnings_defaults = False
+
+
+# Module initialization
+_processoptions(sys.warnoptions)
+if not _warnings_defaults:
+    # Several warning categories are ignored by default in regular builds
+    if not hasattr(sys, 'gettotalrefcount'):
+        filterwarnings("default", category=DeprecationWarning,
+                       module="__main__", append=1)
+        simplefilter("ignore", category=DeprecationWarning, append=1)
+        simplefilter("ignore", category=PendingDeprecationWarning, append=1)
+        simplefilter("ignore", category=ImportWarning, append=1)
+        simplefilter("ignore", category=ResourceWarning, append=1)
+
+del _warnings_defaults

Lib/weakref.py

@@ -0,0 +1,632 @@
+"""Weak reference support for Python.
+
+This module is an implementation of PEP 205:
+
+http://www.python.org/dev/peps/pep-0205/
+"""
+
+# Naming convention: Variables named "wr" are weak reference objects;
+# they are called this instead of "ref" to avoid name collisions with
+# the module-global ref() function imported from _weakref.
+
+from _weakref import (
+     getweakrefcount,
+     getweakrefs,
+     ref,
+     proxy,
+     CallableProxyType,
+     ProxyType,
+     ReferenceType,
+     _remove_dead_weakref)
+
+from _weakrefset import WeakSet, _IterationGuard
+
+import _collections_abc  # Import after _weakref to avoid circular import.
+import sys
+import itertools
+
+ProxyTypes = (ProxyType, CallableProxyType)
+
+__all__ = ["ref", "proxy", "getweakrefcount", "getweakrefs",
+           "WeakKeyDictionary", "ReferenceType", "ProxyType",
+           "CallableProxyType", "ProxyTypes", "WeakValueDictionary",
+           "WeakSet", "WeakMethod", "finalize"]
+
+
+class WeakMethod(ref):
+    """
+    A custom `weakref.ref` subclass which simulates a weak reference to
+    a bound method, working around the lifetime problem of bound methods.
+    """
+
+    __slots__ = "_func_ref", "_meth_type", "_alive", "__weakref__"
+
+    def __new__(cls, meth, callback=None):
+        try:
+            obj = meth.__self__
+            func = meth.__func__
+        except AttributeError:
+            raise TypeError("argument should be a bound method, not {}"
+                            .format(type(meth))) from None
+        def _cb(arg):
+            # The self-weakref trick is needed to avoid creating a reference
+            # cycle.
+            self = self_wr()
+            if self._alive:
+                self._alive = False
+                if callback is not None:
+                    callback(self)
+        self = ref.__new__(cls, obj, _cb)
+        self._func_ref = ref(func, _cb)
+        self._meth_type = type(meth)
+        self._alive = True
+        self_wr = ref(self)
+        return self
+
+    def __call__(self):
+        obj = super().__call__()
+        func = self._func_ref()
+        if obj is None or func is None:
+            return None
+        return self._meth_type(func, obj)
+
+    def __eq__(self, other):
+        if isinstance(other, WeakMethod):
+            if not self._alive or not other._alive:
+                return self is other
+            return ref.__eq__(self, other) and self._func_ref == other._func_ref
+        return False
+
+    def __ne__(self, other):
+        if isinstance(other, WeakMethod):
+            if not self._alive or not other._alive:
+                return self is not other
+            return ref.__ne__(self, other) or self._func_ref != other._func_ref
+        return True
+
+    __hash__ = ref.__hash__
+
+
+class WeakValueDictionary(_collections_abc.MutableMapping):
+    """Mapping class that references values weakly.
+
+    Entries in the dictionary will be discarded when no strong
+    reference to the value exists anymore
+    """
+    # We inherit the constructor without worrying about the input
+    # dictionary; since it uses our .update() method, we get the right
+    # checks (if the other dictionary is a WeakValueDictionary,
+    # objects are unwrapped on the way out, and we always wrap on the
+    # way in).
+
+    def __init__(*args, **kw):
+        if not args:
+            raise TypeError("descriptor '__init__' of 'WeakValueDictionary' "
+                            "object needs an argument")
+        self, *args = args
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
+            self = selfref()
+            if self is not None:
+                if self._iterating:
+                    self._pending_removals.append(wr.key)
+                else:
+                    # Atomic removal is necessary since this function
+                    # can be called asynchronously by the GC
+                    _atomic_removal(d, wr.key)
+        self._remove = remove
+        # A list of keys to be removed
+        self._pending_removals = []
+        self._iterating = set()
+        self.data = d = {}
+        self.update(*args, **kw)
+
+    def _commit_removals(self):
+        l = self._pending_removals
+        d = self.data
+        # We shouldn't encounter any KeyError, because this method should
+        # always be called *before* mutating the dict.
+        while l:
+            key = l.pop()
+            _remove_dead_weakref(d, key)
+
+    def __getitem__(self, key):
+        if self._pending_removals:
+            self._commit_removals()
+        o = self.data[key]()
+        if o is None:
+            raise KeyError(key)
+        else:
+            return o
+
+    def __delitem__(self, key):
+        if self._pending_removals:
+            self._commit_removals()
+        del self.data[key]
+
+    def __len__(self):
+        if self._pending_removals:
+            self._commit_removals()
+        return len(self.data)
+
+    def __contains__(self, key):
+        if self._pending_removals:
+            self._commit_removals()
+        try:
+            o = self.data[key]()
+        except KeyError:
+            return False
+        return o is not None
+
+    def __repr__(self):
+        return "<%s at %#x>" % (self.__class__.__name__, id(self))
+
+    def __setitem__(self, key, value):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data[key] = KeyedRef(value, self._remove, key)
+
+    def copy(self):
+        if self._pending_removals:
+            self._commit_removals()
+        new = WeakValueDictionary()
+        for key, wr in self.data.items():
+            o = wr()
+            if o is not None:
+                new[key] = o
+        return new
+
+    __copy__ = copy
+
+    def __deepcopy__(self, memo):
+        from copy import deepcopy
+        if self._pending_removals:
+            self._commit_removals()
+        new = self.__class__()
+        for key, wr in self.data.items():
+            o = wr()
+            if o is not None:
+                new[deepcopy(key, memo)] = o
+        return new
+
+    def get(self, key, default=None):
+        if self._pending_removals:
+            self._commit_removals()
+        try:
+            wr = self.data[key]
+        except KeyError:
+            return default
+        else:
+            o = wr()
+            if o is None:
+                # This should only happen
+                return default
+            else:
+                return o
+
+    def items(self):
+        if self._pending_removals:
+            self._commit_removals()
+        with _IterationGuard(self):
+            for k, wr in self.data.items():
+                v = wr()
+                if v is not None:
+                    yield k, v
+
+    def keys(self):
+        if self._pending_removals:
+            self._commit_removals()
+        with _IterationGuard(self):
+            for k, wr in self.data.items():
+                if wr() is not None:
+                    yield k
+
+    __iter__ = keys
+
+    def itervaluerefs(self):
+        """Return an iterator that yields the weak references to the values.
+
+        The references are not guaranteed to be 'live' at the time
+        they are used, so the result of calling the references needs
+        to be checked before being used.  This can be used to avoid
+        creating references that will cause the garbage collector to
+        keep the values around longer than needed.
+
+        """
+        if self._pending_removals:
+            self._commit_removals()
+        with _IterationGuard(self):
+            yield from self.data.values()
+
+    def values(self):
+        if self._pending_removals:
+            self._commit_removals()
+        with _IterationGuard(self):
+            for wr in self.data.values():
+                obj = wr()
+                if obj is not None:
+                    yield obj
+
+    def popitem(self):
+        if self._pending_removals:
+            self._commit_removals()
+        while True:
+            key, wr = self.data.popitem()
+            o = wr()
+            if o is not None:
+                return key, o
+
+    def pop(self, key, *args):
+        if self._pending_removals:
+            self._commit_removals()
+        try:
+            o = self.data.pop(key)()
+        except KeyError:
+            o = None
+        if o is None:
+            if args:
+                return args[0]
+            else:
+                raise KeyError(key)
+        else:
+            return o
+
+    def setdefault(self, key, default=None):
+        try:
+            o = self.data[key]()
+        except KeyError:
+            o = None
+        if o is None:
+            if self._pending_removals:
+                self._commit_removals()
+            self.data[key] = KeyedRef(default, self._remove, key)
+            return default
+        else:
+            return o
+
+    def update(*args, **kwargs):
+        if not args:
+            raise TypeError("descriptor 'update' of 'WeakValueDictionary' "
+                            "object needs an argument")
+        self, *args = args
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        dict = args[0] if args else None
+        if self._pending_removals:
+            self._commit_removals()
+        d = self.data
+        if dict is not None:
+            if not hasattr(dict, "items"):
+                dict = type({})(dict)
+            for key, o in dict.items():
+                d[key] = KeyedRef(o, self._remove, key)
+        if len(kwargs):
+            self.update(kwargs)
+
+    def valuerefs(self):
+        """Return a list of weak references to the values.
+
+        The references are not guaranteed to be 'live' at the time
+        they are used, so the result of calling the references needs
+        to be checked before being used.  This can be used to avoid
+        creating references that will cause the garbage collector to
+        keep the values around longer than needed.
+
+        """
+        if self._pending_removals:
+            self._commit_removals()
+        return list(self.data.values())
+
+
+class KeyedRef(ref):
+    """Specialized reference that includes a key corresponding to the value.
+
+    This is used in the WeakValueDictionary to avoid having to create
+    a function object for each key stored in the mapping.  A shared
+    callback object can use the 'key' attribute of a KeyedRef instead
+    of getting a reference to the key from an enclosing scope.
+
+    """
+
+    __slots__ = "key",
+
+    def __new__(type, ob, callback, key):
+        self = ref.__new__(type, ob, callback)
+        self.key = key
+        return self
+
+    def __init__(self, ob, callback, key):
+        super().__init__(ob, callback)
+
+
+class WeakKeyDictionary(_collections_abc.MutableMapping):
+    """ Mapping class that references keys weakly.
+
+    Entries in the dictionary will be discarded when there is no
+    longer a strong reference to the key. This can be used to
+    associate additional data with an object owned by other parts of
+    an application without adding attributes to those objects. This
+    can be especially useful with objects that override attribute
+    accesses.
+    """
+
+    def __init__(self, dict=None):
+        self.data = {}
+        def remove(k, selfref=ref(self)):
+            self = selfref()
+            if self is not None:
+                if self._iterating:
+                    self._pending_removals.append(k)
+                else:
+                    del self.data[k]
+        self._remove = remove
+        # A list of dead weakrefs (keys to be removed)
+        self._pending_removals = []
+        self._iterating = set()
+        self._dirty_len = False
+        if dict is not None:
+            self.update(dict)
+
+    def _commit_removals(self):
+        # NOTE: We don't need to call this method before mutating the dict,
+        # because a dead weakref never compares equal to a live weakref,
+        # even if they happened to refer to equal objects.
+        # However, it means keys may already have been removed.
+        l = self._pending_removals
+        d = self.data
+        while l:
+            try:
+                del d[l.pop()]
+            except KeyError:
+                pass
+
+    def _scrub_removals(self):
+        d = self.data
+        self._pending_removals = [k for k in self._pending_removals if k in d]
+        self._dirty_len = False
+
+    def __delitem__(self, key):
+        self._dirty_len = True
+        del self.data[ref(key)]
+
+    def __getitem__(self, key):
+        return self.data[ref(key)]
+
+    def __len__(self):
+        if self._dirty_len and self._pending_removals:
+            # self._pending_removals may still contain keys which were
+            # explicitly removed, we have to scrub them (see issue #21173).
+            self._scrub_removals()
+        return len(self.data) - len(self._pending_removals)
+
+    def __repr__(self):
+        return "<%s at %#x>" % (self.__class__.__name__, id(self))
+
+    def __setitem__(self, key, value):
+        self.data[ref(key, self._remove)] = value
+
+    def copy(self):
+        new = WeakKeyDictionary()
+        for key, value in self.data.items():
+            o = key()
+            if o is not None:
+                new[o] = value
+        return new
+
+    __copy__ = copy
+
+    def __deepcopy__(self, memo):
+        from copy import deepcopy
+        new = self.__class__()
+        for key, value in self.data.items():
+            o = key()
+            if o is not None:
+                new[o] = deepcopy(value, memo)
+        return new
+
+    def get(self, key, default=None):
+        return self.data.get(ref(key),default)
+
+    def __contains__(self, key):
+        try:
+            wr = ref(key)
+        except TypeError:
+            return False
+        return wr in self.data
+
+    def items(self):
+        with _IterationGuard(self):
+            for wr, value in self.data.items():
+                key = wr()
+                if key is not None:
+                    yield key, value
+
+    def keys(self):
+        with _IterationGuard(self):
+            for wr in self.data:
+                obj = wr()
+                if obj is not None:
+                    yield obj
+
+    __iter__ = keys
+
+    def values(self):
+        with _IterationGuard(self):
+            for wr, value in self.data.items():
+                if wr() is not None:
+                    yield value
+
+    def keyrefs(self):
+        """Return a list of weak references to the keys.
+
+        The references are not guaranteed to be 'live' at the time
+        they are used, so the result of calling the references needs
+        to be checked before being used.  This can be used to avoid
+        creating references that will cause the garbage collector to
+        keep the keys around longer than needed.
+
+        """
+        return list(self.data)
+
+    def popitem(self):
+        self._dirty_len = True
+        while True:
+            key, value = self.data.popitem()
+            o = key()
+            if o is not None:
+                return o, value
+
+    def pop(self, key, *args):
+        self._dirty_len = True
+        return self.data.pop(ref(key), *args)
+
+    def setdefault(self, key, default=None):
+        return self.data.setdefault(ref(key, self._remove),default)
+
+    def update(self, dict=None, **kwargs):
+        d = self.data
+        if dict is not None:
+            if not hasattr(dict, "items"):
+                dict = type({})(dict)
+            for key, value in dict.items():
+                d[ref(key, self._remove)] = value
+        if len(kwargs):
+            self.update(kwargs)
+
+
+class finalize:
+    """Class for finalization of weakrefable objects
+
+    finalize(obj, func, *args, **kwargs) returns a callable finalizer
+    object which will be called when obj is garbage collected. The
+    first time the finalizer is called it evaluates func(*arg, **kwargs)
+    and returns the result. After this the finalizer is dead, and
+    calling it just returns None.
+
+    When the program exits any remaining finalizers for which the
+    atexit attribute is true will be run in reverse order of creation.
+    By default atexit is true.
+    """
+
+    # Finalizer objects don't have any state of their own.  They are
+    # just used as keys to lookup _Info objects in the registry.  This
+    # ensures that they cannot be part of a ref-cycle.
+
+    __slots__ = ()
+    _registry = {}
+    _shutdown = False
+    _index_iter = itertools.count()
+    _dirty = False
+    _registered_with_atexit = False
+
+    class _Info:
+        __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index")
+
+    def __init__(self, obj, func, *args, **kwargs):
+        if not self._registered_with_atexit:
+            # We may register the exit function more than once because
+            # of a thread race, but that is harmless
+            import atexit
+            atexit.register(self._exitfunc)
+            finalize._registered_with_atexit = True
+        info = self._Info()
+        info.weakref = ref(obj, self)
+        info.func = func
+        info.args = args
+        info.kwargs = kwargs or None
+        info.atexit = True
+        info.index = next(self._index_iter)
+        self._registry[self] = info
+        finalize._dirty = True
+
+    def __call__(self, _=None):
+        """If alive then mark as dead and return func(*args, **kwargs);
+        otherwise return None"""
+        info = self._registry.pop(self, None)
+        if info and not self._shutdown:
+            return info.func(*info.args, **(info.kwargs or {}))
+
+    def detach(self):
+        """If alive then mark as dead and return (obj, func, args, kwargs);
+        otherwise return None"""
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is not None and self._registry.pop(self, None):
+            return (obj, info.func, info.args, info.kwargs or {})
+
+    def peek(self):
+        """If alive then return (obj, func, args, kwargs);
+        otherwise return None"""
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is not None:
+            return (obj, info.func, info.args, info.kwargs or {})
+
+    @property
+    def alive(self):
+        """Whether finalizer is alive"""
+        return self in self._registry
+
+    @property
+    def atexit(self):
+        """Whether finalizer should be called at exit"""
+        info = self._registry.get(self)
+        return bool(info) and info.atexit
+
+    @atexit.setter
+    def atexit(self, value):
+        info = self._registry.get(self)
+        if info:
+            info.atexit = bool(value)
+
+    def __repr__(self):
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is None:
+            return '<%s object at %#x; dead>' % (type(self).__name__, id(self))
+        else:
+            return '<%s object at %#x; for %r at %#x>' % \
+                (type(self).__name__, id(self), type(obj).__name__, id(obj))
+
+    @classmethod
+    def _select_for_exit(cls):
+        # Return live finalizers marked for exit, oldest first
+        L = [(f,i) for (f,i) in cls._registry.items() if i.atexit]
+        L.sort(key=lambda item:item[1].index)
+        return [f for (f,i) in L]
+
+    @classmethod
+    def _exitfunc(cls):
+        # At shutdown invoke finalizers for which atexit is true.
+        # This is called once all other non-daemonic threads have been
+        # joined.
+        reenable_gc = False
+        try:
+            if cls._registry:
+                import gc
+                if gc.isenabled():
+                    reenable_gc = True
+                    gc.disable()
+                pending = None
+                while True:
+                    if pending is None or finalize._dirty:
+                        pending = cls._select_for_exit()
+                        finalize._dirty = False
+                    if not pending:
+                        break
+                    f = pending.pop()
+                    try:
+                        # gc is disabled, so (assuming no daemonic
+                        # threads) the following is the only line in
+                        # this function which might trigger creation
+                        # of a new finalizer
+                        f()
+                    except Exception:
+                        sys.excepthook(*sys.exc_info())
+                    assert f not in cls._registry
+        finally:
+            # prevent any more finalizers from executing during shutdown
+            finalize._shutdown = True
+            if reenable_gc:
+                gc.enable()

Lib/xdrlib.py

@@ -0,0 +1,241 @@
+"""Implements (a subset of) Sun XDR -- eXternal Data Representation.
+
+See: RFC 1014
+
+"""
+
+import struct
+from io import BytesIO
+from functools import wraps
+
+__all__ = ["Error", "Packer", "Unpacker", "ConversionError"]
+
+# exceptions
+class Error(Exception):
+    """Exception class for this module. Use:
+
+    except xdrlib.Error as var:
+        # var has the Error instance for the exception
+
+    Public ivars:
+        msg -- contains the message
+
+    """
+    def __init__(self, msg):
+        self.msg = msg
+    def __repr__(self):
+        return repr(self.msg)
+    def __str__(self):
+        return str(self.msg)
+
+
+class ConversionError(Error):
+    pass
+
+def raise_conversion_error(function):
+    """ Wrap any raised struct.errors in a ConversionError. """
+
+    @wraps(function)
+    def result(self, value):
+        try:
+            return function(self, value)
+        except struct.error as e:
+            raise ConversionError(e.args[0]) from None
+    return result
+
+
+class Packer:
+    """Pack various data representations into a buffer."""
+
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.__buf = BytesIO()
+
+    def get_buffer(self):
+        return self.__buf.getvalue()
+    # backwards compatibility
+    get_buf = get_buffer
+
+    @raise_conversion_error
+    def pack_uint(self, x):
+        self.__buf.write(struct.pack('>L', x))
+
+    @raise_conversion_error
+    def pack_int(self, x):
+        self.__buf.write(struct.pack('>l', x))
+
+    pack_enum = pack_int
+
+    def pack_bool(self, x):
+        if x: self.__buf.write(b'\0\0\0\1')
+        else: self.__buf.write(b'\0\0\0\0')
+
+    def pack_uhyper(self, x):
+        try:
+            self.pack_uint(x>>32 & 0xffffffff)
+        except (TypeError, struct.error) as e:
+            raise ConversionError(e.args[0]) from None
+        try:
+            self.pack_uint(x & 0xffffffff)
+        except (TypeError, struct.error) as e:
+            raise ConversionError(e.args[0]) from None
+
+    pack_hyper = pack_uhyper
+
+    @raise_conversion_error
+    def pack_float(self, x):
+        self.__buf.write(struct.pack('>f', x))
+
+    @raise_conversion_error
+    def pack_double(self, x):
+        self.__buf.write(struct.pack('>d', x))
+
+    def pack_fstring(self, n, s):
+        if n < 0:
+            raise ValueError('fstring size must be nonnegative')
+        data = s[:n]
+        n = ((n+3)//4)*4
+        data = data + (n - len(data)) * b'\0'
+        self.__buf.write(data)
+
+    pack_fopaque = pack_fstring
+
+    def pack_string(self, s):
+        n = len(s)
+        self.pack_uint(n)
+        self.pack_fstring(n, s)
+
+    pack_opaque = pack_string
+    pack_bytes = pack_string
+
+    def pack_list(self, list, pack_item):
+        for item in list:
+            self.pack_uint(1)
+            pack_item(item)
+        self.pack_uint(0)
+
+    def pack_farray(self, n, list, pack_item):
+        if len(list) != n:
+            raise ValueError('wrong array size')
+        for item in list:
+            pack_item(item)
+
+    def pack_array(self, list, pack_item):
+        n = len(list)
+        self.pack_uint(n)
+        self.pack_farray(n, list, pack_item)
+
+
+
+class Unpacker:
+    """Unpacks various data representations from the given buffer."""
+
+    def __init__(self, data):
+        self.reset(data)
+
+    def reset(self, data):
+        self.__buf = data
+        self.__pos = 0
+
+    def get_position(self):
+        return self.__pos
+
+    def set_position(self, position):
+        self.__pos = position
+
+    def get_buffer(self):
+        return self.__buf
+
+    def done(self):
+        if self.__pos < len(self.__buf):
+            raise Error('unextracted data remains')
+
+    def unpack_uint(self):
+        i = self.__pos
+        self.__pos = j = i+4
+        data = self.__buf[i:j]
+        if len(data) < 4:
+            raise EOFError
+        return struct.unpack('>L', data)[0]
+
+    def unpack_int(self):
+        i = self.__pos
+        self.__pos = j = i+4
+        data = self.__buf[i:j]
+        if len(data) < 4:
+            raise EOFError
+        return struct.unpack('>l', data)[0]
+
+    unpack_enum = unpack_int
+
+    def unpack_bool(self):
+        return bool(self.unpack_int())
+
+    def unpack_uhyper(self):
+        hi = self.unpack_uint()
+        lo = self.unpack_uint()
+        return int(hi)<<32 | lo
+
+    def unpack_hyper(self):
+        x = self.unpack_uhyper()
+        if x >= 0x8000000000000000:
+            x = x - 0x10000000000000000
+        return x
+
+    def unpack_float(self):
+        i = self.__pos
+        self.__pos = j = i+4
+        data = self.__buf[i:j]
+        if len(data) < 4:
+            raise EOFError
+        return struct.unpack('>f', data)[0]
+
+    def unpack_double(self):
+        i = self.__pos
+        self.__pos = j = i+8
+        data = self.__buf[i:j]
+        if len(data) < 8:
+            raise EOFError
+        return struct.unpack('>d', data)[0]
+
+    def unpack_fstring(self, n):
+        if n < 0:
+            raise ValueError('fstring size must be nonnegative')
+        i = self.__pos
+        j = i + (n+3)//4*4
+        if j > len(self.__buf):
+            raise EOFError
+        self.__pos = j
+        return self.__buf[i:i+n]
+
+    unpack_fopaque = unpack_fstring
+
+    def unpack_string(self):
+        n = self.unpack_uint()
+        return self.unpack_fstring(n)
+
+    unpack_opaque = unpack_string
+    unpack_bytes = unpack_string
+
+    def unpack_list(self, unpack_item):
+        list = []
+        while 1:
+            x = self.unpack_uint()
+            if x == 0: break
+            if x != 1:
+                raise ConversionError('0 or 1 expected, got %r' % (x,))
+            item = unpack_item()
+            list.append(item)
+        return list
+
+    def unpack_farray(self, n, unpack_item):
+        list = []
+        for i in range(n):
+            list.append(unpack_item())
+        return list
+
+    def unpack_array(self, unpack_item):
+        n = self.unpack_uint()
+        return self.unpack_farray(n, unpack_item)

README.md

@@ -1,3 +1,5 @@
+<img src="./logo.png" width="125" height="125" align="right" />
+
 # RustPython
 
 A Python-3 (CPython >= 3.5.0) Interpreter written in Rust :snake: :scream: :metal:.
@@ -83,7 +85,7 @@ Another approach is to checkout the source code: builtin functions and object me
 and easiest way to contribute.
 
 You can also simply run
-`cargo run tests/snippets/whats_left_to_implement.py` to assist in finding any
+`./whats_left.sh` to assist in finding any
 unimplemented method.
 
 # Testing
@@ -103,104 +105,40 @@ There also are some unit tests, you can run those with cargo:
 $ cargo test --all
 ```
 
-# Using another standard library
+# Using a standard library
 
-As of now the standard library is under construction.
+As of now the standard library is under construction. You can
+use a standard library by setting the RUSTPYTHONPATH environment
+variable.
+
+To do this, follow this method:
+
+```shell
+$ export RUSTPYTHONPATH=~/GIT/RustPython/Lib
+$ cargo run -- -c 'import xdrlib'
+```
 
 You can play around
 with other standard libraries for python. For example,
 the [ouroboros library](https://github.com/pybee/ouroboros).
 
-To do this, follow this method:
-
-```shell
-$ cd ~/GIT
-$ git clone git@github.com:pybee/ouroboros.git
-$ export PYTHONPATH=~/GIT/ouroboros/ouroboros
-$ cd RustPython
-$ cargo run -- -c 'import statistics'
-```
-
 # Compiling to WebAssembly
 
-At this stage RustPython only has preliminary support for web assembly. The instructions here are intended for developers or those wishing to run a toy example.
-
-## Setup
-
-To get started, install [wasm-pack](https://rustwasm.github.io/wasm-pack/installer/) and `npm`. ([wasm-bindgen](https://rustwasm.github.io/wasm-bindgen/whirlwind-tour/basic-usage.html) should be installed by `wasm-pack`. if not, install it yourself)
-
-<!-- Using `rustup` add the compile target `wasm32-unknown-emscripten`. To do so you will need to have [rustup](https://rustup.rs/) installed.
-
-```bash
-rustup target add wasm32-unknown-emscripten
-```
-
-Next, install `emsdk`:
-
-```bash
-curl https://s3.amazonaws.com/mozilla-games/emscripten/releases/emsdk-portable.tar.gz | tar -zxv
-cd emsdk-portable/
-./emsdk update
-./emsdk install sdk-incoming-64bit
-./emsdk activate sdk-incoming-64bit
-``` -->
-
-## Build
-
-Move into the `wasm` directory. This directory contains a library crate for interop
-with python to rust to js and back in `wasm/lib`, the demo website found at
-https://rustpython.github.io/demo in `wasm/demo`, and an example of how to use
-the crate as a library in one's own JS app in `wasm/example`.
-
-```sh
-cd wasm
-```
-
-Go to the demo directory. This is the best way of seeing the changes made to either
-the library or the JS demo, as the `rustpython_wasm` module is set to the global
-JS variable `rp` on the website.
-
-```sh
-cd demo
-```
-
-Now, start the webpack development server. It'll compile the crate and then
-the demo app. This will likely take a long time, both the wasm-pack portion and
-the webpack portion (from after it says "Your crate has been correctly compiled"),
-so be patient.
-
-```sh
-npm run dev
-```
-
-You can now open the webpage on https://localhost:8080 and Python code in either
-the text box or browser devtools with:
-
-```js
-rp.pyEval(
-  `
-print(js_vars['a'] * 9)
-`,
-  {
-    vars: {
-      a: 9
-    }
-  }
-);
-```
-
-Alternatively, you can run `npm run build` to build the app once, without watching
-for changes, or `npm run dist` to build the app in release mode, both for the
-crate and webpack.
+[See this doc](wasm/README.md)
 
 # Code style
 
-The code style used is the default rustfmt codestyle. Please format your code accordingly.
+The code style used is the default [rustfmt](https://github.com/rust-lang/rustfmt) codestyle. Please format your code accordingly.
+We also use [clippy](https://github.com/rust-lang/rust-clippy) to detect rust code issues.
 
 # Community
 
 Chat with us on [gitter][gitter].
 
+# Code of conduct
+
+Our code of conduct [can be found here](code-of-conduct.md).
+
 # Credit
 
 The initial work was based on [windelbouwman/rspython](https://github.com/windelbouwman/rspython) and [shinglyu/RustPython](https://github.com/shinglyu/RustPython)

azure-pipelines.yml

@@ -8,8 +8,8 @@ jobs:
     vmImage: 'vs2017-win2016'
   strategy:
     matrix:
-      Python37:
-        python.version: '3.7'
+      Python36:
+        python.version: '3.6'
     maxParallel: 10
 
   steps:

benchmarks/README.md

@@ -0,0 +1,19 @@
+
+# Benchmarking
+
+These are some files to determine performance of rustpython.
+
+# Usage
+
+Install pytest and pytest-benchmark:
+
+    $ pip install pytest-benchmark
+
+Then run:
+
+    $ pytest
+
+# Benchmark source
+
+- https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html
+

benchmarks/bench.rs

@@ -0,0 +1,125 @@
+#![feature(test)]
+
+extern crate cpython;
+extern crate rustpython_parser;
+extern crate rustpython_vm;
+extern crate test;
+
+use rustpython_vm::pyobject::PyResult;
+use rustpython_vm::{compile, VirtualMachine};
+
+#[bench]
+fn bench_tokenization(b: &mut test::Bencher) {
+    use rustpython_parser::lexer::{make_tokenizer, Tok};
+
+    let source = include_str!("./benchmarks/minidom.py");
+
+    b.bytes = source.len() as _;
+    b.iter(|| {
+        let lexer = make_tokenizer(source);
+        for res in lexer {
+            let _token: Tok = res.unwrap().1;
+        }
+    })
+}
+
+#[bench]
+fn bench_rustpy_parse_to_ast(b: &mut test::Bencher) {
+    use rustpython_parser::parser::parse_program;
+
+    let source = include_str!("./benchmarks/minidom.py");
+
+    b.bytes = source.len() as _;
+    b.iter(|| parse_program(source).unwrap())
+}
+
+#[bench]
+fn bench_cpython_parse_to_ast(b: &mut test::Bencher) {
+    let source = include_str!("./benchmarks/minidom.py");
+
+    let gil = cpython::Python::acquire_gil();
+    let python = gil.python();
+
+    let globals = None;
+    let locals = cpython::PyDict::new(python);
+
+    locals.set_item(python, "SOURCE_CODE", source).unwrap();
+
+    let code = "compile(SOURCE_CODE, mode=\"exec\", filename=\"minidom.py\")";
+
+    b.bytes = source.len() as _;
+    b.iter(|| {
+        let res: cpython::PyResult<cpython::PyObject> = python.eval(code, globals, Some(&locals));
+        assert!(res.is_ok());
+    })
+}
+
+#[bench]
+fn bench_cpython_nbody(b: &mut test::Bencher) {
+    let source = include_str!("./benchmarks/nbody.py");
+
+    let gil = cpython::Python::acquire_gil();
+    let python = gil.python();
+
+    let globals = None;
+    let locals = None;
+
+    b.iter(|| {
+        let res: cpython::PyResult<()> = python.run(source, globals, locals);
+        assert!(res.is_ok());
+    })
+}
+
+#[bench]
+fn bench_cpython_mandelbrot(b: &mut test::Bencher) {
+    let source = include_str!("./benchmarks/mandelbrot.py");
+
+    let gil = cpython::Python::acquire_gil();
+    let python = gil.python();
+
+    let globals = None;
+    let locals = None;
+
+    b.iter(|| {
+        let res: cpython::PyResult<()> = python.run(source, globals, locals);
+        assert!(res.is_ok());
+    })
+}
+
+#[bench]
+fn bench_rustpy_nbody(b: &mut test::Bencher) {
+    // NOTE: Take long time.
+    let source = include_str!("./benchmarks/nbody.py");
+
+    let vm = VirtualMachine::new();
+
+    let code = match vm.compile(source, &compile::Mode::Single, "<stdin>".to_string()) {
+        Ok(code) => code,
+        Err(e) => panic!("{:?}", e),
+    };
+
+    b.iter(|| {
+        let scope = vm.new_scope_with_builtins();
+        let res: PyResult = vm.run_code_obj(code.clone(), scope);
+        assert!(res.is_ok());
+    })
+}
+
+#[bench]
+fn bench_rustpy_mandelbrot(b: &mut test::Bencher) {
+    // NOTE: Take long time.
+    let source = include_str!("./benchmarks/mandelbrot.py");
+
+    let vm = VirtualMachine::new();
+
+    let code = match vm.compile(source, &compile::Mode::Single, "<stdin>".to_string()) {
+        Ok(code) => code,
+        Err(e) => panic!("{:?}", e),
+    };
+
+    b.iter(|| {
+        let scope = vm.new_scope_with_builtins();
+        let res: PyResult = vm.run_code_obj(code.clone(), scope);
+        assert!(res.is_ok());
+    })
+}

benchmarks/benchmarks/mandelbrot.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+w = 50.0
+h = 50.0
+
+y = 0.0
+while y < h:
+	x = 0.0
+	while x < w:
+		Zr, Zi, Tr, Ti = 0.0, 0.0, 0.0, 0.0
+		Cr = 2*x/w - 1.5
+		Ci = 2*y/h - 1.0
+
+		i = 0
+		while i < 50 and Tr+Ti <= 4:
+			Zi = 2*Zr*Zi + Ci
+			Zr = Tr - Ti + Cr
+			Tr = Zr * Zr
+			Ti = Zi * Zi
+			i = i+1
+
+		if Tr+Ti <= 4:
+			# print('*', end='')
+			pass
+		else:
+			# print('·', end='')
+			pass
+
+		x = x+1
+
+	# print()
+	y = y+1

benchmarks/benchmarks/nbody.py

@@ -0,0 +1,110 @@
+
+# The Computer Language Benchmarks Game
+# https://salsa.debian.org/benchmarksgame-team/benchmarksgame/
+#
+# originally by Kevin Carson
+# modified by Tupteq, Fredrik Johansson, and Daniel Nanz
+# modified by Maciej Fijalkowski
+# 2to3
+# modified by Andriy Misyura
+
+from math import sqrt
+
+def combinations(l):
+    result = []
+    for x in range(len(l) - 1):
+        ls = l[x+1:]
+        for y in ls:
+            result.append((l[x][0],l[x][1],l[x][2],y[0],y[1],y[2]))
+    return result
+
+PI = 3.14159265358979323
+SOLAR_MASS = 4 * PI * PI
+DAYS_PER_YEAR = 365.24
+
+BODIES = {
+    'sun': ([0.0, 0.0, 0.0], [0.0, 0.0, 0.0], SOLAR_MASS),
+
+    'jupiter': ([4.84143144246472090e+00,
+                 -1.16032004402742839e+00,
+                 -1.03622044471123109e-01],
+                [1.66007664274403694e-03 * DAYS_PER_YEAR,
+                 7.69901118419740425e-03 * DAYS_PER_YEAR,
+                 -6.90460016972063023e-05 * DAYS_PER_YEAR],
+                9.54791938424326609e-04 * SOLAR_MASS),
+
+    'saturn': ([8.34336671824457987e+00,
+                4.12479856412430479e+00,
+                -4.03523417114321381e-01],
+               [-2.76742510726862411e-03 * DAYS_PER_YEAR,
+                4.99852801234917238e-03 * DAYS_PER_YEAR,
+                2.30417297573763929e-05 * DAYS_PER_YEAR],
+               2.85885980666130812e-04 * SOLAR_MASS),
+
+    'uranus': ([1.28943695621391310e+01,
+                -1.51111514016986312e+01,
+                -2.23307578892655734e-01],
+               [2.96460137564761618e-03 * DAYS_PER_YEAR,
+                2.37847173959480950e-03 * DAYS_PER_YEAR,
+                -2.96589568540237556e-05 * DAYS_PER_YEAR],
+               4.36624404335156298e-05 * SOLAR_MASS),
+
+    'neptune': ([1.53796971148509165e+01,
+                 -2.59193146099879641e+01,
+                 1.79258772950371181e-01],
+                [2.68067772490389322e-03 * DAYS_PER_YEAR,
+                 1.62824170038242295e-03 * DAYS_PER_YEAR,
+                 -9.51592254519715870e-05 * DAYS_PER_YEAR],
+                5.15138902046611451e-05 * SOLAR_MASS) }
+
+SYSTEM = tuple(BODIES.values())
+PAIRS = tuple(combinations(SYSTEM))
+
+def advance(dt, n, bodies=SYSTEM, pairs=PAIRS):
+    for i in range(n):
+        for ([x1, y1, z1], v1, m1, [x2, y2, z2], v2, m2) in pairs:
+            dx = x1 - x2
+            dy = y1 - y2
+            dz = z1 - z2
+            dist = sqrt(dx * dx + dy * dy + dz * dz);
+            mag = dt / (dist*dist*dist)
+            b1m = m1 * mag
+            b2m = m2 * mag
+            v1[0] -= dx * b2m
+            v1[1] -= dy * b2m
+            v1[2] -= dz * b2m
+            v2[2] += dz * b1m
+            v2[1] += dy * b1m
+            v2[0] += dx * b1m
+        for (r, [vx, vy, vz], m) in bodies:
+            r[0] += dt * vx
+            r[1] += dt * vy
+            r[2] += dt * vz
+
+def report_energy(bodies=SYSTEM, pairs=PAIRS, e=0.0):
+    for ((x1, y1, z1), v1, m1, (x2, y2, z2), v2, m2) in pairs:
+        dx = x1 - x2
+        dy = y1 - y2
+        dz = z1 - z2
+        e -= (m1 * m2) / ((dx * dx + dy * dy + dz * dz) ** 0.5)
+    for (r, [vx, vy, vz], m) in bodies:
+        e += m * (vx * vx + vy * vy + vz * vz) / 2.
+    # print(f"{e}")
+
+def offset_momentum(ref, bodies=SYSTEM, px=0.0, py=0.0, pz=0.0):
+    for (r, [vx, vy, vz], m) in bodies:
+        px -= vx * m
+        py -= vy * m
+        pz -= vz * m
+    (r, v, m) = ref
+    v[0] = px / m
+    v[1] = py / m
+    v[2] = pz / m
+
+def main(n, ref='sun'):
+    offset_momentum(BODIES[ref])
+    report_energy()
+    advance(0.01, n)
+    report_energy()
+
+main(500)

benchmarks/test_benchmarks.py

@@ -0,0 +1,31 @@
+
+import time
+import sys
+
+import pytest
+import subprocess
+
+from benchmarks import nbody
+
+# Interpreters:
+rustpython_exe = '../target/release/rustpython'
+cpython_exe = sys.executable
+pythons = [
+    cpython_exe,
+    rustpython_exe
+]
+
+# Benchmark scripts:
+benchmarks = [
+    ['benchmarks/nbody.py'],
+    ['benchmarks/mandelbrot.py'],
+]
+
+@pytest.mark.parametrize('exe', pythons)
+@pytest.mark.parametrize('args', benchmarks)
+def test_bench(exe, args, benchmark):
+    def bench():
+        subprocess.run([exe] + args)
+
+    benchmark(bench)
+

bytecode/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "rustpython-bytecode"
+description = "RustPython specific bytecode."
+version = "0.1.0"
+authors = ["RustPython Team"]
+edition = "2018"
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+
+
+[dependencies]
+bitflags = "1.1"
+num-bigint = { version = "0.2", features = ["serde"] }
+num-complex = { version = "0.2", features = ["serde"] }
+serde = { version = "1.0", features = ["derive"] }

bytecode/src/bytecode.rs

@@ -1,27 +1,46 @@
 //! Implement python as a virtual machine with bytecodes. This module
 //! implements bytecode structure.
 
-/*
- * Primitive instruction type, which can be encoded and decoded.
- */
-
+use bitflags::bitflags;
 use num_bigint::BigInt;
 use num_complex::Complex64;
-use rustpython_parser::ast;
+use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::fmt;
 
+/// Sourcode location.
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+pub struct Location {
+    row: usize,
+    column: usize,
+}
+
+impl Location {
+    pub fn new(row: usize, column: usize) -> Self {
+        Location { row, column }
+    }
+
+    pub fn row(&self) -> usize {
+        self.row
+    }
+
+    pub fn column(&self) -> usize {
+        self.column
+    }
+}
+
 /// Primary container of a single code object. Each python function has
 /// a codeobject. Also a module has a codeobject.
-#[derive(Clone, PartialEq)]
+#[derive(Clone, PartialEq, Serialize, Deserialize)]
 pub struct CodeObject {
     pub instructions: Vec<Instruction>,
+    /// Jump targets.
     pub label_map: HashMap<Label, usize>,
-    pub locations: Vec<ast::Location>,
-    pub arg_names: Vec<String>,          // Names of positional arguments
-    pub varargs: Option<Option<String>>, // *args or *
+    pub locations: Vec<Location>,
+    pub arg_names: Vec<String>, // Names of positional arguments
+    pub varargs: Varargs,       // *args or *
     pub kwonlyarg_names: Vec<String>,
-    pub varkeywords: Option<Option<String>>, // **kwargs or **
+    pub varkeywords: Varargs, // **kwargs or **
     pub source_path: String,
     pub first_line_number: usize,
     pub obj_name: String, // Name of the object that created this code object
@@ -29,28 +48,53 @@ pub struct CodeObject {
 }
 
 bitflags! {
+    #[derive(Serialize, Deserialize)]
     pub struct FunctionOpArg: u8 {
         const HAS_DEFAULTS = 0x01;
+        const HAS_KW_ONLY_DEFAULTS = 0x02;
+        const HAS_ANNOTATIONS = 0x04;
     }
 }
 
 pub type Label = usize;
 
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum NameScope {
+    Local,
+    NonLocal,
+    Global,
+}
+
+/// Transforms a value prior to formatting it.
+#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub enum ConversionFlag {
+    /// Converts by calling `str(<value>)`.
+    Str,
+    /// Converts by calling `ascii(<value>)`.
+    Ascii,
+    /// Converts by calling `repr(<value>)`.
+    Repr,
+}
+
 /// A Single bytecode instruction.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Instruction {
     Import {
         name: String,
-        symbol: Option<String>,
+        symbols: Vec<String>,
+        level: usize,
     },
     ImportStar {
         name: String,
+        level: usize,
     },
     LoadName {
         name: String,
+        scope: NameScope,
     },
     StoreName {
         name: String,
+        scope: NameScope,
     },
     DeleteName {
         name: String,
@@ -168,21 +212,22 @@ pub enum Instruction {
     },
     Unpack,
     FormatValue {
-        conversion: Option<ast::ConversionFlag>,
+        conversion: Option<ConversionFlag>,
         spec: String,
     },
+    PopException,
 }
 
 use self::Instruction::*;
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum CallType {
     Positional(usize),
     Keyword(usize),
     Ex(bool),
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Constant {
     Integer { value: BigInt },
     Float { value: f64 },
@@ -196,7 +241,7 @@ pub enum Constant {
     Ellipsis,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum ComparisonOperator {
     Greater,
     GreaterOrEqual,
@@ -210,7 +255,7 @@ pub enum ComparisonOperator {
     IsNot,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum BinaryOperator {
     Power,
     Multiply,
@@ -228,7 +273,7 @@ pub enum BinaryOperator {
     Or,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum UnaryOperator {
     Not,
     Invert,
@@ -236,6 +281,13 @@ pub enum UnaryOperator {
     Plus,
 }
 
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum Varargs {
+    None,
+    Unnamed,
+    Named(String),
+}
+
 /*
 Maintain a stack of blocks on the VM.
 pub enum BlockType {
@@ -247,9 +299,9 @@ pub enum BlockType {
 impl CodeObject {
     pub fn new(
         arg_names: Vec<String>,
-        varargs: Option<Option<String>>,
+        varargs: Varargs,
         kwonlyarg_names: Vec<String>,
-        varkeywords: Option<Option<String>>,
+        varkeywords: Varargs,
         source_path: String,
         first_line_number: usize,
         obj_name: String,
@@ -308,13 +360,27 @@ impl Instruction {
             ($variant:ident, $var1:expr, $var2:expr) => {
                 write!(f, "{:20} ({}, {})\n", stringify!($variant), $var1, $var2)
             };
+            ($variant:ident, $var1:expr, $var2:expr, $var3:expr) => {
+                write!(
+                    f,
+                    "{:20} ({}, {}, {})\n",
+                    stringify!($variant),
+                    $var1,
+                    $var2,
+                    $var3
+                )
+            };
         }
 
         match self {
-            Import { name, symbol } => w!(Import, name, format!("{:?}", symbol)),
-            ImportStar { name } => w!(ImportStar, name),
-            LoadName { name } => w!(LoadName, name),
-            StoreName { name } => w!(StoreName, name),
+            Import {
+                name,
+                symbols,
+                level,
+            } => w!(Import, name, format!("{:?}", symbols), level),
+            ImportStar { name, level } => w!(ImportStar, name, level),
+            LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)),
+            StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)),
             DeleteName { name } => w!(DeleteName, name),
             StoreSubscript => w!(StoreSubscript),
             DeleteSubscript => w!(DeleteSubscript),
@@ -362,6 +428,7 @@ impl Instruction {
             UnpackEx { before, after } => w!(UnpackEx, before, after),
             Unpack => w!(Unpack),
             FormatValue { spec, .. } => w!(FormatValue, spec), // TODO: write conversion
+            PopException => w!(PopException),
         }
     }
 }

bytecode/src/lib.rs

@@ -0,0 +1 @@
+pub mod bytecode;

code-of-conduct.md

@@ -0,0 +1,77 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at windel.bouwman@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq
+

compiler/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "rustpython-compiler"
+version = "0.1.0"
+description = "Compiler for python code into bytecode for the rustpython VM."
+authors = ["RustPython Team"]
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+edition = "2018"
+
+[dependencies]
+rustpython-bytecode = { path = "../bytecode", version = "0.1.0" }
+rustpython-parser = { path = "../parser", version = "0.1.0" }
+num-complex = { version = "0.2", features = ["serde"] }
+log = "0.3"

compiler/src/error.rs

@@ -0,0 +1,67 @@
+use rustpython_parser::error::{ParseError, ParseErrorType};
+use rustpython_parser::lexer::Location;
+
+use std::error::Error;
+use std::fmt;
+
+#[derive(Debug)]
+pub struct CompileError {
+    pub error: CompileErrorType,
+    pub location: Location,
+}
+
+impl From<ParseError> for CompileError {
+    fn from(error: ParseError) -> Self {
+        CompileError {
+            error: CompileErrorType::Parse(error.error),
+            location: error.location,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum CompileErrorType {
+    /// Invalid assignment, cannot store value in target.
+    Assign(&'static str),
+    /// Invalid delete
+    Delete(&'static str),
+    /// Expected an expression got a statement
+    ExpectExpr,
+    /// Parser error
+    Parse(ParseErrorType),
+    SyntaxError(String),
+    /// Multiple `*` detected
+    StarArgs,
+    /// Break statement outside of loop.
+    InvalidBreak,
+    /// Continue statement outside of loop.
+    InvalidContinue,
+    InvalidReturn,
+    InvalidYield,
+}
+
+impl fmt::Display for CompileError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.error {
+            CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target),
+            CompileErrorType::Delete(target) => write!(f, "can't delete {}", target),
+            CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"),
+            CompileErrorType::Parse(err) => write!(f, "{}", err),
+            CompileErrorType::SyntaxError(err) => write!(f, "{}", err),
+            CompileErrorType::StarArgs => write!(f, "Two starred expressions in assignment"),
+            CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
+            CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
+            CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),
+            CompileErrorType::InvalidYield => write!(f, "'yield' outside function"),
+        }?;
+
+        // Print line number:
+        write!(f, " at {}", self.location)
+    }
+}
+
+impl Error for CompileError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        None
+    }
+}

compiler/src/lib.rs

@@ -0,0 +1,9 @@
+//! Compile a Python AST or source code into bytecode consumable by RustPython or
+//! (eventually) CPython.
+
+#[macro_use]
+extern crate log;
+
+pub mod compile;
+pub mod error;
+mod symboltable;

compiler/src/symboltable.rs

@@ -0,0 +1,596 @@
+/* Python code is pre-scanned for symbols in the ast.
+
+This ensures that global and nonlocal keywords are picked up.
+Then the compiler can use the symbol table to generate proper
+load and store instructions for names.
+
+Inspirational file: https://github.com/python/cpython/blob/master/Python/symtable.c
+*/
+
+use crate::error::{CompileError, CompileErrorType};
+use rustpython_parser::ast;
+use rustpython_parser::lexer::Location;
+use std::collections::HashMap;
+
+pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTableError> {
+    let mut builder = SymbolTableBuilder::new();
+    builder.enter_scope();
+    builder.scan_program(program)?;
+    assert_eq!(builder.scopes.len(), 1);
+
+    let symbol_table = builder.scopes.pop().unwrap();
+    analyze_symbol_table(&symbol_table, None)?;
+    Ok(symbol_table)
+}
+
+pub fn statements_to_symbol_table(
+    statements: &[ast::LocatedStatement],
+) -> Result<SymbolScope, SymbolTableError> {
+    let mut builder = SymbolTableBuilder::new();
+    builder.enter_scope();
+    builder.scan_statements(statements)?;
+    assert_eq!(builder.scopes.len(), 1);
+
+    let symbol_table = builder.scopes.pop().unwrap();
+    analyze_symbol_table(&symbol_table, None)?;
+    Ok(symbol_table)
+}
+
+#[derive(Debug)]
+pub enum SymbolRole {
+    Global,
+    Nonlocal,
+    Used,
+    Assigned,
+}
+
+/// Captures all symbols in the current scope, and has a list of subscopes in this scope.
+pub struct SymbolScope {
+    /// A set of symbols present on this scope level.
+    pub symbols: HashMap<String, SymbolRole>,
+
+    /// A list of subscopes in the order as found in the
+    /// AST nodes.
+    pub sub_scopes: Vec<SymbolScope>,
+}
+
+#[derive(Debug)]
+pub struct SymbolTableError {
+    error: String,
+    location: Location,
+}
+
+impl From<SymbolTableError> for CompileError {
+    fn from(error: SymbolTableError) -> Self {
+        CompileError {
+            error: CompileErrorType::SyntaxError(error.error),
+            location: error.location,
+        }
+    }
+}
+
+type SymbolTableResult = Result<(), SymbolTableError>;
+
+impl SymbolScope {
+    pub fn new() -> Self {
+        SymbolScope {
+            symbols: HashMap::new(),
+            sub_scopes: vec![],
+        }
+    }
+
+    pub fn lookup(&self, name: &str) -> Option<&SymbolRole> {
+        self.symbols.get(name)
+    }
+}
+
+impl std::fmt::Debug for SymbolScope {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "SymbolScope({:?} symbols, {:?} sub scopes)",
+            self.symbols.len(),
+            self.sub_scopes.len()
+        )
+    }
+}
+
+/* Perform some sort of analysis on nonlocals, globals etc..
+  See also: https://github.com/python/cpython/blob/master/Python/symtable.c#L410
+*/
+fn analyze_symbol_table(
+    symbol_scope: &SymbolScope,
+    parent_symbol_scope: Option<&SymbolScope>,
+) -> SymbolTableResult {
+    // Analyze sub scopes:
+    for sub_scope in &symbol_scope.sub_scopes {
+        analyze_symbol_table(&sub_scope, Some(symbol_scope))?;
+    }
+
+    // Analyze symbols:
+    for (symbol_name, symbol_role) in &symbol_scope.symbols {
+        analyze_symbol(symbol_name, symbol_role, parent_symbol_scope)?;
+    }
+
+    Ok(())
+}
+
+#[allow(clippy::single_match)]
+fn analyze_symbol(
+    symbol_name: &str,
+    symbol_role: &SymbolRole,
+    parent_symbol_scope: Option<&SymbolScope>,
+) -> SymbolTableResult {
+    match symbol_role {
+        SymbolRole::Nonlocal => {
+            // check if name is defined in parent scope!
+            if let Some(parent_symbol_scope) = parent_symbol_scope {
+                if !parent_symbol_scope.symbols.contains_key(symbol_name) {
+                    return Err(SymbolTableError {
+                        error: format!("no binding for nonlocal '{}' found", symbol_name),
+                        location: Default::default(),
+                    });
+                }
+            } else {
+                return Err(SymbolTableError {
+                    error: format!(
+                        "nonlocal {} defined at place without an enclosing scope",
+                        symbol_name
+                    ),
+                    location: Default::default(),
+                });
+            }
+        }
+        // TODO: add more checks for globals
+        _ => {}
+    }
+    Ok(())
+}
+
+pub struct SymbolTableBuilder {
+    // Scope stack.
+    pub scopes: Vec<SymbolScope>,
+}
+
+impl SymbolTableBuilder {
+    pub fn new() -> Self {
+        SymbolTableBuilder { scopes: vec![] }
+    }
+
+    pub fn enter_scope(&mut self) {
+        let scope = SymbolScope::new();
+        self.scopes.push(scope);
+    }
+
+    fn leave_scope(&mut self) {
+        // Pop scope and add to subscopes of parent scope.
+        let scope = self.scopes.pop().unwrap();
+        self.scopes.last_mut().unwrap().sub_scopes.push(scope);
+    }
+
+    pub fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
+        self.scan_statements(&program.statements)?;
+        Ok(())
+    }
+
+    pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) -> SymbolTableResult {
+        for statement in statements {
+            self.scan_statement(statement)?;
+        }
+        Ok(())
+    }
+
+    fn scan_parameters(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
+        for parameter in parameters {
+            self.scan_parameter(parameter)?;
+        }
+        Ok(())
+    }
+
+    fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
+        self.register_name(&parameter.arg, SymbolRole::Assigned)
+    }
+
+    fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
+        for parameter in parameters {
+            self.scan_parameter_annotation(parameter)?;
+        }
+        Ok(())
+    }
+
+    fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
+        if let Some(annotation) = &parameter.annotation {
+            self.scan_expression(&annotation)?;
+        }
+        Ok(())
+    }
+
+    fn scan_statement(&mut self, statement: &ast::LocatedStatement) -> SymbolTableResult {
+        match &statement.node {
+            ast::Statement::Global { names } => {
+                for name in names {
+                    self.register_name(name, SymbolRole::Global)?;
+                }
+            }
+            ast::Statement::Nonlocal { names } => {
+                for name in names {
+                    self.register_name(name, SymbolRole::Nonlocal)?;
+                }
+            }
+            ast::Statement::FunctionDef {
+                name,
+                body,
+                args,
+                decorator_list,
+                returns,
+            }
+            | ast::Statement::AsyncFunctionDef {
+                name,
+                body,
+                args,
+                decorator_list,
+                returns,
+            } => {
+                self.scan_expressions(decorator_list)?;
+                self.register_name(name, SymbolRole::Assigned)?;
+
+                self.enter_function(args)?;
+
+                self.scan_statements(body)?;
+                if let Some(expression) = returns {
+                    self.scan_expression(expression)?;
+                }
+                self.leave_scope();
+            }
+            ast::Statement::ClassDef {
+                name,
+                body,
+                bases,
+                keywords,
+                decorator_list,
+            } => {
+                self.register_name(name, SymbolRole::Assigned)?;
+                self.enter_scope();
+                self.scan_statements(body)?;
+                self.leave_scope();
+                self.scan_expressions(bases)?;
+                for keyword in keywords {
+                    self.scan_expression(&keyword.value)?;
+                }
+                self.scan_expressions(decorator_list)?;
+            }
+            ast::Statement::Expression { expression } => self.scan_expression(expression)?,
+            ast::Statement::If { test, body, orelse } => {
+                self.scan_expression(test)?;
+                self.scan_statements(body)?;
+                if let Some(code) = orelse {
+                    self.scan_statements(code)?;
+                }
+            }
+            ast::Statement::For {
+                target,
+                iter,
+                body,
+                orelse,
+            }
+            | ast::Statement::AsyncFor {
+                target,
+                iter,
+                body,
+                orelse,
+            } => {
+                self.scan_expression(target)?;
+                self.scan_expression(iter)?;
+                self.scan_statements(body)?;
+                if let Some(code) = orelse {
+                    self.scan_statements(code)?;
+                }
+            }
+            ast::Statement::While { test, body, orelse } => {
+                self.scan_expression(test)?;
+                self.scan_statements(body)?;
+                if let Some(code) = orelse {
+                    self.scan_statements(code)?;
+                }
+            }
+            ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => {
+                // No symbols here.
+            }
+            ast::Statement::Import { import_parts } => {
+                for part in import_parts {
+                    if let Some(alias) = &part.alias {
+                        // `import mymodule as myalias`
+                        self.register_name(alias, SymbolRole::Assigned)?;
+                    } else {
+                        if part.symbols.is_empty() {
+                            // `import module`
+                            self.register_name(&part.module, SymbolRole::Assigned)?;
+                        } else {
+                            // `from mymodule import myimport`
+                            for symbol in &part.symbols {
+                                if let Some(alias) = &symbol.alias {
+                                    // `from mymodule import myimportname as myalias`
+                                    self.register_name(alias, SymbolRole::Assigned)?;
+                                } else {
+                                    self.register_name(&symbol.symbol, SymbolRole::Assigned)?;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            ast::Statement::Return { value } => {
+                if let Some(expression) = value {
+                    self.scan_expression(expression)?;
+                }
+            }
+            ast::Statement::Assert { test, msg } => {
+                self.scan_expression(test)?;
+                if let Some(expression) = msg {
+                    self.scan_expression(expression)?;
+                }
+            }
+            ast::Statement::Delete { targets } => {
+                self.scan_expressions(targets)?;
+            }
+            ast::Statement::Assign { targets, value } => {
+                self.scan_expressions(targets)?;
+                self.scan_expression(value)?;
+            }
+            ast::Statement::AugAssign { target, value, .. } => {
+                self.scan_expression(target)?;
+                self.scan_expression(value)?;
+            }
+            ast::Statement::With { items, body } => {
+                for item in items {
+                    self.scan_expression(&item.context_expr)?;
+                    if let Some(expression) = &item.optional_vars {
+                        self.scan_expression(expression)?;
+                    }
+                }
+                self.scan_statements(body)?;
+            }
+            ast::Statement::Try {
+                body,
+                handlers,
+                orelse,
+                finalbody,
+            } => {
+                self.scan_statements(body)?;
+                for handler in handlers {
+                    if let Some(expression) = &handler.typ {
+                        self.scan_expression(expression)?;
+                    }
+                    if let Some(name) = &handler.name {
+                        self.register_name(name, SymbolRole::Assigned)?;
+                    }
+                    self.scan_statements(&handler.body)?;
+                }
+                if let Some(code) = orelse {
+                    self.scan_statements(code)?;
+                }
+                if let Some(code) = finalbody {
+                    self.scan_statements(code)?;
+                }
+            }
+            ast::Statement::Raise { exception, cause } => {
+                if let Some(expression) = exception {
+                    self.scan_expression(expression)?;
+                }
+                if let Some(expression) = cause {
+                    self.scan_expression(expression)?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn scan_expressions(&mut self, expressions: &[ast::Expression]) -> SymbolTableResult {
+        for expression in expressions {
+            self.scan_expression(expression)?;
+        }
+        Ok(())
+    }
+
+    fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult {
+        match expression {
+            ast::Expression::Binop { a, b, .. } => {
+                self.scan_expression(a)?;
+                self.scan_expression(b)?;
+            }
+            ast::Expression::BoolOp { a, b, .. } => {
+                self.scan_expression(a)?;
+                self.scan_expression(b)?;
+            }
+            ast::Expression::Compare { vals, .. } => {
+                self.scan_expressions(vals)?;
+            }
+            ast::Expression::Subscript { a, b } => {
+                self.scan_expression(a)?;
+                self.scan_expression(b)?;
+            }
+            ast::Expression::Attribute { value, .. } => {
+                self.scan_expression(value)?;
+            }
+            ast::Expression::Dict { elements } => {
+                for (key, value) in elements {
+                    if let Some(key) = key {
+                        self.scan_expression(key)?;
+                    } else {
+                        // dict unpacking marker
+                    }
+                    self.scan_expression(value)?;
+                }
+            }
+            ast::Expression::Await { value } => {
+                self.scan_expression(value)?;
+            }
+            ast::Expression::Yield { value } => {
+                if let Some(expression) = value {
+                    self.scan_expression(expression)?;
+                }
+            }
+            ast::Expression::YieldFrom { value } => {
+                self.scan_expression(value)?;
+            }
+            ast::Expression::Unop { a, .. } => {
+                self.scan_expression(a)?;
+            }
+            ast::Expression::True
+            | ast::Expression::False
+            | ast::Expression::None
+            | ast::Expression::Ellipsis => {}
+            ast::Expression::Number { .. } => {}
+            ast::Expression::Starred { value } => {
+                self.scan_expression(value)?;
+            }
+            ast::Expression::Bytes { .. } => {}
+            ast::Expression::Tuple { elements }
+            | ast::Expression::Set { elements }
+            | ast::Expression::List { elements }
+            | ast::Expression::Slice { elements } => {
+                self.scan_expressions(elements)?;
+            }
+            ast::Expression::Comprehension { kind, generators } => {
+                match **kind {
+                    ast::ComprehensionKind::GeneratorExpression { ref element }
+                    | ast::ComprehensionKind::List { ref element }
+                    | ast::ComprehensionKind::Set { ref element } => {
+                        self.scan_expression(element)?;
+                    }
+                    ast::ComprehensionKind::Dict { ref key, ref value } => {
+                        self.scan_expression(&key)?;
+                        self.scan_expression(&value)?;
+                    }
+                }
+
+                for generator in generators {
+                    self.scan_expression(&generator.target)?;
+                    self.scan_expression(&generator.iter)?;
+                    for if_expr in &generator.ifs {
+                        self.scan_expression(if_expr)?;
+                    }
+                }
+            }
+            ast::Expression::Call {
+                function,
+                args,
+                keywords,
+            } => {
+                self.scan_expression(function)?;
+                self.scan_expressions(args)?;
+                for keyword in keywords {
+                    self.scan_expression(&keyword.value)?;
+                }
+            }
+            ast::Expression::String { value } => {
+                self.scan_string_group(value)?;
+            }
+            ast::Expression::Identifier { name } => {
+                self.register_name(name, SymbolRole::Used)?;
+            }
+            ast::Expression::Lambda { args, body } => {
+                self.enter_function(args)?;
+                self.scan_expression(body)?;
+                self.leave_scope();
+            }
+            ast::Expression::IfExpression { test, body, orelse } => {
+                self.scan_expression(test)?;
+                self.scan_expression(body)?;
+                self.scan_expression(orelse)?;
+            }
+        }
+        Ok(())
+    }
+
+    fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult {
+        // Evaluate eventual default parameters:
+        self.scan_expressions(&args.defaults)?;
+        for kw_default in &args.kw_defaults {
+            if let Some(expression) = kw_default {
+                self.scan_expression(&expression)?;
+            }
+        }
+
+        // Annotations are scanned in outer scope:
+        self.scan_parameters_annotations(&args.args)?;
+        self.scan_parameters_annotations(&args.kwonlyargs)?;
+        if let ast::Varargs::Named(name) = &args.vararg {
+            self.scan_parameter_annotation(name)?;
+        }
+        if let ast::Varargs::Named(name) = &args.kwarg {
+            self.scan_parameter_annotation(name)?;
+        }
+
+        self.enter_scope();
+
+        // Fill scope with parameter names:
+        self.scan_parameters(&args.args)?;
+        self.scan_parameters(&args.kwonlyargs)?;
+        if let ast::Varargs::Named(name) = &args.vararg {
+            self.scan_parameter(name)?;
+        }
+        if let ast::Varargs::Named(name) = &args.kwarg {
+            self.scan_parameter(name)?;
+        }
+        Ok(())
+    }
+
+    fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult {
+        match group {
+            ast::StringGroup::Constant { .. } => {}
+            ast::StringGroup::FormattedValue { value, .. } => {
+                self.scan_expression(value)?;
+            }
+            ast::StringGroup::Joined { values } => {
+                for subgroup in values {
+                    self.scan_string_group(subgroup)?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    #[allow(clippy::single_match)]
+    fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult {
+        let scope_depth = self.scopes.len();
+        let current_scope = self.scopes.last_mut().unwrap();
+        let location = Default::default();
+        if current_scope.symbols.contains_key(name) {
+            // Role already set..
+            match role {
+                SymbolRole::Global => {
+                    return Err(SymbolTableError {
+                        error: format!("name '{}' is used prior to global declaration", name),
+                        location,
+                    })
+                }
+                SymbolRole::Nonlocal => {
+                    return Err(SymbolTableError {
+                        error: format!("name '{}' is used prior to nonlocal declaration", name),
+                        location,
+                    })
+                }
+                _ => {
+                    // Ok?
+                }
+            }
+        } else {
+            match role {
+                SymbolRole::Nonlocal => {
+                    if scope_depth < 2 {
+                        return Err(SymbolTableError {
+                            error: format!("cannot define nonlocal '{}' at top level.", name),
+                            location,
+                        });
+                    }
+                }
+                _ => {
+                    // Ok!
+                }
+            }
+            current_scope.symbols.insert(name.to_string(), role);
+        }
+        Ok(())
+    }
+}

derive/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "rustpython-derive"
+version = "0.1.0"
+description = "Rust language extensions and macros specific to rustpython."
+authors = ["RustPython Team"]
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+edition = "2018"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = { version = "0.15.29", features = ["full"] }
+quote = "0.6.11"
+proc-macro2 = "0.4.27"
+rustpython-compiler = { path = "../compiler", version = "0.1.0" }
+rustpython-bytecode = { path = "../bytecode", version = "0.1.0" }
+bincode =  "1.1"
+proc-macro-hack = "0.5"

derive/src/compile_bytecode.rs

@@ -0,0 +1,174 @@
+//! Parsing and processing for this form:
+//! ```ignore
+//! py_compile_input!(
+//!     // either:
+//!     source = "python_source_code",
+//!     // or
+//!     file = "file/path/relative/to/$CARGO_MANIFEST_DIR",
+//!
+//!     // the mode to compile the code in
+//!     mode = "exec", // or "eval" or "single"
+//!     // the path put into the CodeObject, defaults to "frozen"
+//!     module_name = "frozen",
+//! )
+//! ```
+
+use crate::{extract_spans, Diagnostic};
+use bincode;
+use proc_macro2::{Span, TokenStream as TokenStream2};
+use quote::quote;
+use rustpython_bytecode::bytecode::CodeObject;
+use rustpython_compiler::compile;
+use std::env;
+use std::fs;
+use std::path::PathBuf;
+use syn::parse::{Parse, ParseStream, Result as ParseResult};
+use syn::{self, parse2, Lit, LitByteStr, Meta, Token};
+
+enum CompilationSourceKind {
+    File(PathBuf),
+    SourceCode(String),
+}
+
+struct CompilationSource {
+    kind: CompilationSourceKind,
+    span: (Span, Span),
+}
+
+impl CompilationSource {
+    fn compile(self, mode: &compile::Mode, module_name: String) -> Result<CodeObject, Diagnostic> {
+        let compile = |source| {
+            compile::compile(source, mode, module_name).map_err(|err| {
+                Diagnostic::spans_error(self.span, format!("Compile error: {}", err))
+            })
+        };
+
+        match &self.kind {
+            CompilationSourceKind::File(rel_path) => {
+                let mut path = PathBuf::from(
+                    env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"),
+                );
+                path.push(rel_path);
+                let source = fs::read_to_string(&path).map_err(|err| {
+                    Diagnostic::spans_error(
+                        self.span,
+                        format!("Error reading file {:?}: {}", path, err),
+                    )
+                })?;
+                compile(&source)
+            }
+            CompilationSourceKind::SourceCode(code) => compile(code),
+        }
+    }
+}
+
+/// This is essentially just a comma-separated list of Meta nodes, aka the inside of a MetaList.
+struct PyCompileInput {
+    span: Span,
+    metas: Vec<Meta>,
+}
+
+impl PyCompileInput {
+    fn compile(&self) -> Result<CodeObject, Diagnostic> {
+        let mut module_name = None;
+        let mut mode = None;
+        let mut source: Option<CompilationSource> = None;
+
+        fn assert_source_empty(source: &Option<CompilationSource>) -> Result<(), Diagnostic> {
+            if let Some(source) = source {
+                Err(Diagnostic::spans_error(
+                    source.span.clone(),
+                    "Cannot have more than one source",
+                ))
+            } else {
+                Ok(())
+            }
+        }
+
+        for meta in &self.metas {
+            match meta {
+                Meta::NameValue(name_value) => {
+                    if name_value.ident == "mode" {
+                        mode = Some(match &name_value.lit {
+                            Lit::Str(s) => match s.value().as_str() {
+                                "exec" => compile::Mode::Exec,
+                                "eval" => compile::Mode::Eval,
+                                "single" => compile::Mode::Single,
+                                _ => bail_span!(s, "mode must be exec, eval, or single"),
+                            },
+                            _ => bail_span!(name_value.lit, "mode must be a string"),
+                        })
+                    } else if name_value.ident == "module_name" {
+                        module_name = Some(match &name_value.lit {
+                            Lit::Str(s) => s.value(),
+                            _ => bail_span!(name_value.lit, "module_name must be string"),
+                        })
+                    } else if name_value.ident == "source" {
+                        assert_source_empty(&source)?;
+                        let code = match &name_value.lit {
+                            Lit::Str(s) => s.value(),
+                            _ => bail_span!(name_value.lit, "source must be a string"),
+                        };
+                        source = Some(CompilationSource {
+                            kind: CompilationSourceKind::SourceCode(code),
+                            span: extract_spans(&name_value).unwrap(),
+                        });
+                    } else if name_value.ident == "file" {
+                        assert_source_empty(&source)?;
+                        let path = match &name_value.lit {
+                            Lit::Str(s) => PathBuf::from(s.value()),
+                            _ => bail_span!(name_value.lit, "source must be a string"),
+                        };
+                        source = Some(CompilationSource {
+                            kind: CompilationSourceKind::File(path),
+                            span: extract_spans(&name_value).unwrap(),
+                        });
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        source
+            .ok_or_else(|| {
+                Diagnostic::span_error(
+                    self.span.clone(),
+                    "Must have either file or source in py_compile_bytecode!()",
+                )
+            })?
+            .compile(
+                &mode.unwrap_or(compile::Mode::Exec),
+                module_name.unwrap_or_else(|| "frozen".to_string()),
+            )
+    }
+}
+
+impl Parse for PyCompileInput {
+    fn parse(input: ParseStream) -> ParseResult<Self> {
+        let span = input.cursor().span();
+        let metas = input
+            .parse_terminated::<Meta, Token![,]>(Meta::parse)?
+            .into_iter()
+            .collect();
+        Ok(PyCompileInput { span, metas })
+    }
+}
+
+pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Diagnostic> {
+    let input: PyCompileInput = parse2(input)?;
+
+    let code_obj = input.compile()?;
+
+    let bytes = bincode::serialize(&code_obj).expect("Failed to serialize");
+    let bytes = LitByteStr::new(&bytes, Span::call_site());
+
+    let output = quote! {
+        ({
+            use ::rustpython_vm::__exports::bincode;
+            bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(#bytes)
+                .expect("Deserializing CodeObject failed")
+        })
+    };
+
+    Ok(output)
+}

derive/src/error.rs

@@ -0,0 +1,167 @@
+// Taken from https://github.com/rustwasm/wasm-bindgen/blob/master/crates/backend/src/error.rs
+//
+// Copyright (c) 2014 Alex Crichton
+//
+// Permission is hereby granted, free of charge, to any
+// person obtaining a copy of this software and associated
+// documentation files (the "Software"), to deal in the
+// Software without restriction, including without
+// limitation the rights to use, copy, modify, merge,
+// publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software
+// is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice
+// shall be included in all copies or substantial portions
+// of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+#![allow(dead_code)]
+
+use proc_macro2::*;
+use quote::{ToTokens, TokenStreamExt};
+use syn::parse::Error;
+
+macro_rules! err_span {
+    ($span:expr, $($msg:tt)*) => (
+        $crate::Diagnostic::spanned_error(&$span, format!($($msg)*))
+    )
+}
+
+macro_rules! bail_span {
+    ($($t:tt)*) => (
+        return Err(err_span!($($t)*).into())
+    )
+}
+
+macro_rules! push_err_span {
+    ($diagnostics:expr, $($t:tt)*) => {
+        $diagnostics.push(err_span!($($t)*))
+    };
+}
+
+#[derive(Debug)]
+pub struct Diagnostic {
+    inner: Repr,
+}
+
+#[derive(Debug)]
+enum Repr {
+    Single {
+        text: String,
+        span: Option<(Span, Span)>,
+    },
+    SynError(Error),
+    Multi {
+        diagnostics: Vec<Diagnostic>,
+    },
+}
+
+impl Diagnostic {
+    pub fn error<T: Into<String>>(text: T) -> Diagnostic {
+        Diagnostic {
+            inner: Repr::Single {
+                text: text.into(),
+                span: None,
+            },
+        }
+    }
+
+    pub fn span_error<T: Into<String>>(span: Span, text: T) -> Diagnostic {
+        Diagnostic {
+            inner: Repr::Single {
+                text: text.into(),
+                span: Some((span, span)),
+            },
+        }
+    }
+
+    pub fn spans_error<T: Into<String>>(spans: (Span, Span), text: T) -> Diagnostic {
+        Diagnostic {
+            inner: Repr::Single {
+                text: text.into(),
+                span: Some(spans),
+            },
+        }
+    }
+
+    pub fn spanned_error<T: Into<String>>(node: &ToTokens, text: T) -> Diagnostic {
+        Diagnostic {
+            inner: Repr::Single {
+                text: text.into(),
+                span: extract_spans(node),
+            },
+        }
+    }
+
+    pub fn from_vec(diagnostics: Vec<Diagnostic>) -> Result<(), Diagnostic> {
+        if diagnostics.len() == 0 {
+            Ok(())
+        } else {
+            Err(Diagnostic {
+                inner: Repr::Multi { diagnostics },
+            })
+        }
+    }
+
+    #[allow(unconditional_recursion)]
+    pub fn panic(&self) -> ! {
+        match &self.inner {
+            Repr::Single { text, .. } => panic!("{}", text),
+            Repr::SynError(error) => panic!("{}", error),
+            Repr::Multi { diagnostics } => diagnostics[0].panic(),
+        }
+    }
+}
+
+impl From<Error> for Diagnostic {
+    fn from(err: Error) -> Diagnostic {
+        Diagnostic {
+            inner: Repr::SynError(err),
+        }
+    }
+}
+
+pub fn extract_spans(node: &dyn ToTokens) -> Option<(Span, Span)> {
+    let mut t = TokenStream::new();
+    node.to_tokens(&mut t);
+    let mut tokens = t.into_iter();
+    let start = tokens.next().map(|t| t.span());
+    let end = tokens.last().map(|t| t.span());
+    start.map(|start| (start, end.unwrap_or(start)))
+}
+
+impl ToTokens for Diagnostic {
+    fn to_tokens(&self, dst: &mut TokenStream) {
+        match &self.inner {
+            Repr::Single { text, span } => {
+                let cs2 = (Span::call_site(), Span::call_site());
+                let (start, end) = span.unwrap_or(cs2);
+                dst.append(Ident::new("compile_error", start));
+                dst.append(Punct::new('!', Spacing::Alone));
+                let mut message = TokenStream::new();
+                message.append(Literal::string(text));
+                let mut group = Group::new(Delimiter::Brace, message);
+                group.set_span(end);
+                dst.append(group);
+            }
+            Repr::Multi { diagnostics } => {
+                for diagnostic in diagnostics {
+                    diagnostic.to_tokens(dst);
+                }
+            }
+            Repr::SynError(err) => {
+                err.to_compile_error().to_tokens(dst);
+            }
+        }
+    }
+}

derive/src/from_args.rs

@@ -0,0 +1,221 @@
+use crate::Diagnostic;
+use proc_macro2::TokenStream as TokenStream2;
+use quote::quote;
+use syn::{
+    parse_quote, Attribute, Data, DeriveInput, Expr, Field, Fields, Ident, Lit, Meta, NestedMeta,
+};
+
+/// The kind of the python parameter, this corresponds to the value of Parameter.kind
+/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind)
+enum ParameterKind {
+    PositionalOnly,
+    PositionalOrKeyword,
+    KeywordOnly,
+}
+
+impl ParameterKind {
+    fn from_ident(ident: &Ident) -> Option<ParameterKind> {
+        if ident == "positional_only" {
+            Some(ParameterKind::PositionalOnly)
+        } else if ident == "positional_or_keyword" {
+            Some(ParameterKind::PositionalOrKeyword)
+        } else if ident == "keyword_only" {
+            Some(ParameterKind::KeywordOnly)
+        } else {
+            None
+        }
+    }
+}
+
+struct ArgAttribute {
+    kind: ParameterKind,
+    default: Option<Expr>,
+    optional: bool,
+}
+
+impl ArgAttribute {
+    fn from_attribute(attr: &Attribute) -> Option<Result<ArgAttribute, Diagnostic>> {
+        if !attr.path.is_ident("pyarg") {
+            return None;
+        }
+        let inner = move || match attr.parse_meta()? {
+            Meta::List(list) => {
+                let mut iter = list.nested.iter();
+                let first_arg = iter.next().ok_or_else(|| {
+                    err_span!(list, "There must be at least one argument to #[pyarg()]")
+                })?;
+                let kind = match first_arg {
+                    NestedMeta::Meta(Meta::Word(ident)) => ParameterKind::from_ident(ident),
+                    _ => None,
+                };
+                let kind = kind.ok_or_else(|| {
+                    err_span!(
+                        first_arg,
+                        "The first argument to #[pyarg()] must be the parameter type, either \
+                         'positional_only', 'positional_or_keyword', or 'keyword_only'."
+                    )
+                })?;
+
+                let mut attribute = ArgAttribute {
+                    kind,
+                    default: None,
+                    optional: false,
+                };
+
+                while let Some(arg) = iter.next() {
+                    attribute.parse_argument(arg)?;
+                }
+
+                if attribute.default.is_some() && attribute.optional {
+                    bail_span!(attr, "Can't set both a default value and optional");
+                }
+
+                Ok(attribute)
+            }
+            _ => bail_span!(attr, "pyarg must be a list, like #[pyarg(...)]"),
+        };
+        Some(inner())
+    }
+
+    fn parse_argument(&mut self, arg: &NestedMeta) -> Result<(), Diagnostic> {
+        match arg {
+            NestedMeta::Meta(Meta::Word(ident)) => {
+                if ident == "default" {
+                    if self.default.is_some() {
+                        bail_span!(ident, "Default already set");
+                    }
+                    let expr = parse_quote!(Default::default());
+                    self.default = Some(expr);
+                } else if ident == "optional" {
+                    self.optional = true;
+                } else {
+                    bail_span!(ident, "Unrecognised pyarg attribute");
+                }
+            }
+            NestedMeta::Meta(Meta::NameValue(name_value)) => {
+                if name_value.ident == "default" {
+                    if self.default.is_some() {
+                        bail_span!(name_value, "Default already set");
+                    }
+
+                    match name_value.lit {
+                        Lit::Str(ref val) => {
+                            let expr = val.parse::<Expr>().map_err(|_| {
+                                err_span!(val, "Expected a valid expression for default argument")
+                            })?;
+                            self.default = Some(expr);
+                        }
+                        _ => bail_span!(name_value, "Expected string value for default argument"),
+                    }
+                } else if name_value.ident == "optional" {
+                    match name_value.lit {
+                        Lit::Bool(ref val) => {
+                            self.optional = val.value;
+                        }
+                        _ => bail_span!(
+                            name_value.lit,
+                            "Expected boolean value for optional argument"
+                        ),
+                    }
+                } else {
+                    bail_span!(name_value, "Unrecognised pyarg attribute");
+                }
+            }
+            _ => bail_span!(arg, "Unrecognised pyarg attribute"),
+        }
+
+        Ok(())
+    }
+}
+
+fn generate_field(field: &Field) -> Result<TokenStream2, Diagnostic> {
+    let mut pyarg_attrs = field
+        .attrs
+        .iter()
+        .filter_map(ArgAttribute::from_attribute)
+        .collect::<Result<Vec<_>, _>>()?;
+    let attr = if pyarg_attrs.is_empty() {
+        ArgAttribute {
+            kind: ParameterKind::PositionalOrKeyword,
+            default: None,
+            optional: false,
+        }
+    } else if pyarg_attrs.len() == 1 {
+        pyarg_attrs.remove(0)
+    } else {
+        bail_span!(field, "Multiple pyarg attributes on field");
+    };
+
+    let name = &field.ident;
+    let middle = quote! {
+        .map(|x| ::rustpython_vm::pyobject::TryFromObject::try_from_object(vm, x)).transpose()?
+    };
+    let ending = if let Some(default) = attr.default {
+        quote! {
+            .unwrap_or_else(|| #default)
+        }
+    } else if attr.optional {
+        quote! {
+            .map(::rustpython_vm::function::OptionalArg::Present)
+            .unwrap_or(::rustpython_vm::function::OptionalArg::Missing)
+        }
+    } else {
+        let err = match attr.kind {
+            ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! {
+                ::rustpython_vm::function::ArgumentError::TooFewArgs
+            },
+            ParameterKind::KeywordOnly => quote! {
+                ::rustpython_vm::function::ArgumentError::RequiredKeywordArgument(tringify!(#name))
+            },
+        };
+        quote! {
+            .ok_or_else(|| #err)?
+        }
+    };
+
+    let file_output = match attr.kind {
+        ParameterKind::PositionalOnly => {
+            quote! {
+                #name: args.take_positional()#middle#ending,
+            }
+        }
+        ParameterKind::PositionalOrKeyword => {
+            quote! {
+                #name: args.take_positional_keyword(stringify!(#name))#middle#ending,
+            }
+        }
+        ParameterKind::KeywordOnly => {
+            quote! {
+                #name: args.take_keyword(stringify!(#name))#middle#ending,
+            }
+        }
+    };
+    Ok(file_output)
+}
+
+pub fn impl_from_args(input: DeriveInput) -> Result<TokenStream2, Diagnostic> {
+    let fields = match input.data {
+        Data::Struct(syn::DataStruct {
+            fields: Fields::Named(fields),
+            ..
+        }) => fields
+            .named
+            .iter()
+            .map(generate_field)
+            .collect::<Result<TokenStream2, Diagnostic>>()?,
+        _ => bail_span!(input, "FromArgs input must be a struct with named fields"),
+    };
+
+    let name = input.ident;
+    let output = quote! {
+        impl ::rustpython_vm::function::FromArgs for #name {
+            fn from_args(
+                vm: &::rustpython_vm::VirtualMachine,
+                args: &mut ::rustpython_vm::function::PyFuncArgs
+            ) -> Result<Self, ::rustpython_vm::function::ArgumentError> {
+                Ok(#name { #fields })
+            }
+        }
+    };
+    Ok(output)
+}

derive/src/lib.rs

@@ -0,0 +1,55 @@
+#![recursion_limit = "128"]
+
+extern crate proc_macro;
+
+#[macro_use]
+mod error;
+mod compile_bytecode;
+mod from_args;
+mod pyclass;
+
+use error::{extract_spans, Diagnostic};
+use proc_macro::TokenStream;
+use proc_macro2::TokenStream as TokenStream2;
+use proc_macro_hack::proc_macro_hack;
+use quote::ToTokens;
+use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item};
+
+fn result_to_tokens(result: Result<TokenStream2, Diagnostic>) -> TokenStream {
+    match result {
+        Ok(tokens) => tokens.into(),
+        Err(diagnostic) => diagnostic.into_token_stream().into(),
+    }
+}
+
+#[proc_macro_derive(FromArgs, attributes(pyarg))]
+pub fn derive_from_args(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as DeriveInput);
+    result_to_tokens(from_args::impl_from_args(input))
+}
+
+#[proc_macro_attribute]
+pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream {
+    let attr = parse_macro_input!(attr as AttributeArgs);
+    let item = parse_macro_input!(item as Item);
+    result_to_tokens(pyclass::impl_pyclass(attr, item))
+}
+
+#[proc_macro_attribute]
+pub fn pyimpl(attr: TokenStream, item: TokenStream) -> TokenStream {
+    let attr = parse_macro_input!(attr as AttributeArgs);
+    let item = parse_macro_input!(item as Item);
+    result_to_tokens(pyclass::impl_pyimpl(attr, item))
+}
+
+#[proc_macro_attribute]
+pub fn pystruct_sequence(attr: TokenStream, item: TokenStream) -> TokenStream {
+    let attr = parse_macro_input!(attr as AttributeArgs);
+    let item = parse_macro_input!(item as Item);
+    result_to_tokens(pyclass::impl_pystruct_sequence(attr, item))
+}
+
+#[proc_macro_hack]
+pub fn py_compile_bytecode(input: TokenStream) -> TokenStream {
+    result_to_tokens(compile_bytecode::impl_py_compile_bytecode(input.into()))
+}

derive/src/pyclass.rs

@@ -0,0 +1,463 @@
+use super::Diagnostic;
+use proc_macro2::TokenStream as TokenStream2;
+use quote::quote;
+use std::collections::HashMap;
+use syn::{
+    Attribute, AttributeArgs, Ident, ImplItem, Index, Item, Lit, Meta, MethodSig, NestedMeta,
+};
+
+enum ClassItem {
+    Method {
+        item_ident: Ident,
+        py_name: String,
+    },
+    ClassMethod {
+        item_ident: Ident,
+        py_name: String,
+    },
+    Property {
+        item_ident: Ident,
+        py_name: String,
+        setter: bool,
+    },
+}
+
+fn meta_to_vec(meta: Meta) -> Result<Vec<NestedMeta>, Meta> {
+    match meta {
+        Meta::Word(_) => Ok(Vec::new()),
+        Meta::List(list) => Ok(list.nested.into_iter().collect()),
+        Meta::NameValue(_) => Err(meta),
+    }
+}
+
+impl ClassItem {
+    fn extract_from_syn(
+        attrs: &mut Vec<Attribute>,
+        sig: &MethodSig,
+    ) -> Result<Option<ClassItem>, Diagnostic> {
+        let mut item = None;
+        let mut attr_idx = None;
+        for (i, meta) in attrs
+            .iter()
+            .filter_map(|attr| attr.parse_meta().ok())
+            .enumerate()
+        {
+            let name = meta.name();
+            if name == "pymethod" {
+                if item.is_some() {
+                    bail_span!(
+                        sig.ident,
+                        "You can only have one #[py*] attribute on an impl item"
+                    )
+                }
+                let nesteds = meta_to_vec(meta).map_err(|meta| {
+                    err_span!(
+                        meta,
+                        "#[pymethod = \"...\"] cannot be a name/value, you probably meant \
+                         #[pymethod(name = \"...\")]",
+                    )
+                })?;
+                let mut py_name = None;
+                for meta in nesteds {
+                    let meta = match meta {
+                        NestedMeta::Meta(meta) => meta,
+                        NestedMeta::Literal(_) => continue,
+                    };
+                    match meta {
+                        Meta::NameValue(name_value) => {
+                            if name_value.ident == "name" {
+                                if let Lit::Str(s) = &name_value.lit {
+                                    py_name = Some(s.value());
+                                } else {
+                                    bail_span!(
+                                        &sig.ident,
+                                        "#[pymethod(name = ...)] must be a string"
+                                    );
+                                }
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+                item = Some(ClassItem::Method {
+                    item_ident: sig.ident.clone(),
+                    py_name: py_name.unwrap_or_else(|| sig.ident.to_string()),
+                });
+                attr_idx = Some(i);
+            } else if name == "pyclassmethod" {
+                if item.is_some() {
+                    bail_span!(
+                        sig.ident,
+                        "You can only have one #[py*] attribute on an impl item"
+                    )
+                }
+                let nesteds = meta_to_vec(meta).map_err(|meta| {
+                    err_span!(
+                        meta,
+                        "#[pyclassmethod = \"...\"] cannot be a name/value, you probably meant \
+                         #[pyclassmethod(name = \"...\")]",
+                    )
+                })?;
+                let mut py_name = None;
+                for meta in nesteds {
+                    let meta = match meta {
+                        NestedMeta::Meta(meta) => meta,
+                        NestedMeta::Literal(_) => continue,
+                    };
+                    match meta {
+                        Meta::NameValue(name_value) => {
+                            if name_value.ident == "name" {
+                                if let Lit::Str(s) = &name_value.lit {
+                                    py_name = Some(s.value());
+                                } else {
+                                    bail_span!(
+                                        &sig.ident,
+                                        "#[pyclassmethod(name = ...)] must be a string"
+                                    );
+                                }
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+                item = Some(ClassItem::ClassMethod {
+                    item_ident: sig.ident.clone(),
+                    py_name: py_name.unwrap_or_else(|| sig.ident.to_string()),
+                });
+                attr_idx = Some(i);
+            } else if name == "pyproperty" {
+                if item.is_some() {
+                    bail_span!(
+                        sig.ident,
+                        "You can only have one #[py*] attribute on an impl item"
+                    )
+                }
+                let nesteds = meta_to_vec(meta).map_err(|meta| {
+                    err_span!(
+                        meta,
+                        "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \
+                         #[pyproperty(name = \"...\")]"
+                    )
+                })?;
+                let mut setter = false;
+                let mut py_name = None;
+                for meta in nesteds {
+                    let meta = match meta {
+                        NestedMeta::Meta(meta) => meta,
+                        NestedMeta::Literal(_) => continue,
+                    };
+                    match meta {
+                        Meta::NameValue(name_value) => {
+                            if name_value.ident == "name" {
+                                if let Lit::Str(s) = &name_value.lit {
+                                    py_name = Some(s.value());
+                                } else {
+                                    bail_span!(
+                                        &sig.ident,
+                                        "#[pyproperty(name = ...)] must be a string"
+                                    );
+                                }
+                            }
+                        }
+                        Meta::Word(ident) => {
+                            if ident == "setter" {
+                                setter = true;
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+                let py_name = match py_name {
+                    Some(py_name) => py_name,
+                    None => {
+                        let item_ident = sig.ident.to_string();
+                        if setter {
+                            if item_ident.starts_with("set_") {
+                                let name = &item_ident["set_".len()..];
+                                if name.is_empty() {
+                                    bail_span!(
+                                        &sig.ident,
+                                        "A #[pyproperty(setter)] fn with a set_* name must \
+                                         have something after \"set_\""
+                                    )
+                                } else {
+                                    name.to_string()
+                                }
+                            } else {
+                                bail_span!(
+                                    &sig.ident,
+                                    "A #[pyproperty(setter)] fn must either have a `name` \
+                                     parameter or a fn name along the lines of \"set_*\""
+                                )
+                            }
+                        } else {
+                            item_ident
+                        }
+                    }
+                };
+                item = Some(ClassItem::Property {
+                    py_name,
+                    item_ident: sig.ident.clone(),
+                    setter,
+                });
+                attr_idx = Some(i);
+            }
+        }
+        if let Some(attr_idx) = attr_idx {
+            attrs.remove(attr_idx);
+        }
+        Ok(item)
+    }
+}
+
+pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> Result<TokenStream2, Diagnostic> {
+    let mut imp = if let Item::Impl(imp) = item {
+        imp
+    } else {
+        return Ok(quote!(#item));
+    };
+
+    let mut diagnostics: Vec<Diagnostic> = Vec::new();
+
+    let items = imp
+        .items
+        .iter_mut()
+        .filter_map(|item| {
+            if let ImplItem::Method(meth) = item {
+                ClassItem::extract_from_syn(&mut meth.attrs, &meth.sig)
+                    .map_err(|err| diagnostics.push(err))
+                    .unwrap_or_default()
+            } else {
+                None
+            }
+        })
+        .collect::<Vec<_>>();
+    let ty = &imp.self_ty;
+    let mut properties: HashMap<&str, (Option<&Ident>, Option<&Ident>)> = HashMap::new();
+    for item in items.iter() {
+        match item {
+            ClassItem::Property {
+                ref item_ident,
+                ref py_name,
+                setter,
+            } => {
+                let entry = properties.entry(py_name).or_default();
+                let func = if *setter { &mut entry.1 } else { &mut entry.0 };
+                if func.is_some() {
+                    bail_span!(
+                        item_ident,
+                        "Multiple property accessors with name {:?}",
+                        py_name
+                    )
+                }
+                *func = Some(item_ident);
+            }
+            _ => {}
+        }
+    }
+    let methods = items.iter().filter_map(|item| match item {
+        ClassItem::Method {
+            item_ident,
+            py_name,
+        } => Some(quote! {
+            class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_ident));
+        }),
+        ClassItem::ClassMethod {
+            item_ident,
+            py_name,
+        } => Some(quote! {
+            class.set_str_attr(#py_name, ctx.new_classmethod(Self::#item_ident));
+        }),
+        _ => None,
+    });
+    let properties = properties
+        .iter()
+        .map(|(name, prop)| {
+            let getter = match prop.0 {
+                Some(getter) => getter,
+                None => {
+                    push_err_span!(
+                        diagnostics,
+                        prop.1.unwrap(),
+                        "Property {:?} is missing a getter",
+                        name
+                    );
+                    return TokenStream2::new();
+                }
+            };
+            let add_setter = prop.1.map(|setter| quote!(.add_setter(Self::#setter)));
+            quote! {
+                class.set_str_attr(
+                    #name,
+                    ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx)
+                        .add_getter(Self::#getter)
+                        #add_setter
+                        .create(),
+                );
+            }
+        })
+        .collect::<Vec<_>>();
+
+    Diagnostic::from_vec(diagnostics)?;
+
+    let ret = quote! {
+        #imp
+        impl ::rustpython_vm::pyobject::PyClassImpl for #ty {
+            fn impl_extend_class(
+                ctx: &::rustpython_vm::pyobject::PyContext,
+                class: &::rustpython_vm::obj::objtype::PyClassRef,
+            ) {
+                #(#methods)*
+                #(#properties)*
+            }
+        }
+    };
+    Ok(ret)
+}
+
+fn generate_class_def(
+    ident: &Ident,
+    attr_name: &'static str,
+    attr: AttributeArgs,
+    attrs: &Vec<Attribute>,
+) -> Result<TokenStream2, Diagnostic> {
+    let mut class_name = None;
+    for attr in attr {
+        if let NestedMeta::Meta(meta) = attr {
+            if let Meta::NameValue(name_value) = meta {
+                if name_value.ident == "name" {
+                    if let Lit::Str(s) = name_value.lit {
+                        class_name = Some(s.value());
+                    } else {
+                        bail_span!(
+                            name_value.lit,
+                            "#[{}(name = ...)] must be a string",
+                            attr_name
+                        );
+                    }
+                }
+            }
+        }
+    }
+    let class_name = class_name.unwrap_or_else(|| ident.to_string());
+
+    let mut doc: Option<Vec<String>> = None;
+    for attr in attrs.iter() {
+        if attr.path.is_ident("doc") {
+            let meta = attr.parse_meta().expect("expected doc attr to be a meta");
+            if let Meta::NameValue(name_value) = meta {
+                if let Lit::Str(s) = name_value.lit {
+                    let val = s.value().trim().to_string();
+                    match doc {
+                        Some(ref mut doc) => doc.push(val),
+                        None => doc = Some(vec![val]),
+                    }
+                }
+            }
+        }
+    }
+    let doc = match doc {
+        Some(doc) => {
+            let doc = doc.join("\n");
+            quote!(Some(#doc))
+        }
+        None => quote!(None),
+    };
+
+    let ret = quote! {
+        impl ::rustpython_vm::pyobject::PyClassDef for #ident {
+            const NAME: &'static str = #class_name;
+            const DOC: Option<&'static str> = #doc;
+        }
+    };
+    Ok(ret)
+}
+
+pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result<TokenStream2, Diagnostic> {
+    let (item, ident, attrs) = match item {
+        Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs),
+        Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs),
+        other => bail_span!(
+            other,
+            "#[pyclass] can only be on a struct or enum declaration"
+        ),
+    };
+
+    let class_def = generate_class_def(&ident, "pyclass", attr, &attrs)?;
+
+    let ret = quote! {
+        #item
+        #class_def
+    };
+    Ok(ret)
+}
+
+pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result<TokenStream2, Diagnostic> {
+    let struc = if let Item::Struct(struc) = item {
+        struc
+    } else {
+        bail_span!(
+            item,
+            "#[pystruct_sequence] can only be on a struct declaration"
+        )
+    };
+    let class_def = generate_class_def(&struc.ident, "pystruct_sequence", attr, &struc.attrs)?;
+    let mut properties = Vec::new();
+    let mut field_names = Vec::new();
+    for (i, field) in struc.fields.iter().enumerate() {
+        let idx = Index::from(i);
+        if let Some(ref field_name) = field.ident {
+            let field_name_str = field_name.to_string();
+            // TODO add doc to the generated property
+            let property = quote! {
+                class.set_str_attr(
+                    #field_name_str,
+                    ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx)
+                        .add_getter(|zelf: &::rustpython_vm::obj::objtuple::PyTuple,
+                                     _vm: &::rustpython_vm::vm::VirtualMachine|
+                                     zelf.fast_getitem(#idx))
+                        .create(),
+                );
+            };
+            properties.push(property);
+            field_names.push(quote!(#field_name));
+        } else {
+            field_names.push(quote!(#idx));
+        }
+    }
+
+    let ty = &struc.ident;
+    let ret = quote! {
+        #struc
+        #class_def
+        impl #ty {
+            fn into_struct_sequence(&self,
+                vm: &::rustpython_vm::vm::VirtualMachine,
+                cls: ::rustpython_vm::obj::objtype::PyClassRef,
+            ) -> ::rustpython_vm::pyobject::PyResult<::rustpython_vm::obj::objtuple::PyTupleRef> {
+                let tuple: ::rustpython_vm::obj::objtuple::PyTuple =
+                    vec![#(::rustpython_vm::pyobject::IntoPyObject
+                                ::into_pyobject(self.#field_names, vm)?
+                         ),*].into();
+                ::rustpython_vm::pyobject::PyValue::into_ref_with_type(tuple, vm, cls)
+            }
+        }
+        impl ::rustpython_vm::pyobject::PyClassImpl for #ty {
+            fn impl_extend_class(
+                ctx: &::rustpython_vm::pyobject::PyContext,
+                class: &::rustpython_vm::obj::objtype::PyClassRef,
+            ) {
+                #(#properties)*
+            }
+
+            fn make_class(
+                ctx: &::rustpython_vm::pyobject::PyContext
+            ) -> ::rustpython_vm::obj::objtype::PyClassRef {
+                let py_class = ctx.new_class(<Self as ::rustpython_vm::pyobject::PyClassDef>::NAME, ctx.tuple_type());
+                Self::extend_class(ctx, &py_class);
+                py_class
+            }
+        }
+    };
+    Ok(ret)
+}

docs/builtins.md

@@ -1,30 +0,0 @@
-Byterun
-
-* Builtins are exposed to frame.f_builtins
-* f_builtins is assigned during frame creation,
-    self.f_builtins = f_locals['__builtins__']
-    if hasattr(self.f_builtins, '__dict__'):
-      self.f_builtins = self.f_builtins.__dict__
-* f_locals has a __`____builtins___` field which is directly the `__builtins__` module
-
-
-Jaspy
-
-* The `module()` function creates either a NativeModule or PythonModule
-* The objects in the module are PyType.native
-* The function call is abstracted as a `call` function, which handles different
-
-* IMPORT_NAME depends on `__import__()` in builtins
-
-TODO:
-
-* Implement a new type NativeFunction
-* Wrap a function pointer in NativeFunction
-* Refactor the CALL_FUNCTION case so it can call both python function and native function
-* During frame creation, force push a native function `print` into the namespace
-* Modify LOAD_* so they can search for names in builtins
-
-* Create a module type
-* In VM initialization, load the builtins module into locals
-* During frame creation, create a field that contains the builtins dict
-

docs/compat-test.md

@@ -1,17 +0,0 @@
-https://wiki.python.org/moin/SimplePrograms
-Simple HTTP Server
-bottle.py
-http://pypi-ranking.info/alltime
- - simplejson
- - pep8
- - httplib2
- - argparse
-
-http://learning-python.com/books/lp4e-examples.html
-
-https://docs.python.org/3/tutorial/introduction.html#numbers
-
-http://doc.pypy.org/en/latest/getting-started-dev.html#running-pypy-s-unit-tests
-
-CPython Regression suite
-https://github.com/python/cpython/tree/master/Lib/test

docs/frameTODO.txt

@@ -1,13 +0,0 @@
-Create a frame class
-change the environment to locals
-turn run into run_code -> run_frame
-
-TEST
-
-Create a function class
-implement MAKE_FUNCTION
-implement CALL_FUNCTION
-implement RETURN_VALUE
-
-TEST function
-

docs/notes.md

@@ -1,13 +0,0 @@
-# TODO: other notes should be put into here
-
-# getattr()
-- Required by this opcode [LOAD_ATTR](https://docs.python.org/3/library/dis.html#opcode-LOAD_ATTR)
-- The builtin function: https://docs.python.org/3/library/functions.html?highlight=getattr#getattr
-- 
-
-# Memory management
-- https://docs.python.org/3.6/c-api/memory.html
-
-# Bootstraping
-- http://doc.pypy.org/en/latest/coding-guide.html#our-runtime-interpreter-is-rpython
-- http://www.aosabook.org/en/pypy.html

docs/reference.md

@@ -1,5 +0,0 @@
-Python bytecode: https://docs.python.org/3.4/library/dis.html 
-Python VM in python: https://github.com/nedbat/byterun
-Python VM in JS: https://github.com/koehlma/jaspy
-
-http://www.skulpt.org/

docs/sharing.md

@@ -1,43 +0,0 @@
-# Sharing plan
-
-## Topics
-
-* Python architecture
-  * Compiler -> VM
-* Flavors
-  * Cpython
-  * Pypy
-  * Byterun
-  * JSapy (?)
-* Why Rust 
-  * Security 
-  * Learning
-* Implementation plan
-  * VM first
-  * Compiler second
-
-* Tools for study
-  * dis doc
-  * byterun doc
-  * bytrun code
-  * cpython source
-
----
-* Python VM
-  * Stack machine
-
-*  Load add print
-  * dis
-  * Interpreter loop
-  * Python Types
-
-* Control flow
-  * Jump
-  * If
-  * Loop
-
----
-* Function call
-  * Frame
-* Builtins
-

docs/slides/201708_COSCUP/slide.md

@@ -1,286 +0,0 @@
-class: center, middle
-##Python Interpreter in Rust
-#### 2017/8/6, COSCUP
-#### Shing Lyu
-
-
-???
-top, middle, bottom
-left, center, right
-
----
-### About Me
-* 呂行 Shing Lyu
-* Mozilla engineer
-* Servo team
-
-![rust_and_servo](pic/rust-servo.png)
----
-### Python's architecture
-* Interpreted
-* Garbage Collected
-* Compiler => bytecode => VM
-
-![python](pic/python-logo.png)
-
----
-background-image: url('pic/ice-cream.jpg')
-class: center, middle, bleed, text-bg
-# Flavors
-
-
----
-### Python Flavors
-* CPython (THE python)
-* Jython (JVM)
-* IronPython (.NET)
-* Pypy
-* Educational
-  * Byterun
-  * Jsapy (JS)
-  * Brython (Python in browser)
-
----
-class: center, middle
-
-# Why & How?
----
-### Why rewriting Python in Rust?
-* Memory safety (?)
-* Learn about Python's internal
-* Learn to write Rust from scratch
-* FUN!
-
----
-### Implementation strategy
-* Mostly follow CPython 3.6
-* Focus on the VM first, then the compiler
-  * Use the Python built-in compiler to generate bytecode
-* Focus on learning rather than usability
-
----
-### Milestones
-* Basic arithmetics
-* Variables
-* Control flows (require JUMP)
-* Function call (require call stack)
-* Built-in functions (require native code)
-* Run Python tutorial example code <= We're here
-* Exceptions
-* GC
-* Run popular libraries
-
-
----
-class: center, middle, bleed, text-bg
-background-image: url('pic/car_cutaway.jpg')
-# Python Internals
-
----
-### How Python VM works
-* Stack machine
-  * Call stack and frames
-  * Has a NAMES list and CONSTS list
-  * Has a STACK as workspace
-* Accepts Python bytecode
-* `python -m dis source.py`
-
----
-
-### A simple Python code
-
-```
-#!/usr/bin/env python3
-print(1+1)
-```
-
-Running `python3 -m dis source.py` gives us
-
-```
-  1    0 LOAD_NAME           0 (print)
-       3 LOAD_CONST          2 (2)
-       6 CALL_FUNCTION       1 (1 positional, 0 keyword pair)
-       9 POP_TOP
-      10 LOAD_CONST          1 (None)
-      13 RETURN_VALUE
-
-```
-
----
-
-### LOAD_NAME "print"
-* NAMES = ["print"]
-* CONSTS = [None, 2]
-* STACK:
-
-```
- |                    |
- | print (native code)|
- +--------------------+
-```
----
-### LOAD_CONST 2
-* NAMES = ["print"]
-* CONSTS = [None, 2]
-* STACK:
-
-```
- |                    |
- |          2         |
- | print (native code)|
- +--------------------+
-```
-
----
-
-### CALL_FUNCTION 1
-1. `argument = stack.pop()` (argument == 2)
-2. `function = stack.top()` (function == print)
-3. call `print(2)`
-
-* NAMES = ["print"]
-* CONSTS = [None, 2]
-* STACK:
-
-```
- |                    |
- | print (native code)|
- +--------------------+
-```
----
-### POP_TOP
-* NAMES = ["print"]
-* CONSTS = [None, 2]
-* STACK:
-
-```
- |                    |
- |     (empty)        |
- +--------------------+
-```
-
----
-### LOAD_CONST None
-* NAMES = ["print"]
-* CONSTS = [None, 2]
-* STACK:
-
-```
- |                    |
- |       None         |
- +--------------------+
-```
-
----
-### RETURN_VALUE
-
-(returns top of stack == None)
-
----
-
-class: center, middle, bleed, text-bg
-background-image: url('pic/electronic_parts.jpg')
-# Technical Detail
-
----
-
-### Bytecode format
-* `dis` output format is for human reader
-* Implementing a `dis` format parser is a waste of time
-* Emit JSON bytecode using the [bytecode](https://pypi.python.org/pypi/bytecode/0.5) module
-
-```
-
-code = compile(f,...)  # Python built-in, return a Code object
-
-bytecode.Bytecode()
-   .from_code(code)
-   .to_concrete_bytecode()
-```
-* Load into Rust using `serde_json`
----
-
-### Types
-* Everything is a `PyObject` in CPython
-* We'll need that class hierarchy eventually
-* Use a Rust `enum` for now
-
-```
-pub enum NativeType{
-    NoneType,
-    Boolean(bool),
-    Int(i32),
-    Str(String),
-    Tuple(Vec<NativeType>),
-    ...
-}
-```
-
----
-
-### Testing
-* `assert` is essential to for unittests
-* `assert` raises `AssertionError`
-* Use `panic!()` before we implement exception
-
-```
-assert 1 == 1
-```
-```
-  1           0 LOAD_CONST               0 (1)
-              3 LOAD_CONST               0 (1)
-              6 COMPARE_OP               2 (==)
-              9 POP_JUMP_IF_TRUE        18
-             12 LOAD_GLOBAL              0 (AssertionError)
-             15 RAISE_VARARGS            1
-        >>   18 LOAD_CONST               1 (None)
-             21 RETURN_VALUE
-```
-
----
-### Native Function
-
-* e.g. `print()`
-
-```
-pub enum NativeType {
-    NativeFunction(fn(Vec<NativeType>) -> NativeType),
-    ...
-}
-
-match stack.pop() {
-    NativeFunction(func) => return_val = func(),
-    _ => ...
-}
-
-```
-
----
-
-### Next steps
-* Exceptions
-* Make it run a small but popular tool/library
-* Implement the parser
-* Figure out garbage collection
-* Performance benchmarking
-
----
-### Contribute
-
-## https://github.com/shinglyu/RustPython
-
-![qr_code](pic/repo_QR.png)
-
----
-class: middle, center
-
-# Thank you
-
----
-
-### References
-* [`dis` documentation](https://docs.python.org/3.4/library/dis.html)
-* [byterun](http://www.aosabook.org/en/500L/a-python-interpreter-written-in-python.html)
-* [byterun (GitHub)](https://github.com/nedbat/byterun/)
-* [cpython source code](https://github.com/python/cpython)
-

docs/slides/intro/slide.md

@@ -1,180 +0,0 @@
-class: center, middle
-##Python Interpreter in Rust
-###Introduction
-#### 2017/3/28
-#### Shing Lyu
-
-
-???
-top, middle, bottom
-left, center, right
-
----
-name: toc
-###Agenda
-1. Category 
-1. Category 
-1. Category 
-1. Category 
-1. Category 
-1. Category 
-1. Category 
-
-???
-This is a template
----
-
-### Python's architecture
-* Interpreted
-* Garbage Collected
-* Compiler => bytecode => VM
-
----
-background-image: url('pic/ice-cream.jpg')
-class: bleed
-# Flavors
-
-
----
-### Python Flavors
-* CPython (THE python)
-* Jython (JVM)
-* IronPython (.NET)
-* Pypy
-* Educational
-  * Byterun
-  * Jsapy (JS)
-  * Brython (Python in browser)
-
----
-### Why rewriting Python in Rust?
-* Memory safety
-
-* Learn about Python internal
-* Learn real world Rust
-
----
-### Implementation strategy
-* Focus on the VM first, then the compiler
-  * Reuse the Python built-in compiler to generate bytecode
-* Basic arithmetics
-* Control flows (require JUMP)
-* Function call (require call stack)
-* Built-in functions (require native code)
-* Run popular libraries
-
-
----
-### References
-* [`dis` documentation](https://docs.python.org/3.4/library/dis.html)
-* [byterun](http://www.aosabook.org/en/500L/a-python-interpreter-written-in-python.html)
-* [byterun (GitHub)](https://github.com/nedbat/byterun/)
-* [cpython source code](https://github.com/python/cpython)
-
----
-### How Python VM works
-* Stack machine
-* Accepts Python bytecode
-* `python -m dis source.py`
-
----
-
-### A simple Python code
-
-```
-#!/usr/bin/env python3
-print(1+1)
-```
-
-We run `python3 -m dis source.py`
-
----
-
-### The bytecode
-
-```
-  1    0 LOAD_NAME           0 (print)
-       3 LOAD_CONST          2 (2)
-       6 CALL_FUNCTION       1 (1 positional, 0 keyword pair)
-       9 POP_TOP
-      10 LOAD_CONST          1 (None)
-      13 RETURN_VALUE
-
-```
-
----
-
-### LOAD_NAME "print"
-* NAMES = ["print"]
-* CONSTS = [None, 2]
-* STACK:
-
-```
- |                    |
- | print (native code)|
- +--------------------+
-```
----
-### LOAD_CONST 2
-* NAMES = ["print"]
-* CONSTS = [None, 2]
-* STACK:
-
-```
- |                    |
- |          2         |
- | print (native code)|
- +--------------------+
-```
-
----
-
-### CALL_FUNCTION 1
-1. argument = stack.pop() (argument == 2)
-2. function = stack.top() (function == print)
-3. call print(2)
-
-* NAMES = ["print"]
-* CONSTS = [None, 2]
-* STACK:
-
-```
- |                    |
- | print (native code)|
- +--------------------+
-```
----
-### POP_TOP
-* NAMES = ["print"]
-* CONSTS = [None, 2]
-* STACK:
-
-```
- |                    |
- |     (empty)        |
- +--------------------+
-```
-
----
-### LOAD_CONST 1
-* NAMES = ["print"]
-* CONSTS = [None, 2]
-* STACK:
-
-```
- |                    |
- |       None         |
- +--------------------+
-```
-
----
-### RETURN_VALUE
-
-(returns top of stack == None)
-
----
-
-### Next step
-* Make it run a small but popular tool/library
-* Implement the parser
-* Performance benchmarking

docs/study.md

@@ -1,6 +0,0 @@
-Topic to study
-==================
-* How to save a Rust Iterator on the stack?
-* Study how blocks are handled in CPython.
-  * The `why` var?
-  * Why unwind everything when a `break` happened?

parser/.gitignore

@@ -1,4 +0,0 @@
-src/python.rs
-target/
-Cargo.lock
-

parser/Cargo.toml

@@ -1,17 +1,23 @@
 [package]
-name = "rustpython_parser"
-version = "0.0.1"
-authors = [ "Shing Lyu", "Windel Bouwman" ]
+name = "rustpython-parser"
+version = "0.1.0"
+description = "Parser for python code."
+authors = [ "RustPython Team" ]
 build = "build.rs"
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
 edition = "2018"
 
 [build-dependencies]
-lalrpop="0.15.1"
+lalrpop="0.16.3"
 
 [dependencies]
-lalrpop-util="0.15.1"
+lalrpop-util="0.16.3"
 log="0.4.1"
 regex="0.2.2"
 num-bigint = "0.2"
 num-traits = "0.2"
-
+unicode-xid = "0.1.0"
+unic-emoji-char = "0.9.0"
+serde = { version = "1.0.66", features = ["derive"] }
+wtf8 = "0.0.3"

parser/build.rs

@@ -1,5 +1,5 @@
 use lalrpop;
 
 fn main() {
-    lalrpop::process_root().unwrap();
+    lalrpop::process_root().unwrap()
 }

parser/src/ast.rs

@@ -4,6 +4,8 @@
 
 pub use super::lexer::Location;
 use num_bigint::BigInt;
+use serde::{Deserialize, Serialize};
+
 /*
 #[derive(Debug)]
 
@@ -16,7 +18,7 @@ pub struct Node {
 #[derive(Debug, PartialEq)]
 pub enum Top {
     Program(Program),
-    Statement(LocatedStatement),
+    Statement(Vec<LocatedStatement>),
     Expression(Expression),
 }
 
@@ -25,12 +27,18 @@ pub struct Program {
     pub statements: Vec<LocatedStatement>,
 }
 
+#[derive(Debug, PartialEq)]
+pub struct ImportSymbol {
+    pub symbol: String,
+    pub alias: Option<String>,
+}
+
 #[derive(Debug, PartialEq)]
 pub struct SingleImport {
     pub module: String,
-    // (symbol name in module, name it should be assigned locally)
-    pub symbol: Option<String>,
     pub alias: Option<String>,
+    pub symbols: Vec<ImportSymbol>,
+    pub level: usize,
 }
 
 #[derive(Debug, PartialEq)]
@@ -42,12 +50,13 @@ pub struct Located<T> {
 pub type LocatedStatement = Located<Statement>;
 
 /// Abstract syntax tree nodes for python statements.
+#[allow(clippy::large_enum_variant)]
 #[derive(Debug, PartialEq)]
 pub enum Statement {
     Break,
     Continue,
     Return {
-        value: Option<Vec<Expression>>,
+        value: Option<Box<Expression>>,
     },
     Import {
         import_parts: Vec<SingleImport>,
@@ -94,7 +103,13 @@ pub enum Statement {
     },
     For {
         target: Expression,
-        iter: Vec<Expression>,
+        iter: Expression,
+        body: Vec<LocatedStatement>,
+        orelse: Option<Vec<LocatedStatement>>,
+    },
+    AsyncFor {
+        target: Expression,
+        iter: Expression,
         body: Vec<LocatedStatement>,
         orelse: Option<Vec<LocatedStatement>>,
     },
@@ -114,12 +129,17 @@ pub enum Statement {
         bases: Vec<Expression>,
         keywords: Vec<Keyword>,
         decorator_list: Vec<Expression>,
-        // TODO: docstring: String,
     },
     FunctionDef {
         name: String,
         args: Parameters,
-        // docstring: String,
+        body: Vec<LocatedStatement>,
+        decorator_list: Vec<Expression>,
+        returns: Option<Expression>,
+    },
+    AsyncFunctionDef {
+        name: String,
+        args: Parameters,
         body: Vec<LocatedStatement>,
         decorator_list: Vec<Expression>,
         returns: Option<Expression>,
@@ -152,6 +172,9 @@ pub enum Expression {
         op: UnaryOperator,
         a: Box<Expression>,
     },
+    Await {
+        value: Box<Expression>,
+    },
     Yield {
         value: Option<Box<Expression>>,
     },
@@ -159,9 +182,8 @@ pub enum Expression {
         value: Box<Expression>,
     },
     Compare {
-        a: Box<Expression>,
-        op: Comparison,
-        b: Box<Expression>,
+        vals: Vec<Expression>,
+        ops: Vec<Comparison>,
     },
     Attribute {
         value: Box<Expression>,
@@ -182,7 +204,7 @@ pub enum Expression {
         elements: Vec<Expression>,
     },
     Dict {
-        elements: Vec<(Expression, Expression)>,
+        elements: Vec<(Option<Expression>, Expression)>,
     },
     Set {
         elements: Vec<Expression>,
@@ -230,6 +252,7 @@ impl Expression {
         match self {
             BoolOp { .. } | Binop { .. } | Unop { .. } => "operator",
             Subscript { .. } => "subscript",
+            Await { .. } => "await expression",
             Yield { .. } | YieldFrom { .. } => "yield expression",
             Compare { .. } => "comparison",
             Attribute { .. } => "attribute",
@@ -274,8 +297,8 @@ impl Expression {
 pub struct Parameters {
     pub args: Vec<Parameter>,
     pub kwonlyargs: Vec<Parameter>,
-    pub vararg: Option<Option<Parameter>>, // Optionally we handle optionally named '*args' or '*'
-    pub kwarg: Option<Option<Parameter>>,
+    pub vararg: Varargs, // Optionally we handle optionally named '*args' or '*'
+    pub kwarg: Varargs,
     pub defaults: Vec<Expression>,
     pub kw_defaults: Vec<Option<Expression>>,
 }
@@ -367,7 +390,7 @@ pub enum Number {
 }
 
 /// Transforms a value prior to formatting it.
-#[derive(Copy, Clone, Debug, PartialEq)]
+#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub enum ConversionFlag {
     /// Converts by calling `str(<value>)`.
     Str,
@@ -391,3 +414,28 @@ pub enum StringGroup {
         values: Vec<StringGroup>,
     },
 }
+
+#[derive(Debug, PartialEq)]
+pub enum Varargs {
+    None,
+    Unnamed,
+    Named(Parameter),
+}
+
+impl Default for Varargs {
+    fn default() -> Varargs {
+        Varargs::None
+    }
+}
+
+impl From<Option<Option<Parameter>>> for Varargs {
+    fn from(opt: Option<Option<Parameter>>) -> Varargs {
+        match opt {
+            Some(inner_opt) => match inner_opt {
+                Some(param) => Varargs::Named(param),
+                None => Varargs::Unnamed,
+            },
+            None => Varargs::None,
+        }
+    }
+}

parser/src/error.rs

@@ -3,28 +3,31 @@
 extern crate lalrpop_util;
 use self::lalrpop_util::ParseError as InnerError;
 
-use crate::lexer::{LexicalError, Location};
+use crate::lexer::{LexicalError, LexicalErrorType, Location};
 use crate::token::Tok;
 
 use std::error::Error;
 use std::fmt;
 
-// A token of type `Tok` was observed, with a span given by the two Location values
-type TokSpan = (Location, Tok, Location);
-
 /// Represents an error during parsing
 #[derive(Debug, PartialEq)]
-pub enum ParseError {
+pub struct ParseError {
+    pub error: ParseErrorType,
+    pub location: Location,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum ParseErrorType {
     /// Parser encountered an unexpected end of input
-    EOF(Option<Location>),
+    EOF,
     /// Parser encountered an extra token
-    ExtraToken(TokSpan),
+    ExtraToken(Tok),
     /// Parser encountered an invalid token
-    InvalidToken(Location),
+    InvalidToken,
     /// Parser encountered an unexpected token
-    UnrecognizedToken(TokSpan, Vec<String>),
+    UnrecognizedToken(Tok, Vec<String>),
     /// Maps to `User` type from `lalrpop-util`
-    Other,
+    Lexical(LexicalErrorType),
 }
 
 /// Convert `lalrpop_util::ParseError` to our internal type
@@ -32,15 +35,29 @@ impl From<InnerError<Location, Tok, LexicalError>> for ParseError {
     fn from(err: InnerError<Location, Tok, LexicalError>) -> Self {
         match err {
             // TODO: Are there cases where this isn't an EOF?
-            InnerError::InvalidToken { location } => ParseError::EOF(Some(location)),
-            InnerError::ExtraToken { token } => ParseError::ExtraToken(token),
-            // Inner field is a unit-like enum `LexicalError::StringError` with no useful info
-            InnerError::User { .. } => ParseError::Other,
+            InnerError::InvalidToken { location } => ParseError {
+                error: ParseErrorType::EOF,
+                location,
+            },
+            InnerError::ExtraToken { token } => ParseError {
+                error: ParseErrorType::ExtraToken(token.1),
+                location: token.0,
+            },
+            InnerError::User { error } => ParseError {
+                error: ParseErrorType::Lexical(error.error),
+                location: error.location,
+            },
             InnerError::UnrecognizedToken { token, expected } => {
                 match token {
-                    Some(tok) => ParseError::UnrecognizedToken(tok, expected),
+                    Some(tok) => ParseError {
+                        error: ParseErrorType::UnrecognizedToken(tok.1, expected),
+                        location: tok.0,
+                    },
                     // EOF was observed when it was unexpected
-                    None => ParseError::EOF(None),
+                    None => ParseError {
+                        error: ParseErrorType::EOF,
+                        location: Default::default(),
+                    },
                 }
             }
         }
@@ -48,26 +65,21 @@ impl From<InnerError<Location, Tok, LexicalError>> for ParseError {
 }
 
 impl fmt::Display for ParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{} at {}", self.error, self.location)
+    }
+}
+
+impl fmt::Display for ParseErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match *self {
-            ParseError::EOF(ref location) => {
-                if let Some(l) = location {
-                    write!(f, "Got unexpected EOF at: {:?}", l)
-                } else {
-                    write!(f, "Got unexpected EOF")
-                }
+            ParseErrorType::EOF => write!(f, "Got unexpected EOF"),
+            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {:?}", tok),
+            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
+            ParseErrorType::UnrecognizedToken(ref tok, _) => {
+                write!(f, "Got unexpected token: {:?}", tok)
             }
-            ParseError::ExtraToken(ref t_span) => {
-                write!(f, "Got extraneous token: {:?} at: {:?}", t_span.1, t_span.0)
-            }
-            ParseError::InvalidToken(ref location) => {
-                write!(f, "Got invalid token at: {:?}", location)
-            }
-            ParseError::UnrecognizedToken(ref t_span, _) => {
-                write!(f, "Got unexpected token: {:?} at {:?}", t_span.1, t_span.0)
-            }
-            // This is user defined, it probably means a more useful error should have been given upstream.
-            ParseError::Other => write!(f, "Got unsupported token(s)"),
+            ParseErrorType::Lexical(ref error) => write!(f, "{}", error),
         }
     }
 }

parser/src/fstring.rs

@@ -5,7 +5,7 @@ use std::str;
 use lalrpop_util::ParseError as LalrpopError;
 
 use crate::ast::{ConversionFlag, StringGroup};
-use crate::lexer::{LexicalError, Location, Tok};
+use crate::lexer::{LexicalError, LexicalErrorType, Location, Tok};
 use crate::parser::parse_expression;
 
 use self::FStringError::*;
@@ -25,7 +25,10 @@ pub enum FStringError {
 impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
     fn from(_err: FStringError) -> Self {
         lalrpop_util::ParseError::User {
-            error: LexicalError::StringError,
+            error: LexicalError {
+                error: LexicalErrorType::StringError,
+                location: Default::default(),
+            },
         }
     }
 }

parser/src/lib.rs

@@ -1,11 +1,14 @@
 #[macro_use]
 extern crate log;
+use lalrpop_util::lalrpop_mod;
 
 pub mod ast;
 pub mod error;
 mod fstring;
 pub mod lexer;
 pub mod parser;
-#[cfg_attr(rustfmt, rustfmt_skip)]
-mod python;
+lalrpop_mod!(
+    #[allow(clippy::all)]
+    python
+);
 pub mod token;

parser/src/parser.rs

@@ -35,7 +35,7 @@ pub fn parse_program(source: &str) -> Result<ast::Program, ParseError> {
     do_lalr_parsing!(source, Program, StartProgram)
 }
 
-pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, ParseError> {
+pub fn parse_statement(source: &str) -> Result<Vec<ast::LocatedStatement>, ParseError> {
     do_lalr_parsing!(source, Statement, StartStatement)
 }
 
@@ -44,7 +44,6 @@ pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, ParseError
 /// # Example
 /// ```
 /// extern crate num_bigint;
-/// extern crate rustpython_parser;
 /// use num_bigint::BigInt;
 /// use rustpython_parser::{parser, ast};
 /// let expr = parser::parse_expression("1+2").unwrap();
@@ -180,7 +179,7 @@ mod tests {
         let parse_ast = parse_statement(&source).unwrap();
         assert_eq!(
             parse_ast,
-            ast::LocatedStatement {
+            vec![ast::LocatedStatement {
                 location: ast::Location::new(1, 1),
                 node: ast::Statement::If {
                     test: ast::Expression::Number {
@@ -229,7 +228,7 @@ mod tests {
                         }
                     },]),
                 }
-            }
+            }]
         );
     }
 
@@ -239,7 +238,7 @@ mod tests {
         let parse_ast = parse_statement(&source);
         assert_eq!(
             parse_ast,
-            Ok(ast::LocatedStatement {
+            Ok(vec![ast::LocatedStatement {
                 location: ast::Location::new(1, 1),
                 node: ast::Statement::Expression {
                     expression: ast::Expression::Lambda {
@@ -255,8 +254,8 @@ mod tests {
                                 }
                             ],
                             kwonlyargs: vec![],
-                            vararg: None,
-                            kwarg: None,
+                            vararg: ast::Varargs::None,
+                            kwarg: ast::Varargs::None,
                             defaults: vec![],
                             kw_defaults: vec![],
                         },
@@ -271,7 +270,7 @@ mod tests {
                         })
                     }
                 }
-            })
+            }])
         )
     }
 
@@ -281,7 +280,7 @@ mod tests {
 
         assert_eq!(
             parse_statement(&source),
-            Ok(ast::LocatedStatement {
+            Ok(vec![ast::LocatedStatement {
                 location: ast::Location::new(1, 1),
                 node: ast::Statement::Assign {
                     targets: vec![ast::Expression::Tuple {
@@ -309,7 +308,7 @@ mod tests {
                         ]
                     }
                 }
-            })
+            }])
         )
     }
 
@@ -320,7 +319,7 @@ mod tests {
         );
         assert_eq!(
             parse_statement(&source),
-            Ok(ast::LocatedStatement {
+            Ok(vec![ast::LocatedStatement {
                 location: ast::Location::new(1, 1),
                 node: ast::Statement::ClassDef {
                     name: String::from("Foo"),
@@ -344,8 +343,8 @@ mod tests {
                                         annotation: None,
                                     }],
                                     kwonlyargs: vec![],
-                                    vararg: None,
-                                    kwarg: None,
+                                    vararg: ast::Varargs::None,
+                                    kwarg: ast::Varargs::None,
                                     defaults: vec![],
                                     kw_defaults: vec![],
                                 },
@@ -373,8 +372,8 @@ mod tests {
                                         }
                                     ],
                                     kwonlyargs: vec![],
-                                    vararg: None,
-                                    kwarg: None,
+                                    vararg: ast::Varargs::None,
+                                    kwarg: ast::Varargs::None,
                                     defaults: vec![ast::Expression::String {
                                         value: ast::StringGroup::Constant {
                                             value: "default".to_string()
@@ -393,7 +392,7 @@ mod tests {
                     ],
                     decorator_list: vec![],
                 }
-            })
+            }])
         )
     }
 
@@ -460,26 +459,30 @@ mod tests {
                         },
                         ifs: vec![
                             ast::Expression::Compare {
-                                a: Box::new(ast::Expression::Identifier {
-                                    name: "a".to_string()
-                                }),
-                                op: ast::Comparison::Less,
-                                b: Box::new(ast::Expression::Number {
-                                    value: ast::Number::Integer {
-                                        value: BigInt::from(5)
+                                vals: vec![
+                                    ast::Expression::Identifier {
+                                        name: "a".to_string()
+                                    },
+                                    ast::Expression::Number {
+                                        value: ast::Number::Integer {
+                                            value: BigInt::from(5)
+                                        }
                                     }
-                                }),
+                                ],
+                                ops: vec![ast::Comparison::Less],
                             },
                             ast::Expression::Compare {
-                                a: Box::new(ast::Expression::Identifier {
-                                    name: "a".to_string()
-                                }),
-                                op: ast::Comparison::Greater,
-                                b: Box::new(ast::Expression::Number {
-                                    value: ast::Number::Integer {
-                                        value: BigInt::from(10)
+                                vals: vec![
+                                    ast::Expression::Identifier {
+                                        name: "a".to_string()
+                                    },
+                                    ast::Expression::Number {
+                                        value: ast::Number::Integer {
+                                            value: BigInt::from(10)
+                                        }
                                     }
-                                }),
+                                ],
+                                ops: vec![ast::Comparison::Greater],
                             },
                         ],
                     }

parser/src/python.lalrpop

@@ -2,7 +2,6 @@
 // See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4
 // See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions
 // See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword
-#![allow(unknown_lints,clippy)]
 
 use std::iter::FromIterator;
 
@@ -25,29 +24,32 @@ pub Top: ast::Top = {
 
 Program: ast::Program = {
     <lines:FileLine*> => ast::Program {
-        statements: Vec::from_iter(lines.into_iter().filter_map(|e| e))
+        statements: Vec::from_iter(lines.into_iter().flatten())
     },
 };
 
 // A file line either has a declaration, or an empty newline:
-FileLine: Option<ast::LocatedStatement> = {
-    <s:Statement> => Some(s),
-    "\n" => None,
+FileLine: Vec<ast::LocatedStatement> = {
+    Statement,
+    "\n" => vec![],
 };
 
 Suite: Vec<ast::LocatedStatement> = {
-    <s:SimpleStatement> => vec![s],
-    "\n" indent <s:Statement+> dedent => s,
-};
-
-Statement: ast::LocatedStatement = {
     SimpleStatement,
-    CompoundStatement,
+    "\n" indent <s:Statement+> dedent => s.into_iter().flatten().collect(),
 };
 
-SimpleStatement: ast::LocatedStatement = {
-    <s:SmallStatement> "\n" => s,
-    <s:SmallStatement> ";" => s,
+Statement: Vec<ast::LocatedStatement> = {
+    SimpleStatement,
+    <s:CompoundStatement> => vec![s],
+};
+
+SimpleStatement: Vec<ast::LocatedStatement> = {
+    <s1:SmallStatement> <s2:(";" SmallStatement)*> ";"? "\n" => {
+        let mut statements = vec![s1];
+        statements.extend(s2.into_iter().map(|e| e.1));
+        statements
+    }
 };
 
 SmallStatement: ast::LocatedStatement = {
@@ -103,9 +105,7 @@ ExpressionStatement: ast::LocatedStatement = {
           }
         }
     },
-    <loc:@L> <expr:TestOrStarExprList> <op:AugAssign> <e2:TestList> => {
-        // TODO: this works in most cases:
-        let rhs = e2.into_iter().next().unwrap();
+    <loc:@L> <expr:TestOrStarExprList> <op:AugAssign> <rhs:TestList> => {
         ast::LocatedStatement {
             location: loc,
             node: ast::Statement::AugAssign {
@@ -118,31 +118,17 @@ ExpressionStatement: ast::LocatedStatement = {
 };
 
 AssignSuffix: ast::Expression = {
-    "=" <e:TestList> => {
-        if e.len() > 1 {
-            ast::Expression::Tuple {
-                elements: e
-            }
-        } else {
-            e.into_iter().next().unwrap()
-        }
-    },
+    "=" <e:TestList> => e,
     "=" <e:YieldExpr> => e,
 };
 
 TestOrStarExprList: ast::Expression = {
-    <e:TestOrStarExpr> <e2:("," TestOrStarExpr)*> <comma:","?> => {
-        let mut res = vec![e];
-        res.extend(e2.into_iter().map(|x| x.1));
-
-        // First build tuple from first item:
-        let expr = if (res.len() > 1) || comma.is_some() {
-            ast::Expression::Tuple { elements: res }
+    <elements:OneOrMore<TestOrStarExpr>> <comma:","?> => {
+        if elements.len() == 1 && comma.is_none() {
+            elements.into_iter().next().unwrap()
         } else {
-            res.into_iter().next().unwrap()
-        };
-
-        expr
+            ast::Expression::Tuple { elements }
+        }
     }
 };
 
@@ -183,7 +169,7 @@ FlowStatement: ast::LocatedStatement = {
     <loc:@L> "return" <t:TestList?> => {
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::Return { value: t },
+            node: ast::Statement::Return { value: t.map(Box::new) },
         }
     },
     <loc:@L> <y:YieldExpr> => {
@@ -220,8 +206,9 @@ ImportStatement: ast::LocatedStatement = {
             .map(|(n, a)|
                 ast::SingleImport {
                     module: n.to_string(),
-                    symbol: None,
-                    alias: a.clone()
+                    symbols: vec![],
+                    alias: a.clone(),
+                    level: 0,
                 })
             .collect()
       },
@@ -231,35 +218,30 @@ ImportStatement: ast::LocatedStatement = {
     ast::LocatedStatement {
       location: loc,
       node: ast::Statement::Import {
-        import_parts: i
-            .iter()
-            .map(|(i, a)|
-                ast::SingleImport {
-                    module: n.to_string(),
-                    symbol: Some(i.to_string()),
-                    alias: a.clone()
-                })
-            .collect()
+        import_parts: vec![
+            ast::SingleImport {
+                module: n.0.to_string(),
+                symbols: i.iter()
+                    .map(|(i, a)|
+                        ast::ImportSymbol {
+                            symbol: i.to_string(),
+                            alias: a.clone(),
+                        })
+                    .collect(),
+                alias: None,
+                level: n.1
+        }]
       },
     }
   },
 };
 
-ImportFromLocation: String = {
+ImportFromLocation: (String, usize) = {
     <dots: "."*> <name:DottedName> => {
-       let mut r = "".to_string();
-       for _dot in dots {
-         r.push_str(".");
-       }
-      r.push_str(&name);
-      r
+       (name, dots.len())
     },
     <dots: "."+> => {
-       let mut r = "".to_string();
-       for _dot in dots {
-         r.push_str(".");
-       }
-      r
+       ("".to_string(), dots.len())
     },
 };
 
@@ -310,15 +292,11 @@ NonlocalStatement: ast::LocatedStatement = {
 };
 
 AssertStatement: ast::LocatedStatement = {
-    <loc:@L> "assert" <t:Test> <m: ("," Test)?> => {
+    <loc:@L> "assert" <test:Test> <msg: ("," Test)?> => {
         ast::LocatedStatement {
           location: loc,
           node: ast::Statement::Assert {
-            test: t,
-            msg: match m {
-              Some(e) => Some(e.1),
-              None => None,
-            }
+            test, msg: msg.map(|e| e.1)
           }
         }
     },
@@ -335,12 +313,9 @@ CompoundStatement: ast::LocatedStatement = {
 };
 
 IfStatement: ast::LocatedStatement = {
-    <loc:@L> "if" <t:Test> ":" <s1:Suite> <s2:(@L "elif" Test ":" Suite)*> <s3:("else" ":" Suite)?> => {
+    <loc:@L> "if" <test:Test> ":" <s1:Suite> <s2:(@L "elif" Test ":" Suite)*> <s3:("else" ":" Suite)?> => {
         // Determine last else:
-        let mut last = match s3 {
-            Some(s) => Some(s.2),
-            None => None,
-        };
+        let mut last = s3.map(|s| s.2);
 
         // handle elif:
         for i in s2.into_iter().rev() {
@@ -353,35 +328,30 @@ IfStatement: ast::LocatedStatement = {
 
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::If { test: t, body: s1, orelse: last }
+            node: ast::Statement::If { test, body: s1, orelse: last }
         }
     },
 };
 
 WhileStatement: ast::LocatedStatement = {
-    <loc:@L> "while" <e:Test> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
-        let or_else = match s2 {
-            Some(s) => Some(s.2),
-            None => None,
-        };
+    <loc:@L> "while" <test:Test> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+        let or_else = s2.map(|s| s.2);
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::While { test: e, body: s, orelse: or_else },
+            node: ast::Statement::While { test, body, orelse: or_else },
         }
     },
 };
 
 ForStatement: ast::LocatedStatement = {
-    <loc:@L> "for" <e:ExpressionList> "in" <t:TestList> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
-        let or_else = match s2 {
-            Some(s) => Some(s.2),
-            None => None,
-        };
+    <loc:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+        let orelse = s2.map(|s| s.2);
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::For {
-                target: e,
-                iter: t, body: s, orelse: or_else
+            node: if is_async.is_some() {
+                ast::Statement::AsyncFor { target, iter, body, orelse }
+            } else {
+                ast::Statement::For { target, iter, body, orelse }
             },
         }
     },
@@ -389,14 +359,8 @@ ForStatement: ast::LocatedStatement = {
 
 TryStatement: ast::LocatedStatement = {
     <loc:@L> "try" ":" <body:Suite> <handlers:ExceptClause*> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> => {
-        let or_else = match else_suite {
-            Some(s) => Some(s.2),
-            None => None,
-        };
-        let finalbody = match finally {
-            Some(s) => Some(s.2),
-            None => None,
-        };
+        let or_else = else_suite.map(|s| s.2);
+        let finalbody = finally.map(|s| s.2);
         ast::LocatedStatement {
             location: loc,
             node: ast::Statement::Try {
@@ -437,62 +401,59 @@ WithStatement: ast::LocatedStatement = {
 
 WithItem: ast::WithItem = {
     <t:Test> <n:("as" Expression)?> => {
-        let optional_vars = match n {
-            Some(val) => Some(val.1),
-            None => None,
-        };
+        let optional_vars = n.map(|val| val.1);
         ast::WithItem { context_expr: t, optional_vars }
     },
 };
 
 FuncDef: ast::LocatedStatement = {
-    <d:Decorator*> <loc:@L> "def" <i:Identifier> <a:Parameters> <r:("->" Test)?> ":" <s:Suite> => {
+    <d:Decorator*> <loc:@L> <is_async:"async"?> "def" <i:Identifier> <a:Parameters> <r:("->" Test)?> ":" <s:Suite> => {
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::FunctionDef {
-                name: i,
-                args: a,
-                body: s,
-                decorator_list: d,
-                returns: r.map(|x| x.1),
-            }
+            node: if is_async.is_some() {
+                    ast::Statement::AsyncFunctionDef {
+                        name: i,
+                        args: a,
+                        body: s,
+                        decorator_list: d,
+                        returns: r.map(|x| x.1),
+                    }
+                } else {
+                    ast::Statement::FunctionDef {
+                        name: i,
+                        args: a,
+                        body: s,
+                        decorator_list: d,
+                        returns: r.map(|x| x.1),
+                    }
+                }
         }
     },
 };
 
 Parameters: ast::Parameters = {
-    "(" <a: (TypedArgsList<TypedParameter>)?> ")" => {
-        match a {
-            Some(a) => a,
-            None => Default::default(),
-        }
-    },
+    "(" <a: (ParameterList<TypedParameter>)?> ")" => a.unwrap_or_else(Default::default),
 };
 
-// parameters are (String, None), kwargs are (String, Some(Test)) where Test is
-// the default
 // Note that this is a macro which is used once for function defs, and
 // once for lambda defs.
-TypedArgsList<ArgType>: ast::Parameters = {
-    <param1:TypedParameters<ArgType>> <args2:("," ParameterListStarArgs<ArgType>)?> => {
+ParameterList<ArgType>: ast::Parameters = {
+    <param1:ParameterDefs<ArgType>> <args2:("," ParameterListStarArgs<ArgType>)?> ","? => {
         let (names, default_elements) = param1;
 
         // Now gather rest of parameters:
-        let (vararg, kwonlyargs, kw_defaults, kwarg) = match args2 {
-            Some((_, x)) => x,
-            None => (None, vec![], vec![], None),
-        };
+        let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1);
 
         ast::Parameters {
             args: names,
             kwonlyargs: kwonlyargs,
-            vararg: vararg,
-            kwarg: kwarg,
+            vararg: vararg.into(),
+            kwarg: kwarg.into(),
             defaults: default_elements,
             kw_defaults: kw_defaults,
         }
     },
-    <param1:TypedParameters<ArgType>> <kw:("," KwargParameter<ArgType>)> => {
+    <param1:ParameterDefs<ArgType>> <kw:("," KwargParameter<ArgType>)> ","? => {
         let (names, default_elements) = param1;
 
         // Now gather rest of parameters:
@@ -504,29 +465,29 @@ TypedArgsList<ArgType>: ast::Parameters = {
         ast::Parameters {
             args: names,
             kwonlyargs: kwonlyargs,
-            vararg: vararg,
-            kwarg: kwarg,
+            vararg: vararg.into(),
+            kwarg: kwarg.into(),
             defaults: default_elements,
             kw_defaults: kw_defaults,
         }
     },
-    <params:ParameterListStarArgs<ArgType>> => {
+    <params:ParameterListStarArgs<ArgType>> ","? => {
         let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
         ast::Parameters {
             args: vec![],
             kwonlyargs: kwonlyargs,
-            vararg: vararg,
-            kwarg: kwarg,
+            vararg: vararg.into(),
+            kwarg: kwarg.into(),
             defaults: vec![],
             kw_defaults: kw_defaults,
         }
     },
-    <kw:KwargParameter<ArgType>> => {
+    <kw:KwargParameter<ArgType>> ","? => {
         ast::Parameters {
             args: vec![],
             kwonlyargs: vec![],
-            vararg: None,
-            kwarg: Some(kw),
+            vararg: ast::Varargs::None,
+            kwarg: Some(kw).into(),
             defaults: vec![],
             kw_defaults: vec![],
         }
@@ -535,8 +496,8 @@ TypedArgsList<ArgType>: ast::Parameters = {
 
 // Use inline here to make sure the "," is not creating an ambiguity.
 #[inline]
-TypedParameters<ArgType>: (Vec<ast::Parameter>, Vec<ast::Expression>) = {
-    <param1:TypedParameterDef<ArgType>> <param2:("," TypedParameterDef<ArgType>)*> => {
+ParameterDefs<ArgType>: (Vec<ast::Parameter>, Vec<ast::Expression>) = {
+    <param1:ParameterDef<ArgType>> <param2:("," ParameterDef<ArgType>)*> => {
         // Combine first parameters:
         let mut args = vec![param1];
         args.extend(param2.into_iter().map(|x| x.1));
@@ -564,7 +525,7 @@ TypedParameters<ArgType>: (Vec<ast::Parameter>, Vec<ast::Expression>) = {
     }
 };
 
-TypedParameterDef<ArgType>: (ast::Parameter, Option<ast::Expression>) = {
+ParameterDef<ArgType>: (ast::Parameter, Option<ast::Expression>) = {
     <i:ArgType> => (i, None),
     <i:ArgType> "=" <e:Test> => (i, Some(e)),
 };
@@ -580,8 +541,11 @@ TypedParameter: ast::Parameter = {
     },
 };
 
+// Use inline here to make sure the "," is not creating an ambiguity.
+// TODO: figure out another grammar that makes this inline no longer required.
+#[inline]
 ParameterListStarArgs<ArgType>: (Option<Option<ast::Parameter>>, Vec<ast::Parameter>, Vec<Option<ast::Expression>>, Option<Option<ast::Parameter>>) = {
-    "*" <va:ArgType?> <kw:("," TypedParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> => {
+    "*" <va:ArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> => {
         // Extract keyword arguments:
         let mut kwonlyargs = vec![];
         let mut kw_defaults = vec![];
@@ -590,10 +554,7 @@ ParameterListStarArgs<ArgType>: (Option<Option<ast::Parameter>>, Vec<ast::Parame
             kw_defaults.push(value);
         }
 
-        let kwarg = match kwarg {
-            Some((_, name)) => Some(name),
-            None => None,
-        };
+        let kwarg = kwarg.map(|n| n.1);
 
         (Some(va), kwonlyargs, kw_defaults, kwarg)
     }
@@ -649,66 +610,59 @@ Decorator: ast::Expression = {
 };
 
 YieldExpr: ast::Expression = {
-    "yield" <ex:TestList?> => {
-        ast::Expression::Yield {
-            value: ex.map(|expr| Box::new(
-                if expr.len() > 1 {
-                    ast::Expression::Tuple { elements: expr }
-                } else {
-                    expr.into_iter().next().unwrap()
-                })
-            )
-        }
-    },
-    "yield" "from" <e:Test> => {
-        ast::Expression::YieldFrom {
-            value: Box::new(e),
-        }
-    },
+    "yield" <value:TestList?> => ast::Expression::Yield { value: value.map(Box::new) },
+    "yield" "from" <e:Test> => ast::Expression::YieldFrom { value: Box::new(e) },
 };
 
 Test: ast::Expression = {
-    <e:OrTest> <c: ("if" OrTest "else" Test)?> => {
-        match c {
-            Some(c) => {
-                ast::Expression::IfExpression {
-                    test: Box::new(c.1),
-                    body: Box::new(e),
-                    orelse: Box::new(c.3),
-                }
-            },
-            None => e,
+    <expr:OrTest> <condition: ("if" OrTest "else" Test)?> => {
+        if let Some(c) = condition {
+            ast::Expression::IfExpression {
+                test: Box::new(c.1),
+                body: Box::new(expr),
+                orelse: Box::new(c.3),
+            }
+        } else {
+            expr
         }
     },
-    <e:LambdaDef> => e,
+    LambdaDef,
 };
 
 LambdaDef: ast::Expression = {
-    "lambda" <p:TypedArgsList<UntypedParameter>?> ":" <b:Test> =>
+    "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test> =>
         ast::Expression::Lambda {
             args: p.unwrap_or(Default::default()),
-            body:Box::new(b)
+            body: Box::new(body)
        }
 }
 
 OrTest: ast::Expression = {
-    <e:AndTest> => e,
+    AndTest,
     <e1:OrTest> "or" <e2:AndTest> => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) },
 };
 
 AndTest: ast::Expression = {
-    <e:NotTest> => e,
+    NotTest,
     <e1:AndTest> "and" <e2:NotTest> => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) },
 };
 
 NotTest: ast::Expression = {
     "not" <e:NotTest> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Not },
-    <e:Comparison> => e,
+    Comparison,
 };
 
 Comparison: ast::Expression = {
-    <e1:Comparison> <op:CompOp> <e2:Expression> => ast::Expression::Compare { a: Box::new(e1), op: op, b: Box::new(e2) },
-    <e:Expression> => e,
+    <e:Expression> <comparisons:(CompOp Expression)+> => {
+        let mut vals = vec![e];
+        let mut ops = vec![];
+        for x in comparisons {
+            ops.push(x.0);
+            vals.push(x.1);
+        }
+        ast::Expression::Compare { vals, ops }
+    },
+    Expression,
 };
 
 CompOp: ast::Comparison = {
@@ -726,7 +680,7 @@ CompOp: ast::Comparison = {
 
 Expression: ast::Expression = {
     <e1:Expression> "|" <e2:XorExpression> => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) },
-    <e:XorExpression> => e,
+    XorExpression,
 };
 
 XorExpression: ast::Expression = {
@@ -776,7 +730,7 @@ Factor: ast::Expression = {
     "+" <e:Factor> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Pos },
     "-" <e:Factor> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Neg },
     "~" <e:Factor> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Inv },
-    <e:Power> => e,
+    Power,
 };
 
 Power: ast::Expression = {
@@ -789,14 +743,38 @@ Power: ast::Expression = {
 };
 
 AtomExpr: ast::Expression = {
-    <e:Atom> => e,
-    <f:AtomExpr>  "(" <a:ArgumentList> ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 },
-    <e:AtomExpr>  "[" <s:Subscript> "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) },
-    <e:AtomExpr> "." <n:Identifier> => ast::Expression::Attribute { value: Box::new(e), name: n },
+    <is_await:"await"?> <atom:AtomExpr2> => {
+        if is_await.is_some() {
+            ast::Expression::Await { value: Box::new(atom) }
+        } else {
+            atom
+        }
+    }
+}
+
+AtomExpr2: ast::Expression = {
+    Atom,
+    <f:AtomExpr2> "(" <a:ArgumentList> ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 },
+    <e:AtomExpr2> "[" <s:SubscriptList> "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) },
+    <e:AtomExpr2> "." <n:Identifier> => ast::Expression::Attribute { value: Box::new(e), name: n },
+};
+
+SubscriptList: ast::Expression = {
+    <s1:Subscript> <s2:("," Subscript)*> ","? => {
+        if s2.is_empty() {
+            s1
+        } else {
+            let mut dims = vec![s1];
+            for x in s2 {
+                dims.push(x.1)
+            }
+            ast::Expression::Tuple { elements: dims }
+        }
+    }
 };
 
 Subscript: ast::Expression = {
-    <e:Test> => e,
+    Test,
     <e1:Test?> ":" <e2:Test?> <e3:SliceOp?>  => {
         let s1 = e1.unwrap_or(ast::Expression::None);
         let s2 = e2.unwrap_or(ast::Expression::None);
@@ -810,30 +788,17 @@ SliceOp: ast::Expression = {
 }
 
 Atom: ast::Expression = {
-    <s:StringGroup> => ast::Expression::String { value: s },
-    <b:Bytes> => ast::Expression::Bytes { value: b },
-    <n:Number> => ast::Expression::Number { value: n },
-    <i:Identifier> => ast::Expression::Identifier { name: i },
+    <value:StringGroup> => ast::Expression::String { value },
+    <value:Bytes> => ast::Expression::Bytes { value },
+    <value:Number> => ast::Expression::Number { value },
+    <name:Identifier> => ast::Expression::Identifier { name },
     "[" <e:TestListComp?> "]" => {
         let elements = e.unwrap_or(Vec::new());
         ast::Expression::List { elements }
     },
-    "[" <e:TestListComp2> "]" => {
-        // List comprehension:
-        e
-    },
-    "(" <e:TestList?> <trailing_comma:","?> ")" => {
-         match e {
-             None => ast::Expression::Tuple { elements: Vec::new() },
-             Some(elements) => {
-                 if elements.len() == 1 && trailing_comma.is_none() {
-                   // This is "(e)", which is equivalent to "e"
-                   elements.into_iter().next().unwrap()
-                 } else {
-                   ast::Expression::Tuple { elements }
-                 }
-             }
-         }
+    "[" <e:TestListComp2> "]" => e,
+    "(" <elements:TestList?> ")" => {
+         elements.unwrap_or(ast::Expression::Tuple { elements: Vec::new() })
     },
     "(" <e:Test> <c:CompFor> ")" => {
         ast::Expression::Comprehension {
@@ -852,9 +817,7 @@ Atom: ast::Expression = {
 };
 
 TestListComp: Vec<ast::Expression> = {
-    <e:OneOrMore<TestOrStarExpr>> <_trailing_comma:","?> => {
-        e
-    },
+    <e:OneOrMore<TestOrStarExpr>> <_trailing_comma:","?> => e,
 };
 
 TestListComp2: ast::Expression = {
@@ -866,10 +829,8 @@ TestListComp2: ast::Expression = {
     },
 };
 
-TestDict: Vec<(ast::Expression, ast::Expression)> = {
-    <e1:OneOrMore<DictEntry>> <_trailing_comma:","?> => {
-        e1
-    }
+TestDict: Vec<(Option<ast::Expression>, ast::Expression)> = {
+    <elements:OneOrMore<DictElement>> <_trailing_comma:","?> => elements,
 };
 
 TestDictComp: ast::Expression = {
@@ -885,10 +846,13 @@ DictEntry: (ast::Expression, ast::Expression) = {
     <e1: Test> ":" <e2: Test> => (e1, e2),
 };
 
+DictElement: (Option<ast::Expression>, ast::Expression) = {
+    <e:DictEntry> => (Some(e.0), e.1),
+    "**" <e:Expression> => (None, e),
+};
+
 TestSet: Vec<ast::Expression> = {
-    <e1:OneOrMore<Test>> ","? => {
-        e1
-    }
+    <e1:OneOrMore<Test>> ","? => e1
 };
 
 TestSetComp: ast::Expression = {
@@ -900,31 +864,37 @@ TestSetComp: ast::Expression = {
     }
 };
 
+ExpressionOrStarExpression = {
+    Expression,
+    StarExpr
+};
+
 ExpressionList: ast::Expression = {
-    <e: ExpressionList2> => {
-        if e.len() == 1 {
-            e.into_iter().next().unwrap()
+    <elements: OneOrMore<ExpressionOrStarExpression>> <trailing_comma:","?> => {
+        if elements.len() == 1 && trailing_comma.is_none() {
+            elements.into_iter().next().unwrap()
         } else {
-            ast::Expression::Tuple { elements: e }
+            ast::Expression::Tuple { elements }
         }
     },
 };
 
 ExpressionList2: Vec<ast::Expression> = {
-    <e1:Expression> <e2:("," Expression)*> ","? => {
-        let mut l = vec![e1];
-        l.extend(e2.into_iter().map(|x| x.1));
-        l
-    },
+    <elements:OneOrMore<Expression>> ","? => elements,
 };
 
-#[inline]
-TestList: Vec<ast::Expression> = {
-  <e1:Test> <e2: ("," Test)*> => {
-    let mut l = vec![e1];
-    l.extend(e2.into_iter().map(|x| x.1));
-    l
-  }
+// A test list is one of:
+// - a list of expressions
+// - a single expression
+// - a single expression followed by a trailing comma
+TestList: ast::Expression = {
+    <elements:OneOrMore<Test>> <trailing_comma: ","?> => {
+        if elements.len() == 1 && trailing_comma.is_none() {
+            elements.into_iter().next().unwrap()
+        } else {
+            ast::Expression::Tuple { elements }
+        }
+    }
 };
 
 // Test
@@ -933,27 +903,16 @@ StarExpr: ast::Expression = {
 };
 
 // Comprehensions:
-CompFor: Vec<ast::Comprehension> = {
-    <c:SingleForComprehension+> => c,
-};
+CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c;
 
 SingleForComprehension: ast::Comprehension = {
-    "for" <e:ExpressionList> "in" <i:OrTest> <c2:ComprehensionIf*> => {
-        ast::Comprehension {
-            target: e,
-            iter: i,
-            ifs: c2,
-        }
+    "for" <target:ExpressionList> "in" <iter:OrTest> <c2:ComprehensionIf*> => {
+        ast::Comprehension { target, iter, ifs: c2 }
     }
 };
 
-ExpressionNoCond: ast::Expression = {
-    OrTest,
-};
-
-ComprehensionIf: ast::Expression = {
-    "if" <c:ExpressionNoCond> => c,
-};
+ExpressionNoCond: ast::Expression = OrTest;
+ComprehensionIf: ast::Expression = "if" <c:ExpressionNoCond> => c;
 
 ArgumentList: (Vec<ast::Expression>, Vec<ast::Keyword>) = {
     <e: Comma<FunctionArgument>> => {
@@ -965,8 +924,15 @@ ArgumentList: (Vec<ast::Expression>, Vec<ast::Keyword>) = {
                     keywords.push(ast::Keyword { name: n, value: value });
                 },
                 None => {
-                    if keywords.len() > 0 {
-                        panic!("positional argument follows keyword argument");
+                    // Allow starred args after keyword arguments.
+                    let is_starred = if let ast::Expression::Starred { .. } = &value {
+                        true
+                    } else {
+                        false
+                    };
+
+                    if keywords.len() > 0 && !is_starred {
+                        panic!("positional argument follows keyword argument {:?}", keywords);
                     };
                     args.push(value);
                 },
@@ -1010,8 +976,8 @@ OneOrMore<T>: Vec<T> = {
 };
 
 Number: ast::Number = {
-    <s:int> => { ast::Number::Integer { value: s } },
-    <s:float> => { ast::Number::Float { value: s } },
+    <value:int> => { ast::Number::Integer { value } },
+    <value:float> => { ast::Number::Float { value } },
     <s:complex> => { ast::Number::Complex { real: s.0, imag: s.1 } },
 };
 
@@ -1101,6 +1067,8 @@ extern {
         "and" => lexer::Tok::And,
         "as" => lexer::Tok::As,
         "assert" => lexer::Tok::Assert,
+        "async" => lexer::Tok::Async,
+        "await" => lexer::Tok::Await,
         "break" => lexer::Tok::Break,
         "class" => lexer::Tok::Class,
         "continue" => lexer::Tok::Continue,

parser/src/token.rs

@@ -3,7 +3,7 @@
 use num_bigint::BigInt;
 
 /// Python source code can be tokenized in a sequence of these tokens.
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Tok {
     Name { name: String },
     Int { value: BigInt },
@@ -17,6 +17,7 @@ pub enum Tok {
     StartProgram,
     StartStatement,
     StartExpression,
+    EndOfFile,
     Lpar,
     Rpar,
     Lsqb,
@@ -72,6 +73,8 @@ pub enum Tok {
     And,
     As,
     Assert,
+    Async,
+    Await,
     Break,
     Class,
     Continue,

src/main.rs

@@ -3,24 +3,22 @@ extern crate clap;
 extern crate env_logger;
 #[macro_use]
 extern crate log;
-extern crate rustpython_parser;
-extern crate rustpython_vm;
 extern crate rustyline;
 
 use clap::{App, Arg};
-use rustpython_parser::error::ParseError;
+use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType};
+use rustpython_parser::error::ParseErrorType;
 use rustpython_vm::{
-    compile,
-    error::CompileError,
-    frame::ScopeRef,
+    frame::Scope,
     import,
     obj::objstr,
     print_exception,
-    pyobject::{AttributeProtocol, PyResult},
+    pyobject::{ItemProtocol, PyResult},
     util, VirtualMachine,
 };
+
 use rustyline::{error::ReadlineError, Editor};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 
 fn main() {
     env_logger::init();
@@ -51,50 +49,46 @@ fn main() {
         .get_matches();
 
     // Construct vm:
-    let mut vm = VirtualMachine::new();
+    let vm = VirtualMachine::new();
+
+    let res = import::init_importlib(&vm, true);
+    handle_exception(&vm, res);
 
     // Figure out if a -c option was given:
     let result = if let Some(command) = matches.value_of("c") {
-        run_command(&mut vm, command.to_string())
+        run_command(&vm, command.to_string())
     } else if let Some(module) = matches.value_of("m") {
-        run_module(&mut vm, module)
+        run_module(&vm, module)
     } else {
         // Figure out if a script was passed:
         match matches.value_of("script") {
-            None => run_shell(&mut vm),
-            Some(filename) => run_script(&mut vm, filename),
+            None => run_shell(&vm),
+            Some(filename) => run_script(&vm, filename),
         }
     };
 
     // See if any exception leaked out:
-    handle_exception(&mut vm, result);
+    handle_exception(&vm, result);
 }
 
-fn _run_string(vm: &mut VirtualMachine, source: &str, source_path: String) -> PyResult {
-    let code_obj = compile::compile(
-        source,
-        &compile::Mode::Exec,
-        source_path,
-        vm.ctx.code_type(),
-    )
-    .map_err(|err| {
-        let syntax_error = vm.context().exceptions.syntax_error.clone();
-        vm.new_exception(syntax_error, err.to_string())
-    })?;
+fn _run_string(vm: &VirtualMachine, source: &str, source_path: String) -> PyResult {
+    let code_obj = vm
+        .compile(source, &compile::Mode::Exec, source_path.clone())
+        .map_err(|err| vm.new_syntax_error(&err))?;
     // trace!("Code object: {:?}", code_obj.borrow());
-    let builtins = vm.get_builtin_scope();
-    let vars = vm.context().new_scope(Some(builtins)); // Keep track of local variables
-    vm.run_code_obj(code_obj, vars)
+    let attrs = vm.ctx.new_dict();
+    attrs.set_item("__file__", vm.new_str(source_path), vm)?;
+    vm.run_code_obj(code_obj, Scope::with_builtins(None, attrs, vm))
 }
 
-fn handle_exception(vm: &mut VirtualMachine, result: PyResult) {
+fn handle_exception(vm: &VirtualMachine, result: PyResult) {
     if let Err(err) = result {
         print_exception(vm, &err);
         std::process::exit(1);
     }
 }
 
-fn run_command(vm: &mut VirtualMachine, mut source: String) -> PyResult {
+fn run_command(vm: &VirtualMachine, mut source: String) -> PyResult {
     debug!("Running command {}", source);
 
     // This works around https://github.com/RustPython/RustPython/issues/17
@@ -102,43 +96,97 @@ fn run_command(vm: &mut VirtualMachine, mut source: String) -> PyResult {
     _run_string(vm, &source, "<stdin>".to_string())
 }
 
-fn run_module(vm: &mut VirtualMachine, module: &str) -> PyResult {
+fn run_module(vm: &VirtualMachine, module: &str) -> PyResult {
     debug!("Running module {}", module);
-    let current_path = PathBuf::from(".");
-    import::import_module(vm, current_path, module)
+    vm.import(module, &vm.ctx.new_tuple(vec![]), 0)
 }
 
-fn run_script(vm: &mut VirtualMachine, script_file: &str) -> PyResult {
+fn run_script(vm: &VirtualMachine, script_file: &str) -> PyResult {
     debug!("Running file {}", script_file);
     // Parse an ast from it:
-    let file_path = Path::new(script_file);
-    match util::read_file(file_path) {
+    let file_path = PathBuf::from(script_file);
+    let file_path = if file_path.is_file() {
+        file_path
+    } else if file_path.is_dir() {
+        let main_file_path = file_path.join("__main__.py");
+        if main_file_path.is_file() {
+            main_file_path
+        } else {
+            error!(
+                "can't find '__main__' module in '{}'",
+                file_path.to_str().unwrap()
+            );
+            std::process::exit(1);
+        }
+    } else {
+        error!(
+            "can't open file '{}': No such file or directory",
+            file_path.to_str().unwrap()
+        );
+        std::process::exit(1);
+    };
+
+    let dir = file_path.parent().unwrap().to_str().unwrap().to_string();
+    let sys_path = vm.get_attribute(vm.sys_module.clone(), "path").unwrap();
+    vm.call_method(&sys_path, "insert", vec![vm.new_int(0), vm.new_str(dir)])?;
+
+    match util::read_file(&file_path) {
         Ok(source) => _run_string(vm, &source, file_path.to_str().unwrap().to_string()),
         Err(err) => {
-            error!("Failed reading file: {:?}", err.kind());
+            error!(
+                "Failed reading file '{}': {:?}",
+                file_path.to_str().unwrap(),
+                err.kind()
+            );
             std::process::exit(1);
         }
     }
 }
 
-fn shell_exec(vm: &mut VirtualMachine, source: &str, scope: ScopeRef) -> Result<(), CompileError> {
-    match compile::compile(
-        source,
-        &compile::Mode::Single,
-        "<stdin>".to_string(),
-        vm.ctx.code_type(),
-    ) {
+#[test]
+fn test_run_script() {
+    let vm = VirtualMachine::new();
+
+    // test file run
+    let r = run_script(&vm, "tests/snippets/dir_main/__main__.py");
+    assert!(r.is_ok());
+
+    // test module run
+    let r = run_script(&vm, "tests/snippets/dir_main");
+    assert!(r.is_ok());
+}
+
+fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), CompileError> {
+    match vm.compile(source, &compile::Mode::Single, "<stdin>".to_string()) {
         Ok(code) => {
-            if let Err(err) = vm.run_code_obj(code, scope) {
-                print_exception(vm, &err);
+            match vm.run_code_obj(code, scope.clone()) {
+                Ok(value) => {
+                    // Save non-None values as "_"
+
+                    use rustpython_vm::pyobject::{IdProtocol, IntoPyObject};
+
+                    if !value.is(&vm.get_none()) {
+                        let key = objstr::PyString::from("_").into_pyobject(vm);
+                        scope.globals.set_item(key, value, vm).unwrap();
+                    }
+                }
+
+                Err(err) => {
+                    print_exception(vm, &err);
+                }
             }
+
             Ok(())
         }
         // Don't inject syntax errors for line continuation
-        Err(err @ CompileError::Parse(ParseError::EOF(_))) => Err(err),
+        Err(
+            err @ CompileError {
+                error: CompileErrorType::Parse(ParseErrorType::EOF),
+                ..
+            },
+        ) => Err(err),
         Err(err) => {
-            let syntax_error = vm.context().exceptions.syntax_error.clone();
-            let exc = vm.new_exception(syntax_error, format!("{}", err));
+            let exc = vm.new_syntax_error(&err);
             print_exception(vm, &exc);
             Err(err)
         }
@@ -160,13 +208,20 @@ fn get_history_path() -> PathBuf {
     xdg_dirs.place_cache_file("repl_history.txt").unwrap()
 }
 
-fn run_shell(vm: &mut VirtualMachine) -> PyResult {
+fn get_prompt(vm: &VirtualMachine, prompt_name: &str) -> String {
+    vm.get_attribute(vm.sys_module.clone(), prompt_name)
+        .ok()
+        .as_ref()
+        .map(objstr::get_value)
+        .unwrap_or_else(String::new)
+}
+
+fn run_shell(vm: &VirtualMachine) -> PyResult {
     println!(
-        "Welcome to the magnificent Rust Python {} interpreter",
+        "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}",
         crate_version!()
     );
-    let builtins = vm.get_builtin_scope();
-    let vars = vm.context().new_scope(Some(builtins)); // Keep track of local variables
+    let vars = vm.new_scope_with_builtins();
 
     // Read a single line:
     let mut input = String::new();
@@ -178,25 +233,38 @@ fn run_shell(vm: &mut VirtualMachine) -> PyResult {
         println!("No previous history.");
     }
 
-    let ps1 = &objstr::get_value(&vm.sys_module.get_attr("ps1").unwrap());
-    let ps2 = &objstr::get_value(&vm.sys_module.get_attr("ps2").unwrap());
-    let mut prompt = ps1;
+    let mut continuing = false;
 
     loop {
-        match repl.readline(prompt) {
+        let prompt = if continuing {
+            get_prompt(vm, "ps2")
+        } else {
+            get_prompt(vm, "ps1")
+        };
+        match repl.readline(&prompt) {
             Ok(line) => {
                 debug!("You entered {:?}", line);
                 input.push_str(&line);
-                input.push_str("\n");
-                repl.add_history_entry(line.trim_end().as_ref());
+                input.push('\n');
+                repl.add_history_entry(line.trim_end());
+
+                if continuing {
+                    if line.is_empty() {
+                        continuing = false;
+                    } else {
+                        continue;
+                    }
+                }
 
                 match shell_exec(vm, &input, vars.clone()) {
-                    Err(CompileError::Parse(ParseError::EOF(_))) => {
-                        prompt = ps2;
+                    Err(CompileError {
+                        error: CompileErrorType::Parse(ParseErrorType::EOF),
+                        ..
+                    }) => {
+                        continuing = true;
                         continue;
                     }
                     _ => {
-                        prompt = ps1;
                         input = String::new();
                     }
                 }
@@ -204,7 +272,8 @@ fn run_shell(vm: &mut VirtualMachine) -> PyResult {
             Err(ReadlineError::Interrupted) => {
                 // TODO: Raise a real KeyboardInterrupt exception
                 println!("^C");
-                break;
+                continuing = false;
+                continue;
             }
             Err(ReadlineError::Eof) => {
                 break;

tests/.gitignore

@@ -0,0 +1 @@
+snippets/whats_left_*.py

tests/.travis-runner.sh

@@ -13,6 +13,7 @@ pip install pipenv
 if [ $CODE_COVERAGE = "true" ]
 then
     find . -name '*.gcda' -delete
+    find . -name '*.gcno' -delete
 
     export CARGO_INCREMENTAL=0
     export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads"
@@ -31,7 +32,7 @@ then
     zip -0 ccov.zip `find . \( -name "rustpython*.gc*" \) -print`
 
     # Install grcov
-    curl -L https://github.com/mozilla/grcov/releases/download/v0.4.1/grcov-linux-x86_64.tar.bz2 | tar jxf -
+    curl -L https://github.com/mozilla/grcov/releases/download/v0.4.2/grcov-linux-x86_64.tar.bz2 | tar jxf -
 
     ./grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir "/*" -p "x" > lcov.info
 

tests/Pipfile

@@ -10,4 +10,4 @@ pytest = "*"
 [dev-packages]
 
 [requires]
-python_version = "3"
+python_version = "3.6"

tests/Pipfile.lock

@@ -1,11 +1,11 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "b2d2d68e7d4330ff8d889816c56b9cee4bf54962c86b2c11382108176a201ec8"
+            "sha256": "ce98de5914393363a8cb86a4753b3964caa53a4659a403a3ef357e2086363ef7"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_version": "3"
+            "python_version": "3.6"
         },
         "sources": [
             {
@@ -16,74 +16,99 @@
         ]
     },
     "default": {
-        "aenum": {
-            "hashes": [
-                "sha256:3df9b84cce5dc9ed77c337079f97b66c44c0053eb87d6f4d46b888dc45801e38",
-                "sha256:7a77c205c4bc9d7fe9bd73b3193002d724aebf5909fa0d297534208953891ec8",
-                "sha256:a3208e4b28db3a7b232ff69b934aef2ea1bf27286d9978e1e597d46f490e4687"
-            ],
-            "version": "==2.1.2"
-        },
         "atomicwrites": {
             "hashes": [
-                "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585",
-                "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6"
+                "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
+                "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
             ],
-            "version": "==1.1.5"
+            "version": "==1.3.0"
         },
         "attrs": {
             "hashes": [
-                "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265",
-                "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b"
+                "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
+                "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
             ],
-            "version": "==18.1.0"
+            "version": "==19.1.0"
         },
         "bytecode": {
             "hashes": [
-                "sha256:cc6931151c7f0a542f8cf7619fe1639af3b9529c4678860fa3239397cb0f7de0",
-                "sha256:e464004d4a9eeeca987cb4950dba11b827964b6c90cd331c1f20abd2dab3c962"
+                "sha256:68b1d591c7af0e5c5273e028d3cc0299fbe374dff0cf9149ec7e569be0c573e7",
+                "sha256:c43d5052cbff076bfdf5b0b93ff6c76e461aab628ce47d30637bb200b6b7bb2c"
             ],
             "index": "pypi",
-            "version": "==0.7.0"
+            "version": "==0.8.0"
+        },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7",
+                "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db"
+            ],
+            "version": "==0.18"
         },
         "more-itertools": {
             "hashes": [
-                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
-                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
-                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
+                "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7",
+                "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a"
             ],
-            "version": "==4.3.0"
+            "markers": "python_version > '2.7'",
+            "version": "==7.0.0"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af",
+                "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3"
+            ],
+            "version": "==19.0"
         },
         "pluggy": {
             "hashes": [
-                "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1",
-                "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1"
+                "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc",
+                "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c"
             ],
-            "markers": "python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.2.*'",
-            "version": "==0.7.1"
+            "version": "==0.12.0"
         },
         "py": {
             "hashes": [
-                "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7",
-                "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e"
+                "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
+                "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
             ],
-            "markers": "python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.2.*'",
-            "version": "==1.5.4"
+            "version": "==1.8.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a",
+                "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03"
+            ],
+            "version": "==2.4.0"
         },
         "pytest": {
             "hashes": [
-                "sha256:86a8dbf407e437351cef4dba46736e9c5a6e3c3ac71b2e942209748e76ff2086",
-                "sha256:e74466e97ac14582a8188ff4c53e6cc3810315f342f6096899332ae864c1d432"
+                "sha256:4a784f1d4f2ef198fe9b7aef793e9fa1a3b2f84e822d9b3a64a181293a572d45",
+                "sha256:926855726d8ae8371803f7b2e6ec0a69953d9c6311fa7c3b6c1b929ff92d27da"
             ],
             "index": "pypi",
-            "version": "==3.7.1"
+            "version": "==4.6.3"
         },
         "six": {
             "hashes": [
-                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
-                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
             ],
-            "version": "==1.11.0"
+            "version": "==1.12.0"
+        },
+        "wcwidth": {
+            "hashes": [
+                "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
+                "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
+            ],
+            "version": "==0.1.7"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d",
+                "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3"
+            ],
+            "version": "==0.5.1"
         }
     },
     "develop": {}

tests/README.md

@@ -1,4 +1,3 @@
-
 # Test snippets
 
 This directory contains two sets of test snippets which can be run in
@@ -6,6 +5,10 @@ Python.  The `snippets/` directory contains functional tests, and the
 `benchmarks/` directory contains snippets for use in benchmarking
 RustPython's performance.
 
+## Generates the test for not implemented methods
+
+run using cpython not_impl_gen.py it automatically generate a
+test snippet to check not yet implemented methods
 
 ## Running with CPython + RustPython
 
@@ -15,4 +18,3 @@ compilation to bytecode. When this is done, run the bytecode with rustpython.
 ## Running with RustPython
 
 The other option is to run all snippets with RustPython.
-

tests/generator/not_impl_methods_footer.txt

@@ -0,0 +1,12 @@
+not_implemented = [(name, method)
+                   for name, (val, methods) in expected_methods.items()
+                   for method in methods
+                   if not hasattr(val, method)]
+
+for r in not_implemented:
+    print(r[0], ".", r[1], sep="")
+if not not_implemented:
+    print("Not much \\o/")
+
+if platform.python_implementation() == "CPython":
+    assert len(not_implemented) == 0, "CPython should have all the methods"

tests/generator/not_impl_methods_header.txt

@@ -0,0 +1,7 @@
+# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE
+# EDIT tests/not_impl_gen.py, NOT THIS FILE.
+# RESULTS OF THIS TEST DEPEND ON THE CPYTHON
+# VERSION USED TO RUN not_impl_gen.py
+
+import platform
+

tests/generator/not_impl_modules_footer.txt

@@ -0,0 +1,32 @@
+
+
+rustpymods = list(
+    map(
+        lambda mod: mod[0],
+        filter(
+            lambda mod: (mod[1] == "" or mod[1] == ".py") and "LICENSE" not in mod[0],
+            map(os.path.splitext, os.listdir(libdir)),
+        ),
+    )
+)
+rustpymods += list(sys.builtin_module_names)
+
+rustpymods = dict(map(
+    lambda mod: (
+        mod,
+        set(dir(__import__(mod)))
+        if mod not in ("this", "antigravity")
+        else None,
+    ),
+    rustpymods
+))
+
+for modname, cpymod in cpymods.items():
+    if modname in rustpymods:
+        rustpymod = rustpymods[modname]
+        if rustpymod:
+            for item in cpymod - rustpymod:
+                print(f"{modname}.{item}")
+    else:
+        print(f"{modname} (entire module)")
+

tests/generator/not_impl_modules_header.txt

@@ -0,0 +1,8 @@
+# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE
+# EDIT tests/not_impl_mods_gen.sh, NOT THIS FILE.
+# RESULTS OF THIS TEST DEPEND ON THE CPYTHON
+# VERSION AND PYTHON ENVIRONMENT USED
+# TO RUN not_impl_mods_gen.py
+
+import sys
+import os

tests/not_impl_gen.py

@@ -0,0 +1,105 @@
+# It's recommended to run this with `python3 -I not_impl_gen.py`, to make sure
+# that nothing in your global Python environment interferes with what's being
+# extracted here.
+
+import pkgutil
+import os
+import sys
+
+sys.path = list(
+    filter(
+        lambda path: "site-packages" not in path and "dist-packages" not in path,
+        sys.path,
+    )
+)
+
+
+def attr_is_not_inherited(type_, attr):
+    """
+    returns True if type_'s attr is not inherited from any of its base classes
+    """
+
+    bases = type_.__mro__[1:]
+
+    return getattr(type_, attr) not in (getattr(base, attr, None) for base in bases)
+
+
+def gen_methods(header, footer, output):
+    objects = [
+        bool,
+        bytearray,
+        bytes,
+        complex,
+        dict,
+        float,
+        frozenset,
+        int,
+        list,
+        memoryview,
+        range,
+        set,
+        str,
+        tuple,
+        object,
+    ]
+
+    output.write(header.read())
+    output.write("expected_methods = {\n")
+
+    for obj in objects:
+        output.write(f" '{obj.__name__}': ({obj.__name__}, [\n")
+        output.write(
+            "\n".join(
+                f"    {attr!r},"
+                for attr in dir(obj)
+                if attr_is_not_inherited(obj, attr)
+            )
+        )
+        output.write("\n ])," + ("\n" if objects[-1] == obj else "\n\n"))
+
+    output.write("}\n\n")
+    output.write(footer.read())
+
+def get_module_methods(name):
+    try:
+        return set(dir(__import__(name))) if name not in ("this", "antigravity") else None
+    except ModuleNotFoundError:
+        return None
+
+
+def gen_modules(header, footer, output):
+    output.write(header.read())
+
+    modules = dict(
+        map(
+            lambda mod: (
+                mod.name,
+                # check name b/c modules listed have side effects on import,
+                # e.g. printing something or opening a webpage
+                get_module_methods(mod.name)
+            ),
+            pkgutil.iter_modules(),
+        )
+    )
+
+    print(
+        f"""
+cpymods = {modules!r}
+libdir = {os.path.abspath("../Lib/")!r}
+""",
+        file=output,
+    )
+
+    output.write(footer.read())
+
+
+gen_funcs = {"methods": gen_methods, "modules": gen_modules}
+
+
+for name, gen_func in gen_funcs.items():
+    gen_func(
+        header=open(f"generator/not_impl_{name}_header.txt"),
+        footer=open(f"generator/not_impl_{name}_footer.txt"),
+        output=open(f"snippets/whats_left_{name}.py", "w"),
+    )
+

tests/snippets/append.py

@@ -1,2 +0,0 @@
-x = []
-x.append(1)

tests/snippets/ast_snippet.py

@@ -21,3 +21,10 @@ print(foo.body[0].value.func.id)
 assert foo.body[0].value.func.id == 'print'
 assert foo.body[0].lineno == 3
 assert foo.body[1].lineno == 4
+
+n = ast.parse("3 < 4 > 5\n")
+assert n.body[0].value.left.n == 3
+assert 'Lt' in str(n.body[0].value.ops[0])
+assert 'Gt' in str(n.body[0].value.ops[1])
+assert n.body[0].value.comparators[0].n == 4
+assert n.body[0].value.comparators[1].n == 5