Construct PyObjectRef earlier to avoid the need for Clone on Frame.

.dockerignore

@@ -1,2 +0,0 @@
-target
-**/node_modules

.github/ISSUE_TEMPLATE/report-incompatibility.md

@@ -1,16 +0,0 @@
----
-name: Report incompatibility
-about: Report an incompatibility between RustPython and CPython
-title: ''
-labels: feat
-assignees: ''
-
----
-
-## Feature
-
-<!-- What Python feature is missing from RustPython? Give a short description of the feature and how you ran into its absence. -->
-
-## Python Documentation
-
-<!-- Give a link to the feature in the CPython documentation (https://docs.python.org/3/) in order to assist in its implementation. -->

.github/ISSUE_TEMPLATE/rfc.md

@@ -1,24 +0,0 @@
----
-name: RFC
-about: Make a suggestion in a Request for Comments format to RustPython
-title: "[RFC] "
-labels: RFC
-assignees: ''
-
----
-
-## Summary
-
-<!-- A quick overview of your suggestion -->
-
-## Detailed Explanation
-
-<!-- Elaborate on your suggestion in all its details -->
-
-## Drawbacks, Rationale, and Alternatives
-
-<!-- What drawbacks might this solution have? Why do you feel it is necessary? What other options might there be to solving this problem? -->
-
-## Unresolved Questions
-
-<!-- What would you like feedback on for fleshing out your suggestion? -->

.gitignore

@@ -1,12 +1,8 @@
 /target
-/*/target
+wasm/target
 **/*.rs.bk
 **/*.bytecode
 __pycache__
 **/*.pytest_cache
 .*sw*
 .repl_history.txt
-.vscode
-wasm-pack.log
-.idea/
-tests/snippets/resources

.travis.yml

@@ -1,116 +1,66 @@
 
+language: rust
+
+rust:
+  - stable
+  - beta
+  - nightly
+
+script:
+    - cargo build --verbose --all
+    - cargo test --verbose --all
+
+env:
+    # This is used to only capture the regular nightly test in allow_failures
+    - REGULAR_TEST=true
+
+cache: cargo
+
 matrix:
-  fast_finish: true
+    include:
-  include:
+        # To test the snippets, we use Travis' Python environment (because
-    - name: Run rust tests
+        # installing rust ourselves is a lot easier than installing Python)
-      language: rust
+        - language: python
-      rust: stable
+          python: 3.6
-      cache: cargo
+          cache:
-      script:
+              pip: true
-        - cargo build --verbose --all
+              # Because we're using the Python Travis environment, we can't use
-        - cargo test --verbose --all
+              # the built-in cargo cacher
-      env:
+              directories:
-        # Prevention of cache corruption.
+                  - /home/travis/.cargo
-        # See: https://docs.travis-ci.com/user/caching/#caches-and-build-matrices
+                  - target
-        - JOBCACHE=1
+          env:
-
+              - TRAVIS_RUST_VERSION=stable
-    # To test the snippets, we use Travis' Python environment (because
+              - REGULAR_TEST=false
-    # installing rust ourselves is a lot easier than installing Python)
+          script: tests/.travis-runner.sh
-    - name: python test snippets
+        - language: python
-      language: python
+          python: 3.6
-      python: 3.6
+          cache:
-      cache:
+              pip: true
-        - pip
+              # Because we're using the Python Travis environment, we can't use
-        - cargo
+              # the built-in cargo cacher
-      env:
+              directories:
-        - JOBCACHE=2
+                  - /home/travis/.cargo
-        - TRAVIS_RUST_VERSION=stable
+                  - target
-        - CODE_COVERAGE=false
+          env:
-      script: tests/.travis-runner.sh
+              - TRAVIS_RUST_VERSION=beta
-
+              - REGULAR_TEST=false
-    - name: Check rust code style with rustfmt
+          script: tests/.travis-runner.sh
-      language: rust
+        - name: rustfmt
-      rust: stable
+          language: rust
-      cache: cargo
+          rust: nightly
-      before_script:
+          cache: cargo
-        - rustup component add rustfmt
+          before_script:
-      script:
+              - rustup component add rustfmt-preview
-        - cargo fmt --all -- --check
+          script:
-      env:
+              # Code references the generated python.rs, so put something in
-        - JOBCACHE=3
+              # place to make `cargo fmt` happy.  (We use `echo` rather than
-
+              # `touch` because rustfmt complains about the empty file touch
-    - name: publish documentation
+              # creates.)
-      language: rust
+              - echo > parser/src/python.rs
-      rust: stable
+              - cargo fmt --all -- --check
-      cache: cargo
+          env:
-      script:
+              - REGULAR_TEST=false
-        - cargo doc --no-deps --all
+    allow_failures:
-      if: branch = release
+        - rust: nightly
-      env:
+          env: REGULAR_TEST=true
-        - JOBCACHE=4
-      deploy:
-        - provider: pages
-          repo: RustPython/website
-          target-branch: master
-          local-dir: target/doc
-          skip-cleanup: true
-          # Set in the settings page of your repository, as a secure variable
-          github-token: $WEBSITE_GITHUB_TOKEN
-          keep-history: true
-
-    - name: WASM online demo
-      language: rust
-      rust: stable
-      cache: cargo
-      install:
-        - nvm install node
-        # install wasm-pack
-        - curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
-      script:
-        - cd wasm/demo
-        - npm install
-        - npm run dist
-      if: branch = release
-      env:
-        - JOBCACHE=5
-      deploy:
-        - provider: pages
-          repo: RustPython/demo
-          target-branch: master
-          local-dir: wasm/demo/dist
-          skip-cleanup: true
-          # Set in the settings page of your repository, as a secure variable
-          github-token: $WEBSITE_GITHUB_TOKEN
-          keep-history: true
-
-    - name: Code Coverage
-      language: python
-      python: 3.6
-      cache:
-        - pip
-        - cargo
-      script:
-        - tests/.travis-runner.sh
-      # Only do code coverage on master via a cron job.
-      if: branch = master AND type = cron
-      env:
-        - JOBCACHE=6
-        - TRAVIS_RUST_VERSION=nightly
-        - CODE_COVERAGE=true
-
-    - name: test WASM
-      language: python
-      python: 3.6
-      cache:
-        - pip
-        - cargo
-      addons:
-        firefox: latest
-      install:
-        - nvm install node
-        - pip install pipenv
-      script:
-        - wasm/tests/.travis-runner.sh
-      env:
-        - JOBCACHE=7
-        - TRAVIS_RUST_VERSION=stable

Cargo.toml

@@ -1,29 +1,14 @@
 [package]
 name = "rustpython"
-version = "0.1.0"
+version = "0.0.1"
-authors = ["RustPython Team"]
+authors = ["Windel Bouwman", "Shing Lyu <shing.lyu@gmail.com>"]
-edition = "2018"
-description = "A python interpreter written in rust."
-repository = "https://github.com/RustPython/RustPython"
-license = "MIT"
 
 [workspace]
-members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode"]
-
-[[bench]]
-name = "bench"
-path = "./benchmarks/bench.rs"
-
 
 [dependencies]
 log="0.4.1"
 env_logger="0.5.10"
 clap = "2.31.2"
-rustpython-compiler = {path = "compiler", version = "0.1.0"}
+rustpython_parser = {path = "parser"}
-rustpython-parser = {path = "parser", version = "0.1.0"}
+rustpython_vm = {path = "vm"}
-rustpython-vm = {path = "vm", version = "0.1.0"}
+rustyline = "2.1.0"
-rustyline = "4.1.0"
-xdg = "2.2.0"
-
-[dev-dependencies.cpython]
-version = "0.2"

Dockerfile.bin

@@ -1,9 +0,0 @@
-FROM rust:1.31-slim
-
-WORKDIR /rustpython
-
-COPY . .
-
-RUN cargo build --release
-
-CMD [ "/rustpython/target/release/rustpython" ]

Dockerfile.wasm

@@ -1,19 +0,0 @@
-FROM rust:1.31-slim
-
-RUN apt-get update && apt-get install curl gnupg -y && \
-    curl -o- https://deb.nodesource.com/setup_10.x | bash && \
-    apt-get install nodejs -y && \
-    curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && \
-    npm i -g serve
-
-WORKDIR /rustpython
-
-COPY . .
-
-RUN cd ./wasm/lib/ && \
-    cargo build --release && \
-    cd ../demo && \
-    npm install && \
-    npm run dist
-
-CMD [ "serve", "/rustpython/wasm/demo/dist" ]

Lib/PSF-LICENSE

@@ -1,254 +0,0 @@
-A. HISTORY OF THE SOFTWARE
-==========================
-
-Python was created in the early 1990s by Guido van Rossum at Stichting
-Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
-as a successor of a language called ABC.  Guido remains Python's
-principal author, although it includes many contributions from others.
-
-In 1995, Guido continued his work on Python at the Corporation for
-National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
-in Reston, Virginia where he released several versions of the
-software.
-
-In May 2000, Guido and the Python core development team moved to
-BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
-year, the PythonLabs team moved to Digital Creations, which became
-Zope Corporation.  In 2001, the Python Software Foundation (PSF, see
-https://www.python.org/psf/) was formed, a non-profit organization
-created specifically to own Python-related Intellectual Property.
-Zope Corporation was a sponsoring member of the PSF.
-
-All Python releases are Open Source (see http://www.opensource.org for
-the Open Source Definition).  Historically, most, but not all, Python
-releases have also been GPL-compatible; the table below summarizes
-the various releases.
-
-    Release         Derived     Year        Owner       GPL-
-                    from                                compatible? (1)
-
-    0.9.0 thru 1.2              1991-1995   CWI         yes
-    1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
-    1.6             1.5.2       2000        CNRI        no
-    2.0             1.6         2000        BeOpen.com  no
-    1.6.1           1.6         2001        CNRI        yes (2)
-    2.1             2.0+1.6.1   2001        PSF         no
-    2.0.1           2.0+1.6.1   2001        PSF         yes
-    2.1.1           2.1+2.0.1   2001        PSF         yes
-    2.1.2           2.1.1       2002        PSF         yes
-    2.1.3           2.1.2       2002        PSF         yes
-    2.2 and above   2.1.1       2001-now    PSF         yes
-
-Footnotes:
-
-(1) GPL-compatible doesn't mean that we're distributing Python under
-    the GPL.  All Python licenses, unlike the GPL, let you distribute
-    a modified version without making your changes open source.  The
-    GPL-compatible licenses make it possible to combine Python with
-    other software that is released under the GPL; the others don't.
-
-(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
-    because its license has a choice of law clause.  According to
-    CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
-    is "not incompatible" with the GPL.
-
-Thanks to the many outside volunteers who have worked under Guido's
-direction to make these releases possible.
-
-
-B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
-===============================================================
-
-PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
---------------------------------------------
-
-1. This LICENSE AGREEMENT is between the Python Software Foundation
-("PSF"), and the Individual or Organization ("Licensee") accessing and
-otherwise using this software ("Python") in source or binary form and
-its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, PSF hereby
-grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
-analyze, test, perform and/or display publicly, prepare derivative works,
-distribute, and otherwise use Python alone or in any derivative version,
-provided, however, that PSF's License Agreement and PSF's notice of copyright,
-i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All
-Rights Reserved" are retained in Python alone or in any derivative version
-prepared by Licensee.
-
-3. In the event Licensee prepares a derivative work that is based on
-or incorporates Python or any part thereof, and wants to make
-the derivative work available to others as provided herein, then
-Licensee hereby agrees to include in any such work a brief summary of
-the changes made to Python.
-
-4. PSF is making Python available to Licensee on an "AS IS"
-basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
-OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-6. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-7. Nothing in this License Agreement shall be deemed to create any
-relationship of agency, partnership, or joint venture between PSF and
-Licensee.  This License Agreement does not grant permission to use PSF
-trademarks or trade name in a trademark sense to endorse or promote
-products or services of Licensee, or any third party.
-
-8. By copying, installing or otherwise using Python, Licensee
-agrees to be bound by the terms and conditions of this License
-Agreement.
-
-
-BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
--------------------------------------------
-
-BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
-
-1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
-office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
-Individual or Organization ("Licensee") accessing and otherwise using
-this software in source or binary form and its associated
-documentation ("the Software").
-
-2. Subject to the terms and conditions of this BeOpen Python License
-Agreement, BeOpen hereby grants Licensee a non-exclusive,
-royalty-free, world-wide license to reproduce, analyze, test, perform
-and/or display publicly, prepare derivative works, distribute, and
-otherwise use the Software alone or in any derivative version,
-provided, however, that the BeOpen Python License is retained in the
-Software, alone or in any derivative version prepared by Licensee.
-
-3. BeOpen is making the Software available to Licensee on an "AS IS"
-basis.  BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
-SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
-AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
-DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-5. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-6. This License Agreement shall be governed by and interpreted in all
-respects by the law of the State of California, excluding conflict of
-law provisions.  Nothing in this License Agreement shall be deemed to
-create any relationship of agency, partnership, or joint venture
-between BeOpen and Licensee.  This License Agreement does not grant
-permission to use BeOpen trademarks or trade names in a trademark
-sense to endorse or promote products or services of Licensee, or any
-third party.  As an exception, the "BeOpen Python" logos available at
-http://www.pythonlabs.com/logos.html may be used according to the
-permissions granted on that web page.
-
-7. By copying, installing or otherwise using the software, Licensee
-agrees to be bound by the terms and conditions of this License
-Agreement.
-
-
-CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
----------------------------------------
-
-1. This LICENSE AGREEMENT is between the Corporation for National
-Research Initiatives, having an office at 1895 Preston White Drive,
-Reston, VA 20191 ("CNRI"), and the Individual or Organization
-("Licensee") accessing and otherwise using Python 1.6.1 software in
-source or binary form and its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, CNRI
-hereby grants Licensee a nonexclusive, royalty-free, world-wide
-license to reproduce, analyze, test, perform and/or display publicly,
-prepare derivative works, distribute, and otherwise use Python 1.6.1
-alone or in any derivative version, provided, however, that CNRI's
-License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
-1995-2001 Corporation for National Research Initiatives; All Rights
-Reserved" are retained in Python 1.6.1 alone or in any derivative
-version prepared by Licensee.  Alternately, in lieu of CNRI's License
-Agreement, Licensee may substitute the following text (omitting the
-quotes): "Python 1.6.1 is made available subject to the terms and
-conditions in CNRI's License Agreement.  This Agreement together with
-Python 1.6.1 may be located on the Internet using the following
-unique, persistent identifier (known as a handle): 1895.22/1013.  This
-Agreement may also be obtained from a proxy server on the Internet
-using the following URL: http://hdl.handle.net/1895.22/1013".
-
-3. In the event Licensee prepares a derivative work that is based on
-or incorporates Python 1.6.1 or any part thereof, and wants to make
-the derivative work available to others as provided herein, then
-Licensee hereby agrees to include in any such work a brief summary of
-the changes made to Python 1.6.1.
-
-4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
-basis.  CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
-OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-6. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-7. This License Agreement shall be governed by the federal
-intellectual property law of the United States, including without
-limitation the federal copyright law, and, to the extent such
-U.S. federal law does not apply, by the law of the Commonwealth of
-Virginia, excluding Virginia's conflict of law provisions.
-Notwithstanding the foregoing, with regard to derivative works based
-on Python 1.6.1 that incorporate non-separable material that was
-previously distributed under the GNU General Public License (GPL), the
-law of the Commonwealth of Virginia shall govern this License
-Agreement only as to issues arising under or with respect to
-Paragraphs 4, 5, and 7 of this License Agreement.  Nothing in this
-License Agreement shall be deemed to create any relationship of
-agency, partnership, or joint venture between CNRI and Licensee.  This
-License Agreement does not grant permission to use CNRI trademarks or
-trade name in a trademark sense to endorse or promote products or
-services of Licensee, or any third party.
-
-8. By clicking on the "ACCEPT" button where indicated, or by copying,
-installing or otherwise using Python 1.6.1, Licensee agrees to be
-bound by the terms and conditions of this License Agreement.
-
-        ACCEPT
-
-
-CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
---------------------------------------------------
-
-Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
-The Netherlands.  All rights reserved.
-
-Permission to use, copy, modify, and distribute this software and its
-documentation for any purpose and without fee is hereby granted,
-provided that the above copyright notice appear in all copies and that
-both that copyright notice and this permission notice appear in
-supporting documentation, and that the name of Stichting Mathematisch
-Centrum or CWI not be used in advertising or publicity pertaining to
-distribution of the software without specific, written prior
-permission.
-
-STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
-THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
-FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
-FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
-ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
-OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

Lib/README.md

@@ -1,10 +0,0 @@
-Standard Library for RustPython
-===============================
-
-This directory contains all of the Python files that make up the standard library for RustPython.
-
-Most of these files are copied over from the CPython repository(the 3.7 branch), with slight modifications to allow them 
-to work under RustPython. The current goal is to complete the standard library with as few modifications as possible. 
-Current modifications are just temporary workarounds for bugs/missing feature within the RustPython implementation.
-
-The first target is to run the ``unittest`` module, so we can leverage the CPython test suite.

Lib/_py_abc.py

@@ -1,147 +0,0 @@
-from _weakrefset import WeakSet
-
-
-def get_cache_token():
-    """Returns the current ABC cache token.
-
-    The token is an opaque object (supporting equality testing) identifying the
-    current version of the ABC cache for virtual subclasses. The token changes
-    with every call to ``register()`` on any ABC.
-    """
-    return ABCMeta._abc_invalidation_counter
-
-
-class ABCMeta(type):
-    """Metaclass for defining Abstract Base Classes (ABCs).
-
-    Use this metaclass to create an ABC.  An ABC can be subclassed
-    directly, and then acts as a mix-in class.  You can also register
-    unrelated concrete classes (even built-in classes) and unrelated
-    ABCs as 'virtual subclasses' -- these and their descendants will
-    be considered subclasses of the registering ABC by the built-in
-    issubclass() function, but the registering ABC won't show up in
-    their MRO (Method Resolution Order) nor will method
-    implementations defined by the registering ABC be callable (not
-    even via super()).
-    """
-
-    # A global counter that is incremented each time a class is
-    # registered as a virtual subclass of anything.  It forces the
-    # negative cache to be cleared before its next use.
-    # Note: this counter is private. Use `abc.get_cache_token()` for
-    #       external code.
-    _abc_invalidation_counter = 0
-
-    def __new__(mcls, name, bases, namespace, **kwargs):
-        cls = type.__new__(mcls, name, bases, namespace, **kwargs)
-        # Compute set of abstract method names
-        abstracts = {name
-                     for name, value in namespace.items()
-                     if getattr(value, "__isabstractmethod__", False)}
-        for base in bases:
-            for name in getattr(base, "__abstractmethods__", set()):
-                value = getattr(cls, name, None)
-                if getattr(value, "__isabstractmethod__", False):
-                    abstracts.add(name)
-        cls.__abstractmethods__ = set(abstracts)
-        # Set up inheritance registry
-        cls._abc_registry = WeakSet()
-        cls._abc_cache = WeakSet()
-        cls._abc_negative_cache = WeakSet()
-        cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
-        return cls
-
-    def register(cls, subclass):
-        """Register a virtual subclass of an ABC.
-
-        Returns the subclass, to allow usage as a class decorator.
-        """
-        if not isinstance(subclass, type):
-            raise TypeError("Can only register classes")
-        if issubclass(subclass, cls):
-            return subclass  # Already a subclass
-        # Subtle: test for cycles *after* testing for "already a subclass";
-        # this means we allow X.register(X) and interpret it as a no-op.
-        if issubclass(cls, subclass):
-            # This would create a cycle, which is bad for the algorithm below
-            raise RuntimeError("Refusing to create an inheritance cycle")
-        cls._abc_registry.add(subclass)
-        ABCMeta._abc_invalidation_counter += 1  # Invalidate negative cache
-        return subclass
-
-    def _dump_registry(cls, file=None):
-        """Debug helper to print the ABC registry."""
-        print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file)
-        print(f"Inv. counter: {get_cache_token()}", file=file)
-        for name in cls.__dict__:
-            if name.startswith("_abc_"):
-                value = getattr(cls, name)
-                if isinstance(value, WeakSet):
-                    value = set(value)
-                print(f"{name}: {value!r}", file=file)
-
-    def _abc_registry_clear(cls):
-        """Clear the registry (for debugging or testing)."""
-        cls._abc_registry.clear()
-
-    def _abc_caches_clear(cls):
-        """Clear the caches (for debugging or testing)."""
-        cls._abc_cache.clear()
-        cls._abc_negative_cache.clear()
-
-    def __instancecheck__(cls, instance):
-        """Override for isinstance(instance, cls)."""
-        # Inline the cache checking
-        subclass = instance.__class__
-        if subclass in cls._abc_cache:
-            return True
-        subtype = type(instance)
-        if subtype is subclass:
-            if (cls._abc_negative_cache_version ==
-                ABCMeta._abc_invalidation_counter and
-                subclass in cls._abc_negative_cache):
-                return False
-            # Fall back to the subclass check.
-            return cls.__subclasscheck__(subclass)
-        return any(cls.__subclasscheck__(c) for c in (subclass, subtype))
-
-    def __subclasscheck__(cls, subclass):
-        """Override for issubclass(subclass, cls)."""
-        if not isinstance(subclass, type):
-            raise TypeError('issubclass() arg 1 must be a class')
-        # Check cache
-        if subclass in cls._abc_cache:
-            return True
-        # Check negative cache; may have to invalidate
-        if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter:
-            # Invalidate the negative cache
-            cls._abc_negative_cache = WeakSet()
-            cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
-        elif subclass in cls._abc_negative_cache:
-            return False
-        # Check the subclass hook
-        ok = cls.__subclasshook__(subclass)
-        if ok is not NotImplemented:
-            assert isinstance(ok, bool)
-            if ok:
-                cls._abc_cache.add(subclass)
-            else:
-                cls._abc_negative_cache.add(subclass)
-            return ok
-        # Check if it's a direct subclass
-        if cls in getattr(subclass, '__mro__', ()):
-            cls._abc_cache.add(subclass)
-            return True
-        # Check if it's a subclass of a registered class (recursive)
-        for rcls in cls._abc_registry:
-            if issubclass(subclass, rcls):
-                cls._abc_cache.add(subclass)
-                return True
-        # Check if it's a subclass of a subclass (recursive)
-        for scls in cls.__subclasses__():
-            if issubclass(subclass, scls):
-                cls._abc_cache.add(subclass)
-                return True
-        # No dice; update negative cache
-        cls._abc_negative_cache.add(subclass)
-        return False

Lib/_weakrefset.py

@@ -1,196 +0,0 @@
-# Access WeakSet through the weakref module.
-# This code is separated-out because it is needed
-# by abc.py to load everything else at startup.
-
-from _weakref import ref
-
-__all__ = ['WeakSet']
-
-
-class _IterationGuard:
-    # This context manager registers itself in the current iterators of the
-    # weak container, such as to delay all removals until the context manager
-    # exits.
-    # This technique should be relatively thread-safe (since sets are).
-
-    def __init__(self, weakcontainer):
-        # Don't create cycles
-        self.weakcontainer = ref(weakcontainer)
-
-    def __enter__(self):
-        w = self.weakcontainer()
-        if w is not None:
-            w._iterating.add(self)
-        return self
-
-    def __exit__(self, e, t, b):
-        w = self.weakcontainer()
-        if w is not None:
-            s = w._iterating
-            s.remove(self)
-            if not s:
-                w._commit_removals()
-
-
-class WeakSet:
-    def __init__(self, data=None):
-        self.data = set()
-        def _remove(item, selfref=ref(self)):
-            self = selfref()
-            if self is not None:
-                if self._iterating:
-                    self._pending_removals.append(item)
-                else:
-                    self.data.discard(item)
-        self._remove = _remove
-        # A list of keys to be removed
-        self._pending_removals = []
-        self._iterating = set()
-        if data is not None:
-            self.update(data)
-
-    def _commit_removals(self):
-        l = self._pending_removals
-        discard = self.data.discard
-        while l:
-            discard(l.pop())
-
-    def __iter__(self):
-        with _IterationGuard(self):
-            for itemref in self.data:
-                item = itemref()
-                if item is not None:
-                    # Caveat: the iterator will keep a strong reference to
-                    # `item` until it is resumed or closed.
-                    yield item
-
-    def __len__(self):
-        return len(self.data) - len(self._pending_removals)
-
-    def __contains__(self, item):
-        try:
-            wr = ref(item)
-        except TypeError:
-            return False
-        return wr in self.data
-
-    def __reduce__(self):
-        return (self.__class__, (list(self),),
-                getattr(self, '__dict__', None))
-
-    def add(self, item):
-        if self._pending_removals:
-            self._commit_removals()
-        self.data.add(ref(item, self._remove))
-
-    def clear(self):
-        if self._pending_removals:
-            self._commit_removals()
-        self.data.clear()
-
-    def copy(self):
-        return self.__class__(self)
-
-    def pop(self):
-        if self._pending_removals:
-            self._commit_removals()
-        while True:
-            try:
-                itemref = self.data.pop()
-            except KeyError:
-                raise KeyError('pop from empty WeakSet') from None
-            item = itemref()
-            if item is not None:
-                return item
-
-    def remove(self, item):
-        if self._pending_removals:
-            self._commit_removals()
-        self.data.remove(ref(item))
-
-    def discard(self, item):
-        if self._pending_removals:
-            self._commit_removals()
-        self.data.discard(ref(item))
-
-    def update(self, other):
-        if self._pending_removals:
-            self._commit_removals()
-        for element in other:
-            self.add(element)
-
-    def __ior__(self, other):
-        self.update(other)
-        return self
-
-    def difference(self, other):
-        newset = self.copy()
-        newset.difference_update(other)
-        return newset
-    __sub__ = difference
-
-    def difference_update(self, other):
-        self.__isub__(other)
-    def __isub__(self, other):
-        if self._pending_removals:
-            self._commit_removals()
-        if self is other:
-            self.data.clear()
-        else:
-            self.data.difference_update(ref(item) for item in other)
-        return self
-
-    def intersection(self, other):
-        return self.__class__(item for item in other if item in self)
-    __and__ = intersection
-
-    def intersection_update(self, other):
-        self.__iand__(other)
-    def __iand__(self, other):
-        if self._pending_removals:
-            self._commit_removals()
-        self.data.intersection_update(ref(item) for item in other)
-        return self
-
-    def issubset(self, other):
-        return self.data.issubset(ref(item) for item in other)
-    __le__ = issubset
-
-    def __lt__(self, other):
-        return self.data < set(map(ref, other))
-
-    def issuperset(self, other):
-        return self.data.issuperset(ref(item) for item in other)
-    __ge__ = issuperset
-
-    def __gt__(self, other):
-        return self.data > set(map(ref, other))
-
-    def __eq__(self, other):
-        if not isinstance(other, self.__class__):
-            return NotImplemented
-        return self.data == set(map(ref, other))
-
-    def symmetric_difference(self, other):
-        newset = self.copy()
-        newset.symmetric_difference_update(other)
-        return newset
-    __xor__ = symmetric_difference
-
-    def symmetric_difference_update(self, other):
-        self.__ixor__(other)
-    def __ixor__(self, other):
-        if self._pending_removals:
-            self._commit_removals()
-        if self is other:
-            self.data.clear()
-        else:
-            self.data.symmetric_difference_update(ref(item, self._remove) for item in other)
-        return self
-
-    def union(self, other):
-        return self.__class__(e for s in (self, other) for e in s)
-    __or__ = union
-
-    def isdisjoint(self, other):
-        return len(self.intersection(other)) == 0

Lib/abc.py

@@ -1,170 +0,0 @@
-# Copyright 2007 Google, Inc. All Rights Reserved.
-# Licensed to PSF under a Contributor Agreement.
-
-"""Abstract Base Classes (ABCs) according to PEP 3119."""
-
-
-def abstractmethod(funcobj):
-    """A decorator indicating abstract methods.
-
-    Requires that the metaclass is ABCMeta or derived from it.  A
-    class that has a metaclass derived from ABCMeta cannot be
-    instantiated unless all of its abstract methods are overridden.
-    The abstract methods can be called using any of the normal
-    'super' call mechanisms.
-
-    Usage:
-
-        class C(metaclass=ABCMeta):
-            @abstractmethod
-            def my_abstract_method(self, ...):
-                ...
-    """
-    funcobj.__isabstractmethod__ = True
-    return funcobj
-
-
-class abstractclassmethod(classmethod):
-    """A decorator indicating abstract classmethods.
-
-    Similar to abstractmethod.
-
-    Usage:
-
-        class C(metaclass=ABCMeta):
-            @abstractclassmethod
-            def my_abstract_classmethod(cls, ...):
-                ...
-
-    'abstractclassmethod' is deprecated. Use 'classmethod' with
-    'abstractmethod' instead.
-    """
-
-    __isabstractmethod__ = True
-
-    def __init__(self, callable):
-        callable.__isabstractmethod__ = True
-        super().__init__(callable)
-
-
-class abstractstaticmethod(staticmethod):
-    """A decorator indicating abstract staticmethods.
-
-    Similar to abstractmethod.
-
-    Usage:
-
-        class C(metaclass=ABCMeta):
-            @abstractstaticmethod
-            def my_abstract_staticmethod(...):
-                ...
-
-    'abstractstaticmethod' is deprecated. Use 'staticmethod' with
-    'abstractmethod' instead.
-    """
-
-    __isabstractmethod__ = True
-
-    def __init__(self, callable):
-        callable.__isabstractmethod__ = True
-        super().__init__(callable)
-
-
-class abstractproperty(property):
-    """A decorator indicating abstract properties.
-
-    Requires that the metaclass is ABCMeta or derived from it.  A
-    class that has a metaclass derived from ABCMeta cannot be
-    instantiated unless all of its abstract properties are overridden.
-    The abstract properties can be called using any of the normal
-    'super' call mechanisms.
-
-    Usage:
-
-        class C(metaclass=ABCMeta):
-            @abstractproperty
-            def my_abstract_property(self):
-                ...
-
-    This defines a read-only property; you can also define a read-write
-    abstract property using the 'long' form of property declaration:
-
-        class C(metaclass=ABCMeta):
-            def getx(self): ...
-            def setx(self, value): ...
-            x = abstractproperty(getx, setx)
-
-    'abstractproperty' is deprecated. Use 'property' with 'abstractmethod'
-    instead.
-    """
-
-    __isabstractmethod__ = True
-
-
-try:
-    from _abc import (get_cache_token, _abc_init, _abc_register,
-                      _abc_instancecheck, _abc_subclasscheck, _get_dump,
-                      _reset_registry, _reset_caches)
-except ImportError:
-    from _py_abc import ABCMeta, get_cache_token
-    ABCMeta.__module__ = 'abc'
-else:
-    class ABCMeta(type):
-        """Metaclass for defining Abstract Base Classes (ABCs).
-
-        Use this metaclass to create an ABC.  An ABC can be subclassed
-        directly, and then acts as a mix-in class.  You can also register
-        unrelated concrete classes (even built-in classes) and unrelated
-        ABCs as 'virtual subclasses' -- these and their descendants will
-        be considered subclasses of the registering ABC by the built-in
-        issubclass() function, but the registering ABC won't show up in
-        their MRO (Method Resolution Order) nor will method
-        implementations defined by the registering ABC be callable (not
-        even via super()).
-        """
-        def __new__(mcls, name, bases, namespace, **kwargs):
-            cls = super().__new__(mcls, name, bases, namespace, **kwargs)
-            _abc_init(cls)
-            return cls
-
-        def register(cls, subclass):
-            """Register a virtual subclass of an ABC.
-
-            Returns the subclass, to allow usage as a class decorator.
-            """
-            return _abc_register(cls, subclass)
-
-        def __instancecheck__(cls, instance):
-            """Override for isinstance(instance, cls)."""
-            return _abc_instancecheck(cls, instance)
-
-        def __subclasscheck__(cls, subclass):
-            """Override for issubclass(subclass, cls)."""
-            return _abc_subclasscheck(cls, subclass)
-
-        def _dump_registry(cls, file=None):
-            """Debug helper to print the ABC registry."""
-            print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file)
-            print(f"Inv. counter: {get_cache_token()}", file=file)
-            (_abc_registry, _abc_cache, _abc_negative_cache,
-             _abc_negative_cache_version) = _get_dump(cls)
-            print(f"_abc_registry: {_abc_registry!r}", file=file)
-            print(f"_abc_cache: {_abc_cache!r}", file=file)
-            print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file)
-            print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}",
-                  file=file)
-
-        def _abc_registry_clear(cls):
-            """Clear the registry (for debugging or testing)."""
-            _reset_registry(cls)
-
-        def _abc_caches_clear(cls):
-            """Clear the caches (for debugging or testing)."""
-            _reset_caches(cls)
-
-
-class ABC(metaclass=ABCMeta):
-    """Helper class that provides a standard way to create an ABC using
-    inheritance.
-    """
-    __slots__ = ()

Lib/collections/abc.py

@@ -1,2 +0,0 @@
-from _collections_abc import *
-from _collections_abc import __all__

Lib/functools.py

@@ -1,828 +0,0 @@
-"""functools.py - Tools for working with functions and callable objects
-"""
-# Python module wrapper for _functools C module
-# to allow utilities written in Python to be added
-# to the functools module.
-# Written by Nick Coghlan <ncoghlan at gmail.com>,
-# Raymond Hettinger <python at rcn.com>,
-# and Łukasz Langa <lukasz at langa.pl>.
-#   Copyright (C) 2006-2013 Python Software Foundation.
-# See C source code for _functools credits/copyright
-
-__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
-           'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial',
-           'partialmethod', 'singledispatch']
-
-try:
-    from _functools import reduce
-except ImportError:
-    pass
-from abc import get_cache_token
-from collections import namedtuple
-# import types, weakref  # Deferred to single_dispatch()
-from reprlib import recursive_repr
-from _thread import RLock
-
-
-################################################################################
-### update_wrapper() and wraps() decorator
-################################################################################
-
-# update_wrapper() and wraps() are tools to help write
-# wrapper functions that can handle naive introspection
-
-WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__',
-                       '__annotations__')
-WRAPPER_UPDATES = ('__dict__',)
-def update_wrapper(wrapper,
-                   wrapped,
-                   assigned = WRAPPER_ASSIGNMENTS,
-                   updated = WRAPPER_UPDATES):
-    """Update a wrapper function to look like the wrapped function
-
-       wrapper is the function to be updated
-       wrapped is the original function
-       assigned is a tuple naming the attributes assigned directly
-       from the wrapped function to the wrapper function (defaults to
-       functools.WRAPPER_ASSIGNMENTS)
-       updated is a tuple naming the attributes of the wrapper that
-       are updated with the corresponding attribute from the wrapped
-       function (defaults to functools.WRAPPER_UPDATES)
-    """
-    for attr in assigned:
-        try:
-            value = getattr(wrapped, attr)
-        except AttributeError:
-            pass
-        else:
-            setattr(wrapper, attr, value)
-    for attr in updated:
-        getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
-    # Issue #17482: set __wrapped__ last so we don't inadvertently copy it
-    # from the wrapped function when updating __dict__
-    wrapper.__wrapped__ = wrapped
-    # Return the wrapper so this can be used as a decorator via partial()
-    return wrapper
-
-def wraps(wrapped,
-          assigned = WRAPPER_ASSIGNMENTS,
-          updated = WRAPPER_UPDATES):
-    """Decorator factory to apply update_wrapper() to a wrapper function
-
-       Returns a decorator that invokes update_wrapper() with the decorated
-       function as the wrapper argument and the arguments to wraps() as the
-       remaining arguments. Default arguments are as for update_wrapper().
-       This is a convenience function to simplify applying partial() to
-       update_wrapper().
-    """
-    return partial(update_wrapper, wrapped=wrapped,
-                   assigned=assigned, updated=updated)
-
-
-################################################################################
-### total_ordering class decorator
-################################################################################
-
-# The total ordering functions all invoke the root magic method directly
-# rather than using the corresponding operator.  This avoids possible
-# infinite recursion that could occur when the operator dispatch logic
-# detects a NotImplemented result and then calls a reflected method.
-
-def _gt_from_lt(self, other, NotImplemented=NotImplemented):
-    'Return a > b.  Computed by @total_ordering from (not a < b) and (a != b).'
-    op_result = self.__lt__(other)
-    if op_result is NotImplemented:
-        return op_result
-    return not op_result and self != other
-
-def _le_from_lt(self, other, NotImplemented=NotImplemented):
-    'Return a <= b.  Computed by @total_ordering from (a < b) or (a == b).'
-    op_result = self.__lt__(other)
-    return op_result or self == other
-
-def _ge_from_lt(self, other, NotImplemented=NotImplemented):
-    'Return a >= b.  Computed by @total_ordering from (not a < b).'
-    op_result = self.__lt__(other)
-    if op_result is NotImplemented:
-        return op_result
-    return not op_result
-
-def _ge_from_le(self, other, NotImplemented=NotImplemented):
-    'Return a >= b.  Computed by @total_ordering from (not a <= b) or (a == b).'
-    op_result = self.__le__(other)
-    if op_result is NotImplemented:
-        return op_result
-    return not op_result or self == other
-
-def _lt_from_le(self, other, NotImplemented=NotImplemented):
-    'Return a < b.  Computed by @total_ordering from (a <= b) and (a != b).'
-    op_result = self.__le__(other)
-    if op_result is NotImplemented:
-        return op_result
-    return op_result and self != other
-
-def _gt_from_le(self, other, NotImplemented=NotImplemented):
-    'Return a > b.  Computed by @total_ordering from (not a <= b).'
-    op_result = self.__le__(other)
-    if op_result is NotImplemented:
-        return op_result
-    return not op_result
-
-def _lt_from_gt(self, other, NotImplemented=NotImplemented):
-    'Return a < b.  Computed by @total_ordering from (not a > b) and (a != b).'
-    op_result = self.__gt__(other)
-    if op_result is NotImplemented:
-        return op_result
-    return not op_result and self != other
-
-def _ge_from_gt(self, other, NotImplemented=NotImplemented):
-    'Return a >= b.  Computed by @total_ordering from (a > b) or (a == b).'
-    op_result = self.__gt__(other)
-    return op_result or self == other
-
-def _le_from_gt(self, other, NotImplemented=NotImplemented):
-    'Return a <= b.  Computed by @total_ordering from (not a > b).'
-    op_result = self.__gt__(other)
-    if op_result is NotImplemented:
-        return op_result
-    return not op_result
-
-def _le_from_ge(self, other, NotImplemented=NotImplemented):
-    'Return a <= b.  Computed by @total_ordering from (not a >= b) or (a == b).'
-    op_result = self.__ge__(other)
-    if op_result is NotImplemented:
-        return op_result
-    return not op_result or self == other
-
-def _gt_from_ge(self, other, NotImplemented=NotImplemented):
-    'Return a > b.  Computed by @total_ordering from (a >= b) and (a != b).'
-    op_result = self.__ge__(other)
-    if op_result is NotImplemented:
-        return op_result
-    return op_result and self != other
-
-def _lt_from_ge(self, other, NotImplemented=NotImplemented):
-    'Return a < b.  Computed by @total_ordering from (not a >= b).'
-    op_result = self.__ge__(other)
-    if op_result is NotImplemented:
-        return op_result
-    return not op_result
-
-_convert = {
-    '__lt__': [('__gt__', _gt_from_lt),
-               ('__le__', _le_from_lt),
-               ('__ge__', _ge_from_lt)],
-    '__le__': [('__ge__', _ge_from_le),
-               ('__lt__', _lt_from_le),
-               ('__gt__', _gt_from_le)],
-    '__gt__': [('__lt__', _lt_from_gt),
-               ('__ge__', _ge_from_gt),
-               ('__le__', _le_from_gt)],
-    '__ge__': [('__le__', _le_from_ge),
-               ('__gt__', _gt_from_ge),
-               ('__lt__', _lt_from_ge)]
-}
-
-def total_ordering(cls):
-    """Class decorator that fills in missing ordering methods"""
-    # Find user-defined comparisons (not those inherited from object).
-    roots = {op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)}
-    if not roots:
-        raise ValueError('must define at least one ordering operation: < > <= >=')
-    root = max(roots)       # prefer __lt__ to __le__ to __gt__ to __ge__
-    for opname, opfunc in _convert[root]:
-        if opname not in roots:
-            opfunc.__name__ = opname
-            setattr(cls, opname, opfunc)
-    return cls
-
-
-################################################################################
-### cmp_to_key() function converter
-################################################################################
-
-def cmp_to_key(mycmp):
-    """Convert a cmp= function into a key= function"""
-    class K(object):
-        __slots__ = ['obj']
-        def __init__(self, obj):
-            self.obj = obj
-        def __lt__(self, other):
-            return mycmp(self.obj, other.obj) < 0
-        def __gt__(self, other):
-            return mycmp(self.obj, other.obj) > 0
-        def __eq__(self, other):
-            return mycmp(self.obj, other.obj) == 0
-        def __le__(self, other):
-            return mycmp(self.obj, other.obj) <= 0
-        def __ge__(self, other):
-            return mycmp(self.obj, other.obj) >= 0
-        __hash__ = None
-    return K
-
-try:
-    from _functools import cmp_to_key
-except ImportError:
-    pass
-
-
-################################################################################
-### partial() argument application
-################################################################################
-
-# Purely functional, no descriptor behaviour
-class partial:
-    """New function with partial application of the given arguments
-    and keywords.
-    """
-
-    __slots__ = "func", "args", "keywords", "__dict__", "__weakref__"
-
-    def __new__(*args, **keywords):
-        if not args:
-            raise TypeError("descriptor '__new__' of partial needs an argument")
-        if len(args) < 2:
-            raise TypeError("type 'partial' takes at least one argument")
-        cls, func, *args = args
-        if not callable(func):
-            raise TypeError("the first argument must be callable")
-        args = tuple(args)
-
-        if hasattr(func, "func"):
-            args = func.args + args
-            tmpkw = func.keywords.copy()
-            tmpkw.update(keywords)
-            keywords = tmpkw
-            del tmpkw
-            func = func.func
-
-        self = super(partial, cls).__new__(cls)
-
-        self.func = func
-        self.args = args
-        self.keywords = keywords
-        return self
-
-    def __call__(*args, **keywords):
-        if not args:
-            raise TypeError("descriptor '__call__' of partial needs an argument")
-        self, *args = args
-        newkeywords = self.keywords.copy()
-        newkeywords.update(keywords)
-        return self.func(*self.args, *args, **newkeywords)
-
-    @recursive_repr()
-    def __repr__(self):
-        qualname = type(self).__qualname__
-        args = [repr(self.func)]
-        args.extend(repr(x) for x in self.args)
-        args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items())
-        if type(self).__module__ == "functools":
-            return f"functools.{qualname}({', '.join(args)})"
-        return f"{qualname}({', '.join(args)})"
-
-    def __reduce__(self):
-        return type(self), (self.func,), (self.func, self.args,
-               self.keywords or None, self.__dict__ or None)
-
-    def __setstate__(self, state):
-        if not isinstance(state, tuple):
-            raise TypeError("argument to __setstate__ must be a tuple")
-        if len(state) != 4:
-            raise TypeError(f"expected 4 items in state, got {len(state)}")
-        func, args, kwds, namespace = state
-        if (not callable(func) or not isinstance(args, tuple) or
-           (kwds is not None and not isinstance(kwds, dict)) or
-           (namespace is not None and not isinstance(namespace, dict))):
-            raise TypeError("invalid partial state")
-
-        args = tuple(args) # just in case it's a subclass
-        if kwds is None:
-            kwds = {}
-        elif type(kwds) is not dict: # XXX does it need to be *exactly* dict?
-            kwds = dict(kwds)
-        if namespace is None:
-            namespace = {}
-
-        self.__dict__ = namespace
-        self.func = func
-        self.args = args
-        self.keywords = kwds
-
-try:
-    from _functools import partial
-except ImportError:
-    pass
-
-# Descriptor version
-class partialmethod(object):
-    """Method descriptor with partial application of the given arguments
-    and keywords.
-
-    Supports wrapping existing descriptors and handles non-descriptor
-    callables as instance methods.
-    """
-
-    def __init__(self, func, *args, **keywords):
-        if not callable(func) and not hasattr(func, "__get__"):
-            raise TypeError("{!r} is not callable or a descriptor"
-                                 .format(func))
-
-        # func could be a descriptor like classmethod which isn't callable,
-        # so we can't inherit from partial (it verifies func is callable)
-        if isinstance(func, partialmethod):
-            # flattening is mandatory in order to place cls/self before all
-            # other arguments
-            # it's also more efficient since only one function will be called
-            self.func = func.func
-            self.args = func.args + args
-            self.keywords = func.keywords.copy()
-            self.keywords.update(keywords)
-        else:
-            self.func = func
-            self.args = args
-            self.keywords = keywords
-
-    def __repr__(self):
-        args = ", ".join(map(repr, self.args))
-        keywords = ", ".join("{}={!r}".format(k, v)
-                                 for k, v in self.keywords.items())
-        format_string = "{module}.{cls}({func}, {args}, {keywords})"
-        return format_string.format(module=self.__class__.__module__,
-                                    cls=self.__class__.__qualname__,
-                                    func=self.func,
-                                    args=args,
-                                    keywords=keywords)
-
-    def _make_unbound_method(self):
-        def _method(*args, **keywords):
-            call_keywords = self.keywords.copy()
-            call_keywords.update(keywords)
-            cls_or_self, *rest = args
-            call_args = (cls_or_self,) + self.args + tuple(rest)
-            return self.func(*call_args, **call_keywords)
-        _method.__isabstractmethod__ = self.__isabstractmethod__
-        _method._partialmethod = self
-        return _method
-
-    def __get__(self, obj, cls):
-        get = getattr(self.func, "__get__", None)
-        result = None
-        if get is not None:
-            new_func = get(obj, cls)
-            if new_func is not self.func:
-                # Assume __get__ returning something new indicates the
-                # creation of an appropriate callable
-                result = partial(new_func, *self.args, **self.keywords)
-                try:
-                    result.__self__ = new_func.__self__
-                except AttributeError:
-                    pass
-        if result is None:
-            # If the underlying descriptor didn't do anything, treat this
-            # like an instance method
-            result = self._make_unbound_method().__get__(obj, cls)
-        return result
-
-    @property
-    def __isabstractmethod__(self):
-        return getattr(self.func, "__isabstractmethod__", False)
-
-
-################################################################################
-### LRU Cache function decorator
-################################################################################
-
-_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
-
-class _HashedSeq(list):
-    """ This class guarantees that hash() will be called no more than once
-        per element.  This is important because the lru_cache() will hash
-        the key multiple times on a cache miss.
-
-    """
-
-    __slots__ = 'hashvalue'
-
-    def __init__(self, tup, hash=hash):
-        self[:] = tup
-        self.hashvalue = hash(tup)
-
-    def __hash__(self):
-        return self.hashvalue
-
-def _make_key(args, kwds, typed,
-             kwd_mark = (object(),),
-             fasttypes = {int, str, frozenset, type(None)},
-             tuple=tuple, type=type, len=len):
-    """Make a cache key from optionally typed positional and keyword arguments
-
-    The key is constructed in a way that is flat as possible rather than
-    as a nested structure that would take more memory.
-
-    If there is only a single argument and its data type is known to cache
-    its hash value, then that argument is returned without a wrapper.  This
-    saves space and improves lookup speed.
-
-    """
-    # All of code below relies on kwds preserving the order input by the user.
-    # Formerly, we sorted() the kwds before looping.  The new way is *much*
-    # faster; however, it means that f(x=1, y=2) will now be treated as a
-    # distinct call from f(y=2, x=1) which will be cached separately.
-    key = args
-    if kwds:
-        key += kwd_mark
-        for item in kwds.items():
-            key += item
-    if typed:
-        key += tuple(type(v) for v in args)
-        if kwds:
-            key += tuple(type(v) for v in kwds.values())
-    elif len(key) == 1 and type(key[0]) in fasttypes:
-        return key[0]
-    return _HashedSeq(key)
-
-def lru_cache(maxsize=128, typed=False):
-    """Least-recently-used cache decorator.
-
-    If *maxsize* is set to None, the LRU features are disabled and the cache
-    can grow without bound.
-
-    If *typed* is True, arguments of different types will be cached separately.
-    For example, f(3.0) and f(3) will be treated as distinct calls with
-    distinct results.
-
-    Arguments to the cached function must be hashable.
-
-    View the cache statistics named tuple (hits, misses, maxsize, currsize)
-    with f.cache_info().  Clear the cache and statistics with f.cache_clear().
-    Access the underlying function with f.__wrapped__.
-
-    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
-
-    """
-
-    # Users should only access the lru_cache through its public API:
-    #       cache_info, cache_clear, and f.__wrapped__
-    # The internals of the lru_cache are encapsulated for thread safety and
-    # to allow the implementation to change (including a possible C version).
-
-    # Early detection of an erroneous call to @lru_cache without any arguments
-    # resulting in the inner function being passed to maxsize instead of an
-    # integer or None.
-    if maxsize is not None and not isinstance(maxsize, int):
-        raise TypeError('Expected maxsize to be an integer or None')
-
-    def decorating_function(user_function):
-        wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo)
-        return update_wrapper(wrapper, user_function)
-
-    return decorating_function
-
-def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo):
-    # Constants shared by all lru cache instances:
-    sentinel = object()          # unique object used to signal cache misses
-    make_key = _make_key         # build a key from the function arguments
-    PREV, NEXT, KEY, RESULT = 0, 1, 2, 3   # names for the link fields
-
-    cache = {}
-    hits = misses = 0
-    full = False
-    cache_get = cache.get    # bound method to lookup a key or return None
-    cache_len = cache.__len__  # get cache size without calling len()
-    lock = RLock()           # because linkedlist updates aren't threadsafe
-    root = []                # root of the circular doubly linked list
-    root[:] = [root, root, None, None]     # initialize by pointing to self
-
-    if maxsize == 0:
-
-        def wrapper(*args, **kwds):
-            # No caching -- just a statistics update after a successful call
-            nonlocal misses
-            result = user_function(*args, **kwds)
-            misses += 1
-            return result
-
-    elif maxsize is None:
-
-        def wrapper(*args, **kwds):
-            # Simple caching without ordering or size limit
-            nonlocal hits, misses
-            key = make_key(args, kwds, typed)
-            result = cache_get(key, sentinel)
-            if result is not sentinel:
-                hits += 1
-                return result
-            result = user_function(*args, **kwds)
-            cache[key] = result
-            misses += 1
-            return result
-
-    else:
-
-        def wrapper(*args, **kwds):
-            # Size limited caching that tracks accesses by recency
-            nonlocal root, hits, misses, full
-            key = make_key(args, kwds, typed)
-            with lock:
-                link = cache_get(key)
-                if link is not None:
-                    # Move the link to the front of the circular queue
-                    link_prev, link_next, _key, result = link
-                    link_prev[NEXT] = link_next
-                    link_next[PREV] = link_prev
-                    last = root[PREV]
-                    last[NEXT] = root[PREV] = link
-                    link[PREV] = last
-                    link[NEXT] = root
-                    hits += 1
-                    return result
-            result = user_function(*args, **kwds)
-            with lock:
-                if key in cache:
-                    # Getting here means that this same key was added to the
-                    # cache while the lock was released.  Since the link
-                    # update is already done, we need only return the
-                    # computed result and update the count of misses.
-                    pass
-                elif full:
-                    # Use the old root to store the new key and result.
-                    oldroot = root
-                    oldroot[KEY] = key
-                    oldroot[RESULT] = result
-                    # Empty the oldest link and make it the new root.
-                    # Keep a reference to the old key and old result to
-                    # prevent their ref counts from going to zero during the
-                    # update. That will prevent potentially arbitrary object
-                    # clean-up code (i.e. __del__) from running while we're
-                    # still adjusting the links.
-                    root = oldroot[NEXT]
-                    oldkey = root[KEY]
-                    oldresult = root[RESULT]
-                    root[KEY] = root[RESULT] = None
-                    # Now update the cache dictionary.
-                    del cache[oldkey]
-                    # Save the potentially reentrant cache[key] assignment
-                    # for last, after the root and links have been put in
-                    # a consistent state.
-                    cache[key] = oldroot
-                else:
-                    # Put result in a new link at the front of the queue.
-                    last = root[PREV]
-                    link = [last, root, key, result]
-                    last[NEXT] = root[PREV] = cache[key] = link
-                    # Use the cache_len bound method instead of the len() function
-                    # which could potentially be wrapped in an lru_cache itself.
-                    full = (cache_len() >= maxsize)
-                misses += 1
-            return result
-
-    def cache_info():
-        """Report cache statistics"""
-        with lock:
-            return _CacheInfo(hits, misses, maxsize, cache_len())
-
-    def cache_clear():
-        """Clear the cache and cache statistics"""
-        nonlocal hits, misses, full
-        with lock:
-            cache.clear()
-            root[:] = [root, root, None, None]
-            hits = misses = 0
-            full = False
-
-    wrapper.cache_info = cache_info
-    wrapper.cache_clear = cache_clear
-    return wrapper
-
-try:
-    from _functools import _lru_cache_wrapper
-except ImportError:
-    pass
-
-
-################################################################################
-### singledispatch() - single-dispatch generic function decorator
-################################################################################
-
-def _c3_merge(sequences):
-    """Merges MROs in *sequences* to a single MRO using the C3 algorithm.
-
-    Adapted from http://www.python.org/download/releases/2.3/mro/.
-
-    """
-    result = []
-    while True:
-        sequences = [s for s in sequences if s]   # purge empty sequences
-        if not sequences:
-            return result
-        for s1 in sequences:   # find merge candidates among seq heads
-            candidate = s1[0]
-            for s2 in sequences:
-                if candidate in s2[1:]:
-                    candidate = None
-                    break      # reject the current head, it appears later
-            else:
-                break
-        if candidate is None:
-            raise RuntimeError("Inconsistent hierarchy")
-        result.append(candidate)
-        # remove the chosen candidate
-        for seq in sequences:
-            if seq[0] == candidate:
-                del seq[0]
-
-def _c3_mro(cls, abcs=None):
-    """Computes the method resolution order using extended C3 linearization.
-
-    If no *abcs* are given, the algorithm works exactly like the built-in C3
-    linearization used for method resolution.
-
-    If given, *abcs* is a list of abstract base classes that should be inserted
-    into the resulting MRO. Unrelated ABCs are ignored and don't end up in the
-    result. The algorithm inserts ABCs where their functionality is introduced,
-    i.e. issubclass(cls, abc) returns True for the class itself but returns
-    False for all its direct base classes. Implicit ABCs for a given class
-    (either registered or inferred from the presence of a special method like
-    __len__) are inserted directly after the last ABC explicitly listed in the
-    MRO of said class. If two implicit ABCs end up next to each other in the
-    resulting MRO, their ordering depends on the order of types in *abcs*.
-
-    """
-    for i, base in enumerate(reversed(cls.__bases__)):
-        if hasattr(base, '__abstractmethods__'):
-            boundary = len(cls.__bases__) - i
-            break   # Bases up to the last explicit ABC are considered first.
-    else:
-        boundary = 0
-    abcs = list(abcs) if abcs else []
-    explicit_bases = list(cls.__bases__[:boundary])
-    abstract_bases = []
-    other_bases = list(cls.__bases__[boundary:])
-    for base in abcs:
-        if issubclass(cls, base) and not any(
-                issubclass(b, base) for b in cls.__bases__
-            ):
-            # If *cls* is the class that introduces behaviour described by
-            # an ABC *base*, insert said ABC to its MRO.
-            abstract_bases.append(base)
-    for base in abstract_bases:
-        abcs.remove(base)
-    explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases]
-    abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases]
-    other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases]
-    return _c3_merge(
-        [[cls]] +
-        explicit_c3_mros + abstract_c3_mros + other_c3_mros +
-        [explicit_bases] + [abstract_bases] + [other_bases]
-    )
-
-def _compose_mro(cls, types):
-    """Calculates the method resolution order for a given class *cls*.
-
-    Includes relevant abstract base classes (with their respective bases) from
-    the *types* iterable. Uses a modified C3 linearization algorithm.
-
-    """
-    bases = set(cls.__mro__)
-    # Remove entries which are already present in the __mro__ or unrelated.
-    def is_related(typ):
-        return (typ not in bases and hasattr(typ, '__mro__')
-                                 and issubclass(cls, typ))
-    types = [n for n in types if is_related(n)]
-    # Remove entries which are strict bases of other entries (they will end up
-    # in the MRO anyway.
-    def is_strict_base(typ):
-        for other in types:
-            if typ != other and typ in other.__mro__:
-                return True
-        return False
-    types = [n for n in types if not is_strict_base(n)]
-    # Subclasses of the ABCs in *types* which are also implemented by
-    # *cls* can be used to stabilize ABC ordering.
-    type_set = set(types)
-    mro = []
-    for typ in types:
-        found = []
-        for sub in typ.__subclasses__():
-            if sub not in bases and issubclass(cls, sub):
-                found.append([s for s in sub.__mro__ if s in type_set])
-        if not found:
-            mro.append(typ)
-            continue
-        # Favor subclasses with the biggest number of useful bases
-        found.sort(key=len, reverse=True)
-        for sub in found:
-            for subcls in sub:
-                if subcls not in mro:
-                    mro.append(subcls)
-    return _c3_mro(cls, abcs=mro)
-
-def _find_impl(cls, registry):
-    """Returns the best matching implementation from *registry* for type *cls*.
-
-    Where there is no registered implementation for a specific type, its method
-    resolution order is used to find a more generic implementation.
-
-    Note: if *registry* does not contain an implementation for the base
-    *object* type, this function may return None.
-
-    """
-    mro = _compose_mro(cls, registry.keys())
-    match = None
-    for t in mro:
-        if match is not None:
-            # If *match* is an implicit ABC but there is another unrelated,
-            # equally matching implicit ABC, refuse the temptation to guess.
-            if (t in registry and t not in cls.__mro__
-                              and match not in cls.__mro__
-                              and not issubclass(match, t)):
-                raise RuntimeError("Ambiguous dispatch: {} or {}".format(
-                    match, t))
-            break
-        if t in registry:
-            match = t
-    return registry.get(match)
-
-def singledispatch(func):
-    """Single-dispatch generic function decorator.
-
-    Transforms a function into a generic function, which can have different
-    behaviours depending upon the type of its first argument. The decorated
-    function acts as the default implementation, and additional
-    implementations can be registered using the register() attribute of the
-    generic function.
-    """
-    # There are many programs that use functools without singledispatch, so we
-    # trade-off making singledispatch marginally slower for the benefit of
-    # making start-up of such applications slightly faster.
-    import types, weakref
-
-    registry = {}
-    dispatch_cache = weakref.WeakKeyDictionary()
-    cache_token = None
-
-    def dispatch(cls):
-        """generic_func.dispatch(cls) -> <function implementation>
-
-        Runs the dispatch algorithm to return the best available implementation
-        for the given *cls* registered on *generic_func*.
-
-        """
-        nonlocal cache_token
-        if cache_token is not None:
-            current_token = get_cache_token()
-            if cache_token != current_token:
-                dispatch_cache.clear()
-                cache_token = current_token
-        try:
-            impl = dispatch_cache[cls]
-        except KeyError:
-            try:
-                impl = registry[cls]
-            except KeyError:
-                impl = _find_impl(cls, registry)
-            dispatch_cache[cls] = impl
-        return impl
-
-    def register(cls, func=None):
-        """generic_func.register(cls, func) -> func
-
-        Registers a new implementation for the given *cls* on a *generic_func*.
-
-        """
-        nonlocal cache_token
-        if func is None:
-            if isinstance(cls, type):
-                return lambda f: register(cls, f)
-            ann = getattr(cls, '__annotations__', {})
-            if not ann:
-                raise TypeError(
-                    f"Invalid first argument to `register()`: {cls!r}. "
-                    f"Use either `@register(some_class)` or plain `@register` "
-                    f"on an annotated function."
-                )
-            func = cls
-
-            # only import typing if annotation parsing is necessary
-            from typing import get_type_hints
-            argname, cls = next(iter(get_type_hints(func).items()))
-            assert isinstance(cls, type), (
-                f"Invalid annotation for {argname!r}. {cls!r} is not a class."
-            )
-        registry[cls] = func
-        if cache_token is None and hasattr(cls, '__abstractmethods__'):
-            cache_token = get_cache_token()
-        dispatch_cache.clear()
-        return func
-
-    def wrapper(*args, **kw):
-        return dispatch(args[0].__class__)(*args, **kw)
-
-    registry[object] = func
-    wrapper.register = register
-    wrapper.dispatch = dispatch
-    wrapper.registry = types.MappingProxyType(registry)
-    wrapper._clear_cache = dispatch_cache.clear
-    update_wrapper(wrapper, func)
-    return wrapper

Lib/genericpath.py

@@ -1,151 +0,0 @@
-"""
-Path operations common to more than one OS
-Do not use directly.  The OS specific modules import the appropriate
-functions from this module themselves.
-"""
-import os
-import stat
-
-__all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime',
-           'getsize', 'isdir', 'isfile', 'samefile', 'sameopenfile',
-           'samestat']
-
-
-# Does a path exist?
-# This is false for dangling symbolic links on systems that support them.
-def exists(path):
-    """Test whether a path exists.  Returns False for broken symbolic links"""
-    try:
-        os.stat(path)
-    except (OSError, ValueError):
-        return False
-    return True
-
-
-# This follows symbolic links, so both islink() and isdir() can be true
-# for the same path on systems that support symlinks
-def isfile(path):
-    """Test whether a path is a regular file"""
-    try:
-        st = os.stat(path)
-    except (OSError, ValueError):
-        return False
-    return stat.S_ISREG(st.st_mode)
-
-
-# Is a path a directory?
-# This follows symbolic links, so both islink() and isdir()
-# can be true for the same path on systems that support symlinks
-def isdir(s):
-    """Return true if the pathname refers to an existing directory."""
-    try:
-        st = os.stat(s)
-    except (OSError, ValueError):
-        return False
-    return stat.S_ISDIR(st.st_mode)
-
-
-def getsize(filename):
-    """Return the size of a file, reported by os.stat()."""
-    return os.stat(filename).st_size
-
-
-def getmtime(filename):
-    """Return the last modification time of a file, reported by os.stat()."""
-    return os.stat(filename).st_mtime
-
-
-def getatime(filename):
-    """Return the last access time of a file, reported by os.stat()."""
-    return os.stat(filename).st_atime
-
-
-def getctime(filename):
-    """Return the metadata change time of a file, reported by os.stat()."""
-    return os.stat(filename).st_ctime
-
-
-# Return the longest prefix of all list elements.
-def commonprefix(m):
-    "Given a list of pathnames, returns the longest common leading component"
-    if not m: return ''
-    # Some people pass in a list of pathname parts to operate in an OS-agnostic
-    # fashion; don't try to translate in that case as that's an abuse of the
-    # API and they are already doing what they need to be OS-agnostic and so
-    # they most likely won't be using an os.PathLike object in the sublists.
-    if not isinstance(m[0], (list, tuple)):
-        m = tuple(map(os.fspath, m))
-    s1 = min(m)
-    s2 = max(m)
-    for i, c in enumerate(s1):
-        if c != s2[i]:
-            return s1[:i]
-    return s1
-
-# Are two stat buffers (obtained from stat, fstat or lstat)
-# describing the same file?
-def samestat(s1, s2):
-    """Test whether two stat buffers reference the same file"""
-    return (s1.st_ino == s2.st_ino and
-            s1.st_dev == s2.st_dev)
-
-
-# Are two filenames really pointing to the same file?
-def samefile(f1, f2):
-    """Test whether two pathnames reference the same actual file"""
-    s1 = os.stat(f1)
-    s2 = os.stat(f2)
-    return samestat(s1, s2)
-
-
-# Are two open files really referencing the same file?
-# (Not necessarily the same file descriptor!)
-def sameopenfile(fp1, fp2):
-    """Test whether two open file objects reference the same file"""
-    s1 = os.fstat(fp1)
-    s2 = os.fstat(fp2)
-    return samestat(s1, s2)
-
-
-# Split a path in root and extension.
-# The extension is everything starting at the last dot in the last
-# pathname component; the root is everything before that.
-# It is always true that root + ext == p.
-
-# Generic implementation of splitext, to be parametrized with
-# the separators
-def _splitext(p, sep, altsep, extsep):
-    """Split the extension from a pathname.
-
-    Extension is everything from the last dot to the end, ignoring
-    leading dots.  Returns "(root, ext)"; ext may be empty."""
-    # NOTE: This code must work for text and bytes strings.
-
-    sepIndex = p.rfind(sep)
-    if altsep:
-        altsepIndex = p.rfind(altsep)
-        sepIndex = max(sepIndex, altsepIndex)
-
-    dotIndex = p.rfind(extsep)
-    if dotIndex > sepIndex:
-        # skip all leading dots
-        filenameIndex = sepIndex + 1
-        while filenameIndex < dotIndex:
-            if p[filenameIndex:filenameIndex+1] != extsep:
-                return p[:dotIndex], p[dotIndex:]
-            filenameIndex += 1
-
-    return p, p[:0]
-
-def _check_arg_types(funcname, *args):
-    hasstr = hasbytes = False
-    for s in args:
-        if isinstance(s, str):
-            hasstr = True
-        elif isinstance(s, bytes):
-            hasbytes = True
-        else:
-            raise TypeError('%s() argument must be str or bytes, not %r' %
-                            (funcname, s.__class__.__name__)) from None
-    if hasstr and hasbytes:
-        raise TypeError("Can't mix strings and bytes in path components") from None

Lib/heapq.py

@@ -1,601 +0,0 @@
-"""Heap queue algorithm (a.k.a. priority queue).
-
-Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
-all k, counting elements from 0.  For the sake of comparison,
-non-existing elements are considered to be infinite.  The interesting
-property of a heap is that a[0] is always its smallest element.
-
-Usage:
-
-heap = []            # creates an empty heap
-heappush(heap, item) # pushes a new item on the heap
-item = heappop(heap) # pops the smallest item from the heap
-item = heap[0]       # smallest item on the heap without popping it
-heapify(x)           # transforms list into a heap, in-place, in linear time
-item = heapreplace(heap, item) # pops and returns smallest item, and adds
-                               # new item; the heap size is unchanged
-
-Our API differs from textbook heap algorithms as follows:
-
-- We use 0-based indexing.  This makes the relationship between the
-  index for a node and the indexes for its children slightly less
-  obvious, but is more suitable since Python uses 0-based indexing.
-
-- Our heappop() method returns the smallest item, not the largest.
-
-These two make it possible to view the heap as a regular Python list
-without surprises: heap[0] is the smallest item, and heap.sort()
-maintains the heap invariant!
-"""
-
-# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
-
-__about__ = """Heap queues
-
-[explanation by François Pinard]
-
-Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
-all k, counting elements from 0.  For the sake of comparison,
-non-existing elements are considered to be infinite.  The interesting
-property of a heap is that a[0] is always its smallest element.
-
-The strange invariant above is meant to be an efficient memory
-representation for a tournament.  The numbers below are `k', not a[k]:
-
-                                   0
-
-                  1                                 2
-
-          3               4                5               6
-
-      7       8       9       10      11      12      13      14
-
-    15 16   17 18   19 20   21 22   23 24   25 26   27 28   29 30
-
-
-In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'.  In
-a usual binary tournament we see in sports, each cell is the winner
-over the two cells it tops, and we can trace the winner down the tree
-to see all opponents s/he had.  However, in many computer applications
-of such tournaments, we do not need to trace the history of a winner.
-To be more memory efficient, when a winner is promoted, we try to
-replace it by something else at a lower level, and the rule becomes
-that a cell and the two cells it tops contain three different items,
-but the top cell "wins" over the two topped cells.
-
-If this heap invariant is protected at all time, index 0 is clearly
-the overall winner.  The simplest algorithmic way to remove it and
-find the "next" winner is to move some loser (let's say cell 30 in the
-diagram above) into the 0 position, and then percolate this new 0 down
-the tree, exchanging values, until the invariant is re-established.
-This is clearly logarithmic on the total number of items in the tree.
-By iterating over all items, you get an O(n ln n) sort.
-
-A nice feature of this sort is that you can efficiently insert new
-items while the sort is going on, provided that the inserted items are
-not "better" than the last 0'th element you extracted.  This is
-especially useful in simulation contexts, where the tree holds all
-incoming events, and the "win" condition means the smallest scheduled
-time.  When an event schedule other events for execution, they are
-scheduled into the future, so they can easily go into the heap.  So, a
-heap is a good structure for implementing schedulers (this is what I
-used for my MIDI sequencer :-).
-
-Various structures for implementing schedulers have been extensively
-studied, and heaps are good for this, as they are reasonably speedy,
-the speed is almost constant, and the worst case is not much different
-than the average case.  However, there are other representations which
-are more efficient overall, yet the worst cases might be terrible.
-
-Heaps are also very useful in big disk sorts.  You most probably all
-know that a big sort implies producing "runs" (which are pre-sorted
-sequences, which size is usually related to the amount of CPU memory),
-followed by a merging passes for these runs, which merging is often
-very cleverly organised[1].  It is very important that the initial
-sort produces the longest runs possible.  Tournaments are a good way
-to that.  If, using all the memory available to hold a tournament, you
-replace and percolate items that happen to fit the current run, you'll
-produce runs which are twice the size of the memory for random input,
-and much better for input fuzzily ordered.
-
-Moreover, if you output the 0'th item on disk and get an input which
-may not fit in the current tournament (because the value "wins" over
-the last output value), it cannot fit in the heap, so the size of the
-heap decreases.  The freed memory could be cleverly reused immediately
-for progressively building a second heap, which grows at exactly the
-same rate the first heap is melting.  When the first heap completely
-vanishes, you switch heaps and start a new run.  Clever and quite
-effective!
-
-In a word, heaps are useful memory structures to know.  I use them in
-a few applications, and I think it is good to keep a `heap' module
-around. :-)
-
---------------------
-[1] The disk balancing algorithms which are current, nowadays, are
-more annoying than clever, and this is a consequence of the seeking
-capabilities of the disks.  On devices which cannot seek, like big
-tape drives, the story was quite different, and one had to be very
-clever to ensure (far in advance) that each tape movement will be the
-most effective possible (that is, will best participate at
-"progressing" the merge).  Some tapes were even able to read
-backwards, and this was also used to avoid the rewinding time.
-Believe me, real good tape sorts were quite spectacular to watch!
-From all times, sorting has always been a Great Art! :-)
-"""
-
-__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
-           'nlargest', 'nsmallest', 'heappushpop']
-
-def heappush(heap, item):
-    """Push item onto heap, maintaining the heap invariant."""
-    heap.append(item)
-    _siftdown(heap, 0, len(heap)-1)
-
-def heappop(heap):
-    """Pop the smallest item off the heap, maintaining the heap invariant."""
-    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
-    if heap:
-        returnitem = heap[0]
-        heap[0] = lastelt
-        _siftup(heap, 0)
-        return returnitem
-    return lastelt
-
-def heapreplace(heap, item):
-    """Pop and return the current smallest value, and add the new item.
-
-    This is more efficient than heappop() followed by heappush(), and can be
-    more appropriate when using a fixed-size heap.  Note that the value
-    returned may be larger than item!  That constrains reasonable uses of
-    this routine unless written as part of a conditional replacement:
-
-        if item > heap[0]:
-            item = heapreplace(heap, item)
-    """
-    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
-    heap[0] = item
-    _siftup(heap, 0)
-    return returnitem
-
-def heappushpop(heap, item):
-    """Fast version of a heappush followed by a heappop."""
-    if heap and heap[0] < item:
-        item, heap[0] = heap[0], item
-        _siftup(heap, 0)
-    return item
-
-def heapify(x):
-    """Transform list into a heap, in-place, in O(len(x)) time."""
-    n = len(x)
-    # Transform bottom-up.  The largest index there's any point to looking at
-    # is the largest with a child index in-range, so must have 2*i + 1 < n,
-    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
-    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is
-    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
-    for i in reversed(range(n//2)):
-        _siftup(x, i)
-
-def _heappop_max(heap):
-    """Maxheap version of a heappop."""
-    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
-    if heap:
-        returnitem = heap[0]
-        heap[0] = lastelt
-        _siftup_max(heap, 0)
-        return returnitem
-    return lastelt
-
-def _heapreplace_max(heap, item):
-    """Maxheap version of a heappop followed by a heappush."""
-    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
-    heap[0] = item
-    _siftup_max(heap, 0)
-    return returnitem
-
-def _heapify_max(x):
-    """Transform list into a maxheap, in-place, in O(len(x)) time."""
-    n = len(x)
-    for i in reversed(range(n//2)):
-        _siftup_max(x, i)
-
-# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
-# is the index of a leaf with a possibly out-of-order value.  Restore the
-# heap invariant.
-def _siftdown(heap, startpos, pos):
-    newitem = heap[pos]
-    # Follow the path to the root, moving parents down until finding a place
-    # newitem fits.
-    while pos > startpos:
-        parentpos = (pos - 1) >> 1
-        parent = heap[parentpos]
-        if newitem < parent:
-            heap[pos] = parent
-            pos = parentpos
-            continue
-        break
-    heap[pos] = newitem
-
-# The child indices of heap index pos are already heaps, and we want to make
-# a heap at index pos too.  We do this by bubbling the smaller child of
-# pos up (and so on with that child's children, etc) until hitting a leaf,
-# then using _siftdown to move the oddball originally at index pos into place.
-#
-# We *could* break out of the loop as soon as we find a pos where newitem <=
-# both its children, but turns out that's not a good idea, and despite that
-# many books write the algorithm that way.  During a heap pop, the last array
-# element is sifted in, and that tends to be large, so that comparing it
-# against values starting from the root usually doesn't pay (= usually doesn't
-# get us out of the loop early).  See Knuth, Volume 3, where this is
-# explained and quantified in an exercise.
-#
-# Cutting the # of comparisons is important, since these routines have no
-# way to extract "the priority" from an array element, so that intelligence
-# is likely to be hiding in custom comparison methods, or in array elements
-# storing (priority, record) tuples.  Comparisons are thus potentially
-# expensive.
-#
-# On random arrays of length 1000, making this change cut the number of
-# comparisons made by heapify() a little, and those made by exhaustive
-# heappop() a lot, in accord with theory.  Here are typical results from 3
-# runs (3 just to demonstrate how small the variance is):
-#
-# Compares needed by heapify     Compares needed by 1000 heappops
-# --------------------------     --------------------------------
-# 1837 cut to 1663               14996 cut to 8680
-# 1855 cut to 1659               14966 cut to 8678
-# 1847 cut to 1660               15024 cut to 8703
-#
-# Building the heap by using heappush() 1000 times instead required
-# 2198, 2148, and 2219 compares:  heapify() is more efficient, when
-# you can use it.
-#
-# The total compares needed by list.sort() on the same lists were 8627,
-# 8627, and 8632 (this should be compared to the sum of heapify() and
-# heappop() compares):  list.sort() is (unsurprisingly!) more efficient
-# for sorting.
-
-def _siftup(heap, pos):
-    endpos = len(heap)
-    startpos = pos
-    newitem = heap[pos]
-    # Bubble up the smaller child until hitting a leaf.
-    childpos = 2*pos + 1    # leftmost child position
-    while childpos < endpos:
-        # Set childpos to index of smaller child.
-        rightpos = childpos + 1
-        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
-            childpos = rightpos
-        # Move the smaller child up.
-        heap[pos] = heap[childpos]
-        pos = childpos
-        childpos = 2*pos + 1
-    # The leaf at pos is empty now.  Put newitem there, and bubble it up
-    # to its final resting place (by sifting its parents down).
-    heap[pos] = newitem
-    _siftdown(heap, startpos, pos)
-
-def _siftdown_max(heap, startpos, pos):
-    'Maxheap variant of _siftdown'
-    newitem = heap[pos]
-    # Follow the path to the root, moving parents down until finding a place
-    # newitem fits.
-    while pos > startpos:
-        parentpos = (pos - 1) >> 1
-        parent = heap[parentpos]
-        if parent < newitem:
-            heap[pos] = parent
-            pos = parentpos
-            continue
-        break
-    heap[pos] = newitem
-
-def _siftup_max(heap, pos):
-    'Maxheap variant of _siftup'
-    endpos = len(heap)
-    startpos = pos
-    newitem = heap[pos]
-    # Bubble up the larger child until hitting a leaf.
-    childpos = 2*pos + 1    # leftmost child position
-    while childpos < endpos:
-        # Set childpos to index of larger child.
-        rightpos = childpos + 1
-        if rightpos < endpos and not heap[rightpos] < heap[childpos]:
-            childpos = rightpos
-        # Move the larger child up.
-        heap[pos] = heap[childpos]
-        pos = childpos
-        childpos = 2*pos + 1
-    # The leaf at pos is empty now.  Put newitem there, and bubble it up
-    # to its final resting place (by sifting its parents down).
-    heap[pos] = newitem
-    _siftdown_max(heap, startpos, pos)
-
-def merge(*iterables, key=None, reverse=False):
-    '''Merge multiple sorted inputs into a single sorted output.
-
-    Similar to sorted(itertools.chain(*iterables)) but returns a generator,
-    does not pull the data into memory all at once, and assumes that each of
-    the input streams is already sorted (smallest to largest).
-
-    >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
-    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
-
-    If *key* is not None, applies a key function to each element to determine
-    its sort order.
-
-    >>> list(merge(['dog', 'horse'], ['cat', 'fish', 'kangaroo'], key=len))
-    ['dog', 'cat', 'fish', 'horse', 'kangaroo']
-
-    '''
-
-    h = []
-    h_append = h.append
-
-    if reverse:
-        _heapify = _heapify_max
-        _heappop = _heappop_max
-        _heapreplace = _heapreplace_max
-        direction = -1
-    else:
-        _heapify = heapify
-        _heappop = heappop
-        _heapreplace = heapreplace
-        direction = 1
-
-    if key is None:
-        for order, it in enumerate(map(iter, iterables)):
-            try:
-                next = it.__next__
-                h_append([next(), order * direction, next])
-            except StopIteration:
-                pass
-        _heapify(h)
-        while len(h) > 1:
-            try:
-                while True:
-                    value, order, next = s = h[0]
-                    yield value
-                    s[0] = next()           # raises StopIteration when exhausted
-                    _heapreplace(h, s)      # restore heap condition
-            except StopIteration:
-                _heappop(h)                 # remove empty iterator
-        if h:
-            # fast case when only a single iterator remains
-            value, order, next = h[0]
-            yield value
-            yield from next.__self__
-        return
-
-    for order, it in enumerate(map(iter, iterables)):
-        try:
-            next = it.__next__
-            value = next()
-            h_append([key(value), order * direction, value, next])
-        except StopIteration:
-            pass
-    _heapify(h)
-    while len(h) > 1:
-        try:
-            while True:
-                key_value, order, value, next = s = h[0]
-                yield value
-                value = next()
-                s[0] = key(value)
-                s[2] = value
-                _heapreplace(h, s)
-        except StopIteration:
-            _heappop(h)
-    if h:
-        key_value, order, value, next = h[0]
-        yield value
-        yield from next.__self__
-
-
-# Algorithm notes for nlargest() and nsmallest()
-# ==============================================
-#
-# Make a single pass over the data while keeping the k most extreme values
-# in a heap.  Memory consumption is limited to keeping k values in a list.
-#
-# Measured performance for random inputs:
-#
-#                                   number of comparisons
-#    n inputs     k-extreme values  (average of 5 trials)   % more than min()
-# -------------   ----------------  ---------------------   -----------------
-#      1,000           100                  3,317               231.7%
-#     10,000           100                 14,046                40.5%
-#    100,000           100                105,749                 5.7%
-#  1,000,000           100              1,007,751                 0.8%
-# 10,000,000           100             10,009,401                 0.1%
-#
-# Theoretical number of comparisons for k smallest of n random inputs:
-#
-# Step   Comparisons                  Action
-# ----   --------------------------   ---------------------------
-#  1     1.66 * k                     heapify the first k-inputs
-#  2     n - k                        compare remaining elements to top of heap
-#  3     k * (1 + lg2(k)) * ln(n/k)   replace the topmost value on the heap
-#  4     k * lg2(k) - (k/2)           final sort of the k most extreme values
-#
-# Combining and simplifying for a rough estimate gives:
-#
-#        comparisons = n + k * (log(k, 2) * log(n/k) + log(k, 2) + log(n/k))
-#
-# Computing the number of comparisons for step 3:
-# -----------------------------------------------
-# * For the i-th new value from the iterable, the probability of being in the
-#   k most extreme values is k/i.  For example, the probability of the 101st
-#   value seen being in the 100 most extreme values is 100/101.
-# * If the value is a new extreme value, the cost of inserting it into the
-#   heap is 1 + log(k, 2).
-# * The probability times the cost gives:
-#            (k/i) * (1 + log(k, 2))
-# * Summing across the remaining n-k elements gives:
-#            sum((k/i) * (1 + log(k, 2)) for i in range(k+1, n+1))
-# * This reduces to:
-#            (H(n) - H(k)) * k * (1 + log(k, 2))
-# * Where H(n) is the n-th harmonic number estimated by:
-#            gamma = 0.5772156649
-#            H(n) = log(n, e) + gamma + 1 / (2 * n)
-#   http://en.wikipedia.org/wiki/Harmonic_series_(mathematics)#Rate_of_divergence
-# * Substituting the H(n) formula:
-#            comparisons = k * (1 + log(k, 2)) * (log(n/k, e) + (1/n - 1/k) / 2)
-#
-# Worst-case for step 3:
-# ----------------------
-# In the worst case, the input data is reversed sorted so that every new element
-# must be inserted in the heap:
-#
-#             comparisons = 1.66 * k + log(k, 2) * (n - k)
-#
-# Alternative Algorithms
-# ----------------------
-# Other algorithms were not used because they:
-# 1) Took much more auxiliary memory,
-# 2) Made multiple passes over the data.
-# 3) Made more comparisons in common cases (small k, large n, semi-random input).
-# See the more detailed comparison of approach at:
-# http://code.activestate.com/recipes/577573-compare-algorithms-for-heapqsmallest
-
-def nsmallest(n, iterable, key=None):
-    """Find the n smallest elements in a dataset.
-
-    Equivalent to:  sorted(iterable, key=key)[:n]
-    """
-
-    # Short-cut for n==1 is to use min()
-    if n == 1:
-        it = iter(iterable)
-        sentinel = object()
-        result = min(it, default=sentinel, key=key)
-        return [] if result is sentinel else [result]
-
-    # When n>=size, it's faster to use sorted()
-    try:
-        size = len(iterable)
-    except (TypeError, AttributeError):
-        pass
-    else:
-        if n >= size:
-            return sorted(iterable, key=key)[:n]
-
-    # When key is none, use simpler decoration
-    if key is None:
-        it = iter(iterable)
-        # put the range(n) first so that zip() doesn't
-        # consume one too many elements from the iterator
-        result = [(elem, i) for i, elem in zip(range(n), it)]
-        if not result:
-            return result
-        _heapify_max(result)
-        top = result[0][0]
-        order = n
-        _heapreplace = _heapreplace_max
-        for elem in it:
-            if elem < top:
-                _heapreplace(result, (elem, order))
-                top, _order = result[0]
-                order += 1
-        result.sort()
-        return [elem for (elem, order) in result]
-
-    # General case, slowest method
-    it = iter(iterable)
-    result = [(key(elem), i, elem) for i, elem in zip(range(n), it)]
-    if not result:
-        return result
-    _heapify_max(result)
-    top = result[0][0]
-    order = n
-    _heapreplace = _heapreplace_max
-    for elem in it:
-        k = key(elem)
-        if k < top:
-            _heapreplace(result, (k, order, elem))
-            top, _order, _elem = result[0]
-            order += 1
-    result.sort()
-    return [elem for (k, order, elem) in result]
-
-def nlargest(n, iterable, key=None):
-    """Find the n largest elements in a dataset.
-
-    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
-    """
-
-    # Short-cut for n==1 is to use max()
-    if n == 1:
-        it = iter(iterable)
-        sentinel = object()
-        result = max(it, default=sentinel, key=key)
-        return [] if result is sentinel else [result]
-
-    # When n>=size, it's faster to use sorted()
-    try:
-        size = len(iterable)
-    except (TypeError, AttributeError):
-        pass
-    else:
-        if n >= size:
-            return sorted(iterable, key=key, reverse=True)[:n]
-
-    # When key is none, use simpler decoration
-    if key is None:
-        it = iter(iterable)
-        result = [(elem, i) for i, elem in zip(range(0, -n, -1), it)]
-        if not result:
-            return result
-        heapify(result)
-        top = result[0][0]
-        order = -n
-        _heapreplace = heapreplace
-        for elem in it:
-            if top < elem:
-                _heapreplace(result, (elem, order))
-                top, _order = result[0]
-                order -= 1
-        result.sort(reverse=True)
-        return [elem for (elem, order) in result]
-
-    # General case, slowest method
-    it = iter(iterable)
-    result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)]
-    if not result:
-        return result
-    heapify(result)
-    top = result[0][0]
-    order = -n
-    _heapreplace = heapreplace
-    for elem in it:
-        k = key(elem)
-        if top < k:
-            _heapreplace(result, (k, order, elem))
-            top, _order, _elem = result[0]
-            order -= 1
-    result.sort(reverse=True)
-    return [elem for (k, order, elem) in result]
-
-# If available, use C implementation
-try:
-    from _heapq import *
-except ImportError:
-    pass
-try:
-    from _heapq import _heapreplace_max
-except ImportError:
-    pass
-try:
-    from _heapq import _heapify_max
-except ImportError:
-    pass
-try:
-    from _heapq import _heappop_max
-except ImportError:
-    pass
-
-
-if __name__ == "__main__":
-
-    import doctest
-    print(doctest.testmod())

Lib/io.py

@@ -1 +0,0 @@
-from _io import *

Lib/linecache.py

@@ -1,177 +0,0 @@
-"""Cache lines from Python source files.
-
-This is intended to read lines from modules imported -- hence if a filename
-is not found, it will look down the module search path for a file by
-that name.
-"""
-
-import functools
-import sys
-import os
-import tokenize
-
-__all__ = ["getline", "clearcache", "checkcache"]
-
-def getline(filename, lineno, module_globals=None):
-    lines = getlines(filename, module_globals)
-    if 1 <= lineno <= len(lines):
-        return lines[lineno-1]
-    else:
-        return ''
-
-
-# The cache
-
-# The cache. Maps filenames to either a thunk which will provide source code,
-# or a tuple (size, mtime, lines, fullname) once loaded.
-cache = {}
-
-
-def clearcache():
-    """Clear the cache entirely."""
-
-    global cache
-    cache = {}
-
-
-def getlines(filename, module_globals=None):
-    """Get the lines for a Python source file from the cache.
-    Update the cache if it doesn't contain an entry for this file already."""
-
-    if filename in cache:
-        entry = cache[filename]
-        if len(entry) != 1:
-            return cache[filename][2]
-
-    try:
-        return updatecache(filename, module_globals)
-    except MemoryError:
-        clearcache()
-        return []
-
-
-def checkcache(filename=None):
-    """Discard cache entries that are out of date.
-    (This is not checked upon each call!)"""
-
-    if filename is None:
-        filenames = list(cache.keys())
-    else:
-        if filename in cache:
-            filenames = [filename]
-        else:
-            return
-
-    for filename in filenames:
-        entry = cache[filename]
-        if len(entry) == 1:
-            # lazy cache entry, leave it lazy.
-            continue
-        size, mtime, lines, fullname = entry
-        if mtime is None:
-            continue   # no-op for files loaded via a __loader__
-        try:
-            stat = os.stat(fullname)
-        except OSError:
-            del cache[filename]
-            continue
-        if size != stat.st_size or mtime != stat.st_mtime:
-            del cache[filename]
-
-
-def updatecache(filename, module_globals=None):
-    """Update a cache entry and return its list of lines.
-    If something's wrong, print a message, discard the cache entry,
-    and return an empty list."""
-
-    if filename in cache:
-        if len(cache[filename]) != 1:
-            del cache[filename]
-    if not filename or (filename.startswith('<') and filename.endswith('>')):
-        return []
-
-    fullname = filename
-    try:
-        stat = os.stat(fullname)
-    except OSError:
-        basename = filename
-
-        # Realise a lazy loader based lookup if there is one
-        # otherwise try to lookup right now.
-        if lazycache(filename, module_globals):
-            try:
-                data = cache[filename][0]()
-            except (ImportError, OSError):
-                pass
-            else:
-                if data is None:
-                    # No luck, the PEP302 loader cannot find the source
-                    # for this module.
-                    return []
-                cache[filename] = (
-                    len(data), None,
-                    [line+'\n' for line in data.splitlines()], fullname
-                )
-                return cache[filename][2]
-
-        # Try looking through the module search path, which is only useful
-        # when handling a relative filename.
-        if os.path.isabs(filename):
-            return []
-
-        for dirname in sys.path:
-            try:
-                fullname = os.path.join(dirname, basename)
-            except (TypeError, AttributeError):
-                # Not sufficiently string-like to do anything useful with.
-                continue
-            try:
-                stat = os.stat(fullname)
-                break
-            except OSError:
-                pass
-        else:
-            return []
-    try:
-        with tokenize.open(fullname) as fp:
-            lines = fp.readlines()
-    except OSError:
-        return []
-    if lines and not lines[-1].endswith('\n'):
-        lines[-1] += '\n'
-    size, mtime = stat.st_size, stat.st_mtime
-    cache[filename] = size, mtime, lines, fullname
-    return lines
-
-
-def lazycache(filename, module_globals):
-    """Seed the cache for filename with module_globals.
-
-    The module loader will be asked for the source only when getlines is
-    called, not immediately.
-
-    If there is an entry in the cache already, it is not altered.
-
-    :return: True if a lazy load is registered in the cache,
-        otherwise False. To register such a load a module loader with a
-        get_source method must be found, the filename must be a cachable
-        filename, and the filename must not be already cached.
-    """
-    if filename in cache:
-        if len(cache[filename]) == 1:
-            return True
-        else:
-            return False
-    if not filename or (filename.startswith('<') and filename.endswith('>')):
-        return False
-    # Try for a __loader__, if available
-    if module_globals and '__loader__' in module_globals:
-        name = module_globals.get('__name__')
-        loader = module_globals['__loader__']
-        get_source = getattr(loader, 'get_source', None)
-
-        if name and get_source:
-            get_lines = functools.partial(get_source, name)
-            cache[filename] = (get_lines,)
-            return True
-    return False

Lib/ntpath.py

@@ -1,661 +0,0 @@
-# Module 'ntpath' -- common operations on WinNT/Win95 pathnames
-"""Common pathname manipulations, WindowsNT/95 version.
-
-Instead of importing this module directly, import os and refer to this
-module as os.path.
-"""
-
-# strings representing various path-related bits and pieces
-# These are primarily for export; internally, they are hardcoded.
-# Should be set before imports for resolving cyclic dependency.
-curdir = '.'
-pardir = '..'
-extsep = '.'
-sep = '\\'
-pathsep = ';'
-altsep = '/'
-defpath = '.;C:\\bin'
-devnull = 'nul'
-
-import os
-import sys
-import stat
-import genericpath
-from genericpath import *
-
-__all__ = ["normcase","isabs","join","splitdrive","split","splitext",
-           "basename","dirname","commonprefix","getsize","getmtime",
-           "getatime","getctime", "islink","exists","lexists","isdir","isfile",
-           "ismount", "expanduser","expandvars","normpath","abspath",
-           "curdir","pardir","sep","pathsep","defpath","altsep",
-           "extsep","devnull","realpath","supports_unicode_filenames","relpath",
-           "samefile", "sameopenfile", "samestat", "commonpath"]
-
-def _get_bothseps(path):
-    if isinstance(path, bytes):
-        return b'\\/'
-    else:
-        return '\\/'
-
-# Normalize the case of a pathname and map slashes to backslashes.
-# Other normalizations (such as optimizing '../' away) are not done
-# (this is done by normpath).
-
-def normcase(s):
-    """Normalize case of pathname.
-
-    Makes all characters lowercase and all slashes into backslashes."""
-    s = os.fspath(s)
-    if isinstance(s, bytes):
-        return s.replace(b'/', b'\\').lower()
-    else:
-        return s.replace('/', '\\').lower()
-
-
-# Return whether a path is absolute.
-# Trivial in Posix, harder on Windows.
-# For Windows it is absolute if it starts with a slash or backslash (current
-# volume), or if a pathname after the volume-letter-and-colon or UNC-resource
-# starts with a slash or backslash.
-
-def isabs(s):
-    """Test whether a path is absolute"""
-    s = os.fspath(s)
-    s = splitdrive(s)[1]
-    return len(s) > 0 and s[0] in _get_bothseps(s)
-
-
-# Join two (or more) paths.
-def join(path, *paths):
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        sep = b'\\'
-        seps = b'\\/'
-        colon = b':'
-    else:
-        sep = '\\'
-        seps = '\\/'
-        colon = ':'
-    try:
-        if not paths:
-            path[:0] + sep  #23780: Ensure compatible data type even if p is null.
-        result_drive, result_path = splitdrive(path)
-        for p in map(os.fspath, paths):
-            p_drive, p_path = splitdrive(p)
-            if p_path and p_path[0] in seps:
-                # Second path is absolute
-                if p_drive or not result_drive:
-                    result_drive = p_drive
-                result_path = p_path
-                continue
-            elif p_drive and p_drive != result_drive:
-                if p_drive.lower() != result_drive.lower():
-                    # Different drives => ignore the first path entirely
-                    result_drive = p_drive
-                    result_path = p_path
-                    continue
-                # Same drive in different case
-                result_drive = p_drive
-            # Second path is relative to the first
-            if result_path and result_path[-1] not in seps:
-                result_path = result_path + sep
-            result_path = result_path + p_path
-        ## add separator between UNC and non-absolute path
-        if (result_path and result_path[0] not in seps and
-            result_drive and result_drive[-1:] != colon):
-            return result_drive + sep + result_path
-        return result_drive + result_path
-    except (TypeError, AttributeError, BytesWarning):
-        genericpath._check_arg_types('join', path, *paths)
-        raise
-
-
-# Split a path in a drive specification (a drive letter followed by a
-# colon) and the path specification.
-# It is always true that drivespec + pathspec == p
-def splitdrive(p):
-    """Split a pathname into drive/UNC sharepoint and relative path specifiers.
-    Returns a 2-tuple (drive_or_unc, path); either part may be empty.
-
-    If you assign
-        result = splitdrive(p)
-    It is always true that:
-        result[0] + result[1] == p
-
-    If the path contained a drive letter, drive_or_unc will contain everything
-    up to and including the colon.  e.g. splitdrive("c:/dir") returns ("c:", "/dir")
-
-    If the path contained a UNC path, the drive_or_unc will contain the host name
-    and share up to but not including the fourth directory separator character.
-    e.g. splitdrive("//host/computer/dir") returns ("//host/computer", "/dir")
-
-    Paths cannot contain both a drive letter and a UNC path.
-
-    """
-    p = os.fspath(p)
-    if len(p) >= 2:
-        if isinstance(p, bytes):
-            sep = b'\\'
-            altsep = b'/'
-            colon = b':'
-        else:
-            sep = '\\'
-            altsep = '/'
-            colon = ':'
-        normp = p.replace(altsep, sep)
-        if (normp[0:2] == sep*2) and (normp[2:3] != sep):
-            # is a UNC path:
-            # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path
-            # \\machine\mountpoint\directory\etc\...
-            #           directory ^^^^^^^^^^^^^^^
-            index = normp.find(sep, 2)
-            if index == -1:
-                return p[:0], p
-            index2 = normp.find(sep, index + 1)
-            # a UNC path can't have two slashes in a row
-            # (after the initial two)
-            if index2 == index + 1:
-                return p[:0], p
-            if index2 == -1:
-                index2 = len(p)
-            return p[:index2], p[index2:]
-        if normp[1:2] == colon:
-            return p[:2], p[2:]
-    return p[:0], p
-
-
-# Split a path in head (everything up to the last '/') and tail (the
-# rest).  After the trailing '/' is stripped, the invariant
-# join(head, tail) == p holds.
-# The resulting head won't end in '/' unless it is the root.
-
-def split(p):
-    """Split a pathname.
-
-    Return tuple (head, tail) where tail is everything after the final slash.
-    Either part may be empty."""
-    p = os.fspath(p)
-    seps = _get_bothseps(p)
-    d, p = splitdrive(p)
-    # set i to index beyond p's last slash
-    i = len(p)
-    while i and p[i-1] not in seps:
-        i -= 1
-    head, tail = p[:i], p[i:]  # now tail has no slashes
-    # remove trailing slashes from head, unless it's all slashes
-    head = head.rstrip(seps) or head
-    return d + head, tail
-
-
-# Split a path in root and extension.
-# The extension is everything starting at the last dot in the last
-# pathname component; the root is everything before that.
-# It is always true that root + ext == p.
-
-def splitext(p):
-    p = os.fspath(p)
-    if isinstance(p, bytes):
-        return genericpath._splitext(p, b'\\', b'/', b'.')
-    else:
-        return genericpath._splitext(p, '\\', '/', '.')
-splitext.__doc__ = genericpath._splitext.__doc__
-
-
-# Return the tail (basename) part of a path.
-
-def basename(p):
-    """Returns the final component of a pathname"""
-    return split(p)[1]
-
-
-# Return the head (dirname) part of a path.
-
-def dirname(p):
-    """Returns the directory component of a pathname"""
-    return split(p)[0]
-
-# Is a path a symbolic link?
-# This will always return false on systems where os.lstat doesn't exist.
-
-def islink(path):
-    """Test whether a path is a symbolic link.
-    This will always return false for Windows prior to 6.0.
-    """
-    try:
-        st = os.lstat(path)
-    except (OSError, ValueError, AttributeError):
-        return False
-    return stat.S_ISLNK(st.st_mode)
-
-# Being true for dangling symbolic links is also useful.
-
-def lexists(path):
-    """Test whether a path exists.  Returns True for broken symbolic links"""
-    try:
-        st = os.lstat(path)
-    except (OSError, ValueError):
-        return False
-    return True
-
-# Is a path a mount point?
-# Any drive letter root (eg c:\)
-# Any share UNC (eg \\server\share)
-# Any volume mounted on a filesystem folder
-#
-# No one method detects all three situations. Historically we've lexically
-# detected drive letter roots and share UNCs. The canonical approach to
-# detecting mounted volumes (querying the reparse tag) fails for the most
-# common case: drive letter roots. The alternative which uses GetVolumePathName
-# fails if the drive letter is the result of a SUBST.
-try:
-    from nt import _getvolumepathname
-except ImportError:
-    _getvolumepathname = None
-def ismount(path):
-    """Test whether a path is a mount point (a drive root, the root of a
-    share, or a mounted volume)"""
-    path = os.fspath(path)
-    seps = _get_bothseps(path)
-    path = abspath(path)
-    root, rest = splitdrive(path)
-    if root and root[0] in seps:
-        return (not rest) or (rest in seps)
-    if rest in seps:
-        return True
-
-    if _getvolumepathname:
-        return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps)
-    else:
-        return False
-
-
-# Expand paths beginning with '~' or '~user'.
-# '~' means $HOME; '~user' means that user's home directory.
-# If the path doesn't begin with '~', or if the user or $HOME is unknown,
-# the path is returned unchanged (leaving error reporting to whatever
-# function is called with the expanded path as argument).
-# See also module 'glob' for expansion of *, ? and [...] in pathnames.
-# (A function should also be defined to do full *sh-style environment
-# variable expansion.)
-
-def expanduser(path):
-    """Expand ~ and ~user constructs.
-
-    If user or $HOME is unknown, do nothing."""
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        tilde = b'~'
-    else:
-        tilde = '~'
-    if not path.startswith(tilde):
-        return path
-    i, n = 1, len(path)
-    while i < n and path[i] not in _get_bothseps(path):
-        i += 1
-
-    if 'USERPROFILE' in os.environ:
-        userhome = os.environ['USERPROFILE']
-    elif not 'HOMEPATH' in os.environ:
-        return path
-    else:
-        try:
-            drive = os.environ['HOMEDRIVE']
-        except KeyError:
-            drive = ''
-        userhome = join(drive, os.environ['HOMEPATH'])
-
-    if isinstance(path, bytes):
-        userhome = os.fsencode(userhome)
-
-    if i != 1: #~user
-        userhome = join(dirname(userhome), path[1:i])
-
-    return userhome + path[i:]
-
-
-# Expand paths containing shell variable substitutions.
-# The following rules apply:
-#       - no expansion within single quotes
-#       - '$$' is translated into '$'
-#       - '%%' is translated into '%' if '%%' are not seen in %var1%%var2%
-#       - ${varname} is accepted.
-#       - $varname is accepted.
-#       - %varname% is accepted.
-#       - varnames can be made out of letters, digits and the characters '_-'
-#         (though is not verified in the ${varname} and %varname% cases)
-# XXX With COMMAND.COM you can use any characters in a variable name,
-# XXX except '^|<>='.
-
-def expandvars(path):
-    """Expand shell variables of the forms $var, ${var} and %var%.
-
-    Unknown variables are left unchanged."""
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        if b'$' not in path and b'%' not in path:
-            return path
-        import string
-        varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii')
-        quote = b'\''
-        percent = b'%'
-        brace = b'{'
-        rbrace = b'}'
-        dollar = b'$'
-        environ = getattr(os, 'environb', None)
-    else:
-        if '$' not in path and '%' not in path:
-            return path
-        import string
-        varchars = string.ascii_letters + string.digits + '_-'
-        quote = '\''
-        percent = '%'
-        brace = '{'
-        rbrace = '}'
-        dollar = '$'
-        environ = os.environ
-    res = path[:0]
-    index = 0
-    pathlen = len(path)
-    while index < pathlen:
-        c = path[index:index+1]
-        if c == quote:   # no expansion within single quotes
-            path = path[index + 1:]
-            pathlen = len(path)
-            try:
-                index = path.index(c)
-                res += c + path[:index + 1]
-            except ValueError:
-                res += c + path
-                index = pathlen - 1
-        elif c == percent:  # variable or '%'
-            if path[index + 1:index + 2] == percent:
-                res += c
-                index += 1
-            else:
-                path = path[index+1:]
-                pathlen = len(path)
-                try:
-                    index = path.index(percent)
-                except ValueError:
-                    res += percent + path
-                    index = pathlen - 1
-                else:
-                    var = path[:index]
-                    try:
-                        if environ is None:
-                            value = os.fsencode(os.environ[os.fsdecode(var)])
-                        else:
-                            value = environ[var]
-                    except KeyError:
-                        value = percent + var + percent
-                    res += value
-        elif c == dollar:  # variable or '$$'
-            if path[index + 1:index + 2] == dollar:
-                res += c
-                index += 1
-            elif path[index + 1:index + 2] == brace:
-                path = path[index+2:]
-                pathlen = len(path)
-                try:
-                    index = path.index(rbrace)
-                except ValueError:
-                    res += dollar + brace + path
-                    index = pathlen - 1
-                else:
-                    var = path[:index]
-                    try:
-                        if environ is None:
-                            value = os.fsencode(os.environ[os.fsdecode(var)])
-                        else:
-                            value = environ[var]
-                    except KeyError:
-                        value = dollar + brace + var + rbrace
-                    res += value
-            else:
-                var = path[:0]
-                index += 1
-                c = path[index:index + 1]
-                while c and c in varchars:
-                    var += c
-                    index += 1
-                    c = path[index:index + 1]
-                try:
-                    if environ is None:
-                        value = os.fsencode(os.environ[os.fsdecode(var)])
-                    else:
-                        value = environ[var]
-                except KeyError:
-                    value = dollar + var
-                res += value
-                if c:
-                    index -= 1
-        else:
-            res += c
-        index += 1
-    return res
-
-
-# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B.
-# Previously, this function also truncated pathnames to 8+3 format,
-# but as this module is called "ntpath", that's obviously wrong!
-
-def normpath(path):
-    """Normalize path, eliminating double slashes, etc."""
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        sep = b'\\'
-        altsep = b'/'
-        curdir = b'.'
-        pardir = b'..'
-        special_prefixes = (b'\\\\.\\', b'\\\\?\\')
-    else:
-        sep = '\\'
-        altsep = '/'
-        curdir = '.'
-        pardir = '..'
-        special_prefixes = ('\\\\.\\', '\\\\?\\')
-    if path.startswith(special_prefixes):
-        # in the case of paths with these prefixes:
-        # \\.\ -> device names
-        # \\?\ -> literal paths
-        # do not do any normalization, but return the path unchanged
-        return path
-    path = path.replace(altsep, sep)
-    prefix, path = splitdrive(path)
-
-    # collapse initial backslashes
-    if path.startswith(sep):
-        prefix += sep
-        path = path.lstrip(sep)
-
-    comps = path.split(sep)
-    i = 0
-    while i < len(comps):
-        if not comps[i] or comps[i] == curdir:
-            del comps[i]
-        elif comps[i] == pardir:
-            if i > 0 and comps[i-1] != pardir:
-                del comps[i-1:i+1]
-                i -= 1
-            elif i == 0 and prefix.endswith(sep):
-                del comps[i]
-            else:
-                i += 1
-        else:
-            i += 1
-    # If the path is now empty, substitute '.'
-    if not prefix and not comps:
-        comps.append(curdir)
-    return prefix + sep.join(comps)
-
-def _abspath_fallback(path):
-    """Return the absolute version of a path as a fallback function in case
-    `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for
-    more.
-
-    """
-
-    path = os.fspath(path)
-    if not isabs(path):
-        if isinstance(path, bytes):
-            cwd = os.getcwdb()
-        else:
-            cwd = os.getcwd()
-        path = join(cwd, path)
-    return normpath(path)
-
-# Return an absolute path.
-try:
-    from nt import _getfullpathname
-
-except ImportError: # not running on Windows - mock up something sensible
-    abspath = _abspath_fallback
-
-else:  # use native Windows method on Windows
-    def abspath(path):
-        """Return the absolute version of a path."""
-        try:
-            return normpath(_getfullpathname(path))
-        except (OSError, ValueError):
-            return _abspath_fallback(path)
-
-# realpath is a no-op on systems without islink support
-realpath = abspath
-# Win9x family and earlier have no Unicode filename support.
-supports_unicode_filenames = (hasattr(sys, "getwindowsversion") and
-                              sys.getwindowsversion()[3] >= 2)
-
-def relpath(path, start=None):
-    """Return a relative version of a path"""
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        sep = b'\\'
-        curdir = b'.'
-        pardir = b'..'
-    else:
-        sep = '\\'
-        curdir = '.'
-        pardir = '..'
-
-    if start is None:
-        start = curdir
-
-    if not path:
-        raise ValueError("no path specified")
-
-    start = os.fspath(start)
-    try:
-        start_abs = abspath(normpath(start))
-        path_abs = abspath(normpath(path))
-        start_drive, start_rest = splitdrive(start_abs)
-        path_drive, path_rest = splitdrive(path_abs)
-        if normcase(start_drive) != normcase(path_drive):
-            raise ValueError("path is on mount %r, start on mount %r" % (
-                path_drive, start_drive))
-
-        start_list = [x for x in start_rest.split(sep) if x]
-        path_list = [x for x in path_rest.split(sep) if x]
-        # Work out how much of the filepath is shared by start and path.
-        i = 0
-        for e1, e2 in zip(start_list, path_list):
-            if normcase(e1) != normcase(e2):
-                break
-            i += 1
-
-        rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
-        if not rel_list:
-            return curdir
-        return join(*rel_list)
-    except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning):
-        genericpath._check_arg_types('relpath', path, start)
-        raise
-
-
-# Return the longest common sub-path of the sequence of paths given as input.
-# The function is case-insensitive and 'separator-insensitive', i.e. if the
-# only difference between two paths is the use of '\' versus '/' as separator,
-# they are deemed to be equal.
-#
-# However, the returned path will have the standard '\' separator (even if the
-# given paths had the alternative '/' separator) and will have the case of the
-# first path given in the sequence. Additionally, any trailing separator is
-# stripped from the returned path.
-
-def commonpath(paths):
-    """Given a sequence of path names, returns the longest common sub-path."""
-
-    if not paths:
-        raise ValueError('commonpath() arg is an empty sequence')
-
-    paths = tuple(map(os.fspath, paths))
-    if isinstance(paths[0], bytes):
-        sep = b'\\'
-        altsep = b'/'
-        curdir = b'.'
-    else:
-        sep = '\\'
-        altsep = '/'
-        curdir = '.'
-
-    try:
-        drivesplits = [splitdrive(p.replace(altsep, sep).lower()) for p in paths]
-        split_paths = [p.split(sep) for d, p in drivesplits]
-
-        try:
-            isabs, = set(p[:1] == sep for d, p in drivesplits)
-        except ValueError:
-            raise ValueError("Can't mix absolute and relative paths") from None
-
-        # Check that all drive letters or UNC paths match. The check is made only
-        # now otherwise type errors for mixing strings and bytes would not be
-        # caught.
-        if len(set(d for d, p in drivesplits)) != 1:
-            raise ValueError("Paths don't have the same drive")
-
-        drive, path = splitdrive(paths[0].replace(altsep, sep))
-        common = path.split(sep)
-        common = [c for c in common if c and c != curdir]
-
-        split_paths = [[c for c in s if c and c != curdir] for s in split_paths]
-        s1 = min(split_paths)
-        s2 = max(split_paths)
-        for i, c in enumerate(s1):
-            if c != s2[i]:
-                common = common[:i]
-                break
-        else:
-            common = common[:len(s1)]
-
-        prefix = drive + sep if isabs else drive
-        return prefix + sep.join(common)
-    except (TypeError, AttributeError):
-        genericpath._check_arg_types('commonpath', *paths)
-        raise
-
-
-# determine if two files are in fact the same file
-try:
-    # GetFinalPathNameByHandle is available starting with Windows 6.0.
-    # Windows XP and non-Windows OS'es will mock _getfinalpathname.
-    if sys.getwindowsversion()[:2] >= (6, 0):
-        from nt import _getfinalpathname
-    else:
-        raise ImportError
-except (AttributeError, ImportError):
-    # On Windows XP and earlier, two files are the same if their absolute
-    # pathnames are the same.
-    # Non-Windows operating systems fake this method with an XP
-    # approximation.
-    def _getfinalpathname(f):
-        return normcase(abspath(f))
-
-
-try:
-    # The genericpath.isdir implementation uses os.stat and checks the mode
-    # attribute to tell whether or not the path is a directory.
-    # This is overkill on Windows - just pass the path to GetFileAttributes
-    # and check the attribute from there.
-    from nt import _isdir as isdir
-except ImportError:
-    # Use genericpath.isdir as imported above.
-    pass

Lib/operator.py

@@ -1,464 +1,15 @@
-"""
-Operator Interface
-
-This module exports a set of functions corresponding to the intrinsic
-operators of Python.  For example, operator.add(x, y) is equivalent
-to the expression x+y.  The function names are those used for special
-methods; variants without leading and trailing '__' are also provided
-for convenience.
-
-This is the pure Python implementation of the module.
-"""
-
-__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf',
-           'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand',
-           'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul',
-           'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift',
-           'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le',
-           'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod',
-           'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift',
-           'setitem', 'sub', 'truediv', 'truth', 'xor']
-
-from builtins import abs as _abs
 
 
-# Comparison Operations *******************************************************#
+# Comparison operations:
-
-def lt(a, b):
-    "Same as a < b."
-    return a < b
-
-def le(a, b):
-    "Same as a <= b."
-    return a <= b
 
 def eq(a, b):
-    "Same as a == b."
     return a == b
 
+
 def ne(a, b):
-    "Same as a != b."
     return a != b
 
+
 def ge(a, b):
-    "Same as a >= b."
     return a >= b
 
-def gt(a, b):
-    "Same as a > b."
-    return a > b
-
-# Logical Operations **********************************************************#
-
-def not_(a):
-    "Same as not a."
-    return not a
-
-def truth(a):
-    "Return True if a is true, False otherwise."
-    return True if a else False
-
-def is_(a, b):
-    "Same as a is b."
-    return a is b
-
-def is_not(a, b):
-    "Same as a is not b."
-    return a is not b
-
-# Mathematical/Bitwise Operations *********************************************#
-
-def abs(a):
-    "Same as abs(a)."
-    return _abs(a)
-
-def add(a, b):
-    "Same as a + b."
-    return a + b
-
-def and_(a, b):
-    "Same as a & b."
-    return a & b
-
-def floordiv(a, b):
-    "Same as a // b."
-    return a // b
-
-def index(a):
-    "Same as a.__index__()."
-    return a.__index__()
-
-def inv(a):
-    "Same as ~a."
-    return ~a
-invert = inv
-
-def lshift(a, b):
-    "Same as a << b."
-    return a << b
-
-def mod(a, b):
-    "Same as a % b."
-    return a % b
-
-def mul(a, b):
-    "Same as a * b."
-    return a * b
-
-def matmul(a, b):
-    "Same as a @ b."
-    return a @ b
-
-def neg(a):
-    "Same as -a."
-    return -a
-
-def or_(a, b):
-    "Same as a | b."
-    return a | b
-
-def pos(a):
-    "Same as +a."
-    return +a
-
-def pow(a, b):
-    "Same as a ** b."
-    return a ** b
-
-def rshift(a, b):
-    "Same as a >> b."
-    return a >> b
-
-def sub(a, b):
-    "Same as a - b."
-    return a - b
-
-def truediv(a, b):
-    "Same as a / b."
-    return a / b
-
-def xor(a, b):
-    "Same as a ^ b."
-    return a ^ b
-
-# Sequence Operations *********************************************************#
-
-def concat(a, b):
-    "Same as a + b, for a and b sequences."
-    if not hasattr(a, '__getitem__'):
-        msg = "'%s' object can't be concatenated" % type(a).__name__
-        raise TypeError(msg)
-    return a + b
-
-def contains(a, b):
-    "Same as b in a (note reversed operands)."
-    return b in a
-
-def countOf(a, b):
-    "Return the number of times b occurs in a."
-    count = 0
-    for i in a:
-        if i == b:
-            count += 1
-    return count
-
-def delitem(a, b):
-    "Same as del a[b]."
-    del a[b]
-
-def getitem(a, b):
-    "Same as a[b]."
-    return a[b]
-
-def indexOf(a, b):
-    "Return the first index of b in a."
-    for i, j in enumerate(a):
-        if j == b:
-            return i
-    else:
-        raise ValueError('sequence.index(x): x not in sequence')
-
-def setitem(a, b, c):
-    "Same as a[b] = c."
-    a[b] = c
-
-def length_hint(obj, default=0):
-    """
-    Return an estimate of the number of items in obj.
-    This is useful for presizing containers when building from an iterable.
-
-    If the object supports len(), the result will be exact. Otherwise, it may
-    over- or under-estimate by an arbitrary amount. The result will be an
-    integer >= 0.
-    """
-    if not isinstance(default, int):
-        msg = ("'%s' object cannot be interpreted as an integer" %
-               type(default).__name__)
-        raise TypeError(msg)
-
-    try:
-        return len(obj)
-    except TypeError:
-        pass
-
-    try:
-        hint = type(obj).__length_hint__
-    except AttributeError:
-        return default
-
-    try:
-        val = hint(obj)
-    except TypeError:
-        return default
-    if val is NotImplemented:
-        return default
-    if not isinstance(val, int):
-        msg = ('__length_hint__ must be integer, not %s' %
-               type(val).__name__)
-        raise TypeError(msg)
-    if val < 0:
-        msg = '__length_hint__() should return >= 0'
-        raise ValueError(msg)
-    return val
-
-# Generalized Lookup Objects **************************************************#
-
-class attrgetter:
-    """
-    Return a callable object that fetches the given attribute(s) from its operand.
-    After f = attrgetter('name'), the call f(r) returns r.name.
-    After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date).
-    After h = attrgetter('name.first', 'name.last'), the call h(r) returns
-    (r.name.first, r.name.last).
-    """
-    __slots__ = ('_attrs', '_call')
-
-    def __init__(self, attr, *attrs):
-        if not attrs:
-            if not isinstance(attr, str):
-                raise TypeError('attribute name must be a string')
-            self._attrs = (attr,)
-            names = attr.split('.')
-            def func(obj):
-                for name in names:
-                    obj = getattr(obj, name)
-                return obj
-            self._call = func
-        else:
-            self._attrs = (attr,) + attrs
-            getters = tuple(map(attrgetter, self._attrs))
-            def func(obj):
-                return tuple(getter(obj) for getter in getters)
-            self._call = func
-
-    def __call__(self, obj):
-        return self._call(obj)
-
-    def __repr__(self):
-        return '%s.%s(%s)' % (self.__class__.__module__,
-                              self.__class__.__qualname__,
-                              ', '.join(map(repr, self._attrs)))
-
-    def __reduce__(self):
-        return self.__class__, self._attrs
-
-class itemgetter:
-    """
-    Return a callable object that fetches the given item(s) from its operand.
-    After f = itemgetter(2), the call f(r) returns r[2].
-    After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3])
-    """
-    __slots__ = ('_items', '_call')
-
-    def __init__(self, item, *items):
-        if not items:
-            self._items = (item,)
-            def func(obj):
-                return obj[item]
-            self._call = func
-        else:
-            self._items = items = (item,) + items
-            def func(obj):
-                return tuple(obj[i] for i in items)
-            self._call = func
-
-    def __call__(self, obj):
-        return self._call(obj)
-
-    def __repr__(self):
-        return '%s.%s(%s)' % (self.__class__.__module__,
-                              self.__class__.__name__,
-                              ', '.join(map(repr, self._items)))
-
-    def __reduce__(self):
-        return self.__class__, self._items
-
-class methodcaller:
-    """
-    Return a callable object that calls the given method on its operand.
-    After f = methodcaller('name'), the call f(r) returns r.name().
-    After g = methodcaller('name', 'date', foo=1), the call g(r) returns
-    r.name('date', foo=1).
-    """
-    __slots__ = ('_name', '_args', '_kwargs')
-
-    def __init__(*args, **kwargs):
-        if len(args) < 2:
-            msg = "methodcaller needs at least one argument, the method name"
-            raise TypeError(msg)
-        self = args[0]
-        self._name = args[1]
-        if not isinstance(self._name, str):
-            raise TypeError('method name must be a string')
-        self._args = args[2:]
-        self._kwargs = kwargs
-
-    def __call__(self, obj):
-        return getattr(obj, self._name)(*self._args, **self._kwargs)
-
-    def __repr__(self):
-        args = [repr(self._name)]
-        args.extend(map(repr, self._args))
-        args.extend('%s=%r' % (k, v) for k, v in self._kwargs.items())
-        return '%s.%s(%s)' % (self.__class__.__module__,
-                              self.__class__.__name__,
-                              ', '.join(args))
-
-    def __reduce__(self):
-        if not self._kwargs:
-            return self.__class__, (self._name,) + self._args
-        else:
-            from functools import partial
-            return partial(self.__class__, self._name, **self._kwargs), self._args
-
-
-# In-place Operations *********************************************************#
-
-def iadd(a, b):
-    "Same as a += b."
-    a += b
-    return a
-
-def iand(a, b):
-    "Same as a &= b."
-    a &= b
-    return a
-
-def iconcat(a, b):
-    "Same as a += b, for a and b sequences."
-    if not hasattr(a, '__getitem__'):
-        msg = "'%s' object can't be concatenated" % type(a).__name__
-        raise TypeError(msg)
-    a += b
-    return a
-
-def ifloordiv(a, b):
-    "Same as a //= b."
-    a //= b
-    return a
-
-def ilshift(a, b):
-    "Same as a <<= b."
-    a <<= b
-    return a
-
-def imod(a, b):
-    "Same as a %= b."
-    a %= b
-    return a
-
-def imul(a, b):
-    "Same as a *= b."
-    a *= b
-    return a
-
-def imatmul(a, b):
-    "Same as a @= b."
-    a @= b
-    return a
-
-def ior(a, b):
-    "Same as a |= b."
-    a |= b
-    return a
-
-def ipow(a, b):
-    "Same as a **= b."
-    a **=b
-    return a
-
-def irshift(a, b):
-    "Same as a >>= b."
-    a >>= b
-    return a
-
-def isub(a, b):
-    "Same as a -= b."
-    a -= b
-    return a
-
-def itruediv(a, b):
-    "Same as a /= b."
-    a /= b
-    return a
-
-def ixor(a, b):
-    "Same as a ^= b."
-    a ^= b
-    return a
-
-
-try:
-    from _operator import *
-except ImportError:
-    pass
-else:
-    from _operator import __doc__
-
-# All of these "__func__ = func" assignments have to happen after importing
-# from _operator to make sure they're set to the right function
-__lt__ = lt
-__le__ = le
-__eq__ = eq
-__ne__ = ne
-__ge__ = ge
-__gt__ = gt
-__not__ = not_
-__abs__ = abs
-__add__ = add
-__and__ = and_
-__floordiv__ = floordiv
-__index__ = index
-__inv__ = inv
-__invert__ = invert
-__lshift__ = lshift
-__mod__ = mod
-__mul__ = mul
-__matmul__ = matmul
-__neg__ = neg
-__or__ = or_
-__pos__ = pos
-__pow__ = pow
-__rshift__ = rshift
-__sub__ = sub
-__truediv__ = truediv
-__xor__ = xor
-__concat__ = concat
-__contains__ = contains
-__delitem__ = delitem
-__getitem__ = getitem
-__setitem__ = setitem
-__iadd__ = iadd
-__iand__ = iand
-__iconcat__ = iconcat
-__ifloordiv__ = ifloordiv
-__ilshift__ = ilshift
-__imod__ = imod
-__imul__ = imul
-__imatmul__ = imatmul
-__ior__ = ior
-__ipow__ = ipow
-__irshift__ = irshift
-__isub__ = isub
-__itruediv__ = itruediv
-__ixor__ = ixor

Lib/os.py

@@ -1,135 +0,0 @@
-import sys
-
-from _os import *
-
-
-if name == 'nt':
-    linesep = '\r\n'
-    import ntpath as path
-else:
-    linesep = '\n'
-    import posixpath as path
-
-
-sys.modules['os.path'] = path
-from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, altsep,
-    devnull)
-
-# Change environ to automatically call putenv(), unsetenv if they exist.
-from _collections_abc import MutableMapping
-
-class _Environ(MutableMapping):
-    def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv):
-        self.encodekey = encodekey
-        self.decodekey = decodekey
-        self.encodevalue = encodevalue
-        self.decodevalue = decodevalue
-        self.putenv = putenv
-        self.unsetenv = unsetenv
-        self._data = data
-
-    def __getitem__(self, key):
-        try:
-            value = self._data[self.encodekey(key)]
-        except KeyError:
-            # raise KeyError with the original key value
-            raise KeyError(key) from None
-
-        return self.decodevalue(value)
-
-    def __setitem__(self, key, value):
-        key = self.encodekey(key)
-        value = self.encodevalue(value)
-        self.putenv(key, value)
-        self._data[key] = value
-
-    def __delitem__(self, key):
-        encodedkey = self.encodekey(key)
-        self.unsetenv(encodedkey)
-        try:
-            del self._data[encodedkey]
-        except KeyError:
-            # raise KeyError with the original key value
-            raise KeyError(key) from None
-
-    def __iter__(self):
-        # list() from dict object is an atomic operation
-        keys = list(self._data)
-        for key in keys:
-            yield self.decodekey(key)
-
-    def __len__(self):
-        return len(self._data)
-
-    def __repr__(self):
-        return 'environ({{{}}})'.format(', '.join(
-            ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value))
-            for key, value in self._data.items())))
-
-    def copy(self):
-        return dict(self)
-
-    def setdefault(self, key, value):
-        if key not in self:
-            self[key] = value
-        return self[key]
-
-try:
-    _putenv = putenv
-except NameError:
-    _putenv = lambda key, value: None
-# else:
-#     if "putenv" not in __all__:
-#         __all__.append("putenv")
-
-try:
-    _unsetenv = unsetenv
-except NameError:
-    _unsetenv = lambda key: _putenv(key, "")
-# else:
-#     if "unsetenv" not in __all__:
-#         __all__.append("unsetenv")
-
-def _createenviron():
-    # if name == 'nt':
-    #     # Where Env Var Names Must Be UPPERCASE
-    #     def check_str(value):
-    #         if not isinstance(value, str):
-    #             raise TypeError("str expected, not %s" % type(value).__name__)
-    #         return value
-    #     encode = check_str
-    #     decode = str
-    #     def encodekey(key):
-    #         return encode(key).upper()
-    #     data = {}
-    #     for key, value in environ.items():
-    #         data[encodekey(key)] = value
-    # else:
-    #     # Where Env Var Names Can Be Mixed Case
-    #     encoding = sys.getfilesystemencoding()
-    #     def encode(value):
-    #         if not isinstance(value, str):
-    #             raise TypeError("str expected, not %s" % type(value).__name__)
-    #         return value.encode(encoding, 'surrogateescape')
-    #     def decode(value):
-    #         return value.decode(encoding, 'surrogateescape')
-    #     encodekey = encode
-    decode = str
-    encode = str
-    encodekey = encode
-    data = environ
-    return _Environ(data,
-        encodekey, decode,
-        encode, decode,
-        _putenv, _unsetenv)
-
-# unicode environ
-environ = _createenviron()
-del _createenviron
-
-
-def getenv(key, default=None):
-    """Get an environment variable, return None if it doesn't exist.
-    The optional second argument can specify an alternate default.
-    key, default and the result are str."""
-    return environ.get(key, default)

Lib/posixpath.py

@@ -1,525 +0,0 @@
-"""Common operations on Posix pathnames.
-
-Instead of importing this module directly, import os and refer to
-this module as os.path.  The "os.path" name is an alias for this
-module on Posix systems; on other systems (e.g. Windows),
-os.path provides the same operations in a manner specific to that
-platform, and is an alias to another module (e.g. ntpath).
-
-Some of this can actually be useful on non-Posix systems too, e.g.
-for manipulation of the pathname component of URLs.
-"""
-
-# Strings representing various path-related bits and pieces.
-# These are primarily for export; internally, they are hardcoded.
-# Should be set before imports for resolving cyclic dependency.
-curdir = '.'
-pardir = '..'
-extsep = '.'
-sep = '/'
-pathsep = ':'
-defpath = '/bin:/usr/bin'
-altsep = None
-devnull = '/dev/null'
-
-import os
-import sys
-import stat
-import genericpath
-from genericpath import *
-
-__all__ = ["normcase","isabs","join","splitdrive","split","splitext",
-           "basename","dirname","commonprefix","getsize","getmtime",
-           "getatime","getctime","islink","exists","lexists","isdir","isfile",
-           "ismount", "expanduser","expandvars","normpath","abspath",
-           "samefile","sameopenfile","samestat",
-           "curdir","pardir","sep","pathsep","defpath","altsep","extsep",
-           "devnull","realpath","supports_unicode_filenames","relpath",
-           "commonpath"]
-
-
-def _get_sep(path):
-    if isinstance(path, bytes):
-        return b'/'
-    else:
-        return '/'
-
-# Normalize the case of a pathname.  Trivial in Posix, string.lower on Mac.
-# On MS-DOS this may also turn slashes into backslashes; however, other
-# normalizations (such as optimizing '../' away) are not allowed
-# (another function should be defined to do that).
-
-def normcase(s):
-    """Normalize case of pathname.  Has no effect under Posix"""
-    return os.fspath(s)
-
-
-# Return whether a path is absolute.
-# Trivial in Posix, harder on the Mac or MS-DOS.
-
-def isabs(s):
-    """Test whether a path is absolute"""
-    s = os.fspath(s)
-    sep = _get_sep(s)
-    return s.startswith(sep)
-
-
-# Join pathnames.
-# Ignore the previous parts if a part is absolute.
-# Insert a '/' unless the first part is empty or already ends in '/'.
-
-def join(a, *p):
-    """Join two or more pathname components, inserting '/' as needed.
-    If any component is an absolute path, all previous path components
-    will be discarded.  An empty last part will result in a path that
-    ends with a separator."""
-    a = os.fspath(a)
-    sep = _get_sep(a)
-    path = a
-    try:
-        if not p:
-            path[:0] + sep  #23780: Ensure compatible data type even if p is null.
-        for b in map(os.fspath, p):
-            if b.startswith(sep):
-                path = b
-            elif not path or path.endswith(sep):
-                path += b
-            else:
-                path += sep + b
-    except (TypeError, AttributeError, BytesWarning):
-        genericpath._check_arg_types('join', a, *p)
-        raise
-    return path
-
-
-# Split a path in head (everything up to the last '/') and tail (the
-# rest).  If the path ends in '/', tail will be empty.  If there is no
-# '/' in the path, head  will be empty.
-# Trailing '/'es are stripped from head unless it is the root.
-
-def split(p):
-    """Split a pathname.  Returns tuple "(head, tail)" where "tail" is
-    everything after the final slash.  Either part may be empty."""
-    p = os.fspath(p)
-    sep = _get_sep(p)
-    i = p.rfind(sep) + 1
-    head, tail = p[:i], p[i:]
-    if head and head != sep*len(head):
-        head = head.rstrip(sep)
-    return head, tail
-
-
-# Split a path in root and extension.
-# The extension is everything starting at the last dot in the last
-# pathname component; the root is everything before that.
-# It is always true that root + ext == p.
-
-def splitext(p):
-    p = os.fspath(p)
-    if isinstance(p, bytes):
-        sep = b'/'
-        extsep = b'.'
-    else:
-        sep = '/'
-        extsep = '.'
-    return genericpath._splitext(p, sep, None, extsep)
-splitext.__doc__ = genericpath._splitext.__doc__
-
-# Split a pathname into a drive specification and the rest of the
-# path.  Useful on DOS/Windows/NT; on Unix, the drive is always empty.
-
-def splitdrive(p):
-    """Split a pathname into drive and path. On Posix, drive is always
-    empty."""
-    p = os.fspath(p)
-    return p[:0], p
-
-
-# Return the tail (basename) part of a path, same as split(path)[1].
-
-def basename(p):
-    """Returns the final component of a pathname"""
-    p = os.fspath(p)
-    sep = _get_sep(p)
-    i = p.rfind(sep) + 1
-    return p[i:]
-
-
-# Return the head (dirname) part of a path, same as split(path)[0].
-
-def dirname(p):
-    """Returns the directory component of a pathname"""
-    p = os.fspath(p)
-    sep = _get_sep(p)
-    i = p.rfind(sep) + 1
-    head = p[:i]
-    if head and head != sep*len(head):
-        head = head.rstrip(sep)
-    return head
-
-
-# Is a path a symbolic link?
-# This will always return false on systems where os.lstat doesn't exist.
-
-def islink(path):
-    """Test whether a path is a symbolic link"""
-    try:
-        st = os.lstat(path)
-    except (OSError, ValueError, AttributeError):
-        return False
-    return stat.S_ISLNK(st.st_mode)
-
-# Being true for dangling symbolic links is also useful.
-
-def lexists(path):
-    """Test whether a path exists.  Returns True for broken symbolic links"""
-    try:
-        os.lstat(path)
-    except (OSError, ValueError):
-        return False
-    return True
-
-
-# Is a path a mount point?
-# (Does this work for all UNIXes?  Is it even guaranteed to work by Posix?)
-
-def ismount(path):
-    """Test whether a path is a mount point"""
-    try:
-        s1 = os.lstat(path)
-    except (OSError, ValueError):
-        # It doesn't exist -- so not a mount point. :-)
-        return False
-    else:
-        # A symlink can never be a mount point
-        if stat.S_ISLNK(s1.st_mode):
-            return False
-
-    if isinstance(path, bytes):
-        parent = join(path, b'..')
-    else:
-        parent = join(path, '..')
-    parent = realpath(parent)
-    try:
-        s2 = os.lstat(parent)
-    except (OSError, ValueError):
-        return False
-
-    dev1 = s1.st_dev
-    dev2 = s2.st_dev
-    if dev1 != dev2:
-        return True     # path/.. on a different device as path
-    ino1 = s1.st_ino
-    ino2 = s2.st_ino
-    if ino1 == ino2:
-        return True     # path/.. is the same i-node as path
-    return False
-
-
-# Expand paths beginning with '~' or '~user'.
-# '~' means $HOME; '~user' means that user's home directory.
-# If the path doesn't begin with '~', or if the user or $HOME is unknown,
-# the path is returned unchanged (leaving error reporting to whatever
-# function is called with the expanded path as argument).
-# See also module 'glob' for expansion of *, ? and [...] in pathnames.
-# (A function should also be defined to do full *sh-style environment
-# variable expansion.)
-
-def expanduser(path):
-    """Expand ~ and ~user constructions.  If user or $HOME is unknown,
-    do nothing."""
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        tilde = b'~'
-    else:
-        tilde = '~'
-    if not path.startswith(tilde):
-        return path
-    sep = _get_sep(path)
-    i = path.find(sep, 1)
-    if i < 0:
-        i = len(path)
-    if i == 1:
-        if 'HOME' not in os.environ:
-            import pwd
-            try:
-                userhome = pwd.getpwuid(os.getuid()).pw_dir
-            except KeyError:
-                # bpo-10496: if the current user identifier doesn't exist in the
-                # password database, return the path unchanged
-                return path
-        else:
-            userhome = os.environ['HOME']
-    else:
-        import pwd
-        name = path[1:i]
-        if isinstance(name, bytes):
-            name = str(name, 'ASCII')
-        try:
-            pwent = pwd.getpwnam(name)
-        except KeyError:
-            # bpo-10496: if the user name from the path doesn't exist in the
-            # password database, return the path unchanged
-            return path
-        userhome = pwent.pw_dir
-    if isinstance(path, bytes):
-        userhome = os.fsencode(userhome)
-        root = b'/'
-    else:
-        root = '/'
-    userhome = userhome.rstrip(root)
-    return (userhome + path[i:]) or root
-
-
-# Expand paths containing shell variable substitutions.
-# This expands the forms $variable and ${variable} only.
-# Non-existent variables are left unchanged.
-
-_varprog = None
-_varprogb = None
-
-def expandvars(path):
-    """Expand shell variables of form $var and ${var}.  Unknown variables
-    are left unchanged."""
-    path = os.fspath(path)
-    global _varprog, _varprogb
-    if isinstance(path, bytes):
-        if b'$' not in path:
-            return path
-        if not _varprogb:
-            import re
-            _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII)
-        search = _varprogb.search
-        start = b'{'
-        end = b'}'
-        environ = getattr(os, 'environb', None)
-    else:
-        if '$' not in path:
-            return path
-        if not _varprog:
-            import re
-            _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII)
-        search = _varprog.search
-        start = '{'
-        end = '}'
-        environ = os.environ
-    i = 0
-    while True:
-        m = search(path, i)
-        if not m:
-            break
-        i, j = m.span(0)
-        name = m.group(1)
-        if name.startswith(start) and name.endswith(end):
-            name = name[1:-1]
-        try:
-            if environ is None:
-                value = os.fsencode(os.environ[os.fsdecode(name)])
-            else:
-                value = environ[name]
-        except KeyError:
-            i = j
-        else:
-            tail = path[j:]
-            path = path[:i] + value
-            i = len(path)
-            path += tail
-    return path
-
-
-# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B.
-# It should be understood that this may change the meaning of the path
-# if it contains symbolic links!
-
-def normpath(path):
-    """Normalize path, eliminating double slashes, etc."""
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        sep = b'/'
-        empty = b''
-        dot = b'.'
-        dotdot = b'..'
-    else:
-        sep = '/'
-        empty = ''
-        dot = '.'
-        dotdot = '..'
-    if path == empty:
-        return dot
-    initial_slashes = path.startswith(sep)
-    # POSIX allows one or two initial slashes, but treats three or more
-    # as single slash.
-    if (initial_slashes and
-        path.startswith(sep*2) and not path.startswith(sep*3)):
-        initial_slashes = 2
-    comps = path.split(sep)
-    new_comps = []
-    for comp in comps:
-        if comp in (empty, dot):
-            continue
-        if (comp != dotdot or (not initial_slashes and not new_comps) or
-             (new_comps and new_comps[-1] == dotdot)):
-            new_comps.append(comp)
-        elif new_comps:
-            new_comps.pop()
-    comps = new_comps
-    path = sep.join(comps)
-    if initial_slashes:
-        path = sep*initial_slashes + path
-    return path or dot
-
-
-def abspath(path):
-    """Return an absolute path."""
-    path = os.fspath(path)
-    if not isabs(path):
-        if isinstance(path, bytes):
-            cwd = os.getcwdb()
-        else:
-            cwd = os.getcwd()
-        path = join(cwd, path)
-    return normpath(path)
-
-
-# Return a canonical path (i.e. the absolute location of a file on the
-# filesystem).
-
-def realpath(filename):
-    """Return the canonical path of the specified filename, eliminating any
-symbolic links encountered in the path."""
-    filename = os.fspath(filename)
-    path, ok = _joinrealpath(filename[:0], filename, {})
-    return abspath(path)
-
-# Join two paths, normalizing and eliminating any symbolic links
-# encountered in the second path.
-def _joinrealpath(path, rest, seen):
-    if isinstance(path, bytes):
-        sep = b'/'
-        curdir = b'.'
-        pardir = b'..'
-    else:
-        sep = '/'
-        curdir = '.'
-        pardir = '..'
-
-    if isabs(rest):
-        rest = rest[1:]
-        path = sep
-
-    while rest:
-        name, _, rest = rest.partition(sep)
-        if not name or name == curdir:
-            # current dir
-            continue
-        if name == pardir:
-            # parent dir
-            if path:
-                path, name = split(path)
-                if name == pardir:
-                    path = join(path, pardir, pardir)
-            else:
-                path = pardir
-            continue
-        newpath = join(path, name)
-        if not islink(newpath):
-            path = newpath
-            continue
-        # Resolve the symbolic link
-        if newpath in seen:
-            # Already seen this path
-            path = seen[newpath]
-            if path is not None:
-                # use cached value
-                continue
-            # The symlink is not resolved, so we must have a symlink loop.
-            # Return already resolved part + rest of the path unchanged.
-            return join(newpath, rest), False
-        seen[newpath] = None # not resolved symlink
-        path, ok = _joinrealpath(path, os.readlink(newpath), seen)
-        if not ok:
-            return join(path, rest), False
-        seen[newpath] = path # resolved symlink
-
-    return path, True
-
-
-supports_unicode_filenames = (sys.platform == 'darwin')
-
-def relpath(path, start=None):
-    """Return a relative version of a path"""
-
-    if not path:
-        raise ValueError("no path specified")
-
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        curdir = b'.'
-        sep = b'/'
-        pardir = b'..'
-    else:
-        curdir = '.'
-        sep = '/'
-        pardir = '..'
-
-    if start is None:
-        start = curdir
-    else:
-        start = os.fspath(start)
-
-    try:
-        start_list = [x for x in abspath(start).split(sep) if x]
-        path_list = [x for x in abspath(path).split(sep) if x]
-        # Work out how much of the filepath is shared by start and path.
-        i = len(commonprefix([start_list, path_list]))
-
-        rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
-        if not rel_list:
-            return curdir
-        return join(*rel_list)
-    except (TypeError, AttributeError, BytesWarning, DeprecationWarning):
-        genericpath._check_arg_types('relpath', path, start)
-        raise
-
-
-# Return the longest common sub-path of the sequence of paths given as input.
-# The paths are not normalized before comparing them (this is the
-# responsibility of the caller). Any trailing separator is stripped from the
-# returned path.
-
-def commonpath(paths):
-    """Given a sequence of path names, returns the longest common sub-path."""
-
-    if not paths:
-        raise ValueError('commonpath() arg is an empty sequence')
-
-    paths = tuple(map(os.fspath, paths))
-    if isinstance(paths[0], bytes):
-        sep = b'/'
-        curdir = b'.'
-    else:
-        sep = '/'
-        curdir = '.'
-
-    try:
-        split_paths = [path.split(sep) for path in paths]
-
-        try:
-            isabs, = set(p[:1] == sep for p in paths)
-        except ValueError:
-            raise ValueError("Can't mix absolute and relative paths") from None
-
-        split_paths = [[c for c in s if c and c != curdir] for s in split_paths]
-        s1 = min(split_paths)
-        s2 = max(split_paths)
-        common = s1
-        for i, c in enumerate(s1):
-            if c != s2[i]:
-                common = s1[:i]
-                break
-
-        prefix = sep if isabs else sep[:0]
-        return prefix + sep.join(common)
-    except (TypeError, AttributeError):
-        genericpath._check_arg_types('commonpath', *paths)
-        raise

Lib/re.py

@@ -0,0 +1,11 @@
+
+""" Regular expressions """
+
+
+def match(pattern, string, flags=0):
+    return _compile(pattern, flags).match(string)
+
+
+def _compile(pattern, flags):
+    p = sre_compile.compile(pattern, flags)
+    return p

Lib/reprlib.py

@@ -1,161 +0,0 @@
-"""Redo the builtin repr() (representation) but with limits on most sizes."""
-
-__all__ = ["Repr", "repr", "recursive_repr"]
-
-import builtins
-from itertools import islice
-from _thread import get_ident
-
-def recursive_repr(fillvalue='...'):
-    'Decorator to make a repr function return fillvalue for a recursive call'
-
-    def decorating_function(user_function):
-        repr_running = set()
-
-        def wrapper(self):
-            key = id(self), get_ident()
-            if key in repr_running:
-                return fillvalue
-            repr_running.add(key)
-            try:
-                result = user_function(self)
-            finally:
-                repr_running.discard(key)
-            return result
-
-        # Can't use functools.wraps() here because of bootstrap issues
-        wrapper.__module__ = getattr(user_function, '__module__')
-        wrapper.__doc__ = getattr(user_function, '__doc__')
-        wrapper.__name__ = getattr(user_function, '__name__')
-        wrapper.__qualname__ = getattr(user_function, '__qualname__')
-        wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
-        return wrapper
-
-    return decorating_function
-
-class Repr:
-
-    def __init__(self):
-        self.maxlevel = 6
-        self.maxtuple = 6
-        self.maxlist = 6
-        self.maxarray = 5
-        self.maxdict = 4
-        self.maxset = 6
-        self.maxfrozenset = 6
-        self.maxdeque = 6
-        self.maxstring = 30
-        self.maxlong = 40
-        self.maxother = 30
-
-    def repr(self, x):
-        return self.repr1(x, self.maxlevel)
-
-    def repr1(self, x, level):
-        typename = type(x).__name__
-        if ' ' in typename:
-            parts = typename.split()
-            typename = '_'.join(parts)
-        if hasattr(self, 'repr_' + typename):
-            return getattr(self, 'repr_' + typename)(x, level)
-        else:
-            return self.repr_instance(x, level)
-
-    def _repr_iterable(self, x, level, left, right, maxiter, trail=''):
-        n = len(x)
-        if level <= 0 and n:
-            s = '...'
-        else:
-            newlevel = level - 1
-            repr1 = self.repr1
-            pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)]
-            if n > maxiter:  pieces.append('...')
-            s = ', '.join(pieces)
-            if n == 1 and trail:  right = trail + right
-        return '%s%s%s' % (left, s, right)
-
-    def repr_tuple(self, x, level):
-        return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',')
-
-    def repr_list(self, x, level):
-        return self._repr_iterable(x, level, '[', ']', self.maxlist)
-
-    def repr_array(self, x, level):
-        if not x:
-            return "array('%s')" % x.typecode
-        header = "array('%s', [" % x.typecode
-        return self._repr_iterable(x, level, header, '])', self.maxarray)
-
-    def repr_set(self, x, level):
-        if not x:
-            return 'set()'
-        x = _possibly_sorted(x)
-        return self._repr_iterable(x, level, '{', '}', self.maxset)
-
-    def repr_frozenset(self, x, level):
-        if not x:
-            return 'frozenset()'
-        x = _possibly_sorted(x)
-        return self._repr_iterable(x, level, 'frozenset({', '})',
-                                   self.maxfrozenset)
-
-    def repr_deque(self, x, level):
-        return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque)
-
-    def repr_dict(self, x, level):
-        n = len(x)
-        if n == 0: return '{}'
-        if level <= 0: return '{...}'
-        newlevel = level - 1
-        repr1 = self.repr1
-        pieces = []
-        for key in islice(_possibly_sorted(x), self.maxdict):
-            keyrepr = repr1(key, newlevel)
-            valrepr = repr1(x[key], newlevel)
-            pieces.append('%s: %s' % (keyrepr, valrepr))
-        if n > self.maxdict: pieces.append('...')
-        s = ', '.join(pieces)
-        return '{%s}' % (s,)
-
-    def repr_str(self, x, level):
-        s = builtins.repr(x[:self.maxstring])
-        if len(s) > self.maxstring:
-            i = max(0, (self.maxstring-3)//2)
-            j = max(0, self.maxstring-3-i)
-            s = builtins.repr(x[:i] + x[len(x)-j:])
-            s = s[:i] + '...' + s[len(s)-j:]
-        return s
-
-    def repr_int(self, x, level):
-        s = builtins.repr(x) # XXX Hope this isn't too slow...
-        if len(s) > self.maxlong:
-            i = max(0, (self.maxlong-3)//2)
-            j = max(0, self.maxlong-3-i)
-            s = s[:i] + '...' + s[len(s)-j:]
-        return s
-
-    def repr_instance(self, x, level):
-        try:
-            s = builtins.repr(x)
-            # Bugs in x.__repr__() can cause arbitrary
-            # exceptions -- then make up something
-        except Exception:
-            return '<%s instance at %#x>' % (x.__class__.__name__, id(x))
-        if len(s) > self.maxother:
-            i = max(0, (self.maxother-3)//2)
-            j = max(0, self.maxother-3-i)
-            s = s[:i] + '...' + s[len(s)-j:]
-        return s
-
-
-def _possibly_sorted(x):
-    # Since not all sequences of items can be sorted and comparison
-    # functions may raise arbitrary exceptions, return an unsorted
-    # sequence in that case.
-    try:
-        return sorted(x)
-    except Exception:
-        return list(x)
-
-aRepr = Repr()
-repr = aRepr.repr

Lib/stat.py

@@ -1,179 +0,0 @@
-"""Constants/functions for interpreting results of os.stat() and os.lstat().
-
-Suggested usage: from stat import *
-"""
-
-# Indices for stat struct members in the tuple returned by os.stat()
-
-ST_MODE  = 0
-ST_INO   = 1
-ST_DEV   = 2
-ST_NLINK = 3
-ST_UID   = 4
-ST_GID   = 5
-ST_SIZE  = 6
-ST_ATIME = 7
-ST_MTIME = 8
-ST_CTIME = 9
-
-# Extract bits from the mode
-
-def S_IMODE(mode):
-    """Return the portion of the file's mode that can be set by
-    os.chmod().
-    """
-    return mode & 0o7777
-
-def S_IFMT(mode):
-    """Return the portion of the file's mode that describes the
-    file type.
-    """
-    return mode & 0o170000
-
-# Constants used as S_IFMT() for various file types
-# (not all are implemented on all systems)
-
-S_IFDIR  = 0o040000  # directory
-S_IFCHR  = 0o020000  # character device
-S_IFBLK  = 0o060000  # block device
-S_IFREG  = 0o100000  # regular file
-S_IFIFO  = 0o010000  # fifo (named pipe)
-S_IFLNK  = 0o120000  # symbolic link
-S_IFSOCK = 0o140000  # socket file
-
-# Functions to test for each file type
-
-def S_ISDIR(mode):
-    """Return True if mode is from a directory."""
-    return S_IFMT(mode) == S_IFDIR
-
-def S_ISCHR(mode):
-    """Return True if mode is from a character special device file."""
-    return S_IFMT(mode) == S_IFCHR
-
-def S_ISBLK(mode):
-    """Return True if mode is from a block special device file."""
-    return S_IFMT(mode) == S_IFBLK
-
-def S_ISREG(mode):
-    """Return True if mode is from a regular file."""
-    return S_IFMT(mode) == S_IFREG
-
-def S_ISFIFO(mode):
-    """Return True if mode is from a FIFO (named pipe)."""
-    return S_IFMT(mode) == S_IFIFO
-
-def S_ISLNK(mode):
-    """Return True if mode is from a symbolic link."""
-    return S_IFMT(mode) == S_IFLNK
-
-def S_ISSOCK(mode):
-    """Return True if mode is from a socket."""
-    return S_IFMT(mode) == S_IFSOCK
-
-# Names for permission bits
-
-S_ISUID = 0o4000  # set UID bit
-S_ISGID = 0o2000  # set GID bit
-S_ENFMT = S_ISGID # file locking enforcement
-S_ISVTX = 0o1000  # sticky bit
-S_IREAD = 0o0400  # Unix V7 synonym for S_IRUSR
-S_IWRITE = 0o0200 # Unix V7 synonym for S_IWUSR
-S_IEXEC = 0o0100  # Unix V7 synonym for S_IXUSR
-S_IRWXU = 0o0700  # mask for owner permissions
-S_IRUSR = 0o0400  # read by owner
-S_IWUSR = 0o0200  # write by owner
-S_IXUSR = 0o0100  # execute by owner
-S_IRWXG = 0o0070  # mask for group permissions
-S_IRGRP = 0o0040  # read by group
-S_IWGRP = 0o0020  # write by group
-S_IXGRP = 0o0010  # execute by group
-S_IRWXO = 0o0007  # mask for others (not in group) permissions
-S_IROTH = 0o0004  # read by others
-S_IWOTH = 0o0002  # write by others
-S_IXOTH = 0o0001  # execute by others
-
-# Names for file flags
-
-UF_NODUMP    = 0x00000001  # do not dump file
-UF_IMMUTABLE = 0x00000002  # file may not be changed
-UF_APPEND    = 0x00000004  # file may only be appended to
-UF_OPAQUE    = 0x00000008  # directory is opaque when viewed through a union stack
-UF_NOUNLINK  = 0x00000010  # file may not be renamed or deleted
-UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed
-UF_HIDDEN    = 0x00008000  # OS X: file should not be displayed
-SF_ARCHIVED  = 0x00010000  # file may be archived
-SF_IMMUTABLE = 0x00020000  # file may not be changed
-SF_APPEND    = 0x00040000  # file may only be appended to
-SF_NOUNLINK  = 0x00100000  # file may not be renamed or deleted
-SF_SNAPSHOT  = 0x00200000  # file is a snapshot file
-
-
-_filemode_table = (
-    ((S_IFLNK,         "l"),
-     (S_IFSOCK,        "s"),  # Must appear before IFREG and IFDIR as IFSOCK == IFREG | IFDIR
-     (S_IFREG,         "-"),
-     (S_IFBLK,         "b"),
-     (S_IFDIR,         "d"),
-     (S_IFCHR,         "c"),
-     (S_IFIFO,         "p")),
-
-    ((S_IRUSR,         "r"),),
-    ((S_IWUSR,         "w"),),
-    ((S_IXUSR|S_ISUID, "s"),
-     (S_ISUID,         "S"),
-     (S_IXUSR,         "x")),
-
-    ((S_IRGRP,         "r"),),
-    ((S_IWGRP,         "w"),),
-    ((S_IXGRP|S_ISGID, "s"),
-     (S_ISGID,         "S"),
-     (S_IXGRP,         "x")),
-
-    ((S_IROTH,         "r"),),
-    ((S_IWOTH,         "w"),),
-    ((S_IXOTH|S_ISVTX, "t"),
-     (S_ISVTX,         "T"),
-     (S_IXOTH,         "x"))
-)
-
-def filemode(mode):
-    """Convert a file's mode to a string of the form '-rwxrwxrwx'."""
-    perm = []
-    for table in _filemode_table:
-        for bit, char in table:
-            if mode & bit == bit:
-                perm.append(char)
-                break
-        else:
-            perm.append("-")
-    return "".join(perm)
-
-
-# Windows FILE_ATTRIBUTE constants for interpreting os.stat()'s
-# "st_file_attributes" member
-
-FILE_ATTRIBUTE_ARCHIVE = 32
-FILE_ATTRIBUTE_COMPRESSED = 2048
-FILE_ATTRIBUTE_DEVICE = 64
-FILE_ATTRIBUTE_DIRECTORY = 16
-FILE_ATTRIBUTE_ENCRYPTED = 16384
-FILE_ATTRIBUTE_HIDDEN = 2
-FILE_ATTRIBUTE_INTEGRITY_STREAM = 32768
-FILE_ATTRIBUTE_NORMAL = 128
-FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 8192
-FILE_ATTRIBUTE_NO_SCRUB_DATA = 131072
-FILE_ATTRIBUTE_OFFLINE = 4096
-FILE_ATTRIBUTE_READONLY = 1
-FILE_ATTRIBUTE_REPARSE_POINT = 1024
-FILE_ATTRIBUTE_SPARSE_FILE = 512
-FILE_ATTRIBUTE_SYSTEM = 4
-FILE_ATTRIBUTE_TEMPORARY = 256
-FILE_ATTRIBUTE_VIRTUAL = 65536
-
-
-# If available, use C implementation
-try:
-    from _stat import *
-except ImportError:
-    pass

Lib/this.py

@@ -1,28 +0,0 @@
-s = """Gur Mra bs Clguba, ol Gvz Crgref
-
-Ornhgvshy vf orggre guna htyl.
-Rkcyvpvg vf orggre guna vzcyvpvg.
-Fvzcyr vf orggre guna pbzcyrk.
-Pbzcyrk vf orggre guna pbzcyvpngrq.
-Syng vf orggre guna arfgrq.
-Fcnefr vf orggre guna qrafr.
-Ernqnovyvgl pbhagf.
-Fcrpvny pnfrf nera'g fcrpvny rabhtu gb oernx gur ehyrf.
-Nygubhtu cenpgvpnyvgl orngf chevgl.
-Reebef fubhyq arire cnff fvyragyl.
-Hayrff rkcyvpvgyl fvyraprq.
-Va gur snpr bs nzovthvgl, ershfr gur grzcgngvba gb thrff.
-Gurer fubhyq or bar-- naq cersrenoyl bayl bar --boivbhf jnl gb qb vg.
-Nygubhtu gung jnl znl abg or boivbhf ng svefg hayrff lbh'er Qhgpu.
-Abj vf orggre guna arire.
-Nygubhtu arire vf bsgra orggre guna *evtug* abj.
-Vs gur vzcyrzragngvba vf uneq gb rkcynva, vg'f n onq vqrn.
-Vs gur vzcyrzragngvba vf rnfl gb rkcynva, vg znl or n tbbq vqrn.
-Anzrfcnprf ner bar ubaxvat terng vqrn -- yrg'f qb zber bs gubfr!"""
-
-d = {}
-for c in (65, 97):
-    for i in range(26):
-        d[chr(i+c)] = chr((i+13) % 26 + c)
-
-print("".join([d.get(c, c) for c in s]))

Lib/types.py

@@ -1,295 +0,0 @@
-"""
-Define names for built-in types that aren't directly accessible as a builtin.
-"""
-import sys
-
-# Iterators in Python aren't a matter of type but of protocol.  A large
-# and changing number of builtin types implement *some* flavor of
-# iterator.  Don't check the type!  Use hasattr to check for both
-# "__iter__" and "__next__" attributes instead.
-
-def _f(): pass
-FunctionType = type(_f)
-LambdaType = type(lambda: None)         # Same as FunctionType
-CodeType = type(_f.__code__)
-MappingProxyType = type(type.__dict__)
-SimpleNamespace = type(sys.implementation)
-
-def _g():
-    yield 1
-GeneratorType = type(_g())
-
-# async def _c(): pass
-# _c = _c()
-# CoroutineType = type(_c)
-# _c.close()  # Prevent ResourceWarning
-
-# async def _ag():
-#     yield
-# _ag = _ag()
-# AsyncGeneratorType = type(_ag)
-
-class _C:
-    def _m(self): pass
-MethodType = type(_C()._m)
-
-BuiltinFunctionType = type(len)
-BuiltinMethodType = type([].append)     # Same as BuiltinFunctionType
-
-WrapperDescriptorType = type(object.__init__)
-MethodWrapperType = type(object().__str__)
-MethodDescriptorType = type(str.join)
-ClassMethodDescriptorType = type(dict.__dict__['fromkeys'])
-
-ModuleType = type(sys)
-
-# try:
-#     raise TypeError
-# except TypeError:
-#     tb = sys.exc_info()[2]
-#     TracebackType = type(tb)
-#     FrameType = type(tb.tb_frame)
-#     tb = None; del tb
-
-# For Jython, the following two types are identical
-GetSetDescriptorType = type(FunctionType.__code__)
-# MemberDescriptorType = type(FunctionType.__globals__)
-
-del sys, _f, _g, _C  # Not for export
-
-
-# Provide a PEP 3115 compliant mechanism for class creation
-def new_class(name, bases=(), kwds=None, exec_body=None):
-    """Create a class object dynamically using the appropriate metaclass."""
-    resolved_bases = resolve_bases(bases)
-    meta, ns, kwds = prepare_class(name, resolved_bases, kwds)
-    if exec_body is not None:
-        exec_body(ns)
-    if resolved_bases is not bases:
-        ns['__orig_bases__'] = bases
-    return meta(name, resolved_bases, ns, **kwds)
-
-def resolve_bases(bases):
-    """Resolve MRO entries dynamically as specified by PEP 560."""
-    new_bases = list(bases)
-    updated = False
-    shift = 0
-    for i, base in enumerate(bases):
-        if isinstance(base, type):
-            continue
-        if not hasattr(base, "__mro_entries__"):
-            continue
-        new_base = base.__mro_entries__(bases)
-        updated = True
-        if not isinstance(new_base, tuple):
-            raise TypeError("__mro_entries__ must return a tuple")
-        else:
-            new_bases[i+shift:i+shift+1] = new_base
-            shift += len(new_base) - 1
-    if not updated:
-        return bases
-    return tuple(new_bases)
-
-def prepare_class(name, bases=(), kwds=None):
-    """Call the __prepare__ method of the appropriate metaclass.
-
-    Returns (metaclass, namespace, kwds) as a 3-tuple
-
-    *metaclass* is the appropriate metaclass
-    *namespace* is the prepared class namespace
-    *kwds* is an updated copy of the passed in kwds argument with any
-    'metaclass' entry removed. If no kwds argument is passed in, this will
-    be an empty dict.
-    """
-    if kwds is None:
-        kwds = {}
-    else:
-        kwds = dict(kwds) # Don't alter the provided mapping
-    if 'metaclass' in kwds:
-        meta = kwds.pop('metaclass')
-    else:
-        if bases:
-            meta = type(bases[0])
-        else:
-            meta = type
-    if isinstance(meta, type):
-        # when meta is a type, we first determine the most-derived metaclass
-        # instead of invoking the initial candidate directly
-        meta = _calculate_meta(meta, bases)
-    if hasattr(meta, '__prepare__'):
-        ns = meta.__prepare__(name, bases, **kwds)
-    else:
-        ns = {}
-    return meta, ns, kwds
-
-def _calculate_meta(meta, bases):
-    """Calculate the most derived metaclass."""
-    winner = meta
-    for base in bases:
-        base_meta = type(base)
-        if issubclass(winner, base_meta):
-            continue
-        if issubclass(base_meta, winner):
-            winner = base_meta
-            continue
-        # else:
-        raise TypeError("metaclass conflict: "
-                        "the metaclass of a derived class "
-                        "must be a (non-strict) subclass "
-                        "of the metaclasses of all its bases")
-    return winner
-
-class DynamicClassAttribute:
-    """Route attribute access on a class to __getattr__.
-
-    This is a descriptor, used to define attributes that act differently when
-    accessed through an instance and through a class.  Instance access remains
-    normal, but access to an attribute through a class will be routed to the
-    class's __getattr__ method; this is done by raising AttributeError.
-
-    This allows one to have properties active on an instance, and have virtual
-    attributes on the class with the same name (see Enum for an example).
-
-    """
-    def __init__(self, fget=None, fset=None, fdel=None, doc=None):
-        self.fget = fget
-        self.fset = fset
-        self.fdel = fdel
-        # next two lines make DynamicClassAttribute act the same as property
-        self.__doc__ = doc or fget.__doc__
-        self.overwrite_doc = doc is None
-        # support for abstract methods
-        self.__isabstractmethod__ = bool(getattr(fget, '__isabstractmethod__', False))
-
-    def __get__(self, instance, ownerclass=None):
-        if instance is None:
-            if self.__isabstractmethod__:
-                return self
-            raise AttributeError()
-        elif self.fget is None:
-            raise AttributeError("unreadable attribute")
-        return self.fget(instance)
-
-    def __set__(self, instance, value):
-        if self.fset is None:
-            raise AttributeError("can't set attribute")
-        self.fset(instance, value)
-
-    def __delete__(self, instance):
-        if self.fdel is None:
-            raise AttributeError("can't delete attribute")
-        self.fdel(instance)
-
-    def getter(self, fget):
-        fdoc = fget.__doc__ if self.overwrite_doc else None
-        result = type(self)(fget, self.fset, self.fdel, fdoc or self.__doc__)
-        result.overwrite_doc = self.overwrite_doc
-        return result
-
-    def setter(self, fset):
-        result = type(self)(self.fget, fset, self.fdel, self.__doc__)
-        result.overwrite_doc = self.overwrite_doc
-        return result
-
-    def deleter(self, fdel):
-        result = type(self)(self.fget, self.fset, fdel, self.__doc__)
-        result.overwrite_doc = self.overwrite_doc
-        return result
-
-
-class _GeneratorWrapper:
-    # TODO: Implement this in C.
-    def __init__(self, gen):
-        self.__wrapped = gen
-        self.__isgen = gen.__class__ is GeneratorType
-        self.__name__ = getattr(gen, '__name__', None)
-        self.__qualname__ = getattr(gen, '__qualname__', None)
-    def send(self, val):
-        return self.__wrapped.send(val)
-    def throw(self, tp, *rest):
-        return self.__wrapped.throw(tp, *rest)
-    def close(self):
-        return self.__wrapped.close()
-    @property
-    def gi_code(self):
-        return self.__wrapped.gi_code
-    @property
-    def gi_frame(self):
-        return self.__wrapped.gi_frame
-    @property
-    def gi_running(self):
-        return self.__wrapped.gi_running
-    @property
-    def gi_yieldfrom(self):
-        return self.__wrapped.gi_yieldfrom
-    cr_code = gi_code
-    cr_frame = gi_frame
-    cr_running = gi_running
-    cr_await = gi_yieldfrom
-    def __next__(self):
-        return next(self.__wrapped)
-    def __iter__(self):
-        if self.__isgen:
-            return self.__wrapped
-        return self
-    __await__ = __iter__
-
-def coroutine(func):
-    """Convert regular generator function to a coroutine."""
-
-    if not callable(func):
-        raise TypeError('types.coroutine() expects a callable')
-
-    if (func.__class__ is FunctionType and
-        getattr(func, '__code__', None).__class__ is CodeType):
-
-        co_flags = func.__code__.co_flags
-
-        # Check if 'func' is a coroutine function.
-        # (0x180 == CO_COROUTINE | CO_ITERABLE_COROUTINE)
-        if co_flags & 0x180:
-            return func
-
-        # Check if 'func' is a generator function.
-        # (0x20 == CO_GENERATOR)
-        if co_flags & 0x20:
-            # TODO: Implement this in C.
-            co = func.__code__
-            func.__code__ = CodeType(
-                co.co_argcount, co.co_kwonlyargcount, co.co_nlocals,
-                co.co_stacksize,
-                co.co_flags | 0x100,  # 0x100 == CO_ITERABLE_COROUTINE
-                co.co_code,
-                co.co_consts, co.co_names, co.co_varnames, co.co_filename,
-                co.co_name, co.co_firstlineno, co.co_lnotab, co.co_freevars,
-                co.co_cellvars)
-            return func
-
-    # The following code is primarily to support functions that
-    # return generator-like objects (for instance generators
-    # compiled with Cython).
-
-    # Delay functools and _collections_abc import for speeding up types import.
-    import functools
-    import _collections_abc
-    @functools.wraps(func)
-    def wrapped(*args, **kwargs):
-        coro = func(*args, **kwargs)
-        if (coro.__class__ is CoroutineType or
-            coro.__class__ is GeneratorType and coro.gi_code.co_flags & 0x100):
-            # 'coro' is a native coroutine object or an iterable coroutine
-            return coro
-        if (isinstance(coro, _collections_abc.Generator) and
-            not isinstance(coro, _collections_abc.Coroutine)):
-            # 'coro' is either a pure Python generator iterator, or it
-            # implements collections.abc.Generator (and does not implement
-            # collections.abc.Coroutine).
-            return _GeneratorWrapper(coro)
-        # 'coro' is either an instance of collections.abc.Coroutine or
-        # some other object -- pass it through.
-        return coro
-
-    return wrapped
-
-
-__all__ = [n for n in globals() if n[:1] != '_']

Lib/warnings.py

@@ -1,554 +0,0 @@
-"""Python part of the warnings subsystem."""
-
-import sys
-
-
-__all__ = ["warn", "warn_explicit", "showwarning",
-           "formatwarning", "filterwarnings", "simplefilter",
-           "resetwarnings", "catch_warnings"]
-
-def showwarning(message, category, filename, lineno, file=None, line=None):
-    """Hook to write a warning to a file; replace if you like."""
-    msg = WarningMessage(message, category, filename, lineno, file, line)
-    _showwarnmsg_impl(msg)
-
-def formatwarning(message, category, filename, lineno, line=None):
-    """Function to format a warning the standard way."""
-    msg = WarningMessage(message, category, filename, lineno, None, line)
-    return _formatwarnmsg_impl(msg)
-
-def _showwarnmsg_impl(msg):
-    file = msg.file
-    if file is None:
-        file = sys.stderr
-        if file is None:
-            # sys.stderr is None when run with pythonw.exe:
-            # warnings get lost
-            return
-    text = _formatwarnmsg(msg)
-    try:
-        file.write(text)
-    except OSError:
-        # the file (probably stderr) is invalid - this warning gets lost.
-        pass
-
-def _formatwarnmsg_impl(msg):
-    s =  ("%s:%s: %s: %s\n"
-          % (msg.filename, msg.lineno, msg.category.__name__,
-             msg.message))
-
-    if msg.line is None:
-        try:
-            import linecache
-            line = linecache.getline(msg.filename, msg.lineno)
-        except Exception:
-            # When a warning is logged during Python shutdown, linecache
-            # and the import machinery don't work anymore
-            line = None
-            linecache = None
-    else:
-        line = msg.line
-    if line:
-        line = line.strip()
-        s += "  %s\n" % line
-
-    if msg.source is not None:
-        try:
-            import tracemalloc
-            tb = tracemalloc.get_object_traceback(msg.source)
-        except Exception:
-            # When a warning is logged during Python shutdown, tracemalloc
-            # and the import machinery don't work anymore
-            tb = None
-
-        if tb is not None:
-            s += 'Object allocated at (most recent call last):\n'
-            for frame in tb:
-                s += ('  File "%s", lineno %s\n'
-                      % (frame.filename, frame.lineno))
-
-                try:
-                    if linecache is not None:
-                        line = linecache.getline(frame.filename, frame.lineno)
-                    else:
-                        line = None
-                except Exception:
-                    line = None
-                if line:
-                    line = line.strip()
-                    s += '    %s\n' % line
-    return s
-
-# Keep a reference to check if the function was replaced
-_showwarning_orig = showwarning
-
-def _showwarnmsg(msg):
-    """Hook to write a warning to a file; replace if you like."""
-    try:
-        sw = showwarning
-    except NameError:
-        pass
-    else:
-        if sw is not _showwarning_orig:
-            # warnings.showwarning() was replaced
-            if not callable(sw):
-                raise TypeError("warnings.showwarning() must be set to a "
-                                "function or method")
-
-            sw(msg.message, msg.category, msg.filename, msg.lineno,
-               msg.file, msg.line)
-            return
-    _showwarnmsg_impl(msg)
-
-# Keep a reference to check if the function was replaced
-_formatwarning_orig = formatwarning
-
-def _formatwarnmsg(msg):
-    """Function to format a warning the standard way."""
-    try:
-        fw = formatwarning
-    except NameError:
-        pass
-    else:
-        if fw is not _formatwarning_orig:
-            # warnings.formatwarning() was replaced
-            return fw(msg.message, msg.category,
-                      msg.filename, msg.lineno, line=msg.line)
-    return _formatwarnmsg_impl(msg)
-
-def filterwarnings(action, message="", category=Warning, module="", lineno=0,
-                   append=False):
-    """Insert an entry into the list of warnings filters (at the front).
-
-    'action' -- one of "error", "ignore", "always", "default", "module",
-                or "once"
-    'message' -- a regex that the warning message must match
-    'category' -- a class that the warning must be a subclass of
-    'module' -- a regex that the module name must match
-    'lineno' -- an integer line number, 0 matches all warnings
-    'append' -- if true, append to the list of filters
-    """
-    assert action in ("error", "ignore", "always", "default", "module",
-                      "once"), "invalid action: %r" % (action,)
-    assert isinstance(message, str), "message must be a string"
-    assert isinstance(category, type), "category must be a class"
-    assert issubclass(category, Warning), "category must be a Warning subclass"
-    assert isinstance(module, str), "module must be a string"
-    assert isinstance(lineno, int) and lineno >= 0, \
-           "lineno must be an int >= 0"
-
-    if message or module:
-        import re
-
-    if message:
-        message = re.compile(message, re.I)
-    else:
-        message = None
-    if module:
-        module = re.compile(module)
-    else:
-        module = None
-
-    _add_filter(action, message, category, module, lineno, append=append)
-
-def simplefilter(action, category=Warning, lineno=0, append=False):
-    """Insert a simple entry into the list of warnings filters (at the front).
-
-    A simple filter matches all modules and messages.
-    'action' -- one of "error", "ignore", "always", "default", "module",
-                or "once"
-    'category' -- a class that the warning must be a subclass of
-    'lineno' -- an integer line number, 0 matches all warnings
-    'append' -- if true, append to the list of filters
-    """
-    assert action in ("error", "ignore", "always", "default", "module",
-                      "once"), "invalid action: %r" % (action,)
-    assert isinstance(lineno, int) and lineno >= 0, \
-           "lineno must be an int >= 0"
-    _add_filter(action, None, category, None, lineno, append=append)
-
-def _add_filter(*item, append):
-    # Remove possible duplicate filters, so new one will be placed
-    # in correct place. If append=True and duplicate exists, do nothing.
-    if not append:
-        try:
-            filters.remove(item)
-        except ValueError:
-            pass
-        filters.insert(0, item)
-    else:
-        if item not in filters:
-            filters.append(item)
-    _filters_mutated()
-
-def resetwarnings():
-    """Clear the list of warning filters, so that no filters are active."""
-    filters[:] = []
-    _filters_mutated()
-
-class _OptionError(Exception):
-    """Exception used by option processing helpers."""
-    pass
-
-# Helper to process -W options passed via sys.warnoptions
-def _processoptions(args):
-    for arg in args:
-        try:
-            _setoption(arg)
-        except _OptionError as msg:
-            print("Invalid -W option ignored:", msg, file=sys.stderr)
-
-# Helper for _processoptions()
-def _setoption(arg):
-    import re
-    parts = arg.split(':')
-    if len(parts) > 5:
-        raise _OptionError("too many fields (max 5): %r" % (arg,))
-    while len(parts) < 5:
-        parts.append('')
-    action, message, category, module, lineno = [s.strip()
-                                                 for s in parts]
-    action = _getaction(action)
-    message = re.escape(message)
-    category = _getcategory(category)
-    module = re.escape(module)
-    if module:
-        module = module + '$'
-    if lineno:
-        try:
-            lineno = int(lineno)
-            if lineno < 0:
-                raise ValueError
-        except (ValueError, OverflowError):
-            raise _OptionError("invalid lineno %r" % (lineno,)) from None
-    else:
-        lineno = 0
-    filterwarnings(action, message, category, module, lineno)
-
-# Helper for _setoption()
-def _getaction(action):
-    if not action:
-        return "default"
-    if action == "all": return "always" # Alias
-    for a in ('default', 'always', 'ignore', 'module', 'once', 'error'):
-        if a.startswith(action):
-            return a
-    raise _OptionError("invalid action: %r" % (action,))
-
-# Helper for _setoption()
-def _getcategory(category):
-    import re
-    if not category:
-        return Warning
-    if re.match("^[a-zA-Z0-9_]+$", category):
-        try:
-            cat = eval(category)
-        except NameError:
-            raise _OptionError("unknown warning category: %r" % (category,)) from None
-    else:
-        i = category.rfind(".")
-        module = category[:i]
-        klass = category[i+1:]
-        try:
-            m = __import__(module, None, None, [klass])
-        except ImportError:
-            raise _OptionError("invalid module name: %r" % (module,)) from None
-        try:
-            cat = getattr(m, klass)
-        except AttributeError:
-            raise _OptionError("unknown warning category: %r" % (category,)) from None
-    if not issubclass(cat, Warning):
-        raise _OptionError("invalid warning category: %r" % (category,))
-    return cat
-
-
-def _is_internal_frame(frame):
-    """Signal whether the frame is an internal CPython implementation detail."""
-    filename = frame.f_code.co_filename
-    return 'importlib' in filename and '_bootstrap' in filename
-
-
-def _next_external_frame(frame):
-    """Find the next frame that doesn't involve CPython internals."""
-    frame = frame.f_back
-    while frame is not None and _is_internal_frame(frame):
-        frame = frame.f_back
-    return frame
-
-
-# Code typically replaced by _warnings
-def warn(message, category=None, stacklevel=1, source=None):
-    """Issue a warning, or maybe ignore it or raise an exception."""
-    # Check if message is already a Warning object
-    if isinstance(message, Warning):
-        category = message.__class__
-    # Check category argument
-    if category is None:
-        category = UserWarning
-    if not (isinstance(category, type) and issubclass(category, Warning)):
-        raise TypeError("category must be a Warning subclass, "
-                        "not '{:s}'".format(type(category).__name__))
-    # Get context information
-    try:
-        if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)):
-            # If frame is too small to care or if the warning originated in
-            # internal code, then do not try to hide any frames.
-            frame = sys._getframe(stacklevel)
-        else:
-            frame = sys._getframe(1)
-            # Look for one frame less since the above line starts us off.
-            for x in range(stacklevel-1):
-                frame = _next_external_frame(frame)
-                if frame is None:
-                    raise ValueError
-    except ValueError:
-        globals = sys.__dict__
-        lineno = 1
-    else:
-        globals = frame.f_globals
-        lineno = frame.f_lineno
-    if '__name__' in globals:
-        module = globals['__name__']
-    else:
-        module = "<string>"
-    filename = globals.get('__file__')
-    if filename:
-        fnl = filename.lower()
-        if fnl.endswith(".pyc"):
-            filename = filename[:-1]
-    else:
-        if module == "__main__":
-            try:
-                filename = sys.argv[0]
-            except AttributeError:
-                # embedded interpreters don't have sys.argv, see bug #839151
-                filename = '__main__'
-        if not filename:
-            filename = module
-    registry = globals.setdefault("__warningregistry__", {})
-    warn_explicit(message, category, filename, lineno, module, registry,
-                  globals, source)
-
-def warn_explicit(message, category, filename, lineno,
-                  module=None, registry=None, module_globals=None,
-                  source=None):
-    lineno = int(lineno)
-    if module is None:
-        module = filename or "<unknown>"
-        if module[-3:].lower() == ".py":
-            module = module[:-3] # XXX What about leading pathname?
-    if registry is None:
-        registry = {}
-    if registry.get('version', 0) != _filters_version:
-        registry.clear()
-        registry['version'] = _filters_version
-    if isinstance(message, Warning):
-        text = str(message)
-        category = message.__class__
-    else:
-        text = message
-        message = category(message)
-    key = (text, category, lineno)
-    # Quick test for common case
-    if registry.get(key):
-        return
-    # Search the filters
-    for item in filters:
-        action, msg, cat, mod, ln = item
-        if ((msg is None or msg.match(text)) and
-            issubclass(category, cat) and
-            (mod is None or mod.match(module)) and
-            (ln == 0 or lineno == ln)):
-            break
-    else:
-        action = defaultaction
-    # Early exit actions
-    if action == "ignore":
-        return
-
-    # Prime the linecache for formatting, in case the
-    # "file" is actually in a zipfile or something.
-    import linecache
-    linecache.getlines(filename, module_globals)
-
-    if action == "error":
-        raise message
-    # Other actions
-    if action == "once":
-        registry[key] = 1
-        oncekey = (text, category)
-        if onceregistry.get(oncekey):
-            return
-        onceregistry[oncekey] = 1
-    elif action == "always":
-        pass
-    elif action == "module":
-        registry[key] = 1
-        altkey = (text, category, 0)
-        if registry.get(altkey):
-            return
-        registry[altkey] = 1
-    elif action == "default":
-        registry[key] = 1
-    else:
-        # Unrecognized actions are errors
-        raise RuntimeError(
-              "Unrecognized action (%r) in warnings.filters:\n %s" %
-              (action, item))
-    # Print message and context
-    msg = WarningMessage(message, category, filename, lineno, source)
-    _showwarnmsg(msg)
-
-
-class WarningMessage(object):
-
-    _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
-                        "line", "source")
-
-    def __init__(self, message, category, filename, lineno, file=None,
-                 line=None, source=None):
-        self.message = message
-        self.category = category
-        self.filename = filename
-        self.lineno = lineno
-        self.file = file
-        self.line = line
-        self.source = source
-        self._category_name = category.__name__ if category else None
-
-    def __str__(self):
-        return ("{message : %r, category : %r, filename : %r, lineno : %s, "
-                    "line : %r}" % (self.message, self._category_name,
-                                    self.filename, self.lineno, self.line))
-
-
-class catch_warnings(object):
-
-    """A context manager that copies and restores the warnings filter upon
-    exiting the context.
-
-    The 'record' argument specifies whether warnings should be captured by a
-    custom implementation of warnings.showwarning() and be appended to a list
-    returned by the context manager. Otherwise None is returned by the context
-    manager. The objects appended to the list are arguments whose attributes
-    mirror the arguments to showwarning().
-
-    The 'module' argument is to specify an alternative module to the module
-    named 'warnings' and imported under that name. This argument is only useful
-    when testing the warnings module itself.
-
-    """
-
-    def __init__(self, *, record=False, module=None):
-        """Specify whether to record warnings and if an alternative module
-        should be used other than sys.modules['warnings'].
-
-        For compatibility with Python 3.0, please consider all arguments to be
-        keyword-only.
-
-        """
-        self._record = record
-        self._module = sys.modules['warnings'] if module is None else module
-        self._entered = False
-
-    def __repr__(self):
-        args = []
-        if self._record:
-            args.append("record=True")
-        if self._module is not sys.modules['warnings']:
-            args.append("module=%r" % self._module)
-        name = type(self).__name__
-        return "%s(%s)" % (name, ", ".join(args))
-
-    def __enter__(self):
-        if self._entered:
-            raise RuntimeError("Cannot enter %r twice" % self)
-        self._entered = True
-        self._filters = self._module.filters
-        self._module.filters = self._filters[:]
-        self._module._filters_mutated()
-        self._showwarning = self._module.showwarning
-        self._showwarnmsg_impl = self._module._showwarnmsg_impl
-        if self._record:
-            log = []
-            self._module._showwarnmsg_impl = log.append
-            # Reset showwarning() to the default implementation to make sure
-            # that _showwarnmsg() calls _showwarnmsg_impl()
-            self._module.showwarning = self._module._showwarning_orig
-            return log
-        else:
-            return None
-
-    def __exit__(self, *exc_info):
-        if not self._entered:
-            raise RuntimeError("Cannot exit %r without entering first" % self)
-        self._module.filters = self._filters
-        self._module._filters_mutated()
-        self._module.showwarning = self._showwarning
-        self._module._showwarnmsg_impl = self._showwarnmsg_impl
-
-
-# Private utility function called by _PyErr_WarnUnawaitedCoroutine
-def _warn_unawaited_coroutine(coro):
-    msg_lines = [
-        f"coroutine '{coro.__qualname__}' was never awaited\n"
-    ]
-    if coro.cr_origin is not None:
-        import linecache, traceback
-        def extract():
-            for filename, lineno, funcname in reversed(coro.cr_origin):
-                line = linecache.getline(filename, lineno)
-                yield (filename, lineno, funcname, line)
-        msg_lines.append("Coroutine created at (most recent call last)\n")
-        msg_lines += traceback.format_list(list(extract()))
-    msg = "".join(msg_lines).rstrip("\n")
-    # Passing source= here means that if the user happens to have tracemalloc
-    # enabled and tracking where the coroutine was created, the warning will
-    # contain that traceback. This does mean that if they have *both*
-    # coroutine origin tracking *and* tracemalloc enabled, they'll get two
-    # partially-redundant tracebacks. If we wanted to be clever we could
-    # probably detect this case and avoid it, but for now we don't bother.
-    warn(msg, category=RuntimeWarning, stacklevel=2, source=coro)
-
-
-# filters contains a sequence of filter 5-tuples
-# The components of the 5-tuple are:
-# - an action: error, ignore, always, default, module, or once
-# - a compiled regex that must match the warning message
-# - a class representing the warning category
-# - a compiled regex that must match the module that is being warned
-# - a line number for the line being warning, or 0 to mean any line
-# If either if the compiled regexs are None, match anything.
-try:
-    from _warnings import (filters, _defaultaction, _onceregistry,
-                           warn, warn_explicit, _filters_mutated)
-    defaultaction = _defaultaction
-    onceregistry = _onceregistry
-    _warnings_defaults = True
-except ImportError:
-    filters = []
-    defaultaction = "default"
-    onceregistry = {}
-
-    _filters_version = 1
-
-    def _filters_mutated():
-        global _filters_version
-        _filters_version += 1
-
-    _warnings_defaults = False
-
-
-# Module initialization
-_processoptions(sys.warnoptions)
-if not _warnings_defaults:
-    # Several warning categories are ignored by default in regular builds
-    if not hasattr(sys, 'gettotalrefcount'):
-        filterwarnings("default", category=DeprecationWarning,
-                       module="__main__", append=1)
-        simplefilter("ignore", category=DeprecationWarning, append=1)
-        simplefilter("ignore", category=PendingDeprecationWarning, append=1)
-        simplefilter("ignore", category=ImportWarning, append=1)
-        simplefilter("ignore", category=ResourceWarning, append=1)
-
-del _warnings_defaults

Lib/weakref.py

@@ -1,632 +0,0 @@
-"""Weak reference support for Python.
-
-This module is an implementation of PEP 205:
-
-http://www.python.org/dev/peps/pep-0205/
-"""
-
-# Naming convention: Variables named "wr" are weak reference objects;
-# they are called this instead of "ref" to avoid name collisions with
-# the module-global ref() function imported from _weakref.
-
-from _weakref import (
-     getweakrefcount,
-     getweakrefs,
-     ref,
-     proxy,
-     CallableProxyType,
-     ProxyType,
-     ReferenceType,
-     _remove_dead_weakref)
-
-from _weakrefset import WeakSet, _IterationGuard
-
-import _collections_abc  # Import after _weakref to avoid circular import.
-import sys
-import itertools
-
-ProxyTypes = (ProxyType, CallableProxyType)
-
-__all__ = ["ref", "proxy", "getweakrefcount", "getweakrefs",
-           "WeakKeyDictionary", "ReferenceType", "ProxyType",
-           "CallableProxyType", "ProxyTypes", "WeakValueDictionary",
-           "WeakSet", "WeakMethod", "finalize"]
-
-
-class WeakMethod(ref):
-    """
-    A custom `weakref.ref` subclass which simulates a weak reference to
-    a bound method, working around the lifetime problem of bound methods.
-    """
-
-    __slots__ = "_func_ref", "_meth_type", "_alive", "__weakref__"
-
-    def __new__(cls, meth, callback=None):
-        try:
-            obj = meth.__self__
-            func = meth.__func__
-        except AttributeError:
-            raise TypeError("argument should be a bound method, not {}"
-                            .format(type(meth))) from None
-        def _cb(arg):
-            # The self-weakref trick is needed to avoid creating a reference
-            # cycle.
-            self = self_wr()
-            if self._alive:
-                self._alive = False
-                if callback is not None:
-                    callback(self)
-        self = ref.__new__(cls, obj, _cb)
-        self._func_ref = ref(func, _cb)
-        self._meth_type = type(meth)
-        self._alive = True
-        self_wr = ref(self)
-        return self
-
-    def __call__(self):
-        obj = super().__call__()
-        func = self._func_ref()
-        if obj is None or func is None:
-            return None
-        return self._meth_type(func, obj)
-
-    def __eq__(self, other):
-        if isinstance(other, WeakMethod):
-            if not self._alive or not other._alive:
-                return self is other
-            return ref.__eq__(self, other) and self._func_ref == other._func_ref
-        return False
-
-    def __ne__(self, other):
-        if isinstance(other, WeakMethod):
-            if not self._alive or not other._alive:
-                return self is not other
-            return ref.__ne__(self, other) or self._func_ref != other._func_ref
-        return True
-
-    __hash__ = ref.__hash__
-
-
-class WeakValueDictionary(_collections_abc.MutableMapping):
-    """Mapping class that references values weakly.
-
-    Entries in the dictionary will be discarded when no strong
-    reference to the value exists anymore
-    """
-    # We inherit the constructor without worrying about the input
-    # dictionary; since it uses our .update() method, we get the right
-    # checks (if the other dictionary is a WeakValueDictionary,
-    # objects are unwrapped on the way out, and we always wrap on the
-    # way in).
-
-    def __init__(*args, **kw):
-        if not args:
-            raise TypeError("descriptor '__init__' of 'WeakValueDictionary' "
-                            "object needs an argument")
-        self, *args = args
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
-            self = selfref()
-            if self is not None:
-                if self._iterating:
-                    self._pending_removals.append(wr.key)
-                else:
-                    # Atomic removal is necessary since this function
-                    # can be called asynchronously by the GC
-                    _atomic_removal(d, wr.key)
-        self._remove = remove
-        # A list of keys to be removed
-        self._pending_removals = []
-        self._iterating = set()
-        self.data = d = {}
-        self.update(*args, **kw)
-
-    def _commit_removals(self):
-        l = self._pending_removals
-        d = self.data
-        # We shouldn't encounter any KeyError, because this method should
-        # always be called *before* mutating the dict.
-        while l:
-            key = l.pop()
-            _remove_dead_weakref(d, key)
-
-    def __getitem__(self, key):
-        if self._pending_removals:
-            self._commit_removals()
-        o = self.data[key]()
-        if o is None:
-            raise KeyError(key)
-        else:
-            return o
-
-    def __delitem__(self, key):
-        if self._pending_removals:
-            self._commit_removals()
-        del self.data[key]
-
-    def __len__(self):
-        if self._pending_removals:
-            self._commit_removals()
-        return len(self.data)
-
-    def __contains__(self, key):
-        if self._pending_removals:
-            self._commit_removals()
-        try:
-            o = self.data[key]()
-        except KeyError:
-            return False
-        return o is not None
-
-    def __repr__(self):
-        return "<%s at %#x>" % (self.__class__.__name__, id(self))
-
-    def __setitem__(self, key, value):
-        if self._pending_removals:
-            self._commit_removals()
-        self.data[key] = KeyedRef(value, self._remove, key)
-
-    def copy(self):
-        if self._pending_removals:
-            self._commit_removals()
-        new = WeakValueDictionary()
-        for key, wr in self.data.items():
-            o = wr()
-            if o is not None:
-                new[key] = o
-        return new
-
-    __copy__ = copy
-
-    def __deepcopy__(self, memo):
-        from copy import deepcopy
-        if self._pending_removals:
-            self._commit_removals()
-        new = self.__class__()
-        for key, wr in self.data.items():
-            o = wr()
-            if o is not None:
-                new[deepcopy(key, memo)] = o
-        return new
-
-    def get(self, key, default=None):
-        if self._pending_removals:
-            self._commit_removals()
-        try:
-            wr = self.data[key]
-        except KeyError:
-            return default
-        else:
-            o = wr()
-            if o is None:
-                # This should only happen
-                return default
-            else:
-                return o
-
-    def items(self):
-        if self._pending_removals:
-            self._commit_removals()
-        with _IterationGuard(self):
-            for k, wr in self.data.items():
-                v = wr()
-                if v is not None:
-                    yield k, v
-
-    def keys(self):
-        if self._pending_removals:
-            self._commit_removals()
-        with _IterationGuard(self):
-            for k, wr in self.data.items():
-                if wr() is not None:
-                    yield k
-
-    __iter__ = keys
-
-    def itervaluerefs(self):
-        """Return an iterator that yields the weak references to the values.
-
-        The references are not guaranteed to be 'live' at the time
-        they are used, so the result of calling the references needs
-        to be checked before being used.  This can be used to avoid
-        creating references that will cause the garbage collector to
-        keep the values around longer than needed.
-
-        """
-        if self._pending_removals:
-            self._commit_removals()
-        with _IterationGuard(self):
-            yield from self.data.values()
-
-    def values(self):
-        if self._pending_removals:
-            self._commit_removals()
-        with _IterationGuard(self):
-            for wr in self.data.values():
-                obj = wr()
-                if obj is not None:
-                    yield obj
-
-    def popitem(self):
-        if self._pending_removals:
-            self._commit_removals()
-        while True:
-            key, wr = self.data.popitem()
-            o = wr()
-            if o is not None:
-                return key, o
-
-    def pop(self, key, *args):
-        if self._pending_removals:
-            self._commit_removals()
-        try:
-            o = self.data.pop(key)()
-        except KeyError:
-            o = None
-        if o is None:
-            if args:
-                return args[0]
-            else:
-                raise KeyError(key)
-        else:
-            return o
-
-    def setdefault(self, key, default=None):
-        try:
-            o = self.data[key]()
-        except KeyError:
-            o = None
-        if o is None:
-            if self._pending_removals:
-                self._commit_removals()
-            self.data[key] = KeyedRef(default, self._remove, key)
-            return default
-        else:
-            return o
-
-    def update(*args, **kwargs):
-        if not args:
-            raise TypeError("descriptor 'update' of 'WeakValueDictionary' "
-                            "object needs an argument")
-        self, *args = args
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        dict = args[0] if args else None
-        if self._pending_removals:
-            self._commit_removals()
-        d = self.data
-        if dict is not None:
-            if not hasattr(dict, "items"):
-                dict = type({})(dict)
-            for key, o in dict.items():
-                d[key] = KeyedRef(o, self._remove, key)
-        if len(kwargs):
-            self.update(kwargs)
-
-    def valuerefs(self):
-        """Return a list of weak references to the values.
-
-        The references are not guaranteed to be 'live' at the time
-        they are used, so the result of calling the references needs
-        to be checked before being used.  This can be used to avoid
-        creating references that will cause the garbage collector to
-        keep the values around longer than needed.
-
-        """
-        if self._pending_removals:
-            self._commit_removals()
-        return list(self.data.values())
-
-
-class KeyedRef(ref):
-    """Specialized reference that includes a key corresponding to the value.
-
-    This is used in the WeakValueDictionary to avoid having to create
-    a function object for each key stored in the mapping.  A shared
-    callback object can use the 'key' attribute of a KeyedRef instead
-    of getting a reference to the key from an enclosing scope.
-
-    """
-
-    __slots__ = "key",
-
-    def __new__(type, ob, callback, key):
-        self = ref.__new__(type, ob, callback)
-        self.key = key
-        return self
-
-    def __init__(self, ob, callback, key):
-        super().__init__(ob, callback)
-
-
-class WeakKeyDictionary(_collections_abc.MutableMapping):
-    """ Mapping class that references keys weakly.
-
-    Entries in the dictionary will be discarded when there is no
-    longer a strong reference to the key. This can be used to
-    associate additional data with an object owned by other parts of
-    an application without adding attributes to those objects. This
-    can be especially useful with objects that override attribute
-    accesses.
-    """
-
-    def __init__(self, dict=None):
-        self.data = {}
-        def remove(k, selfref=ref(self)):
-            self = selfref()
-            if self is not None:
-                if self._iterating:
-                    self._pending_removals.append(k)
-                else:
-                    del self.data[k]
-        self._remove = remove
-        # A list of dead weakrefs (keys to be removed)
-        self._pending_removals = []
-        self._iterating = set()
-        self._dirty_len = False
-        if dict is not None:
-            self.update(dict)
-
-    def _commit_removals(self):
-        # NOTE: We don't need to call this method before mutating the dict,
-        # because a dead weakref never compares equal to a live weakref,
-        # even if they happened to refer to equal objects.
-        # However, it means keys may already have been removed.
-        l = self._pending_removals
-        d = self.data
-        while l:
-            try:
-                del d[l.pop()]
-            except KeyError:
-                pass
-
-    def _scrub_removals(self):
-        d = self.data
-        self._pending_removals = [k for k in self._pending_removals if k in d]
-        self._dirty_len = False
-
-    def __delitem__(self, key):
-        self._dirty_len = True
-        del self.data[ref(key)]
-
-    def __getitem__(self, key):
-        return self.data[ref(key)]
-
-    def __len__(self):
-        if self._dirty_len and self._pending_removals:
-            # self._pending_removals may still contain keys which were
-            # explicitly removed, we have to scrub them (see issue #21173).
-            self._scrub_removals()
-        return len(self.data) - len(self._pending_removals)
-
-    def __repr__(self):
-        return "<%s at %#x>" % (self.__class__.__name__, id(self))
-
-    def __setitem__(self, key, value):
-        self.data[ref(key, self._remove)] = value
-
-    def copy(self):
-        new = WeakKeyDictionary()
-        for key, value in self.data.items():
-            o = key()
-            if o is not None:
-                new[o] = value
-        return new
-
-    __copy__ = copy
-
-    def __deepcopy__(self, memo):
-        from copy import deepcopy
-        new = self.__class__()
-        for key, value in self.data.items():
-            o = key()
-            if o is not None:
-                new[o] = deepcopy(value, memo)
-        return new
-
-    def get(self, key, default=None):
-        return self.data.get(ref(key),default)
-
-    def __contains__(self, key):
-        try:
-            wr = ref(key)
-        except TypeError:
-            return False
-        return wr in self.data
-
-    def items(self):
-        with _IterationGuard(self):
-            for wr, value in self.data.items():
-                key = wr()
-                if key is not None:
-                    yield key, value
-
-    def keys(self):
-        with _IterationGuard(self):
-            for wr in self.data:
-                obj = wr()
-                if obj is not None:
-                    yield obj
-
-    __iter__ = keys
-
-    def values(self):
-        with _IterationGuard(self):
-            for wr, value in self.data.items():
-                if wr() is not None:
-                    yield value
-
-    def keyrefs(self):
-        """Return a list of weak references to the keys.
-
-        The references are not guaranteed to be 'live' at the time
-        they are used, so the result of calling the references needs
-        to be checked before being used.  This can be used to avoid
-        creating references that will cause the garbage collector to
-        keep the keys around longer than needed.
-
-        """
-        return list(self.data)
-
-    def popitem(self):
-        self._dirty_len = True
-        while True:
-            key, value = self.data.popitem()
-            o = key()
-            if o is not None:
-                return o, value
-
-    def pop(self, key, *args):
-        self._dirty_len = True
-        return self.data.pop(ref(key), *args)
-
-    def setdefault(self, key, default=None):
-        return self.data.setdefault(ref(key, self._remove),default)
-
-    def update(self, dict=None, **kwargs):
-        d = self.data
-        if dict is not None:
-            if not hasattr(dict, "items"):
-                dict = type({})(dict)
-            for key, value in dict.items():
-                d[ref(key, self._remove)] = value
-        if len(kwargs):
-            self.update(kwargs)
-
-
-class finalize:
-    """Class for finalization of weakrefable objects
-
-    finalize(obj, func, *args, **kwargs) returns a callable finalizer
-    object which will be called when obj is garbage collected. The
-    first time the finalizer is called it evaluates func(*arg, **kwargs)
-    and returns the result. After this the finalizer is dead, and
-    calling it just returns None.
-
-    When the program exits any remaining finalizers for which the
-    atexit attribute is true will be run in reverse order of creation.
-    By default atexit is true.
-    """
-
-    # Finalizer objects don't have any state of their own.  They are
-    # just used as keys to lookup _Info objects in the registry.  This
-    # ensures that they cannot be part of a ref-cycle.
-
-    __slots__ = ()
-    _registry = {}
-    _shutdown = False
-    _index_iter = itertools.count()
-    _dirty = False
-    _registered_with_atexit = False
-
-    class _Info:
-        __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index")
-
-    def __init__(self, obj, func, *args, **kwargs):
-        if not self._registered_with_atexit:
-            # We may register the exit function more than once because
-            # of a thread race, but that is harmless
-            import atexit
-            atexit.register(self._exitfunc)
-            finalize._registered_with_atexit = True
-        info = self._Info()
-        info.weakref = ref(obj, self)
-        info.func = func
-        info.args = args
-        info.kwargs = kwargs or None
-        info.atexit = True
-        info.index = next(self._index_iter)
-        self._registry[self] = info
-        finalize._dirty = True
-
-    def __call__(self, _=None):
-        """If alive then mark as dead and return func(*args, **kwargs);
-        otherwise return None"""
-        info = self._registry.pop(self, None)
-        if info and not self._shutdown:
-            return info.func(*info.args, **(info.kwargs or {}))
-
-    def detach(self):
-        """If alive then mark as dead and return (obj, func, args, kwargs);
-        otherwise return None"""
-        info = self._registry.get(self)
-        obj = info and info.weakref()
-        if obj is not None and self._registry.pop(self, None):
-            return (obj, info.func, info.args, info.kwargs or {})
-
-    def peek(self):
-        """If alive then return (obj, func, args, kwargs);
-        otherwise return None"""
-        info = self._registry.get(self)
-        obj = info and info.weakref()
-        if obj is not None:
-            return (obj, info.func, info.args, info.kwargs or {})
-
-    @property
-    def alive(self):
-        """Whether finalizer is alive"""
-        return self in self._registry
-
-    @property
-    def atexit(self):
-        """Whether finalizer should be called at exit"""
-        info = self._registry.get(self)
-        return bool(info) and info.atexit
-
-    @atexit.setter
-    def atexit(self, value):
-        info = self._registry.get(self)
-        if info:
-            info.atexit = bool(value)
-
-    def __repr__(self):
-        info = self._registry.get(self)
-        obj = info and info.weakref()
-        if obj is None:
-            return '<%s object at %#x; dead>' % (type(self).__name__, id(self))
-        else:
-            return '<%s object at %#x; for %r at %#x>' % \
-                (type(self).__name__, id(self), type(obj).__name__, id(obj))
-
-    @classmethod
-    def _select_for_exit(cls):
-        # Return live finalizers marked for exit, oldest first
-        L = [(f,i) for (f,i) in cls._registry.items() if i.atexit]
-        L.sort(key=lambda item:item[1].index)
-        return [f for (f,i) in L]
-
-    @classmethod
-    def _exitfunc(cls):
-        # At shutdown invoke finalizers for which atexit is true.
-        # This is called once all other non-daemonic threads have been
-        # joined.
-        reenable_gc = False
-        try:
-            if cls._registry:
-                import gc
-                if gc.isenabled():
-                    reenable_gc = True
-                    gc.disable()
-                pending = None
-                while True:
-                    if pending is None or finalize._dirty:
-                        pending = cls._select_for_exit()
-                        finalize._dirty = False
-                    if not pending:
-                        break
-                    f = pending.pop()
-                    try:
-                        # gc is disabled, so (assuming no daemonic
-                        # threads) the following is the only line in
-                        # this function which might trigger creation
-                        # of a new finalizer
-                        f()
-                    except Exception:
-                        sys.excepthook(*sys.exc_info())
-                    assert f not in cls._registry
-        finally:
-            # prevent any more finalizers from executing during shutdown
-            finalize._shutdown = True
-            if reenable_gc:
-                gc.enable()

Lib/xdrlib.py

@@ -1,241 +0,0 @@
-"""Implements (a subset of) Sun XDR -- eXternal Data Representation.
-
-See: RFC 1014
-
-"""
-
-import struct
-from io import BytesIO
-from functools import wraps
-
-__all__ = ["Error", "Packer", "Unpacker", "ConversionError"]
-
-# exceptions
-class Error(Exception):
-    """Exception class for this module. Use:
-
-    except xdrlib.Error as var:
-        # var has the Error instance for the exception
-
-    Public ivars:
-        msg -- contains the message
-
-    """
-    def __init__(self, msg):
-        self.msg = msg
-    def __repr__(self):
-        return repr(self.msg)
-    def __str__(self):
-        return str(self.msg)
-
-
-class ConversionError(Error):
-    pass
-
-def raise_conversion_error(function):
-    """ Wrap any raised struct.errors in a ConversionError. """
-
-    @wraps(function)
-    def result(self, value):
-        try:
-            return function(self, value)
-        except struct.error as e:
-            raise ConversionError(e.args[0]) from None
-    return result
-
-
-class Packer:
-    """Pack various data representations into a buffer."""
-
-    def __init__(self):
-        self.reset()
-
-    def reset(self):
-        self.__buf = BytesIO()
-
-    def get_buffer(self):
-        return self.__buf.getvalue()
-    # backwards compatibility
-    get_buf = get_buffer
-
-    @raise_conversion_error
-    def pack_uint(self, x):
-        self.__buf.write(struct.pack('>L', x))
-
-    @raise_conversion_error
-    def pack_int(self, x):
-        self.__buf.write(struct.pack('>l', x))
-
-    pack_enum = pack_int
-
-    def pack_bool(self, x):
-        if x: self.__buf.write(b'\0\0\0\1')
-        else: self.__buf.write(b'\0\0\0\0')
-
-    def pack_uhyper(self, x):
-        try:
-            self.pack_uint(x>>32 & 0xffffffff)
-        except (TypeError, struct.error) as e:
-            raise ConversionError(e.args[0]) from None
-        try:
-            self.pack_uint(x & 0xffffffff)
-        except (TypeError, struct.error) as e:
-            raise ConversionError(e.args[0]) from None
-
-    pack_hyper = pack_uhyper
-
-    @raise_conversion_error
-    def pack_float(self, x):
-        self.__buf.write(struct.pack('>f', x))
-
-    @raise_conversion_error
-    def pack_double(self, x):
-        self.__buf.write(struct.pack('>d', x))
-
-    def pack_fstring(self, n, s):
-        if n < 0:
-            raise ValueError('fstring size must be nonnegative')
-        data = s[:n]
-        n = ((n+3)//4)*4
-        data = data + (n - len(data)) * b'\0'
-        self.__buf.write(data)
-
-    pack_fopaque = pack_fstring
-
-    def pack_string(self, s):
-        n = len(s)
-        self.pack_uint(n)
-        self.pack_fstring(n, s)
-
-    pack_opaque = pack_string
-    pack_bytes = pack_string
-
-    def pack_list(self, list, pack_item):
-        for item in list:
-            self.pack_uint(1)
-            pack_item(item)
-        self.pack_uint(0)
-
-    def pack_farray(self, n, list, pack_item):
-        if len(list) != n:
-            raise ValueError('wrong array size')
-        for item in list:
-            pack_item(item)
-
-    def pack_array(self, list, pack_item):
-        n = len(list)
-        self.pack_uint(n)
-        self.pack_farray(n, list, pack_item)
-
-
-
-class Unpacker:
-    """Unpacks various data representations from the given buffer."""
-
-    def __init__(self, data):
-        self.reset(data)
-
-    def reset(self, data):
-        self.__buf = data
-        self.__pos = 0
-
-    def get_position(self):
-        return self.__pos
-
-    def set_position(self, position):
-        self.__pos = position
-
-    def get_buffer(self):
-        return self.__buf
-
-    def done(self):
-        if self.__pos < len(self.__buf):
-            raise Error('unextracted data remains')
-
-    def unpack_uint(self):
-        i = self.__pos
-        self.__pos = j = i+4
-        data = self.__buf[i:j]
-        if len(data) < 4:
-            raise EOFError
-        return struct.unpack('>L', data)[0]
-
-    def unpack_int(self):
-        i = self.__pos
-        self.__pos = j = i+4
-        data = self.__buf[i:j]
-        if len(data) < 4:
-            raise EOFError
-        return struct.unpack('>l', data)[0]
-
-    unpack_enum = unpack_int
-
-    def unpack_bool(self):
-        return bool(self.unpack_int())
-
-    def unpack_uhyper(self):
-        hi = self.unpack_uint()
-        lo = self.unpack_uint()
-        return int(hi)<<32 | lo
-
-    def unpack_hyper(self):
-        x = self.unpack_uhyper()
-        if x >= 0x8000000000000000:
-            x = x - 0x10000000000000000
-        return x
-
-    def unpack_float(self):
-        i = self.__pos
-        self.__pos = j = i+4
-        data = self.__buf[i:j]
-        if len(data) < 4:
-            raise EOFError
-        return struct.unpack('>f', data)[0]
-
-    def unpack_double(self):
-        i = self.__pos
-        self.__pos = j = i+8
-        data = self.__buf[i:j]
-        if len(data) < 8:
-            raise EOFError
-        return struct.unpack('>d', data)[0]
-
-    def unpack_fstring(self, n):
-        if n < 0:
-            raise ValueError('fstring size must be nonnegative')
-        i = self.__pos
-        j = i + (n+3)//4*4
-        if j > len(self.__buf):
-            raise EOFError
-        self.__pos = j
-        return self.__buf[i:i+n]
-
-    unpack_fopaque = unpack_fstring
-
-    def unpack_string(self):
-        n = self.unpack_uint()
-        return self.unpack_fstring(n)
-
-    unpack_opaque = unpack_string
-    unpack_bytes = unpack_string
-
-    def unpack_list(self, unpack_item):
-        list = []
-        while 1:
-            x = self.unpack_uint()
-            if x == 0: break
-            if x != 1:
-                raise ConversionError('0 or 1 expected, got %r' % (x,))
-            item = unpack_item()
-            list.append(item)
-        return list
-
-    def unpack_farray(self, n, unpack_item):
-        list = []
-        for i in range(n):
-            list.append(unpack_item())
-        return list
-
-    def unpack_array(self, unpack_item):
-        n = self.unpack_uint()
-        return self.unpack_farray(n, unpack_item)

README.md

@@ -1,20 +1,13 @@
-<img src="./logo.png" width="125" height="125" align="right" />
-
 # RustPython
-
+A Python-3  (CPython >= 3.5.0) Interpreter written in Rust :snake: :scream: :metal:.
-A Python-3 (CPython >= 3.5.0) Interpreter written in Rust :snake: :scream: :metal:.
 
 [![Build Status](https://travis-ci.org/RustPython/RustPython.svg?branch=master)](https://travis-ci.org/RustPython/RustPython)
-[![Build Status](https://dev.azure.com/ryan0463/ryan/_apis/build/status/RustPython.RustPython?branchName=master)](https://dev.azure.com/ryan0463/ryan/_build/latest?definitionId=1&branchName=master)
-[![codecov](https://codecov.io/gh/RustPython/RustPython/branch/master/graph/badge.svg)](https://codecov.io/gh/RustPython/RustPython)
 [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
 [![Contributors](https://img.shields.io/github/contributors/RustPython/RustPython.svg)](https://github.com/RustPython/RustPython/graphs/contributors)
 [![Gitter](https://badges.gitter.im/RustPython/Lobby.svg)](https://gitter.im/rustpython/Lobby)
 
 # Usage
 
-### Check out our [online demo](https://rustpython.github.io/demo/) running on WebAssembly.
-
 To test RustPython, do the following:
 
     $ git clone https://github.com/RustPython/RustPython
@@ -29,13 +22,6 @@ Or use the interactive shell:
     >>>>> 2+2
     4
 
-# Disclaimer
-
-  RustPython is in a development phase and should not be used in production or a fault intolerant setting.
-
-  Our current build supports only a subset of Python syntax.
-
-  Contribution is also more than welcome! See our contribution section for more information on this. 
 
 # Goals
 
@@ -44,48 +30,35 @@ Or use the interactive shell:
 
 # Documentation
 
-Currently along with other areas of the project, documentation is still in an early phase.
+Currently the project is in an early phase, and so is the documentation.
 
-You can read the [online documentation](https://rustpython.github.io/website/rustpython/index.html) for the latest code on master.
+You can generate documentation by running:
-
-You can also generate documentation locally by running:
 
 ```shell
-$ cargo doc # Including documentation for all dependencies
+$ cargo doc
-$ cargo doc --no-deps --all # Excluding all dependencies
 ```
 
 Documentation HTML files can then be found in the `target/doc` directory.
 
-If you wish to update the online documentation, push directly to the `release` branch (or ask a maintainer to do so). This will trigger a Travis build that updates the documentation and WebAssembly demo page.
-
 # Code organization
 
-- `parser/src`: python lexing, parsing and ast
+- `parser`: python lexing, parsing and ast
-- `vm/src`: python virtual machine
+- `vm`: python virtual machine
-  - `builtins.rs`: Builtin functions
-  - `compile.rs`: the python compiler from ast to bytecode
-  - `obj`: python builtin types
 - `src`: using the other subcrates to bring rustpython to life.
 - `docs`: documentation (work in progress)
-- `py_code_object`: CPython bytecode to rustpython bytecode converter (work in progress)
+- `py_code_object`: CPython bytecode to rustpython bytecode convertor (work in progress)
-- `wasm`: Binary crate and resources for WebAssembly build
+- `wasm`: Binary crate and resources for WebAssembly build 
 - `tests`: integration test snippets
 
 # Contributing
 
-Contributions are more than welcome, and in many cases we are happy to guide contributors through PRs or on gitter.
+To start contributing, there are a lot of things that need to be done.
-
-With that in mind, please note this project is maintained by volunteers, some of the best ways to get started are below:
-
 Most tasks are listed in the [issue tracker](https://github.com/RustPython/RustPython/issues).
-Check issues labeled with `good first issue` if you wish to start coding.
+Another approach is to checkout the sourcecode: builtin functions and object methods are often the simplest
-
+and easiest way to contribute. 
-Another approach is to checkout the source code: builtin functions and object methods are often the simplest
-and easiest way to contribute.
 
 You can also simply run
-`./whats_left.sh` to assist in finding any
+`cargo run tests/snippets/todo.py` to assist in finding any
 unimplemented method.
 
 # Testing
@@ -95,50 +68,117 @@ To test rustpython, there is a collection of python snippets located in the
 
 ```shell
 $ cd tests
-$ pipenv install
+$ pipenv shell
-$ pipenv run pytest -v
+$ pytest -v
 ```
 
-There also are some unit tests, you can run those with cargo:
+There also are some unittests, you can run those will cargo:
 
 ```shell
 $ cargo test --all
 ```
 
-# Using a standard library
+# Using another standard library
 
-As of now the standard library is under construction. You can
+As of now the standard library is under construction.
-use a standard library by setting the RUSTPYTHONPATH environment
-variable.
-
-To do this, follow this method:
-
-```shell
-$ export RUSTPYTHONPATH=~/GIT/RustPython/Lib
-$ cargo run -- -c 'import xdrlib'
-```
 
 You can play around
 with other standard libraries for python. For example,
 the [ouroboros library](https://github.com/pybee/ouroboros).
 
+To do this, follow this method:
+
+```shell
+$ cd ~/GIT
+$ git clone git@github.com:pybee/ouroboros.git
+$ export PYTHONPATH=~/GIT/ouroboros/ouroboros
+$ cd RustPython
+$ cargo run -- -c 'import statistics'
+```
+
 # Compiling to WebAssembly
 
-[See this doc](wasm/README.md)
+At this stage RustPython only has preliminary support for web assembly. The instructions here are intended for developers or those wishing to run a toy example.
+
+## Setup
+
+To get started, install [wasm-bingden](https://rustwasm.github.io/wasm-bindgen/whirlwind-tour/basic-usage.html)
+and [wasm-webpack](https://rustwasm.github.io/wasm-pack/installer/). You will also need to have `npm` installed.
+
+<!-- Using `rustup` add the compile target `wasm32-unknown-emscripten`. To do so you will need to have [rustup](https://rustup.rs/) installed.
+
+```bash
+rustup target add wasm32-unknown-emscripten
+```
+
+Next, install `emsdk`:
+
+```bash
+curl https://s3.amazonaws.com/mozilla-games/emscripten/releases/emsdk-portable.tar.gz | tar -zxv
+cd emsdk-portable/
+./emsdk update
+./emsdk install sdk-incoming-64bit
+./emsdk activate sdk-incoming-64bit
+``` -->
+
+
+
+## Build
+
+Move into the `wasm` directory. This contains a custom library crate optimized for wasm build of RustPython.   
+
+```bash
+cd wasm
+```
+
+From here run the build. This can take several minutes depending on the machine.
+
+```
+wasm-pack build
+```
+
+Upon successful build, cd in the the `/pkg` directory and run:
+
+```
+npm link
+```
+
+Now move back out into the `/app` directory. The files here have been adapted from [wasm-pack-template](https://github.com/rustwasm/wasm-pack-template).
+
+Finally, run:
+
+```
+npm install
+```
+
+and you will be able to run the files with:
+
+```
+webpack-dev-server
+```
+
+Open a browser console and see the output of rustpython_wasm. To verify this, modify the line in `app/index.js`
+
+```js
+rp.run_code("print('Hello Python!')\n");
+```
+
+To the following:
+
+```js
+rp.run_code("assert(False)\n");
+```
+
+and you should observe: `Execution failed` in your console output, indicating that the execution of RustPython has failed.
 
 # Code style
 
-The code style used is the default [rustfmt](https://github.com/rust-lang/rustfmt) codestyle. Please format your code accordingly.
+The code style used is the default rustfmt codestyle. Please format your code accordingly.
-We also use [clippy](https://github.com/rust-lang/rust-clippy) to detect rust code issues.
 
 # Community
 
 Chat with us on [gitter][gitter].
 
-# Code of conduct
-
-Our code of conduct [can be found here](code-of-conduct.md).
-
 # Credit
 
 The initial work was based on [windelbouwman/rspython](https://github.com/windelbouwman/rspython) and [shinglyu/RustPython](https://github.com/shinglyu/RustPython)
@@ -152,3 +192,4 @@ These are some useful links to related projects:
 - https://github.com/ProgVal/pythonvm-rust
 - https://github.com/shinglyu/RustPython
 - https://github.com/windelbouwman/rspython
+

azure-pipelines.yml

@@ -1,56 +0,0 @@
-trigger:
-- master
-
-jobs:
-
-- job: 'Test'
-  pool:
-    vmImage: 'vs2017-win2016'
-  strategy:
-    matrix:
-      Python36:
-        python.version: '3.6'
-    maxParallel: 10
-
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '$(python.version)'
-      architecture: 'x64'
-
-  - script: |
-      "C:\Program Files\Git\mingw64\bin\curl.exe" -sSf -o rustup-init.exe https://win.rustup.rs/
-      .\rustup-init.exe -y
-      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-      rustc -V
-      cargo -V
-    displayName: 'Installing Rust'
-
-  - script: |
-      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-      cargo build --verbose --all
-    displayName: 'Build'
-
-  - script: |
-      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-      cargo test --verbose --all
-    displayName: 'Run tests'
-
-  - script: |
-      pip install pipenv
-      pushd tests
-      pipenv install
-      popd
-    displayName: 'Install pipenv and python packages'
-
-  - script: |
-      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-      cargo build --verbose --release
-    displayName: 'Build release'
-
-  - script: |
-      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-      pushd tests
-      pipenv run pytest
-      popd
-    displayName: 'Run snippet tests'

benchmarks/README.md

@@ -1,19 +0,0 @@
-
-# Benchmarking
-
-These are some files to determine performance of rustpython.
-
-# Usage
-
-Install pytest and pytest-benchmark:
-
-    $ pip install pytest-benchmark
-
-Then run:
-
-    $ pytest
-
-# Benchmark source
-
-- https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html
-

benchmarks/bench.rs

@@ -1,125 +0,0 @@
-#![feature(test)]
-
-extern crate cpython;
-extern crate rustpython_parser;
-extern crate rustpython_vm;
-extern crate test;
-
-use rustpython_vm::pyobject::PyResult;
-use rustpython_vm::{compile, VirtualMachine};
-
-#[bench]
-fn bench_tokenization(b: &mut test::Bencher) {
-    use rustpython_parser::lexer::{make_tokenizer, Tok};
-
-    let source = include_str!("./benchmarks/minidom.py");
-
-    b.bytes = source.len() as _;
-    b.iter(|| {
-        let lexer = make_tokenizer(source);
-        for res in lexer {
-            let _token: Tok = res.unwrap().1;
-        }
-    })
-}
-
-#[bench]
-fn bench_rustpy_parse_to_ast(b: &mut test::Bencher) {
-    use rustpython_parser::parser::parse_program;
-
-    let source = include_str!("./benchmarks/minidom.py");
-
-    b.bytes = source.len() as _;
-    b.iter(|| parse_program(source).unwrap())
-}
-
-#[bench]
-fn bench_cpython_parse_to_ast(b: &mut test::Bencher) {
-    let source = include_str!("./benchmarks/minidom.py");
-
-    let gil = cpython::Python::acquire_gil();
-    let python = gil.python();
-
-    let globals = None;
-    let locals = cpython::PyDict::new(python);
-
-    locals.set_item(python, "SOURCE_CODE", source).unwrap();
-
-    let code = "compile(SOURCE_CODE, mode=\"exec\", filename=\"minidom.py\")";
-
-    b.bytes = source.len() as _;
-    b.iter(|| {
-        let res: cpython::PyResult<cpython::PyObject> = python.eval(code, globals, Some(&locals));
-        assert!(res.is_ok());
-    })
-}
-
-#[bench]
-fn bench_cpython_nbody(b: &mut test::Bencher) {
-    let source = include_str!("./benchmarks/nbody.py");
-
-    let gil = cpython::Python::acquire_gil();
-    let python = gil.python();
-
-    let globals = None;
-    let locals = None;
-
-    b.iter(|| {
-        let res: cpython::PyResult<()> = python.run(source, globals, locals);
-        assert!(res.is_ok());
-    })
-}
-
-#[bench]
-fn bench_cpython_mandelbrot(b: &mut test::Bencher) {
-    let source = include_str!("./benchmarks/mandelbrot.py");
-
-    let gil = cpython::Python::acquire_gil();
-    let python = gil.python();
-
-    let globals = None;
-    let locals = None;
-
-    b.iter(|| {
-        let res: cpython::PyResult<()> = python.run(source, globals, locals);
-        assert!(res.is_ok());
-    })
-}
-
-#[bench]
-fn bench_rustpy_nbody(b: &mut test::Bencher) {
-    // NOTE: Take long time.
-    let source = include_str!("./benchmarks/nbody.py");
-
-    let vm = VirtualMachine::new();
-
-    let code = match vm.compile(source, &compile::Mode::Single, "<stdin>".to_string()) {
-        Ok(code) => code,
-        Err(e) => panic!("{:?}", e),
-    };
-
-    b.iter(|| {
-        let scope = vm.new_scope_with_builtins();
-        let res: PyResult = vm.run_code_obj(code.clone(), scope);
-        assert!(res.is_ok());
-    })
-}
-
-#[bench]
-fn bench_rustpy_mandelbrot(b: &mut test::Bencher) {
-    // NOTE: Take long time.
-    let source = include_str!("./benchmarks/mandelbrot.py");
-
-    let vm = VirtualMachine::new();
-
-    let code = match vm.compile(source, &compile::Mode::Single, "<stdin>".to_string()) {
-        Ok(code) => code,
-        Err(e) => panic!("{:?}", e),
-    };
-
-    b.iter(|| {
-        let scope = vm.new_scope_with_builtins();
-        let res: PyResult = vm.run_code_obj(code.clone(), scope);
-        assert!(res.is_ok());
-    })
-}

benchmarks/benchmarks/mandelbrot.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python3
-# coding: utf-8
-
-w = 50.0
-h = 50.0
-
-y = 0.0
-while y < h:
-	x = 0.0
-	while x < w:
-		Zr, Zi, Tr, Ti = 0.0, 0.0, 0.0, 0.0
-		Cr = 2*x/w - 1.5
-		Ci = 2*y/h - 1.0
-
-		i = 0
-		while i < 50 and Tr+Ti <= 4:
-			Zi = 2*Zr*Zi + Ci
-			Zr = Tr - Ti + Cr
-			Tr = Zr * Zr
-			Ti = Zi * Zi
-			i = i+1
-
-		if Tr+Ti <= 4:
-			# print('*', end='')
-			pass
-		else:
-			# print('·', end='')
-			pass
-
-		x = x+1
-
-	# print()
-	y = y+1

benchmarks/benchmarks/nbody.py

@@ -1,110 +0,0 @@
-
-# The Computer Language Benchmarks Game
-# https://salsa.debian.org/benchmarksgame-team/benchmarksgame/
-#
-# originally by Kevin Carson
-# modified by Tupteq, Fredrik Johansson, and Daniel Nanz
-# modified by Maciej Fijalkowski
-# 2to3
-# modified by Andriy Misyura
-
-from math import sqrt
-
-def combinations(l):
-    result = []
-    for x in range(len(l) - 1):
-        ls = l[x+1:]
-        for y in ls:
-            result.append((l[x][0],l[x][1],l[x][2],y[0],y[1],y[2]))
-    return result
-
-PI = 3.14159265358979323
-SOLAR_MASS = 4 * PI * PI
-DAYS_PER_YEAR = 365.24
-
-BODIES = {
-    'sun': ([0.0, 0.0, 0.0], [0.0, 0.0, 0.0], SOLAR_MASS),
-
-    'jupiter': ([4.84143144246472090e+00,
-                 -1.16032004402742839e+00,
-                 -1.03622044471123109e-01],
-                [1.66007664274403694e-03 * DAYS_PER_YEAR,
-                 7.69901118419740425e-03 * DAYS_PER_YEAR,
-                 -6.90460016972063023e-05 * DAYS_PER_YEAR],
-                9.54791938424326609e-04 * SOLAR_MASS),
-
-    'saturn': ([8.34336671824457987e+00,
-                4.12479856412430479e+00,
-                -4.03523417114321381e-01],
-               [-2.76742510726862411e-03 * DAYS_PER_YEAR,
-                4.99852801234917238e-03 * DAYS_PER_YEAR,
-                2.30417297573763929e-05 * DAYS_PER_YEAR],
-               2.85885980666130812e-04 * SOLAR_MASS),
-
-    'uranus': ([1.28943695621391310e+01,
-                -1.51111514016986312e+01,
-                -2.23307578892655734e-01],
-               [2.96460137564761618e-03 * DAYS_PER_YEAR,
-                2.37847173959480950e-03 * DAYS_PER_YEAR,
-                -2.96589568540237556e-05 * DAYS_PER_YEAR],
-               4.36624404335156298e-05 * SOLAR_MASS),
-
-    'neptune': ([1.53796971148509165e+01,
-                 -2.59193146099879641e+01,
-                 1.79258772950371181e-01],
-                [2.68067772490389322e-03 * DAYS_PER_YEAR,
-                 1.62824170038242295e-03 * DAYS_PER_YEAR,
-                 -9.51592254519715870e-05 * DAYS_PER_YEAR],
-                5.15138902046611451e-05 * SOLAR_MASS) }
-
-SYSTEM = tuple(BODIES.values())
-PAIRS = tuple(combinations(SYSTEM))
-
-def advance(dt, n, bodies=SYSTEM, pairs=PAIRS):
-    for i in range(n):
-        for ([x1, y1, z1], v1, m1, [x2, y2, z2], v2, m2) in pairs:
-            dx = x1 - x2
-            dy = y1 - y2
-            dz = z1 - z2
-            dist = sqrt(dx * dx + dy * dy + dz * dz);
-            mag = dt / (dist*dist*dist)
-            b1m = m1 * mag
-            b2m = m2 * mag
-            v1[0] -= dx * b2m
-            v1[1] -= dy * b2m
-            v1[2] -= dz * b2m
-            v2[2] += dz * b1m
-            v2[1] += dy * b1m
-            v2[0] += dx * b1m
-        for (r, [vx, vy, vz], m) in bodies:
-            r[0] += dt * vx
-            r[1] += dt * vy
-            r[2] += dt * vz
-
-def report_energy(bodies=SYSTEM, pairs=PAIRS, e=0.0):
-    for ((x1, y1, z1), v1, m1, (x2, y2, z2), v2, m2) in pairs:
-        dx = x1 - x2
-        dy = y1 - y2
-        dz = z1 - z2
-        e -= (m1 * m2) / ((dx * dx + dy * dy + dz * dz) ** 0.5)
-    for (r, [vx, vy, vz], m) in bodies:
-        e += m * (vx * vx + vy * vy + vz * vz) / 2.
-    # print(f"{e}")
-
-def offset_momentum(ref, bodies=SYSTEM, px=0.0, py=0.0, pz=0.0):
-    for (r, [vx, vy, vz], m) in bodies:
-        px -= vx * m
-        py -= vy * m
-        pz -= vz * m
-    (r, v, m) = ref
-    v[0] = px / m
-    v[1] = py / m
-    v[2] = pz / m
-
-def main(n, ref='sun'):
-    offset_momentum(BODIES[ref])
-    report_energy()
-    advance(0.01, n)
-    report_energy()
-
-main(500)

benchmarks/test_benchmarks.py

@@ -1,31 +0,0 @@
-
-import time
-import sys
-
-import pytest
-import subprocess
-
-from benchmarks import nbody
-
-# Interpreters:
-rustpython_exe = '../target/release/rustpython'
-cpython_exe = sys.executable
-pythons = [
-    cpython_exe,
-    rustpython_exe
-]
-
-# Benchmark scripts:
-benchmarks = [
-    ['benchmarks/nbody.py'],
-    ['benchmarks/mandelbrot.py'],
-]
-
-@pytest.mark.parametrize('exe', pythons)
-@pytest.mark.parametrize('args', benchmarks)
-def test_bench(exe, args, benchmark):
-    def bench():
-        subprocess.run([exe] + args)
-
-    benchmark(bench)
-

bytecode/Cargo.toml

@@ -1,15 +0,0 @@
-[package]
-name = "rustpython-bytecode"
-description = "RustPython specific bytecode."
-version = "0.1.0"
-authors = ["RustPython Team"]
-edition = "2018"
-repository = "https://github.com/RustPython/RustPython"
-license = "MIT"
-
-
-[dependencies]
-bitflags = "1.1"
-num-bigint = { version = "0.2", features = ["serde"] }
-num-complex = { version = "0.2", features = ["serde"] }
-serde = { version = "1.0", features = ["derive"] }

bytecode/src/bytecode.rs

@@ -1,469 +0,0 @@
-//! Implement python as a virtual machine with bytecodes. This module
-//! implements bytecode structure.
-
-use bitflags::bitflags;
-use num_bigint::BigInt;
-use num_complex::Complex64;
-use serde::{Deserialize, Serialize};
-use std::collections::{HashMap, HashSet};
-use std::fmt;
-
-/// Sourcode location.
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-pub struct Location {
-    row: usize,
-    column: usize,
-}
-
-impl Location {
-    pub fn new(row: usize, column: usize) -> Self {
-        Location { row, column }
-    }
-
-    pub fn row(&self) -> usize {
-        self.row
-    }
-
-    pub fn column(&self) -> usize {
-        self.column
-    }
-}
-
-/// Primary container of a single code object. Each python function has
-/// a codeobject. Also a module has a codeobject.
-#[derive(Clone, PartialEq, Serialize, Deserialize)]
-pub struct CodeObject {
-    pub instructions: Vec<Instruction>,
-    /// Jump targets.
-    pub label_map: HashMap<Label, usize>,
-    pub locations: Vec<Location>,
-    pub arg_names: Vec<String>, // Names of positional arguments
-    pub varargs: Varargs,       // *args or *
-    pub kwonlyarg_names: Vec<String>,
-    pub varkeywords: Varargs, // **kwargs or **
-    pub source_path: String,
-    pub first_line_number: usize,
-    pub obj_name: String, // Name of the object that created this code object
-    pub is_generator: bool,
-}
-
-bitflags! {
-    #[derive(Serialize, Deserialize)]
-    pub struct FunctionOpArg: u8 {
-        const HAS_DEFAULTS = 0x01;
-        const HAS_KW_ONLY_DEFAULTS = 0x02;
-        const HAS_ANNOTATIONS = 0x04;
-    }
-}
-
-pub type Label = usize;
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum NameScope {
-    Local,
-    NonLocal,
-    Global,
-}
-
-/// Transforms a value prior to formatting it.
-#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub enum ConversionFlag {
-    /// Converts by calling `str(<value>)`.
-    Str,
-    /// Converts by calling `ascii(<value>)`.
-    Ascii,
-    /// Converts by calling `repr(<value>)`.
-    Repr,
-}
-
-/// A Single bytecode instruction.
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum Instruction {
-    Import {
-        name: String,
-        symbols: Vec<String>,
-        level: usize,
-    },
-    ImportStar {
-        name: String,
-        level: usize,
-    },
-    LoadName {
-        name: String,
-        scope: NameScope,
-    },
-    StoreName {
-        name: String,
-        scope: NameScope,
-    },
-    DeleteName {
-        name: String,
-    },
-    StoreSubscript,
-    DeleteSubscript,
-    StoreAttr {
-        name: String,
-    },
-    DeleteAttr {
-        name: String,
-    },
-    LoadConst {
-        value: Constant,
-    },
-    UnaryOperation {
-        op: UnaryOperator,
-    },
-    BinaryOperation {
-        op: BinaryOperator,
-        inplace: bool,
-    },
-    LoadAttr {
-        name: String,
-    },
-    CompareOperation {
-        op: ComparisonOperator,
-    },
-    Pop,
-    Rotate {
-        amount: usize,
-    },
-    Duplicate,
-    GetIter,
-    Pass,
-    Continue,
-    Break,
-    Jump {
-        target: Label,
-    },
-    JumpIf {
-        target: Label,
-    },
-    JumpIfFalse {
-        target: Label,
-    },
-    MakeFunction {
-        flags: FunctionOpArg,
-    },
-    CallFunction {
-        typ: CallType,
-    },
-    ForIter {
-        target: Label,
-    },
-    ReturnValue,
-    YieldValue,
-    YieldFrom,
-    SetupLoop {
-        start: Label,
-        end: Label,
-    },
-    SetupExcept {
-        handler: Label,
-    },
-    SetupWith {
-        end: Label,
-    },
-    CleanupWith {
-        end: Label,
-    },
-    PopBlock,
-    Raise {
-        argc: usize,
-    },
-    BuildString {
-        size: usize,
-    },
-    BuildTuple {
-        size: usize,
-        unpack: bool,
-    },
-    BuildList {
-        size: usize,
-        unpack: bool,
-    },
-    BuildSet {
-        size: usize,
-        unpack: bool,
-    },
-    BuildMap {
-        size: usize,
-        unpack: bool,
-    },
-    BuildSlice {
-        size: usize,
-    },
-    ListAppend {
-        i: usize,
-    },
-    SetAdd {
-        i: usize,
-    },
-    MapAdd {
-        i: usize,
-    },
-    PrintExpr,
-    LoadBuildClass,
-    UnpackSequence {
-        size: usize,
-    },
-    UnpackEx {
-        before: usize,
-        after: usize,
-    },
-    Unpack,
-    FormatValue {
-        conversion: Option<ConversionFlag>,
-        spec: String,
-    },
-    PopException,
-}
-
-use self::Instruction::*;
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum CallType {
-    Positional(usize),
-    Keyword(usize),
-    Ex(bool),
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum Constant {
-    Integer { value: BigInt },
-    Float { value: f64 },
-    Complex { value: Complex64 },
-    Boolean { value: bool },
-    String { value: String },
-    Bytes { value: Vec<u8> },
-    Code { code: Box<CodeObject> },
-    Tuple { elements: Vec<Constant> },
-    None,
-    Ellipsis,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum ComparisonOperator {
-    Greater,
-    GreaterOrEqual,
-    Less,
-    LessOrEqual,
-    Equal,
-    NotEqual,
-    In,
-    NotIn,
-    Is,
-    IsNot,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum BinaryOperator {
-    Power,
-    Multiply,
-    MatrixMultiply,
-    Divide,
-    FloorDivide,
-    Modulo,
-    Add,
-    Subtract,
-    Subscript,
-    Lshift,
-    Rshift,
-    And,
-    Xor,
-    Or,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum UnaryOperator {
-    Not,
-    Invert,
-    Minus,
-    Plus,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum Varargs {
-    None,
-    Unnamed,
-    Named(String),
-}
-
-/*
-Maintain a stack of blocks on the VM.
-pub enum BlockType {
-    Loop,
-    Except,
-}
-*/
-
-impl CodeObject {
-    pub fn new(
-        arg_names: Vec<String>,
-        varargs: Varargs,
-        kwonlyarg_names: Vec<String>,
-        varkeywords: Varargs,
-        source_path: String,
-        first_line_number: usize,
-        obj_name: String,
-    ) -> CodeObject {
-        CodeObject {
-            instructions: Vec::new(),
-            label_map: HashMap::new(),
-            locations: Vec::new(),
-            arg_names,
-            varargs,
-            kwonlyarg_names,
-            varkeywords,
-            source_path,
-            first_line_number,
-            obj_name,
-            is_generator: false,
-        }
-    }
-
-    pub fn get_constants(&self) -> impl Iterator<Item = &Constant> {
-        self.instructions.iter().filter_map(|x| {
-            if let Instruction::LoadConst { value } = x {
-                Some(value)
-            } else {
-                None
-            }
-        })
-    }
-}
-
-impl fmt::Display for CodeObject {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let label_targets: HashSet<&usize> = self.label_map.values().collect();
-        for (offset, instruction) in self.instructions.iter().enumerate() {
-            let arrow = if label_targets.contains(&offset) {
-                ">>"
-            } else {
-                "  "
-            };
-            write!(f, "          {} {:5} ", arrow, offset)?;
-            instruction.fmt_dis(f, &self.label_map)?;
-        }
-        Ok(())
-    }
-}
-
-impl Instruction {
-    fn fmt_dis(&self, f: &mut fmt::Formatter, label_map: &HashMap<Label, usize>) -> fmt::Result {
-        macro_rules! w {
-            ($variant:ident) => {
-                write!(f, "{:20}\n", stringify!($variant))
-            };
-            ($variant:ident, $var:expr) => {
-                write!(f, "{:20} ({})\n", stringify!($variant), $var)
-            };
-            ($variant:ident, $var1:expr, $var2:expr) => {
-                write!(f, "{:20} ({}, {})\n", stringify!($variant), $var1, $var2)
-            };
-            ($variant:ident, $var1:expr, $var2:expr, $var3:expr) => {
-                write!(
-                    f,
-                    "{:20} ({}, {}, {})\n",
-                    stringify!($variant),
-                    $var1,
-                    $var2,
-                    $var3
-                )
-            };
-        }
-
-        match self {
-            Import {
-                name,
-                symbols,
-                level,
-            } => w!(Import, name, format!("{:?}", symbols), level),
-            ImportStar { name, level } => w!(ImportStar, name, level),
-            LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)),
-            StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)),
-            DeleteName { name } => w!(DeleteName, name),
-            StoreSubscript => w!(StoreSubscript),
-            DeleteSubscript => w!(DeleteSubscript),
-            StoreAttr { name } => w!(StoreAttr, name),
-            DeleteAttr { name } => w!(DeleteAttr, name),
-            LoadConst { value } => w!(LoadConst, value),
-            UnaryOperation { op } => w!(UnaryOperation, format!("{:?}", op)),
-            BinaryOperation { op, inplace } => w!(BinaryOperation, format!("{:?}", op), inplace),
-            LoadAttr { name } => w!(LoadAttr, name),
-            CompareOperation { op } => w!(CompareOperation, format!("{:?}", op)),
-            Pop => w!(Pop),
-            Rotate { amount } => w!(Rotate, amount),
-            Duplicate => w!(Duplicate),
-            GetIter => w!(GetIter),
-            Pass => w!(Pass),
-            Continue => w!(Continue),
-            Break => w!(Break),
-            Jump { target } => w!(Jump, label_map[target]),
-            JumpIf { target } => w!(JumpIf, label_map[target]),
-            JumpIfFalse { target } => w!(JumpIfFalse, label_map[target]),
-            MakeFunction { flags } => w!(MakeFunction, format!("{:?}", flags)),
-            CallFunction { typ } => w!(CallFunction, format!("{:?}", typ)),
-            ForIter { target } => w!(ForIter, label_map[target]),
-            ReturnValue => w!(ReturnValue),
-            YieldValue => w!(YieldValue),
-            YieldFrom => w!(YieldFrom),
-            SetupLoop { start, end } => w!(SetupLoop, label_map[start], label_map[end]),
-            SetupExcept { handler } => w!(SetupExcept, handler),
-            SetupWith { end } => w!(SetupWith, end),
-            CleanupWith { end } => w!(CleanupWith, end),
-            PopBlock => w!(PopBlock),
-            Raise { argc } => w!(Raise, argc),
-            BuildString { size } => w!(BuildString, size),
-            BuildTuple { size, unpack } => w!(BuildTuple, size, unpack),
-            BuildList { size, unpack } => w!(BuildList, size, unpack),
-            BuildSet { size, unpack } => w!(BuildSet, size, unpack),
-            BuildMap { size, unpack } => w!(BuildMap, size, unpack),
-            BuildSlice { size } => w!(BuildSlice, size),
-            ListAppend { i } => w!(ListAppend, i),
-            SetAdd { i } => w!(SetAdd, i),
-            MapAdd { i } => w!(MapAdd, i),
-            PrintExpr => w!(PrintExpr),
-            LoadBuildClass => w!(LoadBuildClass),
-            UnpackSequence { size } => w!(UnpackSequence, size),
-            UnpackEx { before, after } => w!(UnpackEx, before, after),
-            Unpack => w!(Unpack),
-            FormatValue { spec, .. } => w!(FormatValue, spec), // TODO: write conversion
-            PopException => w!(PopException),
-        }
-    }
-}
-
-impl fmt::Display for Constant {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            Constant::Integer { value } => write!(f, "{}", value),
-            Constant::Float { value } => write!(f, "{}", value),
-            Constant::Complex { value } => write!(f, "{}", value),
-            Constant::Boolean { value } => write!(f, "{}", value),
-            Constant::String { value } => write!(f, "{:?}", value),
-            Constant::Bytes { value } => write!(f, "{:?}", value),
-            Constant::Code { code } => write!(f, "{:?}", code),
-            Constant::Tuple { elements } => write!(
-                f,
-                "({})",
-                elements
-                    .iter()
-                    .map(|e| format!("{}", e))
-                    .collect::<Vec<_>>()
-                    .join(", ")
-            ),
-            Constant::None => write!(f, "None"),
-            Constant::Ellipsis => write!(f, "Ellipsis"),
-        }
-    }
-}
-
-impl fmt::Debug for CodeObject {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "<code object {} at ??? file {:?}, line {}>",
-            self.obj_name, self.source_path, self.first_line_number
-        )
-    }
-}

bytecode/src/lib.rs

@@ -1 +0,0 @@
-pub mod bytecode;

code-of-conduct.md

@@ -1,77 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at windel.bouwman@gmail.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
-

compiler/Cargo.toml

@@ -1,14 +0,0 @@
-[package]
-name = "rustpython-compiler"
-version = "0.1.0"
-description = "Compiler for python code into bytecode for the rustpython VM."
-authors = ["RustPython Team"]
-repository = "https://github.com/RustPython/RustPython"
-license = "MIT"
-edition = "2018"
-
-[dependencies]
-rustpython-bytecode = { path = "../bytecode", version = "0.1.0" }
-rustpython-parser = { path = "../parser", version = "0.1.0" }
-num-complex = { version = "0.2", features = ["serde"] }
-log = "0.3"

compiler/src/error.rs

@@ -1,67 +0,0 @@
-use rustpython_parser::error::{ParseError, ParseErrorType};
-use rustpython_parser::lexer::Location;
-
-use std::error::Error;
-use std::fmt;
-
-#[derive(Debug)]
-pub struct CompileError {
-    pub error: CompileErrorType,
-    pub location: Location,
-}
-
-impl From<ParseError> for CompileError {
-    fn from(error: ParseError) -> Self {
-        CompileError {
-            error: CompileErrorType::Parse(error.error),
-            location: error.location,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub enum CompileErrorType {
-    /// Invalid assignment, cannot store value in target.
-    Assign(&'static str),
-    /// Invalid delete
-    Delete(&'static str),
-    /// Expected an expression got a statement
-    ExpectExpr,
-    /// Parser error
-    Parse(ParseErrorType),
-    SyntaxError(String),
-    /// Multiple `*` detected
-    StarArgs,
-    /// Break statement outside of loop.
-    InvalidBreak,
-    /// Continue statement outside of loop.
-    InvalidContinue,
-    InvalidReturn,
-    InvalidYield,
-}
-
-impl fmt::Display for CompileError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match &self.error {
-            CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target),
-            CompileErrorType::Delete(target) => write!(f, "can't delete {}", target),
-            CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"),
-            CompileErrorType::Parse(err) => write!(f, "{}", err),
-            CompileErrorType::SyntaxError(err) => write!(f, "{}", err),
-            CompileErrorType::StarArgs => write!(f, "Two starred expressions in assignment"),
-            CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
-            CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
-            CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),
-            CompileErrorType::InvalidYield => write!(f, "'yield' outside function"),
-        }?;
-
-        // Print line number:
-        write!(f, " at {}", self.location)
-    }
-}
-
-impl Error for CompileError {
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        None
-    }
-}

compiler/src/lib.rs

@@ -1,9 +0,0 @@
-//! Compile a Python AST or source code into bytecode consumable by RustPython or
-//! (eventually) CPython.
-
-#[macro_use]
-extern crate log;
-
-pub mod compile;
-pub mod error;
-mod symboltable;

compiler/src/symboltable.rs

@@ -1,596 +0,0 @@
-/* Python code is pre-scanned for symbols in the ast.
-
-This ensures that global and nonlocal keywords are picked up.
-Then the compiler can use the symbol table to generate proper
-load and store instructions for names.
-
-Inspirational file: https://github.com/python/cpython/blob/master/Python/symtable.c
-*/
-
-use crate::error::{CompileError, CompileErrorType};
-use rustpython_parser::ast;
-use rustpython_parser::lexer::Location;
-use std::collections::HashMap;
-
-pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTableError> {
-    let mut builder = SymbolTableBuilder::new();
-    builder.enter_scope();
-    builder.scan_program(program)?;
-    assert_eq!(builder.scopes.len(), 1);
-
-    let symbol_table = builder.scopes.pop().unwrap();
-    analyze_symbol_table(&symbol_table, None)?;
-    Ok(symbol_table)
-}
-
-pub fn statements_to_symbol_table(
-    statements: &[ast::LocatedStatement],
-) -> Result<SymbolScope, SymbolTableError> {
-    let mut builder = SymbolTableBuilder::new();
-    builder.enter_scope();
-    builder.scan_statements(statements)?;
-    assert_eq!(builder.scopes.len(), 1);
-
-    let symbol_table = builder.scopes.pop().unwrap();
-    analyze_symbol_table(&symbol_table, None)?;
-    Ok(symbol_table)
-}
-
-#[derive(Debug)]
-pub enum SymbolRole {
-    Global,
-    Nonlocal,
-    Used,
-    Assigned,
-}
-
-/// Captures all symbols in the current scope, and has a list of subscopes in this scope.
-pub struct SymbolScope {
-    /// A set of symbols present on this scope level.
-    pub symbols: HashMap<String, SymbolRole>,
-
-    /// A list of subscopes in the order as found in the
-    /// AST nodes.
-    pub sub_scopes: Vec<SymbolScope>,
-}
-
-#[derive(Debug)]
-pub struct SymbolTableError {
-    error: String,
-    location: Location,
-}
-
-impl From<SymbolTableError> for CompileError {
-    fn from(error: SymbolTableError) -> Self {
-        CompileError {
-            error: CompileErrorType::SyntaxError(error.error),
-            location: error.location,
-        }
-    }
-}
-
-type SymbolTableResult = Result<(), SymbolTableError>;
-
-impl SymbolScope {
-    pub fn new() -> Self {
-        SymbolScope {
-            symbols: HashMap::new(),
-            sub_scopes: vec![],
-        }
-    }
-
-    pub fn lookup(&self, name: &str) -> Option<&SymbolRole> {
-        self.symbols.get(name)
-    }
-}
-
-impl std::fmt::Debug for SymbolScope {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(
-            f,
-            "SymbolScope({:?} symbols, {:?} sub scopes)",
-            self.symbols.len(),
-            self.sub_scopes.len()
-        )
-    }
-}
-
-/* Perform some sort of analysis on nonlocals, globals etc..
-  See also: https://github.com/python/cpython/blob/master/Python/symtable.c#L410
-*/
-fn analyze_symbol_table(
-    symbol_scope: &SymbolScope,
-    parent_symbol_scope: Option<&SymbolScope>,
-) -> SymbolTableResult {
-    // Analyze sub scopes:
-    for sub_scope in &symbol_scope.sub_scopes {
-        analyze_symbol_table(&sub_scope, Some(symbol_scope))?;
-    }
-
-    // Analyze symbols:
-    for (symbol_name, symbol_role) in &symbol_scope.symbols {
-        analyze_symbol(symbol_name, symbol_role, parent_symbol_scope)?;
-    }
-
-    Ok(())
-}
-
-#[allow(clippy::single_match)]
-fn analyze_symbol(
-    symbol_name: &str,
-    symbol_role: &SymbolRole,
-    parent_symbol_scope: Option<&SymbolScope>,
-) -> SymbolTableResult {
-    match symbol_role {
-        SymbolRole::Nonlocal => {
-            // check if name is defined in parent scope!
-            if let Some(parent_symbol_scope) = parent_symbol_scope {
-                if !parent_symbol_scope.symbols.contains_key(symbol_name) {
-                    return Err(SymbolTableError {
-                        error: format!("no binding for nonlocal '{}' found", symbol_name),
-                        location: Default::default(),
-                    });
-                }
-            } else {
-                return Err(SymbolTableError {
-                    error: format!(
-                        "nonlocal {} defined at place without an enclosing scope",
-                        symbol_name
-                    ),
-                    location: Default::default(),
-                });
-            }
-        }
-        // TODO: add more checks for globals
-        _ => {}
-    }
-    Ok(())
-}
-
-pub struct SymbolTableBuilder {
-    // Scope stack.
-    pub scopes: Vec<SymbolScope>,
-}
-
-impl SymbolTableBuilder {
-    pub fn new() -> Self {
-        SymbolTableBuilder { scopes: vec![] }
-    }
-
-    pub fn enter_scope(&mut self) {
-        let scope = SymbolScope::new();
-        self.scopes.push(scope);
-    }
-
-    fn leave_scope(&mut self) {
-        // Pop scope and add to subscopes of parent scope.
-        let scope = self.scopes.pop().unwrap();
-        self.scopes.last_mut().unwrap().sub_scopes.push(scope);
-    }
-
-    pub fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
-        self.scan_statements(&program.statements)?;
-        Ok(())
-    }
-
-    pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) -> SymbolTableResult {
-        for statement in statements {
-            self.scan_statement(statement)?;
-        }
-        Ok(())
-    }
-
-    fn scan_parameters(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
-        for parameter in parameters {
-            self.scan_parameter(parameter)?;
-        }
-        Ok(())
-    }
-
-    fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
-        self.register_name(&parameter.arg, SymbolRole::Assigned)
-    }
-
-    fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
-        for parameter in parameters {
-            self.scan_parameter_annotation(parameter)?;
-        }
-        Ok(())
-    }
-
-    fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
-        if let Some(annotation) = &parameter.annotation {
-            self.scan_expression(&annotation)?;
-        }
-        Ok(())
-    }
-
-    fn scan_statement(&mut self, statement: &ast::LocatedStatement) -> SymbolTableResult {
-        match &statement.node {
-            ast::Statement::Global { names } => {
-                for name in names {
-                    self.register_name(name, SymbolRole::Global)?;
-                }
-            }
-            ast::Statement::Nonlocal { names } => {
-                for name in names {
-                    self.register_name(name, SymbolRole::Nonlocal)?;
-                }
-            }
-            ast::Statement::FunctionDef {
-                name,
-                body,
-                args,
-                decorator_list,
-                returns,
-            }
-            | ast::Statement::AsyncFunctionDef {
-                name,
-                body,
-                args,
-                decorator_list,
-                returns,
-            } => {
-                self.scan_expressions(decorator_list)?;
-                self.register_name(name, SymbolRole::Assigned)?;
-
-                self.enter_function(args)?;
-
-                self.scan_statements(body)?;
-                if let Some(expression) = returns {
-                    self.scan_expression(expression)?;
-                }
-                self.leave_scope();
-            }
-            ast::Statement::ClassDef {
-                name,
-                body,
-                bases,
-                keywords,
-                decorator_list,
-            } => {
-                self.register_name(name, SymbolRole::Assigned)?;
-                self.enter_scope();
-                self.scan_statements(body)?;
-                self.leave_scope();
-                self.scan_expressions(bases)?;
-                for keyword in keywords {
-                    self.scan_expression(&keyword.value)?;
-                }
-                self.scan_expressions(decorator_list)?;
-            }
-            ast::Statement::Expression { expression } => self.scan_expression(expression)?,
-            ast::Statement::If { test, body, orelse } => {
-                self.scan_expression(test)?;
-                self.scan_statements(body)?;
-                if let Some(code) = orelse {
-                    self.scan_statements(code)?;
-                }
-            }
-            ast::Statement::For {
-                target,
-                iter,
-                body,
-                orelse,
-            }
-            | ast::Statement::AsyncFor {
-                target,
-                iter,
-                body,
-                orelse,
-            } => {
-                self.scan_expression(target)?;
-                self.scan_expression(iter)?;
-                self.scan_statements(body)?;
-                if let Some(code) = orelse {
-                    self.scan_statements(code)?;
-                }
-            }
-            ast::Statement::While { test, body, orelse } => {
-                self.scan_expression(test)?;
-                self.scan_statements(body)?;
-                if let Some(code) = orelse {
-                    self.scan_statements(code)?;
-                }
-            }
-            ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => {
-                // No symbols here.
-            }
-            ast::Statement::Import { import_parts } => {
-                for part in import_parts {
-                    if let Some(alias) = &part.alias {
-                        // `import mymodule as myalias`
-                        self.register_name(alias, SymbolRole::Assigned)?;
-                    } else {
-                        if part.symbols.is_empty() {
-                            // `import module`
-                            self.register_name(&part.module, SymbolRole::Assigned)?;
-                        } else {
-                            // `from mymodule import myimport`
-                            for symbol in &part.symbols {
-                                if let Some(alias) = &symbol.alias {
-                                    // `from mymodule import myimportname as myalias`
-                                    self.register_name(alias, SymbolRole::Assigned)?;
-                                } else {
-                                    self.register_name(&symbol.symbol, SymbolRole::Assigned)?;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            ast::Statement::Return { value } => {
-                if let Some(expression) = value {
-                    self.scan_expression(expression)?;
-                }
-            }
-            ast::Statement::Assert { test, msg } => {
-                self.scan_expression(test)?;
-                if let Some(expression) = msg {
-                    self.scan_expression(expression)?;
-                }
-            }
-            ast::Statement::Delete { targets } => {
-                self.scan_expressions(targets)?;
-            }
-            ast::Statement::Assign { targets, value } => {
-                self.scan_expressions(targets)?;
-                self.scan_expression(value)?;
-            }
-            ast::Statement::AugAssign { target, value, .. } => {
-                self.scan_expression(target)?;
-                self.scan_expression(value)?;
-            }
-            ast::Statement::With { items, body } => {
-                for item in items {
-                    self.scan_expression(&item.context_expr)?;
-                    if let Some(expression) = &item.optional_vars {
-                        self.scan_expression(expression)?;
-                    }
-                }
-                self.scan_statements(body)?;
-            }
-            ast::Statement::Try {
-                body,
-                handlers,
-                orelse,
-                finalbody,
-            } => {
-                self.scan_statements(body)?;
-                for handler in handlers {
-                    if let Some(expression) = &handler.typ {
-                        self.scan_expression(expression)?;
-                    }
-                    if let Some(name) = &handler.name {
-                        self.register_name(name, SymbolRole::Assigned)?;
-                    }
-                    self.scan_statements(&handler.body)?;
-                }
-                if let Some(code) = orelse {
-                    self.scan_statements(code)?;
-                }
-                if let Some(code) = finalbody {
-                    self.scan_statements(code)?;
-                }
-            }
-            ast::Statement::Raise { exception, cause } => {
-                if let Some(expression) = exception {
-                    self.scan_expression(expression)?;
-                }
-                if let Some(expression) = cause {
-                    self.scan_expression(expression)?;
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn scan_expressions(&mut self, expressions: &[ast::Expression]) -> SymbolTableResult {
-        for expression in expressions {
-            self.scan_expression(expression)?;
-        }
-        Ok(())
-    }
-
-    fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult {
-        match expression {
-            ast::Expression::Binop { a, b, .. } => {
-                self.scan_expression(a)?;
-                self.scan_expression(b)?;
-            }
-            ast::Expression::BoolOp { a, b, .. } => {
-                self.scan_expression(a)?;
-                self.scan_expression(b)?;
-            }
-            ast::Expression::Compare { vals, .. } => {
-                self.scan_expressions(vals)?;
-            }
-            ast::Expression::Subscript { a, b } => {
-                self.scan_expression(a)?;
-                self.scan_expression(b)?;
-            }
-            ast::Expression::Attribute { value, .. } => {
-                self.scan_expression(value)?;
-            }
-            ast::Expression::Dict { elements } => {
-                for (key, value) in elements {
-                    if let Some(key) = key {
-                        self.scan_expression(key)?;
-                    } else {
-                        // dict unpacking marker
-                    }
-                    self.scan_expression(value)?;
-                }
-            }
-            ast::Expression::Await { value } => {
-                self.scan_expression(value)?;
-            }
-            ast::Expression::Yield { value } => {
-                if let Some(expression) = value {
-                    self.scan_expression(expression)?;
-                }
-            }
-            ast::Expression::YieldFrom { value } => {
-                self.scan_expression(value)?;
-            }
-            ast::Expression::Unop { a, .. } => {
-                self.scan_expression(a)?;
-            }
-            ast::Expression::True
-            | ast::Expression::False
-            | ast::Expression::None
-            | ast::Expression::Ellipsis => {}
-            ast::Expression::Number { .. } => {}
-            ast::Expression::Starred { value } => {
-                self.scan_expression(value)?;
-            }
-            ast::Expression::Bytes { .. } => {}
-            ast::Expression::Tuple { elements }
-            | ast::Expression::Set { elements }
-            | ast::Expression::List { elements }
-            | ast::Expression::Slice { elements } => {
-                self.scan_expressions(elements)?;
-            }
-            ast::Expression::Comprehension { kind, generators } => {
-                match **kind {
-                    ast::ComprehensionKind::GeneratorExpression { ref element }
-                    | ast::ComprehensionKind::List { ref element }
-                    | ast::ComprehensionKind::Set { ref element } => {
-                        self.scan_expression(element)?;
-                    }
-                    ast::ComprehensionKind::Dict { ref key, ref value } => {
-                        self.scan_expression(&key)?;
-                        self.scan_expression(&value)?;
-                    }
-                }
-
-                for generator in generators {
-                    self.scan_expression(&generator.target)?;
-                    self.scan_expression(&generator.iter)?;
-                    for if_expr in &generator.ifs {
-                        self.scan_expression(if_expr)?;
-                    }
-                }
-            }
-            ast::Expression::Call {
-                function,
-                args,
-                keywords,
-            } => {
-                self.scan_expression(function)?;
-                self.scan_expressions(args)?;
-                for keyword in keywords {
-                    self.scan_expression(&keyword.value)?;
-                }
-            }
-            ast::Expression::String { value } => {
-                self.scan_string_group(value)?;
-            }
-            ast::Expression::Identifier { name } => {
-                self.register_name(name, SymbolRole::Used)?;
-            }
-            ast::Expression::Lambda { args, body } => {
-                self.enter_function(args)?;
-                self.scan_expression(body)?;
-                self.leave_scope();
-            }
-            ast::Expression::IfExpression { test, body, orelse } => {
-                self.scan_expression(test)?;
-                self.scan_expression(body)?;
-                self.scan_expression(orelse)?;
-            }
-        }
-        Ok(())
-    }
-
-    fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult {
-        // Evaluate eventual default parameters:
-        self.scan_expressions(&args.defaults)?;
-        for kw_default in &args.kw_defaults {
-            if let Some(expression) = kw_default {
-                self.scan_expression(&expression)?;
-            }
-        }
-
-        // Annotations are scanned in outer scope:
-        self.scan_parameters_annotations(&args.args)?;
-        self.scan_parameters_annotations(&args.kwonlyargs)?;
-        if let ast::Varargs::Named(name) = &args.vararg {
-            self.scan_parameter_annotation(name)?;
-        }
-        if let ast::Varargs::Named(name) = &args.kwarg {
-            self.scan_parameter_annotation(name)?;
-        }
-
-        self.enter_scope();
-
-        // Fill scope with parameter names:
-        self.scan_parameters(&args.args)?;
-        self.scan_parameters(&args.kwonlyargs)?;
-        if let ast::Varargs::Named(name) = &args.vararg {
-            self.scan_parameter(name)?;
-        }
-        if let ast::Varargs::Named(name) = &args.kwarg {
-            self.scan_parameter(name)?;
-        }
-        Ok(())
-    }
-
-    fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult {
-        match group {
-            ast::StringGroup::Constant { .. } => {}
-            ast::StringGroup::FormattedValue { value, .. } => {
-                self.scan_expression(value)?;
-            }
-            ast::StringGroup::Joined { values } => {
-                for subgroup in values {
-                    self.scan_string_group(subgroup)?;
-                }
-            }
-        }
-        Ok(())
-    }
-
-    #[allow(clippy::single_match)]
-    fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult {
-        let scope_depth = self.scopes.len();
-        let current_scope = self.scopes.last_mut().unwrap();
-        let location = Default::default();
-        if current_scope.symbols.contains_key(name) {
-            // Role already set..
-            match role {
-                SymbolRole::Global => {
-                    return Err(SymbolTableError {
-                        error: format!("name '{}' is used prior to global declaration", name),
-                        location,
-                    })
-                }
-                SymbolRole::Nonlocal => {
-                    return Err(SymbolTableError {
-                        error: format!("name '{}' is used prior to nonlocal declaration", name),
-                        location,
-                    })
-                }
-                _ => {
-                    // Ok?
-                }
-            }
-        } else {
-            match role {
-                SymbolRole::Nonlocal => {
-                    if scope_depth < 2 {
-                        return Err(SymbolTableError {
-                            error: format!("cannot define nonlocal '{}' at top level.", name),
-                            location,
-                        });
-                    }
-                }
-                _ => {
-                    // Ok!
-                }
-            }
-            current_scope.symbols.insert(name.to_string(), role);
-        }
-        Ok(())
-    }
-}

derive/Cargo.toml

@@ -1,20 +0,0 @@
-[package]
-name = "rustpython-derive"
-version = "0.1.0"
-description = "Rust language extensions and macros specific to rustpython."
-authors = ["RustPython Team"]
-repository = "https://github.com/RustPython/RustPython"
-license = "MIT"
-edition = "2018"
-
-[lib]
-proc-macro = true
-
-[dependencies]
-syn = { version = "0.15.29", features = ["full"] }
-quote = "0.6.11"
-proc-macro2 = "0.4.27"
-rustpython-compiler = { path = "../compiler", version = "0.1.0" }
-rustpython-bytecode = { path = "../bytecode", version = "0.1.0" }
-bincode =  "1.1"
-proc-macro-hack = "0.5"

derive/src/compile_bytecode.rs

@@ -1,174 +0,0 @@
-//! Parsing and processing for this form:
-//! ```ignore
-//! py_compile_input!(
-//!     // either:
-//!     source = "python_source_code",
-//!     // or
-//!     file = "file/path/relative/to/$CARGO_MANIFEST_DIR",
-//!
-//!     // the mode to compile the code in
-//!     mode = "exec", // or "eval" or "single"
-//!     // the path put into the CodeObject, defaults to "frozen"
-//!     module_name = "frozen",
-//! )
-//! ```
-
-use crate::{extract_spans, Diagnostic};
-use bincode;
-use proc_macro2::{Span, TokenStream as TokenStream2};
-use quote::quote;
-use rustpython_bytecode::bytecode::CodeObject;
-use rustpython_compiler::compile;
-use std::env;
-use std::fs;
-use std::path::PathBuf;
-use syn::parse::{Parse, ParseStream, Result as ParseResult};
-use syn::{self, parse2, Lit, LitByteStr, Meta, Token};
-
-enum CompilationSourceKind {
-    File(PathBuf),
-    SourceCode(String),
-}
-
-struct CompilationSource {
-    kind: CompilationSourceKind,
-    span: (Span, Span),
-}
-
-impl CompilationSource {
-    fn compile(self, mode: &compile::Mode, module_name: String) -> Result<CodeObject, Diagnostic> {
-        let compile = |source| {
-            compile::compile(source, mode, module_name).map_err(|err| {
-                Diagnostic::spans_error(self.span, format!("Compile error: {}", err))
-            })
-        };
-
-        match &self.kind {
-            CompilationSourceKind::File(rel_path) => {
-                let mut path = PathBuf::from(
-                    env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"),
-                );
-                path.push(rel_path);
-                let source = fs::read_to_string(&path).map_err(|err| {
-                    Diagnostic::spans_error(
-                        self.span,
-                        format!("Error reading file {:?}: {}", path, err),
-                    )
-                })?;
-                compile(&source)
-            }
-            CompilationSourceKind::SourceCode(code) => compile(code),
-        }
-    }
-}
-
-/// This is essentially just a comma-separated list of Meta nodes, aka the inside of a MetaList.
-struct PyCompileInput {
-    span: Span,
-    metas: Vec<Meta>,
-}
-
-impl PyCompileInput {
-    fn compile(&self) -> Result<CodeObject, Diagnostic> {
-        let mut module_name = None;
-        let mut mode = None;
-        let mut source: Option<CompilationSource> = None;
-
-        fn assert_source_empty(source: &Option<CompilationSource>) -> Result<(), Diagnostic> {
-            if let Some(source) = source {
-                Err(Diagnostic::spans_error(
-                    source.span.clone(),
-                    "Cannot have more than one source",
-                ))
-            } else {
-                Ok(())
-            }
-        }
-
-        for meta in &self.metas {
-            match meta {
-                Meta::NameValue(name_value) => {
-                    if name_value.ident == "mode" {
-                        mode = Some(match &name_value.lit {
-                            Lit::Str(s) => match s.value().as_str() {
-                                "exec" => compile::Mode::Exec,
-                                "eval" => compile::Mode::Eval,
-                                "single" => compile::Mode::Single,
-                                _ => bail_span!(s, "mode must be exec, eval, or single"),
-                            },
-                            _ => bail_span!(name_value.lit, "mode must be a string"),
-                        })
-                    } else if name_value.ident == "module_name" {
-                        module_name = Some(match &name_value.lit {
-                            Lit::Str(s) => s.value(),
-                            _ => bail_span!(name_value.lit, "module_name must be string"),
-                        })
-                    } else if name_value.ident == "source" {
-                        assert_source_empty(&source)?;
-                        let code = match &name_value.lit {
-                            Lit::Str(s) => s.value(),
-                            _ => bail_span!(name_value.lit, "source must be a string"),
-                        };
-                        source = Some(CompilationSource {
-                            kind: CompilationSourceKind::SourceCode(code),
-                            span: extract_spans(&name_value).unwrap(),
-                        });
-                    } else if name_value.ident == "file" {
-                        assert_source_empty(&source)?;
-                        let path = match &name_value.lit {
-                            Lit::Str(s) => PathBuf::from(s.value()),
-                            _ => bail_span!(name_value.lit, "source must be a string"),
-                        };
-                        source = Some(CompilationSource {
-                            kind: CompilationSourceKind::File(path),
-                            span: extract_spans(&name_value).unwrap(),
-                        });
-                    }
-                }
-                _ => {}
-            }
-        }
-
-        source
-            .ok_or_else(|| {
-                Diagnostic::span_error(
-                    self.span.clone(),
-                    "Must have either file or source in py_compile_bytecode!()",
-                )
-            })?
-            .compile(
-                &mode.unwrap_or(compile::Mode::Exec),
-                module_name.unwrap_or_else(|| "frozen".to_string()),
-            )
-    }
-}
-
-impl Parse for PyCompileInput {
-    fn parse(input: ParseStream) -> ParseResult<Self> {
-        let span = input.cursor().span();
-        let metas = input
-            .parse_terminated::<Meta, Token![,]>(Meta::parse)?
-            .into_iter()
-            .collect();
-        Ok(PyCompileInput { span, metas })
-    }
-}
-
-pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Diagnostic> {
-    let input: PyCompileInput = parse2(input)?;
-
-    let code_obj = input.compile()?;
-
-    let bytes = bincode::serialize(&code_obj).expect("Failed to serialize");
-    let bytes = LitByteStr::new(&bytes, Span::call_site());
-
-    let output = quote! {
-        ({
-            use ::rustpython_vm::__exports::bincode;
-            bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(#bytes)
-                .expect("Deserializing CodeObject failed")
-        })
-    };
-
-    Ok(output)
-}

derive/src/error.rs

@@ -1,167 +0,0 @@
-// Taken from https://github.com/rustwasm/wasm-bindgen/blob/master/crates/backend/src/error.rs
-//
-// Copyright (c) 2014 Alex Crichton
-//
-// Permission is hereby granted, free of charge, to any
-// person obtaining a copy of this software and associated
-// documentation files (the "Software"), to deal in the
-// Software without restriction, including without
-// limitation the rights to use, copy, modify, merge,
-// publish, distribute, sublicense, and/or sell copies of
-// the Software, and to permit persons to whom the Software
-// is furnished to do so, subject to the following
-// conditions:
-//
-// The above copyright notice and this permission notice
-// shall be included in all copies or substantial portions
-// of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
-// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
-// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
-// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
-// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-// DEALINGS IN THE SOFTWARE.
-#![allow(dead_code)]
-
-use proc_macro2::*;
-use quote::{ToTokens, TokenStreamExt};
-use syn::parse::Error;
-
-macro_rules! err_span {
-    ($span:expr, $($msg:tt)*) => (
-        $crate::Diagnostic::spanned_error(&$span, format!($($msg)*))
-    )
-}
-
-macro_rules! bail_span {
-    ($($t:tt)*) => (
-        return Err(err_span!($($t)*).into())
-    )
-}
-
-macro_rules! push_err_span {
-    ($diagnostics:expr, $($t:tt)*) => {
-        $diagnostics.push(err_span!($($t)*))
-    };
-}
-
-#[derive(Debug)]
-pub struct Diagnostic {
-    inner: Repr,
-}
-
-#[derive(Debug)]
-enum Repr {
-    Single {
-        text: String,
-        span: Option<(Span, Span)>,
-    },
-    SynError(Error),
-    Multi {
-        diagnostics: Vec<Diagnostic>,
-    },
-}
-
-impl Diagnostic {
-    pub fn error<T: Into<String>>(text: T) -> Diagnostic {
-        Diagnostic {
-            inner: Repr::Single {
-                text: text.into(),
-                span: None,
-            },
-        }
-    }
-
-    pub fn span_error<T: Into<String>>(span: Span, text: T) -> Diagnostic {
-        Diagnostic {
-            inner: Repr::Single {
-                text: text.into(),
-                span: Some((span, span)),
-            },
-        }
-    }
-
-    pub fn spans_error<T: Into<String>>(spans: (Span, Span), text: T) -> Diagnostic {
-        Diagnostic {
-            inner: Repr::Single {
-                text: text.into(),
-                span: Some(spans),
-            },
-        }
-    }
-
-    pub fn spanned_error<T: Into<String>>(node: &ToTokens, text: T) -> Diagnostic {
-        Diagnostic {
-            inner: Repr::Single {
-                text: text.into(),
-                span: extract_spans(node),
-            },
-        }
-    }
-
-    pub fn from_vec(diagnostics: Vec<Diagnostic>) -> Result<(), Diagnostic> {
-        if diagnostics.len() == 0 {
-            Ok(())
-        } else {
-            Err(Diagnostic {
-                inner: Repr::Multi { diagnostics },
-            })
-        }
-    }
-
-    #[allow(unconditional_recursion)]
-    pub fn panic(&self) -> ! {
-        match &self.inner {
-            Repr::Single { text, .. } => panic!("{}", text),
-            Repr::SynError(error) => panic!("{}", error),
-            Repr::Multi { diagnostics } => diagnostics[0].panic(),
-        }
-    }
-}
-
-impl From<Error> for Diagnostic {
-    fn from(err: Error) -> Diagnostic {
-        Diagnostic {
-            inner: Repr::SynError(err),
-        }
-    }
-}
-
-pub fn extract_spans(node: &dyn ToTokens) -> Option<(Span, Span)> {
-    let mut t = TokenStream::new();
-    node.to_tokens(&mut t);
-    let mut tokens = t.into_iter();
-    let start = tokens.next().map(|t| t.span());
-    let end = tokens.last().map(|t| t.span());
-    start.map(|start| (start, end.unwrap_or(start)))
-}
-
-impl ToTokens for Diagnostic {
-    fn to_tokens(&self, dst: &mut TokenStream) {
-        match &self.inner {
-            Repr::Single { text, span } => {
-                let cs2 = (Span::call_site(), Span::call_site());
-                let (start, end) = span.unwrap_or(cs2);
-                dst.append(Ident::new("compile_error", start));
-                dst.append(Punct::new('!', Spacing::Alone));
-                let mut message = TokenStream::new();
-                message.append(Literal::string(text));
-                let mut group = Group::new(Delimiter::Brace, message);
-                group.set_span(end);
-                dst.append(group);
-            }
-            Repr::Multi { diagnostics } => {
-                for diagnostic in diagnostics {
-                    diagnostic.to_tokens(dst);
-                }
-            }
-            Repr::SynError(err) => {
-                err.to_compile_error().to_tokens(dst);
-            }
-        }
-    }
-}

derive/src/from_args.rs

@@ -1,221 +0,0 @@
-use crate::Diagnostic;
-use proc_macro2::TokenStream as TokenStream2;
-use quote::quote;
-use syn::{
-    parse_quote, Attribute, Data, DeriveInput, Expr, Field, Fields, Ident, Lit, Meta, NestedMeta,
-};
-
-/// The kind of the python parameter, this corresponds to the value of Parameter.kind
-/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind)
-enum ParameterKind {
-    PositionalOnly,
-    PositionalOrKeyword,
-    KeywordOnly,
-}
-
-impl ParameterKind {
-    fn from_ident(ident: &Ident) -> Option<ParameterKind> {
-        if ident == "positional_only" {
-            Some(ParameterKind::PositionalOnly)
-        } else if ident == "positional_or_keyword" {
-            Some(ParameterKind::PositionalOrKeyword)
-        } else if ident == "keyword_only" {
-            Some(ParameterKind::KeywordOnly)
-        } else {
-            None
-        }
-    }
-}
-
-struct ArgAttribute {
-    kind: ParameterKind,
-    default: Option<Expr>,
-    optional: bool,
-}
-
-impl ArgAttribute {
-    fn from_attribute(attr: &Attribute) -> Option<Result<ArgAttribute, Diagnostic>> {
-        if !attr.path.is_ident("pyarg") {
-            return None;
-        }
-        let inner = move || match attr.parse_meta()? {
-            Meta::List(list) => {
-                let mut iter = list.nested.iter();
-                let first_arg = iter.next().ok_or_else(|| {
-                    err_span!(list, "There must be at least one argument to #[pyarg()]")
-                })?;
-                let kind = match first_arg {
-                    NestedMeta::Meta(Meta::Word(ident)) => ParameterKind::from_ident(ident),
-                    _ => None,
-                };
-                let kind = kind.ok_or_else(|| {
-                    err_span!(
-                        first_arg,
-                        "The first argument to #[pyarg()] must be the parameter type, either \
-                         'positional_only', 'positional_or_keyword', or 'keyword_only'."
-                    )
-                })?;
-
-                let mut attribute = ArgAttribute {
-                    kind,
-                    default: None,
-                    optional: false,
-                };
-
-                while let Some(arg) = iter.next() {
-                    attribute.parse_argument(arg)?;
-                }
-
-                if attribute.default.is_some() && attribute.optional {
-                    bail_span!(attr, "Can't set both a default value and optional");
-                }
-
-                Ok(attribute)
-            }
-            _ => bail_span!(attr, "pyarg must be a list, like #[pyarg(...)]"),
-        };
-        Some(inner())
-    }
-
-    fn parse_argument(&mut self, arg: &NestedMeta) -> Result<(), Diagnostic> {
-        match arg {
-            NestedMeta::Meta(Meta::Word(ident)) => {
-                if ident == "default" {
-                    if self.default.is_some() {
-                        bail_span!(ident, "Default already set");
-                    }
-                    let expr = parse_quote!(Default::default());
-                    self.default = Some(expr);
-                } else if ident == "optional" {
-                    self.optional = true;
-                } else {
-                    bail_span!(ident, "Unrecognised pyarg attribute");
-                }
-            }
-            NestedMeta::Meta(Meta::NameValue(name_value)) => {
-                if name_value.ident == "default" {
-                    if self.default.is_some() {
-                        bail_span!(name_value, "Default already set");
-                    }
-
-                    match name_value.lit {
-                        Lit::Str(ref val) => {
-                            let expr = val.parse::<Expr>().map_err(|_| {
-                                err_span!(val, "Expected a valid expression for default argument")
-                            })?;
-                            self.default = Some(expr);
-                        }
-                        _ => bail_span!(name_value, "Expected string value for default argument"),
-                    }
-                } else if name_value.ident == "optional" {
-                    match name_value.lit {
-                        Lit::Bool(ref val) => {
-                            self.optional = val.value;
-                        }
-                        _ => bail_span!(
-                            name_value.lit,
-                            "Expected boolean value for optional argument"
-                        ),
-                    }
-                } else {
-                    bail_span!(name_value, "Unrecognised pyarg attribute");
-                }
-            }
-            _ => bail_span!(arg, "Unrecognised pyarg attribute"),
-        }
-
-        Ok(())
-    }
-}
-
-fn generate_field(field: &Field) -> Result<TokenStream2, Diagnostic> {
-    let mut pyarg_attrs = field
-        .attrs
-        .iter()
-        .filter_map(ArgAttribute::from_attribute)
-        .collect::<Result<Vec<_>, _>>()?;
-    let attr = if pyarg_attrs.is_empty() {
-        ArgAttribute {
-            kind: ParameterKind::PositionalOrKeyword,
-            default: None,
-            optional: false,
-        }
-    } else if pyarg_attrs.len() == 1 {
-        pyarg_attrs.remove(0)
-    } else {
-        bail_span!(field, "Multiple pyarg attributes on field");
-    };
-
-    let name = &field.ident;
-    let middle = quote! {
-        .map(|x| ::rustpython_vm::pyobject::TryFromObject::try_from_object(vm, x)).transpose()?
-    };
-    let ending = if let Some(default) = attr.default {
-        quote! {
-            .unwrap_or_else(|| #default)
-        }
-    } else if attr.optional {
-        quote! {
-            .map(::rustpython_vm::function::OptionalArg::Present)
-            .unwrap_or(::rustpython_vm::function::OptionalArg::Missing)
-        }
-    } else {
-        let err = match attr.kind {
-            ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! {
-                ::rustpython_vm::function::ArgumentError::TooFewArgs
-            },
-            ParameterKind::KeywordOnly => quote! {
-                ::rustpython_vm::function::ArgumentError::RequiredKeywordArgument(tringify!(#name))
-            },
-        };
-        quote! {
-            .ok_or_else(|| #err)?
-        }
-    };
-
-    let file_output = match attr.kind {
-        ParameterKind::PositionalOnly => {
-            quote! {
-                #name: args.take_positional()#middle#ending,
-            }
-        }
-        ParameterKind::PositionalOrKeyword => {
-            quote! {
-                #name: args.take_positional_keyword(stringify!(#name))#middle#ending,
-            }
-        }
-        ParameterKind::KeywordOnly => {
-            quote! {
-                #name: args.take_keyword(stringify!(#name))#middle#ending,
-            }
-        }
-    };
-    Ok(file_output)
-}
-
-pub fn impl_from_args(input: DeriveInput) -> Result<TokenStream2, Diagnostic> {
-    let fields = match input.data {
-        Data::Struct(syn::DataStruct {
-            fields: Fields::Named(fields),
-            ..
-        }) => fields
-            .named
-            .iter()
-            .map(generate_field)
-            .collect::<Result<TokenStream2, Diagnostic>>()?,
-        _ => bail_span!(input, "FromArgs input must be a struct with named fields"),
-    };
-
-    let name = input.ident;
-    let output = quote! {
-        impl ::rustpython_vm::function::FromArgs for #name {
-            fn from_args(
-                vm: &::rustpython_vm::VirtualMachine,
-                args: &mut ::rustpython_vm::function::PyFuncArgs
-            ) -> Result<Self, ::rustpython_vm::function::ArgumentError> {
-                Ok(#name { #fields })
-            }
-        }
-    };
-    Ok(output)
-}

derive/src/lib.rs

@@ -1,55 +0,0 @@
-#![recursion_limit = "128"]
-
-extern crate proc_macro;
-
-#[macro_use]
-mod error;
-mod compile_bytecode;
-mod from_args;
-mod pyclass;
-
-use error::{extract_spans, Diagnostic};
-use proc_macro::TokenStream;
-use proc_macro2::TokenStream as TokenStream2;
-use proc_macro_hack::proc_macro_hack;
-use quote::ToTokens;
-use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item};
-
-fn result_to_tokens(result: Result<TokenStream2, Diagnostic>) -> TokenStream {
-    match result {
-        Ok(tokens) => tokens.into(),
-        Err(diagnostic) => diagnostic.into_token_stream().into(),
-    }
-}
-
-#[proc_macro_derive(FromArgs, attributes(pyarg))]
-pub fn derive_from_args(input: TokenStream) -> TokenStream {
-    let input = parse_macro_input!(input as DeriveInput);
-    result_to_tokens(from_args::impl_from_args(input))
-}
-
-#[proc_macro_attribute]
-pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream {
-    let attr = parse_macro_input!(attr as AttributeArgs);
-    let item = parse_macro_input!(item as Item);
-    result_to_tokens(pyclass::impl_pyclass(attr, item))
-}
-
-#[proc_macro_attribute]
-pub fn pyimpl(attr: TokenStream, item: TokenStream) -> TokenStream {
-    let attr = parse_macro_input!(attr as AttributeArgs);
-    let item = parse_macro_input!(item as Item);
-    result_to_tokens(pyclass::impl_pyimpl(attr, item))
-}
-
-#[proc_macro_attribute]
-pub fn pystruct_sequence(attr: TokenStream, item: TokenStream) -> TokenStream {
-    let attr = parse_macro_input!(attr as AttributeArgs);
-    let item = parse_macro_input!(item as Item);
-    result_to_tokens(pyclass::impl_pystruct_sequence(attr, item))
-}
-
-#[proc_macro_hack]
-pub fn py_compile_bytecode(input: TokenStream) -> TokenStream {
-    result_to_tokens(compile_bytecode::impl_py_compile_bytecode(input.into()))
-}

derive/src/pyclass.rs

@@ -1,463 +0,0 @@
-use super::Diagnostic;
-use proc_macro2::TokenStream as TokenStream2;
-use quote::quote;
-use std::collections::HashMap;
-use syn::{
-    Attribute, AttributeArgs, Ident, ImplItem, Index, Item, Lit, Meta, MethodSig, NestedMeta,
-};
-
-enum ClassItem {
-    Method {
-        item_ident: Ident,
-        py_name: String,
-    },
-    ClassMethod {
-        item_ident: Ident,
-        py_name: String,
-    },
-    Property {
-        item_ident: Ident,
-        py_name: String,
-        setter: bool,
-    },
-}
-
-fn meta_to_vec(meta: Meta) -> Result<Vec<NestedMeta>, Meta> {
-    match meta {
-        Meta::Word(_) => Ok(Vec::new()),
-        Meta::List(list) => Ok(list.nested.into_iter().collect()),
-        Meta::NameValue(_) => Err(meta),
-    }
-}
-
-impl ClassItem {
-    fn extract_from_syn(
-        attrs: &mut Vec<Attribute>,
-        sig: &MethodSig,
-    ) -> Result<Option<ClassItem>, Diagnostic> {
-        let mut item = None;
-        let mut attr_idx = None;
-        for (i, meta) in attrs
-            .iter()
-            .filter_map(|attr| attr.parse_meta().ok())
-            .enumerate()
-        {
-            let name = meta.name();
-            if name == "pymethod" {
-                if item.is_some() {
-                    bail_span!(
-                        sig.ident,
-                        "You can only have one #[py*] attribute on an impl item"
-                    )
-                }
-                let nesteds = meta_to_vec(meta).map_err(|meta| {
-                    err_span!(
-                        meta,
-                        "#[pymethod = \"...\"] cannot be a name/value, you probably meant \
-                         #[pymethod(name = \"...\")]",
-                    )
-                })?;
-                let mut py_name = None;
-                for meta in nesteds {
-                    let meta = match meta {
-                        NestedMeta::Meta(meta) => meta,
-                        NestedMeta::Literal(_) => continue,
-                    };
-                    match meta {
-                        Meta::NameValue(name_value) => {
-                            if name_value.ident == "name" {
-                                if let Lit::Str(s) = &name_value.lit {
-                                    py_name = Some(s.value());
-                                } else {
-                                    bail_span!(
-                                        &sig.ident,
-                                        "#[pymethod(name = ...)] must be a string"
-                                    );
-                                }
-                            }
-                        }
-                        _ => {}
-                    }
-                }
-                item = Some(ClassItem::Method {
-                    item_ident: sig.ident.clone(),
-                    py_name: py_name.unwrap_or_else(|| sig.ident.to_string()),
-                });
-                attr_idx = Some(i);
-            } else if name == "pyclassmethod" {
-                if item.is_some() {
-                    bail_span!(
-                        sig.ident,
-                        "You can only have one #[py*] attribute on an impl item"
-                    )
-                }
-                let nesteds = meta_to_vec(meta).map_err(|meta| {
-                    err_span!(
-                        meta,
-                        "#[pyclassmethod = \"...\"] cannot be a name/value, you probably meant \
-                         #[pyclassmethod(name = \"...\")]",
-                    )
-                })?;
-                let mut py_name = None;
-                for meta in nesteds {
-                    let meta = match meta {
-                        NestedMeta::Meta(meta) => meta,
-                        NestedMeta::Literal(_) => continue,
-                    };
-                    match meta {
-                        Meta::NameValue(name_value) => {
-                            if name_value.ident == "name" {
-                                if let Lit::Str(s) = &name_value.lit {
-                                    py_name = Some(s.value());
-                                } else {
-                                    bail_span!(
-                                        &sig.ident,
-                                        "#[pyclassmethod(name = ...)] must be a string"
-                                    );
-                                }
-                            }
-                        }
-                        _ => {}
-                    }
-                }
-                item = Some(ClassItem::ClassMethod {
-                    item_ident: sig.ident.clone(),
-                    py_name: py_name.unwrap_or_else(|| sig.ident.to_string()),
-                });
-                attr_idx = Some(i);
-            } else if name == "pyproperty" {
-                if item.is_some() {
-                    bail_span!(
-                        sig.ident,
-                        "You can only have one #[py*] attribute on an impl item"
-                    )
-                }
-                let nesteds = meta_to_vec(meta).map_err(|meta| {
-                    err_span!(
-                        meta,
-                        "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \
-                         #[pyproperty(name = \"...\")]"
-                    )
-                })?;
-                let mut setter = false;
-                let mut py_name = None;
-                for meta in nesteds {
-                    let meta = match meta {
-                        NestedMeta::Meta(meta) => meta,
-                        NestedMeta::Literal(_) => continue,
-                    };
-                    match meta {
-                        Meta::NameValue(name_value) => {
-                            if name_value.ident == "name" {
-                                if let Lit::Str(s) = &name_value.lit {
-                                    py_name = Some(s.value());
-                                } else {
-                                    bail_span!(
-                                        &sig.ident,
-                                        "#[pyproperty(name = ...)] must be a string"
-                                    );
-                                }
-                            }
-                        }
-                        Meta::Word(ident) => {
-                            if ident == "setter" {
-                                setter = true;
-                            }
-                        }
-                        _ => {}
-                    }
-                }
-                let py_name = match py_name {
-                    Some(py_name) => py_name,
-                    None => {
-                        let item_ident = sig.ident.to_string();
-                        if setter {
-                            if item_ident.starts_with("set_") {
-                                let name = &item_ident["set_".len()..];
-                                if name.is_empty() {
-                                    bail_span!(
-                                        &sig.ident,
-                                        "A #[pyproperty(setter)] fn with a set_* name must \
-                                         have something after \"set_\""
-                                    )
-                                } else {
-                                    name.to_string()
-                                }
-                            } else {
-                                bail_span!(
-                                    &sig.ident,
-                                    "A #[pyproperty(setter)] fn must either have a `name` \
-                                     parameter or a fn name along the lines of \"set_*\""
-                                )
-                            }
-                        } else {
-                            item_ident
-                        }
-                    }
-                };
-                item = Some(ClassItem::Property {
-                    py_name,
-                    item_ident: sig.ident.clone(),
-                    setter,
-                });
-                attr_idx = Some(i);
-            }
-        }
-        if let Some(attr_idx) = attr_idx {
-            attrs.remove(attr_idx);
-        }
-        Ok(item)
-    }
-}
-
-pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> Result<TokenStream2, Diagnostic> {
-    let mut imp = if let Item::Impl(imp) = item {
-        imp
-    } else {
-        return Ok(quote!(#item));
-    };
-
-    let mut diagnostics: Vec<Diagnostic> = Vec::new();
-
-    let items = imp
-        .items
-        .iter_mut()
-        .filter_map(|item| {
-            if let ImplItem::Method(meth) = item {
-                ClassItem::extract_from_syn(&mut meth.attrs, &meth.sig)
-                    .map_err(|err| diagnostics.push(err))
-                    .unwrap_or_default()
-            } else {
-                None
-            }
-        })
-        .collect::<Vec<_>>();
-    let ty = &imp.self_ty;
-    let mut properties: HashMap<&str, (Option<&Ident>, Option<&Ident>)> = HashMap::new();
-    for item in items.iter() {
-        match item {
-            ClassItem::Property {
-                ref item_ident,
-                ref py_name,
-                setter,
-            } => {
-                let entry = properties.entry(py_name).or_default();
-                let func = if *setter { &mut entry.1 } else { &mut entry.0 };
-                if func.is_some() {
-                    bail_span!(
-                        item_ident,
-                        "Multiple property accessors with name {:?}",
-                        py_name
-                    )
-                }
-                *func = Some(item_ident);
-            }
-            _ => {}
-        }
-    }
-    let methods = items.iter().filter_map(|item| match item {
-        ClassItem::Method {
-            item_ident,
-            py_name,
-        } => Some(quote! {
-            class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_ident));
-        }),
-        ClassItem::ClassMethod {
-            item_ident,
-            py_name,
-        } => Some(quote! {
-            class.set_str_attr(#py_name, ctx.new_classmethod(Self::#item_ident));
-        }),
-        _ => None,
-    });
-    let properties = properties
-        .iter()
-        .map(|(name, prop)| {
-            let getter = match prop.0 {
-                Some(getter) => getter,
-                None => {
-                    push_err_span!(
-                        diagnostics,
-                        prop.1.unwrap(),
-                        "Property {:?} is missing a getter",
-                        name
-                    );
-                    return TokenStream2::new();
-                }
-            };
-            let add_setter = prop.1.map(|setter| quote!(.add_setter(Self::#setter)));
-            quote! {
-                class.set_str_attr(
-                    #name,
-                    ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx)
-                        .add_getter(Self::#getter)
-                        #add_setter
-                        .create(),
-                );
-            }
-        })
-        .collect::<Vec<_>>();
-
-    Diagnostic::from_vec(diagnostics)?;
-
-    let ret = quote! {
-        #imp
-        impl ::rustpython_vm::pyobject::PyClassImpl for #ty {
-            fn impl_extend_class(
-                ctx: &::rustpython_vm::pyobject::PyContext,
-                class: &::rustpython_vm::obj::objtype::PyClassRef,
-            ) {
-                #(#methods)*
-                #(#properties)*
-            }
-        }
-    };
-    Ok(ret)
-}
-
-fn generate_class_def(
-    ident: &Ident,
-    attr_name: &'static str,
-    attr: AttributeArgs,
-    attrs: &Vec<Attribute>,
-) -> Result<TokenStream2, Diagnostic> {
-    let mut class_name = None;
-    for attr in attr {
-        if let NestedMeta::Meta(meta) = attr {
-            if let Meta::NameValue(name_value) = meta {
-                if name_value.ident == "name" {
-                    if let Lit::Str(s) = name_value.lit {
-                        class_name = Some(s.value());
-                    } else {
-                        bail_span!(
-                            name_value.lit,
-                            "#[{}(name = ...)] must be a string",
-                            attr_name
-                        );
-                    }
-                }
-            }
-        }
-    }
-    let class_name = class_name.unwrap_or_else(|| ident.to_string());
-
-    let mut doc: Option<Vec<String>> = None;
-    for attr in attrs.iter() {
-        if attr.path.is_ident("doc") {
-            let meta = attr.parse_meta().expect("expected doc attr to be a meta");
-            if let Meta::NameValue(name_value) = meta {
-                if let Lit::Str(s) = name_value.lit {
-                    let val = s.value().trim().to_string();
-                    match doc {
-                        Some(ref mut doc) => doc.push(val),
-                        None => doc = Some(vec![val]),
-                    }
-                }
-            }
-        }
-    }
-    let doc = match doc {
-        Some(doc) => {
-            let doc = doc.join("\n");
-            quote!(Some(#doc))
-        }
-        None => quote!(None),
-    };
-
-    let ret = quote! {
-        impl ::rustpython_vm::pyobject::PyClassDef for #ident {
-            const NAME: &'static str = #class_name;
-            const DOC: Option<&'static str> = #doc;
-        }
-    };
-    Ok(ret)
-}
-
-pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result<TokenStream2, Diagnostic> {
-    let (item, ident, attrs) = match item {
-        Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs),
-        Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs),
-        other => bail_span!(
-            other,
-            "#[pyclass] can only be on a struct or enum declaration"
-        ),
-    };
-
-    let class_def = generate_class_def(&ident, "pyclass", attr, &attrs)?;
-
-    let ret = quote! {
-        #item
-        #class_def
-    };
-    Ok(ret)
-}
-
-pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result<TokenStream2, Diagnostic> {
-    let struc = if let Item::Struct(struc) = item {
-        struc
-    } else {
-        bail_span!(
-            item,
-            "#[pystruct_sequence] can only be on a struct declaration"
-        )
-    };
-    let class_def = generate_class_def(&struc.ident, "pystruct_sequence", attr, &struc.attrs)?;
-    let mut properties = Vec::new();
-    let mut field_names = Vec::new();
-    for (i, field) in struc.fields.iter().enumerate() {
-        let idx = Index::from(i);
-        if let Some(ref field_name) = field.ident {
-            let field_name_str = field_name.to_string();
-            // TODO add doc to the generated property
-            let property = quote! {
-                class.set_str_attr(
-                    #field_name_str,
-                    ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx)
-                        .add_getter(|zelf: &::rustpython_vm::obj::objtuple::PyTuple,
-                                     _vm: &::rustpython_vm::vm::VirtualMachine|
-                                     zelf.fast_getitem(#idx))
-                        .create(),
-                );
-            };
-            properties.push(property);
-            field_names.push(quote!(#field_name));
-        } else {
-            field_names.push(quote!(#idx));
-        }
-    }
-
-    let ty = &struc.ident;
-    let ret = quote! {
-        #struc
-        #class_def
-        impl #ty {
-            fn into_struct_sequence(&self,
-                vm: &::rustpython_vm::vm::VirtualMachine,
-                cls: ::rustpython_vm::obj::objtype::PyClassRef,
-            ) -> ::rustpython_vm::pyobject::PyResult<::rustpython_vm::obj::objtuple::PyTupleRef> {
-                let tuple: ::rustpython_vm::obj::objtuple::PyTuple =
-                    vec![#(::rustpython_vm::pyobject::IntoPyObject
-                                ::into_pyobject(self.#field_names, vm)?
-                         ),*].into();
-                ::rustpython_vm::pyobject::PyValue::into_ref_with_type(tuple, vm, cls)
-            }
-        }
-        impl ::rustpython_vm::pyobject::PyClassImpl for #ty {
-            fn impl_extend_class(
-                ctx: &::rustpython_vm::pyobject::PyContext,
-                class: &::rustpython_vm::obj::objtype::PyClassRef,
-            ) {
-                #(#properties)*
-            }
-
-            fn make_class(
-                ctx: &::rustpython_vm::pyobject::PyContext
-            ) -> ::rustpython_vm::obj::objtype::PyClassRef {
-                let py_class = ctx.new_class(<Self as ::rustpython_vm::pyobject::PyClassDef>::NAME, ctx.tuple_type());
-                Self::extend_class(ctx, &py_class);
-                py_class
-            }
-        }
-    };
-    Ok(ret)
-}

docs/builtins.md

@@ -0,0 +1,30 @@
+Byterun
+
+* Builtins are exposted to frame.f_builtins
+* f_builtins is assigned during frame creation,
+    self.f_builtins = f_locals['__builtins__']
+    if hasattr(self.f_builtins, '__dict__'):
+      self.f_builtins = self.f_builtins.__dict__
+* f_locals has a __`____builtins___` field which is directly the `__builtins__` module
+
+
+Jaspy
+
+* The `module()` function creates either a NativeModule or PythonModule
+* The objects in the module are PyType.native
+* The function call is abstracted as a `call` function, which handles different
+
+* IMPORT_NAME depends on `__import__()` in builtins
+
+TODO:
+
+* Implement a new type NativeFunction
+* Wrap a function pointer in NativeFunction
+* Refactor the CALL_FUNCTION case so it can call both python function and native function
+* During frame creation, force push a nativefunction `print` into the namespace
+* Modify LOAD_* so they can search for names in builtins
+
+* Create a module type
+* In VM initialization, load the builtins module into locals
+* During frame creation, create a field that conatins the builtins dict
+

docs/compat-test.md

@@ -0,0 +1,17 @@
+https://wiki.python.org/moin/SimplePrograms
+Simple HTTP Server
+bottle.py
+http://pypi-ranking.info/alltime
+ - simplejson
+ - pep8
+ - httplib2
+ - argparse
+
+http://learning-python.com/books/lp4e-examples.html
+
+https://docs.python.org/3/tutorial/introduction.html#numbers
+
+http://doc.pypy.org/en/latest/getting-started-dev.html#running-pypy-s-unit-tests
+
+CPython Regression suite
+https://github.com/python/cpython/tree/master/Lib/test

docs/frameTODO.txt

@@ -0,0 +1,13 @@
+Create a frame class
+change the environment to locals
+turn run into run_code -> run_frame
+
+TEST
+
+Create a function class
+implement MAKE_FUNCTION
+implement CALL_FUNCTION
+implement RETURN_VALUE
+
+TEST function
+

docs/notes.md

@@ -0,0 +1,13 @@
+# TODO: other notes should be put into here
+
+# getattr()
+- Required by this opcode [LOAD_ATTR](https://docs.python.org/3/library/dis.html#opcode-LOAD_ATTR)
+- The builtin function: https://docs.python.org/3/library/functions.html?highlight=getattr#getattr
+- 
+
+# Memory management
+- https://docs.python.org/3.6/c-api/memory.html
+
+# Bootstraping
+- http://doc.pypy.org/en/latest/coding-guide.html#our-runtime-interpreter-is-rpython
+- http://www.aosabook.org/en/pypy.html

docs/reference.md

@@ -0,0 +1,5 @@
+Python bytecode: https://docs.python.org/3.4/library/dis.html 
+Python VM in python: https://github.com/nedbat/byterun
+Python VM in JS: https://github.com/koehlma/jaspy
+
+http://www.skulpt.org/

docs/sharing.md

@@ -0,0 +1,43 @@
+# Sharing plan
+
+## Topics
+
+* Python architecture
+  * Compiler -> VM
+* Flavors
+  * Cpython
+  * Pypy
+  * Byterun
+  * JSapy (?)
+* Why Rust 
+  * Security 
+  * Learning
+* Implementation plan
+  * VM first
+  * Compiler second
+
+* Tools for study
+  * dis doc
+  * byterun doc
+  * bytrun code
+  * cpython source
+
+---
+* Python VM
+  * Stack machine
+
+*  Load add print
+  * dis
+  * Interpreter loop
+  * Python Types
+
+* Control flow
+  * Jump
+  * If
+  * Loop
+
+---
+* Function call
+  * Frame
+* Builtins
+

docs/slides/201708_COSCUP/slide.md

@@ -0,0 +1,286 @@
+class: center, middle
+##Python Interpreter in Rust
+#### 2017/8/6, COSCUP
+#### Shing Lyu
+
+
+???
+top, middle, bottom
+left, center, right
+
+---
+### About Me
+* 呂行 Shing Lyu
+* Mozilla engineer
+* Servo team
+
+![rust_and_servo](pic/rust-servo.png)
+---
+### Python's architecture
+* Interpreted
+* Garbage Collected
+* Compiler => bytecode => VM
+
+![python](pic/python-logo.png)
+
+---
+background-image: url('pic/ice-cream.jpg')
+class: center, middle, bleed, text-bg
+# Flavors
+
+
+---
+### Python Flavors
+* CPython (THE python)
+* Jython (JVM)
+* IronPython (.NET)
+* Pypy
+* Educational
+  * Byterun
+  * Jsapy (JS)
+  * Brython (Python in browser)
+
+---
+class: center, middle
+
+# Why & How?
+---
+### Why rewriting Python in Rust?
+* Memory safety (?)
+* Learn about Python's internal
+* Learn to write Rust from scratch
+* FUN!
+
+---
+### Implementation strategy
+* Mostly follow CPython 3.6
+* Focus on the VM first, then the compiler
+  * Use the Python built-in compiler to generate bytecode
+* Focus on learning rather than usability
+
+---
+### Milestones
+* Basic arithmetics
+* Variables
+* Control flows (require JUMP)
+* Function call (require call stack)
+* Built-in functions (require native code)
+* Run Python tutorial example code <= We're here
+* Exceptions
+* GC
+* Run popular libraries
+
+
+---
+class: center, middle, bleed, text-bg
+background-image: url('pic/car_cutaway.jpg')
+# Python Internals
+
+---
+### How Python VM works
+* Stack machine
+  * Call stack and frames
+  * Has a NAMES list and CONSTS list
+  * Has a STACK as workspace
+* Accepts Python bytecode
+* `python -m dis source.py`
+
+---
+
+### A simple Python code
+
+```
+#!/usr/bin/env python3
+print(1+1)
+```
+
+Running `python3 -m dis source.py` gives us
+
+```
+  1    0 LOAD_NAME           0 (print)
+       3 LOAD_CONST          2 (2)
+       6 CALL_FUNCTION       1 (1 positional, 0 keyword pair)
+       9 POP_TOP
+      10 LOAD_CONST          1 (None)
+      13 RETURN_VALUE
+
+```
+
+---
+
+### LOAD_NAME "print"
+* NAMES = ["print"]
+* CONSTS = [None, 2]
+* STACK:
+
+```
+ |                    |
+ | print (native code)|
+ +--------------------+
+```
+---
+### LOAD_CONST 2
+* NAMES = ["print"]
+* CONSTS = [None, 2]
+* STACK:
+
+```
+ |                    |
+ |          2         |
+ | print (native code)|
+ +--------------------+
+```
+
+---
+
+### CALL_FUNCTION 1
+1. `argument = stack.pop()` (argument == 2)
+2. `function = stack.top()` (function == print)
+3. call `print(2)`
+
+* NAMES = ["print"]
+* CONSTS = [None, 2]
+* STACK:
+
+```
+ |                    |
+ | print (native code)|
+ +--------------------+
+```
+---
+### POP_TOP
+* NAMES = ["print"]
+* CONSTS = [None, 2]
+* STACK:
+
+```
+ |                    |
+ |     (empty)        |
+ +--------------------+
+```
+
+---
+### LOAD_CONST None
+* NAMES = ["print"]
+* CONSTS = [None, 2]
+* STACK:
+
+```
+ |                    |
+ |       None         |
+ +--------------------+
+```
+
+---
+### RETURN_VALUE
+
+(returns top of stack == None)
+
+---
+
+class: center, middle, bleed, text-bg
+background-image: url('pic/electronic_parts.jpg')
+# Technical Detail
+
+---
+
+### Bytecode format
+* `dis` output format is for human reader
+* Implementing a `dis` format parser is a waste of time
+* Emit JSON bytecode using the [bytecode](https://pypi.python.org/pypi/bytecode/0.5) module
+
+```
+
+code = compile(f,...)  # Python built-in, return a Code object
+
+bytecode.Bytecode()
+   .from_code(code)
+   .to_concrete_bytecode()
+```
+* Load into Rust using `serde_json`
+---
+
+### Types
+* Everything is a `PyObject` in CPython
+* We'll need that class hierarchy eventually
+* Use a Rust `enum` for now
+
+```
+pub enum NativeType{
+    NoneType,
+    Boolean(bool),
+    Int(i32),
+    Str(String),
+    Tuple(Vec<NativeType>),
+    ...
+}
+```
+
+---
+
+### Testing
+* `assert` is essential to for unittests
+* `assert` raises `AssertionError`
+* Use `panic!()` before we implement exception
+
+```
+assert 1 == 1
+```
+```
+  1           0 LOAD_CONST               0 (1)
+              3 LOAD_CONST               0 (1)
+              6 COMPARE_OP               2 (==)
+              9 POP_JUMP_IF_TRUE        18
+             12 LOAD_GLOBAL              0 (AssertionError)
+             15 RAISE_VARARGS            1
+        >>   18 LOAD_CONST               1 (None)
+             21 RETURN_VALUE
+```
+
+---
+### Native Function
+
+* e.g. `print()`
+
+```
+pub enum NativeType {
+    NativeFunction(fn(Vec<NativeType>) -> NativeType),
+    ...
+}
+
+match stack.pop() {
+    NativeFunction(func) => return_val = func(),
+    _ => ...
+}
+
+```
+
+---
+
+### Next steps
+* Exceptions
+* Make it run a small but popular tool/library
+* Implement the parser
+* Figure out garbage collection
+* Performance benchmarking
+
+---
+### Contribute
+
+## https://github.com/shinglyu/RustPython
+
+![qr_code](pic/repo_QR.png)
+
+---
+class: middle, center
+
+# Thank you
+
+---
+
+### References
+* [`dis` documentation](https://docs.python.org/3.4/library/dis.html)
+* [byterun](http://www.aosabook.org/en/500L/a-python-interpreter-written-in-python.html)
+* [byterun (GitHub)](https://github.com/nedbat/byterun/)
+* [cpython source code](https://github.com/python/cpython)
+

docs/slides/intro/slide.md

@@ -0,0 +1,180 @@
+class: center, middle
+##Python Interpreter in Rust
+###Introduction
+#### 2017/3/28
+#### Shing Lyu
+
+
+???
+top, middle, bottom
+left, center, right
+
+---
+name: toc
+###Agenda
+1. Category 
+1. Category 
+1. Category 
+1. Category 
+1. Category 
+1. Category 
+1. Category 
+
+???
+This is a template
+---
+
+### Python's architecture
+* Interpreted
+* Garbage Collected
+* Compiler => bytecode => VM
+
+---
+background-image: url('pic/ice-cream.jpg')
+class: bleed
+# Flavors
+
+
+---
+### Python Flavors
+* CPython (THE python)
+* Jython (JVM)
+* IronPython (.NET)
+* Pypy
+* Educational
+  * Byterun
+  * Jsapy (JS)
+  * Brython (Python in browser)
+
+---
+### Why rewriting Python in Rust?
+* Memory safety
+
+* Learn about Python internal
+* Learn real world Rust
+
+---
+### Implementation strategy
+* Focus on the VM first, then the compiler
+  * Reuse the Python built-in compiler to generate bytecode
+* Basic arithmetics
+* Control flows (require JUMP)
+* Function call (require call stack)
+* Built-in functions (require native code)
+* Run popular libraries
+
+
+---
+### References
+* [`dis` documentation](https://docs.python.org/3.4/library/dis.html)
+* [byterun](http://www.aosabook.org/en/500L/a-python-interpreter-written-in-python.html)
+* [byterun (GitHub)](https://github.com/nedbat/byterun/)
+* [cpython source code](https://github.com/python/cpython)
+
+---
+### How Python VM works
+* Stack machine
+* Accepts Python bytecode
+* `python -m dis source.py`
+
+---
+
+### A simple Python code
+
+```
+#!/usr/bin/env python3
+print(1+1)
+```
+
+We run `python3 -m dis source.py`
+
+---
+
+### The bytecode
+
+```
+  1    0 LOAD_NAME           0 (print)
+       3 LOAD_CONST          2 (2)
+       6 CALL_FUNCTION       1 (1 positional, 0 keyword pair)
+       9 POP_TOP
+      10 LOAD_CONST          1 (None)
+      13 RETURN_VALUE
+
+```
+
+---
+
+### LOAD_NAME "print"
+* NAMES = ["print"]
+* CONSTS = [None, 2]
+* STACK:
+
+```
+ |                    |
+ | print (native code)|
+ +--------------------+
+```
+---
+### LOAD_CONST 2
+* NAMES = ["print"]
+* CONSTS = [None, 2]
+* STACK:
+
+```
+ |                    |
+ |          2         |
+ | print (native code)|
+ +--------------------+
+```
+
+---
+
+### CALL_FUNCTION 1
+1. argument = stack.pop() (argument == 2)
+2. function = stack.top() (function == print)
+3. call print(2)
+
+* NAMES = ["print"]
+* CONSTS = [None, 2]
+* STACK:
+
+```
+ |                    |
+ | print (native code)|
+ +--------------------+
+```
+---
+### POP_TOP
+* NAMES = ["print"]
+* CONSTS = [None, 2]
+* STACK:
+
+```
+ |                    |
+ |     (empty)        |
+ +--------------------+
+```
+
+---
+### LOAD_CONST 1
+* NAMES = ["print"]
+* CONSTS = [None, 2]
+* STACK:
+
+```
+ |                    |
+ |       None         |
+ +--------------------+
+```
+
+---
+### RETURN_VALUE
+
+(returns top of stack == None)
+
+---
+
+### Next step
+* Make it run a small but popular tool/library
+* Implement the parser
+* Performance benchmarking

docs/study.md

@@ -0,0 +1,6 @@
+Topic to study
+==================
+* How to save a Rust Iterator on the stack?
+* Study how blocks are handled in CPython.
+  * The `why` var?
+  * Why unwind everything when a `break` happened?

parser/.gitignore

@@ -0,0 +1,4 @@
+src/python.rs
+target/
+Cargo.lock
+

parser/Cargo.toml

@@ -1,23 +1,16 @@
 [package]
-name = "rustpython-parser"
+name = "rustpython_parser"
-version = "0.1.0"
+version = "0.0.1"
-description = "Parser for python code."
+authors = [ "Shing Lyu", "Windel Bouwman" ]
-authors = [ "RustPython Team" ]
 build = "build.rs"
-repository = "https://github.com/RustPython/RustPython"
-license = "MIT"
-edition = "2018"
 
 [build-dependencies]
-lalrpop="0.16.3"
+lalrpop="0.15.1"
 
 [dependencies]
-lalrpop-util="0.16.3"
+lalrpop-util="0.15.1"
 log="0.4.1"
 regex="0.2.2"
 num-bigint = "0.2"
 num-traits = "0.2"
-unicode-xid = "0.1.0"
+
-unic-emoji-char = "0.9.0"
-serde = { version = "1.0.66", features = ["derive"] }
-wtf8 = "0.0.3"

parser/build.rs

@@ -1,5 +1,5 @@
-use lalrpop;
+extern crate lalrpop;
 
 fn main() {
-    lalrpop::process_root().unwrap()
+    lalrpop::process_root().unwrap();
 }

parser/src/ast.rs

@@ -4,8 +4,6 @@
 
 pub use super::lexer::Location;
 use num_bigint::BigInt;
-use serde::{Deserialize, Serialize};
-
 /*
 #[derive(Debug)]
 
@@ -18,7 +16,7 @@ pub struct Node {
 #[derive(Debug, PartialEq)]
 pub enum Top {
     Program(Program),
-    Statement(Vec<LocatedStatement>),
+    Statement(LocatedStatement),
     Expression(Expression),
 }
 
@@ -27,18 +25,12 @@ pub struct Program {
     pub statements: Vec<LocatedStatement>,
 }
 
-#[derive(Debug, PartialEq)]
-pub struct ImportSymbol {
-    pub symbol: String,
-    pub alias: Option<String>,
-}
-
 #[derive(Debug, PartialEq)]
 pub struct SingleImport {
     pub module: String,
+    // (symbol name in module, name it should be assigned locally)
+    pub symbol: Option<String>,
     pub alias: Option<String>,
-    pub symbols: Vec<ImportSymbol>,
-    pub level: usize,
 }
 
 #[derive(Debug, PartialEq)]
@@ -50,13 +42,12 @@ pub struct Located<T> {
 pub type LocatedStatement = Located<Statement>;
 
 /// Abstract syntax tree nodes for python statements.
-#[allow(clippy::large_enum_variant)]
 #[derive(Debug, PartialEq)]
 pub enum Statement {
     Break,
     Continue,
     Return {
-        value: Option<Box<Expression>>,
+        value: Option<Vec<Expression>>,
     },
     Import {
         import_parts: Vec<SingleImport>,
@@ -74,9 +65,9 @@ pub enum Statement {
         value: Expression,
     },
     AugAssign {
-        target: Box<Expression>,
+        target: Expression,
         op: Operator,
-        value: Box<Expression>,
+        value: Expression,
     },
     Expression {
         expression: Expression,
@@ -103,13 +94,7 @@ pub enum Statement {
     },
     For {
         target: Expression,
-        iter: Expression,
+        iter: Vec<Expression>,
-        body: Vec<LocatedStatement>,
-        orelse: Option<Vec<LocatedStatement>>,
-    },
-    AsyncFor {
-        target: Expression,
-        iter: Expression,
         body: Vec<LocatedStatement>,
         orelse: Option<Vec<LocatedStatement>>,
     },
@@ -129,20 +114,14 @@ pub enum Statement {
         bases: Vec<Expression>,
         keywords: Vec<Keyword>,
         decorator_list: Vec<Expression>,
+        // TODO: docstring: String,
     },
     FunctionDef {
         name: String,
         args: Parameters,
+        // docstring: String,
         body: Vec<LocatedStatement>,
         decorator_list: Vec<Expression>,
-        returns: Option<Expression>,
-    },
-    AsyncFunctionDef {
-        name: String,
-        args: Parameters,
-        body: Vec<LocatedStatement>,
-        decorator_list: Vec<Expression>,
-        returns: Option<Expression>,
     },
 }
 
@@ -172,9 +151,6 @@ pub enum Expression {
         op: UnaryOperator,
         a: Box<Expression>,
     },
-    Await {
-        value: Box<Expression>,
-    },
     Yield {
         value: Option<Box<Expression>>,
     },
@@ -182,8 +158,9 @@ pub enum Expression {
         value: Box<Expression>,
     },
     Compare {
-        vals: Vec<Expression>,
+        a: Box<Expression>,
-        ops: Vec<Comparison>,
+        op: Comparison,
+        b: Box<Expression>,
     },
     Attribute {
         value: Box<Expression>,
@@ -204,7 +181,7 @@ pub enum Expression {
         elements: Vec<Expression>,
     },
     Dict {
-        elements: Vec<(Option<Expression>, Expression)>,
+        elements: Vec<(Expression, Expression)>,
     },
     Set {
         elements: Vec<Expression>,
@@ -220,7 +197,7 @@ pub enum Expression {
         elements: Vec<Expression>,
     },
     String {
-        value: StringGroup,
+        value: String,
     },
     Bytes {
         value: Vec<u8>,
@@ -240,75 +217,22 @@ pub enum Expression {
     True,
     False,
     None,
-    Ellipsis,
-}
-
-impl Expression {
-    /// Returns a short name for the node suitable for use in error messages.
-    pub fn name(&self) -> &'static str {
-        use self::Expression::*;
-        use self::StringGroup::*;
-
-        match self {
-            BoolOp { .. } | Binop { .. } | Unop { .. } => "operator",
-            Subscript { .. } => "subscript",
-            Await { .. } => "await expression",
-            Yield { .. } | YieldFrom { .. } => "yield expression",
-            Compare { .. } => "comparison",
-            Attribute { .. } => "attribute",
-            Call { .. } => "function call",
-            Number { .. }
-            | String {
-                value: Constant { .. },
-            }
-            | Bytes { .. } => "literal",
-            List { .. } => "list",
-            Tuple { .. } => "tuple",
-            Dict { .. } => "dict display",
-            Set { .. } => "set display",
-            Comprehension { kind, .. } => match **kind {
-                ComprehensionKind::List { .. } => "list comprehension",
-                ComprehensionKind::Dict { .. } => "dict comprehension",
-                ComprehensionKind::Set { .. } => "set comprehension",
-                ComprehensionKind::GeneratorExpression { .. } => "generator expression",
-            },
-            Starred { .. } => "starred",
-            Slice { .. } => "slice",
-            String {
-                value: Joined { .. },
-            }
-            | String {
-                value: FormattedValue { .. },
-            } => "f-string expression",
-            Identifier { .. } => "named expression",
-            Lambda { .. } => "lambda",
-            IfExpression { .. } => "conditional expression",
-            True | False | None => "keyword",
-            Ellipsis => "ellipsis",
-        }
-    }
 }
 
 /*
  * In cpython this is called arguments, but we choose parameters to
- * distinguish between function parameters and actual call arguments.
+ * distuingish between function parameters and actual call arguments.
  */
 #[derive(Debug, PartialEq, Default)]
 pub struct Parameters {
-    pub args: Vec<Parameter>,
+    pub args: Vec<String>,
-    pub kwonlyargs: Vec<Parameter>,
+    pub kwonlyargs: Vec<String>,
-    pub vararg: Varargs, // Optionally we handle optionally named '*args' or '*'
+    pub vararg: Option<Option<String>>, // Optionally we handle optionally named '*args' or '*'
-    pub kwarg: Varargs,
+    pub kwarg: Option<Option<String>>,
     pub defaults: Vec<Expression>,
     pub kw_defaults: Vec<Option<Expression>>,
 }
 
-#[derive(Debug, PartialEq, Default)]
-pub struct Parameter {
-    pub arg: String,
-    pub annotation: Option<Box<Expression>>,
-}
-
 #[derive(Debug, PartialEq)]
 pub enum ComprehensionKind {
     GeneratorExpression { element: Expression },
@@ -388,54 +312,3 @@ pub enum Number {
     Float { value: f64 },
     Complex { real: f64, imag: f64 },
 }
-
-/// Transforms a value prior to formatting it.
-#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub enum ConversionFlag {
-    /// Converts by calling `str(<value>)`.
-    Str,
-    /// Converts by calling `ascii(<value>)`.
-    Ascii,
-    /// Converts by calling `repr(<value>)`.
-    Repr,
-}
-
-#[derive(Debug, PartialEq)]
-pub enum StringGroup {
-    Constant {
-        value: String,
-    },
-    FormattedValue {
-        value: Box<Expression>,
-        conversion: Option<ConversionFlag>,
-        spec: String,
-    },
-    Joined {
-        values: Vec<StringGroup>,
-    },
-}
-
-#[derive(Debug, PartialEq)]
-pub enum Varargs {
-    None,
-    Unnamed,
-    Named(Parameter),
-}
-
-impl Default for Varargs {
-    fn default() -> Varargs {
-        Varargs::None
-    }
-}
-
-impl From<Option<Option<Parameter>>> for Varargs {
-    fn from(opt: Option<Option<Parameter>>) -> Varargs {
-        match opt {
-            Some(inner_opt) => match inner_opt {
-                Some(param) => Varargs::Named(param),
-                None => Varargs::Unnamed,
-            },
-            None => Varargs::None,
-        }
-    }
-}

parser/src/error.rs

@@ -1,91 +0,0 @@
-//! Define internal parse error types
-//! The goal is to provide a matching and a safe error API, maksing errors from LALR
-extern crate lalrpop_util;
-use self::lalrpop_util::ParseError as InnerError;
-
-use crate::lexer::{LexicalError, LexicalErrorType, Location};
-use crate::token::Tok;
-
-use std::error::Error;
-use std::fmt;
-
-/// Represents an error during parsing
-#[derive(Debug, PartialEq)]
-pub struct ParseError {
-    pub error: ParseErrorType,
-    pub location: Location,
-}
-
-#[derive(Debug, PartialEq)]
-pub enum ParseErrorType {
-    /// Parser encountered an unexpected end of input
-    EOF,
-    /// Parser encountered an extra token
-    ExtraToken(Tok),
-    /// Parser encountered an invalid token
-    InvalidToken,
-    /// Parser encountered an unexpected token
-    UnrecognizedToken(Tok, Vec<String>),
-    /// Maps to `User` type from `lalrpop-util`
-    Lexical(LexicalErrorType),
-}
-
-/// Convert `lalrpop_util::ParseError` to our internal type
-impl From<InnerError<Location, Tok, LexicalError>> for ParseError {
-    fn from(err: InnerError<Location, Tok, LexicalError>) -> Self {
-        match err {
-            // TODO: Are there cases where this isn't an EOF?
-            InnerError::InvalidToken { location } => ParseError {
-                error: ParseErrorType::EOF,
-                location,
-            },
-            InnerError::ExtraToken { token } => ParseError {
-                error: ParseErrorType::ExtraToken(token.1),
-                location: token.0,
-            },
-            InnerError::User { error } => ParseError {
-                error: ParseErrorType::Lexical(error.error),
-                location: error.location,
-            },
-            InnerError::UnrecognizedToken { token, expected } => {
-                match token {
-                    Some(tok) => ParseError {
-                        error: ParseErrorType::UnrecognizedToken(tok.1, expected),
-                        location: tok.0,
-                    },
-                    // EOF was observed when it was unexpected
-                    None => ParseError {
-                        error: ParseErrorType::EOF,
-                        location: Default::default(),
-                    },
-                }
-            }
-        }
-    }
-}
-
-impl fmt::Display for ParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{} at {}", self.error, self.location)
-    }
-}
-
-impl fmt::Display for ParseErrorType {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match *self {
-            ParseErrorType::EOF => write!(f, "Got unexpected EOF"),
-            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {:?}", tok),
-            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
-            ParseErrorType::UnrecognizedToken(ref tok, _) => {
-                write!(f, "Got unexpected token: {:?}", tok)
-            }
-            ParseErrorType::Lexical(ref error) => write!(f, "{}", error),
-        }
-    }
-}
-
-impl Error for ParseError {
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        None
-    }
-}

parser/src/fstring.rs

@@ -1,237 +0,0 @@
-use std::iter;
-use std::mem;
-use std::str;
-
-use lalrpop_util::ParseError as LalrpopError;
-
-use crate::ast::{ConversionFlag, StringGroup};
-use crate::lexer::{LexicalError, LexicalErrorType, Location, Tok};
-use crate::parser::parse_expression;
-
-use self::FStringError::*;
-use self::StringGroup::*;
-
-// TODO: consolidate these with ParseError
-#[derive(Debug, PartialEq)]
-pub enum FStringError {
-    UnclosedLbrace,
-    UnopenedRbrace,
-    InvalidExpression,
-    InvalidConversionFlag,
-    EmptyExpression,
-    MismatchedDelimiter,
-}
-
-impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
-    fn from(_err: FStringError) -> Self {
-        lalrpop_util::ParseError::User {
-            error: LexicalError {
-                error: LexicalErrorType::StringError,
-                location: Default::default(),
-            },
-        }
-    }
-}
-
-struct FStringParser<'a> {
-    chars: iter::Peekable<str::Chars<'a>>,
-}
-
-impl<'a> FStringParser<'a> {
-    fn new(source: &'a str) -> Self {
-        Self {
-            chars: source.chars().peekable(),
-        }
-    }
-
-    fn parse_formatted_value(&mut self) -> Result<StringGroup, FStringError> {
-        let mut expression = String::new();
-        let mut spec = String::new();
-        let mut delims = Vec::new();
-        let mut conversion = None;
-
-        while let Some(ch) = self.chars.next() {
-            match ch {
-                '!' if delims.is_empty() => {
-                    conversion = Some(match self.chars.next() {
-                        Some('s') => ConversionFlag::Str,
-                        Some('a') => ConversionFlag::Ascii,
-                        Some('r') => ConversionFlag::Repr,
-                        Some(_) => {
-                            return Err(InvalidConversionFlag);
-                        }
-                        None => {
-                            break;
-                        }
-                    })
-                }
-                ':' if delims.is_empty() => {
-                    while let Some(&next) = self.chars.peek() {
-                        if next != '}' {
-                            spec.push(next);
-                            self.chars.next();
-                        } else {
-                            break;
-                        }
-                    }
-                }
-                '(' | '{' | '[' => {
-                    expression.push(ch);
-                    delims.push(ch);
-                }
-                ')' => {
-                    if delims.pop() != Some('(') {
-                        return Err(MismatchedDelimiter);
-                    }
-                    expression.push(ch);
-                }
-                ']' => {
-                    if delims.pop() != Some('[') {
-                        return Err(MismatchedDelimiter);
-                    }
-                    expression.push(ch);
-                }
-                '}' if !delims.is_empty() => {
-                    if delims.pop() != Some('{') {
-                        return Err(MismatchedDelimiter);
-                    }
-                    expression.push(ch);
-                }
-                '}' => {
-                    if expression.is_empty() {
-                        return Err(EmptyExpression);
-                    }
-                    return Ok(FormattedValue {
-                        value: Box::new(
-                            parse_expression(expression.trim()).map_err(|_| InvalidExpression)?,
-                        ),
-                        conversion,
-                        spec,
-                    });
-                }
-                '"' | '\'' => {
-                    expression.push(ch);
-                    while let Some(next) = self.chars.next() {
-                        expression.push(next);
-                        if next == ch {
-                            break;
-                        }
-                    }
-                }
-                _ => {
-                    expression.push(ch);
-                }
-            }
-        }
-
-        Err(UnclosedLbrace)
-    }
-
-    fn parse(mut self) -> Result<StringGroup, FStringError> {
-        let mut content = String::new();
-        let mut values = vec![];
-
-        while let Some(ch) = self.chars.next() {
-            match ch {
-                '{' => {
-                    if let Some('{') = self.chars.peek() {
-                        self.chars.next();
-                        content.push('{');
-                    } else {
-                        if !content.is_empty() {
-                            values.push(Constant {
-                                value: mem::replace(&mut content, String::new()),
-                            });
-                        }
-
-                        values.push(self.parse_formatted_value()?);
-                    }
-                }
-                '}' => {
-                    if let Some('}') = self.chars.peek() {
-                        self.chars.next();
-                        content.push('}');
-                    } else {
-                        return Err(UnopenedRbrace);
-                    }
-                }
-                _ => {
-                    content.push(ch);
-                }
-            }
-        }
-
-        if !content.is_empty() {
-            values.push(Constant { value: content })
-        }
-
-        Ok(match values.len() {
-            0 => Constant {
-                value: String::new(),
-            },
-            1 => values.into_iter().next().unwrap(),
-            _ => Joined { values },
-        })
-    }
-}
-
-pub fn parse_fstring(source: &str) -> Result<StringGroup, FStringError> {
-    FStringParser::new(source).parse()
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::ast;
-
-    use super::*;
-
-    fn mk_ident(name: &str) -> ast::Expression {
-        ast::Expression::Identifier {
-            name: name.to_owned(),
-        }
-    }
-
-    #[test]
-    fn test_parse_fstring() {
-        let source = String::from("{a}{ b }{{foo}}");
-        let parse_ast = parse_fstring(&source).unwrap();
-
-        assert_eq!(
-            parse_ast,
-            Joined {
-                values: vec![
-                    FormattedValue {
-                        value: Box::new(mk_ident("a")),
-                        conversion: None,
-                        spec: String::new(),
-                    },
-                    FormattedValue {
-                        value: Box::new(mk_ident("b")),
-                        conversion: None,
-                        spec: String::new(),
-                    },
-                    Constant {
-                        value: "{foo}".to_owned()
-                    }
-                ]
-            }
-        );
-    }
-
-    #[test]
-    fn test_parse_empty_fstring() {
-        assert_eq!(
-            parse_fstring(""),
-            Ok(Constant {
-                value: String::new(),
-            }),
-        );
-    }
-
-    #[test]
-    fn test_parse_invalid_fstring() {
-        assert_eq!(parse_fstring("{"), Err(UnclosedLbrace));
-        assert_eq!(parse_fstring("}"), Err(UnopenedRbrace));
-        assert_eq!(parse_fstring("{class}"), Err(InvalidExpression));
-    }
-}

parser/src/lib.rs

@@ -1,14 +1,13 @@
 #[macro_use]
 extern crate log;
-use lalrpop_util::lalrpop_mod;
+
+extern crate num_bigint;
+extern crate num_traits;
 
 pub mod ast;
-pub mod error;
-mod fstring;
 pub mod lexer;
 pub mod parser;
-lalrpop_mod!(
+mod python;
-    #[allow(clippy::all)]
-    python
-);
 pub mod token;
+
+pub use self::parser::parse;

parser/src/parser.rs

@@ -1,10 +1,30 @@
-use std::iter;
+extern crate lalrpop_util;
 
-use crate::ast;
+use std::error::Error;
-use crate::error::ParseError;
+use std::fs::File;
-use crate::lexer;
+use std::io::Read;
-use crate::python;
+use std::iter;
-use crate::token;
+use std::path::Path;
+
+use super::ast;
+use super::lexer;
+use super::python;
+use super::token;
+
+pub fn read_file(filename: &Path) -> Result<String, String> {
+    info!("Loading file {:?}", filename);
+    match File::open(&filename) {
+        Ok(mut file) => {
+            let mut s = String::new();
+
+            match file.read_to_string(&mut s) {
+                Err(why) => Err(String::from("Reading file failed: ") + why.description()),
+                Ok(_) => Ok(s),
+            }
+        }
+        Err(why) => Err(String::from("Opening file failed: ") + why.description()),
+    }
+}
 
 /*
  * Parse python code.
@@ -12,6 +32,13 @@ use crate::token;
  * https://github.com/antlr/grammars-v4/tree/master/python3
  */
 
+pub fn parse(filename: &Path) -> Result<ast::Program, String> {
+    info!("Parsing: {}", filename.display());
+    let txt = read_file(filename)?;
+    debug!("Read contents of file: {}", txt);
+    parse_program(&txt)
+}
+
 macro_rules! do_lalr_parsing {
     ($input: expr, $pat: ident, $tok: ident) => {{
         let lxr = lexer::make_tokenizer($input);
@@ -19,7 +46,7 @@ macro_rules! do_lalr_parsing {
         let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
 
         match python::TopParser::new().parse(tokenizer) {
-            Err(err) => Err(ParseError::from(err)),
+            Err(why) => Err(format!("{:?}", why)),
             Ok(top) => {
                 if let ast::Top::$pat(x) = top {
                     Ok(x)
@@ -31,11 +58,11 @@ macro_rules! do_lalr_parsing {
     }};
 }
 
-pub fn parse_program(source: &str) -> Result<ast::Program, ParseError> {
+pub fn parse_program(source: &str) -> Result<ast::Program, String> {
     do_lalr_parsing!(source, Program, StartProgram)
 }
 
-pub fn parse_statement(source: &str) -> Result<Vec<ast::LocatedStatement>, ParseError> {
+pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, String> {
     do_lalr_parsing!(source, Statement, StartStatement)
 }
 
@@ -44,6 +71,7 @@ pub fn parse_statement(source: &str) -> Result<Vec<ast::LocatedStatement>, Parse
 /// # Example
 /// ```
 /// extern crate num_bigint;
+/// extern crate rustpython_parser;
 /// use num_bigint::BigInt;
 /// use rustpython_parser::{parser, ast};
 /// let expr = parser::parse_expression("1+2").unwrap();
@@ -60,7 +88,7 @@ pub fn parse_statement(source: &str) -> Result<Vec<ast::LocatedStatement>, Parse
 ///     expr);
 ///
 /// ```
-pub fn parse_expression(source: &str) -> Result<ast::Expression, ParseError> {
+pub fn parse_expression(source: &str) -> Result<ast::Expression, String> {
     do_lalr_parsing!(source, Expression, StartExpression)
 }
 
@@ -75,6 +103,7 @@ mod tests {
     #[test]
     fn test_parse_empty() {
         let parse_ast = parse_program(&String::from("\n"));
+
         assert_eq!(parse_ast, Ok(ast::Program { statements: vec![] }))
     }
 
@@ -93,9 +122,7 @@ mod tests {
                                 name: String::from("print"),
                             }),
                             args: vec![ast::Expression::String {
-                                value: ast::StringGroup::Constant {
+                                value: String::from("Hello world"),
-                                    value: String::from("Hello world")
-                                }
                             }],
                             keywords: vec![],
                         },
@@ -121,9 +148,7 @@ mod tests {
                             }),
                             args: vec![
                                 ast::Expression::String {
-                                    value: ast::StringGroup::Constant {
+                                    value: String::from("Hello world"),
-                                        value: String::from("Hello world"),
-                                    }
                                 },
                                 ast::Expression::Number {
                                     value: ast::Number::Integer {
@@ -154,9 +179,7 @@ mod tests {
                                 name: String::from("my_func"),
                             }),
                             args: vec![ast::Expression::String {
-                                value: ast::StringGroup::Constant {
+                                value: String::from("positional"),
-                                    value: String::from("positional"),
-                                }
                             }],
                             keywords: vec![ast::Keyword {
                                 name: Some("keyword".to_string()),
@@ -179,7 +202,7 @@ mod tests {
         let parse_ast = parse_statement(&source).unwrap();
         assert_eq!(
             parse_ast,
-            vec![ast::LocatedStatement {
+            ast::LocatedStatement {
                 location: ast::Location::new(1, 1),
                 node: ast::Statement::If {
                     test: ast::Expression::Number {
@@ -228,7 +251,7 @@ mod tests {
                         }
                     },]),
                 }
-            }]
+            }
         );
     }
 
@@ -238,24 +261,15 @@ mod tests {
         let parse_ast = parse_statement(&source);
         assert_eq!(
             parse_ast,
-            Ok(vec![ast::LocatedStatement {
+            Ok(ast::LocatedStatement {
                 location: ast::Location::new(1, 1),
                 node: ast::Statement::Expression {
                     expression: ast::Expression::Lambda {
                         args: ast::Parameters {
-                            args: vec![
+                            args: vec![String::from("x"), String::from("y")],
-                                ast::Parameter {
-                                    arg: String::from("x"),
-                                    annotation: None,
-                                },
-                                ast::Parameter {
-                                    arg: String::from("y"),
-                                    annotation: None,
-                                }
-                            ],
                             kwonlyargs: vec![],
-                            vararg: ast::Varargs::None,
+                            vararg: None,
-                            kwarg: ast::Varargs::None,
+                            kwarg: None,
                             defaults: vec![],
                             kw_defaults: vec![],
                         },
@@ -270,7 +284,7 @@ mod tests {
                         })
                     }
                 }
-            }])
+            })
         )
     }
 
@@ -280,7 +294,7 @@ mod tests {
 
         assert_eq!(
             parse_statement(&source),
-            Ok(vec![ast::LocatedStatement {
+            Ok(ast::LocatedStatement {
                 location: ast::Location::new(1, 1),
                 node: ast::Statement::Assign {
                     targets: vec![ast::Expression::Tuple {
@@ -308,18 +322,16 @@ mod tests {
                         ]
                     }
                 }
-            }])
+            })
         )
     }
 
     #[test]
     fn test_parse_class() {
-        let source = String::from(
+        let source = String::from("class Foo(A, B):\n def __init__(self):\n  pass\n def method_with_default(self, arg='default'):\n  pass\n");
-            "class Foo(A, B):\n def __init__(self):\n  pass\n def method_with_default(self, arg='default'):\n  pass\n",
-        );
         assert_eq!(
             parse_statement(&source),
-            Ok(vec![ast::LocatedStatement {
+            Ok(ast::LocatedStatement {
                 location: ast::Location::new(1, 1),
                 node: ast::Statement::ClassDef {
                     name: String::from("Foo"),
@@ -338,13 +350,10 @@ mod tests {
                             node: ast::Statement::FunctionDef {
                                 name: String::from("__init__"),
                                 args: ast::Parameters {
-                                    args: vec![ast::Parameter {
+                                    args: vec![String::from("self")],
-                                        arg: String::from("self"),
-                                        annotation: None,
-                                    }],
                                     kwonlyargs: vec![],
-                                    vararg: ast::Varargs::None,
+                                    vararg: None,
-                                    kwarg: ast::Varargs::None,
+                                    kwarg: None,
                                     defaults: vec![],
                                     kw_defaults: vec![],
                                 },
@@ -353,7 +362,6 @@ mod tests {
                                     node: ast::Statement::Pass,
                                 }],
                                 decorator_list: vec![],
-                                returns: None,
                             }
                         },
                         ast::LocatedStatement {
@@ -361,23 +369,12 @@ mod tests {
                             node: ast::Statement::FunctionDef {
                                 name: String::from("method_with_default"),
                                 args: ast::Parameters {
-                                    args: vec![
+                                    args: vec![String::from("self"), String::from("arg"),],
-                                        ast::Parameter {
-                                            arg: String::from("self"),
-                                            annotation: None,
-                                        },
-                                        ast::Parameter {
-                                            arg: String::from("arg"),
-                                            annotation: None,
-                                        }
-                                    ],
                                     kwonlyargs: vec![],
-                                    vararg: ast::Varargs::None,
+                                    vararg: None,
-                                    kwarg: ast::Varargs::None,
+                                    kwarg: None,
                                     defaults: vec![ast::Expression::String {
-                                        value: ast::StringGroup::Constant {
+                                        value: "default".to_string()
-                                            value: "default".to_string()
-                                        }
                                     }],
                                     kw_defaults: vec![],
                                 },
@@ -386,13 +383,12 @@ mod tests {
                                     node: ast::Statement::Pass,
                                 }],
                                 decorator_list: vec![],
-                                returns: None,
                             }
                         }
                     ],
                     decorator_list: vec![],
                 }
-            }])
+            })
         )
     }
 
@@ -459,30 +455,26 @@ mod tests {
                         },
                         ifs: vec![
                             ast::Expression::Compare {
-                                vals: vec![
+                                a: Box::new(ast::Expression::Identifier {
-                                    ast::Expression::Identifier {
+                                    name: "a".to_string()
-                                        name: "a".to_string()
+                                }),
-                                    },
+                                op: ast::Comparison::Less,
-                                    ast::Expression::Number {
+                                b: Box::new(ast::Expression::Number {
-                                        value: ast::Number::Integer {
+                                    value: ast::Number::Integer {
-                                            value: BigInt::from(5)
+                                        value: BigInt::from(5)
-                                        }
                                     }
-                                ],
+                                }),
-                                ops: vec![ast::Comparison::Less],
                             },
                             ast::Expression::Compare {
-                                vals: vec![
+                                a: Box::new(ast::Expression::Identifier {
-                                    ast::Expression::Identifier {
+                                    name: "a".to_string()
-                                        name: "a".to_string()
+                                }),
-                                    },
+                                op: ast::Comparison::Greater,
-                                    ast::Expression::Number {
+                                b: Box::new(ast::Expression::Number {
-                                        value: ast::Number::Integer {
+                                    value: ast::Number::Integer {
-                                            value: BigInt::from(10)
+                                        value: BigInt::from(10)
-                                        }
                                     }
-                                ],
+                                }),
-                                ops: vec![ast::Comparison::Greater],
                             },
                         ],
                     }

parser/src/python.lalrpop

@@ -2,13 +2,11 @@
 // See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4
 // See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions
 // See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword
+#![allow(unknown_lints,clippy)]
 
+use super::ast;
+use super::lexer;
 use std::iter::FromIterator;
-
-use crate::ast;
-use crate::fstring::parse_fstring;
-use crate::lexer;
-
 use num_bigint::BigInt;
 
 grammar;
@@ -24,32 +22,29 @@ pub Top: ast::Top = {
 
 Program: ast::Program = {
     <lines:FileLine*> => ast::Program {
-        statements: Vec::from_iter(lines.into_iter().flatten())
+        statements: Vec::from_iter(lines.into_iter().filter_map(|e| e))
     },
 };
 
 // A file line either has a declaration, or an empty newline:
-FileLine: Vec<ast::LocatedStatement> = {
+FileLine: Option<ast::LocatedStatement> = {
-    Statement,
+    <s:Statement> => Some(s),
-    "\n" => vec![],
+    "\n" => None,
 };
 
 Suite: Vec<ast::LocatedStatement> = {
-    SimpleStatement,
+    <s:SimpleStatement> => vec![s],
-    "\n" indent <s:Statement+> dedent => s.into_iter().flatten().collect(),
+    "\n" indent <s:Statement+> dedent => s,
 };
 
-Statement: Vec<ast::LocatedStatement> = {
+Statement: ast::LocatedStatement = {
     SimpleStatement,
-    <s:CompoundStatement> => vec![s],
+    CompoundStatement,
 };
 
-SimpleStatement: Vec<ast::LocatedStatement> = {
+SimpleStatement: ast::LocatedStatement = {
-    <s1:SmallStatement> <s2:(";" SmallStatement)*> ";"? "\n" => {
+    <s:SmallStatement> "\n" => s,
-        let mut statements = vec![s1];
+    <s:SmallStatement> ";" => s,
-        statements.extend(s2.into_iter().map(|e| e.1));
-        statements
-    }
 };
 
 SmallStatement: ast::LocatedStatement = {
@@ -105,30 +100,42 @@ ExpressionStatement: ast::LocatedStatement = {
           }
         }
     },
-    <loc:@L> <expr:TestOrStarExprList> <op:AugAssign> <rhs:TestList> => {
+    <loc:@L> <expr:TestOrStarExprList> <op:AugAssign> <e2:TestList> => {
+        // TODO: this works in most cases:
+        let rhs = e2.into_iter().next().unwrap();
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::AugAssign {
+            node: ast::Statement::AugAssign { target: expr, op: op, value: rhs },
-                target: Box::new(expr),
-                op,
-                value: Box::new(rhs)
-            },
         }
     },
 };
 
 AssignSuffix: ast::Expression = {
-    "=" <e:TestList> => e,
+    "=" <e:TestList> => {
+        if e.len() > 1 {
+            ast::Expression::Tuple {
+                elements: e
+            }
+        } else {
+            e.into_iter().next().unwrap()
+        }
+    },
     "=" <e:YieldExpr> => e,
 };
 
 TestOrStarExprList: ast::Expression = {
-    <elements:OneOrMore<TestOrStarExpr>> <comma:","?> => {
+    <e:TestOrStarExpr> <e2:("," TestOrStarExpr)*> <comma:","?> => {
-        if elements.len() == 1 && comma.is_none() {
+        let mut res = vec![e];
-            elements.into_iter().next().unwrap()
+        res.extend(e2.into_iter().map(|x| x.1));
+
+        // First build tuple from first item:
+        let expr = if (res.len() > 1) || comma.is_some() {
+            ast::Expression::Tuple { elements: res }
         } else {
-            ast::Expression::Tuple { elements }
+            res.into_iter().next().unwrap()
-        }
+        };
+
+        expr
     }
 };
 
@@ -169,7 +176,7 @@ FlowStatement: ast::LocatedStatement = {
     <loc:@L> "return" <t:TestList?> => {
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::Return { value: t.map(Box::new) },
+            node: ast::Statement::Return { value: t },
         }
     },
     <loc:@L> <y:YieldExpr> => {
@@ -206,9 +213,8 @@ ImportStatement: ast::LocatedStatement = {
             .map(|(n, a)|
                 ast::SingleImport {
                     module: n.to_string(),
-                    symbols: vec![],
+                    symbol: None,
-                    alias: a.clone(),
+                    alias: a.clone()
-                    level: 0,
                 })
             .collect()
       },
@@ -218,30 +224,35 @@ ImportStatement: ast::LocatedStatement = {
     ast::LocatedStatement {
       location: loc,
       node: ast::Statement::Import {
-        import_parts: vec![
+        import_parts: i
-            ast::SingleImport {
+            .iter()
-                module: n.0.to_string(),
+            .map(|(i, a)|
-                symbols: i.iter()
+                ast::SingleImport {
-                    .map(|(i, a)|
+                    module: n.to_string(),
-                        ast::ImportSymbol {
+                    symbol: Some(i.to_string()),
-                            symbol: i.to_string(),
+                    alias: a.clone()
-                            alias: a.clone(),
+                })
-                        })
+            .collect()
-                    .collect(),
-                alias: None,
-                level: n.1
-        }]
       },
     }
   },
 };
 
-ImportFromLocation: (String, usize) = {
+ImportFromLocation: String = {
     <dots: "."*> <name:DottedName> => {
-       (name, dots.len())
+       let mut r = "".to_string();
+       for _dot in dots {
+         r.push_str(".");
+       }
+      r.push_str(&name);
+      r
     },
     <dots: "."+> => {
-       ("".to_string(), dots.len())
+       let mut r = "".to_string();
+       for _dot in dots {
+         r.push_str(".");
+       }
+      r
     },
 };
 
@@ -292,11 +303,15 @@ NonlocalStatement: ast::LocatedStatement = {
 };
 
 AssertStatement: ast::LocatedStatement = {
-    <loc:@L> "assert" <test:Test> <msg: ("," Test)?> => {
+    <loc:@L> "assert" <t:Test> <m: ("," Test)?> => {
         ast::LocatedStatement {
           location: loc,
           node: ast::Statement::Assert {
-            test, msg: msg.map(|e| e.1)
+            test: t,
+            msg: match m {
+              Some(e) => Some(e.1),
+              None => None,
+            }
           }
         }
     },
@@ -313,9 +328,12 @@ CompoundStatement: ast::LocatedStatement = {
 };
 
 IfStatement: ast::LocatedStatement = {
-    <loc:@L> "if" <test:Test> ":" <s1:Suite> <s2:(@L "elif" Test ":" Suite)*> <s3:("else" ":" Suite)?> => {
+    <loc:@L> "if" <t:Test> ":" <s1:Suite> <s2:(@L "elif" Test ":" Suite)*> <s3:("else" ":" Suite)?> => {
         // Determine last else:
-        let mut last = s3.map(|s| s.2);
+        let mut last = match s3 {
+            Some(s) => Some(s.2),
+            None => None,
+        };
 
         // handle elif:
         for i in s2.into_iter().rev() {
@@ -328,30 +346,35 @@ IfStatement: ast::LocatedStatement = {
 
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::If { test, body: s1, orelse: last }
+            node: ast::Statement::If { test: t, body: s1, orelse: last }
         }
     },
 };
 
 WhileStatement: ast::LocatedStatement = {
-    <loc:@L> "while" <test:Test> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+    <loc:@L> "while" <e:Test> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
-        let or_else = s2.map(|s| s.2);
+        let or_else = match s2 {
+            Some(s) => Some(s.2),
+            None => None,
+        };
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::While { test, body, orelse: or_else },
+            node: ast::Statement::While { test: e, body: s, orelse: or_else },
         }
     },
 };
 
 ForStatement: ast::LocatedStatement = {
-    <loc:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+    <loc:@L> "for" <e:ExpressionList> "in" <t:TestList> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
-        let orelse = s2.map(|s| s.2);
+        let or_else = match s2 {
+            Some(s) => Some(s.2),
+            None => None,
+        };
         ast::LocatedStatement {
             location: loc,
-            node: if is_async.is_some() {
+            node: ast::Statement::For {
-                ast::Statement::AsyncFor { target, iter, body, orelse }
+                target: e,
-            } else {
+                iter: t, body: s, orelse: or_else
-                ast::Statement::For { target, iter, body, orelse }
             },
         }
     },
@@ -359,8 +382,14 @@ ForStatement: ast::LocatedStatement = {
 
 TryStatement: ast::LocatedStatement = {
     <loc:@L> "try" ":" <body:Suite> <handlers:ExceptClause*> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> => {
-        let or_else = else_suite.map(|s| s.2);
+        let or_else = match else_suite {
-        let finalbody = finally.map(|s| s.2);
+            Some(s) => Some(s.2),
+            None => None,
+        };
+        let finalbody = match finally {
+            Some(s) => Some(s.2),
+            None => None,
+        };
         ast::LocatedStatement {
             location: loc,
             node: ast::Statement::Try {
@@ -401,59 +430,59 @@ WithStatement: ast::LocatedStatement = {
 
 WithItem: ast::WithItem = {
     <t:Test> <n:("as" Expression)?> => {
-        let optional_vars = n.map(|val| val.1);
+        let optional_vars = match n {
+            Some(val) => Some(val.1),
+            None => None,
+        };
         ast::WithItem { context_expr: t, optional_vars }
     },
 };
 
 FuncDef: ast::LocatedStatement = {
-    <d:Decorator*> <loc:@L> <is_async:"async"?> "def" <i:Identifier> <a:Parameters> <r:("->" Test)?> ":" <s:Suite> => {
+    <d:Decorator*> <loc:@L> "def" <i:Identifier> <a:Parameters> ":" <s:Suite> => {
         ast::LocatedStatement {
             location: loc,
-            node: if is_async.is_some() {
+            node: ast::Statement::FunctionDef {
-                    ast::Statement::AsyncFunctionDef {
+                name: i,
-                        name: i,
+                args: a,
-                        args: a,
+                body: s,
-                        body: s,
+                decorator_list: d,
-                        decorator_list: d,
+            }
-                        returns: r.map(|x| x.1),
-                    }
-                } else {
-                    ast::Statement::FunctionDef {
-                        name: i,
-                        args: a,
-                        body: s,
-                        decorator_list: d,
-                        returns: r.map(|x| x.1),
-                    }
-                }
         }
     },
 };
 
 Parameters: ast::Parameters = {
-    "(" <a: (ParameterList<TypedParameter>)?> ")" => a.unwrap_or_else(Default::default),
+    "(" <a: (TypedArgsList)?> ")" => {
+        match a {
+            Some(a) => a,
+            None => Default::default(),
+        }
+    },
 };
 
-// Note that this is a macro which is used once for function defs, and
+// parameters are (String, None), kwargs are (String, Some(Test)) where Test is
-// once for lambda defs.
+// the default
-ParameterList<ArgType>: ast::Parameters = {
+TypedArgsList: ast::Parameters = {
-    <param1:ParameterDefs<ArgType>> <args2:("," ParameterListStarArgs<ArgType>)?> ","? => {
+    <param1:TypedParameters> <args2:("," ParameterListStarArgs)?> => {
         let (names, default_elements) = param1;
 
         // Now gather rest of parameters:
-        let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1);
+        let (vararg, kwonlyargs, kw_defaults, kwarg) = match args2 {
+            Some((_, x)) => x,
+            None => (None, vec![], vec![], None),
+        };
 
         ast::Parameters {
             args: names,
             kwonlyargs: kwonlyargs,
-            vararg: vararg.into(),
+            vararg: vararg,
-            kwarg: kwarg.into(),
+            kwarg: kwarg,
             defaults: default_elements,
             kw_defaults: kw_defaults,
         }
     },
-    <param1:ParameterDefs<ArgType>> <kw:("," KwargParameter<ArgType>)> ","? => {
+    <param1:TypedParameters> <kw:("," KwargParameter)> => {
         let (names, default_elements) = param1;
 
         // Now gather rest of parameters:
@@ -465,29 +494,29 @@ ParameterList<ArgType>: ast::Parameters = {
         ast::Parameters {
             args: names,
             kwonlyargs: kwonlyargs,
-            vararg: vararg.into(),
+            vararg: vararg,
-            kwarg: kwarg.into(),
+            kwarg: kwarg,
             defaults: default_elements,
             kw_defaults: kw_defaults,
         }
     },
-    <params:ParameterListStarArgs<ArgType>> ","? => {
+    <params:ParameterListStarArgs> => {
         let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
         ast::Parameters {
             args: vec![],
             kwonlyargs: kwonlyargs,
-            vararg: vararg.into(),
+            vararg: vararg,
-            kwarg: kwarg.into(),
+            kwarg: kwarg,
             defaults: vec![],
             kw_defaults: kw_defaults,
         }
     },
-    <kw:KwargParameter<ArgType>> ","? => {
+    <kw:KwargParameter> => {
         ast::Parameters {
             args: vec![],
             kwonlyargs: vec![],
-            vararg: ast::Varargs::None,
+            vararg: None,
-            kwarg: Some(kw).into(),
+            kwarg: Some(kw),
             defaults: vec![],
             kw_defaults: vec![],
         }
@@ -496,8 +525,8 @@ ParameterList<ArgType>: ast::Parameters = {
 
 // Use inline here to make sure the "," is not creating an ambiguity.
 #[inline]
-ParameterDefs<ArgType>: (Vec<ast::Parameter>, Vec<ast::Expression>) = {
+TypedParameters: (Vec<String>, Vec<ast::Expression>) = {
-    <param1:ParameterDef<ArgType>> <param2:("," ParameterDef<ArgType>)*> => {
+    <param1:TypedParameterDef> <param2:("," TypedParameterDef)*> => {
         // Combine first parameters:
         let mut args = vec![param1];
         args.extend(param2.into_iter().map(|x| x.1));
@@ -506,6 +535,7 @@ ParameterDefs<ArgType>: (Vec<ast::Parameter>, Vec<ast::Expression>) = {
         let mut default_elements = vec![];
 
         for (name, default) in args.into_iter() {
+            names.push(name.clone());
             if let Some(default) = default {
                 default_elements.push(default);
             } else {
@@ -514,38 +544,28 @@ ParameterDefs<ArgType>: (Vec<ast::Parameter>, Vec<ast::Expression>) = {
                     // have defaults
                     panic!(
                         "non-default argument follows default argument: {}",
-                        &name.arg
+                        name
                     );
                 }
             }
-            names.push(name);
         }
 
         (names, default_elements)
     }
 };
 
-ParameterDef<ArgType>: (ast::Parameter, Option<ast::Expression>) = {
+TypedParameterDef: (String, Option<ast::Expression>) = {
-    <i:ArgType> => (i, None),
+    <i:TypedParameter> => (i, None),
-    <i:ArgType> "=" <e:Test> => (i, Some(e)),
+    <i:TypedParameter> "=" <e:Test> => (i, Some(e)),
 };
 
-UntypedParameter: ast::Parameter = {
+// TODO: add type annotations here:
-    <i:Identifier> => ast::Parameter { arg: i, annotation: None },
+TypedParameter: String = {
+    Identifier,
 };
 
-TypedParameter: ast::Parameter = {
+ParameterListStarArgs: (Option<Option<String>>, Vec<String>, Vec<Option<ast::Expression>>, Option<Option<String>>) = {
-    <arg:Identifier> <a:(":" Test)?>=> {
+    "*" <va:Identifier?> <kw:("," TypedParameterDef)*> <kwarg:("," KwargParameter)?> => {
-        let annotation = a.map(|x| Box::new(x.1));
-        ast::Parameter { arg, annotation }
-    },
-};
-
-// Use inline here to make sure the "," is not creating an ambiguity.
-// TODO: figure out another grammar that makes this inline no longer required.
-#[inline]
-ParameterListStarArgs<ArgType>: (Option<Option<ast::Parameter>>, Vec<ast::Parameter>, Vec<Option<ast::Expression>>, Option<Option<ast::Parameter>>) = {
-    "*" <va:ArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> => {
         // Extract keyword arguments:
         let mut kwonlyargs = vec![];
         let mut kw_defaults = vec![];
@@ -554,14 +574,17 @@ ParameterListStarArgs<ArgType>: (Option<Option<ast::Parameter>>, Vec<ast::Parame
             kw_defaults.push(value);
         }
 
-        let kwarg = kwarg.map(|n| n.1);
+        let kwarg = match kwarg {
+            Some((_, name)) => Some(name),
+            None => None,
+        };
 
         (Some(va), kwonlyargs, kw_defaults, kwarg)
     }
 };
 
-KwargParameter<ArgType>: Option<ast::Parameter> = {
+KwargParameter: Option<String> = {
-    "**" <kwarg:ArgType?> => {
+    "**" <kwarg:Identifier?> => {
         kwarg
     }
 };
@@ -585,84 +608,82 @@ ClassDef: ast::LocatedStatement = {
     },
 };
 
-Path: ast::Expression = {
-    <n:Identifier> => ast::Expression::Identifier { name: n },
-    <p:Path> "." <n:name> => {
-        ast::Expression::Attribute {
-            value: Box::new(p),
-            name: n,
-        }
-    },
-};
-
 // Decorators:
 Decorator: ast::Expression = {
-    "@" <p:Path> <a: ("(" ArgumentList ")")?> "\n" => {
+    "@" <n:DottedName> <a: ("(" ArgumentList ")")?> "\n" => {
+        let name = ast::Expression::Identifier { name: n };
         match a {
             Some((_, args, _)) => ast::Expression::Call {
-                function: Box::new(p),
+                function: Box::new(name),
                 args: args.0,
                 keywords: args.1,
             },
-            None => p,
+            None => name,
         }
     },
 };
 
 YieldExpr: ast::Expression = {
-    "yield" <value:TestList?> => ast::Expression::Yield { value: value.map(Box::new) },
+    "yield" <ex:TestList?> => {
-    "yield" "from" <e:Test> => ast::Expression::YieldFrom { value: Box::new(e) },
+        ast::Expression::Yield {
+            value: ex.map(|expr| Box::new(
+                if expr.len() > 1 {
+                    ast::Expression::Tuple { elements: expr }
+                } else {
+                    expr.into_iter().next().unwrap()
+                })
+            )
+        }
+    },
+    "yield" "from" <e:Test> => {
+        ast::Expression::YieldFrom {
+            value: Box::new(e),
+        }
+    },
 };
 
 Test: ast::Expression = {
-    <expr:OrTest> <condition: ("if" OrTest "else" Test)?> => {
+    <e:OrTest> <c: ("if" OrTest "else" Test)?> => {
-        if let Some(c) = condition {
+        match c {
-            ast::Expression::IfExpression {
+            Some(c) => {
-                test: Box::new(c.1),
+                ast::Expression::IfExpression {
-                body: Box::new(expr),
+                    test: Box::new(c.1),
-                orelse: Box::new(c.3),
+                    body: Box::new(e),
-            }
+                    orelse: Box::new(c.3),
-        } else {
+                }
-            expr
+            },
+            None => e,
         }
     },
-    LambdaDef,
+    <e:LambdaDef> => e,
 };
 
 LambdaDef: ast::Expression = {
-    "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test> =>
+    "lambda" <p:TypedArgsList?> ":" <b:Expression> =>
         ast::Expression::Lambda {
             args: p.unwrap_or(Default::default()),
-            body: Box::new(body)
+            body:Box::new(b)
        }
 }
 
 OrTest: ast::Expression = {
-    AndTest,
+    <e:AndTest> => e,
     <e1:OrTest> "or" <e2:AndTest> => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) },
 };
 
 AndTest: ast::Expression = {
-    NotTest,
+    <e:NotTest> => e,
     <e1:AndTest> "and" <e2:NotTest> => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) },
 };
 
 NotTest: ast::Expression = {
     "not" <e:NotTest> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Not },
-    Comparison,
+    <e:Comparison> => e,
 };
 
 Comparison: ast::Expression = {
-    <e:Expression> <comparisons:(CompOp Expression)+> => {
+    <e1:Comparison> <op:CompOp> <e2:Expression> => ast::Expression::Compare { a: Box::new(e1), op: op, b: Box::new(e2) },
-        let mut vals = vec![e];
+    <e:Expression> => e,
-        let mut ops = vec![];
-        for x in comparisons {
-            ops.push(x.0);
-            vals.push(x.1);
-        }
-        ast::Expression::Compare { vals, ops }
-    },
-    Expression,
 };
 
 CompOp: ast::Comparison = {
@@ -680,7 +701,7 @@ CompOp: ast::Comparison = {
 
 Expression: ast::Expression = {
     <e1:Expression> "|" <e2:XorExpression> => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) },
-    XorExpression,
+    <e:XorExpression> => e,
 };
 
 XorExpression: ast::Expression = {
@@ -730,7 +751,7 @@ Factor: ast::Expression = {
     "+" <e:Factor> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Pos },
     "-" <e:Factor> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Neg },
     "~" <e:Factor> => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Inv },
-    Power,
+    <e:Power> => e,
 };
 
 Power: ast::Expression = {
@@ -743,38 +764,14 @@ Power: ast::Expression = {
 };
 
 AtomExpr: ast::Expression = {
-    <is_await:"await"?> <atom:AtomExpr2> => {
+    <e:Atom> => e,
-        if is_await.is_some() {
+    <f:AtomExpr>  "(" <a:ArgumentList> ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 },
-            ast::Expression::Await { value: Box::new(atom) }
+    <e:AtomExpr>  "[" <s:Subscript> "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) },
-        } else {
+    <e:AtomExpr> "." <n:Identifier> => ast::Expression::Attribute { value: Box::new(e), name: n },
-            atom
-        }
-    }
-}
-
-AtomExpr2: ast::Expression = {
-    Atom,
-    <f:AtomExpr2> "(" <a:ArgumentList> ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 },
-    <e:AtomExpr2> "[" <s:SubscriptList> "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) },
-    <e:AtomExpr2> "." <n:Identifier> => ast::Expression::Attribute { value: Box::new(e), name: n },
-};
-
-SubscriptList: ast::Expression = {
-    <s1:Subscript> <s2:("," Subscript)*> ","? => {
-        if s2.is_empty() {
-            s1
-        } else {
-            let mut dims = vec![s1];
-            for x in s2 {
-                dims.push(x.1)
-            }
-            ast::Expression::Tuple { elements: dims }
-        }
-    }
 };
 
 Subscript: ast::Expression = {
-    Test,
+    <e:Test> => e,
     <e1:Test?> ":" <e2:Test?> <e3:SliceOp?>  => {
         let s1 = e1.unwrap_or(ast::Expression::None);
         let s2 = e2.unwrap_or(ast::Expression::None);
@@ -788,17 +785,29 @@ SliceOp: ast::Expression = {
 }
 
 Atom: ast::Expression = {
-    <value:StringGroup> => ast::Expression::String { value },
+    StringConstant,
-    <value:Bytes> => ast::Expression::Bytes { value },
+    <n:Number> => ast::Expression::Number { value: n },
-    <value:Number> => ast::Expression::Number { value },
+    <i:Identifier> => ast::Expression::Identifier { name: i },
-    <name:Identifier> => ast::Expression::Identifier { name },
     "[" <e:TestListComp?> "]" => {
         let elements = e.unwrap_or(Vec::new());
         ast::Expression::List { elements }
     },
-    "[" <e:TestListComp2> "]" => e,
+    "[" <e:TestListComp2> "]" => {
-    "(" <elements:TestList?> ")" => {
+        // List comprehension:
-         elements.unwrap_or(ast::Expression::Tuple { elements: Vec::new() })
+        e
+    },
+    "(" <e:TestList?> <trailing_comma:","?> ")" => {
+         match e {
+             None => ast::Expression::Tuple { elements: Vec::new() },
+             Some(elements) => {
+                 if elements.len() == 1 && trailing_comma.is_none() {
+                   // This is "(e)", which is equivalent to "e"
+                   elements.into_iter().next().unwrap()
+                 } else {
+                   ast::Expression::Tuple { elements }
+                 }
+             }
+         }
     },
     "(" <e:Test> <c:CompFor> ")" => {
         ast::Expression::Comprehension {
@@ -813,11 +822,12 @@ Atom: ast::Expression = {
     "True" => ast::Expression::True,
     "False" => ast::Expression::False,
     "None" => ast::Expression::None,
-    "..." => ast::Expression::Ellipsis,
 };
 
 TestListComp: Vec<ast::Expression> = {
-    <e:OneOrMore<TestOrStarExpr>> <_trailing_comma:","?> => e,
+    <e:OneOrMore<TestOrStarExpr>> <_trailing_comma:","?> => {
+        e
+    },
 };
 
 TestListComp2: ast::Expression = {
@@ -829,8 +839,10 @@ TestListComp2: ast::Expression = {
     },
 };
 
-TestDict: Vec<(Option<ast::Expression>, ast::Expression)> = {
+TestDict: Vec<(ast::Expression, ast::Expression)> = {
-    <elements:OneOrMore<DictElement>> <_trailing_comma:","?> => elements,
+    <e1:OneOrMore<DictEntry>> <_trailing_comma:","?> => {
+        e1
+    }
 };
 
 TestDictComp: ast::Expression = {
@@ -846,13 +858,10 @@ DictEntry: (ast::Expression, ast::Expression) = {
     <e1: Test> ":" <e2: Test> => (e1, e2),
 };
 
-DictElement: (Option<ast::Expression>, ast::Expression) = {
-    <e:DictEntry> => (Some(e.0), e.1),
-    "**" <e:Expression> => (None, e),
-};
-
 TestSet: Vec<ast::Expression> = {
-    <e1:OneOrMore<Test>> ","? => e1
+    <e1:OneOrMore<Test>> ","? => {
+        e1
+    }
 };
 
 TestSetComp: ast::Expression = {
@@ -864,37 +873,31 @@ TestSetComp: ast::Expression = {
     }
 };
 
-ExpressionOrStarExpression = {
-    Expression,
-    StarExpr
-};
-
 ExpressionList: ast::Expression = {
-    <elements: OneOrMore<ExpressionOrStarExpression>> <trailing_comma:","?> => {
+    <e: ExpressionList2> => {
-        if elements.len() == 1 && trailing_comma.is_none() {
+        if e.len() == 1 {
-            elements.into_iter().next().unwrap()
+            e.into_iter().next().unwrap()
         } else {
-            ast::Expression::Tuple { elements }
+            ast::Expression::Tuple { elements: e }
         }
     },
 };
 
 ExpressionList2: Vec<ast::Expression> = {
-    <elements:OneOrMore<Expression>> ","? => elements,
+    <e1:Expression> <e2:("," Expression)*> ","? => {
+        let mut l = vec![e1];
+        l.extend(e2.into_iter().map(|x| x.1));
+        l
+    },
 };
 
-// A test list is one of:
+#[inline]
-// - a list of expressions
+TestList: Vec<ast::Expression> = {
-// - a single expression
+  <e1:Test> <e2: ("," Test)*> => {
-// - a single expression followed by a trailing comma
+    let mut l = vec![e1];
-TestList: ast::Expression = {
+    l.extend(e2.into_iter().map(|x| x.1));
-    <elements:OneOrMore<Test>> <trailing_comma: ","?> => {
+    l
-        if elements.len() == 1 && trailing_comma.is_none() {
+  }
-            elements.into_iter().next().unwrap()
-        } else {
-            ast::Expression::Tuple { elements }
-        }
-    }
 };
 
 // Test
@@ -903,16 +906,27 @@ StarExpr: ast::Expression = {
 };
 
 // Comprehensions:
-CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c;
+CompFor: Vec<ast::Comprehension> = {
+    <c:SingleForComprehension+> => c,
+};
 
 SingleForComprehension: ast::Comprehension = {
-    "for" <target:ExpressionList> "in" <iter:OrTest> <c2:ComprehensionIf*> => {
+    "for" <e:ExpressionList> "in" <i:OrTest> <c2:ComprehensionIf*> => {
-        ast::Comprehension { target, iter, ifs: c2 }
+        ast::Comprehension {
+            target: e,
+            iter: i,
+            ifs: c2,
+        }
     }
 };
 
-ExpressionNoCond: ast::Expression = OrTest;
+ExpressionNoCond: ast::Expression = {
-ComprehensionIf: ast::Expression = "if" <c:ExpressionNoCond> => c;
+    OrTest,
+};
+
+ComprehensionIf: ast::Expression = {
+    "if" <c:ExpressionNoCond> => c,
+};
 
 ArgumentList: (Vec<ast::Expression>, Vec<ast::Keyword>) = {
     <e: Comma<FunctionArgument>> => {
@@ -924,15 +938,8 @@ ArgumentList: (Vec<ast::Expression>, Vec<ast::Keyword>) = {
                     keywords.push(ast::Keyword { name: n, value: value });
                 },
                 None => {
-                    // Allow starred args after keyword arguments.
+                    if keywords.len() > 0 {
-                    let is_starred = if let ast::Expression::Starred { .. } = &value {
+                        panic!("positional argument follows keyword argument");
-                        true
-                    } else {
-                        false
-                    };
-
-                    if keywords.len() > 0 && !is_starred {
-                        panic!("positional argument follows keyword argument {:?}", keywords);
                     };
                     args.push(value);
                 },
@@ -976,33 +983,19 @@ OneOrMore<T>: Vec<T> = {
 };
 
 Number: ast::Number = {
-    <value:int> => { ast::Number::Integer { value } },
+    <s:int> => { ast::Number::Integer { value: s } },
-    <value:float> => { ast::Number::Float { value } },
+    <s:float> => { ast::Number::Float { value: s } },
     <s:complex> => { ast::Number::Complex { real: s.0, imag: s.1 } },
 };
 
-StringGroup: ast::StringGroup = {
+StringConstant: ast::Expression = {
-    <s:string+> =>? {
+    <s:string+> => {
-        let mut values = vec![];
+        let glued = s.join("");
-        for (value, is_fstring) in s {
+        ast::Expression::String { value: glued }
-            values.push(if is_fstring {
-                parse_fstring(&value)?
-            } else {
-                ast::StringGroup::Constant { value }
-            })
-        }
-
-        Ok(if values.len() > 1 {
-            ast::StringGroup::Joined { values }
-        } else {
-            values.into_iter().next().unwrap()
-        })
     },
-};
-
-Bytes: Vec<u8> = {
     <s:bytes+> => {
-        s.into_iter().flatten().collect::<Vec<u8>>()
+        let glued = s.into_iter().flatten().collect::<Vec<u8>>();
+        ast::Expression::Bytes { value: glued }
     },
 };
 
@@ -1024,7 +1017,6 @@ extern {
         "~" => lexer::Tok::Tilde,
         ":" => lexer::Tok::Colon,
         "." => lexer::Tok::Dot,
-        "..." => lexer::Tok::Ellipsis,
         "," => lexer::Tok::Comma,
         "*" => lexer::Tok::Star,
         "**" => lexer::Tok::DoubleStar,
@@ -1063,12 +1055,9 @@ extern {
         "<=" => lexer::Tok::LessEqual,
         ">" => lexer::Tok::Greater,
         ">=" => lexer::Tok::GreaterEqual,
-        "->" => lexer::Tok::Rarrow,
         "and" => lexer::Tok::And,
         "as" => lexer::Tok::As,
         "assert" => lexer::Tok::Assert,
-        "async" => lexer::Tok::Async,
-        "await" => lexer::Tok::Await,
         "break" => lexer::Tok::Break,
         "class" => lexer::Tok::Class,
         "continue" => lexer::Tok::Continue,
@@ -1103,7 +1092,7 @@ extern {
         int => lexer::Tok::Int { value: <BigInt> },
         float => lexer::Tok::Float { value: <f64> },
         complex => lexer::Tok::Complex { real: <f64>, imag: <f64> },
-        string => lexer::Tok::String { value: <String>, is_fstring: <bool> },
+        string => lexer::Tok::String { value: <String> },
         bytes => lexer::Tok::Bytes { value: <Vec<u8>> },
         name => lexer::Tok::Name { name: <String> },
         "\n" => lexer::Tok::Newline,

parser/src/token.rs

@@ -3,13 +3,13 @@
 use num_bigint::BigInt;
 
 /// Python source code can be tokenized in a sequence of these tokens.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Debug, PartialEq)]
 pub enum Tok {
     Name { name: String },
     Int { value: BigInt },
     Float { value: f64 },
     Complex { real: f64, imag: f64 },
-    String { value: String, is_fstring: bool },
+    String { value: String },
     Bytes { value: Vec<u8> },
     Newline,
     Indent,
@@ -17,7 +17,6 @@ pub enum Tok {
     StartProgram,
     StartStatement,
     StartExpression,
-    EndOfFile,
     Lpar,
     Rpar,
     Lsqb,
@@ -73,8 +72,6 @@ pub enum Tok {
     And,
     As,
     Assert,
-    Async,
-    Await,
     Break,
     Class,
     Continue,

py_code_object/Cargo.toml

@@ -2,7 +2,6 @@
 name = "py_code_object"
 version = "0.1.0"
 authors = ["Shing Lyu <shing.lyu@gmail.com>"]
-edition = "2018"
 
 [dependencies]
 log = "0.3"

py_code_object/python_compiler/Cargo.toml

@@ -2,7 +2,6 @@
 name = "python_compiler"
 version = "0.1.0"
 authors = ["Shing Lyu <shing.lyu@gmail.com>"]
-edition = "2018"
 
 [dependencies]
 cpython = { git = "https://github.com/dgrunwald/rust-cpython.git" }

py_code_object/src/vm_old.rs

@@ -56,7 +56,7 @@ impl VirtualMachine {
         }
     }
 
-    // Can we get rid of the code parameter?
+    // Can we get rid of the code paramter?
 
     fn make_frame(&self, code: PyCodeObject, callargs: HashMap<String, Rc<NativeType>>, globals: Option<HashMap<String, Rc<NativeType>>>) -> Frame {
         //populate the globals and locals
@@ -345,7 +345,7 @@ impl VirtualMachine {
                 let exception = match argc {
                     1 => curr_frame.stack.pop().unwrap(),
                     0 | 2 | 3 => panic!("Not implemented!"),
-                    _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3")
+                    _ => panic!("Invalid paramter for RAISE_VARARGS, must be between 0 to 3")
                 };
                 panic!("{:?}", exception);
             }

rustfmt.toml

@@ -1 +0,0 @@
-edition = "2018"

src/main.rs

@@ -1,23 +1,23 @@
+//extern crate rustpython_parser;
 #[macro_use]
 extern crate clap;
 extern crate env_logger;
 #[macro_use]
 extern crate log;
+extern crate rustpython_parser;
+extern crate rustpython_vm;
 extern crate rustyline;
 
 use clap::{App, Arg};
-use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType};
+use rustpython_parser::parser;
-use rustpython_parser::error::ParseErrorType;
+use rustpython_vm::obj::objstr;
-use rustpython_vm::{
+use rustpython_vm::print_exception;
-    frame::Scope,
+use rustpython_vm::pyobject::{AttributeProtocol, PyObjectRef, PyResult};
-    import,
+use rustpython_vm::VirtualMachine;
-    obj::objstr,
+use rustpython_vm::{compile, import};
-    print_exception,
+use rustyline::error::ReadlineError;
-    pyobject::{ItemProtocol, PyResult},
+use rustyline::Editor;
-    util, VirtualMachine,
+use std::path::Path;
-};
-
-use rustyline::{error::ReadlineError, Editor};
 use std::path::PathBuf;
 
 fn main() {
@@ -49,231 +49,167 @@ fn main() {
         .get_matches();
 
     // Construct vm:
-    let vm = VirtualMachine::new();
+    let mut vm = VirtualMachine::new();
-
-    let res = import::init_importlib(&vm, true);
-    handle_exception(&vm, res);
 
     // Figure out if a -c option was given:
     let result = if let Some(command) = matches.value_of("c") {
-        run_command(&vm, command.to_string())
+        run_command(&mut vm, command.to_string())
     } else if let Some(module) = matches.value_of("m") {
-        run_module(&vm, module)
+        run_module(&mut vm, module)
     } else {
         // Figure out if a script was passed:
         match matches.value_of("script") {
-            None => run_shell(&vm),
+            None => run_shell(&mut vm),
-            Some(filename) => run_script(&vm, filename),
+            Some(filename) => run_script(&mut vm, &filename.to_string()),
         }
     };
 
     // See if any exception leaked out:
-    handle_exception(&vm, result);
+    handle_exception(&mut vm, result);
 }
 
-fn _run_string(vm: &VirtualMachine, source: &str, source_path: String) -> PyResult {
+fn _run_string(vm: &mut VirtualMachine, source: &str, source_path: Option<String>) -> PyResult {
-    let code_obj = vm
+    let code_obj = compile::compile(vm, &source.to_string(), compile::Mode::Exec, source_path)?;
-        .compile(source, &compile::Mode::Exec, source_path.clone())
-        .map_err(|err| vm.new_syntax_error(&err))?;
     // trace!("Code object: {:?}", code_obj.borrow());
-    let attrs = vm.ctx.new_dict();
+    let builtins = vm.get_builtin_scope();
-    attrs.set_item("__file__", vm.new_str(source_path), vm)?;
+    let vars = vm.context().new_scope(Some(builtins)); // Keep track of local variables
-    vm.run_code_obj(code_obj, Scope::with_builtins(None, attrs, vm))
+    vm.run_code_obj(code_obj, vars)
 }
 
-fn handle_exception(vm: &VirtualMachine, result: PyResult) {
+fn handle_exception(vm: &mut VirtualMachine, result: PyResult) {
-    if let Err(err) = result {
+    match result {
-        print_exception(vm, &err);
+        Ok(_value) => {}
-        std::process::exit(1);
+        Err(err) => {
+            print_exception(vm, &err);
+            std::process::exit(1);
+        }
     }
 }
 
-fn run_command(vm: &VirtualMachine, mut source: String) -> PyResult {
+fn run_command(vm: &mut VirtualMachine, mut source: String) -> PyResult {
     debug!("Running command {}", source);
 
     // This works around https://github.com/RustPython/RustPython/issues/17
-    source.push('\n');
+    source.push_str("\n");
-    _run_string(vm, &source, "<stdin>".to_string())
+    _run_string(vm, &source, None)
 }
 
-fn run_module(vm: &VirtualMachine, module: &str) -> PyResult {
+fn run_module(vm: &mut VirtualMachine, module: &str) -> PyResult {
     debug!("Running module {}", module);
-    vm.import(module, &vm.ctx.new_tuple(vec![]), 0)
+    let current_path = PathBuf::from(".");
+    import::import_module(vm, current_path, module)
 }
 
-fn run_script(vm: &VirtualMachine, script_file: &str) -> PyResult {
+fn run_script(vm: &mut VirtualMachine, script_file: &str) -> PyResult {
     debug!("Running file {}", script_file);
     // Parse an ast from it:
-    let file_path = PathBuf::from(script_file);
+    let filepath = Path::new(script_file);
-    let file_path = if file_path.is_file() {
+    match parser::read_file(filepath) {
-        file_path
+        Ok(source) => _run_string(vm, &source, Some(filepath.to_str().unwrap().to_string())),
-    } else if file_path.is_dir() {
+        Err(msg) => {
-        let main_file_path = file_path.join("__main__.py");
+            error!("Parsing went horribly wrong: {}", msg);
-        if main_file_path.is_file() {
-            main_file_path
-        } else {
-            error!(
-                "can't find '__main__' module in '{}'",
-                file_path.to_str().unwrap()
-            );
-            std::process::exit(1);
-        }
-    } else {
-        error!(
-            "can't open file '{}': No such file or directory",
-            file_path.to_str().unwrap()
-        );
-        std::process::exit(1);
-    };
-
-    let dir = file_path.parent().unwrap().to_str().unwrap().to_string();
-    let sys_path = vm.get_attribute(vm.sys_module.clone(), "path").unwrap();
-    vm.call_method(&sys_path, "insert", vec![vm.new_int(0), vm.new_str(dir)])?;
-
-    match util::read_file(&file_path) {
-        Ok(source) => _run_string(vm, &source, file_path.to_str().unwrap().to_string()),
-        Err(err) => {
-            error!(
-                "Failed reading file '{}': {:?}",
-                file_path.to_str().unwrap(),
-                err.kind()
-            );
             std::process::exit(1);
         }
     }
 }
 
-#[test]
+fn shell_exec(vm: &mut VirtualMachine, source: &str, scope: PyObjectRef) -> bool {
-fn test_run_script() {
+    match compile::compile(vm, &source.to_string(), compile::Mode::Single, None) {
-    let vm = VirtualMachine::new();
-
-    // test file run
-    let r = run_script(&vm, "tests/snippets/dir_main/__main__.py");
-    assert!(r.is_ok());
-
-    // test module run
-    let r = run_script(&vm, "tests/snippets/dir_main");
-    assert!(r.is_ok());
-}
-
-fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), CompileError> {
-    match vm.compile(source, &compile::Mode::Single, "<stdin>".to_string()) {
         Ok(code) => {
-            match vm.run_code_obj(code, scope.clone()) {
+            match vm.run_code_obj(code, scope) {
-                Ok(value) => {
+                Ok(_value) => {
-                    // Save non-None values as "_"
+                    // Printed already.
-
-                    use rustpython_vm::pyobject::{IdProtocol, IntoPyObject};
-
-                    if !value.is(&vm.get_none()) {
-                        let key = objstr::PyString::from("_").into_pyobject(vm);
-                        scope.globals.set_item(key, value, vm).unwrap();
-                    }
                 }
-
                 Err(err) => {
                     print_exception(vm, &err);
                 }
             }
-
-            Ok(())
         }
-        // Don't inject syntax errors for line continuation
-        Err(
-            err @ CompileError {
-                error: CompileErrorType::Parse(ParseErrorType::EOF),
-                ..
-            },
-        ) => Err(err),
         Err(err) => {
-            let exc = vm.new_syntax_error(&err);
+            // Enum rather than special string here.
-            print_exception(vm, &exc);
+            let name = vm.new_str("msg".to_string());
-            Err(err)
+            let msg = match vm.get_attribute(err.clone(), name) {
+                Ok(value) => objstr::get_value(&value),
+                Err(_) => panic!("Expected msg attribute on exception object!"),
+            };
+            if msg == "Unexpected end of input." {
+                return false;
+            } else {
+                print_exception(vm, &err);
+            }
         }
-    }
+    };
+    true
 }
 
-#[cfg(not(unix))]
+fn run_shell(vm: &mut VirtualMachine) -> PyResult {
-fn get_history_path() -> PathBuf {
-    PathBuf::from(".repl_history.txt")
-}
-
-#[cfg(unix)]
-fn get_history_path() -> PathBuf {
-    //work around for windows dependent builds. The xdg crate is unix specific
-    //so access to the BaseDirectories struct breaks builds on python.
-    extern crate xdg;
-
-    let xdg_dirs = xdg::BaseDirectories::with_prefix("rustpython").unwrap();
-    xdg_dirs.place_cache_file("repl_history.txt").unwrap()
-}
-
-fn get_prompt(vm: &VirtualMachine, prompt_name: &str) -> String {
-    vm.get_attribute(vm.sys_module.clone(), prompt_name)
-        .ok()
-        .as_ref()
-        .map(objstr::get_value)
-        .unwrap_or_else(String::new)
-}
-
-fn run_shell(vm: &VirtualMachine) -> PyResult {
     println!(
-        "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}",
+        "Welcome to the magnificent Rust Python {} interpreter",
         crate_version!()
     );
-    let vars = vm.new_scope_with_builtins();
+    let builtins = vm.get_builtin_scope();
+    let vars = vm.context().new_scope(Some(builtins)); // Keep track of local variables
 
     // Read a single line:
     let mut input = String::new();
-    let mut repl = Editor::<()>::new();
+    let mut rl = Editor::<()>::new();
 
-    // Retrieve a `history_path_str` dependent on the OS
+    // TODO: Store the history in a proper XDG directory
-    let repl_history_path_str = &get_history_path();
+    let repl_history_path = ".repl_history.txt";
-    if repl.load_history(repl_history_path_str).is_err() {
+    if rl.load_history(repl_history_path).is_err() {
         println!("No previous history.");
     }
 
-    let mut continuing = false;
-
     loop {
-        let prompt = if continuing {
+        // TODO: modules dont support getattr / setattr yet
-            get_prompt(vm, "ps2")
+        //let prompt = match vm.get_attribute(vm.sys_module.clone(), "ps1") {
-        } else {
+        //        Ok(value) => objstr::get_value(&value),
-            get_prompt(vm, "ps1")
+        //        Err(_) => ">>>>> ".to_string(),
-        };
+        //};
-        match repl.readline(&prompt) {
+
+        // We can customize the prompt:
+        let ps1 = objstr::get_value(&vm.sys_module.get_attr("ps1").unwrap());
+        let ps2 = objstr::get_value(&vm.sys_module.get_attr("ps2").unwrap());
+
+        match rl.readline(&ps1) {
             Ok(line) => {
-                debug!("You entered {:?}", line);
                 input.push_str(&line);
-                input.push('\n');
+                input.push_str("\n");
-                repl.add_history_entry(line.trim_end());
 
-                if continuing {
+                debug!("You entered {:?}", input);
-                    if line.is_empty() {
+                if shell_exec(vm, &input, vars.clone()) {
-                        continuing = false;
+                    // Line was complete.
-                    } else {
+                    rl.add_history_entry(input.trim_right().as_ref());
-                        continue;
+                    input = String::new();
-                    }
+                } else {
-                }
+                    loop {
-
+                        // until an empty line is pressed AND the code is complete
-                match shell_exec(vm, &input, vars.clone()) {
+                        //let prompt = match vm.get_attribute(vm.sys_module.clone(), "ps2") {
-                    Err(CompileError {
+                        //        Ok(value) => objstr::get_value(&value),
-                        error: CompileErrorType::Parse(ParseErrorType::EOF),
+                        //        Err(_) => "..... ".to_string(),
-                        ..
+                        //};
-                    }) => {
+                        match rl.readline(&ps2) {
-                        continuing = true;
+                            Ok(line) => {
-                        continue;
+                                if line.len() == 0 {
-                    }
+                                    if shell_exec(vm, &input, vars.clone()) {
-                    _ => {
+                                        rl.add_history_entry(input.trim_right().as_ref());
-                        input = String::new();
+                                        input = String::new();
+                                        break;
+                                    }
+                                } else {
+                                    input.push_str(&line);
+                                    input.push_str("\n");
+                                }
+                            }
+                            Err(msg) => panic!("Error: {:?}", msg),
+                        }
                     }
                 }
             }
             Err(ReadlineError::Interrupted) => {
                 // TODO: Raise a real KeyboardInterrupt exception
                 println!("^C");
-                continuing = false;
+                break;
-                continue;
             }
             Err(ReadlineError::Eof) => {
                 break;
@@ -284,7 +220,7 @@ fn run_shell(vm: &VirtualMachine) -> PyResult {
             }
         };
     }
-    repl.save_history(repl_history_path_str).unwrap();
+    rl.save_history(repl_history_path).unwrap();
 
     Ok(vm.get_none())
 }

tests/.gitignore

@@ -1 +0,0 @@
-snippets/whats_left_*.py

tests/.travis-runner.sh

@@ -10,33 +10,7 @@ pip install pipenv
 (cd tests; pipenv install)
 
 # Build outside of the test runner
-if [ $CODE_COVERAGE = "true" ]
+cargo build --verbose --release
-then
-    find . -name '*.gcda' -delete
-    find . -name '*.gcno' -delete
-
-    export CARGO_INCREMENTAL=0
-    export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads"
-
-    cargo build --verbose
-else
-    cargo build --verbose --release
-fi
 
 # Run the tests
 (cd tests; pipenv run pytest)
-
-if [ $CODE_COVERAGE = "true" ]
-then
-    cargo test --verbose --all
-    zip -0 ccov.zip `find . \( -name "rustpython*.gc*" \) -print`
-
-    # Install grcov
-    curl -L https://github.com/mozilla/grcov/releases/download/v0.4.2/grcov-linux-x86_64.tar.bz2 | tar jxf -
-
-    ./grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir "/*" -p "x" > lcov.info
-
-    # Install codecov.io reporter
-    curl -s https://codecov.io/bash -o codecov.sh
-    bash codecov.sh -f lcov.info
-fi

tests/Pipfile

@@ -10,4 +10,4 @@ pytest = "*"
 [dev-packages]
 
 [requires]
-python_version = "3.6"
+python_version = "3"

tests/Pipfile.lock

@@ -1,11 +1,11 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "ce98de5914393363a8cb86a4753b3964caa53a4659a403a3ef357e2086363ef7"
+            "sha256": "b2d2d68e7d4330ff8d889816c56b9cee4bf54962c86b2c11382108176a201ec8"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_version": "3.6"
+            "python_version": "3"
         },
         "sources": [
             {
@@ -16,99 +16,74 @@
         ]
     },
     "default": {
+        "aenum": {
+            "hashes": [
+                "sha256:3df9b84cce5dc9ed77c337079f97b66c44c0053eb87d6f4d46b888dc45801e38",
+                "sha256:7a77c205c4bc9d7fe9bd73b3193002d724aebf5909fa0d297534208953891ec8",
+                "sha256:a3208e4b28db3a7b232ff69b934aef2ea1bf27286d9978e1e597d46f490e4687"
+            ],
+            "version": "==2.1.2"
+        },
         "atomicwrites": {
             "hashes": [
-                "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
+                "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585",
-                "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
+                "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6"
             ],
-            "version": "==1.3.0"
+            "version": "==1.1.5"
         },
         "attrs": {
             "hashes": [
-                "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
+                "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265",
-                "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
+                "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b"
             ],
-            "version": "==19.1.0"
+            "version": "==18.1.0"
         },
         "bytecode": {
             "hashes": [
-                "sha256:68b1d591c7af0e5c5273e028d3cc0299fbe374dff0cf9149ec7e569be0c573e7",
+                "sha256:cc6931151c7f0a542f8cf7619fe1639af3b9529c4678860fa3239397cb0f7de0",
-                "sha256:c43d5052cbff076bfdf5b0b93ff6c76e461aab628ce47d30637bb200b6b7bb2c"
+                "sha256:e464004d4a9eeeca987cb4950dba11b827964b6c90cd331c1f20abd2dab3c962"
             ],
             "index": "pypi",
-            "version": "==0.8.0"
+            "version": "==0.7.0"
-        },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7",
-                "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db"
-            ],
-            "version": "==0.18"
         },
         "more-itertools": {
             "hashes": [
-                "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7",
+                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
-                "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a"
+                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
+                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
             ],
-            "markers": "python_version > '2.7'",
+            "version": "==4.3.0"
-            "version": "==7.0.0"
-        },
-        "packaging": {
-            "hashes": [
-                "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af",
-                "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3"
-            ],
-            "version": "==19.0"
         },
         "pluggy": {
             "hashes": [
-                "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc",
+                "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1",
-                "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c"
+                "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1"
             ],
-            "version": "==0.12.0"
+            "markers": "python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.2.*'",
+            "version": "==0.7.1"
         },
         "py": {
             "hashes": [
-                "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
+                "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7",
-                "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
+                "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e"
             ],
-            "version": "==1.8.0"
+            "markers": "python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.2.*'",
-        },
+            "version": "==1.5.4"
-        "pyparsing": {
-            "hashes": [
-                "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a",
-                "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03"
-            ],
-            "version": "==2.4.0"
         },
         "pytest": {
             "hashes": [
-                "sha256:4a784f1d4f2ef198fe9b7aef793e9fa1a3b2f84e822d9b3a64a181293a572d45",
+                "sha256:86a8dbf407e437351cef4dba46736e9c5a6e3c3ac71b2e942209748e76ff2086",
-                "sha256:926855726d8ae8371803f7b2e6ec0a69953d9c6311fa7c3b6c1b929ff92d27da"
+                "sha256:e74466e97ac14582a8188ff4c53e6cc3810315f342f6096899332ae864c1d432"
             ],
             "index": "pypi",
-            "version": "==4.6.3"
+            "version": "==3.7.1"
         },
         "six": {
             "hashes": [
-                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
-                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
+                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
             ],
-            "version": "==1.12.0"
+            "version": "==1.11.0"
-        },
-        "wcwidth": {
-            "hashes": [
-                "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
-                "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
-            ],
-            "version": "==0.1.7"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d",
-                "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3"
-            ],
-            "version": "==0.5.1"
         }
     },
     "develop": {}

tests/README.md

@@ -1,3 +1,4 @@
+
 # Test snippets
 
 This directory contains two sets of test snippets which can be run in
@@ -5,10 +6,6 @@ Python.  The `snippets/` directory contains functional tests, and the
 `benchmarks/` directory contains snippets for use in benchmarking
 RustPython's performance.
 
-## Generates the test for not implemented methods
-
-run using cpython not_impl_gen.py it automatically generate a
-test snippet to check not yet implemented methods
 
 ## Running with CPython + RustPython
 
@@ -18,3 +15,4 @@ compilation to bytecode. When this is done, run the bytecode with rustpython.
 ## Running with RustPython
 
 The other option is to run all snippets with RustPython.
+

tests/generator/not_impl_methods_footer.txt

@@ -1,12 +0,0 @@
-not_implemented = [(name, method)
-                   for name, (val, methods) in expected_methods.items()
-                   for method in methods
-                   if not hasattr(val, method)]
-
-for r in not_implemented:
-    print(r[0], ".", r[1], sep="")
-if not not_implemented:
-    print("Not much \\o/")
-
-if platform.python_implementation() == "CPython":
-    assert len(not_implemented) == 0, "CPython should have all the methods"