mschoi/RustPython
1
0
Fork 0
forked from Rust-related/RustPython
Code Pull Requests Actions Activity

resolve conflicts

Browse Source
This commit is contained in:
yjhmelody
2019-08-17 16:57:10 +08:00
parent bb9945af61 e3344df00e
commit 11ba8468ff
137 changed files with 6293 additions and 1426 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
.travis.yml
View File

@@ -1,9 +1,13 @@
before_cache: |
if command -v cargo; then
! command -v cargo-sweep && cargo install cargo-sweep
cargo sweep -i
cargo sweep -t 15
fi
before_cache:
- |
if command -v cargo; then
if ! command -v cargo-sweep; then
cargo install cargo-sweep
fi
cargo sweep -i
cargo sweep -t 15
fi
- rm -rf ~/.cargo/registry/src
matrix:
fast_finish: true
@@ -145,3 +149,44 @@ matrix:
env:
- JOBCACHE=7
- TRAVIS_RUST_VERSION=stable
- name: Ensure compilation on Redox OS with Redoxer
# language: minimal so that it actually uses bionic rather than xenial;
# rust isn't yet available on bionic
language: minimal
dist: bionic
if: type = cron
cache:
cargo: true
directories:
- $HOME/.redoxer
- $HOME/.cargo
before_install:
# install rust as travis does for language: rust
- curl -sSf https://build.travis-ci.org/files/rustup-init.sh | sh -s --
--default-toolchain=$TRAVIS_RUST_VERSION -y
- export PATH=${TRAVIS_HOME}/.cargo/bin:$PATH
- rustc --version
- rustup --version
- cargo --version
- sudo apt-get update -qq
- sudo apt-get install libfuse-dev
install:
- if ! command -v redoxer; then cargo install redoxfs redoxer; fi
- redoxer install
script:
- bash redox/uncomment-cargo.sh
- redoxer build --verbose
- bash redox/comment-cargo.sh
before_cache:
- |
if ! command -v cargo-sweep; then
rustup install stable
cargo +stable install cargo-sweep
fi
- cargo sweep -t 15
- rm -rf ~/.cargo/registry/src
env:
- JOBCACHE=10
- TRAVIS_RUST_VERSION=nightly

538
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

5
Cargo.toml
View File

@@ -42,8 +42,9 @@ version = "0.2"
name = "rustpython"
path = "src/main.rs"
# Uncommment when you want to compile/check with redoxer
# [patch.crates-io]
[patch.crates-io]
# REDOX START, Uncommment when you want to compile/check with redoxer
# time = { git = "https://gitlab.redox-os.org/redox-os/time.git", branch = "redox-unix" }
# libc = { git = "https://github.com/AdminXVII/libc", branch = "extra-traits-redox" }
# nix = { git = "https://github.com/AdminXVII/nix", branch = "add-redox-support" }
# REDOX END

165
DEVELOPMENT.md
View File

@@ -1,9 +1,91 @@
# RustPython Development Guide and Tips
RustPython attracts developers with interest and experience in Rust, Python,
or WebAssembly. Whether you are familiar with Rust, Python, or
WebAssembly, the goal of this Development Guide is to give you the basics to
get set up for developing RustPython and contributing to this project.
The contents of the Development Guide include:
- [Setting up a development environment](#setting-up-a-development-environment)
- [Code style](#code-style)
- [Testing](#testing)
- [Profiling](#profiling)
- [Code organization](#code-organization)
- [Understanding internals](#understanding-internals)
- [Questions](#questions)
## Setting up a development environment
RustPython requires the following:
- Rust 1.36 or higher
- To check Rust version: `rustc --version`
- If you have `rustup` on your system, enter to update to the latest
stable version: `rustup update stable`
- If you do not have Rust installed, use [rustup](https://rustup.rs/) to
do so.
- CPython version 3.7.4 or higher
- CPython can be installed by your operating system's package manager,
from the [Python website](https://www.python.org/downloads/), or
using a third-party distribution, such as
[Anaconda](https://www.anaconda.com/distribution/).
- [Optional] The Python package, `pytest`, is used for testing Python code
snippets. To install, enter `python3 -m pip install pytest`.
## Code style
The Rust code style used is the default
[rustfmt](https://github.com/rust-lang/rustfmt) codestyle. Please format your
code accordingly. We also use [clippy](https://github.com/rust-lang/rust-clippy)
to detect rust code issues.
Python code should follow the
[PEP 8](https://www.python.org/dev/peps/pep-0008/) style. We also use
[flake8](http://flake8.pycqa.org/en/latest/) to check Python code style.
## Testing
To test RustPython's functionality, a collection of Python snippets is located
in the `tests/snippets` directory and can be run using `pytest`:
```shell
$ cd tests
$ pytest -v
```
Rust unit tests can be run with `cargo`:
```shell
$ cargo test --all
```
## Profiling
To profile RustPython, build it in `release` mode with the `flame-it` feature.
This will generate a file `flamescope.json`, which can be viewed at
https://speedscope.app.
```shell
$ cargo run --release --features flame-it script.py
$ cat flamescope.json
{<json>}
```
You can specify another file name other than the default by using the
`--output-file` option to specify a file name (or `stdout` if you specify `-`).
The `--output-format` option determines the format of the output file.
The speedscope json format (default), text, or raw html can be passed. There
exists a raw html viewer which is currently broken, and we welcome a PR to fix it.
## Code organization
Understanding a new codebase takes time. Here's a brief view of the
repository's structure:
- `bytecode/src`: python bytecode representation in rust structures
- `compiler/src`: python compilation to bytecode
- `derive/src`: Rust language extensions and macros specific to rustpython
- `parser/src`: python lexing, parsing and ast
- `Lib`: Carefully selected / copied files from CPython sourcecode. This is
the python side of the standard library.
@@ -19,42 +101,67 @@
- `wasm`: Binary crate and resources for WebAssembly build
- `tests`: integration test snippets
## Code style
## Understanding Internals
The code style used is the default
[rustfmt](https://github.com/rust-lang/rustfmt) codestyle. Please format your
code accordingly. We also use [clippy](https://github.com/rust-lang/rust-clippy)
to detect rust code issues.
The RustPython workspace includes the `rustpython` top-level crate. The `Cargo.toml`
file in the root of the repo provide configuration of the crate and the
implementation is found in the `src` directory (specifically,
`src/main.rs`).
## Testing
The top-level `rustpython` binary depends on several lower-level crates including:
To test rustpython, there is a collection of python snippets located in the
`tests/snippets` directory. To run those tests do the following:
- `rustpython-parser` (implementation in `parser/src`)
- `rustpython-compiler` (implementation in `compiler/src`)
- `rustpython-vm` (implementation in `vm/src`)
```shell
$ cd tests
$ pytest -v
```
Together, these crates provide the functions of a programming language and
enable a line of code to go through a series of steps:
There also are some unit tests, you can run those with cargo:
- parse the line of source code into tokens
- determine if the tokens are valid syntax
- create an Abstract Syntax Tree (AST)
- compile the AST into bytecode
- execute the bytecode in the virtual machine (VM).
```shell
$ cargo test --all
```
### rustpython-parser
## Profiling
This crate contains the lexer and parser to convert a line of code to
an Abstract Syntax Tree (AST):
To profile rustpython, simply build in release mode with the `flame-it` feature.
This will generate a file `flamescope.json`, which you can then view at
https://speedscope.app.
- Lexer: `parser/lexer.rs` converts Python source code into tokens
- Parser: `parser/parser.rs` takes the tokens generated by the lexer and parses
the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax
tree are Rust structs and enums.
- The Parser relies on `LALRPOP`, a Rust parser generator framework.
- More information on parsers and a tutorial can be found in the
[LALRPOP book](https://lalrpop.github.io/lalrpop/README.html).
- AST: `parser/ast.rs` implements in Rust the Python types and expressions
represented by the AST nodes.
```sh
$ cargo run --release --features flame-it script.py
$ cat flamescope.json
{<json>}
```
### rustpython-compiler
You can also pass the `--output-file` option to choose which file to output to
(or stdout if you specify `-`), and the `--output-format` option to choose if
you want to output in the speedscope json format (default), text, or a raw html
viewer (currently broken).
The `rustpython-compiler` crate's purpose is to transform the AST (Abstract Syntax
Tree) to bytecode. The implementation of the compiler is found in the
`compiler/src` directory. The compiler implements Python's peephole optimizer
implementation, Symbol table, and streams in Rust.
Implementation of bytecode structure in Rust is found in the `bytecode/src`
directory. The `bytecode/src/bytecode.rs` contains the representation of
instructions and operations in Rust. Further information about Python's
bytecode instructions can be found in the
[Python documentation](https://docs.python.org/3/library/dis.html#bytecodes).
### rustpython-vm
The `rustpython-vm` crate has the important job of running the virtual machine that
executes Python's instructions. The `vm/src` directory contains code to
implement the read and evaluation loop that fetches and dispatches
instructions. This directory also contains the implementation of the
Python Standard Library modules in Rust (`vm/src/stdlib`). In Python
everything can be represented as an Object. `vm/src/obj` directory holds
the Rust code used to represent a Python Object and its methods.
## Questions
Have you tried these steps and have a question, please chat with us on
[gitter](https://gitter.im/rustpython/Lobby).

951
Lib/aifc.py Normal file
View File

@@ -0,0 +1,951 @@
"""Stuff to parse AIFF-C and AIFF files.
Unless explicitly stated otherwise, the description below is true
both for AIFF-C files and AIFF files.
An AIFF-C file has the following structure.
+-----------------+
| FORM |
+-----------------+
| <size> |
+----+------------+
| | AIFC |
| +------------+
| | <chunks> |
| | . |
| | . |
| | . |
+----+------------+
An AIFF file has the string "AIFF" instead of "AIFC".
A chunk consists of an identifier (4 bytes) followed by a size (4 bytes,
big endian order), followed by the data. The size field does not include
the size of the 8 byte header.
The following chunk types are recognized.
FVER
<version number of AIFF-C defining document> (AIFF-C only).
MARK
<# of markers> (2 bytes)
list of markers:
<marker ID> (2 bytes, must be > 0)
<position> (4 bytes)
<marker name> ("pstring")
COMM
<# of channels> (2 bytes)
<# of sound frames> (4 bytes)
<size of the samples> (2 bytes)
<sampling frequency> (10 bytes, IEEE 80-bit extended
floating point)
in AIFF-C files only:
<compression type> (4 bytes)
<human-readable version of compression type> ("pstring")
SSND
<offset> (4 bytes, not used by this program)
<blocksize> (4 bytes, not used by this program)
<sound data>
A pstring consists of 1 byte length, a string of characters, and 0 or 1
byte pad to make the total length even.
Usage.
Reading AIFF files:
f = aifc.open(file, 'r')
where file is either the name of a file or an open file pointer.
The open file pointer must have methods read(), seek(), and close().
In some types of audio files, if the setpos() method is not used,
the seek() method is not necessary.
This returns an instance of a class with the following public methods:
getnchannels() -- returns number of audio channels (1 for
mono, 2 for stereo)
getsampwidth() -- returns sample width in bytes
getframerate() -- returns sampling frequency
getnframes() -- returns number of audio frames
getcomptype() -- returns compression type ('NONE' for AIFF files)
getcompname() -- returns human-readable version of
compression type ('not compressed' for AIFF files)
getparams() -- returns a namedtuple consisting of all of the
above in the above order
getmarkers() -- get the list of marks in the audio file or None
if there are no marks
getmark(id) -- get mark with the specified id (raises an error
if the mark does not exist)
readframes(n) -- returns at most n frames of audio
rewind() -- rewind to the beginning of the audio stream
setpos(pos) -- seek to the specified position
tell() -- return the current position
close() -- close the instance (make it unusable)
The position returned by tell(), the position given to setpos() and
the position of marks are all compatible and have nothing to do with
the actual position in the file.
The close() method is called automatically when the class instance
is destroyed.
Writing AIFF files:
f = aifc.open(file, 'w')
where file is either the name of a file or an open file pointer.
The open file pointer must have methods write(), tell(), seek(), and
close().
This returns an instance of a class with the following public methods:
aiff() -- create an AIFF file (AIFF-C default)
aifc() -- create an AIFF-C file
setnchannels(n) -- set the number of channels
setsampwidth(n) -- set the sample width
setframerate(n) -- set the frame rate
setnframes(n) -- set the number of frames
setcomptype(type, name)
-- set the compression type and the
human-readable compression type
setparams(tuple)
-- set all parameters at once
setmark(id, pos, name)
-- add specified mark to the list of marks
tell() -- return current position in output file (useful
in combination with setmark())
writeframesraw(data)
-- write audio frames without pathing up the
file header
writeframes(data)
-- write audio frames and patch up the file header
close() -- patch up the file header and close the
output file
You should set the parameters before the first writeframesraw or
writeframes. The total number of frames does not need to be set,
but when it is set to the correct value, the header does not have to
be patched up.
It is best to first set all parameters, perhaps possibly the
compression type, and then write audio frames using writeframesraw.
When all frames have been written, either call writeframes(b'') or
close() to patch up the sizes in the header.
Marks can be added anytime. If there are any marks, you must call
close() after all frames have been written.
The close() method is called automatically when the class instance
is destroyed.
When a file is opened with the extension '.aiff', an AIFF file is
written, otherwise an AIFF-C file is written. This default can be
changed by calling aiff() or aifc() before the first writeframes or
writeframesraw.
"""
import struct
import builtins
import warnings
__all__ = ["Error", "open", "openfp"]
class Error(Exception):
pass
_AIFC_version = 0xA2805140 # Version 1 of AIFF-C
def _read_long(file):
try:
return struct.unpack('>l', file.read(4))[0]
except struct.error:
raise EOFError from None
def _read_ulong(file):
try:
return struct.unpack('>L', file.read(4))[0]
except struct.error:
raise EOFError from None
def _read_short(file):
try:
return struct.unpack('>h', file.read(2))[0]
except struct.error:
raise EOFError from None
def _read_ushort(file):
try:
return struct.unpack('>H', file.read(2))[0]
except struct.error:
raise EOFError from None
def _read_string(file):
length = ord(file.read(1))
if length == 0:
data = b''
else:
data = file.read(length)
if length & 1 == 0:
dummy = file.read(1)
return data
_HUGE_VAL = 1.79769313486231e+308 # See <limits.h>
def _read_float(f): # 10 bytes
expon = _read_short(f) # 2 bytes
sign = 1
if expon < 0:
sign = -1
expon = expon + 0x8000
himant = _read_ulong(f) # 4 bytes
lomant = _read_ulong(f) # 4 bytes
if expon == himant == lomant == 0:
f = 0.0
elif expon == 0x7FFF:
f = _HUGE_VAL
else:
expon = expon - 16383
f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63)
return sign * f
def _write_short(f, x):
f.write(struct.pack('>h', x))
def _write_ushort(f, x):
f.write(struct.pack('>H', x))
def _write_long(f, x):
f.write(struct.pack('>l', x))
def _write_ulong(f, x):
f.write(struct.pack('>L', x))
def _write_string(f, s):
if len(s) > 255:
raise ValueError("string exceeds maximum pstring length")
f.write(struct.pack('B', len(s)))
f.write(s)
if len(s) & 1 == 0:
f.write(b'\x00')
def _write_float(f, x):
import math
if x < 0:
sign = 0x8000
x = x * -1
else:
sign = 0
if x == 0:
expon = 0
himant = 0
lomant = 0
else:
fmant, expon = math.frexp(x)
if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN
expon = sign|0x7FFF
himant = 0
lomant = 0
else: # Finite
expon = expon + 16382
if expon < 0: # denormalized
fmant = math.ldexp(fmant, expon)
expon = 0
expon = expon | sign
fmant = math.ldexp(fmant, 32)
fsmant = math.floor(fmant)
himant = int(fsmant)
fmant = math.ldexp(fmant - fsmant, 32)
fsmant = math.floor(fmant)
lomant = int(fsmant)
_write_ushort(f, expon)
_write_ulong(f, himant)
_write_ulong(f, lomant)
from chunk import Chunk
from collections import namedtuple
_aifc_params = namedtuple('_aifc_params',
'nchannels sampwidth framerate nframes comptype compname')
_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)'
_aifc_params.sampwidth.__doc__ = 'Sample width in bytes'
_aifc_params.framerate.__doc__ = 'Sampling frequency'
_aifc_params.nframes.__doc__ = 'Number of audio frames'
_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)'
_aifc_params.compname.__doc__ = ("""\
A human-readable version of the compression type
('not compressed' for AIFF files)""")
class Aifc_read:
# Variables used in this class:
#
# These variables are available to the user though appropriate
# methods of this class:
# _file -- the open file with methods read(), close(), and seek()
# set through the __init__() method
# _nchannels -- the number of audio channels
# available through the getnchannels() method
# _nframes -- the number of audio frames
# available through the getnframes() method
# _sampwidth -- the number of bytes per audio sample
# available through the getsampwidth() method
# _framerate -- the sampling frequency
# available through the getframerate() method
# _comptype -- the AIFF-C compression type ('NONE' if AIFF)
# available through the getcomptype() method
# _compname -- the human-readable AIFF-C compression type
# available through the getcomptype() method
# _markers -- the marks in the audio file
# available through the getmarkers() and getmark()
# methods
# _soundpos -- the position in the audio stream
# available through the tell() method, set through the
# setpos() method
#
# These variables are used internally only:
# _version -- the AIFF-C version number
# _decomp -- the decompressor from builtin module cl
# _comm_chunk_read -- 1 iff the COMM chunk has been read
# _aifc -- 1 iff reading an AIFF-C file
# _ssnd_seek_needed -- 1 iff positioned correctly in audio
# file for readframes()
# _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
# _framesize -- size of one frame in the file
_file = None # Set here since __del__ checks it
def initfp(self, file):
self._version = 0
self._convert = None
self._markers = []
self._soundpos = 0
self._file = file
chunk = Chunk(file)
if chunk.getname() != b'FORM':
raise Error('file does not start with FORM id')
formdata = chunk.read(4)
if formdata == b'AIFF':
self._aifc = 0
elif formdata == b'AIFC':
self._aifc = 1
else:
raise Error('not an AIFF or AIFF-C file')
self._comm_chunk_read = 0
self._ssnd_chunk = None
while 1:
self._ssnd_seek_needed = 1
try:
chunk = Chunk(self._file)
except EOFError:
break
chunkname = chunk.getname()
if chunkname == b'COMM':
self._read_comm_chunk(chunk)
self._comm_chunk_read = 1
elif chunkname == b'SSND':
self._ssnd_chunk = chunk
dummy = chunk.read(8)
self._ssnd_seek_needed = 0
elif chunkname == b'FVER':
self._version = _read_ulong(chunk)
elif chunkname == b'MARK':
self._readmark(chunk)
chunk.skip()
if not self._comm_chunk_read or not self._ssnd_chunk:
raise Error('COMM chunk and/or SSND chunk missing')
def __init__(self, f):
if isinstance(f, str):
file_object = builtins.open(f, 'rb')
try:
self.initfp(file_object)
except:
file_object.close()
raise
else:
# assume it is an open file object already
self.initfp(f)
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
#
# User visible methods.
#
def getfp(self):
return self._file
def rewind(self):
self._ssnd_seek_needed = 1
self._soundpos = 0
def close(self):
file = self._file
if file is not None:
self._file = None
file.close()
def tell(self):
return self._soundpos
def getnchannels(self):
return self._nchannels
def getnframes(self):
return self._nframes
def getsampwidth(self):
return self._sampwidth
def getframerate(self):
return self._framerate
def getcomptype(self):
return self._comptype
def getcompname(self):
return self._compname
## def getversion(self):
## return self._version
def getparams(self):
return _aifc_params(self.getnchannels(), self.getsampwidth(),
self.getframerate(), self.getnframes(),
self.getcomptype(), self.getcompname())
def getmarkers(self):
if len(self._markers) == 0:
return None
return self._markers
def getmark(self, id):
for marker in self._markers:
if id == marker[0]:
return marker
raise Error('marker {0!r} does not exist'.format(id))
def setpos(self, pos):
if pos < 0 or pos > self._nframes:
raise Error('position not in range')
self._soundpos = pos
self._ssnd_seek_needed = 1
def readframes(self, nframes):
if self._ssnd_seek_needed:
self._ssnd_chunk.seek(0)
dummy = self._ssnd_chunk.read(8)
pos = self._soundpos * self._framesize
if pos:
self._ssnd_chunk.seek(pos + 8)
self._ssnd_seek_needed = 0
if nframes == 0:
return b''
data = self._ssnd_chunk.read(nframes * self._framesize)
if self._convert and data:
data = self._convert(data)
self._soundpos = self._soundpos + len(data) // (self._nchannels
* self._sampwidth)
return data
#
# Internal methods.
#
def _alaw2lin(self, data):
import audioop
return audioop.alaw2lin(data, 2)
def _ulaw2lin(self, data):
import audioop
return audioop.ulaw2lin(data, 2)
def _adpcm2lin(self, data):
import audioop
if not hasattr(self, '_adpcmstate'):
# first time
self._adpcmstate = None
data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
return data
def _read_comm_chunk(self, chunk):
self._nchannels = _read_short(chunk)
self._nframes = _read_long(chunk)
self._sampwidth = (_read_short(chunk) + 7) // 8
self._framerate = int(_read_float(chunk))
if self._sampwidth <= 0:
raise Error('bad sample width')
if self._nchannels <= 0:
raise Error('bad # of channels')
self._framesize = self._nchannels * self._sampwidth
if self._aifc:
#DEBUG: SGI's soundeditor produces a bad size :-(
kludge = 0
if chunk.chunksize == 18:
kludge = 1
warnings.warn('Warning: bad COMM chunk size')
chunk.chunksize = 23
#DEBUG end
self._comptype = chunk.read(4)
#DEBUG start
if kludge:
length = ord(chunk.file.read(1))
if length & 1 == 0:
length = length + 1
chunk.chunksize = chunk.chunksize + length
chunk.file.seek(-1, 1)
#DEBUG end
self._compname = _read_string(chunk)
if self._comptype != b'NONE':
if self._comptype == b'G722':
self._convert = self._adpcm2lin
elif self._comptype in (b'ulaw', b'ULAW'):
self._convert = self._ulaw2lin
elif self._comptype in (b'alaw', b'ALAW'):
self._convert = self._alaw2lin
else:
raise Error('unsupported compression type')
self._sampwidth = 2
else:
self._comptype = b'NONE'
self._compname = b'not compressed'
def _readmark(self, chunk):
nmarkers = _read_short(chunk)
# Some files appear to contain invalid counts.
# Cope with this by testing for EOF.
try:
for i in range(nmarkers):
id = _read_short(chunk)
pos = _read_long(chunk)
name = _read_string(chunk)
if pos or name:
# some files appear to have
# dummy markers consisting of
# a position 0 and name ''
self._markers.append((id, pos, name))
except EOFError:
w = ('Warning: MARK chunk contains only %s marker%s instead of %s' %
(len(self._markers), '' if len(self._markers) == 1 else 's',
nmarkers))
warnings.warn(w)
class Aifc_write:
# Variables used in this class:
#
# These variables are user settable through appropriate methods
# of this class:
# _file -- the open file with methods write(), close(), tell(), seek()
# set through the __init__() method
# _comptype -- the AIFF-C compression type ('NONE' in AIFF)
# set through the setcomptype() or setparams() method
# _compname -- the human-readable AIFF-C compression type
# set through the setcomptype() or setparams() method
# _nchannels -- the number of audio channels
# set through the setnchannels() or setparams() method
# _sampwidth -- the number of bytes per audio sample
# set through the setsampwidth() or setparams() method
# _framerate -- the sampling frequency
# set through the setframerate() or setparams() method
# _nframes -- the number of audio frames written to the header
# set through the setnframes() or setparams() method
# _aifc -- whether we're writing an AIFF-C file or an AIFF file
# set through the aifc() method, reset through the
# aiff() method
#
# These variables are used internally only:
# _version -- the AIFF-C version number
# _comp -- the compressor from builtin module cl
# _nframeswritten -- the number of audio frames actually written
# _datalength -- the size of the audio samples written to the header
# _datawritten -- the size of the audio samples actually written
_file = None # Set here since __del__ checks it
def __init__(self, f):
if isinstance(f, str):
file_object = builtins.open(f, 'wb')
try:
self.initfp(file_object)
except:
file_object.close()
raise
# treat .aiff file extensions as non-compressed audio
if f.endswith('.aiff'):
self._aifc = 0
else:
# assume it is an open file object already
self.initfp(f)
def initfp(self, file):
self._file = file
self._version = _AIFC_version
self._comptype = b'NONE'
self._compname = b'not compressed'
self._convert = None
self._nchannels = 0
self._sampwidth = 0
self._framerate = 0
self._nframes = 0
self._nframeswritten = 0
self._datawritten = 0
self._datalength = 0
self._markers = []
self._marklength = 0
self._aifc = 1 # AIFF-C is default
def __del__(self):
self.close()
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
#
# User visible methods.
#
def aiff(self):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
self._aifc = 0
def aifc(self):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
self._aifc = 1
def setnchannels(self, nchannels):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if nchannels < 1:
raise Error('bad # of channels')
self._nchannels = nchannels
def getnchannels(self):
if not self._nchannels:
raise Error('number of channels not set')
return self._nchannels
def setsampwidth(self, sampwidth):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if sampwidth < 1 or sampwidth > 4:
raise Error('bad sample width')
self._sampwidth = sampwidth
def getsampwidth(self):
if not self._sampwidth:
raise Error('sample width not set')
return self._sampwidth
def setframerate(self, framerate):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if framerate <= 0:
raise Error('bad frame rate')
self._framerate = framerate
def getframerate(self):
if not self._framerate:
raise Error('frame rate not set')
return self._framerate
def setnframes(self, nframes):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
self._nframes = nframes
def getnframes(self):
return self._nframeswritten
def setcomptype(self, comptype, compname):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if comptype not in (b'NONE', b'ulaw', b'ULAW',
b'alaw', b'ALAW', b'G722'):
raise Error('unsupported compression type')
self._comptype = comptype
self._compname = compname
def getcomptype(self):
return self._comptype
def getcompname(self):
return self._compname
## def setversion(self, version):
## if self._nframeswritten:
## raise Error, 'cannot change parameters after starting to write'
## self._version = version
def setparams(self, params):
nchannels, sampwidth, framerate, nframes, comptype, compname = params
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if comptype not in (b'NONE', b'ulaw', b'ULAW',
b'alaw', b'ALAW', b'G722'):
raise Error('unsupported compression type')
self.setnchannels(nchannels)
self.setsampwidth(sampwidth)
self.setframerate(framerate)
self.setnframes(nframes)
self.setcomptype(comptype, compname)
def getparams(self):
if not self._nchannels or not self._sampwidth or not self._framerate:
raise Error('not all parameters set')
return _aifc_params(self._nchannels, self._sampwidth, self._framerate,
self._nframes, self._comptype, self._compname)
def setmark(self, id, pos, name):
if id <= 0:
raise Error('marker ID must be > 0')
if pos < 0:
raise Error('marker position must be >= 0')
if not isinstance(name, bytes):
raise Error('marker name must be bytes')
for i in range(len(self._markers)):
if id == self._markers[i][0]:
self._markers[i] = id, pos, name
return
self._markers.append((id, pos, name))
def getmark(self, id):
for marker in self._markers:
if id == marker[0]:
return marker
raise Error('marker {0!r} does not exist'.format(id))
def getmarkers(self):
if len(self._markers) == 0:
return None
return self._markers
def tell(self):
return self._nframeswritten
def writeframesraw(self, data):
if not isinstance(data, (bytes, bytearray)):
data = memoryview(data).cast('B')
self._ensure_header_written(len(data))
nframes = len(data) // (self._sampwidth * self._nchannels)
if self._convert:
data = self._convert(data)
self._file.write(data)
self._nframeswritten = self._nframeswritten + nframes
self._datawritten = self._datawritten + len(data)
def writeframes(self, data):
self.writeframesraw(data)
if self._nframeswritten != self._nframes or \
self._datalength != self._datawritten:
self._patchheader()
def close(self):
if self._file is None:
return
try:
self._ensure_header_written(0)
if self._datawritten & 1:
# quick pad to even size
self._file.write(b'\x00')
self._datawritten = self._datawritten + 1
self._writemarkers()
if self._nframeswritten != self._nframes or \
self._datalength != self._datawritten or \
self._marklength:
self._patchheader()
finally:
# Prevent ref cycles
self._convert = None
f = self._file
self._file = None
f.close()
#
# Internal methods.
#
def _lin2alaw(self, data):
import audioop
return audioop.lin2alaw(data, 2)
def _lin2ulaw(self, data):
import audioop
return audioop.lin2ulaw(data, 2)
def _lin2adpcm(self, data):
import audioop
if not hasattr(self, '_adpcmstate'):
self._adpcmstate = None
data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
return data
def _ensure_header_written(self, datasize):
if not self._nframeswritten:
if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
if not self._sampwidth:
self._sampwidth = 2
if self._sampwidth != 2:
raise Error('sample width must be 2 when compressing '
'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)')
if not self._nchannels:
raise Error('# channels not specified')
if not self._sampwidth:
raise Error('sample width not specified')
if not self._framerate:
raise Error('sampling rate not specified')
self._write_header(datasize)
def _init_compression(self):
if self._comptype == b'G722':
self._convert = self._lin2adpcm
elif self._comptype in (b'ulaw', b'ULAW'):
self._convert = self._lin2ulaw
elif self._comptype in (b'alaw', b'ALAW'):
self._convert = self._lin2alaw
def _write_header(self, initlength):
if self._aifc and self._comptype != b'NONE':
self._init_compression()
self._file.write(b'FORM')
if not self._nframes:
self._nframes = initlength // (self._nchannels * self._sampwidth)
self._datalength = self._nframes * self._nchannels * self._sampwidth
if self._datalength & 1:
self._datalength = self._datalength + 1
if self._aifc:
if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'):
self._datalength = self._datalength // 2
if self._datalength & 1:
self._datalength = self._datalength + 1
elif self._comptype == b'G722':
self._datalength = (self._datalength + 3) // 4
if self._datalength & 1:
self._datalength = self._datalength + 1
try:
self._form_length_pos = self._file.tell()
except (AttributeError, OSError):
self._form_length_pos = None
commlength = self._write_form_length(self._datalength)
if self._aifc:
self._file.write(b'AIFC')
self._file.write(b'FVER')
_write_ulong(self._file, 4)
_write_ulong(self._file, self._version)
else:
self._file.write(b'AIFF')
self._file.write(b'COMM')
_write_ulong(self._file, commlength)
_write_short(self._file, self._nchannels)
if self._form_length_pos is not None:
self._nframes_pos = self._file.tell()
_write_ulong(self._file, self._nframes)
if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
_write_short(self._file, 8)
else:
_write_short(self._file, self._sampwidth * 8)
_write_float(self._file, self._framerate)
if self._aifc:
self._file.write(self._comptype)
_write_string(self._file, self._compname)
self._file.write(b'SSND')
if self._form_length_pos is not None:
self._ssnd_length_pos = self._file.tell()
_write_ulong(self._file, self._datalength + 8)
_write_ulong(self._file, 0)
_write_ulong(self._file, 0)
def _write_form_length(self, datalength):
if self._aifc:
commlength = 18 + 5 + len(self._compname)
if commlength & 1:
commlength = commlength + 1
verslength = 12
else:
commlength = 18
verslength = 0
_write_ulong(self._file, 4 + verslength + self._marklength + \
8 + commlength + 16 + datalength)
return commlength
def _patchheader(self):
curpos = self._file.tell()
if self._datawritten & 1:
datalength = self._datawritten + 1
self._file.write(b'\x00')
else:
datalength = self._datawritten
if datalength == self._datalength and \
self._nframes == self._nframeswritten and \
self._marklength == 0:
self._file.seek(curpos, 0)
return
self._file.seek(self._form_length_pos, 0)
dummy = self._write_form_length(datalength)
self._file.seek(self._nframes_pos, 0)
_write_ulong(self._file, self._nframeswritten)
self._file.seek(self._ssnd_length_pos, 0)
_write_ulong(self._file, datalength + 8)
self._file.seek(curpos, 0)
self._nframes = self._nframeswritten
self._datalength = datalength
def _writemarkers(self):
if len(self._markers) == 0:
return
self._file.write(b'MARK')
length = 2
for marker in self._markers:
id, pos, name = marker
length = length + len(name) + 1 + 6
if len(name) & 1 == 0:
length = length + 1
_write_ulong(self._file, length)
self._marklength = length + 8
_write_short(self._file, len(self._markers))
for marker in self._markers:
id, pos, name = marker
_write_short(self._file, id)
_write_ulong(self._file, pos)
_write_string(self._file, name)
def open(f, mode=None):
if mode is None:
if hasattr(f, 'mode'):
mode = f.mode
else:
mode = 'rb'
if mode in ('r', 'rb'):
return Aifc_read(f)
elif mode in ('w', 'wb'):
return Aifc_write(f)
else:
raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
def openfp(f, mode=None):
warnings.warn("aifc.openfp is deprecated since Python 3.7. "
"Use aifc.open instead.", DeprecationWarning, stacklevel=2)
return open(f, mode=mode)
if __name__ == '__main__':
import sys
if not sys.argv[1:]:
sys.argv.append('/usr/demos/data/audio/bach.aiff')
fn = sys.argv[1]
with open(fn, 'r') as f:
print("Reading", fn)
print("nchannels =", f.getnchannels())
print("nframes =", f.getnframes())
print("sampwidth =", f.getsampwidth())
print("framerate =", f.getframerate())
print("comptype =", f.getcomptype())
print("compname =", f.getcompname())
if sys.argv[2:]:
gn = sys.argv[2]
print("Writing", gn)
with open(gn, 'w') as g:
g.setparams(f.getparams())
while 1:
data = f.readframes(1024)
if not data:
break
g.writeframes(data)
print("Done.")

171
Lib/imghdr.py Normal file
View File

@@ -0,0 +1,171 @@
"""Recognize image file formats based on their first few bytes."""
#from os import PathLike
__all__ = ["what"]
# should replace using FileIO into file
from io import FileIO
#-------------------------#
# Recognize image headers #
#-------------------------#
def what(file, h=None):
f = None
try:
if h is None:
# if isinstance(file, (str, PathLike))
if isinstance(file, str): # FIXME(corona10): RustPython doesn't support PathLike yet.
f = FileIO(file, 'rb')
h = f.read(32)
else:
location = file.tell()
h = file.read(32)
file.seek(location)
for tf in tests:
res = tf(h, f)
if res:
return res
finally:
if f: f.close()
return None
#---------------------------------#
# Subroutines per image file type #
#---------------------------------#
tests = []
def test_jpeg(h, f):
"""JPEG data in JFIF or Exif format"""
if h[6:10] in (b'JFIF', b'Exif'):
return 'jpeg'
tests.append(test_jpeg)
def test_png(h, f):
if h.startswith(b'\211PNG\r\n\032\n'):
return 'png'
tests.append(test_png)
def test_gif(h, f):
"""GIF ('87 and '89 variants)"""
if h[:6] in (b'GIF87a', b'GIF89a'):
return 'gif'
tests.append(test_gif)
def test_tiff(h, f):
"""TIFF (can be in Motorola or Intel byte order)"""
if h[:2] in (b'MM', b'II'):
return 'tiff'
tests.append(test_tiff)
def test_rgb(h, f):
"""SGI image library"""
if h.startswith(b'\001\332'):
return 'rgb'
tests.append(test_rgb)
def test_pbm(h, f):
"""PBM (portable bitmap)"""
if len(h) >= 3 and \
h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r':
return 'pbm'
tests.append(test_pbm)
def test_pgm(h, f):
"""PGM (portable graymap)"""
if len(h) >= 3 and \
h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r':
return 'pgm'
tests.append(test_pgm)
def test_ppm(h, f):
"""PPM (portable pixmap)"""
if len(h) >= 3 and \
h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r':
return 'ppm'
tests.append(test_ppm)
def test_rast(h, f):
"""Sun raster file"""
if h.startswith(b'\x59\xA6\x6A\x95'):
return 'rast'
tests.append(test_rast)
def test_xbm(h, f):
"""X bitmap (X10 or X11)"""
if h.startswith(b'#define '):
return 'xbm'
tests.append(test_xbm)
def test_bmp(h, f):
if h.startswith(b'BM'):
return 'bmp'
tests.append(test_bmp)
def test_webp(h, f):
if h.startswith(b'RIFF') and h[8:12] == b'WEBP':
return 'webp'
tests.append(test_webp)
def test_exr(h, f):
if h.startswith(b'\x76\x2f\x31\x01'):
return 'exr'
tests.append(test_exr)
#--------------------#
# Small test program #
#--------------------#
def test():
import sys
recursive = 0
if sys.argv[1:] and sys.argv[1] == '-r':
del sys.argv[1:2]
recursive = 1
try:
if sys.argv[1:]:
testall(sys.argv[1:], recursive, 1)
else:
testall(['.'], recursive, 1)
except KeyboardInterrupt:
sys.stderr.write('\n[Interrupted]\n')
sys.exit(1)
def testall(list, recursive, toplevel):
import sys
import os
for filename in list:
if os.path.isdir(filename):
print(filename + '/:', end=' ')
if recursive or toplevel:
print('recursing down:')
import glob
names = glob.glob(os.path.join(filename, '*'))
testall(names, recursive, 0)
else:
print('*** directory (use -r) ***')
else:
print(filename + ':', end=' ')
sys.stdout.flush()
try:
print(what(filename))
except OSError:
print('*** not found ***')
if __name__ == '__main__':
test()

8
Lib/runpy.py
View File

@@ -11,10 +11,12 @@ importers when locating support scripts as well as when importing modules.
import sys
import __importlib_util
try:
import __importlib_util
except ImportError:
# FIXME replace above with below once we can import importlib
# import importlib.machinery # importlib first so we can test #15386 via -m
# import importlib.util
import importlib.machinery # importlib first so we can test #15386 via -m
import importlib.util as __importlib_util
import types
# FIXME uncomment line below once we can import pkgutil
# from pkgutil import read_code, get_importer

625
Lib/site.py Normal file
View File

@@ -0,0 +1,625 @@
"""Append module search paths for third-party packages to sys.path.
****************************************************************
* This module is automatically imported during initialization. *
****************************************************************
This will append site-specific paths to the module search path. On
Unix (including Mac OSX), it starts with sys.prefix and
sys.exec_prefix (if different) and appends
lib/python<version>/site-packages.
On other platforms (such as Windows), it tries each of the
prefixes directly, as well as with lib/site-packages appended. The
resulting directories, if they exist, are appended to sys.path, and
also inspected for path configuration files.
If a file named "pyvenv.cfg" exists one directory above sys.executable,
sys.prefix and sys.exec_prefix are set to that directory and
it is also checked for site-packages (sys.base_prefix and
sys.base_exec_prefix will always be the "real" prefixes of the Python
installation). If "pyvenv.cfg" (a bootstrap configuration file) contains
the key "include-system-site-packages" set to anything other than "false"
(case-insensitive), the system-level prefixes will still also be
searched for site-packages; otherwise they won't.
All of the resulting site-specific directories, if they exist, are
appended to sys.path, and also inspected for path configuration
files.
A path configuration file is a file whose name has the form
<package>.pth; its contents are additional directories (one per line)
to be added to sys.path. Non-existing directories (or
non-directories) are never added to sys.path; no directory is added to
sys.path more than once. Blank lines and lines beginning with
'#' are skipped. Lines starting with 'import' are executed.
For example, suppose sys.prefix and sys.exec_prefix are set to
/usr/local and there is a directory /usr/local/lib/python2.5/site-packages
with three subdirectories, foo, bar and spam, and two path
configuration files, foo.pth and bar.pth. Assume foo.pth contains the
following:
# foo package configuration
foo
bar
bletch
and bar.pth contains:
# bar package configuration
bar
Then the following directories are added to sys.path, in this order:
/usr/local/lib/python2.5/site-packages/bar
/usr/local/lib/python2.5/site-packages/foo
Note that bletch is omitted because it doesn't exist; bar precedes foo
because bar.pth comes alphabetically before foo.pth; and spam is
omitted because it is not mentioned in either path configuration file.
The readline module is also automatically configured to enable
completion for systems that support it. This can be overridden in
sitecustomize, usercustomize or PYTHONSTARTUP. Starting Python in
isolated mode (-I) disables automatic readline configuration.
After these operations, an attempt is made to import a module
named sitecustomize, which can perform arbitrary additional
site-specific customizations. If this import fails with an
ImportError exception, it is silently ignored.
"""
import sys
import os
import builtins
import _sitebuiltins
# Prefixes for site-packages; add additional prefixes like /usr/local here
PREFIXES = [sys.prefix, sys.exec_prefix]
# Enable per user site-packages directory
# set it to False to disable the feature or True to force the feature
ENABLE_USER_SITE = None
# for distutils.commands.install
# These values are initialized by the getuserbase() and getusersitepackages()
# functions, through the main() function when Python starts.
USER_SITE = None
USER_BASE = None
def makepath(*paths):
dir = os.path.join(*paths)
try:
dir = os.path.abspath(dir)
except OSError:
pass
return dir, os.path.normcase(dir)
def abs_paths():
"""Set all module __file__ and __cached__ attributes to an absolute path"""
for m in set(sys.modules.values()):
if (getattr(getattr(m, '__loader__', None), '__module__', None) not in
('_frozen_importlib', '_frozen_importlib_external')):
continue # don't mess with a PEP 302-supplied __file__
try:
m.__file__ = os.path.abspath(m.__file__)
except (AttributeError, OSError, TypeError):
pass
try:
m.__cached__ = os.path.abspath(m.__cached__)
except (AttributeError, OSError, TypeError):
pass
def removeduppaths():
""" Remove duplicate entries from sys.path along with making them
absolute"""
# This ensures that the initial path provided by the interpreter contains
# only absolute pathnames, even if we're running from the build directory.
L = []
known_paths = set()
for dir in sys.path:
# Filter out duplicate paths (on case-insensitive file systems also
# if they only differ in case); turn relative paths into absolute
# paths.
dir, dircase = makepath(dir)
if dircase not in known_paths:
L.append(dir)
known_paths.add(dircase)
sys.path[:] = L
return known_paths
def _init_pathinfo():
"""Return a set containing all existing file system items from sys.path."""
d = set()
for item in sys.path:
try:
if os.path.exists(item):
_, itemcase = makepath(item)
d.add(itemcase)
except TypeError:
continue
return d
def addpackage(sitedir, name, known_paths):
"""Process a .pth file within the site-packages directory:
For each line in the file, either combine it with sitedir to a path
and add that to known_paths, or execute it if it starts with 'import '.
"""
if known_paths is None:
known_paths = _init_pathinfo()
reset = True
else:
reset = False
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "r")
except OSError:
return
with f:
for n, line in enumerate(f):
if line.startswith("#"):
continue
try:
if line.startswith(("import ", "import\t")):
exec(line)
continue
line = line.rstrip()
dir, dircase = makepath(sitedir, line)
if not dircase in known_paths and os.path.exists(dir):
sys.path.append(dir)
known_paths.add(dircase)
except Exception:
print("Error processing line {:d} of {}:\n".format(n+1, fullname),
file=sys.stderr)
import traceback
for record in traceback.format_exception(*sys.exc_info()):
for line in record.splitlines():
print(' '+line, file=sys.stderr)
print("\nRemainder of file ignored", file=sys.stderr)
break
if reset:
known_paths = None
return known_paths
def addsitedir(sitedir, known_paths=None):
"""Add 'sitedir' argument to sys.path if missing and handle .pth files in
'sitedir'"""
if known_paths is None:
known_paths = _init_pathinfo()
reset = True
else:
reset = False
sitedir, sitedircase = makepath(sitedir)
if not sitedircase in known_paths:
sys.path.append(sitedir) # Add path component
known_paths.add(sitedircase)
try:
names = os.listdir(sitedir)
except OSError:
return
names = [name for name in names if name.endswith(".pth")]
for name in sorted(names):
addpackage(sitedir, name, known_paths)
if reset:
known_paths = None
return known_paths
def check_enableusersite():
"""Check if user site directory is safe for inclusion
The function tests for the command line flag (including environment var),
process uid/gid equal to effective uid/gid.
None: Disabled for security reasons
False: Disabled by user (command line option)
True: Safe and enabled
"""
if sys.flags.no_user_site:
return False
if hasattr(os, "getuid") and hasattr(os, "geteuid"):
# check process uid == effective uid
if os.geteuid() != os.getuid():
return None
if hasattr(os, "getgid") and hasattr(os, "getegid"):
# check process gid == effective gid
if os.getegid() != os.getgid():
return None
return True
# NOTE: sysconfig and it's dependencies are relatively large but site module
# needs very limited part of them.
# To speedup startup time, we have copy of them.
#
# See https://bugs.python.org/issue29585
# Copy of sysconfig._getuserbase()
def _getuserbase():
env_base = os.environ.get("PYTHONUSERBASE", None)
if env_base:
return env_base
def joinuser(*args):
return os.path.expanduser(os.path.join(*args))
if os.name == "nt":
base = os.environ.get("APPDATA") or "~"
return joinuser(base, "Python")
if sys.platform == "darwin" and sys._framework:
return joinuser("~", "Library", sys._framework,
"%d.%d" % sys.version_info[:2])
return joinuser("~", ".local")
# Same to sysconfig.get_path('purelib', os.name+'_user')
def _get_path(userbase):
version = sys.version_info
if os.name == 'nt':
return f'{userbase}\\Python{version[0]}{version[1]}\\site-packages'
if sys.platform == 'darwin' and sys._framework:
return f'{userbase}/lib/python/site-packages'
return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages'
def getuserbase():
"""Returns the `user base` directory path.
The `user base` directory can be used to store data. If the global
variable ``USER_BASE`` is not initialized yet, this function will also set
it.
"""
global USER_BASE
if USER_BASE is None:
USER_BASE = _getuserbase()
return USER_BASE
def getusersitepackages():
"""Returns the user-specific site-packages directory path.
If the global variable ``USER_SITE`` is not initialized yet, this
function will also set it.
"""
global USER_SITE
userbase = getuserbase() # this will also set USER_BASE
if USER_SITE is None:
USER_SITE = _get_path(userbase)
return USER_SITE
def addusersitepackages(known_paths):
"""Add a per user site-package to sys.path
Each user has its own python directory with site-packages in the
home directory.
"""
# get the per user site-package path
# this call will also make sure USER_BASE and USER_SITE are set
user_site = getusersitepackages()
if ENABLE_USER_SITE and os.path.isdir(user_site):
addsitedir(user_site, known_paths)
return known_paths
def getsitepackages(prefixes=None):
"""Returns a list containing all global site-packages directories.
For each directory present in ``prefixes`` (or the global ``PREFIXES``),
this function will find its `site-packages` subdirectory depending on the
system environment, and will return a list of full paths.
"""
sitepackages = []
seen = set()
if prefixes is None:
prefixes = PREFIXES
for prefix in prefixes:
if not prefix or prefix in seen:
continue
seen.add(prefix)
if os.sep == '/':
sitepackages.append(os.path.join(prefix, "lib",
# XXX changed for RustPython
"rustpython%d.%d" % sys.version_info[:2],
"site-packages"))
else:
sitepackages.append(prefix)
sitepackages.append(os.path.join(prefix, "lib", "site-packages"))
return sitepackages
def addsitepackages(known_paths, prefixes=None):
"""Add site-packages to sys.path"""
for sitedir in getsitepackages(prefixes):
if os.path.isdir(sitedir):
addsitedir(sitedir, known_paths)
return known_paths
def setquit():
"""Define new builtins 'quit' and 'exit'.
These are objects which make the interpreter exit when called.
The repr of each object contains a hint at how it works.
"""
if os.sep == '\\':
eof = 'Ctrl-Z plus Return'
else:
eof = 'Ctrl-D (i.e. EOF)'
builtins.quit = _sitebuiltins.Quitter('quit', eof)
builtins.exit = _sitebuiltins.Quitter('exit', eof)
def setcopyright():
"""Set 'copyright' and 'credits' in builtins"""
builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright)
if sys.platform[:4] == 'java':
builtins.credits = _sitebuiltins._Printer(
"credits",
"Jython is maintained by the Jython developers (www.jython.org).")
else:
builtins.credits = _sitebuiltins._Printer("credits", """\
Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
for supporting Python development. See www.python.org for more information.""")
files, dirs = [], []
# Not all modules are required to have a __file__ attribute. See
# PEP 420 for more details.
if hasattr(os, '__file__'):
here = os.path.dirname(os.__file__)
files.extend(["LICENSE.txt", "LICENSE"])
dirs.extend([os.path.join(here, os.pardir), here, os.curdir])
builtins.license = _sitebuiltins._Printer(
"license",
"See https://www.python.org/psf/license/",
files, dirs)
def sethelper():
builtins.help = _sitebuiltins._Helper()
def enablerlcompleter():
"""Enable default readline configuration on interactive prompts, by
registering a sys.__interactivehook__.
If the readline module can be imported, the hook will set the Tab key
as completion key and register ~/.python_history as history file.
This can be overridden in the sitecustomize or usercustomize module,
or in a PYTHONSTARTUP file.
"""
def register_readline():
import atexit
try:
import readline
import rlcompleter
except ImportError:
return
# Reading the initialization (config) file may not be enough to set a
# completion key, so we set one first and then read the file.
readline_doc = getattr(readline, '__doc__', '')
if readline_doc is not None and 'libedit' in readline_doc:
readline.parse_and_bind('bind ^I rl_complete')
else:
readline.parse_and_bind('tab: complete')
try:
readline.read_init_file()
except OSError:
# An OSError here could have many causes, but the most likely one
# is that there's no .inputrc file (or .editrc file in the case of
# Mac OS X + libedit) in the expected location. In that case, we
# want to ignore the exception.
pass
if readline.get_current_history_length() == 0:
# If no history was loaded, default to .python_history.
# The guard is necessary to avoid doubling history size at
# each interpreter exit when readline was already configured
# through a PYTHONSTARTUP hook, see:
# http://bugs.python.org/issue5845#msg198636
history = os.path.join(os.path.expanduser('~'),
'.python_history')
try:
readline.read_history_file(history)
except OSError:
pass
atexit.register(readline.write_history_file, history)
sys.__interactivehook__ = register_readline
def venv(known_paths):
global PREFIXES, ENABLE_USER_SITE
env = os.environ
if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env:
executable = os.environ['__PYVENV_LAUNCHER__']
else:
executable = sys.executable
exe_dir, _ = os.path.split(os.path.abspath(executable))
site_prefix = os.path.dirname(exe_dir)
sys._home = None
conf_basename = 'pyvenv.cfg'
candidate_confs = [
conffile for conffile in (
os.path.join(exe_dir, conf_basename),
os.path.join(site_prefix, conf_basename)
)
if os.path.isfile(conffile)
]
if candidate_confs:
virtual_conf = candidate_confs[0]
system_site = "true"
# Issue 25185: Use UTF-8, as that's what the venv module uses when
# writing the file.
with open(virtual_conf, encoding='utf-8') as f:
for line in f:
if '=' in line:
key, _, value = line.partition('=')
key = key.strip().lower()
value = value.strip()
if key == 'include-system-site-packages':
system_site = value.lower()
elif key == 'home':
sys._home = value
sys.prefix = sys.exec_prefix = site_prefix
# Doing this here ensures venv takes precedence over user-site
addsitepackages(known_paths, [sys.prefix])
# addsitepackages will process site_prefix again if its in PREFIXES,
# but that's ok; known_paths will prevent anything being added twice
if system_site == "true":
PREFIXES.insert(0, sys.prefix)
else:
PREFIXES = [sys.prefix]
ENABLE_USER_SITE = False
return known_paths
def execsitecustomize():
"""Run custom site specific code, if available."""
try:
try:
import sitecustomize
except ImportError as exc:
if exc.name == 'sitecustomize':
pass
else:
raise
except Exception as err:
if sys.flags.verbose:
sys.excepthook(*sys.exc_info())
else:
sys.stderr.write(
"Error in sitecustomize; set PYTHONVERBOSE for traceback:\n"
"%s: %s\n" %
(err.__class__.__name__, err))
def execusercustomize():
"""Run custom user specific code, if available."""
try:
try:
import usercustomize
except ImportError as exc:
if exc.name == 'usercustomize':
pass
else:
raise
except Exception as err:
if sys.flags.verbose:
sys.excepthook(*sys.exc_info())
else:
sys.stderr.write(
"Error in usercustomize; set PYTHONVERBOSE for traceback:\n"
"%s: %s\n" %
(err.__class__.__name__, err))
def main():
"""Add standard site-specific directories to the module search path.
This function is called automatically when this module is imported,
unless the python interpreter was started with the -S flag.
"""
global ENABLE_USER_SITE
orig_path = sys.path[:]
known_paths = removeduppaths()
if orig_path != sys.path:
# removeduppaths() might make sys.path absolute.
# fix __file__ and __cached__ of already imported modules too.
abs_paths()
known_paths = venv(known_paths)
if ENABLE_USER_SITE is None:
ENABLE_USER_SITE = check_enableusersite()
known_paths = addusersitepackages(known_paths)
known_paths = addsitepackages(known_paths)
setquit()
setcopyright()
sethelper()
if not sys.flags.isolated:
enablerlcompleter()
execsitecustomize()
if ENABLE_USER_SITE:
execusercustomize()
# Prevent extending of sys.path when python was started with -S and
# site is imported later.
if not sys.flags.no_site:
main()
def _script():
help = """\
%s [--user-base] [--user-site]
Without arguments print some useful information
With arguments print the value of USER_BASE and/or USER_SITE separated
by '%s'.
Exit codes with --user-base or --user-site:
0 - user site directory is enabled
1 - user site directory is disabled by user
2 - uses site directory is disabled by super user
or for security reasons
>2 - unknown error
"""
args = sys.argv[1:]
if not args:
user_base = getuserbase()
user_site = getusersitepackages()
print("sys.path = [")
for dir in sys.path:
print(" %r," % (dir,))
print("]")
print("USER_BASE: %r (%s)" % (user_base,
"exists" if os.path.isdir(user_base) else "doesn't exist"))
print("USER_SITE: %r (%s)" % (user_site,
"exists" if os.path.isdir(user_site) else "doesn't exist"))
print("ENABLE_USER_SITE: %r" % ENABLE_USER_SITE)
sys.exit(0)
buffer = []
if '--user-base' in args:
buffer.append(USER_BASE)
if '--user-site' in args:
buffer.append(USER_SITE)
if buffer:
print(os.pathsep.join(buffer))
if ENABLE_USER_SITE:
sys.exit(0)
elif ENABLE_USER_SITE is False:
sys.exit(1)
elif ENABLE_USER_SITE is None:
sys.exit(2)
else:
sys.exit(3)
else:
import textwrap
print(textwrap.dedent(help % (sys.argv[0], os.pathsep)))
sys.exit(10)
if __name__ == '__main__':
_script()

257
Lib/sndhdr.py Normal file
View File

@@ -0,0 +1,257 @@
"""Routines to help recognizing sound files.
Function whathdr() recognizes various types of sound file headers.
It understands almost all headers that SOX can decode.
The return tuple contains the following items, in this order:
- file type (as SOX understands it)
- sampling rate (0 if unknown or hard to decode)
- number of channels (0 if unknown or hard to decode)
- number of frames in the file (-1 if unknown or hard to decode)
- number of bits/sample, or 'U' for U-LAW, or 'A' for A-LAW
If the file doesn't have a recognizable type, it returns None.
If the file can't be opened, OSError is raised.
To compute the total time, divide the number of frames by the
sampling rate (a frame contains a sample for each channel).
Function what() calls whathdr(). (It used to also use some
heuristics for raw data, but this doesn't work very well.)
Finally, the function test() is a simple main program that calls
what() for all files mentioned on the argument list. For directory
arguments it calls what() for all files in that directory. Default
argument is "." (testing all files in the current directory). The
option -r tells it to recurse down directories found inside
explicitly given directories.
"""
# The file structure is top-down except that the test program and its
# subroutine come last.
__all__ = ['what', 'whathdr']
from collections import namedtuple
SndHeaders = namedtuple('SndHeaders',
'filetype framerate nchannels nframes sampwidth')
SndHeaders.filetype.__doc__ = ("""The value for type indicates the data type
and will be one of the strings 'aifc', 'aiff', 'au','hcom',
'sndr', 'sndt', 'voc', 'wav', '8svx', 'sb', 'ub', or 'ul'.""")
SndHeaders.framerate.__doc__ = ("""The sampling_rate will be either the actual
value or 0 if unknown or difficult to decode.""")
SndHeaders.nchannels.__doc__ = ("""The number of channels or 0 if it cannot be
determined or if the value is difficult to decode.""")
SndHeaders.nframes.__doc__ = ("""The value for frames will be either the number
of frames or -1.""")
SndHeaders.sampwidth.__doc__ = ("""Either the sample size in bits or
'A' for A-LAW or 'U' for u-LAW.""")
def what(filename):
"""Guess the type of a sound file."""
res = whathdr(filename)
return res
def whathdr(filename):
"""Recognize sound headers."""
with open(filename, 'rb') as f:
h = f.read(512)
for tf in tests:
res = tf(h, f)
if res:
return SndHeaders(*res)
return None
#-----------------------------------#
# Subroutines per sound header type #
#-----------------------------------#
tests = []
def test_aifc(h, f):
import aifc
if not h.startswith(b'FORM'):
return None
if h[8:12] == b'AIFC':
fmt = 'aifc'
elif h[8:12] == b'AIFF':
fmt = 'aiff'
else:
return None
f.seek(0)
try:
a = aifc.open(f, 'r')
except (EOFError, aifc.Error):
return None
return (fmt, a.getframerate(), a.getnchannels(),
a.getnframes(), 8 * a.getsampwidth())
tests.append(test_aifc)
def test_au(h, f):
if h.startswith(b'.snd'):
func = get_long_be
elif h[:4] in (b'\0ds.', b'dns.'):
func = get_long_le
else:
return None
filetype = 'au'
hdr_size = func(h[4:8])
data_size = func(h[8:12])
encoding = func(h[12:16])
rate = func(h[16:20])
nchannels = func(h[20:24])
sample_size = 1 # default
if encoding == 1:
sample_bits = 'U'
elif encoding == 2:
sample_bits = 8
elif encoding == 3:
sample_bits = 16
sample_size = 2
else:
sample_bits = '?'
frame_size = sample_size * nchannels
if frame_size:
nframe = data_size / frame_size
else:
nframe = -1
return filetype, rate, nchannels, nframe, sample_bits
tests.append(test_au)
def test_hcom(h, f):
if h[65:69] != b'FSSD' or h[128:132] != b'HCOM':
return None
divisor = get_long_be(h[144:148])
if divisor:
rate = 22050 / divisor
else:
rate = 0
return 'hcom', rate, 1, -1, 8
tests.append(test_hcom)
def test_voc(h, f):
if not h.startswith(b'Creative Voice File\032'):
return None
sbseek = get_short_le(h[20:22])
rate = 0
if 0 <= sbseek < 500 and h[sbseek] == 1:
ratecode = 256 - h[sbseek+4]
if ratecode:
rate = int(1000000.0 / ratecode)
return 'voc', rate, 1, -1, 8
tests.append(test_voc)
def test_wav(h, f):
import wave
# 'RIFF' <len> 'WAVE' 'fmt ' <len>
if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ':
return None
f.seek(0)
try:
w = wave.open(f, 'r')
except (EOFError, wave.Error):
return None
return ('wav', w.getframerate(), w.getnchannels(),
w.getnframes(), 8*w.getsampwidth())
tests.append(test_wav)
def test_8svx(h, f):
if not h.startswith(b'FORM') or h[8:12] != b'8SVX':
return None
# Should decode it to get #channels -- assume always 1
return '8svx', 0, 1, 0, 8
tests.append(test_8svx)
def test_sndt(h, f):
if h.startswith(b'SOUND'):
nsamples = get_long_le(h[8:12])
rate = get_short_le(h[20:22])
return 'sndt', rate, 1, nsamples, 8
tests.append(test_sndt)
def test_sndr(h, f):
if h.startswith(b'\0\0'):
rate = get_short_le(h[2:4])
if 4000 <= rate <= 25000:
return 'sndr', rate, 1, -1, 8
tests.append(test_sndr)
#-------------------------------------------#
# Subroutines to extract numbers from bytes #
#-------------------------------------------#
def get_long_be(b):
return (b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]
def get_long_le(b):
return (b[3] << 24) | (b[2] << 16) | (b[1] << 8) | b[0]
def get_short_be(b):
return (b[0] << 8) | b[1]
def get_short_le(b):
return (b[1] << 8) | b[0]
#--------------------#
# Small test program #
#--------------------#
def test():
import sys
recursive = 0
if sys.argv[1:] and sys.argv[1] == '-r':
del sys.argv[1:2]
recursive = 1
try:
if sys.argv[1:]:
testall(sys.argv[1:], recursive, 1)
else:
testall(['.'], recursive, 1)
except KeyboardInterrupt:
sys.stderr.write('\n[Interrupted]\n')
sys.exit(1)
def testall(list, recursive, toplevel):
import sys
import os
for filename in list:
if os.path.isdir(filename):
print(filename + '/:', end=' ')
if recursive or toplevel:
print('recursing down:')
import glob
names = glob.glob(os.path.join(filename, '*'))
testall(names, recursive, 0)
else:
print('*** directory (use -r) ***')
else:
print(filename + ':', end=' ')
sys.stdout.flush()
try:
print(what(filename))
except OSError:
print('*** not found ***')
if __name__ == '__main__':
test()

531
Lib/sunau.py Normal file
View File

@@ -0,0 +1,531 @@
"""Stuff to parse Sun and NeXT audio files.
An audio file consists of a header followed by the data. The structure
of the header is as follows.
+---------------+
| magic word |
+---------------+
| header size |
+---------------+
| data size |
+---------------+
| encoding |
+---------------+
| sample rate |
+---------------+
| # of channels |
+---------------+
| info |
| |
+---------------+
The magic word consists of the 4 characters '.snd'. Apart from the
info field, all header fields are 4 bytes in size. They are all
32-bit unsigned integers encoded in big-endian byte order.
The header size really gives the start of the data.
The data size is the physical size of the data. From the other
parameters the number of frames can be calculated.
The encoding gives the way in which audio samples are encoded.
Possible values are listed below.
The info field currently consists of an ASCII string giving a
human-readable description of the audio file. The info field is
padded with NUL bytes to the header size.
Usage.
Reading audio files:
f = sunau.open(file, 'r')
where file is either the name of a file or an open file pointer.
The open file pointer must have methods read(), seek(), and close().
When the setpos() and rewind() methods are not used, the seek()
method is not necessary.
This returns an instance of a class with the following public methods:
getnchannels() -- returns number of audio channels (1 for
mono, 2 for stereo)
getsampwidth() -- returns sample width in bytes
getframerate() -- returns sampling frequency
getnframes() -- returns number of audio frames
getcomptype() -- returns compression type ('NONE' or 'ULAW')
getcompname() -- returns human-readable version of
compression type ('not compressed' matches 'NONE')
getparams() -- returns a namedtuple consisting of all of the
above in the above order
getmarkers() -- returns None (for compatibility with the
aifc module)
getmark(id) -- raises an error since the mark does not
exist (for compatibility with the aifc module)
readframes(n) -- returns at most n frames of audio
rewind() -- rewind to the beginning of the audio stream
setpos(pos) -- seek to the specified position
tell() -- return the current position
close() -- close the instance (make it unusable)
The position returned by tell() and the position given to setpos()
are compatible and have nothing to do with the actual position in the
file.
The close() method is called automatically when the class instance
is destroyed.
Writing audio files:
f = sunau.open(file, 'w')
where file is either the name of a file or an open file pointer.
The open file pointer must have methods write(), tell(), seek(), and
close().
This returns an instance of a class with the following public methods:
setnchannels(n) -- set the number of channels
setsampwidth(n) -- set the sample width
setframerate(n) -- set the frame rate
setnframes(n) -- set the number of frames
setcomptype(type, name)
-- set the compression type and the
human-readable compression type
setparams(tuple)-- set all parameters at once
tell() -- return current position in output file
writeframesraw(data)
-- write audio frames without pathing up the
file header
writeframes(data)
-- write audio frames and patch up the file header
close() -- patch up the file header and close the
output file
You should set the parameters before the first writeframesraw or
writeframes. The total number of frames does not need to be set,
but when it is set to the correct value, the header does not have to
be patched up.
It is best to first set all parameters, perhaps possibly the
compression type, and then write audio frames using writeframesraw.
When all frames have been written, either call writeframes(b'') or
close() to patch up the sizes in the header.
The close() method is called automatically when the class instance
is destroyed.
"""
from collections import namedtuple
import warnings
_sunau_params = namedtuple('_sunau_params',
'nchannels sampwidth framerate nframes comptype compname')
# from <multimedia/audio_filehdr.h>
AUDIO_FILE_MAGIC = 0x2e736e64
AUDIO_FILE_ENCODING_MULAW_8 = 1
AUDIO_FILE_ENCODING_LINEAR_8 = 2
AUDIO_FILE_ENCODING_LINEAR_16 = 3
AUDIO_FILE_ENCODING_LINEAR_24 = 4
AUDIO_FILE_ENCODING_LINEAR_32 = 5
AUDIO_FILE_ENCODING_FLOAT = 6
AUDIO_FILE_ENCODING_DOUBLE = 7
AUDIO_FILE_ENCODING_ADPCM_G721 = 23
AUDIO_FILE_ENCODING_ADPCM_G722 = 24
AUDIO_FILE_ENCODING_ADPCM_G723_3 = 25
AUDIO_FILE_ENCODING_ADPCM_G723_5 = 26
AUDIO_FILE_ENCODING_ALAW_8 = 27
# from <multimedia/audio_hdr.h>
AUDIO_UNKNOWN_SIZE = 0xFFFFFFFF # ((unsigned)(~0))
_simple_encodings = [AUDIO_FILE_ENCODING_MULAW_8,
AUDIO_FILE_ENCODING_LINEAR_8,
AUDIO_FILE_ENCODING_LINEAR_16,
AUDIO_FILE_ENCODING_LINEAR_24,
AUDIO_FILE_ENCODING_LINEAR_32,
AUDIO_FILE_ENCODING_ALAW_8]
class Error(Exception):
pass
def _read_u32(file):
x = 0
for i in range(4):
byte = file.read(1)
if not byte:
raise EOFError
x = x*256 + ord(byte)
return x
def _write_u32(file, x):
data = []
for i in range(4):
d, m = divmod(x, 256)
data.insert(0, int(m))
x = d
file.write(bytes(data))
class Au_read:
def __init__(self, f):
if type(f) == type(''):
import builtins
f = builtins.open(f, 'rb')
self._opened = True
else:
self._opened = False
self.initfp(f)
def __del__(self):
if self._file:
self.close()
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def initfp(self, file):
self._file = file
self._soundpos = 0
magic = int(_read_u32(file))
if magic != AUDIO_FILE_MAGIC:
raise Error('bad magic number')
self._hdr_size = int(_read_u32(file))
if self._hdr_size < 24:
raise Error('header size too small')
if self._hdr_size > 100:
raise Error('header size ridiculously large')
self._data_size = _read_u32(file)
if self._data_size != AUDIO_UNKNOWN_SIZE:
self._data_size = int(self._data_size)
self._encoding = int(_read_u32(file))
if self._encoding not in _simple_encodings:
raise Error('encoding not (yet) supported')
if self._encoding in (AUDIO_FILE_ENCODING_MULAW_8,
AUDIO_FILE_ENCODING_ALAW_8):
self._sampwidth = 2
self._framesize = 1
elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_8:
self._framesize = self._sampwidth = 1
elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_16:
self._framesize = self._sampwidth = 2
elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_24:
self._framesize = self._sampwidth = 3
elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_32:
self._framesize = self._sampwidth = 4
else:
raise Error('unknown encoding')
self._framerate = int(_read_u32(file))
self._nchannels = int(_read_u32(file))
if not self._nchannels:
raise Error('bad # of channels')
self._framesize = self._framesize * self._nchannels
if self._hdr_size > 24:
self._info = file.read(self._hdr_size - 24)
self._info, _, _ = self._info.partition(b'\0')
else:
self._info = b''
try:
self._data_pos = file.tell()
except (AttributeError, OSError):
self._data_pos = None
def getfp(self):
return self._file
def getnchannels(self):
return self._nchannels
def getsampwidth(self):
return self._sampwidth
def getframerate(self):
return self._framerate
def getnframes(self):
if self._data_size == AUDIO_UNKNOWN_SIZE:
return AUDIO_UNKNOWN_SIZE
if self._encoding in _simple_encodings:
return self._data_size // self._framesize
return 0 # XXX--must do some arithmetic here
def getcomptype(self):
if self._encoding == AUDIO_FILE_ENCODING_MULAW_8:
return 'ULAW'
elif self._encoding == AUDIO_FILE_ENCODING_ALAW_8:
return 'ALAW'
else:
return 'NONE'
def getcompname(self):
if self._encoding == AUDIO_FILE_ENCODING_MULAW_8:
return 'CCITT G.711 u-law'
elif self._encoding == AUDIO_FILE_ENCODING_ALAW_8:
return 'CCITT G.711 A-law'
else:
return 'not compressed'
def getparams(self):
return _sunau_params(self.getnchannels(), self.getsampwidth(),
self.getframerate(), self.getnframes(),
self.getcomptype(), self.getcompname())
def getmarkers(self):
return None
def getmark(self, id):
raise Error('no marks')
def readframes(self, nframes):
if self._encoding in _simple_encodings:
if nframes == AUDIO_UNKNOWN_SIZE:
data = self._file.read()
else:
data = self._file.read(nframes * self._framesize)
self._soundpos += len(data) // self._framesize
if self._encoding == AUDIO_FILE_ENCODING_MULAW_8:
import audioop
data = audioop.ulaw2lin(data, self._sampwidth)
return data
return None # XXX--not implemented yet
def rewind(self):
if self._data_pos is None:
raise OSError('cannot seek')
self._file.seek(self._data_pos)
self._soundpos = 0
def tell(self):
return self._soundpos
def setpos(self, pos):
if pos < 0 or pos > self.getnframes():
raise Error('position not in range')
if self._data_pos is None:
raise OSError('cannot seek')
self._file.seek(self._data_pos + pos * self._framesize)
self._soundpos = pos
def close(self):
file = self._file
if file:
self._file = None
if self._opened:
file.close()
class Au_write:
def __init__(self, f):
if type(f) == type(''):
import builtins
f = builtins.open(f, 'wb')
self._opened = True
else:
self._opened = False
self.initfp(f)
def __del__(self):
if self._file:
self.close()
self._file = None
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def initfp(self, file):
self._file = file
self._framerate = 0
self._nchannels = 0
self._sampwidth = 0
self._framesize = 0
self._nframes = AUDIO_UNKNOWN_SIZE
self._nframeswritten = 0
self._datawritten = 0
self._datalength = 0
self._info = b''
self._comptype = 'ULAW' # default is U-law
def setnchannels(self, nchannels):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if nchannels not in (1, 2, 4):
raise Error('only 1, 2, or 4 channels supported')
self._nchannels = nchannels
def getnchannels(self):
if not self._nchannels:
raise Error('number of channels not set')
return self._nchannels
def setsampwidth(self, sampwidth):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if sampwidth not in (1, 2, 3, 4):
raise Error('bad sample width')
self._sampwidth = sampwidth
def getsampwidth(self):
if not self._framerate:
raise Error('sample width not specified')
return self._sampwidth
def setframerate(self, framerate):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
self._framerate = framerate
def getframerate(self):
if not self._framerate:
raise Error('frame rate not set')
return self._framerate
def setnframes(self, nframes):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if nframes < 0:
raise Error('# of frames cannot be negative')
self._nframes = nframes
def getnframes(self):
return self._nframeswritten
def setcomptype(self, type, name):
if type in ('NONE', 'ULAW'):
self._comptype = type
else:
raise Error('unknown compression type')
def getcomptype(self):
return self._comptype
def getcompname(self):
if self._comptype == 'ULAW':
return 'CCITT G.711 u-law'
elif self._comptype == 'ALAW':
return 'CCITT G.711 A-law'
else:
return 'not compressed'
def setparams(self, params):
nchannels, sampwidth, framerate, nframes, comptype, compname = params
self.setnchannels(nchannels)
self.setsampwidth(sampwidth)
self.setframerate(framerate)
self.setnframes(nframes)
self.setcomptype(comptype, compname)
def getparams(self):
return _sunau_params(self.getnchannels(), self.getsampwidth(),
self.getframerate(), self.getnframes(),
self.getcomptype(), self.getcompname())
def tell(self):
return self._nframeswritten
def writeframesraw(self, data):
if not isinstance(data, (bytes, bytearray)):
data = memoryview(data).cast('B')
self._ensure_header_written()
if self._comptype == 'ULAW':
import audioop
data = audioop.lin2ulaw(data, self._sampwidth)
nframes = len(data) // self._framesize
self._file.write(data)
self._nframeswritten = self._nframeswritten + nframes
self._datawritten = self._datawritten + len(data)
def writeframes(self, data):
self.writeframesraw(data)
if self._nframeswritten != self._nframes or \
self._datalength != self._datawritten:
self._patchheader()
def close(self):
if self._file:
try:
self._ensure_header_written()
if self._nframeswritten != self._nframes or \
self._datalength != self._datawritten:
self._patchheader()
self._file.flush()
finally:
file = self._file
self._file = None
if self._opened:
file.close()
#
# private methods
#
def _ensure_header_written(self):
if not self._nframeswritten:
if not self._nchannels:
raise Error('# of channels not specified')
if not self._sampwidth:
raise Error('sample width not specified')
if not self._framerate:
raise Error('frame rate not specified')
self._write_header()
def _write_header(self):
if self._comptype == 'NONE':
if self._sampwidth == 1:
encoding = AUDIO_FILE_ENCODING_LINEAR_8
self._framesize = 1
elif self._sampwidth == 2:
encoding = AUDIO_FILE_ENCODING_LINEAR_16
self._framesize = 2
elif self._sampwidth == 3:
encoding = AUDIO_FILE_ENCODING_LINEAR_24
self._framesize = 3
elif self._sampwidth == 4:
encoding = AUDIO_FILE_ENCODING_LINEAR_32
self._framesize = 4
else:
raise Error('internal error')
elif self._comptype == 'ULAW':
encoding = AUDIO_FILE_ENCODING_MULAW_8
self._framesize = 1
else:
raise Error('internal error')
self._framesize = self._framesize * self._nchannels
_write_u32(self._file, AUDIO_FILE_MAGIC)
header_size = 25 + len(self._info)
header_size = (header_size + 7) & ~7
_write_u32(self._file, header_size)
if self._nframes == AUDIO_UNKNOWN_SIZE:
length = AUDIO_UNKNOWN_SIZE
else:
length = self._nframes * self._framesize
try:
self._form_length_pos = self._file.tell()
except (AttributeError, OSError):
self._form_length_pos = None
_write_u32(self._file, length)
self._datalength = length
_write_u32(self._file, encoding)
_write_u32(self._file, self._framerate)
_write_u32(self._file, self._nchannels)
self._file.write(self._info)
self._file.write(b'\0'*(header_size - len(self._info) - 24))
def _patchheader(self):
if self._form_length_pos is None:
raise OSError('cannot seek')
self._file.seek(self._form_length_pos)
_write_u32(self._file, self._datawritten)
self._datalength = self._datawritten
self._file.seek(0, 2)
def open(f, mode=None):
if mode is None:
if hasattr(f, 'mode'):
mode = f.mode
else:
mode = 'rb'
if mode in ('r', 'rb'):
return Au_read(f)
elif mode in ('w', 'wb'):
return Au_write(f)
else:
raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
def openfp(f, mode=None):
warnings.warn("sunau.openfp is deprecated since Python 3.7. "
"Use sunau.open instead.", DeprecationWarning, stacklevel=2)
return open(f, mode=mode)

3
Lib/unittest/__main__.py
View File

@@ -2,8 +2,7 @@
import sys
if sys.argv[0].endswith("__main__.py"):
# FIXME change to `import os.path` as it was for cpython once `import os.path` works
import os
import os.path
# We change sys.argv[0] to make help message more useful
# use executable without path, unquoted
# (it's just a hint anyway)

4
README.md
View File

@@ -16,6 +16,10 @@ A Python-3 (CPython >= 3.5.0) Interpreter written in Rust :snake: :scream:
#### Check out our [online demo](https://rustpython.github.io/demo/) running on WebAssembly.
RustPython requires Rust 1.36 or higher.
To check Rust version: `rustc --version` If you wish to update,
`rustup update stable`.
To test RustPython, do the following:
$ git clone https://github.com/RustPython/RustPython

4
benchmarks/bench.rs
View File

@@ -94,7 +94,7 @@ fn bench_rustpy_nbody(b: &mut test::Bencher) {
let vm = VirtualMachine::default();
let code = match vm.compile(source, &compile::Mode::Single, "<stdin>".to_string()) {
let code = match vm.compile(source, compile::Mode::Single, "<stdin>".to_string()) {
Ok(code) => code,
Err(e) => panic!("{:?}", e),
};
@@ -113,7 +113,7 @@ fn bench_rustpy_mandelbrot(b: &mut test::Bencher) {
let vm = VirtualMachine::default();
let code = match vm.compile(source, &compile::Mode::Single, "<stdin>".to_string()) {
let code = match vm.compile(source, compile::Mode::Single, "<stdin>".to_string()) {
Ok(code) => code,
Err(e) => panic!("{:?}", e),
};

8
benchmarks/test_benchmarks.py
View File

@@ -1,4 +1,5 @@
import os
import time
import sys
@@ -19,8 +20,11 @@ benchmarks = [
['benchmarks/mandelbrot.py'],
]
@pytest.mark.parametrize('exe', pythons)
@pytest.mark.parametrize('args', benchmarks)
exe_ids = ['cpython', 'rustpython']
benchmark_ids = [benchmark[0].split('/')[-1] for benchmark in benchmarks]
@pytest.mark.parametrize('exe', pythons, ids=exe_ids)
@pytest.mark.parametrize('args', benchmarks, ids=benchmark_ids)
def test_bench(exe, args, benchmark):
def bench():
subprocess.run([exe] + args)

74
bytecode/src/bytecode.rs
View File

@@ -84,10 +84,7 @@ pub enum Instruction {
symbols: Vec<String>,
level: usize,
},
ImportStar {
name: Option<String>,
level: usize,
},
ImportStar,
ImportFrom {
name: String,
},
@@ -138,12 +135,24 @@ pub enum Instruction {
Jump {
target: Label,
},
JumpIf {
/// Pop the top of the stack, and jump if this value is true.
JumpIfTrue {
target: Label,
},
/// Pop the top of the stack, and jump if this value is false.
JumpIfFalse {
target: Label,
},
/// Peek at the top of the stack, and jump if this value is true.
/// Otherwise, pop top of stack.
JumpIfTrueOrPop {
target: Label,
},
/// Peek at the top of the stack, and jump if this value is false.
/// Otherwise, pop top of stack.
JumpIfFalseOrPop {
target: Label,
},
MakeFunction {
flags: FunctionOpArg,
},
@@ -336,10 +345,13 @@ impl CodeObject {
}
})
}
}
impl fmt::Display for CodeObject {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fn display_inner(
&self,
f: &mut fmt::Formatter,
expand_codeobjects: bool,
level: usize,
) -> fmt::Result {
let label_targets: HashSet<&usize> = self.label_map.values().collect();
for (offset, instruction) in self.instructions.iter().enumerate() {
let arrow = if label_targets.contains(&offset) {
@@ -347,15 +359,40 @@ impl fmt::Display for CodeObject {
} else {
" "
};
write!(f, " {} {:5} ", arrow, offset)?;
instruction.fmt_dis(f, &self.label_map)?;
for _ in 0..level {
write!(f, " ")?;
}
write!(f, "{} {:5} ", arrow, offset)?;
instruction.fmt_dis(f, &self.label_map, expand_codeobjects, level)?;
}
Ok(())
}
pub fn display_expand_codeobjects<'a>(&'a self) -> impl fmt::Display + 'a {
struct Display<'a>(&'a CodeObject);
impl fmt::Display for Display<'_> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.0.display_inner(f, true, 1)
}
}
Display(self)
}
}
impl fmt::Display for CodeObject {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.display_inner(f, false, 1)
}
}
impl Instruction {
fn fmt_dis(&self, f: &mut fmt::Formatter, label_map: &HashMap<Label, usize>) -> fmt::Result {
fn fmt_dis(
&self,
f: &mut fmt::Formatter,
label_map: &HashMap<Label, usize>,
expand_codeobjects: bool,
level: usize,
) -> fmt::Result {
macro_rules! w {
($variant:ident) => {
write!(f, "{:20}\n", stringify!($variant))
@@ -389,7 +426,7 @@ impl Instruction {
format!("{:?}", symbols),
level
),
ImportStar { name, level } => w!(ImportStar, format!("{:?}", name), level),
ImportStar => w!(ImportStar),
ImportFrom { name } => w!(ImportFrom, name),
LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)),
StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)),
@@ -398,7 +435,14 @@ impl Instruction {
DeleteSubscript => w!(DeleteSubscript),
StoreAttr { name } => w!(StoreAttr, name),
DeleteAttr { name } => w!(DeleteAttr, name),
LoadConst { value } => w!(LoadConst, value),
LoadConst { value } => match value {
Constant::Code { code } if expand_codeobjects => {
writeln!(f, "LoadConst ({:?}):", code)?;
code.display_inner(f, true, level + 1)?;
Ok(())
}
_ => w!(LoadConst, value),
},
UnaryOperation { op } => w!(UnaryOperation, format!("{:?}", op)),
BinaryOperation { op, inplace } => w!(BinaryOperation, format!("{:?}", op), inplace),
LoadAttr { name } => w!(LoadAttr, name),
@@ -411,8 +455,10 @@ impl Instruction {
Continue => w!(Continue),
Break => w!(Break),
Jump { target } => w!(Jump, label_map[target]),
JumpIf { target } => w!(JumpIf, label_map[target]),
JumpIfTrue { target } => w!(JumpIfTrue, label_map[target]),
JumpIfFalse { target } => w!(JumpIfFalse, label_map[target]),
JumpIfTrueOrPop { target } => w!(JumpIfTrueOrPop, label_map[target]),
JumpIfFalseOrPop { target } => w!(JumpIfFalseOrPop, label_map[target]),
MakeFunction { flags } => w!(MakeFunction, format!("{:?}", flags)),
CallFunction { typ } => w!(CallFunction, format!("{:?}", typ)),
ForIter { target } => w!(ForIter, label_map[target]),

370
compiler/src/compile.rs
View File

@@ -8,16 +8,19 @@
use crate::error::{CompileError, CompileErrorType};
use crate::output_stream::{CodeObjectStream, OutputStream};
use crate::peephole::PeepholeOptimizer;
use crate::symboltable::{make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope};
use crate::symboltable::{
make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope, SymbolTable,
};
use num_complex::Complex64;
use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
use rustpython_parser::{ast, parser};
type BasicOutputStream = PeepholeOptimizer<CodeObjectStream>;
/// Main structure holding the state of compilation.
struct Compiler<O: OutputStream = BasicOutputStream> {
output_stack: Vec<O>,
scope_stack: Vec<SymbolScope>,
symbol_table_stack: Vec<SymbolTable>,
nxt_label: usize,
source_path: Option<String>,
current_source_location: ast::Location,
@@ -30,7 +33,7 @@ struct Compiler<O: OutputStream = BasicOutputStream> {
/// Compile a given sourcecode into a bytecode object.
pub fn compile(
source: &str,
mode: &Mode,
mode: Mode,
source_path: String,
optimize: u8,
) -> Result<CodeObject, CompileError> {
@@ -101,16 +104,34 @@ pub fn compile_program_single(
})
}
#[derive(Clone, Copy)]
pub enum Mode {
Exec,
Eval,
Single,
}
#[derive(Clone, Copy)]
enum EvalContext {
Statement,
Expression,
impl std::str::FromStr for Mode {
type Err = ModeParseError;
fn from_str(s: &str) -> Result<Self, ModeParseError> {
match s {
"exec" => Ok(Mode::Exec),
"eval" => Ok(Mode::Eval),
"single" => Ok(Mode::Single),
_ => Err(ModeParseError { _priv: () }),
}
}
}
#[derive(Debug)]
pub struct ModeParseError {
_priv: (),
}
impl std::fmt::Display for ModeParseError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, r#"mode should be "exec", "eval", or "single""#)
}
}
pub(crate) type Label = usize;
@@ -128,7 +149,7 @@ impl<O: OutputStream> Compiler<O> {
fn new(optimize: u8) -> Self {
Compiler {
output_stack: Vec::new(),
scope_stack: Vec::new(),
symbol_table_stack: Vec::new(),
nxt_label: 0,
source_path: None,
current_source_location: ast::Location::default(),
@@ -163,11 +184,23 @@ impl<O: OutputStream> Compiler<O> {
fn compile_program(
&mut self,
program: &ast::Program,
symbol_scope: SymbolScope,
symbol_table: SymbolTable,
) -> Result<(), CompileError> {
let size_before = self.output_stack.len();
self.scope_stack.push(symbol_scope);
self.compile_statements(&program.statements)?;
self.symbol_table_stack.push(symbol_table);
let (statements, doc) = get_doc(&program.statements);
if let Some(value) = doc {
self.emit(Instruction::LoadConst {
value: bytecode::Constant::String { value },
});
self.emit(Instruction::StoreName {
name: "__doc__".to_owned(),
scope: bytecode::NameScope::Global,
});
}
self.compile_statements(statements)?;
assert_eq!(self.output_stack.len(), size_before);
// Emit None at end:
@@ -181,9 +214,9 @@ impl<O: OutputStream> Compiler<O> {
fn compile_program_single(
&mut self,
program: &ast::Program,
symbol_scope: SymbolScope,
symbol_table: SymbolTable,
) -> Result<(), CompileError> {
self.scope_stack.push(symbol_scope);
self.symbol_table_stack.push(symbol_table);
let mut emitted_return = false;
@@ -220,9 +253,9 @@ impl<O: OutputStream> Compiler<O> {
fn compile_statement_eval(
&mut self,
statements: &[ast::Statement],
symbol_table: SymbolScope,
symbol_table: SymbolTable,
) -> Result<(), CompileError> {
self.scope_stack.push(symbol_table);
self.symbol_table_stack.push(symbol_table);
for statement in statements {
if let ast::StatementType::Expression { ref expression } = statement.node {
self.compile_expression(expression)?;
@@ -246,12 +279,11 @@ impl<O: OutputStream> Compiler<O> {
fn scope_for_name(&self, name: &str) -> bytecode::NameScope {
let symbol = self.lookup_name(name);
if symbol.is_global {
bytecode::NameScope::Global
} else if symbol.is_nonlocal {
bytecode::NameScope::NonLocal
} else {
bytecode::NameScope::Local
match symbol.scope {
SymbolScope::Global => bytecode::NameScope::Global,
SymbolScope::Nonlocal => bytecode::NameScope::NonLocal,
SymbolScope::Unknown => bytecode::NameScope::Local,
SymbolScope::Local => bytecode::NameScope::Local,
}
}
@@ -285,11 +317,15 @@ impl<O: OutputStream> Compiler<O> {
symbols: vec![],
level: 0,
});
if let Some(alias) = &name.alias {
for part in name.symbol.split('.').skip(1) {
self.emit(Instruction::LoadAttr {
name: part.to_owned(),
});
}
self.store_name(alias);
} else {
self.store_name(&name.symbol);
self.store_name(name.symbol.split('.').next().unwrap());
}
}
}
@@ -302,10 +338,12 @@ impl<O: OutputStream> Compiler<O> {
if import_star {
// from .... import *
self.emit(Instruction::ImportStar {
self.emit(Instruction::Import {
name: module.clone(),
symbols: vec!["*".to_owned()],
level: *level,
});
self.emit(Instruction::ImportStar);
} else {
// from mod import a, b as c
// First, determine the fromlist (for import lib):
@@ -350,14 +388,14 @@ impl<O: OutputStream> Compiler<O> {
match orelse {
None => {
// Only if:
self.compile_test(test, None, Some(end_label), EvalContext::Statement)?;
self.compile_jump_if(test, false, end_label)?;
self.compile_statements(body)?;
self.set_label(end_label);
}
Some(statements) => {
// if - else:
let else_label = self.new_label();
self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
self.compile_jump_if(test, false, else_label)?;
self.compile_statements(body)?;
self.emit(Instruction::Jump { target: end_label });
@@ -459,7 +497,7 @@ impl<O: OutputStream> Compiler<O> {
// if some flag, ignore all assert statements!
if self.optimize == 0 {
let end_label = self.new_label();
self.compile_test(test, Some(end_label), None, EvalContext::Statement)?;
self.compile_jump_if(test, true, end_label)?;
self.emit(Instruction::LoadName {
name: String::from("AssertionError"),
scope: bytecode::NameScope::Local,
@@ -1006,7 +1044,7 @@ impl<O: OutputStream> Compiler<O> {
self.set_label(start_label);
self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
self.compile_jump_if(test, false, else_label)?;
let was_in_loop = self.in_loop;
self.in_loop = true;
@@ -1118,12 +1156,9 @@ impl<O: OutputStream> Compiler<O> {
});
// if comparison result is false, we break with this value; if true, try the next one.
// (CPython compresses these three opcodes into JUMP_IF_FALSE_OR_POP)
self.emit(Instruction::Duplicate);
self.emit(Instruction::JumpIfFalse {
self.emit(Instruction::JumpIfFalseOrPop {
target: break_label,
});
self.emit(Instruction::Pop);
}
// handle the last comparison
@@ -1256,66 +1291,120 @@ impl<O: OutputStream> Compiler<O> {
self.emit(Instruction::BinaryOperation { op: i, inplace });
}
fn compile_test(
/// Implement boolean short circuit evaluation logic.
/// https://en.wikipedia.org/wiki/Short-circuit_evaluation
///
/// This means, in a boolean statement 'x and y' the variable y will
/// not be evaluated when x is false.
///
/// The idea is to jump to a label if the expression is either true or false
/// (indicated by the condition parameter).
fn compile_jump_if(
&mut self,
expression: &ast::Expression,
true_label: Option<Label>,
false_label: Option<Label>,
context: EvalContext,
condition: bool,
target_label: Label,
) -> Result<(), CompileError> {
// Compile expression for test, and jump to label if false
match &expression.node {
ast::ExpressionType::BoolOp { a, op, b } => match op {
ast::BooleanOperator::And => {
let f = false_label.unwrap_or_else(|| self.new_label());
self.compile_test(a, None, Some(f), context)?;
self.compile_test(b, true_label, false_label, context)?;
if false_label.is_none() {
self.set_label(f);
}
}
ast::BooleanOperator::Or => {
let t = true_label.unwrap_or_else(|| self.new_label());
self.compile_test(a, Some(t), None, context)?;
self.compile_test(b, true_label, false_label, context)?;
if true_label.is_none() {
self.set_label(t);
}
}
},
_ => {
self.compile_expression(expression)?;
match context {
EvalContext::Statement => {
if let Some(true_label) = true_label {
self.emit(Instruction::JumpIf { target: true_label });
}
if let Some(false_label) = false_label {
self.emit(Instruction::JumpIfFalse {
target: false_label,
});
ast::ExpressionType::BoolOp { op, values } => {
match op {
ast::BooleanOperator::And => {
if condition {
// If all values are true.
let end_label = self.new_label();
let (last_value, values) = values.split_last().unwrap();
// If any of the values is false, we can short-circuit.
for value in values {
self.compile_jump_if(value, false, end_label)?;
}
// It depends upon the last value now: will it be true?
self.compile_jump_if(last_value, true, target_label)?;
self.set_label(end_label);
} else {
// If any value is false, the whole condition is false.
for value in values {
self.compile_jump_if(value, false, target_label)?;
}
}
}
EvalContext::Expression => {
if let Some(true_label) = true_label {
self.emit(Instruction::Duplicate);
self.emit(Instruction::JumpIf { target: true_label });
self.emit(Instruction::Pop);
}
if let Some(false_label) = false_label {
self.emit(Instruction::Duplicate);
self.emit(Instruction::JumpIfFalse {
target: false_label,
});
self.emit(Instruction::Pop);
ast::BooleanOperator::Or => {
if condition {
// If any of the values is true.
for value in values {
self.compile_jump_if(value, true, target_label)?;
}
} else {
// If all of the values are false.
let end_label = self.new_label();
let (last_value, values) = values.split_last().unwrap();
// If any value is true, we can short-circuit:
for value in values {
self.compile_jump_if(value, true, end_label)?;
}
// It all depends upon the last value now!
self.compile_jump_if(last_value, false, target_label)?;
self.set_label(end_label);
}
}
}
}
ast::ExpressionType::Unop {
op: ast::UnaryOperator::Not,
a,
} => {
self.compile_jump_if(a, !condition, target_label)?;
}
_ => {
// Fall back case which always will work!
self.compile_expression(expression)?;
if condition {
self.emit(Instruction::JumpIfTrue {
target: target_label,
});
} else {
self.emit(Instruction::JumpIfFalse {
target: target_label,
});
}
}
}
Ok(())
}
/// Compile a boolean operation as an expression.
/// This means, that the last value remains on the stack.
fn compile_bool_op(
&mut self,
op: &ast::BooleanOperator,
values: &[ast::Expression],
) -> Result<(), CompileError> {
let end_label = self.new_label();
let (last_value, values) = values.split_last().unwrap();
for value in values {
self.compile_expression(value)?;
match op {
ast::BooleanOperator::And => {
self.emit(Instruction::JumpIfFalseOrPop { target: end_label });
}
ast::BooleanOperator::Or => {
self.emit(Instruction::JumpIfTrueOrPop { target: end_label });
}
}
}
// If all values did not qualify, take the value of the last value:
self.compile_expression(last_value)?;
self.set_label(end_label);
Ok(())
}
fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
trace!("Compiling {:?}", expression);
self.set_source_location(&expression.location);
@@ -1327,12 +1416,7 @@ impl<O: OutputStream> Compiler<O> {
args,
keywords,
} => self.compile_call(function, args, keywords)?,
BoolOp { .. } => self.compile_test(
expression,
Option::None,
Option::None,
EvalContext::Expression,
)?,
BoolOp { op, values } => self.compile_bool_op(op, values)?,
Binop { a, op, b } => {
self.compile_expression(a)?;
self.compile_expression(b)?;
@@ -1527,8 +1611,7 @@ impl<O: OutputStream> Compiler<O> {
IfExpression { test, body, orelse } => {
let no_label = self.new_label();
let end_label = self.new_label();
self.compile_test(test, Option::None, Option::None, EvalContext::Expression)?;
self.emit(Instruction::JumpIfFalse { target: no_label });
self.compile_jump_if(test, false, no_label)?;
// True case
self.compile_expression(body)?;
self.emit(Instruction::Jump { target: end_label });
@@ -1745,12 +1828,7 @@ impl<O: OutputStream> Compiler<O> {
// Now evaluate the ifs:
for if_condition in &generator.ifs {
self.compile_test(
if_condition,
None,
Some(start_label),
EvalContext::Statement,
)?
self.compile_jump_if(if_condition, false, start_label)?
}
}
@@ -1831,30 +1909,36 @@ impl<O: OutputStream> Compiler<O> {
}
fn compile_string(&mut self, string: &ast::StringGroup) -> Result<(), CompileError> {
match string {
ast::StringGroup::Joined { values } => {
for value in values {
self.compile_string(value)?;
if let Some(value) = try_get_constant_string(string) {
self.emit(Instruction::LoadConst {
value: bytecode::Constant::String { value },
});
} else {
match string {
ast::StringGroup::Joined { values } => {
for value in values {
self.compile_string(value)?;
}
self.emit(Instruction::BuildString { size: values.len() })
}
ast::StringGroup::Constant { value } => {
self.emit(Instruction::LoadConst {
value: bytecode::Constant::String {
value: value.to_string(),
},
});
}
ast::StringGroup::FormattedValue {
value,
conversion,
spec,
} => {
self.compile_expression(value)?;
self.emit(Instruction::FormatValue {
conversion: conversion.map(compile_conversion_flag),
spec: spec.clone(),
});
}
self.emit(Instruction::BuildString { size: values.len() })
}
ast::StringGroup::Constant { value } => {
self.emit(Instruction::LoadConst {
value: bytecode::Constant::String {
value: value.to_string(),
},
});
}
ast::StringGroup::FormattedValue {
value,
conversion,
spec,
} => {
self.compile_expression(value)?;
self.emit(Instruction::FormatValue {
conversion: conversion.map(compile_conversion_flag),
spec: spec.clone(),
});
}
}
Ok(())
@@ -1862,22 +1946,29 @@ impl<O: OutputStream> Compiler<O> {
// Scope helpers:
fn enter_scope(&mut self) {
// println!("Enter scope {:?}", self.scope_stack);
// println!("Enter scope {:?}", self.symbol_table_stack);
// Enter first subscope!
let scope = self.scope_stack.last_mut().unwrap().sub_scopes.remove(0);
self.scope_stack.push(scope);
let table = self
.symbol_table_stack
.last_mut()
.unwrap()
.sub_tables
.remove(0);
self.symbol_table_stack.push(table);
}
fn leave_scope(&mut self) {
// println!("Leave scope {:?}", self.scope_stack);
let scope = self.scope_stack.pop().unwrap();
assert!(scope.sub_scopes.is_empty());
// println!("Leave scope {:?}", self.symbol_table_stack);
let table = self.symbol_table_stack.pop().unwrap();
assert!(table.sub_tables.is_empty());
}
fn lookup_name(&self, name: &str) -> &Symbol {
// println!("Looking up {:?}", name);
let scope = self.scope_stack.last().unwrap();
scope.lookup(name).unwrap()
let symbol_table = self.symbol_table_stack.last().unwrap();
symbol_table.lookup(name).expect(
"The symbol must be present in the symbol table, even when it is undefined in python.",
)
}
// Low level helper functions:
@@ -1927,13 +2018,11 @@ impl<O: OutputStream> Compiler<O> {
}
fn get_doc(body: &[ast::Statement]) -> (&[ast::Statement], Option<String>) {
if let Some(val) = body.get(0) {
if let Some((val, body_rest)) = body.split_first() {
if let ast::StatementType::Expression { ref expression } = val.node {
if let ast::ExpressionType::String { value } = &expression.node {
if let ast::StringGroup::Constant { ref value } = value {
if let Some((_, body_rest)) = body.split_first() {
return (body_rest, Some(value.to_string()));
}
if let Some(value) = try_get_constant_string(value) {
return (body_rest, Some(value.to_string()));
}
}
}
@@ -1941,6 +2030,27 @@ fn get_doc(body: &[ast::Statement]) -> (&[ast::Statement], Option<String>) {
(body, None)
}
fn try_get_constant_string(string: &ast::StringGroup) -> Option<String> {
fn get_constant_string_inner(out_string: &mut String, string: &ast::StringGroup) -> bool {
match string {
ast::StringGroup::Constant { value } => {
out_string.push_str(&value);
true
}
ast::StringGroup::Joined { values } => values
.iter()
.all(|value| get_constant_string_inner(out_string, value)),
ast::StringGroup::FormattedValue { .. } => false,
}
}
let mut out_string = String::new();
if get_constant_string_inner(&mut out_string, string) {
Some(out_string)
} else {
None
}
}
fn compile_location(location: &ast::Location) -> bytecode::Location {
bytecode::Location::new(location.row(), location.column())
}
@@ -1988,11 +2098,11 @@ mod tests {
LoadConst {
value: Boolean { value: true }
},
JumpIf { target: 1 },
JumpIfTrue { target: 1 },
LoadConst {
value: Boolean { value: false }
},
JumpIf { target: 1 },
JumpIfTrue { target: 1 },
LoadConst {
value: Boolean { value: false }
},
@@ -2042,7 +2152,7 @@ mod tests {
LoadConst {
value: Boolean { value: false }
},
JumpIf { target: 1 },
JumpIfTrue { target: 1 },
LoadConst {
value: Boolean { value: false }
},

453
compiler/src/symboltable.rs
View File

@@ -12,47 +12,59 @@ use indexmap::map::IndexMap;
use rustpython_parser::ast;
use rustpython_parser::location::Location;
pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTableError> {
pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTableError> {
let mut builder: SymbolTableBuilder = Default::default();
builder.enter_scope();
builder.prepare();
builder.scan_program(program)?;
assert_eq!(builder.scopes.len(), 1);
let symbol_table = builder.scopes.pop().unwrap();
analyze_symbol_table(&symbol_table, None)?;
Ok(symbol_table)
builder.finish()
}
pub fn statements_to_symbol_table(
statements: &[ast::Statement],
) -> Result<SymbolScope, SymbolTableError> {
) -> Result<SymbolTable, SymbolTableError> {
let mut builder: SymbolTableBuilder = Default::default();
builder.enter_scope();
builder.prepare();
builder.scan_statements(statements)?;
assert_eq!(builder.scopes.len(), 1);
let symbol_table = builder.scopes.pop().unwrap();
analyze_symbol_table(&symbol_table, None)?;
Ok(symbol_table)
builder.finish()
}
/// Captures all symbols in the current scope, and has a list of subscopes in this scope.
#[derive(Clone, Default)]
pub struct SymbolScope {
pub struct SymbolTable {
/// A set of symbols present on this scope level.
pub symbols: IndexMap<String, Symbol>,
/// A list of subscopes in the order as found in the
/// AST nodes.
pub sub_scopes: Vec<SymbolScope>,
pub sub_tables: Vec<SymbolTable>,
}
impl SymbolTable {
fn new() -> Self {
SymbolTable {
symbols: Default::default(),
sub_tables: vec![],
}
}
}
/// Indicator for a single symbol what the scope of this symbol is.
/// The scope can be unknown, which is unfortunate, but not impossible.
#[derive(Debug, Clone)]
pub enum SymbolScope {
Global,
Nonlocal,
Local,
Unknown,
}
/// A single symbol in a table. Has various properties such as the scope
/// of the symbol, and also the various uses of the symbol.
#[derive(Debug, Clone)]
pub struct Symbol {
pub name: String,
pub is_global: bool,
pub is_local: bool,
pub is_nonlocal: bool,
// pub table: SymbolTableRef,
pub scope: SymbolScope,
pub is_param: bool,
pub is_referenced: bool,
pub is_assigned: bool,
@@ -64,9 +76,8 @@ impl Symbol {
fn new(name: &str) -> Self {
Symbol {
name: name.to_string(),
is_global: false,
is_local: false,
is_nonlocal: false,
// table,
scope: SymbolScope::Unknown,
is_param: false,
is_referenced: false,
is_assigned: false,
@@ -74,6 +85,22 @@ impl Symbol {
is_free: false,
}
}
pub fn is_global(&self) -> bool {
if let SymbolScope::Global = self.scope {
true
} else {
false
}
}
pub fn is_local(&self) -> bool {
if let SymbolScope::Local = self.scope {
true
} else {
false
}
}
}
#[derive(Debug)]
@@ -93,19 +120,19 @@ impl From<SymbolTableError> for CompileError {
type SymbolTableResult = Result<(), SymbolTableError>;
impl SymbolScope {
impl SymbolTable {
pub fn lookup(&self, name: &str) -> Option<&Symbol> {
self.symbols.get(name)
}
}
impl std::fmt::Debug for SymbolScope {
impl std::fmt::Debug for SymbolTable {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
f,
"SymbolScope({:?} symbols, {:?} sub scopes)",
"SymbolTable({:?} symbols, {:?} sub scopes)",
self.symbols.len(),
self.sub_scopes.len()
self.sub_tables.len()
)
}
}
@@ -113,75 +140,127 @@ impl std::fmt::Debug for SymbolScope {
/* Perform some sort of analysis on nonlocals, globals etc..
See also: https://github.com/python/cpython/blob/master/Python/symtable.c#L410
*/
fn analyze_symbol_table(
symbol_scope: &SymbolScope,
parent_symbol_scope: Option<&SymbolScope>,
) -> SymbolTableResult {
// Analyze sub scopes:
for sub_scope in &symbol_scope.sub_scopes {
analyze_symbol_table(&sub_scope, Some(symbol_scope))?;
}
// Analyze symbols:
for symbol in symbol_scope.symbols.values() {
analyze_symbol(symbol, parent_symbol_scope)?;
}
Ok(())
fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult {
let mut analyzer = SymbolTableAnalyzer::default();
analyzer.analyze_symbol_table(symbol_table)
}
fn analyze_symbol(symbol: &Symbol, parent_symbol_scope: Option<&SymbolScope>) -> SymbolTableResult {
if symbol.is_nonlocal {
// check if name is defined in parent scope!
if let Some(parent_symbol_scope) = parent_symbol_scope {
if !parent_symbol_scope.symbols.contains_key(&symbol.name) {
return Err(SymbolTableError {
error: format!("no binding for nonlocal '{}' found", symbol.name),
location: Default::default(),
});
}
} else {
return Err(SymbolTableError {
error: format!(
"nonlocal {} defined at place without an enclosing scope",
symbol.name
),
location: Default::default(),
});
/// Symbol table analysis. Can be used to analyze a fully
/// build symbol table structure. It will mark variables
/// as local variables for example.
#[derive(Default)]
struct SymbolTableAnalyzer {
tables: Vec<SymbolTable>,
}
impl SymbolTableAnalyzer {
fn analyze_symbol_table(&mut self, symbol_table: &mut SymbolTable) -> SymbolTableResult {
// Store a copy to determine the parent.
// TODO: this should be improved to resolve this clone action.
self.tables.push(symbol_table.clone());
// Analyze sub scopes:
for sub_table in &mut symbol_table.sub_tables {
self.analyze_symbol_table(sub_table)?;
}
self.tables.pop();
// Analyze symbols:
for symbol in symbol_table.symbols.values_mut() {
self.analyze_symbol(symbol)?;
}
Ok(())
}
// TODO: add more checks for globals
fn analyze_symbol(&self, symbol: &mut Symbol) -> SymbolTableResult {
match symbol.scope {
SymbolScope::Nonlocal => {
// check if name is defined in parent table!
let parent_symbol_table: Option<&SymbolTable> = self.tables.last();
// symbol.table.borrow().parent.clone();
Ok(())
if let Some(table) = parent_symbol_table {
if !table.symbols.contains_key(&symbol.name) {
return Err(SymbolTableError {
error: format!("no binding for nonlocal '{}' found", symbol.name),
location: Default::default(),
});
}
} else {
return Err(SymbolTableError {
error: format!(
"nonlocal {} defined at place without an enclosing scope",
symbol.name
),
location: Default::default(),
});
}
}
SymbolScope::Global => {
// TODO: add more checks for globals?
}
SymbolScope::Local => {
// all is well
}
SymbolScope::Unknown => {
if symbol.is_assigned {
symbol.scope = SymbolScope::Local;
}
}
}
Ok(())
}
}
#[derive(Debug, Clone)]
enum SymbolRole {
enum SymbolUsage {
Global,
Nonlocal,
Used,
Assigned,
Parameter,
}
#[derive(Default)]
struct SymbolTableBuilder {
// Scope stack.
scopes: Vec<SymbolScope>,
tables: Vec<SymbolTable>,
}
/// Enum to indicate in what mode an expression
/// was used.
/// In cpython this is stored in the AST, but I think this
/// is not logical, since it is not context free.
enum ExpressionContext {
Load,
Store,
Delete,
}
impl SymbolTableBuilder {
fn enter_scope(&mut self) {
let scope = Default::default();
self.scopes.push(scope);
// self.work_scopes.push(Default::default());
fn prepare(&mut self) {
let table = SymbolTable::new();
self.tables.push(table);
}
fn leave_scope(&mut self) {
// Pop scope and add to subscopes of parent scope.
// let work_scope = self.work_scopes.pop().unwrap();
let scope = self.scopes.pop().unwrap();
self.scopes.last_mut().unwrap().sub_scopes.push(scope);
fn finish(&mut self) -> Result<SymbolTable, SymbolTableError> {
assert_eq!(self.tables.len(), 1);
let mut symbol_table = self.tables.pop().unwrap();
analyze_symbol_table(&mut symbol_table)?;
Ok(symbol_table)
}
fn enter_block(&mut self) {
// let parent = Some(self.tables.last().unwrap().clone());
let table = SymbolTable::new();
self.tables.push(table);
}
fn leave_block(&mut self) {
// Pop symbol table and add to sub table of parent table.
let table = self.tables.pop().unwrap();
self.tables.last_mut().unwrap().sub_tables.push(table);
}
fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
@@ -204,7 +283,7 @@ impl SymbolTableBuilder {
}
fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
self.register_name(&parameter.arg, SymbolRole::Assigned)
self.register_name(&parameter.arg, SymbolUsage::Parameter)
}
fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
@@ -216,7 +295,7 @@ impl SymbolTableBuilder {
fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
if let Some(annotation) = &parameter.annotation {
self.scan_expression(&annotation)?;
self.scan_expression(&annotation, &ExpressionContext::Load)?;
}
Ok(())
}
@@ -226,12 +305,12 @@ impl SymbolTableBuilder {
match &statement.node {
Global { names } => {
for name in names {
self.register_name(name, SymbolRole::Global)?;
self.register_name(name, SymbolUsage::Global)?;
}
}
Nonlocal { names } => {
for name in names {
self.register_name(name, SymbolRole::Nonlocal)?;
self.register_name(name, SymbolUsage::Nonlocal)?;
}
}
FunctionDef {
@@ -242,16 +321,14 @@ impl SymbolTableBuilder {
returns,
..
} => {
self.scan_expressions(decorator_list)?;
self.register_name(name, SymbolRole::Assigned)?;
self.enter_function(args)?;
self.scan_statements(body)?;
self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
self.register_name(name, SymbolUsage::Assigned)?;
if let Some(expression) = returns {
self.scan_expression(expression)?;
self.scan_expression(expression, &ExpressionContext::Load)?;
}
self.leave_scope();
self.enter_function(args)?;
self.scan_statements(body)?;
self.leave_block();
}
ClassDef {
name,
@@ -260,19 +337,21 @@ impl SymbolTableBuilder {
keywords,
decorator_list,
} => {
self.register_name(name, SymbolRole::Assigned)?;
self.enter_scope();
self.register_name(name, SymbolUsage::Assigned)?;
self.enter_block();
self.scan_statements(body)?;
self.leave_scope();
self.scan_expressions(bases)?;
self.leave_block();
self.scan_expressions(bases, &ExpressionContext::Load)?;
for keyword in keywords {
self.scan_expression(&keyword.value)?;
self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
}
self.scan_expressions(decorator_list)?;
self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
}
Expression { expression } => {
self.scan_expression(expression, &ExpressionContext::Load)?
}
Expression { expression } => self.scan_expression(expression)?,
If { test, body, orelse } => {
self.scan_expression(test)?;
self.scan_expression(test, &ExpressionContext::Load)?;
self.scan_statements(body)?;
if let Some(code) = orelse {
self.scan_statements(code)?;
@@ -285,15 +364,15 @@ impl SymbolTableBuilder {
orelse,
..
} => {
self.scan_expression(target)?;
self.scan_expression(iter)?;
self.scan_expression(target, &ExpressionContext::Store)?;
self.scan_expression(iter, &ExpressionContext::Load)?;
self.scan_statements(body)?;
if let Some(code) = orelse {
self.scan_statements(code)?;
}
}
While { test, body, orelse } => {
self.scan_expression(test)?;
self.scan_expression(test, &ExpressionContext::Load)?;
self.scan_statements(body)?;
if let Some(code) = orelse {
self.scan_statements(code)?;
@@ -306,51 +385,54 @@ impl SymbolTableBuilder {
for name in names {
if let Some(alias) = &name.alias {
// `import mymodule as myalias`
self.register_name(alias, SymbolRole::Assigned)?;
self.register_name(alias, SymbolUsage::Assigned)?;
} else {
// `import module`
self.register_name(&name.symbol, SymbolRole::Assigned)?;
self.register_name(
name.symbol.split('.').next().unwrap(),
SymbolUsage::Assigned,
)?;
}
}
}
Return { value } => {
if let Some(expression) = value {
self.scan_expression(expression)?;
self.scan_expression(expression, &ExpressionContext::Load)?;
}
}
Assert { test, msg } => {
self.scan_expression(test)?;
self.scan_expression(test, &ExpressionContext::Load)?;
if let Some(expression) = msg {
self.scan_expression(expression)?;
self.scan_expression(expression, &ExpressionContext::Load)?;
}
}
Delete { targets } => {
self.scan_expressions(targets)?;
self.scan_expressions(targets, &ExpressionContext::Delete)?;
}
Assign { targets, value } => {
self.scan_expressions(targets)?;
self.scan_expression(value)?;
self.scan_expressions(targets, &ExpressionContext::Store)?;
self.scan_expression(value, &ExpressionContext::Load)?;
}
AugAssign { target, value, .. } => {
self.scan_expression(target)?;
self.scan_expression(value)?;
self.scan_expression(target, &ExpressionContext::Store)?;
self.scan_expression(value, &ExpressionContext::Load)?;
}
AnnAssign {
target,
annotation,
value,
} => {
self.scan_expression(target)?;
self.scan_expression(annotation)?;
self.scan_expression(target, &ExpressionContext::Store)?;
self.scan_expression(annotation, &ExpressionContext::Load)?;
if let Some(value) = value {
self.scan_expression(value)?;
self.scan_expression(value, &ExpressionContext::Load)?;
}
}
With { items, body, .. } => {
for item in items {
self.scan_expression(&item.context_expr)?;
self.scan_expression(&item.context_expr, &ExpressionContext::Load)?;
if let Some(expression) = &item.optional_vars {
self.scan_expression(expression)?;
self.scan_expression(expression, &ExpressionContext::Store)?;
}
}
self.scan_statements(body)?;
@@ -364,10 +446,10 @@ impl SymbolTableBuilder {
self.scan_statements(body)?;
for handler in handlers {
if let Some(expression) = &handler.typ {
self.scan_expression(expression)?;
self.scan_expression(expression, &ExpressionContext::Load)?;
}
if let Some(name) = &handler.name {
self.register_name(name, SymbolRole::Assigned)?;
self.register_name(name, SymbolUsage::Assigned)?;
}
self.scan_statements(&handler.body)?;
}
@@ -380,95 +462,102 @@ impl SymbolTableBuilder {
}
Raise { exception, cause } => {
if let Some(expression) = exception {
self.scan_expression(expression)?;
self.scan_expression(expression, &ExpressionContext::Load)?;
}
if let Some(expression) = cause {
self.scan_expression(expression)?;
self.scan_expression(expression, &ExpressionContext::Load)?;
}
}
}
Ok(())
}
fn scan_expressions(&mut self, expressions: &[ast::Expression]) -> SymbolTableResult {
fn scan_expressions(
&mut self,
expressions: &[ast::Expression],
context: &ExpressionContext,
) -> SymbolTableResult {
for expression in expressions {
self.scan_expression(expression)?;
self.scan_expression(expression, context)?;
}
Ok(())
}
fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult {
fn scan_expression(
&mut self,
expression: &ast::Expression,
context: &ExpressionContext,
) -> SymbolTableResult {
use ast::ExpressionType::*;
match &expression.node {
Binop { a, b, .. } => {
self.scan_expression(a)?;
self.scan_expression(b)?;
self.scan_expression(a, context)?;
self.scan_expression(b, context)?;
}
BoolOp { a, b, .. } => {
self.scan_expression(a)?;
self.scan_expression(b)?;
BoolOp { values, .. } => {
self.scan_expressions(values, context)?;
}
Compare { vals, .. } => {
self.scan_expressions(vals)?;
self.scan_expressions(vals, context)?;
}
Subscript { a, b } => {
self.scan_expression(a)?;
self.scan_expression(b)?;
self.scan_expression(a, context)?;
self.scan_expression(b, context)?;
}
Attribute { value, .. } => {
self.scan_expression(value)?;
self.scan_expression(value, context)?;
}
Dict { elements } => {
for (key, value) in elements {
if let Some(key) = key {
self.scan_expression(key)?;
self.scan_expression(key, context)?;
} else {
// dict unpacking marker
}
self.scan_expression(value)?;
self.scan_expression(value, context)?;
}
}
Await { value } => {
self.scan_expression(value)?;
self.scan_expression(value, context)?;
}
Yield { value } => {
if let Some(expression) = value {
self.scan_expression(expression)?;
self.scan_expression(expression, context)?;
}
}
YieldFrom { value } => {
self.scan_expression(value)?;
self.scan_expression(value, context)?;
}
Unop { a, .. } => {
self.scan_expression(a)?;
self.scan_expression(a, context)?;
}
True | False | None | Ellipsis => {}
Number { .. } => {}
Starred { value } => {
self.scan_expression(value)?;
self.scan_expression(value, context)?;
}
Bytes { .. } => {}
Tuple { elements } | Set { elements } | List { elements } | Slice { elements } => {
self.scan_expressions(elements)?;
self.scan_expressions(elements, &ExpressionContext::Load)?;
}
Comprehension { kind, generators } => {
match **kind {
ast::ComprehensionKind::GeneratorExpression { ref element }
| ast::ComprehensionKind::List { ref element }
| ast::ComprehensionKind::Set { ref element } => {
self.scan_expression(element)?;
self.scan_expression(element, &ExpressionContext::Load)?;
}
ast::ComprehensionKind::Dict { ref key, ref value } => {
self.scan_expression(&key)?;
self.scan_expression(&value)?;
self.scan_expression(&key, &ExpressionContext::Load)?;
self.scan_expression(&value, &ExpressionContext::Load)?;
}
}
for generator in generators {
self.scan_expression(&generator.target)?;
self.scan_expression(&generator.iter)?;
self.scan_expression(&generator.target, &ExpressionContext::Store)?;
self.scan_expression(&generator.iter, &ExpressionContext::Load)?;
for if_expr in &generator.ifs {
self.scan_expression(if_expr)?;
self.scan_expression(if_expr, &ExpressionContext::Load)?;
}
}
}
@@ -477,27 +566,38 @@ impl SymbolTableBuilder {
args,
keywords,
} => {
self.scan_expression(function)?;
self.scan_expressions(args)?;
self.scan_expression(function, &ExpressionContext::Load)?;
self.scan_expressions(args, &ExpressionContext::Load)?;
for keyword in keywords {
self.scan_expression(&keyword.value)?;
self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
}
}
String { value } => {
self.scan_string_group(value)?;
}
Identifier { name } => {
self.register_name(name, SymbolRole::Used)?;
// Determine the contextual usage of this symbol:
match context {
ExpressionContext::Delete => {
self.register_name(name, SymbolUsage::Used)?;
}
ExpressionContext::Load => {
self.register_name(name, SymbolUsage::Used)?;
}
ExpressionContext::Store => {
self.register_name(name, SymbolUsage::Assigned)?;
}
}
}
Lambda { args, body } => {
self.enter_function(args)?;
self.scan_expression(body)?;
self.leave_scope();
self.scan_expression(body, &ExpressionContext::Load)?;
self.leave_block();
}
IfExpression { test, body, orelse } => {
self.scan_expression(test)?;
self.scan_expression(body)?;
self.scan_expression(orelse)?;
self.scan_expression(test, &ExpressionContext::Load)?;
self.scan_expression(body, &ExpressionContext::Load)?;
self.scan_expression(orelse, &ExpressionContext::Load)?;
}
}
Ok(())
@@ -505,10 +605,10 @@ impl SymbolTableBuilder {
fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult {
// Evaluate eventual default parameters:
self.scan_expressions(&args.defaults)?;
self.scan_expressions(&args.defaults, &ExpressionContext::Load)?;
for kw_default in &args.kw_defaults {
if let Some(expression) = kw_default {
self.scan_expression(&expression)?;
self.scan_expression(&expression, &ExpressionContext::Load)?;
}
}
@@ -522,7 +622,7 @@ impl SymbolTableBuilder {
self.scan_parameter_annotation(name)?;
}
self.enter_scope();
self.enter_block();
// Fill scope with parameter names:
self.scan_parameters(&args.args)?;
@@ -540,7 +640,7 @@ impl SymbolTableBuilder {
match group {
ast::StringGroup::Constant { .. } => {}
ast::StringGroup::FormattedValue { value, .. } => {
self.scan_expression(value)?;
self.scan_expression(value, &ExpressionContext::Load)?;
}
ast::StringGroup::Joined { values } => {
for subgroup in values {
@@ -552,22 +652,22 @@ impl SymbolTableBuilder {
}
#[allow(clippy::single_match)]
fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult {
let scope_depth = self.scopes.len();
let current_scope = self.scopes.last_mut().unwrap();
fn register_name(&mut self, name: &str, role: SymbolUsage) -> SymbolTableResult {
let scope_depth = self.tables.len();
let table = self.tables.last_mut().unwrap();
let location = Default::default();
// Some checks:
if current_scope.symbols.contains_key(name) {
if table.symbols.contains_key(name) {
// Role already set..
match role {
SymbolRole::Global => {
SymbolUsage::Global => {
return Err(SymbolTableError {
error: format!("name '{}' is used prior to global declaration", name),
location,
})
}
SymbolRole::Nonlocal => {
SymbolUsage::Nonlocal => {
return Err(SymbolTableError {
error: format!("name '{}' is used prior to nonlocal declaration", name),
location,
@@ -581,7 +681,7 @@ impl SymbolTableBuilder {
// Some more checks:
match role {
SymbolRole::Nonlocal => {
SymbolUsage::Nonlocal => {
if scope_depth < 2 {
return Err(SymbolTableError {
error: format!("cannot define nonlocal '{}' at top level.", name),
@@ -595,25 +695,38 @@ impl SymbolTableBuilder {
}
// Insert symbol when required:
if !current_scope.symbols.contains_key(name) {
if !table.symbols.contains_key(name) {
let symbol = Symbol::new(name);
current_scope.symbols.insert(name.to_string(), symbol);
table.symbols.insert(name.to_string(), symbol);
}
// Set proper flags on symbol:
let symbol = current_scope.symbols.get_mut(name).unwrap();
let symbol = table.symbols.get_mut(name).unwrap();
match role {
SymbolRole::Nonlocal => {
symbol.is_nonlocal = true;
SymbolUsage::Nonlocal => {
if let SymbolScope::Unknown = symbol.scope {
symbol.scope = SymbolScope::Nonlocal;
} else {
return Err(SymbolTableError {
error: format!("Symbol {} scope cannot be set to nonlocal, since its scope was already determined otherwise.", name),
location,
});
}
}
SymbolRole::Assigned => {
SymbolUsage::Parameter | SymbolUsage::Assigned => {
symbol.is_assigned = true;
// symbol.is_local = true;
}
SymbolRole::Global => {
symbol.is_global = true;
SymbolUsage::Global => {
if let SymbolScope::Unknown = symbol.scope {
symbol.scope = SymbolScope::Global;
} else {
return Err(SymbolTableError {
error: format!("Symbol {} scope cannot be set to global, since its scope was already determined otherwise.", name),
location,
});
}
}
SymbolRole::Used => {
SymbolUsage::Used => {
symbol.is_referenced = true;
}
}

7
crawl_sourcecode.py
View File

@@ -12,6 +12,7 @@ $ diff cpython.txt rustpython.txt
import ast
import sys
import symtable
import dis
filename = sys.argv[1]
print('Crawling file:', filename)
@@ -67,3 +68,9 @@ def print_table(table, indent=0):
table = symtable.symtable(source, 'a', 'exec')
print_table(table)
print()
print('======== dis.dis ========')
print()
co = compile(source, filename, 'exec')
print(dis.dis(co))

20
derive/src/compile_bytecode.rs
View File

@@ -41,7 +41,7 @@ impl CompilationSource {
fn compile_string(
&self,
source: &str,
mode: &compile::Mode,
mode: compile::Mode,
module_name: String,
) -> Result<CodeObject, Diagnostic> {
compile::compile(source, mode, module_name, 0)
@@ -50,7 +50,7 @@ impl CompilationSource {
fn compile(
&self,
mode: &compile::Mode,
mode: compile::Mode,
module_name: String,
) -> Result<HashMap<String, FrozenModule>, Diagnostic> {
Ok(match &self.kind {
@@ -94,7 +94,7 @@ impl CompilationSource {
&self,
path: &Path,
parent: String,
mode: &compile::Mode,
mode: compile::Mode,
) -> Result<HashMap<String, FrozenModule>, Diagnostic> {
let mut code_map = HashMap::new();
let paths = fs::read_dir(&path).map_err(|err| {
@@ -169,15 +169,13 @@ impl PyCompileInput {
for meta in &self.metas {
if let Meta::NameValue(name_value) = meta {
if name_value.ident == "mode" {
mode = Some(match &name_value.lit {
Lit::Str(s) => match s.value().as_str() {
"exec" => compile::Mode::Exec,
"eval" => compile::Mode::Eval,
"single" => compile::Mode::Single,
_ => bail_span!(s, "mode must be exec, eval, or single"),
match &name_value.lit {
Lit::Str(s) => match s.value().parse() {
Ok(mode_val) => mode = Some(mode_val),
Err(e) => bail_span!(s, "{}", e),
},
_ => bail_span!(name_value.lit, "mode must be a string"),
})
}
} else if name_value.ident == "module_name" {
module_name = Some(match &name_value.lit {
Lit::Str(s) => s.value(),
@@ -225,7 +223,7 @@ impl PyCompileInput {
)
})?
.compile(
&mode.unwrap_or(compile::Mode::Exec),
mode.unwrap_or(compile::Mode::Exec),
module_name.unwrap_or_else(|| "frozen".to_string()),
)
}

10
derive/src/pyclass.rs
View File

@@ -424,10 +424,12 @@ pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result<TokenSt
vm: &::rustpython_vm::vm::VirtualMachine,
cls: ::rustpython_vm::obj::objtype::PyClassRef,
) -> ::rustpython_vm::pyobject::PyResult<::rustpython_vm::obj::objtuple::PyTupleRef> {
let tuple: ::rustpython_vm::obj::objtuple::PyTuple =
vec![#(::rustpython_vm::pyobject::IntoPyObject
::into_pyobject(self.#field_names, vm)?
),*].into();
let tuple = ::rustpython_vm::obj::objtuple::PyTuple::from(
vec![#(::rustpython_vm::pyobject::IntoPyObject::into_pyobject(
::std::clone::Clone::clone(&self.#field_names),
vm,
)?),*],
);
::rustpython_vm::pyobject::PyValue::into_ref_with_type(tuple, vm, cls)
}
}

92
examples/dis.rs Normal file
View File

@@ -0,0 +1,92 @@
/// This an example usage of the rustpython_compiler crate.
/// This program reads, parses, and compiles a file you provide
/// to RustPython bytecode, and then displays the output in the
/// `dis.dis` format.
///
/// example usage:
/// $ cargo run --release --example dis demo*.py
#[macro_use]
extern crate clap;
extern crate env_logger;
#[macro_use]
extern crate log;
use clap::{App, Arg};
use rustpython_compiler::compile;
use std::error::Error;
use std::fs;
use std::path::Path;
fn main() {
env_logger::init();
let app = App::new("dis")
.version(crate_version!())
.author(crate_authors!())
.about("Compiles and disassembles python script files for viewing their bytecode.")
.arg(
Arg::with_name("scripts")
.help("Scripts to scan")
.multiple(true)
.required(true),
)
.arg(
Arg::with_name("mode")
.help("The mode to compile the scripts in")
.long("mode")
.short("m")
.default_value("exec")
.possible_values(&["exec", "single", "eval"])
.takes_value(true),
)
.arg(
Arg::with_name("no_expand")
.help(
"Don't expand CodeObject LoadConst instructions to show \
the instructions inside",
)
.long("no-expand")
.short("x"),
)
.arg(
Arg::with_name("optimize")
.help("The amount of optimization to apply to the compiled bytecode")
.short("O")
.multiple(true),
);
let matches = app.get_matches();
let mode = matches.value_of_lossy("mode").unwrap().parse().unwrap();
let expand_codeobjects = !matches.is_present("no_expand");
let optimize = matches.occurrences_of("optimize") as u8;
let scripts = matches.values_of_os("scripts").unwrap();
for script in scripts.map(Path::new) {
if script.exists() && script.is_file() {
let res = display_script(script, mode, optimize, expand_codeobjects);
if let Err(e) = res {
error!("Error while compiling {:?}: {}", script, e);
}
} else {
eprintln!("{:?} is not a file.", script);
}
}
}
fn display_script(
path: &Path,
mode: compile::Mode,
optimize: u8,
expand_codeobjects: bool,
) -> Result<(), Box<dyn Error>> {
let source = fs::read_to_string(path)?;
let code = compile::compile(&source, mode, path.to_string_lossy().into_owned(), optimize)?;
println!("{}:", path.display());
if expand_codeobjects {
println!("{}", code.display_expand_codeobjects());
} else {
println!("{}", code);
}
Ok(())
}

57
examples/parse_folder.rs
View File

@@ -14,8 +14,8 @@ extern crate log;
use clap::{App, Arg};
use rustpython_parser::{ast, parser};
use std::path::{Path, PathBuf};
use std::time::Instant;
use std::path::Path;
use std::time::{Duration, Instant};
fn main() {
env_logger::init();
@@ -61,30 +61,45 @@ fn parse_folder(path: &Path) -> std::io::Result<Vec<ParsedFile>> {
}
if metadata.is_file() && path.extension().and_then(|s| s.to_str()) == Some("py") {
let result = parse_python_file(&path);
match &result {
let parsed_file = parse_python_file(&path);
match &parsed_file.result {
Ok(_) => {}
Err(y) => error!("Erreur in file {:?} {:?}", path, y),
}
res.push(ParsedFile {
filename: Box::new(path),
result,
});
res.push(parsed_file);
}
}
Ok(res)
}
fn parse_python_file(filename: &Path) -> ParseResult {
fn parse_python_file(filename: &Path) -> ParsedFile {
info!("Parsing file {:?}", filename);
let source = std::fs::read_to_string(filename).map_err(|e| e.to_string())?;
parser::parse_program(&source).map_err(|e| e.to_string())
match std::fs::read_to_string(filename) {
Err(e) => ParsedFile {
// filename: Box::new(filename.to_path_buf()),
// code: "".to_string(),
num_lines: 0,
result: Err(e.to_string()),
},
Ok(source) => {
let num_lines = source.to_string().lines().count();
let result = parser::parse_program(&source).map_err(|e| e.to_string());
ParsedFile {
// filename: Box::new(filename.to_path_buf()),
// code: source.to_string(),
num_lines,
result,
}
}
}
}
fn statistics(results: ScanResult) {
// println!("Processed {:?} files", res.len());
println!("Scanned a total of {} files", results.parsed_files.len());
let total = results.parsed_files.len();
let total: usize = results.parsed_files.len();
let total_lines: usize = results.parsed_files.iter().map(|p| p.num_lines).sum();
let failed = results
.parsed_files
.iter()
@@ -103,9 +118,19 @@ fn statistics(results: ScanResult) {
let duration = results.t2 - results.t1;
println!("Total time spend: {:?}", duration);
println!(
"File processing rate: {} files/second",
(total * 1_000_000) as f64 / duration.as_micros() as f64
"Processed {} files. That's {} files/second",
total,
rate(total, duration)
);
println!(
"Processed {} lines of python code. That's {} lines/second",
total_lines,
rate(total_lines, duration)
);
}
fn rate(counter: usize, duration: Duration) -> f64 {
(counter * 1_000_000) as f64 / duration.as_micros() as f64
}
struct ScanResult {
@@ -115,7 +140,9 @@ struct ScanResult {
}
struct ParsedFile {
filename: Box<PathBuf>,
// filename: Box<PathBuf>,
// code: String,
num_lines: usize,
result: ParseResult,
}

3
parser/src/ast.rs
View File

@@ -148,9 +148,8 @@ pub type Expression = Located<ExpressionType>;
#[derive(Debug, PartialEq)]
pub enum ExpressionType {
BoolOp {
a: Box<Expression>,
op: BooleanOperator,
b: Box<Expression>,
values: Vec<Expression>,
},
Binop {
a: Box<Expression>,

107
parser/src/lexer.rs
View File

@@ -340,18 +340,7 @@ where
/// Lex a hex/octal/decimal/binary number without a decimal point.
fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> LexResult {
let mut value_text = String::new();
loop {
if let Some(c) = self.take_number(radix) {
value_text.push(c);
} else if self.chr0 == Some('_') {
self.next_char();
} else {
break;
}
}
let value_text = self.radix_run(radix);
let end_pos = self.get_pos();
let value = BigInt::from_str_radix(&value_text, radix).map_err(|e| LexicalError {
error: LexicalErrorType::OtherError(format!("{:?}", e)),
@@ -360,24 +349,19 @@ where
Ok((start_pos, Tok::Int { value }, end_pos))
}
/// Lex a normal number, that is, no octal, hex or binary number.
fn lex_normal_number(&mut self) -> LexResult {
let start_pos = self.get_pos();
let mut value_text = String::new();
// Normal number:
while let Some(c) = self.take_number(10) {
value_text.push(c);
}
let mut value_text = self.radix_run(10);
// If float:
if self.chr0 == Some('.') || self.at_exponent() {
// Take '.':
if self.chr0 == Some('.') {
value_text.push(self.next_char().unwrap());
while let Some(c) = self.take_number(10) {
value_text.push(c);
}
value_text.push_str(&self.radix_run(10));
}
// 1e6 for example:
@@ -389,9 +373,7 @@ where
value_text.push(self.next_char().unwrap());
}
while let Some(c) = self.take_number(10) {
value_text.push(c);
}
value_text.push_str(&self.radix_run(10));
}
let value = f64::from_str(&value_text).unwrap();
@@ -426,6 +408,57 @@ where
}
}
/// Consume a sequence of numbers with the given radix,
/// the digits can be decorated with underscores
/// like this: '1_2_3_4' == '1234'
fn radix_run(&mut self, radix: u32) -> String {
let mut value_text = String::new();
loop {
if let Some(c) = self.take_number(radix) {
value_text.push(c);
} else if self.chr0 == Some('_') && Lexer::<T>::is_digit_of_radix(self.chr1, radix) {
self.next_char();
} else {
break;
}
}
value_text
}
/// Consume a single character with the given radix.
fn take_number(&mut self, radix: u32) -> Option<char> {
let take_char = Lexer::<T>::is_digit_of_radix(self.chr0, radix);
if take_char {
Some(self.next_char().unwrap())
} else {
None
}
}
/// Test if a digit is of a certain radix.
fn is_digit_of_radix(c: Option<char>, radix: u32) -> bool {
match radix {
2 => match c {
Some('0'..='1') => true,
_ => false,
},
8 => match c {
Some('0'..='7') => true,
_ => false,
},
10 => match c {
Some('0'..='9') => true,
_ => false,
},
16 => match c {
Some('0'..='9') | Some('a'..='f') | Some('A'..='F') => true,
_ => false,
},
x => unimplemented!("Radix not implemented: {}", x),
}
}
/// Test if we face '[eE][-+]?[0-9]+'
fn at_exponent(&self) -> bool {
match self.chr0 {
@@ -626,34 +659,6 @@ where
}
}
fn take_number(&mut self, radix: u32) -> Option<char> {
let take_char = match radix {
2 => match self.chr0 {
Some('0'..='1') => true,
_ => false,
},
8 => match self.chr0 {
Some('0'..='7') => true,
_ => false,
},
10 => match self.chr0 {
Some('0'..='9') => true,
_ => false,
},
16 => match self.chr0 {
Some('0'..='9') | Some('a'..='f') | Some('A'..='F') => true,
_ => false,
},
x => unimplemented!("Radix not implemented: {}", x),
};
if take_char {
Some(self.next_char().unwrap())
} else {
None
}
}
/// This is the main entry point. Call this function to retrieve the next token.
/// This function is used by the iterator implementation.
fn inner_next(&mut self) -> LexResult {

36
parser/src/python.lalrpop
View File

@@ -247,7 +247,7 @@ ImportDots: usize = {
ImportAsNames: Vec<ast::ImportSymbol> = {
<i:OneOrMore<ImportAsAlias<Identifier>>> => i,
"(" <i:OneOrMore<ImportAsAlias<Identifier>>> ")" => i,
"(" <i:OneOrMore<ImportAsAlias<Identifier>>> ","? ")" => i,
"*" => {
// Star import all
vec![ast::ImportSymbol { symbol: "*".to_string(), alias: None }]
@@ -659,18 +659,32 @@ LambdaDef: ast::Expression = {
}
OrTest: ast::Expression = {
AndTest,
<e1:OrTest> <location:@L> "or" <e2:AndTest> => ast::Expression {
location,
node: ast::ExpressionType::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) }
<e1:AndTest> <location:@L> <e2:("or" AndTest)*> => {
if e2.is_empty() {
e1
} else {
let mut values = vec![e1];
values.extend(e2.into_iter().map(|e| e.1));
ast::Expression {
location,
node: ast::ExpressionType::BoolOp { op: ast::BooleanOperator::Or, values }
}
}
},
};
AndTest: ast::Expression = {
NotTest,
<e1:AndTest> <location:@L> "and" <e2:NotTest> => ast::Expression {
location,
node: ast::ExpressionType::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) }
<e1:NotTest> <location:@L> <e2:("and" NotTest)*> => {
if e2.is_empty() {
e1
} else {
let mut values = vec![e1];
values.extend(e2.into_iter().map(|e| e.1));
ast::Expression {
location,
node: ast::ExpressionType::BoolOp { op: ast::BooleanOperator::And, values }
}
}
},
};
@@ -952,11 +966,11 @@ Atom: ast::Expression = {
};
ListLiteralValues: Vec<ast::Expression> = {
<e:OneOrMore<TestOrStarExpr>> <_trailing_comma:","?> => e,
<e:OneOrMore<TestOrStarExpr>> ","? => e,
};
DictLiteralValues: Vec<(Option<ast::Expression>, ast::Expression)> = {
<elements:OneOrMore<DictElement>> <_trailing_comma:","?> => elements,
<elements:OneOrMore<DictElement>> ","? => elements,
};
DictEntry: (ast::Expression, ast::Expression) = {

15
redox/comment-cargo.sh Executable file
View File

@@ -0,0 +1,15 @@
#!/bin/bash
set -e
cargo=${1:-Cargo.toml}
tmpfile=$(mktemp)
awk '
/REDOX START/{redox=1; print; next}
/REDOX END/{redox=0}
{if (redox) print "#", $0; else print}
' "$cargo" >"$tmpfile"
mv "$tmpfile" "$cargo"

1
redox/recipe.sh
View File

@@ -1,4 +1,5 @@
GIT=https://github.com/RustPython/RustPython
BRANCH=redox-release
CARGOFLAGS=--no-default-features
export BUILDTIME_RUSTPYTHONPATH=/lib/rustpython/

15
redox/uncomment-cargo.sh Executable file
View File

@@ -0,0 +1,15 @@
#!/bin/bash
set -e
cargo=${1:-Cargo.toml}
tmpfile=$(mktemp)
awk '
/REDOX START/{redox=1; print; next}
/REDOX END/{redox=0}
{if (redox) sub(/^#\s*/, ""); print}
' "$cargo" >"$tmpfile"
mv "$tmpfile" "$cargo"

17
src/main.rs
View File

@@ -312,11 +312,20 @@ fn run_rustpython(vm: &VirtualMachine, matches: &ArgMatches) -> PyResult<()> {
}
let scope = vm.new_scope_with_builtins();
let main_module = vm.ctx.new_module("__main__", scope.globals.clone());
let main_module = vm.new_module("__main__", scope.globals.clone());
vm.get_attribute(vm.sys_module.clone(), "modules")?
.set_item("__main__", main_module, vm)?;
let site_result = vm.import("site", &vm.ctx.new_tuple(vec![]), 0);
if site_result.is_err() {
warn!(
"Failed to import site, consider adding the Lib directory to your RUSTPYTHONPATH \
environment variable",
);
}
// Figure out if a -c option was given:
if let Some(command) = matches.value_of("c") {
run_command(&vm, scope, command.to_string())?;
@@ -333,7 +342,7 @@ fn run_rustpython(vm: &VirtualMachine, matches: &ArgMatches) -> PyResult<()> {
fn _run_string(vm: &VirtualMachine, scope: Scope, source: &str, source_path: String) -> PyResult {
let code_obj = vm
.compile(source, &compile::Mode::Exec, source_path.clone())
.compile(source, compile::Mode::Exec, source_path.clone())
.map_err(|err| vm.new_syntax_error(&err))?;
// trace!("Code object: {:?}", code_obj.borrow());
scope
@@ -359,7 +368,7 @@ fn run_module(vm: &VirtualMachine, module: &str) -> PyResult<()> {
debug!("Running module {}", module);
let runpy = vm.import("runpy", &vm.ctx.new_tuple(vec![]), 0)?;
let run_module_as_main = vm.get_attribute(runpy, "_run_module_as_main")?;
vm.invoke(run_module_as_main, vec![vm.new_str(module.to_owned())])?;
vm.invoke(&run_module_as_main, vec![vm.new_str(module.to_owned())])?;
Ok(())
}
@@ -426,7 +435,7 @@ fn test_run_script() {
}
fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), CompileError> {
match vm.compile(source, &compile::Mode::Single, "<stdin>".to_string()) {
match vm.compile(source, compile::Mode::Single, "<stdin>".to_string()) {
Ok(code) => {
match vm.run_code_obj(code, scope.clone()) {
Ok(value) => {

4
tests/snippets/builtin_complex.py
View File

@@ -145,3 +145,7 @@ assert '(1+1j)' == str(1+1j)
assert '(1-1j)' == str(1-1j)
assert '(1+1j)' == repr(1+1j)
assert '(1-1j)' == repr(1-1j)
# __getnewargs__
assert (3 + 5j).__getnewargs__() == (3.0, 5.0)
assert (5j).__getnewargs__() == (0.0, 5.0)

1
tests/snippets/builtin_divmod.py
View File

@@ -3,6 +3,7 @@ from testutils import assert_raises
assert divmod(11, 3) == (3, 2)
assert divmod(8,11) == (0, 8)
assert divmod(0.873, 0.252) == (3.0, 0.11699999999999999)
assert divmod(-86340, 86400) == (-1, 60)
assert_raises(ZeroDivisionError, lambda: divmod(5, 0), 'divmod by zero')
assert_raises(ZeroDivisionError, lambda: divmod(5.0, 0.0), 'divmod by zero')

21
tests/snippets/builtin_round.py Normal file
View File

@@ -0,0 +1,21 @@
from testutils import assertRaises
assert round(0) == 0
assert isinstance(round(0), int)
assert round(0.0) == 0
assert isinstance(round(0.0), int)
assert round(0, None) == 0
assert isinstance(round(0, None), int)
assert round(0.0, None) == 0
assert isinstance(round(0, None), int)
assert round(0, 0) == 0
assert isinstance(round(0, 0), int)
assert round(0.0, 0) == 0.0 # Cannot check the type
assert isinstance(round(0.0, 0), float)
with assertRaises(TypeError):
round(0, 0.0)
with assertRaises(TypeError):
round(0.0, 0.0)

9
tests/snippets/class.py
View File

@@ -155,3 +155,12 @@ assert T4.t1.__doc__ == "t1"
cm = classmethod(lambda cls: cls)
assert cm.__func__(int) is int
assert str(super(int, 5)) == "<super: <class 'int'>, <int object>>"
class T5(int):
pass
assert str(super(int, T5(5))) == "<super: <class 'int'>, <T5 object>>"
#assert str(super(type, None)) == "<super: <class 'type'>, NULL>"

17
tests/snippets/ellipsis.py
View File

@@ -11,6 +11,23 @@ assert b is c
assert b is d
assert d is e
assert Ellipsis.__repr__() == 'Ellipsis'
assert Ellipsis.__reduce__() == 'Ellipsis'
assert type(Ellipsis).__new__(type(Ellipsis)) == Ellipsis
assert type(Ellipsis).__reduce__(Ellipsis) == 'Ellipsis'
try:
type(Ellipsis).__new__(type(1))
except TypeError:
pass
else:
assert False, '`Ellipsis.__new__` should only accept `type(Ellipsis)` as argument'
try:
type(Ellipsis).__reduce__(1)
except TypeError:
pass
else:
assert False, '`Ellipsis.__reduce__` should only accept `Ellipsis` as argument'
assert Ellipsis is ...
Ellipsis = 2
assert Ellipsis is not ...

16
tests/snippets/floats.py
View File

@@ -151,12 +151,20 @@ assert float(1.2) == 1.2
assert math.trunc(1.2) == 1
assert_raises(OverflowError, float('inf').__trunc__)
assert_raises(ValueError, float('nan').__trunc__)
assert 0.5.__round__() == 0.0
assert 1.5.__round__() == 2.0
assert isinstance(0.5.__round__(), int)
assert isinstance(1.5.__round__(), int)
assert 0.5.__round__() == 0
assert 1.5.__round__() == 2
assert isinstance(0.5.__round__(0), float)
assert isinstance(1.5.__round__(0), float)
assert 0.5.__round__(0) == 0.0
assert 1.5.__round__(0) == 2.0
assert 0.5.__round__(None) == 0.0
assert 1.5.__round__(None) == 2.0
assert isinstance(0.5.__round__(None), int)
assert isinstance(1.5.__round__(None), int)
assert 0.5.__round__(None) == 0
assert 1.5.__round__(None) == 2
assert_raises(TypeError, lambda: 0.5.__round__(0.0))
assert_raises(TypeError, lambda: 1.5.__round__(0.0))
assert_raises(OverflowError, float('inf').__round__)
assert_raises(ValueError, float('nan').__round__)

18
tests/snippets/function.py
View File

@@ -1,4 +1,3 @@
__name__ = "function"
@@ -72,3 +71,20 @@ def f6():
f6()
def f7():
try:
def t() -> void: # noqa: F821
pass
except NameError:
return True
return False
assert f7()
def f8() -> int:
return 10
assert f8() == 10

BIN
tests/snippets/imghdrdata/python.bmp Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.1 KiB

BIN
tests/snippets/imghdrdata/python.exr Normal file
View File

Binary file not shown.

BIN
tests/snippets/imghdrdata/python.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 610 B

BIN
tests/snippets/imghdrdata/python.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 543 B

3
tests/snippets/imghdrdata/python.pbm Normal file
View File

@@ -0,0 +1,3 @@
P4
16 16
ûñ¿úßÕ­±[ñ¥a_ÁX°°ðððð?ÿÿ

BIN
tests/snippets/imghdrdata/python.pgm Normal file
View File

Binary file not shown.

BIN
tests/snippets/imghdrdata/python.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1020 B

BIN
tests/snippets/imghdrdata/python.ppm Normal file
View File

Binary file not shown.

BIN
tests/snippets/imghdrdata/python.ras Normal file
View File

Binary file not shown.

BIN
tests/snippets/imghdrdata/python.sgi Normal file
View File

Binary file not shown.

BIN
tests/snippets/imghdrdata/python.tiff Normal file
View File

Binary file not shown.

BIN
tests/snippets/imghdrdata/python.webp Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 432 B

6
tests/snippets/imghdrdata/python.xbm Normal file
View File

@@ -0,0 +1,6 @@
#define python_width 16
#define python_height 16
static char python_bits[] = {
0xDF, 0xFE, 0x8F, 0xFD, 0x5F, 0xFB, 0xAB, 0xFE, 0xB5, 0x8D, 0xDA, 0x8F,
0xA5, 0x86, 0xFA, 0x83, 0x1A, 0x80, 0x0D, 0x80, 0x0D, 0x80, 0x0F, 0xE0,
0x0F, 0xF8, 0x0F, 0xF8, 0x0F, 0xFC, 0xFF, 0xFF, };

34
tests/snippets/ints.py
View File

@@ -36,7 +36,17 @@ assert (2).__rsub__(1) == -1
assert (2).__mul__(1) == 2
assert (2).__rmul__(1) == 2
assert (2).__truediv__(1) == 2.0
with assertRaises(ZeroDivisionError):
(2).__truediv__(0)
assert (2).__rtruediv__(1) == 0.5
assert (-2).__floordiv__(3) == -1
with assertRaises(ZeroDivisionError):
(2).__floordiv__(0)
assert (-3).__rfloordiv__(2) == -1
assert (-2).__divmod__(3) == (-1, 1)
with assertRaises(ZeroDivisionError):
(2).__divmod__(0)
assert (-3).__rdivmod__(2) == (-1, -1)
assert (2).__pow__(3) == 8
assert (10).__pow__(-1) == 0.1
assert (2).__rpow__(3) == 9
@@ -90,6 +100,10 @@ assert int() == 0
assert int("101", 2) == 5
assert int("101", base=2) == 5
assert int(1) == 1
assert int(' 1') == 1
assert int('1 ') == 1
assert int(' 1 ') == 1
assert int('10', base=0) == 10
assert int.from_bytes(b'\x00\x10', 'big') == 16
assert int.from_bytes(b'\x00\x10', 'little') == 4096
@@ -103,6 +117,13 @@ assert (-1024).to_bytes(4, 'little', signed=True) == b'\x00\xfc\xff\xff'
assert (2147483647).to_bytes(8, 'big', signed=False) == b'\x00\x00\x00\x00\x7f\xff\xff\xff'
assert (-2147483648).to_bytes(8, 'little', signed=True) == b'\x00\x00\x00\x80\xff\xff\xff\xff'
with assertRaises(ValueError):
# check base first
int(' 1 ', base=1)
with assertRaises(ValueError):
int(' 1 ', base=37)
with assertRaises(TypeError):
int(base=2)
@@ -146,3 +167,16 @@ class F(float):
return 3
assert int(F(1.2)) == 3
assert isinstance((0).__round__(), int)
assert isinstance((1).__round__(), int)
assert (0).__round__() == 0
assert (1).__round__() == 1
assert isinstance((0).__round__(0), int)
assert isinstance((1).__round__(0), int)
assert (0).__round__(0) == 0
assert (1).__round__(0) == 1
assert_raises(TypeError, lambda: (0).__round__(None))
assert_raises(TypeError, lambda: (1).__round__(None))
assert_raises(TypeError, lambda: (0).__round__(0.0))
assert_raises(TypeError, lambda: (1).__round__(0.0))

67
tests/snippets/json_snippet.py
View File

@@ -1,5 +1,6 @@
from testutils import assert_raises
import json
from io import StringIO
def round_trip_test(obj):
# serde_json and Python's json module produce slightly differently spaced
@@ -7,63 +8,129 @@ def round_trip_test(obj):
# proxy
return obj == json.loads(json.dumps(obj))
def json_dump(obj):
f = StringIO()
json.dump(obj, f)
f.seek(0)
return f.getvalue()
def json_load(obj):
f = StringIO(obj)
return json.load(f)
assert '"string"' == json.dumps("string")
assert '"string"' == json_dump("string")
assert "1" == json.dumps(1)
assert "1" == json_dump(1)
assert "1.0" == json.dumps(1.0)
assert "1.0" == json_dump(1.0)
assert "true" == json.dumps(True)
assert "true" == json_dump(True)
assert "false" == json.dumps(False)
assert "false" == json_dump(False)
assert 'null' == json.dumps(None)
assert 'null' == json_dump(None)
assert '[]' == json.dumps([])
assert '[]' == json_dump([])
assert '[1]' == json.dumps([1])
assert '[1]' == json_dump([1])
assert '[[1]]' == json.dumps([[1]])
assert '[[1]]' == json_dump([[1]])
assert round_trip_test([1, "string", 1.0, True])
assert '[]' == json.dumps(())
assert '[]' == json_dump(())
assert '[1]' == json.dumps((1,))
assert '[1]' == json_dump((1,))
assert '[[1]]' == json.dumps(((1,),))
assert '[[1]]' == json_dump(((1,),))
# tuples don't round-trip through json
assert [1, "string", 1.0, True] == json.loads(json.dumps((1, "string", 1.0, True)))
assert '{}' == json.dumps({})
assert '{}' == json_dump({})
assert round_trip_test({'a': 'b'})
# should reject non-str keys in jsons
assert_raises(json.JSONDecodeError, lambda: json.loads('{3: "abc"}'))
assert_raises(json.JSONDecodeError, lambda: json_load('{3: "abc"}'))
# should serialize non-str keys as strings
assert json.dumps({'3': 'abc'}) == json.dumps({3: 'abc'})
assert 1 == json.loads("1")
assert 1 == json_load("1")
assert -1 == json.loads("-1")
assert -1 == json_load("-1")
assert 1.0 == json.loads("1.0")
assert 1.0 == json_load("1.0")
assert -1.0 == json.loads("-1.0")
assert -1.0 == json_load("-1.0")
assert "str" == json.loads('"str"')
assert "str" == json_load('"str"')
assert True is json.loads('true')
assert True is json_load('true')
assert False is json.loads('false')
assert False is json_load('false')
assert None is json.loads('null')
assert None is json_load('null')
assert [] == json.loads('[]')
assert [] == json_load('[]')
assert ['a'] == json.loads('["a"]')
assert ['a'] == json_load('["a"]')
assert [['a'], 'b'] == json.loads('[["a"], "b"]')
assert [['a'], 'b'] == json_load('[["a"], "b"]')
class String(str): pass
assert "string" == json.loads(String('"string"'))
assert "string" == json_load(String('"string"'))
assert '"string"' == json.dumps(String("string"))
assert '"string"' == json_dump(String("string"))
class Int(int): pass
class Float(float): pass
assert '1' == json.dumps(Int(1))
assert '1' == json_dump(Int(1))
assert '0.5' == json.dumps(Float(0.5))
assert '0.5' == json_dump(Float(0.5))
class List(list): pass
class Tuple(tuple): pass
class Dict(dict): pass
assert '[1]' == json.dumps(List([1]))
assert '[1]' == json_dump(List([1]))
assert json.dumps((1, "string", 1.0, True)) == json.dumps(Tuple((1, "string", 1.0, True)))
assert json_dump((1, "string", 1.0, True)) == json_dump(Tuple((1, "string", 1.0, True)))
assert json.dumps({'a': 'b'}) == json.dumps(Dict({'a': 'b'}))
assert json_dump({'a': 'b'}) == json_dump(Dict({'a': 'b'}))
# big ints should not crash VM
# TODO: test for correct output when actual serialization implemented and doesn’t throw

15
tests/snippets/math_module.py
View File

@@ -83,6 +83,13 @@ assert str(math.frexp(0.0)) == str((+0.0, 0))
assert str(math.frexp(-0.0)) == str((-0.0, 0))
assert math.frexp(1) == (0.5, 1)
assert math.frexp(1.5) == (0.75, 1)
assert_raises(TypeError, lambda: math.frexp(None))
assert str(math.ldexp(+0.0, 0)) == str(0.0)
assert str(math.ldexp(-0.0, 0)) == str(-0.0)
assert math.ldexp(0.5, 1) == 1
assert math.ldexp(0.75, 1) == 1.5
assert_raises(TypeError, lambda: math.ldexp(None, None))
assert math.frexp(float('inf')) == (float('inf'), 0)
assert str(math.frexp(float('nan'))) == str((float('nan'), 0))
@@ -97,3 +104,11 @@ assert math.gcd(1, -1) == 1
assert math.gcd(-1, -1) == 1
assert math.gcd(125, -255) == 5
assert_raises(TypeError, lambda: math.gcd(1.1, 2))
assert math.factorial(0) == 1
assert math.factorial(1) == 1
assert math.factorial(2) == 2
assert math.factorial(3) == 6
assert math.factorial(10) == 3628800
assert math.factorial(20) == 2432902008176640000
assert_raises(ValueError, lambda: math.factorial(-1))

30
tests/snippets/numbers.py
View File

@@ -1,3 +1,5 @@
from testutils import assertRaises
x = 5
x.__init__(6)
assert x == 5
@@ -36,9 +38,37 @@ assert int(0).__invert__() == -1
assert int(-3).__invert__() == 2
assert int(4).__invert__() == -5
assert int(0).__ror__(0) == 0
assert int(1).__ror__(0) == 1
assert int(0).__ror__(1) == 1
assert int(1).__ror__(1) == 1
assert int(3).__ror__(-3) == -1
assert int(3).__ror__(4) == 7
assert int(0).__rand__(0) == 0
assert int(1).__rand__(0) == 0
assert int(0).__rand__(1) == 0
assert int(1).__rand__(1) == 1
assert int(3).__rand__(-3) == 1
assert int(3).__rand__(4) == 0
assert int(0).__rxor__(0) == 0
assert int(1).__rxor__(0) == 1
assert int(0).__rxor__(1) == 1
assert int(1).__rxor__(1) == 0
assert int(3).__rxor__(-3) == -2
assert int(3).__rxor__(4) == 7
assert int(4).__lshift__(1) == 8
assert int(4).__rshift__(1) == 2
assert int(4).__rlshift__(1) == 16
assert int(4).__rrshift__(1) == 0
# Test underscores in numbers:
assert 1_2 == 12
assert 1_2_3 == 123
assert 1_2.3_4 == 12.34
assert 1_2.3_4e0_0 == 12.34
with assertRaises(SyntaxError):
eval('1__2')

4
tests/snippets/object.py
View File

@@ -7,6 +7,10 @@ myobj = MyObject()
assert myobj == myobj
assert not myobj != myobj
object.__subclasshook__() == NotImplemented
object.__subclasshook__(1) == NotImplemented
object.__subclasshook__(1, 2) == NotImplemented
assert MyObject().__eq__(MyObject()) == NotImplemented
assert MyObject().__ne__(MyObject()) == NotImplemented
assert MyObject().__lt__(MyObject()) == NotImplemented

6
tests/snippets/property.py
View File

@@ -59,7 +59,11 @@ with assertRaises(AttributeError):
with assertRaises(TypeError):
property.__new__(object)
# assert p.__doc__ is None
assert p.__doc__ is None
# Test property instance __doc__ attribute:
p.__doc__ = '222'
assert p.__doc__ == '222'
p1 = property("a", "b", "c")

15
tests/snippets/short_circuit_evaluations.py Normal file
View File

@@ -0,0 +1,15 @@
# Test various cases of short circuit evaluation:
run = 1
timeTaken = 33
r = (11, 22, run, run != 1 and "s" or "", timeTaken)
print(r)
assert r == (11, 22, 1, '', 33)
run = 0
r = (11, 22, run, run != 1 and "s" or "", timeTaken)
print(r)
assert r == (11, 22, 0, 's', 33)

79
tests/snippets/stdlib_functools.py Normal file
View File

@@ -0,0 +1,79 @@
from functools import reduce
from testutils import assertRaises
class Squares:
def __init__(self, max):
self.max = max
self.sofar = []
def __len__(self):
return len(self.sofar)
def __getitem__(self, i):
if not 0 <= i < self.max: raise IndexError
n = len(self.sofar)
while n <= i:
self.sofar.append(n*n)
n += 1
return self.sofar[i]
def add(a, b):
return a + b
assert reduce(add, ['a', 'b', 'c']) == 'abc'
assert reduce(add, ['a', 'b', 'c'], str(42)) == '42abc'
assert reduce(add, [['a', 'c'], [], ['d', 'w']], []) == ['a','c','d','w']
assert reduce(add, [['a', 'c'], [], ['d', 'w']], []) == ['a','c','d','w']
assert reduce(lambda x, y: x*y, range(2, 21), 1) == 2432902008176640000
assert reduce(add, Squares(10)) == 285
assert reduce(add, Squares(10), 0) == 285
assert reduce(add, Squares(0), 0) == 0
assert reduce(42, "1") == "1"
assert reduce(42, "", "1") == "1"
with assertRaises(TypeError):
reduce()
with assertRaises(TypeError):
reduce(42, 42)
with assertRaises(TypeError):
reduce(42, 42, 42)
class TestFailingIter:
def __iter__(self):
raise RuntimeError
with assertRaises(RuntimeError):
reduce(add, TestFailingIter())
assert reduce(add, [], None) == None
assert reduce(add, [], 42) == 42
class BadSeq:
def __getitem__(self, index):
raise ValueError
with assertRaises(ValueError):
reduce(42, BadSeq())
# Test reduce()'s use of iterators.
class SequenceClass:
def __init__(self, n):
self.n = n
def __getitem__(self, i):
if 0 <= i < self.n:
return i
else:
raise IndexError
assert reduce(add, SequenceClass(5)) == 10
assert reduce(add, SequenceClass(5), 42) == 52
with assertRaises(TypeError):
reduce(add, SequenceClass(0))
assert reduce(add, SequenceClass(0), 42) == 42
assert reduce(add, SequenceClass(1)) == 0
assert reduce(add, SequenceClass(1), 42) == 42
d = {"one": 1, "two": 2, "three": 3}
assert reduce(add, d) == "".join(d.keys())

13
tests/snippets/stdlib_io.py
View File

@@ -1,5 +1,6 @@
from io import BufferedReader, FileIO, StringIO, BytesIO
import os
from testutils import assertRaises
fi = FileIO('README.md')
assert fi.seekable()
@@ -23,3 +24,15 @@ fd = os.open('README.md', os.O_RDONLY)
with FileIO(fd) as fio:
res2 = fio.read()
assert res == res2
fi = FileIO('README.md')
fi.read()
fi.close()
with assertRaises(ValueError):
fi.read()
with FileIO('README.md') as fio:
nres = fio.read(1)
assert len(nres) == 1
nres = fio.read(2)
assert len(nres) == 2

2
tests/snippets/stdlib_os.py
View File

@@ -266,8 +266,6 @@ if "win" not in sys.platform:
assert isinstance(os.getppid(), int)
assert isinstance(os.getpgid(os.getpid()), int)
assert os.getppid() < os.getpid()
if os.getuid() != 0:
assert_raises(PermissionError, lambda: os.setgid(42))
assert_raises(PermissionError, lambda: os.setegid(42))

8
tests/snippets/stdlib_socket.py
View File

@@ -119,6 +119,12 @@ with assertRaises(OSError):
with assertRaises(OSError):
socket.inet_aton("test")
with assertRaises(OverflowError):
socket.htonl(-1)
assert socket.htonl(0)==0
assert socket.htonl(10)==167772160
assert socket.inet_aton("127.0.0.1")==b"\x7f\x00\x00\x01"
assert socket.inet_aton("255.255.255.255")==b"\xff\xff\xff\xff"
@@ -129,3 +135,5 @@ assert socket.inet_ntoa(b"\xff\xff\xff\xff")=="255.255.255.255"
with assertRaises(OSError):
socket.inet_ntoa(b"\xff\xff\xff\xff\xff")
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
pass

31
tests/snippets/stdlib_subprocess.py
View File

@@ -1,6 +1,7 @@
import subprocess
import time
import sys
import signal
from testutils import assertRaises
@@ -27,9 +28,33 @@ assert p.returncode == 0
p = subprocess.Popen(["echo", "test"], stdout=subprocess.PIPE)
p.wait()
if "win" not in sys.platform:
is_unix = "win" not in sys.platform or "darwin" in sys.platform
if is_unix:
# unix
assert p.stdout.read() == b"test\n"
test_output = b"test\n"
else:
# windows
assert p.stdout.read() == b"test\r\n"
test_output = b"test\r\n"
assert p.stdout.read() == test_output
p = subprocess.Popen(["sleep", "2"])
p.terminate()
p.wait()
if is_unix:
assert p.returncode == -signal.SIGTERM
else:
assert p.returncode == 1
p = subprocess.Popen(["sleep", "2"])
p.kill()
p.wait()
if is_unix:
assert p.returncode == -signal.SIGKILL
else:
assert p.returncode == 1
p = subprocess.Popen(["echo", "test"], stdout=subprocess.PIPE)
(stdout, stderr) = p.communicate()
assert stdout == test_output

62
tests/snippets/stdlib_zlib.py Normal file
View File

@@ -0,0 +1,62 @@
import zlib
from testutils import assert_raises
# checksum functions
assert zlib.crc32(b"123") == 2286445522
assert zlib.crc32(b"123", 1) == 2307525093
assert zlib.crc32(b"123", 2) == 2345449404
assert zlib.crc32(b"123", 3) == 2316230027
assert zlib.crc32(b"123", 4) == 2403453710
assert zlib.crc32(b"123", 5) == 2390991161
assert zlib.crc32(b"123", 6) == 2361728864
assert zlib.crc32(b"123", -123) == 3515918521
assert zlib.crc32(b"123", -122) == 3554023136
assert zlib.crc32(b"123", -121) == 3524558039
assert zlib.crc32(b"123", -120) == 3645389802
assert zlib.crc32(b"123", -119) == 3632943581
assert zlib.crc32(b"123", -118) == 3670863748
assert zlib.crc32(b"123") == zlib.crc32(b"123", 0)
assert zlib.adler32(b"456") == 20906144
assert zlib.adler32(b"456", 1) == 20906144
assert zlib.adler32(b"456", 2) == 21102753
assert zlib.adler32(b"456", 3) == 21299362
assert zlib.adler32(b"456", 4) == 21495971
assert zlib.adler32(b"456", 5) == 21692580
assert zlib.adler32(b"456", 6) == 21889189
assert zlib.adler32(b"456", -123) == 393267
assert zlib.adler32(b"456", -122) == 589876
assert zlib.adler32(b"456", -121) == 786485
assert zlib.adler32(b"456", -120) == 983094
assert zlib.adler32(b"456", -119) == 1179703
assert zlib.adler32(b"456", -118) == 1376312
assert zlib.adler32(b"456") == zlib.adler32(b"456", 1)
# compression
lorem = bytes("Lorem ipsum dolor sit amet", "utf-8")
compressed_lorem_list = [
b"x\x01\x01\x1a\x00\xe5\xffLorem ipsum dolor sit amet\x83\xd5\t\xc5",
b"x\x01\xf3\xc9/J\xcdU\xc8,(.\xcdUH\xc9\xcf\xc9/R(\xce,QH\xccM-\x01\x00\x83\xd5\t\xc5",
b"x^\xf3\xc9/J\xcdU\xc8,(.\xcdUH\xc9\xcf\xc9/R(\xce,QH\xccM-\x01\x00\x83\xd5\t\xc5",
b"x^\xf3\xc9/J\xcdU\xc8,(.\xcdUH\xc9\xcf\xc9/R(\xce,QH\xccM-\x01\x00\x83\xd5\t\xc5",
b"x^\xf3\xc9/J\xcdU\xc8,(.\xcdUH\xc9\xcf\xc9/R(\xce,QH\xccM-\x01\x00\x83\xd5\t\xc5",
b"x^\xf3\xc9/J\xcdU\xc8,(.\xcdUH\xc9\xcf\xc9/R(\xce,QH\xccM-\x01\x00\x83\xd5\t\xc5",
b"x\x9c\xf3\xc9/J\xcdU\xc8,(.\xcdUH\xc9\xcf\xc9/R(\xce,QH\xccM-\x01\x00\x83\xd5\t\xc5",
b"x\xda\xf3\xc9/J\xcdU\xc8,(.\xcdUH\xc9\xcf\xc9/R(\xce,QH\xccM-\x01\x00\x83\xd5\t\xc5",
b"x\xda\xf3\xc9/J\xcdU\xc8,(.\xcdUH\xc9\xcf\xc9/R(\xce,QH\xccM-\x01\x00\x83\xd5\t\xc5",
b"x\xda\xf3\xc9/J\xcdU\xc8,(.\xcdUH\xc9\xcf\xc9/R(\xce,QH\xccM-\x01\x00\x83\xd5\t\xc5",
]
for level, text in enumerate(compressed_lorem_list):
assert zlib.compress(lorem, level) == text
# default level
assert zlib.compress(lorem) == zlib.compress(lorem, -1) == zlib.compress(lorem, 6)
# decompression
for text in compressed_lorem_list:
assert zlib.decompress(text) == lorem
assert_raises(zlib.error, lambda: zlib.compress(b"123", -40))
assert_raises(zlib.error, lambda: zlib.compress(b"123", 10))

10
tests/snippets/strings.py
View File

@@ -212,6 +212,13 @@ assert "%(first)s %(second)s" % {'second': 'World!', 'first': "Hello,"} == "Hell
assert "%(key())s" % {'key()': 'aaa'}
assert "%s %a %r" % (f, f, f) == "str(Foo) repr(Foo) repr(Foo)"
assert "repr() shows quotes: %r; str() doesn't: %s" % ("test1", "test2") == "repr() shows quotes: 'test1'; str() doesn't: test2"
assert "%f" % (1.2345) == "1.234500"
assert "%+f" % (1.2345) == "+1.234500"
assert "% f" % (1.2345) == " 1.234500"
assert "%f" % (-1.2345) == "-1.234500"
assert "%f" % (1.23456789012) == "1.234568"
assert "%f" % (123) == "123.000000"
assert "%f" % (-123) == "-123.000000"
assert_raises(TypeError, lambda: "My name is %s and I'm %(age)d years old" % ("Foo", 25), msg="format requires a mapping")
assert_raises(TypeError, lambda: "My name is %(name)s" % "Foo", msg="format requires a mapping")
@@ -293,3 +300,6 @@ assert next(str_iter_reversed) == "2"
assert next(str_iter_reversed) == "1"
assert next(str_iter_reversed, None) == None
assert_raises(StopIteration, lambda: next(str_iter_reversed))
assert str.__rmod__('%i', 30) == NotImplemented
assert_raises(TypeError, lambda: str.__rmod__(30, '%i'))

27
tests/snippets/test_imghdr.py Normal file
View File

@@ -0,0 +1,27 @@
# unittest for modified imghdr.py
# Should be replace it into https://github.com/python/cpython/blob/master/Lib/test/test_imghdr.py
import os
import imghdr
TEST_FILES = (
#('python.png', 'png'),
('python.gif', 'gif'),
('python.bmp', 'bmp'),
('python.ppm', 'ppm'),
('python.pgm', 'pgm'),
('python.pbm', 'pbm'),
('python.jpg', 'jpeg'),
('python.ras', 'rast'),
#('python.sgi', 'rgb'),
('python.tiff', 'tiff'),
('python.xbm', 'xbm'),
('python.webp', 'webp'),
('python.exr', 'exr'),
)
resource_dir = os.path.join(os.path.dirname(__file__), 'imghdrdata')
for fname, expected in TEST_FILES:
res = imghdr.what(os.path.join(resource_dir, fname))
assert res == expected

9
vm/Cargo.toml
View File

@@ -29,6 +29,7 @@ num-bigint = { version = "0.2", features = ["serde"] }
num-traits = "=0.2.6"
num-integer = "=0.1.39"
num-rational = "0.2.1"
num-iter = "0.1"
rand = "0.5"
log = "0.3"
rustpython-derive = {path = "../derive", version = "0.1.0"}
@@ -72,5 +73,13 @@ flamer = { version = "0.3", optional = true }
pwd = "1"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
crc32fast = "1.2.0"
adler32 = "1.0.3"
flate2 = { version = "1.0", features = ["zlib"], default-features = false }
libz-sys = "1.0.25"
gethostname = "0.2.0"
subprocess = "0.1.18"
num_cpus = "1.0"
[target."cfg(windows)".dependencies]
kernel32-sys = "0.2.2"

10
vm/Lib/_bootstrap_external.py
View File

@@ -26,6 +26,16 @@ _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY
+ _CASE_INSENSITIVE_PLATFORMS_STR_KEY)
def _w_long(x):
"""Convert a 32-bit integer to little-endian."""
return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
def _r_long(int_bytes):
"""Convert 4 bytes in little-endian to an integer."""
return int.from_bytes(int_bytes, 'little')
def _make_relax_case():
if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY):

63
vm/src/builtins.rs
View File

@@ -21,7 +21,6 @@ use crate::obj::objtype::{self, PyClassRef};
#[cfg(feature = "rustpython-compiler")]
use rustpython_compiler::compile;
use crate::eval::get_compile_mode;
use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs};
use crate::pyobject::{
Either, IdProtocol, IntoPyObject, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue,
@@ -38,7 +37,7 @@ fn builtin_abs(x: PyObjectRef, vm: &VirtualMachine) -> PyResult {
let method = vm.get_method_or_type_error(x.clone(), "__abs__", || {
format!("bad operand type for abs(): '{}'", x.class().name)
})?;
vm.invoke(method, PyFuncArgs::new(vec![], vec![]))
vm.invoke(&method, PyFuncArgs::new(vec![], vec![]))
}
fn builtin_all(iterable: PyIterable<bool>, vm: &VirtualMachine) -> PyResult<bool> {
@@ -125,9 +124,13 @@ fn builtin_compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult<PyCodeRef
Either::B(bytes) => str::from_utf8(&bytes).unwrap().to_string(),
};
let mode = get_compile_mode(vm, &args.mode.value)?;
let mode = args
.mode
.as_str()
.parse::<compile::Mode>()
.map_err(|err| vm.new_value_error(err.to_string()))?;
vm.compile(&source, &mode, args.filename.value.to_string())
vm.compile(&source, mode, args.filename.value.to_string())
.map_err(|err| vm.new_syntax_error(&err))
}
@@ -175,7 +178,7 @@ fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
} else if objtype::isinstance(source, &vm.ctx.str_type()) {
let mode = compile::Mode::Eval;
let source = objstr::get_value(source);
vm.compile(&source, &mode, "<string>".to_string())
vm.compile(&source, mode, "<string>".to_string())
.map_err(|err| vm.new_syntax_error(&err))?
} else {
return Err(vm.new_type_error("code argument must be str or code object".to_string()));
@@ -202,7 +205,7 @@ fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
let code_obj = if objtype::isinstance(source, &vm.ctx.str_type()) {
let mode = compile::Mode::Exec;
let source = objstr::get_value(source);
vm.compile(&source, &mode, "<string>".to_string())
vm.compile(&source, mode, "<string>".to_string())
.map_err(|err| vm.new_syntax_error(&err))?
} else if let Ok(code_obj) = PyCodeRef::try_from_object(vm, source.clone()) {
code_obj
@@ -392,7 +395,7 @@ fn builtin_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
let method = vm.get_method_or_type_error(obj.clone(), "__len__", || {
format!("object of type '{}' has no len()", obj.class().name)
})?;
vm.invoke(method, PyFuncArgs::default())
vm.invoke(&method, PyFuncArgs::default())
}
fn builtin_locals(vm: &VirtualMachine) -> PyDictRef {
@@ -425,15 +428,15 @@ fn builtin_max(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
let mut x = candidates_iter.next().unwrap();
// TODO: this key function looks pretty duplicate. Maybe we can create
// a local function?
let mut x_key = if let Some(f) = &key_func {
vm.invoke(f.clone(), vec![x.clone()])?
let mut x_key = if let Some(ref f) = &key_func {
vm.invoke(f, vec![x.clone()])?
} else {
x.clone()
};
for y in candidates_iter {
let y_key = if let Some(f) = &key_func {
vm.invoke(f.clone(), vec![y.clone()])?
let y_key = if let Some(ref f) = &key_func {
vm.invoke(f, vec![y.clone()])?
} else {
y.clone()
};
@@ -473,15 +476,15 @@ fn builtin_min(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
let mut x = candidates_iter.next().unwrap();
// TODO: this key function looks pretty duplicate. Maybe we can create
// a local function?
let mut x_key = if let Some(f) = &key_func {
vm.invoke(f.clone(), vec![x.clone()])?
let mut x_key = if let Some(ref f) = &key_func {
vm.invoke(f, vec![x.clone()])?
} else {
x.clone()
};
for y in candidates_iter {
let y_key = if let Some(f) = &key_func {
vm.invoke(f.clone(), vec![y.clone()])?
let y_key = if let Some(ref f) = &key_func {
vm.invoke(f, vec![y.clone()])?
} else {
y.clone()
};
@@ -708,7 +711,7 @@ fn builtin_reversed(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
arg_check!(vm, args, required = [(obj, None)]);
if let Some(reversed_method) = vm.get_method(obj.clone(), "__reversed__") {
vm.invoke(reversed_method?, PyFuncArgs::default())
vm.invoke(&reversed_method?, PyFuncArgs::default())
} else {
vm.get_method_or_type_error(obj.clone(), "__getitem__", || {
"argument to reversed() must be a sequence".to_string()
@@ -731,9 +734,19 @@ fn builtin_round(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
optional = [(ndigits, None)]
);
if let Some(ndigits) = ndigits {
let ndigits = vm.call_method(ndigits, "__int__", vec![])?;
let rounded = vm.call_method(number, "__round__", vec![ndigits])?;
Ok(rounded)
if objtype::isinstance(ndigits, &vm.ctx.int_type()) {
let ndigits = vm.call_method(ndigits, "__int__", vec![])?;
let rounded = vm.call_method(number, "__round__", vec![ndigits])?;
Ok(rounded)
} else if vm.ctx.none().is(ndigits) {
let rounded = &vm.call_method(number, "__round__", vec![])?;
Ok(vm.ctx.new_int(objint::get_value(rounded).clone()))
} else {
Err(vm.new_type_error(format!(
"'{}' object cannot be interpreted as an integer",
ndigits.class().name
)))
}
} else {
// without a parameter, the result type is coerced to int
let rounded = &vm.call_method(number, "__round__", vec![])?;
@@ -774,7 +787,7 @@ fn builtin_sum(iterable: PyIterable, start: OptionalArg, vm: &VirtualMachine) ->
// Should be renamed to builtin___import__?
fn builtin_import(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
vm.invoke(vm.import_func.borrow().clone(), args)
vm.invoke(&vm.import_func.borrow(), args)
}
fn builtin_vars(obj: OptionalArg, vm: &VirtualMachine) -> PyResult {
@@ -949,7 +962,11 @@ pub fn builtin_build_class_(
if objtype::issubclass(&base.class(), &metaclass) {
metaclass = base.class();
} else if !objtype::issubclass(&metaclass, &base.class()) {
return Err(vm.new_type_error("metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases".to_string()));
return Err(vm.new_type_error(
"metaclass conflict: the metaclass of a derived class must be a (non-strict) \
subclass of the metaclasses of all its bases"
.to_owned(),
));
}
}
@@ -957,13 +974,13 @@ pub fn builtin_build_class_(
// Prepare uses full __getattribute__ resolution chain.
let prepare = vm.get_attribute(metaclass.clone().into_object(), "__prepare__")?;
let namespace = vm.invoke(prepare, vec![name_obj.clone(), bases.clone()])?;
let namespace = vm.invoke(&prepare, vec![name_obj.clone(), bases.clone()])?;
let namespace: PyDictRef = TryFromObject::try_from_object(vm, namespace)?;
let cells = vm.ctx.new_dict();
vm.invoke_with_locals(function, cells.clone(), namespace.clone())?;
vm.invoke_with_locals(&function, cells.clone(), namespace.clone())?;
namespace.set_item("__name__", name_obj.clone(), vm)?;
namespace.set_item("__qualname__", qualified_name.into_object(), vm)?;

74
vm/src/cformat.rs
View File

@@ -209,6 +209,44 @@ impl CFormatSpec {
self.fill_string(format!("{}{}", prefix, magnitude_string), ' ', None)
}
}
pub fn format_float(&self, num: f64) -> String {
let magnitude = num.abs();
let sign_string = if num.is_sign_positive() {
if self.flags.contains(CConversionFlags::SIGN_CHAR) {
"+"
} else if self.flags.contains(CConversionFlags::BLANK_SIGN) {
" "
} else {
""
}
} else {
"-"
};
// TODO: Support precision
let magnitude_string = format!("{:.6}", magnitude);
if self.flags.contains(CConversionFlags::ZERO_PAD) {
let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) {
'0'
} else {
' '
};
format!(
"{}{}",
sign_string,
self.fill_string(
magnitude_string,
fill_char,
Some(sign_string.chars().count())
)
)
} else {
self.fill_string(format!("{}{}", sign_string, magnitude_string), ' ', None)
}
}
}
#[derive(Debug, PartialEq)]
@@ -763,6 +801,42 @@ mod tests {
);
}
#[test]
fn test_parse_and_format_float() {
assert_eq!(
"%f".parse::<CFormatSpec>()
.unwrap()
.format_float(f64::from(1.2345)),
"1.234500".to_string()
);
assert_eq!(
"%+f"
.parse::<CFormatSpec>()
.unwrap()
.format_float(f64::from(1.2345)),
"+1.234500".to_string()
);
assert_eq!(
"% f"
.parse::<CFormatSpec>()
.unwrap()
.format_float(f64::from(1.2345)),
" 1.234500".to_string()
);
assert_eq!(
"%f".parse::<CFormatSpec>()
.unwrap()
.format_float(f64::from(-1.2345)),
"-1.234500".to_string()
);
assert_eq!(
"%f".parse::<CFormatSpec>()
.unwrap()
.format_float(f64::from(1.2345678901)),
"1.234568".to_string()
);
}
#[test]
fn test_format_parse() {
let fmt = "Hello, my name is %s and I'm %d years old";

13
vm/src/eval.rs
View File

@@ -4,7 +4,7 @@ use crate::vm::VirtualMachine;
use rustpython_compiler::compile;
pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) -> PyResult {
match vm.compile(source, &compile::Mode::Eval, source_path.to_string()) {
match vm.compile(source, compile::Mode::Eval, source_path.to_string()) {
Ok(bytecode) => {
debug!("Code object: {:?}", bytecode);
vm.run_code_obj(bytecode, scope)
@@ -13,17 +13,6 @@ pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str)
}
}
pub fn get_compile_mode(vm: &VirtualMachine, mode: &str) -> PyResult<compile::Mode> {
match mode {
"exec" => Ok(compile::Mode::Exec),
"eval" => Ok(compile::Mode::Eval),
"single" => Ok(compile::Mode::Single),
_ => {
Err(vm.new_value_error("compile() mode must be 'exec', 'eval' or 'single'".to_string()))
}
}
}
#[cfg(test)]
mod tests {
use super::eval;

3
vm/src/exceptions.rs
View File

@@ -3,7 +3,8 @@ use crate::obj::objsequence;
use crate::obj::objtuple::{PyTuple, PyTupleRef};
use crate::obj::objtype;
use crate::obj::objtype::PyClassRef;
use crate::pyobject::{create_type, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol};
use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol};
use crate::types::create_type;
use crate::vm::VirtualMachine;
use itertools::Itertools;
use std::fs::File;

53
vm/src/frame.rs
View File

@@ -198,10 +198,7 @@ impl Frame {
ref symbols,
ref level,
} => self.import(vm, name, symbols, *level),
bytecode::Instruction::ImportStar {
ref name,
ref level,
} => self.import_star(vm, name, *level),
bytecode::Instruction::ImportStar => self.import_star(vm),
bytecode::Instruction::ImportFrom { ref name } => self.import_from(vm, name),
bytecode::Instruction::LoadName {
ref name,
@@ -488,7 +485,7 @@ impl Frame {
// Call function:
let func_ref = self.pop_value();
let value = vm.invoke(func_ref, args)?;
let value = vm.invoke(&func_ref, args)?;
self.push_value(value);
Ok(None)
}
@@ -496,7 +493,7 @@ impl Frame {
self.jump(*target);
Ok(None)
}
bytecode::Instruction::JumpIf { target } => {
bytecode::Instruction::JumpIfTrue { target } => {
let obj = self.pop_value();
let value = objbool::boolval(vm, obj)?;
if value {
@@ -514,6 +511,28 @@ impl Frame {
Ok(None)
}
bytecode::Instruction::JumpIfTrueOrPop { target } => {
let obj = self.last_value();
let value = objbool::boolval(vm, obj)?;
if value {
self.jump(*target);
} else {
self.pop_value();
}
Ok(None)
}
bytecode::Instruction::JumpIfFalseOrPop { target } => {
let obj = self.last_value();
let value = objbool::boolval(vm, obj)?;
if !value {
self.jump(*target);
} else {
self.pop_value();
}
Ok(None)
}
bytecode::Instruction::Raise { argc } => {
let cause = match argc {
2 => self.get_exception(vm, true)?,
@@ -577,7 +596,7 @@ impl Frame {
if !expr.is(&vm.get_none()) {
let repr = vm.to_repr(&expr)?;
// TODO: implement sys.displayhook
if let Ok(print) = vm.get_attribute(vm.builtins.clone(), "print") {
if let Ok(ref print) = vm.get_attribute(vm.builtins.clone(), "print") {
vm.invoke(print, vec![repr.into_object()])?;
}
}
@@ -718,25 +737,23 @@ impl Frame {
// Load attribute, and transform any error into import error.
let obj = vm
.get_attribute(module, name)
.map_err(|_| vm.new_import_error(format!("cannot import name '{}'", name)));
self.push_value(obj?);
.map_err(|_| vm.new_import_error(format!("cannot import name '{}'", name)))?;
self.push_value(obj);
Ok(None)
}
#[cfg_attr(feature = "flame-it", flame("Frame"))]
fn import_star(
&self,
vm: &VirtualMachine,
module: &Option<String>,
level: usize,
) -> FrameResult {
let module = module.clone().unwrap_or_default();
let module = vm.import(&module, &vm.ctx.new_tuple(vec![]), level)?;
fn import_star(&self, vm: &VirtualMachine) -> FrameResult {
let module = self.pop_value();
// Grab all the names from the module and put them in the context
if let Some(dict) = &module.dict {
for (k, v) in dict {
self.scope.store_name(&vm, &objstr::get_value(&k), v);
let k = vm.to_str(&k)?;
let k = k.as_str();
if !k.starts_with('_') {
self.scope.store_name(&vm, k, v);
}
}
}
Ok(None)

12
vm/src/function.rs
View File

@@ -393,6 +393,18 @@ impl<T> OptionalArg<T> {
}
}
pub type OptionalOption<T> = OptionalArg<Option<T>>;
impl<T> OptionalOption<T> {
#[inline]
pub fn flat_option(self) -> Option<T> {
match self {
Present(Some(value)) => Some(value),
_ => None,
}
}
}
impl<T> FromArgs for OptionalArg<T>
where
T: TryFromObject,

8
vm/src/import.rs
View File

@@ -17,14 +17,14 @@ pub fn init_importlib(vm: &VirtualMachine, external: bool) -> PyResult {
let importlib = import_frozen(vm, "_frozen_importlib")?;
let impmod = import_builtin(vm, "_imp")?;
let install = vm.get_attribute(importlib.clone(), "_install")?;
vm.invoke(install, vec![vm.sys_module.clone(), impmod])?;
vm.invoke(&install, vec![vm.sys_module.clone(), impmod])?;
vm.import_func
.replace(vm.get_attribute(importlib.clone(), "__import__")?);
if external && cfg!(feature = "rustpython-compiler") {
flame_guard!("install_external");
let install_external =
vm.get_attribute(importlib.clone(), "_install_external_importers")?;
vm.invoke(install_external, vec![])?;
vm.invoke(&install_external, vec![])?;
// Set pyc magic number to commit hash. Should be changed when bytecode will be more stable.
let importlib_external =
vm.import("_frozen_importlib_external", &vm.ctx.new_tuple(vec![]), 0)?;
@@ -68,7 +68,7 @@ pub fn import_file(
) -> PyResult {
let code_obj = compile::compile(
&content,
&compile::Mode::Exec,
compile::Mode::Exec,
file_path,
vm.settings.optimize,
)
@@ -87,7 +87,7 @@ pub fn import_codeobj(
if set_file_attr {
attrs.set_item("__file__", vm.new_str(code_obj.source_path.to_owned()), vm)?;
}
let module = vm.ctx.new_module(module_name, attrs.clone());
let module = vm.new_module(module_name, attrs.clone());
// Store module in cache to prevent infinite loop with mutual importing libs:
let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules")?;

1
vm/src/lib.rs
View File

@@ -68,6 +68,7 @@ pub mod scope;
pub mod stdlib;
mod sysmodule;
mod traceback;
pub mod types;
pub mod util;
mod version;
mod vm;

2
vm/src/macros.rs
View File

@@ -117,7 +117,7 @@ macro_rules! no_kwargs {
#[macro_export]
macro_rules! py_module {
( $vm:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => {{
let module = $vm.ctx.new_module($module_name, $vm.ctx.new_dict());
let module = $vm.new_module($module_name, $vm.ctx.new_dict());
$vm.set_attr(&module, "__name__", $vm.ctx.new_str($module_name.to_string())).unwrap();
$(
$vm.set_attr(&module, $name, $value).unwrap();

6
vm/src/obj/objbool.rs
View File

@@ -28,7 +28,7 @@ pub fn boolval(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<bool> {
Some(method_or_err) => {
// If descriptor returns Error, propagate it further
let method = method_or_err?;
let bool_obj = vm.invoke(method, PyFuncArgs::default())?;
let bool_obj = vm.invoke(&method, PyFuncArgs::default())?;
match bool_obj.payload::<PyInt>() {
Some(int_obj) => !int_obj.as_bigint().is_zero(),
None => {
@@ -42,7 +42,7 @@ pub fn boolval(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<bool> {
None => match vm.get_method(obj.clone(), "__len__") {
Some(method_or_err) => {
let method = method_or_err?;
let bool_obj = vm.invoke(method, PyFuncArgs::default())?;
let bool_obj = vm.invoke(&method, PyFuncArgs::default())?;
match bool_obj.payload::<PyInt>() {
Some(int_obj) => !int_obj.as_bigint().is_zero(),
None => {
@@ -66,7 +66,7 @@ Returns True when the argument x is true, False otherwise.
The builtins True and False are the only two instances of the class bool.
The class bool is a subclass of the class int, and cannot be subclassed.";
let bool_type = &context.bool_type;
let bool_type = &context.types.bool_type;
extend_class!(context, bool_type, {
"__new__" => context.new_rustfunc(bool_new),
"__repr__" => context.new_rustfunc(bool_repr),

6
vm/src/obj/objbytearray.rs
View File

@@ -78,14 +78,14 @@ impl PyValue for PyByteArray {
/// Fill bytearray class methods dictionary.
pub fn init(context: &PyContext) {
PyByteArrayRef::extend_class(context, &context.bytearray_type);
let bytearray_type = &context.bytearray_type;
PyByteArrayRef::extend_class(context, &context.types.bytearray_type);
let bytearray_type = &context.types.bytearray_type;
extend_class!(context, bytearray_type, {
"fromhex" => context.new_rustfunc(PyByteArrayRef::fromhex),
"maketrans" => context.new_rustfunc(PyByteInner::maketrans),
});
PyByteArrayIterator::extend_class(context, &context.bytearrayiterator_type);
PyByteArrayIterator::extend_class(context, &context.types.bytearrayiterator_type);
}
#[pyimpl]

15
vm/src/obj/objbytes.rs
View File

@@ -11,7 +11,8 @@ use std::ops::Deref;
use crate::function::OptionalArg;
use crate::pyobject::{
PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue,
TryFromObject,
};
use super::objbyteinner::{
@@ -59,6 +60,12 @@ impl PyBytes {
}
}
impl IntoPyObject for Vec<u8> {
fn into_pyobject(self, vm: &VirtualMachine) -> PyResult {
Ok(vm.ctx.new_bytes(self))
}
}
impl Deref for PyBytes {
type Target = [u8];
@@ -78,14 +85,14 @@ pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref<Target = Vec<u8>> + 'a
}
pub fn init(context: &PyContext) {
PyBytesRef::extend_class(context, &context.bytes_type);
let bytes_type = &context.bytes_type;
PyBytesRef::extend_class(context, &context.types.bytes_type);
let bytes_type = &context.types.bytes_type;
extend_class!(context, bytes_type, {
"fromhex" => context.new_rustfunc(PyBytesRef::fromhex),
"maketrans" => context.new_rustfunc(PyByteInner::maketrans),
});
PyBytesIterator::extend_class(context, &context.bytesiterator_type);
PyBytesIterator::extend_class(context, &context.types.bytesiterator_type);
}
#[pyimpl]

2
vm/src/obj/objclassmethod.rs
View File

@@ -65,5 +65,5 @@ impl PyClassMethod {
}
pub fn init(context: &PyContext) {
PyClassMethod::extend_class(context, &context.classmethod_type);
PyClassMethod::extend_class(context, &context.types.classmethod_type);
}

2
vm/src/obj/objcode.rs
View File

@@ -81,7 +81,7 @@ impl PyCodeRef {
}
pub fn init(context: &PyContext) {
extend_class!(context, &context.code_type, {
extend_class!(context, &context.types.code_type, {
"__new__" => context.new_rustfunc(PyCodeRef::new),
"__repr__" => context.new_rustfunc(PyCodeRef::repr),

10
vm/src/obj/objcomplex.rs
View File

@@ -41,7 +41,7 @@ impl From<Complex64> for PyComplex {
}
pub fn init(context: &PyContext) {
PyComplex::extend_class(context, &context.complex_type);
PyComplex::extend_class(context, &context.types.complex_type);
}
pub fn get_value(obj: &PyObjectRef) -> Complex64 {
@@ -258,4 +258,12 @@ impl PyComplex {
let ret = Wrapping(re_hash) + Wrapping(im_hash) * Wrapping(pyhash::IMAG);
ret.0
}
#[pymethod(name = "__getnewargs__")]
fn complex_getnewargs(&self, vm: &VirtualMachine) -> PyResult {
let Complex64 { re, im } = self.value;
Ok(vm
.ctx
.new_tuple(vec![vm.ctx.new_float(re), vm.ctx.new_float(im)]))
}
}

20
vm/src/obj/objdict.rs
View File

@@ -210,7 +210,7 @@ impl PyDictRef {
}
if let Some(method_or_err) = vm.get_method(self.clone().into_object(), "__missing__") {
let method = method_or_err?;
return vm.invoke(method, vec![key]);
return vm.invoke(&method, vec![key]);
}
Err(vm.new_key_error(key.clone()))
}
@@ -423,7 +423,7 @@ macro_rules! dict_iterator {
impl PyValue for $name {
fn class(vm: &VirtualMachine) -> PyClassRef {
vm.ctx.$class.clone()
vm.ctx.types.$class.clone()
}
}
@@ -473,7 +473,7 @@ macro_rules! dict_iterator {
impl PyValue for $iter_name {
fn class(vm: &VirtualMachine) -> PyClassRef {
vm.ctx.$iter_class.clone()
vm.ctx.types.$iter_class.clone()
}
}
};
@@ -511,7 +511,7 @@ dict_iterator! {
}
pub fn init(context: &PyContext) {
extend_class!(context, &context.dict_type, {
extend_class!(context, &context.types.dict_type, {
"__bool__" => context.new_rustfunc(PyDictRef::bool),
"__len__" => context.new_rustfunc(PyDictRef::len),
"__contains__" => context.new_rustfunc(PyDictRef::contains),
@@ -536,10 +536,10 @@ pub fn init(context: &PyContext) {
"popitem" => context.new_rustfunc(PyDictRef::popitem),
});
PyDictKeys::extend_class(context, &context.dictkeys_type);
PyDictKeyIterator::extend_class(context, &context.dictkeyiterator_type);
PyDictValues::extend_class(context, &context.dictvalues_type);
PyDictValueIterator::extend_class(context, &context.dictvalueiterator_type);
PyDictItems::extend_class(context, &context.dictitems_type);
PyDictItemIterator::extend_class(context, &context.dictitemiterator_type);
PyDictKeys::extend_class(context, &context.types.dictkeys_type);
PyDictKeyIterator::extend_class(context, &context.types.dictkeyiterator_type);
PyDictValues::extend_class(context, &context.types.dictvalues_type);
PyDictValueIterator::extend_class(context, &context.types.dictvalueiterator_type);
PyDictItems::extend_class(context, &context.types.dictitems_type);
PyDictItemIterator::extend_class(context, &context.types.dictitemiterator_type);
}

28
vm/src/obj/objellipsis.rs
View File

@@ -1,20 +1,30 @@
use crate::function::PyFuncArgs;
use crate::pyobject::{PyContext, PyResult};
use crate::obj::objtype::{issubclass, PyClassRef};
use crate::pyobject::{PyContext, PyEllipsisRef, PyResult};
use crate::vm::VirtualMachine;
pub fn init(context: &PyContext) {
extend_class!(context, &context.ellipsis_type, {
"__new__" => context.new_rustfunc(ellipsis_new),
"__repr__" => context.new_rustfunc(ellipsis_repr)
"__repr__" => context.new_rustfunc(ellipsis_repr),
"__reduce__" => context.new_rustfunc(ellipsis_reduce),
});
}
fn ellipsis_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
arg_check!(vm, args, required = [(_cls, None)]);
Ok(vm.ctx.ellipsis())
fn ellipsis_new(cls: PyClassRef, vm: &VirtualMachine) -> PyResult {
if issubclass(&cls, &vm.ctx.ellipsis_type) {
Ok(vm.ctx.ellipsis())
} else {
Err(vm.new_type_error(format!(
"ellipsis.__new__({ty}): {ty} is not a subtype of ellipsis",
ty = cls,
)))
}
}
fn ellipsis_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
arg_check!(vm, args, required = [(_cls, None)]);
Ok(vm.new_str("Ellipsis".to_string()))
fn ellipsis_repr(_self: PyEllipsisRef, _vm: &VirtualMachine) -> String {
"Ellipsis".to_string()
}
fn ellipsis_reduce(_self: PyEllipsisRef, _vm: &VirtualMachine) -> String {
"Ellipsis".to_string()
}

4
vm/src/obj/objenumerate.rs
View File

@@ -68,8 +68,8 @@ impl PyEnumerate {
}
pub fn init(context: &PyContext) {
PyEnumerate::extend_class(context, &context.enumerate_type);
extend_class!(context, &context.enumerate_type, {
PyEnumerate::extend_class(context, &context.types.enumerate_type);
extend_class!(context, &context.types.enumerate_type, {
"__new__" => context.new_rustfunc(enumerate_new),
});
}

6
vm/src/obj/objfilter.rs
View File

@@ -52,7 +52,7 @@ impl PyFilter {
} else {
// the predicate itself can raise StopIteration which does stop the filter
// iteration
vm.invoke(predicate.clone(), vec![next_obj.clone()])?
vm.invoke(&predicate, vec![next_obj.clone()])?
};
if objbool::boolval(vm, predicate_value)? {
return Ok(next_obj);
@@ -67,8 +67,8 @@ impl PyFilter {
}
pub fn init(context: &PyContext) {
PyFilter::extend_class(context, &context.filter_type);
extend_class!(context, &context.filter_type, {
PyFilter::extend_class(context, &context.types.filter_type);
extend_class!(context, &context.types.filter_type, {
"__new__" => context.new_rustfunc(filter_new),
});
}

39
vm/src/obj/objfloat.rs
View File

@@ -440,25 +440,38 @@ impl PyFloat {
OptionalArg::Missing => None,
OptionalArg::Present(ref value) => {
if !vm.get_none().is(value) {
let ndigits = if objtype::isinstance(value, &vm.ctx.int_type()) {
objint::get_value(value)
} else {
if !objtype::isinstance(value, &vm.ctx.int_type()) {
return Err(vm.new_type_error(format!(
"TypeError: '{}' object cannot be interpreted as an integer",
"'{}' object cannot be interpreted as an integer",
value.class().name
)));
};
if ndigits.is_zero() {
None
} else {
Some(ndigits)
}
// Only accept int type ndigits
let ndigits = objint::get_value(value);
Some(ndigits)
} else {
None
}
}
};
if ndigits.is_none() {
if let Some(ndigits) = ndigits {
if ndigits.is_zero() {
let fract = self.value.fract();
let value = if (fract.abs() - 0.5).abs() < std::f64::EPSILON {
if self.value.trunc() % 2.0 == 0.0 {
self.value - fract
} else {
self.value + fract
}
} else {
self.value.round()
};
Ok(vm.ctx.new_float(value))
} else {
Ok(vm.ctx.not_implemented())
}
} else {
let fract = self.value.fract();
let value = if (fract.abs() - 0.5).abs() < std::f64::EPSILON {
if self.value.trunc() % 2.0 == 0.0 {
@@ -471,8 +484,6 @@ impl PyFloat {
};
let int = try_to_bigint(value, vm)?;
Ok(vm.ctx.new_int(int))
} else {
Ok(vm.ctx.not_implemented())
}
}
@@ -612,12 +623,12 @@ pub fn make_float(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult<f64> {
obj.class().name
)
})?;
let result = vm.invoke(method, vec![])?;
let result = vm.invoke(&method, vec![])?;
Ok(get_value(&result))
}
}
#[rustfmt::skip] // to avoid line splitting
pub fn init(context: &PyContext) {
PyFloat::extend_class(context, &context.float_type);
PyFloat::extend_class(context, &context.types.float_type);
}

2
vm/src/obj/objframe.rs
View File

@@ -9,7 +9,7 @@ use crate::pyobject::{PyContext, PyObjectRef, PyResult};
use crate::vm::VirtualMachine;
pub fn init(context: &PyContext) {
extend_class!(context, &context.frame_type, {
extend_class!(context, &context.types.frame_type, {
"__new__" => context.new_rustfunc(FrameRef::new),
"__repr__" => context.new_rustfunc(FrameRef::repr),
"f_locals" => context.new_property(FrameRef::flocals),

6
vm/src/obj/objfunction.rs
View File

@@ -42,7 +42,7 @@ impl PyValue for PyFunction {
impl PyFunctionRef {
fn call(self, args: Args, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult {
vm.invoke(self.into_object(), (&args, &kwargs))
vm.invoke(&self.into_object(), (&args, &kwargs))
}
fn code(self, _vm: &VirtualMachine) -> PyCodeRef {
@@ -78,7 +78,7 @@ impl PyValue for PyMethod {
}
pub fn init(context: &PyContext) {
let function_type = &context.function_type;
let function_type = &context.types.function_type;
extend_class!(context, function_type, {
"__get__" => context.new_rustfunc(bind_method),
"__call__" => context.new_rustfunc(PyFunctionRef::call),
@@ -87,7 +87,7 @@ pub fn init(context: &PyContext) {
"__kwdefaults__" => context.new_property(PyFunctionRef::kwdefaults),
});
let builtin_function_or_method_type = &context.builtin_function_or_method_type;
let builtin_function_or_method_type = &context.types.builtin_function_or_method_type;
extend_class!(context, builtin_function_or_method_type, {
"__get__" => context.new_rustfunc(bind_method)
});

2
vm/src/obj/objgenerator.rs
View File

@@ -75,5 +75,5 @@ fn handle_execution_result(result: ExecutionResult, vm: &VirtualMachine) -> PyRe
}
pub fn init(ctx: &PyContext) {
PyGenerator::extend_class(ctx, &ctx.generator_type);
PyGenerator::extend_class(ctx, &ctx.types.generator_type);
}

205
vm/src/obj/objint.rs
View File

@@ -6,18 +6,19 @@ use num_traits::{One, Pow, Signed, ToPrimitive, Zero};
use crate::format::FormatSpec;
use crate::function::{KwArgs, OptionalArg, PyFuncArgs};
use crate::obj::objtype::PyClassRef;
use crate::pyhash;
use crate::pyobject::{
IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
TypeProtocol,
IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue,
TryFromObject, TypeProtocol,
};
use crate::vm::VirtualMachine;
use super::objbyteinner::PyByteInner;
use super::objbytes::PyBytes;
use super::objint;
use super::objstr::{PyString, PyStringRef};
use super::objtype;
use crate::obj::objtype::PyClassRef;
/// int(x=0) -> integer
/// int(x, base=10) -> integer
@@ -139,13 +140,42 @@ fn inner_pow(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult {
}
fn inner_mod(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult {
if int2.value != BigInt::zero() {
Ok(vm.ctx.new_int(&int1.value % &int2.value))
} else {
if int2.value.is_zero() {
Err(vm.new_zero_division_error("integer modulo by zero".to_string()))
} else {
Ok(vm.ctx.new_int(&int1.value % &int2.value))
}
}
fn inner_floordiv(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult {
if int2.value.is_zero() {
Err(vm.new_zero_division_error("integer division by zero".to_string()))
} else {
Ok(vm.ctx.new_int(int1.value.div_floor(&int2.value)))
}
}
fn inner_divmod(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult {
if int2.value.is_zero() {
Err(vm.new_zero_division_error("integer division or modulo by zero".to_string()))
} else {
let (div, modulo) = int1.value.div_mod_floor(&int2.value);
Ok(vm
.ctx
.new_tuple(vec![vm.ctx.new_int(div), vm.ctx.new_int(modulo)]))
}
}
fn inner_lshift(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult {
let n_bits = get_shift_amount(int2, vm)?;
Ok(vm.ctx.new_int(&int1.value << n_bits))
}
fn inner_rshift(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult {
let n_bits = get_shift_amount(int2, vm)?;
Ok(vm.ctx.new_int(&int1.value >> n_bits))
}
#[pyimpl]
impl PyInt {
#[pymethod(name = "__eq__")]
@@ -269,13 +299,18 @@ impl PyInt {
#[pymethod(name = "__floordiv__")]
fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if objtype::isinstance(&other, &vm.ctx.int_type()) {
let v2 = get_value(&other);
if *v2 != BigInt::zero() {
let modulo = (&self.value % v2 + v2) % v2;
Ok(vm.ctx.new_int((&self.value - modulo) / v2))
} else {
Err(vm.new_zero_division_error("integer floordiv by zero".to_string()))
}
let other = get_py_int(&other);
inner_floordiv(self, &other, &vm)
} else {
Ok(vm.ctx.not_implemented())
}
}
#[pymethod(name = "__rfloordiv__")]
fn rfloordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if objtype::isinstance(&other, &vm.ctx.int_type()) {
let other = get_py_int(&other);
inner_floordiv(&other, self, &vm)
} else {
Ok(vm.ctx.not_implemented())
}
@@ -287,18 +322,18 @@ impl PyInt {
return Ok(vm.ctx.not_implemented());
}
if let Some(n_bits) = get_value(&other).to_usize() {
return Ok(vm.ctx.new_int((&self.value) << n_bits));
let other = get_py_int(&other);
inner_lshift(self, other, vm)
}
#[pymethod(name = "__rlshift__")]
fn rlshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if !objtype::isinstance(&other, &vm.ctx.int_type()) {
return Ok(vm.ctx.not_implemented());
}
// i2 failed `to_usize()` conversion
match get_value(&other) {
v if *v < BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())),
v if *v > BigInt::from(usize::max_value()) => {
Err(vm.new_overflow_error("the number is too large to convert to int".to_string()))
}
_ => panic!("Failed converting {} to rust usize", get_value(&other)),
}
let other = get_py_int(&other);
inner_lshift(other, self, vm)
}
#[pymethod(name = "__rshift__")]
@@ -307,18 +342,18 @@ impl PyInt {
return Ok(vm.ctx.not_implemented());
}
if let Some(n_bits) = get_value(&other).to_usize() {
return Ok(vm.ctx.new_int((&self.value) >> n_bits));
let other = get_py_int(&other);
inner_rshift(self, other, vm)
}
#[pymethod(name = "__rrshift__")]
fn rrshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if !objtype::isinstance(&other, &vm.ctx.int_type()) {
return Ok(vm.ctx.not_implemented());
}
// i2 failed `to_usize()` conversion
match get_value(&other) {
v if *v < BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())),
v if *v > BigInt::from(usize::max_value()) => {
Err(vm.new_overflow_error("the number is too large to convert to int".to_string()))
}
_ => panic!("Failed converting {} to rust usize", get_value(&other)),
}
let other = get_py_int(&other);
inner_rshift(other, self, vm)
}
#[pymethod(name = "__xor__")]
@@ -344,6 +379,11 @@ impl PyInt {
}
}
#[pymethod(name = "__ror__")]
fn ror(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef {
self.or(other, vm)
}
#[pymethod(name = "__and__")]
pub fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef {
if objtype::isinstance(&other, &vm.ctx.int_type()) {
@@ -354,10 +394,15 @@ impl PyInt {
}
}
#[pymethod(name = "__rand__")]
fn rand(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef {
self.and(other, vm)
}
#[pymethod(name = "__pow__")]
fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if objtype::isinstance(&other, &vm.ctx.int_type()) {
let other = other.payload::<PyInt>().unwrap();
let other = get_py_int(&other);
inner_pow(self, &other, vm)
} else {
Ok(vm.ctx.not_implemented())
@@ -367,7 +412,7 @@ impl PyInt {
#[pymethod(name = "__rpow__")]
fn rpow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if objtype::isinstance(&other, &vm.ctx.int_type()) {
let other = other.payload::<PyInt>().unwrap();
let other = get_py_int(&other);
inner_pow(&other, self, vm)
} else {
Ok(vm.ctx.not_implemented())
@@ -377,7 +422,7 @@ impl PyInt {
#[pymethod(name = "__mod__")]
fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if objtype::isinstance(&other, &vm.ctx.int_type()) {
let other = other.payload::<PyInt>().unwrap();
let other = get_py_int(&other);
inner_mod(self, &other, vm)
} else {
Ok(vm.ctx.not_implemented())
@@ -387,7 +432,7 @@ impl PyInt {
#[pymethod(name = "__rmod__")]
fn rmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if objtype::isinstance(&other, &vm.ctx.int_type()) {
let other = other.payload::<PyInt>().unwrap();
let other = get_py_int(&other);
inner_mod(&other, self, vm)
} else {
Ok(vm.ctx.not_implemented())
@@ -397,15 +442,18 @@ impl PyInt {
#[pymethod(name = "__divmod__")]
fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if objtype::isinstance(&other, &vm.ctx.int_type()) {
let v2 = get_value(&other);
if *v2 != BigInt::zero() {
let (r1, r2) = self.value.div_rem(v2);
Ok(vm
.ctx
.new_tuple(vec![vm.ctx.new_int(r1), vm.ctx.new_int(r2)]))
} else {
Err(vm.new_zero_division_error("integer divmod by zero".to_string()))
}
let other = get_py_int(&other);
inner_divmod(self, &other, vm)
} else {
Ok(vm.ctx.not_implemented())
}
}
#[pymethod(name = "__rdivmod__")]
fn rdivmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if objtype::isinstance(&other, &vm.ctx.int_type()) {
let other = get_py_int(&other);
inner_divmod(&other, self, vm)
} else {
Ok(vm.ctx.not_implemented())
}
@@ -431,8 +479,29 @@ impl PyInt {
zelf: PyRef<Self>,
_precision: OptionalArg<PyObjectRef>,
_vm: &VirtualMachine,
) -> PyIntRef {
zelf
) -> PyResult<PyIntRef> {
let _ndigits = match _precision {
OptionalArg::Missing => None,
OptionalArg::Present(ref value) => {
if !_vm.get_none().is(value) {
if !objtype::isinstance(value, &_vm.ctx.int_type()) {
return Err(_vm.new_type_error(format!(
"'{}' object cannot be interpreted as an integer",
value.class().name
)));
};
// Only accept int type _ndigits
let _ndigits = objint::get_value(value);
Some(_ndigits)
} else {
return Err(_vm.new_type_error(format!(
"'{}' object cannot be interpreted as an integer",
value.class().name
)));
}
}
};
Ok(zelf)
}
#[pymethod(name = "__int__")]
@@ -644,7 +713,7 @@ impl IntOptions {
fn get_int_value(self, vm: &VirtualMachine) -> PyResult<BigInt> {
if let OptionalArg::Present(val) = self.val_options {
let base = if let OptionalArg::Present(base) = self.base {
if !objtype::isinstance(&val, &vm.ctx.str_type) {
if !objtype::isinstance(&val, &vm.ctx.str_type()) {
return Err(vm.new_type_error(
"int() can't convert non-string with explicit base".to_string(),
));
@@ -667,10 +736,16 @@ fn int_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResul
}
// Casting function:
pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult<BigInt> {
pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, mut base: u32) -> PyResult<BigInt> {
if base == 0 {
base = 10
} else if base < 2 || base > 36 {
return Err(vm.new_value_error("int() base must be >= 2 and <= 36, or 0".to_string()));
}
match_class!(obj.clone(),
s @ PyString => {
i32::from_str_radix(s.as_str(), base)
i32::from_str_radix(s.as_str().trim(), base)
.map(BigInt::from)
.map_err(|_|vm.new_value_error(format!(
"invalid literal for int() with base {}: '{}'",
@@ -681,7 +756,7 @@ pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult<Big
let method = vm.get_method_or_type_error(obj.clone(), "__int__", || {
format!("int() argument must be a string or a number, not '{}'", obj.class().name)
})?;
let result = vm.invoke(method, PyFuncArgs::default())?;
let result = vm.invoke(&method, PyFuncArgs::default())?;
match result.payload::<PyInt>() {
Some(int_obj) => Ok(int_obj.as_bigint().clone()),
None => Err(vm.new_type_error(format!(
@@ -693,11 +768,11 @@ pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult<Big
// Retrieve inner int value:
pub fn get_value(obj: &PyObjectRef) -> &BigInt {
&obj.payload::<PyInt>().unwrap().value
&get_py_int(obj).value
}
pub fn get_float_value(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> {
obj.payload::<PyInt>().unwrap().float(vm)
get_py_int(obj).float(vm)
}
#[inline]
@@ -732,9 +807,27 @@ fn div_ints(vm: &VirtualMachine, i1: &BigInt, i2: &BigInt) -> PyResult {
}
}
fn get_shift_amount(amount: &PyInt, vm: &VirtualMachine) -> PyResult<usize> {
if let Some(n_bits) = amount.value.to_usize() {
Ok(n_bits)
} else {
match &amount.value {
v if *v < BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())),
v if *v > BigInt::from(usize::max_value()) => {
Err(vm.new_overflow_error("the number is too large to convert to int".to_string()))
}
_ => panic!("Failed converting {} to rust usize", amount.value),
}
}
}
fn get_py_int(obj: &PyObjectRef) -> &PyInt {
&obj.payload::<PyInt>().unwrap()
}
pub fn init(context: &PyContext) {
PyInt::extend_class(context, &context.int_type);
extend_class!(context, &context.int_type, {
PyInt::extend_class(context, &context.types.int_type);
extend_class!(context, &context.types.int_type, {
"__new__" => context.new_rustfunc(int_new),
});
}

4
vm/src/obj/objiter.rs
View File

@@ -20,7 +20,7 @@ use super::objtype::PyClassRef;
pub fn get_iter(vm: &VirtualMachine, iter_target: &PyObjectRef) -> PyResult {
if let Some(method_or_err) = vm.get_method(iter_target.clone(), "__iter__") {
let method = method_or_err?;
vm.invoke(method, vec![])
vm.invoke(&method, vec![])
} else {
vm.get_method_or_type_error(iter_target.clone(), "__getitem__", || {
format!("Cannot iterate over {}", iter_target.class().name)
@@ -122,5 +122,5 @@ impl PySequenceIterator {
}
pub fn init(context: &PyContext) {
PySequenceIterator::extend_class(context, &context.iter_type);
PySequenceIterator::extend_class(context, &context.types.iter_type);
}

8
vm/src/obj/objlist.rs
View File

@@ -755,7 +755,7 @@ fn do_sort(
for x in values.iter() {
keys.push(match &key_func {
None => x.clone(),
Some(ref func) => vm.invoke((*func).clone(), vec![x.clone()])?,
Some(ref func) => vm.invoke(func, vec![x.clone()])?,
});
}
@@ -859,7 +859,7 @@ impl PyListReverseIterator {
#[rustfmt::skip] // to avoid line splitting
pub fn init(context: &PyContext) {
let list_type = &context.list_type;
let list_type = &context.types.list_type;
let list_doc = "Built-in mutable sequence.\n\n\
If no argument is given, the constructor creates a new empty list.\n\
@@ -901,6 +901,6 @@ pub fn init(context: &PyContext) {
"remove" => context.new_rustfunc(PyListRef::remove)
});
PyListIterator::extend_class(context, &context.listiterator_type);
PyListReverseIterator::extend_class(context, &context.listreverseiterator_type);
PyListIterator::extend_class(context, &context.types.listiterator_type);
PyListReverseIterator::extend_class(context, &context.types.listreverseiterator_type);
}

6
vm/src/obj/objmap.rs
View File

@@ -51,7 +51,7 @@ impl PyMap {
.collect::<Result<Vec<_>, _>>()?;
// the mapper itself can raise StopIteration which does stop the map iteration
vm.invoke(self.mapper.clone(), next_objs)
vm.invoke(&self.mapper, next_objs)
}
#[pymethod(name = "__iter__")]
@@ -61,8 +61,8 @@ impl PyMap {
}
pub fn init(context: &PyContext) {
PyMap::extend_class(context, &context.map_type);
extend_class!(context, &context.map_type, {
PyMap::extend_class(context, &context.types.map_type);
extend_class!(context, &context.types.map_type, {
"__new__" => context.new_rustfunc(map_new),
});
}

4
vm/src/obj/objmappingproxy.rs
View File

@@ -13,7 +13,7 @@ pub type PyMappingProxyRef = PyRef<PyMappingProxy>;
impl PyValue for PyMappingProxy {
fn class(vm: &VirtualMachine) -> PyClassRef {
vm.ctx.mappingproxy_type.clone()
vm.ctx.types.mappingproxy_type.clone()
}
}
@@ -38,5 +38,5 @@ impl PyMappingProxy {
}
pub fn init(context: &PyContext) {
PyMappingProxy::extend_class(context, &context.mappingproxy_type)
PyMappingProxy::extend_class(context, &context.types.mappingproxy_type)
}

2
vm/src/obj/objmemory.rs
View File

@@ -47,5 +47,5 @@ impl PyValue for PyMemoryView {
}
pub fn init(ctx: &PyContext) {
PyMemoryView::extend_class(ctx, &ctx.memoryview_type)
PyMemoryView::extend_class(ctx, &ctx.types.memoryview_type)
}

70
vm/src/obj/objmodule.rs
View File

@@ -1,6 +1,8 @@
use crate::function::OptionalOption;
use crate::obj::objdict::PyDictRef;
use crate::obj::objstr::PyStringRef;
use crate::obj::objtype::PyClassRef;
use crate::pyobject::{PyContext, PyRef, PyResult, PyValue};
use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue};
use crate::vm::VirtualMachine;
#[derive(Debug)]
@@ -17,15 +19,71 @@ impl PyValue for PyModule {
}
}
pub fn init_module_dict(
vm: &VirtualMachine,
module_dict: &PyDictRef,
name: PyObjectRef,
doc: PyObjectRef,
) {
module_dict
.set_item("__name__", name, vm)
.expect("Failed to set __name__ on module");
module_dict
.set_item("__doc__", doc, vm)
.expect("Failed to set __doc__ on module");
module_dict
.set_item("__package__", vm.get_none(), vm)
.expect("Failed to set __package__ on module");
module_dict
.set_item("__loader__", vm.get_none(), vm)
.expect("Failed to set __loader__ on module");
module_dict
.set_item("__spec__", vm.get_none(), vm)
.expect("Failed to set __spec__ on module");
}
impl PyModuleRef {
fn init(self, name: PyStringRef, vm: &VirtualMachine) -> PyResult {
vm.set_attr(&self.into_object(), "__name__", name)?;
Ok(vm.get_none())
fn new(
cls: PyClassRef,
name: PyStringRef,
doc: OptionalOption<PyStringRef>,
vm: &VirtualMachine,
) -> PyResult<PyModuleRef> {
let zelf = PyModule {
name: name.as_str().to_owned(),
}
.into_ref_with_type(vm, cls)?;
init_module_dict(
vm,
zelf.as_object().dict.as_ref().unwrap(),
name.into_object(),
doc.flat_option()
.map_or_else(|| vm.get_none(), PyRef::into_object),
);
Ok(zelf)
}
fn getattribute(self, name: PyStringRef, vm: &VirtualMachine) -> PyResult {
vm.generic_getattribute(self.as_object().clone(), name.clone())?
.ok_or_else(|| {
vm.new_attribute_error(format!(
"module '{}' has no attribute '{}'",
self.name, name,
))
})
}
fn repr(self, vm: &VirtualMachine) -> PyResult {
let importlib = vm.import("_frozen_importlib", &vm.ctx.new_tuple(vec![]), 0)?;
let module_repr = vm.get_attribute(importlib, "_module_repr")?;
vm.invoke(&module_repr, vec![self.into_object()])
}
}
pub fn init(context: &PyContext) {
extend_class!(&context, &context.module_type, {
"__init__" => context.new_rustfunc(PyModuleRef::init),
extend_class!(&context, &context.types.module_type, {
"__new__" => context.new_rustfunc(PyModuleRef::new),
"__getattribute__" => context.new_rustfunc(PyModuleRef::getattribute),
"__repr__" => context.new_rustfunc(PyModuleRef::repr),
});
}

2
vm/src/obj/objnamespace.rs
View File

@@ -28,5 +28,5 @@ impl PyNamespace {
}
pub fn init(context: &PyContext) {
PyNamespace::extend_class(context, &context.namespace_type);
PyNamespace::extend_class(context, &context.types.namespace_type);
}

4
vm/src/obj/objnone.rs
View File

@@ -64,7 +64,7 @@ impl PyNoneRef {
if let Ok(property) = PyPropertyRef::try_from_object(vm, descriptor.clone()) {
property.instance_binding_get(obj, vm)
} else {
vm.invoke(get_func, vec![descriptor, obj, cls])
vm.invoke(&get_func, vec![descriptor, obj, cls])
}
}
@@ -95,7 +95,7 @@ impl PyNoneRef {
Ok(attr)
}
} else if let Some(getter) = class_get_attr(&cls, "__getattr__") {
vm.invoke(getter, vec![self.into_object(), name.into_object()])
vm.invoke(&getter, vec![self.into_object(), name.into_object()])
} else {
Err(vm.new_attribute_error(format!("{} has no attribute '{}'", self.as_object(), name)))
}

54
vm/src/obj/objobject.rs
View File

@@ -24,7 +24,7 @@ impl PyValue for PyInstance {
pub fn new_instance(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult {
// more or less __new__ operator
let cls = PyClassRef::try_from_object(vm, args.shift())?;
let dict = if cls.is(&vm.ctx.object) {
let dict = if cls.is(&vm.ctx.object()) {
None
} else {
Some(vm.ctx.new_dict())
@@ -72,7 +72,7 @@ fn object_setattr(
if let Some(attr) = objtype::class_get_attr(&cls, &attr_name.value) {
if let Some(descriptor) = objtype::class_get_attr(&attr.class(), "__set__") {
return vm
.invoke(descriptor, vec![attr, obj.clone(), value])
.invoke(&descriptor, vec![attr, obj.clone(), value])
.map(|_| ());
}
}
@@ -94,7 +94,7 @@ fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine)
if let Some(attr) = objtype::class_get_attr(&cls, &attr_name.value) {
if let Some(descriptor) = objtype::class_get_attr(&attr.class(), "__delete__") {
return vm.invoke(descriptor, vec![attr, obj.clone()]).map(|_| ());
return vm.invoke(&descriptor, vec![attr, obj.clone()]).map(|_| ());
}
}
@@ -118,6 +118,10 @@ fn object_repr(zelf: PyObjectRef, _vm: &VirtualMachine) -> String {
format!("<{} object at 0x{:x}>", zelf.class().name, zelf.get_id())
}
fn object_subclasshook(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult {
Ok(vm.ctx.not_implemented())
}
pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyList> {
let attributes: PyAttributes = objtype::get_attributes(obj.class());
@@ -126,7 +130,7 @@ pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyList> {
// Get instance attributes:
if let Some(object_dict) = &obj.dict {
vm.invoke(
vm.get_attribute(dict.clone().into_object(), "update")?,
&vm.get_attribute(dict.clone().into_object(), "update")?,
object_dict.clone().into_object(),
)?;
}
@@ -149,7 +153,7 @@ fn object_format(
}
pub fn init(context: &PyContext) {
let object = &context.object;
let object = &context.types.object_type;
let object_doc = "The most base type";
extend_class!(context, object, {
@@ -179,7 +183,8 @@ pub fn init(context: &PyContext) {
"__repr__" => context.new_rustfunc(object_repr),
"__format__" => context.new_rustfunc(object_format),
"__getattribute__" => context.new_rustfunc(object_getattribute),
"__doc__" => context.new_str(object_doc.to_string())
"__subclasshook__" => context.new_classmethod(object_subclasshook),
"__doc__" => context.new_str(object_doc.to_string()),
});
}
@@ -218,39 +223,8 @@ fn object_dict_setter(
))
}
fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMachine) -> PyResult {
let name = &name_str.value;
fn object_getattribute(obj: PyObjectRef, name: PyStringRef, vm: &VirtualMachine) -> PyResult {
vm_trace!("object.__getattribute__({:?}, {:?})", obj, name);
let cls = obj.class();
if let Some(attr) = objtype::class_get_attr(&cls, &name) {
let attr_class = attr.class();
if objtype::class_has_attr(&attr_class, "__set__") {
if let Some(descriptor) = objtype::class_get_attr(&attr_class, "__get__") {
return vm.invoke(descriptor, vec![attr, obj, cls.into_object()]);
}
}
}
if let Some(obj_attr) = object_getattr(&obj, &name, &vm)? {
Ok(obj_attr)
} else if let Some(attr) = objtype::class_get_attr(&cls, &name) {
vm.call_get_descriptor(attr, obj)
} else if let Some(getter) = objtype::class_get_attr(&cls, "__getattr__") {
vm.invoke(getter, vec![obj, name_str.into_object()])
} else {
Err(vm.new_attribute_error(format!("{} has no attribute '{}'", obj, name)))
}
}
fn object_getattr(
obj: &PyObjectRef,
attr_name: &str,
vm: &VirtualMachine,
) -> PyResult<Option<PyObjectRef>> {
if let Some(ref dict) = obj.dict {
dict.get_item_option(attr_name, vm)
} else {
Ok(None)
}
vm.generic_getattribute(obj.clone(), name.clone())?
.ok_or_else(|| vm.new_attribute_error(format!("{} has no attribute '{}'", obj, name)))
}

70
vm/src/obj/objproperty.rs
View File

@@ -1,7 +1,6 @@
/*! Python `property` descriptor class.
*/
use crate::function::{IntoPyNativeFunc, OptionalArg, PyFuncArgs};
use crate::obj::objtype::PyClassRef;
use crate::pyobject::{
@@ -9,6 +8,7 @@ use crate::pyobject::{
TypeProtocol,
};
use crate::vm::VirtualMachine;
use std::cell::RefCell;
// Read-only property, doesn't have __set__ or __delete__
#[pyclass]
@@ -37,7 +37,7 @@ impl PyReadOnlyProperty {
if obj.is(vm.ctx.none.as_object()) {
Ok(zelf.into_object())
} else {
vm.invoke(zelf.getter.clone(), obj)
vm.invoke(&zelf.getter, obj)
}
}
}
@@ -80,7 +80,7 @@ pub struct PyProperty {
getter: Option<PyObjectRef>,
setter: Option<PyObjectRef>,
deleter: Option<PyObjectRef>,
doc: Option<PyObjectRef>,
doc: RefCell<Option<PyObjectRef>>,
}
impl PyValue for PyProperty {
@@ -107,20 +107,14 @@ impl PyProperty {
);
fn into_option(vm: &VirtualMachine, arg: Option<&PyObjectRef>) -> Option<PyObjectRef> {
arg.and_then(|arg| {
if vm.ctx.none().is(arg) {
None
} else {
Some(arg.clone())
}
})
arg.and_then(|arg| py_none_to_option(vm, arg))
}
PyProperty {
getter: into_option(vm, fget),
setter: into_option(vm, fset),
deleter: into_option(vm, fdel),
doc: into_option(vm, doc),
doc: RefCell::new(into_option(vm, doc)),
}
.into_ref_with_type(vm, cls)
}
@@ -129,8 +123,8 @@ impl PyProperty {
// specialised version that doesn't check for None
pub(crate) fn instance_binding_get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if let Some(getter) = self.getter.as_ref() {
vm.invoke(getter.clone(), obj)
if let Some(ref getter) = self.getter.as_ref() {
vm.invoke(getter, obj)
} else {
Err(vm.new_attribute_error("unreadable attribute".to_string()))
}
@@ -147,7 +141,7 @@ impl PyProperty {
if obj.is(vm.ctx.none.as_object()) {
Ok(zelf.into_object())
} else {
vm.invoke(getter.clone(), obj)
vm.invoke(&getter, obj)
}
} else {
Err(vm.new_attribute_error("unreadable attribute".to_string()))
@@ -156,8 +150,8 @@ impl PyProperty {
#[pymethod(name = "__set__")]
fn set(&self, obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if let Some(setter) = self.setter.as_ref() {
vm.invoke(setter.clone(), vec![obj, value])
if let Some(ref setter) = self.setter.as_ref() {
vm.invoke(setter, vec![obj, value])
} else {
Err(vm.new_attribute_error("can't set attribute".to_string()))
}
@@ -165,8 +159,8 @@ impl PyProperty {
#[pymethod(name = "__delete__")]
fn delete(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult {
if let Some(deleter) = self.deleter.as_ref() {
vm.invoke(deleter.clone(), obj)
if let Some(ref deleter) = self.deleter.as_ref() {
vm.invoke(deleter, obj)
} else {
Err(vm.new_attribute_error("can't delete attribute".to_string()))
}
@@ -189,6 +183,15 @@ impl PyProperty {
self.deleter.clone()
}
fn doc_getter(&self, _vm: &VirtualMachine) -> Option<PyObjectRef> {
self.doc.borrow().clone()
}
fn doc_setter(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult {
self.doc.replace(py_none_to_option(vm, &value));
Ok(vm.get_none())
}
// Python builder functions
#[pymethod]
@@ -201,7 +204,7 @@ impl PyProperty {
getter: getter.or_else(|| zelf.getter.clone()),
setter: zelf.setter.clone(),
deleter: zelf.deleter.clone(),
doc: None,
doc: RefCell::new(None),
}
.into_ref_with_type(vm, TypeProtocol::class(&zelf))
}
@@ -216,7 +219,7 @@ impl PyProperty {
getter: zelf.getter.clone(),
setter: setter.or_else(|| zelf.setter.clone()),
deleter: zelf.deleter.clone(),
doc: None,
doc: RefCell::new(None),
}
.into_ref_with_type(vm, TypeProtocol::class(&zelf))
}
@@ -231,12 +234,21 @@ impl PyProperty {
getter: zelf.getter.clone(),
setter: zelf.setter.clone(),
deleter: deleter.or_else(|| zelf.deleter.clone()),
doc: None,
doc: RefCell::new(None),
}
.into_ref_with_type(vm, TypeProtocol::class(&zelf))
}
}
/// Take a python object and turn it into an option object, where python None maps to rust None.
fn py_none_to_option(vm: &VirtualMachine, value: &PyObjectRef) -> Option<PyObjectRef> {
if vm.ctx.none().is(value) {
None
} else {
Some(value.clone())
}
}
pub struct PropertyBuilder<'a> {
ctx: &'a PyContext,
getter: Option<PyObjectRef>,
@@ -276,7 +288,7 @@ impl<'a> PropertyBuilder<'a> {
getter: self.getter.clone(),
setter: self.setter.clone(),
deleter: None,
doc: None,
doc: RefCell::new(None),
};
PyObject::new(payload, self.ctx.property_type(), None)
@@ -293,6 +305,16 @@ impl<'a> PropertyBuilder<'a> {
}
pub fn init(context: &PyContext) {
PyReadOnlyProperty::extend_class(context, &context.readonly_property_type);
PyProperty::extend_class(context, &context.property_type);
PyReadOnlyProperty::extend_class(context, &context.types.readonly_property_type);
PyProperty::extend_class(context, &context.types.property_type);
// This is a bit unfortunate, but this instance attribute overlaps with the
// class __doc__ string..
extend_class!(context, &context.types.property_type, {
"__doc__" =>
PropertyBuilder::new(context)
.add_getter(PyProperty::doc_getter)
.add_setter(PyProperty::doc_setter)
.create(),
});
}

Some files were not shown because too many files have changed in this diff Show More