From 4dc31e21ec39510b49a86ba568263f8f0236ded6 Mon Sep 17 00:00:00 2001 From: JimJeon Date: Sat, 29 Jun 2019 16:15:37 +0900 Subject: [PATCH 001/143] Fix tests for float.__round__ Added type checkings for float.__round__ tests and fixed tests when ndigits is zero. --- tests/snippets/floats.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 101d58cc92..9af25565bf 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -151,12 +151,18 @@ assert float(1.2) == 1.2 assert math.trunc(1.2) == 1 assert_raises(OverflowError, float('inf').__trunc__) assert_raises(ValueError, float('nan').__trunc__) -assert 0.5.__round__() == 0.0 -assert 1.5.__round__() == 2.0 +assert isinstance(0.5.__round__(), int) +assert isinstance(1.5.__round__(), int) +assert 0.5.__round__() == 0 +assert 1.5.__round__() == 2 +assert isinstance(0.5.__round__(0), float) +assert isinstance(1.5.__round__(0), float) assert 0.5.__round__(0) == 0.0 assert 1.5.__round__(0) == 2.0 -assert 0.5.__round__(None) == 0.0 -assert 1.5.__round__(None) == 2.0 +assert isinstance(0.5.__round__(None), int) +assert isinstance(1.5.__round__(None), int) +assert 0.5.__round__(None) == 0 +assert 1.5.__round__(None) == 2 assert_raises(OverflowError, float('inf').__round__) assert_raises(ValueError, float('nan').__round__) From be887526c1616657d6844c8ba0beecec806f948f Mon Sep 17 00:00:00 2001 From: JimJeon Date: Sat, 29 Jun 2019 17:34:28 +0900 Subject: [PATCH 002/143] Fix implements of `round` - Fix implements when ndigits argument of builtin_round is None - Fix implements when ndigits argument of float_round is 0 --- tests/snippets/builtin_round.py | 4 ++++ vm/src/builtins.rs | 16 +++++++++++++--- vm/src/obj/objfloat.rs | 26 ++++++++++++++++++-------- 3 files changed, 35 insertions(+), 11 deletions(-) create mode 100644 tests/snippets/builtin_round.py diff --git a/tests/snippets/builtin_round.py b/tests/snippets/builtin_round.py new file mode 100644 index 0000000000..8abb3d88f9 --- /dev/null +++ b/tests/snippets/builtin_round.py @@ -0,0 +1,4 @@ +assert round(0.0, 0) == 0.0 +assert round(0, 0) == 0 +with assertRaises(TypeError): + round(0, 0.0) \ No newline at end of file diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 55f85baaa0..1d1d8d73bc 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -733,9 +733,19 @@ fn builtin_round(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { optional = [(ndigits, None)] ); if let Some(ndigits) = ndigits { - let ndigits = vm.call_method(ndigits, "__int__", vec![])?; - let rounded = vm.call_method(number, "__round__", vec![ndigits])?; - Ok(rounded) + if objtype::isinstance(ndigits, &vm.ctx.int_type()) { + let ndigits = vm.call_method(ndigits, "__int__", vec![])?; + let rounded = vm.call_method(number, "__round__", vec![ndigits])?; + Ok(rounded) + } else if vm.ctx.none().is(ndigits) { + let rounded = &vm.call_method(number, "__round__", vec![])?; + Ok(vm.ctx.new_int(objint::get_value(rounded).clone())) + } else { + Err(vm.new_type_error(format!( + "'{}' object cannot be interpreted as an integer", + ndigits.class().name + ))) + } } else { // without a parameter, the result type is coerced to int let rounded = &vm.call_method(number, "__round__", vec![])?; diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index cfac52aa9f..e0b0cfbd66 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -448,17 +448,29 @@ impl PyFloat { value.class().name ))); }; - if ndigits.is_zero() { - None - } else { - Some(ndigits) - } + Some(ndigits) } else { None } } }; - if ndigits.is_none() { + if let Some(ndigits) = ndigits { + if ndigits.is_zero() { + let fract = self.value.fract(); + let value = if (fract.abs() - 0.5).abs() < std::f64::EPSILON { + if self.value.trunc() % 2.0 == 0.0 { + self.value - fract + } else { + self.value + fract + } + } else { + self.value.round() + }; + Ok(vm.ctx.new_float(value)) + } else { + Ok(vm.ctx.not_implemented()) + } + } else { let fract = self.value.fract(); let value = if (fract.abs() - 0.5).abs() < std::f64::EPSILON { if self.value.trunc() % 2.0 == 0.0 { @@ -471,8 +483,6 @@ impl PyFloat { }; let int = try_to_bigint(value, vm)?; Ok(vm.ctx.new_int(int)) - } else { - Ok(vm.ctx.not_implemented()) } } From a3ad97c786615a958c124a95677f10f1982523a8 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 9 Aug 2019 13:19:39 +0300 Subject: [PATCH 003/143] Add Popen.terminate --- tests/snippets/stdlib_subprocess.py | 5 +++++ vm/src/stdlib/os.rs | 2 +- vm/src/stdlib/subprocess.rs | 10 +++++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/snippets/stdlib_subprocess.py b/tests/snippets/stdlib_subprocess.py index fa1f3c30cd..9e7d7fbfce 100644 --- a/tests/snippets/stdlib_subprocess.py +++ b/tests/snippets/stdlib_subprocess.py @@ -1,6 +1,7 @@ import subprocess import time import sys +import signal from testutils import assertRaises @@ -33,3 +34,7 @@ if "win" not in sys.platform: else: # windows assert p.stdout.read() == b"test\r\n" + +p = subprocess.Popen(["sleep", "2"]) +p.terminate() +assert p.poll() == -signal.SIGTERM diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index bb8bdec660..c2fff8c2e2 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -159,7 +159,7 @@ pub fn os_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(raw_file_number(handle))) } -fn convert_io_error(vm: &VirtualMachine, err: io::Error) -> PyObjectRef { +pub fn convert_io_error(vm: &VirtualMachine, err: io::Error) -> PyObjectRef { let os_error = match err.kind() { ErrorKind::NotFound => { let exc_type = vm.ctx.exceptions.file_not_found_error.clone(); diff --git a/vm/src/stdlib/subprocess.rs b/vm/src/stdlib/subprocess.rs index dd49e247ec..168a54e4f5 100644 --- a/vm/src/stdlib/subprocess.rs +++ b/vm/src/stdlib/subprocess.rs @@ -11,7 +11,7 @@ use crate::obj::objstr::{self, PyStringRef}; use crate::obj::objtype::PyClassRef; use crate::pyobject::{Either, IntoPyObject, PyObjectRef, PyRef, PyResult, PyValue}; use crate::stdlib::io::io_open; -use crate::stdlib::os::{raw_file_number, rust_file}; +use crate::stdlib::os::{convert_io_error, raw_file_number, rust_file}; use crate::vm::VirtualMachine; #[derive(Debug)] @@ -149,6 +149,13 @@ impl PopenRef { fn stderr(self, vm: &VirtualMachine) -> PyResult { convert_to_file_io(&self.process.borrow().stderr, "rb".to_string(), vm) } + + fn terminate(self, vm: &VirtualMachine) -> PyResult<()> { + self.process + .borrow_mut() + .terminate() + .map_err(|err| convert_io_error(vm, err)) + } } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { @@ -165,6 +172,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "stdin" => ctx.new_property(PopenRef::stdin), "stdout" => ctx.new_property(PopenRef::stdout), "stderr" => ctx.new_property(PopenRef::stderr), + "terminate" => ctx.new_rustfunc(PopenRef::terminate), }); let module = py_module!(vm, "subprocess", { From 572e3b6e9322e9d3267d8e6ea7f2e6adea2f3a10 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 9 Aug 2019 13:21:51 +0300 Subject: [PATCH 004/143] Add Popen.kill --- tests/snippets/stdlib_subprocess.py | 4 ++++ vm/src/stdlib/subprocess.rs | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/tests/snippets/stdlib_subprocess.py b/tests/snippets/stdlib_subprocess.py index 9e7d7fbfce..0421610cc9 100644 --- a/tests/snippets/stdlib_subprocess.py +++ b/tests/snippets/stdlib_subprocess.py @@ -38,3 +38,7 @@ else: p = subprocess.Popen(["sleep", "2"]) p.terminate() assert p.poll() == -signal.SIGTERM + +p = subprocess.Popen(["sleep", "2"]) +p.kill() +assert p.poll() == -signal.SIGKILL diff --git a/vm/src/stdlib/subprocess.rs b/vm/src/stdlib/subprocess.rs index 168a54e4f5..e214fc9fa4 100644 --- a/vm/src/stdlib/subprocess.rs +++ b/vm/src/stdlib/subprocess.rs @@ -156,6 +156,13 @@ impl PopenRef { .terminate() .map_err(|err| convert_io_error(vm, err)) } + + fn kill(self, vm: &VirtualMachine) -> PyResult<()> { + self.process + .borrow_mut() + .kill() + .map_err(|err| convert_io_error(vm, err)) + } } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { @@ -173,6 +180,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "stdout" => ctx.new_property(PopenRef::stdout), "stderr" => ctx.new_property(PopenRef::stderr), "terminate" => ctx.new_rustfunc(PopenRef::terminate), + "kill" => ctx.new_rustfunc(PopenRef::kill), }); let module = py_module!(vm, "subprocess", { From 93701098f94b4d19600158226be14c71fe239556 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 9 Aug 2019 14:27:28 +0300 Subject: [PATCH 005/143] Add Popen.communicate --- tests/snippets/stdlib_subprocess.py | 10 ++++++++-- vm/src/obj/objbytes.rs | 9 ++++++++- vm/src/obj/objtuple.rs | 25 ++++++++++++++++++++++++- vm/src/stdlib/subprocess.rs | 13 +++++++++++++ 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/tests/snippets/stdlib_subprocess.py b/tests/snippets/stdlib_subprocess.py index 0421610cc9..f80a3775d6 100644 --- a/tests/snippets/stdlib_subprocess.py +++ b/tests/snippets/stdlib_subprocess.py @@ -30,10 +30,12 @@ p.wait() if "win" not in sys.platform: # unix - assert p.stdout.read() == b"test\n" + test_output = b"test\n" else: # windows - assert p.stdout.read() == b"test\r\n" + test_output = b"test\r\n" + +assert p.stdout.read() == test_output p = subprocess.Popen(["sleep", "2"]) p.terminate() @@ -42,3 +44,7 @@ assert p.poll() == -signal.SIGTERM p = subprocess.Popen(["sleep", "2"]) p.kill() assert p.poll() == -signal.SIGKILL + +p = subprocess.Popen(["echo", "test"], stdout=subprocess.PIPE) +(stdout, stderr) = p.communicate() +assert stdout == test_output diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8e3149b17f..c67a044b2b 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -11,7 +11,8 @@ use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{ - PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, }; use super::objbyteinner::{ @@ -59,6 +60,12 @@ impl PyBytes { } } +impl IntoPyObject for Vec { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self)) + } +} + impl Deref for PyBytes { type Target = [u8]; diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index e13d3160fe..5be1232c04 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -3,7 +3,9 @@ use std::fmt; use crate::function::OptionalArg; use crate::pyhash; -use crate::pyobject::{IdProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, +}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; @@ -37,6 +39,27 @@ impl PyValue for PyTuple { } } +impl IntoPyObject for (A,) +where + A: IntoPyObject, +{ + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_tuple(vec![self.0.into_pyobject(vm)?])) + } +} + +impl IntoPyObject for (A, B) +where + A: IntoPyObject, + B: IntoPyObject, +{ + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_tuple(vec![self.0.into_pyobject(vm)?, self.1.into_pyobject(vm)?])) + } +} + impl PyTuple { pub fn fast_getitem(&self, idx: usize) -> PyObjectRef { self.elements[idx].clone() diff --git a/vm/src/stdlib/subprocess.rs b/vm/src/stdlib/subprocess.rs index e214fc9fa4..1e7b1e2624 100644 --- a/vm/src/stdlib/subprocess.rs +++ b/vm/src/stdlib/subprocess.rs @@ -5,6 +5,7 @@ use std::time::Duration; use subprocess; use crate::function::OptionalArg; +use crate::obj::objbytes::PyBytesRef; use crate::obj::objlist::PyListRef; use crate::obj::objsequence; use crate::obj::objstr::{self, PyStringRef}; @@ -163,6 +164,17 @@ impl PopenRef { .kill() .map_err(|err| convert_io_error(vm, err)) } + + fn communicate( + self, + stdin: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<(Option>, Option>)> { + self.process + .borrow_mut() + .communicate_bytes(stdin.into_option().as_ref().map(|bytes| bytes.get_value())) + .map_err(|err| convert_io_error(vm, err)) + } } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { @@ -181,6 +193,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "stderr" => ctx.new_property(PopenRef::stderr), "terminate" => ctx.new_rustfunc(PopenRef::terminate), "kill" => ctx.new_rustfunc(PopenRef::kill), + "communicate" => ctx.new_rustfunc(PopenRef::communicate), }); let module = py_module!(vm, "subprocess", { From 77b34c9b958fd9d357983fcfb7b7d7b674497d90 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 4 Aug 2019 15:40:17 -0500 Subject: [PATCH 006/143] Add redox to travis using redoxer --- .travis.yml | 17 +++++++++++++++++ Cargo.toml | 5 +++-- redox/comment-cargo.sh | 15 +++++++++++++++ redox/uncomment-cargo.sh | 15 +++++++++++++++ 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100755 redox/comment-cargo.sh create mode 100755 redox/uncomment-cargo.sh diff --git a/.travis.yml b/.travis.yml index 228aba298e..b176992ccb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -145,3 +145,20 @@ matrix: env: - JOBCACHE=7 - TRAVIS_RUST_VERSION=stable + + - name: Ensure compilation on Redox OS with Redoxer + language: rust + rust: stable + cache: + cargo: true + directories: + - $HOME/.redoxer + install: + - cargo install redoxfs redoxer + - redoxer install + script: + - bash redox/uncomment-cargo.sh + - redoxer build --verbose + - bash redox/comment-cargo.sh + env: + - JOBCACHE=10 diff --git a/Cargo.toml b/Cargo.toml index b55f474973..56d5a21385 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,8 +42,9 @@ version = "0.2" name = "rustpython" path = "src/main.rs" -# Uncommment when you want to compile/check with redoxer -# [patch.crates-io] +[patch.crates-io] +# REDOX START, Uncommment when you want to compile/check with redoxer # time = { git = "https://gitlab.redox-os.org/redox-os/time.git", branch = "redox-unix" } # libc = { git = "https://github.com/AdminXVII/libc", branch = "extra-traits-redox" } # nix = { git = "https://github.com/AdminXVII/nix", branch = "add-redox-support" } +# REDOX END diff --git a/redox/comment-cargo.sh b/redox/comment-cargo.sh new file mode 100755 index 0000000000..e69c53f057 --- /dev/null +++ b/redox/comment-cargo.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +cargo=$(realpath "${1:-Cargo.toml}") + +tmpfile=$(mktemp) + +awk ' +/REDOX START/{redox=1; print; next} +/REDOX END/{redox=0} +{if (redox) print "#", $0; else print} +' "$cargo" >"$tmpfile" + +mv "$tmpfile" "$cargo" diff --git a/redox/uncomment-cargo.sh b/redox/uncomment-cargo.sh new file mode 100755 index 0000000000..f0e9cf252f --- /dev/null +++ b/redox/uncomment-cargo.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +cargo=$(realpath "${1:-Cargo.toml}") + +tmpfile=$(mktemp) + +awk ' +/REDOX START/{redox=1; print; next} +/REDOX END/{redox=0} +{if (redox) sub(/^#\s*/, ""); print} +' "$cargo" >"$tmpfile" + +mv "$tmpfile" "$cargo" From fd39a830c449a79f6ec2e07b574faa01f09f4f26 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 4 Aug 2019 15:46:07 -0500 Subject: [PATCH 007/143] Fix openpty on Redox --- vm/src/stdlib/os.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index bb8bdec660..4d20fa5c84 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -8,7 +8,7 @@ use std::{env, fs}; #[cfg(unix)] use nix::errno::Errno; -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "redox")))] use nix::pty::openpty; #[cfg(unix)] use nix::unistd::{self, Gid, Pid, Uid}; @@ -853,7 +853,7 @@ fn os_seteuid(euid: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { unistd::seteuid(Uid::from_raw(euid)).map_err(|err| convert_nix_error(vm, err)) } -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "redox")))] pub fn os_openpty(vm: &VirtualMachine) -> PyResult { match openpty(None, None) { Ok(r) => Ok(vm From 7c93617ac57c6f84558997eee1470fe04956b55a Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 4 Aug 2019 20:58:37 -0500 Subject: [PATCH 008/143] Fix travis redox --- .travis.yml | 31 +++++++++++++++++++++++-------- redox/comment-cargo.sh | 2 +- redox/uncomment-cargo.sh | 2 +- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index b176992ccb..372b71f8e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,11 @@ -before_cache: | - if command -v cargo; then - ! command -v cargo-sweep && cargo install cargo-sweep - cargo sweep -i - cargo sweep -t 15 - fi +before_cache: + - | + if command -v cargo; then + ! command -v cargo-sweep && cargo install cargo-sweep + cargo sweep -i + cargo sweep -t 15 + fi + - rm -rf ~/.cargo/registry/src matrix: fast_finish: true @@ -148,17 +150,30 @@ matrix: - name: Ensure compilation on Redox OS with Redoxer language: rust - rust: stable + rust: nightly + dist: bionic cache: cargo: true directories: - $HOME/.redoxer + - $HOME/.cargo/bin + before_install: + - sudo apt-get update -qq + - sudo apt-get install libfuse-dev install: - - cargo install redoxfs redoxer + - "! command -v redoxer && cargo install redoxfs redoxer" - redoxer install script: - bash redox/uncomment-cargo.sh - redoxer build --verbose - bash redox/comment-cargo.sh + before_cache: + - | + if ! command -v cargo-sweep; then + rustup install stable + cargo +stable install cargo-sweep + fi + - cargo sweep -t 15 + - rm -rf ~/.cargo/registry/src env: - JOBCACHE=10 diff --git a/redox/comment-cargo.sh b/redox/comment-cargo.sh index e69c53f057..edb03f51c9 100755 --- a/redox/comment-cargo.sh +++ b/redox/comment-cargo.sh @@ -2,7 +2,7 @@ set -e -cargo=$(realpath "${1:-Cargo.toml}") +cargo=${1:-Cargo.toml} tmpfile=$(mktemp) diff --git a/redox/uncomment-cargo.sh b/redox/uncomment-cargo.sh index f0e9cf252f..1cc19ca478 100755 --- a/redox/uncomment-cargo.sh +++ b/redox/uncomment-cargo.sh @@ -2,7 +2,7 @@ set -e -cargo=$(realpath "${1:-Cargo.toml}") +cargo=${1:-Cargo.toml} tmpfile=$(mktemp) From 4cda5e289bb27249562fe81e9064bc10c25516ef Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 5 Aug 2019 17:22:32 -0500 Subject: [PATCH 009/143] Use language: minimal --- .travis.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 372b71f8e6..9883401713 100644 --- a/.travis.yml +++ b/.travis.yml @@ -149,8 +149,9 @@ matrix: - TRAVIS_RUST_VERSION=stable - name: Ensure compilation on Redox OS with Redoxer - language: rust - rust: nightly + # language: minimal so that it actually uses bionic rather than xenial; + # rust isn't yet available on bionic + language: minimal dist: bionic cache: cargo: true @@ -158,6 +159,14 @@ matrix: - $HOME/.redoxer - $HOME/.cargo/bin before_install: + # install rust as travis does for language: rust + - curl -sSf https://build.travis-ci.org/files/rustup-init.sh | sh -s -- + --default-toolchain=$TRAVIS_RUST_VERSION -y + - export PATH=${TRAVIS_HOME}/.cargo/bin:$PATH + - rustc --version + - rustup --version + - cargo --version + - sudo apt-get update -qq - sudo apt-get install libfuse-dev install: @@ -177,3 +186,4 @@ matrix: - rm -rf ~/.cargo/registry/src env: - JOBCACHE=10 + - TRAVIS_RUST_VERSION=nightly From e4daf9167eada607cc34249ba8968f706385c899 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 5 Aug 2019 17:45:59 -0500 Subject: [PATCH 010/143] Add all of ~/.cargo to build cache --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9883401713..c82074b05e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -157,7 +157,7 @@ matrix: cargo: true directories: - $HOME/.redoxer - - $HOME/.cargo/bin + - $HOME/.cargo before_install: # install rust as travis does for language: rust - curl -sSf https://build.travis-ci.org/files/rustup-init.sh | sh -s -- From 04a8839ac9a00b1e2a7f5d3e05f4d26000d55b4a Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 5 Aug 2019 17:54:26 -0500 Subject: [PATCH 011/143] Fix cargo install redoxer --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c82074b05e..0c0785c441 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,9 @@ before_cache: - | if command -v cargo; then - ! command -v cargo-sweep && cargo install cargo-sweep + if ! command -v cargo-sweep; then + cargo install cargo-sweep + fi cargo sweep -i cargo sweep -t 15 fi @@ -170,7 +172,7 @@ matrix: - sudo apt-get update -qq - sudo apt-get install libfuse-dev install: - - "! command -v redoxer && cargo install redoxfs redoxer" + - if ! command -v redoxer; then cargo install redoxfs redoxer; fi - redoxer install script: - bash redox/uncomment-cargo.sh From f7246a3f517a7de462c37839140b75ad864b54b8 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 9 Aug 2019 11:27:39 -0500 Subject: [PATCH 012/143] Fix socket.rs --- vm/src/stdlib/socket.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 22f431a82b..9040c14fff 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -4,7 +4,7 @@ use std::io::Read; use std::io::Write; use std::net::{Ipv4Addr, SocketAddr, TcpListener, TcpStream, ToSocketAddrs, UdpSocket}; -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "redox")))] use nix::unistd::sethostname; use gethostname::gethostname; @@ -388,7 +388,7 @@ fn socket_gethostname(vm: &VirtualMachine) -> PyResult { .map_err(|err| vm.new_os_error(err.into_string().unwrap())) } -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "redox")))] fn socket_sethostname(hostname: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { sethostname(hostname.value.as_str()).map_err(|err| convert_nix_error(vm, err)) } @@ -449,6 +449,7 @@ fn extend_module_platform_specific(_vm: &VirtualMachine, module: PyObjectRef) -> fn extend_module_platform_specific(vm: &VirtualMachine, module: PyObjectRef) -> PyObjectRef { let ctx = &vm.ctx; + #[cfg(not(target_os = "redox"))] extend_module!(vm, module, { "sethostname" => ctx.new_rustfunc(socket_sethostname), }); From a7b08ad9a2025a2db2e45c08d948e32c23eb9faa Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 9 Aug 2019 16:33:28 -0500 Subject: [PATCH 013/143] Only check Redox on release or cron job --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0c0785c441..d5226c6f89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -155,6 +155,7 @@ matrix: # rust isn't yet available on bionic language: minimal dist: bionic + if: branch = release OR type = cron cache: cargo: true directories: From f8bde1217f4c4625cd677d35f9021aeeebd1b327 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 9 Aug 2019 16:34:33 -0500 Subject: [PATCH 014/143] Use release branch in redox recipe --- redox/recipe.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/redox/recipe.sh b/redox/recipe.sh index 2428d9b5f5..24611f0e5d 100644 --- a/redox/recipe.sh +++ b/redox/recipe.sh @@ -1,4 +1,5 @@ GIT=https://github.com/RustPython/RustPython +BRANCH=release CARGOFLAGS=--no-default-features export BUILDTIME_RUSTPYTHONPATH=/lib/rustpython/ From 0e77f0e65418ec000f4176132e0f0424288fd975 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 9 Aug 2019 21:36:42 -0500 Subject: [PATCH 015/143] Fix compilation of nested boolean operations --- compiler/src/compile.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index a13d3617e8..aafab376e6 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -1303,11 +1303,9 @@ impl Compiler { self.emit(Instruction::Pop); } if let Some(false_label) = false_label { - self.emit(Instruction::Duplicate); self.emit(Instruction::JumpIfFalse { target: false_label, }); - self.emit(Instruction::Pop); } } } From c44b1c843f59c8b5301b1c9529c05bdef7c207b8 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 10 Aug 2019 09:53:39 +0300 Subject: [PATCH 016/143] Make test less flaky --- tests/snippets/stdlib_subprocess.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/snippets/stdlib_subprocess.py b/tests/snippets/stdlib_subprocess.py index f80a3775d6..8d0dfe9205 100644 --- a/tests/snippets/stdlib_subprocess.py +++ b/tests/snippets/stdlib_subprocess.py @@ -39,11 +39,13 @@ assert p.stdout.read() == test_output p = subprocess.Popen(["sleep", "2"]) p.terminate() -assert p.poll() == -signal.SIGTERM +p.wait() +assert p.returncode == -signal.SIGTERM p = subprocess.Popen(["sleep", "2"]) p.kill() -assert p.poll() == -signal.SIGKILL +p.wait() +assert p.returncode == -signal.SIGKILL p = subprocess.Popen(["echo", "test"], stdout=subprocess.PIPE) (stdout, stderr) = p.communicate() From 608572b2b527fc5a89daf2d4903a85689baabbbc Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 10 Aug 2019 09:58:40 +0300 Subject: [PATCH 017/143] Add Popen.pid --- vm/src/stdlib/subprocess.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vm/src/stdlib/subprocess.rs b/vm/src/stdlib/subprocess.rs index 1e7b1e2624..006d463a49 100644 --- a/vm/src/stdlib/subprocess.rs +++ b/vm/src/stdlib/subprocess.rs @@ -175,6 +175,10 @@ impl PopenRef { .communicate_bytes(stdin.into_option().as_ref().map(|bytes| bytes.get_value())) .map_err(|err| convert_io_error(vm, err)) } + + fn pid(self, _vm: &VirtualMachine) -> Option { + self.process.borrow().pid() + } } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { @@ -194,6 +198,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "terminate" => ctx.new_rustfunc(PopenRef::terminate), "kill" => ctx.new_rustfunc(PopenRef::kill), "communicate" => ctx.new_rustfunc(PopenRef::communicate), + "pid" => ctx.new_property(PopenRef::pid), }); let module = py_module!(vm, "subprocess", { From 12c59dacfb77aafd13713593cfc2082366155020 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 10 Aug 2019 10:13:58 +0300 Subject: [PATCH 018/143] Add Popen cwd support --- vm/src/stdlib/subprocess.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vm/src/stdlib/subprocess.rs b/vm/src/stdlib/subprocess.rs index 006d463a49..82cd42a5e1 100644 --- a/vm/src/stdlib/subprocess.rs +++ b/vm/src/stdlib/subprocess.rs @@ -1,4 +1,5 @@ use std::cell::RefCell; +use std::ffi::OsString; use std::fs::File; use std::time::Duration; @@ -38,6 +39,8 @@ struct PopenArgs { stdout: Option, #[pyarg(positional_or_keyword, default = "None")] stderr: Option, + #[pyarg(positional_or_keyword, default = "None")] + cwd: Option, } impl IntoPyObject for subprocess::ExitStatus { @@ -96,6 +99,7 @@ impl PopenRef { .map(|x| objstr::get_value(x)) .collect(), }; + let cwd = args.cwd.map(|x| OsString::from(x.as_str())); let process = subprocess::Popen::create( &command_list, @@ -103,6 +107,7 @@ impl PopenRef { stdin, stdout, stderr, + cwd, ..Default::default() }, ) From ba31f4fde73ba53bb0bab34f28e128e508727b87 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 10 Aug 2019 19:40:52 +0300 Subject: [PATCH 019/143] Ignore clippy warning --- vm/src/stdlib/subprocess.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/vm/src/stdlib/subprocess.rs b/vm/src/stdlib/subprocess.rs index 82cd42a5e1..3af308f7c5 100644 --- a/vm/src/stdlib/subprocess.rs +++ b/vm/src/stdlib/subprocess.rs @@ -170,6 +170,7 @@ impl PopenRef { .map_err(|err| convert_io_error(vm, err)) } + #[allow(clippy::type_complexity)] fn communicate( self, stdin: OptionalArg, From f790de24018efe98738c9de9d4a7d47ba284edc7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 10 Aug 2019 22:28:33 +0300 Subject: [PATCH 020/143] Fix windows test --- tests/snippets/stdlib_subprocess.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/snippets/stdlib_subprocess.py b/tests/snippets/stdlib_subprocess.py index 8d0dfe9205..1202d8ef71 100644 --- a/tests/snippets/stdlib_subprocess.py +++ b/tests/snippets/stdlib_subprocess.py @@ -40,12 +40,18 @@ assert p.stdout.read() == test_output p = subprocess.Popen(["sleep", "2"]) p.terminate() p.wait() -assert p.returncode == -signal.SIGTERM +if "win" not in sys.platform: + assert p.returncode == -signal.SIGTERM +else: + assert p.returncode == 1 p = subprocess.Popen(["sleep", "2"]) p.kill() p.wait() -assert p.returncode == -signal.SIGKILL +if "win" not in sys.platform: + assert p.returncode == -signal.SIGKILL +else: + assert p.returncode == 1 p = subprocess.Popen(["echo", "test"], stdout=subprocess.PIPE) (stdout, stderr) = p.communicate() From 36c263f3647bc4c8156360c24e66a196a3670612 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 11 Aug 2019 13:30:28 +0900 Subject: [PATCH 021/143] Fix stdlib_subprocess for macOS --- tests/snippets/stdlib_subprocess.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/snippets/stdlib_subprocess.py b/tests/snippets/stdlib_subprocess.py index 1202d8ef71..9e04f49347 100644 --- a/tests/snippets/stdlib_subprocess.py +++ b/tests/snippets/stdlib_subprocess.py @@ -28,7 +28,9 @@ assert p.returncode == 0 p = subprocess.Popen(["echo", "test"], stdout=subprocess.PIPE) p.wait() -if "win" not in sys.platform: +is_unix = "win" not in sys.platform or "darwin" in sys.platform + +if is_unix: # unix test_output = b"test\n" else: @@ -40,7 +42,7 @@ assert p.stdout.read() == test_output p = subprocess.Popen(["sleep", "2"]) p.terminate() p.wait() -if "win" not in sys.platform: +if is_unix: assert p.returncode == -signal.SIGTERM else: assert p.returncode == 1 @@ -48,7 +50,7 @@ else: p = subprocess.Popen(["sleep", "2"]) p.kill() p.wait() -if "win" not in sys.platform: +if is_unix: assert p.returncode == -signal.SIGKILL else: assert p.returncode == 1 From a400f6631cee18ab03c2f42174d7a789f9c5fe30 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 11 Aug 2019 09:29:21 +0200 Subject: [PATCH 022/143] Improve lexing of numbers with underscores. --- examples/parse_folder.rs | 53 ++++++++++++++----- parser/src/lexer.rs | 107 ++++++++++++++++++++------------------ parser/src/python.lalrpop | 6 +-- tests/snippets/numbers.py | 11 ++++ 4 files changed, 110 insertions(+), 67 deletions(-) diff --git a/examples/parse_folder.rs b/examples/parse_folder.rs index 513333b953..33530708dd 100644 --- a/examples/parse_folder.rs +++ b/examples/parse_folder.rs @@ -15,7 +15,7 @@ use clap::{App, Arg}; use rustpython_parser::{ast, parser}; use std::path::{Path, PathBuf}; -use std::time::Instant; +use std::time::{Duration, Instant}; fn main() { env_logger::init(); @@ -61,30 +61,45 @@ fn parse_folder(path: &Path) -> std::io::Result> { } if metadata.is_file() && path.extension().and_then(|s| s.to_str()) == Some("py") { - let result = parse_python_file(&path); - match &result { + let parsed_file = parse_python_file(&path); + match &parsed_file.result { Ok(_) => {} Err(y) => error!("Erreur in file {:?} {:?}", path, y), } - res.push(ParsedFile { - filename: Box::new(path), - result, - }); + + res.push(parsed_file); } } Ok(res) } -fn parse_python_file(filename: &Path) -> ParseResult { +fn parse_python_file(filename: &Path) -> ParsedFile { info!("Parsing file {:?}", filename); - let source = std::fs::read_to_string(filename).map_err(|e| e.to_string())?; - parser::parse_program(&source).map_err(|e| e.to_string()) + match std::fs::read_to_string(filename) { + Err(e) => ParsedFile { + filename: Box::new(filename.to_path_buf()), + code: "".to_string(), + num_lines: 0, + result: Err(e.to_string()), + }, + Ok(source) => { + let num_lines = source.to_string().lines().count(); + let result = parser::parse_program(&source).map_err(|e| e.to_string()); + ParsedFile { + filename: Box::new(filename.to_path_buf()), + code: source.to_string(), + num_lines, + result, + } + } + } } fn statistics(results: ScanResult) { // println!("Processed {:?} files", res.len()); println!("Scanned a total of {} files", results.parsed_files.len()); - let total = results.parsed_files.len(); + let total: usize = results.parsed_files.len(); + let total_lines: usize = results.parsed_files.iter().map(|p| p.num_lines).sum(); let failed = results .parsed_files .iter() @@ -103,9 +118,19 @@ fn statistics(results: ScanResult) { let duration = results.t2 - results.t1; println!("Total time spend: {:?}", duration); println!( - "File processing rate: {} files/second", - (total * 1_000_000) as f64 / duration.as_micros() as f64 + "Processed {} files. That's {} files/second", + total, + rate(total, duration) ); + println!( + "Processed {} lines of python code. That's {} lines/second", + total_lines, + rate(total_lines, duration) + ); +} + +fn rate(counter: usize, duration: Duration) -> f64 { + (counter * 1_000_000) as f64 / duration.as_micros() as f64 } struct ScanResult { @@ -116,6 +141,8 @@ struct ScanResult { struct ParsedFile { filename: Box, + code: String, + num_lines: usize, result: ParseResult, } diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index ae83a1d8d1..4051621042 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -340,18 +340,7 @@ where /// Lex a hex/octal/decimal/binary number without a decimal point. fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> LexResult { - let mut value_text = String::new(); - - loop { - if let Some(c) = self.take_number(radix) { - value_text.push(c); - } else if self.chr0 == Some('_') { - self.next_char(); - } else { - break; - } - } - + let value_text = self.radix_run(radix); let end_pos = self.get_pos(); let value = BigInt::from_str_radix(&value_text, radix).map_err(|e| LexicalError { error: LexicalErrorType::OtherError(format!("{:?}", e)), @@ -360,24 +349,19 @@ where Ok((start_pos, Tok::Int { value }, end_pos)) } + /// Lex a normal number, that is, no octal, hex or binary number. fn lex_normal_number(&mut self) -> LexResult { let start_pos = self.get_pos(); - let mut value_text = String::new(); - // Normal number: - while let Some(c) = self.take_number(10) { - value_text.push(c); - } + let mut value_text = self.radix_run(10); // If float: if self.chr0 == Some('.') || self.at_exponent() { // Take '.': if self.chr0 == Some('.') { value_text.push(self.next_char().unwrap()); - while let Some(c) = self.take_number(10) { - value_text.push(c); - } + value_text.push_str(&self.radix_run(10)); } // 1e6 for example: @@ -389,9 +373,7 @@ where value_text.push(self.next_char().unwrap()); } - while let Some(c) = self.take_number(10) { - value_text.push(c); - } + value_text.push_str(&self.radix_run(10)); } let value = f64::from_str(&value_text).unwrap(); @@ -426,6 +408,57 @@ where } } + /// Consume a sequence of numbers with the given radix, + /// the digits can be decorated with underscores + /// like this: '1_2_3_4' == '1234' + fn radix_run(&mut self, radix: u32) -> String { + let mut value_text = String::new(); + loop { + if let Some(c) = self.take_number(radix) { + value_text.push(c); + } else if self.chr0 == Some('_') && Lexer::::is_digit_of_radix(&self.chr1, radix) { + self.next_char(); + } else { + break; + } + } + value_text + } + + /// Consume a single character with the given radix. + fn take_number(&mut self, radix: u32) -> Option { + let take_char = Lexer::::is_digit_of_radix(&self.chr0, radix); + + if take_char { + Some(self.next_char().unwrap()) + } else { + None + } + } + + /// Test if a digit is of a certain radix. + fn is_digit_of_radix(c: &Option, radix: u32) -> bool { + match radix { + 2 => match c { + Some('0'..='1') => true, + _ => false, + }, + 8 => match c { + Some('0'..='7') => true, + _ => false, + }, + 10 => match c { + Some('0'..='9') => true, + _ => false, + }, + 16 => match c { + Some('0'..='9') | Some('a'..='f') | Some('A'..='F') => true, + _ => false, + }, + x => unimplemented!("Radix not implemented: {}", x), + } + } + /// Test if we face '[eE][-+]?[0-9]+' fn at_exponent(&self) -> bool { match self.chr0 { @@ -626,34 +659,6 @@ where } } - fn take_number(&mut self, radix: u32) -> Option { - let take_char = match radix { - 2 => match self.chr0 { - Some('0'..='1') => true, - _ => false, - }, - 8 => match self.chr0 { - Some('0'..='7') => true, - _ => false, - }, - 10 => match self.chr0 { - Some('0'..='9') => true, - _ => false, - }, - 16 => match self.chr0 { - Some('0'..='9') | Some('a'..='f') | Some('A'..='F') => true, - _ => false, - }, - x => unimplemented!("Radix not implemented: {}", x), - }; - - if take_char { - Some(self.next_char().unwrap()) - } else { - None - } - } - /// This is the main entry point. Call this function to retrieve the next token. /// This function is used by the iterator implementation. fn inner_next(&mut self) -> LexResult { diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 6af9fea61c..8929dc5427 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -247,7 +247,7 @@ ImportDots: usize = { ImportAsNames: Vec = { >> => i, - "(" >> ")" => i, + "(" >> ","? ")" => i, "*" => { // Star import all vec![ast::ImportSymbol { symbol: "*".to_string(), alias: None }] @@ -952,11 +952,11 @@ Atom: ast::Expression = { }; ListLiteralValues: Vec = { - > <_trailing_comma:","?> => e, + > ","? => e, }; DictLiteralValues: Vec<(Option, ast::Expression)> = { - > <_trailing_comma:","?> => elements, + > ","? => elements, }; DictEntry: (ast::Expression, ast::Expression) = { diff --git a/tests/snippets/numbers.py b/tests/snippets/numbers.py index c36602ee16..b90168d417 100644 --- a/tests/snippets/numbers.py +++ b/tests/snippets/numbers.py @@ -1,3 +1,5 @@ +from testutils import assertRaises + x = 5 x.__init__(6) assert x == 5 @@ -42,3 +44,12 @@ assert int(0).__rxor__(1) == 1 assert int(1).__rxor__(1) == 0 assert int(3).__rxor__(-3) == -2 assert int(3).__rxor__(4) == 7 + +# Test underscores in numbers: +assert 1_2 == 12 +assert 1_2_3 == 123 +assert 1_2.3_4 == 12.34 +assert 1_2.3_4e0_0 == 12.34 + +with assertRaises(SyntaxError): + eval('1__2') From dbc562c61311b1d99ccfd8f17125089e2b400867 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 11 Aug 2019 09:57:40 +0200 Subject: [PATCH 023/143] Implement clippy hint. --- examples/parse_folder.rs | 14 +++++++------- parser/src/lexer.rs | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/parse_folder.rs b/examples/parse_folder.rs index 33530708dd..ad0c5f8599 100644 --- a/examples/parse_folder.rs +++ b/examples/parse_folder.rs @@ -14,7 +14,7 @@ extern crate log; use clap::{App, Arg}; use rustpython_parser::{ast, parser}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::time::{Duration, Instant}; fn main() { @@ -77,8 +77,8 @@ fn parse_python_file(filename: &Path) -> ParsedFile { info!("Parsing file {:?}", filename); match std::fs::read_to_string(filename) { Err(e) => ParsedFile { - filename: Box::new(filename.to_path_buf()), - code: "".to_string(), + // filename: Box::new(filename.to_path_buf()), + // code: "".to_string(), num_lines: 0, result: Err(e.to_string()), }, @@ -86,8 +86,8 @@ fn parse_python_file(filename: &Path) -> ParsedFile { let num_lines = source.to_string().lines().count(); let result = parser::parse_program(&source).map_err(|e| e.to_string()); ParsedFile { - filename: Box::new(filename.to_path_buf()), - code: source.to_string(), + // filename: Box::new(filename.to_path_buf()), + // code: source.to_string(), num_lines, result, } @@ -140,8 +140,8 @@ struct ScanResult { } struct ParsedFile { - filename: Box, - code: String, + // filename: Box, + // code: String, num_lines: usize, result: ParseResult, } diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 4051621042..0f2e28bd29 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -416,7 +416,7 @@ where loop { if let Some(c) = self.take_number(radix) { value_text.push(c); - } else if self.chr0 == Some('_') && Lexer::::is_digit_of_radix(&self.chr1, radix) { + } else if self.chr0 == Some('_') && Lexer::::is_digit_of_radix(self.chr1, radix) { self.next_char(); } else { break; @@ -427,7 +427,7 @@ where /// Consume a single character with the given radix. fn take_number(&mut self, radix: u32) -> Option { - let take_char = Lexer::::is_digit_of_radix(&self.chr0, radix); + let take_char = Lexer::::is_digit_of_radix(self.chr0, radix); if take_char { Some(self.next_char().unwrap()) @@ -437,7 +437,7 @@ where } /// Test if a digit is of a certain radix. - fn is_digit_of_radix(c: &Option, radix: u32) -> bool { + fn is_digit_of_radix(c: Option, radix: u32) -> bool { match radix { 2 => match c { Some('0'..='1') => true, From b24444e9bc71ada14c3b705635c95752d2cd1497 Mon Sep 17 00:00:00 2001 From: yjhmelody <465402634@qq.com> Date: Sun, 11 Aug 2019 19:48:18 +0800 Subject: [PATCH 024/143] fix int fn --- vm/src/obj/objint.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index a158eb344d..9f41edf048 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -6,6 +6,7 @@ use num_traits::{One, Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; use crate::function::{KwArgs, OptionalArg, PyFuncArgs}; +use crate::obj::objtype::PyClassRef; use crate::pyhash; use crate::pyobject::{ IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, @@ -17,7 +18,6 @@ use super::objbyteinner::PyByteInner; use super::objbytes::PyBytes; use super::objstr::{PyString, PyStringRef}; use super::objtype; -use crate::obj::objtype::PyClassRef; /// int(x=0) -> integer /// int(x, base=10) -> integer @@ -667,10 +667,16 @@ fn int_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResul } // Casting function: -pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult { +pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, mut base: u32) -> PyResult { + if base == 0 { + base = 10 + } else if base < 2 || base > 36 { + return Err(vm.new_value_error(format!("int() base must be >= 2 and <= 36, or 0"))); + } + match_class!(obj.clone(), s @ PyString => { - i32::from_str_radix(s.as_str(), base) + i32::from_str_radix(s.as_str().trim(), base) .map(BigInt::from) .map_err(|_|vm.new_value_error(format!( "invalid literal for int() with base {}: '{}'", From 7c42a18ab67a1659384b40a78cdb505d5c9b9b2b Mon Sep 17 00:00:00 2001 From: yjhmelody <465402634@qq.com> Date: Mon, 12 Aug 2019 00:10:27 +0800 Subject: [PATCH 025/143] add test cases for int fn --- tests/snippets/ints.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index 0b6eb7f761..06f9068f38 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -90,6 +90,10 @@ assert int() == 0 assert int("101", 2) == 5 assert int("101", base=2) == 5 assert int(1) == 1 +assert int(' 1') == 1 +assert int('1 ') == 1 +assert int(' 1 ') == 1 +assert int('10', base=0) == 10 assert int.from_bytes(b'\x00\x10', 'big') == 16 assert int.from_bytes(b'\x00\x10', 'little') == 4096 @@ -103,6 +107,13 @@ assert (-1024).to_bytes(4, 'little', signed=True) == b'\x00\xfc\xff\xff' assert (2147483647).to_bytes(8, 'big', signed=False) == b'\x00\x00\x00\x00\x7f\xff\xff\xff' assert (-2147483648).to_bytes(8, 'little', signed=True) == b'\x00\x00\x00\x80\xff\xff\xff\xff' +with assertRaises(ValueError): + # check base first + int(' 1 ', base=1) + +with assertRaises(ValueError): + int(' 1 ', base=37) + with assertRaises(TypeError): int(base=2) From fb5ebea3897071865a6669bda37f54741efd9a8d Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sun, 11 Aug 2019 12:29:10 -0500 Subject: [PATCH 026/143] Revert "Fix compilation of nested boolean operations" --- compiler/src/compile.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index aafab376e6..a13d3617e8 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -1303,9 +1303,11 @@ impl Compiler { self.emit(Instruction::Pop); } if let Some(false_label) = false_label { + self.emit(Instruction::Duplicate); self.emit(Instruction::JumpIfFalse { target: false_label, }); + self.emit(Instruction::Pop); } } } From 36d8147e8e39566f5c1f242a6b5f85e4413e4267 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 11 Aug 2019 20:20:15 +0200 Subject: [PATCH 027/143] Improve the situation regarding boolean operations. --- bytecode/src/bytecode.rs | 18 +- compiler/src/compile.rs | 187 ++++++++++++-------- compiler/src/symboltable.rs | 5 +- parser/src/ast.rs | 3 +- parser/src/python.lalrpop | 30 +++- tests/snippets/short_circuit_evaluations.py | 15 ++ vm/src/frame.rs | 24 ++- vm/src/stdlib/ast.rs | 7 +- 8 files changed, 192 insertions(+), 97 deletions(-) create mode 100644 tests/snippets/short_circuit_evaluations.py diff --git a/bytecode/src/bytecode.rs b/bytecode/src/bytecode.rs index 0d5008b839..2c3d431636 100644 --- a/bytecode/src/bytecode.rs +++ b/bytecode/src/bytecode.rs @@ -138,12 +138,24 @@ pub enum Instruction { Jump { target: Label, }, - JumpIf { + /// Pop the top of the stack, and jump if this value is true. + JumpIfTrue { target: Label, }, + /// Pop the top of the stack, and jump if this value is false. JumpIfFalse { target: Label, }, + /// Peek at the top of the stack, and jump if this value is true. + /// Otherwise, pop top of stack. + JumpIfTrueOrPop { + target: Label, + }, + /// Peek at the top of the stack, and jump if this value is false. + /// Otherwise, pop top of stack. + JumpIfFalseOrPop { + target: Label, + }, MakeFunction { flags: FunctionOpArg, }, @@ -411,8 +423,10 @@ impl Instruction { Continue => w!(Continue), Break => w!(Break), Jump { target } => w!(Jump, label_map[target]), - JumpIf { target } => w!(JumpIf, label_map[target]), + JumpIfTrue { target } => w!(JumpIfTrue, label_map[target]), JumpIfFalse { target } => w!(JumpIfFalse, label_map[target]), + JumpIfTrueOrPop { target } => w!(JumpIfTrueOrPop, label_map[target]), + JumpIfFalseOrPop { target } => w!(JumpIfFalseOrPop, label_map[target]), MakeFunction { flags } => w!(MakeFunction, format!("{:?}", flags)), CallFunction { typ } => w!(CallFunction, format!("{:?}", typ)), ForIter { target } => w!(ForIter, label_map[target]), diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index a13d3617e8..a0f20c1e08 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -15,6 +15,7 @@ use rustpython_parser::{ast, parser}; type BasicOutputStream = PeepholeOptimizer; +/// Main structure holding the state of compilation. struct Compiler { output_stack: Vec, scope_stack: Vec, @@ -107,12 +108,6 @@ pub enum Mode { Single, } -#[derive(Clone, Copy)] -enum EvalContext { - Statement, - Expression, -} - pub(crate) type Label = usize; impl Default for Compiler @@ -350,14 +345,14 @@ impl Compiler { match orelse { None => { // Only if: - self.compile_test(test, None, Some(end_label), EvalContext::Statement)?; + self.compile_jump_if(test, false, end_label)?; self.compile_statements(body)?; self.set_label(end_label); } Some(statements) => { // if - else: let else_label = self.new_label(); - self.compile_test(test, None, Some(else_label), EvalContext::Statement)?; + self.compile_jump_if(test, false, else_label)?; self.compile_statements(body)?; self.emit(Instruction::Jump { target: end_label }); @@ -459,7 +454,7 @@ impl Compiler { // if some flag, ignore all assert statements! if self.optimize == 0 { let end_label = self.new_label(); - self.compile_test(test, Some(end_label), None, EvalContext::Statement)?; + self.compile_jump_if(test, true, end_label)?; self.emit(Instruction::LoadName { name: String::from("AssertionError"), scope: bytecode::NameScope::Local, @@ -1006,7 +1001,7 @@ impl Compiler { self.set_label(start_label); - self.compile_test(test, None, Some(else_label), EvalContext::Statement)?; + self.compile_jump_if(test, false, else_label)?; let was_in_loop = self.in_loop; self.in_loop = true; @@ -1118,12 +1113,9 @@ impl Compiler { }); // if comparison result is false, we break with this value; if true, try the next one. - // (CPython compresses these three opcodes into JUMP_IF_FALSE_OR_POP) - self.emit(Instruction::Duplicate); - self.emit(Instruction::JumpIfFalse { + self.emit(Instruction::JumpIfFalseOrPop { target: break_label, }); - self.emit(Instruction::Pop); } // handle the last comparison @@ -1256,66 +1248,120 @@ impl Compiler { self.emit(Instruction::BinaryOperation { op: i, inplace }); } - fn compile_test( + /// Implement boolean short circuit evaluation logic. + /// https://en.wikipedia.org/wiki/Short-circuit_evaluation + /// + /// This means, in a boolean statement 'x and y' the variable y will + /// not be evaluated when x is false. + /// + /// The idea is to jump to a label if the expression is either true or false + /// (indicated by the condition parameter). + fn compile_jump_if( &mut self, expression: &ast::Expression, - true_label: Option