From 31bdfe2cd730e6ade88d1628b5b482dba18fb9ad Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Sat, 4 Aug 2018 11:31:48 -0400 Subject: [PATCH 1/6] Move performance snippets to a separate directory --- tests/README.md | 5 ++++- tests/{snippets => benchmarks}/perf_add.py | 0 tests/{snippets => benchmarks}/perf_fib.py | 0 3 files changed, 4 insertions(+), 1 deletion(-) rename tests/{snippets => benchmarks}/perf_add.py (100%) rename tests/{snippets => benchmarks}/perf_fib.py (100%) diff --git a/tests/README.md b/tests/README.md index f394a4493..a225ba3b6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,7 +1,10 @@ # Test snippets -This directory contains a set of test snippets which can be run in python. +This directory contains two sets of test snippets which can be run in +Python. The `snippets/` directory contains functional tests, and the +`benchmarks/` directory contains snippets for use in benchmarking +RustPython's performance. ## Running with CPython + RustPython diff --git a/tests/snippets/perf_add.py b/tests/benchmarks/perf_add.py similarity index 100% rename from tests/snippets/perf_add.py rename to tests/benchmarks/perf_add.py diff --git a/tests/snippets/perf_fib.py b/tests/benchmarks/perf_fib.py similarity index 100% rename from tests/snippets/perf_fib.py rename to tests/benchmarks/perf_fib.py From 402ca31423eaf1af4188440bd5bf01572c596380 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Sat, 4 Aug 2018 11:36:22 -0400 Subject: [PATCH 2/6] Suppress cargo/jobserver log messages in snippet testing They make the output unmanageably large, and don't provide any useful information for debugging. --- tests/test_snippets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_snippets.py b/tests/test_snippets.py index d855d0c61..96c6548a4 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -54,7 +54,7 @@ def run_via_cpython_bytecode(filename): # Step2: run cpython bytecode: env = os.environ.copy() - env['RUST_LOG'] = 'debug' + env['RUST_LOG'] = 'debug,cargo=error,jobserver=error' env['RUST_BACKTRACE'] = '1' with pushd(CPYTHON_RUNNER_DIR): subprocess.check_call(['cargo', 'run', bytecode_filename], env=env) @@ -62,7 +62,7 @@ def run_via_cpython_bytecode(filename): def run_via_rustpython(filename): env = os.environ.copy() - env['RUST_LOG'] = 'trace' + env['RUST_LOG'] = 'trace,cargo=error,jobserver=error' env['RUST_BACKTRACE'] = '1' with pushd(RUSTPYTHON_RUNNER_DIR): subprocess.check_call(['cargo', 'run', '--release', filename], env=env) From c857d7574865453f87648cd12f2cf8a2b0204de3 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Sat, 4 Aug 2018 11:49:16 -0400 Subject: [PATCH 3/6] Restore running of benchmarks in test_snippets.py This distinguishes them from regular snippet tests, and runs them without trace logging (which was causing major slowdowns previously). --- tests/test_snippets.py | 48 ++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 96c6548a4..7805cba4e 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -10,13 +10,23 @@ import glob import logging import subprocess import contextlib +import enum import compile_code +class _TestType(enum.Enum): + functional = 1 + benchmark = 2 + + logger = logging.getLogger('tests') ROOT_DIR = '..' -TEST_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'tests', 'snippets')) +TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, 'tests')) +TEST_DIRS = { + _TestType.functional: os.path.join(TEST_ROOT, 'snippets'), + _TestType.benchmark: os.path.join(TEST_ROOT, 'benchmarks'), +} CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object')) RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) @@ -29,14 +39,14 @@ def pushd(path): os.chdir(old_dir) -def perform_test(filename, method): +def perform_test(filename, method, test_type): logger.info('Running %s via %s', filename, method) if method == 'cpython': run_via_cpython(filename) elif method == 'cpython_bytecode': - run_via_cpython_bytecode(filename) + run_via_cpython_bytecode(filename, test_type) elif method == 'rustpython': - run_via_rustpython(filename) + run_via_rustpython(filename, test_type) else: raise NotImplementedError(method) @@ -46,7 +56,7 @@ def run_via_cpython(filename): subprocess.check_call([sys.executable, filename]) -def run_via_cpython_bytecode(filename): +def run_via_cpython_bytecode(filename, test_type): # Step1: Create bytecode file: bytecode_filename = filename + '.bytecode' with open(bytecode_filename, 'w') as f: @@ -54,21 +64,23 @@ def run_via_cpython_bytecode(filename): # Step2: run cpython bytecode: env = os.environ.copy() - env['RUST_LOG'] = 'debug,cargo=error,jobserver=error' + log_level = 'info' if test_type == _TestType.benchmark else 'debug' + env['RUST_LOG'] = '{},cargo=error,jobserver=error'.format(log_level) env['RUST_BACKTRACE'] = '1' with pushd(CPYTHON_RUNNER_DIR): subprocess.check_call(['cargo', 'run', bytecode_filename], env=env) -def run_via_rustpython(filename): +def run_via_rustpython(filename, test_type): env = os.environ.copy() - env['RUST_LOG'] = 'trace,cargo=error,jobserver=error' + log_level = 'info' if test_type == _TestType.benchmark else 'trace' + env['RUST_LOG'] = '{},cargo=error,jobserver=error'.format(log_level) env['RUST_BACKTRACE'] = '1' with pushd(RUSTPYTHON_RUNNER_DIR): subprocess.check_call(['cargo', 'run', '--release', filename], env=env) -def create_test_function(cls, filename, method): +def create_test_function(cls, filename, method, test_type): """ Create a test function for a single snippet """ core_test_directory, snippet_filename = os.path.split(filename) test_function_name = 'test_{}_'.format(method) \ @@ -76,7 +88,7 @@ def create_test_function(cls, filename, method): .replace('.', '_').replace('-', '_') def test_function(self): - perform_test(filename, method) + perform_test(filename, method, test_type) if hasattr(cls, test_function_name): raise ValueError('Duplicate test case {}'.format(test_function_name)) @@ -86,21 +98,21 @@ def create_test_function(cls, filename, method): def populate(method): def wrapper(cls): """ Decorator function which can populate a unittest.TestCase class """ - for filename in get_test_files(): - create_test_function(cls, filename, method) + for test_type, filename in get_test_files(): + create_test_function(cls, filename, method, test_type) return cls return wrapper def get_test_files(): """ Retrieve test files """ - for filepath in sorted(glob.iglob(os.path.join( - TEST_DIR, '*.py'))): - filename = os.path.split(filepath)[1] - if filename.startswith('xfail_'): - continue + for test_type, test_dir in TEST_DIRS.items(): + for filepath in sorted(glob.iglob(os.path.join(test_dir, '*.py'))): + filename = os.path.split(filepath)[1] + if filename.startswith('xfail_'): + continue - yield os.path.abspath(filepath) + yield test_type, os.path.abspath(filepath) @populate('cpython') From 9e8c5fb960dcde0d42251141c5c62cff95c335d7 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Sat, 4 Aug 2018 10:34:29 -0400 Subject: [PATCH 4/6] First attempt to run snippet tests in Travis --- .travis.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.travis.yml b/.travis.yml index 19a7268e0..51da0a9d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,3 +8,29 @@ rust: script: - cargo build --verbose --all - cargo test --verbose --all + +cache: cargo + +matrix: + include: + # To test the snippets, we use Travis' Python environment (because + # installing rust ourselves is a lot easier than installing Python) + - language: python + python: 3.6 + cache: + pip: true + # Because we're using the Python Travis environment, we can't use + # the built-in cargo cacher + directories: + - /home/travis/.cargo + - target + before_script: + - curl -sSf https://build.travis-ci.org/files/rustup-init.sh | sh -s -- --default-toolchain=stable -y + - export PATH=$HOME/.cargo/bin:$PATH + install: + - pip install pipenv + - (cd tests; pipenv install) + script: + # Build outside of the test runner + - cargo build --verbose --release + - (cd tests; pipenv run pytest) From a9d6c8db873a36f905199429494edde051825dbe Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Sat, 4 Aug 2018 13:42:00 -0400 Subject: [PATCH 5/6] Run snippet tests for stable and beta Rusts --- .travis.yml | 25 +++++++++++++++---------- tests/.travis-runner.sh | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) create mode 100755 tests/.travis-runner.sh diff --git a/.travis.yml b/.travis.yml index 51da0a9d6..d99dadcf5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,13 +24,18 @@ matrix: directories: - /home/travis/.cargo - target - before_script: - - curl -sSf https://build.travis-ci.org/files/rustup-init.sh | sh -s -- --default-toolchain=stable -y - - export PATH=$HOME/.cargo/bin:$PATH - install: - - pip install pipenv - - (cd tests; pipenv install) - script: - # Build outside of the test runner - - cargo build --verbose --release - - (cd tests; pipenv run pytest) + env: + - TRAVIS_RUST_VERSION=stable + script: tests/.travis-runner.sh + - language: python + python: 3.6 + cache: + pip: true + # Because we're using the Python Travis environment, we can't use + # the built-in cargo cacher + directories: + - /home/travis/.cargo + - target + env: + - TRAVIS_RUST_VERSION=beta + script: tests/.travis-runner.sh diff --git a/tests/.travis-runner.sh b/tests/.travis-runner.sh new file mode 100755 index 000000000..680b18ea4 --- /dev/null +++ b/tests/.travis-runner.sh @@ -0,0 +1,16 @@ +#!/bin/sh -eux +# This script is intended to be run in Travis from the root of the repository + +# Install Rust +curl -sSf https://build.travis-ci.org/files/rustup-init.sh | sh -s -- --default-toolchain=$TRAVIS_RUST_VERSION -y +export PATH=$HOME/.cargo/bin:$PATH + +# Install pipenv +pip install pipenv +(cd tests; pipenv install) + +# Build outside of the test runner +cargo build --verbose --release + +# Run the tests +(cd tests; pipenv run pytest) From dc633ef8188eca01437337953943eae4e4667c5a Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Sat, 4 Aug 2018 13:55:23 -0400 Subject: [PATCH 6/6] Comment or move all currently-failing snippet tests This allows us to make the snippet tests merge-blockers, whilst improving over time. --- tests/{snippets => not_yet_working}/append.py | 0 tests/{snippets => not_yet_working}/cast.py | 0 tests/{snippets => not_yet_working}/floats.py | 0 tests/{snippets => not_yet_working}/function_args.py | 0 tests/{benchmarks => not_yet_working}/perf_fib.py | 0 tests/snippets/basic_types.py | 4 ++-- tests/snippets/math.py | 2 +- 7 files changed, 3 insertions(+), 3 deletions(-) rename tests/{snippets => not_yet_working}/append.py (100%) rename tests/{snippets => not_yet_working}/cast.py (100%) rename tests/{snippets => not_yet_working}/floats.py (100%) rename tests/{snippets => not_yet_working}/function_args.py (100%) rename tests/{benchmarks => not_yet_working}/perf_fib.py (100%) diff --git a/tests/snippets/append.py b/tests/not_yet_working/append.py similarity index 100% rename from tests/snippets/append.py rename to tests/not_yet_working/append.py diff --git a/tests/snippets/cast.py b/tests/not_yet_working/cast.py similarity index 100% rename from tests/snippets/cast.py rename to tests/not_yet_working/cast.py diff --git a/tests/snippets/floats.py b/tests/not_yet_working/floats.py similarity index 100% rename from tests/snippets/floats.py rename to tests/not_yet_working/floats.py diff --git a/tests/snippets/function_args.py b/tests/not_yet_working/function_args.py similarity index 100% rename from tests/snippets/function_args.py rename to tests/not_yet_working/function_args.py diff --git a/tests/benchmarks/perf_fib.py b/tests/not_yet_working/perf_fib.py similarity index 100% rename from tests/benchmarks/perf_fib.py rename to tests/not_yet_working/perf_fib.py diff --git a/tests/snippets/basic_types.py b/tests/snippets/basic_types.py index 34c35e117..024658c91 100644 --- a/tests/snippets/basic_types.py +++ b/tests/snippets/basic_types.py @@ -4,10 +4,10 @@ print(None) # print(True) # LOAD_NAME??? print(1) # print(1L) # Long -print(1.1) +# print(1.1) # ComplexType print("abc") -print(u"abc") +# print(u"abc") # Structural below # print((1, 2)) # Tuple can be any length, but fixed after declared # x = (1,2) diff --git a/tests/snippets/math.py b/tests/snippets/math.py index fb71da752..92fee575e 100644 --- a/tests/snippets/math.py +++ b/tests/snippets/math.py @@ -11,7 +11,7 @@ a = 4 assert a ** 3 == 64 assert a * 3 == 12 assert a / 2 == 2 -assert a % 3 == 1 +# assert a % 3 == 1 assert a - 3 == 1 assert -a == -4 assert +a == 4