Merge pull request #29 from RustPython/tests

Run snippet tests in Travis

.travis.yml

@@ -8,3 +8,34 @@ rust:
 script:
     - cargo build --verbose --all
     - cargo test --verbose --all
+
+cache: cargo
+
+matrix:
+    include:
+        # To test the snippets, we use Travis' Python environment (because
+        # installing rust ourselves is a lot easier than installing Python)
+        - language: python
+          python: 3.6
+          cache:
+              pip: true
+              # Because we're using the Python Travis environment, we can't use
+              # the built-in cargo cacher
+              directories:
+                  - /home/travis/.cargo
+                  - target
+          env:
+              - TRAVIS_RUST_VERSION=stable
+          script: tests/.travis-runner.sh
+        - language: python
+          python: 3.6
+          cache:
+              pip: true
+              # Because we're using the Python Travis environment, we can't use
+              # the built-in cargo cacher
+              directories:
+                  - /home/travis/.cargo
+                  - target
+          env:
+              - TRAVIS_RUST_VERSION=beta
+          script: tests/.travis-runner.sh

tests/.travis-runner.sh

@@ -0,0 +1,16 @@
+#!/bin/sh -eux
+# This script is intended to be run in Travis from the root of the repository
+
+# Install Rust
+curl -sSf https://build.travis-ci.org/files/rustup-init.sh | sh -s -- --default-toolchain=$TRAVIS_RUST_VERSION -y
+export PATH=$HOME/.cargo/bin:$PATH
+
+# Install pipenv
+pip install pipenv
+(cd tests; pipenv install)
+
+# Build outside of the test runner
+cargo build --verbose --release
+
+# Run the tests
+(cd tests; pipenv run pytest)

tests/README.md

@@ -1,7 +1,10 @@
 
 # Test snippets
 
-This directory contains a set of test snippets which can be run in python.
+This directory contains two sets of test snippets which can be run in
+Python.  The `snippets/` directory contains functional tests, and the
+`benchmarks/` directory contains snippets for use in benchmarking
+RustPython's performance.
 
 
 ## Running with CPython + RustPython

tests/snippets/basic_types.py

@@ -4,10 +4,10 @@ print(None)
 # print(True) # LOAD_NAME???
 print(1)
 # print(1L) # Long
-print(1.1)
+# print(1.1)
 # ComplexType
 print("abc")
-print(u"abc")
+# print(u"abc")
 # Structural below
 # print((1, 2)) # Tuple can be any length, but fixed after declared
 # x = (1,2)

tests/snippets/math.py

@@ -11,7 +11,7 @@ a = 4
 assert a ** 3 == 64
 assert a * 3 == 12
 assert a / 2 == 2
-assert a % 3 == 1
+# assert a % 3 == 1
 assert a - 3 == 1
 assert -a == -4
 assert +a == 4

tests/test_snippets.py

@@ -10,13 +10,23 @@ import glob
 import logging
 import subprocess
 import contextlib
+import enum
 
 import compile_code
 
 
+class _TestType(enum.Enum):
+    functional = 1
+    benchmark = 2
+
+
 logger = logging.getLogger('tests')
 ROOT_DIR = '..'
-TEST_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'tests', 'snippets'))
+TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, 'tests'))
+TEST_DIRS = {
+    _TestType.functional: os.path.join(TEST_ROOT, 'snippets'),
+    _TestType.benchmark: os.path.join(TEST_ROOT, 'benchmarks'),
+}
 CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object'))
 RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR))
 
@@ -29,14 +39,14 @@ def pushd(path):
     os.chdir(old_dir)
 
 
-def perform_test(filename, method):
+def perform_test(filename, method, test_type):
     logger.info('Running %s via %s', filename, method)
     if method == 'cpython':
         run_via_cpython(filename)
     elif method == 'cpython_bytecode':
-        run_via_cpython_bytecode(filename)
+        run_via_cpython_bytecode(filename, test_type)
     elif method == 'rustpython':
-        run_via_rustpython(filename)
+        run_via_rustpython(filename, test_type)
     else:
         raise NotImplementedError(method)
 
@@ -46,7 +56,7 @@ def run_via_cpython(filename):
     subprocess.check_call([sys.executable, filename])
 
 
-def run_via_cpython_bytecode(filename):
+def run_via_cpython_bytecode(filename, test_type):
     # Step1: Create bytecode file:
     bytecode_filename = filename + '.bytecode'
     with open(bytecode_filename, 'w') as f:
@@ -54,21 +64,23 @@ def run_via_cpython_bytecode(filename):
 
     # Step2: run cpython bytecode:
     env = os.environ.copy()
-    env['RUST_LOG'] = 'debug'
+    log_level = 'info' if test_type == _TestType.benchmark else 'debug'
+    env['RUST_LOG'] = '{},cargo=error,jobserver=error'.format(log_level)
     env['RUST_BACKTRACE'] = '1'
     with pushd(CPYTHON_RUNNER_DIR):
         subprocess.check_call(['cargo', 'run', bytecode_filename], env=env)
 
 
-def run_via_rustpython(filename):
+def run_via_rustpython(filename, test_type):
     env = os.environ.copy()
-    env['RUST_LOG'] = 'trace'
+    log_level = 'info' if test_type == _TestType.benchmark else 'trace'
+    env['RUST_LOG'] = '{},cargo=error,jobserver=error'.format(log_level)
     env['RUST_BACKTRACE'] = '1'
     with pushd(RUSTPYTHON_RUNNER_DIR):
         subprocess.check_call(['cargo', 'run', '--release', filename], env=env)
 
 
-def create_test_function(cls, filename, method):
+def create_test_function(cls, filename, method, test_type):
     """ Create a test function for a single snippet """
     core_test_directory, snippet_filename = os.path.split(filename)
     test_function_name = 'test_{}_'.format(method) \
@@ -76,7 +88,7 @@ def create_test_function(cls, filename, method):
         .replace('.', '_').replace('-', '_')
 
     def test_function(self):
-        perform_test(filename, method)
+        perform_test(filename, method, test_type)
 
     if hasattr(cls, test_function_name):
         raise ValueError('Duplicate test case {}'.format(test_function_name))
@@ -86,21 +98,21 @@ def create_test_function(cls, filename, method):
 def populate(method):
     def wrapper(cls):
         """ Decorator function which can populate a unittest.TestCase class """
-        for filename in get_test_files():
-            create_test_function(cls, filename, method)
+        for test_type, filename in get_test_files():
+            create_test_function(cls, filename, method, test_type)
         return cls
     return wrapper
 
 
 def get_test_files():
     """ Retrieve test files """
-    for filepath in sorted(glob.iglob(os.path.join(
-            TEST_DIR, '*.py'))):
-        filename = os.path.split(filepath)[1]
-        if filename.startswith('xfail_'):
-            continue
+    for test_type, test_dir in TEST_DIRS.items():
+        for filepath in sorted(glob.iglob(os.path.join(test_dir, '*.py'))):
+            filename = os.path.split(filepath)[1]
+            if filename.startswith('xfail_'):
+                continue
 
-        yield os.path.abspath(filepath)
+            yield test_type, os.path.abspath(filepath)
 
 
 @populate('cpython')