RustPython/scripts/fix_test.py

"""
An automated script to mark failures in python test suite.
It adds @unittest.expectedFailure to the test functions that are failing in RustPython, but not in CPython.
As well as marking the test with a TODO comment.

How to use:
1. Copy a specific test from the CPython repository to the RustPython repository.
2. Remove all unexpected failures from the test and skip the tests that hang
3. Run python ./scripts/fix_test.py --test test_venv --path ./Lib/test/test_venv.py or equivalent for the test from the project root.
4. Ensure that there are no unexpected successes in the test.
5. Actually fix the test.
"""
import argparse
import ast
import itertools
import platform
from pathlib import Path

def parse_args():
    parser = argparse.ArgumentParser(description="Fix test.")
    parser.add_argument("--path", type=Path, help="Path to test file")
    parser.add_argument("--force", action="store_true", help="Force modification")
    parser.add_argument("--platform", action="store_true", help="Platform specific failure")

    args = parser.parse_args()
    return args

class Test:
    name: str = ""
    path: str = ""
    result: str = ""

    def __str__(self):
        return f"Test(name={self.name}, path={self.path}, result={self.result})"

class TestResult:
    tests_result: str = ""
    tests = []
    stdout = ""

    def __str__(self):
        return f"TestResult(tests_result={self.tests_result},tests={len(self.tests)})"


def parse_results(result):
    lines = result.stdout.splitlines()
    test_results = TestResult()
    test_results.stdout = result.stdout
    in_test_results = False
    for line in lines:
        if line == "Run tests sequentially":
            in_test_results = True
        elif line.startswith("-----------"):
            in_test_results = False
        if in_test_results and not line.startswith("tests") and not line.startswith("["):
            line = line.split(" ")
            if line != [] and len(line) > 3:
                test = Test()
                test.name = line[0]
                test.path = line[1].strip("(").strip(")")
                test.result = " ".join(line[3:]).lower()
                test_results.tests.append(test)
        else:
            if "== Tests result: " in line:
                res = line.split("== Tests result: ")[1]
                res = res.split(" ")[0]
                test_results.tests_result = res
    return test_results

def path_to_test(path) -> list[str]:
    return path.split(".")[2:]

def modify_test(file: str, test: list[str], for_platform: bool = False) -> str:
    a = ast.parse(file)
    lines = file.splitlines()
    fixture = "@unittest.expectedFailure"
    for node in ast.walk(a):
        if isinstance(node, ast.FunctionDef):
            if node.name == test[-1]:
                assert not for_platform
                indent = " " * node.col_offset
                lines.insert(node.lineno - 1, indent + fixture)
                lines.insert(node.lineno - 1, indent + "# TODO: RUSTPYTHON")
                break
    return "\n".join(lines)

def modify_test_v2(file: str, test: list[str], for_platform: bool = False) -> str:
    a = ast.parse(file)
    lines = file.splitlines()
    fixture = "@unittest.expectedFailure"
    for key, node in ast.iter_fields(a):
        if key == "body":
            for i, n in enumerate(node):
                match n:
                    case ast.ClassDef():
                        if len(test) == 2 and test[0] == n.name:
                            # look through body for function def
                            for i, fn in enumerate(n.body):
                                match fn:
                                    case ast.FunctionDef():
                                        if fn.name == test[-1]:
                                            assert not for_platform
                                            indent = " " * fn.col_offset
                                            lines.insert(fn.lineno - 1, indent + fixture)
                                            lines.insert(fn.lineno - 1, indent + "# TODO: RUSTPYTHON")
                                            break
                    case ast.FunctionDef():
                        if n.name == test[0] and len(test) == 1:
                            assert not for_platform
                            indent = " " * n.col_offset
                            lines.insert(n.lineno - 1, indent + fixture)
                            lines.insert(n.lineno - 1, indent + "# TODO: RUSTPYTHON")
                            break
                if i > 500:
                    exit()
    return "\n".join(lines)

def run_test(test_name):
    print(f"Running test: {test_name}")
    rustpython_location = "./target/release/rustpython"
    import subprocess
    result = subprocess.run([rustpython_location, "-m", "test", "-v", test_name], capture_output=True, text=True)
    return parse_results(result)


if __name__ == "__main__":
    args = parse_args()
    test_name = args.path.stem
    tests = run_test(test_name)
    f = open(args.path).read()
    for test in tests.tests:
        if test.result == "fail" or test.result == "error":
            print("Modifying test:", test.name)
            f = modify_test_v2(f, path_to_test(test.path), args.platform)
    with open(args.path, "w") as file:
        # TODO: Find validation method, and make --force override it
        file.write(f)