Rust-related/RustPython
1
0
Fork 1
mirror of https://github.com/RustPython/RustPython.git synced 2026-06-02 19:39:49 +09:00
Code Issues Packages Projects Releases Wiki Activity

stdlib compatability checking scripts (#5697)

Browse Source 
* stdlib compat checking scripts

Signed-off-by: Ashwin Naren <arihant2math@gmail.com>

* update output

---------

Signed-off-by: Ashwin Naren <arihant2math@gmail.com>
This commit is contained in:
Ashwin Naren
2025-06-26 21:49:53 -07:00
committed by GitHub
parent a18029ee89
commit 5a81533f61
4 changed files with 373 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
scripts/checklist_template.md Normal file
View File

@@ -0,0 +1,21 @@
{% macro display_line(i) %}- {% if i.completed == True %}[x] {% elif i.completed == False %}[ ] {% endif %}{{ i.name }}{% if i.pr != None %} {{ i.pr }}{% endif %}{% endmacro %}
# List of libraries
{% for lib in update_libs %}{{ display_line(lib) }}
{% endfor %}
# List of un-added libraries
These libraries are not added yet. Pure python one will be possible while others are not.
{% for lib in add_libs %}{{ display_line(lib) }}
{% endfor %}
# List of tests without python libraries
{% for lib in update_tests %}{{ display_line(lib) }}
{% endfor %}
# List of un-added tests without python libraries
{% for lib in add_tests %}{{ display_line(lib) }}
{% endfor %}

76
scripts/find_eq.py Normal file
View File

@@ -0,0 +1,76 @@
# Run differential queries to find equivalent files in cpython and rustpython
# Arguments
# --cpython: Path to cpython source code
# --print-diff: Print the diff between the files
# --color: Output color
# --files: Optional globbing pattern to match files in cpython source code
# --checklist: output as checklist
import argparse
import difflib
import pathlib
parser = argparse.ArgumentParser(description="Find equivalent files in cpython and rustpython")
parser.add_argument("--cpython", type=pathlib.Path, required=True, help="Path to cpython source code")
parser.add_argument("--print-diff", action="store_true", help="Print the diff between the files")
parser.add_argument("--color", action="store_true", help="Output color")
parser.add_argument("--files", type=str, default="*.py", help="Optional globbing pattern to match files in cpython source code")
args = parser.parse_args()
if not args.cpython.exists():
raise FileNotFoundError(f"Path {args.cpython} does not exist")
if not args.cpython.is_dir():
raise NotADirectoryError(f"Path {args.cpython} is not a directory")
if not args.cpython.is_absolute():
args.cpython = args.cpython.resolve()
cpython_lib = args.cpython / "Lib"
rustpython_lib = pathlib.Path(__file__).parent.parent / "Lib"
assert rustpython_lib.exists(), "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place"
# walk through the cpython lib directory
cpython_files = []
for path in cpython_lib.rglob(args.files):
if path.is_file():
# remove the cpython lib path from the file path
path = path.relative_to(cpython_lib)
cpython_files.append(path)
for path in cpython_files:
# check if the file exists in the rustpython lib directory
rustpython_path = rustpython_lib / path
if rustpython_path.exists():
# open both files and compare them
try:
with open(cpython_lib / path, "r") as cpython_file:
cpython_code = cpython_file.read()
with open(rustpython_lib / path, "r") as rustpython_file:
rustpython_code = rustpython_file.read()
# compare the files
diff = difflib.unified_diff(cpython_code.splitlines(), rustpython_code.splitlines(), lineterm="", fromfile=str(path), tofile=str(path))
# print the diff if there are differences
diff = list(diff)
if len(diff) > 0:
if args.print_diff:
print("Differences:")
for line in diff:
print(line)
else:
print(f"File is not identical: {path}")
else:
print(f"File is identical: {path}")
except Exception as e:
print(f"Unable to check file {path}: {e}")
else:
print(f"File not found in RustPython: {path}")
# check for files in rustpython lib directory that are not in cpython lib directory
rustpython_files = []
for path in rustpython_lib.rglob(args.files):
if path.is_file():
# remove the rustpython lib path from the file path
path = path.relative_to(rustpython_lib)
rustpython_files.append(path)
if path not in cpython_files:
print(f"File not found in CPython: {path}")

228
scripts/generate_checklist.py Normal file
View File

@@ -0,0 +1,228 @@
# Arguments
# --cpython: Path to cpython source code
# --updated-libs: Libraries that have been updated in RustPython
import argparse
import dataclasses
import difflib
import pathlib
from typing import Optional
import warnings
import requests
from jinja2 import Environment, FileSystemLoader
parser = argparse.ArgumentParser(description="Find equivalent files in cpython and rustpython")
parser.add_argument("--cpython", type=pathlib.Path, required=True, help="Path to cpython source code")
parser.add_argument("--notes", type=pathlib.Path, required=False, help="Path to notes file")
args = parser.parse_args()
def check_pr(pr_id: str) -> bool:
if pr_id.startswith("#"):
pr_id = pr_id[1:]
int_pr_id = int(pr_id)
req = f"https://api.github.com/repos/RustPython/RustPython/pulls/{int_pr_id}"
response = requests.get(req).json()
return response["merged_at"] is not None
@dataclasses.dataclass
class LibUpdate:
pr: Optional[str] = None
done: bool = True
def parse_updated_lib_issue(issue_body: str) -> dict[str, LibUpdate]:
lines = issue_body.splitlines()
updated_libs = {}
for line in lines:
if line.strip().startswith("- "):
line = line.strip()[2:]
out = line.split(" ")
out = [x for x in out if x]
assert len(out) < 3
if len(out) == 1:
updated_libs[out[0]] = LibUpdate()
elif len(out) == 2:
updated_libs[out[0]] = LibUpdate(out[1], check_pr(out[1]))
return updated_libs
def get_updated_libs() -> dict[str, LibUpdate]:
issue_id = "5736"
req = f"https://api.github.com/repos/RustPython/RustPython/issues/{issue_id}"
response = requests.get(req).json()
return parse_updated_lib_issue(response["body"])
updated_libs = get_updated_libs()
if not args.cpython.exists():
raise FileNotFoundError(f"Path {args.cpython} does not exist")
if not args.cpython.is_dir():
raise NotADirectoryError(f"Path {args.cpython} is not a directory")
if not args.cpython.is_absolute():
args.cpython = args.cpython.resolve()
notes: dict = {}
if args.notes:
# check if the file exists in the rustpython lib directory
notes_path = args.notes
if notes_path.exists():
with open(notes_path) as f:
for line in f:
line = line.strip()
if not line.startswith("//") and line:
line_split = line.split(" ")
if len(line_split) > 1:
rest = " ".join(line_split[1:])
if line_split[0] in notes:
notes[line_split[0]].append(rest)
else:
notes[line_split[0]] = [rest]
else:
raise ValueError(f"Invalid note: {line}")
else:
raise FileNotFoundError(f"Path {notes_path} does not exist")
cpython_lib = args.cpython / "Lib"
rustpython_lib = pathlib.Path(__file__).parent.parent / "Lib"
assert rustpython_lib.exists(), "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place"
ignored_objs = [
"__pycache__",
"test"
]
# loop through the top-level directories in the cpython lib directory
libs = []
for path in cpython_lib.iterdir():
if path.is_dir() and path.name not in ignored_objs:
# add the directory name to the list of libraries
libs.append(path.name)
elif path.is_file() and path.name.endswith(".py") and path.name not in ignored_objs:
# add the file name to the list of libraries
libs.append(path.name)
tests = []
cpython_lib_test = cpython_lib / "test"
for path in cpython_lib_test.iterdir():
if path.is_dir() and path.name not in ignored_objs and path.name.startswith("test_"):
# add the directory name to the list of libraries
tests.append(path.name)
elif path.is_file() and path.name.endswith(".py") and path.name not in ignored_objs and path.name.startswith("test_"):
# add the file name to the list of libraries
file_name = path.name.replace("test_", "")
if file_name not in libs and file_name.replace(".py", "") not in libs:
tests.append(path.name)
def check_diff(file1, file2):
try:
with open(file1, "r") as f1, open(file2, "r") as f2:
f1_lines = f1.readlines()
f2_lines = f2.readlines()
diff = difflib.unified_diff(f1_lines, f2_lines, lineterm="")
diff_lines = list(diff)
return len(diff_lines)
except UnicodeDecodeError:
return False
def check_completion_pr(display_name):
for lib in updated_libs:
if lib == str(display_name):
return updated_libs[lib].done, updated_libs[lib].pr
return False, None
def check_test_completion(rustpython_path, cpython_path):
if rustpython_path.exists() and rustpython_path.is_file():
if cpython_path.exists() and cpython_path.is_file():
if not rustpython_path.exists() or not rustpython_path.is_file():
return False
elif check_diff(rustpython_path, cpython_path) > 0:
return False
return True
return False
def check_lib_completion(rustpython_path, cpython_path):
test_name = "test_" + rustpython_path.name
rustpython_test_path = rustpython_lib / "test" / test_name
cpython_test_path = cpython_lib / "test" / test_name
if cpython_test_path.exists() and not check_test_completion(rustpython_test_path, cpython_test_path):
return False
if rustpython_path.exists() and rustpython_path.is_file():
if check_diff(rustpython_path, cpython_path) > 0:
return False
return True
return False
def handle_notes(display_path) -> list[str]:
if str(display_path) in notes:
res = notes[str(display_path)]
# remove the note from the notes list
del notes[str(display_path)]
return res
return []
@dataclasses.dataclass
class Output:
name: str
pr: Optional[str]
completed: Optional[bool]
notes: list[str]
update_libs_output = []
add_libs_output = []
for path in libs:
# check if the file exists in the rustpython lib directory
rustpython_path = rustpython_lib / path
# remove the file extension if it exists
display_path = pathlib.Path(path).with_suffix("")
(completed, pr) = check_completion_pr(display_path)
if rustpython_path.exists():
if not completed:
# check if the file exists in the cpython lib directory
cpython_path = cpython_lib / path
# check if the file exists in the rustpython lib directory
if rustpython_path.exists() and rustpython_path.is_file():
completed = check_lib_completion(rustpython_path, cpython_path)
update_libs_output.append(Output(str(display_path), pr, completed, handle_notes(display_path)))
else:
if pr is not None and completed:
update_libs_output.append(Output(str(display_path), pr, None, handle_notes(display_path)))
else:
add_libs_output.append(Output(str(display_path), pr, None, handle_notes(display_path)))
update_tests_output = []
add_tests_output = []
for path in tests:
# check if the file exists in the rustpython lib directory
rustpython_path = rustpython_lib / "test" / path
# remove the file extension if it exists
display_path = pathlib.Path(path).with_suffix("")
(completed, pr) = check_completion_pr(display_path)
if rustpython_path.exists():
if not completed:
# check if the file exists in the cpython lib directory
cpython_path = cpython_lib / "test" / path
# check if the file exists in the rustpython lib directory
if rustpython_path.exists() and rustpython_path.is_file():
completed = check_lib_completion(rustpython_path, cpython_path)
update_tests_output.append(Output(str(display_path), pr, completed, handle_notes(display_path)))
else:
if pr is not None and completed:
update_tests_output.append(Output(str(display_path), pr, None, handle_notes(display_path)))
else:
add_tests_output.append(Output(str(display_path), pr, None, handle_notes(display_path)))
for note in notes:
# add a warning for each note that is not attached to a file
for n in notes[note]:
warnings.warn(f"Unattached Note: {note} - {n}")
env = Environment(loader=FileSystemLoader('.'))
template = env.get_template("checklist_template.md")
output = template.render(
update_libs=update_libs_output,
add_libs=add_libs_output,
update_tests=update_tests_output,
add_tests=add_tests_output
)
print(output)

48
scripts/notes.txt Normal file
View File

@@ -0,0 +1,48 @@
__future__ Related test is `test_future_stmt`
abc `_collections_abc.py`
abc `_py_abc.py`
code Related test is `test_code_module`
codecs `_pycodecs.py`
collections See also #3418
ctypes #5572
datetime `_pydatetime.py`
decimal `_pydecimal.py`
dis See also #3846
importlib #4565
io `_pyio.py`
io #3960
io #4702
locale #3850
mailbox #4072
multiprocessing #3965
os Blocker: Some tests requires async comprehension
os #3960
os #4053
pickle #3876
pickle `_compat_pickle.py`
pickle `test/pickletester.py` supports `test_pickle.py`
pickle `test/test_picklebuffer.py`
pydoc `pydoc_data`
queue See also #3608
re Don't forget sre files `sre_compile.py`, `sre_constants.py`, `sre_parse.py`
shutil #3960
site Don't forget `_sitebuiltins.py`
venv #3960
warnings #4013
// test
test_array #3876
test_gc #4158
test_marshal #3458
test_mmap #3847
test_posix #4496
test_property #3430
test_set #3992
test_structseq #4063
test_super #3865
test_support #4538
test_syntax #4469
test_sys #4541
test_time #3850
test_time #4157