#!/usr/bin/env python """ Show dependency information for a module. Usage: python scripts/update_lib deps dis python scripts/update_lib deps dataclasses python scripts/update_lib deps dis --depth 2 python scripts/update_lib deps all # Show all modules' dependencies """ import argparse import pathlib import sys sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) def get_all_modules(cpython_prefix: str) -> list[str]: """Get all top-level module names from cpython/Lib/. Includes private modules (_*) that are not hard_deps of other modules. Returns: Sorted list of module names (without .py extension) """ from update_lib.deps import resolve_hard_dep_parent lib_dir = pathlib.Path(cpython_prefix) / "Lib" if not lib_dir.exists(): return [] modules = set() for entry in lib_dir.iterdir(): # Skip hidden files if entry.name.startswith("."): continue # Skip test directory if entry.name == "test": continue if entry.is_file() and entry.suffix == ".py": name = entry.stem elif entry.is_dir() and (entry / "__init__.py").exists(): name = entry.name else: continue # Skip modules that are hard_deps of other modules # e.g., _pydatetime is a hard_dep of datetime, pydoc_data is a hard_dep of pydoc if resolve_hard_dep_parent(name, cpython_prefix) is not None: continue modules.add(name) return sorted(modules) def format_deps_tree( cpython_prefix: str, lib_prefix: str, max_depth: int, *, name: str | None = None, soft_deps: set[str] | None = None, hard_deps: set[str] | None = None, _depth: int = 0, _visited: set[str] | None = None, _indent: str = "", ) -> list[str]: """Format soft dependencies as a tree with up-to-date status. Args: cpython_prefix: CPython directory prefix lib_prefix: Local Lib directory prefix max_depth: Maximum recursion depth name: Module name (used to compute deps if soft_deps not provided) soft_deps: Pre-computed soft dependencies (optional) hard_deps: Hard dependencies to show under the module (root level only) _depth: Current depth (internal) _visited: Already visited modules (internal) _indent: Current indentation (internal) Returns: List of formatted lines """ from update_lib.deps import ( get_lib_paths, get_rust_deps, get_soft_deps, is_up_to_date, ) lines = [] if _visited is None: _visited = set() # Compute deps from name if not provided if soft_deps is None: soft_deps = get_soft_deps(name, cpython_prefix) if name else set() soft_deps = sorted(soft_deps) if not soft_deps and not hard_deps: return lines # Separate up-to-date and outdated modules up_to_date_deps = [] outdated_deps = [] dup_deps = [] for dep in soft_deps: # Skip if library doesn't exist in cpython lib_paths = get_lib_paths(dep, cpython_prefix) if not any(p.exists() for p in lib_paths): continue up_to_date = is_up_to_date(dep, cpython_prefix, lib_prefix) if up_to_date: # Up-to-date modules collected compactly, no dup tracking needed up_to_date_deps.append(dep) elif dep in _visited: # Only track dup for outdated modules dup_deps.append(dep) else: outdated_deps.append(dep) # Show outdated modules with expansion for dep in outdated_deps: dep_native = get_rust_deps(dep, cpython_prefix) native_suffix = ( f" (native: {', '.join(sorted(dep_native))})" if dep_native else "" ) lines.append(f"{_indent}- [ ] {dep}{native_suffix}") _visited.add(dep) # Show hard_deps under this module (only at root level, i.e., when hard_deps is provided) if hard_deps and dep in soft_deps: for hd in sorted(hard_deps): hd_up_to_date = is_up_to_date(hd, cpython_prefix, lib_prefix) hd_marker = "[x]" if hd_up_to_date else "[ ]" lines.append(f"{_indent} - {hd_marker} {hd}") hard_deps = None # Only show once # Recurse if within depth limit if _depth < max_depth - 1: lines.extend( format_deps_tree( cpython_prefix, lib_prefix, max_depth, name=dep, _depth=_depth + 1, _visited=_visited, _indent=_indent + " ", ) ) # Show duplicates compactly (only for outdated) if dup_deps: lines.append(f"{_indent}- [ ] {', '.join(dup_deps)}") # Show up-to-date modules compactly on one line if up_to_date_deps: lines.append(f"{_indent}- [x] {', '.join(up_to_date_deps)}") return lines def format_deps( name: str, cpython_prefix: str, lib_prefix: str, max_depth: int = 10, _visited: set[str] | None = None, ) -> list[str]: """Format all dependency information for a module. Args: name: Module name cpython_prefix: CPython directory prefix lib_prefix: Local Lib directory prefix max_depth: Maximum recursion depth _visited: Shared visited set for deduplication across modules Returns: List of formatted lines """ from update_lib.deps import ( DEPENDENCIES, count_test_todos, find_dependent_tests_tree, get_lib_paths, get_test_paths, is_path_synced, is_test_up_to_date, resolve_hard_dep_parent, ) if _visited is None: _visited = set() lines = [] # Resolve test_ prefix to module (e.g., test_pydoc -> pydoc) if name.startswith("test_"): module_name = name[5:] # strip "test_" lines.append(f"(redirecting {name} -> {module_name})") name = module_name # Resolve hard_dep to parent module (e.g., pydoc_data -> pydoc) parent = resolve_hard_dep_parent(name, cpython_prefix) if parent: lines.append(f"(redirecting {name} -> {parent})") name = parent # lib paths (only show existing) lib_paths = get_lib_paths(name, cpython_prefix) existing_lib_paths = [p for p in lib_paths if p.exists()] for p in existing_lib_paths: synced = is_path_synced(p, cpython_prefix, lib_prefix) marker = "[x]" if synced else "[ ]" lines.append(f"{marker} lib: {p}") # test paths (only show existing) test_paths = get_test_paths(name, cpython_prefix) existing_test_paths = [p for p in test_paths if p.exists()] for p in existing_test_paths: test_name = p.stem if p.is_file() else p.name synced = is_test_up_to_date(test_name, cpython_prefix, lib_prefix) marker = "[x]" if synced else "[ ]" todo_count = count_test_todos(test_name, lib_prefix) todo_suffix = f" (TODO: {todo_count})" if todo_count > 0 else "" lines.append(f"{marker} test: {p}{todo_suffix}") # If no lib or test paths exist, module doesn't exist if not existing_lib_paths and not existing_test_paths: lines.append(f"(module '{name}' not found)") return lines # Collect all hard_deps (explicit from DEPENDENCIES + implicit from lib_paths) dep_info = DEPENDENCIES.get(name, {}) explicit_hard_deps = dep_info.get("hard_deps", []) # Get implicit hard_deps from lib_paths (e.g., _pydecimal.py for decimal) all_hard_deps = set() for hd in explicit_hard_deps: # Remove .py extension if present all_hard_deps.add(hd[:-3] if hd.endswith(".py") else hd) for p in existing_lib_paths: dep_name = p.stem if p.is_file() else p.name if dep_name != name: # Skip the main module itself all_hard_deps.add(dep_name) lines.append("\ndependencies:") lines.extend( format_deps_tree( cpython_prefix, lib_prefix, max_depth, soft_deps={name}, _visited=_visited, hard_deps=all_hard_deps, ) ) # Show dependent tests as tree (depth 2: module + direct importers + their importers) tree = find_dependent_tests_tree(name, lib_prefix=lib_prefix, max_depth=2) lines.extend(_format_dependent_tests_tree(tree, cpython_prefix, lib_prefix)) return lines def _format_dependent_tests_tree( tree: dict, cpython_prefix: str, lib_prefix: str, indent: str = "", ) -> list[str]: """Format dependent tests tree for display.""" from update_lib.deps import is_up_to_date lines = [] module = tree["module"] tests = tree["tests"] children = tree["children"] if indent == "": # Root level # Count total tests in tree def count_tests(t: dict) -> int: total = len(t.get("tests", [])) for c in t.get("children", []): total += count_tests(c) return total total = count_tests(tree) if total == 0 and not children: lines.append(f"\ndependent tests: (no tests depend on {module})") return lines lines.append(f"\ndependent tests: ({total} tests)") # Check if module is up-to-date synced = is_up_to_date(module.split(".")[0], cpython_prefix, lib_prefix) marker = "[x]" if synced else "[ ]" # Format this node if tests: test_str = " ".join(tests) if indent == "": lines.append(f"- {marker} {module}: {test_str}") else: lines.append(f"{indent}- {marker} {module}: {test_str}") elif indent != "" and children: # Has children but no direct tests lines.append(f"{indent}- {marker} {module}:") # Format children child_indent = indent + " " if indent else " " for child in children: lines.extend( _format_dependent_tests_tree( child, cpython_prefix, lib_prefix, child_indent ) ) return lines def _resolve_module_name( name: str, cpython_prefix: str, lib_prefix: str, ) -> list[str]: """Resolve module name through redirects. Returns a list of module names (usually 1, but test support files may expand to multiple). """ import pathlib from update_lib.deps import ( _build_test_import_graph, get_lib_paths, get_test_paths, resolve_hard_dep_parent, resolve_test_to_lib, ) # Resolve test to library group (e.g., test_urllib2 -> urllib) if name.startswith("test_"): lib_group = resolve_test_to_lib(name) if lib_group: return [lib_group] name = name[5:] # Resolve hard_dep to parent parent = resolve_hard_dep_parent(name, cpython_prefix) if parent: return [parent] # Check if it's a valid module lib_paths = get_lib_paths(name, cpython_prefix) test_paths = get_test_paths(name, cpython_prefix) if any(p.exists() for p in lib_paths) or any(p.exists() for p in test_paths): return [name] # Check for test support files (e.g., string_tests -> bytes, str, userstring) test_support_path = pathlib.Path(cpython_prefix) / "Lib" / "test" / f"{name}.py" if test_support_path.exists(): test_dir = pathlib.Path(lib_prefix) / "test" if test_dir.exists(): import_graph, _ = _build_test_import_graph(test_dir) importing_tests = [] for file_key, imports in import_graph.items(): if name in imports and file_key.startswith("test_"): importing_tests.append(file_key) if importing_tests: # Resolve test names to module names (test_bytes -> bytes) return sorted(set(t[5:] for t in importing_tests)) return [name] def show_deps( names: list[str], cpython_prefix: str, lib_prefix: str, max_depth: int = 10, ) -> None: """Show all dependency information for modules.""" # Expand "all" to all module names expanded_names = [] for name in set(names): if name == "all": expanded_names.extend(get_all_modules(cpython_prefix)) else: expanded_names.append(name) # Resolve and deduplicate names (preserving order) seen: set[str] = set() resolved_names: list[str] = [] for name in expanded_names: for resolved in _resolve_module_name(name, cpython_prefix, lib_prefix): if resolved not in seen: seen.add(resolved) resolved_names.append(resolved) # Shared visited set across all modules visited: set[str] = set() for i, name in enumerate(resolved_names): if i > 0: print() # blank line between modules for line in format_deps(name, cpython_prefix, lib_prefix, max_depth, visited): print(line) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "names", nargs="+", help="Module names (e.g., dis, dataclasses) or 'all' for all modules", ) parser.add_argument( "--cpython", default="cpython", help="CPython directory prefix (default: cpython)", ) parser.add_argument( "--lib", default="Lib", help="Local Lib directory prefix (default: Lib)", ) parser.add_argument( "--depth", type=int, default=10, help="Maximum recursion depth for soft_deps tree (default: 10)", ) args = parser.parse_args(argv) # When user does `./update_lib/ deps "foo bar" "baz"` # We still want to get a list of # ["foo", "bar", "baz"], not ["foo bar", "baz"] args.names = [name for names in args.names for name in names.split()] try: show_deps(args.names, args.cpython, args.lib, args.depth) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) return 1 if __name__ == "__main__": sys.exit(main())