Merge pull request #3158 from RustPython/pip-warnings

Miscellaneous pip fixes

Cargo.lock

@@ -1749,7 +1749,6 @@ dependencies = [
  "python3-sys",
  "rustpython-compiler",
  "rustpython-parser",
- "rustpython-pylib",
  "rustpython-stdlib",
  "rustpython-vm",
  "rustyline",

Cargo.toml

@@ -22,6 +22,7 @@ flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
 freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
 threading = ["rustpython-vm/threading"]
+pylib = ["rustpython-vm/pylib"]
 zlib = ["rustpython-stdlib/zlib"]
 ssl = ["rustpython-stdlib/ssl"]
 ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
@@ -34,7 +35,6 @@ rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" }
 rustpython-parser = { path = "parser", version = "0.1.1" }
 rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
 rustpython-stdlib = {path = "stdlib", optional = true, default-features = false, features = ["compile-parse"]}
-pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1.0", optional = true }
 dirs = { package = "dirs-next", version = "2.0.0" }
 num-traits = "0.2.8"
 cfg-if = "1.0"
@@ -66,6 +66,10 @@ path = "src/main.rs"
 [profile.dev.package."*"]
 opt-level = 3
 
+[profile.test]
+opt-level = 3
+lto = "thin"
+
 [profile.bench]
 lto = true
 codegen-units = 1

Lib/distutils/command/install.py

@@ -78,6 +78,11 @@ if HAS_USER_SITE:
         'data'   : '$userbase',
         }
 
+# XXX RUSTPYTHON: replace python with rustpython in all these paths
+for group in INSTALL_SCHEMES.values():
+    for key in group.keys():
+        group[key] = group[key].replace("Python", "RustPython").replace("python", "rustpython")
+
 # The keys to an installation scheme; if any new types of files are to be
 # installed, be sure to add an entry to every installation scheme above,
 # and to SCHEME_KEYS here.

Lib/distutils/sysconfig.py

@@ -13,7 +13,6 @@ import _imp
 import os
 import re
 import sys
-import fnmatch
 
 from .errors import DistutilsPlatformError
 
@@ -24,36 +23,47 @@ BASE_PREFIX = os.path.normpath(sys.base_prefix)
 BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix)
 
 # Path to the base directory of the project. On Windows the binary may
-# live in project/PCBuild/win32 or project/PCBuild/amd64.
+# live in project/PCbuild/win32 or project/PCbuild/amd64.
 # set for cross builds
 if "_PYTHON_PROJECT_BASE" in os.environ:
     project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"])
 else:
-    project_base = os.path.dirname(os.path.abspath(sys.executable))
-if (os.name == 'nt' and
-    project_base.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))):
-    project_base = os.path.dirname(os.path.dirname(project_base))
+    if sys.executable:
+        project_base = os.path.dirname(os.path.abspath(sys.executable))
+    else:
+        # sys.executable can be empty if argv[0] has been changed and Python is
+        # unable to retrieve the real program name
+        project_base = os.getcwd()
+
 
 # python_build: (Boolean) if true, we're either building Python or
 # building an extension with an un-installed Python, so we use
 # different (hard-wired) directories.
-# Setup.local is available for Makefile builds including VPATH builds,
-# Setup.dist is available on Windows
 def _is_python_source_dir(d):
-    for fn in ("Setup.dist", "Setup.local"):
+    for fn in ("Setup", "Setup.local"):
         if os.path.isfile(os.path.join(d, "Modules", fn)):
             return True
     return False
+
 _sys_home = getattr(sys, '_home', None)
-if (_sys_home and os.name == 'nt' and
-    _sys_home.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))):
-    _sys_home = os.path.dirname(os.path.dirname(_sys_home))
+
+if os.name == 'nt':
+    def _fix_pcbuild(d):
+        if d and os.path.normcase(d).startswith(
+                os.path.normcase(os.path.join(PREFIX, "PCbuild"))):
+            return PREFIX
+        return d
+    project_base = _fix_pcbuild(project_base)
+    _sys_home = _fix_pcbuild(_sys_home)
+
 def _python_build():
     if _sys_home:
         return _is_python_source_dir(_sys_home)
     return _is_python_source_dir(project_base)
+
 python_build = _python_build()
 
+
 # Calculate the build qualifier flags if they are defined.  Adding the flags
 # to the include and lib directories only makes sense for an installation, not
 # an in-source build.
@@ -100,11 +110,13 @@ def get_python_inc(plat_specific=0, prefix=None):
                 incdir = os.path.join(get_config_var('srcdir'), 'Include')
                 return os.path.normpath(incdir)
         python_dir = 'python' + get_python_version() + build_flags
-        if not python_build and plat_specific:
-            import sysconfig
-            return sysconfig.get_path('platinclude')
         return os.path.join(prefix, "include", python_dir)
     elif os.name == "nt":
+        if python_build:
+            # Include both the include and PC dir to ensure we can find
+            # pyconfig.h
+            return (os.path.join(prefix, "include") + os.path.pathsep +
+                    os.path.join(prefix, "PC"))
         return os.path.join(prefix, "include")
     else:
         raise DistutilsPlatformError(
@@ -126,7 +138,6 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
     If 'prefix' is supplied, use it instead of sys.base_prefix or
     sys.base_exec_prefix -- i.e., ignore 'plat_specific'.
     """
-    is_default_prefix = not prefix or os.path.normpath(prefix) in ('/usr', '/usr/local')
     if prefix is None:
         if standard_lib:
             prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
@@ -134,16 +145,18 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
             prefix = plat_specific and EXEC_PREFIX or PREFIX
 
     if os.name == "posix":
-        libpython = os.path.join(prefix,
-                                 "lib", "python" + get_python_version())
+        if plat_specific or standard_lib:
+            # Platform-specific modules (any module from a non-pure-Python
+            # module distribution) or standard Python library modules.
+            libdir = sys.platlibdir
+        else:
+            # Pure Python
+            libdir = "lib"
+        libpython = os.path.join(prefix, libdir,
+                                 # XXX RUSTPYTHON: changed from python->rustpython
+                                 "rustpython" + get_python_version())
         if standard_lib:
             return libpython
-        elif (is_default_prefix and
-              'PYTHONUSERBASE' not in os.environ and
-              'VIRTUAL_ENV' not in os.environ and
-              'real_prefix' not in sys.__dict__ and
-              sys.prefix == sys.base_prefix):
-            return os.path.join(prefix, "lib", "python3", "dist-packages")
         else:
             return os.path.join(libpython, "site-packages")
     elif os.name == "nt":
@@ -181,11 +194,9 @@ def customize_compiler(compiler):
                 _osx_support.customize_compiler(_config_vars)
                 _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
 
-        (cc, cxx, opt, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags,
-         configure_cppflags, configure_cflags, configure_ldflags) = \
-            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
-                            'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS',
-                            'CONFIGURE_CPPFLAGS', 'CONFIGURE_CFLAGS', 'CONFIGURE_LDFLAGS')
+        (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS')
 
         if 'CC' in os.environ:
             newcc = os.environ['CC']
@@ -198,10 +209,6 @@ def customize_compiler(compiler):
             cc = newcc
         if 'CXX' in os.environ:
             cxx = os.environ['CXX']
-        if fnmatch.filter([cc, cxx], '*-4.[0-8]'):
-            configure_cflags = configure_cflags.replace('-fstack-protector-strong', '-fstack-protector')
-            ldshared = ldshared.replace('-fstack-protector-strong', '-fstack-protector')
-            cflags = cflags.replace('-fstack-protector-strong', '-fstack-protector')
         if 'LDSHARED' in os.environ:
             ldshared = os.environ['LDSHARED']
         if 'CPP' in os.environ:
@@ -210,22 +217,13 @@ def customize_compiler(compiler):
             cpp = cc + " -E"           # not always
         if 'LDFLAGS' in os.environ:
             ldshared = ldshared + ' ' + os.environ['LDFLAGS']
-        elif configure_ldflags:
-            ldshared = ldshared + ' ' + configure_ldflags
         if 'CFLAGS' in os.environ:
-            cflags = opt + ' ' + os.environ['CFLAGS']
+            cflags = cflags + ' ' + os.environ['CFLAGS']
             ldshared = ldshared + ' ' + os.environ['CFLAGS']
-        elif configure_cflags:
-            cflags = opt + ' ' + configure_cflags
-            ldshared = ldshared + ' ' + configure_cflags
         if 'CPPFLAGS' in os.environ:
             cpp = cpp + ' ' + os.environ['CPPFLAGS']
             cflags = cflags + ' ' + os.environ['CPPFLAGS']
             ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
-        elif configure_cppflags:
-            cpp = cpp + ' ' + configure_cppflags
-            cflags = cflags + ' ' + configure_cppflags
-            ldshared = ldshared + ' ' + configure_cppflags
         if 'AR' in os.environ:
             ar = os.environ['AR']
         if 'ARFLAGS' in os.environ:

Lib/test/test_zipfile.py

@@ -1605,8 +1605,6 @@ class OtherTests(unittest.TestCase):
             self.assertEqual(zf.filelist[0].filename, "foo.txt")
             self.assertEqual(zf.filelist[1].filename, "\xf6.txt")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_read_after_write_unicode_filenames(self):
         with zipfile.ZipFile(TESTFN2, 'w') as zipfp:
             zipfp.writestr('приклад', b'sample')
@@ -1815,8 +1813,6 @@ class OtherTests(unittest.TestCase):
         self.assertEqual(zipfile.sizeEndCentDir64, 56)
         self.assertEqual(zipfile.sizeEndCentDir64Locator, 20)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_comments(self):
         """Check that comments on the archive are handled properly."""
 
@@ -2872,7 +2868,7 @@ class TestPath(unittest.TestCase):
             a, b, g = root.iterdir()
             with a.open() as strm:
                 data = strm.read()
-            assert data == b"content of a"
+            assert data == "content of a"
 
     def test_read(self):
         for alpharep in self.zipfile_alpharep():

Lib/tokenize.py

@@ -24,7 +24,10 @@ __author__ = 'Ka-Ping Yee <ping@lfw.org>'
 __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
                'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
                'Michael Foord')
-from builtins import open as _builtin_open
+try:
+    from builtins import open as _builtin_open
+except ImportError:
+    pass
 from codecs import lookup, BOM_UTF8
 import collections
 from io import TextIOWrapper

Lib/zipfile.py

@@ -4,7 +4,6 @@ Read and write ZIP files.
 XXX references to utf-8 need further investigation.
 """
 import binascii
-import functools
 import importlib.util
 import io
 import itertools
@@ -26,7 +25,6 @@ except ImportError:
     import _dummy_thread as threading
 import time
 import contextlib
-from collections import OrderedDict
 
 try:
     import zlib # We may need its compression method
@@ -47,7 +45,8 @@ except ImportError:
 
 __all__ = ["BadZipFile", "BadZipfile", "error",
            "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA",
-           "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile"]
+           "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile",
+           "Path"]
 
 class BadZipFile(Exception):
     pass
@@ -387,11 +386,11 @@ class ZipInfo (object):
         self.volume = 0                 # Volume number of file header
         self.internal_attr = 0          # Internal attributes
         self.external_attr = 0          # External file attributes
+        self.compress_size = 0          # Size of the compressed file
+        self.file_size = 0              # Size of the uncompressed file
         # Other attributes are set by class ZipFile:
         # header_offset         Byte offset to the file header
         # CRC                   CRC-32 of the uncompressed file
-        # compress_size         Size of the compressed file
-        # file_size             Size of the uncompressed file
 
     def __repr__(self):
         result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)]
@@ -476,44 +475,23 @@ class ZipInfo (object):
             if ln+4 > len(extra):
                 raise BadZipFile("Corrupt extra field %04x (size=%d)" % (tp, ln))
             if tp == 0x0001:
-                if ln >= 24:
-                    counts = unpack('<QQQ', extra[4:28])
-                elif ln == 16:
-                    counts = unpack('<QQ', extra[4:20])
-                elif ln == 8:
-                    counts = unpack('<Q', extra[4:12])
-                elif ln == 0:
-                    counts = ()
-                else:
-                    raise BadZipFile("Corrupt extra field %04x (size=%d)" % (tp, ln))
-
-                idx = 0
-
+                data = extra[4:ln+4]
                 # ZIP64 extension (large files and/or large archives)
-                if self.file_size in (0xffffffffffffffff, 0xffffffff):
-                    if len(counts) <= idx:
-                        raise BadZipFile(
-                            "Corrupt zip64 extra field. File size not found."
-                        )
-                    self.file_size = counts[idx]
-                    idx += 1
-
-                if self.compress_size == 0xFFFFFFFF:
-                    if len(counts) <= idx:
-                        raise BadZipFile(
-                            "Corrupt zip64 extra field. Compress size not found."
-                        )
-                    self.compress_size = counts[idx]
-                    idx += 1
-
-                if self.header_offset == 0xffffffff:
-                    if len(counts) <= idx:
-                        raise BadZipFile(
-                            "Corrupt zip64 extra field. Header offset not found."
-                        )
-                    old = self.header_offset
-                    self.header_offset = counts[idx]
-                    idx+=1
+                try:
+                    if self.file_size in (0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF):
+                        field = "File size"
+                        self.file_size, = unpack('<Q', data[:8])
+                        data = data[8:]
+                    if self.compress_size == 0xFFFF_FFFF:
+                        field = "Compress size"
+                        self.compress_size, = unpack('<Q', data[:8])
+                        data = data[8:]
+                    if self.header_offset == 0xFFFF_FFFF:
+                        field = "Header offset"
+                        self.header_offset, = unpack('<Q', data[:8])
+                except struct.error:
+                    raise BadZipFile(f"Corrupt zip64 extra field. "
+                                     f"{field} not found.") from None
 
             extra = extra[ln+4:]
 
@@ -921,12 +899,16 @@ class ZipExtFile(io.BufferedIOBase):
         return self._readbuffer[self._offset: self._offset + 512]
 
     def readable(self):
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
         return True
 
     def read(self, n=-1):
         """Read and return up to n bytes.
         If the argument is omitted, None, or negative, data is read and returned until EOF is reached.
         """
+        if self.closed:
+            raise ValueError("read from closed file.")
         if n is None or n < 0:
             buf = self._readbuffer[self._offset:]
             self._readbuffer = b''
@@ -1063,9 +1045,13 @@ class ZipExtFile(io.BufferedIOBase):
             super().close()
 
     def seekable(self):
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
         return self._seekable
 
     def seek(self, offset, whence=0):
+        if self.closed:
+            raise ValueError("seek on closed file.")
         if not self._seekable:
             raise io.UnsupportedOperation("underlying stream is not seekable")
         curr_pos = self.tell()
@@ -1114,6 +1100,8 @@ class ZipExtFile(io.BufferedIOBase):
         return self.tell()
 
     def tell(self):
+        if self.closed:
+            raise ValueError("tell on closed file.")
         if not self._seekable:
             raise io.UnsupportedOperation("underlying stream is not seekable")
         filepos = self._orig_file_size - self._left - len(self._readbuffer) + self._offset
@@ -1555,7 +1543,7 @@ class ZipFile:
                 # strong encryption
                 raise NotImplementedError("strong encryption (flag bit 6)")
 
-            if zinfo.flag_bits & 0x800:
+            if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & 0x800:
                 # UTF-8 filename
                 fname_str = fname.decode("utf-8")
             else:
@@ -1593,9 +1581,7 @@ class ZipFile:
                              "another write handle open on it. "
                              "Close the first handle before opening another.")
 
-        # Sizes and CRC are overwritten with correct data after processing the file
-        if not hasattr(zinfo, 'file_size'):
-            zinfo.file_size = 0
+        # Size and CRC are overwritten with correct data after processing the file
         zinfo.compress_size = 0
         zinfo.CRC = 0
 
@@ -1891,25 +1877,15 @@ class ZipFile:
 
             extract_version = max(min_version, zinfo.extract_version)
             create_version = max(min_version, zinfo.create_version)
-            try:
-                filename, flag_bits = zinfo._encodeFilenameFlags()
-                centdir = struct.pack(structCentralDir,
-                                      stringCentralDir, create_version,
-                                      zinfo.create_system, extract_version, zinfo.reserved,
-                                      flag_bits, zinfo.compress_type, dostime, dosdate,
-                                      zinfo.CRC, compress_size, file_size,
-                                      len(filename), len(extra_data), len(zinfo.comment),
-                                      0, zinfo.internal_attr, zinfo.external_attr,
-                                      header_offset)
-            except DeprecationWarning:
-                print((structCentralDir, stringCentralDir, create_version,
-                       zinfo.create_system, extract_version, zinfo.reserved,
-                       zinfo.flag_bits, zinfo.compress_type, dostime, dosdate,
-                       zinfo.CRC, compress_size, file_size,
-                       len(zinfo.filename), len(extra_data), len(zinfo.comment),
-                       0, zinfo.internal_attr, zinfo.external_attr,
-                       header_offset), file=sys.stderr)
-                raise
+            filename, flag_bits = zinfo._encodeFilenameFlags()
+            centdir = struct.pack(structCentralDir,
+                                  stringCentralDir, create_version,
+                                  zinfo.create_system, extract_version, zinfo.reserved,
+                                  flag_bits, zinfo.compress_type, dostime, dosdate,
+                                  zinfo.CRC, compress_size, file_size,
+                                  len(filename), len(extra_data), len(zinfo.comment),
+                                  0, zinfo.internal_attr, zinfo.external_attr,
+                                  header_offset)
             self.fp.write(centdir)
             self.fp.write(filename)
             self.fp.write(extra_data)
@@ -1951,6 +1927,8 @@ class ZipFile:
                              centDirSize, centDirOffset, len(self._comment))
         self.fp.write(endrec)
         self.fp.write(self._comment)
+        if self.mode == "a":
+            self.fp.truncate()
         self.fp.flush()
 
     def _fpclose(self, fp):
@@ -2134,24 +2112,6 @@ class PyZipFile(ZipFile):
         return (fname, archivename)
 
 
-def _unique_everseen(iterable, key=None):
-    "List unique elements, preserving order. Remember all elements ever seen."
-    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
-    # unique_everseen('ABBCcAD', str.lower) --> A B C D
-    seen = set()
-    seen_add = seen.add
-    if key is None:
-        for element in itertools.filterfalse(seen.__contains__, iterable):
-            seen_add(element)
-            yield element
-    else:
-        for element in iterable:
-            k = key(element)
-            if k not in seen:
-                seen_add(k)
-                yield element
-
-
 def _parents(path):
     """
     Given a path with elements separated by
@@ -2193,6 +2153,18 @@ def _ancestry(path):
         path, tail = posixpath.split(path)
 
 
+_dedupe = dict.fromkeys
+"""Deduplicate an iterable in original order"""
+
+
+def _difference(minuend, subtrahend):
+    """
+    Return items in minuend not in subtrahend, retaining order
+    with O(1) lookup.
+    """
+    return itertools.filterfalse(set(subtrahend).__contains__, minuend)
+
+
 class CompleteDirs(ZipFile):
     """
     A ZipFile subclass that ensures that implied directories
@@ -2202,13 +2174,8 @@ class CompleteDirs(ZipFile):
     @staticmethod
     def _implied_dirs(names):
         parents = itertools.chain.from_iterable(map(_parents, names))
-        # Deduplicate entries in original order
-        implied_dirs = OrderedDict.fromkeys(
-            p + posixpath.sep for p in parents
-            # Cast names to a set for O(1) lookups
-            if p + posixpath.sep not in set(names)
-        )
-        return implied_dirs
+        as_dirs = (p + posixpath.sep for p in parents)
+        return _dedupe(_difference(as_dirs, names))
 
     def namelist(self):
         names = super(CompleteDirs, self).namelist()
@@ -2337,20 +2304,31 @@ class Path:
         self.root = FastLookup.make(root)
         self.at = at
 
-    @property
-    def open(self):
-        return functools.partial(self.root.open, self.at)
+    def open(self, mode='r', *args, **kwargs):
+        """
+        Open this entry as text or binary following the semantics
+        of ``pathlib.Path.open()`` by passing arguments through
+        to io.TextIOWrapper().
+        """
+        pwd = kwargs.pop('pwd', None)
+        zip_mode = mode[0]
+        stream = self.root.open(self.at, zip_mode, pwd=pwd)
+        if 'b' in mode:
+            if args or kwargs:
+                raise ValueError("encoding args invalid for binary operation")
+            return stream
+        return io.TextIOWrapper(stream, *args, **kwargs)
 
     @property
     def name(self):
         return posixpath.basename(self.at.rstrip("/"))
 
     def read_text(self, *args, **kwargs):
-        with self.open() as strm:
-            return io.TextIOWrapper(strm, *args, **kwargs).read()
+        with self.open('r', *args, **kwargs) as strm:
+            return strm.read()
 
     def read_bytes(self):
-        with self.open() as strm:
+        with self.open('rb') as strm:
             return strm.read()
 
     def _is_child(self, path):

src/lib.rs

@@ -45,13 +45,14 @@ extern crate log;
 
 use clap::{App, AppSettings, Arg, ArgMatches};
 use rustpython_vm::{
-    builtins::PyInt, compile, exceptions::print_exception, match_class, scope::Scope, stdlib::sys,
-    InitParameter, Interpreter, ItemProtocol, PyResult, PySettings, TypeProtocol, VirtualMachine,
+    builtins::PyDictRef, builtins::PyInt, compile, exceptions::print_exception, match_class,
+    scope::Scope, stdlib::sys, InitParameter, Interpreter, ItemProtocol, PyObjectRef, PyResult,
+    PySettings, TryFromObject, TypeProtocol, VirtualMachine,
 };
 
 use std::convert::TryInto;
 use std::env;
-use std::path::PathBuf;
+use std::path::Path;
 use std::process;
 use std::str::FromStr;
 
@@ -93,13 +94,7 @@ where
     };
 
     let interp = Interpreter::new_with_init(settings, |vm| {
-        #[cfg(feature = "stdlib")]
-        {
-            let stdlib = rustpython_stdlib::get_module_inits();
-            for (name, init) in stdlib.into_iter() {
-                vm.add_native_module(name, init);
-            }
-        }
+        add_stdlib(vm);
         init(vm);
         init_param
     });
@@ -312,6 +307,17 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> {
     app.get_matches()
 }
 
+fn add_stdlib(vm: &mut VirtualMachine) {
+    let _ = vm;
+    #[cfg(feature = "stdlib")]
+    {
+        let stdlib = rustpython_stdlib::get_module_inits();
+        for (name, init) in stdlib.into_iter() {
+            vm.add_native_module(name, init);
+        }
+    }
+}
+
 /// Create settings by examining command line arguments and environment
 /// variables.
 fn create_settings(matches: &ArgMatches) -> PySettings {
@@ -327,6 +333,9 @@ fn create_settings(matches: &ArgMatches) -> PySettings {
     };
     let ignore_environment = settings.ignore_environment || settings.isolated;
 
+    // when rustpython-vm/pylib is enabled, PySettings::default().path_list has pylib::LIB_PATH
+    let maybe_pylib = settings.path_list.pop();
+
     // add the current directory to sys.path
     settings.path_list.push("".to_owned());
 
@@ -336,8 +345,7 @@ fn create_settings(matches: &ArgMatches) -> PySettings {
             std::env::split_paths(paths).map(|path| path.into_os_string().into_string().unwrap()),
         )
     } else {
-        #[cfg(all(feature = "pylib", not(feature = "freeze-stdlib")))]
-        settings.path_list.push(pylib::LIB_PATH.to_owned());
+        settings.path_list.extend(maybe_pylib);
     }
 
     if !ignore_environment {
@@ -524,7 +532,7 @@ fn write_profile(matches: &ArgMatches) -> Result<(), Box<dyn std::error::Error>>
     Ok(())
 }
 
-fn run_rustpython(vm: &VirtualMachine, matches: &ArgMatches) -> PyResult<()> {
+fn setup_main_module(vm: &VirtualMachine) -> PyResult<Scope> {
     let scope = vm.new_scope_with_builtins();
     let main_module = vm.new_module("__main__", scope.globals.clone(), None);
     main_module
@@ -542,6 +550,12 @@ fn run_rustpython(vm: &VirtualMachine, matches: &ArgMatches) -> PyResult<()> {
     vm.get_attribute(vm.sys_module.clone(), "modules")?
         .set_item("__main__", main_module, vm)?;
 
+    Ok(scope)
+}
+
+fn run_rustpython(vm: &VirtualMachine, matches: &ArgMatches) -> PyResult<()> {
+    let scope = setup_main_module(vm)?;
+
     let site_result = vm.import("site", None, 0);
 
     if site_result.is_err() {
@@ -613,58 +627,91 @@ fn run_module(vm: &VirtualMachine, module: &str) -> PyResult<()> {
     Ok(())
 }
 
-fn run_script(vm: &VirtualMachine, scope: Scope, script_file: &str) -> PyResult<()> {
-    debug!("Running file {}", script_file);
-    let mut file_path = PathBuf::from(script_file);
-    let file_meta = file_path.metadata().unwrap_or_else(|e| {
-        error!("can't open file '{}': {}", file_path.display(), e);
-        process::exit(1);
-    });
-    if file_meta.is_dir() {
-        file_path.push("__main__.py");
-        if !file_path.is_file() {
-            error!("can't find '__main__' module in '{}'", file_path.display());
-            process::exit(1);
+fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> {
+    let path_importer_cache = vm.get_attribute(vm.sys_module.clone(), "path_importer_cache")?;
+    let path_importer_cache = PyDictRef::try_from_object(vm, path_importer_cache)?;
+    if let Some(importer) = path_importer_cache.get_item_option(path, vm)? {
+        return Ok(Some(importer));
+    }
+    let path = vm.ctx.new_utf8_str(path);
+    let path_hooks = vm.get_attribute(vm.sys_module.clone(), "path_hooks")?;
+    let mut importer = None;
+    for path_hook in vm.extract_elements(&path_hooks)? {
+        match vm.invoke(&path_hook, (path.clone(),)) {
+            Ok(imp) => {
+                importer = Some(imp);
+                break;
+            }
+            Err(e) if e.isinstance(&vm.ctx.exceptions.import_error) => continue,
+            Err(e) => return Err(e),
         }
     }
+    Ok(if let Some(imp) = importer {
+        let imp = path_importer_cache.get_or_insert(vm, path, || imp.clone())?;
+        Some(imp)
+    } else {
+        None
+    })
+}
 
-    let dir = file_path.parent().unwrap().to_str().unwrap().to_owned();
+fn insert_sys_path(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<()> {
     let sys_path = vm.get_attribute(vm.sys_module.clone(), "path").unwrap();
-    vm.call_method(&sys_path, "insert", (0, dir))?;
+    vm.call_method(&sys_path, "insert", (0, obj))?;
+    Ok(())
+}
 
-    match std::fs::read_to_string(&file_path) {
+fn run_script(vm: &VirtualMachine, scope: Scope, script_file: &str) -> PyResult<()> {
+    debug!("Running file {}", script_file);
+    if get_importer(script_file, vm)?.is_some() {
+        insert_sys_path(vm, vm.ctx.new_utf8_str(script_file))?;
+        let runpy = vm.import("runpy", None, 0)?;
+        let run_module_as_main = vm.get_attribute(runpy, "_run_module_as_main")?;
+        vm.invoke(
+            &run_module_as_main,
+            (vm.ctx.new_utf8_str("__main__"), false),
+        )?;
+        return Ok(());
+    }
+    let dir = Path::new(script_file).parent().unwrap().to_str().unwrap();
+    insert_sys_path(vm, vm.ctx.new_utf8_str(dir))?;
+
+    match std::fs::read_to_string(script_file) {
         Ok(source) => {
-            _run_string(vm, scope, &source, file_path.to_str().unwrap().to_owned())?;
+            _run_string(vm, scope, &source, script_file.to_owned())?;
         }
         Err(err) => {
-            error!(
-                "Failed reading file '{}': {:?}",
-                file_path.to_str().unwrap(),
-                err.kind()
-            );
+            error!("Failed reading file '{}': {}", script_file, err);
             process::exit(1);
         }
     }
     Ok(())
 }
 
-#[test]
-fn test_run_script() {
-    Interpreter::default().enter(|vm| {
-        // test file run
-        let r = run_script(
-            vm,
-            vm.new_scope_with_builtins(),
-            "extra_tests/snippets/dir_main/__main__.py",
-        );
-        assert!(r.is_ok());
+#[cfg(test)]
+mod tests {
+    use super::*;
 
-        // test module run
-        let r = run_script(
-            vm,
-            vm.new_scope_with_builtins(),
-            "extra_tests/snippets/dir_main",
-        );
-        assert!(r.is_ok());
-    })
+    fn interpreter() -> Interpreter {
+        Interpreter::new_with_init(PySettings::default(), |vm| {
+            add_stdlib(vm);
+            InitParameter::External
+        })
+    }
+
+    #[test]
+    fn test_run_script() {
+        interpreter().enter(|vm| {
+            vm.unwrap_pyresult((|| {
+                let scope = setup_main_module(vm)?;
+                // test file run
+                run_script(vm, scope, "extra_tests/snippets/dir_main/__main__.py")?;
+
+                let scope = setup_main_module(vm)?;
+                // test module run
+                run_script(vm, scope, "extra_tests/snippets/dir_main")?;
+
+                Ok(())
+            })());
+        })
+    }
 }

vm/src/builtins/dict.rs

@@ -302,6 +302,15 @@ impl PyDict {
             .setdefault(vm, key, || default.unwrap_or_none(vm))
     }
 
+    pub fn get_or_insert(
+        &self,
+        vm: &VirtualMachine,
+        key: PyObjectRef,
+        default: impl FnOnce() -> PyObjectRef,
+    ) -> PyResult {
+        self.entries.setdefault(vm, key, default)
+    }
+
     #[pymethod]
     pub fn copy(&self) -> PyDict {
         PyDict {

vm/src/builtins/function.rs

@@ -317,6 +317,7 @@ impl PyFunction {
         }
     }
 
+    #[inline(always)]
     pub fn invoke(&self, func_args: FuncArgs, vm: &VirtualMachine) -> PyResult {
         self.invoke_with_locals(func_args, None, vm)
     }

vm/src/frame.rs

@@ -202,7 +202,7 @@ impl Frame {
 }
 
 impl FrameRef {
-    #[inline]
+    #[inline(always)]
     fn with_exec<R>(&self, f: impl FnOnce(ExecutingFrame) -> R) -> R {
         let mut state = self.state.lock();
         let exec = ExecutingFrame {

vm/src/function.rs

@@ -594,6 +594,7 @@ impl<F, T, R, VM> IntoPyNativeFunc<(T, R, VM)> for F
 where
     F: PyNativeFuncInternal<T, R, VM>,
 {
+    #[inline(always)]
     fn call(&self, vm: &VirtualMachine, args: FuncArgs) -> PyResult {
         self.call_(vm, args)
     }

vm/src/stdlib/sys.rs

@@ -655,6 +655,7 @@ settrace() -- set the global debug tracing function
     let base_prefix = option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or(prefix);
     let exec_prefix = option_env!("RUSTPYTHON_EXECPREFIX").unwrap_or(prefix);
     let base_exec_prefix = option_env!("RUSTPYTHON_BASEEXECPREFIX").unwrap_or(exec_prefix);
+    let platlibdir = option_env!("RUSTPYTHON_PLATLIBDIR").unwrap_or("lib");
 
     extend_module!(vm, module, {
       "__name__" => ctx.new_ascii_literal(ascii!("sys")),
@@ -702,6 +703,7 @@ settrace() -- set the global debug tracing function
       "base_prefix" => ctx.new_utf8_str(base_prefix),
       "exec_prefix" => ctx.new_utf8_str(exec_prefix),
       "base_exec_prefix" => ctx.new_utf8_str(base_exec_prefix),
+      "platlibdir" => ctx.new_utf8_str(platlibdir),
       "exit" => named_function!(ctx, sys, exit),
       "abiflags" => ctx.new_utf8_str(ABIFLAGS.to_owned()),
       "audit" => named_function!(ctx, sys, audit),

vm/src/vm.rs

@@ -495,6 +495,7 @@ impl VirtualMachine {
         self.run_frame_full(frame)
     }
 
+    #[inline(always)]
     pub fn run_frame_full(&self, frame: FrameRef) -> PyResult {
         match self.run_frame(frame)? {
             ExecutionResult::Return(value) => Ok(value),
@@ -804,7 +805,8 @@ impl VirtualMachine {
         self.new_exception_empty(stop_iteration_type)
     }
 
-    // TODO: #[track_caller] when stabilized
+    #[track_caller]
+    #[cold]
     fn _py_panic_failed(&self, exc: PyBaseExceptionRef, msg: &str) -> ! {
         #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
         {
@@ -831,13 +833,21 @@ impl VirtualMachine {
             panic!("{}; exception backtrace above", msg)
         }
     }
+    #[track_caller]
     pub fn unwrap_pyresult<T>(&self, result: PyResult<T>) -> T {
-        result.unwrap_or_else(|exc| {
-            self._py_panic_failed(exc, "called `vm.unwrap_pyresult()` on an `Err` value")
-        })
+        match result {
+            Ok(x) => x,
+            Err(exc) => {
+                self._py_panic_failed(exc, "called `vm.unwrap_pyresult()` on an `Err` value")
+            }
+        }
     }
+    #[track_caller]
     pub fn expect_pyresult<T>(&self, result: PyResult<T>, msg: &str) -> T {
-        result.unwrap_or_else(|exc| self._py_panic_failed(exc, msg))
+        match result {
+            Ok(x) => x,
+            Err(exc) => self._py_panic_failed(exc, msg),
+        }
     }
 
     pub fn new_scope_with_builtins(&self) -> Scope {
@@ -1196,7 +1206,7 @@ impl VirtualMachine {
         }
     }
 
-    #[inline]
+    #[inline(always)]
     pub fn invoke<T>(&self, func_ref: &PyObjectRef, args: T) -> PyResult
     where
         T: IntoFuncArgs,