From 95b5508f526d022101a59bd8aeae97c3800e5c27 Mon Sep 17 00:00:00 2001 From: ChJR Date: Thu, 15 Aug 2019 18:42:45 +0900 Subject: [PATCH 01/10] Refactoring int.__lshift__ int.__rshift__ methods. --- vm/src/obj/objint.rs | 58 ++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index e7533752a..ccdf208da 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -165,6 +165,36 @@ fn inner_divmod(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { } } +fn inner_lshift(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { + if let Some(n_bits) = int2.value.to_usize() { + return Ok(vm.ctx.new_int(&int1.value << n_bits)); + } + + // i2 failed `to_usize()` conversion + match &int2.value { + v if v < &BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), + v if v > &BigInt::from(usize::max_value()) => { + Err(vm.new_overflow_error("the number is too large to convert to int".to_string())) + } + _ => panic!("Failed converting {} to rust usize", int2.value), + } +} + +fn inner_rshift(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { + if let Some(n_bits) = int2.value.to_usize() { + return Ok(vm.ctx.new_int(&int1.value >> n_bits)); + } + + // i2 failed `to_usize()` conversion + match &int2.value { + v if v < &BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), + v if v > &BigInt::from(usize::max_value()) => { + Err(vm.new_overflow_error("the number is too large to convert to int".to_string())) + } + _ => panic!("Failed converting {} to rust usize", int2.value), + } +} + #[pyimpl] impl PyInt { #[pymethod(name = "__eq__")] @@ -311,18 +341,8 @@ impl PyInt { return Ok(vm.ctx.not_implemented()); } - if let Some(n_bits) = get_value(&other).to_usize() { - return Ok(vm.ctx.new_int((&self.value) << n_bits)); - } - - // i2 failed `to_usize()` conversion - match get_value(&other) { - v if *v < BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), - v if *v > BigInt::from(usize::max_value()) => { - Err(vm.new_overflow_error("the number is too large to convert to int".to_string())) - } - _ => panic!("Failed converting {} to rust usize", get_value(&other)), - } + let other = other.payload::().unwrap(); + inner_lshift(self, other, vm) } #[pymethod(name = "__rshift__")] @@ -331,18 +351,8 @@ impl PyInt { return Ok(vm.ctx.not_implemented()); } - if let Some(n_bits) = get_value(&other).to_usize() { - return Ok(vm.ctx.new_int((&self.value) >> n_bits)); - } - - // i2 failed `to_usize()` conversion - match get_value(&other) { - v if *v < BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), - v if *v > BigInt::from(usize::max_value()) => { - Err(vm.new_overflow_error("the number is too large to convert to int".to_string())) - } - _ => panic!("Failed converting {} to rust usize", get_value(&other)), - } + let other = other.payload::().unwrap(); + inner_rshift(self, other, vm) } #[pymethod(name = "__xor__")] From f4b0195643453e398d8bca1b22311e7f60e1d559 Mon Sep 17 00:00:00 2001 From: ChJR Date: Thu, 15 Aug 2019 18:51:09 +0900 Subject: [PATCH 02/10] Add int.__rlshift__, int.__rrshift__ methods. --- vm/src/obj/objint.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index ccdf208da..9a2a892a3 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -345,6 +345,16 @@ impl PyInt { inner_lshift(self, other, vm) } + #[pymethod(name = "__rlshift__")] + fn rlshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if !objtype::isinstance(&other, &vm.ctx.int_type()) { + return Ok(vm.ctx.not_implemented()); + } + + let other = other.payload::().unwrap(); + inner_lshift(other, self, vm) + } + #[pymethod(name = "__rshift__")] fn rshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { @@ -355,6 +365,16 @@ impl PyInt { inner_rshift(self, other, vm) } + #[pymethod(name = "__rrshift__")] + fn rrshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if !objtype::isinstance(&other, &vm.ctx.int_type()) { + return Ok(vm.ctx.not_implemented()); + } + + let other = other.payload::().unwrap(); + inner_rshift(other, self, vm) + } + #[pymethod(name = "__xor__")] pub fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { From 151823aadd13f0d5ea07a74921bb261f33d9e2cd Mon Sep 17 00:00:00 2001 From: ChJR Date: Thu, 15 Aug 2019 19:01:16 +0900 Subject: [PATCH 03/10] Add tests for int.__(x)xshift__ methods. --- tests/snippets/numbers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/snippets/numbers.py b/tests/snippets/numbers.py index bded8f30c..1e6b03777 100644 --- a/tests/snippets/numbers.py +++ b/tests/snippets/numbers.py @@ -59,6 +59,11 @@ assert int(1).__rxor__(1) == 0 assert int(3).__rxor__(-3) == -2 assert int(3).__rxor__(4) == 7 +assert int(4).__lshift__(1) == 8 +assert int(4).__rshift__(1) == 2 +assert int(4).__rlshift__(1) == 16 +assert int(4).__rrshift__(1) == 0 + # Test underscores in numbers: assert 1_2 == 12 assert 1_2_3 == 123 From b8e86b24e6f6bcf66121b063451bf1750ce7d8a7 Mon Sep 17 00:00:00 2001 From: ChJR Date: Fri, 16 Aug 2019 02:21:50 +0900 Subject: [PATCH 04/10] Change int.inner_xshift wrong match guard routine. --- vm/src/obj/objint.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 9a2a892a3..2e2b3d898 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -172,8 +172,8 @@ fn inner_lshift(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { // i2 failed `to_usize()` conversion match &int2.value { - v if v < &BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), - v if v > &BigInt::from(usize::max_value()) => { + v if *v < BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), + v if *v > BigInt::from(usize::max_value()) => { Err(vm.new_overflow_error("the number is too large to convert to int".to_string())) } _ => panic!("Failed converting {} to rust usize", int2.value), @@ -187,8 +187,8 @@ fn inner_rshift(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { // i2 failed `to_usize()` conversion match &int2.value { - v if v < &BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), - v if v > &BigInt::from(usize::max_value()) => { + v if *v < BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), + v if *v > BigInt::from(usize::max_value()) => { Err(vm.new_overflow_error("the number is too large to convert to int".to_string())) } _ => panic!("Failed converting {} to rust usize", int2.value), From a0146c5fa6a10b974452241fbb544a81899d466b Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Thu, 15 Aug 2019 00:12:06 -0500 Subject: [PATCH 05/10] Add site module and run it on startup --- Lib/site.py | 624 ++++++++++++++++++++++++++++++++++++++++++ derive/src/pyclass.rs | 10 +- src/main.rs | 9 + vm/src/stdlib/re.rs | 19 +- vm/src/sysmodule.rs | 24 ++ 5 files changed, 678 insertions(+), 8 deletions(-) create mode 100644 Lib/site.py diff --git a/Lib/site.py b/Lib/site.py new file mode 100644 index 000000000..4595244e2 --- /dev/null +++ b/Lib/site.py @@ -0,0 +1,624 @@ +"""Append module search paths for third-party packages to sys.path. + +**************************************************************** +* This module is automatically imported during initialization. * +**************************************************************** + +This will append site-specific paths to the module search path. On +Unix (including Mac OSX), it starts with sys.prefix and +sys.exec_prefix (if different) and appends +lib/python/site-packages. +On other platforms (such as Windows), it tries each of the +prefixes directly, as well as with lib/site-packages appended. The +resulting directories, if they exist, are appended to sys.path, and +also inspected for path configuration files. + +If a file named "pyvenv.cfg" exists one directory above sys.executable, +sys.prefix and sys.exec_prefix are set to that directory and +it is also checked for site-packages (sys.base_prefix and +sys.base_exec_prefix will always be the "real" prefixes of the Python +installation). If "pyvenv.cfg" (a bootstrap configuration file) contains +the key "include-system-site-packages" set to anything other than "false" +(case-insensitive), the system-level prefixes will still also be +searched for site-packages; otherwise they won't. + +All of the resulting site-specific directories, if they exist, are +appended to sys.path, and also inspected for path configuration +files. + +A path configuration file is a file whose name has the form +.pth; its contents are additional directories (one per line) +to be added to sys.path. Non-existing directories (or +non-directories) are never added to sys.path; no directory is added to +sys.path more than once. Blank lines and lines beginning with +'#' are skipped. Lines starting with 'import' are executed. + +For example, suppose sys.prefix and sys.exec_prefix are set to +/usr/local and there is a directory /usr/local/lib/python2.5/site-packages +with three subdirectories, foo, bar and spam, and two path +configuration files, foo.pth and bar.pth. Assume foo.pth contains the +following: + + # foo package configuration + foo + bar + bletch + +and bar.pth contains: + + # bar package configuration + bar + +Then the following directories are added to sys.path, in this order: + + /usr/local/lib/python2.5/site-packages/bar + /usr/local/lib/python2.5/site-packages/foo + +Note that bletch is omitted because it doesn't exist; bar precedes foo +because bar.pth comes alphabetically before foo.pth; and spam is +omitted because it is not mentioned in either path configuration file. + +The readline module is also automatically configured to enable +completion for systems that support it. This can be overridden in +sitecustomize, usercustomize or PYTHONSTARTUP. Starting Python in +isolated mode (-I) disables automatic readline configuration. + +After these operations, an attempt is made to import a module +named sitecustomize, which can perform arbitrary additional +site-specific customizations. If this import fails with an +ImportError exception, it is silently ignored. +""" + +import sys +import os +import builtins +import _sitebuiltins + +# Prefixes for site-packages; add additional prefixes like /usr/local here +PREFIXES = [sys.prefix, sys.exec_prefix] +# Enable per user site-packages directory +# set it to False to disable the feature or True to force the feature +ENABLE_USER_SITE = None + +# for distutils.commands.install +# These values are initialized by the getuserbase() and getusersitepackages() +# functions, through the main() function when Python starts. +USER_SITE = None +USER_BASE = None + + +def makepath(*paths): + dir = os.path.join(*paths) + try: + dir = os.path.abspath(dir) + except OSError: + pass + return dir, os.path.normcase(dir) + + +def abs_paths(): + """Set all module __file__ and __cached__ attributes to an absolute path""" + for m in set(sys.modules.values()): + if (getattr(getattr(m, '__loader__', None), '__module__', None) not in + ('_frozen_importlib', '_frozen_importlib_external')): + continue # don't mess with a PEP 302-supplied __file__ + try: + m.__file__ = os.path.abspath(m.__file__) + except (AttributeError, OSError, TypeError): + pass + try: + m.__cached__ = os.path.abspath(m.__cached__) + except (AttributeError, OSError, TypeError): + pass + + +def removeduppaths(): + """ Remove duplicate entries from sys.path along with making them + absolute""" + # This ensures that the initial path provided by the interpreter contains + # only absolute pathnames, even if we're running from the build directory. + L = [] + known_paths = set() + for dir in sys.path: + # Filter out duplicate paths (on case-insensitive file systems also + # if they only differ in case); turn relative paths into absolute + # paths. + dir, dircase = makepath(dir) + if dircase not in known_paths: + L.append(dir) + known_paths.add(dircase) + sys.path[:] = L + return known_paths + + +def _init_pathinfo(): + """Return a set containing all existing file system items from sys.path.""" + d = set() + for item in sys.path: + try: + if os.path.exists(item): + _, itemcase = makepath(item) + d.add(itemcase) + except TypeError: + continue + return d + + +def addpackage(sitedir, name, known_paths): + """Process a .pth file within the site-packages directory: + For each line in the file, either combine it with sitedir to a path + and add that to known_paths, or execute it if it starts with 'import '. + """ + if known_paths is None: + known_paths = _init_pathinfo() + reset = True + else: + reset = False + fullname = os.path.join(sitedir, name) + try: + f = open(fullname, "r") + except OSError: + return + with f: + for n, line in enumerate(f): + if line.startswith("#"): + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception: + print("Error processing line {:d} of {}:\n".format(n+1, fullname), + file=sys.stderr) + import traceback + for record in traceback.format_exception(*sys.exc_info()): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break + if reset: + known_paths = None + return known_paths + + +def addsitedir(sitedir, known_paths=None): + """Add 'sitedir' argument to sys.path if missing and handle .pth files in + 'sitedir'""" + if known_paths is None: + known_paths = _init_pathinfo() + reset = True + else: + reset = False + sitedir, sitedircase = makepath(sitedir) + if not sitedircase in known_paths: + sys.path.append(sitedir) # Add path component + known_paths.add(sitedircase) + try: + names = os.listdir(sitedir) + except OSError: + return + names = [name for name in names if name.endswith(".pth")] + for name in sorted(names): + addpackage(sitedir, name, known_paths) + if reset: + known_paths = None + return known_paths + + +def check_enableusersite(): + """Check if user site directory is safe for inclusion + + The function tests for the command line flag (including environment var), + process uid/gid equal to effective uid/gid. + + None: Disabled for security reasons + False: Disabled by user (command line option) + True: Safe and enabled + """ + if sys.flags.no_user_site: + return False + + if hasattr(os, "getuid") and hasattr(os, "geteuid"): + # check process uid == effective uid + if os.geteuid() != os.getuid(): + return None + if hasattr(os, "getgid") and hasattr(os, "getegid"): + # check process gid == effective gid + if os.getegid() != os.getgid(): + return None + + return True + + +# NOTE: sysconfig and it's dependencies are relatively large but site module +# needs very limited part of them. +# To speedup startup time, we have copy of them. +# +# See https://bugs.python.org/issue29585 + +# Copy of sysconfig._getuserbase() +def _getuserbase(): + env_base = os.environ.get("PYTHONUSERBASE", None) + if env_base: + return env_base + + def joinuser(*args): + return os.path.expanduser(os.path.join(*args)) + + if os.name == "nt": + base = os.environ.get("APPDATA") or "~" + return joinuser(base, "Python") + + if sys.platform == "darwin" and sys._framework: + return joinuser("~", "Library", sys._framework, + "%d.%d" % sys.version_info[:2]) + + return joinuser("~", ".local") + + +# Same to sysconfig.get_path('purelib', os.name+'_user') +def _get_path(userbase): + version = sys.version_info + + if os.name == 'nt': + return f'{userbase}\\Python{version[0]}{version[1]}\\site-packages' + + if sys.platform == 'darwin' and sys._framework: + return f'{userbase}/lib/python/site-packages' + + return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' + + +def getuserbase(): + """Returns the `user base` directory path. + + The `user base` directory can be used to store data. If the global + variable ``USER_BASE`` is not initialized yet, this function will also set + it. + """ + global USER_BASE + if USER_BASE is None: + USER_BASE = _getuserbase() + return USER_BASE + + +def getusersitepackages(): + """Returns the user-specific site-packages directory path. + + If the global variable ``USER_SITE`` is not initialized yet, this + function will also set it. + """ + global USER_SITE + userbase = getuserbase() # this will also set USER_BASE + + if USER_SITE is None: + USER_SITE = _get_path(userbase) + + return USER_SITE + +def addusersitepackages(known_paths): + """Add a per user site-package to sys.path + + Each user has its own python directory with site-packages in the + home directory. + """ + # get the per user site-package path + # this call will also make sure USER_BASE and USER_SITE are set + user_site = getusersitepackages() + + if ENABLE_USER_SITE and os.path.isdir(user_site): + addsitedir(user_site, known_paths) + return known_paths + +def getsitepackages(prefixes=None): + """Returns a list containing all global site-packages directories. + + For each directory present in ``prefixes`` (or the global ``PREFIXES``), + this function will find its `site-packages` subdirectory depending on the + system environment, and will return a list of full paths. + """ + sitepackages = [] + seen = set() + + if prefixes is None: + prefixes = PREFIXES + + for prefix in prefixes: + if not prefix or prefix in seen: + continue + seen.add(prefix) + + if os.sep == '/': + sitepackages.append(os.path.join(prefix, "lib", + "python%d.%d" % sys.version_info[:2], + "site-packages")) + else: + sitepackages.append(prefix) + sitepackages.append(os.path.join(prefix, "lib", "site-packages")) + return sitepackages + +def addsitepackages(known_paths, prefixes=None): + """Add site-packages to sys.path""" + for sitedir in getsitepackages(prefixes): + if os.path.isdir(sitedir): + addsitedir(sitedir, known_paths) + + return known_paths + +def setquit(): + """Define new builtins 'quit' and 'exit'. + + These are objects which make the interpreter exit when called. + The repr of each object contains a hint at how it works. + + """ + if os.sep == '\\': + eof = 'Ctrl-Z plus Return' + else: + eof = 'Ctrl-D (i.e. EOF)' + + builtins.quit = _sitebuiltins.Quitter('quit', eof) + builtins.exit = _sitebuiltins.Quitter('exit', eof) + + +def setcopyright(): + """Set 'copyright' and 'credits' in builtins""" + builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) + if sys.platform[:4] == 'java': + builtins.credits = _sitebuiltins._Printer( + "credits", + "Jython is maintained by the Jython developers (www.jython.org).") + else: + builtins.credits = _sitebuiltins._Printer("credits", """\ + Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands + for supporting Python development. See www.python.org for more information.""") + files, dirs = [], [] + # Not all modules are required to have a __file__ attribute. See + # PEP 420 for more details. + if hasattr(os, '__file__'): + here = os.path.dirname(os.__file__) + files.extend(["LICENSE.txt", "LICENSE"]) + dirs.extend([os.path.join(here, os.pardir), here, os.curdir]) + builtins.license = _sitebuiltins._Printer( + "license", + "See https://www.python.org/psf/license/", + files, dirs) + + +def sethelper(): + builtins.help = _sitebuiltins._Helper() + +def enablerlcompleter(): + """Enable default readline configuration on interactive prompts, by + registering a sys.__interactivehook__. + + If the readline module can be imported, the hook will set the Tab key + as completion key and register ~/.python_history as history file. + This can be overridden in the sitecustomize or usercustomize module, + or in a PYTHONSTARTUP file. + """ + def register_readline(): + import atexit + try: + import readline + import rlcompleter + except ImportError: + return + + # Reading the initialization (config) file may not be enough to set a + # completion key, so we set one first and then read the file. + readline_doc = getattr(readline, '__doc__', '') + if readline_doc is not None and 'libedit' in readline_doc: + readline.parse_and_bind('bind ^I rl_complete') + else: + readline.parse_and_bind('tab: complete') + + try: + readline.read_init_file() + except OSError: + # An OSError here could have many causes, but the most likely one + # is that there's no .inputrc file (or .editrc file in the case of + # Mac OS X + libedit) in the expected location. In that case, we + # want to ignore the exception. + pass + + if readline.get_current_history_length() == 0: + # If no history was loaded, default to .python_history. + # The guard is necessary to avoid doubling history size at + # each interpreter exit when readline was already configured + # through a PYTHONSTARTUP hook, see: + # http://bugs.python.org/issue5845#msg198636 + history = os.path.join(os.path.expanduser('~'), + '.python_history') + try: + readline.read_history_file(history) + except OSError: + pass + atexit.register(readline.write_history_file, history) + + sys.__interactivehook__ = register_readline + +def venv(known_paths): + global PREFIXES, ENABLE_USER_SITE + + env = os.environ + if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env: + executable = os.environ['__PYVENV_LAUNCHER__'] + else: + executable = sys.executable + exe_dir, _ = os.path.split(os.path.abspath(executable)) + site_prefix = os.path.dirname(exe_dir) + sys._home = None + conf_basename = 'pyvenv.cfg' + candidate_confs = [ + conffile for conffile in ( + os.path.join(exe_dir, conf_basename), + os.path.join(site_prefix, conf_basename) + ) + if os.path.isfile(conffile) + ] + + if candidate_confs: + virtual_conf = candidate_confs[0] + system_site = "true" + # Issue 25185: Use UTF-8, as that's what the venv module uses when + # writing the file. + with open(virtual_conf, encoding='utf-8') as f: + for line in f: + if '=' in line: + key, _, value = line.partition('=') + key = key.strip().lower() + value = value.strip() + if key == 'include-system-site-packages': + system_site = value.lower() + elif key == 'home': + sys._home = value + + sys.prefix = sys.exec_prefix = site_prefix + + # Doing this here ensures venv takes precedence over user-site + addsitepackages(known_paths, [sys.prefix]) + + # addsitepackages will process site_prefix again if its in PREFIXES, + # but that's ok; known_paths will prevent anything being added twice + if system_site == "true": + PREFIXES.insert(0, sys.prefix) + else: + PREFIXES = [sys.prefix] + ENABLE_USER_SITE = False + + return known_paths + + +def execsitecustomize(): + """Run custom site specific code, if available.""" + try: + try: + import sitecustomize + except ImportError as exc: + if exc.name == 'sitecustomize': + pass + else: + raise + except Exception as err: + if sys.flags.verbose: + sys.excepthook(*sys.exc_info()) + else: + sys.stderr.write( + "Error in sitecustomize; set PYTHONVERBOSE for traceback:\n" + "%s: %s\n" % + (err.__class__.__name__, err)) + + +def execusercustomize(): + """Run custom user specific code, if available.""" + try: + try: + import usercustomize + except ImportError as exc: + if exc.name == 'usercustomize': + pass + else: + raise + except Exception as err: + if sys.flags.verbose: + sys.excepthook(*sys.exc_info()) + else: + sys.stderr.write( + "Error in usercustomize; set PYTHONVERBOSE for traceback:\n" + "%s: %s\n" % + (err.__class__.__name__, err)) + + +def main(): + """Add standard site-specific directories to the module search path. + + This function is called automatically when this module is imported, + unless the python interpreter was started with the -S flag. + """ + global ENABLE_USER_SITE + + orig_path = sys.path[:] + known_paths = removeduppaths() + if orig_path != sys.path: + # removeduppaths() might make sys.path absolute. + # fix __file__ and __cached__ of already imported modules too. + abs_paths() + + known_paths = venv(known_paths) + if ENABLE_USER_SITE is None: + ENABLE_USER_SITE = check_enableusersite() + known_paths = addusersitepackages(known_paths) + known_paths = addsitepackages(known_paths) + setquit() + setcopyright() + sethelper() + if not sys.flags.isolated: + enablerlcompleter() + execsitecustomize() + if ENABLE_USER_SITE: + execusercustomize() + +# Prevent extending of sys.path when python was started with -S and +# site is imported later. +if not sys.flags.no_site: + main() + +def _script(): + help = """\ + %s [--user-base] [--user-site] + + Without arguments print some useful information + With arguments print the value of USER_BASE and/or USER_SITE separated + by '%s'. + + Exit codes with --user-base or --user-site: + 0 - user site directory is enabled + 1 - user site directory is disabled by user + 2 - uses site directory is disabled by super user + or for security reasons + >2 - unknown error + """ + args = sys.argv[1:] + if not args: + user_base = getuserbase() + user_site = getusersitepackages() + print("sys.path = [") + for dir in sys.path: + print(" %r," % (dir,)) + print("]") + print("USER_BASE: %r (%s)" % (user_base, + "exists" if os.path.isdir(user_base) else "doesn't exist")) + print("USER_SITE: %r (%s)" % (user_site, + "exists" if os.path.isdir(user_site) else "doesn't exist")) + print("ENABLE_USER_SITE: %r" % ENABLE_USER_SITE) + sys.exit(0) + + buffer = [] + if '--user-base' in args: + buffer.append(USER_BASE) + if '--user-site' in args: + buffer.append(USER_SITE) + + if buffer: + print(os.pathsep.join(buffer)) + if ENABLE_USER_SITE: + sys.exit(0) + elif ENABLE_USER_SITE is False: + sys.exit(1) + elif ENABLE_USER_SITE is None: + sys.exit(2) + else: + sys.exit(3) + else: + import textwrap + print(textwrap.dedent(help % (sys.argv[0], os.pathsep))) + sys.exit(10) + +if __name__ == '__main__': + _script() diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index c16304d17..f4f0ec255 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -424,10 +424,12 @@ pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result ::rustpython_vm::pyobject::PyResult<::rustpython_vm::obj::objtuple::PyTupleRef> { - let tuple: ::rustpython_vm::obj::objtuple::PyTuple = - vec![#(::rustpython_vm::pyobject::IntoPyObject - ::into_pyobject(self.#field_names, vm)? - ),*].into(); + let tuple = ::rustpython_vm::obj::objtuple::PyTuple::from( + vec![#(::rustpython_vm::pyobject::IntoPyObject::into_pyobject( + ::std::clone::Clone::clone(&self.#field_names), + vm, + )?),*], + ); ::rustpython_vm::pyobject::PyValue::into_ref_with_type(tuple, vm, cls) } } diff --git a/src/main.rs b/src/main.rs index dcd55aa1d..5610711b2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -317,6 +317,15 @@ fn run_rustpython(vm: &VirtualMachine, matches: &ArgMatches) -> PyResult<()> { vm.get_attribute(vm.sys_module.clone(), "modules")? .set_item("__main__", main_module, vm)?; + let site_result = vm.import("site", &vm.ctx.new_tuple(vec![]), 0); + + if site_result.is_err() { + warn!( + "Failed to import site, consider adding the Lib directory to your RUSTPYTHONPATH \ + environment variable", + ); + } + // Figure out if a -c option was given: if let Some(command) = matches.value_of("c") { run_command(&vm, scope, command.to_string())?; diff --git a/vm/src/stdlib/re.rs b/vm/src/stdlib/re.rs index b44f5701d..49fba3185 100644 --- a/vm/src/stdlib/re.rs +++ b/vm/src/stdlib/re.rs @@ -188,12 +188,18 @@ fn do_findall(vm: &VirtualMachine, pattern: &PyPattern, search_text: PyStringRef let out = pattern .regex .captures_iter(search_text.as_str().as_bytes()) - .map(|captures| { - if captures.len() == 1 { - let full = captures.get(1).unwrap().as_bytes(); + .map(|captures| match captures.len() { + 1 => { + let full = captures.get(0).unwrap().as_bytes(); let full = String::from_utf8_lossy(full).into_owned(); vm.new_str(full) - } else { + } + 2 => { + let capture = captures.get(1).unwrap().as_bytes(); + let capture = String::from_utf8_lossy(capture).into_owned(); + vm.new_str(capture) + } + _ => { let out = captures .iter() .skip(1) @@ -354,6 +360,11 @@ impl PyPattern { ) -> PyResult { do_split(vm, self, search_text, maxsplit.into_option()) } + + #[pymethod] + fn findall(&self, search_text: PyStringRef, vm: &VirtualMachine) -> PyResult { + do_findall(vm, self, search_text) + } } #[pyimpl] diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 66bcef0cc..2527e9d10 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -173,6 +173,16 @@ fn sys_exit(code: OptionalArg, _vm: &VirtualMachine) -> PyResult<()> { std::process::exit(code) } +#[pystruct_sequence(name = "version_info")] +#[derive(Default, Debug)] +struct VersionInfo { + major: usize, + minor: usize, + micro: usize, + releaselevel: String, + serial: usize, +} + pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { let ctx = &vm.ctx; @@ -181,6 +191,17 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectR .into_struct_sequence(vm, flags_type) .unwrap(); + let version_info_type = VersionInfo::make_class(ctx); + let version_info = VersionInfo { + major: env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(), + minor: env!("CARGO_PKG_VERSION_MINOR").parse().unwrap(), + micro: env!("CARGO_PKG_VERSION_PATCH").parse().unwrap(), + releaselevel: "alpha".to_owned(), + serial: 0, + } + .into_struct_sequence(vm, version_info_type) + .unwrap(); + // TODO Add crate version to this namespace let implementation = py_namespace!(vm, { "name" => ctx.new_str("RustPython".to_string()), @@ -302,6 +323,7 @@ settrace() -- set the global debug tracing function let prefix = option_env!("RUSTPYTHON_PREFIX").unwrap_or("/usr/local"); let base_prefix = option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or(prefix); + let exec_prefix = option_env!("RUSTPYTHON_EXECPREFIX").unwrap_or(prefix); extend_module!(vm, module, { "__name__" => ctx.new_str(String::from("sys")), @@ -337,9 +359,11 @@ settrace() -- set the global debug tracing function "setprofile" => ctx.new_rustfunc(sys_setprofile), "settrace" => ctx.new_rustfunc(sys_settrace), "version" => vm.new_str(version::get_version()), + "version_info" => version_info, "exc_info" => ctx.new_rustfunc(sys_exc_info), "prefix" => ctx.new_str(prefix.to_string()), "base_prefix" => ctx.new_str(base_prefix.to_string()), + "exec_prefix" => ctx.new_str(exec_prefix.to_string()), "exit" => ctx.new_rustfunc(sys_exit), }); From 6f72969202d3f7b390fa2f7b3bc9068a951c7bfc Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Thu, 15 Aug 2019 00:13:03 -0500 Subject: [PATCH 06/10] Change to use .local/lib/rustpython --- Lib/site.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index 4595244e2..170dbc8c1 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -335,7 +335,8 @@ def getsitepackages(prefixes=None): if os.sep == '/': sitepackages.append(os.path.join(prefix, "lib", - "python%d.%d" % sys.version_info[:2], + # XXX changed for RustPython + "rustpython%d.%d" % sys.version_info[:2], "site-packages")) else: sitepackages.append(prefix) From 8b8f542e23e39c2f0135cee099ffc9327d1c5f06 Mon Sep 17 00:00:00 2001 From: Hyunji Kim Date: Thu, 15 Aug 2019 11:29:48 +0900 Subject: [PATCH 07/10] add __enter__ and __exit__ for socket --- tests/snippets/stdlib_socket.py | 2 ++ vm/src/stdlib/socket.rs | 49 ++++++++++++++++++++------------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/tests/snippets/stdlib_socket.py b/tests/snippets/stdlib_socket.py index f34c83e47..d5727bb25 100644 --- a/tests/snippets/stdlib_socket.py +++ b/tests/snippets/stdlib_socket.py @@ -135,3 +135,5 @@ assert socket.inet_ntoa(b"\xff\xff\xff\xff")=="255.255.255.255" with assertRaises(OSError): socket.inet_ntoa(b"\xff\xff\xff\xff\xff") +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + pass diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 1a111d7ea..c83b75656 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -11,6 +11,7 @@ use gethostname::gethostname; use byteorder::{BigEndian, ByteOrder}; +use crate::function::PyFuncArgs; use crate::obj::objbytes::PyBytesRef; use crate::obj::objint::PyIntRef; use crate::obj::objstr::PyStringRef; @@ -181,6 +182,14 @@ impl SocketRef { Socket::new(family, kind).into_ref_with_type(vm, cls) } + fn enter(self, _vm: &VirtualMachine) -> SocketRef { + self + } + + fn exit(self, _args: PyFuncArgs, _vm: &VirtualMachine) { + self.close(_vm) + } + fn connect(self, address: Address, vm: &VirtualMachine) -> PyResult<()> { let address_string = address.get_address_string(); @@ -425,29 +434,31 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let socket = py_class!(ctx, "socket", ctx.object(), { - "__new__" => ctx.new_rustfunc(SocketRef::new), - "connect" => ctx.new_rustfunc(SocketRef::connect), - "recv" => ctx.new_rustfunc(SocketRef::recv), - "send" => ctx.new_rustfunc(SocketRef::send), - "bind" => ctx.new_rustfunc(SocketRef::bind), - "accept" => ctx.new_rustfunc(SocketRef::accept), - "listen" => ctx.new_rustfunc(SocketRef::listen), - "close" => ctx.new_rustfunc(SocketRef::close), - "getsockname" => ctx.new_rustfunc(SocketRef::getsockname), - "sendto" => ctx.new_rustfunc(SocketRef::sendto), - "recvfrom" => ctx.new_rustfunc(SocketRef::recvfrom), - "fileno" => ctx.new_rustfunc(SocketRef::fileno), + "__new__" => ctx.new_rustfunc(SocketRef::new), + "__enter__" => ctx.new_rustfunc(SocketRef::enter), + "__exit__" => ctx.new_rustfunc(SocketRef::exit), + "connect" => ctx.new_rustfunc(SocketRef::connect), + "recv" => ctx.new_rustfunc(SocketRef::recv), + "send" => ctx.new_rustfunc(SocketRef::send), + "bind" => ctx.new_rustfunc(SocketRef::bind), + "accept" => ctx.new_rustfunc(SocketRef::accept), + "listen" => ctx.new_rustfunc(SocketRef::listen), + "close" => ctx.new_rustfunc(SocketRef::close), + "getsockname" => ctx.new_rustfunc(SocketRef::getsockname), + "sendto" => ctx.new_rustfunc(SocketRef::sendto), + "recvfrom" => ctx.new_rustfunc(SocketRef::recvfrom), + "fileno" => ctx.new_rustfunc(SocketRef::fileno), }); let module = py_module!(vm, "socket", { "AF_INET" => ctx.new_int(AddressFamily::Inet as i32), "SOCK_STREAM" => ctx.new_int(SocketKind::Stream as i32), - "SOCK_DGRAM" => ctx.new_int(SocketKind::Dgram as i32), - "socket" => socket, - "inet_aton" => ctx.new_rustfunc(socket_inet_aton), - "inet_ntoa" => ctx.new_rustfunc(socket_inet_ntoa), - "gethostname" => ctx.new_rustfunc(socket_gethostname), - "htonl" => ctx.new_rustfunc(socket_htonl), + "SOCK_DGRAM" => ctx.new_int(SocketKind::Dgram as i32), + "socket" => socket, + "inet_aton" => ctx.new_rustfunc(socket_inet_aton), + "inet_ntoa" => ctx.new_rustfunc(socket_inet_ntoa), + "gethostname" => ctx.new_rustfunc(socket_gethostname), + "htonl" => ctx.new_rustfunc(socket_htonl), }); extend_module_platform_specific(vm, module) @@ -464,7 +475,7 @@ fn extend_module_platform_specific(vm: &VirtualMachine, module: PyObjectRef) -> #[cfg(not(target_os = "redox"))] extend_module!(vm, module, { - "sethostname" => ctx.new_rustfunc(socket_sethostname), + "sethostname" => ctx.new_rustfunc(socket_sethostname), }); module From 440a25d222a2a17e73f783007d56a6bbc8f70b3a Mon Sep 17 00:00:00 2001 From: ChJR Date: Fri, 16 Aug 2019 14:57:07 +0900 Subject: [PATCH 08/10] Refactoring int.inner_xshift methods. --- vm/src/obj/objint.rs | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 2e2b3d898..26b469414 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -166,33 +166,13 @@ fn inner_divmod(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { } fn inner_lshift(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { - if let Some(n_bits) = int2.value.to_usize() { - return Ok(vm.ctx.new_int(&int1.value << n_bits)); - } - - // i2 failed `to_usize()` conversion - match &int2.value { - v if *v < BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), - v if *v > BigInt::from(usize::max_value()) => { - Err(vm.new_overflow_error("the number is too large to convert to int".to_string())) - } - _ => panic!("Failed converting {} to rust usize", int2.value), - } + let n_bits = get_shift_amount(int2, vm)?; + Ok(vm.ctx.new_int(&int1.value << n_bits)) } fn inner_rshift(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { - if let Some(n_bits) = int2.value.to_usize() { - return Ok(vm.ctx.new_int(&int1.value >> n_bits)); - } - - // i2 failed `to_usize()` conversion - match &int2.value { - v if *v < BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), - v if *v > BigInt::from(usize::max_value()) => { - Err(vm.new_overflow_error("the number is too large to convert to int".to_string())) - } - _ => panic!("Failed converting {} to rust usize", int2.value), - } + let n_bits = get_shift_amount(int2, vm)?; + Ok(vm.ctx.new_int(&int1.value >> n_bits)) } #[pyimpl] @@ -805,6 +785,20 @@ fn div_ints(vm: &VirtualMachine, i1: &BigInt, i2: &BigInt) -> PyResult { } } +fn get_shift_amount(amount: &PyInt, vm: &VirtualMachine) -> PyResult { + if let Some(n_bits) = amount.value.to_usize() { + Ok(n_bits) + } else { + match &amount.value { + v if *v < BigInt::zero() => Err(vm.new_value_error("negative shift count".to_string())), + v if *v > BigInt::from(usize::max_value()) => { + Err(vm.new_overflow_error("the number is too large to convert to int".to_string())) + } + _ => panic!("Failed converting {} to rust usize", amount.value), + } + } +} + pub fn init(context: &PyContext) { PyInt::extend_class(context, &context.int_type); extend_class!(context, &context.int_type, { From 1dceae9205ae3385cdafe368d8ef43bd753f2a34 Mon Sep 17 00:00:00 2001 From: ChJR Date: Fri, 16 Aug 2019 15:40:32 +0900 Subject: [PATCH 09/10] Add str.__rmod__() method. #190 (#1262) * Add str.__rmod__ method. * Add tests for str.__rmod__ method. * Improve test for str.__rmod__ method. * Change str.__rmod__ method return value type. * Format with rustfmt. * Remove not required code of str.__rmod__ method. * Improve with clippy. --- tests/snippets/strings.py | 3 +++ vm/src/obj/objstr.rs | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index ed5bfb972..087fe26cb 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -293,3 +293,6 @@ assert next(str_iter_reversed) == "2" assert next(str_iter_reversed) == "1" assert next(str_iter_reversed, None) == None assert_raises(StopIteration, lambda: next(str_iter_reversed)) + +assert str.__rmod__('%i', 30) == NotImplemented +assert_raises(TypeError, lambda: str.__rmod__(30, '%i')) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 15db1c47e..be83f2888 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -518,6 +518,11 @@ impl PyString { do_cformat(vm, format_string, values.clone()) } + #[pymethod(name = "__rmod__")] + fn rmod(&self, _values: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.not_implemented()) + } + #[pymethod] fn format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if args.args.is_empty() { From 3972239ae550e71471a15b490324bf484baebf11 Mon Sep 17 00:00:00 2001 From: ChJR Date: Fri, 16 Aug 2019 16:16:25 +0900 Subject: [PATCH 10/10] Simplify error formatting of str.__mod__ method. --- vm/src/obj/objstr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index ff609eaad..662438717 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -514,7 +514,7 @@ impl PyString { fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { let format_string_text = &self.value; let format_string = CFormatString::from_str(format_string_text) - .map_err(|err| vm.new_value_error(format!("{}", err)))?; + .map_err(|err| vm.new_value_error(err.to_string()))?; do_cformat(vm, format_string, values.clone()) }