From e4be8829947b87b25f0710d65560ca7a096fb700 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 13 Nov 2024 15:48:22 -0600 Subject: [PATCH 1/2] Miscellaneous cli-related parity fixes --- Cargo.lock | 2 - Cargo.toml | 2 - Lib/test/test_cmd_line.py | 16 --- Lib/test/test_repl.py | 2 - src/interpreter.rs | 2 +- src/lib.rs | 96 ++++++++-------- src/settings.rs | 218 +++++++++++++++++------------------- src/shell.rs | 11 +- vm/Cargo.toml | 5 +- vm/src/function/argument.rs | 1 + vm/src/stdlib/builtins.rs | 4 +- vm/src/stdlib/imp.rs | 2 +- vm/src/stdlib/io.rs | 14 +-- vm/src/stdlib/sys.rs | 2 +- vm/src/vm/compile.rs | 14 ++- vm/src/vm/mod.rs | 44 ++++++-- vm/src/vm/setting.rs | 18 ++- 17 files changed, 225 insertions(+), 228 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b995c7cd..a26f97ecf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1876,7 +1876,6 @@ dependencies = [ name = "rustpython" version = "0.4.0" dependencies = [ - "atty", "cfg-if", "clap", "criterion", @@ -2194,7 +2193,6 @@ version = "0.4.0" dependencies = [ "ahash", "ascii", - "atty", "bitflags 2.6.0", "bstr", "caseless", diff --git a/Cargo.toml b/Cargo.toml index 6e90643ef..5e870e6aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] rustpython-vm = { workspace = true, features = ["compiler"] } rustpython-parser = { workspace = true } -atty = { workspace = true } cfg-if = { workspace = true } log = { workspace = true } flame = { workspace = true, optional = true } @@ -141,7 +140,6 @@ rustpython-format= { version = "0.4.0" } ahash = "0.8.11" ascii = "1.0" -atty = "0.2.14" bitflags = "2.4.1" bstr = "1" cfg-if = "1.0" diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index bda1f7822..f7247705d 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -177,8 +177,6 @@ class CmdLineTest(unittest.TestCase): # All good if module is located and run successfully assert_python_ok('-m', 'timeit', '-n', '1') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_run_module_bug1764407(self): # -m and -i need to play well together # Runs the timeit module and checks the __main__ @@ -335,8 +333,6 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(stdout, expected) self.assertEqual(p.returncode, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_non_interactive_output_buffering(self): code = textwrap.dedent(""" import sys @@ -352,8 +348,6 @@ class CmdLineTest(unittest.TestCase): 'False False False\n' 'False False True\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unbuffered_output(self): # Test expected operation of the '-u' switch for stream in ('stdout', 'stderr'): @@ -447,8 +441,6 @@ class CmdLineTest(unittest.TestCase): stdout, stderr = proc.communicate() self.assertEqual(stdout.rstrip(), expected) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform.startswith('win'), "TODO: RUSTPYTHON windows has \n troubles") def test_stdin_readline(self): # Issue #11272: check that sys.stdin.readline() replaces '\r\n' by '\n' # on Windows (sys.stdin is opened in binary mode) @@ -456,16 +448,12 @@ class CmdLineTest(unittest.TestCase): "import sys; print(repr(sys.stdin.readline()))", b"'abc\\n'") - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform.startswith('win'), "TODO: RUSTPYTHON windows has \n troubles") def test_builtin_input(self): # Issue #11272: check that input() strips newlines ('\n' or '\r\n') self.check_input( "print(repr(input()))", b"'abc'") - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform.startswith('win'), "TODO: RUSTPYTHON windows has \n troubles") def test_output_newline(self): # Issue 13119 Newline for print() should be \r\n on Windows. code = """if 1: @@ -632,8 +620,6 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(err.splitlines().count(b'Unknown option: -a'), 1) self.assertEqual(b'', out) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') def test_isolatedmode(self): @@ -662,8 +648,6 @@ class CmdLineTest(unittest.TestCase): cwd=tmpdir) self.assertEqual(out.strip(), b"ok") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_sys_flags_set(self): # Issue 31845: a startup refactoring broke reading flags from env vars for value, expected in (("", 0), ("1", 1), ("text", 1), ("2", 2)): diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index a63c1213c..03bf8d8b5 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -92,8 +92,6 @@ class TestInteractiveInterpreter(unittest.TestCase): output = kill_python(p) self.assertEqual(p.returncode, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_close_stdin(self): user_input = dedent(''' import os diff --git a/src/interpreter.rs b/src/interpreter.rs index 984fd8af8..9d4dc4ee9 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -16,7 +16,7 @@ pub type InitHook = Box; /// use rustpython_vm::Settings; /// // Override your settings here. /// let mut settings = Settings::default(); -/// settings.debug = true; +/// settings.debug = 1; /// // You may want to add paths to `rustpython_vm::Settings::path_list` to allow import python libraries. /// settings.path_list.push("".to_owned()); // add current working directory /// let interpreter = rustpython::InterpreterConfig::new() diff --git a/src/lib.rs b/src/lib.rs index 0b5e88ec4..326a3f174 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,13 +50,14 @@ mod interpreter; mod settings; mod shell; -use atty::Stream; use rustpython_vm::{scope::Scope, PyResult, VirtualMachine}; -use std::{env, process::ExitCode}; +use std::env; +use std::io::IsTerminal; +use std::process::ExitCode; pub use interpreter::InterpreterConfig; pub use rustpython_vm as vm; -pub use settings::{opts_with_clap, RunMode}; +pub use settings::{opts_with_clap, InstallPipMode, RunMode}; /// The main cli of the `rustpython` interpreter. This function will return `std::process::ExitCode` /// based on the return code of the python code ran through the cli. @@ -73,9 +74,6 @@ pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { let (settings, run_mode) = opts_with_clap(); - // Be quiet if "quiet" arg is set OR stdin is not connected to a terminal - let quiet_var = settings.quiet || !atty::is(Stream::Stdin); - // don't translate newlines (\r\n <=> \n) #[cfg(windows)] { @@ -97,7 +95,7 @@ pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { config = config.init_hook(Box::new(init)); let interp = config.interpreter(); - let exitcode = interp.run(move |vm| run_rustpython(vm, run_mode, quiet_var)); + let exitcode = interp.run(move |vm| run_rustpython(vm, run_mode)); ExitCode::from(exitcode) } @@ -117,7 +115,6 @@ fn setup_main_module(vm: &VirtualMachine) -> PyResult { Ok(scope) } -#[cfg(feature = "ssl")] fn get_pip(scope: Scope, vm: &VirtualMachine) -> PyResult<()> { let get_getpip = rustpython_vm::py_compile!( source = r#"\ @@ -128,7 +125,7 @@ __import__("io").TextIOWrapper( mode = "eval" ); eprintln!("downloading get-pip.py..."); - let getpip_code = vm.run_code_obj(vm.ctx.new_code(get_getpip), scope.clone())?; + let getpip_code = vm.run_code_obj(vm.ctx.new_code(get_getpip), vm.new_scope_with_builtins())?; let getpip_code: rustpython_vm::builtins::PyStrRef = getpip_code .downcast() .expect("TextIOWrapper.read() should return str"); @@ -137,29 +134,21 @@ __import__("io").TextIOWrapper( Ok(()) } -#[cfg(feature = "ssl")] -fn ensurepip(_: Scope, vm: &VirtualMachine) -> PyResult<()> { - vm.run_module("ensurepip") -} - -fn install_pip(_installer: &str, _scope: Scope, vm: &VirtualMachine) -> PyResult<()> { - #[cfg(feature = "ssl")] - { - match _installer { - "ensurepip" => ensurepip(_scope, vm), - "get-pip" => get_pip(_scope, vm), - _ => unreachable!(), - } +fn install_pip(installer: InstallPipMode, scope: Scope, vm: &VirtualMachine) -> PyResult<()> { + if cfg!(not(feature = "ssl")) { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.system_error.to_owned(), + "install-pip requires rustpython be build with '--features=ssl'".to_owned(), + )); } - #[cfg(not(feature = "ssl"))] - Err(vm.new_exception_msg( - vm.ctx.exceptions.system_error.to_owned(), - "install-pip requires rustpython be build with '--features=ssl'".to_owned(), - )) + match installer { + InstallPipMode::Ensurepip => vm.run_module("ensurepip"), + InstallPipMode::GetPip => get_pip(scope, vm), + } } -fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode, quiet: bool) -> PyResult<()> { +fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { #[cfg(feature = "flame-it")] let main_guard = flame::start_guard("RustPython main"); @@ -183,33 +172,46 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode, quiet: bool) -> PyResu ); } - match run_mode { + let is_repl = matches!(run_mode, RunMode::Repl); + if !vm.state.settings.quiet + && (vm.state.settings.verbose > 0 || (is_repl && std::io::stdin().is_terminal())) + { + eprintln!( + "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", + env!("CARGO_PKG_VERSION") + ); + eprintln!( + "RustPython {}.{}.{}", + vm::version::MAJOR, + vm::version::MINOR, + vm::version::MICRO, + ); + + eprintln!("Type \"help\", \"copyright\", \"credits\" or \"license\" for more information."); + } + let res = match run_mode { RunMode::Command(command) => { debug!("Running command {}", command); - vm.run_code_string(scope, &command, "".to_owned())?; + vm.run_code_string(scope.clone(), &command, "".to_owned()) + .map(drop) } RunMode::Module(module) => { debug!("Running module {}", module); - vm.run_module(&module)?; + vm.run_module(&module) } - RunMode::InstallPip(installer) => { - install_pip(&installer, scope, vm)?; - } - RunMode::ScriptInteractive(script, interactive) => { - if let Some(script) = script { - debug!("Running script {}", &script); - vm.run_script(scope.clone(), &script)?; - } else if !quiet { - println!( - "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", - crate_version!() - ); - } - if interactive { - shell::run_shell(vm, scope)?; - } + RunMode::InstallPip(installer) => install_pip(installer, scope.clone(), vm), + RunMode::Script(script) => { + debug!("Running script {}", &script); + vm.run_script(scope.clone(), &script) } + RunMode::Repl => Ok(()), + }; + if is_repl || vm.state.settings.inspect { + shell::run_shell(vm, scope)?; + } else { + res?; } + #[cfg(feature = "flame-it")] { main_guard.end(); diff --git a/src/settings.rs b/src/settings.rs index 9a6aca85e..f6b7f21f2 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -1,12 +1,18 @@ use clap::{App, AppSettings, Arg, ArgMatches}; use rustpython_vm::Settings; -use std::{env, str::FromStr}; +use std::env; pub enum RunMode { - ScriptInteractive(Option, bool), + Script(String), Command(String), Module(String), - InstallPip(String), + InstallPip(InstallPipMode), + Repl, +} + +pub enum InstallPipMode { + Ensurepip, + GetPip, } pub fn opts_with_clap() -> (Settings, RunMode) { @@ -58,8 +64,9 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> { .multiple(true) .value_name("get-pip args") .min_values(0) - .help("install the pip package manager for rustpython; \ - requires rustpython be build with the ssl feature enabled." + .help( + "install the pip package manager for rustpython; \ + requires rustpython be build with the ssl feature enabled.", ), ) .arg( @@ -74,45 +81,58 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> { .multiple(true) .help("Give the verbosity (can be applied multiple times)"), ) - .arg(Arg::with_name("debug").short("d").help("Debug the parser.")) + .arg( + Arg::with_name("debug") + .short("d") + .multiple(true) + .help("Debug the parser."), + ) .arg( Arg::with_name("quiet") .short("q") + .multiple(true) .help("Be quiet at startup."), ) .arg( Arg::with_name("inspect") .short("i") + .multiple(true) .help("Inspect interactively after running the script."), ) .arg( Arg::with_name("no-user-site") .short("s") + .multiple(true) .help("don't add user site directory to sys.path."), ) .arg( Arg::with_name("no-site") .short("S") + .multiple(true) .help("don't imply 'import site' on initialization"), ) .arg( Arg::with_name("dont-write-bytecode") .short("B") + .multiple(true) .help("don't write .pyc files on import"), ) .arg( Arg::with_name("safe-path") .short("P") + .multiple(true) .help("don’t prepend a potentially unsafe path to sys.path"), ) .arg( Arg::with_name("ignore-environment") .short("E") + .multiple(true) .help("Ignore environment variables PYTHON* such as PYTHONPATH"), ) .arg( Arg::with_name("isolate") .short("I") + .multiple(true) .help("isolate Python from the user's environment (implies -E and -s)"), ) .arg( @@ -136,22 +156,22 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> { .long("check-hash-based-pycs") .takes_value(true) .number_of_values(1) - .default_value("default") - .help("always|default|never\ncontrol how Python invalidates hash-based .pyc files"), + .possible_values(&["always", "default", "never"]) + .help("control how Python invalidates hash-based .pyc files"), ) .arg( Arg::with_name("bytes-warning") .short("b") .multiple(true) - .help("issue warnings about using bytes where strings are usually expected (-bb: issue errors)"), - ).arg( - Arg::with_name("unbuffered") - .short("u") .help( - "force the stdout and stderr streams to be unbuffered; \ - this option has no effect on stdin; also PYTHONUNBUFFERED=x", + "issue warnings about using bytes where strings \ + are usually expected (-bb: issue errors)", ), - ); + ) + .arg(Arg::with_name("unbuffered").short("u").multiple(true).help( + "force the stdout and stderr streams to be unbuffered; \ + this option has no effect on stdin; also PYTHONUNBUFFERED=x", + )); #[cfg(feature = "flame-it")] let app = app .arg( @@ -174,67 +194,49 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> { fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { let mut settings = Settings::default(); settings.isolated = matches.is_present("isolate"); - settings.ignore_environment = matches.is_present("ignore-environment"); + let ignore_environment = settings.isolated || matches.is_present("ignore-environment"); + settings.ignore_environment = ignore_environment; settings.interactive = !matches.is_present("c") && !matches.is_present("m") && (!matches.is_present("script") || matches.is_present("inspect")); settings.bytes_warning = matches.occurrences_of("bytes-warning"); settings.import_site = !matches.is_present("no-site"); - let ignore_environment = settings.ignore_environment || settings.isolated; - if !ignore_environment { settings.path_list.extend(get_paths("RUSTPYTHONPATH")); settings.path_list.extend(get_paths("PYTHONPATH")); } // Now process command line flags: - if matches.is_present("debug") || (!ignore_environment && env::var_os("PYTHONDEBUG").is_some()) - { - settings.debug = true; - } - if matches.is_present("inspect") - || (!ignore_environment && env::var_os("PYTHONINSPECT").is_some()) - { - settings.inspect = true; - } - - if matches.is_present("optimize") { - settings.optimize = matches.occurrences_of("optimize").try_into().unwrap(); - } else if !ignore_environment { - if let Ok(value) = get_env_var_value("PYTHONOPTIMIZE") { - settings.optimize = value; + let count_flag = |arg, env| { + let mut val = matches.occurrences_of(arg) as u8; + if !ignore_environment { + if let Some(value) = get_env_var_value(env) { + val = std::cmp::max(val, value); + } } - } + val + }; - if matches.is_present("verbose") { - settings.verbose = matches.occurrences_of("verbose").try_into().unwrap(); - } else if !ignore_environment { - if let Ok(value) = get_env_var_value("PYTHONVERBOSE") { - settings.verbose = value; - } - } + settings.optimize = count_flag("optimize", "PYTHONOPTIMIZE"); + settings.verbose = count_flag("verbose", "PYTHONVERBOSE"); + settings.debug = count_flag("debug", "PYTHONDEBUG"); - if matches.is_present("no-user-site") - || matches.is_present("isolate") - || (!ignore_environment && env::var_os("PYTHONNOUSERSITE").is_some()) - { - settings.user_site_directory = false; - } + let bool_env_var = |env| !ignore_environment && env::var_os(env).is_some_and(|v| !v.is_empty()); + let bool_flag = |arg, env| matches.is_present(arg) || bool_env_var(env); - if matches.is_present("quiet") { - settings.quiet = true; - } + settings.user_site_directory = + !(settings.isolated || bool_flag("no-user-site", "PYTHONNOUSERSITE")); + settings.quiet = matches.is_present("quiet"); + settings.write_bytecode = !bool_flag("dont-write-bytecode", "PYTHONDONTWRITEBYTECODE"); + settings.safe_path = settings.isolated || bool_flag("safe-path", "PYTHONSAFEPATH"); + settings.inspect = bool_flag("inspect", "PYTHONINSPECT"); + settings.buffered_stdio = !bool_flag("unbuffered", "PYTHONUNBUFFERED"); - if matches.is_present("dont-write-bytecode") - || (!ignore_environment && env::var_os("PYTHONDONTWRITEBYTECODE").is_some()) - { - settings.write_bytecode = false; - } if !ignore_environment && env::var_os("PYTHONINTMAXSTRDIGITS").is_some() { settings.int_max_str_digits = match env::var("PYTHONINTMAXSTRDIGITS").unwrap().parse() { - Ok(digits) if digits == 0 || digits >= 640 => digits, + Ok(digits @ (0 | 640..)) => digits, _ => { error!("Fatal Python error: config_init_int_max_str_digits: PYTHONINTMAXSTRDIGITS: invalid limit; must be >= 640 or 0 for unlimited.\nPython runtime state: preinitialized"); std::process::exit(1); @@ -242,54 +244,44 @@ fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { }; } - if matches.is_present("safe-path") - || (!ignore_environment && env::var_os("PYTHONSAFEPATH").is_some()) - { - settings.safe_path = true; - } - - matches + settings.check_hash_pycs_mode = matches .value_of("check-hash-based-pycs") - .unwrap_or("default") - .clone_into(&mut settings.check_hash_pycs_mode); + .map(|val| val.parse().unwrap()) + .unwrap_or_default(); - let mut dev_mode = false; - let mut warn_default_encoding = false; - if let Some(xopts) = matches.values_of("implementation-option") { - settings.xoptions.extend(xopts.map(|s| { - let mut parts = s.splitn(2, '='); - let name = parts.next().unwrap().to_owned(); - let value = parts.next().map(ToOwned::to_owned); - if name == "dev" { - dev_mode = true + let xopts = matches + .values_of("implementation-option") + .unwrap_or_default() + .map(|s| { + let (name, value) = s.split_once('=').unzip(); + let name = name.unwrap_or(s); + match name { + "dev" => settings.dev_mode = true, + "warn_default_encoding" => settings.warn_default_encoding = true, + "no_sig_int" => settings.install_signal_handlers = false, + "int_max_str_digits" => { + settings.int_max_str_digits = match value.unwrap().parse() { + Ok(digits) if digits == 0 || digits >= 640 => digits, + _ => { + error!( + "Fatal Python error: config_init_int_max_str_digits: \ + -X int_max_str_digits: \ + invalid limit; must be >= 640 or 0 for unlimited.\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; + } + _ => {} } - if name == "warn_default_encoding" { - warn_default_encoding = true - } - if name == "no_sig_int" { - settings.install_signal_handlers = false; - } - if name == "int_max_str_digits" { - settings.int_max_str_digits = match value.as_ref().unwrap().parse() { - Ok(digits) if digits == 0 || digits >= 640 => digits, - _ => { + (name.to_owned(), value.map(str::to_owned)) + }); + settings.xoptions.extend(xopts); - error!("Fatal Python error: config_init_int_max_str_digits: -X int_max_str_digits: invalid limit; must be >= 640 or 0 for unlimited.\nPython runtime state: preinitialized"); - std::process::exit(1); - }, - }; - } - (name, value) - })); - } - settings.dev_mode = dev_mode; - if warn_default_encoding - || (!ignore_environment && env::var_os("PYTHONWARNDEFAULTENCODING").is_some()) - { - settings.warn_default_encoding = true; - } + settings.warn_default_encoding |= bool_env_var("PYTHONWARNDEFAULTENCODING"); - if dev_mode { + if settings.dev_mode { settings.warnoptions.push("default".to_owned()) } if settings.bytes_warning > 0 { @@ -320,25 +312,20 @@ fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { settings.isolated = true; let mut args: Vec<_> = get_pip_args.map(ToOwned::to_owned).collect(); if args.is_empty() { - args.push("ensurepip".to_owned()); - args.push("--upgrade".to_owned()); - args.push("--default-pip".to_owned()); + args.extend(["ensurepip", "--upgrade", "--default-pip"].map(str::to_owned)); } - let installer = args[0].clone(); - let mode = match installer.as_str() { - "ensurepip" | "get-pip" => RunMode::InstallPip(installer), + let mode = match &*args[0] { + "ensurepip" => InstallPipMode::Ensurepip, + "get-pip" => InstallPipMode::GetPip, _ => panic!("--install-pip takes ensurepip or get-pip as first argument"), }; - (mode, args) + (RunMode::InstallPip(mode), args) } else if let Some(argv) = matches.values_of("script") { let argv: Vec<_> = argv.map(ToOwned::to_owned).collect(); let script = argv[0].clone(); - ( - RunMode::ScriptInteractive(Some(script), matches.is_present("inspect")), - argv, - ) + (RunMode::Script(script), argv) } else { - (RunMode::ScriptInteractive(None, true), vec!["".to_owned()]) + (RunMode::Repl, vec!["".to_owned()]) }; let hash_seed = match env::var("PYTHONHASHSEED") { @@ -358,8 +345,13 @@ fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { } /// Get environment variable and turn it into integer. -fn get_env_var_value(name: &str) -> Result { - env::var(name).map(|value| u8::from_str(&value).unwrap_or(1)) +fn get_env_var_value(name: &str) -> Option { + env::var_os(name).filter(|v| !v.is_empty()).map(|value| { + value + .to_str() + .and_then(|v| v.parse::().ok()) + .unwrap_or(1) + }) } /// Helper function to retrieve a sequence of paths from an environment variable. diff --git a/src/shell.rs b/src/shell.rs index 75d980b44..91becb0bb 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -6,7 +6,7 @@ use rustpython_vm::{ compiler::{self, CompileError, CompileErrorType}, readline::{Readline, ReadlineResult}, scope::Scope, - version, AsObject, PyResult, VirtualMachine, + AsObject, PyResult, VirtualMachine, }; enum ShellExecResult { @@ -93,15 +93,6 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { let mut continuing = false; - println!( - "RustPython {}.{}.{}", - version::MAJOR, - version::MINOR, - version::MICRO, - ); - - println!("Type \"help\", \"copyright\", \"credits\" or \"license\" for more information."); - loop { let prompt_name = if continuing { "ps2" } else { "ps1" }; let prompt = vm diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 4bcd8512e..dd81414eb 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -42,7 +42,6 @@ rustpython-sre_engine = { workspace = true } ascii = { workspace = true } ahash = { workspace = true } -atty = { workspace = true } bitflags = { workspace = true } bstr = { workspace = true } cfg-if = { workspace = true } @@ -68,6 +67,8 @@ paste = { workspace = true } rand = { workspace = true } serde = { workspace = true, optional = true } static_assertions = { workspace = true } +strum = { workspace = true } +strum_macros = { workspace = true } thiserror = { workspace = true } thread_local = { workspace = true } memchr = { workspace = true } @@ -94,8 +95,6 @@ unic-ucd-ident = "0.9.0" rustix = { workspace = true } exitcode = "1.1.2" uname = "0.1.1" -strum = { workspace = true } -strum_macros = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] rustyline = { workspace = true } diff --git a/vm/src/function/argument.rs b/vm/src/function/argument.rs index 880591860..bc60cfa25 100644 --- a/vm/src/function/argument.rs +++ b/vm/src/function/argument.rs @@ -54,6 +54,7 @@ into_func_args_from_tuple!((v1, T1), (v2, T2)); into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3)); into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4)); into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4), (v5, T5)); +into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4), (v5, T5), (v6, T6)); /// The `FuncArgs` struct is one of the most used structs then creating /// a rust function that can be called from python. It holds both positional diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 27b55bc6c..b151f5832 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -7,6 +7,8 @@ pub use builtins::{ascii, print, reversed}; #[pymodule] mod builtins { + use std::io::IsTerminal; + use crate::{ builtins::{ enumerate::PyReverseSequenceIterator, @@ -432,7 +434,7 @@ mod builtins { }; // everything is normalish, we can just rely on rustyline to use stdin/stdout - if fd_matches(&stdin, 0) && fd_matches(&stdout, 1) && atty::is(atty::Stream::Stdin) { + if fd_matches(&stdin, 0) && fd_matches(&stdout, 1) && std::io::stdin().is_terminal() { let prompt = prompt.as_ref().map_or("", |s| s.as_str()); let mut readline = Readline::new(()); match readline.readline(prompt) { diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index ab711bccc..d8727e74f 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -88,7 +88,7 @@ mod _imp { #[pyattr] fn check_hash_based_pycs(vm: &VirtualMachine) -> PyStrRef { vm.ctx - .new_str(vm.state.settings.check_hash_pycs_mode.clone()) + .new_str(vm.state.settings.check_hash_pycs_mode.to_string()) } #[pyfunction] diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index fb31ae487..165dc621d 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -15,7 +15,7 @@ use crate::{ convert::{IntoPyException, ToPyException}, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, }; -pub use _io::io_open as open; +pub use _io::{io_open as open, OpenArgs}; impl ToPyException for std::io::Error { fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { @@ -3822,17 +3822,17 @@ mod _io { #[derive(FromArgs)] pub struct OpenArgs { #[pyarg(any, default = "-1")] - buffering: isize, + pub buffering: isize, #[pyarg(any, default)] - encoding: Option, + pub encoding: Option, #[pyarg(any, default)] - errors: Option, + pub errors: Option, #[pyarg(any, default)] - newline: Option, + pub newline: Option, #[pyarg(any, default = "true")] - closefd: bool, + pub closefd: bool, #[pyarg(any, default)] - opener: Option, + pub opener: Option, } impl Default for OpenArgs { fn default() -> Self { diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs index 590f1d1ce..59414b5fd 100644 --- a/vm/src/stdlib/sys.rs +++ b/vm/src/stdlib/sys.rs @@ -823,7 +823,7 @@ mod sys { impl Flags { fn from_settings(settings: &Settings) -> Self { Self { - debug: settings.debug as u8, + debug: settings.debug, inspect: settings.inspect as u8, interactive: settings.interactive as u8, optimize: settings.optimize, diff --git a/vm/src/vm/compile.rs b/vm/src/vm/compile.rs index 1088568d6..b7c888ab1 100644 --- a/vm/src/vm/compile.rs +++ b/vm/src/vm/compile.rs @@ -35,12 +35,14 @@ impl VirtualMachine { return Ok(()); } - let dir = std::path::Path::new(path) - .parent() - .unwrap() - .to_str() - .unwrap(); - self.insert_sys_path(self.new_pyobj(dir))?; + if !self.state.settings.safe_path { + let dir = std::path::Path::new(path) + .parent() + .unwrap() + .to_str() + .unwrap(); + self.insert_sys_path(self.new_pyobj(dir))?; + } match std::fs::read_to_string(path) { Ok(source) => { diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 1a5f40e76..e8bde4e79 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -51,7 +51,7 @@ use std::{ pub use context::Context; pub use interpreter::Interpreter; pub(crate) use method::PyMethod; -pub use setting::Settings; +pub use setting::{CheckHashPycsMode, Settings}; pub const MAX_MEMORY_SIZE: usize = isize::MAX as usize; @@ -301,17 +301,39 @@ impl VirtualMachine { #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { - // this isn't fully compatible with CPython; it imports "io" and sets - // builtins.open to io.OpenWrapper, but this is easier, since it doesn't - // require the Python stdlib to be present let io = import::import_builtin(self, "_io")?; - let set_stdio = |name, fd, mode: &str| { - let stdio = crate::stdlib::io::open( + let set_stdio = |name, fd, write| { + let buffered_stdio = self.state.settings.buffered_stdio; + let unbuffered = write && !buffered_stdio; + let buf = crate::stdlib::io::open( self.ctx.new_int(fd).into(), - Some(mode), - Default::default(), + Some(if write { "wb" } else { "rb" }), + crate::stdlib::io::OpenArgs { + buffering: if unbuffered { 0 } else { -1 }, + ..Default::default() + }, self, )?; + let raw = if unbuffered { + buf.clone() + } else { + buf.get_attr("raw", self)? + }; + raw.set_attr("name", self.ctx.new_str(format!("<{name}>")), self)?; + let isatty = self.call_method(&raw, "isatty", ())?.is_true(self)?; + let write_through = !buffered_stdio; + let line_buffering = buffered_stdio && (isatty || fd == 2); + + let newline = if cfg!(windows) { None } else { Some("\n") }; + + let stdio = self.call_method( + &io, + "TextIOWrapper", + (buf, (), (), newline, line_buffering, write_through), + )?; + let mode = if write { "w" } else { "r" }; + stdio.set_attr("mode", self.ctx.new_str(mode), self)?; + let dunder_name = self.ctx.intern_str(format!("__{name}__")); self.sys_module.set_attr( dunder_name, // e.g. __stdin__ @@ -321,9 +343,9 @@ impl VirtualMachine { self.sys_module.set_attr(name, stdio, self)?; Ok(()) }; - set_stdio("stdin", 0, "r")?; - set_stdio("stdout", 1, "w")?; - set_stdio("stderr", 2, "w")?; + set_stdio("stdin", 0, false)?; + set_stdio("stdout", 1, true)?; + set_stdio("stderr", 2, true)?; let io_open = io.get_attr("open", self)?; self.builtins.set_attr("open", io_open, self)?; diff --git a/vm/src/vm/setting.rs b/vm/src/vm/setting.rs index 36bb9cee6..1abd26cf4 100644 --- a/vm/src/vm/setting.rs +++ b/vm/src/vm/setting.rs @@ -73,14 +73,13 @@ pub struct Settings { // int configure_c_stdio; /// -u, PYTHONUNBUFFERED=x - // TODO: use this; can TextIOWrapper even work with a non-buffered? pub buffered_stdio: bool, // wchar_t *stdio_encoding; pub utf8_mode: u8, // wchar_t *stdio_errors; /// --check-hash-based-pycs - pub check_hash_pycs_mode: String, + pub check_hash_pycs_mode: CheckHashPycsMode, // int use_frozen_modules; /// -P @@ -98,7 +97,7 @@ pub struct Settings { // wchar_t *home; // wchar_t *platlibdir; /// -d command line switch - pub debug: bool, + pub debug: u8, /// -O optimization switch counter pub optimize: u8, @@ -115,6 +114,15 @@ pub struct Settings { pub profile_format: Option, } +#[derive(Debug, Default, Copy, Clone, strum_macros::Display, strum_macros::EnumString)] +#[strum(serialize_all = "lowercase")] +pub enum CheckHashPycsMode { + #[default] + Default, + Always, + Never, +} + impl Settings { pub fn with_path(mut self, path: String) -> Self { self.path_list.push(path); @@ -126,7 +134,7 @@ impl Settings { impl Default for Settings { fn default() -> Self { Settings { - debug: false, + debug: 0, inspect: false, interactive: false, optimize: 0, @@ -148,7 +156,7 @@ impl Default for Settings { argv: vec![], hash_seed: None, buffered_stdio: true, - check_hash_pycs_mode: "default".to_owned(), + check_hash_pycs_mode: CheckHashPycsMode::Default, allow_external_library: cfg!(feature = "importlib"), utf8_mode: 1, int_max_str_digits: 4300, From c6da4ffcdd6d7ba1f18845584728a64dd634e8b6 Mon Sep 17 00:00:00 2001 From: Noa Date: Tue, 3 Dec 2024 17:29:33 -0600 Subject: [PATCH 2/2] Try to fix universal write on windows --- Lib/test/test_calendar.py | 4 +--- Lib/test/test_gzip.py | 1 - Lib/test/test_httpservers.py | 2 -- vm/src/stdlib/io.rs | 8 +++----- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index a53766d45..24e472b5f 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -845,9 +845,7 @@ class LeapdaysTestCase(unittest.TestCase): def conv(s): - # XXX RUSTPYTHON TODO: TextIOWrapper newline translation - return s.encode() - # return s.replace('\n', os.linesep).encode() + return s.replace('\n', os.linesep).encode() class CommandLineTestCase(unittest.TestCase): def run_ok(self, *args): diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 9d66385ab..6a709d3f6 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -725,7 +725,6 @@ class TestOpen(BaseTest): with self.assertRaises(ValueError): gzip.open(self.filename, "rb", newline="\n") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_encoding(self): # Test non-default encoding. uncompressed = data1.decode("ascii") * 50 diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 570abcec6..59700ac79 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -616,8 +616,6 @@ class CGIHTTPServerTestCase(BaseTestCase): pass linesep = os.linesep.encode('ascii') - # TODO: RUSTPYTHON - linesep = b'\n' def setUp(self): BaseTestCase.setUp(self) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 165dc621d..3c35b05e1 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -2750,15 +2750,13 @@ mod _io { let data = obj.as_str(); let replace_nl = match textio.newline { + Newlines::Lf => Some("\n"), Newlines::Cr => Some("\r"), Newlines::Crlf => Some("\r\n"), + Newlines::Universal if cfg!(windows) => Some("\r\n"), _ => None, }; - let has_lf = if replace_nl.is_some() || textio.line_buffering { - data.contains('\n') - } else { - false - }; + let has_lf = (replace_nl.is_some() || textio.line_buffering) && data.contains('\n'); let flush = textio.line_buffering && (has_lf || data.contains('\r')); let chunk = if let Some(replace_nl) = replace_nl { if has_lf {