From ead42f290351148ad6f0a1ec20d1eac4641145e9 Mon Sep 17 00:00:00 2001 From: Dean Li Date: Sun, 7 Mar 2021 14:10:42 +0000 Subject: [PATCH 1/5] Implement dir_fd in os.open Now pass `test_posix.py PosixTester.test_open_dir_fd` Implement `dir_fd` in `make_path` and `os.open(path, flags, mode=0o777, *, dir_fd=None)` --- Cargo.lock | 13 ++++++++++++- vm/Cargo.toml | 1 + vm/src/stdlib/os.rs | 47 +++++++++++++++++++++++++++------------------ 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 725e2ad50..8d2d1081b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -749,6 +749,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "filepath" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd50318458226958db06f173610f54f01d26f0a9ccfa3d38bd60f70bc9939ff" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fixedbitset" version = "0.2.0" @@ -1101,7 +1111,7 @@ checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" [[package]] name = "lock_api" version = "0.4.2" -source = "git+https://github.com/coolreader18/parking_lot?branch=remutex-is_owned#391cf555e94c05450cea2e88b42c95c6631ab163" +source = "git+https://github.com/Amanieu/parking_lot#2b3032f95d4aa5b1ccdf41b937c60603f569763c" dependencies = [ "scopeguard", ] @@ -1986,6 +1996,7 @@ dependencies = [ "digest", "dns-lookup", "exitcode", + "filepath", "flame", "flamer", "flate2", diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 29e06964f..c0eb0d237 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -84,6 +84,7 @@ atty = "0.2" static_assertions = "1.1" half = "1.6" memchr = "2" +filepath = "0.1.1" # RustPython crates implementing functionality based on CPython mt19937 = "2.0" diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index a8626ab05..7b41c63d3 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -7,6 +7,7 @@ use std::time::{Duration, SystemTime}; use std::{env, fs}; use crossbeam_utils::atomic::AtomicCell; +use filepath::FilePath; use num_bigint::BigInt; use super::errno::errors; @@ -157,16 +158,24 @@ impl TryFromObject for PyPathLike { } } -fn make_path<'a>( - vm: &VirtualMachine, - path: &'a PyPathLike, - dir_fd: &DirFd, -) -> PyResult<&'a ffi::OsStr> { - if dir_fd.0.is_some() { - Err(vm.new_os_error("dir_fd not supported yet".to_owned())) - } else { - Ok(path.path.as_os_str()) +fn make_path(vm: &VirtualMachine, path: &PyPathLike, dir_fd: &DirFd) -> PyResult { + let path = path.path.to_path_buf(); + if dir_fd.0.is_none() | path.is_absolute() { + return Ok(path.into_os_string()); } + + if cfg!(target_os = "wasi") { + return Err(vm.new_os_error("dir_fd not supported on wasi yet".to_owned())); + } + + let dir_path = match rust_file(dir_fd.0.unwrap().into()).path() { + Ok(dir_path) => dir_path, + Err(_) => { + return Err(vm.new_os_error(format!("Cannot determine path of dir_fd: {:?}", dir_fd.0))); + } + }; + let p: PathBuf = vec![dir_path, path].iter().collect(); + Ok(p.into_os_string()) } impl IntoPyException for io::Error { @@ -349,7 +358,7 @@ mod _os { vm: &VirtualMachine, ) -> PyResult { let fname = make_path(vm, &name, &dir_fd)?; - if osstr_contains_nul(fname) { + if osstr_contains_nul(&fname) { return Err(vm.new_value_error("embedded null character".to_owned())); } @@ -504,14 +513,14 @@ mod _os { fn remove(path: PyPathLike, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { let path = make_path(vm, &path, &dir_fd)?; let is_junction = cfg!(windows) - && fs::symlink_metadata(path).map_or(false, |meta| { + && fs::symlink_metadata(&path).map_or(false, |meta| { let ty = meta.file_type(); ty.is_dir() && ty.is_symlink() }); let res = if is_junction { - fs::remove_dir(path) + fs::remove_dir(&path) } else { - fs::remove_file(path) + fs::remove_file(&path) }; res.map_err(|err| err.into_pyexception(vm)) } @@ -1219,7 +1228,7 @@ mod _os { pub(super) fn support_funcs(vm: &VirtualMachine) -> Vec { let mut supports = super::platform::support_funcs(vm); supports.extend(vec![ - SupportFunc::new(vm, "open", open, None, Some(false), None), + SupportFunc::new(vm, "open", open, None, Some(true), None), SupportFunc::new( vm, "access", @@ -1795,10 +1804,10 @@ mod posix { let path = make_path(vm, &path, &dir_fd)?; let body = move || { use std::os::unix::fs::PermissionsExt; - let meta = fs_metadata(path, follow_symlinks.0)?; + let meta = fs_metadata(&path, follow_symlinks.0)?; let mut permissions = meta.permissions(); permissions.set_mode(mode); - fs::set_permissions(path, permissions) + fs::set_permissions(&path, permissions) }; body().map_err(|err| err.into_pyexception(vm)) } @@ -2696,14 +2705,14 @@ mod nt { const S_IWRITE: u32 = 128; let path = make_path(vm, &path, &dir_fd)?; let metadata = if follow_symlinks.0 { - fs::metadata(path) + fs::metadata(&path) } else { - fs::symlink_metadata(path) + fs::symlink_metadata(&path) }; let meta = metadata.map_err(|err| err.into_pyexception(vm))?; let mut permissions = meta.permissions(); permissions.set_readonly(mode & S_IWRITE != 0); - fs::set_permissions(path, permissions).map_err(|err| err.into_pyexception(vm)) + fs::set_permissions(&path, permissions).map_err(|err| err.into_pyexception(vm)) } // cwait is available on MSVC only (according to CPython) From 9e6a7ae1a1dbc0f96341003c836c9ce77d93dd0c Mon Sep 17 00:00:00 2001 From: Dean Li Date: Sat, 13 Mar 2021 12:09:41 +0800 Subject: [PATCH 2/5] Exclude trait FilePath for wasi --- vm/Cargo.toml | 2 +- vm/src/stdlib/os.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index c0eb0d237..3183339bf 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -84,7 +84,6 @@ atty = "0.2" static_assertions = "1.1" half = "1.6" memchr = "2" -filepath = "0.1.1" # RustPython crates implementing functionality based on CPython mt19937 = "2.0" @@ -124,6 +123,7 @@ openssl = { version = "0.10.32", features = ["vendored"], optional = true } openssl-sys = { version = "0.9", optional = true } openssl-probe = { version = "0.1", optional = true } which = "4.0" +filepath = "0.1.1" [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] num_cpus = "1" diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 7b41c63d3..6967913a5 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -7,6 +7,7 @@ use std::time::{Duration, SystemTime}; use std::{env, fs}; use crossbeam_utils::atomic::AtomicCell; +#[cfg(not(target_os = "wasi"))] use filepath::FilePath; use num_bigint::BigInt; From cdcc191c0b934cc611e7658edcac3398788028d2 Mon Sep 17 00:00:00 2001 From: Dean Li Date: Sat, 13 Mar 2021 14:47:29 +0800 Subject: [PATCH 3/5] Use `cfg_if!` for trait FilePath --- vm/src/stdlib/os.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6967913a5..644075436 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -165,18 +165,20 @@ fn make_path(vm: &VirtualMachine, path: &PyPathLike, dir_fd: &DirFd) -> PyResult return Ok(path.into_os_string()); } - if cfg!(target_os = "wasi") { - return Err(vm.new_os_error("dir_fd not supported on wasi yet".to_owned())); - } - - let dir_path = match rust_file(dir_fd.0.unwrap().into()).path() { - Ok(dir_path) => dir_path, - Err(_) => { - return Err(vm.new_os_error(format!("Cannot determine path of dir_fd: {:?}", dir_fd.0))); + cfg_if::cfg_if! { + if #[cfg(target_os = "wasi")] { + return Err(vm.new_os_error("dir_fd not supported on wasi yet".to_owned())); + } else { + let dir_path = match rust_file(dir_fd.0.unwrap().into()).path() { + Ok(dir_path) => dir_path, + Err(_) => { + return Err(vm.new_os_error(format!("Cannot determine path of dir_fd: {:?}", dir_fd.0))); + } + }; + let p: PathBuf = vec![dir_path, path].iter().collect(); + Ok(p.into_os_string()) } - }; - let p: PathBuf = vec![dir_path, path].iter().collect(); - Ok(p.into_os_string()) + } } impl IntoPyException for io::Error { From d6d689dc1fc37bc600880895e2722a41207a22c3 Mon Sep 17 00:00:00 2001 From: Dean Li Date: Tue, 16 Mar 2021 20:14:11 +0800 Subject: [PATCH 4/5] os.open: make `make_path` return Cow --- vm/src/stdlib/os.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 644075436..41bf2f0d3 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -159,10 +159,14 @@ impl TryFromObject for PyPathLike { } } -fn make_path(vm: &VirtualMachine, path: &PyPathLike, dir_fd: &DirFd) -> PyResult { - let path = path.path.to_path_buf(); +fn make_path<'a>( + vm: &VirtualMachine, + path: &'a PyPathLike, + dir_fd: &DirFd, +) -> PyResult> { + let path = &path.path; if dir_fd.0.is_none() | path.is_absolute() { - return Ok(path.into_os_string()); + return Ok(std::borrow::Cow::Borrowed(path.as_os_str().into())); } cfg_if::cfg_if! { @@ -175,8 +179,8 @@ fn make_path(vm: &VirtualMachine, path: &PyPathLike, dir_fd: &DirFd) -> PyResult return Err(vm.new_os_error(format!("Cannot determine path of dir_fd: {:?}", dir_fd.0))); } }; - let p: PathBuf = vec![dir_path, path].iter().collect(); - Ok(p.into_os_string()) + let p: PathBuf = vec![dir_path, path.to_path_buf()].iter().collect(); + Ok(p.into_os_string().into()) } } } From 9963bf9fe46a19ca768e3846c7d07bedd1b64f6e Mon Sep 17 00:00:00 2001 From: Dean Li Date: Tue, 16 Mar 2021 20:14:31 +0800 Subject: [PATCH 5/5] Move filepath behind feature gate --- vm/Cargo.toml | 4 +++- vm/src/stdlib/os.rs | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 3183339bf..f312d55cf 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -123,7 +123,6 @@ openssl = { version = "0.10.32", features = ["vendored"], optional = true } openssl-sys = { version = "0.9", optional = true } openssl-probe = { version = "0.1", optional = true } which = "4.0" -filepath = "0.1.1" [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] num_cpus = "1" @@ -148,5 +147,8 @@ features = [ [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen = "0.2" +[target.'cfg(any(target_os = "linux", target_os = "macos", windows))'.dependencies] +filepath = "0.1.1" + [build-dependencies] itertools = "0.9" diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 41bf2f0d3..6041e2c55 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -7,7 +7,7 @@ use std::time::{Duration, SystemTime}; use std::{env, fs}; use crossbeam_utils::atomic::AtomicCell; -#[cfg(not(target_os = "wasi"))] +#[cfg(any(target_os = "linux", target_os = "macos", windows))] use filepath::FilePath; use num_bigint::BigInt; @@ -165,14 +165,12 @@ fn make_path<'a>( dir_fd: &DirFd, ) -> PyResult> { let path = &path.path; - if dir_fd.0.is_none() | path.is_absolute() { - return Ok(std::borrow::Cow::Borrowed(path.as_os_str().into())); + if dir_fd.0.is_none() || path.is_absolute() { + return Ok(path.as_os_str().into()); } cfg_if::cfg_if! { - if #[cfg(target_os = "wasi")] { - return Err(vm.new_os_error("dir_fd not supported on wasi yet".to_owned())); - } else { + if #[cfg(any(target_os = "linux", target_os = "macos", windows))] { let dir_path = match rust_file(dir_fd.0.unwrap().into()).path() { Ok(dir_path) => dir_path, Err(_) => { @@ -181,6 +179,8 @@ fn make_path<'a>( }; let p: PathBuf = vec![dir_path, path.to_path_buf()].iter().collect(); Ok(p.into_os_string().into()) + } else { + return Err(vm.new_os_error("dir_fd not supported on wasi yet".to_owned())); } } }