From 7a890256aae9bb7c0d97187b155f3bd7971bc63c Mon Sep 17 00:00:00 2001 From: Dean Li Date: Sun, 4 Apr 2021 22:29:04 +0800 Subject: [PATCH 1/6] Fix skip condition for `test_fd` --- Lib/test/test_os.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 0b95d2b49..92e2244f7 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3958,8 +3958,8 @@ class TestScandir(unittest.TestCase): self.assertIs(type(entry.name), bytes) self.assertIs(type(entry.path), bytes) - @unittest.skipUnless(os.listdir in os.supports_fd, - 'fd support for listdir required for this test.') + @unittest.skipUnless(os.scandir in os.supports_fd, + 'fd support for scandir required for this test.') def test_fd(self): self.assertIn(os.scandir, os.supports_fd) self.create_file('file.txt') From a2b97f0f96af7008779096a4fc28079198a21755 Mon Sep 17 00:00:00 2001 From: Dean Li Date: Sun, 4 Apr 2021 20:52:38 +0800 Subject: [PATCH 2/6] os: update support features for `stat` and `fstat` --- vm/src/stdlib/os.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index a0b925cf3..a2ac884cf 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -900,7 +900,7 @@ mod _os { #[pyfunction] fn stat( file: Either, - dir_fd: super::DirFd, + dir_fd: DirFd, follow_symlinks: FollowSymlinks, vm: &VirtualMachine, ) -> PyResult { @@ -1259,8 +1259,8 @@ mod _os { SupportFunc::new(vm, "replace", rename, Some(false), Some(false), None), // TODO: Fix replace SupportFunc::new(vm, "rmdir", rmdir, Some(false), Some(false), None), SupportFunc::new(vm, "scandir", scandir, Some(false), None, None), - SupportFunc::new(vm, "stat", stat, Some(false), Some(false), Some(false)), - SupportFunc::new(vm, "fstat", stat, Some(false), Some(false), Some(false)), + SupportFunc::new(vm, "stat", stat, Some(true), Some(true), Some(true)), + SupportFunc::new(vm, "fstat", stat, Some(true), Some(true), Some(true)), SupportFunc::new(vm, "symlink", platform::symlink, None, Some(false), None), // truncate Some None None SupportFunc::new(vm, "unlink", remove, Some(false), Some(false), None), From 204dccc477907b8336b67985860448165552914d Mon Sep 17 00:00:00 2001 From: Dean Li Date: Sun, 4 Apr 2021 22:29:41 +0800 Subject: [PATCH 3/6] os: Add support_fd for listdir Use same approach as make_path to add support_fd for `listdir` After this patch, `os.fwalk` is available. Noted that, test coverage doesn't increase due to the lack of 3rd argument in `os.symlink`. --- vm/src/stdlib/os.rs | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index a2ac884cf..f1a6c5ed4 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -555,8 +555,30 @@ mod _os { } #[pyfunction] - fn listdir(path: PyPathLike, vm: &VirtualMachine) -> PyResult { - let dir_iter = fs::read_dir(&path.path).map_err(|err| err.into_pyexception(vm))?; + fn listdir(path: Either, vm: &VirtualMachine) -> PyResult { + let path = match path { + Either::A(path) => path, + Either::B(fno) => { + cfg_if::cfg_if! { + if #[cfg(any(target_os = "linux", target_os = "macos", windows))] { + let file = rust_file(fno); + match file.path() { + Ok(path) => PyPathLike::new_str(path.to_string_lossy().to_string()), + Err(_) => { + return Err(vm.new_os_error(format!( + "Cannot determine path of fno: {:?}", + fno + ))); + } + } + } else { + return Err(vm.new_os_error("dir_fd not supported on wasi yet".to_owned())); + } + } + } + }; + + let dir_iter = fs::read_dir(&path).map_err(|err| err.into_pyexception(vm))?; let res: PyResult> = dir_iter .map(|entry| match entry { Ok(entry_path) => path.mode.process_path(entry_path.file_name(), vm), @@ -1248,7 +1270,7 @@ mod _os { ), SupportFunc::new(vm, "chdir", chdir, Some(false), None, None), // chflags Some, None Some - SupportFunc::new(vm, "listdir", listdir, Some(false), None, None), + SupportFunc::new(vm, "listdir", listdir, Some(true), None, None), SupportFunc::new(vm, "mkdir", mkdir, Some(false), Some(false), None), // mkfifo Some Some None // mknod Some Some None From 84e1cf87ab5131f58527de6c305050294112b7bf Mon Sep 17 00:00:00 2001 From: Dean Li Date: Mon, 5 Apr 2021 10:09:12 +0800 Subject: [PATCH 4/6] Extract path_from_fd from make_path and listdir --- vm/src/stdlib/os.rs | 60 +++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index f1a6c5ed4..f540b4428 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -159,6 +159,22 @@ impl TryFromObject for PyPathLike { } } +fn path_from_fd(raw_fd: i64) -> Result { + cfg_if::cfg_if! { + if #[cfg(any(target_os = "linux", target_os = "macos", windows))] { + let path = match rust_file(raw_fd).path() { + Ok(path) => path, + Err(_) => { + return Err(format!("Cannot determine path of fd: {:?}", raw_fd)); + } + }; + Ok(path) + } else { + Err(vm.new_os_error("fd not supported on wasi yet".to_owned())); + } + } +} + fn make_path<'a>( vm: &VirtualMachine, path: &'a PyPathLike, @@ -168,21 +184,14 @@ fn make_path<'a>( if dir_fd.0.is_none() || path.is_absolute() { return Ok(path.as_os_str().into()); } - - cfg_if::cfg_if! { - if #[cfg(any(target_os = "linux", target_os = "macos", windows))] { - let dir_path = match rust_file(dir_fd.0.unwrap().into()).path() { - Ok(dir_path) => dir_path, - Err(_) => { - return Err(vm.new_os_error(format!("Cannot determine path of dir_fd: {:?}", dir_fd.0))); - } - }; - let p: PathBuf = vec![dir_path, path.to_path_buf()].iter().collect(); - Ok(p.into_os_string().into()) - } else { - return Err(vm.new_os_error("dir_fd not supported on wasi yet".to_owned())); + let dir_path = match path_from_fd(dir_fd.0.unwrap().into()) { + Ok(path) => path, + Err(e) => { + return Err(vm.new_os_error(e)); } - } + }; + let p: PathBuf = vec![dir_path, path.to_path_buf()].iter().collect(); + Ok(p.into_os_string().into()) } impl IntoPyException for io::Error { @@ -558,26 +567,13 @@ mod _os { fn listdir(path: Either, vm: &VirtualMachine) -> PyResult { let path = match path { Either::A(path) => path, - Either::B(fno) => { - cfg_if::cfg_if! { - if #[cfg(any(target_os = "linux", target_os = "macos", windows))] { - let file = rust_file(fno); - match file.path() { - Ok(path) => PyPathLike::new_str(path.to_string_lossy().to_string()), - Err(_) => { - return Err(vm.new_os_error(format!( - "Cannot determine path of fno: {:?}", - fno - ))); - } - } - } else { - return Err(vm.new_os_error("dir_fd not supported on wasi yet".to_owned())); - } + Either::B(fno) => match path_from_fd(fno) { + Ok(path) => PyPathLike::new_str(path.to_string_lossy().to_string()), + Err(e) => { + return Err(vm.new_os_error(e)); } - } + }, }; - let dir_iter = fs::read_dir(&path).map_err(|err| err.into_pyexception(vm))?; let res: PyResult> = dir_iter .map(|entry| match entry { From 5dae7aacdadbebc57b603ee56c1ed289de82f9ce Mon Sep 17 00:00:00 2001 From: Dean Li Date: Mon, 5 Apr 2021 11:13:39 +0800 Subject: [PATCH 5/6] Remove vm in path_from_fd --- vm/src/stdlib/os.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index f540b4428..c053cda61 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -170,7 +170,7 @@ fn path_from_fd(raw_fd: i64) -> Result { }; Ok(path) } else { - Err(vm.new_os_error("fd not supported on wasi yet".to_owned())); + Err("fd not supported on wasi yet"); } } } From b065c059d24e094e0e19443562ad81e942e0bfaf Mon Sep 17 00:00:00 2001 From: Dean Li Date: Mon, 5 Apr 2021 13:21:38 +0800 Subject: [PATCH 6/6] os: Don't consume `File` in path_from_fd --- Lib/test/test_os.py | 1 + vm/src/stdlib/os.rs | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 92e2244f7..f440e807b 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3958,6 +3958,7 @@ class TestScandir(unittest.TestCase): self.assertIs(type(entry.name), bytes) self.assertIs(type(entry.path), bytes) + # TODO: RUSTPYTHON (scandir needs to have supports_fd) @unittest.skipUnless(os.scandir in os.supports_fd, 'fd support for scandir required for this test.') def test_fd(self): diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index c053cda61..f3053a4fc 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -162,15 +162,17 @@ impl TryFromObject for PyPathLike { fn path_from_fd(raw_fd: i64) -> Result { cfg_if::cfg_if! { if #[cfg(any(target_os = "linux", target_os = "macos", windows))] { - let path = match rust_file(raw_fd).path() { + let file = rust_file(raw_fd); + let path = match file.path() { Ok(path) => path, - Err(_) => { - return Err(format!("Cannot determine path of fd: {:?}", raw_fd)); + Err(e) => { + return Err(format!("{:?} Cannot determine path of fd: {:?}", e, raw_fd)); } }; + raw_file_number(file); // Do not consume `raw_fd` Ok(path) } else { - Err("fd not supported on wasi yet"); + Err("fd not supported on wasi yet".to_owned()); } } }