From b80db73af6e4224f845815a208bab819d12c5b20 Mon Sep 17 00:00:00 2001 From: Dean Li Date: Wed, 11 Aug 2021 20:58:13 +0800 Subject: [PATCH 1/3] os: update TestSendfile from cpython --- Lib/test/test_os.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f508f6e58..2ac32c0cb 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3050,7 +3050,6 @@ class SendfileTestServer(asyncore.dispatcher, threading.Thread): raise -@unittest.skip("TODO: RUSTPYTHON") @unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()") class TestSendfile(unittest.TestCase): @@ -3178,11 +3177,12 @@ class TestSendfile(unittest.TestCase): def test_keywords(self): # Keyword arguments should be supported - os.sendfile(out=self.sockno, offset=0, count=4096, - **{'in': self.fileno}) + os.sendfile(out_fd=self.sockno, in_fd=self.fileno, + offset=0, count=4096) if self.SUPPORT_HEADERS_TRAILERS: - os.sendfile(self.sockno, self.fileno, offset=0, count=4096, - headers=(), trailers=(), flags=0) + os.sendfile(out_fd=self.sockno, in_fd=self.fileno, + offset=0, count=4096, + headers=(), trailers=(), flags=0) # --- headers / trailers tests From 8f4719222cdf0dfa841f69ba1ebee06844aded87 Mon Sep 17 00:00:00 2001 From: Dean Li Date: Wed, 11 Aug 2021 21:10:12 +0800 Subject: [PATCH 2/3] os: fix sendfile arguments --- vm/src/stdlib/os.rs | 85 ++++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 24 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 70912a1a8..6550b8657 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -517,34 +517,68 @@ mod _os { fd.map(|fd| fd.0).map_err(|e| e.into_pyexception(vm)) } + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[derive(FromArgs)] + struct SendFileArgs { + #[pyarg(any)] + out_fd: i32, + #[pyarg(any)] + in_fd: i32, + #[pyarg(any)] + offset: i64, + #[pyarg(any)] + count: i64, + #[cfg(target_os = "macos")] + #[pyarg(any, optional)] + headers: OptionalArg, + #[cfg(target_os = "macos")] + #[pyarg(any, optional)] + trailers: OptionalArg, + #[cfg(target_os = "macos")] + #[allow(dead_code)] + #[pyarg(any, default)] + // TODO: not implemented + flags: OptionalArg, + } + #[cfg(target_os = "linux")] #[pyfunction] - fn sendfile(out_fd: i32, in_fd: i32, offset: i64, count: u64, vm: &VirtualMachine) -> PyResult { - let mut file_offset = offset; + fn sendfile(args: SendFileArgs, vm: &VirtualMachine) -> PyResult { + let mut file_offset = args.offset; - let res = - nix::sys::sendfile::sendfile(out_fd, in_fd, Some(&mut file_offset), count as usize) - .map_err(|err| err.into_pyexception(vm))?; + let res = nix::sys::sendfile::sendfile( + args.out_fd, + args.in_fd, + Some(&mut file_offset), + args.count as usize, + ) + .map_err(|err| err.into_pyexception(vm))?; Ok(vm.ctx.new_int(res as u64)) } #[cfg(target_os = "macos")] - #[pyfunction] - #[allow(clippy::too_many_arguments)] - fn sendfile( - out_fd: i32, - in_fd: i32, - offset: i64, - count: i64, - headers: OptionalArg, - trailers: OptionalArg, - _flags: OptionalArg, + fn _extract_vec_bytes( + x: OptionalArg, vm: &VirtualMachine, - ) -> PyResult { - let headers = match headers.into_option() { - Some(x) => Some(vm.extract_elements::(&x)?), + ) -> PyResult>> { + let inner = match x.into_option() { + Some(v) => { + let v = vm.extract_elements::(&v)?; + if v.is_empty() { + None + } else { + Some(v) + } + } None => None, }; + Ok(inner) + } + + #[cfg(target_os = "macos")] + #[pyfunction] + fn sendfile(args: SendFileArgs, vm: &VirtualMachine) -> PyResult { + let headers = _extract_vec_bytes(args.headers, vm)?; let headers = headers .as_ref() @@ -554,10 +588,7 @@ mod _os { .map(|v| v.iter().map(|borrowed| &**borrowed).collect::>()); let headers = headers.as_deref(); - let trailers = match trailers.into_option() { - Some(x) => Some(vm.extract_elements::(&x)?), - None => None, - }; + let trailers = _extract_vec_bytes(args.trailers, vm)?; let trailers = trailers .as_ref() @@ -567,8 +598,14 @@ mod _os { .map(|v| v.iter().map(|borrowed| &**borrowed).collect::>()); let trailers = trailers.as_deref(); - let (res, written) = - nix::sys::sendfile::sendfile(in_fd, out_fd, offset, Some(count), headers, trailers); + let (res, written) = nix::sys::sendfile::sendfile( + args.in_fd, + args.out_fd, + args.offset, + Some(args.count), + headers, + trailers, + ); res.map_err(|err| err.into_pyexception(vm))?; Ok(vm.ctx.new_int(written as u64)) } From 3bb4e5d8080d0ee6795f1540153ad040ec37ae9c Mon Sep 17 00:00:00 2001 From: Dean Li Date: Sat, 14 Aug 2021 00:17:15 +0800 Subject: [PATCH 3/3] os: fix sendfile for macos In python, `count` in `sendfile` doesn't include nbytes in headers or trailers. But in `nix::sys::sendfile::sendfile` does. From nix doc, ``` If any headers are specified and `count` is non-zero, the length of the headers will be counted in the limit of total bytes sent. ``` --- vm/src/stdlib/os.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6550b8657..b62a25209 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -579,6 +579,11 @@ mod _os { #[pyfunction] fn sendfile(args: SendFileArgs, vm: &VirtualMachine) -> PyResult { let headers = _extract_vec_bytes(args.headers, vm)?; + let count = headers + .as_ref() + .map(|v| v.iter().map(|s| s.len()).sum()) + .unwrap_or(0) as i64 + + args.count; let headers = headers .as_ref() @@ -602,7 +607,7 @@ mod _os { args.in_fd, args.out_fd, args.offset, - Some(args.count), + Some(count), headers, trailers, );