Merge pull request #3111 from youknowone/posix

split `posix` and `nt` files from `os`
This commit is contained in:
Jeong YunWon
2021-09-24 12:59:44 +09:00
committed by GitHub
9 changed files with 2365 additions and 2306 deletions

View File

@@ -9,8 +9,6 @@ cfg_if::cfg_if! {
}
}
#[cfg(unix)]
use crate::stdlib::os::{errno_err, PathOrFd};
use crate::{PyObjectRef, PyResult, TryFromObject, VirtualMachine};
pub(crate) use _io::io_open as open;
@@ -89,7 +87,7 @@ mod _io {
PyThreadMutex, PyThreadMutexGuard,
};
use crate::common::rc::PyRc;
use crate::exceptions::{self, IntoPyException, PyBaseExceptionRef};
use crate::exceptions::{self, PyBaseExceptionRef};
use crate::function::{ArgIterable, FuncArgs, OptionalArg, OptionalOption};
use crate::slots::SlotConstructor;
use crate::utils::Either;
@@ -159,6 +157,7 @@ mod _io {
fn os_err(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef {
#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
{
use crate::exceptions::IntoPyException;
err.into_pyexception(vm)
}
#[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
@@ -3530,8 +3529,11 @@ mod _io {
// check file descriptor validity
#[cfg(unix)]
if let Ok(PathOrFd::Fd(fd)) = PathOrFd::try_from_object(vm, file.clone()) {
nix::fcntl::fcntl(fd, nix::fcntl::F_GETFD).map_err(|_| errno_err(vm))?;
if let Ok(crate::stdlib::os::PathOrFd::Fd(fd)) =
TryFromObject::try_from_object(vm, file.clone())
{
nix::fcntl::fcntl(fd, nix::fcntl::F_GETFD)
.map_err(|_| crate::stdlib::os::errno_err(vm))?;
}
// Construct a FileIO (subclass of RawIOBase)

View File

@@ -1,8 +1,3 @@
use crate::vm::VirtualMachine;
use crate::PyObjectRef;
use std::borrow::Cow;
use std::collections::HashMap;
mod array;
#[cfg(feature = "rustpython-ast")]
pub(crate) mod ast;
@@ -52,6 +47,16 @@ mod zlib;
#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
#[macro_use]
pub(crate) mod os;
#[cfg(windows)]
pub(crate) mod nt;
#[cfg(unix)]
pub(crate) mod posix;
#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
#[cfg(not(any(unix, windows)))]
pub(crate) mod posix_compat;
#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
#[cfg(not(any(unix, windows)))]
pub(crate) use posix_compat as posix;
#[cfg(not(target_arch = "wasm32"))]
mod faulthandler;
@@ -83,6 +88,11 @@ mod winapi;
#[cfg(windows)]
mod winreg;
use crate::vm::VirtualMachine;
use crate::PyObjectRef;
use std::borrow::Cow;
use std::collections::HashMap;
pub type StdlibInitFunc = Box<py_dyn_fn!(dyn Fn(&VirtualMachine) -> PyObjectRef)>;
pub type StdlibMap = HashMap<Cow<'static, str>, StdlibInitFunc, ahash::RandomState>;
@@ -152,12 +162,9 @@ pub fn get_module_inits() -> StdlibMap {
{
"symtable" => symtable::make_module,
}
#[cfg(any(unix, windows, target_os = "wasi"))]
{
os::MODULE_NAME => os::make_module,
}
#[cfg(any(unix, target_os = "wasi"))]
{
"posix" => posix::make_module,
"fcntl" => fcntl::make_module,
}
// disable some modules on WASM
@@ -195,6 +202,7 @@ pub fn get_module_inits() -> StdlibMap {
// Windows-only
#[cfg(windows)]
{
"nt" => nt::make_module,
"msvcrt" => msvcrt::make_module,
"_winapi" => winapi::make_module,
"winreg" => winreg::make_module,

View File

@@ -1,7 +1,7 @@
use super::os::errno_err;
use crate::{
builtins::{PyBytes, PyStrRef},
PyObjectRef, PyRef, PyResult, VirtualMachine,
suppress_iph, PyObjectRef, PyRef, PyResult, VirtualMachine,
};
use itertools::Itertools;
use winapi::{

430
vm/src/stdlib/nt.rs Normal file
View File

@@ -0,0 +1,430 @@
use crate::{PyObjectRef, VirtualMachine};
pub(crate) use module::raw_set_handle_inheritable;
pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef {
let module = module::make_module(vm);
super::os::extend_module(vm, &module);
module
}
#[pymodule(name = "nt")]
pub(crate) mod module {
use crate::{
builtins::{PyStrRef, PyTupleRef},
crt_fd::Fd,
exceptions::IntoPyException,
function::OptionalArg,
stdlib::os::{
errno_err, DirFd, FollowSymlinks, PyPathLike, SupportFunc, TargetIsDirectory, _os,
errno,
},
suppress_iph,
utils::Either,
PyResult, TryFromObject, VirtualMachine,
};
use std::io;
use std::{env, fs};
#[cfg(target_env = "msvc")]
use crate::builtins::PyListRef;
use crate::{builtins::PyDictRef, ItemProtocol};
use winapi::{um, vc::vcruntime::intptr_t};
#[pyattr]
use libc::{O_BINARY, O_TEMPORARY};
#[pyfunction]
pub(super) fn access(path: PyPathLike, mode: u8, vm: &VirtualMachine) -> PyResult<bool> {
use um::{fileapi, winnt};
let attr = unsafe { fileapi::GetFileAttributesW(path.to_widecstring(vm)?.as_ptr()) };
Ok(attr != fileapi::INVALID_FILE_ATTRIBUTES
&& (mode & 2 == 0
|| attr & winnt::FILE_ATTRIBUTE_READONLY == 0
|| attr & winnt::FILE_ATTRIBUTE_DIRECTORY != 0))
}
#[derive(FromArgs)]
pub(super) struct SimlinkArgs {
#[pyarg(any)]
src: PyPathLike,
#[pyarg(any)]
dst: PyPathLike,
#[pyarg(flatten)]
target_is_directory: TargetIsDirectory,
#[pyarg(flatten)]
_dir_fd: DirFd<{ _os::SYMLINK_DIR_FD as usize }>,
}
#[pyfunction]
pub(super) fn symlink(args: SimlinkArgs, vm: &VirtualMachine) -> PyResult<()> {
use std::os::windows::fs as win_fs;
let dir = args.target_is_directory.target_is_directory
|| args
.dst
.path
.parent()
.and_then(|dst_parent| dst_parent.join(&args.src).symlink_metadata().ok())
.map_or(false, |meta| meta.is_dir());
let res = if dir {
win_fs::symlink_dir(args.src.path, args.dst.path)
} else {
win_fs::symlink_file(args.src.path, args.dst.path)
};
res.map_err(|err| err.into_pyexception(vm))
}
#[pyfunction]
fn set_inheritable(fd: i32, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> {
let handle = Fd(fd).to_raw_handle().map_err(|e| e.into_pyexception(vm))?;
set_handle_inheritable(handle as _, inheritable, vm)
}
#[pyattr]
fn environ(vm: &VirtualMachine) -> PyDictRef {
let environ = vm.ctx.new_dict();
for (key, value) in env::vars() {
environ
.set_item(vm.ctx.new_utf8_str(key), vm.ctx.new_utf8_str(value), vm)
.unwrap();
}
environ
}
#[pyfunction]
fn chmod(
path: PyPathLike,
dir_fd: DirFd<0>,
mode: u32,
follow_symlinks: FollowSymlinks,
vm: &VirtualMachine,
) -> PyResult<()> {
const S_IWRITE: u32 = 128;
let [] = dir_fd.0;
let metadata = if follow_symlinks.0 {
fs::metadata(&path)
} else {
fs::symlink_metadata(&path)
};
let meta = metadata.map_err(|err| err.into_pyexception(vm))?;
let mut permissions = meta.permissions();
permissions.set_readonly(mode & S_IWRITE == 0);
fs::set_permissions(&path, permissions).map_err(|err| err.into_pyexception(vm))
}
// cwait is available on MSVC only (according to CPython)
#[cfg(target_env = "msvc")]
extern "C" {
fn _cwait(termstat: *mut i32, procHandle: intptr_t, action: i32) -> intptr_t;
}
#[cfg(target_env = "msvc")]
#[pyfunction]
fn waitpid(pid: intptr_t, opt: i32, vm: &VirtualMachine) -> PyResult<(intptr_t, i32)> {
let mut status = 0;
let pid = unsafe { suppress_iph!(_cwait(&mut status, pid, opt)) };
if pid == -1 {
Err(errno_err(vm))
} else {
Ok((pid, status << 8))
}
}
#[cfg(target_env = "msvc")]
#[pyfunction]
fn wait(vm: &VirtualMachine) -> PyResult<(intptr_t, i32)> {
waitpid(-1, 0, vm)
}
#[pyfunction]
fn kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> {
{
use um::{handleapi, processthreadsapi, wincon, winnt};
let sig = sig as u32;
let pid = pid as u32;
if sig == wincon::CTRL_C_EVENT || sig == wincon::CTRL_BREAK_EVENT {
let ret = unsafe { wincon::GenerateConsoleCtrlEvent(sig, pid) };
let res = if ret == 0 { Err(errno_err(vm)) } else { Ok(()) };
return res;
}
let h = unsafe { processthreadsapi::OpenProcess(winnt::PROCESS_ALL_ACCESS, 0, pid) };
if h.is_null() {
return Err(errno_err(vm));
}
let ret = unsafe { processthreadsapi::TerminateProcess(h, sig) };
let res = if ret == 0 { Err(errno_err(vm)) } else { Ok(()) };
unsafe { handleapi::CloseHandle(h) };
res
}
}
#[pyfunction]
fn get_terminal_size(
fd: OptionalArg<i32>,
vm: &VirtualMachine,
) -> PyResult<_os::PyTerminalSize> {
let (columns, lines) = {
use um::{handleapi, processenv, winbase, wincon};
let stdhandle = match fd {
OptionalArg::Present(0) => winbase::STD_INPUT_HANDLE,
OptionalArg::Present(1) | OptionalArg::Missing => winbase::STD_OUTPUT_HANDLE,
OptionalArg::Present(2) => winbase::STD_ERROR_HANDLE,
_ => return Err(vm.new_value_error("bad file descriptor".to_owned())),
};
let h = unsafe { processenv::GetStdHandle(stdhandle) };
if h.is_null() {
return Err(vm.new_os_error("handle cannot be retrieved".to_owned()));
}
if h == handleapi::INVALID_HANDLE_VALUE {
return Err(errno_err(vm));
}
let mut csbi = wincon::CONSOLE_SCREEN_BUFFER_INFO::default();
let ret = unsafe { wincon::GetConsoleScreenBufferInfo(h, &mut csbi) };
if ret == 0 {
return Err(errno_err(vm));
}
let w = csbi.srWindow;
(
(w.Right - w.Left + 1) as usize,
(w.Bottom - w.Top + 1) as usize,
)
};
Ok(_os::PyTerminalSize { columns, lines })
}
#[cfg(target_env = "msvc")]
type InvalidParamHandler = extern "C" fn(
*const libc::wchar_t,
*const libc::wchar_t,
*const libc::wchar_t,
libc::c_uint,
libc::uintptr_t,
);
#[cfg(target_env = "msvc")]
extern "C" {
#[doc(hidden)]
pub fn _set_thread_local_invalid_parameter_handler(
pNew: InvalidParamHandler,
) -> InvalidParamHandler;
}
#[cfg(target_env = "msvc")]
#[doc(hidden)]
pub extern "C" fn silent_iph_handler(
_: *const libc::wchar_t,
_: *const libc::wchar_t,
_: *const libc::wchar_t,
_: libc::c_uint,
_: libc::uintptr_t,
) {
}
#[cfg(target_env = "msvc")]
extern "C" {
fn _wexecv(cmdname: *const u16, argv: *const *const u16) -> intptr_t;
}
#[cfg(target_env = "msvc")]
#[pyfunction]
fn execv(
path: PyStrRef,
argv: Either<PyListRef, PyTupleRef>,
vm: &VirtualMachine,
) -> PyResult<()> {
use std::iter::once;
let make_widestring = |s: &str| {
widestring::WideCString::from_os_str(s).map_err(|err| err.into_pyexception(vm))
};
let path = make_widestring(path.as_str())?;
let argv = vm.extract_elements_func(argv.as_object(), |obj| {
let arg = PyStrRef::try_from_object(vm, obj)?;
make_widestring(arg.as_str())
})?;
let first = argv
.first()
.ok_or_else(|| vm.new_value_error("execv() arg 2 must not be empty".to_owned()))?;
if first.is_empty() {
return Err(
vm.new_value_error("execv() arg 2 first element cannot be empty".to_owned())
);
}
let argv_execv: Vec<*const u16> = argv
.iter()
.map(|v| v.as_ptr())
.chain(once(std::ptr::null()))
.collect();
if (unsafe { suppress_iph!(_wexecv(path.as_ptr(), argv_execv.as_ptr())) } == -1) {
Err(errno_err(vm))
} else {
Ok(())
}
}
#[pyfunction]
fn _getfinalpathname(path: PyPathLike, vm: &VirtualMachine) -> PyResult {
let real = path
.as_ref()
.canonicalize()
.map_err(|e| e.into_pyexception(vm))?;
path.mode.process_path(real, vm)
}
#[pyfunction]
fn _getfullpathname(path: PyPathLike, vm: &VirtualMachine) -> PyResult {
let wpath = path.to_widecstring(vm)?;
let mut buffer = vec![0u16; winapi::shared::minwindef::MAX_PATH];
let ret = unsafe {
um::fileapi::GetFullPathNameW(
wpath.as_ptr(),
buffer.len() as _,
buffer.as_mut_ptr(),
std::ptr::null_mut(),
)
};
if ret == 0 {
return Err(errno_err(vm));
}
if ret as usize > buffer.len() {
buffer.resize(ret as usize, 0);
let ret = unsafe {
um::fileapi::GetFullPathNameW(
wpath.as_ptr(),
buffer.len() as _,
buffer.as_mut_ptr(),
std::ptr::null_mut(),
)
};
if ret == 0 {
return Err(errno_err(vm));
}
}
let buffer = widestring::WideCString::from_vec_with_nul(buffer).unwrap();
path.mode.process_path(buffer.to_os_string(), vm)
}
#[pyfunction]
fn _getvolumepathname(path: PyPathLike, vm: &VirtualMachine) -> PyResult {
let wide = path.to_widecstring(vm)?;
let buflen = std::cmp::max(wide.len(), winapi::shared::minwindef::MAX_PATH);
let mut buffer = vec![0u16; buflen];
let ret = unsafe {
um::fileapi::GetVolumePathNameW(wide.as_ptr(), buffer.as_mut_ptr(), buflen as _)
};
if ret == 0 {
return Err(errno_err(vm));
}
let buffer = widestring::WideCString::from_vec_with_nul(buffer).unwrap();
path.mode.process_path(buffer.to_os_string(), vm)
}
#[pyfunction]
fn _getdiskusage(path: PyPathLike, vm: &VirtualMachine) -> PyResult<(u64, u64)> {
use um::fileapi::GetDiskFreeSpaceExW;
use winapi::shared::{ntdef::ULARGE_INTEGER, winerror};
let wpath = path.to_widecstring(vm)?;
let mut _free_to_me = ULARGE_INTEGER::default();
let mut total = ULARGE_INTEGER::default();
let mut free = ULARGE_INTEGER::default();
let ret =
unsafe { GetDiskFreeSpaceExW(wpath.as_ptr(), &mut _free_to_me, &mut total, &mut free) };
if ret != 0 {
return unsafe { Ok((*total.QuadPart(), *free.QuadPart())) };
}
let err = io::Error::last_os_error();
if err.raw_os_error() == Some(winerror::ERROR_DIRECTORY as i32) {
if let Some(parent) = path.as_ref().parent() {
let parent = widestring::WideCString::from_os_str(parent).unwrap();
let ret = unsafe {
GetDiskFreeSpaceExW(parent.as_ptr(), &mut _free_to_me, &mut total, &mut free)
};
if ret == 0 {
return Err(errno_err(vm));
} else {
return unsafe { Ok((*total.QuadPart(), *free.QuadPart())) };
}
}
}
return Err(err.into_pyexception(vm));
}
#[pyfunction]
fn get_handle_inheritable(handle: intptr_t, vm: &VirtualMachine) -> PyResult<bool> {
let mut flags = 0;
if unsafe { um::handleapi::GetHandleInformation(handle as _, &mut flags) } == 0 {
Err(errno_err(vm))
} else {
Ok(flags & um::winbase::HANDLE_FLAG_INHERIT != 0)
}
}
pub(crate) fn raw_set_handle_inheritable(
handle: intptr_t,
inheritable: bool,
) -> io::Result<()> {
use um::winbase::HANDLE_FLAG_INHERIT;
let flags = if inheritable { HANDLE_FLAG_INHERIT } else { 0 };
let res =
unsafe { um::handleapi::SetHandleInformation(handle as _, HANDLE_FLAG_INHERIT, flags) };
if res == 0 {
Err(errno())
} else {
Ok(())
}
}
#[pyfunction]
fn set_handle_inheritable(
handle: intptr_t,
inheritable: bool,
vm: &VirtualMachine,
) -> PyResult<()> {
raw_set_handle_inheritable(handle, inheritable).map_err(|e| e.into_pyexception(vm))
}
#[pyfunction]
fn mkdir(
path: PyPathLike,
mode: OptionalArg<i32>,
dir_fd: DirFd<{ _os::MKDIR_DIR_FD as usize }>,
vm: &VirtualMachine,
) -> PyResult<()> {
let mode = mode.unwrap_or(0o777);
let [] = dir_fd.0;
let _ = mode;
let wide = path.to_widecstring(vm)?;
let res = unsafe { um::fileapi::CreateDirectoryW(wide.as_ptr(), std::ptr::null_mut()) };
if res == 0 {
return Err(errno_err(vm));
}
Ok(())
}
pub(crate) fn support_funcs() -> Vec<SupportFunc> {
Vec::new()
}
}
#[cfg(all(windows, target_env = "msvc"))]
#[macro_export]
macro_rules! suppress_iph {
($e:expr) => {{
let old = $crate::stdlib::nt::module::_set_thread_local_invalid_parameter_handler(
$crate::stdlib::nt::module::silent_iph_handler,
);
let ret = $e;
$crate::stdlib::nt::module::_set_thread_local_invalid_parameter_handler(old);
ret
}};
}

File diff suppressed because it is too large Load Diff

1658
vm/src/stdlib/posix.rs Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,75 @@
//! `posix` compatible module for `not(any(unix, windows))`
use crate::{PyObjectRef, VirtualMachine};
pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef {
let module = module::make_module(vm);
super::os::extend_module(vm, &module);
module
}
#[pymodule(name = "posix")]
pub(crate) mod module {
use crate::{
builtins::PyStrRef,
stdlib::os::{DirFd, PyPathLike, SupportFunc, TargetIsDirectory, _os},
PyResult, VirtualMachine,
};
use std::env;
#[cfg(unix)]
use std::os::unix::ffi as ffi_ext;
#[cfg(target_os = "wasi")]
use std::os::wasi::ffi as ffi_ext;
#[pyfunction]
pub(super) fn access(_path: PyStrRef, _mode: u8, vm: &VirtualMachine) -> PyResult<bool> {
os_unimpl("os.access", vm)
}
#[derive(FromArgs)]
#[allow(unused)]
pub(super) struct SimlinkArgs {
#[pyarg(any)]
src: PyPathLike,
#[pyarg(any)]
dst: PyPathLike,
#[pyarg(flatten)]
_target_is_directory: TargetIsDirectory,
#[pyarg(flatten)]
_dir_fd: DirFd<{ _os::SYMLINK_DIR_FD as usize }>,
}
#[pyfunction]
pub(super) fn symlink(_args: SimlinkArgs, vm: &VirtualMachine) -> PyResult<()> {
os_unimpl("os.symlink", vm)
}
#[cfg(target_os = "wasi")]
#[pyattr]
fn environ(vm: &VirtualMachine) -> crate::builtins::PyDictRef {
use crate::ItemProtocol;
use ffi_ext::OsStringExt;
let environ = vm.ctx.new_dict();
for (key, value) in env::vars_os() {
environ
.set_item(
vm.ctx.new_bytes(key.into_vec()),
vm.ctx.new_bytes(value.into_vec()),
vm,
)
.unwrap();
}
environ
}
#[allow(dead_code)]
fn os_unimpl<T>(func: &str, vm: &VirtualMachine) -> PyResult<T> {
Err(vm.new_os_error(format!("{} is not supported on this platform", func)))
}
pub(crate) fn support_funcs() -> Vec<SupportFunc> {
Vec::new()
}
}

View File

@@ -28,6 +28,9 @@ mod _posixsubprocess {
}
}
use super::os::PyPathLike;
use super::posix;
use crate::{PyObjectRef, PyResult, PySequence, TryFromObject, VirtualMachine};
use nix::{errno::Errno, unistd};
use std::convert::Infallible as Never;
#[cfg(not(target_os = "redox"))]
@@ -37,10 +40,6 @@ use std::io::{self, prelude::*};
#[cfg(not(target_os = "redox"))]
use std::os::unix::io::AsRawFd;
use super::os;
use crate::VirtualMachine;
use crate::{PyObjectRef, PyResult, PySequence, TryFromObject};
macro_rules! gen_args {
($($field:ident: $t:ty),*$(,)?) => {
#[derive(FromArgs)]
@@ -55,7 +54,7 @@ struct CStrPathLike {
}
impl TryFromObject for CStrPathLike {
fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
let s = os::PyPathLike::try_from_object(vm, obj)?.into_cstring(vm)?;
let s = PyPathLike::try_from_object(vm, obj)?.into_cstring(vm)?;
Ok(CStrPathLike { s })
}
}
@@ -93,7 +92,7 @@ fn exec(args: &ForkExecArgs, procargs: ProcArgs) -> ! {
fn exec_inner(args: &ForkExecArgs, procargs: ProcArgs) -> nix::Result<Never> {
for &fd in args.fds_to_keep.as_slice() {
if fd != args.errpipe_write {
os::raw_set_inheritable(fd, true)?
posix::raw_set_inheritable(fd, true)?
}
}
@@ -106,7 +105,7 @@ fn exec_inner(args: &ForkExecArgs, procargs: ProcArgs) -> nix::Result<Never> {
let c2pwrite = if args.c2pwrite == 0 {
let fd = unistd::dup(args.c2pwrite)?;
os::raw_set_inheritable(fd, true)?;
posix::raw_set_inheritable(fd, true)?;
fd
} else {
args.c2pwrite
@@ -115,12 +114,12 @@ fn exec_inner(args: &ForkExecArgs, procargs: ProcArgs) -> nix::Result<Never> {
let mut errwrite = args.errwrite;
while errwrite == 0 || errwrite == 1 {
errwrite = unistd::dup(errwrite)?;
os::raw_set_inheritable(errwrite, true)?;
posix::raw_set_inheritable(errwrite, true)?;
}
let dup_into_stdio = |fd, io_fd| {
if fd == io_fd {
os::raw_set_inheritable(fd, true)
posix::raw_set_inheritable(fd, true)
} else if fd != -1 {
unistd::dup2(fd, io_fd).map(drop)
} else {

View File

@@ -1778,7 +1778,7 @@ fn _socket_dup(x: PyObjectRef, vm: &VirtualMachine) -> PyResult<RawSocket> {
let newsock = sock.try_clone().map_err(|e| e.into_pyexception(vm))?;
let fd = into_sock_fileno(newsock);
#[cfg(windows)]
super::os::raw_set_handle_inheritable(fd as _, false).map_err(|e| e.into_pyexception(vm))?;
super::nt::raw_set_handle_inheritable(fd as _, false).map_err(|e| e.into_pyexception(vm))?;
Ok(fd)
}