forked from Rust-related/RustPython
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
5343c0bd0b |
@@ -1,5 +0,0 @@
|
||||
[target.'cfg(target_env = "msvc")']
|
||||
rustflags = "-C link-arg=/STACK:8000000"
|
||||
|
||||
[target.'cfg(all(target_os = "windows", not(target_env = "msvc")))']
|
||||
rustflags = "-C link-args=-Wl,--stack,8000000"
|
||||
298
.cspell.json
298
.cspell.json
@@ -1,298 +0,0 @@
|
||||
// See: https://github.com/streetsidesoftware/cspell/tree/master/packages/cspell
|
||||
{
|
||||
"version": "0.2",
|
||||
// language - current active spelling language
|
||||
"language": "en",
|
||||
// dictionaries - list of the names of the dictionaries to use
|
||||
"dictionaries": [
|
||||
"en_US",
|
||||
"softwareTerms",
|
||||
"c",
|
||||
"cpp",
|
||||
"python",
|
||||
"python-custom",
|
||||
"rust",
|
||||
"unix",
|
||||
"posix",
|
||||
"winapi"
|
||||
],
|
||||
// dictionaryDefinitions - this list defines any custom dictionaries to use
|
||||
"dictionaryDefinitions": [],
|
||||
"ignorePaths": [
|
||||
"**/__pycache__/**",
|
||||
"Lib/**"
|
||||
],
|
||||
// words - list of words to be always considered correct
|
||||
"words": [
|
||||
// Rust
|
||||
"ahash",
|
||||
"bidi",
|
||||
"biguint",
|
||||
"bindgen",
|
||||
"bitflags",
|
||||
"bstr",
|
||||
"byteorder",
|
||||
"chrono",
|
||||
"consts",
|
||||
"cstring",
|
||||
"flate2",
|
||||
"fract",
|
||||
"hasher",
|
||||
"idents",
|
||||
"indexmap",
|
||||
"insta",
|
||||
"keccak",
|
||||
"lalrpop",
|
||||
"libc",
|
||||
"libz",
|
||||
"longlong",
|
||||
"Manually",
|
||||
"maplit",
|
||||
"memmap",
|
||||
"metas",
|
||||
"modpow",
|
||||
"nanos",
|
||||
"peekable",
|
||||
"powc",
|
||||
"powf",
|
||||
"prepended",
|
||||
"punct",
|
||||
"replacen",
|
||||
"rsplitn",
|
||||
"rustc",
|
||||
"rustfmt",
|
||||
"seekfrom",
|
||||
"splitn",
|
||||
"subsec",
|
||||
"timsort",
|
||||
"trai",
|
||||
"ulonglong",
|
||||
"unic",
|
||||
"unistd",
|
||||
"winapi",
|
||||
"winsock",
|
||||
// Python
|
||||
"abstractmethods",
|
||||
"aiter",
|
||||
"anext",
|
||||
"arrayiterator",
|
||||
"arraytype",
|
||||
"asend",
|
||||
"athrow",
|
||||
"basicsize",
|
||||
"cformat",
|
||||
"classcell",
|
||||
"closesocket",
|
||||
"codepoint",
|
||||
"codepoints",
|
||||
"cpython",
|
||||
"decompressor",
|
||||
"defaultaction",
|
||||
"descr",
|
||||
"dictcomp",
|
||||
"dictitems",
|
||||
"dictkeys",
|
||||
"dictview",
|
||||
"docstring",
|
||||
"docstrings",
|
||||
"dunder",
|
||||
"eventmask",
|
||||
"fdel",
|
||||
"fget",
|
||||
"fileencoding",
|
||||
"fillchar",
|
||||
"finallyhandler",
|
||||
"frombytes",
|
||||
"fromhex",
|
||||
"fromunicode",
|
||||
"fset",
|
||||
"fspath",
|
||||
"fstring",
|
||||
"fstrings",
|
||||
"genexpr",
|
||||
"getattro",
|
||||
"getformat",
|
||||
"getnewargs",
|
||||
"getweakrefcount",
|
||||
"getweakrefs",
|
||||
"hostnames",
|
||||
"idiv",
|
||||
"impls",
|
||||
"infj",
|
||||
"instancecheck",
|
||||
"instanceof",
|
||||
"isabstractmethod",
|
||||
"itemiterator",
|
||||
"itemsize",
|
||||
"iternext",
|
||||
"keyiterator",
|
||||
"kwarg",
|
||||
"kwargs",
|
||||
"linearization",
|
||||
"linearize",
|
||||
"listcomp",
|
||||
"mappingproxy",
|
||||
"maxsplit",
|
||||
"memoryview",
|
||||
"memoryviewiterator",
|
||||
"metaclass",
|
||||
"metaclasses",
|
||||
"metatype",
|
||||
"mro",
|
||||
"mros",
|
||||
"nanj",
|
||||
"ndigits",
|
||||
"ndim",
|
||||
"nonbytes",
|
||||
"origname",
|
||||
"posixsubprocess",
|
||||
"pyexpat",
|
||||
"PYTHONDEBUG",
|
||||
"PYTHONHOME",
|
||||
"PYTHONINSPECT",
|
||||
"PYTHONOPTIMIZE",
|
||||
"PYTHONPATH",
|
||||
"PYTHONPATH",
|
||||
"PYTHONVERBOSE",
|
||||
"PYTHONWARNINGS",
|
||||
"qualname",
|
||||
"radd",
|
||||
"rdiv",
|
||||
"rdivmod",
|
||||
"reconstructor",
|
||||
"reversevalueiterator",
|
||||
"rfloordiv",
|
||||
"rlshift",
|
||||
"rmod",
|
||||
"rpow",
|
||||
"rrshift",
|
||||
"rsub",
|
||||
"rtruediv",
|
||||
"scproxy",
|
||||
"setattro",
|
||||
"setcomp",
|
||||
"showwarnmsg",
|
||||
"warnmsg",
|
||||
"stacklevel",
|
||||
"subclasscheck",
|
||||
"subclasshook",
|
||||
"unionable",
|
||||
"unraisablehook",
|
||||
"valueiterator",
|
||||
"vararg",
|
||||
"varargs",
|
||||
"varnames",
|
||||
"warningregistry",
|
||||
"warnopts",
|
||||
"weakproxy",
|
||||
"xopts",
|
||||
// RustPython
|
||||
"baseclass",
|
||||
"Bytecode",
|
||||
"cfgs",
|
||||
"codegen",
|
||||
"dedentations",
|
||||
"dedents",
|
||||
"deduped",
|
||||
"downcasted",
|
||||
"dumpable",
|
||||
"GetSet",
|
||||
"internable",
|
||||
"makeunicodedata",
|
||||
"miri",
|
||||
"notrace",
|
||||
"pyarg",
|
||||
"pyarg",
|
||||
"pyargs",
|
||||
"PyAttr",
|
||||
"pyc",
|
||||
"PyClass",
|
||||
"PyClassMethod",
|
||||
"PyException",
|
||||
"PyFunction",
|
||||
"pygetset",
|
||||
"pyimpl",
|
||||
"pymember",
|
||||
"PyMethod",
|
||||
"PyModule",
|
||||
"pyname",
|
||||
"pyobj",
|
||||
"PyObject",
|
||||
"pypayload",
|
||||
"PyProperty",
|
||||
"pyref",
|
||||
"PyResult",
|
||||
"pyslot",
|
||||
"PyStaticMethod",
|
||||
"pystr",
|
||||
"pystruct",
|
||||
"pystructseq",
|
||||
"pytrace",
|
||||
"reducelib",
|
||||
"richcompare",
|
||||
"RustPython",
|
||||
"struc",
|
||||
"tracebacks",
|
||||
"typealiases",
|
||||
"Unconstructible",
|
||||
"unhashable",
|
||||
"uninit",
|
||||
"unraisable",
|
||||
"wasi",
|
||||
"zelf",
|
||||
// cpython
|
||||
"argtypes",
|
||||
"asdl",
|
||||
"asname",
|
||||
"augassign",
|
||||
"badsyntax",
|
||||
"basetype",
|
||||
"boolop",
|
||||
"bxor",
|
||||
"cellarg",
|
||||
"cellvar",
|
||||
"cellvars",
|
||||
"cmpop",
|
||||
"dictoffset",
|
||||
"elts",
|
||||
"excepthandler",
|
||||
"finalbody",
|
||||
"freevar",
|
||||
"freevars",
|
||||
"fromlist",
|
||||
"heaptype",
|
||||
"IMMUTABLETYPE",
|
||||
"kwonlyarg",
|
||||
"kwonlyargs",
|
||||
"linearise",
|
||||
"maxdepth",
|
||||
"mult",
|
||||
"nkwargs",
|
||||
"orelse",
|
||||
"patma",
|
||||
"posonlyarg",
|
||||
"posonlyargs",
|
||||
"prec",
|
||||
"stackdepth",
|
||||
"unaryop",
|
||||
"unparse",
|
||||
"unparser",
|
||||
"VARKEYWORDS",
|
||||
"varkwarg",
|
||||
"wbits",
|
||||
"withitem",
|
||||
"withs"
|
||||
],
|
||||
// flagWords - list of words to be always considered incorrect
|
||||
"flagWords": [
|
||||
],
|
||||
"ignoreRegExpList": [
|
||||
],
|
||||
// languageSettings - allow for per programming language configuration settings.
|
||||
"languageSettings": [
|
||||
{
|
||||
"languageId": "python",
|
||||
"locale": "en"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"image": "mcr.microsoft.com/devcontainers/universal:2",
|
||||
"features": {
|
||||
"ghcr.io/devcontainers/features/rust:1": {}
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@
|
||||
.vscode
|
||||
wasm-pack.log
|
||||
.idea/
|
||||
extra_tests/snippets/resources
|
||||
tests/snippets/resources
|
||||
|
||||
flame-graph.html
|
||||
flame.txt
|
||||
|
||||
7
.gitattributes
vendored
7
.gitattributes
vendored
@@ -1,6 +1 @@
|
||||
Lib/** linguist-vendored
|
||||
Cargo.lock linguist-generated -merge
|
||||
*.snap linguist-generated -merge
|
||||
vm/src/stdlib/ast/gen.rs linguist-generated -merge
|
||||
Lib/*.py text working-tree-encoding=UTF-8 eol=LF
|
||||
**/*.rs text working-tree-encoding=UTF-8 eol=LF
|
||||
Lib/* linguist-vendored
|
||||
|
||||
16
.github/ISSUE_TEMPLATE/empty.md
vendored
16
.github/ISSUE_TEMPLATE/empty.md
vendored
@@ -1,16 +0,0 @@
|
||||
---
|
||||
name: Generic issue template
|
||||
about: which is not covered by other templates
|
||||
title: ''
|
||||
labels:
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
<!-- Short description of the issue. -->
|
||||
|
||||
## Details
|
||||
|
||||
<!-- Whatever you want to share -->
|
||||
16
.github/ISSUE_TEMPLATE/feature-request.md
vendored
16
.github/ISSUE_TEMPLATE/feature-request.md
vendored
@@ -1,16 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Request a feature to use RustPython (as a Rust library)
|
||||
title: ''
|
||||
labels: C-enhancement
|
||||
assignees: 'youknowone'
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
<!-- Short description of the request. Please use incompatibility form to report missing features as Python interpreter -->
|
||||
|
||||
## Expected use case
|
||||
|
||||
<!-- By sharing detailed use case, we can understand the requirements better! If it will be used by open source projects, please also share the project URL. -->
|
||||
24
.github/ISSUE_TEMPLATE/report-bug.md
vendored
24
.github/ISSUE_TEMPLATE/report-bug.md
vendored
@@ -1,24 +0,0 @@
|
||||
---
|
||||
name: Report bugs
|
||||
about: Report a bug not related to CPython compatibility
|
||||
title: ''
|
||||
labels: C-bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
<!-- Short description of the bug -->
|
||||
|
||||
## Expected
|
||||
|
||||
<!-- What's the expected result? Using ``` ``` block is preferred for text. -->
|
||||
|
||||
## Actual
|
||||
|
||||
<!-- What's the actual result? Using ``` ``` block is preferred for text. -->
|
||||
|
||||
## Python Documentation
|
||||
|
||||
<!-- If applicable. -->
|
||||
@@ -2,7 +2,7 @@
|
||||
name: Report incompatibility
|
||||
about: Report an incompatibility between RustPython and CPython
|
||||
title: ''
|
||||
labels: C-compat
|
||||
labels: feat
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
@@ -11,6 +11,6 @@ assignees: ''
|
||||
|
||||
<!-- What Python feature is missing from RustPython? Give a short description of the feature and how you ran into its absence. -->
|
||||
|
||||
## Python Documentation or reference to CPython source code
|
||||
## Python Documentation
|
||||
|
||||
<!-- Give a link to the feature in the CPython documentation (https://docs.python.org/3/) in order to assist in its implementation. -->
|
||||
|
||||
425
.github/workflows/ci.yaml
vendored
425
.github/workflows/ci.yaml
vendored
@@ -1,117 +1,15 @@
|
||||
on:
|
||||
push:
|
||||
branches: [main, release]
|
||||
pull_request:
|
||||
types: [unlabeled, opened, synchronize, reopened]
|
||||
merge_group:
|
||||
branches: [master, release]
|
||||
pull_request:
|
||||
|
||||
name: CI
|
||||
|
||||
# Cancel previous workflows if they are the same workflow on same ref (branch/tags)
|
||||
# with the same event (push/pull_request) even they are in progress.
|
||||
# This setting will help reduce the number of duplicated workflows.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
|
||||
# Skip additional tests on Windows. They are checked on Linux and MacOS.
|
||||
WINDOWS_SKIPS: >-
|
||||
test_glob
|
||||
test_importlib
|
||||
test_io
|
||||
test_os
|
||||
test_pathlib
|
||||
test_posixpath
|
||||
test_shutil
|
||||
test_venv
|
||||
# configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417
|
||||
# socketserver: seems related to configparser crash.
|
||||
MACOS_SKIPS: >-
|
||||
test_configparser
|
||||
test_socketserver
|
||||
# PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently
|
||||
# only run on Linux to speed up the CI.
|
||||
PLATFORM_INDEPENDENT_TESTS: >-
|
||||
test_argparse
|
||||
test_array
|
||||
test_asyncgen
|
||||
test_binop
|
||||
test_bisect
|
||||
test_bool
|
||||
test_bytes
|
||||
test_call
|
||||
test_class
|
||||
test_cmath
|
||||
test_collections
|
||||
test_complex
|
||||
test_contains
|
||||
test_copy
|
||||
test_dataclasses
|
||||
test_decimal
|
||||
test_decorators
|
||||
test_defaultdict
|
||||
test_deque
|
||||
test_dict
|
||||
test_dictcomps
|
||||
test_dictviews
|
||||
test_dis
|
||||
test_enumerate
|
||||
test_exception_variations
|
||||
test_exceptions
|
||||
test_float
|
||||
test_format
|
||||
test_fractions
|
||||
test_genericalias
|
||||
test_genericclass
|
||||
test_grammar
|
||||
test_range
|
||||
test_index
|
||||
test_int
|
||||
test_int_literal
|
||||
test_isinstance
|
||||
test_iter
|
||||
test_iterlen
|
||||
test_itertools
|
||||
test_json
|
||||
test_keyword
|
||||
test_keywordonlyarg
|
||||
test_list
|
||||
test_long
|
||||
test_longexp
|
||||
test_math
|
||||
test_operator
|
||||
test_ordered_dict
|
||||
test_pow
|
||||
test_raise
|
||||
test_richcmp
|
||||
test_scope
|
||||
test_set
|
||||
test_slice
|
||||
test_sort
|
||||
test_string
|
||||
test_string_literals
|
||||
test_strtod
|
||||
test_structseq
|
||||
test_subclassinit
|
||||
test_super
|
||||
test_syntax
|
||||
test_tuple
|
||||
test_types
|
||||
test_unary
|
||||
test_unicode
|
||||
test_unpack
|
||||
test_weakref
|
||||
test_yield_from
|
||||
# Python version targeted by the CI.
|
||||
PYTHON_VERSION: "3.12.0"
|
||||
CARGO_ARGS: --all --features ssl
|
||||
|
||||
jobs:
|
||||
rust_tests:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
env:
|
||||
RUST_BACKTRACE: full
|
||||
name: Run rust tests
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
@@ -119,123 +17,25 @@ jobs:
|
||||
os: [macos-latest, ubuntu-latest, windows-latest]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
|
||||
- uses: actions/checkout@master
|
||||
- name: Set up the Windows environment
|
||||
shell: bash
|
||||
run: |
|
||||
cargo install --target-dir=target -v cargo-vcpkg
|
||||
cargo vcpkg -v build
|
||||
powershell.exe scripts/symlinks-to-hardlinks.ps1
|
||||
if: runner.os == 'Windows'
|
||||
- name: Set up the Mac environment
|
||||
run: brew install autoconf automake libtool
|
||||
if: runner.os == 'macOS'
|
||||
|
||||
- name: run clippy
|
||||
run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --exclude rustpython_wasm -- -Dwarnings
|
||||
|
||||
- name: Cache cargo dependencies
|
||||
uses: actions/cache@v1
|
||||
with:
|
||||
key: ${{ runner.os }}-rust_tests-${{ hashFiles('Cargo.lock') }}
|
||||
path: target
|
||||
restore-keys: |
|
||||
${{ runner.os }}-rust_tests-
|
||||
- name: run rust tests
|
||||
run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }}
|
||||
if: runner.os != 'macOS'
|
||||
# temp skip ssl linking for Mac to avoid CI failure
|
||||
- name: run rust tests (MacOS no ssl)
|
||||
run: cargo test --workspace --exclude rustpython_wasm --verbose --no-default-features --features threading,stdlib,zlib,importlib,encodings,jit
|
||||
if: runner.os == 'macOS'
|
||||
|
||||
- name: check compilation without threading
|
||||
run: cargo check ${{ env.CARGO_ARGS }}
|
||||
|
||||
- name: prepare AppleSilicon build
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
target: aarch64-apple-darwin
|
||||
if: runner.os == 'macOS'
|
||||
- name: Check compilation for Apple Silicon
|
||||
run: cargo check --target aarch64-apple-darwin
|
||||
if: runner.os == 'macOS'
|
||||
- name: prepare iOS build
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: aarch64-apple-ios
|
||||
if: runner.os == 'macOS'
|
||||
- name: Check compilation for iOS
|
||||
run: cargo check --target aarch64-apple-ios
|
||||
if: runner.os == 'macOS'
|
||||
|
||||
exotic_targets:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
name: Ensure compilation on various targets
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: i686-unknown-linux-gnu
|
||||
|
||||
- name: Install gcc-multilib and musl-tools
|
||||
run: sudo apt-get update && sudo apt-get install gcc-multilib musl-tools
|
||||
- name: Check compilation for x86 32bit
|
||||
run: cargo check --target i686-unknown-linux-gnu
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: aarch64-linux-android
|
||||
|
||||
- name: Check compilation for android
|
||||
run: cargo check --target aarch64-linux-android
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: aarch64-unknown-linux-gnu
|
||||
|
||||
- name: Install gcc-aarch64-linux-gnu
|
||||
run: sudo apt install gcc-aarch64-linux-gnu
|
||||
- name: Check compilation for aarch64 linux gnu
|
||||
run: cargo check --target aarch64-unknown-linux-gnu
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: i686-unknown-linux-musl
|
||||
|
||||
- name: Check compilation for musl
|
||||
run: cargo check --target i686-unknown-linux-musl
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: x86_64-unknown-freebsd
|
||||
|
||||
- name: Check compilation for freebsd
|
||||
run: cargo check --target x86_64-unknown-freebsd
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: wasm32-unknown-unknown
|
||||
|
||||
- name: Check compilation for wasm32
|
||||
run: cargo check --target wasm32-unknown-unknown --no-default-features
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: x86_64-unknown-freebsd
|
||||
|
||||
- name: Check compilation for freeBSD
|
||||
run: cargo check --target x86_64-unknown-freebsd
|
||||
|
||||
- name: Prepare repository for redox compilation
|
||||
run: bash scripts/redox/uncomment-cargo.sh
|
||||
- name: Check compilation for Redox
|
||||
uses: coolreader18/redoxer-action@v1
|
||||
with:
|
||||
command: check
|
||||
command: test
|
||||
args: --verbose ${{ env.CARGO_ARGS }}
|
||||
|
||||
snippets_cpython:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
env:
|
||||
RUST_BACKTRACE: full
|
||||
name: Run snippets and cpython tests
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
@@ -243,143 +43,113 @@ jobs:
|
||||
os: [macos-latest, ubuntu-latest, windows-latest]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
- uses: actions/checkout@master
|
||||
- name: Set up the Windows environment
|
||||
shell: bash
|
||||
run: |
|
||||
cargo install cargo-vcpkg
|
||||
cargo vcpkg build
|
||||
powershell.exe scripts/symlinks-to-hardlinks.ps1
|
||||
if: runner.os == 'Windows'
|
||||
- name: Set up the Mac environment
|
||||
run: brew install autoconf automake libtool openssl@3
|
||||
if: runner.os == 'macOS'
|
||||
|
||||
- name: build rustpython
|
||||
run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }}
|
||||
- uses: actions/setup-python@v4
|
||||
- name: Cache cargo dependencies
|
||||
uses: actions/cache@v1
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
key: ${{ runner.os }}-snippets-${{ hashFiles('Cargo.lock') }}
|
||||
path: target
|
||||
restore-keys: |
|
||||
${{ runner.os }}-snippets-
|
||||
- name: build rustpython
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: build
|
||||
args: --release --verbose ${{ env.CARGO_ARGS }}
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: 3.8
|
||||
- name: Install pipenv
|
||||
run: |
|
||||
python -V
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install pipenv
|
||||
- run: pipenv install
|
||||
working-directory: ./tests
|
||||
- name: run snippets
|
||||
run: python -m pip install -r requirements.txt && pytest -v
|
||||
working-directory: ./extra_tests
|
||||
- if: runner.os == 'Linux'
|
||||
name: run cpython platform-independent tests
|
||||
run:
|
||||
target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }}
|
||||
- if: runner.os == 'Linux'
|
||||
name: run cpython platform-dependent tests (Linux)
|
||||
run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }}
|
||||
- if: runner.os == 'macOS'
|
||||
name: run cpython platform-dependent tests (MacOS)
|
||||
run: target/release/rustpython -m test -j 1 all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.MACOS_SKIPS }}
|
||||
- if: runner.os == 'Windows'
|
||||
name: run cpython platform-dependent tests (windows partial - fixme)
|
||||
run:
|
||||
target/release/rustpython -m test -j 1 all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }}
|
||||
- if: runner.os != 'Windows'
|
||||
name: check that --install-pip succeeds
|
||||
run: |
|
||||
mkdir site-packages
|
||||
target/release/rustpython --install-pip ensurepip --user
|
||||
- if: runner.os != 'Windows'
|
||||
name: Check that ensurepip succeeds.
|
||||
run: |
|
||||
target/release/rustpython -m ensurepip
|
||||
target/release/rustpython -c "import pip"
|
||||
- if: runner.os != 'Windows'
|
||||
name: Check if pip inside venv is functional
|
||||
run: |
|
||||
target/release/rustpython -m venv testvenv
|
||||
testvenv/bin/rustpython -m pip install wheel
|
||||
- name: Check whats_left is not broken
|
||||
run: python -I whats_left.py
|
||||
run: pipenv run pytest -v
|
||||
working-directory: ./tests
|
||||
- name: run cpython tests
|
||||
run: target/release/rustpython -m test -v
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
if: runner.os != 'Windows'
|
||||
|
||||
lint:
|
||||
format:
|
||||
name: Check Rust code with rustfmt and clippy
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/checkout@master
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
components: rustfmt, clippy
|
||||
profile: minimal
|
||||
toolchain: stable
|
||||
components: rustfmt
|
||||
override: true
|
||||
- name: run rustfmt
|
||||
run: cargo fmt --check
|
||||
- name: run clippy on wasm
|
||||
run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
|
||||
- uses: actions/setup-python@v4
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
- name: install ruff
|
||||
run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778
|
||||
- name: run python lint
|
||||
run: ruff extra_tests wasm examples --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source
|
||||
- name: install prettier
|
||||
run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH
|
||||
- name: check wasm code with prettier
|
||||
# prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506
|
||||
run: cd wasm && git ls-files -z | xargs -0 prettier --check -u
|
||||
command: fmt
|
||||
args: --all -- --check
|
||||
- name: run clippy
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: clippy
|
||||
args: ${{ env.CARGO_ARGS }} -- -Dwarnings
|
||||
|
||||
miri:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
name: Run tests under miri
|
||||
lint:
|
||||
name: Lint Python code with flake8
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@master
|
||||
- uses: actions/checkout@master
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
toolchain: nightly
|
||||
components: miri
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Run tests under miri
|
||||
# miri-ignore-leaks because the type-object circular reference means that there will always be
|
||||
# a memory leak, at least until we have proper cyclic gc
|
||||
run: MIRIFLAGS='-Zmiri-ignore-leaks' cargo +nightly miri test -p rustpython-vm -- miri_test
|
||||
python-version: 3.8
|
||||
- name: install flake8
|
||||
run: python -m pip install flake8
|
||||
- name: run lint
|
||||
run: flake8 . --count --exclude=./.*,./Lib,./vm/Lib --select=E9,F63,F7,F82 --show-source --statistics
|
||||
|
||||
wasm:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
name: Check the WASM package and demo
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: actions/checkout@master
|
||||
- name: Cache cargo dependencies
|
||||
uses: actions/cache@v1
|
||||
with:
|
||||
key: ${{ runner.os }}-wasm-${{ hashFiles('**/Cargo.lock') }}
|
||||
path: target
|
||||
restore-keys: |
|
||||
${{ runner.os }}-wasm-
|
||||
- name: install wasm-pack
|
||||
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
|
||||
- name: install geckodriver
|
||||
run: |
|
||||
wget https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz
|
||||
wget https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckodriver-v0.24.0-linux32.tar.gz
|
||||
mkdir geckodriver
|
||||
tar -xzf geckodriver-v0.30.0-linux64.tar.gz -C geckodriver
|
||||
- uses: actions/setup-python@v4
|
||||
tar -xzf geckodriver-v0.24.0-linux32.tar.gz -C geckodriver
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
- run: python -m pip install -r requirements.txt
|
||||
python-version: 3.8
|
||||
- name: Install pipenv
|
||||
run: |
|
||||
python -V
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install pipenv
|
||||
- run: pipenv install
|
||||
working-directory: ./wasm/tests
|
||||
- uses: actions/setup-node@v3
|
||||
- uses: actions/setup-node@v1
|
||||
- name: run test
|
||||
run: |
|
||||
export PATH=$PATH:`pwd`/../../geckodriver
|
||||
npm install
|
||||
npm run test
|
||||
env:
|
||||
NODE_OPTIONS: "--openssl-legacy-provider"
|
||||
working-directory: ./wasm/demo
|
||||
- name: build notebook demo
|
||||
if: github.ref == 'refs/heads/release'
|
||||
run: |
|
||||
npm install
|
||||
npm run dist
|
||||
mv dist ../demo/dist/notebook
|
||||
env:
|
||||
NODE_OPTIONS: "--openssl-legacy-provider"
|
||||
working-directory: ./wasm/notebook
|
||||
- name: Deploy demo to Github Pages
|
||||
if: success() && github.ref == 'refs/heads/release'
|
||||
uses: peaceiris/actions-gh-pages@v2
|
||||
@@ -389,22 +159,3 @@ jobs:
|
||||
EXTERNAL_REPOSITORY: RustPython/demo
|
||||
PUBLISH_BRANCH: master
|
||||
|
||||
wasm-wasi:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
name: Run snippets and cpython tests on wasm-wasi
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: wasm32-wasi
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Setup Wasmer
|
||||
uses: wasmerio/setup-wasmer@v2
|
||||
- name: Install clang
|
||||
run: sudo apt-get update && sudo apt-get install clang -y
|
||||
- name: build rustpython
|
||||
run: cargo build --release --target wasm32-wasi --features freeze-stdlib,stdlib --verbose
|
||||
- name: run snippets
|
||||
run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py
|
||||
|
||||
172
.github/workflows/cron-ci.yaml
vendored
172
.github/workflows/cron-ci.yaml
vendored
@@ -1,53 +1,74 @@
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 6'
|
||||
workflow_dispatch:
|
||||
|
||||
name: Periodic checks/tasks
|
||||
|
||||
env:
|
||||
CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
|
||||
PYTHON_VERSION: "3.12.0"
|
||||
|
||||
jobs:
|
||||
# codecov collects code coverage data from the rust tests, python snippets and python test suite.
|
||||
# This is done using cargo-llvm-cov, which is a wrapper around llvm-cov.
|
||||
redox:
|
||||
name: Check compilation on Redox
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: redoxos/redoxer:latest
|
||||
steps:
|
||||
- uses: actions/checkout@master
|
||||
- name: prepare repository for redoxer compilation
|
||||
run: bash scripts/redox/uncomment-cargo.sh
|
||||
- name: compile for redox
|
||||
run: redoxer build --verbose
|
||||
|
||||
codecov:
|
||||
name: Collect code coverage data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: taiki-e/install-action@cargo-llvm-cov
|
||||
- uses: actions/setup-python@v4
|
||||
- uses: actions/checkout@master
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
- run: sudo apt-get update && sudo apt-get -y install lcov
|
||||
- name: Run cargo-llvm-cov with Rust tests.
|
||||
run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
|
||||
- name: Run cargo-llvm-cov with Python snippets.
|
||||
run: python scripts/cargo-llvm-cov.py
|
||||
continue-on-error: true
|
||||
- name: Run cargo-llvm-cov with Python test suite.
|
||||
run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed
|
||||
continue-on-error: true
|
||||
- name: Prepare code coverage data
|
||||
run: cargo llvm-cov report --lcov --output-path='codecov.lcov'
|
||||
- name: Upload to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
toolchain: nightly
|
||||
override: true
|
||||
- uses: actions-rs/cargo@v1
|
||||
with:
|
||||
file: ./codecov.lcov
|
||||
command: build
|
||||
args: --verbose
|
||||
env:
|
||||
CARGO_INCREMENTAL: '0'
|
||||
RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests' # -Cpanic=abort
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: 3.8
|
||||
- name: Install pipenv
|
||||
run: |
|
||||
python -V
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install pipenv
|
||||
- run: pipenv install
|
||||
working-directory: ./tests
|
||||
- name: run snippets
|
||||
run: pipenv run pytest -v
|
||||
working-directory: ./tests
|
||||
env:
|
||||
RUSTPYTHON_DEBUG: 'true'
|
||||
- name: run cpython tests
|
||||
run: cargo run -- -m test -v
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
- uses: actions-rs/grcov@v0.1
|
||||
id: coverage
|
||||
- name: upload to Codecov
|
||||
uses: codecov/codecov-action@v1
|
||||
with:
|
||||
file: ${{ steps.coverage.outputs.report }}
|
||||
|
||||
testdata:
|
||||
name: Collect regression test data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/checkout@master
|
||||
- name: build rustpython
|
||||
run: cargo build --release --verbose
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: build
|
||||
args: --release --verbose --all
|
||||
- name: collect tests data
|
||||
run: cargo run --release extra_tests/jsontests.py
|
||||
run: cargo run --release tests/jsontests.py
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
- name: upload tests data to the website
|
||||
@@ -61,88 +82,7 @@ jobs:
|
||||
|
||||
git clone git@github.com:RustPython/rustpython.github.io.git website
|
||||
cd website
|
||||
cp ../extra_tests/cpython_tests_results.json ./_data/regrtests_results.json
|
||||
cp ../tests/cpython_tests_results.json ./_data/regrtests_results.json
|
||||
git add ./_data/regrtests_results.json
|
||||
if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results" --author="$GITHUB_ACTOR"; then
|
||||
git push
|
||||
fi
|
||||
|
||||
whatsleft:
|
||||
name: Collect what is left data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
- name: build rustpython
|
||||
run: cargo build --release --verbose
|
||||
- name: Collect what is left data
|
||||
run: |
|
||||
chmod +x ./whats_left.py
|
||||
./whats_left.py > whats_left.temp
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
- name: Upload data to the website
|
||||
env:
|
||||
SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
|
||||
GITHUB_ACTOR: ${{ github.actor }}
|
||||
run: |
|
||||
echo "$SSHKEY" >~/github_key
|
||||
chmod 600 ~/github_key
|
||||
export GIT_SSH_COMMAND="ssh -i ~/github_key"
|
||||
|
||||
git clone git@github.com:RustPython/rustpython.github.io.git website
|
||||
cd website
|
||||
[ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp
|
||||
cp ../whats_left.temp ./_data/whats_left.temp
|
||||
git add -A
|
||||
if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then
|
||||
git push
|
||||
fi
|
||||
|
||||
benchmark:
|
||||
name: Collect benchmark data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.9
|
||||
- run: cargo install cargo-criterion
|
||||
- name: build benchmarks
|
||||
run: cargo build --release --benches
|
||||
- name: collect execution benchmark data
|
||||
run: cargo criterion --bench execution
|
||||
- name: collect microbenchmarks data
|
||||
run: cargo criterion --bench microbenchmarks
|
||||
- name: restructure generated files
|
||||
run: |
|
||||
cd ./target/criterion/reports
|
||||
find -type d -name cpython | xargs rm -rf
|
||||
find -type d -name rustpython | xargs rm -rf
|
||||
find -mindepth 2 -maxdepth 2 -name violin.svg | xargs rm -rf
|
||||
find -type f -not -name violin.svg | xargs rm -rf
|
||||
for file in $(find -type f -name violin.svg); do mv $file $(echo $file | sed -E "s_\./([^/]+)/([^/]+)/violin\.svg_./\1/\2.svg_"); done
|
||||
find -mindepth 2 -maxdepth 2 -type d | xargs rm -rf
|
||||
cd ..
|
||||
mv reports/* .
|
||||
rmdir reports
|
||||
- name: upload benchmark data to the website
|
||||
env:
|
||||
SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
|
||||
run: |
|
||||
echo "$SSHKEY" >~/github_key
|
||||
chmod 600 ~/github_key
|
||||
export GIT_SSH_COMMAND="ssh -i ~/github_key"
|
||||
|
||||
git clone git@github.com:RustPython/rustpython.github.io.git website
|
||||
cd website
|
||||
rm -rf ./assets/criterion
|
||||
cp -r ../target/criterion ./assets/criterion
|
||||
git add ./assets/criterion
|
||||
if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then
|
||||
git push
|
||||
fi
|
||||
git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results" --author="$GITHUB_ACTOR"
|
||||
git push
|
||||
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@@ -9,13 +9,10 @@ __pycache__
|
||||
.vscode
|
||||
wasm-pack.log
|
||||
.idea/
|
||||
tests/snippets/resources
|
||||
|
||||
flame-graph.html
|
||||
flame.txt
|
||||
flamescope.json
|
||||
/wapm.lock
|
||||
/wapm_packages
|
||||
/.cargo/config
|
||||
|
||||
extra_tests/snippets/resources
|
||||
extra_tests/not_impl.py
|
||||
|
||||
7
.gitpod.Dockerfile
vendored
7
.gitpod.Dockerfile
vendored
@@ -11,11 +11,4 @@ RUN rm -rf ~/.rustup && \
|
||||
curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && \
|
||||
rustup target add wasm32-unknown-unknown
|
||||
|
||||
RUN sudo apt-get -q update \
|
||||
&& sudo apt-get install -yq \
|
||||
libpython3.6 \
|
||||
rust-lldb \
|
||||
&& sudo rm -rf /var/lib/apt/lists/*
|
||||
ENV RUST_LLDB=/usr/bin/lldb-8
|
||||
|
||||
USER root
|
||||
|
||||
@@ -1,6 +1,2 @@
|
||||
image:
|
||||
file: .gitpod.Dockerfile
|
||||
|
||||
vscode:
|
||||
extensions:
|
||||
- vadimcn.vscode-lldb@1.5.3:vTh/rWhvJ5nQpeAVsD20QA==
|
||||
8
.mailmap
8
.mailmap
@@ -1,8 +0,0 @@
|
||||
#
|
||||
# This list is used by git-shortlog to aggregate contributions. It is
|
||||
# necessary when either the author's full name is not always written
|
||||
# the same way, and/or the same author contributes from different
|
||||
# email addresses.
|
||||
#
|
||||
|
||||
Noa <coolreader18@gmail.com> <33094578+coolreader18@users.noreply.github.com>
|
||||
@@ -1,16 +0,0 @@
|
||||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug Rust Code",
|
||||
//"preLaunchTask": "cargo",
|
||||
"program": "${workspaceFolder}/target/debug/rustpython",
|
||||
"cwd": "${workspaceFolder}",
|
||||
//"valuesFormatting": "parseText"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"cpp.buildConfigurations": [
|
||||
{
|
||||
"name": "",
|
||||
"directory": ""
|
||||
},
|
||||
]
|
||||
}
|
||||
298
.vscode/launch.json
vendored
298
.vscode/launch.json
vendored
@@ -1,298 +0,0 @@
|
||||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug executable 'rustpython'",
|
||||
"preLaunchTask": "Build RustPython Debug",
|
||||
"program": "target/debug/rustpython",
|
||||
"args": [],
|
||||
"env": {
|
||||
"RUST_BACKTRACE": "1"
|
||||
},
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug executable 'rustpython' without SSL",
|
||||
"preLaunchTask": "Build RustPython Debug without SSL",
|
||||
"program": "target/debug/rustpython",
|
||||
"args": [],
|
||||
"env": {
|
||||
"RUST_BACKTRACE": "1"
|
||||
},
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug benchmark 'execution'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--bench=execution",
|
||||
"--package=rustpython"
|
||||
],
|
||||
"filter": {
|
||||
"name": "execution",
|
||||
"kind": "bench"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug benchmark 'microbenchmarks'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--bench=microbenchmarks",
|
||||
"--package=rustpython"
|
||||
],
|
||||
"filter": {
|
||||
"name": "microbenchmarks",
|
||||
"kind": "bench"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-pylib'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-pylib"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-pylib",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-bytecode'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-bytecode"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-bytecode",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-compiler'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-compiler"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-compiler",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-compiler-core'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-compiler-core"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-compiler-core",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-ast'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-ast"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-ast",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-parser'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-parser"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-parser",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-vm'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-vm"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-vm",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-common'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-common"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-common",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-jit'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-jit"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-jit",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug integration test 'integration'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--test=integration",
|
||||
"--package=rustpython-jit"
|
||||
],
|
||||
"filter": {
|
||||
"name": "integration",
|
||||
"kind": "test"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython_wasm'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython_wasm"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython_wasm",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
}
|
||||
]
|
||||
}
|
||||
36
.vscode/tasks.json
vendored
36
.vscode/tasks.json
vendored
@@ -1,36 +0,0 @@
|
||||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "Build RustPython Debug without SSL",
|
||||
"type": "shell",
|
||||
"command": "cargo",
|
||||
"args": [
|
||||
"build",
|
||||
],
|
||||
"problemMatcher": [
|
||||
"$rustc",
|
||||
],
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true,
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build RustPython Debug",
|
||||
"type": "shell",
|
||||
"command": "cargo",
|
||||
"args": [
|
||||
"build",
|
||||
"--features=ssl"
|
||||
],
|
||||
"problemMatcher": [
|
||||
"$rustc",
|
||||
],
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true,
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
3364
Cargo.lock
generated
3364
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
169
Cargo.toml
169
Cargo.toml
@@ -1,165 +1,54 @@
|
||||
[package]
|
||||
name = "rustpython"
|
||||
version = "0.3.1"
|
||||
version = "0.1.2"
|
||||
authors = ["RustPython Team"]
|
||||
edition = "2021"
|
||||
rust-version = "1.67.1"
|
||||
edition = "2018"
|
||||
description = "A python interpreter written in rust."
|
||||
repository = "https://github.com/RustPython/RustPython"
|
||||
license = "MIT"
|
||||
include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
|
||||
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"compiler", "compiler/core", "compiler/codegen",
|
||||
".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "wasm/lib", "derive-impl",
|
||||
]
|
||||
members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "examples/freeze"]
|
||||
|
||||
[workspace.dependencies]
|
||||
rustpython-compiler-core = { path = "compiler/core", version = "0.3.1" }
|
||||
rustpython-compiler = { path = "compiler", version = "0.3.1" }
|
||||
rustpython-codegen = { path = "compiler/codegen", version = "0.3.1" }
|
||||
rustpython-common = { path = "common", version = "0.3.1" }
|
||||
rustpython-derive = { path = "derive", version = "0.3.1" }
|
||||
rustpython-derive-impl = { path = "derive-impl", version = "0.3.1" }
|
||||
rustpython-jit = { path = "jit", version = "0.3.1" }
|
||||
rustpython-vm = { path = "vm", default-features = false, version = "0.3.1" }
|
||||
rustpython-pylib = { path = "pylib", version = "0.3.1" }
|
||||
rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.3.1" }
|
||||
rustpython-sre_engine = { path = "vm/sre_engine", version = "0.3.1" }
|
||||
rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
|
||||
|
||||
rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
|
||||
rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
|
||||
rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
|
||||
rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
|
||||
rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
|
||||
# rustpython-literal = { path = "../RustPython-parser/literal" }
|
||||
# rustpython-parser-core = { path = "../RustPython-parser/core" }
|
||||
# rustpython-parser = { path = "../RustPython-parser/parser" }
|
||||
# rustpython-ast = { path = "../RustPython-parser/ast" }
|
||||
# rustpython-format = { path = "../RustPython-parser/format" }
|
||||
|
||||
ahash = "0.8.11"
|
||||
ascii = "1.0"
|
||||
atty = "0.2.14"
|
||||
bitflags = "2.4.1"
|
||||
bstr = "0.2.17"
|
||||
cfg-if = "1.0"
|
||||
chrono = "0.4.37"
|
||||
crossbeam-utils = "0.8.19"
|
||||
flame = "0.2.2"
|
||||
glob = "0.3"
|
||||
hex = "0.4.3"
|
||||
indexmap = { version = "2.2.6", features = ["std"] }
|
||||
insta = "1.38.0"
|
||||
itertools = "0.11.0"
|
||||
is-macro = "0.3.0"
|
||||
libc = "0.2.153"
|
||||
log = "0.4.16"
|
||||
nix = { version = "0.27", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
|
||||
malachite-bigint = "0.2.0"
|
||||
malachite-q = "0.4.4"
|
||||
malachite-base = "0.4.4"
|
||||
memchr = "2.7.2"
|
||||
num-complex = "0.4.0"
|
||||
num-integer = "0.1.44"
|
||||
num-traits = "0.2"
|
||||
num_enum = "0.7"
|
||||
once_cell = "1.19.0"
|
||||
parking_lot = "0.12.1"
|
||||
paste = "1.0.7"
|
||||
rand = "0.8.5"
|
||||
rustyline = "14.0.0"
|
||||
serde = { version = "1.0.133", default-features = false }
|
||||
schannel = "0.1.22"
|
||||
static_assertions = "1.1"
|
||||
syn = "1.0.109"
|
||||
thiserror = "1.0"
|
||||
thread_local = "1.1.4"
|
||||
unicode_names2 = "1.1.0"
|
||||
widestring = "1.1.0"
|
||||
[[bench]]
|
||||
name = "bench"
|
||||
path = "./benchmarks/bench.rs"
|
||||
|
||||
[features]
|
||||
default = ["threading", "stdlib", "zlib", "importlib"]
|
||||
importlib = ["rustpython-vm/importlib"]
|
||||
encodings = ["rustpython-vm/encodings"]
|
||||
stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"]
|
||||
flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
|
||||
freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
|
||||
jit = ["rustpython-vm/jit"]
|
||||
threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
|
||||
zlib = ["stdlib", "rustpython-stdlib/zlib"]
|
||||
bz2 = ["stdlib", "rustpython-stdlib/bz2"]
|
||||
ssl = ["rustpython-stdlib/ssl"]
|
||||
ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
|
||||
freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
|
||||
|
||||
ssl = ["rustpython-vm/ssl"]
|
||||
|
||||
[dependencies]
|
||||
rustpython-compiler = { workspace = true }
|
||||
rustpython-pylib = { workspace = true, optional = true }
|
||||
rustpython-stdlib = { workspace = true, optional = true }
|
||||
rustpython-vm = { workspace = true, features = ["compiler"] }
|
||||
rustpython-parser = { workspace = true }
|
||||
log = "0.4"
|
||||
env_logger = "0.7"
|
||||
clap = "2.33"
|
||||
rustpython-compiler = {path = "compiler", version = "0.1.1"}
|
||||
rustpython-parser = {path = "parser", version = "0.1.1"}
|
||||
rustpython-vm = {path = "vm", version = "0.1.1"}
|
||||
dirs = { package = "dirs-next", version = "1.0" }
|
||||
num-traits = "0.2.8"
|
||||
cfg-if = "0.1"
|
||||
|
||||
atty = { workspace = true }
|
||||
cfg-if = { workspace = true }
|
||||
log = { workspace = true }
|
||||
flame = { workspace = true, optional = true }
|
||||
flame = { version = "0.2", optional = true }
|
||||
flamescope = { version = "0.1", optional = true }
|
||||
|
||||
clap = "2.34"
|
||||
dirs = { package = "dirs-next", version = "2.0.0" }
|
||||
env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
|
||||
flamescope = { version = "0.1.2", optional = true }
|
||||
[target.'cfg(not(target_os = "wasi"))'.dependencies]
|
||||
rustyline = "6.0"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
libc = { workspace = true }
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
rustyline = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { version = "0.3.5", features = ["html_reports"] }
|
||||
pyo3 = { version = "0.20.2", features = ["auto-initialize"] }
|
||||
|
||||
[[bench]]
|
||||
name = "execution"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "microbenchmarks"
|
||||
harness = false
|
||||
[dev-dependencies.cpython]
|
||||
version = "0.2"
|
||||
|
||||
[[bin]]
|
||||
name = "rustpython"
|
||||
path = "src/main.rs"
|
||||
|
||||
[profile.dev.package."*"]
|
||||
opt-level = 3
|
||||
|
||||
[profile.test]
|
||||
opt-level = 3
|
||||
# https://github.com/rust-lang/rust/issues/92869
|
||||
# lto = "thin"
|
||||
|
||||
[profile.bench]
|
||||
lto = "thin"
|
||||
codegen-units = 1
|
||||
opt-level = 3
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
|
||||
[patch.crates-io]
|
||||
# REDOX START, Uncomment when you want to compile/check with redoxer
|
||||
# REDOX START, Uncommment when you want to compile/check with redoxer
|
||||
# # following patches are just waiting on a new version to be released to crates.io
|
||||
# nix = { git = "https://github.com/nix-rust/nix" }
|
||||
# crossbeam-utils = { git = "https://github.com/crossbeam-rs/crossbeam" }
|
||||
# socket2 = { git = "https://github.com/alexcrichton/socket2-rs" }
|
||||
# REDOX END
|
||||
|
||||
# Used only on Windows to build the vcpkg dependencies
|
||||
[package.metadata.vcpkg]
|
||||
git = "https://github.com/microsoft/vcpkg"
|
||||
# The revision of the vcpkg repository to use
|
||||
# https://github.com/microsoft/vcpkg/tags
|
||||
rev = "2024.02.14"
|
||||
|
||||
[package.metadata.vcpkg.target]
|
||||
x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] }
|
||||
|
||||
109
DEVELOPMENT.md
109
DEVELOPMENT.md
@@ -19,21 +19,17 @@ The contents of the Development Guide include:
|
||||
|
||||
RustPython requires the following:
|
||||
|
||||
- Rust latest stable version (e.g 1.69.0 as of Apr 20 2023)
|
||||
- Rust latest stable version (e.g 1.38.0 at Oct 1st 2019)
|
||||
- To check Rust version: `rustc --version`
|
||||
- If you have `rustup` on your system, enter to update to the latest
|
||||
stable version: `rustup update stable`
|
||||
- If you do not have Rust installed, use [rustup](https://rustup.rs/) to
|
||||
do so.
|
||||
- CPython version 3.12 or higher
|
||||
- CPython version 3.7.4 or higher
|
||||
- CPython can be installed by your operating system's package manager,
|
||||
from the [Python website](https://www.python.org/downloads/), or
|
||||
using a third-party distribution, such as
|
||||
[Anaconda](https://www.anaconda.com/distribution/).
|
||||
- [macOS] In case of libffi-sys compilation error, make sure autoconf, automake,
|
||||
libtool are installed
|
||||
- To install with [Homebrew](https://brew.sh), enter
|
||||
`brew install autoconf automake libtool`
|
||||
- [Optional] The Python package, `pytest`, is used for testing Python code
|
||||
snippets. To install, enter `python3 -m pip install pytest`.
|
||||
|
||||
@@ -41,58 +37,27 @@ RustPython requires the following:
|
||||
|
||||
The Rust code style used is the default
|
||||
[rustfmt](https://github.com/rust-lang/rustfmt) codestyle. Please format your
|
||||
code accordingly, or run `cargo fmt` to autoformat it. We also use
|
||||
[clippy](https://github.com/rust-lang/rust-clippy) to lint Rust code, which
|
||||
you can check yourself with `cargo clippy`.
|
||||
code accordingly. We also use [clippy](https://github.com/rust-lang/rust-clippy)
|
||||
to detect rust code issues.
|
||||
|
||||
Custom Python code (i.e. code not copied from CPython's standard library) should
|
||||
follow the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style. We also use
|
||||
[ruff](https://beta.ruff.rs/docs/) to check Python code style.
|
||||
|
||||
In addition to language specific tools, [cspell](https://github.com/streetsidesoftware/cspell),
|
||||
a code spell checker, is used in order to ensure correct spellings for code.
|
||||
Python code should follow the
|
||||
[PEP 8](https://www.python.org/dev/peps/pep-0008/) style. We also use
|
||||
[flake8](http://flake8.pycqa.org/en/latest/) to check Python code style.
|
||||
|
||||
## Testing
|
||||
|
||||
To test RustPython's functionality, a collection of Python snippets is located
|
||||
in the `extra_tests/snippets` directory and can be run using `pytest`:
|
||||
in the `tests/snippets` directory and can be run using `pytest`:
|
||||
|
||||
```shell
|
||||
$ cd extra_tests
|
||||
$ cd tests
|
||||
$ pytest -v
|
||||
```
|
||||
|
||||
Rust unit tests can be run with `cargo`:
|
||||
|
||||
```shell
|
||||
$ cargo test --workspace --exclude rustpython_wasm
|
||||
```
|
||||
|
||||
Python unit tests can be run by compiling RustPython and running the test module:
|
||||
|
||||
```shell
|
||||
$ cargo run --release -- -m test
|
||||
```
|
||||
|
||||
There are a few test options that are especially useful:
|
||||
|
||||
- `-j <n>` enables parallel testing (which is a lot faster), where `<n>` is the
|
||||
number of threads to be used, ideally the same as number of cores on your CPU.
|
||||
If you don't know, `-j 4` or `-j 8` are good options.
|
||||
- `-v` enables verbose mode, adding additional information about the tests being
|
||||
run.
|
||||
- `<test_name>` specifies a single test to run instead of running all tests.
|
||||
|
||||
For example, to run all tests in parallel:
|
||||
|
||||
```shell
|
||||
$ cargo run --release -- -m test -j 4
|
||||
```
|
||||
|
||||
To run only `test_cmath` (located at `Lib/test/test_cmath`) verbosely:
|
||||
|
||||
```shell
|
||||
$ cargo run --release -- -m test test_cmath -v
|
||||
$ cargo test --all
|
||||
```
|
||||
|
||||
## Profiling
|
||||
@@ -118,29 +83,34 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i
|
||||
Understanding a new codebase takes time. Here's a brief view of the
|
||||
repository's structure:
|
||||
|
||||
- `bytecode/src`: python bytecode representation in rust structures
|
||||
- `compiler/src`: python compilation to bytecode
|
||||
- `core/src`: python bytecode representation in rust structures
|
||||
- `parser/src`: python lexing, parsing and ast
|
||||
- `derive/src`: Rust language extensions and macros specific to rustpython
|
||||
- `parser/src`: python lexing, parsing and ast
|
||||
- `Lib`: Carefully selected / copied files from CPython sourcecode. This is
|
||||
the python side of the standard library.
|
||||
- `test`: CPython test suite
|
||||
- `vm/src`: python virtual machine
|
||||
- `builtins`: Builtin functions and types
|
||||
- `builtins.rs`: Builtin functions
|
||||
- `compile.rs`: the python compiler from ast to bytecode
|
||||
- `obj`: python builtin types
|
||||
- `stdlib`: Standard library parts implemented in rust.
|
||||
- `src`: using the other subcrates to bring rustpython to life.
|
||||
- `docs`: documentation (work in progress)
|
||||
- `py_code_object`: CPython bytecode to rustpython bytecode converter (work in
|
||||
progress)
|
||||
- `wasm`: Binary crate and resources for WebAssembly build
|
||||
- `extra_tests`: extra integration test snippets as a supplement to `Lib/test`
|
||||
- `tests`: integration test snippets
|
||||
|
||||
## Understanding Internals
|
||||
|
||||
The RustPython workspace includes the `rustpython` top-level crate. The `Cargo.toml`
|
||||
file in the root of the repo provide configuration of the crate and the
|
||||
implementation is found in the `src` directory (specifically, `src/lib.rs`).
|
||||
implementation is found in the `src` directory (specifically,
|
||||
`src/main.rs`).
|
||||
|
||||
The top-level `rustpython` binary depends on several lower-level crates including:
|
||||
|
||||
- `rustpython-parser` (implementation in `compiler/parser/src`)
|
||||
- `rustpython-parser` (implementation in `parser/src`)
|
||||
- `rustpython-compiler` (implementation in `compiler/src`)
|
||||
- `rustpython-vm` (implementation in `vm/src`)
|
||||
|
||||
@@ -158,26 +128,25 @@ enable a line of code to go through a series of steps:
|
||||
This crate contains the lexer and parser to convert a line of code to
|
||||
an Abstract Syntax Tree (AST):
|
||||
|
||||
- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens
|
||||
- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses
|
||||
- Lexer: `parser/lexer.rs` converts Python source code into tokens
|
||||
- Parser: `parser/parser.rs` takes the tokens generated by the lexer and parses
|
||||
the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax
|
||||
tree are Rust structs and enums.
|
||||
- The Parser relies on `LALRPOP`, a Rust parser generator framework. The
|
||||
LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`.
|
||||
- The Parser relies on `LALRPOP`, a Rust parser generator framework.
|
||||
- More information on parsers and a tutorial can be found in the
|
||||
[LALRPOP book](https://lalrpop.github.io/lalrpop/).
|
||||
- AST: `compiler/ast/` implements in Rust the Python types and expressions
|
||||
[LALRPOP book](https://lalrpop.github.io/lalrpop/README.html).
|
||||
- AST: `parser/ast.rs` implements in Rust the Python types and expressions
|
||||
represented by the AST nodes.
|
||||
|
||||
### rustpython-compiler
|
||||
|
||||
The `rustpython-compiler` crate's purpose is to transform the AST (Abstract Syntax
|
||||
Tree) to bytecode. The implementation of the compiler is found in the
|
||||
`compiler/src` directory. The compiler implements Python's symbol table,
|
||||
ast->bytecode compiler, and bytecode optimizer in Rust.
|
||||
`compiler/src` directory. The compiler implements Python's peephole optimizer
|
||||
implementation, Symbol table, and streams in Rust.
|
||||
|
||||
Implementation of bytecode structure in Rust is found in the `compiler/core/src`
|
||||
directory. `compiler/core/src/bytecode.rs` contains the representation of
|
||||
Implementation of bytecode structure in Rust is found in the `bytecode/src`
|
||||
directory. The `bytecode/src/bytecode.rs` contains the representation of
|
||||
instructions and operations in Rust. Further information about Python's
|
||||
bytecode instructions can be found in the
|
||||
[Python documentation](https://docs.python.org/3/library/dis.html#bytecodes).
|
||||
@@ -189,20 +158,10 @@ executes Python's instructions. The `vm/src` directory contains code to
|
||||
implement the read and evaluation loop that fetches and dispatches
|
||||
instructions. This directory also contains the implementation of the
|
||||
Python Standard Library modules in Rust (`vm/src/stdlib`). In Python
|
||||
everything can be represented as an object. The `vm/src/builtins` directory holds
|
||||
the Rust code used to represent different Python objects and their methods. The
|
||||
core implementation of what a Python object is can be found in
|
||||
`vm/src/object/core.rs`.
|
||||
|
||||
### Code generation
|
||||
|
||||
There are some code generations involved in building RustPython:
|
||||
|
||||
- some part of the AST code is generated from `vm/src/stdlib/ast/gen.rs` to `compiler/ast/src/ast_gen.rs`.
|
||||
- the `__doc__` attributes are generated by the
|
||||
[__doc__](https://github.com/RustPython/__doc__) project which is then included as the `rustpython-doc` crate.
|
||||
everything can be represented as an Object. `vm/src/obj` directory holds
|
||||
the Rust code used to represent a Python Object and its methods.
|
||||
|
||||
## Questions
|
||||
|
||||
Have you tried these steps and have a question, please chat with us on
|
||||
[Discord](https://discord.gg/vru8NypEhv).
|
||||
[gitter](https://gitter.im/rustpython/Lobby).
|
||||
|
||||
45
Lib/__future__.py
vendored
45
Lib/__future__.py
vendored
@@ -42,7 +42,7 @@ CompilerFlag is the (bitfield) flag that should be passed in the fourth
|
||||
argument to the builtin function compile() to enable the feature in
|
||||
dynamically compiled code. This flag is stored in the .compiler_flag
|
||||
attribute on _Future instances. These values must match the appropriate
|
||||
#defines of CO_xxx flags in Include/cpython/compile.h.
|
||||
#defines of CO_xxx flags in Include/compile.h.
|
||||
|
||||
No feature line is ever to be deleted from this file.
|
||||
"""
|
||||
@@ -57,29 +57,25 @@ all_feature_names = [
|
||||
"unicode_literals",
|
||||
"barry_as_FLUFL",
|
||||
"generator_stop",
|
||||
"annotations",
|
||||
]
|
||||
|
||||
__all__ = ["all_feature_names"] + all_feature_names
|
||||
|
||||
# The CO_xxx symbols are defined here under the same names defined in
|
||||
# code.h and used by compile.h, so that an editor search will find them here.
|
||||
# However, they're not exported in __all__, because they don't really belong to
|
||||
# The CO_xxx symbols are defined here under the same names used by
|
||||
# compile.h, so that an editor search will find them here. However,
|
||||
# they're not exported in __all__, because they don't really belong to
|
||||
# this module.
|
||||
CO_NESTED = 0x0010 # nested_scopes
|
||||
CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000)
|
||||
CO_FUTURE_DIVISION = 0x20000 # division
|
||||
CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default
|
||||
CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement
|
||||
CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function
|
||||
CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals
|
||||
CO_FUTURE_BARRY_AS_BDFL = 0x400000
|
||||
CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators
|
||||
CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime
|
||||
|
||||
CO_NESTED = 0x0010 # nested_scopes
|
||||
CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000)
|
||||
CO_FUTURE_DIVISION = 0x2000 # division
|
||||
CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
|
||||
CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement
|
||||
CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function
|
||||
CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
|
||||
CO_FUTURE_BARRY_AS_BDFL = 0x40000
|
||||
CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators
|
||||
|
||||
class _Feature:
|
||||
|
||||
def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
|
||||
self.optional = optionalRelease
|
||||
self.mandatory = mandatoryRelease
|
||||
@@ -90,6 +86,7 @@ class _Feature:
|
||||
|
||||
This is a 5-tuple, of the same form as sys.version_info.
|
||||
"""
|
||||
|
||||
return self.optional
|
||||
|
||||
def getMandatoryRelease(self):
|
||||
@@ -98,6 +95,7 @@ class _Feature:
|
||||
This is a 5-tuple, of the same form as sys.version_info, or, if
|
||||
the feature was dropped, is None.
|
||||
"""
|
||||
|
||||
return self.mandatory
|
||||
|
||||
def __repr__(self):
|
||||
@@ -105,7 +103,6 @@ class _Feature:
|
||||
self.mandatory,
|
||||
self.compiler_flag))
|
||||
|
||||
|
||||
nested_scopes = _Feature((2, 1, 0, "beta", 1),
|
||||
(2, 2, 0, "alpha", 0),
|
||||
CO_NESTED)
|
||||
@@ -135,13 +132,9 @@ unicode_literals = _Feature((2, 6, 0, "alpha", 2),
|
||||
CO_FUTURE_UNICODE_LITERALS)
|
||||
|
||||
barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2),
|
||||
(4, 0, 0, "alpha", 0),
|
||||
CO_FUTURE_BARRY_AS_BDFL)
|
||||
(3, 9, 0, "alpha", 0),
|
||||
CO_FUTURE_BARRY_AS_BDFL)
|
||||
|
||||
generator_stop = _Feature((3, 5, 0, "beta", 1),
|
||||
(3, 7, 0, "alpha", 0),
|
||||
CO_FUTURE_GENERATOR_STOP)
|
||||
|
||||
annotations = _Feature((3, 7, 0, "beta", 1),
|
||||
(3, 11, 0, "alpha", 0),
|
||||
CO_FUTURE_ANNOTATIONS)
|
||||
(3, 7, 0, "alpha", 0),
|
||||
CO_FUTURE_GENERATOR_STOP)
|
||||
|
||||
16
Lib/__hello__.py
vendored
16
Lib/__hello__.py
vendored
@@ -1,16 +0,0 @@
|
||||
initialized = True
|
||||
|
||||
class TestFrozenUtf8_1:
|
||||
"""\u00b6"""
|
||||
|
||||
class TestFrozenUtf8_2:
|
||||
"""\u03c0"""
|
||||
|
||||
class TestFrozenUtf8_4:
|
||||
"""\U0001f600"""
|
||||
|
||||
def main():
|
||||
print("Hello world!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,7 +0,0 @@
|
||||
initialized = True
|
||||
|
||||
def main():
|
||||
print("Hello world!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,7 +0,0 @@
|
||||
initialized = True
|
||||
|
||||
def main():
|
||||
print("Hello world!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
806
Lib/_pycodecs.py → Lib/_codecs.py
vendored
806
Lib/_pycodecs.py → Lib/_codecs.py
vendored
File diff suppressed because it is too large
Load Diff
264
Lib/_collections_abc.py
vendored
264
Lib/_collections_abc.py
vendored
@@ -6,41 +6,9 @@
|
||||
Unit tests are in test_collections.
|
||||
"""
|
||||
|
||||
############ Maintenance notes #########################################
|
||||
#
|
||||
# ABCs are different from other standard library modules in that they
|
||||
# specify compliance tests. In general, once an ABC has been published,
|
||||
# new methods (either abstract or concrete) cannot be added.
|
||||
#
|
||||
# Though classes that inherit from an ABC would automatically receive a
|
||||
# new mixin method, registered classes would become non-compliant and
|
||||
# violate the contract promised by ``isinstance(someobj, SomeABC)``.
|
||||
#
|
||||
# Though irritating, the correct procedure for adding new abstract or
|
||||
# mixin methods is to create a new ABC as a subclass of the previous
|
||||
# ABC. For example, union(), intersection(), and difference() cannot
|
||||
# be added to Set but could go into a new ABC that extends Set.
|
||||
#
|
||||
# Because they are so hard to change, new ABCs should have their APIs
|
||||
# carefully thought through prior to publication.
|
||||
#
|
||||
# Since ABCMeta only checks for the presence of methods, it is possible
|
||||
# to alter the signature of a method by adding optional arguments
|
||||
# or changing parameters names. This is still a bit dubious but at
|
||||
# least it won't cause isinstance() to return an incorrect result.
|
||||
#
|
||||
#
|
||||
#######################################################################
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
import sys
|
||||
|
||||
GenericAlias = type(list[int])
|
||||
EllipsisType = type(...)
|
||||
def _f(): pass
|
||||
FunctionType = type(_f)
|
||||
del _f
|
||||
|
||||
__all__ = ["Awaitable", "Coroutine",
|
||||
"AsyncIterable", "AsyncIterator", "AsyncGenerator",
|
||||
"Hashable", "Iterable", "Iterator", "Generator", "Reversible",
|
||||
@@ -49,7 +17,7 @@ __all__ = ["Awaitable", "Coroutine",
|
||||
"Mapping", "MutableMapping",
|
||||
"MappingView", "KeysView", "ItemsView", "ValuesView",
|
||||
"Sequence", "MutableSequence",
|
||||
"ByteString", "Buffer",
|
||||
"ByteString",
|
||||
]
|
||||
|
||||
# This module has been renamed from collections.abc to _collections_abc to
|
||||
@@ -92,14 +60,15 @@ _coro = _coro()
|
||||
coroutine = type(_coro)
|
||||
_coro.close() # Prevent ResourceWarning
|
||||
del _coro
|
||||
## asynchronous generator ##
|
||||
async def _ag(): yield
|
||||
_ag = _ag()
|
||||
async_generator = type(_ag)
|
||||
del _ag
|
||||
# XXX RustPython TODO: async generators
|
||||
# ## asynchronous generator ##
|
||||
# async def _ag(): yield
|
||||
# _ag = _ag()
|
||||
# async_generator = type(_ag)
|
||||
# del _ag
|
||||
|
||||
|
||||
### ONE-TRICK PONIES ###
|
||||
# ## ONE-TRICK PONIES ###
|
||||
|
||||
def _check_methods(C, *methods):
|
||||
mro = C.__mro__
|
||||
@@ -142,8 +111,6 @@ class Awaitable(metaclass=ABCMeta):
|
||||
return _check_methods(C, "__await__")
|
||||
return NotImplemented
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
|
||||
class Coroutine(Awaitable):
|
||||
|
||||
@@ -203,8 +170,6 @@ class AsyncIterable(metaclass=ABCMeta):
|
||||
return _check_methods(C, "__aiter__")
|
||||
return NotImplemented
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
|
||||
class AsyncIterator(AsyncIterable):
|
||||
|
||||
@@ -273,7 +238,7 @@ class AsyncGenerator(AsyncIterator):
|
||||
return NotImplemented
|
||||
|
||||
|
||||
AsyncGenerator.register(async_generator)
|
||||
# AsyncGenerator.register(async_generator)
|
||||
|
||||
|
||||
class Iterable(metaclass=ABCMeta):
|
||||
@@ -291,8 +256,6 @@ class Iterable(metaclass=ABCMeta):
|
||||
return _check_methods(C, "__iter__")
|
||||
return NotImplemented
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
|
||||
class Iterator(Iterable):
|
||||
|
||||
@@ -312,10 +275,9 @@ class Iterator(Iterable):
|
||||
return _check_methods(C, '__iter__', '__next__')
|
||||
return NotImplemented
|
||||
|
||||
|
||||
Iterator.register(bytes_iterator)
|
||||
Iterator.register(bytearray_iterator)
|
||||
#Iterator.register(callable_iterator)
|
||||
# Iterator.register(callable_iterator)
|
||||
Iterator.register(dict_keyiterator)
|
||||
Iterator.register(dict_valueiterator)
|
||||
Iterator.register(dict_itemiterator)
|
||||
@@ -392,10 +354,8 @@ class Generator(Iterator):
|
||||
'send', 'throw', 'close')
|
||||
return NotImplemented
|
||||
|
||||
|
||||
Generator.register(generator)
|
||||
|
||||
|
||||
class Sized(metaclass=ABCMeta):
|
||||
|
||||
__slots__ = ()
|
||||
@@ -425,9 +385,6 @@ class Container(metaclass=ABCMeta):
|
||||
return _check_methods(C, "__contains__")
|
||||
return NotImplemented
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
|
||||
class Collection(Sized, Iterable, Container):
|
||||
|
||||
__slots__ = ()
|
||||
@@ -438,106 +395,6 @@ class Collection(Sized, Iterable, Container):
|
||||
return _check_methods(C, "__len__", "__iter__", "__contains__")
|
||||
return NotImplemented
|
||||
|
||||
|
||||
class Buffer(metaclass=ABCMeta):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@abstractmethod
|
||||
def __buffer__(self, flags: int, /) -> memoryview:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def __subclasshook__(cls, C):
|
||||
if cls is Buffer:
|
||||
return _check_methods(C, "__buffer__")
|
||||
return NotImplemented
|
||||
|
||||
|
||||
class _CallableGenericAlias(GenericAlias):
|
||||
""" Represent `Callable[argtypes, resulttype]`.
|
||||
|
||||
This sets ``__args__`` to a tuple containing the flattened ``argtypes``
|
||||
followed by ``resulttype``.
|
||||
|
||||
Example: ``Callable[[int, str], float]`` sets ``__args__`` to
|
||||
``(int, str, float)``.
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def __new__(cls, origin, args):
|
||||
if not (isinstance(args, tuple) and len(args) == 2):
|
||||
raise TypeError(
|
||||
"Callable must be used as Callable[[arg, ...], result].")
|
||||
t_args, t_result = args
|
||||
if isinstance(t_args, (tuple, list)):
|
||||
args = (*t_args, t_result)
|
||||
elif not _is_param_expr(t_args):
|
||||
raise TypeError(f"Expected a list of types, an ellipsis, "
|
||||
f"ParamSpec, or Concatenate. Got {t_args}")
|
||||
return super().__new__(cls, origin, args)
|
||||
|
||||
def __repr__(self):
|
||||
if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]):
|
||||
return super().__repr__()
|
||||
return (f'collections.abc.Callable'
|
||||
f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], '
|
||||
f'{_type_repr(self.__args__[-1])}]')
|
||||
|
||||
def __reduce__(self):
|
||||
args = self.__args__
|
||||
if not (len(args) == 2 and _is_param_expr(args[0])):
|
||||
args = list(args[:-1]), args[-1]
|
||||
return _CallableGenericAlias, (Callable, args)
|
||||
|
||||
def __getitem__(self, item):
|
||||
# Called during TypeVar substitution, returns the custom subclass
|
||||
# rather than the default types.GenericAlias object. Most of the
|
||||
# code is copied from typing's _GenericAlias and the builtin
|
||||
# types.GenericAlias.
|
||||
if not isinstance(item, tuple):
|
||||
item = (item,)
|
||||
|
||||
new_args = super().__getitem__(item).__args__
|
||||
|
||||
# args[0] occurs due to things like Z[[int, str, bool]] from PEP 612
|
||||
if not isinstance(new_args[0], (tuple, list)):
|
||||
t_result = new_args[-1]
|
||||
t_args = new_args[:-1]
|
||||
new_args = (t_args, t_result)
|
||||
return _CallableGenericAlias(Callable, tuple(new_args))
|
||||
|
||||
def _is_param_expr(obj):
|
||||
"""Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or
|
||||
``_ConcatenateGenericAlias`` from typing.py
|
||||
"""
|
||||
if obj is Ellipsis:
|
||||
return True
|
||||
if isinstance(obj, list):
|
||||
return True
|
||||
obj = type(obj)
|
||||
names = ('ParamSpec', '_ConcatenateGenericAlias')
|
||||
return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names)
|
||||
|
||||
def _type_repr(obj):
|
||||
"""Return the repr() of an object, special-casing types (internal helper).
|
||||
|
||||
Copied from :mod:`typing` since collections.abc
|
||||
shouldn't depend on that module.
|
||||
(Keep this roughly in sync with the typing version.)
|
||||
"""
|
||||
if isinstance(obj, type):
|
||||
if obj.__module__ == 'builtins':
|
||||
return obj.__qualname__
|
||||
return f'{obj.__module__}.{obj.__qualname__}'
|
||||
if obj is Ellipsis:
|
||||
return '...'
|
||||
if isinstance(obj, FunctionType):
|
||||
return obj.__name__
|
||||
return repr(obj)
|
||||
|
||||
|
||||
class Callable(metaclass=ABCMeta):
|
||||
|
||||
__slots__ = ()
|
||||
@@ -552,13 +409,12 @@ class Callable(metaclass=ABCMeta):
|
||||
return _check_methods(C, "__call__")
|
||||
return NotImplemented
|
||||
|
||||
__class_getitem__ = classmethod(_CallableGenericAlias)
|
||||
|
||||
|
||||
### SETS ###
|
||||
|
||||
|
||||
class Set(Collection):
|
||||
|
||||
"""A set is a finite, iterable container.
|
||||
|
||||
This class provides concrete generic implementations of all
|
||||
@@ -686,7 +542,6 @@ class Set(Collection):
|
||||
hx = hash(x)
|
||||
h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167
|
||||
h &= MASK
|
||||
h ^= (h >> 11) ^ (h >> 25)
|
||||
h = h * 69069 + 907133923
|
||||
h &= MASK
|
||||
if h > MAX:
|
||||
@@ -695,7 +550,6 @@ class Set(Collection):
|
||||
h = 590923713
|
||||
return h
|
||||
|
||||
|
||||
Set.register(frozenset)
|
||||
|
||||
|
||||
@@ -778,25 +632,24 @@ class MutableSet(Set):
|
||||
self.discard(value)
|
||||
return self
|
||||
|
||||
|
||||
MutableSet.register(set)
|
||||
|
||||
|
||||
### MAPPINGS ###
|
||||
|
||||
|
||||
class Mapping(Collection):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
"""A Mapping is a generic container for associating key/value
|
||||
pairs.
|
||||
|
||||
This class provides concrete generic implementations of all
|
||||
methods except for __getitem__, __iter__, and __len__.
|
||||
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
# Tell ABCMeta.__new__ that this class should have TPFLAGS_MAPPING set.
|
||||
__abc_tpflags__ = 1 << 6 # Py_TPFLAGS_MAPPING
|
||||
|
||||
@abstractmethod
|
||||
def __getitem__(self, key):
|
||||
raise KeyError
|
||||
@@ -851,15 +704,13 @@ class MappingView(Sized):
|
||||
def __repr__(self):
|
||||
return '{0.__class__.__name__}({0._mapping!r})'.format(self)
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
|
||||
class KeysView(MappingView, Set):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@classmethod
|
||||
def _from_iterable(cls, it):
|
||||
def _from_iterable(self, it):
|
||||
return set(it)
|
||||
|
||||
def __contains__(self, key):
|
||||
@@ -868,7 +719,6 @@ class KeysView(MappingView, Set):
|
||||
def __iter__(self):
|
||||
yield from self._mapping
|
||||
|
||||
|
||||
KeysView.register(dict_keys)
|
||||
|
||||
|
||||
@@ -877,7 +727,7 @@ class ItemsView(MappingView, Set):
|
||||
__slots__ = ()
|
||||
|
||||
@classmethod
|
||||
def _from_iterable(cls, it):
|
||||
def _from_iterable(self, it):
|
||||
return set(it)
|
||||
|
||||
def __contains__(self, item):
|
||||
@@ -893,7 +743,6 @@ class ItemsView(MappingView, Set):
|
||||
for key in self._mapping:
|
||||
yield (key, self._mapping[key])
|
||||
|
||||
|
||||
ItemsView.register(dict_items)
|
||||
|
||||
|
||||
@@ -912,20 +761,21 @@ class ValuesView(MappingView, Collection):
|
||||
for key in self._mapping:
|
||||
yield self._mapping[key]
|
||||
|
||||
|
||||
ValuesView.register(dict_values)
|
||||
|
||||
|
||||
class MutableMapping(Mapping):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
"""A MutableMapping is a generic container for associating
|
||||
key/value pairs.
|
||||
|
||||
This class provides concrete generic implementations of all
|
||||
methods except for __getitem__, __setitem__, __delitem__,
|
||||
__iter__, and __len__.
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def __setitem__(self, key, value):
|
||||
@@ -971,21 +821,34 @@ class MutableMapping(Mapping):
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def update(self, other=(), /, **kwds):
|
||||
def update(*args, **kwds):
|
||||
''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.
|
||||
If E present and has a .keys() method, does: for k in E: D[k] = E[k]
|
||||
If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v
|
||||
In either case, this is followed by: for k, v in F.items(): D[k] = v
|
||||
'''
|
||||
if isinstance(other, Mapping):
|
||||
for key in other:
|
||||
self[key] = other[key]
|
||||
elif hasattr(other, "keys"):
|
||||
for key in other.keys():
|
||||
self[key] = other[key]
|
||||
else:
|
||||
for key, value in other:
|
||||
self[key] = value
|
||||
if not args:
|
||||
raise TypeError("descriptor 'update' of 'MutableMapping' object "
|
||||
"needs an argument")
|
||||
self, *args = args
|
||||
if len(args) > 1:
|
||||
raise TypeError('update expected at most 1 arguments, got %d' %
|
||||
len(args))
|
||||
if args:
|
||||
other = args[0]
|
||||
try:
|
||||
mapping_inst = isinstance(other, Mapping)
|
||||
except TypeError:
|
||||
mapping_inst = False
|
||||
if mapping_inst:
|
||||
for key in other:
|
||||
self[key] = other[key]
|
||||
elif hasattr(other, "keys"):
|
||||
for key in other.keys():
|
||||
self[key] = other[key]
|
||||
else:
|
||||
for key, value in other:
|
||||
self[key] = value
|
||||
for key, value in kwds.items():
|
||||
self[key] = value
|
||||
|
||||
@@ -997,13 +860,14 @@ class MutableMapping(Mapping):
|
||||
self[key] = default
|
||||
return default
|
||||
|
||||
|
||||
MutableMapping.register(dict)
|
||||
|
||||
|
||||
### SEQUENCES ###
|
||||
|
||||
|
||||
class Sequence(Reversible, Collection):
|
||||
|
||||
"""All the operations on a read-only sequence.
|
||||
|
||||
Concrete subclasses must override __new__ or __init__,
|
||||
@@ -1012,9 +876,6 @@ class Sequence(Reversible, Collection):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
# Tell ABCMeta.__new__ that this class should have TPFLAGS_SEQUENCE set.
|
||||
__abc_tpflags__ = 1 << 5 # Py_TPFLAGS_SEQUENCE
|
||||
|
||||
@abstractmethod
|
||||
def __getitem__(self, index):
|
||||
raise IndexError
|
||||
@@ -1055,10 +916,10 @@ class Sequence(Reversible, Collection):
|
||||
while stop is None or i < stop:
|
||||
try:
|
||||
v = self[i]
|
||||
if v is value or v == value:
|
||||
return i
|
||||
except IndexError:
|
||||
break
|
||||
if v is value or v == value:
|
||||
return i
|
||||
i += 1
|
||||
raise ValueError
|
||||
|
||||
@@ -1071,27 +932,9 @@ Sequence.register(str)
|
||||
Sequence.register(range)
|
||||
Sequence.register(memoryview)
|
||||
|
||||
class _DeprecateByteStringMeta(ABCMeta):
|
||||
def __new__(cls, name, bases, namespace, **kwargs):
|
||||
if name != "ByteString":
|
||||
import warnings
|
||||
|
||||
warnings._deprecated(
|
||||
"collections.abc.ByteString",
|
||||
remove=(3, 14),
|
||||
)
|
||||
return super().__new__(cls, name, bases, namespace, **kwargs)
|
||||
class ByteString(Sequence):
|
||||
|
||||
def __instancecheck__(cls, instance):
|
||||
import warnings
|
||||
|
||||
warnings._deprecated(
|
||||
"collections.abc.ByteString",
|
||||
remove=(3, 14),
|
||||
)
|
||||
return super().__instancecheck__(instance)
|
||||
|
||||
class ByteString(Sequence, metaclass=_DeprecateByteStringMeta):
|
||||
"""This unifies bytes and bytearray.
|
||||
|
||||
XXX Should add all their methods.
|
||||
@@ -1104,13 +947,15 @@ ByteString.register(bytearray)
|
||||
|
||||
|
||||
class MutableSequence(Sequence):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
"""All the operations on a read-write sequence.
|
||||
|
||||
Concrete subclasses must provide __new__ or __init__,
|
||||
__getitem__, __setitem__, __delitem__, __len__, and insert().
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def __setitem__(self, index, value):
|
||||
@@ -1168,6 +1013,5 @@ class MutableSequence(Sequence):
|
||||
self.extend(values)
|
||||
return self
|
||||
|
||||
|
||||
MutableSequence.register(list)
|
||||
MutableSequence.register(bytearray) # Multiply inheriting, see ByteString
|
||||
|
||||
8
Lib/_compat_pickle.py
vendored
8
Lib/_compat_pickle.py
vendored
@@ -148,14 +148,6 @@ except NameError:
|
||||
else:
|
||||
PYTHON2_EXCEPTIONS += ("WindowsError",)
|
||||
|
||||
# NOTE: RUSTPYTHON exceptions
|
||||
try:
|
||||
JitError
|
||||
except NameError:
|
||||
pass
|
||||
else:
|
||||
PYTHON2_EXCEPTIONS += ("JitError",)
|
||||
|
||||
for excname in PYTHON2_EXCEPTIONS:
|
||||
NAME_MAPPING[("exceptions", excname)] = ("builtins", excname)
|
||||
|
||||
|
||||
162
Lib/_compression.py
vendored
162
Lib/_compression.py
vendored
@@ -1,162 +0,0 @@
|
||||
"""Internal classes used by the gzip, lzma and bz2 modules"""
|
||||
|
||||
import io
|
||||
import sys
|
||||
|
||||
BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size
|
||||
|
||||
|
||||
class BaseStream(io.BufferedIOBase):
|
||||
"""Mode-checking helper functions."""
|
||||
|
||||
def _check_not_closed(self):
|
||||
if self.closed:
|
||||
raise ValueError("I/O operation on closed file")
|
||||
|
||||
def _check_can_read(self):
|
||||
if not self.readable():
|
||||
raise io.UnsupportedOperation("File not open for reading")
|
||||
|
||||
def _check_can_write(self):
|
||||
if not self.writable():
|
||||
raise io.UnsupportedOperation("File not open for writing")
|
||||
|
||||
def _check_can_seek(self):
|
||||
if not self.readable():
|
||||
raise io.UnsupportedOperation("Seeking is only supported "
|
||||
"on files open for reading")
|
||||
if not self.seekable():
|
||||
raise io.UnsupportedOperation("The underlying file object "
|
||||
"does not support seeking")
|
||||
|
||||
|
||||
class DecompressReader(io.RawIOBase):
|
||||
"""Adapts the decompressor API to a RawIOBase reader API"""
|
||||
|
||||
def readable(self):
|
||||
return True
|
||||
|
||||
def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
|
||||
self._fp = fp
|
||||
self._eof = False
|
||||
self._pos = 0 # Current offset in decompressed stream
|
||||
|
||||
# Set to size of decompressed stream once it is known, for SEEK_END
|
||||
self._size = -1
|
||||
|
||||
# Save the decompressor factory and arguments.
|
||||
# If the file contains multiple compressed streams, each
|
||||
# stream will need a separate decompressor object. A new decompressor
|
||||
# object is also needed when implementing a backwards seek().
|
||||
self._decomp_factory = decomp_factory
|
||||
self._decomp_args = decomp_args
|
||||
self._decompressor = self._decomp_factory(**self._decomp_args)
|
||||
|
||||
# Exception class to catch from decompressor signifying invalid
|
||||
# trailing data to ignore
|
||||
self._trailing_error = trailing_error
|
||||
|
||||
def close(self):
|
||||
self._decompressor = None
|
||||
return super().close()
|
||||
|
||||
def seekable(self):
|
||||
return self._fp.seekable()
|
||||
|
||||
def readinto(self, b):
|
||||
with memoryview(b) as view, view.cast("B") as byte_view:
|
||||
data = self.read(len(byte_view))
|
||||
byte_view[:len(data)] = data
|
||||
return len(data)
|
||||
|
||||
def read(self, size=-1):
|
||||
if size < 0:
|
||||
return self.readall()
|
||||
|
||||
if not size or self._eof:
|
||||
return b""
|
||||
data = None # Default if EOF is encountered
|
||||
# Depending on the input data, our call to the decompressor may not
|
||||
# return any data. In this case, try again after reading another block.
|
||||
while True:
|
||||
if self._decompressor.eof:
|
||||
rawblock = (self._decompressor.unused_data or
|
||||
self._fp.read(BUFFER_SIZE))
|
||||
if not rawblock:
|
||||
break
|
||||
# Continue to next stream.
|
||||
self._decompressor = self._decomp_factory(
|
||||
**self._decomp_args)
|
||||
try:
|
||||
data = self._decompressor.decompress(rawblock, size)
|
||||
except self._trailing_error:
|
||||
# Trailing data isn't a valid compressed stream; ignore it.
|
||||
break
|
||||
else:
|
||||
if self._decompressor.needs_input:
|
||||
rawblock = self._fp.read(BUFFER_SIZE)
|
||||
if not rawblock:
|
||||
raise EOFError("Compressed file ended before the "
|
||||
"end-of-stream marker was reached")
|
||||
else:
|
||||
rawblock = b""
|
||||
data = self._decompressor.decompress(rawblock, size)
|
||||
if data:
|
||||
break
|
||||
if not data:
|
||||
self._eof = True
|
||||
self._size = self._pos
|
||||
return b""
|
||||
self._pos += len(data)
|
||||
return data
|
||||
|
||||
def readall(self):
|
||||
chunks = []
|
||||
# sys.maxsize means the max length of output buffer is unlimited,
|
||||
# so that the whole input buffer can be decompressed within one
|
||||
# .decompress() call.
|
||||
while data := self.read(sys.maxsize):
|
||||
chunks.append(data)
|
||||
|
||||
return b"".join(chunks)
|
||||
|
||||
# Rewind the file to the beginning of the data stream.
|
||||
def _rewind(self):
|
||||
self._fp.seek(0)
|
||||
self._eof = False
|
||||
self._pos = 0
|
||||
self._decompressor = self._decomp_factory(**self._decomp_args)
|
||||
|
||||
def seek(self, offset, whence=io.SEEK_SET):
|
||||
# Recalculate offset as an absolute file position.
|
||||
if whence == io.SEEK_SET:
|
||||
pass
|
||||
elif whence == io.SEEK_CUR:
|
||||
offset = self._pos + offset
|
||||
elif whence == io.SEEK_END:
|
||||
# Seeking relative to EOF - we need to know the file's size.
|
||||
if self._size < 0:
|
||||
while self.read(io.DEFAULT_BUFFER_SIZE):
|
||||
pass
|
||||
offset = self._size + offset
|
||||
else:
|
||||
raise ValueError("Invalid value for whence: {}".format(whence))
|
||||
|
||||
# Make it so that offset is the number of bytes to skip forward.
|
||||
if offset < self._pos:
|
||||
self._rewind()
|
||||
else:
|
||||
offset -= self._pos
|
||||
|
||||
# Read and discard data until we reach the desired position.
|
||||
while offset > 0:
|
||||
data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset))
|
||||
if not data:
|
||||
break
|
||||
offset -= len(data)
|
||||
|
||||
return self._pos
|
||||
|
||||
def tell(self):
|
||||
"""Return the current file position."""
|
||||
return self._pos
|
||||
66
Lib/_dummy_os.py
vendored
66
Lib/_dummy_os.py
vendored
@@ -1,66 +0,0 @@
|
||||
"""
|
||||
A shim of the os module containing only simple path-related utilities
|
||||
"""
|
||||
|
||||
try:
|
||||
from os import *
|
||||
except ImportError:
|
||||
import abc
|
||||
|
||||
def __getattr__(name):
|
||||
raise OSError("no os specific module found")
|
||||
|
||||
def _shim():
|
||||
import _dummy_os, sys
|
||||
sys.modules['os'] = _dummy_os
|
||||
sys.modules['os.path'] = _dummy_os.path
|
||||
|
||||
import posixpath as path
|
||||
import sys
|
||||
sys.modules['os.path'] = path
|
||||
del sys
|
||||
|
||||
sep = path.sep
|
||||
|
||||
|
||||
def fspath(path):
|
||||
"""Return the path representation of a path-like object.
|
||||
|
||||
If str or bytes is passed in, it is returned unchanged. Otherwise the
|
||||
os.PathLike interface is used to get the path representation. If the
|
||||
path representation is not str or bytes, TypeError is raised. If the
|
||||
provided path is not str, bytes, or os.PathLike, TypeError is raised.
|
||||
"""
|
||||
if isinstance(path, (str, bytes)):
|
||||
return path
|
||||
|
||||
# Work from the object's type to match method resolution of other magic
|
||||
# methods.
|
||||
path_type = type(path)
|
||||
try:
|
||||
path_repr = path_type.__fspath__(path)
|
||||
except AttributeError:
|
||||
if hasattr(path_type, '__fspath__'):
|
||||
raise
|
||||
else:
|
||||
raise TypeError("expected str, bytes or os.PathLike object, "
|
||||
"not " + path_type.__name__)
|
||||
if isinstance(path_repr, (str, bytes)):
|
||||
return path_repr
|
||||
else:
|
||||
raise TypeError("expected {}.__fspath__() to return str or bytes, "
|
||||
"not {}".format(path_type.__name__,
|
||||
type(path_repr).__name__))
|
||||
|
||||
class PathLike(abc.ABC):
|
||||
|
||||
"""Abstract base class for implementing the file system path protocol."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def __fspath__(self):
|
||||
"""Return the file system path representation of the object."""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def __subclasshook__(cls, subclass):
|
||||
return hasattr(subclass, '__fspath__')
|
||||
42
Lib/_dummy_thread.py
vendored
42
Lib/_dummy_thread.py
vendored
@@ -14,8 +14,7 @@ Suggested usage is::
|
||||
# Exports only things specified by thread documentation;
|
||||
# skipping obsolete synonyms allocate(), start_new(), exit_thread().
|
||||
__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
|
||||
'interrupt_main', 'LockType', 'RLock',
|
||||
'_count']
|
||||
'interrupt_main', 'LockType']
|
||||
|
||||
# A dummy value
|
||||
TIMEOUT_MAX = 2**31
|
||||
@@ -86,10 +85,6 @@ def _set_sentinel():
|
||||
"""Dummy implementation of _thread._set_sentinel()."""
|
||||
return LockType()
|
||||
|
||||
def _count():
|
||||
"""Dummy implementation of _thread._count()."""
|
||||
return 0
|
||||
|
||||
class LockType(object):
|
||||
"""Class implementing dummy implementation of _thread.LockType.
|
||||
|
||||
@@ -145,9 +140,6 @@ class LockType(object):
|
||||
def locked(self):
|
||||
return self.locked_status
|
||||
|
||||
def _at_fork_reinit(self):
|
||||
self.locked_status = False
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s %s.%s object at %s>" % (
|
||||
"locked" if self.locked_status else "unlocked",
|
||||
@@ -169,35 +161,3 @@ def interrupt_main():
|
||||
else:
|
||||
global _interrupt
|
||||
_interrupt = True
|
||||
|
||||
class RLock:
|
||||
def __init__(self):
|
||||
self.locked_count = 0
|
||||
|
||||
def acquire(self, waitflag=None, timeout=-1):
|
||||
self.locked_count += 1
|
||||
return True
|
||||
|
||||
__enter__ = acquire
|
||||
|
||||
def __exit__(self, typ, val, tb):
|
||||
self.release()
|
||||
|
||||
def release(self):
|
||||
if not self.locked_count:
|
||||
raise error
|
||||
self.locked_count -= 1
|
||||
return True
|
||||
|
||||
def locked(self):
|
||||
return self.locked_status != 0
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s %s.%s object owner=%s count=%s at %s>" % (
|
||||
"locked" if self.locked_count else "unlocked",
|
||||
self.__class__.__module__,
|
||||
self.__class__.__qualname__,
|
||||
get_ident() if self.locked_count else 0,
|
||||
self.locked_count,
|
||||
hex(id(self))
|
||||
)
|
||||
|
||||
35
Lib/_markupbase.py
vendored
35
Lib/_markupbase.py
vendored
@@ -29,6 +29,10 @@ class ParserBase:
|
||||
raise RuntimeError(
|
||||
"_markupbase.ParserBase must be subclassed")
|
||||
|
||||
def error(self, message):
|
||||
raise NotImplementedError(
|
||||
"subclasses of ParserBase must override error()")
|
||||
|
||||
def reset(self):
|
||||
self.lineno = 1
|
||||
self.offset = 0
|
||||
@@ -127,11 +131,12 @@ class ParserBase:
|
||||
# also in data attribute specifications of attlist declaration
|
||||
# also link type declaration subsets in linktype declarations
|
||||
# also link attribute specification lists in link declarations
|
||||
raise AssertionError("unsupported '[' char in %s declaration" % decltype)
|
||||
self.error("unsupported '[' char in %s declaration" % decltype)
|
||||
else:
|
||||
raise AssertionError("unexpected '[' char in declaration")
|
||||
self.error("unexpected '[' char in declaration")
|
||||
else:
|
||||
raise AssertionError("unexpected %r char in declaration" % rawdata[j])
|
||||
self.error(
|
||||
"unexpected %r char in declaration" % rawdata[j])
|
||||
if j < 0:
|
||||
return j
|
||||
return -1 # incomplete
|
||||
@@ -151,9 +156,7 @@ class ParserBase:
|
||||
# look for MS Office ]> ending
|
||||
match= _msmarkedsectionclose.search(rawdata, i+3)
|
||||
else:
|
||||
raise AssertionError(
|
||||
'unknown status keyword %r in marked section' % rawdata[i+3:j]
|
||||
)
|
||||
self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
|
||||
if not match:
|
||||
return -1
|
||||
if report:
|
||||
@@ -165,7 +168,7 @@ class ParserBase:
|
||||
def parse_comment(self, i, report=1):
|
||||
rawdata = self.rawdata
|
||||
if rawdata[i:i+4] != '<!--':
|
||||
raise AssertionError('unexpected call to parse_comment()')
|
||||
self.error('unexpected call to parse_comment()')
|
||||
match = _commentclose.search(rawdata, i+4)
|
||||
if not match:
|
||||
return -1
|
||||
@@ -189,9 +192,7 @@ class ParserBase:
|
||||
return -1
|
||||
if s != "<!":
|
||||
self.updatepos(declstartpos, j + 1)
|
||||
raise AssertionError(
|
||||
"unexpected char in internal subset (in %r)" % s
|
||||
)
|
||||
self.error("unexpected char in internal subset (in %r)" % s)
|
||||
if (j + 2) == n:
|
||||
# end of buffer; incomplete
|
||||
return -1
|
||||
@@ -208,9 +209,8 @@ class ParserBase:
|
||||
return -1
|
||||
if name not in {"attlist", "element", "entity", "notation"}:
|
||||
self.updatepos(declstartpos, j + 2)
|
||||
raise AssertionError(
|
||||
"unknown declaration %r in internal subset" % name
|
||||
)
|
||||
self.error(
|
||||
"unknown declaration %r in internal subset" % name)
|
||||
# handle the individual names
|
||||
meth = getattr(self, "_parse_doctype_" + name)
|
||||
j = meth(j, declstartpos)
|
||||
@@ -234,14 +234,14 @@ class ParserBase:
|
||||
if rawdata[j] == ">":
|
||||
return j
|
||||
self.updatepos(declstartpos, j)
|
||||
raise AssertionError("unexpected char after internal subset")
|
||||
self.error("unexpected char after internal subset")
|
||||
else:
|
||||
return -1
|
||||
elif c.isspace():
|
||||
j = j + 1
|
||||
else:
|
||||
self.updatepos(declstartpos, j)
|
||||
raise AssertionError("unexpected char %r in internal subset" % c)
|
||||
self.error("unexpected char %r in internal subset" % c)
|
||||
# end of buffer reached
|
||||
return -1
|
||||
|
||||
@@ -387,9 +387,8 @@ class ParserBase:
|
||||
return name.lower(), m.end()
|
||||
else:
|
||||
self.updatepos(declstartpos, i)
|
||||
raise AssertionError(
|
||||
"expected name token at %r" % rawdata[declstartpos:declstartpos+20]
|
||||
)
|
||||
self.error("expected name token at %r"
|
||||
% rawdata[declstartpos:declstartpos+20])
|
||||
|
||||
# To be overridden -- handlers for unknown objects
|
||||
def unknown_decl(self, data):
|
||||
|
||||
574
Lib/_osx_support.py
vendored
574
Lib/_osx_support.py
vendored
@@ -1,574 +0,0 @@
|
||||
"""Shared OS X support functions."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
__all__ = [
|
||||
'compiler_fixup',
|
||||
'customize_config_vars',
|
||||
'customize_compiler',
|
||||
'get_platform_osx',
|
||||
]
|
||||
|
||||
# configuration variables that may contain universal build flags,
|
||||
# like "-arch" or "-isdkroot", that may need customization for
|
||||
# the user environment
|
||||
_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS',
|
||||
'BLDSHARED', 'LDSHARED', 'CC', 'CXX',
|
||||
'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS',
|
||||
'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS')
|
||||
|
||||
# configuration variables that may contain compiler calls
|
||||
_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX')
|
||||
|
||||
# prefix added to original configuration variable names
|
||||
_INITPRE = '_OSX_SUPPORT_INITIAL_'
|
||||
|
||||
|
||||
def _find_executable(executable, path=None):
|
||||
"""Tries to find 'executable' in the directories listed in 'path'.
|
||||
|
||||
A string listing directories separated by 'os.pathsep'; defaults to
|
||||
os.environ['PATH']. Returns the complete filename or None if not found.
|
||||
"""
|
||||
if path is None:
|
||||
path = os.environ['PATH']
|
||||
|
||||
paths = path.split(os.pathsep)
|
||||
base, ext = os.path.splitext(executable)
|
||||
|
||||
if (sys.platform == 'win32') and (ext != '.exe'):
|
||||
executable = executable + '.exe'
|
||||
|
||||
if not os.path.isfile(executable):
|
||||
for p in paths:
|
||||
f = os.path.join(p, executable)
|
||||
if os.path.isfile(f):
|
||||
# the file exists, we have a shot at spawn working
|
||||
return f
|
||||
return None
|
||||
else:
|
||||
return executable
|
||||
|
||||
|
||||
def _read_output(commandstring, capture_stderr=False):
|
||||
"""Output from successful command execution or None"""
|
||||
# Similar to os.popen(commandstring, "r").read(),
|
||||
# but without actually using os.popen because that
|
||||
# function is not usable during python bootstrap.
|
||||
# tempfile is also not available then.
|
||||
import contextlib
|
||||
try:
|
||||
import tempfile
|
||||
fp = tempfile.NamedTemporaryFile()
|
||||
except ImportError:
|
||||
fp = open("/tmp/_osx_support.%s"%(
|
||||
os.getpid(),), "w+b")
|
||||
|
||||
with contextlib.closing(fp) as fp:
|
||||
if capture_stderr:
|
||||
cmd = "%s >'%s' 2>&1" % (commandstring, fp.name)
|
||||
else:
|
||||
cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
|
||||
return fp.read().decode('utf-8').strip() if not os.system(cmd) else None
|
||||
|
||||
|
||||
def _find_build_tool(toolname):
|
||||
"""Find a build tool on current path or using xcrun"""
|
||||
return (_find_executable(toolname)
|
||||
or _read_output("/usr/bin/xcrun -find %s" % (toolname,))
|
||||
or ''
|
||||
)
|
||||
|
||||
_SYSTEM_VERSION = None
|
||||
|
||||
def _get_system_version():
|
||||
"""Return the OS X system version as a string"""
|
||||
# Reading this plist is a documented way to get the system
|
||||
# version (see the documentation for the Gestalt Manager)
|
||||
# We avoid using platform.mac_ver to avoid possible bootstrap issues during
|
||||
# the build of Python itself (distutils is used to build standard library
|
||||
# extensions).
|
||||
|
||||
global _SYSTEM_VERSION
|
||||
|
||||
if _SYSTEM_VERSION is None:
|
||||
_SYSTEM_VERSION = ''
|
||||
try:
|
||||
f = open('/System/Library/CoreServices/SystemVersion.plist', encoding="utf-8")
|
||||
except OSError:
|
||||
# We're on a plain darwin box, fall back to the default
|
||||
# behaviour.
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
m = re.search(r'<key>ProductUserVisibleVersion</key>\s*'
|
||||
r'<string>(.*?)</string>', f.read())
|
||||
finally:
|
||||
f.close()
|
||||
if m is not None:
|
||||
_SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2])
|
||||
# else: fall back to the default behaviour
|
||||
|
||||
return _SYSTEM_VERSION
|
||||
|
||||
_SYSTEM_VERSION_TUPLE = None
|
||||
def _get_system_version_tuple():
|
||||
"""
|
||||
Return the macOS system version as a tuple
|
||||
|
||||
The return value is safe to use to compare
|
||||
two version numbers.
|
||||
"""
|
||||
global _SYSTEM_VERSION_TUPLE
|
||||
if _SYSTEM_VERSION_TUPLE is None:
|
||||
osx_version = _get_system_version()
|
||||
if osx_version:
|
||||
try:
|
||||
_SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.'))
|
||||
except ValueError:
|
||||
_SYSTEM_VERSION_TUPLE = ()
|
||||
|
||||
return _SYSTEM_VERSION_TUPLE
|
||||
|
||||
|
||||
def _remove_original_values(_config_vars):
|
||||
"""Remove original unmodified values for testing"""
|
||||
# This is needed for higher-level cross-platform tests of get_platform.
|
||||
for k in list(_config_vars):
|
||||
if k.startswith(_INITPRE):
|
||||
del _config_vars[k]
|
||||
|
||||
def _save_modified_value(_config_vars, cv, newvalue):
|
||||
"""Save modified and original unmodified value of configuration var"""
|
||||
|
||||
oldvalue = _config_vars.get(cv, '')
|
||||
if (oldvalue != newvalue) and (_INITPRE + cv not in _config_vars):
|
||||
_config_vars[_INITPRE + cv] = oldvalue
|
||||
_config_vars[cv] = newvalue
|
||||
|
||||
|
||||
_cache_default_sysroot = None
|
||||
def _default_sysroot(cc):
|
||||
""" Returns the root of the default SDK for this system, or '/' """
|
||||
global _cache_default_sysroot
|
||||
|
||||
if _cache_default_sysroot is not None:
|
||||
return _cache_default_sysroot
|
||||
|
||||
contents = _read_output('%s -c -E -v - </dev/null' % (cc,), True)
|
||||
in_incdirs = False
|
||||
for line in contents.splitlines():
|
||||
if line.startswith("#include <...>"):
|
||||
in_incdirs = True
|
||||
elif line.startswith("End of search list"):
|
||||
in_incdirs = False
|
||||
elif in_incdirs:
|
||||
line = line.strip()
|
||||
if line == '/usr/include':
|
||||
_cache_default_sysroot = '/'
|
||||
elif line.endswith(".sdk/usr/include"):
|
||||
_cache_default_sysroot = line[:-12]
|
||||
if _cache_default_sysroot is None:
|
||||
_cache_default_sysroot = '/'
|
||||
|
||||
return _cache_default_sysroot
|
||||
|
||||
def _supports_universal_builds():
|
||||
"""Returns True if universal builds are supported on this system"""
|
||||
# As an approximation, we assume that if we are running on 10.4 or above,
|
||||
# then we are running with an Xcode environment that supports universal
|
||||
# builds, in particular -isysroot and -arch arguments to the compiler. This
|
||||
# is in support of allowing 10.4 universal builds to run on 10.3.x systems.
|
||||
|
||||
osx_version = _get_system_version_tuple()
|
||||
return bool(osx_version >= (10, 4)) if osx_version else False
|
||||
|
||||
def _supports_arm64_builds():
|
||||
"""Returns True if arm64 builds are supported on this system"""
|
||||
# There are two sets of systems supporting macOS/arm64 builds:
|
||||
# 1. macOS 11 and later, unconditionally
|
||||
# 2. macOS 10.15 with Xcode 12.2 or later
|
||||
# For now the second category is ignored.
|
||||
osx_version = _get_system_version_tuple()
|
||||
return osx_version >= (11, 0) if osx_version else False
|
||||
|
||||
|
||||
def _find_appropriate_compiler(_config_vars):
|
||||
"""Find appropriate C compiler for extension module builds"""
|
||||
|
||||
# Issue #13590:
|
||||
# The OSX location for the compiler varies between OSX
|
||||
# (or rather Xcode) releases. With older releases (up-to 10.5)
|
||||
# the compiler is in /usr/bin, with newer releases the compiler
|
||||
# can only be found inside Xcode.app if the "Command Line Tools"
|
||||
# are not installed.
|
||||
#
|
||||
# Furthermore, the compiler that can be used varies between
|
||||
# Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2'
|
||||
# as the compiler, after that 'clang' should be used because
|
||||
# gcc-4.2 is either not present, or a copy of 'llvm-gcc' that
|
||||
# miscompiles Python.
|
||||
|
||||
# skip checks if the compiler was overridden with a CC env variable
|
||||
if 'CC' in os.environ:
|
||||
return _config_vars
|
||||
|
||||
# The CC config var might contain additional arguments.
|
||||
# Ignore them while searching.
|
||||
cc = oldcc = _config_vars['CC'].split()[0]
|
||||
if not _find_executable(cc):
|
||||
# Compiler is not found on the shell search PATH.
|
||||
# Now search for clang, first on PATH (if the Command LIne
|
||||
# Tools have been installed in / or if the user has provided
|
||||
# another location via CC). If not found, try using xcrun
|
||||
# to find an uninstalled clang (within a selected Xcode).
|
||||
|
||||
# NOTE: Cannot use subprocess here because of bootstrap
|
||||
# issues when building Python itself (and os.popen is
|
||||
# implemented on top of subprocess and is therefore not
|
||||
# usable as well)
|
||||
|
||||
cc = _find_build_tool('clang')
|
||||
|
||||
elif os.path.basename(cc).startswith('gcc'):
|
||||
# Compiler is GCC, check if it is LLVM-GCC
|
||||
data = _read_output("'%s' --version"
|
||||
% (cc.replace("'", "'\"'\"'"),))
|
||||
if data and 'llvm-gcc' in data:
|
||||
# Found LLVM-GCC, fall back to clang
|
||||
cc = _find_build_tool('clang')
|
||||
|
||||
if not cc:
|
||||
raise SystemError(
|
||||
"Cannot locate working compiler")
|
||||
|
||||
if cc != oldcc:
|
||||
# Found a replacement compiler.
|
||||
# Modify config vars using new compiler, if not already explicitly
|
||||
# overridden by an env variable, preserving additional arguments.
|
||||
for cv in _COMPILER_CONFIG_VARS:
|
||||
if cv in _config_vars and cv not in os.environ:
|
||||
cv_split = _config_vars[cv].split()
|
||||
cv_split[0] = cc if cv != 'CXX' else cc + '++'
|
||||
_save_modified_value(_config_vars, cv, ' '.join(cv_split))
|
||||
|
||||
return _config_vars
|
||||
|
||||
|
||||
def _remove_universal_flags(_config_vars):
|
||||
"""Remove all universal build arguments from config vars"""
|
||||
|
||||
for cv in _UNIVERSAL_CONFIG_VARS:
|
||||
# Do not alter a config var explicitly overridden by env var
|
||||
if cv in _config_vars and cv not in os.environ:
|
||||
flags = _config_vars[cv]
|
||||
flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII)
|
||||
flags = re.sub(r'-isysroot\s*\S+', ' ', flags)
|
||||
_save_modified_value(_config_vars, cv, flags)
|
||||
|
||||
return _config_vars
|
||||
|
||||
|
||||
def _remove_unsupported_archs(_config_vars):
|
||||
"""Remove any unsupported archs from config vars"""
|
||||
# Different Xcode releases support different sets for '-arch'
|
||||
# flags. In particular, Xcode 4.x no longer supports the
|
||||
# PPC architectures.
|
||||
#
|
||||
# This code automatically removes '-arch ppc' and '-arch ppc64'
|
||||
# when these are not supported. That makes it possible to
|
||||
# build extensions on OSX 10.7 and later with the prebuilt
|
||||
# 32-bit installer on the python.org website.
|
||||
|
||||
# skip checks if the compiler was overridden with a CC env variable
|
||||
if 'CC' in os.environ:
|
||||
return _config_vars
|
||||
|
||||
if re.search(r'-arch\s+ppc', _config_vars['CFLAGS']) is not None:
|
||||
# NOTE: Cannot use subprocess here because of bootstrap
|
||||
# issues when building Python itself
|
||||
status = os.system(
|
||||
"""echo 'int main{};' | """
|
||||
"""'%s' -c -arch ppc -x c -o /dev/null /dev/null 2>/dev/null"""
|
||||
%(_config_vars['CC'].replace("'", "'\"'\"'"),))
|
||||
if status:
|
||||
# The compile failed for some reason. Because of differences
|
||||
# across Xcode and compiler versions, there is no reliable way
|
||||
# to be sure why it failed. Assume here it was due to lack of
|
||||
# PPC support and remove the related '-arch' flags from each
|
||||
# config variables not explicitly overridden by an environment
|
||||
# variable. If the error was for some other reason, we hope the
|
||||
# failure will show up again when trying to compile an extension
|
||||
# module.
|
||||
for cv in _UNIVERSAL_CONFIG_VARS:
|
||||
if cv in _config_vars and cv not in os.environ:
|
||||
flags = _config_vars[cv]
|
||||
flags = re.sub(r'-arch\s+ppc\w*\s', ' ', flags)
|
||||
_save_modified_value(_config_vars, cv, flags)
|
||||
|
||||
return _config_vars
|
||||
|
||||
|
||||
def _override_all_archs(_config_vars):
|
||||
"""Allow override of all archs with ARCHFLAGS env var"""
|
||||
# NOTE: This name was introduced by Apple in OSX 10.5 and
|
||||
# is used by several scripting languages distributed with
|
||||
# that OS release.
|
||||
if 'ARCHFLAGS' in os.environ:
|
||||
arch = os.environ['ARCHFLAGS']
|
||||
for cv in _UNIVERSAL_CONFIG_VARS:
|
||||
if cv in _config_vars and '-arch' in _config_vars[cv]:
|
||||
flags = _config_vars[cv]
|
||||
flags = re.sub(r'-arch\s+\w+\s', ' ', flags)
|
||||
flags = flags + ' ' + arch
|
||||
_save_modified_value(_config_vars, cv, flags)
|
||||
|
||||
return _config_vars
|
||||
|
||||
|
||||
def _check_for_unavailable_sdk(_config_vars):
|
||||
"""Remove references to any SDKs not available"""
|
||||
# If we're on OSX 10.5 or later and the user tries to
|
||||
# compile an extension using an SDK that is not present
|
||||
# on the current machine it is better to not use an SDK
|
||||
# than to fail. This is particularly important with
|
||||
# the standalone Command Line Tools alternative to a
|
||||
# full-blown Xcode install since the CLT packages do not
|
||||
# provide SDKs. If the SDK is not present, it is assumed
|
||||
# that the header files and dev libs have been installed
|
||||
# to /usr and /System/Library by either a standalone CLT
|
||||
# package or the CLT component within Xcode.
|
||||
cflags = _config_vars.get('CFLAGS', '')
|
||||
m = re.search(r'-isysroot\s*(\S+)', cflags)
|
||||
if m is not None:
|
||||
sdk = m.group(1)
|
||||
if not os.path.exists(sdk):
|
||||
for cv in _UNIVERSAL_CONFIG_VARS:
|
||||
# Do not alter a config var explicitly overridden by env var
|
||||
if cv in _config_vars and cv not in os.environ:
|
||||
flags = _config_vars[cv]
|
||||
flags = re.sub(r'-isysroot\s*\S+(?:\s|$)', ' ', flags)
|
||||
_save_modified_value(_config_vars, cv, flags)
|
||||
|
||||
return _config_vars
|
||||
|
||||
|
||||
def compiler_fixup(compiler_so, cc_args):
|
||||
"""
|
||||
This function will strip '-isysroot PATH' and '-arch ARCH' from the
|
||||
compile flags if the user has specified one them in extra_compile_flags.
|
||||
|
||||
This is needed because '-arch ARCH' adds another architecture to the
|
||||
build, without a way to remove an architecture. Furthermore GCC will
|
||||
barf if multiple '-isysroot' arguments are present.
|
||||
"""
|
||||
stripArch = stripSysroot = False
|
||||
|
||||
compiler_so = list(compiler_so)
|
||||
|
||||
if not _supports_universal_builds():
|
||||
# OSX before 10.4.0, these don't support -arch and -isysroot at
|
||||
# all.
|
||||
stripArch = stripSysroot = True
|
||||
else:
|
||||
stripArch = '-arch' in cc_args
|
||||
stripSysroot = any(arg for arg in cc_args if arg.startswith('-isysroot'))
|
||||
|
||||
if stripArch or 'ARCHFLAGS' in os.environ:
|
||||
while True:
|
||||
try:
|
||||
index = compiler_so.index('-arch')
|
||||
# Strip this argument and the next one:
|
||||
del compiler_so[index:index+2]
|
||||
except ValueError:
|
||||
break
|
||||
|
||||
elif not _supports_arm64_builds():
|
||||
# Look for "-arch arm64" and drop that
|
||||
for idx in reversed(range(len(compiler_so))):
|
||||
if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64":
|
||||
del compiler_so[idx:idx+2]
|
||||
|
||||
if 'ARCHFLAGS' in os.environ and not stripArch:
|
||||
# User specified different -arch flags in the environ,
|
||||
# see also distutils.sysconfig
|
||||
compiler_so = compiler_so + os.environ['ARCHFLAGS'].split()
|
||||
|
||||
if stripSysroot:
|
||||
while True:
|
||||
indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
|
||||
if not indices:
|
||||
break
|
||||
index = indices[0]
|
||||
if compiler_so[index] == '-isysroot':
|
||||
# Strip this argument and the next one:
|
||||
del compiler_so[index:index+2]
|
||||
else:
|
||||
# It's '-isysroot/some/path' in one arg
|
||||
del compiler_so[index:index+1]
|
||||
|
||||
# Check if the SDK that is used during compilation actually exists,
|
||||
# the universal build requires the usage of a universal SDK and not all
|
||||
# users have that installed by default.
|
||||
sysroot = None
|
||||
argvar = cc_args
|
||||
indices = [i for i,x in enumerate(cc_args) if x.startswith('-isysroot')]
|
||||
if not indices:
|
||||
argvar = compiler_so
|
||||
indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
|
||||
|
||||
for idx in indices:
|
||||
if argvar[idx] == '-isysroot':
|
||||
sysroot = argvar[idx+1]
|
||||
break
|
||||
else:
|
||||
sysroot = argvar[idx][len('-isysroot'):]
|
||||
break
|
||||
|
||||
if sysroot and not os.path.isdir(sysroot):
|
||||
sys.stderr.write(f"Compiling with an SDK that doesn't seem to exist: {sysroot}\n")
|
||||
sys.stderr.write("Please check your Xcode installation\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
return compiler_so
|
||||
|
||||
|
||||
def customize_config_vars(_config_vars):
|
||||
"""Customize Python build configuration variables.
|
||||
|
||||
Called internally from sysconfig with a mutable mapping
|
||||
containing name/value pairs parsed from the configured
|
||||
makefile used to build this interpreter. Returns
|
||||
the mapping updated as needed to reflect the environment
|
||||
in which the interpreter is running; in the case of
|
||||
a Python from a binary installer, the installed
|
||||
environment may be very different from the build
|
||||
environment, i.e. different OS levels, different
|
||||
built tools, different available CPU architectures.
|
||||
|
||||
This customization is performed whenever
|
||||
distutils.sysconfig.get_config_vars() is first
|
||||
called. It may be used in environments where no
|
||||
compilers are present, i.e. when installing pure
|
||||
Python dists. Customization of compiler paths
|
||||
and detection of unavailable archs is deferred
|
||||
until the first extension module build is
|
||||
requested (in distutils.sysconfig.customize_compiler).
|
||||
|
||||
Currently called from distutils.sysconfig
|
||||
"""
|
||||
|
||||
if not _supports_universal_builds():
|
||||
# On Mac OS X before 10.4, check if -arch and -isysroot
|
||||
# are in CFLAGS or LDFLAGS and remove them if they are.
|
||||
# This is needed when building extensions on a 10.3 system
|
||||
# using a universal build of python.
|
||||
_remove_universal_flags(_config_vars)
|
||||
|
||||
# Allow user to override all archs with ARCHFLAGS env var
|
||||
_override_all_archs(_config_vars)
|
||||
|
||||
# Remove references to sdks that are not found
|
||||
_check_for_unavailable_sdk(_config_vars)
|
||||
|
||||
return _config_vars
|
||||
|
||||
|
||||
def customize_compiler(_config_vars):
|
||||
"""Customize compiler path and configuration variables.
|
||||
|
||||
This customization is performed when the first
|
||||
extension module build is requested
|
||||
in distutils.sysconfig.customize_compiler.
|
||||
"""
|
||||
|
||||
# Find a compiler to use for extension module builds
|
||||
_find_appropriate_compiler(_config_vars)
|
||||
|
||||
# Remove ppc arch flags if not supported here
|
||||
_remove_unsupported_archs(_config_vars)
|
||||
|
||||
# Allow user to override all archs with ARCHFLAGS env var
|
||||
_override_all_archs(_config_vars)
|
||||
|
||||
return _config_vars
|
||||
|
||||
|
||||
def get_platform_osx(_config_vars, osname, release, machine):
|
||||
"""Filter values for get_platform()"""
|
||||
# called from get_platform() in sysconfig and distutils.util
|
||||
#
|
||||
# For our purposes, we'll assume that the system version from
|
||||
# distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set
|
||||
# to. This makes the compatibility story a bit more sane because the
|
||||
# machine is going to compile and link as if it were
|
||||
# MACOSX_DEPLOYMENT_TARGET.
|
||||
|
||||
macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '')
|
||||
macrelease = _get_system_version() or macver
|
||||
macver = macver or macrelease
|
||||
|
||||
if macver:
|
||||
release = macver
|
||||
osname = "macosx"
|
||||
|
||||
# Use the original CFLAGS value, if available, so that we
|
||||
# return the same machine type for the platform string.
|
||||
# Otherwise, distutils may consider this a cross-compiling
|
||||
# case and disallow installs.
|
||||
cflags = _config_vars.get(_INITPRE+'CFLAGS',
|
||||
_config_vars.get('CFLAGS', ''))
|
||||
if macrelease:
|
||||
try:
|
||||
macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
|
||||
except ValueError:
|
||||
macrelease = (10, 3)
|
||||
else:
|
||||
# assume no universal support
|
||||
macrelease = (10, 3)
|
||||
|
||||
if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
|
||||
# The universal build will build fat binaries, but not on
|
||||
# systems before 10.4
|
||||
|
||||
machine = 'fat'
|
||||
|
||||
archs = re.findall(r'-arch\s+(\S+)', cflags)
|
||||
archs = tuple(sorted(set(archs)))
|
||||
|
||||
if len(archs) == 1:
|
||||
machine = archs[0]
|
||||
elif archs == ('arm64', 'x86_64'):
|
||||
machine = 'universal2'
|
||||
elif archs == ('i386', 'ppc'):
|
||||
machine = 'fat'
|
||||
elif archs == ('i386', 'x86_64'):
|
||||
machine = 'intel'
|
||||
elif archs == ('i386', 'ppc', 'x86_64'):
|
||||
machine = 'fat3'
|
||||
elif archs == ('ppc64', 'x86_64'):
|
||||
machine = 'fat64'
|
||||
elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'):
|
||||
machine = 'universal'
|
||||
else:
|
||||
raise ValueError(
|
||||
"Don't know machine value for archs=%r" % (archs,))
|
||||
|
||||
elif machine == 'i386':
|
||||
# On OSX the machine type returned by uname is always the
|
||||
# 32-bit variant, even if the executable architecture is
|
||||
# the 64-bit variant
|
||||
if sys.maxsize >= 2**32:
|
||||
machine = 'x86_64'
|
||||
|
||||
elif machine in ('PowerPC', 'Power_Macintosh'):
|
||||
# Pick a sane name for the PPC architecture.
|
||||
# See 'i386' case
|
||||
if sys.maxsize >= 2**32:
|
||||
machine = 'ppc64'
|
||||
else:
|
||||
machine = 'ppc'
|
||||
|
||||
return (osname, release, machine)
|
||||
6
Lib/_py_abc.py
vendored
6
Lib/_py_abc.py
vendored
@@ -32,8 +32,8 @@ class ABCMeta(type):
|
||||
# external code.
|
||||
_abc_invalidation_counter = 0
|
||||
|
||||
def __new__(mcls, name, bases, namespace, /, **kwargs):
|
||||
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
|
||||
def __new__(mcls, name, bases, namespace, **kwargs):
|
||||
cls = type.__new__(mcls, name, bases, namespace, **kwargs)
|
||||
# Compute set of abstract method names
|
||||
abstracts = {name
|
||||
for name, value in namespace.items()
|
||||
@@ -43,7 +43,7 @@ class ABCMeta(type):
|
||||
value = getattr(cls, name, None)
|
||||
if getattr(value, "__isabstractmethod__", False):
|
||||
abstracts.add(name)
|
||||
cls.__abstractmethods__ = frozenset(abstracts)
|
||||
cls.__abstractmethods__ = set(abstracts)
|
||||
# Set up inheritance registry
|
||||
cls._abc_registry = WeakSet()
|
||||
cls._abc_cache = WeakSet()
|
||||
|
||||
2719
Lib/_pyio.py
vendored
2719
Lib/_pyio.py
vendored
File diff suppressed because it is too large
Load Diff
1297
Lib/_sre.py
vendored
Normal file
1297
Lib/_sre.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
13
Lib/_weakrefset.py
vendored
13
Lib/_weakrefset.py
vendored
@@ -3,7 +3,6 @@
|
||||
# by abc.py to load everything else at startup.
|
||||
|
||||
from _weakref import ref
|
||||
from types import GenericAlias
|
||||
|
||||
__all__ = ['WeakSet']
|
||||
|
||||
@@ -51,14 +50,10 @@ class WeakSet:
|
||||
self.update(data)
|
||||
|
||||
def _commit_removals(self):
|
||||
pop = self._pending_removals.pop
|
||||
l = self._pending_removals
|
||||
discard = self.data.discard
|
||||
while True:
|
||||
try:
|
||||
item = pop()
|
||||
except IndexError:
|
||||
return
|
||||
discard(item)
|
||||
while l:
|
||||
discard(l.pop())
|
||||
|
||||
def __iter__(self):
|
||||
with _IterationGuard(self):
|
||||
@@ -202,5 +197,3 @@ class WeakSet:
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.data)
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
94
Lib/abc.py
vendored
94
Lib/abc.py
vendored
@@ -11,14 +11,13 @@ def abstractmethod(funcobj):
|
||||
class that has a metaclass derived from ABCMeta cannot be
|
||||
instantiated unless all of its abstract methods are overridden.
|
||||
The abstract methods can be called using any of the normal
|
||||
'super' call mechanisms. abstractmethod() may be used to declare
|
||||
abstract methods for properties and descriptors.
|
||||
'super' call mechanisms.
|
||||
|
||||
Usage:
|
||||
|
||||
class C(metaclass=ABCMeta):
|
||||
@abstractmethod
|
||||
def my_abstract_method(self, arg1, arg2, argN):
|
||||
def my_abstract_method(self, ...):
|
||||
...
|
||||
"""
|
||||
funcobj.__isabstractmethod__ = True
|
||||
@@ -28,14 +27,17 @@ def abstractmethod(funcobj):
|
||||
class abstractclassmethod(classmethod):
|
||||
"""A decorator indicating abstract classmethods.
|
||||
|
||||
Deprecated, use 'classmethod' with 'abstractmethod' instead:
|
||||
Similar to abstractmethod.
|
||||
|
||||
class C(ABC):
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
Usage:
|
||||
|
||||
class C(metaclass=ABCMeta):
|
||||
@abstractclassmethod
|
||||
def my_abstract_classmethod(cls, ...):
|
||||
...
|
||||
|
||||
'abstractclassmethod' is deprecated. Use 'classmethod' with
|
||||
'abstractmethod' instead.
|
||||
"""
|
||||
|
||||
__isabstractmethod__ = True
|
||||
@@ -48,14 +50,17 @@ class abstractclassmethod(classmethod):
|
||||
class abstractstaticmethod(staticmethod):
|
||||
"""A decorator indicating abstract staticmethods.
|
||||
|
||||
Deprecated, use 'staticmethod' with 'abstractmethod' instead:
|
||||
Similar to abstractmethod.
|
||||
|
||||
class C(ABC):
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
Usage:
|
||||
|
||||
class C(metaclass=ABCMeta):
|
||||
@abstractstaticmethod
|
||||
def my_abstract_staticmethod(...):
|
||||
...
|
||||
|
||||
'abstractstaticmethod' is deprecated. Use 'staticmethod' with
|
||||
'abstractmethod' instead.
|
||||
"""
|
||||
|
||||
__isabstractmethod__ = True
|
||||
@@ -68,14 +73,29 @@ class abstractstaticmethod(staticmethod):
|
||||
class abstractproperty(property):
|
||||
"""A decorator indicating abstract properties.
|
||||
|
||||
Deprecated, use 'property' with 'abstractmethod' instead:
|
||||
Requires that the metaclass is ABCMeta or derived from it. A
|
||||
class that has a metaclass derived from ABCMeta cannot be
|
||||
instantiated unless all of its abstract properties are overridden.
|
||||
The abstract properties can be called using any of the normal
|
||||
'super' call mechanisms.
|
||||
|
||||
class C(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
Usage:
|
||||
|
||||
class C(metaclass=ABCMeta):
|
||||
@abstractproperty
|
||||
def my_abstract_property(self):
|
||||
...
|
||||
|
||||
This defines a read-only property; you can also define a read-write
|
||||
abstract property using the 'long' form of property declaration:
|
||||
|
||||
class C(metaclass=ABCMeta):
|
||||
def getx(self): ...
|
||||
def setx(self, value): ...
|
||||
x = abstractproperty(getx, setx)
|
||||
|
||||
'abstractproperty' is deprecated. Use 'property' with 'abstractmethod'
|
||||
instead.
|
||||
"""
|
||||
|
||||
__isabstractmethod__ = True
|
||||
@@ -85,10 +105,6 @@ try:
|
||||
from _abc import (get_cache_token, _abc_init, _abc_register,
|
||||
_abc_instancecheck, _abc_subclasscheck, _get_dump,
|
||||
_reset_registry, _reset_caches)
|
||||
# TODO: RUSTPYTHON missing _abc module implementation.
|
||||
except ModuleNotFoundError:
|
||||
from _py_abc import ABCMeta, get_cache_token
|
||||
ABCMeta.__module__ = 'abc'
|
||||
except ImportError:
|
||||
from _py_abc import ABCMeta, get_cache_token
|
||||
ABCMeta.__module__ = 'abc'
|
||||
@@ -106,7 +122,7 @@ else:
|
||||
implementations defined by the registering ABC be callable (not
|
||||
even via super()).
|
||||
"""
|
||||
def __new__(mcls, name, bases, namespace, /, **kwargs):
|
||||
def __new__(mcls, name, bases, namespace, **kwargs):
|
||||
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
|
||||
_abc_init(cls)
|
||||
return cls
|
||||
@@ -147,44 +163,6 @@ else:
|
||||
_reset_caches(cls)
|
||||
|
||||
|
||||
def update_abstractmethods(cls):
|
||||
"""Recalculate the set of abstract methods of an abstract class.
|
||||
|
||||
If a class has had one of its abstract methods implemented after the
|
||||
class was created, the method will not be considered implemented until
|
||||
this function is called. Alternatively, if a new abstract method has been
|
||||
added to the class, it will only be considered an abstract method of the
|
||||
class after this function is called.
|
||||
|
||||
This function should be called before any use is made of the class,
|
||||
usually in class decorators that add methods to the subject class.
|
||||
|
||||
Returns cls, to allow usage as a class decorator.
|
||||
|
||||
If cls is not an instance of ABCMeta, does nothing.
|
||||
"""
|
||||
if not hasattr(cls, '__abstractmethods__'):
|
||||
# We check for __abstractmethods__ here because cls might by a C
|
||||
# implementation or a python implementation (especially during
|
||||
# testing), and we want to handle both cases.
|
||||
return cls
|
||||
|
||||
abstracts = set()
|
||||
# Check the existing abstract methods of the parents, keep only the ones
|
||||
# that are not implemented.
|
||||
for scls in cls.__bases__:
|
||||
for name in getattr(scls, '__abstractmethods__', ()):
|
||||
value = getattr(cls, name, None)
|
||||
if getattr(value, "__isabstractmethod__", False):
|
||||
abstracts.add(name)
|
||||
# Also add any other newly added abstract methods.
|
||||
for name, value in cls.__dict__.items():
|
||||
if getattr(value, "__isabstractmethod__", False):
|
||||
abstracts.add(name)
|
||||
cls.__abstractmethods__ = frozenset(abstracts)
|
||||
return cls
|
||||
|
||||
|
||||
class ABC(metaclass=ABCMeta):
|
||||
"""Helper class that provides a standard way to create an ABC using
|
||||
inheritance.
|
||||
|
||||
65
Lib/aifc.py
vendored
65
Lib/aifc.py
vendored
@@ -138,11 +138,7 @@ import struct
|
||||
import builtins
|
||||
import warnings
|
||||
|
||||
__all__ = ["Error", "open"]
|
||||
|
||||
|
||||
warnings._deprecated(__name__, remove=(3, 13))
|
||||
|
||||
__all__ = ["Error", "open", "openfp"]
|
||||
|
||||
class Error(Exception):
|
||||
pass
|
||||
@@ -255,9 +251,7 @@ def _write_float(f, x):
|
||||
_write_ulong(f, himant)
|
||||
_write_ulong(f, lomant)
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
from chunk import Chunk
|
||||
from chunk import Chunk
|
||||
from collections import namedtuple
|
||||
|
||||
_aifc_params = namedtuple('_aifc_params',
|
||||
@@ -453,33 +447,21 @@ class Aifc_read:
|
||||
#
|
||||
|
||||
def _alaw2lin(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
return audioop.alaw2lin(data, 2)
|
||||
|
||||
def _ulaw2lin(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
return audioop.ulaw2lin(data, 2)
|
||||
|
||||
def _adpcm2lin(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
if not hasattr(self, '_adpcmstate'):
|
||||
# first time
|
||||
self._adpcmstate = None
|
||||
data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
|
||||
return data
|
||||
|
||||
def _sowt2lin(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
return audioop.byteswap(data, 2)
|
||||
|
||||
def _read_comm_chunk(self, chunk):
|
||||
self._nchannels = _read_short(chunk)
|
||||
self._nframes = _read_long(chunk)
|
||||
@@ -515,8 +497,6 @@ class Aifc_read:
|
||||
self._convert = self._ulaw2lin
|
||||
elif self._comptype in (b'alaw', b'ALAW'):
|
||||
self._convert = self._alaw2lin
|
||||
elif self._comptype in (b'sowt', b'SOWT'):
|
||||
self._convert = self._sowt2lin
|
||||
else:
|
||||
raise Error('unsupported compression type')
|
||||
self._sampwidth = 2
|
||||
@@ -679,7 +659,7 @@ class Aifc_write:
|
||||
if self._nframeswritten:
|
||||
raise Error('cannot change parameters after starting to write')
|
||||
if comptype not in (b'NONE', b'ulaw', b'ULAW',
|
||||
b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
|
||||
b'alaw', b'ALAW', b'G722'):
|
||||
raise Error('unsupported compression type')
|
||||
self._comptype = comptype
|
||||
self._compname = compname
|
||||
@@ -700,7 +680,7 @@ class Aifc_write:
|
||||
if self._nframeswritten:
|
||||
raise Error('cannot change parameters after starting to write')
|
||||
if comptype not in (b'NONE', b'ulaw', b'ULAW',
|
||||
b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
|
||||
b'alaw', b'ALAW', b'G722'):
|
||||
raise Error('unsupported compression type')
|
||||
self.setnchannels(nchannels)
|
||||
self.setsampwidth(sampwidth)
|
||||
@@ -784,43 +764,28 @@ class Aifc_write:
|
||||
#
|
||||
|
||||
def _lin2alaw(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
return audioop.lin2alaw(data, 2)
|
||||
|
||||
def _lin2ulaw(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
return audioop.lin2ulaw(data, 2)
|
||||
|
||||
def _lin2adpcm(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
if not hasattr(self, '_adpcmstate'):
|
||||
self._adpcmstate = None
|
||||
data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
|
||||
return data
|
||||
|
||||
def _lin2sowt(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
return audioop.byteswap(data, 2)
|
||||
|
||||
def _ensure_header_written(self, datasize):
|
||||
if not self._nframeswritten:
|
||||
if self._comptype in (b'ULAW', b'ulaw',
|
||||
b'ALAW', b'alaw', b'G722',
|
||||
b'sowt', b'SOWT'):
|
||||
if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
|
||||
if not self._sampwidth:
|
||||
self._sampwidth = 2
|
||||
if self._sampwidth != 2:
|
||||
raise Error('sample width must be 2 when compressing '
|
||||
'with ulaw/ULAW, alaw/ALAW, sowt/SOWT '
|
||||
'or G7.22 (ADPCM)')
|
||||
'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)')
|
||||
if not self._nchannels:
|
||||
raise Error('# channels not specified')
|
||||
if not self._sampwidth:
|
||||
@@ -836,8 +801,6 @@ class Aifc_write:
|
||||
self._convert = self._lin2ulaw
|
||||
elif self._comptype in (b'alaw', b'ALAW'):
|
||||
self._convert = self._lin2alaw
|
||||
elif self._comptype in (b'sowt', b'SOWT'):
|
||||
self._convert = self._lin2sowt
|
||||
|
||||
def _write_header(self, initlength):
|
||||
if self._aifc and self._comptype != b'NONE':
|
||||
@@ -957,6 +920,10 @@ def open(f, mode=None):
|
||||
else:
|
||||
raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
|
||||
|
||||
def openfp(f, mode=None):
|
||||
warnings.warn("aifc.openfp is deprecated since Python 3.7. "
|
||||
"Use aifc.open instead.", DeprecationWarning, stacklevel=2)
|
||||
return open(f, mode=mode)
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
|
||||
4
Lib/antigravity.py
vendored
4
Lib/antigravity.py
vendored
@@ -11,7 +11,7 @@ def geohash(latitude, longitude, datedow):
|
||||
37.857713 -122.544543
|
||||
|
||||
'''
|
||||
# https://xkcd.com/426/
|
||||
h = hashlib.md5(datedow, usedforsecurity=False).hexdigest()
|
||||
# http://xkcd.com/426/
|
||||
h = hashlib.md5(datedow).hexdigest()
|
||||
p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])]
|
||||
print('%d%s %d%s' % (latitude, p[1:], longitude, q[1:]))
|
||||
|
||||
352
Lib/argparse.py
vendored
352
Lib/argparse.py
vendored
@@ -1,5 +1,4 @@
|
||||
# Author: Steven J. Bethard <steven.bethard@gmail.com>.
|
||||
# New maintainer as of 29 August 2019: Raymond Hettinger <raymond.hettinger@gmail.com>
|
||||
|
||||
"""Command-line parsing library
|
||||
|
||||
@@ -67,7 +66,6 @@ __all__ = [
|
||||
'ArgumentParser',
|
||||
'ArgumentError',
|
||||
'ArgumentTypeError',
|
||||
'BooleanOptionalAction',
|
||||
'FileType',
|
||||
'HelpFormatter',
|
||||
'ArgumentDefaultsHelpFormatter',
|
||||
@@ -89,8 +87,6 @@ import os as _os
|
||||
import re as _re
|
||||
import sys as _sys
|
||||
|
||||
import warnings
|
||||
|
||||
from gettext import gettext as _, ngettext
|
||||
|
||||
SUPPRESS = '==SUPPRESS=='
|
||||
@@ -131,7 +127,7 @@ class _AttributeHolder(object):
|
||||
return '%s(%s)' % (type_name, ', '.join(arg_strings))
|
||||
|
||||
def _get_kwargs(self):
|
||||
return list(self.__dict__.items())
|
||||
return sorted(self.__dict__.items())
|
||||
|
||||
def _get_args(self):
|
||||
return []
|
||||
@@ -153,7 +149,6 @@ def _copy_items(items):
|
||||
# Formatting Help
|
||||
# ===============
|
||||
|
||||
|
||||
class HelpFormatter(object):
|
||||
"""Formatter for generating usage messages and argument help strings.
|
||||
|
||||
@@ -169,12 +164,15 @@ class HelpFormatter(object):
|
||||
|
||||
# default setting for width
|
||||
if width is None:
|
||||
import shutil
|
||||
width = shutil.get_terminal_size().columns
|
||||
try:
|
||||
width = int(_os.environ['COLUMNS'])
|
||||
except (KeyError, ValueError):
|
||||
width = 80
|
||||
width -= 2
|
||||
|
||||
self._prog = prog
|
||||
self._indent_increment = indent_increment
|
||||
self._max_help_position = max_help_position
|
||||
self._max_help_position = min(max_help_position,
|
||||
max(width - 20, indent_increment * 2))
|
||||
self._width = width
|
||||
@@ -267,7 +265,7 @@ class HelpFormatter(object):
|
||||
invocations.append(get_invocation(subaction))
|
||||
|
||||
# update the maximum item length
|
||||
invocation_length = max(map(len, invocations))
|
||||
invocation_length = max([len(s) for s in invocations])
|
||||
action_length = invocation_length + self._current_indent
|
||||
self._action_max_length = max(self._action_max_length,
|
||||
action_length)
|
||||
@@ -345,22 +343,21 @@ class HelpFormatter(object):
|
||||
def get_lines(parts, indent, prefix=None):
|
||||
lines = []
|
||||
line = []
|
||||
indent_length = len(indent)
|
||||
if prefix is not None:
|
||||
line_len = len(prefix) - 1
|
||||
else:
|
||||
line_len = indent_length - 1
|
||||
line_len = len(indent) - 1
|
||||
for part in parts:
|
||||
if line_len + 1 + len(part) > text_width and line:
|
||||
lines.append(indent + ' '.join(line))
|
||||
line = []
|
||||
line_len = indent_length - 1
|
||||
line_len = len(indent) - 1
|
||||
line.append(part)
|
||||
line_len += len(part) + 1
|
||||
if line:
|
||||
lines.append(indent + ' '.join(line))
|
||||
if prefix is not None:
|
||||
lines[0] = lines[0][indent_length:]
|
||||
lines[0] = lines[0][len(indent):]
|
||||
return lines
|
||||
|
||||
# if prog is short, follow it with optionals or positionals
|
||||
@@ -396,44 +393,27 @@ class HelpFormatter(object):
|
||||
group_actions = set()
|
||||
inserts = {}
|
||||
for group in groups:
|
||||
if not group._group_actions:
|
||||
raise ValueError(f'empty group {group}')
|
||||
|
||||
try:
|
||||
start = actions.index(group._group_actions[0])
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
group_action_count = len(group._group_actions)
|
||||
end = start + group_action_count
|
||||
end = start + len(group._group_actions)
|
||||
if actions[start:end] == group._group_actions:
|
||||
|
||||
suppressed_actions_count = 0
|
||||
for action in group._group_actions:
|
||||
group_actions.add(action)
|
||||
if action.help is SUPPRESS:
|
||||
suppressed_actions_count += 1
|
||||
|
||||
exposed_actions_count = group_action_count - suppressed_actions_count
|
||||
|
||||
if not group.required:
|
||||
if start in inserts:
|
||||
inserts[start] += ' ['
|
||||
else:
|
||||
inserts[start] = '['
|
||||
if end in inserts:
|
||||
inserts[end] += ']'
|
||||
else:
|
||||
inserts[end] = ']'
|
||||
elif exposed_actions_count > 1:
|
||||
inserts[end] = ']'
|
||||
else:
|
||||
if start in inserts:
|
||||
inserts[start] += ' ('
|
||||
else:
|
||||
inserts[start] = '('
|
||||
if end in inserts:
|
||||
inserts[end] += ')'
|
||||
else:
|
||||
inserts[end] = ')'
|
||||
inserts[end] = ')'
|
||||
for i in range(start + 1, end):
|
||||
inserts[i] = '|'
|
||||
|
||||
@@ -470,7 +450,7 @@ class HelpFormatter(object):
|
||||
# if the Optional doesn't take a value, format is:
|
||||
# -s or --long
|
||||
if action.nargs == 0:
|
||||
part = action.format_usage()
|
||||
part = '%s' % option_string
|
||||
|
||||
# if the Optional takes a value, format is:
|
||||
# -s ARGS or --long ARGS
|
||||
@@ -499,6 +479,7 @@ class HelpFormatter(object):
|
||||
text = _re.sub(r'(%s) ' % open, r'\1', text)
|
||||
text = _re.sub(r' (%s)' % close, r'\1', text)
|
||||
text = _re.sub(r'%s *%s' % (open, close), r'', text)
|
||||
text = _re.sub(r'\(([^|]*)\)', r'\1', text)
|
||||
text = text.strip()
|
||||
|
||||
# return the text
|
||||
@@ -540,13 +521,12 @@ class HelpFormatter(object):
|
||||
parts = [action_header]
|
||||
|
||||
# if there was help for the action, add lines of help text
|
||||
if action.help and action.help.strip():
|
||||
if action.help:
|
||||
help_text = self._expand_help(action)
|
||||
if help_text:
|
||||
help_lines = self._split_lines(help_text, help_width)
|
||||
parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
|
||||
for line in help_lines[1:]:
|
||||
parts.append('%*s%s\n' % (help_position, '', line))
|
||||
help_lines = self._split_lines(help_text, help_width)
|
||||
parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
|
||||
for line in help_lines[1:]:
|
||||
parts.append('%*s%s\n' % (help_position, '', line))
|
||||
|
||||
# or add a newline if the description doesn't end with one
|
||||
elif not action_header.endswith('\n'):
|
||||
@@ -606,11 +586,7 @@ class HelpFormatter(object):
|
||||
elif action.nargs == OPTIONAL:
|
||||
result = '[%s]' % get_metavar(1)
|
||||
elif action.nargs == ZERO_OR_MORE:
|
||||
metavar = get_metavar(1)
|
||||
if len(metavar) == 2:
|
||||
result = '[%s [%s ...]]' % metavar
|
||||
else:
|
||||
result = '[%s ...]' % metavar
|
||||
result = '[%s [%s ...]]' % get_metavar(2)
|
||||
elif action.nargs == ONE_OR_MORE:
|
||||
result = '%s [%s ...]' % get_metavar(2)
|
||||
elif action.nargs == REMAINDER:
|
||||
@@ -620,10 +596,7 @@ class HelpFormatter(object):
|
||||
elif action.nargs == SUPPRESS:
|
||||
result = ''
|
||||
else:
|
||||
try:
|
||||
formats = ['%s' for _ in range(action.nargs)]
|
||||
except TypeError:
|
||||
raise ValueError("invalid nargs value") from None
|
||||
formats = ['%s' for _ in range(action.nargs)]
|
||||
result = ' '.join(formats) % get_metavar(action.nargs)
|
||||
return result
|
||||
|
||||
@@ -704,19 +677,8 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
|
||||
"""
|
||||
|
||||
def _get_help_string(self, action):
|
||||
"""
|
||||
Add the default value to the option help message.
|
||||
|
||||
ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't
|
||||
already present. This code will do that, detecting cornercases to
|
||||
prevent duplicates or cases where it wouldn't make sense to the end
|
||||
user.
|
||||
"""
|
||||
help = action.help
|
||||
if help is None:
|
||||
help = ''
|
||||
|
||||
if '%(default)' not in help:
|
||||
if '%(default)' not in action.help:
|
||||
if action.default is not SUPPRESS:
|
||||
defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
|
||||
if action.option_strings or action.nargs in defaulting_nargs:
|
||||
@@ -724,7 +686,6 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
|
||||
return help
|
||||
|
||||
|
||||
|
||||
class MetavarTypeHelpFormatter(HelpFormatter):
|
||||
"""Help message formatter which uses the argument 'type' as the default
|
||||
metavar value (instead of the argument 'dest')
|
||||
@@ -740,6 +701,7 @@ class MetavarTypeHelpFormatter(HelpFormatter):
|
||||
return action.type.__name__
|
||||
|
||||
|
||||
|
||||
# =====================
|
||||
# Options and Arguments
|
||||
# =====================
|
||||
@@ -748,13 +710,11 @@ def _get_action_name(argument):
|
||||
if argument is None:
|
||||
return None
|
||||
elif argument.option_strings:
|
||||
return '/'.join(argument.option_strings)
|
||||
return '/'.join(argument.option_strings)
|
||||
elif argument.metavar not in (None, SUPPRESS):
|
||||
return argument.metavar
|
||||
elif argument.dest not in (None, SUPPRESS):
|
||||
return argument.dest
|
||||
elif argument.choices:
|
||||
return '{' + ','.join(argument.choices) + '}'
|
||||
else:
|
||||
return None
|
||||
|
||||
@@ -774,7 +734,7 @@ class ArgumentError(Exception):
|
||||
if self.argument_name is None:
|
||||
format = '%(message)s'
|
||||
else:
|
||||
format = _('argument %(argument_name)s: %(message)s')
|
||||
format = 'argument %(argument_name)s: %(message)s'
|
||||
return format % dict(message=self.message,
|
||||
argument_name=self.argument_name)
|
||||
|
||||
@@ -870,79 +830,15 @@ class Action(_AttributeHolder):
|
||||
'default',
|
||||
'type',
|
||||
'choices',
|
||||
'required',
|
||||
'help',
|
||||
'metavar',
|
||||
]
|
||||
return [(name, getattr(self, name)) for name in names]
|
||||
|
||||
def format_usage(self):
|
||||
return self.option_strings[0]
|
||||
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
raise NotImplementedError(_('.__call__() not defined'))
|
||||
|
||||
|
||||
# FIXME: remove together with `BooleanOptionalAction` deprecated arguments.
|
||||
_deprecated_default = object()
|
||||
|
||||
class BooleanOptionalAction(Action):
|
||||
def __init__(self,
|
||||
option_strings,
|
||||
dest,
|
||||
default=None,
|
||||
type=_deprecated_default,
|
||||
choices=_deprecated_default,
|
||||
required=False,
|
||||
help=None,
|
||||
metavar=_deprecated_default):
|
||||
|
||||
_option_strings = []
|
||||
for option_string in option_strings:
|
||||
_option_strings.append(option_string)
|
||||
|
||||
if option_string.startswith('--'):
|
||||
option_string = '--no-' + option_string[2:]
|
||||
_option_strings.append(option_string)
|
||||
|
||||
# We need `_deprecated` special value to ban explicit arguments that
|
||||
# match default value. Like:
|
||||
# parser.add_argument('-f', action=BooleanOptionalAction, type=int)
|
||||
for field_name in ('type', 'choices', 'metavar'):
|
||||
if locals()[field_name] is not _deprecated_default:
|
||||
warnings._deprecated(
|
||||
field_name,
|
||||
"{name!r} is deprecated as of Python 3.12 and will be "
|
||||
"removed in Python {remove}.",
|
||||
remove=(3, 14))
|
||||
|
||||
if type is _deprecated_default:
|
||||
type = None
|
||||
if choices is _deprecated_default:
|
||||
choices = None
|
||||
if metavar is _deprecated_default:
|
||||
metavar = None
|
||||
|
||||
super().__init__(
|
||||
option_strings=_option_strings,
|
||||
dest=dest,
|
||||
nargs=0,
|
||||
default=default,
|
||||
type=type,
|
||||
choices=choices,
|
||||
required=required,
|
||||
help=help,
|
||||
metavar=metavar)
|
||||
|
||||
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
if option_string in self.option_strings:
|
||||
setattr(namespace, self.dest, not option_string.startswith('--no-'))
|
||||
|
||||
def format_usage(self):
|
||||
return ' | '.join(self.option_strings)
|
||||
|
||||
|
||||
class _StoreAction(Action):
|
||||
|
||||
def __init__(self,
|
||||
@@ -957,7 +853,7 @@ class _StoreAction(Action):
|
||||
help=None,
|
||||
metavar=None):
|
||||
if nargs == 0:
|
||||
raise ValueError('nargs for store actions must be != 0; if you '
|
||||
raise ValueError('nargs for store actions must be > 0; if you '
|
||||
'have nothing to store, actions such as store '
|
||||
'true or store const may be more appropriate')
|
||||
if const is not None and nargs != OPTIONAL:
|
||||
@@ -983,7 +879,7 @@ class _StoreConstAction(Action):
|
||||
def __init__(self,
|
||||
option_strings,
|
||||
dest,
|
||||
const=None,
|
||||
const,
|
||||
default=None,
|
||||
required=False,
|
||||
help=None,
|
||||
@@ -1049,7 +945,7 @@ class _AppendAction(Action):
|
||||
help=None,
|
||||
metavar=None):
|
||||
if nargs == 0:
|
||||
raise ValueError('nargs for append actions must be != 0; if arg '
|
||||
raise ValueError('nargs for append actions must be > 0; if arg '
|
||||
'strings are not supplying the value to append, '
|
||||
'the append const action may be more appropriate')
|
||||
if const is not None and nargs != OPTIONAL:
|
||||
@@ -1078,7 +974,7 @@ class _AppendConstAction(Action):
|
||||
def __init__(self,
|
||||
option_strings,
|
||||
dest,
|
||||
const=None,
|
||||
const,
|
||||
default=None,
|
||||
required=False,
|
||||
help=None,
|
||||
@@ -1210,13 +1106,6 @@ class _SubParsersAction(Action):
|
||||
|
||||
aliases = kwargs.pop('aliases', ())
|
||||
|
||||
if name in self._name_parser_map:
|
||||
raise ArgumentError(self, _('conflicting subparser: %s') % name)
|
||||
for alias in aliases:
|
||||
if alias in self._name_parser_map:
|
||||
raise ArgumentError(
|
||||
self, _('conflicting subparser alias: %s') % alias)
|
||||
|
||||
# create a pseudo-action to hold the choice help
|
||||
if 'help' in kwargs:
|
||||
help = kwargs.pop('help')
|
||||
@@ -1268,12 +1157,6 @@ class _SubParsersAction(Action):
|
||||
vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
|
||||
getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
|
||||
|
||||
class _ExtendAction(_AppendAction):
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
items = getattr(namespace, self.dest, None)
|
||||
items = _copy_items(items)
|
||||
items.extend(values)
|
||||
setattr(namespace, self.dest, items)
|
||||
|
||||
# ==============
|
||||
# Type classes
|
||||
@@ -1306,9 +1189,9 @@ class FileType(object):
|
||||
# the special argument "-" means sys.std{in,out}
|
||||
if string == '-':
|
||||
if 'r' in self._mode:
|
||||
return _sys.stdin.buffer if 'b' in self._mode else _sys.stdin
|
||||
elif any(c in self._mode for c in 'wax'):
|
||||
return _sys.stdout.buffer if 'b' in self._mode else _sys.stdout
|
||||
return _sys.stdin
|
||||
elif 'w' in self._mode:
|
||||
return _sys.stdout
|
||||
else:
|
||||
msg = _('argument "-" with mode %r') % self._mode
|
||||
raise ValueError(msg)
|
||||
@@ -1318,9 +1201,8 @@ class FileType(object):
|
||||
return open(string, self._mode, self._bufsize, self._encoding,
|
||||
self._errors)
|
||||
except OSError as e:
|
||||
args = {'filename': string, 'error': e}
|
||||
message = _("can't open '%(filename)s': %(error)s")
|
||||
raise ArgumentTypeError(message % args)
|
||||
message = _("can't open '%s': %s")
|
||||
raise ArgumentTypeError(message % (string, e))
|
||||
|
||||
def __repr__(self):
|
||||
args = self._mode, self._bufsize
|
||||
@@ -1383,7 +1265,6 @@ class _ActionsContainer(object):
|
||||
self.register('action', 'help', _HelpAction)
|
||||
self.register('action', 'version', _VersionAction)
|
||||
self.register('action', 'parsers', _SubParsersAction)
|
||||
self.register('action', 'extend', _ExtendAction)
|
||||
|
||||
# raise an exception if the conflict handler is invalid
|
||||
self._get_handler()
|
||||
@@ -1476,10 +1357,6 @@ class _ActionsContainer(object):
|
||||
if not callable(type_func):
|
||||
raise ValueError('%r is not callable' % (type_func,))
|
||||
|
||||
if type_func is FileType:
|
||||
raise ValueError('%r is a FileType class object, instance of it'
|
||||
' must be passed' % (type_func,))
|
||||
|
||||
# raise an error if the metavar does not match the type
|
||||
if hasattr(self, "_get_formatter"):
|
||||
try:
|
||||
@@ -1594,8 +1471,10 @@ class _ActionsContainer(object):
|
||||
|
||||
# strings starting with two prefix characters are long options
|
||||
option_strings.append(option_string)
|
||||
if len(option_string) > 1 and option_string[1] in self.prefix_chars:
|
||||
long_option_strings.append(option_string)
|
||||
if option_string[0] in self.prefix_chars:
|
||||
if len(option_string) > 1:
|
||||
if option_string[1] in self.prefix_chars:
|
||||
long_option_strings.append(option_string)
|
||||
|
||||
# infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
|
||||
dest = kwargs.pop('dest', None)
|
||||
@@ -1697,14 +1576,6 @@ class _ArgumentGroup(_ActionsContainer):
|
||||
super(_ArgumentGroup, self)._remove_action(action)
|
||||
self._group_actions.remove(action)
|
||||
|
||||
def add_argument_group(self, *args, **kwargs):
|
||||
warnings.warn(
|
||||
"Nesting argument groups is deprecated.",
|
||||
category=DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
return super().add_argument_group(*args, **kwargs)
|
||||
|
||||
|
||||
class _MutuallyExclusiveGroup(_ArgumentGroup):
|
||||
|
||||
@@ -1725,21 +1596,12 @@ class _MutuallyExclusiveGroup(_ArgumentGroup):
|
||||
self._container._remove_action(action)
|
||||
self._group_actions.remove(action)
|
||||
|
||||
def add_mutually_exclusive_group(self, *args, **kwargs):
|
||||
warnings.warn(
|
||||
"Nesting mutually exclusive groups is deprecated.",
|
||||
category=DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
return super().add_mutually_exclusive_group(*args, **kwargs)
|
||||
|
||||
|
||||
class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
"""Object for parsing command line strings into Python objects.
|
||||
|
||||
Keyword Arguments:
|
||||
- prog -- The name of the program (default:
|
||||
``os.path.basename(sys.argv[0])``)
|
||||
- prog -- The name of the program (default: sys.argv[0])
|
||||
- usage -- A usage message (default: auto-generated from arguments)
|
||||
- description -- A description of what the program does
|
||||
- epilog -- Text following the argument descriptions
|
||||
@@ -1752,8 +1614,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
- conflict_handler -- String indicating how to handle conflicts
|
||||
- add_help -- Add a -h/-help option
|
||||
- allow_abbrev -- Allow long options to be abbreviated unambiguously
|
||||
- exit_on_error -- Determines whether or not ArgumentParser exits with
|
||||
error info when an error occurs
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
@@ -1768,14 +1628,19 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
argument_default=None,
|
||||
conflict_handler='error',
|
||||
add_help=True,
|
||||
allow_abbrev=True,
|
||||
exit_on_error=True):
|
||||
|
||||
superinit = super(ArgumentParser, self).__init__
|
||||
superinit(description=description,
|
||||
prefix_chars=prefix_chars,
|
||||
argument_default=argument_default,
|
||||
conflict_handler=conflict_handler)
|
||||
allow_abbrev=True):
|
||||
_ActionsContainer.__init__(self,
|
||||
description=description,
|
||||
prefix_chars=prefix_chars,
|
||||
argument_default=argument_default,
|
||||
conflict_handler=conflict_handler)
|
||||
# FIXME: get multiple inheritance method resolution right so we can use
|
||||
# what's below instead of the modified version above
|
||||
# superinit = super(ArgumentParser, self).__init__
|
||||
# superinit(description=description,
|
||||
# prefix_chars=prefix_chars,
|
||||
# argument_default=argument_default,
|
||||
# conflict_handler=conflict_handler)
|
||||
|
||||
# default setting for prog
|
||||
if prog is None:
|
||||
@@ -1788,11 +1653,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
self.fromfile_prefix_chars = fromfile_prefix_chars
|
||||
self.add_help = add_help
|
||||
self.allow_abbrev = allow_abbrev
|
||||
self.exit_on_error = exit_on_error
|
||||
|
||||
add_group = self.add_argument_group
|
||||
self._positionals = add_group(_('positional arguments'))
|
||||
self._optionals = add_group(_('options'))
|
||||
self._optionals = add_group(_('optional arguments'))
|
||||
self._subparsers = None
|
||||
|
||||
# register types
|
||||
@@ -1919,18 +1783,15 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
setattr(namespace, dest, self._defaults[dest])
|
||||
|
||||
# parse the arguments and exit if there are any errors
|
||||
if self.exit_on_error:
|
||||
try:
|
||||
namespace, args = self._parse_known_args(args, namespace)
|
||||
except ArgumentError as err:
|
||||
self.error(str(err))
|
||||
else:
|
||||
try:
|
||||
namespace, args = self._parse_known_args(args, namespace)
|
||||
|
||||
if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
|
||||
args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
|
||||
delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
|
||||
return namespace, args
|
||||
if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
|
||||
args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
|
||||
delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
|
||||
return namespace, args
|
||||
except ArgumentError:
|
||||
err = _sys.exc_info()[1]
|
||||
self.error(str(err))
|
||||
|
||||
def _parse_known_args(self, arg_strings, namespace):
|
||||
# replace arg strings that are file references
|
||||
@@ -2026,11 +1887,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
# arguments, try to parse more single-dash options out
|
||||
# of the tail of the option string
|
||||
chars = self.prefix_chars
|
||||
if (
|
||||
arg_count == 0
|
||||
and option_string[1] not in chars
|
||||
and explicit_arg != ''
|
||||
):
|
||||
if arg_count == 0 and option_string[1] not in chars:
|
||||
action_tuples.append((action, [], option_string))
|
||||
char = option_string[0]
|
||||
option_string = char + explicit_arg[0]
|
||||
@@ -2194,16 +2051,15 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
# replace arguments referencing files with the file content
|
||||
else:
|
||||
try:
|
||||
with open(arg_string[1:],
|
||||
encoding=_sys.getfilesystemencoding(),
|
||||
errors=_sys.getfilesystemencodeerrors()) as args_file:
|
||||
with open(arg_string[1:]) as args_file:
|
||||
arg_strings = []
|
||||
for arg_line in args_file.read().splitlines():
|
||||
for arg in self.convert_arg_line_to_args(arg_line):
|
||||
arg_strings.append(arg)
|
||||
arg_strings = self._read_args_from_files(arg_strings)
|
||||
new_arg_strings.extend(arg_strings)
|
||||
except OSError as err:
|
||||
except OSError:
|
||||
err = _sys.exc_info()[1]
|
||||
self.error(str(err))
|
||||
|
||||
# return the modified argument list
|
||||
@@ -2224,11 +2080,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
OPTIONAL: _('expected at most one argument'),
|
||||
ONE_OR_MORE: _('expected at least one argument'),
|
||||
}
|
||||
msg = nargs_errors.get(action.nargs)
|
||||
if msg is None:
|
||||
msg = ngettext('expected %s argument',
|
||||
default = ngettext('expected %s argument',
|
||||
'expected %s arguments',
|
||||
action.nargs) % action.nargs
|
||||
msg = nargs_errors.get(action.nargs, default)
|
||||
raise ArgumentError(action, msg)
|
||||
|
||||
# return the number of arguments matched
|
||||
@@ -2275,23 +2130,24 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
action = self._option_string_actions[option_string]
|
||||
return action, option_string, explicit_arg
|
||||
|
||||
# search through all possible prefixes of the option string
|
||||
# and all actions in the parser for possible interpretations
|
||||
option_tuples = self._get_option_tuples(arg_string)
|
||||
if self.allow_abbrev:
|
||||
# search through all possible prefixes of the option string
|
||||
# and all actions in the parser for possible interpretations
|
||||
option_tuples = self._get_option_tuples(arg_string)
|
||||
|
||||
# if multiple actions match, the option string was ambiguous
|
||||
if len(option_tuples) > 1:
|
||||
options = ', '.join([option_string
|
||||
for action, option_string, explicit_arg in option_tuples])
|
||||
args = {'option': arg_string, 'matches': options}
|
||||
msg = _('ambiguous option: %(option)s could match %(matches)s')
|
||||
self.error(msg % args)
|
||||
# if multiple actions match, the option string was ambiguous
|
||||
if len(option_tuples) > 1:
|
||||
options = ', '.join([option_string
|
||||
for action, option_string, explicit_arg in option_tuples])
|
||||
args = {'option': arg_string, 'matches': options}
|
||||
msg = _('ambiguous option: %(option)s could match %(matches)s')
|
||||
self.error(msg % args)
|
||||
|
||||
# if exactly one action matched, this segmentation is good,
|
||||
# so return the parsed action
|
||||
elif len(option_tuples) == 1:
|
||||
option_tuple, = option_tuples
|
||||
return option_tuple
|
||||
# if exactly one action matched, this segmentation is good,
|
||||
# so return the parsed action
|
||||
elif len(option_tuples) == 1:
|
||||
option_tuple, = option_tuples
|
||||
return option_tuple
|
||||
|
||||
# if it was not found as an option, but it looks like a negative
|
||||
# number, it was meant to be positional
|
||||
@@ -2315,17 +2171,16 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
# split at the '='
|
||||
chars = self.prefix_chars
|
||||
if option_string[0] in chars and option_string[1] in chars:
|
||||
if self.allow_abbrev:
|
||||
if '=' in option_string:
|
||||
option_prefix, explicit_arg = option_string.split('=', 1)
|
||||
else:
|
||||
option_prefix = option_string
|
||||
explicit_arg = None
|
||||
for option_string in self._option_string_actions:
|
||||
if option_string.startswith(option_prefix):
|
||||
action = self._option_string_actions[option_string]
|
||||
tup = action, option_string, explicit_arg
|
||||
result.append(tup)
|
||||
if '=' in option_string:
|
||||
option_prefix, explicit_arg = option_string.split('=', 1)
|
||||
else:
|
||||
option_prefix = option_string
|
||||
explicit_arg = None
|
||||
for option_string in self._option_string_actions:
|
||||
if option_string.startswith(option_prefix):
|
||||
action = self._option_string_actions[option_string]
|
||||
tup = action, option_string, explicit_arg
|
||||
result.append(tup)
|
||||
|
||||
# single character options can be concatenated with their arguments
|
||||
# but multiple character options always have to have their argument
|
||||
@@ -2510,11 +2365,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
not action.option_strings):
|
||||
if action.default is not None:
|
||||
value = action.default
|
||||
self._check_value(action, value)
|
||||
else:
|
||||
# since arg_strings is always [] at this point
|
||||
# there is no need to use self._check_value(action, value)
|
||||
value = arg_strings
|
||||
self._check_value(action, value)
|
||||
|
||||
# single argument or optional argument produces a single value
|
||||
elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
|
||||
@@ -2555,8 +2408,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
result = type_func(arg_string)
|
||||
|
||||
# ArgumentTypeErrors indicate errors
|
||||
except ArgumentTypeError as err:
|
||||
msg = str(err)
|
||||
except ArgumentTypeError:
|
||||
name = getattr(action.type, '__name__', repr(action.type))
|
||||
msg = str(_sys.exc_info()[1])
|
||||
raise ArgumentError(action, msg)
|
||||
|
||||
# TypeErrors or ValueErrors also indicate errors
|
||||
@@ -2627,11 +2481,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
|
||||
def _print_message(self, message, file=None):
|
||||
if message:
|
||||
file = file or _sys.stderr
|
||||
try:
|
||||
file.write(message)
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
if file is None:
|
||||
file = _sys.stderr
|
||||
file.write(message)
|
||||
|
||||
# ===============
|
||||
# Exiting methods
|
||||
|
||||
1604
Lib/ast.py
vendored
1604
Lib/ast.py
vendored
File diff suppressed because it is too large
Load Diff
307
Lib/asynchat.py
vendored
307
Lib/asynchat.py
vendored
@@ -1,307 +0,0 @@
|
||||
# -*- Mode: Python; tab-width: 4 -*-
|
||||
# Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp
|
||||
# Author: Sam Rushing <rushing@nightmare.com>
|
||||
|
||||
# ======================================================================
|
||||
# Copyright 1996 by Sam Rushing
|
||||
#
|
||||
# All Rights Reserved
|
||||
#
|
||||
# Permission to use, copy, modify, and distribute this software and
|
||||
# its documentation for any purpose and without fee is hereby
|
||||
# granted, provided that the above copyright notice appear in all
|
||||
# copies and that both that copyright notice and this permission
|
||||
# notice appear in supporting documentation, and that the name of Sam
|
||||
# Rushing not be used in advertising or publicity pertaining to
|
||||
# distribution of the software without specific, written prior
|
||||
# permission.
|
||||
#
|
||||
# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
|
||||
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
|
||||
# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
|
||||
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
|
||||
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
|
||||
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
|
||||
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
# ======================================================================
|
||||
|
||||
r"""A class supporting chat-style (command/response) protocols.
|
||||
|
||||
This class adds support for 'chat' style protocols - where one side
|
||||
sends a 'command', and the other sends a response (examples would be
|
||||
the common internet protocols - smtp, nntp, ftp, etc..).
|
||||
|
||||
The handle_read() method looks at the input stream for the current
|
||||
'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n'
|
||||
for multi-line output), calling self.found_terminator() on its
|
||||
receipt.
|
||||
|
||||
for example:
|
||||
Say you build an async nntp client using this class. At the start
|
||||
of the connection, you'll have self.terminator set to '\r\n', in
|
||||
order to process the single-line greeting. Just before issuing a
|
||||
'LIST' command you'll set it to '\r\n.\r\n'. The output of the LIST
|
||||
command will be accumulated (using your own 'collect_incoming_data'
|
||||
method) up to the terminator, and then control will be returned to
|
||||
you - by calling your self.found_terminator() method.
|
||||
"""
|
||||
import asyncore
|
||||
from collections import deque
|
||||
|
||||
|
||||
class async_chat(asyncore.dispatcher):
|
||||
"""This is an abstract class. You must derive from this class, and add
|
||||
the two methods collect_incoming_data() and found_terminator()"""
|
||||
|
||||
# these are overridable defaults
|
||||
|
||||
ac_in_buffer_size = 65536
|
||||
ac_out_buffer_size = 65536
|
||||
|
||||
# we don't want to enable the use of encoding by default, because that is a
|
||||
# sign of an application bug that we don't want to pass silently
|
||||
|
||||
use_encoding = 0
|
||||
encoding = 'latin-1'
|
||||
|
||||
def __init__(self, sock=None, map=None):
|
||||
# for string terminator matching
|
||||
self.ac_in_buffer = b''
|
||||
|
||||
# we use a list here rather than io.BytesIO for a few reasons...
|
||||
# del lst[:] is faster than bio.truncate(0)
|
||||
# lst = [] is faster than bio.truncate(0)
|
||||
self.incoming = []
|
||||
|
||||
# we toss the use of the "simple producer" and replace it with
|
||||
# a pure deque, which the original fifo was a wrapping of
|
||||
self.producer_fifo = deque()
|
||||
asyncore.dispatcher.__init__(self, sock, map)
|
||||
|
||||
def collect_incoming_data(self, data):
|
||||
raise NotImplementedError("must be implemented in subclass")
|
||||
|
||||
def _collect_incoming_data(self, data):
|
||||
self.incoming.append(data)
|
||||
|
||||
def _get_data(self):
|
||||
d = b''.join(self.incoming)
|
||||
del self.incoming[:]
|
||||
return d
|
||||
|
||||
def found_terminator(self):
|
||||
raise NotImplementedError("must be implemented in subclass")
|
||||
|
||||
def set_terminator(self, term):
|
||||
"""Set the input delimiter.
|
||||
|
||||
Can be a fixed string of any length, an integer, or None.
|
||||
"""
|
||||
if isinstance(term, str) and self.use_encoding:
|
||||
term = bytes(term, self.encoding)
|
||||
elif isinstance(term, int) and term < 0:
|
||||
raise ValueError('the number of received bytes must be positive')
|
||||
self.terminator = term
|
||||
|
||||
def get_terminator(self):
|
||||
return self.terminator
|
||||
|
||||
# grab some more data from the socket,
|
||||
# throw it to the collector method,
|
||||
# check for the terminator,
|
||||
# if found, transition to the next state.
|
||||
|
||||
def handle_read(self):
|
||||
|
||||
try:
|
||||
data = self.recv(self.ac_in_buffer_size)
|
||||
except BlockingIOError:
|
||||
return
|
||||
except OSError as why:
|
||||
self.handle_error()
|
||||
return
|
||||
|
||||
if isinstance(data, str) and self.use_encoding:
|
||||
data = bytes(str, self.encoding)
|
||||
self.ac_in_buffer = self.ac_in_buffer + data
|
||||
|
||||
# Continue to search for self.terminator in self.ac_in_buffer,
|
||||
# while calling self.collect_incoming_data. The while loop
|
||||
# is necessary because we might read several data+terminator
|
||||
# combos with a single recv(4096).
|
||||
|
||||
while self.ac_in_buffer:
|
||||
lb = len(self.ac_in_buffer)
|
||||
terminator = self.get_terminator()
|
||||
if not terminator:
|
||||
# no terminator, collect it all
|
||||
self.collect_incoming_data(self.ac_in_buffer)
|
||||
self.ac_in_buffer = b''
|
||||
elif isinstance(terminator, int):
|
||||
# numeric terminator
|
||||
n = terminator
|
||||
if lb < n:
|
||||
self.collect_incoming_data(self.ac_in_buffer)
|
||||
self.ac_in_buffer = b''
|
||||
self.terminator = self.terminator - lb
|
||||
else:
|
||||
self.collect_incoming_data(self.ac_in_buffer[:n])
|
||||
self.ac_in_buffer = self.ac_in_buffer[n:]
|
||||
self.terminator = 0
|
||||
self.found_terminator()
|
||||
else:
|
||||
# 3 cases:
|
||||
# 1) end of buffer matches terminator exactly:
|
||||
# collect data, transition
|
||||
# 2) end of buffer matches some prefix:
|
||||
# collect data to the prefix
|
||||
# 3) end of buffer does not match any prefix:
|
||||
# collect data
|
||||
terminator_len = len(terminator)
|
||||
index = self.ac_in_buffer.find(terminator)
|
||||
if index != -1:
|
||||
# we found the terminator
|
||||
if index > 0:
|
||||
# don't bother reporting the empty string
|
||||
# (source of subtle bugs)
|
||||
self.collect_incoming_data(self.ac_in_buffer[:index])
|
||||
self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
|
||||
# This does the Right Thing if the terminator
|
||||
# is changed here.
|
||||
self.found_terminator()
|
||||
else:
|
||||
# check for a prefix of the terminator
|
||||
index = find_prefix_at_end(self.ac_in_buffer, terminator)
|
||||
if index:
|
||||
if index != lb:
|
||||
# we found a prefix, collect up to the prefix
|
||||
self.collect_incoming_data(self.ac_in_buffer[:-index])
|
||||
self.ac_in_buffer = self.ac_in_buffer[-index:]
|
||||
break
|
||||
else:
|
||||
# no prefix, collect it all
|
||||
self.collect_incoming_data(self.ac_in_buffer)
|
||||
self.ac_in_buffer = b''
|
||||
|
||||
def handle_write(self):
|
||||
self.initiate_send()
|
||||
|
||||
def handle_close(self):
|
||||
self.close()
|
||||
|
||||
def push(self, data):
|
||||
if not isinstance(data, (bytes, bytearray, memoryview)):
|
||||
raise TypeError('data argument must be byte-ish (%r)',
|
||||
type(data))
|
||||
sabs = self.ac_out_buffer_size
|
||||
if len(data) > sabs:
|
||||
for i in range(0, len(data), sabs):
|
||||
self.producer_fifo.append(data[i:i+sabs])
|
||||
else:
|
||||
self.producer_fifo.append(data)
|
||||
self.initiate_send()
|
||||
|
||||
def push_with_producer(self, producer):
|
||||
self.producer_fifo.append(producer)
|
||||
self.initiate_send()
|
||||
|
||||
def readable(self):
|
||||
"predicate for inclusion in the readable for select()"
|
||||
# cannot use the old predicate, it violates the claim of the
|
||||
# set_terminator method.
|
||||
|
||||
# return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
|
||||
return 1
|
||||
|
||||
def writable(self):
|
||||
"predicate for inclusion in the writable for select()"
|
||||
return self.producer_fifo or (not self.connected)
|
||||
|
||||
def close_when_done(self):
|
||||
"automatically close this channel once the outgoing queue is empty"
|
||||
self.producer_fifo.append(None)
|
||||
|
||||
def initiate_send(self):
|
||||
while self.producer_fifo and self.connected:
|
||||
first = self.producer_fifo[0]
|
||||
# handle empty string/buffer or None entry
|
||||
if not first:
|
||||
del self.producer_fifo[0]
|
||||
if first is None:
|
||||
self.handle_close()
|
||||
return
|
||||
|
||||
# handle classic producer behavior
|
||||
obs = self.ac_out_buffer_size
|
||||
try:
|
||||
data = first[:obs]
|
||||
except TypeError:
|
||||
data = first.more()
|
||||
if data:
|
||||
self.producer_fifo.appendleft(data)
|
||||
else:
|
||||
del self.producer_fifo[0]
|
||||
continue
|
||||
|
||||
if isinstance(data, str) and self.use_encoding:
|
||||
data = bytes(data, self.encoding)
|
||||
|
||||
# send the data
|
||||
try:
|
||||
num_sent = self.send(data)
|
||||
except OSError:
|
||||
self.handle_error()
|
||||
return
|
||||
|
||||
if num_sent:
|
||||
if num_sent < len(data) or obs < len(first):
|
||||
self.producer_fifo[0] = first[num_sent:]
|
||||
else:
|
||||
del self.producer_fifo[0]
|
||||
# we tried to send some actual data
|
||||
return
|
||||
|
||||
def discard_buffers(self):
|
||||
# Emergencies only!
|
||||
self.ac_in_buffer = b''
|
||||
del self.incoming[:]
|
||||
self.producer_fifo.clear()
|
||||
|
||||
|
||||
class simple_producer:
|
||||
|
||||
def __init__(self, data, buffer_size=512):
|
||||
self.data = data
|
||||
self.buffer_size = buffer_size
|
||||
|
||||
def more(self):
|
||||
if len(self.data) > self.buffer_size:
|
||||
result = self.data[:self.buffer_size]
|
||||
self.data = self.data[self.buffer_size:]
|
||||
return result
|
||||
else:
|
||||
result = self.data
|
||||
self.data = b''
|
||||
return result
|
||||
|
||||
|
||||
# Given 'haystack', see if any prefix of 'needle' is at its end. This
|
||||
# assumes an exact match has already been checked. Return the number of
|
||||
# characters matched.
|
||||
# for example:
|
||||
# f_p_a_e("qwerty\r", "\r\n") => 1
|
||||
# f_p_a_e("qwertydkjf", "\r\n") => 0
|
||||
# f_p_a_e("qwerty\r\n", "\r\n") => <undefined>
|
||||
|
||||
# this could maybe be made faster with a computed regex?
|
||||
# [answer: no; circa Python-2.0, Jan 2001]
|
||||
# new python: 28961/s
|
||||
# old python: 18307/s
|
||||
# re: 12820/s
|
||||
# regex: 14035/s
|
||||
|
||||
def find_prefix_at_end(haystack, needle):
|
||||
l = len(needle) - 1
|
||||
while l and not haystack.endswith(needle[:l]):
|
||||
l -= 1
|
||||
return l
|
||||
@@ -184,9 +184,6 @@ class Future:
|
||||
context['source_traceback'] = self._source_traceback
|
||||
self._loop.call_exception_handler(context)
|
||||
|
||||
def __class_getitem__(cls, type):
|
||||
return cls
|
||||
|
||||
def cancel(self):
|
||||
"""Cancel the future and schedule callbacks.
|
||||
|
||||
|
||||
@@ -81,9 +81,6 @@ class Queue:
|
||||
def __str__(self):
|
||||
return '<{} {}>'.format(type(self).__name__, self._format())
|
||||
|
||||
def __class_getitem__(cls, type):
|
||||
return cls
|
||||
|
||||
def _format(self):
|
||||
result = 'maxsize={!r}'.format(self._maxsize)
|
||||
if getattr(self, '_queue', None):
|
||||
|
||||
@@ -4,7 +4,6 @@ __all__ = ['Task', 'create_task',
|
||||
'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED',
|
||||
'wait', 'wait_for', 'as_completed', 'sleep', 'async',
|
||||
'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe',
|
||||
'all_tasks'
|
||||
]
|
||||
|
||||
import concurrent.futures
|
||||
|
||||
@@ -9,7 +9,8 @@ if sys.platform != 'win32': # pragma: no cover
|
||||
|
||||
import _winapi
|
||||
import itertools
|
||||
import msvcrt
|
||||
# XXX RustPython TODO: msvcrt
|
||||
# import msvcrt
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
|
||||
642
Lib/asyncore.py
vendored
642
Lib/asyncore.py
vendored
@@ -1,642 +0,0 @@
|
||||
# -*- Mode: Python -*-
|
||||
# Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp
|
||||
# Author: Sam Rushing <rushing@nightmare.com>
|
||||
|
||||
# ======================================================================
|
||||
# Copyright 1996 by Sam Rushing
|
||||
#
|
||||
# All Rights Reserved
|
||||
#
|
||||
# Permission to use, copy, modify, and distribute this software and
|
||||
# its documentation for any purpose and without fee is hereby
|
||||
# granted, provided that the above copyright notice appear in all
|
||||
# copies and that both that copyright notice and this permission
|
||||
# notice appear in supporting documentation, and that the name of Sam
|
||||
# Rushing not be used in advertising or publicity pertaining to
|
||||
# distribution of the software without specific, written prior
|
||||
# permission.
|
||||
#
|
||||
# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
|
||||
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
|
||||
# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
|
||||
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
|
||||
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
|
||||
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
|
||||
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
# ======================================================================
|
||||
|
||||
"""Basic infrastructure for asynchronous socket service clients and servers.
|
||||
|
||||
There are only two ways to have a program on a single processor do "more
|
||||
than one thing at a time". Multi-threaded programming is the simplest and
|
||||
most popular way to do it, but there is another very different technique,
|
||||
that lets you have nearly all the advantages of multi-threading, without
|
||||
actually using multiple threads. it's really only practical if your program
|
||||
is largely I/O bound. If your program is CPU bound, then pre-emptive
|
||||
scheduled threads are probably what you really need. Network servers are
|
||||
rarely CPU-bound, however.
|
||||
|
||||
If your operating system supports the select() system call in its I/O
|
||||
library (and nearly all do), then you can use it to juggle multiple
|
||||
communication channels at once; doing other work while your I/O is taking
|
||||
place in the "background." Although this strategy can seem strange and
|
||||
complex, especially at first, it is in many ways easier to understand and
|
||||
control than multi-threaded programming. The module documented here solves
|
||||
many of the difficult problems for you, making the task of building
|
||||
sophisticated high-performance network servers and clients a snap.
|
||||
"""
|
||||
|
||||
import select
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import warnings
|
||||
|
||||
import os
|
||||
from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
|
||||
ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
|
||||
errorcode
|
||||
|
||||
_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
|
||||
EBADF})
|
||||
|
||||
try:
|
||||
socket_map
|
||||
except NameError:
|
||||
socket_map = {}
|
||||
|
||||
def _strerror(err):
|
||||
try:
|
||||
return os.strerror(err)
|
||||
except (ValueError, OverflowError, NameError):
|
||||
if err in errorcode:
|
||||
return errorcode[err]
|
||||
return "Unknown error %s" %err
|
||||
|
||||
class ExitNow(Exception):
|
||||
pass
|
||||
|
||||
_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit)
|
||||
|
||||
def read(obj):
|
||||
try:
|
||||
obj.handle_read_event()
|
||||
except _reraised_exceptions:
|
||||
raise
|
||||
except:
|
||||
obj.handle_error()
|
||||
|
||||
def write(obj):
|
||||
try:
|
||||
obj.handle_write_event()
|
||||
except _reraised_exceptions:
|
||||
raise
|
||||
except:
|
||||
obj.handle_error()
|
||||
|
||||
def _exception(obj):
|
||||
try:
|
||||
obj.handle_expt_event()
|
||||
except _reraised_exceptions:
|
||||
raise
|
||||
except:
|
||||
obj.handle_error()
|
||||
|
||||
def readwrite(obj, flags):
|
||||
try:
|
||||
if flags & select.POLLIN:
|
||||
obj.handle_read_event()
|
||||
if flags & select.POLLOUT:
|
||||
obj.handle_write_event()
|
||||
if flags & select.POLLPRI:
|
||||
obj.handle_expt_event()
|
||||
if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
|
||||
obj.handle_close()
|
||||
except OSError as e:
|
||||
if e.args[0] not in _DISCONNECTED:
|
||||
obj.handle_error()
|
||||
else:
|
||||
obj.handle_close()
|
||||
except _reraised_exceptions:
|
||||
raise
|
||||
except:
|
||||
obj.handle_error()
|
||||
|
||||
def poll(timeout=0.0, map=None):
|
||||
if map is None:
|
||||
map = socket_map
|
||||
if map:
|
||||
r = []; w = []; e = []
|
||||
for fd, obj in list(map.items()):
|
||||
is_r = obj.readable()
|
||||
is_w = obj.writable()
|
||||
if is_r:
|
||||
r.append(fd)
|
||||
# accepting sockets should not be writable
|
||||
if is_w and not obj.accepting:
|
||||
w.append(fd)
|
||||
if is_r or is_w:
|
||||
e.append(fd)
|
||||
if [] == r == w == e:
|
||||
time.sleep(timeout)
|
||||
return
|
||||
|
||||
r, w, e = select.select(r, w, e, timeout)
|
||||
|
||||
for fd in r:
|
||||
obj = map.get(fd)
|
||||
if obj is None:
|
||||
continue
|
||||
read(obj)
|
||||
|
||||
for fd in w:
|
||||
obj = map.get(fd)
|
||||
if obj is None:
|
||||
continue
|
||||
write(obj)
|
||||
|
||||
for fd in e:
|
||||
obj = map.get(fd)
|
||||
if obj is None:
|
||||
continue
|
||||
_exception(obj)
|
||||
|
||||
def poll2(timeout=0.0, map=None):
|
||||
# Use the poll() support added to the select module in Python 2.0
|
||||
if map is None:
|
||||
map = socket_map
|
||||
if timeout is not None:
|
||||
# timeout is in milliseconds
|
||||
timeout = int(timeout*1000)
|
||||
pollster = select.poll()
|
||||
if map:
|
||||
for fd, obj in list(map.items()):
|
||||
flags = 0
|
||||
if obj.readable():
|
||||
flags |= select.POLLIN | select.POLLPRI
|
||||
# accepting sockets should not be writable
|
||||
if obj.writable() and not obj.accepting:
|
||||
flags |= select.POLLOUT
|
||||
if flags:
|
||||
pollster.register(fd, flags)
|
||||
|
||||
r = pollster.poll(timeout)
|
||||
for fd, flags in r:
|
||||
obj = map.get(fd)
|
||||
if obj is None:
|
||||
continue
|
||||
readwrite(obj, flags)
|
||||
|
||||
poll3 = poll2 # Alias for backward compatibility
|
||||
|
||||
def loop(timeout=30.0, use_poll=False, map=None, count=None):
|
||||
if map is None:
|
||||
map = socket_map
|
||||
|
||||
if use_poll and hasattr(select, 'poll'):
|
||||
poll_fun = poll2
|
||||
else:
|
||||
poll_fun = poll
|
||||
|
||||
if count is None:
|
||||
while map:
|
||||
poll_fun(timeout, map)
|
||||
|
||||
else:
|
||||
while map and count > 0:
|
||||
poll_fun(timeout, map)
|
||||
count = count - 1
|
||||
|
||||
class dispatcher:
|
||||
|
||||
debug = False
|
||||
connected = False
|
||||
accepting = False
|
||||
connecting = False
|
||||
closing = False
|
||||
addr = None
|
||||
ignore_log_types = frozenset({'warning'})
|
||||
|
||||
def __init__(self, sock=None, map=None):
|
||||
if map is None:
|
||||
self._map = socket_map
|
||||
else:
|
||||
self._map = map
|
||||
|
||||
self._fileno = None
|
||||
|
||||
if sock:
|
||||
# Set to nonblocking just to make sure for cases where we
|
||||
# get a socket from a blocking source.
|
||||
sock.setblocking(0)
|
||||
self.set_socket(sock, map)
|
||||
self.connected = True
|
||||
# The constructor no longer requires that the socket
|
||||
# passed be connected.
|
||||
try:
|
||||
self.addr = sock.getpeername()
|
||||
except OSError as err:
|
||||
if err.args[0] in (ENOTCONN, EINVAL):
|
||||
# To handle the case where we got an unconnected
|
||||
# socket.
|
||||
self.connected = False
|
||||
else:
|
||||
# The socket is broken in some unknown way, alert
|
||||
# the user and remove it from the map (to prevent
|
||||
# polling of broken sockets).
|
||||
self.del_channel(map)
|
||||
raise
|
||||
else:
|
||||
self.socket = None
|
||||
|
||||
def __repr__(self):
|
||||
status = [self.__class__.__module__+"."+self.__class__.__qualname__]
|
||||
if self.accepting and self.addr:
|
||||
status.append('listening')
|
||||
elif self.connected:
|
||||
status.append('connected')
|
||||
if self.addr is not None:
|
||||
try:
|
||||
status.append('%s:%d' % self.addr)
|
||||
except TypeError:
|
||||
status.append(repr(self.addr))
|
||||
return '<%s at %#x>' % (' '.join(status), id(self))
|
||||
|
||||
def add_channel(self, map=None):
|
||||
#self.log_info('adding channel %s' % self)
|
||||
if map is None:
|
||||
map = self._map
|
||||
map[self._fileno] = self
|
||||
|
||||
def del_channel(self, map=None):
|
||||
fd = self._fileno
|
||||
if map is None:
|
||||
map = self._map
|
||||
if fd in map:
|
||||
#self.log_info('closing channel %d:%s' % (fd, self))
|
||||
del map[fd]
|
||||
self._fileno = None
|
||||
|
||||
def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM):
|
||||
self.family_and_type = family, type
|
||||
sock = socket.socket(family, type)
|
||||
sock.setblocking(0)
|
||||
self.set_socket(sock)
|
||||
|
||||
def set_socket(self, sock, map=None):
|
||||
self.socket = sock
|
||||
self._fileno = sock.fileno()
|
||||
self.add_channel(map)
|
||||
|
||||
def set_reuse_addr(self):
|
||||
# try to re-use a server port if possible
|
||||
try:
|
||||
self.socket.setsockopt(
|
||||
socket.SOL_SOCKET, socket.SO_REUSEADDR,
|
||||
self.socket.getsockopt(socket.SOL_SOCKET,
|
||||
socket.SO_REUSEADDR) | 1
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# ==================================================
|
||||
# predicates for select()
|
||||
# these are used as filters for the lists of sockets
|
||||
# to pass to select().
|
||||
# ==================================================
|
||||
|
||||
def readable(self):
|
||||
return True
|
||||
|
||||
def writable(self):
|
||||
return True
|
||||
|
||||
# ==================================================
|
||||
# socket object methods.
|
||||
# ==================================================
|
||||
|
||||
def listen(self, num):
|
||||
self.accepting = True
|
||||
if os.name == 'nt' and num > 5:
|
||||
num = 5
|
||||
return self.socket.listen(num)
|
||||
|
||||
def bind(self, addr):
|
||||
self.addr = addr
|
||||
return self.socket.bind(addr)
|
||||
|
||||
def connect(self, address):
|
||||
self.connected = False
|
||||
self.connecting = True
|
||||
err = self.socket.connect_ex(address)
|
||||
if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \
|
||||
or err == EINVAL and os.name == 'nt':
|
||||
self.addr = address
|
||||
return
|
||||
if err in (0, EISCONN):
|
||||
self.addr = address
|
||||
self.handle_connect_event()
|
||||
else:
|
||||
raise OSError(err, errorcode[err])
|
||||
|
||||
def accept(self):
|
||||
# XXX can return either an address pair or None
|
||||
try:
|
||||
conn, addr = self.socket.accept()
|
||||
except TypeError:
|
||||
return None
|
||||
except OSError as why:
|
||||
if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
|
||||
return None
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
return conn, addr
|
||||
|
||||
def send(self, data):
|
||||
try:
|
||||
result = self.socket.send(data)
|
||||
return result
|
||||
except OSError as why:
|
||||
if why.args[0] == EWOULDBLOCK:
|
||||
return 0
|
||||
elif why.args[0] in _DISCONNECTED:
|
||||
self.handle_close()
|
||||
return 0
|
||||
else:
|
||||
raise
|
||||
|
||||
def recv(self, buffer_size):
|
||||
try:
|
||||
data = self.socket.recv(buffer_size)
|
||||
if not data:
|
||||
# a closed connection is indicated by signaling
|
||||
# a read condition, and having recv() return 0.
|
||||
self.handle_close()
|
||||
return b''
|
||||
else:
|
||||
return data
|
||||
except OSError as why:
|
||||
# winsock sometimes raises ENOTCONN
|
||||
if why.args[0] in _DISCONNECTED:
|
||||
self.handle_close()
|
||||
return b''
|
||||
else:
|
||||
raise
|
||||
|
||||
def close(self):
|
||||
self.connected = False
|
||||
self.accepting = False
|
||||
self.connecting = False
|
||||
self.del_channel()
|
||||
if self.socket is not None:
|
||||
try:
|
||||
self.socket.close()
|
||||
except OSError as why:
|
||||
if why.args[0] not in (ENOTCONN, EBADF):
|
||||
raise
|
||||
|
||||
# log and log_info may be overridden to provide more sophisticated
|
||||
# logging and warning methods. In general, log is for 'hit' logging
|
||||
# and 'log_info' is for informational, warning and error logging.
|
||||
|
||||
def log(self, message):
|
||||
sys.stderr.write('log: %s\n' % str(message))
|
||||
|
||||
def log_info(self, message, type='info'):
|
||||
if type not in self.ignore_log_types:
|
||||
print('%s: %s' % (type, message))
|
||||
|
||||
def handle_read_event(self):
|
||||
if self.accepting:
|
||||
# accepting sockets are never connected, they "spawn" new
|
||||
# sockets that are connected
|
||||
self.handle_accept()
|
||||
elif not self.connected:
|
||||
if self.connecting:
|
||||
self.handle_connect_event()
|
||||
self.handle_read()
|
||||
else:
|
||||
self.handle_read()
|
||||
|
||||
def handle_connect_event(self):
|
||||
err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
|
||||
if err != 0:
|
||||
raise OSError(err, _strerror(err))
|
||||
self.handle_connect()
|
||||
self.connected = True
|
||||
self.connecting = False
|
||||
|
||||
def handle_write_event(self):
|
||||
if self.accepting:
|
||||
# Accepting sockets shouldn't get a write event.
|
||||
# We will pretend it didn't happen.
|
||||
return
|
||||
|
||||
if not self.connected:
|
||||
if self.connecting:
|
||||
self.handle_connect_event()
|
||||
self.handle_write()
|
||||
|
||||
def handle_expt_event(self):
|
||||
# handle_expt_event() is called if there might be an error on the
|
||||
# socket, or if there is OOB data
|
||||
# check for the error condition first
|
||||
err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
|
||||
if err != 0:
|
||||
# we can get here when select.select() says that there is an
|
||||
# exceptional condition on the socket
|
||||
# since there is an error, we'll go ahead and close the socket
|
||||
# like we would in a subclassed handle_read() that received no
|
||||
# data
|
||||
self.handle_close()
|
||||
else:
|
||||
self.handle_expt()
|
||||
|
||||
def handle_error(self):
|
||||
nil, t, v, tbinfo = compact_traceback()
|
||||
|
||||
# sometimes a user repr method will crash.
|
||||
try:
|
||||
self_repr = repr(self)
|
||||
except:
|
||||
self_repr = '<__repr__(self) failed for object at %0x>' % id(self)
|
||||
|
||||
self.log_info(
|
||||
'uncaptured python exception, closing channel %s (%s:%s %s)' % (
|
||||
self_repr,
|
||||
t,
|
||||
v,
|
||||
tbinfo
|
||||
),
|
||||
'error'
|
||||
)
|
||||
self.handle_close()
|
||||
|
||||
def handle_expt(self):
|
||||
self.log_info('unhandled incoming priority event', 'warning')
|
||||
|
||||
def handle_read(self):
|
||||
self.log_info('unhandled read event', 'warning')
|
||||
|
||||
def handle_write(self):
|
||||
self.log_info('unhandled write event', 'warning')
|
||||
|
||||
def handle_connect(self):
|
||||
self.log_info('unhandled connect event', 'warning')
|
||||
|
||||
def handle_accept(self):
|
||||
pair = self.accept()
|
||||
if pair is not None:
|
||||
self.handle_accepted(*pair)
|
||||
|
||||
def handle_accepted(self, sock, addr):
|
||||
sock.close()
|
||||
self.log_info('unhandled accepted event', 'warning')
|
||||
|
||||
def handle_close(self):
|
||||
self.log_info('unhandled close event', 'warning')
|
||||
self.close()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# adds simple buffered output capability, useful for simple clients.
|
||||
# [for more sophisticated usage use asynchat.async_chat]
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class dispatcher_with_send(dispatcher):
|
||||
|
||||
def __init__(self, sock=None, map=None):
|
||||
dispatcher.__init__(self, sock, map)
|
||||
self.out_buffer = b''
|
||||
|
||||
def initiate_send(self):
|
||||
num_sent = 0
|
||||
num_sent = dispatcher.send(self, self.out_buffer[:65536])
|
||||
self.out_buffer = self.out_buffer[num_sent:]
|
||||
|
||||
def handle_write(self):
|
||||
self.initiate_send()
|
||||
|
||||
def writable(self):
|
||||
return (not self.connected) or len(self.out_buffer)
|
||||
|
||||
def send(self, data):
|
||||
if self.debug:
|
||||
self.log_info('sending %s' % repr(data))
|
||||
self.out_buffer = self.out_buffer + data
|
||||
self.initiate_send()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# used for debugging.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def compact_traceback():
|
||||
t, v, tb = sys.exc_info()
|
||||
tbinfo = []
|
||||
if not tb: # Must have a traceback
|
||||
raise AssertionError("traceback does not exist")
|
||||
while tb:
|
||||
tbinfo.append((
|
||||
tb.tb_frame.f_code.co_filename,
|
||||
tb.tb_frame.f_code.co_name,
|
||||
str(tb.tb_lineno)
|
||||
))
|
||||
tb = tb.tb_next
|
||||
|
||||
# just to be safe
|
||||
del tb
|
||||
|
||||
file, function, line = tbinfo[-1]
|
||||
info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])
|
||||
return (file, function, line), t, v, info
|
||||
|
||||
def close_all(map=None, ignore_all=False):
|
||||
if map is None:
|
||||
map = socket_map
|
||||
for x in list(map.values()):
|
||||
try:
|
||||
x.close()
|
||||
except OSError as x:
|
||||
if x.args[0] == EBADF:
|
||||
pass
|
||||
elif not ignore_all:
|
||||
raise
|
||||
except _reraised_exceptions:
|
||||
raise
|
||||
except:
|
||||
if not ignore_all:
|
||||
raise
|
||||
map.clear()
|
||||
|
||||
# Asynchronous File I/O:
|
||||
#
|
||||
# After a little research (reading man pages on various unixen, and
|
||||
# digging through the linux kernel), I've determined that select()
|
||||
# isn't meant for doing asynchronous file i/o.
|
||||
# Heartening, though - reading linux/mm/filemap.c shows that linux
|
||||
# supports asynchronous read-ahead. So _MOST_ of the time, the data
|
||||
# will be sitting in memory for us already when we go to read it.
|
||||
#
|
||||
# What other OS's (besides NT) support async file i/o? [VMS?]
|
||||
#
|
||||
# Regardless, this is useful for pipes, and stdin/stdout...
|
||||
|
||||
if os.name == 'posix':
|
||||
class file_wrapper:
|
||||
# Here we override just enough to make a file
|
||||
# look like a socket for the purposes of asyncore.
|
||||
# The passed fd is automatically os.dup()'d
|
||||
|
||||
def __init__(self, fd):
|
||||
self.fd = os.dup(fd)
|
||||
|
||||
def __del__(self):
|
||||
if self.fd >= 0:
|
||||
warnings.warn("unclosed file %r" % self, ResourceWarning,
|
||||
source=self)
|
||||
self.close()
|
||||
|
||||
def recv(self, *args):
|
||||
return os.read(self.fd, *args)
|
||||
|
||||
def send(self, *args):
|
||||
return os.write(self.fd, *args)
|
||||
|
||||
def getsockopt(self, level, optname, buflen=None):
|
||||
if (level == socket.SOL_SOCKET and
|
||||
optname == socket.SO_ERROR and
|
||||
not buflen):
|
||||
return 0
|
||||
raise NotImplementedError("Only asyncore specific behaviour "
|
||||
"implemented.")
|
||||
|
||||
read = recv
|
||||
write = send
|
||||
|
||||
def close(self):
|
||||
if self.fd < 0:
|
||||
return
|
||||
fd = self.fd
|
||||
self.fd = -1
|
||||
os.close(fd)
|
||||
|
||||
def fileno(self):
|
||||
return self.fd
|
||||
|
||||
class file_dispatcher(dispatcher):
|
||||
|
||||
def __init__(self, fd, map=None):
|
||||
dispatcher.__init__(self, None, map)
|
||||
self.connected = True
|
||||
try:
|
||||
fd = fd.fileno()
|
||||
except AttributeError:
|
||||
pass
|
||||
self.set_file(fd)
|
||||
# set it to non-blocking mode
|
||||
os.set_blocking(fd, False)
|
||||
|
||||
def set_file(self, fd):
|
||||
self.socket = file_wrapper(fd)
|
||||
self._fileno = self.socket.fileno()
|
||||
self.add_channel()
|
||||
9
Lib/atexit.py
vendored
Normal file
9
Lib/atexit.py
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
# Dummy implementation of atexit
|
||||
|
||||
|
||||
def register(func, *args, **kwargs):
|
||||
return func
|
||||
|
||||
|
||||
def unregister(func):
|
||||
pass
|
||||
159
Lib/base64.py
vendored
159
Lib/base64.py
vendored
@@ -1,4 +1,4 @@
|
||||
#! /usr/bin/env python3
|
||||
#! /usr/bin/python3.6
|
||||
|
||||
"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings"""
|
||||
|
||||
@@ -16,7 +16,7 @@ __all__ = [
|
||||
'encode', 'decode', 'encodebytes', 'decodebytes',
|
||||
# Generalized interface for other encodings
|
||||
'b64encode', 'b64decode', 'b32encode', 'b32decode',
|
||||
'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode',
|
||||
'b16encode', 'b16decode',
|
||||
# Base85 and Ascii85 encodings
|
||||
'b85encode', 'b85decode', 'a85encode', 'a85decode',
|
||||
# Standard Base64 encoding
|
||||
@@ -76,16 +76,15 @@ def b64decode(s, altchars=None, validate=False):
|
||||
normal base-64 alphabet nor the alternative alphabet are discarded prior
|
||||
to the padding check. If validate is True, these non-alphabet characters
|
||||
in the input result in a binascii.Error.
|
||||
For more information about the strict base64 check, see:
|
||||
|
||||
https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64
|
||||
"""
|
||||
s = _bytes_from_decode_data(s)
|
||||
if altchars is not None:
|
||||
altchars = _bytes_from_decode_data(altchars)
|
||||
assert len(altchars) == 2, repr(altchars)
|
||||
s = s.translate(bytes.maketrans(altchars, b'+/'))
|
||||
return binascii.a2b_base64(s, strict_mode=validate)
|
||||
if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s):
|
||||
raise binascii.Error('Non-base64 digit found')
|
||||
return binascii.a2b_base64(s)
|
||||
|
||||
|
||||
def standard_b64encode(s):
|
||||
@@ -136,40 +135,19 @@ def urlsafe_b64decode(s):
|
||||
|
||||
|
||||
# Base32 encoding/decoding must be done in Python
|
||||
_B32_ENCODE_DOCSTRING = '''
|
||||
Encode the bytes-like objects using {encoding} and return a bytes object.
|
||||
'''
|
||||
_B32_DECODE_DOCSTRING = '''
|
||||
Decode the {encoding} encoded bytes-like object or ASCII string s.
|
||||
|
||||
Optional casefold is a flag specifying whether a lowercase alphabet is
|
||||
acceptable as input. For security purposes, the default is False.
|
||||
{extra_args}
|
||||
The result is returned as a bytes object. A binascii.Error is raised if
|
||||
the input is incorrectly padded or if there are non-alphabet
|
||||
characters present in the input.
|
||||
'''
|
||||
_B32_DECODE_MAP01_DOCSTRING = '''
|
||||
RFC 3548 allows for optional mapping of the digit 0 (zero) to the
|
||||
letter O (oh), and for optional mapping of the digit 1 (one) to
|
||||
either the letter I (eye) or letter L (el). The optional argument
|
||||
map01 when not None, specifies which letter the digit 1 should be
|
||||
mapped to (when map01 is not None, the digit 0 is always mapped to
|
||||
the letter O). For security purposes the default is None, so that
|
||||
0 and 1 are not allowed in the input.
|
||||
'''
|
||||
_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
|
||||
_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV'
|
||||
_b32tab2 = {}
|
||||
_b32rev = {}
|
||||
_b32tab2 = None
|
||||
_b32rev = None
|
||||
|
||||
def _b32encode(alphabet, s):
|
||||
def b32encode(s):
|
||||
"""Encode the bytes-like object s using Base32 and return a bytes object.
|
||||
"""
|
||||
global _b32tab2
|
||||
# Delay the initialization of the table to not waste memory
|
||||
# if the function is never called
|
||||
if alphabet not in _b32tab2:
|
||||
b32tab = [bytes((i,)) for i in alphabet]
|
||||
_b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab]
|
||||
if _b32tab2 is None:
|
||||
b32tab = [bytes((i,)) for i in _b32alphabet]
|
||||
_b32tab2 = [a + b for a in b32tab for b in b32tab]
|
||||
b32tab = None
|
||||
|
||||
if not isinstance(s, bytes_types):
|
||||
@@ -180,9 +158,9 @@ def _b32encode(alphabet, s):
|
||||
s = s + b'\0' * (5 - leftover) # Don't use += !
|
||||
encoded = bytearray()
|
||||
from_bytes = int.from_bytes
|
||||
b32tab2 = _b32tab2[alphabet]
|
||||
b32tab2 = _b32tab2
|
||||
for i in range(0, len(s), 5):
|
||||
c = from_bytes(s[i: i + 5]) # big endian
|
||||
c = from_bytes(s[i: i + 5], 'big')
|
||||
encoded += (b32tab2[c >> 30] + # bits 1 - 10
|
||||
b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20
|
||||
b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30
|
||||
@@ -199,12 +177,29 @@ def _b32encode(alphabet, s):
|
||||
encoded[-1:] = b'='
|
||||
return bytes(encoded)
|
||||
|
||||
def _b32decode(alphabet, s, casefold=False, map01=None):
|
||||
def b32decode(s, casefold=False, map01=None):
|
||||
"""Decode the Base32 encoded bytes-like object or ASCII string s.
|
||||
|
||||
Optional casefold is a flag specifying whether a lowercase alphabet is
|
||||
acceptable as input. For security purposes, the default is False.
|
||||
|
||||
RFC 3548 allows for optional mapping of the digit 0 (zero) to the
|
||||
letter O (oh), and for optional mapping of the digit 1 (one) to
|
||||
either the letter I (eye) or letter L (el). The optional argument
|
||||
map01 when not None, specifies which letter the digit 1 should be
|
||||
mapped to (when map01 is not None, the digit 0 is always mapped to
|
||||
the letter O). For security purposes the default is None, so that
|
||||
0 and 1 are not allowed in the input.
|
||||
|
||||
The result is returned as a bytes object. A binascii.Error is raised if
|
||||
the input is incorrectly padded or if there are non-alphabet
|
||||
characters present in the input.
|
||||
"""
|
||||
global _b32rev
|
||||
# Delay the initialization of the table to not waste memory
|
||||
# if the function is never called
|
||||
if alphabet not in _b32rev:
|
||||
_b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)}
|
||||
if _b32rev is None:
|
||||
_b32rev = {v: k for k, v in enumerate(_b32alphabet)}
|
||||
s = _bytes_from_decode_data(s)
|
||||
if len(s) % 8:
|
||||
raise binascii.Error('Incorrect padding')
|
||||
@@ -225,7 +220,7 @@ def _b32decode(alphabet, s, casefold=False, map01=None):
|
||||
padchars = l - len(s)
|
||||
# Now decode the full quanta
|
||||
decoded = bytearray()
|
||||
b32rev = _b32rev[alphabet]
|
||||
b32rev = _b32rev
|
||||
for i in range(0, len(s), 8):
|
||||
quanta = s[i: i + 8]
|
||||
acc = 0
|
||||
@@ -234,38 +229,18 @@ def _b32decode(alphabet, s, casefold=False, map01=None):
|
||||
acc = (acc << 5) + b32rev[c]
|
||||
except KeyError:
|
||||
raise binascii.Error('Non-base32 digit found') from None
|
||||
decoded += acc.to_bytes(5) # big endian
|
||||
decoded += acc.to_bytes(5, 'big')
|
||||
# Process the last, partial quanta
|
||||
if l % 8 or padchars not in {0, 1, 3, 4, 6}:
|
||||
raise binascii.Error('Incorrect padding')
|
||||
if padchars and decoded:
|
||||
acc <<= 5 * padchars
|
||||
last = acc.to_bytes(5) # big endian
|
||||
last = acc.to_bytes(5, 'big')
|
||||
leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1
|
||||
decoded[-5:] = last[:leftover]
|
||||
return bytes(decoded)
|
||||
|
||||
|
||||
def b32encode(s):
|
||||
return _b32encode(_b32alphabet, s)
|
||||
b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32')
|
||||
|
||||
def b32decode(s, casefold=False, map01=None):
|
||||
return _b32decode(_b32alphabet, s, casefold, map01)
|
||||
b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32',
|
||||
extra_args=_B32_DECODE_MAP01_DOCSTRING)
|
||||
|
||||
def b32hexencode(s):
|
||||
return _b32encode(_b32hexalphabet, s)
|
||||
b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex')
|
||||
|
||||
def b32hexdecode(s, casefold=False):
|
||||
# base32hex does not have the 01 mapping
|
||||
return _b32decode(_b32hexalphabet, s, casefold)
|
||||
b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex',
|
||||
extra_args='')
|
||||
|
||||
|
||||
# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
|
||||
# lowercase. The RFC also recommends against accepting input case
|
||||
# insensitively.
|
||||
@@ -345,7 +320,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
|
||||
global _a85chars, _a85chars2
|
||||
# Delay the initialization of tables to not waste memory
|
||||
# if the function is never called
|
||||
if _a85chars2 is None:
|
||||
if _a85chars is None:
|
||||
_a85chars = [bytes((i,)) for i in range(33, 118)]
|
||||
_a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]
|
||||
|
||||
@@ -453,7 +428,7 @@ def b85encode(b, pad=False):
|
||||
global _b85chars, _b85chars2
|
||||
# Delay the initialization of tables to not waste memory
|
||||
# if the function is never called
|
||||
if _b85chars2 is None:
|
||||
if _b85chars is None:
|
||||
_b85chars = [bytes((i,)) for i in _b85alphabet]
|
||||
_b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
|
||||
return _85encode(b, _b85chars, _b85chars2, pad)
|
||||
@@ -508,8 +483,14 @@ MAXBINSIZE = (MAXLINESIZE//4)*3
|
||||
|
||||
def encode(input, output):
|
||||
"""Encode a file; input and output are binary files."""
|
||||
while s := input.read(MAXBINSIZE):
|
||||
while len(s) < MAXBINSIZE and (ns := input.read(MAXBINSIZE-len(s))):
|
||||
while True:
|
||||
s = input.read(MAXBINSIZE)
|
||||
if not s:
|
||||
break
|
||||
while len(s) < MAXBINSIZE:
|
||||
ns = input.read(MAXBINSIZE-len(s))
|
||||
if not ns:
|
||||
break
|
||||
s += ns
|
||||
line = binascii.b2a_base64(s)
|
||||
output.write(line)
|
||||
@@ -517,7 +498,10 @@ def encode(input, output):
|
||||
|
||||
def decode(input, output):
|
||||
"""Decode a file; input and output are binary files."""
|
||||
while line := input.readline():
|
||||
while True:
|
||||
line = input.readline()
|
||||
if not line:
|
||||
break
|
||||
s = binascii.a2b_base64(line)
|
||||
output.write(s)
|
||||
|
||||
@@ -547,34 +531,49 @@ def encodebytes(s):
|
||||
pieces.append(binascii.b2a_base64(chunk))
|
||||
return b"".join(pieces)
|
||||
|
||||
def encodestring(s):
|
||||
"""Legacy alias of encodebytes()."""
|
||||
import warnings
|
||||
warnings.warn("encodestring() is a deprecated alias since 3.1, "
|
||||
"use encodebytes()",
|
||||
DeprecationWarning, 2)
|
||||
return encodebytes(s)
|
||||
|
||||
|
||||
def decodebytes(s):
|
||||
"""Decode a bytestring of base-64 data into a bytes object."""
|
||||
_input_type_check(s)
|
||||
return binascii.a2b_base64(s)
|
||||
|
||||
def decodestring(s):
|
||||
"""Legacy alias of decodebytes()."""
|
||||
import warnings
|
||||
warnings.warn("decodestring() is a deprecated alias since Python 3.1, "
|
||||
"use decodebytes()",
|
||||
DeprecationWarning, 2)
|
||||
return decodebytes(s)
|
||||
|
||||
|
||||
# Usable as a script...
|
||||
def main():
|
||||
"""Small main program"""
|
||||
import sys, getopt
|
||||
usage = f"""usage: {sys.argv[0]} [-h|-d|-e|-u] [file|-]
|
||||
-h: print this help message and exit
|
||||
-d, -u: decode
|
||||
-e: encode (default)"""
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'hdeu')
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'deut')
|
||||
except getopt.error as msg:
|
||||
sys.stdout = sys.stderr
|
||||
print(msg)
|
||||
print(usage)
|
||||
print("""usage: %s [-d|-e|-u|-t] [file|-]
|
||||
-d, -u: decode
|
||||
-e: encode (default)
|
||||
-t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0])
|
||||
sys.exit(2)
|
||||
func = encode
|
||||
for o, a in opts:
|
||||
if o == '-e': func = encode
|
||||
if o == '-d': func = decode
|
||||
if o == '-u': func = decode
|
||||
if o == '-h': print(usage); return
|
||||
if o == '-t': test(); return
|
||||
if args and args[0] != '-':
|
||||
with open(args[0], 'rb') as f:
|
||||
func(f, sys.stdout.buffer)
|
||||
@@ -582,5 +581,15 @@ def main():
|
||||
func(sys.stdin.buffer, sys.stdout.buffer)
|
||||
|
||||
|
||||
def test():
|
||||
s0 = b"Aladdin:open sesame"
|
||||
print(repr(s0))
|
||||
s1 = encodebytes(s0)
|
||||
print(repr(s1))
|
||||
s2 = decodebytes(s1)
|
||||
print(repr(s2))
|
||||
assert s0 == s2
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
75
Lib/bdb.py
vendored
75
Lib/bdb.py
vendored
@@ -34,8 +34,6 @@ class Bdb:
|
||||
self.fncache = {}
|
||||
self.frame_returning = None
|
||||
|
||||
self._load_breaks()
|
||||
|
||||
def canonic(self, filename):
|
||||
"""Return canonical form of filename.
|
||||
|
||||
@@ -119,7 +117,7 @@ class Bdb:
|
||||
"""Invoke user function and return trace function for call event.
|
||||
|
||||
If the debugger stops on this function call, invoke
|
||||
self.user_call(). Raise BdbQuit if self.quitting is set.
|
||||
self.user_call(). Raise BbdQuit if self.quitting is set.
|
||||
Return self.trace_dispatch to continue tracing in this scope.
|
||||
"""
|
||||
# XXX 'arg' is no longer used
|
||||
@@ -367,12 +365,6 @@ class Bdb:
|
||||
# Call self.get_*break*() to see the breakpoints or better
|
||||
# for bp in Breakpoint.bpbynumber: if bp: bp.bpprint().
|
||||
|
||||
def _add_to_breaks(self, filename, lineno):
|
||||
"""Add breakpoint to breaks, if not already there."""
|
||||
bp_linenos = self.breaks.setdefault(filename, [])
|
||||
if lineno not in bp_linenos:
|
||||
bp_linenos.append(lineno)
|
||||
|
||||
def set_break(self, filename, lineno, temporary=False, cond=None,
|
||||
funcname=None):
|
||||
"""Set a new breakpoint for filename:lineno.
|
||||
@@ -385,21 +377,12 @@ class Bdb:
|
||||
line = linecache.getline(filename, lineno)
|
||||
if not line:
|
||||
return 'Line %s:%d does not exist' % (filename, lineno)
|
||||
self._add_to_breaks(filename, lineno)
|
||||
list = self.breaks.setdefault(filename, [])
|
||||
if lineno not in list:
|
||||
list.append(lineno)
|
||||
bp = Breakpoint(filename, lineno, temporary, cond, funcname)
|
||||
return None
|
||||
|
||||
def _load_breaks(self):
|
||||
"""Apply all breakpoints (set in other instances) to this one.
|
||||
|
||||
Populates this instance's breaks list from the Breakpoint class's
|
||||
list, which can have breakpoints set by another Bdb instance. This
|
||||
is necessary for interactive sessions to keep the breakpoints
|
||||
active across multiple calls to run().
|
||||
"""
|
||||
for (filename, lineno) in Breakpoint.bplist.keys():
|
||||
self._add_to_breaks(filename, lineno)
|
||||
|
||||
def _prune_breaks(self, filename, lineno):
|
||||
"""Prune breakpoints for filename:lineno.
|
||||
|
||||
@@ -570,12 +553,9 @@ class Bdb:
|
||||
rv = frame.f_locals['__return__']
|
||||
s += '->'
|
||||
s += reprlib.repr(rv)
|
||||
if lineno is not None:
|
||||
line = linecache.getline(filename, lineno, frame.f_globals)
|
||||
if line:
|
||||
s += lprefix + line.strip()
|
||||
else:
|
||||
s += f'{lprefix}Warning: lineno is None'
|
||||
line = linecache.getline(filename, lineno, frame.f_globals)
|
||||
if line:
|
||||
s += lprefix + line.strip()
|
||||
return s
|
||||
|
||||
# The following methods can be called by clients to use
|
||||
@@ -631,11 +611,26 @@ class Bdb:
|
||||
|
||||
# This method is more useful to debug a single function call.
|
||||
|
||||
def runcall(self, func, /, *args, **kwds):
|
||||
def runcall(*args, **kwds):
|
||||
"""Debug a single function call.
|
||||
|
||||
Return the result of the function call.
|
||||
"""
|
||||
if len(args) >= 2:
|
||||
self, func, *args = args
|
||||
elif not args:
|
||||
raise TypeError("descriptor 'runcall' of 'Bdb' object "
|
||||
"needs an argument")
|
||||
elif 'func' in kwds:
|
||||
func = kwds.pop('func')
|
||||
self, *args = args
|
||||
import warnings
|
||||
warnings.warn("Passing 'func' as keyword argument is deprecated",
|
||||
DeprecationWarning, stacklevel=2)
|
||||
else:
|
||||
raise TypeError('runcall expected at least 1 positional argument, '
|
||||
'got %d' % (len(args)-1))
|
||||
|
||||
self.reset()
|
||||
sys.settrace(self.trace_dispatch)
|
||||
res = None
|
||||
@@ -647,6 +642,7 @@ class Bdb:
|
||||
self.quitting = True
|
||||
sys.settrace(None)
|
||||
return res
|
||||
runcall.__text_signature__ = '($self, func, /, *args, **kwds)'
|
||||
|
||||
|
||||
def set_trace():
|
||||
@@ -701,12 +697,6 @@ class Breakpoint:
|
||||
else:
|
||||
self.bplist[file, line] = [self]
|
||||
|
||||
@staticmethod
|
||||
def clearBreakpoints():
|
||||
Breakpoint.next = 1
|
||||
Breakpoint.bplist = {}
|
||||
Breakpoint.bpbynumber = [None]
|
||||
|
||||
def deleteMe(self):
|
||||
"""Delete the breakpoint from the list associated to a file:line.
|
||||
|
||||
@@ -808,18 +798,15 @@ def checkfuncname(b, frame):
|
||||
return True
|
||||
|
||||
|
||||
# Determines if there is an effective (active) breakpoint at this
|
||||
# line of code. Returns breakpoint number or 0 if none
|
||||
def effective(file, line, frame):
|
||||
"""Return (active breakpoint, delete temporary flag) or (None, None) as
|
||||
breakpoint to act upon.
|
||||
"""Determine which breakpoint for this file:line is to be acted upon.
|
||||
|
||||
The "active breakpoint" is the first entry in bplist[line, file] (which
|
||||
must exist) that is enabled, for which checkfuncname is True, and that
|
||||
has neither a False condition nor a positive ignore count. The flag,
|
||||
meaning that a temporary breakpoint should be deleted, is False only
|
||||
when the condiion cannot be evaluated (in which case, ignore count is
|
||||
ignored).
|
||||
|
||||
If no such entry exists, then (None, None) is returned.
|
||||
Called only if we know there is a breakpoint at this location. Return
|
||||
the breakpoint that was triggered and a boolean that indicates if it is
|
||||
ok to delete a temporary breakpoint. Return (None, None) if there is no
|
||||
matching breakpoint.
|
||||
"""
|
||||
possibles = Breakpoint.bplist[file, line]
|
||||
for b in possibles:
|
||||
|
||||
138
Lib/bisect.py
vendored
138
Lib/bisect.py
vendored
@@ -1,118 +1,92 @@
|
||||
"""Bisection algorithms."""
|
||||
|
||||
|
||||
def insort_right(a, x, lo=0, hi=None, *, key=None):
|
||||
def insort_right(a, x, lo=0, hi=None):
|
||||
"""Insert item x in list a, and keep it sorted assuming a is sorted.
|
||||
|
||||
If x is already in a, insert it to the right of the rightmost x.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
|
||||
A custom key function can be supplied to customize the sort order.
|
||||
"""
|
||||
if key is None:
|
||||
lo = bisect_right(a, x, lo, hi)
|
||||
else:
|
||||
lo = bisect_right(a, key(x), lo, hi, key=key)
|
||||
a.insert(lo, x)
|
||||
|
||||
|
||||
def bisect_right(a, x, lo=0, hi=None, *, key=None):
|
||||
"""Return the index where to insert item x in list a, assuming a is sorted.
|
||||
|
||||
The return value i is such that all e in a[:i] have e <= x, and all e in
|
||||
a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will
|
||||
insert just after the rightmost x already there.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
|
||||
A custom key function can be supplied to customize the sort order.
|
||||
"""
|
||||
|
||||
if lo < 0:
|
||||
raise ValueError('lo must be non-negative')
|
||||
if hi is None:
|
||||
hi = len(a)
|
||||
# Note, the comparison uses "<" to match the
|
||||
# __lt__() logic in list.sort() and in heapq.
|
||||
if key is None:
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
if x < a[mid]:
|
||||
hi = mid
|
||||
else:
|
||||
lo = mid + 1
|
||||
else:
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
if x < key(a[mid]):
|
||||
hi = mid
|
||||
else:
|
||||
lo = mid + 1
|
||||
while lo < hi:
|
||||
mid = (lo+hi)//2
|
||||
if x < a[mid]: hi = mid
|
||||
else: lo = mid+1
|
||||
a.insert(lo, x)
|
||||
|
||||
insort = insort_right # backward compatibility
|
||||
|
||||
def bisect_right(a, x, lo=0, hi=None):
|
||||
"""Return the index where to insert item x in list a, assuming a is sorted.
|
||||
|
||||
The return value i is such that all e in a[:i] have e <= x, and all e in
|
||||
a[i:] have e > x. So if x already appears in the list, a.insert(x) will
|
||||
insert just after the rightmost x already there.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
"""
|
||||
|
||||
if lo < 0:
|
||||
raise ValueError('lo must be non-negative')
|
||||
if hi is None:
|
||||
hi = len(a)
|
||||
while lo < hi:
|
||||
mid = (lo+hi)//2
|
||||
if x < a[mid]: hi = mid
|
||||
else: lo = mid+1
|
||||
return lo
|
||||
|
||||
bisect = bisect_right # backward compatibility
|
||||
|
||||
def insort_left(a, x, lo=0, hi=None, *, key=None):
|
||||
def insort_left(a, x, lo=0, hi=None):
|
||||
"""Insert item x in list a, and keep it sorted assuming a is sorted.
|
||||
|
||||
If x is already in a, insert it to the left of the leftmost x.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
|
||||
A custom key function can be supplied to customize the sort order.
|
||||
"""
|
||||
|
||||
if key is None:
|
||||
lo = bisect_left(a, x, lo, hi)
|
||||
else:
|
||||
lo = bisect_left(a, key(x), lo, hi, key=key)
|
||||
a.insert(lo, x)
|
||||
|
||||
def bisect_left(a, x, lo=0, hi=None, *, key=None):
|
||||
"""Return the index where to insert item x in list a, assuming a is sorted.
|
||||
|
||||
The return value i is such that all e in a[:i] have e < x, and all e in
|
||||
a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will
|
||||
insert just before the leftmost x already there.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
|
||||
A custom key function can be supplied to customize the sort order.
|
||||
"""
|
||||
|
||||
if lo < 0:
|
||||
raise ValueError('lo must be non-negative')
|
||||
if hi is None:
|
||||
hi = len(a)
|
||||
# Note, the comparison uses "<" to match the
|
||||
# __lt__() logic in list.sort() and in heapq.
|
||||
if key is None:
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
if a[mid] < x:
|
||||
lo = mid + 1
|
||||
else:
|
||||
hi = mid
|
||||
else:
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
if key(a[mid]) < x:
|
||||
lo = mid + 1
|
||||
else:
|
||||
hi = mid
|
||||
return lo
|
||||
while lo < hi:
|
||||
mid = (lo+hi)//2
|
||||
if a[mid] < x: lo = mid+1
|
||||
else: hi = mid
|
||||
a.insert(lo, x)
|
||||
|
||||
|
||||
def bisect_left(a, x, lo=0, hi=None):
|
||||
"""Return the index where to insert item x in list a, assuming a is sorted.
|
||||
|
||||
The return value i is such that all e in a[:i] have e < x, and all e in
|
||||
a[i:] have e >= x. So if x already appears in the list, a.insert(x) will
|
||||
insert just before the leftmost x already there.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
"""
|
||||
|
||||
if lo < 0:
|
||||
raise ValueError('lo must be non-negative')
|
||||
if hi is None:
|
||||
hi = len(a)
|
||||
while lo < hi:
|
||||
mid = (lo+hi)//2
|
||||
if a[mid] < x: lo = mid+1
|
||||
else: hi = mid
|
||||
return lo
|
||||
|
||||
# Overwrite above definitions with a fast C implementation
|
||||
try:
|
||||
from _bisect import *
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Create aliases
|
||||
bisect = bisect_right
|
||||
insort = insort_right
|
||||
|
||||
344
Lib/bz2.py
vendored
344
Lib/bz2.py
vendored
@@ -1,344 +0,0 @@
|
||||
"""Interface to the libbzip2 compression library.
|
||||
|
||||
This module provides a file interface, classes for incremental
|
||||
(de)compression, and functions for one-shot (de)compression.
|
||||
"""
|
||||
|
||||
__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor",
|
||||
"open", "compress", "decompress"]
|
||||
|
||||
__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
|
||||
|
||||
from builtins import open as _builtin_open
|
||||
import io
|
||||
import os
|
||||
import _compression
|
||||
|
||||
from _bz2 import BZ2Compressor, BZ2Decompressor
|
||||
|
||||
|
||||
_MODE_CLOSED = 0
|
||||
_MODE_READ = 1
|
||||
# Value 2 no longer used
|
||||
_MODE_WRITE = 3
|
||||
|
||||
|
||||
class BZ2File(_compression.BaseStream):
|
||||
|
||||
"""A file object providing transparent bzip2 (de)compression.
|
||||
|
||||
A BZ2File can act as a wrapper for an existing file object, or refer
|
||||
directly to a named file on disk.
|
||||
|
||||
Note that BZ2File provides a *binary* file interface - data read is
|
||||
returned as bytes, and data to be written should be given as bytes.
|
||||
"""
|
||||
|
||||
def __init__(self, filename, mode="r", *, compresslevel=9):
|
||||
"""Open a bzip2-compressed file.
|
||||
|
||||
If filename is a str, bytes, or PathLike object, it gives the
|
||||
name of the file to be opened. Otherwise, it should be a file
|
||||
object, which will be used to read or write the compressed data.
|
||||
|
||||
mode can be 'r' for reading (default), 'w' for (over)writing,
|
||||
'x' for creating exclusively, or 'a' for appending. These can
|
||||
equivalently be given as 'rb', 'wb', 'xb', and 'ab'.
|
||||
|
||||
If mode is 'w', 'x' or 'a', compresslevel can be a number between 1
|
||||
and 9 specifying the level of compression: 1 produces the least
|
||||
compression, and 9 (default) produces the most compression.
|
||||
|
||||
If mode is 'r', the input file may be the concatenation of
|
||||
multiple compressed streams.
|
||||
"""
|
||||
self._fp = None
|
||||
self._closefp = False
|
||||
self._mode = _MODE_CLOSED
|
||||
|
||||
if not (1 <= compresslevel <= 9):
|
||||
raise ValueError("compresslevel must be between 1 and 9")
|
||||
|
||||
if mode in ("", "r", "rb"):
|
||||
mode = "rb"
|
||||
mode_code = _MODE_READ
|
||||
elif mode in ("w", "wb"):
|
||||
mode = "wb"
|
||||
mode_code = _MODE_WRITE
|
||||
self._compressor = BZ2Compressor(compresslevel)
|
||||
elif mode in ("x", "xb"):
|
||||
mode = "xb"
|
||||
mode_code = _MODE_WRITE
|
||||
self._compressor = BZ2Compressor(compresslevel)
|
||||
elif mode in ("a", "ab"):
|
||||
mode = "ab"
|
||||
mode_code = _MODE_WRITE
|
||||
self._compressor = BZ2Compressor(compresslevel)
|
||||
else:
|
||||
raise ValueError("Invalid mode: %r" % (mode,))
|
||||
|
||||
if isinstance(filename, (str, bytes, os.PathLike)):
|
||||
self._fp = _builtin_open(filename, mode)
|
||||
self._closefp = True
|
||||
self._mode = mode_code
|
||||
elif hasattr(filename, "read") or hasattr(filename, "write"):
|
||||
self._fp = filename
|
||||
self._mode = mode_code
|
||||
else:
|
||||
raise TypeError("filename must be a str, bytes, file or PathLike object")
|
||||
|
||||
if self._mode == _MODE_READ:
|
||||
raw = _compression.DecompressReader(self._fp,
|
||||
BZ2Decompressor, trailing_error=OSError)
|
||||
self._buffer = io.BufferedReader(raw)
|
||||
else:
|
||||
self._pos = 0
|
||||
|
||||
def close(self):
|
||||
"""Flush and close the file.
|
||||
|
||||
May be called more than once without error. Once the file is
|
||||
closed, any other operation on it will raise a ValueError.
|
||||
"""
|
||||
if self._mode == _MODE_CLOSED:
|
||||
return
|
||||
try:
|
||||
if self._mode == _MODE_READ:
|
||||
self._buffer.close()
|
||||
elif self._mode == _MODE_WRITE:
|
||||
self._fp.write(self._compressor.flush())
|
||||
self._compressor = None
|
||||
finally:
|
||||
try:
|
||||
if self._closefp:
|
||||
self._fp.close()
|
||||
finally:
|
||||
self._fp = None
|
||||
self._closefp = False
|
||||
self._mode = _MODE_CLOSED
|
||||
self._buffer = None
|
||||
|
||||
@property
|
||||
def closed(self):
|
||||
"""True if this file is closed."""
|
||||
return self._mode == _MODE_CLOSED
|
||||
|
||||
def fileno(self):
|
||||
"""Return the file descriptor for the underlying file."""
|
||||
self._check_not_closed()
|
||||
return self._fp.fileno()
|
||||
|
||||
def seekable(self):
|
||||
"""Return whether the file supports seeking."""
|
||||
return self.readable() and self._buffer.seekable()
|
||||
|
||||
def readable(self):
|
||||
"""Return whether the file was opened for reading."""
|
||||
self._check_not_closed()
|
||||
return self._mode == _MODE_READ
|
||||
|
||||
def writable(self):
|
||||
"""Return whether the file was opened for writing."""
|
||||
self._check_not_closed()
|
||||
return self._mode == _MODE_WRITE
|
||||
|
||||
def peek(self, n=0):
|
||||
"""Return buffered data without advancing the file position.
|
||||
|
||||
Always returns at least one byte of data, unless at EOF.
|
||||
The exact number of bytes returned is unspecified.
|
||||
"""
|
||||
self._check_can_read()
|
||||
# Relies on the undocumented fact that BufferedReader.peek()
|
||||
# always returns at least one byte (except at EOF), independent
|
||||
# of the value of n
|
||||
return self._buffer.peek(n)
|
||||
|
||||
def read(self, size=-1):
|
||||
"""Read up to size uncompressed bytes from the file.
|
||||
|
||||
If size is negative or omitted, read until EOF is reached.
|
||||
Returns b'' if the file is already at EOF.
|
||||
"""
|
||||
self._check_can_read()
|
||||
return self._buffer.read(size)
|
||||
|
||||
def read1(self, size=-1):
|
||||
"""Read up to size uncompressed bytes, while trying to avoid
|
||||
making multiple reads from the underlying stream. Reads up to a
|
||||
buffer's worth of data if size is negative.
|
||||
|
||||
Returns b'' if the file is at EOF.
|
||||
"""
|
||||
self._check_can_read()
|
||||
if size < 0:
|
||||
size = io.DEFAULT_BUFFER_SIZE
|
||||
return self._buffer.read1(size)
|
||||
|
||||
def readinto(self, b):
|
||||
"""Read bytes into b.
|
||||
|
||||
Returns the number of bytes read (0 for EOF).
|
||||
"""
|
||||
self._check_can_read()
|
||||
return self._buffer.readinto(b)
|
||||
|
||||
def readline(self, size=-1):
|
||||
"""Read a line of uncompressed bytes from the file.
|
||||
|
||||
The terminating newline (if present) is retained. If size is
|
||||
non-negative, no more than size bytes will be read (in which
|
||||
case the line may be incomplete). Returns b'' if already at EOF.
|
||||
"""
|
||||
if not isinstance(size, int):
|
||||
if not hasattr(size, "__index__"):
|
||||
raise TypeError("Integer argument expected")
|
||||
size = size.__index__()
|
||||
self._check_can_read()
|
||||
return self._buffer.readline(size)
|
||||
|
||||
def readlines(self, size=-1):
|
||||
"""Read a list of lines of uncompressed bytes from the file.
|
||||
|
||||
size can be specified to control the number of lines read: no
|
||||
further lines will be read once the total size of the lines read
|
||||
so far equals or exceeds size.
|
||||
"""
|
||||
if not isinstance(size, int):
|
||||
if not hasattr(size, "__index__"):
|
||||
raise TypeError("Integer argument expected")
|
||||
size = size.__index__()
|
||||
self._check_can_read()
|
||||
return self._buffer.readlines(size)
|
||||
|
||||
def write(self, data):
|
||||
"""Write a byte string to the file.
|
||||
|
||||
Returns the number of uncompressed bytes written, which is
|
||||
always the length of data in bytes. Note that due to buffering,
|
||||
the file on disk may not reflect the data written until close()
|
||||
is called.
|
||||
"""
|
||||
self._check_can_write()
|
||||
if isinstance(data, (bytes, bytearray)):
|
||||
length = len(data)
|
||||
else:
|
||||
# accept any data that supports the buffer protocol
|
||||
data = memoryview(data)
|
||||
length = data.nbytes
|
||||
|
||||
compressed = self._compressor.compress(data)
|
||||
self._fp.write(compressed)
|
||||
self._pos += length
|
||||
return length
|
||||
|
||||
def writelines(self, seq):
|
||||
"""Write a sequence of byte strings to the file.
|
||||
|
||||
Returns the number of uncompressed bytes written.
|
||||
seq can be any iterable yielding byte strings.
|
||||
|
||||
Line separators are not added between the written byte strings.
|
||||
"""
|
||||
return _compression.BaseStream.writelines(self, seq)
|
||||
|
||||
def seek(self, offset, whence=io.SEEK_SET):
|
||||
"""Change the file position.
|
||||
|
||||
The new position is specified by offset, relative to the
|
||||
position indicated by whence. Values for whence are:
|
||||
|
||||
0: start of stream (default); offset must not be negative
|
||||
1: current stream position
|
||||
2: end of stream; offset must not be positive
|
||||
|
||||
Returns the new file position.
|
||||
|
||||
Note that seeking is emulated, so depending on the parameters,
|
||||
this operation may be extremely slow.
|
||||
"""
|
||||
self._check_can_seek()
|
||||
return self._buffer.seek(offset, whence)
|
||||
|
||||
def tell(self):
|
||||
"""Return the current file position."""
|
||||
self._check_not_closed()
|
||||
if self._mode == _MODE_READ:
|
||||
return self._buffer.tell()
|
||||
return self._pos
|
||||
|
||||
|
||||
def open(filename, mode="rb", compresslevel=9,
|
||||
encoding=None, errors=None, newline=None):
|
||||
"""Open a bzip2-compressed file in binary or text mode.
|
||||
|
||||
The filename argument can be an actual filename (a str, bytes, or
|
||||
PathLike object), or an existing file object to read from or write
|
||||
to.
|
||||
|
||||
The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or
|
||||
"ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode.
|
||||
The default mode is "rb", and the default compresslevel is 9.
|
||||
|
||||
For binary mode, this function is equivalent to the BZ2File
|
||||
constructor: BZ2File(filename, mode, compresslevel). In this case,
|
||||
the encoding, errors and newline arguments must not be provided.
|
||||
|
||||
For text mode, a BZ2File object is created, and wrapped in an
|
||||
io.TextIOWrapper instance with the specified encoding, error
|
||||
handling behavior, and line ending(s).
|
||||
|
||||
"""
|
||||
if "t" in mode:
|
||||
if "b" in mode:
|
||||
raise ValueError("Invalid mode: %r" % (mode,))
|
||||
else:
|
||||
if encoding is not None:
|
||||
raise ValueError("Argument 'encoding' not supported in binary mode")
|
||||
if errors is not None:
|
||||
raise ValueError("Argument 'errors' not supported in binary mode")
|
||||
if newline is not None:
|
||||
raise ValueError("Argument 'newline' not supported in binary mode")
|
||||
|
||||
bz_mode = mode.replace("t", "")
|
||||
binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel)
|
||||
|
||||
if "t" in mode:
|
||||
encoding = io.text_encoding(encoding)
|
||||
return io.TextIOWrapper(binary_file, encoding, errors, newline)
|
||||
else:
|
||||
return binary_file
|
||||
|
||||
|
||||
def compress(data, compresslevel=9):
|
||||
"""Compress a block of data.
|
||||
|
||||
compresslevel, if given, must be a number between 1 and 9.
|
||||
|
||||
For incremental compression, use a BZ2Compressor object instead.
|
||||
"""
|
||||
comp = BZ2Compressor(compresslevel)
|
||||
return comp.compress(data) + comp.flush()
|
||||
|
||||
|
||||
def decompress(data):
|
||||
"""Decompress a block of data.
|
||||
|
||||
For incremental decompression, use a BZ2Decompressor object instead.
|
||||
"""
|
||||
results = []
|
||||
while data:
|
||||
decomp = BZ2Decompressor()
|
||||
try:
|
||||
res = decomp.decompress(data)
|
||||
except OSError:
|
||||
if results:
|
||||
break # Leftover data is not a valid bzip2 stream; ignore it.
|
||||
else:
|
||||
raise # Error on the first iteration; bail out.
|
||||
results.append(res)
|
||||
if not decomp.eof:
|
||||
raise ValueError("Compressed data ended before the "
|
||||
"end-of-stream marker was reached")
|
||||
data = decomp.unused_data
|
||||
return b"".join(results)
|
||||
247
Lib/calendar.py
vendored
247
Lib/calendar.py
vendored
@@ -7,22 +7,15 @@ set the first day of the week (0=Monday, 6=Sunday)."""
|
||||
|
||||
import sys
|
||||
import datetime
|
||||
from enum import IntEnum, global_enum
|
||||
import locale as _locale
|
||||
from itertools import repeat
|
||||
import warnings
|
||||
|
||||
__all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
|
||||
"firstweekday", "isleap", "leapdays", "weekday", "monthrange",
|
||||
"monthcalendar", "prmonth", "month", "prcal", "calendar",
|
||||
"timegm", "month_name", "month_abbr", "day_name", "day_abbr",
|
||||
"Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar",
|
||||
"LocaleHTMLCalendar", "weekheader",
|
||||
"Day", "Month", "JANUARY", "FEBRUARY", "MARCH",
|
||||
"APRIL", "MAY", "JUNE", "JULY",
|
||||
"AUGUST", "SEPTEMBER", "OCTOBER", "NOVEMBER", "DECEMBER",
|
||||
"MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY",
|
||||
"SATURDAY", "SUNDAY"]
|
||||
"LocaleHTMLCalendar", "weekheader"]
|
||||
|
||||
# Exception raised for bad input (with string parameter for details)
|
||||
error = ValueError
|
||||
@@ -42,46 +35,9 @@ class IllegalWeekdayError(ValueError):
|
||||
return "bad weekday number %r; must be 0 (Monday) to 6 (Sunday)" % self.weekday
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
if name in ('January', 'February'):
|
||||
warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead",
|
||||
DeprecationWarning, stacklevel=2)
|
||||
if name == 'January':
|
||||
return 1
|
||||
else:
|
||||
return 2
|
||||
|
||||
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
||||
|
||||
|
||||
# Constants for months
|
||||
@global_enum
|
||||
class Month(IntEnum):
|
||||
JANUARY = 1
|
||||
FEBRUARY = 2
|
||||
MARCH = 3
|
||||
APRIL = 4
|
||||
MAY = 5
|
||||
JUNE = 6
|
||||
JULY = 7
|
||||
AUGUST = 8
|
||||
SEPTEMBER = 9
|
||||
OCTOBER = 10
|
||||
NOVEMBER = 11
|
||||
DECEMBER = 12
|
||||
|
||||
|
||||
# Constants for days
|
||||
@global_enum
|
||||
class Day(IntEnum):
|
||||
MONDAY = 0
|
||||
TUESDAY = 1
|
||||
WEDNESDAY = 2
|
||||
THURSDAY = 3
|
||||
FRIDAY = 4
|
||||
SATURDAY = 5
|
||||
SUNDAY = 6
|
||||
|
||||
# Constants for months referenced later
|
||||
January = 1
|
||||
February = 2
|
||||
|
||||
# Number of days per month (except for February in leap years)
|
||||
mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
|
||||
@@ -137,6 +93,9 @@ day_abbr = _localized_day('%a')
|
||||
month_name = _localized_month('%B')
|
||||
month_abbr = _localized_month('%b')
|
||||
|
||||
# Constants for weekdays
|
||||
(MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY) = range(7)
|
||||
|
||||
|
||||
def isleap(year):
|
||||
"""Return True for leap years, False for non-leap years."""
|
||||
@@ -152,10 +111,9 @@ def leapdays(y1, y2):
|
||||
|
||||
|
||||
def weekday(year, month, day):
|
||||
"""Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31)."""
|
||||
if not datetime.MINYEAR <= year <= datetime.MAXYEAR:
|
||||
year = 2000 + year % 400
|
||||
return Day(datetime.date(year, month, day).weekday())
|
||||
"""Return weekday (0-6 ~ Mon-Sun) for year (1970-...), month (1-12),
|
||||
day (1-31)."""
|
||||
return datetime.date(year, month, day).weekday()
|
||||
|
||||
|
||||
def monthrange(year, month):
|
||||
@@ -164,28 +122,10 @@ def monthrange(year, month):
|
||||
if not 1 <= month <= 12:
|
||||
raise IllegalMonthError(month)
|
||||
day1 = weekday(year, month, 1)
|
||||
ndays = mdays[month] + (month == FEBRUARY and isleap(year))
|
||||
ndays = mdays[month] + (month == February and isleap(year))
|
||||
return day1, ndays
|
||||
|
||||
|
||||
def _monthlen(year, month):
|
||||
return mdays[month] + (month == FEBRUARY and isleap(year))
|
||||
|
||||
|
||||
def _prevmonth(year, month):
|
||||
if month == 1:
|
||||
return year-1, 12
|
||||
else:
|
||||
return year, month-1
|
||||
|
||||
|
||||
def _nextmonth(year, month):
|
||||
if month == 12:
|
||||
return year+1, 1
|
||||
else:
|
||||
return year, month+1
|
||||
|
||||
|
||||
class Calendar(object):
|
||||
"""
|
||||
Base calendar class. This class doesn't do any formatting. It simply
|
||||
@@ -217,8 +157,28 @@ class Calendar(object):
|
||||
values and will always iterate through complete weeks, so it will yield
|
||||
dates outside the specified month.
|
||||
"""
|
||||
for y, m, d in self.itermonthdays3(year, month):
|
||||
yield datetime.date(y, m, d)
|
||||
date = datetime.date(year, month, 1)
|
||||
# Go back to the beginning of the week
|
||||
days = (date.weekday() - self.firstweekday) % 7
|
||||
date -= datetime.timedelta(days=days)
|
||||
oneday = datetime.timedelta(days=1)
|
||||
while True:
|
||||
yield date
|
||||
try:
|
||||
date += oneday
|
||||
except OverflowError:
|
||||
# Adding one day could fail after datetime.MAXYEAR
|
||||
break
|
||||
if date.month != month and date.weekday() == self.firstweekday:
|
||||
break
|
||||
|
||||
def itermonthdays2(self, year, month):
|
||||
"""
|
||||
Like itermonthdates(), but will yield (day number, weekday number)
|
||||
tuples. For days outside the specified month the day number is 0.
|
||||
"""
|
||||
for i, d in enumerate(self.itermonthdays(year, month), self.firstweekday):
|
||||
yield d, i % 7
|
||||
|
||||
def itermonthdays(self, year, month):
|
||||
"""
|
||||
@@ -232,40 +192,6 @@ class Calendar(object):
|
||||
days_after = (self.firstweekday - day1 - ndays) % 7
|
||||
yield from repeat(0, days_after)
|
||||
|
||||
def itermonthdays2(self, year, month):
|
||||
"""
|
||||
Like itermonthdates(), but will yield (day number, weekday number)
|
||||
tuples. For days outside the specified month the day number is 0.
|
||||
"""
|
||||
for i, d in enumerate(self.itermonthdays(year, month), self.firstweekday):
|
||||
yield d, i % 7
|
||||
|
||||
def itermonthdays3(self, year, month):
|
||||
"""
|
||||
Like itermonthdates(), but will yield (year, month, day) tuples. Can be
|
||||
used for dates outside of datetime.date range.
|
||||
"""
|
||||
day1, ndays = monthrange(year, month)
|
||||
days_before = (day1 - self.firstweekday) % 7
|
||||
days_after = (self.firstweekday - day1 - ndays) % 7
|
||||
y, m = _prevmonth(year, month)
|
||||
end = _monthlen(y, m) + 1
|
||||
for d in range(end-days_before, end):
|
||||
yield y, m, d
|
||||
for d in range(1, ndays + 1):
|
||||
yield year, month, d
|
||||
y, m = _nextmonth(year, month)
|
||||
for d in range(1, days_after + 1):
|
||||
yield y, m, d
|
||||
|
||||
def itermonthdays4(self, year, month):
|
||||
"""
|
||||
Like itermonthdates(), but will yield (year, month, day, day_of_week) tuples.
|
||||
Can be used for dates outside of datetime.date range.
|
||||
"""
|
||||
for i, (y, m, d) in enumerate(self.itermonthdays3(year, month)):
|
||||
yield y, m, d, (self.firstweekday + i) % 7
|
||||
|
||||
def monthdatescalendar(self, year, month):
|
||||
"""
|
||||
Return a matrix (list of lists) representing a month's calendar.
|
||||
@@ -299,7 +225,10 @@ class Calendar(object):
|
||||
Each month contains between 4 and 6 weeks and each week contains 1-7
|
||||
days. Days are datetime.date objects.
|
||||
"""
|
||||
months = [self.monthdatescalendar(year, m) for m in Month]
|
||||
months = [
|
||||
self.monthdatescalendar(year, i)
|
||||
for i in range(January, January+12)
|
||||
]
|
||||
return [months[i:i+width] for i in range(0, len(months), width) ]
|
||||
|
||||
def yeardays2calendar(self, year, width=3):
|
||||
@@ -309,7 +238,10 @@ class Calendar(object):
|
||||
(day number, weekday number) tuples. Day numbers outside this month are
|
||||
zero.
|
||||
"""
|
||||
months = [self.monthdays2calendar(year, m) for m in Month]
|
||||
months = [
|
||||
self.monthdays2calendar(year, i)
|
||||
for i in range(January, January+12)
|
||||
]
|
||||
return [months[i:i+width] for i in range(0, len(months), width) ]
|
||||
|
||||
def yeardayscalendar(self, year, width=3):
|
||||
@@ -318,7 +250,10 @@ class Calendar(object):
|
||||
yeardatescalendar()). Entries in the week lists are day numbers.
|
||||
Day numbers outside this month are zero.
|
||||
"""
|
||||
months = [self.monthdayscalendar(year, m) for m in Month]
|
||||
months = [
|
||||
self.monthdayscalendar(year, i)
|
||||
for i in range(January, January+12)
|
||||
]
|
||||
return [months[i:i+width] for i in range(0, len(months), width) ]
|
||||
|
||||
|
||||
@@ -332,7 +267,7 @@ class TextCalendar(Calendar):
|
||||
"""
|
||||
Print a single week (no newline).
|
||||
"""
|
||||
print(self.formatweek(theweek, width), end='')
|
||||
print(self.formatweek(theweek, width), end=' ')
|
||||
|
||||
def formatday(self, day, weekday, width):
|
||||
"""
|
||||
@@ -436,7 +371,7 @@ class TextCalendar(Calendar):
|
||||
|
||||
def pryear(self, theyear, w=0, l=0, c=6, m=3):
|
||||
"""Print a year's calendar."""
|
||||
print(self.formatyear(theyear, w, l, c, m), end='')
|
||||
print(self.formatyear(theyear, w, l, c, m))
|
||||
|
||||
|
||||
class HTMLCalendar(Calendar):
|
||||
@@ -447,31 +382,12 @@ class HTMLCalendar(Calendar):
|
||||
# CSS classes for the day <td>s
|
||||
cssclasses = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"]
|
||||
|
||||
# CSS classes for the day <th>s
|
||||
cssclasses_weekday_head = cssclasses
|
||||
|
||||
# CSS class for the days before and after current month
|
||||
cssclass_noday = "noday"
|
||||
|
||||
# CSS class for the month's head
|
||||
cssclass_month_head = "month"
|
||||
|
||||
# CSS class for the month
|
||||
cssclass_month = "month"
|
||||
|
||||
# CSS class for the year's table head
|
||||
cssclass_year_head = "year"
|
||||
|
||||
# CSS class for the whole year table
|
||||
cssclass_year = "year"
|
||||
|
||||
def formatday(self, day, weekday):
|
||||
"""
|
||||
Return a day as a table cell.
|
||||
"""
|
||||
if day == 0:
|
||||
# day outside month
|
||||
return '<td class="%s"> </td>' % self.cssclass_noday
|
||||
return '<td class="noday"> </td>' # day outside month
|
||||
else:
|
||||
return '<td class="%s">%d</td>' % (self.cssclasses[weekday], day)
|
||||
|
||||
@@ -486,8 +402,7 @@ class HTMLCalendar(Calendar):
|
||||
"""
|
||||
Return a weekday name as a table header.
|
||||
"""
|
||||
return '<th class="%s">%s</th>' % (
|
||||
self.cssclasses_weekday_head[day], day_abbr[day])
|
||||
return '<th class="%s">%s</th>' % (self.cssclasses[day], day_abbr[day])
|
||||
|
||||
def formatweekheader(self):
|
||||
"""
|
||||
@@ -504,8 +419,7 @@ class HTMLCalendar(Calendar):
|
||||
s = '%s %s' % (month_name[themonth], theyear)
|
||||
else:
|
||||
s = '%s' % month_name[themonth]
|
||||
return '<tr><th colspan="7" class="%s">%s</th></tr>' % (
|
||||
self.cssclass_month_head, s)
|
||||
return '<tr><th colspan="7" class="month">%s</th></tr>' % s
|
||||
|
||||
def formatmonth(self, theyear, themonth, withyear=True):
|
||||
"""
|
||||
@@ -513,8 +427,7 @@ class HTMLCalendar(Calendar):
|
||||
"""
|
||||
v = []
|
||||
a = v.append
|
||||
a('<table border="0" cellpadding="0" cellspacing="0" class="%s">' % (
|
||||
self.cssclass_month))
|
||||
a('<table border="0" cellpadding="0" cellspacing="0" class="month">')
|
||||
a('\n')
|
||||
a(self.formatmonthname(theyear, themonth, withyear=withyear))
|
||||
a('\n')
|
||||
@@ -534,12 +447,10 @@ class HTMLCalendar(Calendar):
|
||||
v = []
|
||||
a = v.append
|
||||
width = max(width, 1)
|
||||
a('<table border="0" cellpadding="0" cellspacing="0" class="%s">' %
|
||||
self.cssclass_year)
|
||||
a('<table border="0" cellpadding="0" cellspacing="0" class="year">')
|
||||
a('\n')
|
||||
a('<tr><th colspan="%d" class="%s">%s</th></tr>' % (
|
||||
width, self.cssclass_year_head, theyear))
|
||||
for i in range(JANUARY, JANUARY+12, width):
|
||||
a('<tr><th colspan="%d" class="year">%s</th></tr>' % (width, theyear))
|
||||
for i in range(January, January+12, width):
|
||||
# months in this row
|
||||
months = range(i, min(i+width, 13))
|
||||
a('<tr>')
|
||||
@@ -578,67 +489,71 @@ class HTMLCalendar(Calendar):
|
||||
class different_locale:
|
||||
def __init__(self, locale):
|
||||
self.locale = locale
|
||||
self.oldlocale = None
|
||||
|
||||
def __enter__(self):
|
||||
self.oldlocale = _locale.setlocale(_locale.LC_TIME, None)
|
||||
self.oldlocale = _locale.getlocale(_locale.LC_TIME)
|
||||
_locale.setlocale(_locale.LC_TIME, self.locale)
|
||||
|
||||
def __exit__(self, *args):
|
||||
if self.oldlocale is None:
|
||||
return
|
||||
_locale.setlocale(_locale.LC_TIME, self.oldlocale)
|
||||
|
||||
|
||||
def _get_default_locale():
|
||||
locale = _locale.setlocale(_locale.LC_TIME, None)
|
||||
if locale == "C":
|
||||
with different_locale(""):
|
||||
# The LC_TIME locale does not seem to be configured:
|
||||
# get the user preferred locale.
|
||||
locale = _locale.setlocale(_locale.LC_TIME, None)
|
||||
return locale
|
||||
|
||||
|
||||
class LocaleTextCalendar(TextCalendar):
|
||||
"""
|
||||
This class can be passed a locale name in the constructor and will return
|
||||
month and weekday names in the specified locale.
|
||||
month and weekday names in the specified locale. If this locale includes
|
||||
an encoding all strings containing month and weekday names will be returned
|
||||
as unicode.
|
||||
"""
|
||||
|
||||
def __init__(self, firstweekday=0, locale=None):
|
||||
TextCalendar.__init__(self, firstweekday)
|
||||
if locale is None:
|
||||
locale = _get_default_locale()
|
||||
locale = _locale.getdefaultlocale()
|
||||
self.locale = locale
|
||||
|
||||
def formatweekday(self, day, width):
|
||||
with different_locale(self.locale):
|
||||
return super().formatweekday(day, width)
|
||||
if width >= 9:
|
||||
names = day_name
|
||||
else:
|
||||
names = day_abbr
|
||||
name = names[day]
|
||||
return name[:width].center(width)
|
||||
|
||||
def formatmonthname(self, theyear, themonth, width, withyear=True):
|
||||
with different_locale(self.locale):
|
||||
return super().formatmonthname(theyear, themonth, width, withyear)
|
||||
s = month_name[themonth]
|
||||
if withyear:
|
||||
s = "%s %r" % (s, theyear)
|
||||
return s.center(width)
|
||||
|
||||
|
||||
class LocaleHTMLCalendar(HTMLCalendar):
|
||||
"""
|
||||
This class can be passed a locale name in the constructor and will return
|
||||
month and weekday names in the specified locale.
|
||||
month and weekday names in the specified locale. If this locale includes
|
||||
an encoding all strings containing month and weekday names will be returned
|
||||
as unicode.
|
||||
"""
|
||||
def __init__(self, firstweekday=0, locale=None):
|
||||
HTMLCalendar.__init__(self, firstweekday)
|
||||
if locale is None:
|
||||
locale = _get_default_locale()
|
||||
locale = _locale.getdefaultlocale()
|
||||
self.locale = locale
|
||||
|
||||
def formatweekday(self, day):
|
||||
with different_locale(self.locale):
|
||||
return super().formatweekday(day)
|
||||
s = day_abbr[day]
|
||||
return '<th class="%s">%s</th>' % (self.cssclasses[day], s)
|
||||
|
||||
def formatmonthname(self, theyear, themonth, withyear=True):
|
||||
with different_locale(self.locale):
|
||||
return super().formatmonthname(theyear, themonth, withyear)
|
||||
s = month_name[themonth]
|
||||
if withyear:
|
||||
s = '%s %s' % (s, theyear)
|
||||
return '<tr><th colspan="7" class="month">%s</th></tr>' % s
|
||||
|
||||
|
||||
# Support for old module level interface
|
||||
c = TextCalendar()
|
||||
@@ -723,7 +638,7 @@ def main(args):
|
||||
parser.add_argument(
|
||||
"-L", "--locale",
|
||||
default=None,
|
||||
help="locale to use for month and weekday names"
|
||||
help="locale to be used from month and weekday names"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-e", "--encoding",
|
||||
|
||||
1012
Lib/cgi.py
vendored
1012
Lib/cgi.py
vendored
File diff suppressed because it is too large
Load Diff
332
Lib/cgitb.py
vendored
332
Lib/cgitb.py
vendored
@@ -1,332 +0,0 @@
|
||||
"""More comprehensive traceback formatting for Python scripts.
|
||||
|
||||
To enable this module, do:
|
||||
|
||||
import cgitb; cgitb.enable()
|
||||
|
||||
at the top of your script. The optional arguments to enable() are:
|
||||
|
||||
display - if true, tracebacks are displayed in the web browser
|
||||
logdir - if set, tracebacks are written to files in this directory
|
||||
context - number of lines of source code to show for each stack frame
|
||||
format - 'text' or 'html' controls the output format
|
||||
|
||||
By default, tracebacks are displayed but not saved, the context is 5 lines
|
||||
and the output format is 'html' (for backwards compatibility with the
|
||||
original use of this module)
|
||||
|
||||
Alternatively, if you have caught an exception and want cgitb to display it
|
||||
for you, call cgitb.handler(). The optional argument to handler() is a
|
||||
3-item tuple (etype, evalue, etb) just like the value of sys.exc_info().
|
||||
The default handler displays output as HTML.
|
||||
|
||||
"""
|
||||
import inspect
|
||||
import keyword
|
||||
import linecache
|
||||
import os
|
||||
import pydoc
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import tokenize
|
||||
import traceback
|
||||
import warnings
|
||||
from html import escape as html_escape
|
||||
|
||||
warnings._deprecated(__name__, remove=(3, 13))
|
||||
|
||||
|
||||
def reset():
|
||||
"""Return a string that resets the CGI and browser to a known state."""
|
||||
return '''<!--: spam
|
||||
Content-Type: text/html
|
||||
|
||||
<body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> -->
|
||||
<body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> --> -->
|
||||
</font> </font> </font> </script> </object> </blockquote> </pre>
|
||||
</table> </table> </table> </table> </table> </font> </font> </font>'''
|
||||
|
||||
__UNDEF__ = [] # a special sentinel object
|
||||
def small(text):
|
||||
if text:
|
||||
return '<small>' + text + '</small>'
|
||||
else:
|
||||
return ''
|
||||
|
||||
def strong(text):
|
||||
if text:
|
||||
return '<strong>' + text + '</strong>'
|
||||
else:
|
||||
return ''
|
||||
|
||||
def grey(text):
|
||||
if text:
|
||||
return '<font color="#909090">' + text + '</font>'
|
||||
else:
|
||||
return ''
|
||||
|
||||
def lookup(name, frame, locals):
|
||||
"""Find the value for a given name in the given environment."""
|
||||
if name in locals:
|
||||
return 'local', locals[name]
|
||||
if name in frame.f_globals:
|
||||
return 'global', frame.f_globals[name]
|
||||
if '__builtins__' in frame.f_globals:
|
||||
builtins = frame.f_globals['__builtins__']
|
||||
if isinstance(builtins, dict):
|
||||
if name in builtins:
|
||||
return 'builtin', builtins[name]
|
||||
else:
|
||||
if hasattr(builtins, name):
|
||||
return 'builtin', getattr(builtins, name)
|
||||
return None, __UNDEF__
|
||||
|
||||
def scanvars(reader, frame, locals):
|
||||
"""Scan one logical line of Python and look up values of variables used."""
|
||||
vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__
|
||||
for ttype, token, start, end, line in tokenize.generate_tokens(reader):
|
||||
if ttype == tokenize.NEWLINE: break
|
||||
if ttype == tokenize.NAME and token not in keyword.kwlist:
|
||||
if lasttoken == '.':
|
||||
if parent is not __UNDEF__:
|
||||
value = getattr(parent, token, __UNDEF__)
|
||||
vars.append((prefix + token, prefix, value))
|
||||
else:
|
||||
where, value = lookup(token, frame, locals)
|
||||
vars.append((token, where, value))
|
||||
elif token == '.':
|
||||
prefix += lasttoken + '.'
|
||||
parent = value
|
||||
else:
|
||||
parent, prefix = None, ''
|
||||
lasttoken = token
|
||||
return vars
|
||||
|
||||
def html(einfo, context=5):
|
||||
"""Return a nice HTML document describing a given traceback."""
|
||||
etype, evalue, etb = einfo
|
||||
if isinstance(etype, type):
|
||||
etype = etype.__name__
|
||||
pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
|
||||
date = time.ctime(time.time())
|
||||
head = f'''
|
||||
<body bgcolor="#f0f0f8">
|
||||
<table width="100%" cellspacing=0 cellpadding=2 border=0 summary="heading">
|
||||
<tr bgcolor="#6622aa">
|
||||
<td valign=bottom> <br>
|
||||
<font color="#ffffff" face="helvetica, arial"> <br>
|
||||
<big><big><strong>{html_escape(str(etype))}</strong></big></big></font></td>
|
||||
<td align=right valign=bottom>
|
||||
<font color="#ffffff" face="helvetica, arial">{pyver}<br>{date}</font></td>
|
||||
</tr></table>
|
||||
<p>A problem occurred in a Python script. Here is the sequence of
|
||||
function calls leading up to the error, in the order they occurred.</p>'''
|
||||
|
||||
indent = '<tt>' + small(' ' * 5) + ' </tt>'
|
||||
frames = []
|
||||
records = inspect.getinnerframes(etb, context)
|
||||
for frame, file, lnum, func, lines, index in records:
|
||||
if file:
|
||||
file = os.path.abspath(file)
|
||||
link = '<a href="file://%s">%s</a>' % (file, pydoc.html.escape(file))
|
||||
else:
|
||||
file = link = '?'
|
||||
args, varargs, varkw, locals = inspect.getargvalues(frame)
|
||||
call = ''
|
||||
if func != '?':
|
||||
call = 'in ' + strong(pydoc.html.escape(func))
|
||||
if func != "<module>":
|
||||
call += inspect.formatargvalues(args, varargs, varkw, locals,
|
||||
formatvalue=lambda value: '=' + pydoc.html.repr(value))
|
||||
|
||||
highlight = {}
|
||||
def reader(lnum=[lnum]):
|
||||
highlight[lnum[0]] = 1
|
||||
try: return linecache.getline(file, lnum[0])
|
||||
finally: lnum[0] += 1
|
||||
vars = scanvars(reader, frame, locals)
|
||||
|
||||
rows = ['<tr><td bgcolor="#d8bbff">%s%s %s</td></tr>' %
|
||||
('<big> </big>', link, call)]
|
||||
if index is not None:
|
||||
i = lnum - index
|
||||
for line in lines:
|
||||
num = small(' ' * (5-len(str(i))) + str(i)) + ' '
|
||||
if i in highlight:
|
||||
line = '<tt>=>%s%s</tt>' % (num, pydoc.html.preformat(line))
|
||||
rows.append('<tr><td bgcolor="#ffccee">%s</td></tr>' % line)
|
||||
else:
|
||||
line = '<tt> %s%s</tt>' % (num, pydoc.html.preformat(line))
|
||||
rows.append('<tr><td>%s</td></tr>' % grey(line))
|
||||
i += 1
|
||||
|
||||
done, dump = {}, []
|
||||
for name, where, value in vars:
|
||||
if name in done: continue
|
||||
done[name] = 1
|
||||
if value is not __UNDEF__:
|
||||
if where in ('global', 'builtin'):
|
||||
name = ('<em>%s</em> ' % where) + strong(name)
|
||||
elif where == 'local':
|
||||
name = strong(name)
|
||||
else:
|
||||
name = where + strong(name.split('.')[-1])
|
||||
dump.append('%s = %s' % (name, pydoc.html.repr(value)))
|
||||
else:
|
||||
dump.append(name + ' <em>undefined</em>')
|
||||
|
||||
rows.append('<tr><td>%s</td></tr>' % small(grey(', '.join(dump))))
|
||||
frames.append('''
|
||||
<table width="100%%" cellspacing=0 cellpadding=0 border=0>
|
||||
%s</table>''' % '\n'.join(rows))
|
||||
|
||||
exception = ['<p>%s: %s' % (strong(pydoc.html.escape(str(etype))),
|
||||
pydoc.html.escape(str(evalue)))]
|
||||
for name in dir(evalue):
|
||||
if name[:1] == '_': continue
|
||||
value = pydoc.html.repr(getattr(evalue, name))
|
||||
exception.append('\n<br>%s%s =\n%s' % (indent, name, value))
|
||||
|
||||
return head + ''.join(frames) + ''.join(exception) + '''
|
||||
|
||||
|
||||
<!-- The above is a description of an error in a Python program, formatted
|
||||
for a web browser because the 'cgitb' module was enabled. In case you
|
||||
are not reading this in a web browser, here is the original traceback:
|
||||
|
||||
%s
|
||||
-->
|
||||
''' % pydoc.html.escape(
|
||||
''.join(traceback.format_exception(etype, evalue, etb)))
|
||||
|
||||
def text(einfo, context=5):
|
||||
"""Return a plain text document describing a given traceback."""
|
||||
etype, evalue, etb = einfo
|
||||
if isinstance(etype, type):
|
||||
etype = etype.__name__
|
||||
pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
|
||||
date = time.ctime(time.time())
|
||||
head = "%s\n%s\n%s\n" % (str(etype), pyver, date) + '''
|
||||
A problem occurred in a Python script. Here is the sequence of
|
||||
function calls leading up to the error, in the order they occurred.
|
||||
'''
|
||||
|
||||
frames = []
|
||||
records = inspect.getinnerframes(etb, context)
|
||||
for frame, file, lnum, func, lines, index in records:
|
||||
file = file and os.path.abspath(file) or '?'
|
||||
args, varargs, varkw, locals = inspect.getargvalues(frame)
|
||||
call = ''
|
||||
if func != '?':
|
||||
call = 'in ' + func
|
||||
if func != "<module>":
|
||||
call += inspect.formatargvalues(args, varargs, varkw, locals,
|
||||
formatvalue=lambda value: '=' + pydoc.text.repr(value))
|
||||
|
||||
highlight = {}
|
||||
def reader(lnum=[lnum]):
|
||||
highlight[lnum[0]] = 1
|
||||
try: return linecache.getline(file, lnum[0])
|
||||
finally: lnum[0] += 1
|
||||
vars = scanvars(reader, frame, locals)
|
||||
|
||||
rows = [' %s %s' % (file, call)]
|
||||
if index is not None:
|
||||
i = lnum - index
|
||||
for line in lines:
|
||||
num = '%5d ' % i
|
||||
rows.append(num+line.rstrip())
|
||||
i += 1
|
||||
|
||||
done, dump = {}, []
|
||||
for name, where, value in vars:
|
||||
if name in done: continue
|
||||
done[name] = 1
|
||||
if value is not __UNDEF__:
|
||||
if where == 'global': name = 'global ' + name
|
||||
elif where != 'local': name = where + name.split('.')[-1]
|
||||
dump.append('%s = %s' % (name, pydoc.text.repr(value)))
|
||||
else:
|
||||
dump.append(name + ' undefined')
|
||||
|
||||
rows.append('\n'.join(dump))
|
||||
frames.append('\n%s\n' % '\n'.join(rows))
|
||||
|
||||
exception = ['%s: %s' % (str(etype), str(evalue))]
|
||||
for name in dir(evalue):
|
||||
value = pydoc.text.repr(getattr(evalue, name))
|
||||
exception.append('\n%s%s = %s' % (" "*4, name, value))
|
||||
|
||||
return head + ''.join(frames) + ''.join(exception) + '''
|
||||
|
||||
The above is a description of an error in a Python program. Here is
|
||||
the original traceback:
|
||||
|
||||
%s
|
||||
''' % ''.join(traceback.format_exception(etype, evalue, etb))
|
||||
|
||||
class Hook:
|
||||
"""A hook to replace sys.excepthook that shows tracebacks in HTML."""
|
||||
|
||||
def __init__(self, display=1, logdir=None, context=5, file=None,
|
||||
format="html"):
|
||||
self.display = display # send tracebacks to browser if true
|
||||
self.logdir = logdir # log tracebacks to files if not None
|
||||
self.context = context # number of source code lines per frame
|
||||
self.file = file or sys.stdout # place to send the output
|
||||
self.format = format
|
||||
|
||||
def __call__(self, etype, evalue, etb):
|
||||
self.handle((etype, evalue, etb))
|
||||
|
||||
def handle(self, info=None):
|
||||
info = info or sys.exc_info()
|
||||
if self.format == "html":
|
||||
self.file.write(reset())
|
||||
|
||||
formatter = (self.format=="html") and html or text
|
||||
plain = False
|
||||
try:
|
||||
doc = formatter(info, self.context)
|
||||
except: # just in case something goes wrong
|
||||
doc = ''.join(traceback.format_exception(*info))
|
||||
plain = True
|
||||
|
||||
if self.display:
|
||||
if plain:
|
||||
doc = pydoc.html.escape(doc)
|
||||
self.file.write('<pre>' + doc + '</pre>\n')
|
||||
else:
|
||||
self.file.write(doc + '\n')
|
||||
else:
|
||||
self.file.write('<p>A problem occurred in a Python script.\n')
|
||||
|
||||
if self.logdir is not None:
|
||||
suffix = ['.txt', '.html'][self.format=="html"]
|
||||
(fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir)
|
||||
|
||||
try:
|
||||
with os.fdopen(fd, 'w') as file:
|
||||
file.write(doc)
|
||||
msg = '%s contains the description of this error.' % path
|
||||
except:
|
||||
msg = 'Tried to save traceback to %s, but failed.' % path
|
||||
|
||||
if self.format == 'html':
|
||||
self.file.write('<p>%s</p>\n' % msg)
|
||||
else:
|
||||
self.file.write(msg + '\n')
|
||||
try:
|
||||
self.file.flush()
|
||||
except: pass
|
||||
|
||||
handler = Hook().handle
|
||||
def enable(display=1, logdir=None, context=5, format="html"):
|
||||
"""Install an exception handler that formats tracebacks as HTML.
|
||||
|
||||
The optional argument 'display' can be set to 0 to suppress sending the
|
||||
traceback to the browser, and 'logdir' can be set to a directory to cause
|
||||
tracebacks to be written to files there."""
|
||||
sys.excepthook = Hook(display=display, logdir=logdir,
|
||||
context=context, format=format)
|
||||
6
Lib/chunk.py
vendored
6
Lib/chunk.py
vendored
@@ -48,10 +48,6 @@ specifies whether or not chunks are aligned on 2-byte boundaries. The
|
||||
default is 1, i.e. aligned.
|
||||
"""
|
||||
|
||||
import warnings
|
||||
|
||||
warnings._deprecated(__name__, remove=(3, 13))
|
||||
|
||||
class Chunk:
|
||||
def __init__(self, file, align=True, bigendian=True, inclheader=False):
|
||||
import struct
|
||||
@@ -68,7 +64,7 @@ class Chunk:
|
||||
try:
|
||||
self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0]
|
||||
except struct.error:
|
||||
raise EOFError from None
|
||||
raise EOFError
|
||||
if inclheader:
|
||||
self.chunksize = self.chunksize - 8 # subtract header
|
||||
self.size_read = 0
|
||||
|
||||
10
Lib/cmd.py
vendored
10
Lib/cmd.py
vendored
@@ -310,10 +310,10 @@ class Cmd:
|
||||
names = self.get_names()
|
||||
cmds_doc = []
|
||||
cmds_undoc = []
|
||||
topics = set()
|
||||
help = {}
|
||||
for name in names:
|
||||
if name[:5] == 'help_':
|
||||
topics.add(name[5:])
|
||||
help[name[5:]]=1
|
||||
names.sort()
|
||||
# There can be duplicates if routines overridden
|
||||
prevname = ''
|
||||
@@ -323,16 +323,16 @@ class Cmd:
|
||||
continue
|
||||
prevname = name
|
||||
cmd=name[3:]
|
||||
if cmd in topics:
|
||||
if cmd in help:
|
||||
cmds_doc.append(cmd)
|
||||
topics.remove(cmd)
|
||||
del help[cmd]
|
||||
elif getattr(self, name).__doc__:
|
||||
cmds_doc.append(cmd)
|
||||
else:
|
||||
cmds_undoc.append(cmd)
|
||||
self.stdout.write("%s\n"%str(self.doc_leader))
|
||||
self.print_topics(self.doc_header, cmds_doc, 15,80)
|
||||
self.print_topics(self.misc_header, sorted(topics),15,80)
|
||||
self.print_topics(self.misc_header, list(help.keys()),15,80)
|
||||
self.print_topics(self.undoc_header, cmds_undoc, 15,80)
|
||||
|
||||
def print_topics(self, header, cmds, cmdlen, maxcol):
|
||||
|
||||
9
Lib/code.py
vendored
9
Lib/code.py
vendored
@@ -7,6 +7,7 @@
|
||||
|
||||
import sys
|
||||
import traceback
|
||||
import argparse
|
||||
from codeop import CommandCompiler, compile_command
|
||||
|
||||
__all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact",
|
||||
@@ -40,7 +41,7 @@ class InteractiveInterpreter:
|
||||
|
||||
Arguments are as for compile_command().
|
||||
|
||||
One of several things can happen:
|
||||
One several things can happen:
|
||||
|
||||
1) The input is incorrect; compile_command() raised an
|
||||
exception (SyntaxError or OverflowError). A syntax traceback
|
||||
@@ -106,7 +107,6 @@ class InteractiveInterpreter:
|
||||
|
||||
"""
|
||||
type, value, tb = sys.exc_info()
|
||||
sys.last_exc = value
|
||||
sys.last_type = type
|
||||
sys.last_value = value
|
||||
sys.last_traceback = tb
|
||||
@@ -120,7 +120,7 @@ class InteractiveInterpreter:
|
||||
else:
|
||||
# Stuff in the right filename
|
||||
value = SyntaxError(msg, (filename, lineno, offset, line))
|
||||
sys.last_exc = sys.last_value = value
|
||||
sys.last_value = value
|
||||
if sys.excepthook is sys.__excepthook__:
|
||||
lines = traceback.format_exception_only(type, value)
|
||||
self.write(''.join(lines))
|
||||
@@ -139,7 +139,6 @@ class InteractiveInterpreter:
|
||||
"""
|
||||
sys.last_type, sys.last_value, last_tb = ei = sys.exc_info()
|
||||
sys.last_traceback = last_tb
|
||||
sys.last_exc = ei[1]
|
||||
try:
|
||||
lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next)
|
||||
if sys.excepthook is sys.__excepthook__:
|
||||
@@ -304,8 +303,6 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-q', action='store_true',
|
||||
help="don't print version and copyright messages")
|
||||
|
||||
34
Lib/codecs.py
vendored
34
Lib/codecs.py
vendored
@@ -83,7 +83,7 @@ BOM64_BE = BOM_UTF32_BE
|
||||
class CodecInfo(tuple):
|
||||
"""Codec details when looking up the codec registry"""
|
||||
|
||||
# Private API to allow Python 3.4 to denylist the known non-Unicode
|
||||
# Private API to allow Python 3.4 to blacklist the known non-Unicode
|
||||
# codecs in the standard library. A more general mechanism to
|
||||
# reliably distinguish test encodings from other codecs will hopefully
|
||||
# be defined for Python 3.5
|
||||
@@ -386,7 +386,7 @@ class StreamWriter(Codec):
|
||||
|
||||
def reset(self):
|
||||
|
||||
""" Resets the codec buffers used for keeping internal state.
|
||||
""" Flushes and resets the codec buffers used for keeping state.
|
||||
|
||||
Calling this method should ensure that the data on the
|
||||
output is put into a clean state, that allows appending
|
||||
@@ -620,7 +620,7 @@ class StreamReader(Codec):
|
||||
|
||||
def reset(self):
|
||||
|
||||
""" Resets the codec buffers used for keeping internal state.
|
||||
""" Resets the codec buffers used for keeping state.
|
||||
|
||||
Note that no stream repositioning should take place.
|
||||
This method is primarily intended to be able to recover
|
||||
@@ -838,7 +838,7 @@ class StreamRecoder:
|
||||
|
||||
def writelines(self, list):
|
||||
|
||||
data = b''.join(list)
|
||||
data = ''.join(list)
|
||||
data, bytesdecoded = self.decode(data, self.errors)
|
||||
return self.writer.write(data)
|
||||
|
||||
@@ -847,12 +847,6 @@ class StreamRecoder:
|
||||
self.reader.reset()
|
||||
self.writer.reset()
|
||||
|
||||
def seek(self, offset, whence=0):
|
||||
# Seeks must be propagated to both the readers and writers
|
||||
# as they might need to reset their internal buffers.
|
||||
self.reader.seek(offset, whence)
|
||||
self.writer.seek(offset, whence)
|
||||
|
||||
def __getattr__(self, name,
|
||||
getattr=getattr):
|
||||
|
||||
@@ -868,7 +862,7 @@ class StreamRecoder:
|
||||
|
||||
### Shortcuts
|
||||
|
||||
def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
|
||||
def open(filename, mode='r', encoding=None, errors='strict', buffering=1):
|
||||
|
||||
""" Open an encoded file using the given mode and return
|
||||
a wrapped version providing transparent encoding/decoding.
|
||||
@@ -889,8 +883,7 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
|
||||
encoding error occurs.
|
||||
|
||||
buffering has the same meaning as for the builtin open() API.
|
||||
It defaults to -1 which means that the default buffer size will
|
||||
be used.
|
||||
It defaults to line buffered.
|
||||
|
||||
The returned wrapped file object provides an extra attribute
|
||||
.encoding which allows querying the used encoding. This
|
||||
@@ -905,16 +898,11 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
|
||||
file = builtins.open(filename, mode, buffering)
|
||||
if encoding is None:
|
||||
return file
|
||||
|
||||
try:
|
||||
info = lookup(encoding)
|
||||
srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
|
||||
# Add attributes to simplify introspection
|
||||
srw.encoding = encoding
|
||||
return srw
|
||||
except:
|
||||
file.close()
|
||||
raise
|
||||
info = lookup(encoding)
|
||||
srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
|
||||
# Add attributes to simplify introspection
|
||||
srw.encoding = encoding
|
||||
return srw
|
||||
|
||||
def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'):
|
||||
|
||||
|
||||
97
Lib/codeop.py
vendored
97
Lib/codeop.py
vendored
@@ -10,6 +10,30 @@ and:
|
||||
syntax error (OverflowError and ValueError can be produced by
|
||||
malformed literals).
|
||||
|
||||
Approach:
|
||||
|
||||
First, check if the source consists entirely of blank lines and
|
||||
comments; if so, replace it with 'pass', because the built-in
|
||||
parser doesn't always do the right thing for these.
|
||||
|
||||
Compile three times: as is, with \n, and with \n\n appended. If it
|
||||
compiles as is, it's complete. If it compiles with one \n appended,
|
||||
we expect more. If it doesn't compile either way, we compare the
|
||||
error we get when compiling with \n or \n\n appended. If the errors
|
||||
are the same, the code is broken. But if the errors are different, we
|
||||
expect more. Not intuitive; not even guaranteed to hold in future
|
||||
releases; but this matches the compiler's behavior from Python 1.4
|
||||
through 2.2, at least.
|
||||
|
||||
Caveat:
|
||||
|
||||
It is possible (but not likely) that the parser stops parsing with a
|
||||
successful outcome before reaching the end of the source; in this
|
||||
case, trailing symbols may be ignored instead of causing an error.
|
||||
For example, a backslash followed by two newlines may be followed by
|
||||
arbitrary garbage. This will be fixed once the API for the parser is
|
||||
better.
|
||||
|
||||
The two interfaces are:
|
||||
|
||||
compile_command(source, filename, symbol):
|
||||
@@ -33,61 +57,49 @@ Compile():
|
||||
"""
|
||||
|
||||
import __future__
|
||||
import warnings
|
||||
|
||||
_features = [getattr(__future__, fname)
|
||||
for fname in __future__.all_feature_names]
|
||||
|
||||
__all__ = ["compile_command", "Compile", "CommandCompiler"]
|
||||
|
||||
# The following flags match the values from Include/cpython/compile.h
|
||||
# Caveat emptor: These flags are undocumented on purpose and depending
|
||||
# on their effect outside the standard library is **unsupported**.
|
||||
PyCF_DONT_IMPLY_DEDENT = 0x200
|
||||
PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
|
||||
PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h
|
||||
|
||||
def _maybe_compile(compiler, source, filename, symbol):
|
||||
# Check for source consisting of only blank lines and comments.
|
||||
# Check for source consisting of only blank lines and comments
|
||||
for line in source.split("\n"):
|
||||
line = line.strip()
|
||||
if line and line[0] != '#':
|
||||
break # Leave it alone.
|
||||
break # Leave it alone
|
||||
else:
|
||||
if symbol != "eval":
|
||||
source = "pass" # Replace it with a 'pass' statement
|
||||
|
||||
# Disable compiler warnings when checking for incomplete input.
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
|
||||
try:
|
||||
compiler(source, filename, symbol)
|
||||
except SyntaxError: # Let other compile() errors propagate.
|
||||
try:
|
||||
compiler(source + "\n", filename, symbol)
|
||||
return None
|
||||
except SyntaxError as e:
|
||||
if "incomplete input" in str(e):
|
||||
return None
|
||||
# fallthrough
|
||||
err = err1 = err2 = None
|
||||
code = code1 = code2 = None
|
||||
|
||||
return compiler(source, filename, symbol, incomplete_input=False)
|
||||
try:
|
||||
code = compiler(source, filename, symbol)
|
||||
except SyntaxError as err:
|
||||
pass
|
||||
|
||||
def _is_syntax_error(err1, err2):
|
||||
rep1 = repr(err1)
|
||||
rep2 = repr(err2)
|
||||
if "was never closed" in rep1 and "was never closed" in rep2:
|
||||
return False
|
||||
if rep1 == rep2:
|
||||
return True
|
||||
return False
|
||||
try:
|
||||
code1 = compiler(source + "\n", filename, symbol)
|
||||
except SyntaxError as e:
|
||||
err1 = e
|
||||
|
||||
def _compile(source, filename, symbol, incomplete_input=True):
|
||||
flags = 0
|
||||
if incomplete_input:
|
||||
flags |= PyCF_ALLOW_INCOMPLETE_INPUT
|
||||
flags |= PyCF_DONT_IMPLY_DEDENT
|
||||
return compile(source, filename, symbol, flags)
|
||||
try:
|
||||
code2 = compiler(source + "\n\n", filename, symbol)
|
||||
except SyntaxError as e:
|
||||
err2 = e
|
||||
|
||||
if code:
|
||||
return code
|
||||
if not code1 and repr(err1) == repr(err2):
|
||||
raise err1
|
||||
|
||||
def _compile(source, filename, symbol):
|
||||
return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT)
|
||||
|
||||
def compile_command(source, filename="<input>", symbol="single"):
|
||||
r"""Compile a command and determine whether it is incomplete.
|
||||
@@ -97,8 +109,7 @@ def compile_command(source, filename="<input>", symbol="single"):
|
||||
source -- the source string; may contain \n characters
|
||||
filename -- optional filename from which source was read; default
|
||||
"<input>"
|
||||
symbol -- optional grammar start symbol; "single" (default), "exec"
|
||||
or "eval"
|
||||
symbol -- optional grammar start symbol; "single" (default) or "eval"
|
||||
|
||||
Return value / exceptions raised:
|
||||
|
||||
@@ -116,14 +127,10 @@ class Compile:
|
||||
statement, it "remembers" and compiles all subsequent program texts
|
||||
with the statement in force."""
|
||||
def __init__(self):
|
||||
self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
|
||||
self.flags = PyCF_DONT_IMPLY_DEDENT
|
||||
|
||||
def __call__(self, source, filename, symbol, **kwargs):
|
||||
flags = self.flags
|
||||
if kwargs.get('incomplete_input', True) is False:
|
||||
flags &= ~PyCF_DONT_IMPLY_DEDENT
|
||||
flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT
|
||||
codeob = compile(source, filename, symbol, flags, True)
|
||||
def __call__(self, source, filename, symbol):
|
||||
codeob = compile(source, filename, symbol, self.flags, 1)
|
||||
for feature in _features:
|
||||
if codeob.co_flags & feature.compiler_flag:
|
||||
self.flags |= feature.compiler_flag
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,58 +1,20 @@
|
||||
from reprlib import recursive_repr as _recursive_repr
|
||||
|
||||
class defaultdict(dict):
|
||||
def __init__(self, *args, **kwargs):
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if len(args) >= 1:
|
||||
default_factory = args[0]
|
||||
if default_factory is not None and not callable(default_factory):
|
||||
raise TypeError("first argument must be callable or None")
|
||||
args = args[1:]
|
||||
else:
|
||||
default_factory = None
|
||||
super().__init__(*args, **kwargs)
|
||||
self = dict.__new__(cls, *args, **kwargs)
|
||||
self.default_factory = default_factory
|
||||
return self
|
||||
|
||||
def __missing__(self, key):
|
||||
if self.default_factory is not None:
|
||||
val = self.default_factory()
|
||||
if self.default_factory:
|
||||
return self.default_factory()
|
||||
else:
|
||||
raise KeyError(key)
|
||||
self[key] = val
|
||||
return val
|
||||
|
||||
@_recursive_repr()
|
||||
def __repr_factory(factory):
|
||||
return repr(factory)
|
||||
|
||||
def __repr__(self):
|
||||
return f"{type(self).__name__}({defaultdict.__repr_factory(self.default_factory)}, {dict.__repr__(self)})"
|
||||
return f"defaultdict({self.default_factory}, {dict.__repr__(self)})"
|
||||
|
||||
def copy(self):
|
||||
return type(self)(self.default_factory, self)
|
||||
|
||||
__copy__ = copy
|
||||
|
||||
def __reduce__(self):
|
||||
if self.default_factory is not None:
|
||||
args = self.default_factory,
|
||||
else:
|
||||
args = ()
|
||||
return type(self), args, None, None, iter(self.items())
|
||||
|
||||
def __or__(self, other):
|
||||
if not isinstance(other, dict):
|
||||
return NotImplemented
|
||||
|
||||
new = defaultdict(self.default_factory, self)
|
||||
new.update(other)
|
||||
return new
|
||||
|
||||
def __ror__(self, other):
|
||||
if not isinstance(other, dict):
|
||||
return NotImplemented
|
||||
|
||||
new = defaultdict(self.default_factory, other)
|
||||
new.update(self)
|
||||
return new
|
||||
|
||||
defaultdict.__module__ = 'collections'
|
||||
|
||||
@@ -1,3 +1,2 @@
|
||||
from _collections_abc import *
|
||||
from _collections_abc import __all__
|
||||
from _collections_abc import _CallableGenericAlias
|
||||
|
||||
28
Lib/colorsys.py
vendored
28
Lib/colorsys.py
vendored
@@ -1,14 +1,10 @@
|
||||
"""Conversion functions between RGB and other color systems.
|
||||
|
||||
This modules provides two functions for each color system ABC:
|
||||
|
||||
rgb_to_abc(r, g, b) --> a, b, c
|
||||
abc_to_rgb(a, b, c) --> r, g, b
|
||||
|
||||
All inputs and outputs are triples of floats in the range [0.0...1.0]
|
||||
(with the exception of I and Q, which covers a slightly larger range).
|
||||
Inputs outside the valid range may cause exceptions or invalid outputs.
|
||||
|
||||
Supported color systems:
|
||||
RGB: Red, Green, Blue components
|
||||
YIQ: Luminance, Chrominance (used by composite video signals)
|
||||
@@ -75,18 +71,17 @@ def yiq_to_rgb(y, i, q):
|
||||
def rgb_to_hls(r, g, b):
|
||||
maxc = max(r, g, b)
|
||||
minc = min(r, g, b)
|
||||
sumc = (maxc+minc)
|
||||
rangec = (maxc-minc)
|
||||
l = sumc/2.0
|
||||
# XXX Can optimize (maxc+minc) and (maxc-minc)
|
||||
l = (minc+maxc)/2.0
|
||||
if minc == maxc:
|
||||
return 0.0, l, 0.0
|
||||
if l <= 0.5:
|
||||
s = rangec / sumc
|
||||
s = (maxc-minc) / (maxc+minc)
|
||||
else:
|
||||
s = rangec / (2.0-maxc-minc) # Not always 2.0-sumc: gh-106498.
|
||||
rc = (maxc-r) / rangec
|
||||
gc = (maxc-g) / rangec
|
||||
bc = (maxc-b) / rangec
|
||||
s = (maxc-minc) / (2.0-maxc-minc)
|
||||
rc = (maxc-r) / (maxc-minc)
|
||||
gc = (maxc-g) / (maxc-minc)
|
||||
bc = (maxc-b) / (maxc-minc)
|
||||
if r == maxc:
|
||||
h = bc-gc
|
||||
elif g == maxc:
|
||||
@@ -125,14 +120,13 @@ def _v(m1, m2, hue):
|
||||
def rgb_to_hsv(r, g, b):
|
||||
maxc = max(r, g, b)
|
||||
minc = min(r, g, b)
|
||||
rangec = (maxc-minc)
|
||||
v = maxc
|
||||
if minc == maxc:
|
||||
return 0.0, 0.0, v
|
||||
s = rangec / maxc
|
||||
rc = (maxc-r) / rangec
|
||||
gc = (maxc-g) / rangec
|
||||
bc = (maxc-b) / rangec
|
||||
s = (maxc-minc) / maxc
|
||||
rc = (maxc-r) / (maxc-minc)
|
||||
gc = (maxc-g) / (maxc-minc)
|
||||
bc = (maxc-b) / (maxc-minc)
|
||||
if r == maxc:
|
||||
h = bc-gc
|
||||
elif g == maxc:
|
||||
|
||||
286
Lib/compileall.py
vendored
286
Lib/compileall.py
vendored
@@ -4,7 +4,7 @@ When called as a script with arguments, this compiles the directories
|
||||
given as arguments recursively; the -l option prevents it from
|
||||
recursing into directories.
|
||||
|
||||
Without arguments, it compiles all modules on sys.path, without
|
||||
Without arguments, if compiles all modules on sys.path, without
|
||||
recursing into subdirectories. (Even though it should do so for
|
||||
packages -- for now, you'll have to deal with packages separately.)
|
||||
|
||||
@@ -15,14 +15,16 @@ import sys
|
||||
import importlib.util
|
||||
import py_compile
|
||||
import struct
|
||||
import filecmp
|
||||
|
||||
try:
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
except ImportError:
|
||||
ProcessPoolExecutor = None
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
__all__ = ["compile_dir","compile_file","compile_path"]
|
||||
|
||||
def _walk_dir(dir, maxlevels, quiet=0):
|
||||
def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
|
||||
if quiet < 2 and isinstance(dir, os.PathLike):
|
||||
dir = os.fspath(dir)
|
||||
if not quiet:
|
||||
@@ -38,94 +40,59 @@ def _walk_dir(dir, maxlevels, quiet=0):
|
||||
if name == '__pycache__':
|
||||
continue
|
||||
fullname = os.path.join(dir, name)
|
||||
if ddir is not None:
|
||||
dfile = os.path.join(ddir, name)
|
||||
else:
|
||||
dfile = None
|
||||
if not os.path.isdir(fullname):
|
||||
yield fullname
|
||||
elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
|
||||
os.path.isdir(fullname) and not os.path.islink(fullname)):
|
||||
yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
|
||||
quiet=quiet)
|
||||
yield from _walk_dir(fullname, ddir=dfile,
|
||||
maxlevels=maxlevels - 1, quiet=quiet)
|
||||
|
||||
def compile_dir(dir, maxlevels=None, ddir=None, force=False,
|
||||
rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
|
||||
invalidation_mode=None, *, stripdir=None,
|
||||
prependdir=None, limit_sl_dest=None, hardlink_dupes=False):
|
||||
def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
|
||||
quiet=0, legacy=False, optimize=-1, workers=1):
|
||||
"""Byte-compile all modules in the given directory tree.
|
||||
|
||||
Arguments (only dir is required):
|
||||
|
||||
dir: the directory to byte-compile
|
||||
maxlevels: maximum recursion level (default `sys.getrecursionlimit()`)
|
||||
maxlevels: maximum recursion level (default 10)
|
||||
ddir: the directory that will be prepended to the path to the
|
||||
file as it is compiled into each byte-code file.
|
||||
force: if True, force compilation, even if timestamps are up-to-date
|
||||
quiet: full output with False or 0, errors only with 1,
|
||||
no output with 2
|
||||
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
|
||||
optimize: int or list of optimization levels or -1 for level of
|
||||
the interpreter. Multiple levels leads to multiple compiled
|
||||
files each with one optimization level.
|
||||
optimize: optimization level or -1 for level of the interpreter
|
||||
workers: maximum number of parallel workers
|
||||
invalidation_mode: how the up-to-dateness of the pyc will be checked
|
||||
stripdir: part of path to left-strip from source file path
|
||||
prependdir: path to prepend to beginning of original file path, applied
|
||||
after stripdir
|
||||
limit_sl_dest: ignore symlinks if they are pointing outside of
|
||||
the defined path
|
||||
hardlink_dupes: hardlink duplicated pyc files
|
||||
"""
|
||||
ProcessPoolExecutor = None
|
||||
if ddir is not None and (stripdir is not None or prependdir is not None):
|
||||
raise ValueError(("Destination dir (ddir) cannot be used "
|
||||
"in combination with stripdir or prependdir"))
|
||||
if ddir is not None:
|
||||
stripdir = dir
|
||||
prependdir = ddir
|
||||
ddir = None
|
||||
if workers < 0:
|
||||
if workers is not None and workers < 0:
|
||||
raise ValueError('workers must be greater or equal to 0')
|
||||
if workers != 1:
|
||||
# Check if this is a system where ProcessPoolExecutor can function.
|
||||
from concurrent.futures.process import _check_system_limits
|
||||
try:
|
||||
_check_system_limits()
|
||||
except NotImplementedError:
|
||||
workers = 1
|
||||
else:
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
if maxlevels is None:
|
||||
maxlevels = sys.getrecursionlimit()
|
||||
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
|
||||
|
||||
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
|
||||
ddir=ddir)
|
||||
success = True
|
||||
if workers != 1 and ProcessPoolExecutor is not None:
|
||||
# If workers == 0, let ProcessPoolExecutor choose
|
||||
if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
|
||||
workers = workers or None
|
||||
with ProcessPoolExecutor(max_workers=workers) as executor:
|
||||
results = executor.map(partial(compile_file,
|
||||
ddir=ddir, force=force,
|
||||
rx=rx, quiet=quiet,
|
||||
legacy=legacy,
|
||||
optimize=optimize,
|
||||
invalidation_mode=invalidation_mode,
|
||||
stripdir=stripdir,
|
||||
prependdir=prependdir,
|
||||
limit_sl_dest=limit_sl_dest,
|
||||
hardlink_dupes=hardlink_dupes),
|
||||
optimize=optimize),
|
||||
files)
|
||||
success = min(results, default=True)
|
||||
else:
|
||||
for file in files:
|
||||
if not compile_file(file, ddir, force, rx, quiet,
|
||||
legacy, optimize, invalidation_mode,
|
||||
stripdir=stripdir, prependdir=prependdir,
|
||||
limit_sl_dest=limit_sl_dest,
|
||||
hardlink_dupes=hardlink_dupes):
|
||||
legacy, optimize):
|
||||
success = False
|
||||
return success
|
||||
|
||||
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
||||
legacy=False, optimize=-1,
|
||||
invalidation_mode=None, *, stripdir=None, prependdir=None,
|
||||
limit_sl_dest=None, hardlink_dupes=False):
|
||||
legacy=False, optimize=-1):
|
||||
"""Byte-compile one file.
|
||||
|
||||
Arguments (only fullname is required):
|
||||
@@ -137,114 +104,49 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
||||
quiet: full output with False or 0, errors only with 1,
|
||||
no output with 2
|
||||
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
|
||||
optimize: int or list of optimization levels or -1 for level of
|
||||
the interpreter. Multiple levels leads to multiple compiled
|
||||
files each with one optimization level.
|
||||
invalidation_mode: how the up-to-dateness of the pyc will be checked
|
||||
stripdir: part of path to left-strip from source file path
|
||||
prependdir: path to prepend to beginning of original file path, applied
|
||||
after stripdir
|
||||
limit_sl_dest: ignore symlinks if they are pointing outside of
|
||||
the defined path.
|
||||
hardlink_dupes: hardlink duplicated pyc files
|
||||
optimize: optimization level or -1 for level of the interpreter
|
||||
"""
|
||||
|
||||
if ddir is not None and (stripdir is not None or prependdir is not None):
|
||||
raise ValueError(("Destination dir (ddir) cannot be used "
|
||||
"in combination with stripdir or prependdir"))
|
||||
|
||||
success = True
|
||||
fullname = os.fspath(fullname)
|
||||
stripdir = os.fspath(stripdir) if stripdir is not None else None
|
||||
if quiet < 2 and isinstance(fullname, os.PathLike):
|
||||
fullname = os.fspath(fullname)
|
||||
name = os.path.basename(fullname)
|
||||
|
||||
dfile = None
|
||||
|
||||
if ddir is not None:
|
||||
dfile = os.path.join(ddir, name)
|
||||
|
||||
if stripdir is not None:
|
||||
fullname_parts = fullname.split(os.path.sep)
|
||||
stripdir_parts = stripdir.split(os.path.sep)
|
||||
ddir_parts = list(fullname_parts)
|
||||
|
||||
for spart, opart in zip(stripdir_parts, fullname_parts):
|
||||
if spart == opart:
|
||||
ddir_parts.remove(spart)
|
||||
|
||||
dfile = os.path.join(*ddir_parts)
|
||||
|
||||
if prependdir is not None:
|
||||
if dfile is None:
|
||||
dfile = os.path.join(prependdir, fullname)
|
||||
else:
|
||||
dfile = os.path.join(prependdir, dfile)
|
||||
|
||||
if isinstance(optimize, int):
|
||||
optimize = [optimize]
|
||||
|
||||
# Use set() to remove duplicates.
|
||||
# Use sorted() to create pyc files in a deterministic order.
|
||||
optimize = sorted(set(optimize))
|
||||
|
||||
if hardlink_dupes and len(optimize) < 2:
|
||||
raise ValueError("Hardlinking of duplicated bytecode makes sense "
|
||||
"only for more than one optimization level")
|
||||
|
||||
else:
|
||||
dfile = None
|
||||
if rx is not None:
|
||||
mo = rx.search(fullname)
|
||||
if mo:
|
||||
return success
|
||||
|
||||
if limit_sl_dest is not None and os.path.islink(fullname):
|
||||
if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents:
|
||||
return success
|
||||
|
||||
opt_cfiles = {}
|
||||
|
||||
if os.path.isfile(fullname):
|
||||
for opt_level in optimize:
|
||||
if legacy:
|
||||
opt_cfiles[opt_level] = fullname + 'c'
|
||||
if legacy:
|
||||
cfile = fullname + 'c'
|
||||
else:
|
||||
if optimize >= 0:
|
||||
opt = optimize if optimize >= 1 else ''
|
||||
cfile = importlib.util.cache_from_source(
|
||||
fullname, optimization=opt)
|
||||
else:
|
||||
if opt_level >= 0:
|
||||
opt = opt_level if opt_level >= 1 else ''
|
||||
cfile = (importlib.util.cache_from_source(
|
||||
fullname, optimization=opt))
|
||||
opt_cfiles[opt_level] = cfile
|
||||
else:
|
||||
cfile = importlib.util.cache_from_source(fullname)
|
||||
opt_cfiles[opt_level] = cfile
|
||||
|
||||
cfile = importlib.util.cache_from_source(fullname)
|
||||
cache_dir = os.path.dirname(cfile)
|
||||
head, tail = name[:-3], name[-3:]
|
||||
if tail == '.py':
|
||||
if not force:
|
||||
try:
|
||||
mtime = int(os.stat(fullname).st_mtime)
|
||||
expect = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER,
|
||||
0, mtime & 0xFFFF_FFFF)
|
||||
for cfile in opt_cfiles.values():
|
||||
with open(cfile, 'rb') as chandle:
|
||||
actual = chandle.read(12)
|
||||
if expect != actual:
|
||||
break
|
||||
else:
|
||||
expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER,
|
||||
mtime)
|
||||
with open(cfile, 'rb') as chandle:
|
||||
actual = chandle.read(8)
|
||||
if expect == actual:
|
||||
return success
|
||||
except OSError:
|
||||
pass
|
||||
if not quiet:
|
||||
print('Compiling {!r}...'.format(fullname))
|
||||
try:
|
||||
for index, opt_level in enumerate(optimize):
|
||||
cfile = opt_cfiles[opt_level]
|
||||
ok = py_compile.compile(fullname, cfile, dfile, True,
|
||||
optimize=opt_level,
|
||||
invalidation_mode=invalidation_mode)
|
||||
if index > 0 and hardlink_dupes:
|
||||
previous_cfile = opt_cfiles[optimize[index - 1]]
|
||||
if filecmp.cmp(cfile, previous_cfile, shallow=False):
|
||||
os.unlink(cfile)
|
||||
os.link(previous_cfile, cfile)
|
||||
ok = py_compile.compile(fullname, cfile, dfile, True,
|
||||
optimize=optimize)
|
||||
except py_compile.PyCompileError as err:
|
||||
success = False
|
||||
if quiet >= 2:
|
||||
@@ -254,8 +156,9 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
||||
else:
|
||||
print('*** ', end='')
|
||||
# escape non-printable characters in msg
|
||||
encoding = sys.stdout.encoding or sys.getdefaultencoding()
|
||||
msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding)
|
||||
msg = err.msg.encode(sys.stdout.encoding,
|
||||
errors='backslashreplace')
|
||||
msg = msg.decode(sys.stdout.encoding)
|
||||
print(msg)
|
||||
except (SyntaxError, UnicodeError, OSError) as e:
|
||||
success = False
|
||||
@@ -272,8 +175,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
||||
return success
|
||||
|
||||
def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
|
||||
legacy=False, optimize=-1,
|
||||
invalidation_mode=None):
|
||||
legacy=False, optimize=-1):
|
||||
"""Byte-compile all module on sys.path.
|
||||
|
||||
Arguments (all optional):
|
||||
@@ -284,7 +186,6 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
|
||||
quiet: as for compile_dir() (default 0)
|
||||
legacy: as for compile_dir() (default False)
|
||||
optimize: as for compile_dir() (default -1)
|
||||
invalidation_mode: as for compiler_dir()
|
||||
"""
|
||||
success = True
|
||||
for dir in sys.path:
|
||||
@@ -292,16 +193,9 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
|
||||
if quiet < 2:
|
||||
print('Skipping current directory')
|
||||
else:
|
||||
success = success and compile_dir(
|
||||
dir,
|
||||
maxlevels,
|
||||
None,
|
||||
force,
|
||||
quiet=quiet,
|
||||
legacy=legacy,
|
||||
optimize=optimize,
|
||||
invalidation_mode=invalidation_mode,
|
||||
)
|
||||
success = success and compile_dir(dir, maxlevels, None,
|
||||
force, quiet=quiet,
|
||||
legacy=legacy, optimize=optimize)
|
||||
return success
|
||||
|
||||
|
||||
@@ -312,7 +206,7 @@ def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Utilities to support installing Python libraries.')
|
||||
parser.add_argument('-l', action='store_const', const=0,
|
||||
default=None, dest='maxlevels',
|
||||
default=10, dest='maxlevels',
|
||||
help="don't recurse into subdirectories")
|
||||
parser.add_argument('-r', type=int, dest='recursion',
|
||||
help=('control the maximum recursion level. '
|
||||
@@ -330,20 +224,6 @@ def main():
|
||||
'compile-time tracebacks and in runtime '
|
||||
'tracebacks in cases where the source file is '
|
||||
'unavailable'))
|
||||
parser.add_argument('-s', metavar='STRIPDIR', dest='stripdir',
|
||||
default=None,
|
||||
help=('part of path to left-strip from path '
|
||||
'to source file - for example buildroot. '
|
||||
'`-d` and `-s` options cannot be '
|
||||
'specified together.'))
|
||||
parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir',
|
||||
default=None,
|
||||
help=('path to add as prefix to path '
|
||||
'to source file - for example / to make '
|
||||
'it absolute when some part is removed '
|
||||
'by `-s` option. '
|
||||
'`-d` and `-p` options cannot be '
|
||||
'specified together.'))
|
||||
parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
|
||||
help=('skip files matching the regular expression; '
|
||||
'the regexp is searched for in the full path '
|
||||
@@ -358,23 +238,6 @@ def main():
|
||||
'to the equivalent of -l sys.path'))
|
||||
parser.add_argument('-j', '--workers', default=1,
|
||||
type=int, help='Run compileall concurrently')
|
||||
invalidation_modes = [mode.name.lower().replace('_', '-')
|
||||
for mode in py_compile.PycInvalidationMode]
|
||||
parser.add_argument('--invalidation-mode',
|
||||
choices=sorted(invalidation_modes),
|
||||
help=('set .pyc invalidation mode; defaults to '
|
||||
'"checked-hash" if the SOURCE_DATE_EPOCH '
|
||||
'environment variable is set, and '
|
||||
'"timestamp" otherwise.'))
|
||||
parser.add_argument('-o', action='append', type=int, dest='opt_levels',
|
||||
help=('Optimization levels to run compilation with. '
|
||||
'Default is -1 which uses the optimization level '
|
||||
'of the Python interpreter itself (see -O).'))
|
||||
parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',
|
||||
help='Ignore symlinks pointing outsite of the DIR')
|
||||
parser.add_argument('--hardlink-dupes', action='store_true',
|
||||
dest='hardlink_dupes',
|
||||
help='Hardlink duplicated pyc files')
|
||||
|
||||
args = parser.parse_args()
|
||||
compile_dests = args.compile_dest
|
||||
@@ -383,31 +246,16 @@ def main():
|
||||
import re
|
||||
args.rx = re.compile(args.rx)
|
||||
|
||||
if args.limit_sl_dest == "":
|
||||
args.limit_sl_dest = None
|
||||
|
||||
if args.recursion is not None:
|
||||
maxlevels = args.recursion
|
||||
else:
|
||||
maxlevels = args.maxlevels
|
||||
|
||||
if args.opt_levels is None:
|
||||
args.opt_levels = [-1]
|
||||
|
||||
if len(args.opt_levels) == 1 and args.hardlink_dupes:
|
||||
parser.error(("Hardlinking of duplicated bytecode makes sense "
|
||||
"only for more than one optimization level."))
|
||||
|
||||
if args.ddir is not None and (
|
||||
args.stripdir is not None or args.prependdir is not None
|
||||
):
|
||||
parser.error("-d cannot be used in combination with -s or -p")
|
||||
|
||||
# if flist is provided then load it
|
||||
if args.flist:
|
||||
try:
|
||||
with (sys.stdin if args.flist=='-' else
|
||||
open(args.flist, encoding="utf-8")) as f:
|
||||
with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
|
||||
for line in f:
|
||||
compile_dests.append(line.strip())
|
||||
except OSError:
|
||||
@@ -415,11 +263,8 @@ def main():
|
||||
print("Error reading file list {}".format(args.flist))
|
||||
return False
|
||||
|
||||
if args.invalidation_mode:
|
||||
ivl_mode = args.invalidation_mode.replace('-', '_').upper()
|
||||
invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]
|
||||
else:
|
||||
invalidation_mode = None
|
||||
if args.workers is not None:
|
||||
args.workers = args.workers or None
|
||||
|
||||
success = True
|
||||
try:
|
||||
@@ -427,30 +272,17 @@ def main():
|
||||
for dest in compile_dests:
|
||||
if os.path.isfile(dest):
|
||||
if not compile_file(dest, args.ddir, args.force, args.rx,
|
||||
args.quiet, args.legacy,
|
||||
invalidation_mode=invalidation_mode,
|
||||
stripdir=args.stripdir,
|
||||
prependdir=args.prependdir,
|
||||
optimize=args.opt_levels,
|
||||
limit_sl_dest=args.limit_sl_dest,
|
||||
hardlink_dupes=args.hardlink_dupes):
|
||||
args.quiet, args.legacy):
|
||||
success = False
|
||||
else:
|
||||
if not compile_dir(dest, maxlevels, args.ddir,
|
||||
args.force, args.rx, args.quiet,
|
||||
args.legacy, workers=args.workers,
|
||||
invalidation_mode=invalidation_mode,
|
||||
stripdir=args.stripdir,
|
||||
prependdir=args.prependdir,
|
||||
optimize=args.opt_levels,
|
||||
limit_sl_dest=args.limit_sl_dest,
|
||||
hardlink_dupes=args.hardlink_dupes):
|
||||
args.legacy, workers=args.workers):
|
||||
success = False
|
||||
return success
|
||||
else:
|
||||
return compile_path(legacy=args.legacy, force=args.force,
|
||||
quiet=args.quiet,
|
||||
invalidation_mode=invalidation_mode)
|
||||
quiet=args.quiet)
|
||||
except KeyboardInterrupt:
|
||||
if args.quiet < 2:
|
||||
print("\n[interrupted]")
|
||||
|
||||
@@ -10,44 +10,9 @@ from concurrent.futures._base import (FIRST_COMPLETED,
|
||||
ALL_COMPLETED,
|
||||
CancelledError,
|
||||
TimeoutError,
|
||||
InvalidStateError,
|
||||
BrokenExecutor,
|
||||
Future,
|
||||
Executor,
|
||||
wait,
|
||||
as_completed)
|
||||
|
||||
__all__ = (
|
||||
'FIRST_COMPLETED',
|
||||
'FIRST_EXCEPTION',
|
||||
'ALL_COMPLETED',
|
||||
'CancelledError',
|
||||
'TimeoutError',
|
||||
'BrokenExecutor',
|
||||
'Future',
|
||||
'Executor',
|
||||
'wait',
|
||||
'as_completed',
|
||||
'ProcessPoolExecutor',
|
||||
'ThreadPoolExecutor',
|
||||
)
|
||||
|
||||
|
||||
def __dir__():
|
||||
return __all__ + ('__author__', '__doc__')
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
global ProcessPoolExecutor, ThreadPoolExecutor
|
||||
|
||||
if name == 'ProcessPoolExecutor':
|
||||
from .process import ProcessPoolExecutor as pe
|
||||
ProcessPoolExecutor = pe
|
||||
return pe
|
||||
|
||||
if name == 'ThreadPoolExecutor':
|
||||
from .thread import ThreadPoolExecutor as te
|
||||
ThreadPoolExecutor = te
|
||||
return te
|
||||
|
||||
raise AttributeError(f"module {__name__} has no attribute {name}")
|
||||
from concurrent.futures.process import ProcessPoolExecutor
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
@@ -7,7 +7,6 @@ import collections
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
|
||||
FIRST_COMPLETED = 'FIRST_COMPLETED'
|
||||
FIRST_EXCEPTION = 'FIRST_EXCEPTION'
|
||||
@@ -54,10 +53,6 @@ class TimeoutError(Error):
|
||||
"""The operation exceeded the given deadline."""
|
||||
pass
|
||||
|
||||
class InvalidStateError(Error):
|
||||
"""The operation is not allowed in this state."""
|
||||
pass
|
||||
|
||||
class _Waiter(object):
|
||||
"""Provides the event that wait() and as_completed() block on."""
|
||||
def __init__(self):
|
||||
@@ -175,29 +170,6 @@ def _create_and_install_waiters(fs, return_when):
|
||||
|
||||
return waiter
|
||||
|
||||
|
||||
def _yield_finished_futures(fs, waiter, ref_collect):
|
||||
"""
|
||||
Iterate on the list *fs*, yielding finished futures one by one in
|
||||
reverse order.
|
||||
Before yielding a future, *waiter* is removed from its waiters
|
||||
and the future is removed from each set in the collection of sets
|
||||
*ref_collect*.
|
||||
|
||||
The aim of this function is to avoid keeping stale references after
|
||||
the future is yielded and before the iterator resumes.
|
||||
"""
|
||||
while fs:
|
||||
f = fs[-1]
|
||||
for futures_set in ref_collect:
|
||||
futures_set.remove(f)
|
||||
with f._condition:
|
||||
f._waiters.remove(waiter)
|
||||
del f
|
||||
# Careful not to keep a reference to the popped value
|
||||
yield fs.pop()
|
||||
|
||||
|
||||
def as_completed(fs, timeout=None):
|
||||
"""An iterator over the given futures that yields each as it completes.
|
||||
|
||||
@@ -217,30 +189,28 @@ def as_completed(fs, timeout=None):
|
||||
before the given timeout.
|
||||
"""
|
||||
if timeout is not None:
|
||||
end_time = timeout + time.monotonic()
|
||||
end_time = timeout + time.time()
|
||||
|
||||
fs = set(fs)
|
||||
total_futures = len(fs)
|
||||
with _AcquireFutures(fs):
|
||||
finished = set(
|
||||
f for f in fs
|
||||
if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
|
||||
pending = fs - finished
|
||||
waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
|
||||
finished = list(finished)
|
||||
|
||||
try:
|
||||
yield from _yield_finished_futures(finished, waiter,
|
||||
ref_collect=(fs,))
|
||||
yield from finished
|
||||
|
||||
while pending:
|
||||
if timeout is None:
|
||||
wait_timeout = None
|
||||
else:
|
||||
wait_timeout = end_time - time.monotonic()
|
||||
wait_timeout = end_time - time.time()
|
||||
if wait_timeout < 0:
|
||||
raise TimeoutError(
|
||||
'%d (of %d) futures unfinished' % (
|
||||
len(pending), total_futures))
|
||||
len(pending), len(fs)))
|
||||
|
||||
waiter.event.wait(wait_timeout)
|
||||
|
||||
@@ -249,13 +219,11 @@ def as_completed(fs, timeout=None):
|
||||
waiter.finished_futures = []
|
||||
waiter.event.clear()
|
||||
|
||||
# reverse to keep finishing order
|
||||
finished.reverse()
|
||||
yield from _yield_finished_futures(finished, waiter,
|
||||
ref_collect=(fs, pending))
|
||||
for future in finished:
|
||||
yield future
|
||||
pending.remove(future)
|
||||
|
||||
finally:
|
||||
# Remove waiter from unfinished futures
|
||||
for f in fs:
|
||||
with f._condition:
|
||||
f._waiters.remove(waiter)
|
||||
@@ -284,14 +252,13 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
|
||||
A named 2-tuple of sets. The first set, named 'done', contains the
|
||||
futures that completed (is finished or cancelled) before the wait
|
||||
completed. The second set, named 'not_done', contains uncompleted
|
||||
futures. Duplicate futures given to *fs* are removed and will be
|
||||
returned only once.
|
||||
futures.
|
||||
"""
|
||||
fs = set(fs)
|
||||
with _AcquireFutures(fs):
|
||||
done = {f for f in fs
|
||||
if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]}
|
||||
not_done = fs - done
|
||||
done = set(f for f in fs
|
||||
if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
|
||||
not_done = set(fs) - done
|
||||
|
||||
if (return_when == FIRST_COMPLETED) and done:
|
||||
return DoneAndNotDoneFutures(done, not_done)
|
||||
elif (return_when == FIRST_EXCEPTION) and done:
|
||||
@@ -310,7 +277,7 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
|
||||
f._waiters.remove(waiter)
|
||||
|
||||
done.update(waiter.finished_futures)
|
||||
return DoneAndNotDoneFutures(done, fs - done)
|
||||
return DoneAndNotDoneFutures(done, set(fs) - done)
|
||||
|
||||
class Future(object):
|
||||
"""Represents the result of an asynchronous computation."""
|
||||
@@ -381,17 +348,13 @@ class Future(object):
|
||||
return self._state == RUNNING
|
||||
|
||||
def done(self):
|
||||
"""Return True if the future was cancelled or finished executing."""
|
||||
"""Return True of the future was cancelled or finished executing."""
|
||||
with self._condition:
|
||||
return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]
|
||||
|
||||
def __get_result(self):
|
||||
if self._exception:
|
||||
try:
|
||||
raise self._exception
|
||||
finally:
|
||||
# Break a reference cycle with the exception in self._exception
|
||||
self = None
|
||||
raise self._exception
|
||||
else:
|
||||
return self._result
|
||||
|
||||
@@ -410,10 +373,7 @@ class Future(object):
|
||||
if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]:
|
||||
self._done_callbacks.append(fn)
|
||||
return
|
||||
try:
|
||||
fn(self)
|
||||
except Exception:
|
||||
LOGGER.exception('exception calling callback for %r', self)
|
||||
fn(self)
|
||||
|
||||
def result(self, timeout=None):
|
||||
"""Return the result of the call that the future represents.
|
||||
@@ -431,24 +391,20 @@ class Future(object):
|
||||
timeout.
|
||||
Exception: If the call raised then that exception will be raised.
|
||||
"""
|
||||
try:
|
||||
with self._condition:
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self.__get_result()
|
||||
with self._condition:
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self.__get_result()
|
||||
|
||||
self._condition.wait(timeout)
|
||||
self._condition.wait(timeout)
|
||||
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self.__get_result()
|
||||
else:
|
||||
raise TimeoutError()
|
||||
finally:
|
||||
# Break a reference cycle with the exception in self._exception
|
||||
self = None
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self.__get_result()
|
||||
else:
|
||||
raise TimeoutError()
|
||||
|
||||
def exception(self, timeout=None):
|
||||
"""Return the exception raised by the call that the future represents.
|
||||
@@ -530,8 +486,6 @@ class Future(object):
|
||||
Should only be used by Executor implementations and unit tests.
|
||||
"""
|
||||
with self._condition:
|
||||
if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
|
||||
raise InvalidStateError('{}: {!r}'.format(self._state, self))
|
||||
self._result = result
|
||||
self._state = FINISHED
|
||||
for waiter in self._waiters:
|
||||
@@ -545,8 +499,6 @@ class Future(object):
|
||||
Should only be used by Executor implementations and unit tests.
|
||||
"""
|
||||
with self._condition:
|
||||
if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
|
||||
raise InvalidStateError('{}: {!r}'.format(self._state, self))
|
||||
self._exception = exception
|
||||
self._state = FINISHED
|
||||
for waiter in self._waiters:
|
||||
@@ -554,12 +506,10 @@ class Future(object):
|
||||
self._condition.notify_all()
|
||||
self._invoke_callbacks()
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
class Executor(object):
|
||||
"""This is an abstract base class for concrete asynchronous executors."""
|
||||
|
||||
def submit(self, fn, /, *args, **kwargs):
|
||||
def submit(self, fn, *args, **kwargs):
|
||||
"""Submits a callable to be executed with the given arguments.
|
||||
|
||||
Schedules the callable to be executed as fn(*args, **kwargs) and returns
|
||||
@@ -593,7 +543,7 @@ class Executor(object):
|
||||
Exception: If fn(*args) raises for any values.
|
||||
"""
|
||||
if timeout is not None:
|
||||
end_time = timeout + time.monotonic()
|
||||
end_time = timeout + time.time()
|
||||
|
||||
fs = [self.submit(fn, *args) for args in zip(*iterables)]
|
||||
|
||||
@@ -601,20 +551,17 @@ class Executor(object):
|
||||
# before the first iterator value is required.
|
||||
def result_iterator():
|
||||
try:
|
||||
# reverse to keep finishing order
|
||||
fs.reverse()
|
||||
while fs:
|
||||
# Careful not to keep a reference to the popped future
|
||||
for future in fs:
|
||||
if timeout is None:
|
||||
yield fs.pop().result()
|
||||
yield future.result()
|
||||
else:
|
||||
yield fs.pop().result(end_time - time.monotonic())
|
||||
yield future.result(end_time - time.time())
|
||||
finally:
|
||||
for future in fs:
|
||||
future.cancel()
|
||||
return result_iterator()
|
||||
|
||||
def shutdown(self, wait=True, *, cancel_futures=False):
|
||||
def shutdown(self, wait=True):
|
||||
"""Clean-up the resources associated with the Executor.
|
||||
|
||||
It is safe to call this method several times. Otherwise, no other
|
||||
@@ -624,9 +571,6 @@ class Executor(object):
|
||||
wait: If True then shutdown will not return until all running
|
||||
futures have finished executing and the resources used by the
|
||||
executor have been reclaimed.
|
||||
cancel_futures: If True then shutdown will cancel all pending
|
||||
futures. Futures that are completed or running will not be
|
||||
cancelled.
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -636,9 +580,3 @@ class Executor(object):
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.shutdown(wait=True)
|
||||
return False
|
||||
|
||||
|
||||
class BrokenExecutor(RuntimeError):
|
||||
"""
|
||||
Raised when a executor has become non-functional after a severe failure.
|
||||
"""
|
||||
|
||||
@@ -3,15 +3,15 @@
|
||||
|
||||
"""Implements ProcessPoolExecutor.
|
||||
|
||||
The following diagram and text describe the data-flow through the system:
|
||||
The follow diagram and text describe the data-flow through the system:
|
||||
|
||||
|======================= In-process =====================|== Out-of-process ==|
|
||||
|
||||
+----------+ +----------+ +--------+ +-----------+ +---------+
|
||||
| | => | Work Ids | | | | Call Q | | Process |
|
||||
| | +----------+ | | +-----------+ | Pool |
|
||||
| | | ... | | | | ... | +---------+
|
||||
| | | 6 | => | | => | 5, call() | => | |
|
||||
| | => | Work Ids | => | | => | Call Q | => | |
|
||||
| | +----------+ | | +-----------+ | |
|
||||
| | | ... | | | | ... | | |
|
||||
| | | 6 | | | | 5, call() | | |
|
||||
| | | 7 | | | | ... | | |
|
||||
| Process | | ... | | Local | +-----------+ | Process |
|
||||
| Pool | +----------+ | Worker | | #1..n |
|
||||
@@ -45,74 +45,52 @@ Process #1..n:
|
||||
|
||||
__author__ = 'Brian Quinlan (brian@sweetapp.com)'
|
||||
|
||||
import atexit
|
||||
import os
|
||||
from concurrent.futures import _base
|
||||
import queue
|
||||
import multiprocessing as mp
|
||||
import multiprocessing.connection
|
||||
from multiprocessing.queues import Queue
|
||||
from queue import Full
|
||||
import multiprocessing
|
||||
from multiprocessing import SimpleQueue
|
||||
from multiprocessing.connection import wait
|
||||
import threading
|
||||
import weakref
|
||||
from functools import partial
|
||||
import itertools
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
# Workers are created as daemon threads and processes. This is done to allow the
|
||||
# interpreter to exit when there are still idle processes in a
|
||||
# ProcessPoolExecutor's process pool (i.e. shutdown() was not called). However,
|
||||
# allowing workers to die with the interpreter has two undesirable properties:
|
||||
# - The workers would still be running during interpreter shutdown,
|
||||
# meaning that they would fail in unpredictable ways.
|
||||
# - The workers could be killed while evaluating a work item, which could
|
||||
# be bad if the callable being evaluated has external side-effects e.g.
|
||||
# writing to a file.
|
||||
#
|
||||
# To work around this problem, an exit handler is installed which tells the
|
||||
# workers to exit when their work queues are empty and then waits until the
|
||||
# threads/processes finish.
|
||||
|
||||
_threads_wakeups = weakref.WeakKeyDictionary()
|
||||
_global_shutdown = False
|
||||
|
||||
|
||||
class _ThreadWakeup:
|
||||
def __init__(self):
|
||||
self._closed = False
|
||||
self._reader, self._writer = mp.Pipe(duplex=False)
|
||||
|
||||
def close(self):
|
||||
if not self._closed:
|
||||
self._closed = True
|
||||
self._writer.close()
|
||||
self._reader.close()
|
||||
|
||||
def wakeup(self):
|
||||
if not self._closed:
|
||||
self._writer.send_bytes(b"")
|
||||
|
||||
def clear(self):
|
||||
if not self._closed:
|
||||
while self._reader.poll():
|
||||
self._reader.recv_bytes()
|
||||
|
||||
_threads_queues = weakref.WeakKeyDictionary()
|
||||
_shutdown = False
|
||||
|
||||
def _python_exit():
|
||||
global _global_shutdown
|
||||
_global_shutdown = True
|
||||
items = list(_threads_wakeups.items())
|
||||
for _, thread_wakeup in items:
|
||||
# call not protected by ProcessPoolExecutor._shutdown_lock
|
||||
thread_wakeup.wakeup()
|
||||
for t, _ in items:
|
||||
global _shutdown
|
||||
_shutdown = True
|
||||
items = list(_threads_queues.items())
|
||||
for t, q in items:
|
||||
q.put(None)
|
||||
for t, q in items:
|
||||
t.join()
|
||||
|
||||
# Register for `_python_exit()` to be called just before joining all
|
||||
# non-daemon threads. This is used instead of `atexit.register()` for
|
||||
# compatibility with subinterpreters, which no longer support daemon threads.
|
||||
# See bpo-39812 for context.
|
||||
threading._register_atexit(_python_exit)
|
||||
|
||||
# Controls how many more calls than processes will be queued in the call queue.
|
||||
# A smaller number will mean that processes spend more time idle waiting for
|
||||
# work while a larger number will make Future.cancel() succeed less frequently
|
||||
# (Futures in the call queue cannot be cancelled).
|
||||
EXTRA_QUEUED_CALLS = 1
|
||||
|
||||
|
||||
# On Windows, WaitForMultipleObjects is used to wait for processes to finish.
|
||||
# It can wait on, at most, 63 objects. There is an overhead of two objects:
|
||||
# - the result queue reader
|
||||
# - the thread wakeup reader
|
||||
_MAX_WINDOWS_WORKERS = 63 - 2
|
||||
|
||||
# Hack to embed stringification of remote traceback in local traceback
|
||||
|
||||
class _RemoteTraceback(Exception):
|
||||
@@ -126,9 +104,6 @@ class _ExceptionWithTraceback:
|
||||
tb = traceback.format_exception(type(exc), exc, tb)
|
||||
tb = ''.join(tb)
|
||||
self.exc = exc
|
||||
# Traceback object needs to be garbage-collected as its frames
|
||||
# contain references to all the objects in the exception scope
|
||||
self.exc.__traceback__ = None
|
||||
self.tb = '\n"""\n%s"""' % tb
|
||||
def __reduce__(self):
|
||||
return _rebuild_exc, (self.exc, self.tb)
|
||||
@@ -157,32 +132,6 @@ class _CallItem(object):
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
|
||||
|
||||
class _SafeQueue(Queue):
|
||||
"""Safe Queue set exception to the future object linked to a job"""
|
||||
def __init__(self, max_size=0, *, ctx, pending_work_items, shutdown_lock,
|
||||
thread_wakeup):
|
||||
self.pending_work_items = pending_work_items
|
||||
self.shutdown_lock = shutdown_lock
|
||||
self.thread_wakeup = thread_wakeup
|
||||
super().__init__(max_size, ctx=ctx)
|
||||
|
||||
def _on_queue_feeder_error(self, e, obj):
|
||||
if isinstance(obj, _CallItem):
|
||||
tb = traceback.format_exception(type(e), e, e.__traceback__)
|
||||
e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb)))
|
||||
work_item = self.pending_work_items.pop(obj.work_id, None)
|
||||
with self.shutdown_lock:
|
||||
self.thread_wakeup.wakeup()
|
||||
# work_item can be None if another process terminated. In this
|
||||
# case, the executor_manager_thread fails all work_items
|
||||
# with BrokenProcessPool
|
||||
if work_item is not None:
|
||||
work_item.future.set_exception(e)
|
||||
else:
|
||||
super()._on_queue_feeder_error(e, obj)
|
||||
|
||||
|
||||
def _get_chunks(*iterables, chunksize):
|
||||
""" Iterates over zip()ed iterables in chunks. """
|
||||
it = zip(*iterables)
|
||||
@@ -192,7 +141,6 @@ def _get_chunks(*iterables, chunksize):
|
||||
return
|
||||
yield chunk
|
||||
|
||||
|
||||
def _process_chunk(fn, chunk):
|
||||
""" Processes a chunk of an iterable passed to map.
|
||||
|
||||
@@ -204,38 +152,19 @@ def _process_chunk(fn, chunk):
|
||||
"""
|
||||
return [fn(*args) for args in chunk]
|
||||
|
||||
|
||||
def _sendback_result(result_queue, work_id, result=None, exception=None):
|
||||
"""Safely send back the given result or exception"""
|
||||
try:
|
||||
result_queue.put(_ResultItem(work_id, result=result,
|
||||
exception=exception))
|
||||
except BaseException as e:
|
||||
exc = _ExceptionWithTraceback(e, e.__traceback__)
|
||||
result_queue.put(_ResultItem(work_id, exception=exc))
|
||||
|
||||
|
||||
def _process_worker(call_queue, result_queue, initializer, initargs):
|
||||
def _process_worker(call_queue, result_queue):
|
||||
"""Evaluates calls from call_queue and places the results in result_queue.
|
||||
|
||||
This worker is run in a separate process.
|
||||
|
||||
Args:
|
||||
call_queue: A ctx.Queue of _CallItems that will be read and
|
||||
call_queue: A multiprocessing.Queue of _CallItems that will be read and
|
||||
evaluated by the worker.
|
||||
result_queue: A ctx.Queue of _ResultItems that will written
|
||||
result_queue: A multiprocessing.Queue of _ResultItems that will written
|
||||
to by the worker.
|
||||
initializer: A callable initializer, or None
|
||||
initargs: A tuple of args for the initializer
|
||||
shutdown: A multiprocessing.Event that will be set as a signal to the
|
||||
worker that it should exit when call_queue is empty.
|
||||
"""
|
||||
if initializer is not None:
|
||||
try:
|
||||
initializer(*initargs)
|
||||
except BaseException:
|
||||
_base.LOGGER.critical('Exception in initializer:', exc_info=True)
|
||||
# The parent will notice that the process stopped and
|
||||
# mark the pool broken
|
||||
return
|
||||
while True:
|
||||
call_item = call_queue.get(block=True)
|
||||
if call_item is None:
|
||||
@@ -246,303 +175,171 @@ def _process_worker(call_queue, result_queue, initializer, initargs):
|
||||
r = call_item.fn(*call_item.args, **call_item.kwargs)
|
||||
except BaseException as e:
|
||||
exc = _ExceptionWithTraceback(e, e.__traceback__)
|
||||
_sendback_result(result_queue, call_item.work_id, exception=exc)
|
||||
result_queue.put(_ResultItem(call_item.work_id, exception=exc))
|
||||
else:
|
||||
_sendback_result(result_queue, call_item.work_id, result=r)
|
||||
del r
|
||||
result_queue.put(_ResultItem(call_item.work_id,
|
||||
result=r))
|
||||
|
||||
# Liberate the resource as soon as possible, to avoid holding onto
|
||||
# open files or shared memory that is not needed anymore
|
||||
del call_item
|
||||
def _add_call_item_to_queue(pending_work_items,
|
||||
work_ids,
|
||||
call_queue):
|
||||
"""Fills call_queue with _WorkItems from pending_work_items.
|
||||
|
||||
|
||||
class _ExecutorManagerThread(threading.Thread):
|
||||
"""Manages the communication between this process and the worker processes.
|
||||
|
||||
The manager is run in a local thread.
|
||||
This function never blocks.
|
||||
|
||||
Args:
|
||||
executor: A reference to the ProcessPoolExecutor that owns
|
||||
this thread. A weakref will be own by the manager as well as
|
||||
references to internal objects used to introspect the state of
|
||||
the executor.
|
||||
pending_work_items: A dict mapping work ids to _WorkItems e.g.
|
||||
{5: <_WorkItem...>, 6: <_WorkItem...>, ...}
|
||||
work_ids: A queue.Queue of work ids e.g. Queue([5, 6, ...]). Work ids
|
||||
are consumed and the corresponding _WorkItems from
|
||||
pending_work_items are transformed into _CallItems and put in
|
||||
call_queue.
|
||||
call_queue: A multiprocessing.Queue that will be filled with _CallItems
|
||||
derived from _WorkItems.
|
||||
"""
|
||||
while True:
|
||||
if call_queue.full():
|
||||
return
|
||||
try:
|
||||
work_id = work_ids.get(block=False)
|
||||
except queue.Empty:
|
||||
return
|
||||
else:
|
||||
work_item = pending_work_items[work_id]
|
||||
|
||||
def __init__(self, executor):
|
||||
# Store references to necessary internals of the executor.
|
||||
|
||||
# A _ThreadWakeup to allow waking up the queue_manager_thread from the
|
||||
# main Thread and avoid deadlocks caused by permanently locked queues.
|
||||
self.thread_wakeup = executor._executor_manager_thread_wakeup
|
||||
self.shutdown_lock = executor._shutdown_lock
|
||||
|
||||
# A weakref.ref to the ProcessPoolExecutor that owns this thread. Used
|
||||
# to determine if the ProcessPoolExecutor has been garbage collected
|
||||
# and that the manager can exit.
|
||||
# When the executor gets garbage collected, the weakref callback
|
||||
# will wake up the queue management thread so that it can terminate
|
||||
# if there is no pending work item.
|
||||
def weakref_cb(_,
|
||||
thread_wakeup=self.thread_wakeup,
|
||||
shutdown_lock=self.shutdown_lock):
|
||||
mp.util.debug('Executor collected: triggering callback for'
|
||||
' QueueManager wakeup')
|
||||
with shutdown_lock:
|
||||
thread_wakeup.wakeup()
|
||||
|
||||
self.executor_reference = weakref.ref(executor, weakref_cb)
|
||||
|
||||
# A list of the ctx.Process instances used as workers.
|
||||
self.processes = executor._processes
|
||||
|
||||
# A ctx.Queue that will be filled with _CallItems derived from
|
||||
# _WorkItems for processing by the process workers.
|
||||
self.call_queue = executor._call_queue
|
||||
|
||||
# A ctx.SimpleQueue of _ResultItems generated by the process workers.
|
||||
self.result_queue = executor._result_queue
|
||||
|
||||
# A queue.Queue of work ids e.g. Queue([5, 6, ...]).
|
||||
self.work_ids_queue = executor._work_ids
|
||||
|
||||
# A dict mapping work ids to _WorkItems e.g.
|
||||
# {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
|
||||
self.pending_work_items = executor._pending_work_items
|
||||
|
||||
super().__init__()
|
||||
|
||||
def run(self):
|
||||
# Main loop for the executor manager thread.
|
||||
|
||||
while True:
|
||||
self.add_call_item_to_queue()
|
||||
|
||||
result_item, is_broken, cause = self.wait_result_broken_or_wakeup()
|
||||
|
||||
if is_broken:
|
||||
self.terminate_broken(cause)
|
||||
return
|
||||
if result_item is not None:
|
||||
self.process_result_item(result_item)
|
||||
# Delete reference to result_item to avoid keeping references
|
||||
# while waiting on new results.
|
||||
del result_item
|
||||
|
||||
# attempt to increment idle process count
|
||||
executor = self.executor_reference()
|
||||
if executor is not None:
|
||||
executor._idle_worker_semaphore.release()
|
||||
del executor
|
||||
|
||||
if self.is_shutting_down():
|
||||
self.flag_executor_shutting_down()
|
||||
|
||||
# Since no new work items can be added, it is safe to shutdown
|
||||
# this thread if there are no pending work items.
|
||||
if not self.pending_work_items:
|
||||
self.join_executor_internals()
|
||||
return
|
||||
|
||||
def add_call_item_to_queue(self):
|
||||
# Fills call_queue with _WorkItems from pending_work_items.
|
||||
# This function never blocks.
|
||||
while True:
|
||||
if self.call_queue.full():
|
||||
return
|
||||
try:
|
||||
work_id = self.work_ids_queue.get(block=False)
|
||||
except queue.Empty:
|
||||
return
|
||||
if work_item.future.set_running_or_notify_cancel():
|
||||
call_queue.put(_CallItem(work_id,
|
||||
work_item.fn,
|
||||
work_item.args,
|
||||
work_item.kwargs),
|
||||
block=True)
|
||||
else:
|
||||
work_item = self.pending_work_items[work_id]
|
||||
del pending_work_items[work_id]
|
||||
continue
|
||||
|
||||
if work_item.future.set_running_or_notify_cancel():
|
||||
self.call_queue.put(_CallItem(work_id,
|
||||
work_item.fn,
|
||||
work_item.args,
|
||||
work_item.kwargs),
|
||||
block=True)
|
||||
else:
|
||||
del self.pending_work_items[work_id]
|
||||
continue
|
||||
def _queue_management_worker(executor_reference,
|
||||
processes,
|
||||
pending_work_items,
|
||||
work_ids_queue,
|
||||
call_queue,
|
||||
result_queue):
|
||||
"""Manages the communication between this process and the worker processes.
|
||||
|
||||
def wait_result_broken_or_wakeup(self):
|
||||
# Wait for a result to be ready in the result_queue while checking
|
||||
# that all worker processes are still running, or for a wake up
|
||||
# signal send. The wake up signals come either from new tasks being
|
||||
# submitted, from the executor being shutdown/gc-ed, or from the
|
||||
# shutdown of the python interpreter.
|
||||
result_reader = self.result_queue._reader
|
||||
assert not self.thread_wakeup._closed
|
||||
wakeup_reader = self.thread_wakeup._reader
|
||||
readers = [result_reader, wakeup_reader]
|
||||
worker_sentinels = [p.sentinel for p in list(self.processes.values())]
|
||||
ready = mp.connection.wait(readers + worker_sentinels)
|
||||
This function is run in a local thread.
|
||||
|
||||
cause = None
|
||||
is_broken = True
|
||||
result_item = None
|
||||
if result_reader in ready:
|
||||
try:
|
||||
result_item = result_reader.recv()
|
||||
is_broken = False
|
||||
except BaseException as e:
|
||||
cause = traceback.format_exception(type(e), e, e.__traceback__)
|
||||
Args:
|
||||
executor_reference: A weakref.ref to the ProcessPoolExecutor that owns
|
||||
this thread. Used to determine if the ProcessPoolExecutor has been
|
||||
garbage collected and that this function can exit.
|
||||
process: A list of the multiprocessing.Process instances used as
|
||||
workers.
|
||||
pending_work_items: A dict mapping work ids to _WorkItems e.g.
|
||||
{5: <_WorkItem...>, 6: <_WorkItem...>, ...}
|
||||
work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]).
|
||||
call_queue: A multiprocessing.Queue that will be filled with _CallItems
|
||||
derived from _WorkItems for processing by the process workers.
|
||||
result_queue: A multiprocessing.Queue of _ResultItems generated by the
|
||||
process workers.
|
||||
"""
|
||||
executor = None
|
||||
|
||||
elif wakeup_reader in ready:
|
||||
is_broken = False
|
||||
def shutting_down():
|
||||
return _shutdown or executor is None or executor._shutdown_thread
|
||||
|
||||
with self.shutdown_lock:
|
||||
self.thread_wakeup.clear()
|
||||
def shutdown_worker():
|
||||
# This is an upper bound
|
||||
nb_children_alive = sum(p.is_alive() for p in processes.values())
|
||||
for i in range(0, nb_children_alive):
|
||||
call_queue.put_nowait(None)
|
||||
# Release the queue's resources as soon as possible.
|
||||
call_queue.close()
|
||||
# If .join() is not called on the created processes then
|
||||
# some multiprocessing.Queue methods may deadlock on Mac OS X.
|
||||
for p in processes.values():
|
||||
p.join()
|
||||
|
||||
return result_item, is_broken, cause
|
||||
reader = result_queue._reader
|
||||
|
||||
def process_result_item(self, result_item):
|
||||
# Process the received a result_item. This can be either the PID of a
|
||||
# worker that exited gracefully or a _ResultItem
|
||||
while True:
|
||||
_add_call_item_to_queue(pending_work_items,
|
||||
work_ids_queue,
|
||||
call_queue)
|
||||
|
||||
sentinels = [p.sentinel for p in processes.values()]
|
||||
assert sentinels
|
||||
ready = wait([reader] + sentinels)
|
||||
if reader in ready:
|
||||
result_item = reader.recv()
|
||||
else:
|
||||
# Mark the process pool broken so that submits fail right now.
|
||||
executor = executor_reference()
|
||||
if executor is not None:
|
||||
executor._broken = True
|
||||
executor._shutdown_thread = True
|
||||
executor = None
|
||||
# All futures in flight must be marked failed
|
||||
for work_id, work_item in pending_work_items.items():
|
||||
work_item.future.set_exception(
|
||||
BrokenProcessPool(
|
||||
"A process in the process pool was "
|
||||
"terminated abruptly while the future was "
|
||||
"running or pending."
|
||||
))
|
||||
# Delete references to object. See issue16284
|
||||
del work_item
|
||||
pending_work_items.clear()
|
||||
# Terminate remaining workers forcibly: the queues or their
|
||||
# locks may be in a dirty state and block forever.
|
||||
for p in processes.values():
|
||||
p.terminate()
|
||||
shutdown_worker()
|
||||
return
|
||||
if isinstance(result_item, int):
|
||||
# Clean shutdown of a worker using its PID
|
||||
# (avoids marking the executor broken)
|
||||
assert self.is_shutting_down()
|
||||
p = self.processes.pop(result_item)
|
||||
assert shutting_down()
|
||||
p = processes.pop(result_item)
|
||||
p.join()
|
||||
if not self.processes:
|
||||
self.join_executor_internals()
|
||||
if not processes:
|
||||
shutdown_worker()
|
||||
return
|
||||
else:
|
||||
# Received a _ResultItem so mark the future as completed.
|
||||
work_item = self.pending_work_items.pop(result_item.work_id, None)
|
||||
elif result_item is not None:
|
||||
work_item = pending_work_items.pop(result_item.work_id, None)
|
||||
# work_item can be None if another process terminated (see above)
|
||||
if work_item is not None:
|
||||
if result_item.exception:
|
||||
work_item.future.set_exception(result_item.exception)
|
||||
else:
|
||||
work_item.future.set_result(result_item.result)
|
||||
|
||||
def is_shutting_down(self):
|
||||
# Check whether we should start shutting down the executor.
|
||||
executor = self.executor_reference()
|
||||
# Delete references to object. See issue16284
|
||||
del work_item
|
||||
# Check whether we should start shutting down.
|
||||
executor = executor_reference()
|
||||
# No more work items can be added if:
|
||||
# - The interpreter is shutting down OR
|
||||
# - The executor that owns this worker has been collected OR
|
||||
# - The executor that owns this worker has been shutdown.
|
||||
return (_global_shutdown or executor is None
|
||||
or executor._shutdown_thread)
|
||||
|
||||
def terminate_broken(self, cause):
|
||||
# Terminate the executor because it is in a broken state. The cause
|
||||
# argument can be used to display more information on the error that
|
||||
# lead the executor into becoming broken.
|
||||
|
||||
# Mark the process pool broken so that submits fail right now.
|
||||
executor = self.executor_reference()
|
||||
if executor is not None:
|
||||
executor._broken = ('A child process terminated '
|
||||
'abruptly, the process pool is not '
|
||||
'usable anymore')
|
||||
executor._shutdown_thread = True
|
||||
executor = None
|
||||
|
||||
# All pending tasks are to be marked failed with the following
|
||||
# BrokenProcessPool error
|
||||
bpe = BrokenProcessPool("A process in the process pool was "
|
||||
"terminated abruptly while the future was "
|
||||
"running or pending.")
|
||||
if cause is not None:
|
||||
bpe.__cause__ = _RemoteTraceback(
|
||||
f"\n'''\n{''.join(cause)}'''")
|
||||
|
||||
# Mark pending tasks as failed.
|
||||
for work_id, work_item in self.pending_work_items.items():
|
||||
work_item.future.set_exception(bpe)
|
||||
# Delete references to object. See issue16284
|
||||
del work_item
|
||||
self.pending_work_items.clear()
|
||||
|
||||
# Terminate remaining workers forcibly: the queues or their
|
||||
# locks may be in a dirty state and block forever.
|
||||
for p in self.processes.values():
|
||||
p.terminate()
|
||||
|
||||
# clean up resources
|
||||
self.join_executor_internals()
|
||||
|
||||
def flag_executor_shutting_down(self):
|
||||
# Flag the executor as shutting down and cancel remaining tasks if
|
||||
# requested as early as possible if it is not gc-ed yet.
|
||||
executor = self.executor_reference()
|
||||
if executor is not None:
|
||||
executor._shutdown_thread = True
|
||||
# Cancel pending work items if requested.
|
||||
if executor._cancel_pending_futures:
|
||||
# Cancel all pending futures and update pending_work_items
|
||||
# to only have futures that are currently running.
|
||||
new_pending_work_items = {}
|
||||
for work_id, work_item in self.pending_work_items.items():
|
||||
if not work_item.future.cancel():
|
||||
new_pending_work_items[work_id] = work_item
|
||||
self.pending_work_items = new_pending_work_items
|
||||
# Drain work_ids_queue since we no longer need to
|
||||
# add items to the call queue.
|
||||
while True:
|
||||
try:
|
||||
self.work_ids_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
# Make sure we do this only once to not waste time looping
|
||||
# on running processes over and over.
|
||||
executor._cancel_pending_futures = False
|
||||
|
||||
def shutdown_workers(self):
|
||||
n_children_to_stop = self.get_n_children_alive()
|
||||
n_sentinels_sent = 0
|
||||
# Send the right number of sentinels, to make sure all children are
|
||||
# properly terminated.
|
||||
while (n_sentinels_sent < n_children_to_stop
|
||||
and self.get_n_children_alive() > 0):
|
||||
for i in range(n_children_to_stop - n_sentinels_sent):
|
||||
try:
|
||||
self.call_queue.put_nowait(None)
|
||||
n_sentinels_sent += 1
|
||||
except queue.Full:
|
||||
break
|
||||
|
||||
def join_executor_internals(self):
|
||||
self.shutdown_workers()
|
||||
# Release the queue's resources as soon as possible.
|
||||
self.call_queue.close()
|
||||
self.call_queue.join_thread()
|
||||
with self.shutdown_lock:
|
||||
self.thread_wakeup.close()
|
||||
# If .join() is not called on the created processes then
|
||||
# some ctx.Queue methods may deadlock on Mac OS X.
|
||||
for p in self.processes.values():
|
||||
p.join()
|
||||
|
||||
def get_n_children_alive(self):
|
||||
# This is an upper bound on the number of children alive.
|
||||
return sum(p.is_alive() for p in self.processes.values())
|
||||
|
||||
if shutting_down():
|
||||
try:
|
||||
# Since no new work items can be added, it is safe to shutdown
|
||||
# this thread if there are no pending work items.
|
||||
if not pending_work_items:
|
||||
shutdown_worker()
|
||||
return
|
||||
except Full:
|
||||
# This is not a problem: we will eventually be woken up (in
|
||||
# result_queue.get()) and be able to send a sentinel again.
|
||||
pass
|
||||
executor = None
|
||||
|
||||
_system_limits_checked = False
|
||||
_system_limited = None
|
||||
|
||||
|
||||
def _check_system_limits():
|
||||
global _system_limits_checked, _system_limited
|
||||
if _system_limits_checked:
|
||||
if _system_limited:
|
||||
raise NotImplementedError(_system_limited)
|
||||
_system_limits_checked = True
|
||||
try:
|
||||
import multiprocessing.synchronize
|
||||
except ImportError:
|
||||
_system_limited = (
|
||||
"This Python build lacks multiprocessing.synchronize, usually due "
|
||||
"to named semaphores being unavailable on this platform."
|
||||
)
|
||||
raise NotImplementedError(_system_limited)
|
||||
try:
|
||||
nsems_max = os.sysconf("SC_SEM_NSEMS_MAX")
|
||||
except (AttributeError, ValueError):
|
||||
@@ -556,24 +353,11 @@ def _check_system_limits():
|
||||
# minimum number of semaphores available
|
||||
# according to POSIX
|
||||
return
|
||||
_system_limited = ("system provides too few semaphores (%d"
|
||||
" available, 256 necessary)" % nsems_max)
|
||||
_system_limited = "system provides too few semaphores (%d available, 256 necessary)" % nsems_max
|
||||
raise NotImplementedError(_system_limited)
|
||||
|
||||
|
||||
def _chain_from_iterable_of_lists(iterable):
|
||||
"""
|
||||
Specialized implementation of itertools.chain.from_iterable.
|
||||
Each item in *iterable* should be a list. This function is
|
||||
careful not to keep references to yielded objects.
|
||||
"""
|
||||
for element in iterable:
|
||||
element.reverse()
|
||||
while element:
|
||||
yield element.pop()
|
||||
|
||||
|
||||
class BrokenProcessPool(_base.BrokenExecutor):
|
||||
class BrokenProcessPool(RuntimeError):
|
||||
"""
|
||||
Raised when a process in a ProcessPoolExecutor terminated abruptly
|
||||
while a future was in the running state.
|
||||
@@ -581,143 +365,82 @@ class BrokenProcessPool(_base.BrokenExecutor):
|
||||
|
||||
|
||||
class ProcessPoolExecutor(_base.Executor):
|
||||
def __init__(self, max_workers=None, mp_context=None,
|
||||
initializer=None, initargs=()):
|
||||
def __init__(self, max_workers=None):
|
||||
"""Initializes a new ProcessPoolExecutor instance.
|
||||
|
||||
Args:
|
||||
max_workers: The maximum number of processes that can be used to
|
||||
execute the given calls. If None or not given then as many
|
||||
worker processes will be created as the machine has processors.
|
||||
mp_context: A multiprocessing context to launch the workers. This
|
||||
object should provide SimpleQueue, Queue and Process.
|
||||
initializer: A callable used to initialize worker processes.
|
||||
initargs: A tuple of arguments to pass to the initializer.
|
||||
"""
|
||||
_check_system_limits()
|
||||
|
||||
if max_workers is None:
|
||||
self._max_workers = os.cpu_count() or 1
|
||||
if sys.platform == 'win32':
|
||||
self._max_workers = min(_MAX_WINDOWS_WORKERS,
|
||||
self._max_workers)
|
||||
else:
|
||||
if max_workers <= 0:
|
||||
raise ValueError("max_workers must be greater than 0")
|
||||
elif (sys.platform == 'win32' and
|
||||
max_workers > _MAX_WINDOWS_WORKERS):
|
||||
raise ValueError(
|
||||
f"max_workers must be <= {_MAX_WINDOWS_WORKERS}")
|
||||
|
||||
self._max_workers = max_workers
|
||||
|
||||
if mp_context is None:
|
||||
mp_context = mp.get_context()
|
||||
self._mp_context = mp_context
|
||||
|
||||
# https://github.com/python/cpython/issues/90622
|
||||
self._safe_to_dynamically_spawn_children = (
|
||||
self._mp_context.get_start_method(allow_none=False) != "fork")
|
||||
|
||||
if initializer is not None and not callable(initializer):
|
||||
raise TypeError("initializer must be a callable")
|
||||
self._initializer = initializer
|
||||
self._initargs = initargs
|
||||
|
||||
# Management thread
|
||||
self._executor_manager_thread = None
|
||||
|
||||
# Make the call queue slightly larger than the number of processes to
|
||||
# prevent the worker processes from idling. But don't make it too big
|
||||
# because futures in the call queue cannot be cancelled.
|
||||
self._call_queue = multiprocessing.Queue(self._max_workers +
|
||||
EXTRA_QUEUED_CALLS)
|
||||
# Killed worker processes can produce spurious "broken pipe"
|
||||
# tracebacks in the queue's own worker thread. But we detect killed
|
||||
# processes anyway, so silence the tracebacks.
|
||||
self._call_queue._ignore_epipe = True
|
||||
self._result_queue = SimpleQueue()
|
||||
self._work_ids = queue.Queue()
|
||||
self._queue_management_thread = None
|
||||
# Map of pids to processes
|
||||
self._processes = {}
|
||||
|
||||
# Shutdown is a two-step process.
|
||||
self._shutdown_thread = False
|
||||
self._shutdown_lock = threading.Lock()
|
||||
self._idle_worker_semaphore = threading.Semaphore(0)
|
||||
self._broken = False
|
||||
self._queue_count = 0
|
||||
self._pending_work_items = {}
|
||||
self._cancel_pending_futures = False
|
||||
|
||||
# _ThreadWakeup is a communication channel used to interrupt the wait
|
||||
# of the main loop of executor_manager_thread from another thread (e.g.
|
||||
# when calling executor.submit or executor.shutdown). We do not use the
|
||||
# _result_queue to send wakeup signals to the executor_manager_thread
|
||||
# as it could result in a deadlock if a worker process dies with the
|
||||
# _result_queue write lock still acquired.
|
||||
#
|
||||
# _shutdown_lock must be locked to access _ThreadWakeup.
|
||||
self._executor_manager_thread_wakeup = _ThreadWakeup()
|
||||
|
||||
# Create communication channels for the executor
|
||||
# Make the call queue slightly larger than the number of processes to
|
||||
# prevent the worker processes from idling. But don't make it too big
|
||||
# because futures in the call queue cannot be cancelled.
|
||||
queue_size = self._max_workers + EXTRA_QUEUED_CALLS
|
||||
self._call_queue = _SafeQueue(
|
||||
max_size=queue_size, ctx=self._mp_context,
|
||||
pending_work_items=self._pending_work_items,
|
||||
shutdown_lock=self._shutdown_lock,
|
||||
thread_wakeup=self._executor_manager_thread_wakeup)
|
||||
# Killed worker processes can produce spurious "broken pipe"
|
||||
# tracebacks in the queue's own worker thread. But we detect killed
|
||||
# processes anyway, so silence the tracebacks.
|
||||
self._call_queue._ignore_epipe = True
|
||||
self._result_queue = mp_context.SimpleQueue()
|
||||
self._work_ids = queue.Queue()
|
||||
|
||||
def _start_executor_manager_thread(self):
|
||||
if self._executor_manager_thread is None:
|
||||
def _start_queue_management_thread(self):
|
||||
# When the executor gets lost, the weakref callback will wake up
|
||||
# the queue management thread.
|
||||
def weakref_cb(_, q=self._result_queue):
|
||||
q.put(None)
|
||||
if self._queue_management_thread is None:
|
||||
# Start the processes so that their sentinels are known.
|
||||
if not self._safe_to_dynamically_spawn_children: # ie, using fork.
|
||||
self._launch_processes()
|
||||
self._executor_manager_thread = _ExecutorManagerThread(self)
|
||||
self._executor_manager_thread.start()
|
||||
_threads_wakeups[self._executor_manager_thread] = \
|
||||
self._executor_manager_thread_wakeup
|
||||
self._adjust_process_count()
|
||||
self._queue_management_thread = threading.Thread(
|
||||
target=_queue_management_worker,
|
||||
args=(weakref.ref(self, weakref_cb),
|
||||
self._processes,
|
||||
self._pending_work_items,
|
||||
self._work_ids,
|
||||
self._call_queue,
|
||||
self._result_queue))
|
||||
self._queue_management_thread.daemon = True
|
||||
self._queue_management_thread.start()
|
||||
_threads_queues[self._queue_management_thread] = self._result_queue
|
||||
|
||||
def _adjust_process_count(self):
|
||||
# if there's an idle process, we don't need to spawn a new one.
|
||||
if self._idle_worker_semaphore.acquire(blocking=False):
|
||||
return
|
||||
|
||||
process_count = len(self._processes)
|
||||
if process_count < self._max_workers:
|
||||
# Assertion disabled as this codepath is also used to replace a
|
||||
# worker that unexpectedly dies, even when using the 'fork' start
|
||||
# method. That means there is still a potential deadlock bug. If a
|
||||
# 'fork' mp_context worker dies, we'll be forking a new one when
|
||||
# we know a thread is running (self._executor_manager_thread).
|
||||
#assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622'
|
||||
self._spawn_process()
|
||||
|
||||
def _launch_processes(self):
|
||||
# https://github.com/python/cpython/issues/90622
|
||||
assert not self._executor_manager_thread, (
|
||||
'Processes cannot be fork()ed after the thread has started, '
|
||||
'deadlock in the child processes could result.')
|
||||
for _ in range(len(self._processes), self._max_workers):
|
||||
self._spawn_process()
|
||||
p = multiprocessing.Process(
|
||||
target=_process_worker,
|
||||
args=(self._call_queue,
|
||||
self._result_queue))
|
||||
p.start()
|
||||
self._processes[p.pid] = p
|
||||
|
||||
def _spawn_process(self):
|
||||
p = self._mp_context.Process(
|
||||
target=_process_worker,
|
||||
args=(self._call_queue,
|
||||
self._result_queue,
|
||||
self._initializer,
|
||||
self._initargs))
|
||||
p.start()
|
||||
self._processes[p.pid] = p
|
||||
|
||||
def submit(self, fn, /, *args, **kwargs):
|
||||
def submit(self, fn, *args, **kwargs):
|
||||
with self._shutdown_lock:
|
||||
if self._broken:
|
||||
raise BrokenProcessPool(self._broken)
|
||||
raise BrokenProcessPool('A child process terminated '
|
||||
'abruptly, the process pool is not usable anymore')
|
||||
if self._shutdown_thread:
|
||||
raise RuntimeError('cannot schedule new futures after shutdown')
|
||||
if _global_shutdown:
|
||||
raise RuntimeError('cannot schedule new futures after '
|
||||
'interpreter shutdown')
|
||||
|
||||
f = _base.Future()
|
||||
w = _WorkItem(f, fn, args, kwargs)
|
||||
@@ -726,11 +449,9 @@ class ProcessPoolExecutor(_base.Executor):
|
||||
self._work_ids.put(self._queue_count)
|
||||
self._queue_count += 1
|
||||
# Wake up queue management thread
|
||||
self._executor_manager_thread_wakeup.wakeup()
|
||||
self._result_queue.put(None)
|
||||
|
||||
if self._safe_to_dynamically_spawn_children:
|
||||
self._adjust_process_count()
|
||||
self._start_executor_manager_thread()
|
||||
self._start_queue_management_thread()
|
||||
return f
|
||||
submit.__doc__ = _base.Executor.submit.__doc__
|
||||
|
||||
@@ -761,26 +482,22 @@ class ProcessPoolExecutor(_base.Executor):
|
||||
results = super().map(partial(_process_chunk, fn),
|
||||
_get_chunks(*iterables, chunksize=chunksize),
|
||||
timeout=timeout)
|
||||
return _chain_from_iterable_of_lists(results)
|
||||
return itertools.chain.from_iterable(results)
|
||||
|
||||
def shutdown(self, wait=True, *, cancel_futures=False):
|
||||
def shutdown(self, wait=True):
|
||||
with self._shutdown_lock:
|
||||
self._cancel_pending_futures = cancel_futures
|
||||
self._shutdown_thread = True
|
||||
if self._executor_manager_thread_wakeup is not None:
|
||||
# Wake up queue management thread
|
||||
self._executor_manager_thread_wakeup.wakeup()
|
||||
|
||||
if self._executor_manager_thread is not None and wait:
|
||||
self._executor_manager_thread.join()
|
||||
if self._queue_management_thread:
|
||||
# Wake up queue management thread
|
||||
self._result_queue.put(None)
|
||||
if wait:
|
||||
self._queue_management_thread.join()
|
||||
# To reduce the risk of opening too many files, remove references to
|
||||
# objects that use file descriptors.
|
||||
self._executor_manager_thread = None
|
||||
self._queue_management_thread = None
|
||||
self._call_queue = None
|
||||
if self._result_queue is not None and wait:
|
||||
self._result_queue.close()
|
||||
self._result_queue = None
|
||||
self._processes = None
|
||||
self._executor_manager_thread_wakeup = None
|
||||
|
||||
shutdown.__doc__ = _base.Executor.shutdown.__doc__
|
||||
|
||||
atexit.register(_python_exit)
|
||||
|
||||
@@ -5,44 +5,40 @@
|
||||
|
||||
__author__ = 'Brian Quinlan (brian@sweetapp.com)'
|
||||
|
||||
import atexit
|
||||
from concurrent.futures import _base
|
||||
import itertools
|
||||
import queue
|
||||
import threading
|
||||
import types
|
||||
import weakref
|
||||
import os
|
||||
|
||||
# Workers are created as daemon threads. This is done to allow the interpreter
|
||||
# to exit when there are still idle threads in a ThreadPoolExecutor's thread
|
||||
# pool (i.e. shutdown() was not called). However, allowing workers to die with
|
||||
# the interpreter has two undesirable properties:
|
||||
# - The workers would still be running during interpreter shutdown,
|
||||
# meaning that they would fail in unpredictable ways.
|
||||
# - The workers could be killed while evaluating a work item, which could
|
||||
# be bad if the callable being evaluated has external side-effects e.g.
|
||||
# writing to a file.
|
||||
#
|
||||
# To work around this problem, an exit handler is installed which tells the
|
||||
# workers to exit when their work queues are empty and then waits until the
|
||||
# threads finish.
|
||||
|
||||
_threads_queues = weakref.WeakKeyDictionary()
|
||||
_shutdown = False
|
||||
# Lock that ensures that new workers are not created while the interpreter is
|
||||
# shutting down. Must be held while mutating _threads_queues and _shutdown.
|
||||
_global_shutdown_lock = threading.Lock()
|
||||
|
||||
def _python_exit():
|
||||
global _shutdown
|
||||
with _global_shutdown_lock:
|
||||
_shutdown = True
|
||||
_shutdown = True
|
||||
items = list(_threads_queues.items())
|
||||
for t, q in items:
|
||||
q.put(None)
|
||||
for t, q in items:
|
||||
t.join()
|
||||
|
||||
# Register for `_python_exit()` to be called just before joining all
|
||||
# non-daemon threads. This is used instead of `atexit.register()` for
|
||||
# compatibility with subinterpreters, which no longer support daemon threads.
|
||||
# See bpo-39812 for context.
|
||||
threading._register_atexit(_python_exit)
|
||||
|
||||
# At fork, reinitialize the `_global_shutdown_lock` lock in the child process
|
||||
# TODO RUSTPYTHON - _at_fork_reinit is not implemented yet
|
||||
if hasattr(os, 'register_at_fork') and hasattr(_global_shutdown_lock, '_at_fork_reinit'):
|
||||
os.register_at_fork(before=_global_shutdown_lock.acquire,
|
||||
after_in_child=_global_shutdown_lock._at_fork_reinit,
|
||||
after_in_parent=_global_shutdown_lock.release)
|
||||
|
||||
atexit.register(_python_exit)
|
||||
|
||||
class _WorkItem(object):
|
||||
def __init__(self, future, fn, args, kwargs):
|
||||
@@ -57,26 +53,12 @@ class _WorkItem(object):
|
||||
|
||||
try:
|
||||
result = self.fn(*self.args, **self.kwargs)
|
||||
except BaseException as exc:
|
||||
self.future.set_exception(exc)
|
||||
# Break a reference cycle with the exception 'exc'
|
||||
self = None
|
||||
except BaseException as e:
|
||||
self.future.set_exception(e)
|
||||
else:
|
||||
self.future.set_result(result)
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
|
||||
def _worker(executor_reference, work_queue, initializer, initargs):
|
||||
if initializer is not None:
|
||||
try:
|
||||
initializer(*initargs)
|
||||
except BaseException:
|
||||
_base.LOGGER.critical('Exception in initializer:', exc_info=True)
|
||||
executor = executor_reference()
|
||||
if executor is not None:
|
||||
executor._initializer_failed()
|
||||
return
|
||||
def _worker(executor_reference, work_queue):
|
||||
try:
|
||||
while True:
|
||||
work_item = work_queue.get(block=True)
|
||||
@@ -84,24 +66,13 @@ def _worker(executor_reference, work_queue, initializer, initargs):
|
||||
work_item.run()
|
||||
# Delete references to object. See issue16284
|
||||
del work_item
|
||||
|
||||
# attempt to increment idle count
|
||||
executor = executor_reference()
|
||||
if executor is not None:
|
||||
executor._idle_semaphore.release()
|
||||
del executor
|
||||
continue
|
||||
|
||||
executor = executor_reference()
|
||||
# Exit if:
|
||||
# - The interpreter is shutting down OR
|
||||
# - The executor that owns the worker has been collected OR
|
||||
# - The executor that owns the worker has been shutdown.
|
||||
if _shutdown or executor is None or executor._shutdown:
|
||||
# Flag the executor as shutting down as early as possible if it
|
||||
# is not gc-ed yet.
|
||||
if executor is not None:
|
||||
executor._shutdown = True
|
||||
# Notice other workers
|
||||
work_queue.put(None)
|
||||
return
|
||||
@@ -109,66 +80,33 @@ def _worker(executor_reference, work_queue, initializer, initargs):
|
||||
except BaseException:
|
||||
_base.LOGGER.critical('Exception in worker', exc_info=True)
|
||||
|
||||
|
||||
class BrokenThreadPool(_base.BrokenExecutor):
|
||||
"""
|
||||
Raised when a worker thread in a ThreadPoolExecutor failed initializing.
|
||||
"""
|
||||
|
||||
|
||||
class ThreadPoolExecutor(_base.Executor):
|
||||
|
||||
# Used to assign unique thread names when thread_name_prefix is not supplied.
|
||||
_counter = itertools.count().__next__
|
||||
|
||||
def __init__(self, max_workers=None, thread_name_prefix='',
|
||||
initializer=None, initargs=()):
|
||||
def __init__(self, max_workers=None, thread_name_prefix=''):
|
||||
"""Initializes a new ThreadPoolExecutor instance.
|
||||
|
||||
Args:
|
||||
max_workers: The maximum number of threads that can be used to
|
||||
execute the given calls.
|
||||
thread_name_prefix: An optional name prefix to give our threads.
|
||||
initializer: A callable used to initialize worker threads.
|
||||
initargs: A tuple of arguments to pass to the initializer.
|
||||
"""
|
||||
if max_workers is None:
|
||||
# ThreadPoolExecutor is often used to:
|
||||
# * CPU bound task which releases GIL
|
||||
# * I/O bound task (which releases GIL, of course)
|
||||
#
|
||||
# We use cpu_count + 4 for both types of tasks.
|
||||
# But we limit it to 32 to avoid consuming surprisingly large resource
|
||||
# on many core machine.
|
||||
max_workers = min(32, (os.cpu_count() or 1) + 4)
|
||||
# Use this number because ThreadPoolExecutor is often
|
||||
# used to overlap I/O instead of CPU work.
|
||||
max_workers = (os.cpu_count() or 1) * 5
|
||||
if max_workers <= 0:
|
||||
raise ValueError("max_workers must be greater than 0")
|
||||
|
||||
if initializer is not None and not callable(initializer):
|
||||
raise TypeError("initializer must be a callable")
|
||||
|
||||
self._max_workers = max_workers
|
||||
self._work_queue = queue.SimpleQueue()
|
||||
self._idle_semaphore = threading.Semaphore(0)
|
||||
self._work_queue = queue.Queue()
|
||||
self._threads = set()
|
||||
self._broken = False
|
||||
self._shutdown = False
|
||||
self._shutdown_lock = threading.Lock()
|
||||
self._thread_name_prefix = (thread_name_prefix or
|
||||
("ThreadPoolExecutor-%d" % self._counter()))
|
||||
self._initializer = initializer
|
||||
self._initargs = initargs
|
||||
|
||||
def submit(self, fn, /, *args, **kwargs):
|
||||
with self._shutdown_lock, _global_shutdown_lock:
|
||||
if self._broken:
|
||||
raise BrokenThreadPool(self._broken)
|
||||
self._thread_name_prefix = thread_name_prefix
|
||||
|
||||
def submit(self, fn, *args, **kwargs):
|
||||
with self._shutdown_lock:
|
||||
if self._shutdown:
|
||||
raise RuntimeError('cannot schedule new futures after shutdown')
|
||||
if _shutdown:
|
||||
raise RuntimeError('cannot schedule new futures after '
|
||||
'interpreter shutdown')
|
||||
|
||||
f = _base.Future()
|
||||
w = _WorkItem(f, fn, args, kwargs)
|
||||
@@ -179,57 +117,27 @@ class ThreadPoolExecutor(_base.Executor):
|
||||
submit.__doc__ = _base.Executor.submit.__doc__
|
||||
|
||||
def _adjust_thread_count(self):
|
||||
# if idle threads are available, don't spin new threads
|
||||
if self._idle_semaphore.acquire(timeout=0):
|
||||
return
|
||||
|
||||
# When the executor gets lost, the weakref callback will wake up
|
||||
# the worker threads.
|
||||
def weakref_cb(_, q=self._work_queue):
|
||||
q.put(None)
|
||||
|
||||
# TODO(bquinlan): Should avoid creating new threads if there are more
|
||||
# idle threads than items in the work queue.
|
||||
num_threads = len(self._threads)
|
||||
if num_threads < self._max_workers:
|
||||
thread_name = '%s_%d' % (self._thread_name_prefix or self,
|
||||
num_threads)
|
||||
t = threading.Thread(name=thread_name, target=_worker,
|
||||
args=(weakref.ref(self, weakref_cb),
|
||||
self._work_queue,
|
||||
self._initializer,
|
||||
self._initargs))
|
||||
self._work_queue))
|
||||
t.daemon = True
|
||||
t.start()
|
||||
self._threads.add(t)
|
||||
_threads_queues[t] = self._work_queue
|
||||
|
||||
def _initializer_failed(self):
|
||||
with self._shutdown_lock:
|
||||
self._broken = ('A thread initializer failed, the thread pool '
|
||||
'is not usable anymore')
|
||||
# Drain work queue and mark pending futures failed
|
||||
while True:
|
||||
try:
|
||||
work_item = self._work_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
if work_item is not None:
|
||||
work_item.future.set_exception(BrokenThreadPool(self._broken))
|
||||
|
||||
def shutdown(self, wait=True, *, cancel_futures=False):
|
||||
def shutdown(self, wait=True):
|
||||
with self._shutdown_lock:
|
||||
self._shutdown = True
|
||||
if cancel_futures:
|
||||
# Drain all work items from the queue, and then cancel their
|
||||
# associated futures.
|
||||
while True:
|
||||
try:
|
||||
work_item = self._work_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
if work_item is not None:
|
||||
work_item.future.cancel()
|
||||
|
||||
# Send a wake-up to prevent threads calling
|
||||
# _work_queue.get(block=True) from permanently blocking.
|
||||
self._work_queue.put(None)
|
||||
if wait:
|
||||
for t in self._threads:
|
||||
|
||||
204
Lib/configparser.py
vendored
204
Lib/configparser.py
vendored
@@ -19,37 +19,36 @@ ConfigParser -- responsible for parsing a list of
|
||||
inline_comment_prefixes=None, strict=True,
|
||||
empty_lines_in_values=True, default_section='DEFAULT',
|
||||
interpolation=<unset>, converters=<unset>):
|
||||
|
||||
Create the parser. When `defaults` is given, it is initialized into the
|
||||
Create the parser. When `defaults' is given, it is initialized into the
|
||||
dictionary or intrinsic defaults. The keys must be strings, the values
|
||||
must be appropriate for %()s string interpolation.
|
||||
|
||||
When `dict_type` is given, it will be used to create the dictionary
|
||||
When `dict_type' is given, it will be used to create the dictionary
|
||||
objects for the list of sections, for the options within a section, and
|
||||
for the default values.
|
||||
|
||||
When `delimiters` is given, it will be used as the set of substrings
|
||||
When `delimiters' is given, it will be used as the set of substrings
|
||||
that divide keys from values.
|
||||
|
||||
When `comment_prefixes` is given, it will be used as the set of
|
||||
When `comment_prefixes' is given, it will be used as the set of
|
||||
substrings that prefix comments in empty lines. Comments can be
|
||||
indented.
|
||||
|
||||
When `inline_comment_prefixes` is given, it will be used as the set of
|
||||
When `inline_comment_prefixes' is given, it will be used as the set of
|
||||
substrings that prefix comments in non-empty lines.
|
||||
|
||||
When `strict` is True, the parser won't allow for any section or option
|
||||
duplicates while reading from a single source (file, string or
|
||||
dictionary). Default is True.
|
||||
|
||||
When `empty_lines_in_values` is False (default: True), each empty line
|
||||
When `empty_lines_in_values' is False (default: True), each empty line
|
||||
marks the end of an option. Otherwise, internal empty lines of
|
||||
a multiline option are kept as part of the value.
|
||||
|
||||
When `allow_no_value` is True (default: False), options without
|
||||
When `allow_no_value' is True (default: False), options without
|
||||
values are accepted; the value presented for these is None.
|
||||
|
||||
When `default_section` is given, the name of the special section is
|
||||
When `default_section' is given, the name of the special section is
|
||||
named accordingly. By default it is called ``"DEFAULT"`` but this can
|
||||
be customized to point to any other valid section name. Its current
|
||||
value can be retrieved using the ``parser_instance.default_section``
|
||||
@@ -57,9 +56,9 @@ ConfigParser -- responsible for parsing a list of
|
||||
|
||||
When `interpolation` is given, it should be an Interpolation subclass
|
||||
instance. It will be used as the handler for option value
|
||||
pre-processing when using getters. RawConfigParser objects don't do
|
||||
pre-processing when using getters. RawConfigParser object s don't do
|
||||
any sort of interpolation, whereas ConfigParser uses an instance of
|
||||
BasicInterpolation. The library also provides a ``zc.buildout``
|
||||
BasicInterpolation. The library also provides a ``zc.buildbot``
|
||||
inspired ExtendedInterpolation implementation.
|
||||
|
||||
When `converters` is given, it should be a dictionary where each key
|
||||
@@ -81,14 +80,14 @@ ConfigParser -- responsible for parsing a list of
|
||||
Return list of configuration options for the named section.
|
||||
|
||||
read(filenames, encoding=None)
|
||||
Read and parse the iterable of named configuration files, given by
|
||||
Read and parse the list of named configuration files, given by
|
||||
name. A single filename is also allowed. Non-existing files
|
||||
are ignored. Return list of successfully read files.
|
||||
|
||||
read_file(f, filename=None)
|
||||
Read and parse one configuration file, given as a file object.
|
||||
The filename defaults to f.name; it is only used in error
|
||||
messages (if f has no `name` attribute, the string `<???>` is used).
|
||||
messages (if f has no `name' attribute, the string `<???>' is used).
|
||||
|
||||
read_string(string)
|
||||
Read configuration from a given string.
|
||||
@@ -104,9 +103,9 @@ ConfigParser -- responsible for parsing a list of
|
||||
Return a string value for the named option. All % interpolations are
|
||||
expanded in the return values, based on the defaults passed into the
|
||||
constructor and the DEFAULT section. Additional substitutions may be
|
||||
provided using the `vars` argument, which must be a dictionary whose
|
||||
contents override any pre-existing defaults. If `option` is a key in
|
||||
`vars`, the value from `vars` is used.
|
||||
provided using the `vars' argument, which must be a dictionary whose
|
||||
contents override any pre-existing defaults. If `option' is a key in
|
||||
`vars', the value from `vars' is used.
|
||||
|
||||
getint(section, options, raw=False, vars=None, fallback=_UNSET)
|
||||
Like get(), but convert value to an integer.
|
||||
@@ -135,30 +134,28 @@ ConfigParser -- responsible for parsing a list of
|
||||
|
||||
write(fp, space_around_delimiters=True)
|
||||
Write the configuration state in .ini format. If
|
||||
`space_around_delimiters` is True (the default), delimiters
|
||||
`space_around_delimiters' is True (the default), delimiters
|
||||
between keys and values are surrounded by spaces.
|
||||
"""
|
||||
|
||||
from collections.abc import MutableMapping
|
||||
from collections import ChainMap as _ChainMap
|
||||
from collections import OrderedDict as _default_dict, ChainMap as _ChainMap
|
||||
import functools
|
||||
import io
|
||||
import itertools
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
__all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError",
|
||||
__all__ = ["NoSectionError", "DuplicateOptionError", "DuplicateSectionError",
|
||||
"NoOptionError", "InterpolationError", "InterpolationDepthError",
|
||||
"InterpolationMissingOptionError", "InterpolationSyntaxError",
|
||||
"ParsingError", "MissingSectionHeaderError",
|
||||
"ConfigParser", "RawConfigParser",
|
||||
"ConfigParser", "SafeConfigParser", "RawConfigParser",
|
||||
"Interpolation", "BasicInterpolation", "ExtendedInterpolation",
|
||||
"LegacyInterpolation", "SectionProxy", "ConverterMapping",
|
||||
"DEFAULTSECT", "MAX_INTERPOLATION_DEPTH")
|
||||
"DEFAULTSECT", "MAX_INTERPOLATION_DEPTH"]
|
||||
|
||||
_default_dict = dict
|
||||
DEFAULTSECT = "DEFAULT"
|
||||
|
||||
MAX_INTERPOLATION_DEPTH = 10
|
||||
@@ -298,12 +295,41 @@ class InterpolationDepthError(InterpolationError):
|
||||
class ParsingError(Error):
|
||||
"""Raised when a configuration file does not follow legal syntax."""
|
||||
|
||||
def __init__(self, source):
|
||||
super().__init__(f'Source contains parsing errors: {source!r}')
|
||||
def __init__(self, source=None, filename=None):
|
||||
# Exactly one of `source'/`filename' arguments has to be given.
|
||||
# `filename' kept for compatibility.
|
||||
if filename and source:
|
||||
raise ValueError("Cannot specify both `filename' and `source'. "
|
||||
"Use `source'.")
|
||||
elif not filename and not source:
|
||||
raise ValueError("Required argument `source' not given.")
|
||||
elif filename:
|
||||
source = filename
|
||||
Error.__init__(self, 'Source contains parsing errors: %r' % source)
|
||||
self.source = source
|
||||
self.errors = []
|
||||
self.args = (source, )
|
||||
|
||||
@property
|
||||
def filename(self):
|
||||
"""Deprecated, use `source'."""
|
||||
warnings.warn(
|
||||
"The 'filename' attribute will be removed in future versions. "
|
||||
"Use 'source' instead.",
|
||||
DeprecationWarning, stacklevel=2
|
||||
)
|
||||
return self.source
|
||||
|
||||
@filename.setter
|
||||
def filename(self, value):
|
||||
"""Deprecated, user `source'."""
|
||||
warnings.warn(
|
||||
"The 'filename' attribute will be removed in future versions. "
|
||||
"Use 'source' instead.",
|
||||
DeprecationWarning, stacklevel=2
|
||||
)
|
||||
self.source = value
|
||||
|
||||
def append(self, lineno, line):
|
||||
self.errors.append((lineno, line))
|
||||
self.message += '\n\t[line %2d]: %s' % (lineno, line)
|
||||
@@ -324,7 +350,7 @@ class MissingSectionHeaderError(ParsingError):
|
||||
|
||||
|
||||
# Used in parser getters to indicate the default behaviour when a specific
|
||||
# option is not found it to raise an exception. Created to enable `None` as
|
||||
# option is not found it to raise an exception. Created to enable `None' as
|
||||
# a valid fallback value.
|
||||
_UNSET = object()
|
||||
|
||||
@@ -358,7 +384,7 @@ class BasicInterpolation(Interpolation):
|
||||
would resolve the "%(dir)s" to the value of dir. All reference
|
||||
expansions are done late, on demand. If a user needs to use a bare % in
|
||||
a configuration file, she can escape it by writing %%. Other % usage
|
||||
is considered a user error and raises `InterpolationSyntaxError`."""
|
||||
is considered a user error and raises `InterpolationSyntaxError'."""
|
||||
|
||||
_KEYCRE = re.compile(r"%\(([^)]+)\)s")
|
||||
|
||||
@@ -419,7 +445,7 @@ class BasicInterpolation(Interpolation):
|
||||
|
||||
class ExtendedInterpolation(Interpolation):
|
||||
"""Advanced variant of interpolation, supports the syntax used by
|
||||
`zc.buildout`. Enables interpolation between sections."""
|
||||
`zc.buildout'. Enables interpolation between sections."""
|
||||
|
||||
_KEYCRE = re.compile(r"\$\{([^}]+)\}")
|
||||
|
||||
@@ -497,15 +523,6 @@ class LegacyInterpolation(Interpolation):
|
||||
|
||||
_KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
warnings.warn(
|
||||
"LegacyInterpolation has been deprecated since Python 3.2 "
|
||||
"and will be removed from the configparser module in Python 3.13. "
|
||||
"Use BasicInterpolation or ExtendedInterpolation instead.",
|
||||
DeprecationWarning, stacklevel=2
|
||||
)
|
||||
|
||||
def before_get(self, parser, section, option, value, vars):
|
||||
rawval = value
|
||||
depth = MAX_INTERPOLATION_DEPTH
|
||||
@@ -544,7 +561,7 @@ class RawConfigParser(MutableMapping):
|
||||
# Regular expressions for parsing section headers and options
|
||||
_SECT_TMPL = r"""
|
||||
\[ # [
|
||||
(?P<header>.+) # very permissive!
|
||||
(?P<header>[^]]+) # very permissive!
|
||||
\] # ]
|
||||
"""
|
||||
_OPT_TMPL = r"""
|
||||
@@ -592,6 +609,9 @@ class RawConfigParser(MutableMapping):
|
||||
self._converters = ConverterMapping(self)
|
||||
self._proxies = self._dict()
|
||||
self._proxies[default_section] = SectionProxy(self, default_section)
|
||||
if defaults:
|
||||
for key, value in defaults.items():
|
||||
self._defaults[self.optionxform(key)] = value
|
||||
self._delimiters = tuple(delimiters)
|
||||
if delimiters == ('=', ':'):
|
||||
self._optcre = self.OPTCRE_NV if allow_no_value else self.OPTCRE
|
||||
@@ -614,15 +634,8 @@ class RawConfigParser(MutableMapping):
|
||||
self._interpolation = self._DEFAULT_INTERPOLATION
|
||||
if self._interpolation is None:
|
||||
self._interpolation = Interpolation()
|
||||
if not isinstance(self._interpolation, Interpolation):
|
||||
raise TypeError(
|
||||
f"interpolation= must be None or an instance of Interpolation;"
|
||||
f" got an object of type {type(self._interpolation)}"
|
||||
)
|
||||
if converters is not _UNSET:
|
||||
self._converters.update(converters)
|
||||
if defaults:
|
||||
self._read_defaults(defaults)
|
||||
|
||||
def defaults(self):
|
||||
return self._defaults
|
||||
@@ -663,20 +676,19 @@ class RawConfigParser(MutableMapping):
|
||||
return list(opts.keys())
|
||||
|
||||
def read(self, filenames, encoding=None):
|
||||
"""Read and parse a filename or an iterable of filenames.
|
||||
"""Read and parse a filename or a list of filenames.
|
||||
|
||||
Files that cannot be opened are silently ignored; this is
|
||||
designed so that you can specify an iterable of potential
|
||||
designed so that you can specify a list of potential
|
||||
configuration file locations (e.g. current directory, user's
|
||||
home directory, systemwide directory), and all existing
|
||||
configuration files in the iterable will be read. A single
|
||||
configuration files in the list will be read. A single
|
||||
filename may also be given.
|
||||
|
||||
Return list of successfully read files.
|
||||
"""
|
||||
if isinstance(filenames, (str, bytes, os.PathLike)):
|
||||
if isinstance(filenames, str):
|
||||
filenames = [filenames]
|
||||
encoding = io.text_encoding(encoding)
|
||||
read_ok = []
|
||||
for filename in filenames:
|
||||
try:
|
||||
@@ -684,18 +696,16 @@ class RawConfigParser(MutableMapping):
|
||||
self._read(fp, filename)
|
||||
except OSError:
|
||||
continue
|
||||
if isinstance(filename, os.PathLike):
|
||||
filename = os.fspath(filename)
|
||||
read_ok.append(filename)
|
||||
return read_ok
|
||||
|
||||
def read_file(self, f, source=None):
|
||||
"""Like read() but the argument must be a file-like object.
|
||||
|
||||
The `f` argument must be iterable, returning one line at a time.
|
||||
Optional second argument is the `source` specifying the name of the
|
||||
file being read. If not given, it is taken from f.name. If `f` has no
|
||||
`name` attribute, `<???>` is used.
|
||||
The `f' argument must be iterable, returning one line at a time.
|
||||
Optional second argument is the `source' specifying the name of the
|
||||
file being read. If not given, it is taken from f.name. If `f' has no
|
||||
`name' attribute, `<???>' is used.
|
||||
"""
|
||||
if source is None:
|
||||
try:
|
||||
@@ -719,7 +729,7 @@ class RawConfigParser(MutableMapping):
|
||||
All types held in the dictionary are converted to strings during
|
||||
reading, including section names, option names and keys.
|
||||
|
||||
Optional second argument is the `source` specifying the name of the
|
||||
Optional second argument is the `source' specifying the name of the
|
||||
dictionary being read.
|
||||
"""
|
||||
elements_added = set()
|
||||
@@ -740,18 +750,27 @@ class RawConfigParser(MutableMapping):
|
||||
elements_added.add((section, key))
|
||||
self.set(section, key, value)
|
||||
|
||||
def readfp(self, fp, filename=None):
|
||||
"""Deprecated, use read_file instead."""
|
||||
warnings.warn(
|
||||
"This method will be removed in future versions. "
|
||||
"Use 'parser.read_file()' instead.",
|
||||
DeprecationWarning, stacklevel=2
|
||||
)
|
||||
self.read_file(fp, source=filename)
|
||||
|
||||
def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET):
|
||||
"""Get an option value for a given section.
|
||||
|
||||
If `vars` is provided, it must be a dictionary. The option is looked up
|
||||
in `vars` (if provided), `section`, and in `DEFAULTSECT` in that order.
|
||||
If the key is not found and `fallback` is provided, it is used as
|
||||
a fallback value. `None` can be provided as a `fallback` value.
|
||||
If `vars' is provided, it must be a dictionary. The option is looked up
|
||||
in `vars' (if provided), `section', and in `DEFAULTSECT' in that order.
|
||||
If the key is not found and `fallback' is provided, it is used as
|
||||
a fallback value. `None' can be provided as a `fallback' value.
|
||||
|
||||
If interpolation is enabled and the optional argument `raw` is False,
|
||||
If interpolation is enabled and the optional argument `raw' is False,
|
||||
all interpolations are expanded in the return values.
|
||||
|
||||
Arguments `raw`, `vars`, and `fallback` are keyword only.
|
||||
Arguments `raw', `vars', and `fallback' are keyword only.
|
||||
|
||||
The section DEFAULT is special.
|
||||
"""
|
||||
@@ -811,8 +830,8 @@ class RawConfigParser(MutableMapping):
|
||||
|
||||
All % interpolations are expanded in the return values, based on the
|
||||
defaults passed into the constructor, unless the optional argument
|
||||
`raw` is true. Additional substitutions may be provided using the
|
||||
`vars` argument, which must be a dictionary whose contents overrides
|
||||
`raw' is true. Additional substitutions may be provided using the
|
||||
`vars' argument, which must be a dictionary whose contents overrides
|
||||
any pre-existing defaults.
|
||||
|
||||
The section DEFAULT is special.
|
||||
@@ -825,7 +844,6 @@ class RawConfigParser(MutableMapping):
|
||||
except KeyError:
|
||||
if section != self.default_section:
|
||||
raise NoSectionError(section)
|
||||
orig_keys = list(d.keys())
|
||||
# Update with the entry specific variables
|
||||
if vars:
|
||||
for key, value in vars.items():
|
||||
@@ -834,7 +852,7 @@ class RawConfigParser(MutableMapping):
|
||||
section, option, d[option], d)
|
||||
if raw:
|
||||
value_getter = lambda option: d[option]
|
||||
return [(option, value_getter(option)) for option in orig_keys]
|
||||
return [(option, value_getter(option)) for option in d.keys()]
|
||||
|
||||
def popitem(self):
|
||||
"""Remove a section from the parser and return it as
|
||||
@@ -854,8 +872,8 @@ class RawConfigParser(MutableMapping):
|
||||
|
||||
def has_option(self, section, option):
|
||||
"""Check for the existence of a given option in a given section.
|
||||
If the specified `section` is None or an empty string, DEFAULT is
|
||||
assumed. If the specified `section` does not exist, returns False."""
|
||||
If the specified `section' is None or an empty string, DEFAULT is
|
||||
assumed. If the specified `section' does not exist, returns False."""
|
||||
if not section or section == self.default_section:
|
||||
option = self.optionxform(option)
|
||||
return option in self._defaults
|
||||
@@ -883,11 +901,8 @@ class RawConfigParser(MutableMapping):
|
||||
def write(self, fp, space_around_delimiters=True):
|
||||
"""Write an .ini-format representation of the configuration state.
|
||||
|
||||
If `space_around_delimiters` is True (the default), delimiters
|
||||
If `space_around_delimiters' is True (the default), delimiters
|
||||
between keys and values are surrounded by spaces.
|
||||
|
||||
Please note that comments in the original configuration file are not
|
||||
preserved when writing the configuration back.
|
||||
"""
|
||||
if space_around_delimiters:
|
||||
d = " {} ".format(self._delimiters[0])
|
||||
@@ -901,7 +916,7 @@ class RawConfigParser(MutableMapping):
|
||||
self._sections[section].items(), d)
|
||||
|
||||
def _write_section(self, fp, section_name, section_items, delimiter):
|
||||
"""Write a single section to the specified `fp`."""
|
||||
"""Write a single section to the specified `fp'."""
|
||||
fp.write("[{}]\n".format(section_name))
|
||||
for key, value in section_items:
|
||||
value = self._interpolation.before_write(self, section_name, key,
|
||||
@@ -944,8 +959,7 @@ class RawConfigParser(MutableMapping):
|
||||
def __setitem__(self, key, value):
|
||||
# To conform with the mapping protocol, overwrites existing values in
|
||||
# the section.
|
||||
if key in self and self[key] is value:
|
||||
return
|
||||
|
||||
# XXX this is not atomic if read_dict fails at any point. Then again,
|
||||
# no update method in configparser is atomic in this implementation.
|
||||
if key == self.default_section:
|
||||
@@ -975,8 +989,8 @@ class RawConfigParser(MutableMapping):
|
||||
"""Parse a sectioned configuration file.
|
||||
|
||||
Each section in a configuration file contains a header, indicated by
|
||||
a name in square brackets (`[]`), plus key/value options, indicated by
|
||||
`name` and `value` delimited with a specific substring (`=` or `:` by
|
||||
a name in square brackets (`[]'), plus key/value options, indicated by
|
||||
`name' and `value' delimited with a specific substring (`=' or `:' by
|
||||
default).
|
||||
|
||||
Values can span multiple lines, as long as they are indented deeper
|
||||
@@ -984,9 +998,9 @@ class RawConfigParser(MutableMapping):
|
||||
lines may be treated as parts of multiline values or ignored.
|
||||
|
||||
Configuration files may include comments, prefixed by specific
|
||||
characters (`#` and `;` by default). Comments may appear on their own
|
||||
characters (`#' and `;' by default). Comments may appear on their own
|
||||
in an otherwise empty line or may be entered in lines holding values or
|
||||
section names. Please note that comments get stripped off when reading configuration files.
|
||||
section names.
|
||||
"""
|
||||
elements_added = set()
|
||||
cursect = None # None, or a dictionary
|
||||
@@ -1105,12 +1119,6 @@ class RawConfigParser(MutableMapping):
|
||||
section,
|
||||
name, val)
|
||||
|
||||
def _read_defaults(self, defaults):
|
||||
"""Read the defaults passed in the initializer.
|
||||
Note: values can be non-string."""
|
||||
for key, value in defaults.items():
|
||||
self._defaults[self.optionxform(key)] = value
|
||||
|
||||
def _handle_error(self, exc, fpname, lineno, line):
|
||||
if not exc:
|
||||
exc = ParsingError(fpname)
|
||||
@@ -1127,7 +1135,7 @@ class RawConfigParser(MutableMapping):
|
||||
sectiondict = self._sections[section]
|
||||
except KeyError:
|
||||
if section != self.default_section:
|
||||
raise NoSectionError(section) from None
|
||||
raise NoSectionError(section)
|
||||
# Update with the entry specific variables
|
||||
vardict = {}
|
||||
if vars:
|
||||
@@ -1188,18 +1196,18 @@ class ConfigParser(RawConfigParser):
|
||||
self._validate_value_types(section=section)
|
||||
super().add_section(section)
|
||||
|
||||
def _read_defaults(self, defaults):
|
||||
"""Reads the defaults passed in the initializer, implicitly converting
|
||||
values to strings like the rest of the API.
|
||||
|
||||
Does not perform interpolation for backwards compatibility.
|
||||
"""
|
||||
try:
|
||||
hold_interpolation = self._interpolation
|
||||
self._interpolation = Interpolation()
|
||||
self.read_dict({self.default_section: defaults})
|
||||
finally:
|
||||
self._interpolation = hold_interpolation
|
||||
class SafeConfigParser(ConfigParser):
|
||||
"""ConfigParser alias for backwards compatibility purposes."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
warnings.warn(
|
||||
"The SafeConfigParser class has been renamed to ConfigParser "
|
||||
"in Python 3.2. This alias will be removed in future versions."
|
||||
" Use ConfigParser directly instead.",
|
||||
DeprecationWarning, stacklevel=2
|
||||
)
|
||||
|
||||
|
||||
class SectionProxy(MutableMapping):
|
||||
|
||||
228
Lib/contextlib.py
vendored
228
Lib/contextlib.py
vendored
@@ -1,25 +1,20 @@
|
||||
"""Utilities for with-statement contexts. See PEP 343."""
|
||||
import abc
|
||||
import os
|
||||
import sys
|
||||
import _collections_abc
|
||||
from collections import deque
|
||||
from functools import wraps
|
||||
from types import MethodType, GenericAlias
|
||||
|
||||
__all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext",
|
||||
"AbstractContextManager", "AbstractAsyncContextManager",
|
||||
"AsyncExitStack", "ContextDecorator", "ExitStack",
|
||||
"redirect_stdout", "redirect_stderr", "suppress", "aclosing",
|
||||
"chdir"]
|
||||
"redirect_stdout", "redirect_stderr", "suppress"]
|
||||
|
||||
|
||||
class AbstractContextManager(abc.ABC):
|
||||
|
||||
"""An abstract base class for context managers."""
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
def __enter__(self):
|
||||
"""Return `self` upon entering the runtime context."""
|
||||
return self
|
||||
@@ -40,8 +35,6 @@ class AbstractAsyncContextManager(abc.ABC):
|
||||
|
||||
"""An abstract base class for asynchronous context managers."""
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
async def __aenter__(self):
|
||||
"""Return `self` upon entering the runtime context."""
|
||||
return self
|
||||
@@ -82,22 +75,6 @@ class ContextDecorator(object):
|
||||
return inner
|
||||
|
||||
|
||||
class AsyncContextDecorator(object):
|
||||
"A base class or mixin that enables async context managers to work as decorators."
|
||||
|
||||
def _recreate_cm(self):
|
||||
"""Return a recreated instance of self.
|
||||
"""
|
||||
return self
|
||||
|
||||
def __call__(self, func):
|
||||
@wraps(func)
|
||||
async def inner(*args, **kwds):
|
||||
async with self._recreate_cm():
|
||||
return await func(*args, **kwds)
|
||||
return inner
|
||||
|
||||
|
||||
class _GeneratorContextManagerBase:
|
||||
"""Shared functionality for @contextmanager and @asynccontextmanager."""
|
||||
|
||||
@@ -115,20 +92,18 @@ class _GeneratorContextManagerBase:
|
||||
# for the class instead.
|
||||
# See http://bugs.python.org/issue19404 for more details.
|
||||
|
||||
|
||||
class _GeneratorContextManager(_GeneratorContextManagerBase,
|
||||
AbstractContextManager,
|
||||
ContextDecorator):
|
||||
"""Helper for @contextmanager decorator."""
|
||||
|
||||
def _recreate_cm(self):
|
||||
# _GCMB instances are one-shot context managers, so the
|
||||
# _GCM instances are one-shot context managers, so the
|
||||
# CM must be recreated each time a decorated function is
|
||||
# called
|
||||
return self.__class__(self.func, self.args, self.kwds)
|
||||
|
||||
|
||||
class _GeneratorContextManager(
|
||||
_GeneratorContextManagerBase,
|
||||
AbstractContextManager,
|
||||
ContextDecorator,
|
||||
):
|
||||
"""Helper for @contextmanager decorator."""
|
||||
|
||||
def __enter__(self):
|
||||
# do not keep args and kwds alive unnecessarily
|
||||
# they are only needed for recreation, which is not possible anymore
|
||||
@@ -138,8 +113,8 @@ class _GeneratorContextManager(
|
||||
except StopIteration:
|
||||
raise RuntimeError("generator didn't yield") from None
|
||||
|
||||
def __exit__(self, typ, value, traceback):
|
||||
if typ is None:
|
||||
def __exit__(self, type, value, traceback):
|
||||
if type is None:
|
||||
try:
|
||||
next(self.gen)
|
||||
except StopIteration:
|
||||
@@ -150,9 +125,9 @@ class _GeneratorContextManager(
|
||||
if value is None:
|
||||
# Need to force instantiation so we can reliably
|
||||
# tell if we get the same exception back
|
||||
value = typ()
|
||||
value = type()
|
||||
try:
|
||||
self.gen.throw(typ, value, traceback)
|
||||
self.gen.throw(type, value, traceback)
|
||||
except StopIteration as exc:
|
||||
# Suppress StopIteration *unless* it's the same exception that
|
||||
# was passed to throw(). This prevents a StopIteration
|
||||
@@ -161,100 +136,75 @@ class _GeneratorContextManager(
|
||||
except RuntimeError as exc:
|
||||
# Don't re-raise the passed in exception. (issue27122)
|
||||
if exc is value:
|
||||
exc.__traceback__ = traceback
|
||||
return False
|
||||
# Avoid suppressing if a StopIteration exception
|
||||
# Likewise, avoid suppressing if a StopIteration exception
|
||||
# was passed to throw() and later wrapped into a RuntimeError
|
||||
# (see PEP 479 for sync generators; async generators also
|
||||
# have this behavior). But do this only if the exception wrapped
|
||||
# by the RuntimeError is actually Stop(Async)Iteration (see
|
||||
# issue29692).
|
||||
if (
|
||||
isinstance(value, StopIteration)
|
||||
and exc.__cause__ is value
|
||||
):
|
||||
value.__traceback__ = traceback
|
||||
# (see PEP 479).
|
||||
if type is StopIteration and exc.__cause__ is value:
|
||||
return False
|
||||
raise
|
||||
except BaseException as exc:
|
||||
except:
|
||||
# only re-raise if it's *not* the exception that was
|
||||
# passed to throw(), because __exit__() must not raise
|
||||
# an exception unless __exit__() itself failed. But throw()
|
||||
# has to raise the exception to signal propagation, so this
|
||||
# fixes the impedance mismatch between the throw() protocol
|
||||
# and the __exit__() protocol.
|
||||
if exc is not value:
|
||||
raise
|
||||
exc.__traceback__ = traceback
|
||||
return False
|
||||
#
|
||||
# This cannot use 'except BaseException as exc' (as in the
|
||||
# async implementation) to maintain compatibility with
|
||||
# Python 2, where old-style class exceptions are not caught
|
||||
# by 'except BaseException'.
|
||||
if sys.exc_info()[1] is value:
|
||||
return False
|
||||
raise
|
||||
raise RuntimeError("generator didn't stop after throw()")
|
||||
|
||||
class _AsyncGeneratorContextManager(
|
||||
_GeneratorContextManagerBase,
|
||||
AbstractAsyncContextManager,
|
||||
AsyncContextDecorator,
|
||||
):
|
||||
"""Helper for @asynccontextmanager decorator."""
|
||||
|
||||
class _AsyncGeneratorContextManager(_GeneratorContextManagerBase,
|
||||
AbstractAsyncContextManager):
|
||||
"""Helper for @asynccontextmanager."""
|
||||
|
||||
async def __aenter__(self):
|
||||
# do not keep args and kwds alive unnecessarily
|
||||
# they are only needed for recreation, which is not possible anymore
|
||||
del self.args, self.kwds, self.func
|
||||
try:
|
||||
return await anext(self.gen)
|
||||
return await self.gen.__anext__()
|
||||
except StopAsyncIteration:
|
||||
raise RuntimeError("generator didn't yield") from None
|
||||
|
||||
async def __aexit__(self, typ, value, traceback):
|
||||
if typ is None:
|
||||
try:
|
||||
await anext(self.gen)
|
||||
await self.gen.__anext__()
|
||||
except StopAsyncIteration:
|
||||
return False
|
||||
return
|
||||
else:
|
||||
raise RuntimeError("generator didn't stop")
|
||||
else:
|
||||
if value is None:
|
||||
# Need to force instantiation so we can reliably
|
||||
# tell if we get the same exception back
|
||||
value = typ()
|
||||
# See _GeneratorContextManager.__exit__ for comments on subtleties
|
||||
# in this implementation
|
||||
try:
|
||||
await self.gen.athrow(typ, value, traceback)
|
||||
raise RuntimeError("generator didn't stop after throw()")
|
||||
except StopAsyncIteration as exc:
|
||||
# Suppress StopIteration *unless* it's the same exception that
|
||||
# was passed to throw(). This prevents a StopIteration
|
||||
# raised inside the "with" statement from being suppressed.
|
||||
return exc is not value
|
||||
except RuntimeError as exc:
|
||||
# Don't re-raise the passed in exception. (issue27122)
|
||||
if exc is value:
|
||||
exc.__traceback__ = traceback
|
||||
return False
|
||||
# Avoid suppressing if a Stop(Async)Iteration exception
|
||||
# was passed to athrow() and later wrapped into a RuntimeError
|
||||
# Avoid suppressing if a StopIteration exception
|
||||
# was passed to throw() and later wrapped into a RuntimeError
|
||||
# (see PEP 479 for sync generators; async generators also
|
||||
# have this behavior). But do this only if the exception wrapped
|
||||
# by the RuntimeError is actually Stop(Async)Iteration (see
|
||||
# by the RuntimeError is actully Stop(Async)Iteration (see
|
||||
# issue29692).
|
||||
if (
|
||||
isinstance(value, (StopIteration, StopAsyncIteration))
|
||||
and exc.__cause__ is value
|
||||
):
|
||||
value.__traceback__ = traceback
|
||||
return False
|
||||
if isinstance(value, (StopIteration, StopAsyncIteration)):
|
||||
if exc.__cause__ is value:
|
||||
return False
|
||||
raise
|
||||
except BaseException as exc:
|
||||
# only re-raise if it's *not* the exception that was
|
||||
# passed to throw(), because __exit__() must not raise
|
||||
# an exception unless __exit__() itself failed. But throw()
|
||||
# has to raise the exception to signal propagation, so this
|
||||
# fixes the impedance mismatch between the throw() protocol
|
||||
# and the __exit__() protocol.
|
||||
if exc is not value:
|
||||
raise
|
||||
exc.__traceback__ = traceback
|
||||
return False
|
||||
raise RuntimeError("generator didn't stop after athrow()")
|
||||
|
||||
|
||||
def contextmanager(func):
|
||||
@@ -348,32 +298,6 @@ class closing(AbstractContextManager):
|
||||
self.thing.close()
|
||||
|
||||
|
||||
class aclosing(AbstractAsyncContextManager):
|
||||
"""Async context manager for safely finalizing an asynchronously cleaned-up
|
||||
resource such as an async generator, calling its ``aclose()`` method.
|
||||
|
||||
Code like this:
|
||||
|
||||
async with aclosing(<module>.fetch(<arguments>)) as agen:
|
||||
<block>
|
||||
|
||||
is equivalent to this:
|
||||
|
||||
agen = <module>.fetch(<arguments>)
|
||||
try:
|
||||
<block>
|
||||
finally:
|
||||
await agen.aclose()
|
||||
|
||||
"""
|
||||
def __init__(self, thing):
|
||||
self.thing = thing
|
||||
async def __aenter__(self):
|
||||
return self.thing
|
||||
async def __aexit__(self, *exc_info):
|
||||
await self.thing.aclose()
|
||||
|
||||
|
||||
class _RedirectStream(AbstractContextManager):
|
||||
|
||||
_stream = None
|
||||
@@ -449,10 +373,12 @@ class _BaseExitStack:
|
||||
|
||||
@staticmethod
|
||||
def _create_exit_wrapper(cm, cm_exit):
|
||||
return MethodType(cm_exit, cm)
|
||||
def _exit_wrapper(exc_type, exc, tb):
|
||||
return cm_exit(cm, exc_type, exc, tb)
|
||||
return _exit_wrapper
|
||||
|
||||
@staticmethod
|
||||
def _create_cb_wrapper(callback, /, *args, **kwds):
|
||||
def _create_cb_wrapper(callback, *args, **kwds):
|
||||
def _exit_wrapper(exc_type, exc, tb):
|
||||
callback(*args, **kwds)
|
||||
return _exit_wrapper
|
||||
@@ -495,18 +421,13 @@ class _BaseExitStack:
|
||||
"""
|
||||
# We look up the special methods on the type to match the with
|
||||
# statement.
|
||||
cls = type(cm)
|
||||
try:
|
||||
_enter = cls.__enter__
|
||||
_exit = cls.__exit__
|
||||
except AttributeError:
|
||||
raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does "
|
||||
f"not support the context manager protocol") from None
|
||||
result = _enter(cm)
|
||||
_cm_type = type(cm)
|
||||
_exit = _cm_type.__exit__
|
||||
result = _cm_type.__enter__(cm)
|
||||
self._push_cm_exit(cm, _exit)
|
||||
return result
|
||||
|
||||
def callback(self, callback, /, *args, **kwds):
|
||||
def callback(self, callback, *args, **kwds):
|
||||
"""Registers an arbitrary callback and arguments.
|
||||
|
||||
Cannot suppress exceptions.
|
||||
@@ -522,6 +443,7 @@ class _BaseExitStack:
|
||||
def _push_cm_exit(self, cm, cm_exit):
|
||||
"""Helper to correctly register callbacks to __exit__ methods."""
|
||||
_exit_wrapper = self._create_exit_wrapper(cm, cm_exit)
|
||||
_exit_wrapper.__self__ = cm
|
||||
self._push_exit_callback(_exit_wrapper, True)
|
||||
|
||||
def _push_exit_callback(self, callback, is_sync=True):
|
||||
@@ -553,10 +475,10 @@ class ExitStack(_BaseExitStack, AbstractContextManager):
|
||||
# Context may not be correct, so find the end of the chain
|
||||
while 1:
|
||||
exc_context = new_exc.__context__
|
||||
if exc_context is None or exc_context is old_exc:
|
||||
if exc_context is old_exc:
|
||||
# Context is already set correctly (see issue 20317)
|
||||
return
|
||||
if exc_context is frame_exc:
|
||||
if exc_context is None or exc_context is frame_exc:
|
||||
break
|
||||
new_exc = exc_context
|
||||
# Change the end of the chain to point to the exception
|
||||
@@ -613,10 +535,12 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
|
||||
|
||||
@staticmethod
|
||||
def _create_async_exit_wrapper(cm, cm_exit):
|
||||
return MethodType(cm_exit, cm)
|
||||
async def _exit_wrapper(exc_type, exc, tb):
|
||||
return await cm_exit(cm, exc_type, exc, tb)
|
||||
return _exit_wrapper
|
||||
|
||||
@staticmethod
|
||||
def _create_async_cb_wrapper(callback, /, *args, **kwds):
|
||||
def _create_async_cb_wrapper(callback, *args, **kwds):
|
||||
async def _exit_wrapper(exc_type, exc, tb):
|
||||
await callback(*args, **kwds)
|
||||
return _exit_wrapper
|
||||
@@ -627,15 +551,9 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
|
||||
If successful, also pushes its __aexit__ method as a callback and
|
||||
returns the result of the __aenter__ method.
|
||||
"""
|
||||
cls = type(cm)
|
||||
try:
|
||||
_enter = cls.__aenter__
|
||||
_exit = cls.__aexit__
|
||||
except AttributeError:
|
||||
raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does "
|
||||
f"not support the asynchronous context manager protocol"
|
||||
) from None
|
||||
result = await _enter(cm)
|
||||
_cm_type = type(cm)
|
||||
_exit = _cm_type.__aexit__
|
||||
result = await _cm_type.__aenter__(cm)
|
||||
self._push_async_cm_exit(cm, _exit)
|
||||
return result
|
||||
|
||||
@@ -657,7 +575,7 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
|
||||
self._push_async_cm_exit(exit, exit_method)
|
||||
return exit # Allow use as a decorator
|
||||
|
||||
def push_async_callback(self, callback, /, *args, **kwds):
|
||||
def push_async_callback(self, callback, *args, **kwds):
|
||||
"""Registers an arbitrary coroutine function and arguments.
|
||||
|
||||
Cannot suppress exceptions.
|
||||
@@ -678,6 +596,7 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
|
||||
"""Helper to correctly register coroutine function to __aexit__
|
||||
method."""
|
||||
_exit_wrapper = self._create_async_exit_wrapper(cm, cm_exit)
|
||||
_exit_wrapper.__self__ = cm
|
||||
self._push_exit_callback(_exit_wrapper, False)
|
||||
|
||||
async def __aenter__(self):
|
||||
@@ -693,10 +612,10 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
|
||||
# Context may not be correct, so find the end of the chain
|
||||
while 1:
|
||||
exc_context = new_exc.__context__
|
||||
if exc_context is None or exc_context is old_exc:
|
||||
if exc_context is old_exc:
|
||||
# Context is already set correctly (see issue 20317)
|
||||
return
|
||||
if exc_context is frame_exc:
|
||||
if exc_context is None or exc_context is frame_exc:
|
||||
break
|
||||
new_exc = exc_context
|
||||
# Change the end of the chain to point to the exception
|
||||
@@ -737,7 +656,7 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
|
||||
return received_exc and suppressed_exc
|
||||
|
||||
|
||||
class nullcontext(AbstractContextManager, AbstractAsyncContextManager):
|
||||
class nullcontext(AbstractContextManager):
|
||||
"""Context manager that does no additional processing.
|
||||
|
||||
Used as a stand-in for a normal context manager, when a particular
|
||||
@@ -756,24 +675,3 @@ class nullcontext(AbstractContextManager, AbstractAsyncContextManager):
|
||||
|
||||
def __exit__(self, *excinfo):
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self.enter_result
|
||||
|
||||
async def __aexit__(self, *excinfo):
|
||||
pass
|
||||
|
||||
|
||||
class chdir(AbstractContextManager):
|
||||
"""Non thread-safe context manager to change the current working directory."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self._old_cwd = []
|
||||
|
||||
def __enter__(self):
|
||||
self._old_cwd.append(os.getcwd())
|
||||
os.chdir(self.path)
|
||||
|
||||
def __exit__(self, *excinfo):
|
||||
os.chdir(self._old_cwd.pop())
|
||||
|
||||
4
Lib/contextvars.py
vendored
4
Lib/contextvars.py
vendored
@@ -1,4 +0,0 @@
|
||||
from _contextvars import Context, ContextVar, Token, copy_context
|
||||
|
||||
|
||||
__all__ = ('Context', 'ContextVar', 'Token', 'copy_context')
|
||||
67
Lib/copy.py
vendored
67
Lib/copy.py
vendored
@@ -39,8 +39,8 @@ Python's deep copy operation avoids these problems by:
|
||||
set of components copied
|
||||
|
||||
This version does not copy types like module, class, function, method,
|
||||
nor stack trace, stack frame, nor file, socket, window, nor any
|
||||
similar types.
|
||||
nor stack trace, stack frame, nor file, socket, window, nor array, nor
|
||||
any similar types.
|
||||
|
||||
Classes can use the same interfaces to control copying that they use
|
||||
to control pickling: they can define methods called __getinitargs__(),
|
||||
@@ -56,6 +56,11 @@ class Error(Exception):
|
||||
pass
|
||||
error = Error # backward compatibility
|
||||
|
||||
try:
|
||||
from org.python.core import PyStringMap
|
||||
except ImportError:
|
||||
PyStringMap = None
|
||||
|
||||
__all__ = ["Error", "copy", "deepcopy"]
|
||||
|
||||
def copy(x):
|
||||
@@ -70,20 +75,24 @@ def copy(x):
|
||||
if copier:
|
||||
return copier(x)
|
||||
|
||||
if issubclass(cls, type):
|
||||
try:
|
||||
issc = issubclass(cls, type)
|
||||
except TypeError: # cls is not a class
|
||||
issc = False
|
||||
if issc:
|
||||
# treat it as a regular class:
|
||||
return _copy_immutable(x)
|
||||
|
||||
copier = getattr(cls, "__copy__", None)
|
||||
if copier is not None:
|
||||
if copier:
|
||||
return copier(x)
|
||||
|
||||
reductor = dispatch_table.get(cls)
|
||||
if reductor is not None:
|
||||
if reductor:
|
||||
rv = reductor(x)
|
||||
else:
|
||||
reductor = getattr(x, "__reduce_ex__", None)
|
||||
if reductor is not None:
|
||||
if reductor:
|
||||
rv = reductor(4)
|
||||
else:
|
||||
reductor = getattr(x, "__reduce__", None)
|
||||
@@ -101,11 +110,13 @@ _copy_dispatch = d = {}
|
||||
|
||||
def _copy_immutable(x):
|
||||
return x
|
||||
for t in (types.NoneType, int, float, bool, complex, str, tuple,
|
||||
bytes, frozenset, type, range, slice, property,
|
||||
types.BuiltinFunctionType, types.EllipsisType,
|
||||
types.NotImplementedType, types.FunctionType, types.CodeType,
|
||||
weakref.ref):
|
||||
for t in (type(None), int, float, bool, complex, str, tuple,
|
||||
bytes, frozenset, type, range, slice,
|
||||
types.BuiltinFunctionType, type(Ellipsis), type(NotImplemented),
|
||||
types.FunctionType, weakref.ref):
|
||||
d[t] = _copy_immutable
|
||||
t = getattr(types, "CodeType", None)
|
||||
if t is not None:
|
||||
d[t] = _copy_immutable
|
||||
|
||||
d[list] = list.copy
|
||||
@@ -113,6 +124,9 @@ d[dict] = dict.copy
|
||||
d[set] = set.copy
|
||||
d[bytearray] = bytearray.copy
|
||||
|
||||
if PyStringMap is not None:
|
||||
d[PyStringMap] = PyStringMap.copy
|
||||
|
||||
del d, t
|
||||
|
||||
def deepcopy(x, memo=None, _nil=[]):
|
||||
@@ -132,14 +146,18 @@ def deepcopy(x, memo=None, _nil=[]):
|
||||
cls = type(x)
|
||||
|
||||
copier = _deepcopy_dispatch.get(cls)
|
||||
if copier is not None:
|
||||
if copier:
|
||||
y = copier(x, memo)
|
||||
else:
|
||||
if issubclass(cls, type):
|
||||
try:
|
||||
issc = issubclass(cls, type)
|
||||
except TypeError: # cls is not a class (old Boost; see SF #502085)
|
||||
issc = 0
|
||||
if issc:
|
||||
y = _deepcopy_atomic(x, memo)
|
||||
else:
|
||||
copier = getattr(x, "__deepcopy__", None)
|
||||
if copier is not None:
|
||||
if copier:
|
||||
y = copier(memo)
|
||||
else:
|
||||
reductor = dispatch_table.get(cls)
|
||||
@@ -147,7 +165,7 @@ def deepcopy(x, memo=None, _nil=[]):
|
||||
rv = reductor(x)
|
||||
else:
|
||||
reductor = getattr(x, "__reduce_ex__", None)
|
||||
if reductor is not None:
|
||||
if reductor:
|
||||
rv = reductor(4)
|
||||
else:
|
||||
reductor = getattr(x, "__reduce__", None)
|
||||
@@ -171,22 +189,23 @@ _deepcopy_dispatch = d = {}
|
||||
|
||||
def _deepcopy_atomic(x, memo):
|
||||
return x
|
||||
d[types.NoneType] = _deepcopy_atomic
|
||||
d[types.EllipsisType] = _deepcopy_atomic
|
||||
d[types.NotImplementedType] = _deepcopy_atomic
|
||||
d[type(None)] = _deepcopy_atomic
|
||||
d[type(Ellipsis)] = _deepcopy_atomic
|
||||
d[type(NotImplemented)] = _deepcopy_atomic
|
||||
d[int] = _deepcopy_atomic
|
||||
d[float] = _deepcopy_atomic
|
||||
d[bool] = _deepcopy_atomic
|
||||
d[complex] = _deepcopy_atomic
|
||||
d[bytes] = _deepcopy_atomic
|
||||
d[str] = _deepcopy_atomic
|
||||
d[types.CodeType] = _deepcopy_atomic
|
||||
try:
|
||||
d[types.CodeType] = _deepcopy_atomic
|
||||
except AttributeError:
|
||||
pass
|
||||
d[type] = _deepcopy_atomic
|
||||
d[range] = _deepcopy_atomic
|
||||
d[types.BuiltinFunctionType] = _deepcopy_atomic
|
||||
d[types.FunctionType] = _deepcopy_atomic
|
||||
d[weakref.ref] = _deepcopy_atomic
|
||||
d[property] = _deepcopy_atomic
|
||||
|
||||
def _deepcopy_list(x, memo, deepcopy=deepcopy):
|
||||
y = []
|
||||
@@ -221,6 +240,8 @@ def _deepcopy_dict(x, memo, deepcopy=deepcopy):
|
||||
y[deepcopy(key, memo)] = deepcopy(value, memo)
|
||||
return y
|
||||
d[dict] = _deepcopy_dict
|
||||
if PyStringMap is not None:
|
||||
d[PyStringMap] = _deepcopy_dict
|
||||
|
||||
def _deepcopy_method(x, memo): # Copy instance methods
|
||||
return type(x)(x.__func__, deepcopy(x.__self__, memo))
|
||||
@@ -246,7 +267,7 @@ def _keep_alive(x, memo):
|
||||
|
||||
def _reconstruct(x, memo, func, args,
|
||||
state=None, listiter=None, dictiter=None,
|
||||
*, deepcopy=deepcopy):
|
||||
deepcopy=deepcopy):
|
||||
deep = memo is not None
|
||||
if deep and args:
|
||||
args = (deepcopy(arg, memo) for arg in args)
|
||||
@@ -289,4 +310,4 @@ def _reconstruct(x, memo, func, args,
|
||||
y[key] = value
|
||||
return y
|
||||
|
||||
del types, weakref
|
||||
del types, weakref, PyStringMap
|
||||
|
||||
15
Lib/copyreg.py
vendored
15
Lib/copyreg.py
vendored
@@ -53,8 +53,7 @@ _HEAPTYPE = 1<<9
|
||||
|
||||
def _reduce_ex(self, proto):
|
||||
assert proto < 2
|
||||
cls = self.__class__
|
||||
for base in cls.__mro__:
|
||||
for base in self.__class__.__mro__:
|
||||
if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE:
|
||||
break
|
||||
else:
|
||||
@@ -62,18 +61,16 @@ def _reduce_ex(self, proto):
|
||||
if base is object:
|
||||
state = None
|
||||
else:
|
||||
if base is cls:
|
||||
raise TypeError(f"cannot pickle {cls.__name__!r} object")
|
||||
if base is self.__class__:
|
||||
raise TypeError("can't pickle %s objects" % base.__name__)
|
||||
state = base(self)
|
||||
args = (cls, base, state)
|
||||
args = (self.__class__, base, state)
|
||||
try:
|
||||
getstate = self.__getstate__
|
||||
except AttributeError:
|
||||
if getattr(self, "__slots__", None):
|
||||
raise TypeError(f"cannot pickle {cls.__name__!r} object: "
|
||||
f"a class that defines __slots__ without "
|
||||
f"defining __getstate__ cannot be pickled "
|
||||
f"with protocol {proto}") from None
|
||||
raise TypeError("a class that defines __slots__ without "
|
||||
"defining __getstate__ cannot be pickled")
|
||||
try:
|
||||
dict = self.__dict__
|
||||
except AttributeError:
|
||||
|
||||
48
Lib/csv.py
vendored
48
Lib/csv.py
vendored
@@ -4,22 +4,17 @@ csv.py - read/write/investigate CSV files
|
||||
"""
|
||||
|
||||
import re
|
||||
import types
|
||||
from _csv import Error, __version__, writer, reader, register_dialect, \
|
||||
unregister_dialect, get_dialect, list_dialects, \
|
||||
field_size_limit, \
|
||||
from _csv import Error, reader, \
|
||||
QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \
|
||||
QUOTE_STRINGS, QUOTE_NOTNULL, \
|
||||
__doc__
|
||||
from _csv import Dialect as _Dialect
|
||||
|
||||
from collections import OrderedDict
|
||||
from io import StringIO
|
||||
|
||||
__all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE",
|
||||
"QUOTE_STRINGS", "QUOTE_NOTNULL",
|
||||
"Error", "Dialect", "__doc__", "excel", "excel_tab",
|
||||
"field_size_limit", "reader", "writer",
|
||||
"register_dialect", "get_dialect", "list_dialects", "Sniffer",
|
||||
"Sniffer",
|
||||
"unregister_dialect", "__version__", "DictReader", "DictWriter",
|
||||
"unix_dialect"]
|
||||
|
||||
@@ -62,12 +57,10 @@ class excel(Dialect):
|
||||
skipinitialspace = False
|
||||
lineterminator = '\r\n'
|
||||
quoting = QUOTE_MINIMAL
|
||||
register_dialect("excel", excel)
|
||||
|
||||
class excel_tab(excel):
|
||||
"""Describe the usual properties of Excel-generated TAB-delimited files."""
|
||||
delimiter = '\t'
|
||||
register_dialect("excel-tab", excel_tab)
|
||||
|
||||
class unix_dialect(Dialect):
|
||||
"""Describe the usual properties of Unix-generated CSV files."""
|
||||
@@ -77,14 +70,11 @@ class unix_dialect(Dialect):
|
||||
skipinitialspace = False
|
||||
lineterminator = '\n'
|
||||
quoting = QUOTE_ALL
|
||||
register_dialect("unix", unix_dialect)
|
||||
|
||||
|
||||
class DictReader:
|
||||
def __init__(self, f, fieldnames=None, restkey=None, restval=None,
|
||||
dialect="excel", *args, **kwds):
|
||||
if fieldnames is not None and iter(fieldnames) is fieldnames:
|
||||
fieldnames = list(fieldnames)
|
||||
self._fieldnames = fieldnames # list of keys for the dict
|
||||
self.restkey = restkey # key to catch long rows
|
||||
self.restval = restval # default value for short rows
|
||||
@@ -121,7 +111,7 @@ class DictReader:
|
||||
# values
|
||||
while row == []:
|
||||
row = next(self.reader)
|
||||
d = dict(zip(self.fieldnames, row))
|
||||
d = OrderedDict(zip(self.fieldnames, row))
|
||||
lf = len(self.fieldnames)
|
||||
lr = len(row)
|
||||
if lf < lr:
|
||||
@@ -131,18 +121,13 @@ class DictReader:
|
||||
d[key] = self.restval
|
||||
return d
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
|
||||
class DictWriter:
|
||||
def __init__(self, f, fieldnames, restval="", extrasaction="raise",
|
||||
dialect="excel", *args, **kwds):
|
||||
if fieldnames is not None and iter(fieldnames) is fieldnames:
|
||||
fieldnames = list(fieldnames)
|
||||
self.fieldnames = fieldnames # list of keys for the dict
|
||||
self.restval = restval # for writing short dicts
|
||||
extrasaction = extrasaction.lower()
|
||||
if extrasaction not in ("raise", "ignore"):
|
||||
if extrasaction.lower() not in ("raise", "ignore"):
|
||||
raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'"
|
||||
% extrasaction)
|
||||
self.extrasaction = extrasaction
|
||||
@@ -150,7 +135,7 @@ class DictWriter:
|
||||
|
||||
def writeheader(self):
|
||||
header = dict(zip(self.fieldnames, self.fieldnames))
|
||||
return self.writerow(header)
|
||||
self.writerow(header)
|
||||
|
||||
def _dict_to_list(self, rowdict):
|
||||
if self.extrasaction == "raise":
|
||||
@@ -166,8 +151,11 @@ class DictWriter:
|
||||
def writerows(self, rowdicts):
|
||||
return self.writer.writerows(map(self._dict_to_list, rowdicts))
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
# Guard Sniffer's type checking against builds that exclude complex()
|
||||
try:
|
||||
complex
|
||||
except NameError:
|
||||
complex = float
|
||||
|
||||
class Sniffer:
|
||||
'''
|
||||
@@ -416,10 +404,14 @@ class Sniffer:
|
||||
continue # skip rows that have irregular number of columns
|
||||
|
||||
for col in list(columnTypes.keys()):
|
||||
thisType = complex
|
||||
try:
|
||||
thisType(row[col])
|
||||
except (ValueError, OverflowError):
|
||||
|
||||
for thisType in [int, float, complex]:
|
||||
try:
|
||||
thisType(row[col])
|
||||
break
|
||||
except (ValueError, OverflowError):
|
||||
pass
|
||||
else:
|
||||
# fallback to length of string
|
||||
thisType = len(row[col])
|
||||
|
||||
@@ -435,7 +427,7 @@ class Sniffer:
|
||||
# on whether it's a header
|
||||
hasHeader = 0
|
||||
for col, colType in columnTypes.items():
|
||||
if isinstance(colType, int): # it's a length
|
||||
if type(colType) == type(0): # it's a length
|
||||
if len(header[col]) != colType:
|
||||
hasHeader += 1
|
||||
else:
|
||||
|
||||
@@ -1,577 +0,0 @@
|
||||
"""create and manipulate C data types in Python"""
|
||||
|
||||
import os as _os, sys as _sys
|
||||
import types as _types
|
||||
|
||||
__version__ = "1.1.0"
|
||||
|
||||
from _ctypes import Union, Structure, Array
|
||||
from _ctypes import _Pointer
|
||||
from _ctypes import CFuncPtr as _CFuncPtr
|
||||
from _ctypes import __version__ as _ctypes_version
|
||||
from _ctypes import RTLD_LOCAL, RTLD_GLOBAL
|
||||
from _ctypes import ArgumentError
|
||||
from _ctypes import SIZEOF_TIME_T
|
||||
|
||||
from struct import calcsize as _calcsize
|
||||
|
||||
if __version__ != _ctypes_version:
|
||||
raise Exception("Version number mismatch", __version__, _ctypes_version)
|
||||
|
||||
if _os.name == "nt":
|
||||
from _ctypes import FormatError
|
||||
|
||||
DEFAULT_MODE = RTLD_LOCAL
|
||||
if _os.name == "posix" and _sys.platform == "darwin":
|
||||
# On OS X 10.3, we use RTLD_GLOBAL as default mode
|
||||
# because RTLD_LOCAL does not work at least on some
|
||||
# libraries. OS X 10.3 is Darwin 7, so we check for
|
||||
# that.
|
||||
|
||||
if int(_os.uname().release.split('.')[0]) < 8:
|
||||
DEFAULT_MODE = RTLD_GLOBAL
|
||||
|
||||
from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \
|
||||
FUNCFLAG_PYTHONAPI as _FUNCFLAG_PYTHONAPI, \
|
||||
FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \
|
||||
FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR
|
||||
|
||||
# WINOLEAPI -> HRESULT
|
||||
# WINOLEAPI_(type)
|
||||
#
|
||||
# STDMETHODCALLTYPE
|
||||
#
|
||||
# STDMETHOD(name)
|
||||
# STDMETHOD_(type, name)
|
||||
#
|
||||
# STDAPICALLTYPE
|
||||
|
||||
def create_string_buffer(init, size=None):
|
||||
"""create_string_buffer(aBytes) -> character array
|
||||
create_string_buffer(anInteger) -> character array
|
||||
create_string_buffer(aBytes, anInteger) -> character array
|
||||
"""
|
||||
if isinstance(init, bytes):
|
||||
if size is None:
|
||||
size = len(init)+1
|
||||
_sys.audit("ctypes.create_string_buffer", init, size)
|
||||
buftype = c_char * size
|
||||
buf = buftype()
|
||||
buf.value = init
|
||||
return buf
|
||||
elif isinstance(init, int):
|
||||
_sys.audit("ctypes.create_string_buffer", None, init)
|
||||
buftype = c_char * init
|
||||
buf = buftype()
|
||||
return buf
|
||||
raise TypeError(init)
|
||||
|
||||
# Alias to create_string_buffer() for backward compatibility
|
||||
c_buffer = create_string_buffer
|
||||
|
||||
_c_functype_cache = {}
|
||||
def CFUNCTYPE(restype, *argtypes, **kw):
|
||||
"""CFUNCTYPE(restype, *argtypes,
|
||||
use_errno=False, use_last_error=False) -> function prototype.
|
||||
|
||||
restype: the result type
|
||||
argtypes: a sequence specifying the argument types
|
||||
|
||||
The function prototype can be called in different ways to create a
|
||||
callable object:
|
||||
|
||||
prototype(integer address) -> foreign function
|
||||
prototype(callable) -> create and return a C callable function from callable
|
||||
prototype(integer index, method name[, paramflags]) -> foreign function calling a COM method
|
||||
prototype((ordinal number, dll object)[, paramflags]) -> foreign function exported by ordinal
|
||||
prototype((function name, dll object)[, paramflags]) -> foreign function exported by name
|
||||
"""
|
||||
flags = _FUNCFLAG_CDECL
|
||||
if kw.pop("use_errno", False):
|
||||
flags |= _FUNCFLAG_USE_ERRNO
|
||||
if kw.pop("use_last_error", False):
|
||||
flags |= _FUNCFLAG_USE_LASTERROR
|
||||
if kw:
|
||||
raise ValueError("unexpected keyword argument(s) %s" % kw.keys())
|
||||
|
||||
try:
|
||||
return _c_functype_cache[(restype, argtypes, flags)]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
class CFunctionType(_CFuncPtr):
|
||||
_argtypes_ = argtypes
|
||||
_restype_ = restype
|
||||
_flags_ = flags
|
||||
_c_functype_cache[(restype, argtypes, flags)] = CFunctionType
|
||||
return CFunctionType
|
||||
|
||||
if _os.name == "nt":
|
||||
from _ctypes import LoadLibrary as _dlopen
|
||||
from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL
|
||||
|
||||
_win_functype_cache = {}
|
||||
def WINFUNCTYPE(restype, *argtypes, **kw):
|
||||
# docstring set later (very similar to CFUNCTYPE.__doc__)
|
||||
flags = _FUNCFLAG_STDCALL
|
||||
if kw.pop("use_errno", False):
|
||||
flags |= _FUNCFLAG_USE_ERRNO
|
||||
if kw.pop("use_last_error", False):
|
||||
flags |= _FUNCFLAG_USE_LASTERROR
|
||||
if kw:
|
||||
raise ValueError("unexpected keyword argument(s) %s" % kw.keys())
|
||||
|
||||
try:
|
||||
return _win_functype_cache[(restype, argtypes, flags)]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
class WinFunctionType(_CFuncPtr):
|
||||
_argtypes_ = argtypes
|
||||
_restype_ = restype
|
||||
_flags_ = flags
|
||||
_win_functype_cache[(restype, argtypes, flags)] = WinFunctionType
|
||||
return WinFunctionType
|
||||
if WINFUNCTYPE.__doc__:
|
||||
WINFUNCTYPE.__doc__ = CFUNCTYPE.__doc__.replace("CFUNCTYPE", "WINFUNCTYPE")
|
||||
|
||||
elif _os.name == "posix":
|
||||
from _ctypes import dlopen as _dlopen
|
||||
|
||||
from _ctypes import sizeof, byref, addressof, alignment, resize
|
||||
from _ctypes import get_errno, set_errno
|
||||
from _ctypes import _SimpleCData
|
||||
|
||||
def _check_size(typ, typecode=None):
|
||||
# Check if sizeof(ctypes_type) against struct.calcsize. This
|
||||
# should protect somewhat against a misconfigured libffi.
|
||||
from struct import calcsize
|
||||
if typecode is None:
|
||||
# Most _type_ codes are the same as used in struct
|
||||
typecode = typ._type_
|
||||
actual, required = sizeof(typ), calcsize(typecode)
|
||||
if actual != required:
|
||||
raise SystemError("sizeof(%s) wrong: %d instead of %d" % \
|
||||
(typ, actual, required))
|
||||
|
||||
class py_object(_SimpleCData):
|
||||
_type_ = "O"
|
||||
def __repr__(self):
|
||||
try:
|
||||
return super().__repr__()
|
||||
except ValueError:
|
||||
return "%s(<NULL>)" % type(self).__name__
|
||||
_check_size(py_object, "P")
|
||||
|
||||
class c_short(_SimpleCData):
|
||||
_type_ = "h"
|
||||
_check_size(c_short)
|
||||
|
||||
class c_ushort(_SimpleCData):
|
||||
_type_ = "H"
|
||||
_check_size(c_ushort)
|
||||
|
||||
class c_long(_SimpleCData):
|
||||
_type_ = "l"
|
||||
_check_size(c_long)
|
||||
|
||||
class c_ulong(_SimpleCData):
|
||||
_type_ = "L"
|
||||
_check_size(c_ulong)
|
||||
|
||||
if _calcsize("i") == _calcsize("l"):
|
||||
# if int and long have the same size, make c_int an alias for c_long
|
||||
c_int = c_long
|
||||
c_uint = c_ulong
|
||||
else:
|
||||
class c_int(_SimpleCData):
|
||||
_type_ = "i"
|
||||
_check_size(c_int)
|
||||
|
||||
class c_uint(_SimpleCData):
|
||||
_type_ = "I"
|
||||
_check_size(c_uint)
|
||||
|
||||
class c_float(_SimpleCData):
|
||||
_type_ = "f"
|
||||
_check_size(c_float)
|
||||
|
||||
class c_double(_SimpleCData):
|
||||
_type_ = "d"
|
||||
_check_size(c_double)
|
||||
|
||||
class c_longdouble(_SimpleCData):
|
||||
_type_ = "g"
|
||||
if sizeof(c_longdouble) == sizeof(c_double):
|
||||
c_longdouble = c_double
|
||||
|
||||
if _calcsize("l") == _calcsize("q"):
|
||||
# if long and long long have the same size, make c_longlong an alias for c_long
|
||||
c_longlong = c_long
|
||||
c_ulonglong = c_ulong
|
||||
else:
|
||||
class c_longlong(_SimpleCData):
|
||||
_type_ = "q"
|
||||
_check_size(c_longlong)
|
||||
|
||||
class c_ulonglong(_SimpleCData):
|
||||
_type_ = "Q"
|
||||
## def from_param(cls, val):
|
||||
## return ('d', float(val), val)
|
||||
## from_param = classmethod(from_param)
|
||||
_check_size(c_ulonglong)
|
||||
|
||||
class c_ubyte(_SimpleCData):
|
||||
_type_ = "B"
|
||||
c_ubyte.__ctype_le__ = c_ubyte.__ctype_be__ = c_ubyte
|
||||
# backward compatibility:
|
||||
##c_uchar = c_ubyte
|
||||
_check_size(c_ubyte)
|
||||
|
||||
class c_byte(_SimpleCData):
|
||||
_type_ = "b"
|
||||
c_byte.__ctype_le__ = c_byte.__ctype_be__ = c_byte
|
||||
_check_size(c_byte)
|
||||
|
||||
class c_char(_SimpleCData):
|
||||
_type_ = "c"
|
||||
c_char.__ctype_le__ = c_char.__ctype_be__ = c_char
|
||||
_check_size(c_char)
|
||||
|
||||
class c_char_p(_SimpleCData):
|
||||
_type_ = "z"
|
||||
def __repr__(self):
|
||||
return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value)
|
||||
_check_size(c_char_p, "P")
|
||||
|
||||
class c_void_p(_SimpleCData):
|
||||
_type_ = "P"
|
||||
c_voidp = c_void_p # backwards compatibility (to a bug)
|
||||
_check_size(c_void_p)
|
||||
|
||||
class c_bool(_SimpleCData):
|
||||
_type_ = "?"
|
||||
|
||||
from _ctypes import POINTER, pointer, _pointer_type_cache
|
||||
|
||||
class c_wchar_p(_SimpleCData):
|
||||
_type_ = "Z"
|
||||
def __repr__(self):
|
||||
return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value)
|
||||
|
||||
class c_wchar(_SimpleCData):
|
||||
_type_ = "u"
|
||||
|
||||
def _reset_cache():
|
||||
_pointer_type_cache.clear()
|
||||
_c_functype_cache.clear()
|
||||
if _os.name == "nt":
|
||||
_win_functype_cache.clear()
|
||||
# _SimpleCData.c_wchar_p_from_param
|
||||
POINTER(c_wchar).from_param = c_wchar_p.from_param
|
||||
# _SimpleCData.c_char_p_from_param
|
||||
POINTER(c_char).from_param = c_char_p.from_param
|
||||
_pointer_type_cache[None] = c_void_p
|
||||
|
||||
def create_unicode_buffer(init, size=None):
|
||||
"""create_unicode_buffer(aString) -> character array
|
||||
create_unicode_buffer(anInteger) -> character array
|
||||
create_unicode_buffer(aString, anInteger) -> character array
|
||||
"""
|
||||
if isinstance(init, str):
|
||||
if size is None:
|
||||
if sizeof(c_wchar) == 2:
|
||||
# UTF-16 requires a surrogate pair (2 wchar_t) for non-BMP
|
||||
# characters (outside [U+0000; U+FFFF] range). +1 for trailing
|
||||
# NUL character.
|
||||
size = sum(2 if ord(c) > 0xFFFF else 1 for c in init) + 1
|
||||
else:
|
||||
# 32-bit wchar_t (1 wchar_t per Unicode character). +1 for
|
||||
# trailing NUL character.
|
||||
size = len(init) + 1
|
||||
_sys.audit("ctypes.create_unicode_buffer", init, size)
|
||||
buftype = c_wchar * size
|
||||
buf = buftype()
|
||||
buf.value = init
|
||||
return buf
|
||||
elif isinstance(init, int):
|
||||
_sys.audit("ctypes.create_unicode_buffer", None, init)
|
||||
buftype = c_wchar * init
|
||||
buf = buftype()
|
||||
return buf
|
||||
raise TypeError(init)
|
||||
|
||||
|
||||
# XXX Deprecated
|
||||
def SetPointerType(pointer, cls):
|
||||
if _pointer_type_cache.get(cls, None) is not None:
|
||||
raise RuntimeError("This type already exists in the cache")
|
||||
if id(pointer) not in _pointer_type_cache:
|
||||
raise RuntimeError("What's this???")
|
||||
pointer.set_type(cls)
|
||||
_pointer_type_cache[cls] = pointer
|
||||
del _pointer_type_cache[id(pointer)]
|
||||
|
||||
# XXX Deprecated
|
||||
def ARRAY(typ, len):
|
||||
return typ * len
|
||||
|
||||
################################################################
|
||||
|
||||
|
||||
class CDLL(object):
|
||||
"""An instance of this class represents a loaded dll/shared
|
||||
library, exporting functions using the standard C calling
|
||||
convention (named 'cdecl' on Windows).
|
||||
|
||||
The exported functions can be accessed as attributes, or by
|
||||
indexing with the function name. Examples:
|
||||
|
||||
<obj>.qsort -> callable object
|
||||
<obj>['qsort'] -> callable object
|
||||
|
||||
Calling the functions releases the Python GIL during the call and
|
||||
reacquires it afterwards.
|
||||
"""
|
||||
_func_flags_ = _FUNCFLAG_CDECL
|
||||
_func_restype_ = c_int
|
||||
# default values for repr
|
||||
_name = '<uninitialized>'
|
||||
_handle = 0
|
||||
_FuncPtr = None
|
||||
|
||||
def __init__(self, name, mode=DEFAULT_MODE, handle=None,
|
||||
use_errno=False,
|
||||
use_last_error=False,
|
||||
winmode=None):
|
||||
self._name = name
|
||||
flags = self._func_flags_
|
||||
if use_errno:
|
||||
flags |= _FUNCFLAG_USE_ERRNO
|
||||
if use_last_error:
|
||||
flags |= _FUNCFLAG_USE_LASTERROR
|
||||
if _sys.platform.startswith("aix"):
|
||||
"""When the name contains ".a(" and ends with ")",
|
||||
e.g., "libFOO.a(libFOO.so)" - this is taken to be an
|
||||
archive(member) syntax for dlopen(), and the mode is adjusted.
|
||||
Otherwise, name is presented to dlopen() as a file argument.
|
||||
"""
|
||||
if name and name.endswith(")") and ".a(" in name:
|
||||
mode |= ( _os.RTLD_MEMBER | _os.RTLD_NOW )
|
||||
if _os.name == "nt":
|
||||
if winmode is not None:
|
||||
mode = winmode
|
||||
else:
|
||||
import nt
|
||||
mode = nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS
|
||||
if '/' in name or '\\' in name:
|
||||
self._name = nt._getfullpathname(self._name)
|
||||
mode |= nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR
|
||||
|
||||
class _FuncPtr(_CFuncPtr):
|
||||
_flags_ = flags
|
||||
_restype_ = self._func_restype_
|
||||
self._FuncPtr = _FuncPtr
|
||||
|
||||
if handle is None:
|
||||
self._handle = _dlopen(self._name, mode)
|
||||
else:
|
||||
self._handle = handle
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s '%s', handle %x at %#x>" % \
|
||||
(self.__class__.__name__, self._name,
|
||||
(self._handle & (_sys.maxsize*2 + 1)),
|
||||
id(self) & (_sys.maxsize*2 + 1))
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name.startswith('__') and name.endswith('__'):
|
||||
raise AttributeError(name)
|
||||
func = self.__getitem__(name)
|
||||
setattr(self, name, func)
|
||||
return func
|
||||
|
||||
def __getitem__(self, name_or_ordinal):
|
||||
func = self._FuncPtr((name_or_ordinal, self))
|
||||
if not isinstance(name_or_ordinal, int):
|
||||
func.__name__ = name_or_ordinal
|
||||
return func
|
||||
|
||||
class PyDLL(CDLL):
|
||||
"""This class represents the Python library itself. It allows
|
||||
accessing Python API functions. The GIL is not released, and
|
||||
Python exceptions are handled correctly.
|
||||
"""
|
||||
_func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
|
||||
|
||||
if _os.name == "nt":
|
||||
|
||||
class WinDLL(CDLL):
|
||||
"""This class represents a dll exporting functions using the
|
||||
Windows stdcall calling convention.
|
||||
"""
|
||||
_func_flags_ = _FUNCFLAG_STDCALL
|
||||
|
||||
# XXX Hm, what about HRESULT as normal parameter?
|
||||
# Mustn't it derive from c_long then?
|
||||
from _ctypes import _check_HRESULT, _SimpleCData
|
||||
class HRESULT(_SimpleCData):
|
||||
_type_ = "l"
|
||||
# _check_retval_ is called with the function's result when it
|
||||
# is used as restype. It checks for the FAILED bit, and
|
||||
# raises an OSError if it is set.
|
||||
#
|
||||
# The _check_retval_ method is implemented in C, so that the
|
||||
# method definition itself is not included in the traceback
|
||||
# when it raises an error - that is what we want (and Python
|
||||
# doesn't have a way to raise an exception in the caller's
|
||||
# frame).
|
||||
_check_retval_ = _check_HRESULT
|
||||
|
||||
class OleDLL(CDLL):
|
||||
"""This class represents a dll exporting functions using the
|
||||
Windows stdcall calling convention, and returning HRESULT.
|
||||
HRESULT error values are automatically raised as OSError
|
||||
exceptions.
|
||||
"""
|
||||
_func_flags_ = _FUNCFLAG_STDCALL
|
||||
_func_restype_ = HRESULT
|
||||
|
||||
class LibraryLoader(object):
|
||||
def __init__(self, dlltype):
|
||||
self._dlltype = dlltype
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name[0] == '_':
|
||||
raise AttributeError(name)
|
||||
try:
|
||||
dll = self._dlltype(name)
|
||||
except OSError:
|
||||
raise AttributeError(name)
|
||||
setattr(self, name, dll)
|
||||
return dll
|
||||
|
||||
def __getitem__(self, name):
|
||||
return getattr(self, name)
|
||||
|
||||
def LoadLibrary(self, name):
|
||||
return self._dlltype(name)
|
||||
|
||||
__class_getitem__ = classmethod(_types.GenericAlias)
|
||||
|
||||
cdll = LibraryLoader(CDLL)
|
||||
pydll = LibraryLoader(PyDLL)
|
||||
|
||||
if _os.name == "nt":
|
||||
pythonapi = PyDLL("python dll", None, _sys.dllhandle)
|
||||
elif _sys.platform == "cygwin":
|
||||
pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
|
||||
else:
|
||||
pythonapi = PyDLL(None)
|
||||
|
||||
|
||||
if _os.name == "nt":
|
||||
windll = LibraryLoader(WinDLL)
|
||||
oledll = LibraryLoader(OleDLL)
|
||||
|
||||
GetLastError = windll.kernel32.GetLastError
|
||||
from _ctypes import get_last_error, set_last_error
|
||||
|
||||
def WinError(code=None, descr=None):
|
||||
if code is None:
|
||||
code = GetLastError()
|
||||
if descr is None:
|
||||
descr = FormatError(code).strip()
|
||||
return OSError(None, descr, None, code)
|
||||
|
||||
if sizeof(c_uint) == sizeof(c_void_p):
|
||||
c_size_t = c_uint
|
||||
c_ssize_t = c_int
|
||||
elif sizeof(c_ulong) == sizeof(c_void_p):
|
||||
c_size_t = c_ulong
|
||||
c_ssize_t = c_long
|
||||
elif sizeof(c_ulonglong) == sizeof(c_void_p):
|
||||
c_size_t = c_ulonglong
|
||||
c_ssize_t = c_longlong
|
||||
|
||||
# functions
|
||||
|
||||
from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr
|
||||
|
||||
## void *memmove(void *, const void *, size_t);
|
||||
memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr)
|
||||
|
||||
## void *memset(void *, int, size_t)
|
||||
memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr)
|
||||
|
||||
def PYFUNCTYPE(restype, *argtypes):
|
||||
class CFunctionType(_CFuncPtr):
|
||||
_argtypes_ = argtypes
|
||||
_restype_ = restype
|
||||
_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
|
||||
return CFunctionType
|
||||
|
||||
_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
|
||||
def cast(obj, typ):
|
||||
return _cast(obj, obj, typ)
|
||||
|
||||
_string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
|
||||
def string_at(ptr, size=-1):
|
||||
"""string_at(addr[, size]) -> string
|
||||
|
||||
Return the string at addr."""
|
||||
return _string_at(ptr, size)
|
||||
|
||||
try:
|
||||
from _ctypes import _wstring_at_addr
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
_wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr)
|
||||
def wstring_at(ptr, size=-1):
|
||||
"""wstring_at(addr[, size]) -> string
|
||||
|
||||
Return the string at addr."""
|
||||
return _wstring_at(ptr, size)
|
||||
|
||||
|
||||
if _os.name == "nt": # COM stuff
|
||||
def DllGetClassObject(rclsid, riid, ppv):
|
||||
try:
|
||||
ccom = __import__("comtypes.server.inprocserver", globals(), locals(), ['*'])
|
||||
except ImportError:
|
||||
return -2147221231 # CLASS_E_CLASSNOTAVAILABLE
|
||||
else:
|
||||
return ccom.DllGetClassObject(rclsid, riid, ppv)
|
||||
|
||||
def DllCanUnloadNow():
|
||||
try:
|
||||
ccom = __import__("comtypes.server.inprocserver", globals(), locals(), ['*'])
|
||||
except ImportError:
|
||||
return 0 # S_OK
|
||||
return ccom.DllCanUnloadNow()
|
||||
|
||||
from ctypes._endian import BigEndianStructure, LittleEndianStructure
|
||||
from ctypes._endian import BigEndianUnion, LittleEndianUnion
|
||||
|
||||
# Fill in specifically-sized types
|
||||
c_int8 = c_byte
|
||||
c_uint8 = c_ubyte
|
||||
for kind in [c_short, c_int, c_long, c_longlong]:
|
||||
if sizeof(kind) == 2: c_int16 = kind
|
||||
elif sizeof(kind) == 4: c_int32 = kind
|
||||
elif sizeof(kind) == 8: c_int64 = kind
|
||||
for kind in [c_ushort, c_uint, c_ulong, c_ulonglong]:
|
||||
if sizeof(kind) == 2: c_uint16 = kind
|
||||
elif sizeof(kind) == 4: c_uint32 = kind
|
||||
elif sizeof(kind) == 8: c_uint64 = kind
|
||||
del(kind)
|
||||
|
||||
if SIZEOF_TIME_T == 8:
|
||||
c_time_t = c_int64
|
||||
elif SIZEOF_TIME_T == 4:
|
||||
c_time_t = c_int32
|
||||
else:
|
||||
raise SystemError(f"Unexpected sizeof(time_t): {SIZEOF_TIME_T=}")
|
||||
|
||||
_reset_cache()
|
||||
@@ -1,327 +0,0 @@
|
||||
"""
|
||||
Lib/ctypes.util.find_library() support for AIX
|
||||
Similar approach as done for Darwin support by using separate files
|
||||
but unlike Darwin - no extension such as ctypes.macholib.*
|
||||
|
||||
dlopen() is an interface to AIX initAndLoad() - primary documentation at:
|
||||
https://www.ibm.com/support/knowledgecenter/en/ssw_aix_61/com.ibm.aix.basetrf1/dlopen.htm
|
||||
https://www.ibm.com/support/knowledgecenter/en/ssw_aix_61/com.ibm.aix.basetrf1/load.htm
|
||||
|
||||
AIX supports two styles for dlopen(): svr4 (System V Release 4) which is common on posix
|
||||
platforms, but also a BSD style - aka SVR3.
|
||||
|
||||
From AIX 5.3 Difference Addendum (December 2004)
|
||||
2.9 SVR4 linking affinity
|
||||
Nowadays, there are two major object file formats used by the operating systems:
|
||||
XCOFF: The COFF enhanced by IBM and others. The original COFF (Common
|
||||
Object File Format) was the base of SVR3 and BSD 4.2 systems.
|
||||
ELF: Executable and Linking Format that was developed by AT&T and is a
|
||||
base for SVR4 UNIX.
|
||||
|
||||
While the shared library content is identical on AIX - one is located as a filepath name
|
||||
(svr4 style) and the other is located as a member of an archive (and the archive
|
||||
is located as a filepath name).
|
||||
|
||||
The key difference arises when supporting multiple abi formats (i.e., 32 and 64 bit).
|
||||
For svr4 either only one ABI is supported, or there are two directories, or there
|
||||
are different file names. The most common solution for multiple ABI is multiple
|
||||
directories.
|
||||
|
||||
For the XCOFF (aka AIX) style - one directory (one archive file) is sufficient
|
||||
as multiple shared libraries can be in the archive - even sharing the same name.
|
||||
In documentation the archive is also referred to as the "base" and the shared
|
||||
library object is referred to as the "member".
|
||||
|
||||
For dlopen() on AIX (read initAndLoad()) the calls are similar.
|
||||
Default activity occurs when no path information is provided. When path
|
||||
information is provided dlopen() does not search any other directories.
|
||||
|
||||
For SVR4 - the shared library name is the name of the file expected: libFOO.so
|
||||
For AIX - the shared library is expressed as base(member). The search is for the
|
||||
base (e.g., libFOO.a) and once the base is found the shared library - identified by
|
||||
member (e.g., libFOO.so, or shr.o) is located and loaded.
|
||||
|
||||
The mode bit RTLD_MEMBER tells initAndLoad() that it needs to use the AIX (SVR3)
|
||||
naming style.
|
||||
"""
|
||||
__author__ = "Michael Felt <aixtools@felt.demon.nl>"
|
||||
|
||||
import re
|
||||
from os import environ, path
|
||||
from sys import executable
|
||||
from ctypes import c_void_p, sizeof
|
||||
from subprocess import Popen, PIPE, DEVNULL
|
||||
|
||||
# Executable bit size - 32 or 64
|
||||
# Used to filter the search in an archive by size, e.g., -X64
|
||||
AIX_ABI = sizeof(c_void_p) * 8
|
||||
|
||||
|
||||
from sys import maxsize
|
||||
def _last_version(libnames, sep):
|
||||
def _num_version(libname):
|
||||
# "libxyz.so.MAJOR.MINOR" => [MAJOR, MINOR]
|
||||
parts = libname.split(sep)
|
||||
nums = []
|
||||
try:
|
||||
while parts:
|
||||
nums.insert(0, int(parts.pop()))
|
||||
except ValueError:
|
||||
pass
|
||||
return nums or [maxsize]
|
||||
return max(reversed(libnames), key=_num_version)
|
||||
|
||||
def get_ld_header(p):
|
||||
# "nested-function, but placed at module level
|
||||
ld_header = None
|
||||
for line in p.stdout:
|
||||
if line.startswith(('/', './', '../')):
|
||||
ld_header = line
|
||||
elif "INDEX" in line:
|
||||
return ld_header.rstrip('\n')
|
||||
return None
|
||||
|
||||
def get_ld_header_info(p):
|
||||
# "nested-function, but placed at module level
|
||||
# as an ld_header was found, return known paths, archives and members
|
||||
# these lines start with a digit
|
||||
info = []
|
||||
for line in p.stdout:
|
||||
if re.match("[0-9]", line):
|
||||
info.append(line)
|
||||
else:
|
||||
# blank line (separator), consume line and end for loop
|
||||
break
|
||||
return info
|
||||
|
||||
def get_ld_headers(file):
|
||||
"""
|
||||
Parse the header of the loader section of executable and archives
|
||||
This function calls /usr/bin/dump -H as a subprocess
|
||||
and returns a list of (ld_header, ld_header_info) tuples.
|
||||
"""
|
||||
# get_ld_headers parsing:
|
||||
# 1. Find a line that starts with /, ./, or ../ - set as ld_header
|
||||
# 2. If "INDEX" in occurs in a following line - return ld_header
|
||||
# 3. get info (lines starting with [0-9])
|
||||
ldr_headers = []
|
||||
p = Popen(["/usr/bin/dump", f"-X{AIX_ABI}", "-H", file],
|
||||
universal_newlines=True, stdout=PIPE, stderr=DEVNULL)
|
||||
# be sure to read to the end-of-file - getting all entries
|
||||
while ld_header := get_ld_header(p):
|
||||
ldr_headers.append((ld_header, get_ld_header_info(p)))
|
||||
p.stdout.close()
|
||||
p.wait()
|
||||
return ldr_headers
|
||||
|
||||
def get_shared(ld_headers):
|
||||
"""
|
||||
extract the shareable objects from ld_headers
|
||||
character "[" is used to strip off the path information.
|
||||
Note: the "[" and "]" characters that are part of dump -H output
|
||||
are not removed here.
|
||||
"""
|
||||
shared = []
|
||||
for (line, _) in ld_headers:
|
||||
# potential member lines contain "["
|
||||
# otherwise, no processing needed
|
||||
if "[" in line:
|
||||
# Strip off trailing colon (:)
|
||||
shared.append(line[line.index("["):-1])
|
||||
return shared
|
||||
|
||||
def get_one_match(expr, lines):
|
||||
"""
|
||||
Must be only one match, otherwise result is None.
|
||||
When there is a match, strip leading "[" and trailing "]"
|
||||
"""
|
||||
# member names in the ld_headers output are between square brackets
|
||||
expr = rf'\[({expr})\]'
|
||||
matches = list(filter(None, (re.search(expr, line) for line in lines)))
|
||||
if len(matches) == 1:
|
||||
return matches[0].group(1)
|
||||
else:
|
||||
return None
|
||||
|
||||
# additional processing to deal with AIX legacy names for 64-bit members
|
||||
def get_legacy(members):
|
||||
"""
|
||||
This routine provides historical aka legacy naming schemes started
|
||||
in AIX4 shared library support for library members names.
|
||||
e.g., in /usr/lib/libc.a the member name shr.o for 32-bit binary and
|
||||
shr_64.o for 64-bit binary.
|
||||
"""
|
||||
if AIX_ABI == 64:
|
||||
# AIX 64-bit member is one of shr64.o, shr_64.o, or shr4_64.o
|
||||
expr = r'shr4?_?64\.o'
|
||||
member = get_one_match(expr, members)
|
||||
if member:
|
||||
return member
|
||||
else:
|
||||
# 32-bit legacy names - both shr.o and shr4.o exist.
|
||||
# shr.o is the preferred name so we look for shr.o first
|
||||
# i.e., shr4.o is returned only when shr.o does not exist
|
||||
for name in ['shr.o', 'shr4.o']:
|
||||
member = get_one_match(re.escape(name), members)
|
||||
if member:
|
||||
return member
|
||||
return None
|
||||
|
||||
def get_version(name, members):
|
||||
"""
|
||||
Sort list of members and return highest numbered version - if it exists.
|
||||
This function is called when an unversioned libFOO.a(libFOO.so) has
|
||||
not been found.
|
||||
|
||||
Versioning for the member name is expected to follow
|
||||
GNU LIBTOOL conventions: the highest version (x, then X.y, then X.Y.z)
|
||||
* find [libFoo.so.X]
|
||||
* find [libFoo.so.X.Y]
|
||||
* find [libFoo.so.X.Y.Z]
|
||||
|
||||
Before the GNU convention became the standard scheme regardless of
|
||||
binary size AIX packagers used GNU convention "as-is" for 32-bit
|
||||
archive members but used an "distinguishing" name for 64-bit members.
|
||||
This scheme inserted either 64 or _64 between libFOO and .so
|
||||
- generally libFOO_64.so, but occasionally libFOO64.so
|
||||
"""
|
||||
# the expression ending for versions must start as
|
||||
# '.so.[0-9]', i.e., *.so.[at least one digit]
|
||||
# while multiple, more specific expressions could be specified
|
||||
# to search for .so.X, .so.X.Y and .so.X.Y.Z
|
||||
# after the first required 'dot' digit
|
||||
# any combination of additional 'dot' digits pairs are accepted
|
||||
# anything more than libFOO.so.digits.digits.digits
|
||||
# should be seen as a member name outside normal expectations
|
||||
exprs = [rf'lib{name}\.so\.[0-9]+[0-9.]*',
|
||||
rf'lib{name}_?64\.so\.[0-9]+[0-9.]*']
|
||||
for expr in exprs:
|
||||
versions = []
|
||||
for line in members:
|
||||
m = re.search(expr, line)
|
||||
if m:
|
||||
versions.append(m.group(0))
|
||||
if versions:
|
||||
return _last_version(versions, '.')
|
||||
return None
|
||||
|
||||
def get_member(name, members):
|
||||
"""
|
||||
Return an archive member matching the request in name.
|
||||
Name is the library name without any prefix like lib, suffix like .so,
|
||||
or version number.
|
||||
Given a list of members find and return the most appropriate result
|
||||
Priority is given to generic libXXX.so, then a versioned libXXX.so.a.b.c
|
||||
and finally, legacy AIX naming scheme.
|
||||
"""
|
||||
# look first for a generic match - prepend lib and append .so
|
||||
expr = rf'lib{name}\.so'
|
||||
member = get_one_match(expr, members)
|
||||
if member:
|
||||
return member
|
||||
elif AIX_ABI == 64:
|
||||
expr = rf'lib{name}64\.so'
|
||||
member = get_one_match(expr, members)
|
||||
if member:
|
||||
return member
|
||||
# since an exact match with .so as suffix was not found
|
||||
# look for a versioned name
|
||||
# If a versioned name is not found, look for AIX legacy member name
|
||||
member = get_version(name, members)
|
||||
if member:
|
||||
return member
|
||||
else:
|
||||
return get_legacy(members)
|
||||
|
||||
def get_libpaths():
|
||||
"""
|
||||
On AIX, the buildtime searchpath is stored in the executable.
|
||||
as "loader header information".
|
||||
The command /usr/bin/dump -H extracts this info.
|
||||
Prefix searched libraries with LD_LIBRARY_PATH (preferred),
|
||||
or LIBPATH if defined. These paths are appended to the paths
|
||||
to libraries the python executable is linked with.
|
||||
This mimics AIX dlopen() behavior.
|
||||
"""
|
||||
libpaths = environ.get("LD_LIBRARY_PATH")
|
||||
if libpaths is None:
|
||||
libpaths = environ.get("LIBPATH")
|
||||
if libpaths is None:
|
||||
libpaths = []
|
||||
else:
|
||||
libpaths = libpaths.split(":")
|
||||
objects = get_ld_headers(executable)
|
||||
for (_, lines) in objects:
|
||||
for line in lines:
|
||||
# the second (optional) argument is PATH if it includes a /
|
||||
path = line.split()[1]
|
||||
if "/" in path:
|
||||
libpaths.extend(path.split(":"))
|
||||
return libpaths
|
||||
|
||||
def find_shared(paths, name):
|
||||
"""
|
||||
paths is a list of directories to search for an archive.
|
||||
name is the abbreviated name given to find_library().
|
||||
Process: search "paths" for archive, and if an archive is found
|
||||
return the result of get_member().
|
||||
If an archive is not found then return None
|
||||
"""
|
||||
for dir in paths:
|
||||
# /lib is a symbolic link to /usr/lib, skip it
|
||||
if dir == "/lib":
|
||||
continue
|
||||
# "lib" is prefixed to emulate compiler name resolution,
|
||||
# e.g., -lc to libc
|
||||
base = f'lib{name}.a'
|
||||
archive = path.join(dir, base)
|
||||
if path.exists(archive):
|
||||
members = get_shared(get_ld_headers(archive))
|
||||
member = get_member(re.escape(name), members)
|
||||
if member is not None:
|
||||
return (base, member)
|
||||
else:
|
||||
return (None, None)
|
||||
return (None, None)
|
||||
|
||||
def find_library(name):
|
||||
"""AIX implementation of ctypes.util.find_library()
|
||||
Find an archive member that will dlopen(). If not available,
|
||||
also search for a file (or link) with a .so suffix.
|
||||
|
||||
AIX supports two types of schemes that can be used with dlopen().
|
||||
The so-called SystemV Release4 (svr4) format is commonly suffixed
|
||||
with .so while the (default) AIX scheme has the library (archive)
|
||||
ending with the suffix .a
|
||||
As an archive has multiple members (e.g., 32-bit and 64-bit) in one file
|
||||
the argument passed to dlopen must include both the library and
|
||||
the member names in a single string.
|
||||
|
||||
find_library() looks first for an archive (.a) with a suitable member.
|
||||
If no archive+member pair is found, look for a .so file.
|
||||
"""
|
||||
|
||||
libpaths = get_libpaths()
|
||||
(base, member) = find_shared(libpaths, name)
|
||||
if base is not None:
|
||||
return f"{base}({member})"
|
||||
|
||||
# To get here, a member in an archive has not been found
|
||||
# In other words, either:
|
||||
# a) a .a file was not found
|
||||
# b) a .a file did not have a suitable member
|
||||
# So, look for a .so file
|
||||
# Check libpaths for .so file
|
||||
# Note, the installation must prepare a link from a .so
|
||||
# to a versioned file
|
||||
# This is common practice by GNU libtool on other platforms
|
||||
soname = f"lib{name}.so"
|
||||
for dir in libpaths:
|
||||
# /lib is a symbolic link to /usr/lib, skip it
|
||||
if dir == "/lib":
|
||||
continue
|
||||
shlib = path.join(dir, soname)
|
||||
if path.exists(shlib):
|
||||
return soname
|
||||
# if we are here, we have not found anything plausible
|
||||
return None
|
||||
@@ -1,78 +0,0 @@
|
||||
import sys
|
||||
from ctypes import *
|
||||
|
||||
_array_type = type(Array)
|
||||
|
||||
def _other_endian(typ):
|
||||
"""Return the type with the 'other' byte order. Simple types like
|
||||
c_int and so on already have __ctype_be__ and __ctype_le__
|
||||
attributes which contain the types, for more complicated types
|
||||
arrays and structures are supported.
|
||||
"""
|
||||
# check _OTHER_ENDIAN attribute (present if typ is primitive type)
|
||||
if hasattr(typ, _OTHER_ENDIAN):
|
||||
return getattr(typ, _OTHER_ENDIAN)
|
||||
# if typ is array
|
||||
if isinstance(typ, _array_type):
|
||||
return _other_endian(typ._type_) * typ._length_
|
||||
# if typ is structure
|
||||
if issubclass(typ, Structure):
|
||||
return typ
|
||||
raise TypeError("This type does not support other endian: %s" % typ)
|
||||
|
||||
class _swapped_meta:
|
||||
def __setattr__(self, attrname, value):
|
||||
if attrname == "_fields_":
|
||||
fields = []
|
||||
for desc in value:
|
||||
name = desc[0]
|
||||
typ = desc[1]
|
||||
rest = desc[2:]
|
||||
fields.append((name, _other_endian(typ)) + rest)
|
||||
value = fields
|
||||
super().__setattr__(attrname, value)
|
||||
class _swapped_struct_meta(_swapped_meta, type(Structure)): pass
|
||||
class _swapped_union_meta(_swapped_meta, type(Union)): pass
|
||||
|
||||
################################################################
|
||||
|
||||
# Note: The Structure metaclass checks for the *presence* (not the
|
||||
# value!) of a _swapped_bytes_ attribute to determine the bit order in
|
||||
# structures containing bit fields.
|
||||
|
||||
if sys.byteorder == "little":
|
||||
_OTHER_ENDIAN = "__ctype_be__"
|
||||
|
||||
LittleEndianStructure = Structure
|
||||
|
||||
class BigEndianStructure(Structure, metaclass=_swapped_struct_meta):
|
||||
"""Structure with big endian byte order"""
|
||||
__slots__ = ()
|
||||
_swappedbytes_ = None
|
||||
|
||||
LittleEndianUnion = Union
|
||||
|
||||
class BigEndianUnion(Union, metaclass=_swapped_union_meta):
|
||||
"""Union with big endian byte order"""
|
||||
__slots__ = ()
|
||||
_swappedbytes_ = None
|
||||
|
||||
elif sys.byteorder == "big":
|
||||
_OTHER_ENDIAN = "__ctype_le__"
|
||||
|
||||
BigEndianStructure = Structure
|
||||
|
||||
class LittleEndianStructure(Structure, metaclass=_swapped_struct_meta):
|
||||
"""Structure with little endian byte order"""
|
||||
__slots__ = ()
|
||||
_swappedbytes_ = None
|
||||
|
||||
BigEndianUnion = Union
|
||||
|
||||
class LittleEndianUnion(Union, metaclass=_swapped_union_meta):
|
||||
"""Union with little endian byte order"""
|
||||
__slots__ = ()
|
||||
_swappedbytes_ = None
|
||||
|
||||
else:
|
||||
raise RuntimeError("Invalid byteorder")
|
||||
@@ -1,7 +0,0 @@
|
||||
Files in this directory come from Bob Ippolito's py2app.
|
||||
|
||||
License: Any components of the py2app suite may be distributed under
|
||||
the MIT or PSF open source licenses.
|
||||
|
||||
This is version 1.0, SVN revision 789, from 2006/01/25.
|
||||
The main repository is http://svn.red-bean.com/bob/macholib/trunk/macholib/
|
||||
@@ -1,9 +0,0 @@
|
||||
"""
|
||||
Enough Mach-O to make your head spin.
|
||||
|
||||
See the relevant header files in /usr/include/mach-o
|
||||
|
||||
And also Apple's documentation.
|
||||
"""
|
||||
|
||||
__version__ = '1.0'
|
||||
@@ -1,165 +0,0 @@
|
||||
"""
|
||||
dyld emulation
|
||||
"""
|
||||
|
||||
import os
|
||||
from ctypes.macholib.framework import framework_info
|
||||
from ctypes.macholib.dylib import dylib_info
|
||||
from itertools import *
|
||||
try:
|
||||
from _ctypes import _dyld_shared_cache_contains_path
|
||||
except ImportError:
|
||||
def _dyld_shared_cache_contains_path(*args):
|
||||
raise NotImplementedError
|
||||
|
||||
__all__ = [
|
||||
'dyld_find', 'framework_find',
|
||||
'framework_info', 'dylib_info',
|
||||
]
|
||||
|
||||
# These are the defaults as per man dyld(1)
|
||||
#
|
||||
DEFAULT_FRAMEWORK_FALLBACK = [
|
||||
os.path.expanduser("~/Library/Frameworks"),
|
||||
"/Library/Frameworks",
|
||||
"/Network/Library/Frameworks",
|
||||
"/System/Library/Frameworks",
|
||||
]
|
||||
|
||||
DEFAULT_LIBRARY_FALLBACK = [
|
||||
os.path.expanduser("~/lib"),
|
||||
"/usr/local/lib",
|
||||
"/lib",
|
||||
"/usr/lib",
|
||||
]
|
||||
|
||||
def dyld_env(env, var):
|
||||
if env is None:
|
||||
env = os.environ
|
||||
rval = env.get(var)
|
||||
if rval is None:
|
||||
return []
|
||||
return rval.split(':')
|
||||
|
||||
def dyld_image_suffix(env=None):
|
||||
if env is None:
|
||||
env = os.environ
|
||||
return env.get('DYLD_IMAGE_SUFFIX')
|
||||
|
||||
def dyld_framework_path(env=None):
|
||||
return dyld_env(env, 'DYLD_FRAMEWORK_PATH')
|
||||
|
||||
def dyld_library_path(env=None):
|
||||
return dyld_env(env, 'DYLD_LIBRARY_PATH')
|
||||
|
||||
def dyld_fallback_framework_path(env=None):
|
||||
return dyld_env(env, 'DYLD_FALLBACK_FRAMEWORK_PATH')
|
||||
|
||||
def dyld_fallback_library_path(env=None):
|
||||
return dyld_env(env, 'DYLD_FALLBACK_LIBRARY_PATH')
|
||||
|
||||
def dyld_image_suffix_search(iterator, env=None):
|
||||
"""For a potential path iterator, add DYLD_IMAGE_SUFFIX semantics"""
|
||||
suffix = dyld_image_suffix(env)
|
||||
if suffix is None:
|
||||
return iterator
|
||||
def _inject(iterator=iterator, suffix=suffix):
|
||||
for path in iterator:
|
||||
if path.endswith('.dylib'):
|
||||
yield path[:-len('.dylib')] + suffix + '.dylib'
|
||||
else:
|
||||
yield path + suffix
|
||||
yield path
|
||||
return _inject()
|
||||
|
||||
def dyld_override_search(name, env=None):
|
||||
# If DYLD_FRAMEWORK_PATH is set and this dylib_name is a
|
||||
# framework name, use the first file that exists in the framework
|
||||
# path if any. If there is none go on to search the DYLD_LIBRARY_PATH
|
||||
# if any.
|
||||
|
||||
framework = framework_info(name)
|
||||
|
||||
if framework is not None:
|
||||
for path in dyld_framework_path(env):
|
||||
yield os.path.join(path, framework['name'])
|
||||
|
||||
# If DYLD_LIBRARY_PATH is set then use the first file that exists
|
||||
# in the path. If none use the original name.
|
||||
for path in dyld_library_path(env):
|
||||
yield os.path.join(path, os.path.basename(name))
|
||||
|
||||
def dyld_executable_path_search(name, executable_path=None):
|
||||
# If we haven't done any searching and found a library and the
|
||||
# dylib_name starts with "@executable_path/" then construct the
|
||||
# library name.
|
||||
if name.startswith('@executable_path/') and executable_path is not None:
|
||||
yield os.path.join(executable_path, name[len('@executable_path/'):])
|
||||
|
||||
def dyld_default_search(name, env=None):
|
||||
yield name
|
||||
|
||||
framework = framework_info(name)
|
||||
|
||||
if framework is not None:
|
||||
fallback_framework_path = dyld_fallback_framework_path(env)
|
||||
for path in fallback_framework_path:
|
||||
yield os.path.join(path, framework['name'])
|
||||
|
||||
fallback_library_path = dyld_fallback_library_path(env)
|
||||
for path in fallback_library_path:
|
||||
yield os.path.join(path, os.path.basename(name))
|
||||
|
||||
if framework is not None and not fallback_framework_path:
|
||||
for path in DEFAULT_FRAMEWORK_FALLBACK:
|
||||
yield os.path.join(path, framework['name'])
|
||||
|
||||
if not fallback_library_path:
|
||||
for path in DEFAULT_LIBRARY_FALLBACK:
|
||||
yield os.path.join(path, os.path.basename(name))
|
||||
|
||||
def dyld_find(name, executable_path=None, env=None):
|
||||
"""
|
||||
Find a library or framework using dyld semantics
|
||||
"""
|
||||
for path in dyld_image_suffix_search(chain(
|
||||
dyld_override_search(name, env),
|
||||
dyld_executable_path_search(name, executable_path),
|
||||
dyld_default_search(name, env),
|
||||
), env):
|
||||
|
||||
if os.path.isfile(path):
|
||||
return path
|
||||
try:
|
||||
if _dyld_shared_cache_contains_path(path):
|
||||
return path
|
||||
except NotImplementedError:
|
||||
pass
|
||||
|
||||
raise ValueError("dylib %s could not be found" % (name,))
|
||||
|
||||
def framework_find(fn, executable_path=None, env=None):
|
||||
"""
|
||||
Find a framework using dyld semantics in a very loose manner.
|
||||
|
||||
Will take input such as:
|
||||
Python
|
||||
Python.framework
|
||||
Python.framework/Versions/Current
|
||||
"""
|
||||
error = None
|
||||
try:
|
||||
return dyld_find(fn, executable_path=executable_path, env=env)
|
||||
except ValueError as e:
|
||||
error = e
|
||||
fmwk_index = fn.rfind('.framework')
|
||||
if fmwk_index == -1:
|
||||
fmwk_index = len(fn)
|
||||
fn += '.framework'
|
||||
fn = os.path.join(fn, os.path.basename(fn[:fmwk_index]))
|
||||
try:
|
||||
return dyld_find(fn, executable_path=executable_path, env=env)
|
||||
except ValueError:
|
||||
raise error
|
||||
finally:
|
||||
error = None
|
||||
@@ -1,42 +0,0 @@
|
||||
"""
|
||||
Generic dylib path manipulation
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
__all__ = ['dylib_info']
|
||||
|
||||
DYLIB_RE = re.compile(r"""(?x)
|
||||
(?P<location>^.*)(?:^|/)
|
||||
(?P<name>
|
||||
(?P<shortname>\w+?)
|
||||
(?:\.(?P<version>[^._]+))?
|
||||
(?:_(?P<suffix>[^._]+))?
|
||||
\.dylib$
|
||||
)
|
||||
""")
|
||||
|
||||
def dylib_info(filename):
|
||||
"""
|
||||
A dylib name can take one of the following four forms:
|
||||
Location/Name.SomeVersion_Suffix.dylib
|
||||
Location/Name.SomeVersion.dylib
|
||||
Location/Name_Suffix.dylib
|
||||
Location/Name.dylib
|
||||
|
||||
returns None if not found or a mapping equivalent to:
|
||||
dict(
|
||||
location='Location',
|
||||
name='Name.SomeVersion_Suffix.dylib',
|
||||
shortname='Name',
|
||||
version='SomeVersion',
|
||||
suffix='Suffix',
|
||||
)
|
||||
|
||||
Note that SomeVersion and Suffix are optional and may be None
|
||||
if not present.
|
||||
"""
|
||||
is_dylib = DYLIB_RE.match(filename)
|
||||
if not is_dylib:
|
||||
return None
|
||||
return is_dylib.groupdict()
|
||||
@@ -1,2 +0,0 @@
|
||||
#!/bin/sh
|
||||
svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
|
||||
@@ -1 +0,0 @@
|
||||
svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
|
||||
@@ -1,42 +0,0 @@
|
||||
"""
|
||||
Generic framework path manipulation
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
__all__ = ['framework_info']
|
||||
|
||||
STRICT_FRAMEWORK_RE = re.compile(r"""(?x)
|
||||
(?P<location>^.*)(?:^|/)
|
||||
(?P<name>
|
||||
(?P<shortname>\w+).framework/
|
||||
(?:Versions/(?P<version>[^/]+)/)?
|
||||
(?P=shortname)
|
||||
(?:_(?P<suffix>[^_]+))?
|
||||
)$
|
||||
""")
|
||||
|
||||
def framework_info(filename):
|
||||
"""
|
||||
A framework name can take one of the following four forms:
|
||||
Location/Name.framework/Versions/SomeVersion/Name_Suffix
|
||||
Location/Name.framework/Versions/SomeVersion/Name
|
||||
Location/Name.framework/Name_Suffix
|
||||
Location/Name.framework/Name
|
||||
|
||||
returns None if not found, or a mapping equivalent to:
|
||||
dict(
|
||||
location='Location',
|
||||
name='Name.framework/Versions/SomeVersion/Name_Suffix',
|
||||
shortname='Name',
|
||||
version='SomeVersion',
|
||||
suffix='Suffix',
|
||||
)
|
||||
|
||||
Note that SomeVersion and Suffix are optional and may be None
|
||||
if not present
|
||||
"""
|
||||
is_framework = STRICT_FRAMEWORK_RE.match(filename)
|
||||
if not is_framework:
|
||||
return None
|
||||
return is_framework.groupdict()
|
||||
@@ -1,16 +0,0 @@
|
||||
import os
|
||||
import unittest
|
||||
from test import support
|
||||
from test.support import import_helper
|
||||
|
||||
|
||||
# skip tests if _ctypes was not built
|
||||
ctypes = import_helper.import_module('ctypes')
|
||||
ctypes_symbols = dir(ctypes)
|
||||
|
||||
def need_symbol(name):
|
||||
return unittest.skipUnless(name in ctypes_symbols,
|
||||
'{!r} is required'.format(name))
|
||||
|
||||
def load_tests(*args):
|
||||
return support.load_package_tests(os.path.dirname(__file__), *args)
|
||||
@@ -1,4 +0,0 @@
|
||||
from ctypes.test import load_tests
|
||||
import unittest
|
||||
|
||||
unittest.main()
|
||||
@@ -1,73 +0,0 @@
|
||||
import unittest
|
||||
import test.support
|
||||
from ctypes import *
|
||||
|
||||
class AnonTest(unittest.TestCase):
|
||||
|
||||
def test_anon(self):
|
||||
class ANON(Union):
|
||||
_fields_ = [("a", c_int),
|
||||
("b", c_int)]
|
||||
|
||||
class Y(Structure):
|
||||
_fields_ = [("x", c_int),
|
||||
("_", ANON),
|
||||
("y", c_int)]
|
||||
_anonymous_ = ["_"]
|
||||
|
||||
self.assertEqual(Y.a.offset, sizeof(c_int))
|
||||
self.assertEqual(Y.b.offset, sizeof(c_int))
|
||||
|
||||
self.assertEqual(ANON.a.offset, 0)
|
||||
self.assertEqual(ANON.b.offset, 0)
|
||||
|
||||
def test_anon_nonseq(self):
|
||||
# TypeError: _anonymous_ must be a sequence
|
||||
self.assertRaises(TypeError,
|
||||
lambda: type(Structure)("Name",
|
||||
(Structure,),
|
||||
{"_fields_": [], "_anonymous_": 42}))
|
||||
|
||||
def test_anon_nonmember(self):
|
||||
# AttributeError: type object 'Name' has no attribute 'x'
|
||||
self.assertRaises(AttributeError,
|
||||
lambda: type(Structure)("Name",
|
||||
(Structure,),
|
||||
{"_fields_": [],
|
||||
"_anonymous_": ["x"]}))
|
||||
|
||||
@test.support.cpython_only
|
||||
def test_issue31490(self):
|
||||
# There shouldn't be an assertion failure in case the class has an
|
||||
# attribute whose name is specified in _anonymous_ but not in _fields_.
|
||||
|
||||
# AttributeError: 'x' is specified in _anonymous_ but not in _fields_
|
||||
with self.assertRaises(AttributeError):
|
||||
class Name(Structure):
|
||||
_fields_ = []
|
||||
_anonymous_ = ["x"]
|
||||
x = 42
|
||||
|
||||
def test_nested(self):
|
||||
class ANON_S(Structure):
|
||||
_fields_ = [("a", c_int)]
|
||||
|
||||
class ANON_U(Union):
|
||||
_fields_ = [("_", ANON_S),
|
||||
("b", c_int)]
|
||||
_anonymous_ = ["_"]
|
||||
|
||||
class Y(Structure):
|
||||
_fields_ = [("x", c_int),
|
||||
("_", ANON_U),
|
||||
("y", c_int)]
|
||||
_anonymous_ = ["_"]
|
||||
|
||||
self.assertEqual(Y.x.offset, 0)
|
||||
self.assertEqual(Y.a.offset, sizeof(c_int))
|
||||
self.assertEqual(Y.b.offset, sizeof(c_int))
|
||||
self.assertEqual(Y._.offset, sizeof(c_int))
|
||||
self.assertEqual(Y.y.offset, sizeof(c_int) * 2)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,64 +0,0 @@
|
||||
import unittest
|
||||
from ctypes import *
|
||||
from binascii import hexlify
|
||||
import re
|
||||
|
||||
def dump(obj):
|
||||
# helper function to dump memory contents in hex, with a hyphen
|
||||
# between the bytes.
|
||||
h = hexlify(memoryview(obj)).decode()
|
||||
return re.sub(r"(..)", r"\1-", h)[:-1]
|
||||
|
||||
|
||||
class Value(Structure):
|
||||
_fields_ = [("val", c_byte)]
|
||||
|
||||
class Container(Structure):
|
||||
_fields_ = [("pvalues", POINTER(Value))]
|
||||
|
||||
class Test(unittest.TestCase):
|
||||
def test(self):
|
||||
# create an array of 4 values
|
||||
val_array = (Value * 4)()
|
||||
|
||||
# create a container, which holds a pointer to the pvalues array.
|
||||
c = Container()
|
||||
c.pvalues = val_array
|
||||
|
||||
# memory contains 4 NUL bytes now, that's correct
|
||||
self.assertEqual("00-00-00-00", dump(val_array))
|
||||
|
||||
# set the values of the array through the pointer:
|
||||
for i in range(4):
|
||||
c.pvalues[i].val = i + 1
|
||||
|
||||
values = [c.pvalues[i].val for i in range(4)]
|
||||
|
||||
# These are the expected results: here s the bug!
|
||||
self.assertEqual(
|
||||
(values, dump(val_array)),
|
||||
([1, 2, 3, 4], "01-02-03-04")
|
||||
)
|
||||
|
||||
def test_2(self):
|
||||
|
||||
val_array = (Value * 4)()
|
||||
|
||||
# memory contains 4 NUL bytes now, that's correct
|
||||
self.assertEqual("00-00-00-00", dump(val_array))
|
||||
|
||||
ptr = cast(val_array, POINTER(Value))
|
||||
# set the values of the array through the pointer:
|
||||
for i in range(4):
|
||||
ptr[i].val = i + 1
|
||||
|
||||
values = [ptr[i].val for i in range(4)]
|
||||
|
||||
# These are the expected results: here s the bug!
|
||||
self.assertEqual(
|
||||
(values, dump(val_array)),
|
||||
([1, 2, 3, 4], "01-02-03-04")
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,238 +0,0 @@
|
||||
import unittest
|
||||
from test.support import bigmemtest, _2G
|
||||
import sys
|
||||
from ctypes import *
|
||||
|
||||
from ctypes.test import need_symbol
|
||||
|
||||
formats = "bBhHiIlLqQfd"
|
||||
|
||||
formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \
|
||||
c_long, c_ulonglong, c_float, c_double, c_longdouble
|
||||
|
||||
class ArrayTestCase(unittest.TestCase):
|
||||
def test_simple(self):
|
||||
# create classes holding simple numeric types, and check
|
||||
# various properties.
|
||||
|
||||
init = list(range(15, 25))
|
||||
|
||||
for fmt in formats:
|
||||
alen = len(init)
|
||||
int_array = ARRAY(fmt, alen)
|
||||
|
||||
ia = int_array(*init)
|
||||
# length of instance ok?
|
||||
self.assertEqual(len(ia), alen)
|
||||
|
||||
# slot values ok?
|
||||
values = [ia[i] for i in range(alen)]
|
||||
self.assertEqual(values, init)
|
||||
|
||||
# out-of-bounds accesses should be caught
|
||||
with self.assertRaises(IndexError): ia[alen]
|
||||
with self.assertRaises(IndexError): ia[-alen-1]
|
||||
|
||||
# change the items
|
||||
from operator import setitem
|
||||
new_values = list(range(42, 42+alen))
|
||||
[setitem(ia, n, new_values[n]) for n in range(alen)]
|
||||
values = [ia[i] for i in range(alen)]
|
||||
self.assertEqual(values, new_values)
|
||||
|
||||
# are the items initialized to 0?
|
||||
ia = int_array()
|
||||
values = [ia[i] for i in range(alen)]
|
||||
self.assertEqual(values, [0] * alen)
|
||||
|
||||
# Too many initializers should be caught
|
||||
self.assertRaises(IndexError, int_array, *range(alen*2))
|
||||
|
||||
CharArray = ARRAY(c_char, 3)
|
||||
|
||||
ca = CharArray(b"a", b"b", b"c")
|
||||
|
||||
# Should this work? It doesn't:
|
||||
# CharArray("abc")
|
||||
self.assertRaises(TypeError, CharArray, "abc")
|
||||
|
||||
self.assertEqual(ca[0], b"a")
|
||||
self.assertEqual(ca[1], b"b")
|
||||
self.assertEqual(ca[2], b"c")
|
||||
self.assertEqual(ca[-3], b"a")
|
||||
self.assertEqual(ca[-2], b"b")
|
||||
self.assertEqual(ca[-1], b"c")
|
||||
|
||||
self.assertEqual(len(ca), 3)
|
||||
|
||||
# cannot delete items
|
||||
from operator import delitem
|
||||
self.assertRaises(TypeError, delitem, ca, 0)
|
||||
|
||||
def test_step_overflow(self):
|
||||
a = (c_int * 5)()
|
||||
a[3::sys.maxsize] = (1,)
|
||||
self.assertListEqual(a[3::sys.maxsize], [1])
|
||||
a = (c_char * 5)()
|
||||
a[3::sys.maxsize] = b"A"
|
||||
self.assertEqual(a[3::sys.maxsize], b"A")
|
||||
a = (c_wchar * 5)()
|
||||
a[3::sys.maxsize] = u"X"
|
||||
self.assertEqual(a[3::sys.maxsize], u"X")
|
||||
|
||||
def test_numeric_arrays(self):
|
||||
|
||||
alen = 5
|
||||
|
||||
numarray = ARRAY(c_int, alen)
|
||||
|
||||
na = numarray()
|
||||
values = [na[i] for i in range(alen)]
|
||||
self.assertEqual(values, [0] * alen)
|
||||
|
||||
na = numarray(*[c_int()] * alen)
|
||||
values = [na[i] for i in range(alen)]
|
||||
self.assertEqual(values, [0]*alen)
|
||||
|
||||
na = numarray(1, 2, 3, 4, 5)
|
||||
values = [i for i in na]
|
||||
self.assertEqual(values, [1, 2, 3, 4, 5])
|
||||
|
||||
na = numarray(*map(c_int, (1, 2, 3, 4, 5)))
|
||||
values = [i for i in na]
|
||||
self.assertEqual(values, [1, 2, 3, 4, 5])
|
||||
|
||||
def test_classcache(self):
|
||||
self.assertIsNot(ARRAY(c_int, 3), ARRAY(c_int, 4))
|
||||
self.assertIs(ARRAY(c_int, 3), ARRAY(c_int, 3))
|
||||
|
||||
def test_from_address(self):
|
||||
# Failed with 0.9.8, reported by JUrner
|
||||
p = create_string_buffer(b"foo")
|
||||
sz = (c_char * 3).from_address(addressof(p))
|
||||
self.assertEqual(sz[:], b"foo")
|
||||
self.assertEqual(sz[::], b"foo")
|
||||
self.assertEqual(sz[::-1], b"oof")
|
||||
self.assertEqual(sz[::3], b"f")
|
||||
self.assertEqual(sz[1:4:2], b"o")
|
||||
self.assertEqual(sz.value, b"foo")
|
||||
|
||||
@need_symbol('create_unicode_buffer')
|
||||
def test_from_addressW(self):
|
||||
p = create_unicode_buffer("foo")
|
||||
sz = (c_wchar * 3).from_address(addressof(p))
|
||||
self.assertEqual(sz[:], "foo")
|
||||
self.assertEqual(sz[::], "foo")
|
||||
self.assertEqual(sz[::-1], "oof")
|
||||
self.assertEqual(sz[::3], "f")
|
||||
self.assertEqual(sz[1:4:2], "o")
|
||||
self.assertEqual(sz.value, "foo")
|
||||
|
||||
def test_cache(self):
|
||||
# Array types are cached internally in the _ctypes extension,
|
||||
# in a WeakValueDictionary. Make sure the array type is
|
||||
# removed from the cache when the itemtype goes away. This
|
||||
# test will not fail, but will show a leak in the testsuite.
|
||||
|
||||
# Create a new type:
|
||||
class my_int(c_int):
|
||||
pass
|
||||
# Create a new array type based on it:
|
||||
t1 = my_int * 1
|
||||
t2 = my_int * 1
|
||||
self.assertIs(t1, t2)
|
||||
|
||||
def test_subclass(self):
|
||||
class T(Array):
|
||||
_type_ = c_int
|
||||
_length_ = 13
|
||||
class U(T):
|
||||
pass
|
||||
class V(U):
|
||||
pass
|
||||
class W(V):
|
||||
pass
|
||||
class X(T):
|
||||
_type_ = c_short
|
||||
class Y(T):
|
||||
_length_ = 187
|
||||
|
||||
for c in [T, U, V, W]:
|
||||
self.assertEqual(c._type_, c_int)
|
||||
self.assertEqual(c._length_, 13)
|
||||
self.assertEqual(c()._type_, c_int)
|
||||
self.assertEqual(c()._length_, 13)
|
||||
|
||||
self.assertEqual(X._type_, c_short)
|
||||
self.assertEqual(X._length_, 13)
|
||||
self.assertEqual(X()._type_, c_short)
|
||||
self.assertEqual(X()._length_, 13)
|
||||
|
||||
self.assertEqual(Y._type_, c_int)
|
||||
self.assertEqual(Y._length_, 187)
|
||||
self.assertEqual(Y()._type_, c_int)
|
||||
self.assertEqual(Y()._length_, 187)
|
||||
|
||||
def test_bad_subclass(self):
|
||||
with self.assertRaises(AttributeError):
|
||||
class T(Array):
|
||||
pass
|
||||
with self.assertRaises(AttributeError):
|
||||
class T(Array):
|
||||
_type_ = c_int
|
||||
with self.assertRaises(AttributeError):
|
||||
class T(Array):
|
||||
_length_ = 13
|
||||
|
||||
def test_bad_length(self):
|
||||
with self.assertRaises(ValueError):
|
||||
class T(Array):
|
||||
_type_ = c_int
|
||||
_length_ = - sys.maxsize * 2
|
||||
with self.assertRaises(ValueError):
|
||||
class T(Array):
|
||||
_type_ = c_int
|
||||
_length_ = -1
|
||||
with self.assertRaises(TypeError):
|
||||
class T(Array):
|
||||
_type_ = c_int
|
||||
_length_ = 1.87
|
||||
with self.assertRaises(OverflowError):
|
||||
class T(Array):
|
||||
_type_ = c_int
|
||||
_length_ = sys.maxsize * 2
|
||||
|
||||
def test_zero_length(self):
|
||||
# _length_ can be zero.
|
||||
class T(Array):
|
||||
_type_ = c_int
|
||||
_length_ = 0
|
||||
|
||||
def test_empty_element_struct(self):
|
||||
class EmptyStruct(Structure):
|
||||
_fields_ = []
|
||||
|
||||
obj = (EmptyStruct * 2)() # bpo37188: Floating point exception
|
||||
self.assertEqual(sizeof(obj), 0)
|
||||
|
||||
def test_empty_element_array(self):
|
||||
class EmptyArray(Array):
|
||||
_type_ = c_int
|
||||
_length_ = 0
|
||||
|
||||
obj = (EmptyArray * 2)() # bpo37188: Floating point exception
|
||||
self.assertEqual(sizeof(obj), 0)
|
||||
|
||||
def test_bpo36504_signed_int_overflow(self):
|
||||
# The overflow check in PyCArrayType_new() could cause signed integer
|
||||
# overflow.
|
||||
with self.assertRaises(OverflowError):
|
||||
c_char * sys.maxsize * 2
|
||||
|
||||
@unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
|
||||
@bigmemtest(size=_2G, memuse=1, dry_run=False)
|
||||
def test_large_array(self, size):
|
||||
c_char * size
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -1,231 +0,0 @@
|
||||
import unittest
|
||||
from ctypes import *
|
||||
from ctypes.test import need_symbol
|
||||
import _ctypes_test
|
||||
|
||||
dll = CDLL(_ctypes_test.__file__)
|
||||
|
||||
try:
|
||||
CALLBACK_FUNCTYPE = WINFUNCTYPE
|
||||
except NameError:
|
||||
# fake to enable this test on Linux
|
||||
CALLBACK_FUNCTYPE = CFUNCTYPE
|
||||
|
||||
class POINT(Structure):
|
||||
_fields_ = [("x", c_int), ("y", c_int)]
|
||||
|
||||
class BasicWrapTestCase(unittest.TestCase):
|
||||
def wrap(self, param):
|
||||
return param
|
||||
|
||||
@need_symbol('c_wchar')
|
||||
def test_wchar_parm(self):
|
||||
f = dll._testfunc_i_bhilfd
|
||||
f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
|
||||
result = f(self.wrap(1), self.wrap("x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0))
|
||||
self.assertEqual(result, 139)
|
||||
self.assertIs(type(result), int)
|
||||
|
||||
def test_pointers(self):
|
||||
f = dll._testfunc_p_p
|
||||
f.restype = POINTER(c_int)
|
||||
f.argtypes = [POINTER(c_int)]
|
||||
|
||||
# This only works if the value c_int(42) passed to the
|
||||
# function is still alive while the pointer (the result) is
|
||||
# used.
|
||||
|
||||
v = c_int(42)
|
||||
|
||||
self.assertEqual(pointer(v).contents.value, 42)
|
||||
result = f(self.wrap(pointer(v)))
|
||||
self.assertEqual(type(result), POINTER(c_int))
|
||||
self.assertEqual(result.contents.value, 42)
|
||||
|
||||
# This on works...
|
||||
result = f(self.wrap(pointer(v)))
|
||||
self.assertEqual(result.contents.value, v.value)
|
||||
|
||||
p = pointer(c_int(99))
|
||||
result = f(self.wrap(p))
|
||||
self.assertEqual(result.contents.value, 99)
|
||||
|
||||
def test_shorts(self):
|
||||
f = dll._testfunc_callback_i_if
|
||||
|
||||
args = []
|
||||
expected = [262144, 131072, 65536, 32768, 16384, 8192, 4096, 2048,
|
||||
1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1]
|
||||
|
||||
def callback(v):
|
||||
args.append(v)
|
||||
return v
|
||||
|
||||
CallBack = CFUNCTYPE(c_int, c_int)
|
||||
|
||||
cb = CallBack(callback)
|
||||
f(self.wrap(2**18), self.wrap(cb))
|
||||
self.assertEqual(args, expected)
|
||||
|
||||
################################################################
|
||||
|
||||
def test_callbacks(self):
|
||||
f = dll._testfunc_callback_i_if
|
||||
f.restype = c_int
|
||||
f.argtypes = None
|
||||
|
||||
MyCallback = CFUNCTYPE(c_int, c_int)
|
||||
|
||||
def callback(value):
|
||||
#print "called back with", value
|
||||
return value
|
||||
|
||||
cb = MyCallback(callback)
|
||||
|
||||
result = f(self.wrap(-10), self.wrap(cb))
|
||||
self.assertEqual(result, -18)
|
||||
|
||||
# test with prototype
|
||||
f.argtypes = [c_int, MyCallback]
|
||||
cb = MyCallback(callback)
|
||||
|
||||
result = f(self.wrap(-10), self.wrap(cb))
|
||||
self.assertEqual(result, -18)
|
||||
|
||||
result = f(self.wrap(-10), self.wrap(cb))
|
||||
self.assertEqual(result, -18)
|
||||
|
||||
AnotherCallback = CALLBACK_FUNCTYPE(c_int, c_int, c_int, c_int, c_int)
|
||||
|
||||
# check that the prototype works: we call f with wrong
|
||||
# argument types
|
||||
cb = AnotherCallback(callback)
|
||||
self.assertRaises(ArgumentError, f, self.wrap(-10), self.wrap(cb))
|
||||
|
||||
def test_callbacks_2(self):
|
||||
# Can also use simple datatypes as argument type specifiers
|
||||
# for the callback function.
|
||||
# In this case the call receives an instance of that type
|
||||
f = dll._testfunc_callback_i_if
|
||||
f.restype = c_int
|
||||
|
||||
MyCallback = CFUNCTYPE(c_int, c_int)
|
||||
|
||||
f.argtypes = [c_int, MyCallback]
|
||||
|
||||
def callback(value):
|
||||
#print "called back with", value
|
||||
self.assertEqual(type(value), int)
|
||||
return value
|
||||
|
||||
cb = MyCallback(callback)
|
||||
result = f(self.wrap(-10), self.wrap(cb))
|
||||
self.assertEqual(result, -18)
|
||||
|
||||
@need_symbol('c_longlong')
|
||||
def test_longlong_callbacks(self):
|
||||
|
||||
f = dll._testfunc_callback_q_qf
|
||||
f.restype = c_longlong
|
||||
|
||||
MyCallback = CFUNCTYPE(c_longlong, c_longlong)
|
||||
|
||||
f.argtypes = [c_longlong, MyCallback]
|
||||
|
||||
def callback(value):
|
||||
self.assertIsInstance(value, int)
|
||||
return value & 0x7FFFFFFF
|
||||
|
||||
cb = MyCallback(callback)
|
||||
|
||||
self.assertEqual(13577625587, int(f(self.wrap(1000000000000), self.wrap(cb))))
|
||||
|
||||
def test_byval(self):
|
||||
# without prototype
|
||||
ptin = POINT(1, 2)
|
||||
ptout = POINT()
|
||||
# EXPORT int _testfunc_byval(point in, point *pout)
|
||||
result = dll._testfunc_byval(ptin, byref(ptout))
|
||||
got = result, ptout.x, ptout.y
|
||||
expected = 3, 1, 2
|
||||
self.assertEqual(got, expected)
|
||||
|
||||
# with prototype
|
||||
ptin = POINT(101, 102)
|
||||
ptout = POINT()
|
||||
dll._testfunc_byval.argtypes = (POINT, POINTER(POINT))
|
||||
dll._testfunc_byval.restype = c_int
|
||||
result = dll._testfunc_byval(self.wrap(ptin), byref(ptout))
|
||||
got = result, ptout.x, ptout.y
|
||||
expected = 203, 101, 102
|
||||
self.assertEqual(got, expected)
|
||||
|
||||
def test_struct_return_2H(self):
|
||||
class S2H(Structure):
|
||||
_fields_ = [("x", c_short),
|
||||
("y", c_short)]
|
||||
dll.ret_2h_func.restype = S2H
|
||||
dll.ret_2h_func.argtypes = [S2H]
|
||||
inp = S2H(99, 88)
|
||||
s2h = dll.ret_2h_func(self.wrap(inp))
|
||||
self.assertEqual((s2h.x, s2h.y), (99*2, 88*3))
|
||||
|
||||
# Test also that the original struct was unmodified (i.e. was passed by
|
||||
# value)
|
||||
self.assertEqual((inp.x, inp.y), (99, 88))
|
||||
|
||||
def test_struct_return_8H(self):
|
||||
class S8I(Structure):
|
||||
_fields_ = [("a", c_int),
|
||||
("b", c_int),
|
||||
("c", c_int),
|
||||
("d", c_int),
|
||||
("e", c_int),
|
||||
("f", c_int),
|
||||
("g", c_int),
|
||||
("h", c_int)]
|
||||
dll.ret_8i_func.restype = S8I
|
||||
dll.ret_8i_func.argtypes = [S8I]
|
||||
inp = S8I(9, 8, 7, 6, 5, 4, 3, 2)
|
||||
s8i = dll.ret_8i_func(self.wrap(inp))
|
||||
self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
|
||||
(9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
|
||||
|
||||
def test_recursive_as_param(self):
|
||||
from ctypes import c_int
|
||||
|
||||
class A(object):
|
||||
pass
|
||||
|
||||
a = A()
|
||||
a._as_parameter_ = a
|
||||
with self.assertRaises(RecursionError):
|
||||
c_int.from_param(a)
|
||||
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
class AsParamWrapper(object):
|
||||
def __init__(self, param):
|
||||
self._as_parameter_ = param
|
||||
|
||||
class AsParamWrapperTestCase(BasicWrapTestCase):
|
||||
wrap = AsParamWrapper
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
class AsParamPropertyWrapper(object):
|
||||
def __init__(self, param):
|
||||
self._param = param
|
||||
|
||||
def getParameter(self):
|
||||
return self._param
|
||||
_as_parameter_ = property(getParameter)
|
||||
|
||||
class AsParamPropertyWrapperTestCase(BasicWrapTestCase):
|
||||
wrap = AsParamPropertyWrapper
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -1,297 +0,0 @@
|
||||
from ctypes import *
|
||||
from ctypes.test import need_symbol
|
||||
from test import support
|
||||
import unittest
|
||||
import os
|
||||
|
||||
import _ctypes_test
|
||||
|
||||
class BITS(Structure):
|
||||
_fields_ = [("A", c_int, 1),
|
||||
("B", c_int, 2),
|
||||
("C", c_int, 3),
|
||||
("D", c_int, 4),
|
||||
("E", c_int, 5),
|
||||
("F", c_int, 6),
|
||||
("G", c_int, 7),
|
||||
("H", c_int, 8),
|
||||
("I", c_int, 9),
|
||||
|
||||
("M", c_short, 1),
|
||||
("N", c_short, 2),
|
||||
("O", c_short, 3),
|
||||
("P", c_short, 4),
|
||||
("Q", c_short, 5),
|
||||
("R", c_short, 6),
|
||||
("S", c_short, 7)]
|
||||
|
||||
func = CDLL(_ctypes_test.__file__).unpack_bitfields
|
||||
func.argtypes = POINTER(BITS), c_char
|
||||
|
||||
##for n in "ABCDEFGHIMNOPQRS":
|
||||
## print n, hex(getattr(BITS, n).size), getattr(BITS, n).offset
|
||||
|
||||
class C_Test(unittest.TestCase):
|
||||
|
||||
def test_ints(self):
|
||||
for i in range(512):
|
||||
for name in "ABCDEFGHI":
|
||||
b = BITS()
|
||||
setattr(b, name, i)
|
||||
self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii')))
|
||||
|
||||
# bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior
|
||||
@support.skip_if_sanitizer(ub=True)
|
||||
def test_shorts(self):
|
||||
b = BITS()
|
||||
name = "M"
|
||||
if func(byref(b), name.encode('ascii')) == 999:
|
||||
self.skipTest("Compiler does not support signed short bitfields")
|
||||
for i in range(256):
|
||||
for name in "MNOPQRS":
|
||||
b = BITS()
|
||||
setattr(b, name, i)
|
||||
self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii')))
|
||||
|
||||
signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong)
|
||||
unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong)
|
||||
int_types = unsigned_int_types + signed_int_types
|
||||
|
||||
class BitFieldTest(unittest.TestCase):
|
||||
|
||||
def test_longlong(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_longlong, 1),
|
||||
("b", c_longlong, 62),
|
||||
("c", c_longlong, 1)]
|
||||
|
||||
self.assertEqual(sizeof(X), sizeof(c_longlong))
|
||||
x = X()
|
||||
x.a, x.b, x.c = -1, 7, -1
|
||||
self.assertEqual((x.a, x.b, x.c), (-1, 7, -1))
|
||||
|
||||
def test_ulonglong(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_ulonglong, 1),
|
||||
("b", c_ulonglong, 62),
|
||||
("c", c_ulonglong, 1)]
|
||||
|
||||
self.assertEqual(sizeof(X), sizeof(c_longlong))
|
||||
x = X()
|
||||
self.assertEqual((x.a, x.b, x.c), (0, 0, 0))
|
||||
x.a, x.b, x.c = 7, 7, 7
|
||||
self.assertEqual((x.a, x.b, x.c), (1, 7, 1))
|
||||
|
||||
def test_signed(self):
|
||||
for c_typ in signed_int_types:
|
||||
class X(Structure):
|
||||
_fields_ = [("dummy", c_typ),
|
||||
("a", c_typ, 3),
|
||||
("b", c_typ, 3),
|
||||
("c", c_typ, 1)]
|
||||
self.assertEqual(sizeof(X), sizeof(c_typ)*2)
|
||||
|
||||
x = X()
|
||||
self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0))
|
||||
x.a = -1
|
||||
self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0))
|
||||
x.a, x.b = 0, -1
|
||||
self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0))
|
||||
|
||||
|
||||
def test_unsigned(self):
|
||||
for c_typ in unsigned_int_types:
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_typ, 3),
|
||||
("b", c_typ, 3),
|
||||
("c", c_typ, 1)]
|
||||
self.assertEqual(sizeof(X), sizeof(c_typ))
|
||||
|
||||
x = X()
|
||||
self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0))
|
||||
x.a = -1
|
||||
self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0))
|
||||
x.a, x.b = 0, -1
|
||||
self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0))
|
||||
|
||||
|
||||
def fail_fields(self, *fields):
|
||||
return self.get_except(type(Structure), "X", (),
|
||||
{"_fields_": fields})
|
||||
|
||||
def test_nonint_types(self):
|
||||
# bit fields are not allowed on non-integer types.
|
||||
result = self.fail_fields(("a", c_char_p, 1))
|
||||
self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char_p'))
|
||||
|
||||
result = self.fail_fields(("a", c_void_p, 1))
|
||||
self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_void_p'))
|
||||
|
||||
if c_int != c_long:
|
||||
result = self.fail_fields(("a", POINTER(c_int), 1))
|
||||
self.assertEqual(result, (TypeError, 'bit fields not allowed for type LP_c_int'))
|
||||
|
||||
result = self.fail_fields(("a", c_char, 1))
|
||||
self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char'))
|
||||
|
||||
class Dummy(Structure):
|
||||
_fields_ = []
|
||||
|
||||
result = self.fail_fields(("a", Dummy, 1))
|
||||
self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy'))
|
||||
|
||||
@need_symbol('c_wchar')
|
||||
def test_c_wchar(self):
|
||||
result = self.fail_fields(("a", c_wchar, 1))
|
||||
self.assertEqual(result,
|
||||
(TypeError, 'bit fields not allowed for type c_wchar'))
|
||||
|
||||
def test_single_bitfield_size(self):
|
||||
for c_typ in int_types:
|
||||
result = self.fail_fields(("a", c_typ, -1))
|
||||
self.assertEqual(result, (ValueError, 'number of bits invalid for bit field'))
|
||||
|
||||
result = self.fail_fields(("a", c_typ, 0))
|
||||
self.assertEqual(result, (ValueError, 'number of bits invalid for bit field'))
|
||||
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_typ, 1)]
|
||||
self.assertEqual(sizeof(X), sizeof(c_typ))
|
||||
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_typ, sizeof(c_typ)*8)]
|
||||
self.assertEqual(sizeof(X), sizeof(c_typ))
|
||||
|
||||
result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1))
|
||||
self.assertEqual(result, (ValueError, 'number of bits invalid for bit field'))
|
||||
|
||||
def test_multi_bitfields_size(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_short, 1),
|
||||
("b", c_short, 14),
|
||||
("c", c_short, 1)]
|
||||
self.assertEqual(sizeof(X), sizeof(c_short))
|
||||
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_short, 1),
|
||||
("a1", c_short),
|
||||
("b", c_short, 14),
|
||||
("c", c_short, 1)]
|
||||
self.assertEqual(sizeof(X), sizeof(c_short)*3)
|
||||
self.assertEqual(X.a.offset, 0)
|
||||
self.assertEqual(X.a1.offset, sizeof(c_short))
|
||||
self.assertEqual(X.b.offset, sizeof(c_short)*2)
|
||||
self.assertEqual(X.c.offset, sizeof(c_short)*2)
|
||||
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_short, 3),
|
||||
("b", c_short, 14),
|
||||
("c", c_short, 14)]
|
||||
self.assertEqual(sizeof(X), sizeof(c_short)*3)
|
||||
self.assertEqual(X.a.offset, sizeof(c_short)*0)
|
||||
self.assertEqual(X.b.offset, sizeof(c_short)*1)
|
||||
self.assertEqual(X.c.offset, sizeof(c_short)*2)
|
||||
|
||||
|
||||
def get_except(self, func, *args, **kw):
|
||||
try:
|
||||
func(*args, **kw)
|
||||
except Exception as detail:
|
||||
return detail.__class__, str(detail)
|
||||
|
||||
def test_mixed_1(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_byte, 4),
|
||||
("b", c_int, 4)]
|
||||
if os.name == "nt":
|
||||
self.assertEqual(sizeof(X), sizeof(c_int)*2)
|
||||
else:
|
||||
self.assertEqual(sizeof(X), sizeof(c_int))
|
||||
|
||||
def test_mixed_2(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_byte, 4),
|
||||
("b", c_int, 32)]
|
||||
self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int))
|
||||
|
||||
def test_mixed_3(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_byte, 4),
|
||||
("b", c_ubyte, 4)]
|
||||
self.assertEqual(sizeof(X), sizeof(c_byte))
|
||||
|
||||
def test_mixed_4(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_short, 4),
|
||||
("b", c_short, 4),
|
||||
("c", c_int, 24),
|
||||
("d", c_short, 4),
|
||||
("e", c_short, 4),
|
||||
("f", c_int, 24)]
|
||||
# MSVC does NOT combine c_short and c_int into one field, GCC
|
||||
# does (unless GCC is run with '-mms-bitfields' which
|
||||
# produces code compatible with MSVC).
|
||||
if os.name == "nt":
|
||||
self.assertEqual(sizeof(X), sizeof(c_int) * 4)
|
||||
else:
|
||||
self.assertEqual(sizeof(X), sizeof(c_int) * 2)
|
||||
|
||||
def test_anon_bitfields(self):
|
||||
# anonymous bit-fields gave a strange error message
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_byte, 4),
|
||||
("b", c_ubyte, 4)]
|
||||
class Y(Structure):
|
||||
_anonymous_ = ["_"]
|
||||
_fields_ = [("_", X)]
|
||||
|
||||
@need_symbol('c_uint32')
|
||||
def test_uint32(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_uint32, 32)]
|
||||
x = X()
|
||||
x.a = 10
|
||||
self.assertEqual(x.a, 10)
|
||||
x.a = 0xFDCBA987
|
||||
self.assertEqual(x.a, 0xFDCBA987)
|
||||
|
||||
@need_symbol('c_uint64')
|
||||
def test_uint64(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_uint64, 64)]
|
||||
x = X()
|
||||
x.a = 10
|
||||
self.assertEqual(x.a, 10)
|
||||
x.a = 0xFEDCBA9876543211
|
||||
self.assertEqual(x.a, 0xFEDCBA9876543211)
|
||||
|
||||
@need_symbol('c_uint32')
|
||||
def test_uint32_swap_little_endian(self):
|
||||
# Issue #23319
|
||||
class Little(LittleEndianStructure):
|
||||
_fields_ = [("a", c_uint32, 24),
|
||||
("b", c_uint32, 4),
|
||||
("c", c_uint32, 4)]
|
||||
b = bytearray(4)
|
||||
x = Little.from_buffer(b)
|
||||
x.a = 0xabcdef
|
||||
x.b = 1
|
||||
x.c = 2
|
||||
self.assertEqual(b, b'\xef\xcd\xab\x21')
|
||||
|
||||
@need_symbol('c_uint32')
|
||||
def test_uint32_swap_big_endian(self):
|
||||
# Issue #23319
|
||||
class Big(BigEndianStructure):
|
||||
_fields_ = [("a", c_uint32, 24),
|
||||
("b", c_uint32, 4),
|
||||
("c", c_uint32, 4)]
|
||||
b = bytearray(4)
|
||||
x = Big.from_buffer(b)
|
||||
x.a = 0xabcdef
|
||||
x.b = 1
|
||||
x.c = 2
|
||||
self.assertEqual(b, b'\xab\xcd\xef\x12')
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,73 +0,0 @@
|
||||
from ctypes import *
|
||||
from ctypes.test import need_symbol
|
||||
import unittest
|
||||
|
||||
class StringBufferTestCase(unittest.TestCase):
|
||||
|
||||
def test_buffer(self):
|
||||
b = create_string_buffer(32)
|
||||
self.assertEqual(len(b), 32)
|
||||
self.assertEqual(sizeof(b), 32 * sizeof(c_char))
|
||||
self.assertIs(type(b[0]), bytes)
|
||||
|
||||
b = create_string_buffer(b"abc")
|
||||
self.assertEqual(len(b), 4) # trailing nul char
|
||||
self.assertEqual(sizeof(b), 4 * sizeof(c_char))
|
||||
self.assertIs(type(b[0]), bytes)
|
||||
self.assertEqual(b[0], b"a")
|
||||
self.assertEqual(b[:], b"abc\0")
|
||||
self.assertEqual(b[::], b"abc\0")
|
||||
self.assertEqual(b[::-1], b"\0cba")
|
||||
self.assertEqual(b[::2], b"ac")
|
||||
self.assertEqual(b[::5], b"a")
|
||||
|
||||
self.assertRaises(TypeError, create_string_buffer, "abc")
|
||||
|
||||
def test_buffer_interface(self):
|
||||
self.assertEqual(len(bytearray(create_string_buffer(0))), 0)
|
||||
self.assertEqual(len(bytearray(create_string_buffer(1))), 1)
|
||||
|
||||
@need_symbol('c_wchar')
|
||||
def test_unicode_buffer(self):
|
||||
b = create_unicode_buffer(32)
|
||||
self.assertEqual(len(b), 32)
|
||||
self.assertEqual(sizeof(b), 32 * sizeof(c_wchar))
|
||||
self.assertIs(type(b[0]), str)
|
||||
|
||||
b = create_unicode_buffer("abc")
|
||||
self.assertEqual(len(b), 4) # trailing nul char
|
||||
self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
|
||||
self.assertIs(type(b[0]), str)
|
||||
self.assertEqual(b[0], "a")
|
||||
self.assertEqual(b[:], "abc\0")
|
||||
self.assertEqual(b[::], "abc\0")
|
||||
self.assertEqual(b[::-1], "\0cba")
|
||||
self.assertEqual(b[::2], "ac")
|
||||
self.assertEqual(b[::5], "a")
|
||||
|
||||
self.assertRaises(TypeError, create_unicode_buffer, b"abc")
|
||||
|
||||
@need_symbol('c_wchar')
|
||||
def test_unicode_conversion(self):
|
||||
b = create_unicode_buffer("abc")
|
||||
self.assertEqual(len(b), 4) # trailing nul char
|
||||
self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
|
||||
self.assertIs(type(b[0]), str)
|
||||
self.assertEqual(b[0], "a")
|
||||
self.assertEqual(b[:], "abc\0")
|
||||
self.assertEqual(b[::], "abc\0")
|
||||
self.assertEqual(b[::-1], "\0cba")
|
||||
self.assertEqual(b[::2], "ac")
|
||||
self.assertEqual(b[::5], "a")
|
||||
|
||||
@need_symbol('c_wchar')
|
||||
def test_create_unicode_buffer_non_bmp(self):
|
||||
expected = 5 if sizeof(c_wchar) == 2 else 3
|
||||
for s in '\U00010000\U00100000', '\U00010000\U0010ffff':
|
||||
b = create_unicode_buffer(s)
|
||||
self.assertEqual(len(b), expected)
|
||||
self.assertEqual(b[-1], '\0')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,66 +0,0 @@
|
||||
"""Test where byte objects are accepted"""
|
||||
import unittest
|
||||
import sys
|
||||
from ctypes import *
|
||||
|
||||
class BytesTest(unittest.TestCase):
|
||||
def test_c_char(self):
|
||||
x = c_char(b"x")
|
||||
self.assertRaises(TypeError, c_char, "x")
|
||||
x.value = b"y"
|
||||
with self.assertRaises(TypeError):
|
||||
x.value = "y"
|
||||
c_char.from_param(b"x")
|
||||
self.assertRaises(TypeError, c_char.from_param, "x")
|
||||
self.assertIn('xbd', repr(c_char.from_param(b"\xbd")))
|
||||
(c_char * 3)(b"a", b"b", b"c")
|
||||
self.assertRaises(TypeError, c_char * 3, "a", "b", "c")
|
||||
|
||||
def test_c_wchar(self):
|
||||
x = c_wchar("x")
|
||||
self.assertRaises(TypeError, c_wchar, b"x")
|
||||
x.value = "y"
|
||||
with self.assertRaises(TypeError):
|
||||
x.value = b"y"
|
||||
c_wchar.from_param("x")
|
||||
self.assertRaises(TypeError, c_wchar.from_param, b"x")
|
||||
(c_wchar * 3)("a", "b", "c")
|
||||
self.assertRaises(TypeError, c_wchar * 3, b"a", b"b", b"c")
|
||||
|
||||
def test_c_char_p(self):
|
||||
c_char_p(b"foo bar")
|
||||
self.assertRaises(TypeError, c_char_p, "foo bar")
|
||||
|
||||
def test_c_wchar_p(self):
|
||||
c_wchar_p("foo bar")
|
||||
self.assertRaises(TypeError, c_wchar_p, b"foo bar")
|
||||
|
||||
def test_struct(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_char * 3)]
|
||||
|
||||
x = X(b"abc")
|
||||
self.assertRaises(TypeError, X, "abc")
|
||||
self.assertEqual(x.a, b"abc")
|
||||
self.assertEqual(type(x.a), bytes)
|
||||
|
||||
def test_struct_W(self):
|
||||
class X(Structure):
|
||||
_fields_ = [("a", c_wchar * 3)]
|
||||
|
||||
x = X("abc")
|
||||
self.assertRaises(TypeError, X, b"abc")
|
||||
self.assertEqual(x.a, "abc")
|
||||
self.assertEqual(type(x.a), str)
|
||||
|
||||
@unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
|
||||
def test_BSTR(self):
|
||||
from _ctypes import _SimpleCData
|
||||
class BSTR(_SimpleCData):
|
||||
_type_ = "X"
|
||||
|
||||
BSTR("abc")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -1,364 +0,0 @@
|
||||
import sys, unittest, struct, math, ctypes
|
||||
from binascii import hexlify
|
||||
|
||||
from ctypes import *
|
||||
|
||||
def bin(s):
|
||||
return hexlify(memoryview(s)).decode().upper()
|
||||
|
||||
# Each *simple* type that supports different byte orders has an
|
||||
# __ctype_be__ attribute that specifies the same type in BIG ENDIAN
|
||||
# byte order, and a __ctype_le__ attribute that is the same type in
|
||||
# LITTLE ENDIAN byte order.
|
||||
#
|
||||
# For Structures and Unions, these types are created on demand.
|
||||
|
||||
class Test(unittest.TestCase):
|
||||
@unittest.skip('test disabled')
|
||||
def test_X(self):
|
||||
print(sys.byteorder, file=sys.stderr)
|
||||
for i in range(32):
|
||||
bits = BITS()
|
||||
setattr(bits, "i%s" % i, 1)
|
||||
dump(bits)
|
||||
|
||||
def test_slots(self):
|
||||
class BigPoint(BigEndianStructure):
|
||||
__slots__ = ()
|
||||
_fields_ = [("x", c_int), ("y", c_int)]
|
||||
|
||||
class LowPoint(LittleEndianStructure):
|
||||
__slots__ = ()
|
||||
_fields_ = [("x", c_int), ("y", c_int)]
|
||||
|
||||
big = BigPoint()
|
||||
little = LowPoint()
|
||||
big.x = 4
|
||||
big.y = 2
|
||||
little.x = 2
|
||||
little.y = 4
|
||||
with self.assertRaises(AttributeError):
|
||||
big.z = 42
|
||||
with self.assertRaises(AttributeError):
|
||||
little.z = 24
|
||||
|
||||
def test_endian_short(self):
|
||||
if sys.byteorder == "little":
|
||||
self.assertIs(c_short.__ctype_le__, c_short)
|
||||
self.assertIs(c_short.__ctype_be__.__ctype_le__, c_short)
|
||||
else:
|
||||
self.assertIs(c_short.__ctype_be__, c_short)
|
||||
self.assertIs(c_short.__ctype_le__.__ctype_be__, c_short)
|
||||
s = c_short.__ctype_be__(0x1234)
|
||||
self.assertEqual(bin(struct.pack(">h", 0x1234)), "1234")
|
||||
self.assertEqual(bin(s), "1234")
|
||||
self.assertEqual(s.value, 0x1234)
|
||||
|
||||
s = c_short.__ctype_le__(0x1234)
|
||||
self.assertEqual(bin(struct.pack("<h", 0x1234)), "3412")
|
||||
self.assertEqual(bin(s), "3412")
|
||||
self.assertEqual(s.value, 0x1234)
|
||||
|
||||
s = c_ushort.__ctype_be__(0x1234)
|
||||
self.assertEqual(bin(struct.pack(">h", 0x1234)), "1234")
|
||||
self.assertEqual(bin(s), "1234")
|
||||
self.assertEqual(s.value, 0x1234)
|
||||
|
||||
s = c_ushort.__ctype_le__(0x1234)
|
||||
self.assertEqual(bin(struct.pack("<h", 0x1234)), "3412")
|
||||
self.assertEqual(bin(s), "3412")
|
||||
self.assertEqual(s.value, 0x1234)
|
||||
|
||||
def test_endian_int(self):
|
||||
if sys.byteorder == "little":
|
||||
self.assertIs(c_int.__ctype_le__, c_int)
|
||||
self.assertIs(c_int.__ctype_be__.__ctype_le__, c_int)
|
||||
else:
|
||||
self.assertIs(c_int.__ctype_be__, c_int)
|
||||
self.assertIs(c_int.__ctype_le__.__ctype_be__, c_int)
|
||||
|
||||
s = c_int.__ctype_be__(0x12345678)
|
||||
self.assertEqual(bin(struct.pack(">i", 0x12345678)), "12345678")
|
||||
self.assertEqual(bin(s), "12345678")
|
||||
self.assertEqual(s.value, 0x12345678)
|
||||
|
||||
s = c_int.__ctype_le__(0x12345678)
|
||||
self.assertEqual(bin(struct.pack("<i", 0x12345678)), "78563412")
|
||||
self.assertEqual(bin(s), "78563412")
|
||||
self.assertEqual(s.value, 0x12345678)
|
||||
|
||||
s = c_uint.__ctype_be__(0x12345678)
|
||||
self.assertEqual(bin(struct.pack(">I", 0x12345678)), "12345678")
|
||||
self.assertEqual(bin(s), "12345678")
|
||||
self.assertEqual(s.value, 0x12345678)
|
||||
|
||||
s = c_uint.__ctype_le__(0x12345678)
|
||||
self.assertEqual(bin(struct.pack("<I", 0x12345678)), "78563412")
|
||||
self.assertEqual(bin(s), "78563412")
|
||||
self.assertEqual(s.value, 0x12345678)
|
||||
|
||||
def test_endian_longlong(self):
|
||||
if sys.byteorder == "little":
|
||||
self.assertIs(c_longlong.__ctype_le__, c_longlong)
|
||||
self.assertIs(c_longlong.__ctype_be__.__ctype_le__, c_longlong)
|
||||
else:
|
||||
self.assertIs(c_longlong.__ctype_be__, c_longlong)
|
||||
self.assertIs(c_longlong.__ctype_le__.__ctype_be__, c_longlong)
|
||||
|
||||
s = c_longlong.__ctype_be__(0x1234567890ABCDEF)
|
||||
self.assertEqual(bin(struct.pack(">q", 0x1234567890ABCDEF)), "1234567890ABCDEF")
|
||||
self.assertEqual(bin(s), "1234567890ABCDEF")
|
||||
self.assertEqual(s.value, 0x1234567890ABCDEF)
|
||||
|
||||
s = c_longlong.__ctype_le__(0x1234567890ABCDEF)
|
||||
self.assertEqual(bin(struct.pack("<q", 0x1234567890ABCDEF)), "EFCDAB9078563412")
|
||||
self.assertEqual(bin(s), "EFCDAB9078563412")
|
||||
self.assertEqual(s.value, 0x1234567890ABCDEF)
|
||||
|
||||
s = c_ulonglong.__ctype_be__(0x1234567890ABCDEF)
|
||||
self.assertEqual(bin(struct.pack(">Q", 0x1234567890ABCDEF)), "1234567890ABCDEF")
|
||||
self.assertEqual(bin(s), "1234567890ABCDEF")
|
||||
self.assertEqual(s.value, 0x1234567890ABCDEF)
|
||||
|
||||
s = c_ulonglong.__ctype_le__(0x1234567890ABCDEF)
|
||||
self.assertEqual(bin(struct.pack("<Q", 0x1234567890ABCDEF)), "EFCDAB9078563412")
|
||||
self.assertEqual(bin(s), "EFCDAB9078563412")
|
||||
self.assertEqual(s.value, 0x1234567890ABCDEF)
|
||||
|
||||
def test_endian_float(self):
|
||||
if sys.byteorder == "little":
|
||||
self.assertIs(c_float.__ctype_le__, c_float)
|
||||
self.assertIs(c_float.__ctype_be__.__ctype_le__, c_float)
|
||||
else:
|
||||
self.assertIs(c_float.__ctype_be__, c_float)
|
||||
self.assertIs(c_float.__ctype_le__.__ctype_be__, c_float)
|
||||
s = c_float(math.pi)
|
||||
self.assertEqual(bin(struct.pack("f", math.pi)), bin(s))
|
||||
# Hm, what's the precision of a float compared to a double?
|
||||
self.assertAlmostEqual(s.value, math.pi, places=6)
|
||||
s = c_float.__ctype_le__(math.pi)
|
||||
self.assertAlmostEqual(s.value, math.pi, places=6)
|
||||
self.assertEqual(bin(struct.pack("<f", math.pi)), bin(s))
|
||||
s = c_float.__ctype_be__(math.pi)
|
||||
self.assertAlmostEqual(s.value, math.pi, places=6)
|
||||
self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
|
||||
|
||||
def test_endian_double(self):
|
||||
if sys.byteorder == "little":
|
||||
self.assertIs(c_double.__ctype_le__, c_double)
|
||||
self.assertIs(c_double.__ctype_be__.__ctype_le__, c_double)
|
||||
else:
|
||||
self.assertIs(c_double.__ctype_be__, c_double)
|
||||
self.assertIs(c_double.__ctype_le__.__ctype_be__, c_double)
|
||||
s = c_double(math.pi)
|
||||
self.assertEqual(s.value, math.pi)
|
||||
self.assertEqual(bin(struct.pack("d", math.pi)), bin(s))
|
||||
s = c_double.__ctype_le__(math.pi)
|
||||
self.assertEqual(s.value, math.pi)
|
||||
self.assertEqual(bin(struct.pack("<d", math.pi)), bin(s))
|
||||
s = c_double.__ctype_be__(math.pi)
|
||||
self.assertEqual(s.value, math.pi)
|
||||
self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s))
|
||||
|
||||
def test_endian_other(self):
|
||||
self.assertIs(c_byte.__ctype_le__, c_byte)
|
||||
self.assertIs(c_byte.__ctype_be__, c_byte)
|
||||
|
||||
self.assertIs(c_ubyte.__ctype_le__, c_ubyte)
|
||||
self.assertIs(c_ubyte.__ctype_be__, c_ubyte)
|
||||
|
||||
self.assertIs(c_char.__ctype_le__, c_char)
|
||||
self.assertIs(c_char.__ctype_be__, c_char)
|
||||
|
||||
def test_struct_fields_unsupported_byte_order(self):
|
||||
|
||||
fields = [
|
||||
("a", c_ubyte),
|
||||
("b", c_byte),
|
||||
("c", c_short),
|
||||
("d", c_ushort),
|
||||
("e", c_int),
|
||||
("f", c_uint),
|
||||
("g", c_long),
|
||||
("h", c_ulong),
|
||||
("i", c_longlong),
|
||||
("k", c_ulonglong),
|
||||
("l", c_float),
|
||||
("m", c_double),
|
||||
("n", c_char),
|
||||
("b1", c_byte, 3),
|
||||
("b2", c_byte, 3),
|
||||
("b3", c_byte, 2),
|
||||
("a", c_int * 3 * 3 * 3)
|
||||
]
|
||||
|
||||
# these fields do not support different byte order:
|
||||
for typ in c_wchar, c_void_p, POINTER(c_int):
|
||||
with self.assertRaises(TypeError):
|
||||
class T(BigEndianStructure if sys.byteorder == "little" else LittleEndianStructure):
|
||||
_fields_ = fields + [("x", typ)]
|
||||
|
||||
|
||||
def test_struct_struct(self):
|
||||
# nested structures with different byteorders
|
||||
|
||||
# create nested structures with given byteorders and set memory to data
|
||||
|
||||
for nested, data in (
|
||||
(BigEndianStructure, b'\0\0\0\1\0\0\0\2'),
|
||||
(LittleEndianStructure, b'\1\0\0\0\2\0\0\0'),
|
||||
):
|
||||
for parent in (
|
||||
BigEndianStructure,
|
||||
LittleEndianStructure,
|
||||
Structure,
|
||||
):
|
||||
class NestedStructure(nested):
|
||||
_fields_ = [("x", c_uint32),
|
||||
("y", c_uint32)]
|
||||
|
||||
class TestStructure(parent):
|
||||
_fields_ = [("point", NestedStructure)]
|
||||
|
||||
self.assertEqual(len(data), sizeof(TestStructure))
|
||||
ptr = POINTER(TestStructure)
|
||||
s = cast(data, ptr)[0]
|
||||
del ctypes._pointer_type_cache[TestStructure]
|
||||
self.assertEqual(s.point.x, 1)
|
||||
self.assertEqual(s.point.y, 2)
|
||||
|
||||
def test_struct_field_alignment(self):
|
||||
# standard packing in struct uses no alignment.
|
||||
# So, we have to align using pad bytes.
|
||||
#
|
||||
# Unaligned accesses will crash Python (on those platforms that
|
||||
# don't allow it, like sparc solaris).
|
||||
if sys.byteorder == "little":
|
||||
base = BigEndianStructure
|
||||
fmt = ">bxhid"
|
||||
else:
|
||||
base = LittleEndianStructure
|
||||
fmt = "<bxhid"
|
||||
|
||||
class S(base):
|
||||
_fields_ = [("b", c_byte),
|
||||
("h", c_short),
|
||||
("i", c_int),
|
||||
("d", c_double)]
|
||||
|
||||
s1 = S(0x12, 0x1234, 0x12345678, 3.14)
|
||||
s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
|
||||
self.assertEqual(bin(s1), bin(s2))
|
||||
|
||||
def test_unaligned_nonnative_struct_fields(self):
|
||||
if sys.byteorder == "little":
|
||||
base = BigEndianStructure
|
||||
fmt = ">b h xi xd"
|
||||
else:
|
||||
base = LittleEndianStructure
|
||||
fmt = "<b h xi xd"
|
||||
|
||||
class S(base):
|
||||
_pack_ = 1
|
||||
_fields_ = [("b", c_byte),
|
||||
("h", c_short),
|
||||
|
||||
("_1", c_byte),
|
||||
("i", c_int),
|
||||
|
||||
("_2", c_byte),
|
||||
("d", c_double)]
|
||||
|
||||
s1 = S()
|
||||
s1.b = 0x12
|
||||
s1.h = 0x1234
|
||||
s1.i = 0x12345678
|
||||
s1.d = 3.14
|
||||
s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
|
||||
self.assertEqual(bin(s1), bin(s2))
|
||||
|
||||
def test_unaligned_native_struct_fields(self):
|
||||
if sys.byteorder == "little":
|
||||
fmt = "<b h xi xd"
|
||||
else:
|
||||
base = LittleEndianStructure
|
||||
fmt = ">b h xi xd"
|
||||
|
||||
class S(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [("b", c_byte),
|
||||
|
||||
("h", c_short),
|
||||
|
||||
("_1", c_byte),
|
||||
("i", c_int),
|
||||
|
||||
("_2", c_byte),
|
||||
("d", c_double)]
|
||||
|
||||
s1 = S()
|
||||
s1.b = 0x12
|
||||
s1.h = 0x1234
|
||||
s1.i = 0x12345678
|
||||
s1.d = 3.14
|
||||
s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
|
||||
self.assertEqual(bin(s1), bin(s2))
|
||||
|
||||
def test_union_fields_unsupported_byte_order(self):
|
||||
|
||||
fields = [
|
||||
("a", c_ubyte),
|
||||
("b", c_byte),
|
||||
("c", c_short),
|
||||
("d", c_ushort),
|
||||
("e", c_int),
|
||||
("f", c_uint),
|
||||
("g", c_long),
|
||||
("h", c_ulong),
|
||||
("i", c_longlong),
|
||||
("k", c_ulonglong),
|
||||
("l", c_float),
|
||||
("m", c_double),
|
||||
("n", c_char),
|
||||
("b1", c_byte, 3),
|
||||
("b2", c_byte, 3),
|
||||
("b3", c_byte, 2),
|
||||
("a", c_int * 3 * 3 * 3)
|
||||
]
|
||||
|
||||
# these fields do not support different byte order:
|
||||
for typ in c_wchar, c_void_p, POINTER(c_int):
|
||||
with self.assertRaises(TypeError):
|
||||
class T(BigEndianUnion if sys.byteorder == "little" else LittleEndianUnion):
|
||||
_fields_ = fields + [("x", typ)]
|
||||
|
||||
def test_union_struct(self):
|
||||
# nested structures in unions with different byteorders
|
||||
|
||||
# create nested structures in unions with given byteorders and set memory to data
|
||||
|
||||
for nested, data in (
|
||||
(BigEndianStructure, b'\0\0\0\1\0\0\0\2'),
|
||||
(LittleEndianStructure, b'\1\0\0\0\2\0\0\0'),
|
||||
):
|
||||
for parent in (
|
||||
BigEndianUnion,
|
||||
LittleEndianUnion,
|
||||
Union,
|
||||
):
|
||||
class NestedStructure(nested):
|
||||
_fields_ = [("x", c_uint32),
|
||||
("y", c_uint32)]
|
||||
|
||||
class TestUnion(parent):
|
||||
_fields_ = [("point", NestedStructure)]
|
||||
|
||||
self.assertEqual(len(data), sizeof(TestUnion))
|
||||
ptr = POINTER(TestUnion)
|
||||
s = cast(data, ptr)[0]
|
||||
del ctypes._pointer_type_cache[TestUnion]
|
||||
self.assertEqual(s.point.x, 1)
|
||||
self.assertEqual(s.point.y, 2)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user