forked from Rust-related/RustPython
Compare commits
8 Commits
update_tes
...
release
| Author | SHA1 | Date | |
|---|---|---|---|
|
21e0ad6a2d | ||
|
|
8d510405f9 | ||
|
be42508cc6 | ||
|
|
f4c432b532 | ||
|
|
e0bfd18114 | ||
|
|
89d0e7f1e9 | ||
|
|
c8026f980e | ||
|
|
1929bc0953 |
@@ -1,5 +0,0 @@
|
||||
[target.'cfg(target_env = "msvc")']
|
||||
rustflags = "-C link-arg=/STACK:8000000"
|
||||
|
||||
[target.'cfg(all(target_os = "windows", not(target_env = "msvc")))']
|
||||
rustflags = "-C link-args=-Wl,--stack,8000000"
|
||||
299
.cspell.json
299
.cspell.json
@@ -1,299 +0,0 @@
|
||||
// See: https://github.com/streetsidesoftware/cspell/tree/master/packages/cspell
|
||||
{
|
||||
"version": "0.2",
|
||||
// language - current active spelling language
|
||||
"language": "en",
|
||||
// dictionaries - list of the names of the dictionaries to use
|
||||
"dictionaries": [
|
||||
"en_US",
|
||||
"softwareTerms",
|
||||
"c",
|
||||
"cpp",
|
||||
"python",
|
||||
"python-custom",
|
||||
"rust",
|
||||
"unix",
|
||||
"posix",
|
||||
"winapi"
|
||||
],
|
||||
// dictionaryDefinitions - this list defines any custom dictionaries to use
|
||||
"dictionaryDefinitions": [],
|
||||
"ignorePaths": [
|
||||
"**/__pycache__/**",
|
||||
"Lib/**"
|
||||
],
|
||||
// words - list of words to be always considered correct
|
||||
"words": [
|
||||
// Rust
|
||||
"ahash",
|
||||
"bidi",
|
||||
"biguint",
|
||||
"bindgen",
|
||||
"bitflags",
|
||||
"bstr",
|
||||
"byteorder",
|
||||
"chrono",
|
||||
"consts",
|
||||
"cstring",
|
||||
"flate2",
|
||||
"fract",
|
||||
"hasher",
|
||||
"idents",
|
||||
"indexmap",
|
||||
"insta",
|
||||
"keccak",
|
||||
"lalrpop",
|
||||
"libc",
|
||||
"libz",
|
||||
"longlong",
|
||||
"Manually",
|
||||
"maplit",
|
||||
"memmap",
|
||||
"metas",
|
||||
"modpow",
|
||||
"nanos",
|
||||
"objclass",
|
||||
"peekable",
|
||||
"powc",
|
||||
"powf",
|
||||
"prepended",
|
||||
"punct",
|
||||
"replacen",
|
||||
"rsplitn",
|
||||
"rustc",
|
||||
"rustfmt",
|
||||
"seekfrom",
|
||||
"splitn",
|
||||
"subsec",
|
||||
"timsort",
|
||||
"trai",
|
||||
"ulonglong",
|
||||
"unic",
|
||||
"unistd",
|
||||
"winapi",
|
||||
"winsock",
|
||||
// Python
|
||||
"abstractmethods",
|
||||
"aiter",
|
||||
"anext",
|
||||
"arrayiterator",
|
||||
"arraytype",
|
||||
"asend",
|
||||
"athrow",
|
||||
"basicsize",
|
||||
"cformat",
|
||||
"classcell",
|
||||
"closesocket",
|
||||
"codepoint",
|
||||
"codepoints",
|
||||
"cpython",
|
||||
"decompressor",
|
||||
"defaultaction",
|
||||
"descr",
|
||||
"dictcomp",
|
||||
"dictitems",
|
||||
"dictkeys",
|
||||
"dictview",
|
||||
"docstring",
|
||||
"docstrings",
|
||||
"dunder",
|
||||
"eventmask",
|
||||
"fdel",
|
||||
"fget",
|
||||
"fileencoding",
|
||||
"fillchar",
|
||||
"finallyhandler",
|
||||
"frombytes",
|
||||
"fromhex",
|
||||
"fromunicode",
|
||||
"fset",
|
||||
"fspath",
|
||||
"fstring",
|
||||
"fstrings",
|
||||
"genexpr",
|
||||
"getattro",
|
||||
"getformat",
|
||||
"getnewargs",
|
||||
"getweakrefcount",
|
||||
"getweakrefs",
|
||||
"hostnames",
|
||||
"idiv",
|
||||
"impls",
|
||||
"infj",
|
||||
"instancecheck",
|
||||
"instanceof",
|
||||
"isabstractmethod",
|
||||
"itemiterator",
|
||||
"itemsize",
|
||||
"iternext",
|
||||
"keyiterator",
|
||||
"kwarg",
|
||||
"kwargs",
|
||||
"linearization",
|
||||
"linearize",
|
||||
"listcomp",
|
||||
"mappingproxy",
|
||||
"maxsplit",
|
||||
"memoryview",
|
||||
"memoryviewiterator",
|
||||
"metaclass",
|
||||
"metaclasses",
|
||||
"metatype",
|
||||
"mro",
|
||||
"mros",
|
||||
"nanj",
|
||||
"ndigits",
|
||||
"ndim",
|
||||
"nonbytes",
|
||||
"origname",
|
||||
"posixsubprocess",
|
||||
"pyexpat",
|
||||
"PYTHONDEBUG",
|
||||
"PYTHONHOME",
|
||||
"PYTHONINSPECT",
|
||||
"PYTHONOPTIMIZE",
|
||||
"PYTHONPATH",
|
||||
"PYTHONPATH",
|
||||
"PYTHONVERBOSE",
|
||||
"PYTHONWARNINGS",
|
||||
"qualname",
|
||||
"radd",
|
||||
"rdiv",
|
||||
"rdivmod",
|
||||
"reconstructor",
|
||||
"reversevalueiterator",
|
||||
"rfloordiv",
|
||||
"rlshift",
|
||||
"rmod",
|
||||
"rpow",
|
||||
"rrshift",
|
||||
"rsub",
|
||||
"rtruediv",
|
||||
"scproxy",
|
||||
"setattro",
|
||||
"setcomp",
|
||||
"showwarnmsg",
|
||||
"warnmsg",
|
||||
"stacklevel",
|
||||
"subclasscheck",
|
||||
"subclasshook",
|
||||
"unionable",
|
||||
"unraisablehook",
|
||||
"valueiterator",
|
||||
"vararg",
|
||||
"varargs",
|
||||
"varnames",
|
||||
"warningregistry",
|
||||
"warnopts",
|
||||
"weakproxy",
|
||||
"xopts",
|
||||
// RustPython
|
||||
"baseclass",
|
||||
"Bytecode",
|
||||
"cfgs",
|
||||
"codegen",
|
||||
"dedentations",
|
||||
"dedents",
|
||||
"deduped",
|
||||
"downcasted",
|
||||
"dumpable",
|
||||
"GetSet",
|
||||
"internable",
|
||||
"makeunicodedata",
|
||||
"miri",
|
||||
"notrace",
|
||||
"pyarg",
|
||||
"pyarg",
|
||||
"pyargs",
|
||||
"PyAttr",
|
||||
"pyc",
|
||||
"PyClass",
|
||||
"PyClassMethod",
|
||||
"PyException",
|
||||
"PyFunction",
|
||||
"pygetset",
|
||||
"pyimpl",
|
||||
"pymember",
|
||||
"PyMethod",
|
||||
"PyModule",
|
||||
"pyname",
|
||||
"pyobj",
|
||||
"PyObject",
|
||||
"pypayload",
|
||||
"PyProperty",
|
||||
"pyref",
|
||||
"PyResult",
|
||||
"pyslot",
|
||||
"PyStaticMethod",
|
||||
"pystr",
|
||||
"pystruct",
|
||||
"pystructseq",
|
||||
"pytrace",
|
||||
"reducelib",
|
||||
"richcompare",
|
||||
"RustPython",
|
||||
"struc",
|
||||
"tracebacks",
|
||||
"typealiases",
|
||||
"Unconstructible",
|
||||
"unhashable",
|
||||
"uninit",
|
||||
"unraisable",
|
||||
"wasi",
|
||||
"zelf",
|
||||
// cpython
|
||||
"argtypes",
|
||||
"asdl",
|
||||
"asname",
|
||||
"augassign",
|
||||
"badsyntax",
|
||||
"basetype",
|
||||
"boolop",
|
||||
"bxor",
|
||||
"cellarg",
|
||||
"cellvar",
|
||||
"cellvars",
|
||||
"cmpop",
|
||||
"dictoffset",
|
||||
"elts",
|
||||
"excepthandler",
|
||||
"finalbody",
|
||||
"freevar",
|
||||
"freevars",
|
||||
"fromlist",
|
||||
"heaptype",
|
||||
"IMMUTABLETYPE",
|
||||
"kwonlyarg",
|
||||
"kwonlyargs",
|
||||
"linearise",
|
||||
"maxdepth",
|
||||
"mult",
|
||||
"nkwargs",
|
||||
"orelse",
|
||||
"patma",
|
||||
"posonlyarg",
|
||||
"posonlyargs",
|
||||
"prec",
|
||||
"stackdepth",
|
||||
"unaryop",
|
||||
"unparse",
|
||||
"unparser",
|
||||
"VARKEYWORDS",
|
||||
"varkwarg",
|
||||
"wbits",
|
||||
"withitem",
|
||||
"withs"
|
||||
],
|
||||
// flagWords - list of words to be always considered incorrect
|
||||
"flagWords": [
|
||||
],
|
||||
"ignoreRegExpList": [
|
||||
],
|
||||
// languageSettings - allow for per programming language configuration settings.
|
||||
"languageSettings": [
|
||||
{
|
||||
"languageId": "python",
|
||||
"locale": "en"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"image": "mcr.microsoft.com/devcontainers/universal:2",
|
||||
"features": {
|
||||
"ghcr.io/devcontainers/features/rust:1": {}
|
||||
}
|
||||
}
|
||||
7
.gitattributes
vendored
7
.gitattributes
vendored
@@ -1,6 +1 @@
|
||||
Lib/** linguist-vendored
|
||||
Cargo.lock linguist-generated -merge
|
||||
*.snap linguist-generated -merge
|
||||
vm/src/stdlib/ast/gen.rs linguist-generated -merge
|
||||
Lib/*.py text working-tree-encoding=UTF-8 eol=LF
|
||||
**/*.rs text working-tree-encoding=UTF-8 eol=LF
|
||||
Lib/* linguist-vendored
|
||||
|
||||
16
.github/ISSUE_TEMPLATE/empty.md
vendored
16
.github/ISSUE_TEMPLATE/empty.md
vendored
@@ -1,16 +0,0 @@
|
||||
---
|
||||
name: Generic issue template
|
||||
about: which is not covered by other templates
|
||||
title: ''
|
||||
labels:
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
<!-- Short description of the issue. -->
|
||||
|
||||
## Details
|
||||
|
||||
<!-- Whatever you want to share -->
|
||||
16
.github/ISSUE_TEMPLATE/feature-request.md
vendored
16
.github/ISSUE_TEMPLATE/feature-request.md
vendored
@@ -1,16 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Request a feature to use RustPython (as a Rust library)
|
||||
title: ''
|
||||
labels: C-enhancement
|
||||
assignees: 'youknowone'
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
<!-- Short description of the request. Please use incompatibility form to report missing features as Python interpreter -->
|
||||
|
||||
## Expected use case
|
||||
|
||||
<!-- By sharing detailed use case, we can understand the requirements better! If it will be used by open source projects, please also share the project URL. -->
|
||||
24
.github/ISSUE_TEMPLATE/report-bug.md
vendored
24
.github/ISSUE_TEMPLATE/report-bug.md
vendored
@@ -1,24 +0,0 @@
|
||||
---
|
||||
name: Report bugs
|
||||
about: Report a bug not related to CPython compatibility
|
||||
title: ''
|
||||
labels: C-bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
<!-- Short description of the bug -->
|
||||
|
||||
## Expected
|
||||
|
||||
<!-- What's the expected result? Using ``` ``` block is preferred for text. -->
|
||||
|
||||
## Actual
|
||||
|
||||
<!-- What's the actual result? Using ``` ``` block is preferred for text. -->
|
||||
|
||||
## Python Documentation
|
||||
|
||||
<!-- If applicable. -->
|
||||
@@ -2,7 +2,7 @@
|
||||
name: Report incompatibility
|
||||
about: Report an incompatibility between RustPython and CPython
|
||||
title: ''
|
||||
labels: C-compat
|
||||
labels: feat
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
@@ -11,6 +11,6 @@ assignees: ''
|
||||
|
||||
<!-- What Python feature is missing from RustPython? Give a short description of the feature and how you ran into its absence. -->
|
||||
|
||||
## Python Documentation or reference to CPython source code
|
||||
## Python Documentation
|
||||
|
||||
<!-- Give a link to the feature in the CPython documentation (https://docs.python.org/3/) in order to assist in its implementation. -->
|
||||
|
||||
13
.github/dependabot.yml
vendored
13
.github/dependabot.yml
vendored
@@ -1,13 +0,0 @@
|
||||
# Keep GitHub Actions up to date with GitHub's Dependabot...
|
||||
# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
|
||||
# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
groups:
|
||||
github-actions:
|
||||
patterns:
|
||||
- "*" # Group all Actions updates into a single larger pull request
|
||||
schedule:
|
||||
interval: weekly
|
||||
23
.github/workflows/ai-review.yml
vendored
23
.github/workflows/ai-review.yml
vendored
@@ -1,23 +0,0 @@
|
||||
name: PR Review
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
review:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: AI Code Review
|
||||
uses: gitea-actions/ai-reviewer@v0.6
|
||||
with:
|
||||
access-token: ${{ secrets.ACCESS_TOKEN }}
|
||||
full-context-model: "gpt-4o"
|
||||
full-context-api-key: ${{ secrets.OPENAI_API_KEY }}
|
||||
single-chunk-model: "claude-3-5-sonnet"
|
||||
single-chunk-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
exclude-files: "*.md,*.yaml"
|
||||
459
.github/workflows/ci.yaml
vendored
459
.github/workflows/ci.yaml
vendored
@@ -1,117 +1,24 @@
|
||||
on:
|
||||
push:
|
||||
branches: [main, release]
|
||||
branches: [master, release]
|
||||
pull_request:
|
||||
types: [unlabeled, opened, synchronize, reopened]
|
||||
merge_group:
|
||||
|
||||
name: CI
|
||||
|
||||
# Cancel previous workflows if they are the same workflow on same ref (branch/tags)
|
||||
# with the same event (push/pull_request) even they are in progress.
|
||||
# This setting will help reduce the number of duplicated workflows.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl
|
||||
# Skip additional tests on Windows. They are checked on Linux and MacOS.
|
||||
WINDOWS_SKIPS: >-
|
||||
test_datetime
|
||||
test_glob
|
||||
test_importlib
|
||||
test_io
|
||||
test_os
|
||||
test_pathlib
|
||||
test_posixpath
|
||||
test_venv
|
||||
# configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417
|
||||
# socketserver: seems related to configparser crash.
|
||||
MACOS_SKIPS: >-
|
||||
test_configparser
|
||||
test_socketserver
|
||||
# PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently
|
||||
# only run on Linux to speed up the CI.
|
||||
PLATFORM_INDEPENDENT_TESTS: >-
|
||||
test_argparse
|
||||
test_array
|
||||
test_asyncgen
|
||||
test_binop
|
||||
test_bisect
|
||||
test_bool
|
||||
test_bytes
|
||||
test_call
|
||||
test_class
|
||||
test_cmath
|
||||
test_collections
|
||||
test_complex
|
||||
test_contains
|
||||
test_copy
|
||||
test_dataclasses
|
||||
test_decimal
|
||||
test_decorators
|
||||
test_defaultdict
|
||||
test_deque
|
||||
test_dict
|
||||
test_dictcomps
|
||||
test_dictviews
|
||||
test_dis
|
||||
test_enumerate
|
||||
test_exception_variations
|
||||
test_exceptions
|
||||
test_float
|
||||
test_format
|
||||
test_fractions
|
||||
test_genericalias
|
||||
test_genericclass
|
||||
test_grammar
|
||||
test_range
|
||||
test_index
|
||||
test_int
|
||||
test_int_literal
|
||||
test_isinstance
|
||||
test_iter
|
||||
test_iterlen
|
||||
test_itertools
|
||||
test_json
|
||||
test_keyword
|
||||
test_keywordonlyarg
|
||||
test_list
|
||||
test_long
|
||||
test_longexp
|
||||
test_math
|
||||
test_operator
|
||||
test_ordered_dict
|
||||
test_pow
|
||||
test_raise
|
||||
test_richcmp
|
||||
test_scope
|
||||
test_set
|
||||
test_slice
|
||||
test_sort
|
||||
test_string
|
||||
test_string_literals
|
||||
test_strtod
|
||||
test_structseq
|
||||
test_subclassinit
|
||||
test_super
|
||||
test_syntax
|
||||
test_tuple
|
||||
test_types
|
||||
test_unary
|
||||
test_unicode
|
||||
test_unpack
|
||||
test_weakref
|
||||
test_yield_from
|
||||
# Python version targeted by the CI.
|
||||
PYTHON_VERSION: "3.12.3"
|
||||
CARGO_ARGS: --features "ssl jit"
|
||||
NON_WASM_PACKAGES: >
|
||||
-p rustpython-bytecode
|
||||
-p rustpython-common
|
||||
-p rustpython-compiler
|
||||
-p rustpython-parser
|
||||
-p rustpython-vm
|
||||
-p rustpython-jit
|
||||
-p rustpython-derive
|
||||
-p rustpython
|
||||
|
||||
jobs:
|
||||
rust_tests:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
env:
|
||||
RUST_BACKTRACE: full
|
||||
name: Run rust tests
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
@@ -119,121 +26,36 @@ jobs:
|
||||
os: [macos-latest, ubuntu-latest, windows-latest]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
|
||||
- uses: actions/checkout@master
|
||||
- uses: actions-rs/toolchain@v1
|
||||
- name: Set up the Windows environment
|
||||
shell: bash
|
||||
run: |
|
||||
cargo install --target-dir=target -v cargo-vcpkg
|
||||
cargo vcpkg -v build
|
||||
choco install llvm
|
||||
powershell.exe scripts/symlinks-to-hardlinks.ps1
|
||||
if: runner.os == 'Windows'
|
||||
- name: Set up the Mac environment
|
||||
run: brew install autoconf automake libtool
|
||||
if: runner.os == 'macOS'
|
||||
|
||||
- name: run clippy
|
||||
run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --exclude rustpython_wasm -- -Dwarnings
|
||||
|
||||
- name: Cache cargo dependencies
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-debug_opt3-${{ hashFiles('**/Cargo.lock') }}
|
||||
- name: run rust tests
|
||||
run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }}
|
||||
if: runner.os != 'macOS'
|
||||
- name: run rust tests
|
||||
run: cargo test --workspace --exclude rustpython_wasm --exclude rustpython-jit --verbose --features threading ${{ env.CARGO_ARGS }}
|
||||
if: runner.os == 'macOS'
|
||||
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: test
|
||||
args: --verbose ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }}
|
||||
- name: check compilation without threading
|
||||
run: cargo check ${{ env.CARGO_ARGS }}
|
||||
|
||||
- name: Test example projects
|
||||
run:
|
||||
cargo run --manifest-path example_projects/barebone/Cargo.toml
|
||||
cargo run --manifest-path example_projects/frozen_stdlib/Cargo.toml
|
||||
if: runner.os == 'Linux'
|
||||
|
||||
- name: prepare AppleSilicon build
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: aarch64-apple-darwin
|
||||
if: runner.os == 'macOS'
|
||||
- name: Check compilation for Apple Silicon
|
||||
run: cargo check --target aarch64-apple-darwin
|
||||
if: runner.os == 'macOS'
|
||||
- name: prepare iOS build
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: aarch64-apple-ios
|
||||
if: runner.os == 'macOS'
|
||||
- name: Check compilation for iOS
|
||||
run: cargo check --target aarch64-apple-ios
|
||||
if: runner.os == 'macOS'
|
||||
|
||||
exotic_targets:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
name: Ensure compilation on various targets
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: i686-unknown-linux-gnu
|
||||
|
||||
- name: Install gcc-multilib and musl-tools
|
||||
run: sudo apt-get update && sudo apt-get install gcc-multilib musl-tools
|
||||
- name: Check compilation for x86 32bit
|
||||
run: cargo check --target i686-unknown-linux-gnu
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: aarch64-linux-android
|
||||
|
||||
- name: Check compilation for android
|
||||
run: cargo check --target aarch64-linux-android
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: aarch64-unknown-linux-gnu
|
||||
|
||||
- name: Install gcc-aarch64-linux-gnu
|
||||
run: sudo apt install gcc-aarch64-linux-gnu
|
||||
- name: Check compilation for aarch64 linux gnu
|
||||
run: cargo check --target aarch64-unknown-linux-gnu
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: i686-unknown-linux-musl
|
||||
|
||||
- name: Check compilation for musl
|
||||
run: cargo check --target i686-unknown-linux-musl
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: x86_64-unknown-freebsd
|
||||
|
||||
- name: Check compilation for freebsd
|
||||
run: cargo check --target x86_64-unknown-freebsd
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: x86_64-unknown-freebsd
|
||||
|
||||
- name: Check compilation for freeBSD
|
||||
run: cargo check --target x86_64-unknown-freebsd
|
||||
|
||||
- name: Prepare repository for redox compilation
|
||||
run: bash scripts/redox/uncomment-cargo.sh
|
||||
- name: Check compilation for Redox
|
||||
uses: coolreader18/redoxer-action@v1
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: check
|
||||
args: ${{ env.CARGO_ARGS }} --no-default-features
|
||||
|
||||
snippets_cpython:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
env:
|
||||
RUST_BACKTRACE: full
|
||||
name: Run snippets and cpython tests
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
@@ -241,182 +63,173 @@ jobs:
|
||||
os: [macos-latest, ubuntu-latest, windows-latest]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@master
|
||||
- uses: actions-rs/toolchain@v1
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
python-version: 3.8
|
||||
- name: Set up the Windows environment
|
||||
shell: bash
|
||||
run: |
|
||||
cargo install cargo-vcpkg
|
||||
cargo vcpkg build
|
||||
choco install llvm
|
||||
powershell.exe scripts/symlinks-to-hardlinks.ps1
|
||||
if: runner.os == 'Windows'
|
||||
- name: Set up the Mac environment
|
||||
run: brew install autoconf automake libtool openssl@3
|
||||
run: brew install autoconf automake libtool
|
||||
if: runner.os == 'macOS'
|
||||
- name: build rustpython
|
||||
run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }}
|
||||
if: runner.os == 'macOS'
|
||||
- name: build rustpython
|
||||
run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit
|
||||
- name: Cache cargo dependencies
|
||||
uses: actions/cache@v2
|
||||
# cache gets corrupted for some reason on mac
|
||||
if: runner.os != 'macOS'
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
- name: run snippets
|
||||
run: python -m pip install -r requirements.txt && pytest -v
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-release-${{ hashFiles('**/Cargo.lock') }}
|
||||
- name: build rustpython
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: build
|
||||
args: --release --verbose ${{ env.CARGO_ARGS }}
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: 3.8
|
||||
- name: Install pipenv
|
||||
run: |
|
||||
python -V
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install pipenv
|
||||
- run: pipenv install --python 3.8
|
||||
working-directory: ./extra_tests
|
||||
- if: runner.os == 'Linux'
|
||||
name: run cpython platform-independent tests
|
||||
- name: run snippets
|
||||
run: pipenv run pytest -v
|
||||
working-directory: ./extra_tests
|
||||
- name: run cpython tests
|
||||
run: target/release/rustpython -m test -v
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
if: runner.os == 'Linux'
|
||||
- name: run cpython tests (macOS lightweight)
|
||||
run:
|
||||
target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }}
|
||||
- if: runner.os == 'Linux'
|
||||
name: run cpython platform-dependent tests (Linux)
|
||||
run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }}
|
||||
- if: runner.os == 'macOS'
|
||||
name: run cpython platform-dependent tests (MacOS)
|
||||
run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.MACOS_SKIPS }}
|
||||
- if: runner.os == 'Windows'
|
||||
name: run cpython platform-dependent tests (windows partial - fixme)
|
||||
target/release/rustpython -m test -v -x
|
||||
test_argparse test_json test_bytes test_bytearray test_long test_unicode test_array
|
||||
test_asyncgen test_list test_complex test_json test_set test_dis test_calendar
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
if: runner.os == 'macOS'
|
||||
- name: run cpython tests (windows partial - fixme)
|
||||
run:
|
||||
target/release/rustpython -m test -j 1 --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }}
|
||||
- if: runner.os != 'Windows'
|
||||
name: check that --install-pip succeeds
|
||||
run: |
|
||||
mkdir site-packages
|
||||
target/release/rustpython --install-pip ensurepip --user
|
||||
target/release/rustpython -m pip install six
|
||||
- if: runner.os != 'Windows'
|
||||
name: Check that ensurepip succeeds.
|
||||
run: |
|
||||
target/release/rustpython -m ensurepip
|
||||
target/release/rustpython -c "import pip"
|
||||
- if: runner.os != 'Windows'
|
||||
name: Check if pip inside venv is functional
|
||||
run: |
|
||||
target/release/rustpython -m venv testvenv
|
||||
testvenv/bin/rustpython -m pip install wheel
|
||||
- name: Check whats_left is not broken
|
||||
run: python -I whats_left.py
|
||||
target/release/rustpython -m test -v -x
|
||||
test_argparse test_json test_bytes test_long test_pwd test_bool test_cgi test_complex
|
||||
test_exception_hierarchy test_glob test_iter test_list test_os test_pathlib
|
||||
test_py_compile test_set test_shutil test_sys test_unicode test_unittest test_venv
|
||||
test_zipimport test_importlib test_io
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
if: runner.os == 'Windows'
|
||||
|
||||
lint:
|
||||
name: Check Rust code with rustfmt and clippy
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/checkout@master
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
components: rustfmt, clippy
|
||||
profile: minimal
|
||||
toolchain: stable
|
||||
components: rustfmt
|
||||
override: true
|
||||
- name: run rustfmt
|
||||
run: cargo fmt --check
|
||||
- name: run clippy on wasm
|
||||
run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
|
||||
- uses: actions/setup-python@v5
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
- name: install ruff
|
||||
run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778
|
||||
- name: run python lint
|
||||
run: ruff extra_tests wasm examples --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source
|
||||
- name: install prettier
|
||||
run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH
|
||||
- name: check wasm code with prettier
|
||||
# prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506
|
||||
run: cd wasm && git ls-files -z | xargs -0 prettier --check -u
|
||||
|
||||
command: fmt
|
||||
args: --all -- --check
|
||||
- name: run clippy
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: clippy
|
||||
args: ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }} -- -Dwarnings
|
||||
- name: run clippy on wasm
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: clippy
|
||||
args: --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: 3.8
|
||||
- name: install flake8
|
||||
run: python -m pip install flake8
|
||||
- name: run lint
|
||||
run: flake8 . --count --exclude=./.*,./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source --statistics
|
||||
miri:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
name: Run tests under miri
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@master
|
||||
- uses: actions/checkout@master
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
profile: minimal
|
||||
toolchain: nightly
|
||||
components: miri
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
override: true
|
||||
- name: Run tests under miri
|
||||
# miri-ignore-leaks because the type-object circular reference means that there will always be
|
||||
# a memory leak, at least until we have proper cyclic gc
|
||||
run: MIRIFLAGS='-Zmiri-ignore-leaks' cargo +nightly miri test -p rustpython-vm -- miri_test
|
||||
|
||||
wasm:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
name: Check the WASM package and demo
|
||||
needs: rust_tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: actions/checkout@master
|
||||
- name: Cache cargo dependencies
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-wasm_opt3-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-debug_opt3-${{ hashFiles('**/Cargo.lock') }}
|
||||
- name: install wasm-pack
|
||||
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
|
||||
- name: install geckodriver
|
||||
run: |
|
||||
wget https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz
|
||||
wget https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckodriver-v0.24.0-linux32.tar.gz
|
||||
mkdir geckodriver
|
||||
tar -xzf geckodriver-v0.34.0-linux64.tar.gz -C geckodriver
|
||||
- uses: actions/setup-python@v5
|
||||
tar -xzf geckodriver-v0.24.0-linux32.tar.gz -C geckodriver
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
- run: python -m pip install -r requirements.txt
|
||||
python-version: 3.8
|
||||
- name: Install pipenv
|
||||
run: |
|
||||
python -V
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install pipenv
|
||||
- run: pipenv install
|
||||
working-directory: ./wasm/tests
|
||||
- uses: actions/setup-node@v4
|
||||
- uses: actions/setup-node@v1
|
||||
- name: run test
|
||||
run: |
|
||||
export PATH=$PATH:`pwd`/../../geckodriver
|
||||
npm install
|
||||
npm run test
|
||||
env:
|
||||
NODE_OPTIONS: "--openssl-legacy-provider"
|
||||
working-directory: ./wasm/demo
|
||||
- uses: mwilliamson/setup-wabt-action@v3
|
||||
with: { wabt-version: "1.0.30" }
|
||||
- name: check wasm32-unknown without js
|
||||
run: |
|
||||
cargo build --release --manifest-path wasm/wasm-unknown-test/Cargo.toml --target wasm32-unknown-unknown --verbose
|
||||
if wasm-objdump -xj Import target/wasm32-unknown-unknown/release/wasm_unknown_test.wasm; then
|
||||
echo "ERROR: wasm32-unknown module expects imports from the host environment" >2
|
||||
fi
|
||||
- name: build notebook demo
|
||||
if: github.ref == 'refs/heads/release'
|
||||
run: |
|
||||
npm install
|
||||
npm run dist
|
||||
mv dist ../demo/dist/notebook
|
||||
env:
|
||||
NODE_OPTIONS: "--openssl-legacy-provider"
|
||||
working-directory: ./wasm/notebook
|
||||
- name: Deploy demo to Github Pages
|
||||
if: success() && github.ref == 'refs/heads/release'
|
||||
uses: peaceiris/actions-gh-pages@v4
|
||||
uses: peaceiris/actions-gh-pages@v2
|
||||
env:
|
||||
ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
|
||||
PUBLISH_DIR: ./wasm/demo/dist
|
||||
EXTERNAL_REPOSITORY: RustPython/demo
|
||||
PUBLISH_BRANCH: master
|
||||
|
||||
wasm-wasi:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
name: Run snippets and cpython tests on wasm-wasi
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: wasm32-wasi
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Setup Wasmer
|
||||
uses: wasmerio/setup-wasmer@v3
|
||||
- name: Install clang
|
||||
run: sudo apt-get update && sudo apt-get install clang -y
|
||||
- name: build rustpython
|
||||
run: cargo build --release --target wasm32-wasi --features freeze-stdlib,stdlib --verbose
|
||||
- name: run snippets
|
||||
run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py
|
||||
- name: run cpython unittest
|
||||
run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py
|
||||
|
||||
168
.github/workflows/cron-ci.yaml
vendored
168
.github/workflows/cron-ci.yaml
vendored
@@ -1,51 +1,72 @@
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 6'
|
||||
workflow_dispatch:
|
||||
|
||||
name: Periodic checks/tasks
|
||||
|
||||
env:
|
||||
CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
|
||||
PYTHON_VERSION: "3.12.0"
|
||||
|
||||
jobs:
|
||||
# codecov collects code coverage data from the rust tests, python snippets and python test suite.
|
||||
# This is done using cargo-llvm-cov, which is a wrapper around llvm-cov.
|
||||
redox:
|
||||
name: Check compilation on Redox
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: redoxos/redoxer:latest
|
||||
steps:
|
||||
- uses: actions/checkout@master
|
||||
- name: prepare repository for redoxer compilation
|
||||
run: bash scripts/redox/uncomment-cargo.sh
|
||||
- name: compile for redox
|
||||
run: redoxer build --verbose
|
||||
|
||||
codecov:
|
||||
name: Collect code coverage data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: taiki-e/install-action@cargo-llvm-cov
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@master
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
- run: sudo apt-get update && sudo apt-get -y install lcov
|
||||
- name: Run cargo-llvm-cov with Rust tests.
|
||||
run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
|
||||
- name: Run cargo-llvm-cov with Python snippets.
|
||||
run: python scripts/cargo-llvm-cov.py
|
||||
continue-on-error: true
|
||||
- name: Run cargo-llvm-cov with Python test suite.
|
||||
run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed
|
||||
continue-on-error: true
|
||||
- name: Prepare code coverage data
|
||||
run: cargo llvm-cov report --lcov --output-path='codecov.lcov'
|
||||
- name: Upload to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
toolchain: nightly
|
||||
override: true
|
||||
- uses: actions-rs/cargo@v1
|
||||
with:
|
||||
file: ./codecov.lcov
|
||||
command: build
|
||||
args: --verbose
|
||||
env:
|
||||
CARGO_INCREMENTAL: '0'
|
||||
RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests' # -Cpanic=abort
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: 3.8
|
||||
- name: Install pipenv
|
||||
run: |
|
||||
python -V
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install pipenv
|
||||
- run: pipenv install
|
||||
working-directory: ./extra_tests
|
||||
- name: run snippets
|
||||
run: pipenv run pytest -v
|
||||
working-directory: ./extra_tests
|
||||
env:
|
||||
RUSTPYTHON_DEBUG: 'true'
|
||||
- name: run cpython tests
|
||||
run: cargo run -- -m test -v
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
- uses: actions-rs/grcov@v0.1
|
||||
id: coverage
|
||||
- name: upload to Codecov
|
||||
uses: codecov/codecov-action@v1
|
||||
with:
|
||||
file: ${{ steps.coverage.outputs.report }}
|
||||
|
||||
testdata:
|
||||
name: Collect regression test data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/checkout@master
|
||||
- name: build rustpython
|
||||
run: cargo build --release --verbose
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: build
|
||||
args: --release --verbose
|
||||
- name: collect tests data
|
||||
run: cargo run --release extra_tests/jsontests.py
|
||||
env:
|
||||
@@ -63,86 +84,5 @@ jobs:
|
||||
cd website
|
||||
cp ../extra_tests/cpython_tests_results.json ./_data/regrtests_results.json
|
||||
git add ./_data/regrtests_results.json
|
||||
if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results" --author="$GITHUB_ACTOR"; then
|
||||
git push
|
||||
fi
|
||||
|
||||
whatsleft:
|
||||
name: Collect what is left data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
- name: build rustpython
|
||||
run: cargo build --release --verbose
|
||||
- name: Collect what is left data
|
||||
run: |
|
||||
chmod +x ./whats_left.py
|
||||
./whats_left.py > whats_left.temp
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
- name: Upload data to the website
|
||||
env:
|
||||
SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
|
||||
GITHUB_ACTOR: ${{ github.actor }}
|
||||
run: |
|
||||
echo "$SSHKEY" >~/github_key
|
||||
chmod 600 ~/github_key
|
||||
export GIT_SSH_COMMAND="ssh -i ~/github_key"
|
||||
|
||||
git clone git@github.com:RustPython/rustpython.github.io.git website
|
||||
cd website
|
||||
[ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp
|
||||
cp ../whats_left.temp ./_data/whats_left.temp
|
||||
git add -A
|
||||
if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then
|
||||
git push
|
||||
fi
|
||||
|
||||
benchmark:
|
||||
name: Collect benchmark data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.9
|
||||
- run: cargo install cargo-criterion
|
||||
- name: build benchmarks
|
||||
run: cargo build --release --benches
|
||||
- name: collect execution benchmark data
|
||||
run: cargo criterion --bench execution
|
||||
- name: collect microbenchmarks data
|
||||
run: cargo criterion --bench microbenchmarks
|
||||
- name: restructure generated files
|
||||
run: |
|
||||
cd ./target/criterion/reports
|
||||
find -type d -name cpython | xargs rm -rf
|
||||
find -type d -name rustpython | xargs rm -rf
|
||||
find -mindepth 2 -maxdepth 2 -name violin.svg | xargs rm -rf
|
||||
find -type f -not -name violin.svg | xargs rm -rf
|
||||
for file in $(find -type f -name violin.svg); do mv $file $(echo $file | sed -E "s_\./([^/]+)/([^/]+)/violin\.svg_./\1/\2.svg_"); done
|
||||
find -mindepth 2 -maxdepth 2 -type d | xargs rm -rf
|
||||
cd ..
|
||||
mv reports/* .
|
||||
rmdir reports
|
||||
- name: upload benchmark data to the website
|
||||
env:
|
||||
SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
|
||||
run: |
|
||||
echo "$SSHKEY" >~/github_key
|
||||
chmod 600 ~/github_key
|
||||
export GIT_SSH_COMMAND="ssh -i ~/github_key"
|
||||
|
||||
git clone git@github.com:RustPython/rustpython.github.io.git website
|
||||
cd website
|
||||
rm -rf ./assets/criterion
|
||||
cp -r ../target/criterion ./assets/criterion
|
||||
git add ./assets/criterion
|
||||
if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then
|
||||
git push
|
||||
fi
|
||||
git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results" --author="$GITHUB_ACTOR"
|
||||
git push
|
||||
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
@@ -9,8 +9,7 @@ __pycache__
|
||||
.vscode
|
||||
wasm-pack.log
|
||||
.idea/
|
||||
.envrc
|
||||
.python-version
|
||||
extra_tests/snippets/resources
|
||||
|
||||
flame-graph.html
|
||||
flame.txt
|
||||
@@ -18,6 +17,3 @@ flamescope.json
|
||||
/wapm.lock
|
||||
/wapm_packages
|
||||
/.cargo/config
|
||||
|
||||
extra_tests/snippets/resources
|
||||
extra_tests/not_impl.py
|
||||
|
||||
8
.mailmap
8
.mailmap
@@ -1,8 +0,0 @@
|
||||
#
|
||||
# This list is used by git-shortlog to aggregate contributions. It is
|
||||
# necessary when either the author's full name is not always written
|
||||
# the same way, and/or the same author contributes from different
|
||||
# email addresses.
|
||||
#
|
||||
|
||||
Noa <coolreader18@gmail.com> <33094578+coolreader18@users.noreply.github.com>
|
||||
298
.vscode/launch.json
vendored
298
.vscode/launch.json
vendored
@@ -1,298 +0,0 @@
|
||||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug executable 'rustpython'",
|
||||
"preLaunchTask": "Build RustPython Debug",
|
||||
"program": "target/debug/rustpython",
|
||||
"args": [],
|
||||
"env": {
|
||||
"RUST_BACKTRACE": "1"
|
||||
},
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug executable 'rustpython' without SSL",
|
||||
"preLaunchTask": "Build RustPython Debug without SSL",
|
||||
"program": "target/debug/rustpython",
|
||||
"args": [],
|
||||
"env": {
|
||||
"RUST_BACKTRACE": "1"
|
||||
},
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug benchmark 'execution'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--bench=execution",
|
||||
"--package=rustpython"
|
||||
],
|
||||
"filter": {
|
||||
"name": "execution",
|
||||
"kind": "bench"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug benchmark 'microbenchmarks'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--bench=microbenchmarks",
|
||||
"--package=rustpython"
|
||||
],
|
||||
"filter": {
|
||||
"name": "microbenchmarks",
|
||||
"kind": "bench"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-pylib'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-pylib"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-pylib",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-bytecode'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-bytecode"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-bytecode",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-compiler'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-compiler"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-compiler",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-compiler-core'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-compiler-core"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-compiler-core",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-ast'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-ast"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-ast",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-parser'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-parser"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-parser",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-vm'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-vm"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-vm",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-common'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-common"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-common",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython-jit'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython-jit"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython-jit",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug integration test 'integration'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--test=integration",
|
||||
"--package=rustpython-jit"
|
||||
],
|
||||
"filter": {
|
||||
"name": "integration",
|
||||
"kind": "test"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"name": "Debug unit tests in library 'rustpython_wasm'",
|
||||
"cargo": {
|
||||
"args": [
|
||||
"test",
|
||||
"--no-run",
|
||||
"--lib",
|
||||
"--package=rustpython_wasm"
|
||||
],
|
||||
"filter": {
|
||||
"name": "rustpython_wasm",
|
||||
"kind": "lib"
|
||||
}
|
||||
},
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}"
|
||||
}
|
||||
]
|
||||
}
|
||||
36
.vscode/tasks.json
vendored
36
.vscode/tasks.json
vendored
@@ -1,36 +0,0 @@
|
||||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "Build RustPython Debug without SSL",
|
||||
"type": "shell",
|
||||
"command": "cargo",
|
||||
"args": [
|
||||
"build",
|
||||
],
|
||||
"problemMatcher": [
|
||||
"$rustc",
|
||||
],
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true,
|
||||
},
|
||||
},
|
||||
{
|
||||
"label": "Build RustPython Debug",
|
||||
"type": "shell",
|
||||
"command": "cargo",
|
||||
"args": [
|
||||
"build",
|
||||
"--features=ssl"
|
||||
],
|
||||
"problemMatcher": [
|
||||
"$rustc",
|
||||
],
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true,
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
2785
Cargo.lock
generated
2785
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
199
Cargo.toml
199
Cargo.toml
@@ -1,54 +1,50 @@
|
||||
[package]
|
||||
name = "rustpython"
|
||||
version = "0.1.2"
|
||||
authors = ["RustPython Team"]
|
||||
edition = "2018"
|
||||
description = "A python interpreter written in rust."
|
||||
repository = "https://github.com/RustPython/RustPython"
|
||||
license = "MIT"
|
||||
include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
|
||||
version.workspace = true
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[workspace]
|
||||
members = [
|
||||
".", "ast", "bytecode", "common", "compiler", "compiler/porcelain",
|
||||
"derive", "jit", "parser", "vm", "vm/pylib-crate", "wasm/lib",
|
||||
]
|
||||
|
||||
[features]
|
||||
default = ["threading", "stdlib", "zlib", "importlib"]
|
||||
importlib = ["rustpython-vm/importlib"]
|
||||
encodings = ["rustpython-vm/encodings"]
|
||||
stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"]
|
||||
default = ["threading", "pylib"]
|
||||
flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
|
||||
freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
|
||||
freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
|
||||
jit = ["rustpython-vm/jit"]
|
||||
threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
|
||||
zlib = ["stdlib", "rustpython-stdlib/zlib"]
|
||||
bz2 = ["stdlib", "rustpython-stdlib/bz2"]
|
||||
sqlite = ["rustpython-stdlib/sqlite"]
|
||||
ssl = ["rustpython-stdlib/ssl"]
|
||||
ssl-vendor = ["ssl", "rustpython-stdlib/ssl-vendor"]
|
||||
threading = ["rustpython-vm/threading"]
|
||||
|
||||
ssl = ["rustpython-vm/ssl"]
|
||||
|
||||
[dependencies]
|
||||
rustpython-compiler = { workspace = true }
|
||||
rustpython-pylib = { workspace = true, optional = true }
|
||||
rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
|
||||
rustpython-vm = { workspace = true, features = ["compiler"] }
|
||||
rustpython-parser = { workspace = true }
|
||||
log = "0.4"
|
||||
env_logger = "0.7"
|
||||
clap = "2.33"
|
||||
rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" }
|
||||
rustpython-parser = { path = "parser", version = "0.1.1" }
|
||||
rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
|
||||
pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1.0", default-features = false, optional = true }
|
||||
dirs = { package = "dirs-next", version = "1.0" }
|
||||
num-traits = "0.2.8"
|
||||
cfg-if = "0.1"
|
||||
libc = "0.2"
|
||||
|
||||
cfg-if = { workspace = true }
|
||||
log = { workspace = true }
|
||||
flame = { workspace = true, optional = true }
|
||||
flame = { version = "0.2", optional = true }
|
||||
flamescope = { version = "0.1", optional = true }
|
||||
|
||||
clap = "2.34"
|
||||
dirs = { package = "dirs-next", version = "2.0.0" }
|
||||
env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
|
||||
flamescope = { version = "0.1.2", optional = true }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
libc = { workspace = true }
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
rustyline = { workspace = true }
|
||||
[target.'cfg(not(target_os = "wasi"))'.dependencies]
|
||||
rustyline = "6.0"
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { version = "0.3.5", features = ["html_reports"] }
|
||||
pyo3 = { version = "0.22", features = ["auto-initialize"] }
|
||||
cpython = "0.5.0"
|
||||
criterion = "0.3"
|
||||
|
||||
[[bench]]
|
||||
name = "execution"
|
||||
@@ -65,133 +61,16 @@ path = "src/main.rs"
|
||||
[profile.dev.package."*"]
|
||||
opt-level = 3
|
||||
|
||||
[profile.test]
|
||||
opt-level = 3
|
||||
# https://github.com/rust-lang/rust/issues/92869
|
||||
# lto = "thin"
|
||||
|
||||
[profile.bench]
|
||||
lto = "thin"
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
opt-level = 3
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
|
||||
[patch.crates-io]
|
||||
# REDOX START, Uncomment when you want to compile/check with redoxer
|
||||
# REDOX START, Uncommment when you want to compile/check with redoxer
|
||||
# # following patches are just waiting on a new version to be released to crates.io
|
||||
# nix = { git = "https://github.com/nix-rust/nix" }
|
||||
# crossbeam-utils = { git = "https://github.com/crossbeam-rs/crossbeam" }
|
||||
# socket2 = { git = "https://github.com/alexcrichton/socket2-rs" }
|
||||
# REDOX END
|
||||
|
||||
# Used only on Windows to build the vcpkg dependencies
|
||||
[package.metadata.vcpkg]
|
||||
git = "https://github.com/microsoft/vcpkg"
|
||||
# The revision of the vcpkg repository to use
|
||||
# https://github.com/microsoft/vcpkg/tags
|
||||
rev = "2024.02.14"
|
||||
|
||||
[package.metadata.vcpkg.target]
|
||||
x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] }
|
||||
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"compiler", "compiler/core", "compiler/codegen",
|
||||
".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl",
|
||||
"wasm/lib",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.4.0"
|
||||
authors = ["RustPython Team"]
|
||||
edition = "2021"
|
||||
rust-version = "1.80.0"
|
||||
repository = "https://github.com/RustPython/RustPython"
|
||||
license = "MIT"
|
||||
|
||||
[workspace.dependencies]
|
||||
rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
|
||||
rustpython-compiler = { path = "compiler", version = "0.4.0" }
|
||||
rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }
|
||||
rustpython-common = { path = "common", version = "0.4.0" }
|
||||
rustpython-derive = { path = "derive", version = "0.4.0" }
|
||||
rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" }
|
||||
rustpython-jit = { path = "jit", version = "0.4.0" }
|
||||
rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" }
|
||||
rustpython-pylib = { path = "pylib", version = "0.4.0" }
|
||||
rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" }
|
||||
rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
|
||||
rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
|
||||
|
||||
rustpython-literal = { version = "0.4.0" }
|
||||
rustpython-parser-core = { version = "0.4.0" }
|
||||
rustpython-parser = { version = "0.4.0" }
|
||||
rustpython-ast = { version = "0.4.0" }
|
||||
rustpython-format= { version = "0.4.0" }
|
||||
# rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
|
||||
# rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
|
||||
# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
|
||||
# rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
|
||||
# rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "00d2f1d1a7522ef9c85c10dfa5f0bb7178dee655" }
|
||||
# rustpython-literal = { path = "../RustPython-parser/literal" }
|
||||
# rustpython-parser-core = { path = "../RustPython-parser/core" }
|
||||
# rustpython-parser = { path = "../RustPython-parser/parser" }
|
||||
# rustpython-ast = { path = "../RustPython-parser/ast" }
|
||||
# rustpython-format = { path = "../RustPython-parser/format" }
|
||||
|
||||
ahash = "0.8.11"
|
||||
ascii = "1.0"
|
||||
bitflags = "2.4.1"
|
||||
bstr = "1"
|
||||
cfg-if = "1.0"
|
||||
chrono = "0.4.37"
|
||||
crossbeam-utils = "0.8.19"
|
||||
flame = "0.2.2"
|
||||
getrandom = "0.2.12"
|
||||
glob = "0.3"
|
||||
hex = "0.4.3"
|
||||
indexmap = { version = "2.2.6", features = ["std"] }
|
||||
insta = "1.38.0"
|
||||
itertools = "0.11.0"
|
||||
is-macro = "0.3.0"
|
||||
junction = "1.0.0"
|
||||
libc = "0.2.153"
|
||||
log = "0.4.16"
|
||||
nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
|
||||
malachite-bigint = "0.2.0"
|
||||
malachite-q = "0.4.4"
|
||||
malachite-base = "0.4.4"
|
||||
memchr = "2.7.2"
|
||||
num-complex = "0.4.0"
|
||||
num-integer = "0.1.44"
|
||||
num-traits = "0.2"
|
||||
num_enum = { version = "0.7", default-features = false }
|
||||
once_cell = "1.19.0"
|
||||
parking_lot = "0.12.1"
|
||||
paste = "1.0.7"
|
||||
rand = "0.8.5"
|
||||
rustix = { version = "0.38", features = ["event"] }
|
||||
rustyline = "14.0.0"
|
||||
serde = { version = "1.0.133", default-features = false }
|
||||
schannel = "0.1.22"
|
||||
static_assertions = "1.1"
|
||||
strum = "0.26"
|
||||
strum_macros = "0.26"
|
||||
syn = "1.0.109"
|
||||
thiserror = "1.0"
|
||||
thread_local = "1.1.4"
|
||||
unicode_names2 = "1.2.0"
|
||||
widestring = "1.1.0"
|
||||
windows-sys = "0.52.0"
|
||||
wasm-bindgen = "0.2.92"
|
||||
|
||||
# Lints
|
||||
|
||||
[workspace.lints.rust]
|
||||
unsafe_code = "allow"
|
||||
|
||||
[workspace.lints.clippy]
|
||||
perf = "warn"
|
||||
style = "warn"
|
||||
complexity = "warn"
|
||||
suspicious = "warn"
|
||||
correctness = "warn"
|
||||
|
||||
100
DEVELOPMENT.md
100
DEVELOPMENT.md
@@ -19,13 +19,13 @@ The contents of the Development Guide include:
|
||||
|
||||
RustPython requires the following:
|
||||
|
||||
- Rust latest stable version (e.g 1.69.0 as of Apr 20 2023)
|
||||
- Rust latest stable version (e.g 1.38.0 at Oct 1st 2019)
|
||||
- To check Rust version: `rustc --version`
|
||||
- If you have `rustup` on your system, enter to update to the latest
|
||||
stable version: `rustup update stable`
|
||||
- If you do not have Rust installed, use [rustup](https://rustup.rs/) to
|
||||
do so.
|
||||
- CPython version 3.12 or higher
|
||||
- CPython version 3.7.4 or higher
|
||||
- CPython can be installed by your operating system's package manager,
|
||||
from the [Python website](https://www.python.org/downloads/), or
|
||||
using a third-party distribution, such as
|
||||
@@ -41,16 +41,12 @@ RustPython requires the following:
|
||||
|
||||
The Rust code style used is the default
|
||||
[rustfmt](https://github.com/rust-lang/rustfmt) codestyle. Please format your
|
||||
code accordingly, or run `cargo fmt` to autoformat it. We also use
|
||||
[clippy](https://github.com/rust-lang/rust-clippy) to lint Rust code, which
|
||||
you can check yourself with `cargo clippy`.
|
||||
code accordingly. We also use [clippy](https://github.com/rust-lang/rust-clippy)
|
||||
to detect rust code issues.
|
||||
|
||||
Custom Python code (i.e. code not copied from CPython's standard library) should
|
||||
follow the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style. We also use
|
||||
[ruff](https://beta.ruff.rs/docs/) to check Python code style.
|
||||
|
||||
In addition to language specific tools, [cspell](https://github.com/streetsidesoftware/cspell),
|
||||
a code spell checker, is used in order to ensure correct spellings for code.
|
||||
Python code should follow the
|
||||
[PEP 8](https://www.python.org/dev/peps/pep-0008/) style. We also use
|
||||
[flake8](http://flake8.pycqa.org/en/latest/) to check Python code style.
|
||||
|
||||
## Testing
|
||||
|
||||
@@ -65,34 +61,7 @@ $ pytest -v
|
||||
Rust unit tests can be run with `cargo`:
|
||||
|
||||
```shell
|
||||
$ cargo test --workspace --exclude rustpython_wasm
|
||||
```
|
||||
|
||||
Python unit tests can be run by compiling RustPython and running the test module:
|
||||
|
||||
```shell
|
||||
$ cargo run --release -- -m test
|
||||
```
|
||||
|
||||
There are a few test options that are especially useful:
|
||||
|
||||
- `-j <n>` enables parallel testing (which is a lot faster), where `<n>` is the
|
||||
number of threads to be used, ideally the same as number of cores on your CPU.
|
||||
If you don't know, `-j 4` or `-j 8` are good options.
|
||||
- `-v` enables verbose mode, adding additional information about the tests being
|
||||
run.
|
||||
- `<test_name>` specifies a single test to run instead of running all tests.
|
||||
|
||||
For example, to run all tests in parallel:
|
||||
|
||||
```shell
|
||||
$ cargo run --release -- -m test -j 4
|
||||
```
|
||||
|
||||
To run only `test_cmath` (located at `Lib/test/test_cmath`) verbosely:
|
||||
|
||||
```shell
|
||||
$ cargo run --release -- -m test test_cmath -v
|
||||
$ cargo test --all
|
||||
```
|
||||
|
||||
## Profiling
|
||||
@@ -118,29 +87,35 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i
|
||||
Understanding a new codebase takes time. Here's a brief view of the
|
||||
repository's structure:
|
||||
|
||||
- `bytecode/src`: python bytecode representation in rust structures
|
||||
- `compiler/src`: python compilation to bytecode
|
||||
- `core/src`: python bytecode representation in rust structures
|
||||
- `parser/src`: python lexing, parsing and ast
|
||||
- `derive/src`: Rust language extensions and macros specific to rustpython
|
||||
- `parser/src`: python lexing, parsing and ast
|
||||
- `Lib`: Carefully selected / copied files from CPython sourcecode. This is
|
||||
the python side of the standard library.
|
||||
- `test`: CPython test suite
|
||||
- `vm/src`: python virtual machine
|
||||
- `builtins`: Builtin functions and types
|
||||
- `builtins.rs`: Builtin functions
|
||||
- `compile.rs`: the python compiler from ast to bytecode
|
||||
- `obj`: python builtin types
|
||||
- `stdlib`: Standard library parts implemented in rust.
|
||||
- `src`: using the other subcrates to bring rustpython to life.
|
||||
- `docs`: documentation (work in progress)
|
||||
- `py_code_object`: CPython bytecode to rustpython bytecode converter (work in
|
||||
progress)
|
||||
- `wasm`: Binary crate and resources for WebAssembly build
|
||||
- `extra_tests`: extra integration test snippets as a supplement to `Lib/test`
|
||||
- `extra_tests`: extra integration test snippets as supplement of `Lib/test`
|
||||
|
||||
## Understanding Internals
|
||||
|
||||
The RustPython workspace includes the `rustpython` top-level crate. The `Cargo.toml`
|
||||
file in the root of the repo provide configuration of the crate and the
|
||||
implementation is found in the `src` directory (specifically, `src/lib.rs`).
|
||||
implementation is found in the `src` directory (specifically,
|
||||
`src/main.rs`).
|
||||
|
||||
The top-level `rustpython` binary depends on several lower-level crates including:
|
||||
|
||||
- `rustpython-parser` (implementation in `compiler/parser/src`)
|
||||
- `rustpython-parser` (implementation in `parser/src`)
|
||||
- `rustpython-compiler` (implementation in `compiler/src`)
|
||||
- `rustpython-vm` (implementation in `vm/src`)
|
||||
|
||||
@@ -158,26 +133,25 @@ enable a line of code to go through a series of steps:
|
||||
This crate contains the lexer and parser to convert a line of code to
|
||||
an Abstract Syntax Tree (AST):
|
||||
|
||||
- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens
|
||||
- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses
|
||||
- Lexer: `parser/lexer.rs` converts Python source code into tokens
|
||||
- Parser: `parser/parser.rs` takes the tokens generated by the lexer and parses
|
||||
the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax
|
||||
tree are Rust structs and enums.
|
||||
- The Parser relies on `LALRPOP`, a Rust parser generator framework. The
|
||||
LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`.
|
||||
- The Parser relies on `LALRPOP`, a Rust parser generator framework.
|
||||
- More information on parsers and a tutorial can be found in the
|
||||
[LALRPOP book](https://lalrpop.github.io/lalrpop/).
|
||||
- AST: `compiler/ast/` implements in Rust the Python types and expressions
|
||||
[LALRPOP book](https://lalrpop.github.io/lalrpop/README.html).
|
||||
- AST: `parser/ast.rs` implements in Rust the Python types and expressions
|
||||
represented by the AST nodes.
|
||||
|
||||
### rustpython-compiler
|
||||
|
||||
The `rustpython-compiler` crate's purpose is to transform the AST (Abstract Syntax
|
||||
Tree) to bytecode. The implementation of the compiler is found in the
|
||||
`compiler/src` directory. The compiler implements Python's symbol table,
|
||||
ast->bytecode compiler, and bytecode optimizer in Rust.
|
||||
`compiler/src` directory. The compiler implements Python's peephole optimizer
|
||||
implementation, Symbol table, and streams in Rust.
|
||||
|
||||
Implementation of bytecode structure in Rust is found in the `compiler/core/src`
|
||||
directory. `compiler/core/src/bytecode.rs` contains the representation of
|
||||
Implementation of bytecode structure in Rust is found in the `bytecode/src`
|
||||
directory. The `bytecode/src/bytecode.rs` contains the representation of
|
||||
instructions and operations in Rust. Further information about Python's
|
||||
bytecode instructions can be found in the
|
||||
[Python documentation](https://docs.python.org/3/library/dis.html#bytecodes).
|
||||
@@ -189,20 +163,10 @@ executes Python's instructions. The `vm/src` directory contains code to
|
||||
implement the read and evaluation loop that fetches and dispatches
|
||||
instructions. This directory also contains the implementation of the
|
||||
Python Standard Library modules in Rust (`vm/src/stdlib`). In Python
|
||||
everything can be represented as an object. The `vm/src/builtins` directory holds
|
||||
the Rust code used to represent different Python objects and their methods. The
|
||||
core implementation of what a Python object is can be found in
|
||||
`vm/src/object/core.rs`.
|
||||
|
||||
### Code generation
|
||||
|
||||
There are some code generations involved in building RustPython:
|
||||
|
||||
- some part of the AST code is generated from `vm/src/stdlib/ast/gen.rs` to `compiler/ast/src/ast_gen.rs`.
|
||||
- the `__doc__` attributes are generated by the
|
||||
[__doc__](https://github.com/RustPython/__doc__) project which is then included as the `rustpython-doc` crate.
|
||||
everything can be represented as an Object. `vm/src/obj` directory holds
|
||||
the Rust code used to represent a Python Object and its methods.
|
||||
|
||||
## Questions
|
||||
|
||||
Have you tried these steps and have a question, please chat with us on
|
||||
[Discord](https://discord.gg/vru8NypEhv).
|
||||
[gitter](https://gitter.im/rustpython/Lobby).
|
||||
|
||||
49
Lib/__future__.py
vendored
49
Lib/__future__.py
vendored
@@ -33,7 +33,7 @@ in releases at or after that, modules no longer need
|
||||
to use the feature in question, but may continue to use such imports.
|
||||
|
||||
MandatoryRelease may also be None, meaning that a planned feature got
|
||||
dropped or that the release version is undetermined.
|
||||
dropped.
|
||||
|
||||
Instances of class _Feature have two corresponding methods,
|
||||
.getOptionalRelease() and .getMandatoryRelease().
|
||||
@@ -42,7 +42,7 @@ CompilerFlag is the (bitfield) flag that should be passed in the fourth
|
||||
argument to the builtin function compile() to enable the feature in
|
||||
dynamically compiled code. This flag is stored in the .compiler_flag
|
||||
attribute on _Future instances. These values must match the appropriate
|
||||
#defines of CO_xxx flags in Include/cpython/compile.h.
|
||||
#defines of CO_xxx flags in Include/compile.h.
|
||||
|
||||
No feature line is ever to be deleted from this file.
|
||||
"""
|
||||
@@ -57,29 +57,25 @@ all_feature_names = [
|
||||
"unicode_literals",
|
||||
"barry_as_FLUFL",
|
||||
"generator_stop",
|
||||
"annotations",
|
||||
]
|
||||
|
||||
__all__ = ["all_feature_names"] + all_feature_names
|
||||
|
||||
# The CO_xxx symbols are defined here under the same names defined in
|
||||
# code.h and used by compile.h, so that an editor search will find them here.
|
||||
# However, they're not exported in __all__, because they don't really belong to
|
||||
# The CO_xxx symbols are defined here under the same names used by
|
||||
# compile.h, so that an editor search will find them here. However,
|
||||
# they're not exported in __all__, because they don't really belong to
|
||||
# this module.
|
||||
CO_NESTED = 0x0010 # nested_scopes
|
||||
CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000)
|
||||
CO_FUTURE_DIVISION = 0x20000 # division
|
||||
CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default
|
||||
CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement
|
||||
CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function
|
||||
CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals
|
||||
CO_FUTURE_BARRY_AS_BDFL = 0x400000
|
||||
CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators
|
||||
CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime
|
||||
|
||||
CO_NESTED = 0x0010 # nested_scopes
|
||||
CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000)
|
||||
CO_FUTURE_DIVISION = 0x2000 # division
|
||||
CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
|
||||
CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement
|
||||
CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function
|
||||
CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
|
||||
CO_FUTURE_BARRY_AS_BDFL = 0x40000
|
||||
CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators
|
||||
|
||||
class _Feature:
|
||||
|
||||
def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
|
||||
self.optional = optionalRelease
|
||||
self.mandatory = mandatoryRelease
|
||||
@@ -90,14 +86,16 @@ class _Feature:
|
||||
|
||||
This is a 5-tuple, of the same form as sys.version_info.
|
||||
"""
|
||||
|
||||
return self.optional
|
||||
|
||||
def getMandatoryRelease(self):
|
||||
"""Return release in which this feature will become mandatory.
|
||||
|
||||
This is a 5-tuple, of the same form as sys.version_info, or, if
|
||||
the feature was dropped, or the release date is undetermined, is None.
|
||||
the feature was dropped, is None.
|
||||
"""
|
||||
|
||||
return self.mandatory
|
||||
|
||||
def __repr__(self):
|
||||
@@ -105,7 +103,6 @@ class _Feature:
|
||||
self.mandatory,
|
||||
self.compiler_flag))
|
||||
|
||||
|
||||
nested_scopes = _Feature((2, 1, 0, "beta", 1),
|
||||
(2, 2, 0, "alpha", 0),
|
||||
CO_NESTED)
|
||||
@@ -135,13 +132,9 @@ unicode_literals = _Feature((2, 6, 0, "alpha", 2),
|
||||
CO_FUTURE_UNICODE_LITERALS)
|
||||
|
||||
barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2),
|
||||
(4, 0, 0, "alpha", 0),
|
||||
CO_FUTURE_BARRY_AS_BDFL)
|
||||
(3, 9, 0, "alpha", 0),
|
||||
CO_FUTURE_BARRY_AS_BDFL)
|
||||
|
||||
generator_stop = _Feature((3, 5, 0, "beta", 1),
|
||||
(3, 7, 0, "alpha", 0),
|
||||
CO_FUTURE_GENERATOR_STOP)
|
||||
|
||||
annotations = _Feature((3, 7, 0, "beta", 1),
|
||||
None,
|
||||
CO_FUTURE_ANNOTATIONS)
|
||||
(3, 7, 0, "alpha", 0),
|
||||
CO_FUTURE_GENERATOR_STOP)
|
||||
|
||||
16
Lib/__hello__.py
vendored
16
Lib/__hello__.py
vendored
@@ -1,16 +0,0 @@
|
||||
initialized = True
|
||||
|
||||
class TestFrozenUtf8_1:
|
||||
"""\u00b6"""
|
||||
|
||||
class TestFrozenUtf8_2:
|
||||
"""\u03c0"""
|
||||
|
||||
class TestFrozenUtf8_4:
|
||||
"""\U0001f600"""
|
||||
|
||||
def main():
|
||||
print("Hello world!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,7 +0,0 @@
|
||||
initialized = True
|
||||
|
||||
def main():
|
||||
print("Hello world!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,7 +0,0 @@
|
||||
initialized = True
|
||||
|
||||
def main():
|
||||
print("Hello world!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
765
Lib/_pycodecs.py → Lib/_codecs.py
vendored
765
Lib/_pycodecs.py → Lib/_codecs.py
vendored
File diff suppressed because it is too large
Load Diff
251
Lib/_collections_abc.py
vendored
251
Lib/_collections_abc.py
vendored
@@ -6,41 +6,9 @@
|
||||
Unit tests are in test_collections.
|
||||
"""
|
||||
|
||||
############ Maintenance notes #########################################
|
||||
#
|
||||
# ABCs are different from other standard library modules in that they
|
||||
# specify compliance tests. In general, once an ABC has been published,
|
||||
# new methods (either abstract or concrete) cannot be added.
|
||||
#
|
||||
# Though classes that inherit from an ABC would automatically receive a
|
||||
# new mixin method, registered classes would become non-compliant and
|
||||
# violate the contract promised by ``isinstance(someobj, SomeABC)``.
|
||||
#
|
||||
# Though irritating, the correct procedure for adding new abstract or
|
||||
# mixin methods is to create a new ABC as a subclass of the previous
|
||||
# ABC. For example, union(), intersection(), and difference() cannot
|
||||
# be added to Set but could go into a new ABC that extends Set.
|
||||
#
|
||||
# Because they are so hard to change, new ABCs should have their APIs
|
||||
# carefully thought through prior to publication.
|
||||
#
|
||||
# Since ABCMeta only checks for the presence of methods, it is possible
|
||||
# to alter the signature of a method by adding optional arguments
|
||||
# or changing parameters names. This is still a bit dubious but at
|
||||
# least it won't cause isinstance() to return an incorrect result.
|
||||
#
|
||||
#
|
||||
#######################################################################
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
import sys
|
||||
|
||||
GenericAlias = type(list[int])
|
||||
EllipsisType = type(...)
|
||||
def _f(): pass
|
||||
FunctionType = type(_f)
|
||||
del _f
|
||||
|
||||
__all__ = ["Awaitable", "Coroutine",
|
||||
"AsyncIterable", "AsyncIterator", "AsyncGenerator",
|
||||
"Hashable", "Iterable", "Iterator", "Generator", "Reversible",
|
||||
@@ -49,7 +17,7 @@ __all__ = ["Awaitable", "Coroutine",
|
||||
"Mapping", "MutableMapping",
|
||||
"MappingView", "KeysView", "ItemsView", "ValuesView",
|
||||
"Sequence", "MutableSequence",
|
||||
"ByteString", "Buffer",
|
||||
"ByteString",
|
||||
]
|
||||
|
||||
# This module has been renamed from collections.abc to _collections_abc to
|
||||
@@ -99,7 +67,7 @@ async_generator = type(_ag)
|
||||
del _ag
|
||||
|
||||
|
||||
### ONE-TRICK PONIES ###
|
||||
# ## ONE-TRICK PONIES ###
|
||||
|
||||
def _check_methods(C, *methods):
|
||||
mro = C.__mro__
|
||||
@@ -142,8 +110,6 @@ class Awaitable(metaclass=ABCMeta):
|
||||
return _check_methods(C, "__await__")
|
||||
return NotImplemented
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
|
||||
class Coroutine(Awaitable):
|
||||
|
||||
@@ -203,8 +169,6 @@ class AsyncIterable(metaclass=ABCMeta):
|
||||
return _check_methods(C, "__aiter__")
|
||||
return NotImplemented
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
|
||||
class AsyncIterator(AsyncIterable):
|
||||
|
||||
@@ -291,8 +255,6 @@ class Iterable(metaclass=ABCMeta):
|
||||
return _check_methods(C, "__iter__")
|
||||
return NotImplemented
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
|
||||
class Iterator(Iterable):
|
||||
|
||||
@@ -312,10 +274,9 @@ class Iterator(Iterable):
|
||||
return _check_methods(C, '__iter__', '__next__')
|
||||
return NotImplemented
|
||||
|
||||
|
||||
Iterator.register(bytes_iterator)
|
||||
Iterator.register(bytearray_iterator)
|
||||
#Iterator.register(callable_iterator)
|
||||
# Iterator.register(callable_iterator)
|
||||
Iterator.register(dict_keyiterator)
|
||||
Iterator.register(dict_valueiterator)
|
||||
Iterator.register(dict_itemiterator)
|
||||
@@ -392,10 +353,8 @@ class Generator(Iterator):
|
||||
'send', 'throw', 'close')
|
||||
return NotImplemented
|
||||
|
||||
|
||||
Generator.register(generator)
|
||||
|
||||
|
||||
class Sized(metaclass=ABCMeta):
|
||||
|
||||
__slots__ = ()
|
||||
@@ -425,9 +384,6 @@ class Container(metaclass=ABCMeta):
|
||||
return _check_methods(C, "__contains__")
|
||||
return NotImplemented
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
|
||||
class Collection(Sized, Iterable, Container):
|
||||
|
||||
__slots__ = ()
|
||||
@@ -438,106 +394,6 @@ class Collection(Sized, Iterable, Container):
|
||||
return _check_methods(C, "__len__", "__iter__", "__contains__")
|
||||
return NotImplemented
|
||||
|
||||
|
||||
class Buffer(metaclass=ABCMeta):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@abstractmethod
|
||||
def __buffer__(self, flags: int, /) -> memoryview:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def __subclasshook__(cls, C):
|
||||
if cls is Buffer:
|
||||
return _check_methods(C, "__buffer__")
|
||||
return NotImplemented
|
||||
|
||||
|
||||
class _CallableGenericAlias(GenericAlias):
|
||||
""" Represent `Callable[argtypes, resulttype]`.
|
||||
|
||||
This sets ``__args__`` to a tuple containing the flattened ``argtypes``
|
||||
followed by ``resulttype``.
|
||||
|
||||
Example: ``Callable[[int, str], float]`` sets ``__args__`` to
|
||||
``(int, str, float)``.
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def __new__(cls, origin, args):
|
||||
if not (isinstance(args, tuple) and len(args) == 2):
|
||||
raise TypeError(
|
||||
"Callable must be used as Callable[[arg, ...], result].")
|
||||
t_args, t_result = args
|
||||
if isinstance(t_args, (tuple, list)):
|
||||
args = (*t_args, t_result)
|
||||
elif not _is_param_expr(t_args):
|
||||
raise TypeError(f"Expected a list of types, an ellipsis, "
|
||||
f"ParamSpec, or Concatenate. Got {t_args}")
|
||||
return super().__new__(cls, origin, args)
|
||||
|
||||
def __repr__(self):
|
||||
if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]):
|
||||
return super().__repr__()
|
||||
return (f'collections.abc.Callable'
|
||||
f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], '
|
||||
f'{_type_repr(self.__args__[-1])}]')
|
||||
|
||||
def __reduce__(self):
|
||||
args = self.__args__
|
||||
if not (len(args) == 2 and _is_param_expr(args[0])):
|
||||
args = list(args[:-1]), args[-1]
|
||||
return _CallableGenericAlias, (Callable, args)
|
||||
|
||||
def __getitem__(self, item):
|
||||
# Called during TypeVar substitution, returns the custom subclass
|
||||
# rather than the default types.GenericAlias object. Most of the
|
||||
# code is copied from typing's _GenericAlias and the builtin
|
||||
# types.GenericAlias.
|
||||
if not isinstance(item, tuple):
|
||||
item = (item,)
|
||||
|
||||
new_args = super().__getitem__(item).__args__
|
||||
|
||||
# args[0] occurs due to things like Z[[int, str, bool]] from PEP 612
|
||||
if not isinstance(new_args[0], (tuple, list)):
|
||||
t_result = new_args[-1]
|
||||
t_args = new_args[:-1]
|
||||
new_args = (t_args, t_result)
|
||||
return _CallableGenericAlias(Callable, tuple(new_args))
|
||||
|
||||
def _is_param_expr(obj):
|
||||
"""Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or
|
||||
``_ConcatenateGenericAlias`` from typing.py
|
||||
"""
|
||||
if obj is Ellipsis:
|
||||
return True
|
||||
if isinstance(obj, list):
|
||||
return True
|
||||
obj = type(obj)
|
||||
names = ('ParamSpec', '_ConcatenateGenericAlias')
|
||||
return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names)
|
||||
|
||||
def _type_repr(obj):
|
||||
"""Return the repr() of an object, special-casing types (internal helper).
|
||||
|
||||
Copied from :mod:`typing` since collections.abc
|
||||
shouldn't depend on that module.
|
||||
(Keep this roughly in sync with the typing version.)
|
||||
"""
|
||||
if isinstance(obj, type):
|
||||
if obj.__module__ == 'builtins':
|
||||
return obj.__qualname__
|
||||
return f'{obj.__module__}.{obj.__qualname__}'
|
||||
if obj is Ellipsis:
|
||||
return '...'
|
||||
if isinstance(obj, FunctionType):
|
||||
return obj.__name__
|
||||
return repr(obj)
|
||||
|
||||
|
||||
class Callable(metaclass=ABCMeta):
|
||||
|
||||
__slots__ = ()
|
||||
@@ -552,13 +408,12 @@ class Callable(metaclass=ABCMeta):
|
||||
return _check_methods(C, "__call__")
|
||||
return NotImplemented
|
||||
|
||||
__class_getitem__ = classmethod(_CallableGenericAlias)
|
||||
|
||||
|
||||
### SETS ###
|
||||
|
||||
|
||||
class Set(Collection):
|
||||
|
||||
"""A set is a finite, iterable container.
|
||||
|
||||
This class provides concrete generic implementations of all
|
||||
@@ -686,7 +541,6 @@ class Set(Collection):
|
||||
hx = hash(x)
|
||||
h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167
|
||||
h &= MASK
|
||||
h ^= (h >> 11) ^ (h >> 25)
|
||||
h = h * 69069 + 907133923
|
||||
h &= MASK
|
||||
if h > MAX:
|
||||
@@ -695,7 +549,6 @@ class Set(Collection):
|
||||
h = 590923713
|
||||
return h
|
||||
|
||||
|
||||
Set.register(frozenset)
|
||||
|
||||
|
||||
@@ -778,25 +631,24 @@ class MutableSet(Set):
|
||||
self.discard(value)
|
||||
return self
|
||||
|
||||
|
||||
MutableSet.register(set)
|
||||
|
||||
|
||||
### MAPPINGS ###
|
||||
|
||||
|
||||
class Mapping(Collection):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
"""A Mapping is a generic container for associating key/value
|
||||
pairs.
|
||||
|
||||
This class provides concrete generic implementations of all
|
||||
methods except for __getitem__, __iter__, and __len__.
|
||||
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
# Tell ABCMeta.__new__ that this class should have TPFLAGS_MAPPING set.
|
||||
__abc_tpflags__ = 1 << 6 # Py_TPFLAGS_MAPPING
|
||||
|
||||
@abstractmethod
|
||||
def __getitem__(self, key):
|
||||
raise KeyError
|
||||
@@ -851,15 +703,13 @@ class MappingView(Sized):
|
||||
def __repr__(self):
|
||||
return '{0.__class__.__name__}({0._mapping!r})'.format(self)
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
|
||||
class KeysView(MappingView, Set):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@classmethod
|
||||
def _from_iterable(cls, it):
|
||||
def _from_iterable(self, it):
|
||||
return set(it)
|
||||
|
||||
def __contains__(self, key):
|
||||
@@ -868,7 +718,6 @@ class KeysView(MappingView, Set):
|
||||
def __iter__(self):
|
||||
yield from self._mapping
|
||||
|
||||
|
||||
KeysView.register(dict_keys)
|
||||
|
||||
|
||||
@@ -877,7 +726,7 @@ class ItemsView(MappingView, Set):
|
||||
__slots__ = ()
|
||||
|
||||
@classmethod
|
||||
def _from_iterable(cls, it):
|
||||
def _from_iterable(self, it):
|
||||
return set(it)
|
||||
|
||||
def __contains__(self, item):
|
||||
@@ -893,7 +742,6 @@ class ItemsView(MappingView, Set):
|
||||
for key in self._mapping:
|
||||
yield (key, self._mapping[key])
|
||||
|
||||
|
||||
ItemsView.register(dict_items)
|
||||
|
||||
|
||||
@@ -912,20 +760,21 @@ class ValuesView(MappingView, Collection):
|
||||
for key in self._mapping:
|
||||
yield self._mapping[key]
|
||||
|
||||
|
||||
ValuesView.register(dict_values)
|
||||
|
||||
|
||||
class MutableMapping(Mapping):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
"""A MutableMapping is a generic container for associating
|
||||
key/value pairs.
|
||||
|
||||
This class provides concrete generic implementations of all
|
||||
methods except for __getitem__, __setitem__, __delitem__,
|
||||
__iter__, and __len__.
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def __setitem__(self, key, value):
|
||||
@@ -971,21 +820,34 @@ class MutableMapping(Mapping):
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def update(self, other=(), /, **kwds):
|
||||
def update(*args, **kwds):
|
||||
''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.
|
||||
If E present and has a .keys() method, does: for k in E: D[k] = E[k]
|
||||
If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v
|
||||
In either case, this is followed by: for k, v in F.items(): D[k] = v
|
||||
'''
|
||||
if isinstance(other, Mapping):
|
||||
for key in other:
|
||||
self[key] = other[key]
|
||||
elif hasattr(other, "keys"):
|
||||
for key in other.keys():
|
||||
self[key] = other[key]
|
||||
else:
|
||||
for key, value in other:
|
||||
self[key] = value
|
||||
if not args:
|
||||
raise TypeError("descriptor 'update' of 'MutableMapping' object "
|
||||
"needs an argument")
|
||||
self, *args = args
|
||||
if len(args) > 1:
|
||||
raise TypeError('update expected at most 1 arguments, got %d' %
|
||||
len(args))
|
||||
if args:
|
||||
other = args[0]
|
||||
try:
|
||||
mapping_inst = isinstance(other, Mapping)
|
||||
except TypeError:
|
||||
mapping_inst = False
|
||||
if mapping_inst:
|
||||
for key in other:
|
||||
self[key] = other[key]
|
||||
elif hasattr(other, "keys"):
|
||||
for key in other.keys():
|
||||
self[key] = other[key]
|
||||
else:
|
||||
for key, value in other:
|
||||
self[key] = value
|
||||
for key, value in kwds.items():
|
||||
self[key] = value
|
||||
|
||||
@@ -997,13 +859,14 @@ class MutableMapping(Mapping):
|
||||
self[key] = default
|
||||
return default
|
||||
|
||||
|
||||
MutableMapping.register(dict)
|
||||
|
||||
|
||||
### SEQUENCES ###
|
||||
|
||||
|
||||
class Sequence(Reversible, Collection):
|
||||
|
||||
"""All the operations on a read-only sequence.
|
||||
|
||||
Concrete subclasses must override __new__ or __init__,
|
||||
@@ -1012,9 +875,6 @@ class Sequence(Reversible, Collection):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
# Tell ABCMeta.__new__ that this class should have TPFLAGS_SEQUENCE set.
|
||||
__abc_tpflags__ = 1 << 5 # Py_TPFLAGS_SEQUENCE
|
||||
|
||||
@abstractmethod
|
||||
def __getitem__(self, index):
|
||||
raise IndexError
|
||||
@@ -1055,10 +915,10 @@ class Sequence(Reversible, Collection):
|
||||
while stop is None or i < stop:
|
||||
try:
|
||||
v = self[i]
|
||||
if v is value or v == value:
|
||||
return i
|
||||
except IndexError:
|
||||
break
|
||||
if v is value or v == value:
|
||||
return i
|
||||
i += 1
|
||||
raise ValueError
|
||||
|
||||
@@ -1071,27 +931,9 @@ Sequence.register(str)
|
||||
Sequence.register(range)
|
||||
Sequence.register(memoryview)
|
||||
|
||||
class _DeprecateByteStringMeta(ABCMeta):
|
||||
def __new__(cls, name, bases, namespace, **kwargs):
|
||||
if name != "ByteString":
|
||||
import warnings
|
||||
|
||||
warnings._deprecated(
|
||||
"collections.abc.ByteString",
|
||||
remove=(3, 14),
|
||||
)
|
||||
return super().__new__(cls, name, bases, namespace, **kwargs)
|
||||
class ByteString(Sequence):
|
||||
|
||||
def __instancecheck__(cls, instance):
|
||||
import warnings
|
||||
|
||||
warnings._deprecated(
|
||||
"collections.abc.ByteString",
|
||||
remove=(3, 14),
|
||||
)
|
||||
return super().__instancecheck__(instance)
|
||||
|
||||
class ByteString(Sequence, metaclass=_DeprecateByteStringMeta):
|
||||
"""This unifies bytes and bytearray.
|
||||
|
||||
XXX Should add all their methods.
|
||||
@@ -1104,13 +946,15 @@ ByteString.register(bytearray)
|
||||
|
||||
|
||||
class MutableSequence(Sequence):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
"""All the operations on a read-write sequence.
|
||||
|
||||
Concrete subclasses must provide __new__ or __init__,
|
||||
__getitem__, __setitem__, __delitem__, __len__, and insert().
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def __setitem__(self, index, value):
|
||||
@@ -1168,6 +1012,5 @@ class MutableSequence(Sequence):
|
||||
self.extend(values)
|
||||
return self
|
||||
|
||||
|
||||
MutableSequence.register(list)
|
||||
MutableSequence.register(bytearray) # Multiply inheriting, see ByteString
|
||||
|
||||
8
Lib/_compat_pickle.py
vendored
8
Lib/_compat_pickle.py
vendored
@@ -148,14 +148,6 @@ except NameError:
|
||||
else:
|
||||
PYTHON2_EXCEPTIONS += ("WindowsError",)
|
||||
|
||||
# NOTE: RUSTPYTHON exceptions
|
||||
try:
|
||||
JitError
|
||||
except NameError:
|
||||
pass
|
||||
else:
|
||||
PYTHON2_EXCEPTIONS += ("JitError",)
|
||||
|
||||
for excname in PYTHON2_EXCEPTIONS:
|
||||
NAME_MAPPING[("exceptions", excname)] = ("builtins", excname)
|
||||
|
||||
|
||||
162
Lib/_compression.py
vendored
162
Lib/_compression.py
vendored
@@ -1,162 +0,0 @@
|
||||
"""Internal classes used by the gzip, lzma and bz2 modules"""
|
||||
|
||||
import io
|
||||
import sys
|
||||
|
||||
BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size
|
||||
|
||||
|
||||
class BaseStream(io.BufferedIOBase):
|
||||
"""Mode-checking helper functions."""
|
||||
|
||||
def _check_not_closed(self):
|
||||
if self.closed:
|
||||
raise ValueError("I/O operation on closed file")
|
||||
|
||||
def _check_can_read(self):
|
||||
if not self.readable():
|
||||
raise io.UnsupportedOperation("File not open for reading")
|
||||
|
||||
def _check_can_write(self):
|
||||
if not self.writable():
|
||||
raise io.UnsupportedOperation("File not open for writing")
|
||||
|
||||
def _check_can_seek(self):
|
||||
if not self.readable():
|
||||
raise io.UnsupportedOperation("Seeking is only supported "
|
||||
"on files open for reading")
|
||||
if not self.seekable():
|
||||
raise io.UnsupportedOperation("The underlying file object "
|
||||
"does not support seeking")
|
||||
|
||||
|
||||
class DecompressReader(io.RawIOBase):
|
||||
"""Adapts the decompressor API to a RawIOBase reader API"""
|
||||
|
||||
def readable(self):
|
||||
return True
|
||||
|
||||
def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
|
||||
self._fp = fp
|
||||
self._eof = False
|
||||
self._pos = 0 # Current offset in decompressed stream
|
||||
|
||||
# Set to size of decompressed stream once it is known, for SEEK_END
|
||||
self._size = -1
|
||||
|
||||
# Save the decompressor factory and arguments.
|
||||
# If the file contains multiple compressed streams, each
|
||||
# stream will need a separate decompressor object. A new decompressor
|
||||
# object is also needed when implementing a backwards seek().
|
||||
self._decomp_factory = decomp_factory
|
||||
self._decomp_args = decomp_args
|
||||
self._decompressor = self._decomp_factory(**self._decomp_args)
|
||||
|
||||
# Exception class to catch from decompressor signifying invalid
|
||||
# trailing data to ignore
|
||||
self._trailing_error = trailing_error
|
||||
|
||||
def close(self):
|
||||
self._decompressor = None
|
||||
return super().close()
|
||||
|
||||
def seekable(self):
|
||||
return self._fp.seekable()
|
||||
|
||||
def readinto(self, b):
|
||||
with memoryview(b) as view, view.cast("B") as byte_view:
|
||||
data = self.read(len(byte_view))
|
||||
byte_view[:len(data)] = data
|
||||
return len(data)
|
||||
|
||||
def read(self, size=-1):
|
||||
if size < 0:
|
||||
return self.readall()
|
||||
|
||||
if not size or self._eof:
|
||||
return b""
|
||||
data = None # Default if EOF is encountered
|
||||
# Depending on the input data, our call to the decompressor may not
|
||||
# return any data. In this case, try again after reading another block.
|
||||
while True:
|
||||
if self._decompressor.eof:
|
||||
rawblock = (self._decompressor.unused_data or
|
||||
self._fp.read(BUFFER_SIZE))
|
||||
if not rawblock:
|
||||
break
|
||||
# Continue to next stream.
|
||||
self._decompressor = self._decomp_factory(
|
||||
**self._decomp_args)
|
||||
try:
|
||||
data = self._decompressor.decompress(rawblock, size)
|
||||
except self._trailing_error:
|
||||
# Trailing data isn't a valid compressed stream; ignore it.
|
||||
break
|
||||
else:
|
||||
if self._decompressor.needs_input:
|
||||
rawblock = self._fp.read(BUFFER_SIZE)
|
||||
if not rawblock:
|
||||
raise EOFError("Compressed file ended before the "
|
||||
"end-of-stream marker was reached")
|
||||
else:
|
||||
rawblock = b""
|
||||
data = self._decompressor.decompress(rawblock, size)
|
||||
if data:
|
||||
break
|
||||
if not data:
|
||||
self._eof = True
|
||||
self._size = self._pos
|
||||
return b""
|
||||
self._pos += len(data)
|
||||
return data
|
||||
|
||||
def readall(self):
|
||||
chunks = []
|
||||
# sys.maxsize means the max length of output buffer is unlimited,
|
||||
# so that the whole input buffer can be decompressed within one
|
||||
# .decompress() call.
|
||||
while data := self.read(sys.maxsize):
|
||||
chunks.append(data)
|
||||
|
||||
return b"".join(chunks)
|
||||
|
||||
# Rewind the file to the beginning of the data stream.
|
||||
def _rewind(self):
|
||||
self._fp.seek(0)
|
||||
self._eof = False
|
||||
self._pos = 0
|
||||
self._decompressor = self._decomp_factory(**self._decomp_args)
|
||||
|
||||
def seek(self, offset, whence=io.SEEK_SET):
|
||||
# Recalculate offset as an absolute file position.
|
||||
if whence == io.SEEK_SET:
|
||||
pass
|
||||
elif whence == io.SEEK_CUR:
|
||||
offset = self._pos + offset
|
||||
elif whence == io.SEEK_END:
|
||||
# Seeking relative to EOF - we need to know the file's size.
|
||||
if self._size < 0:
|
||||
while self.read(io.DEFAULT_BUFFER_SIZE):
|
||||
pass
|
||||
offset = self._size + offset
|
||||
else:
|
||||
raise ValueError("Invalid value for whence: {}".format(whence))
|
||||
|
||||
# Make it so that offset is the number of bytes to skip forward.
|
||||
if offset < self._pos:
|
||||
self._rewind()
|
||||
else:
|
||||
offset -= self._pos
|
||||
|
||||
# Read and discard data until we reach the desired position.
|
||||
while offset > 0:
|
||||
data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset))
|
||||
if not data:
|
||||
break
|
||||
offset -= len(data)
|
||||
|
||||
return self._pos
|
||||
|
||||
def tell(self):
|
||||
"""Return the current file position."""
|
||||
return self._pos
|
||||
66
Lib/_dummy_os.py
vendored
66
Lib/_dummy_os.py
vendored
@@ -1,66 +0,0 @@
|
||||
"""
|
||||
A shim of the os module containing only simple path-related utilities
|
||||
"""
|
||||
|
||||
try:
|
||||
from os import *
|
||||
except ImportError:
|
||||
import abc
|
||||
|
||||
def __getattr__(name):
|
||||
raise OSError("no os specific module found")
|
||||
|
||||
def _shim():
|
||||
import _dummy_os, sys
|
||||
sys.modules['os'] = _dummy_os
|
||||
sys.modules['os.path'] = _dummy_os.path
|
||||
|
||||
import posixpath as path
|
||||
import sys
|
||||
sys.modules['os.path'] = path
|
||||
del sys
|
||||
|
||||
sep = path.sep
|
||||
|
||||
|
||||
def fspath(path):
|
||||
"""Return the path representation of a path-like object.
|
||||
|
||||
If str or bytes is passed in, it is returned unchanged. Otherwise the
|
||||
os.PathLike interface is used to get the path representation. If the
|
||||
path representation is not str or bytes, TypeError is raised. If the
|
||||
provided path is not str, bytes, or os.PathLike, TypeError is raised.
|
||||
"""
|
||||
if isinstance(path, (str, bytes)):
|
||||
return path
|
||||
|
||||
# Work from the object's type to match method resolution of other magic
|
||||
# methods.
|
||||
path_type = type(path)
|
||||
try:
|
||||
path_repr = path_type.__fspath__(path)
|
||||
except AttributeError:
|
||||
if hasattr(path_type, '__fspath__'):
|
||||
raise
|
||||
else:
|
||||
raise TypeError("expected str, bytes or os.PathLike object, "
|
||||
"not " + path_type.__name__)
|
||||
if isinstance(path_repr, (str, bytes)):
|
||||
return path_repr
|
||||
else:
|
||||
raise TypeError("expected {}.__fspath__() to return str or bytes, "
|
||||
"not {}".format(path_type.__name__,
|
||||
type(path_repr).__name__))
|
||||
|
||||
class PathLike(abc.ABC):
|
||||
|
||||
"""Abstract base class for implementing the file system path protocol."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def __fspath__(self):
|
||||
"""Return the file system path representation of the object."""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def __subclasshook__(cls, subclass):
|
||||
return hasattr(subclass, '__fspath__')
|
||||
10
Lib/_dummy_thread.py
vendored
10
Lib/_dummy_thread.py
vendored
@@ -14,8 +14,7 @@ Suggested usage is::
|
||||
# Exports only things specified by thread documentation;
|
||||
# skipping obsolete synonyms allocate(), start_new(), exit_thread().
|
||||
__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
|
||||
'interrupt_main', 'LockType', 'RLock',
|
||||
'_count']
|
||||
'interrupt_main', 'LockType', 'RLock']
|
||||
|
||||
# A dummy value
|
||||
TIMEOUT_MAX = 2**31
|
||||
@@ -86,10 +85,6 @@ def _set_sentinel():
|
||||
"""Dummy implementation of _thread._set_sentinel()."""
|
||||
return LockType()
|
||||
|
||||
def _count():
|
||||
"""Dummy implementation of _thread._count()."""
|
||||
return 0
|
||||
|
||||
class LockType(object):
|
||||
"""Class implementing dummy implementation of _thread.LockType.
|
||||
|
||||
@@ -145,9 +140,6 @@ class LockType(object):
|
||||
def locked(self):
|
||||
return self.locked_status
|
||||
|
||||
def _at_fork_reinit(self):
|
||||
self.locked_status = False
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s %s.%s object at %s>" % (
|
||||
"locked" if self.locked_status else "unlocked",
|
||||
|
||||
35
Lib/_markupbase.py
vendored
35
Lib/_markupbase.py
vendored
@@ -29,6 +29,10 @@ class ParserBase:
|
||||
raise RuntimeError(
|
||||
"_markupbase.ParserBase must be subclassed")
|
||||
|
||||
def error(self, message):
|
||||
raise NotImplementedError(
|
||||
"subclasses of ParserBase must override error()")
|
||||
|
||||
def reset(self):
|
||||
self.lineno = 1
|
||||
self.offset = 0
|
||||
@@ -127,11 +131,12 @@ class ParserBase:
|
||||
# also in data attribute specifications of attlist declaration
|
||||
# also link type declaration subsets in linktype declarations
|
||||
# also link attribute specification lists in link declarations
|
||||
raise AssertionError("unsupported '[' char in %s declaration" % decltype)
|
||||
self.error("unsupported '[' char in %s declaration" % decltype)
|
||||
else:
|
||||
raise AssertionError("unexpected '[' char in declaration")
|
||||
self.error("unexpected '[' char in declaration")
|
||||
else:
|
||||
raise AssertionError("unexpected %r char in declaration" % rawdata[j])
|
||||
self.error(
|
||||
"unexpected %r char in declaration" % rawdata[j])
|
||||
if j < 0:
|
||||
return j
|
||||
return -1 # incomplete
|
||||
@@ -151,9 +156,7 @@ class ParserBase:
|
||||
# look for MS Office ]> ending
|
||||
match= _msmarkedsectionclose.search(rawdata, i+3)
|
||||
else:
|
||||
raise AssertionError(
|
||||
'unknown status keyword %r in marked section' % rawdata[i+3:j]
|
||||
)
|
||||
self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
|
||||
if not match:
|
||||
return -1
|
||||
if report:
|
||||
@@ -165,7 +168,7 @@ class ParserBase:
|
||||
def parse_comment(self, i, report=1):
|
||||
rawdata = self.rawdata
|
||||
if rawdata[i:i+4] != '<!--':
|
||||
raise AssertionError('unexpected call to parse_comment()')
|
||||
self.error('unexpected call to parse_comment()')
|
||||
match = _commentclose.search(rawdata, i+4)
|
||||
if not match:
|
||||
return -1
|
||||
@@ -189,9 +192,7 @@ class ParserBase:
|
||||
return -1
|
||||
if s != "<!":
|
||||
self.updatepos(declstartpos, j + 1)
|
||||
raise AssertionError(
|
||||
"unexpected char in internal subset (in %r)" % s
|
||||
)
|
||||
self.error("unexpected char in internal subset (in %r)" % s)
|
||||
if (j + 2) == n:
|
||||
# end of buffer; incomplete
|
||||
return -1
|
||||
@@ -208,9 +209,8 @@ class ParserBase:
|
||||
return -1
|
||||
if name not in {"attlist", "element", "entity", "notation"}:
|
||||
self.updatepos(declstartpos, j + 2)
|
||||
raise AssertionError(
|
||||
"unknown declaration %r in internal subset" % name
|
||||
)
|
||||
self.error(
|
||||
"unknown declaration %r in internal subset" % name)
|
||||
# handle the individual names
|
||||
meth = getattr(self, "_parse_doctype_" + name)
|
||||
j = meth(j, declstartpos)
|
||||
@@ -234,14 +234,14 @@ class ParserBase:
|
||||
if rawdata[j] == ">":
|
||||
return j
|
||||
self.updatepos(declstartpos, j)
|
||||
raise AssertionError("unexpected char after internal subset")
|
||||
self.error("unexpected char after internal subset")
|
||||
else:
|
||||
return -1
|
||||
elif c.isspace():
|
||||
j = j + 1
|
||||
else:
|
||||
self.updatepos(declstartpos, j)
|
||||
raise AssertionError("unexpected char %r in internal subset" % c)
|
||||
self.error("unexpected char %r in internal subset" % c)
|
||||
# end of buffer reached
|
||||
return -1
|
||||
|
||||
@@ -387,9 +387,8 @@ class ParserBase:
|
||||
return name.lower(), m.end()
|
||||
else:
|
||||
self.updatepos(declstartpos, i)
|
||||
raise AssertionError(
|
||||
"expected name token at %r" % rawdata[declstartpos:declstartpos+20]
|
||||
)
|
||||
self.error("expected name token at %r"
|
||||
% rawdata[declstartpos:declstartpos+20])
|
||||
|
||||
# To be overridden -- handlers for unknown objects
|
||||
def unknown_decl(self, data):
|
||||
|
||||
132
Lib/_osx_support.py
vendored
132
Lib/_osx_support.py
vendored
@@ -52,7 +52,7 @@ def _find_executable(executable, path=None):
|
||||
return executable
|
||||
|
||||
|
||||
def _read_output(commandstring, capture_stderr=False):
|
||||
def _read_output(commandstring):
|
||||
"""Output from successful command execution or None"""
|
||||
# Similar to os.popen(commandstring, "r").read(),
|
||||
# but without actually using os.popen because that
|
||||
@@ -67,10 +67,7 @@ def _read_output(commandstring, capture_stderr=False):
|
||||
os.getpid(),), "w+b")
|
||||
|
||||
with contextlib.closing(fp) as fp:
|
||||
if capture_stderr:
|
||||
cmd = "%s >'%s' 2>&1" % (commandstring, fp.name)
|
||||
else:
|
||||
cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
|
||||
cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
|
||||
return fp.read().decode('utf-8').strip() if not os.system(cmd) else None
|
||||
|
||||
|
||||
@@ -96,7 +93,7 @@ def _get_system_version():
|
||||
if _SYSTEM_VERSION is None:
|
||||
_SYSTEM_VERSION = ''
|
||||
try:
|
||||
f = open('/System/Library/CoreServices/SystemVersion.plist', encoding="utf-8")
|
||||
f = open('/System/Library/CoreServices/SystemVersion.plist')
|
||||
except OSError:
|
||||
# We're on a plain darwin box, fall back to the default
|
||||
# behaviour.
|
||||
@@ -113,26 +110,6 @@ def _get_system_version():
|
||||
|
||||
return _SYSTEM_VERSION
|
||||
|
||||
_SYSTEM_VERSION_TUPLE = None
|
||||
def _get_system_version_tuple():
|
||||
"""
|
||||
Return the macOS system version as a tuple
|
||||
|
||||
The return value is safe to use to compare
|
||||
two version numbers.
|
||||
"""
|
||||
global _SYSTEM_VERSION_TUPLE
|
||||
if _SYSTEM_VERSION_TUPLE is None:
|
||||
osx_version = _get_system_version()
|
||||
if osx_version:
|
||||
try:
|
||||
_SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.'))
|
||||
except ValueError:
|
||||
_SYSTEM_VERSION_TUPLE = ()
|
||||
|
||||
return _SYSTEM_VERSION_TUPLE
|
||||
|
||||
|
||||
def _remove_original_values(_config_vars):
|
||||
"""Remove original unmodified values for testing"""
|
||||
# This is needed for higher-level cross-platform tests of get_platform.
|
||||
@@ -148,33 +125,6 @@ def _save_modified_value(_config_vars, cv, newvalue):
|
||||
_config_vars[_INITPRE + cv] = oldvalue
|
||||
_config_vars[cv] = newvalue
|
||||
|
||||
|
||||
_cache_default_sysroot = None
|
||||
def _default_sysroot(cc):
|
||||
""" Returns the root of the default SDK for this system, or '/' """
|
||||
global _cache_default_sysroot
|
||||
|
||||
if _cache_default_sysroot is not None:
|
||||
return _cache_default_sysroot
|
||||
|
||||
contents = _read_output('%s -c -E -v - </dev/null' % (cc,), True)
|
||||
in_incdirs = False
|
||||
for line in contents.splitlines():
|
||||
if line.startswith("#include <...>"):
|
||||
in_incdirs = True
|
||||
elif line.startswith("End of search list"):
|
||||
in_incdirs = False
|
||||
elif in_incdirs:
|
||||
line = line.strip()
|
||||
if line == '/usr/include':
|
||||
_cache_default_sysroot = '/'
|
||||
elif line.endswith(".sdk/usr/include"):
|
||||
_cache_default_sysroot = line[:-12]
|
||||
if _cache_default_sysroot is None:
|
||||
_cache_default_sysroot = '/'
|
||||
|
||||
return _cache_default_sysroot
|
||||
|
||||
def _supports_universal_builds():
|
||||
"""Returns True if universal builds are supported on this system"""
|
||||
# As an approximation, we assume that if we are running on 10.4 or above,
|
||||
@@ -182,18 +132,14 @@ def _supports_universal_builds():
|
||||
# builds, in particular -isysroot and -arch arguments to the compiler. This
|
||||
# is in support of allowing 10.4 universal builds to run on 10.3.x systems.
|
||||
|
||||
osx_version = _get_system_version_tuple()
|
||||
osx_version = _get_system_version()
|
||||
if osx_version:
|
||||
try:
|
||||
osx_version = tuple(int(i) for i in osx_version.split('.'))
|
||||
except ValueError:
|
||||
osx_version = ''
|
||||
return bool(osx_version >= (10, 4)) if osx_version else False
|
||||
|
||||
def _supports_arm64_builds():
|
||||
"""Returns True if arm64 builds are supported on this system"""
|
||||
# There are two sets of systems supporting macOS/arm64 builds:
|
||||
# 1. macOS 11 and later, unconditionally
|
||||
# 2. macOS 10.15 with Xcode 12.2 or later
|
||||
# For now the second category is ignored.
|
||||
osx_version = _get_system_version_tuple()
|
||||
return osx_version >= (11, 0) if osx_version else False
|
||||
|
||||
|
||||
def _find_appropriate_compiler(_config_vars):
|
||||
"""Find appropriate C compiler for extension module builds"""
|
||||
@@ -265,7 +211,7 @@ def _remove_universal_flags(_config_vars):
|
||||
if cv in _config_vars and cv not in os.environ:
|
||||
flags = _config_vars[cv]
|
||||
flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII)
|
||||
flags = re.sub(r'-isysroot\s*\S+', ' ', flags)
|
||||
flags = re.sub('-isysroot [^ \t]*', ' ', flags)
|
||||
_save_modified_value(_config_vars, cv, flags)
|
||||
|
||||
return _config_vars
|
||||
@@ -341,7 +287,7 @@ def _check_for_unavailable_sdk(_config_vars):
|
||||
# to /usr and /System/Library by either a standalone CLT
|
||||
# package or the CLT component within Xcode.
|
||||
cflags = _config_vars.get('CFLAGS', '')
|
||||
m = re.search(r'-isysroot\s*(\S+)', cflags)
|
||||
m = re.search(r'-isysroot\s+(\S+)', cflags)
|
||||
if m is not None:
|
||||
sdk = m.group(1)
|
||||
if not os.path.exists(sdk):
|
||||
@@ -349,7 +295,7 @@ def _check_for_unavailable_sdk(_config_vars):
|
||||
# Do not alter a config var explicitly overridden by env var
|
||||
if cv in _config_vars and cv not in os.environ:
|
||||
flags = _config_vars[cv]
|
||||
flags = re.sub(r'-isysroot\s*\S+(?:\s|$)', ' ', flags)
|
||||
flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags)
|
||||
_save_modified_value(_config_vars, cv, flags)
|
||||
|
||||
return _config_vars
|
||||
@@ -374,7 +320,7 @@ def compiler_fixup(compiler_so, cc_args):
|
||||
stripArch = stripSysroot = True
|
||||
else:
|
||||
stripArch = '-arch' in cc_args
|
||||
stripSysroot = any(arg for arg in cc_args if arg.startswith('-isysroot'))
|
||||
stripSysroot = '-isysroot' in cc_args
|
||||
|
||||
if stripArch or 'ARCHFLAGS' in os.environ:
|
||||
while True:
|
||||
@@ -385,12 +331,6 @@ def compiler_fixup(compiler_so, cc_args):
|
||||
except ValueError:
|
||||
break
|
||||
|
||||
elif not _supports_arm64_builds():
|
||||
# Look for "-arch arm64" and drop that
|
||||
for idx in reversed(range(len(compiler_so))):
|
||||
if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64":
|
||||
del compiler_so[idx:idx+2]
|
||||
|
||||
if 'ARCHFLAGS' in os.environ and not stripArch:
|
||||
# User specified different -arch flags in the environ,
|
||||
# see also distutils.sysconfig
|
||||
@@ -398,39 +338,29 @@ def compiler_fixup(compiler_so, cc_args):
|
||||
|
||||
if stripSysroot:
|
||||
while True:
|
||||
indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
|
||||
if not indices:
|
||||
break
|
||||
index = indices[0]
|
||||
if compiler_so[index] == '-isysroot':
|
||||
try:
|
||||
index = compiler_so.index('-isysroot')
|
||||
# Strip this argument and the next one:
|
||||
del compiler_so[index:index+2]
|
||||
else:
|
||||
# It's '-isysroot/some/path' in one arg
|
||||
del compiler_so[index:index+1]
|
||||
except ValueError:
|
||||
break
|
||||
|
||||
# Check if the SDK that is used during compilation actually exists,
|
||||
# the universal build requires the usage of a universal SDK and not all
|
||||
# users have that installed by default.
|
||||
sysroot = None
|
||||
argvar = cc_args
|
||||
indices = [i for i,x in enumerate(cc_args) if x.startswith('-isysroot')]
|
||||
if not indices:
|
||||
argvar = compiler_so
|
||||
indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
|
||||
|
||||
for idx in indices:
|
||||
if argvar[idx] == '-isysroot':
|
||||
sysroot = argvar[idx+1]
|
||||
break
|
||||
else:
|
||||
sysroot = argvar[idx][len('-isysroot'):]
|
||||
break
|
||||
if '-isysroot' in cc_args:
|
||||
idx = cc_args.index('-isysroot')
|
||||
sysroot = cc_args[idx+1]
|
||||
elif '-isysroot' in compiler_so:
|
||||
idx = compiler_so.index('-isysroot')
|
||||
sysroot = compiler_so[idx+1]
|
||||
|
||||
if sysroot and not os.path.isdir(sysroot):
|
||||
sys.stderr.write(f"Compiling with an SDK that doesn't seem to exist: {sysroot}\n")
|
||||
sys.stderr.write("Please check your Xcode installation\n")
|
||||
sys.stderr.flush()
|
||||
from distutils import log
|
||||
log.warn("Compiling with an SDK that doesn't seem to exist: %s",
|
||||
sysroot)
|
||||
log.warn("Please check your Xcode installation")
|
||||
|
||||
return compiler_so
|
||||
|
||||
@@ -481,7 +411,7 @@ def customize_compiler(_config_vars):
|
||||
|
||||
This customization is performed when the first
|
||||
extension module build is requested
|
||||
in distutils.sysconfig.customize_compiler.
|
||||
in distutils.sysconfig.customize_compiler).
|
||||
"""
|
||||
|
||||
# Find a compiler to use for extension module builds
|
||||
@@ -524,10 +454,10 @@ def get_platform_osx(_config_vars, osname, release, machine):
|
||||
try:
|
||||
macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
|
||||
except ValueError:
|
||||
macrelease = (10, 3)
|
||||
macrelease = (10, 0)
|
||||
else:
|
||||
# assume no universal support
|
||||
macrelease = (10, 3)
|
||||
macrelease = (10, 0)
|
||||
|
||||
if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
|
||||
# The universal build will build fat binaries, but not on
|
||||
@@ -540,8 +470,6 @@ def get_platform_osx(_config_vars, osname, release, machine):
|
||||
|
||||
if len(archs) == 1:
|
||||
machine = archs[0]
|
||||
elif archs == ('arm64', 'x86_64'):
|
||||
machine = 'universal2'
|
||||
elif archs == ('i386', 'ppc'):
|
||||
machine = 'fat'
|
||||
elif archs == ('i386', 'x86_64'):
|
||||
|
||||
4
Lib/_py_abc.py
vendored
4
Lib/_py_abc.py
vendored
@@ -32,7 +32,7 @@ class ABCMeta(type):
|
||||
# external code.
|
||||
_abc_invalidation_counter = 0
|
||||
|
||||
def __new__(mcls, name, bases, namespace, /, **kwargs):
|
||||
def __new__(mcls, name, bases, namespace, **kwargs):
|
||||
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
|
||||
# Compute set of abstract method names
|
||||
abstracts = {name
|
||||
@@ -43,7 +43,7 @@ class ABCMeta(type):
|
||||
value = getattr(cls, name, None)
|
||||
if getattr(value, "__isabstractmethod__", False):
|
||||
abstracts.add(name)
|
||||
cls.__abstractmethods__ = frozenset(abstracts)
|
||||
cls.__abstractmethods__ = set(abstracts)
|
||||
# Set up inheritance registry
|
||||
cls._abc_registry = WeakSet()
|
||||
cls._abc_cache = WeakSet()
|
||||
|
||||
2643
Lib/_pydatetime.py
vendored
2643
Lib/_pydatetime.py
vendored
File diff suppressed because it is too large
Load Diff
27
Lib/_pydecimal.py
vendored
27
Lib/_pydecimal.py
vendored
@@ -734,23 +734,18 @@ class Decimal(object):
|
||||
|
||||
"""
|
||||
if isinstance(f, int): # handle integer inputs
|
||||
sign = 0 if f >= 0 else 1
|
||||
k = 0
|
||||
coeff = str(abs(f))
|
||||
elif isinstance(f, float):
|
||||
if _math.isinf(f) or _math.isnan(f):
|
||||
return cls(repr(f))
|
||||
if _math.copysign(1.0, f) == 1.0:
|
||||
sign = 0
|
||||
else:
|
||||
sign = 1
|
||||
n, d = abs(f).as_integer_ratio()
|
||||
k = d.bit_length() - 1
|
||||
coeff = str(n*5**k)
|
||||
else:
|
||||
return cls(f)
|
||||
if not isinstance(f, float):
|
||||
raise TypeError("argument must be int or float.")
|
||||
|
||||
result = _dec_from_triple(sign, coeff, -k)
|
||||
if _math.isinf(f) or _math.isnan(f):
|
||||
return cls(repr(f))
|
||||
if _math.copysign(1.0, f) == 1.0:
|
||||
sign = 0
|
||||
else:
|
||||
sign = 1
|
||||
n, d = abs(f).as_integer_ratio()
|
||||
k = d.bit_length() - 1
|
||||
result = _dec_from_triple(sign, str(n*5**k), -k)
|
||||
if cls is Decimal:
|
||||
return result
|
||||
else:
|
||||
|
||||
142
Lib/_pyio.py
vendored
142
Lib/_pyio.py
vendored
@@ -36,44 +36,8 @@ BlockingIOError = BlockingIOError
|
||||
# Does io.IOBase finalizer log the exception if the close() method fails?
|
||||
# The exception is ignored silently by default in release build.
|
||||
_IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode)
|
||||
# Does open() check its 'errors' argument?
|
||||
_CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE
|
||||
|
||||
|
||||
def text_encoding(encoding, stacklevel=2):
|
||||
"""
|
||||
A helper function to choose the text encoding.
|
||||
|
||||
When encoding is not None, this function returns it.
|
||||
Otherwise, this function returns the default text encoding
|
||||
(i.e. "locale" or "utf-8" depends on UTF-8 mode).
|
||||
|
||||
This function emits an EncodingWarning if *encoding* is None and
|
||||
sys.flags.warn_default_encoding is true.
|
||||
|
||||
This can be used in APIs with an encoding=None parameter
|
||||
that pass it to TextIOWrapper or open.
|
||||
However, please consider using encoding="utf-8" for new APIs.
|
||||
"""
|
||||
if encoding is None:
|
||||
if sys.flags.utf8_mode:
|
||||
encoding = "utf-8"
|
||||
else:
|
||||
encoding = "locale"
|
||||
if sys.flags.warn_default_encoding:
|
||||
import warnings
|
||||
warnings.warn("'encoding' argument not specified.",
|
||||
EncodingWarning, stacklevel + 1)
|
||||
return encoding
|
||||
|
||||
|
||||
# Wrapper for builtins.open
|
||||
#
|
||||
# Trick so that open() won't become a bound method when stored
|
||||
# as a class variable (as dbm.dumb does).
|
||||
#
|
||||
# See init_set_builtins_open() in Python/pylifecycle.c.
|
||||
@staticmethod
|
||||
def open(file, mode="r", buffering=-1, encoding=None, errors=None,
|
||||
newline=None, closefd=True, opener=None):
|
||||
|
||||
@@ -105,6 +69,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
|
||||
'b' binary mode
|
||||
't' text mode (default)
|
||||
'+' open a disk file for updating (reading and writing)
|
||||
'U' universal newline mode (deprecated)
|
||||
========= ===============================================================
|
||||
|
||||
The default mode is 'rt' (open for reading text). For binary random
|
||||
@@ -120,6 +85,10 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
|
||||
returned as strings, the bytes having been first decoded using a
|
||||
platform-dependent encoding or using the specified encoding if given.
|
||||
|
||||
'U' mode is deprecated and will raise an exception in future versions
|
||||
of Python. It has no effect in Python 3. Use newline to control
|
||||
universal newlines mode.
|
||||
|
||||
buffering is an optional integer used to set the buffering policy.
|
||||
Pass 0 to switch buffering off (only allowed in binary mode), 1 to select
|
||||
line buffering (only usable in text mode), and an integer > 1 to indicate
|
||||
@@ -205,7 +174,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
|
||||
if errors is not None and not isinstance(errors, str):
|
||||
raise TypeError("invalid errors: %r" % errors)
|
||||
modes = set(mode)
|
||||
if modes - set("axrwb+t") or len(mode) > len(modes):
|
||||
if modes - set("axrwb+tU") or len(mode) > len(modes):
|
||||
raise ValueError("invalid mode: %r" % mode)
|
||||
creating = "x" in modes
|
||||
reading = "r" in modes
|
||||
@@ -214,6 +183,13 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
|
||||
updating = "+" in modes
|
||||
text = "t" in modes
|
||||
binary = "b" in modes
|
||||
if "U" in modes:
|
||||
if creating or writing or appending or updating:
|
||||
raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'")
|
||||
import warnings
|
||||
warnings.warn("'U' mode is deprecated",
|
||||
DeprecationWarning, 2)
|
||||
reading = True
|
||||
if text and binary:
|
||||
raise ValueError("can't have text and binary mode at once")
|
||||
if creating + reading + writing + appending > 1:
|
||||
@@ -270,7 +246,6 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
|
||||
result = buffer
|
||||
if binary:
|
||||
return result
|
||||
encoding = text_encoding(encoding)
|
||||
text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
|
||||
result = text
|
||||
text.mode = mode
|
||||
@@ -303,6 +278,29 @@ except AttributeError:
|
||||
open_code = _open_code_with_warning
|
||||
|
||||
|
||||
class DocDescriptor:
|
||||
"""Helper for builtins.open.__doc__
|
||||
"""
|
||||
def __get__(self, obj, typ=None):
|
||||
return (
|
||||
"open(file, mode='r', buffering=-1, encoding=None, "
|
||||
"errors=None, newline=None, closefd=True)\n\n" +
|
||||
open.__doc__)
|
||||
|
||||
class OpenWrapper:
|
||||
"""Wrapper for builtins.open
|
||||
|
||||
Trick so that open won't become a bound method when stored
|
||||
as a class variable (as dbm.dumb does).
|
||||
|
||||
See initstdio() in Python/pylifecycle.c.
|
||||
"""
|
||||
__doc__ = DocDescriptor()
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
return open(*args, **kwargs)
|
||||
|
||||
|
||||
# In normal operation, both `UnsupportedOperation`s should be bound to the
|
||||
# same object.
|
||||
try:
|
||||
@@ -314,7 +312,8 @@ except AttributeError:
|
||||
|
||||
class IOBase(metaclass=abc.ABCMeta):
|
||||
|
||||
"""The abstract base class for all I/O classes.
|
||||
"""The abstract base class for all I/O classes, acting on streams of
|
||||
bytes. There is no public constructor.
|
||||
|
||||
This class provides dummy implementations for many methods that
|
||||
derived classes can override selectively; the default implementations
|
||||
@@ -803,9 +802,6 @@ class _BufferedIOMixin(BufferedIOBase):
|
||||
return pos
|
||||
|
||||
def truncate(self, pos=None):
|
||||
self._checkClosed()
|
||||
self._checkWritable()
|
||||
|
||||
# Flush the stream. We're mixing buffered I/O with lower-level I/O,
|
||||
# and a flush may be necessary to synch both views of the current
|
||||
# file state.
|
||||
@@ -1129,7 +1125,6 @@ class BufferedReader(_BufferedIOMixin):
|
||||
do at most one raw read to satisfy it. We never return more
|
||||
than self.buffer_size.
|
||||
"""
|
||||
self._checkClosed("peek of closed file")
|
||||
with self._read_lock:
|
||||
return self._peek_unlocked(size)
|
||||
|
||||
@@ -1148,7 +1143,6 @@ class BufferedReader(_BufferedIOMixin):
|
||||
"""Reads up to size bytes, with at most one read() system call."""
|
||||
# Returns up to size bytes. If at least one byte is buffered, we
|
||||
# only return buffered bytes. Otherwise, we do one raw read.
|
||||
self._checkClosed("read of closed file")
|
||||
if size < 0:
|
||||
size = self.buffer_size
|
||||
if size == 0:
|
||||
@@ -1166,8 +1160,6 @@ class BufferedReader(_BufferedIOMixin):
|
||||
def _readinto(self, buf, read1):
|
||||
"""Read data into *buf* with at most one system call."""
|
||||
|
||||
self._checkClosed("readinto of closed file")
|
||||
|
||||
# Need to create a memoryview object of type 'b', otherwise
|
||||
# we may not be able to assign bytes to it, and slicing it
|
||||
# would create a new object.
|
||||
@@ -1212,13 +1204,11 @@ class BufferedReader(_BufferedIOMixin):
|
||||
return written
|
||||
|
||||
def tell(self):
|
||||
# GH-95782: Keep return value non-negative
|
||||
return max(_BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos, 0)
|
||||
return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
|
||||
|
||||
def seek(self, pos, whence=0):
|
||||
if whence not in valid_seek_flags:
|
||||
raise ValueError("invalid whence value")
|
||||
self._checkClosed("seek of closed file")
|
||||
with self._read_lock:
|
||||
if whence == 1:
|
||||
pos -= len(self._read_buf) - self._read_pos
|
||||
@@ -1581,7 +1571,7 @@ class FileIO(RawIOBase):
|
||||
raise IsADirectoryError(errno.EISDIR,
|
||||
os.strerror(errno.EISDIR), file)
|
||||
except AttributeError:
|
||||
# Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR
|
||||
# Ignore the AttribueError if stat.S_ISDIR or errno.EISDIR
|
||||
# don't exist.
|
||||
pass
|
||||
self._blksize = getattr(fdfstat, 'st_blksize', 0)
|
||||
@@ -1826,7 +1816,7 @@ class TextIOBase(IOBase):
|
||||
"""Base class for text I/O.
|
||||
|
||||
This class provides a character and line based interface to stream
|
||||
I/O.
|
||||
I/O. There is no public constructor.
|
||||
"""
|
||||
|
||||
def read(self, size=-1):
|
||||
@@ -1978,7 +1968,7 @@ class TextIOWrapper(TextIOBase):
|
||||
r"""Character and line based layer over a BufferedIOBase object, buffer.
|
||||
|
||||
encoding gives the name of the encoding that the stream will be
|
||||
decoded or encoded with. It defaults to locale.getencoding().
|
||||
decoded or encoded with. It defaults to locale.getpreferredencoding(False).
|
||||
|
||||
errors determines the strictness of encoding and decoding (see the
|
||||
codecs.register) and defaults to "strict".
|
||||
@@ -2009,10 +1999,19 @@ class TextIOWrapper(TextIOBase):
|
||||
def __init__(self, buffer, encoding=None, errors=None, newline=None,
|
||||
line_buffering=False, write_through=False):
|
||||
self._check_newline(newline)
|
||||
encoding = text_encoding(encoding)
|
||||
|
||||
if encoding == "locale":
|
||||
encoding = self._get_locale_encoding()
|
||||
if encoding is None:
|
||||
try:
|
||||
encoding = os.device_encoding(buffer.fileno())
|
||||
except (AttributeError, UnsupportedOperation):
|
||||
pass
|
||||
if encoding is None:
|
||||
try:
|
||||
import locale
|
||||
except ImportError:
|
||||
# Importing locale may fail if Python is being built
|
||||
encoding = "ascii"
|
||||
else:
|
||||
encoding = locale.getpreferredencoding(False)
|
||||
|
||||
if not isinstance(encoding, str):
|
||||
raise ValueError("invalid encoding: %r" % encoding)
|
||||
@@ -2027,8 +2026,6 @@ class TextIOWrapper(TextIOBase):
|
||||
else:
|
||||
if not isinstance(errors, str):
|
||||
raise ValueError("invalid errors: %r" % errors)
|
||||
if _CHECK_ERRORS:
|
||||
codecs.lookup_error(errors)
|
||||
|
||||
self._buffer = buffer
|
||||
self._decoded_chars = '' # buffer for text returned from decoder
|
||||
@@ -2145,8 +2142,6 @@ class TextIOWrapper(TextIOBase):
|
||||
else:
|
||||
if not isinstance(encoding, str):
|
||||
raise TypeError("invalid encoding: %r" % encoding)
|
||||
if encoding == "locale":
|
||||
encoding = self._get_locale_encoding()
|
||||
|
||||
if newline is Ellipsis:
|
||||
newline = self._readnl
|
||||
@@ -2214,9 +2209,8 @@ class TextIOWrapper(TextIOBase):
|
||||
self.buffer.write(b)
|
||||
if self._line_buffering and (haslf or "\r" in s):
|
||||
self.flush()
|
||||
if self._snapshot is not None:
|
||||
self._set_decoded_chars('')
|
||||
self._snapshot = None
|
||||
self._set_decoded_chars('')
|
||||
self._snapshot = None
|
||||
if self._decoder:
|
||||
self._decoder.reset()
|
||||
return length
|
||||
@@ -2252,15 +2246,6 @@ class TextIOWrapper(TextIOBase):
|
||||
self._decoded_chars_used += len(chars)
|
||||
return chars
|
||||
|
||||
def _get_locale_encoding(self):
|
||||
try:
|
||||
import locale
|
||||
except ImportError:
|
||||
# Importing locale may fail if Python is being built
|
||||
return "utf-8"
|
||||
else:
|
||||
return locale.getencoding()
|
||||
|
||||
def _rewind_decoded_chars(self, n):
|
||||
"""Rewind the _decoded_chars buffer."""
|
||||
if self._decoded_chars_used < n:
|
||||
@@ -2310,7 +2295,7 @@ class TextIOWrapper(TextIOBase):
|
||||
return not eof
|
||||
|
||||
def _pack_cookie(self, position, dec_flags=0,
|
||||
bytes_to_feed=0, need_eof=False, chars_to_skip=0):
|
||||
bytes_to_feed=0, need_eof=0, chars_to_skip=0):
|
||||
# The meaning of a tell() cookie is: seek to position, set the
|
||||
# decoder flags to dec_flags, read bytes_to_feed bytes, feed them
|
||||
# into the decoder with need_eof as the EOF flag, then skip
|
||||
@@ -2324,7 +2309,7 @@ class TextIOWrapper(TextIOBase):
|
||||
rest, dec_flags = divmod(rest, 1<<64)
|
||||
rest, bytes_to_feed = divmod(rest, 1<<64)
|
||||
need_eof, chars_to_skip = divmod(rest, 1<<64)
|
||||
return position, dec_flags, bytes_to_feed, bool(need_eof), chars_to_skip
|
||||
return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip
|
||||
|
||||
def tell(self):
|
||||
if not self._seekable:
|
||||
@@ -2398,7 +2383,7 @@ class TextIOWrapper(TextIOBase):
|
||||
# (a point where the decoder has nothing buffered, so seek()
|
||||
# can safely start from there and advance to this location).
|
||||
bytes_fed = 0
|
||||
need_eof = False
|
||||
need_eof = 0
|
||||
# Chars decoded since `start_pos`
|
||||
chars_decoded = 0
|
||||
for i in range(skip_bytes, len(next_input)):
|
||||
@@ -2415,7 +2400,7 @@ class TextIOWrapper(TextIOBase):
|
||||
else:
|
||||
# We didn't get enough decoded data; signal EOF to get more.
|
||||
chars_decoded += len(decoder.decode(b'', final=True))
|
||||
need_eof = True
|
||||
need_eof = 1
|
||||
if chars_decoded < chars_to_skip:
|
||||
raise OSError("can't reconstruct logical file position")
|
||||
|
||||
@@ -2530,9 +2515,8 @@ class TextIOWrapper(TextIOBase):
|
||||
# Read everything.
|
||||
result = (self._get_decoded_chars() +
|
||||
decoder.decode(self.buffer.read(), final=True))
|
||||
if self._snapshot is not None:
|
||||
self._set_decoded_chars('')
|
||||
self._snapshot = None
|
||||
self._set_decoded_chars('')
|
||||
self._snapshot = None
|
||||
return result
|
||||
else:
|
||||
# Keep reading chunks until we have size characters to return.
|
||||
|
||||
3
Lib/_sitebuiltins.py
vendored
3
Lib/_sitebuiltins.py
vendored
@@ -10,6 +10,7 @@ The objects used by the site module to add custom builtins.
|
||||
|
||||
import sys
|
||||
|
||||
|
||||
class Quitter(object):
|
||||
def __init__(self, name, eof):
|
||||
self.name = name
|
||||
@@ -47,7 +48,7 @@ class _Printer(object):
|
||||
data = None
|
||||
for filename in self.__filenames:
|
||||
try:
|
||||
with open(filename, encoding='utf-8') as fp:
|
||||
with open(filename, "r") as fp:
|
||||
data = fp.read()
|
||||
break
|
||||
except OSError:
|
||||
|
||||
1310
Lib/_sre.py
vendored
Normal file
1310
Lib/_sre.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
565
Lib/_strptime.py
vendored
565
Lib/_strptime.py
vendored
@@ -1,565 +0,0 @@
|
||||
"""Strptime-related classes and functions.
|
||||
|
||||
CLASSES:
|
||||
LocaleTime -- Discovers and stores locale-specific time information
|
||||
TimeRE -- Creates regexes for pattern matching a string of text containing
|
||||
time information
|
||||
|
||||
FUNCTIONS:
|
||||
_getlang -- Figure out what language is being used for the locale
|
||||
strptime -- Calculates the time struct represented by the passed-in string
|
||||
|
||||
"""
|
||||
import time
|
||||
import locale
|
||||
import calendar
|
||||
from re import compile as re_compile
|
||||
from re import IGNORECASE
|
||||
from re import escape as re_escape
|
||||
from datetime import (date as datetime_date,
|
||||
timedelta as datetime_timedelta,
|
||||
timezone as datetime_timezone)
|
||||
from _thread import allocate_lock as _thread_allocate_lock
|
||||
|
||||
__all__ = []
|
||||
|
||||
def _getlang():
|
||||
# Figure out what the current language is set to.
|
||||
return locale.getlocale(locale.LC_TIME)
|
||||
|
||||
class LocaleTime(object):
|
||||
"""Stores and handles locale-specific information related to time.
|
||||
|
||||
ATTRIBUTES:
|
||||
f_weekday -- full weekday names (7-item list)
|
||||
a_weekday -- abbreviated weekday names (7-item list)
|
||||
f_month -- full month names (13-item list; dummy value in [0], which
|
||||
is added by code)
|
||||
a_month -- abbreviated month names (13-item list, dummy value in
|
||||
[0], which is added by code)
|
||||
am_pm -- AM/PM representation (2-item list)
|
||||
LC_date_time -- format string for date/time representation (string)
|
||||
LC_date -- format string for date representation (string)
|
||||
LC_time -- format string for time representation (string)
|
||||
timezone -- daylight- and non-daylight-savings timezone representation
|
||||
(2-item list of sets)
|
||||
lang -- Language used by instance (2-item tuple)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Set all attributes.
|
||||
|
||||
Order of methods called matters for dependency reasons.
|
||||
|
||||
The locale language is set at the offset and then checked again before
|
||||
exiting. This is to make sure that the attributes were not set with a
|
||||
mix of information from more than one locale. This would most likely
|
||||
happen when using threads where one thread calls a locale-dependent
|
||||
function while another thread changes the locale while the function in
|
||||
the other thread is still running. Proper coding would call for
|
||||
locks to prevent changing the locale while locale-dependent code is
|
||||
running. The check here is done in case someone does not think about
|
||||
doing this.
|
||||
|
||||
Only other possible issue is if someone changed the timezone and did
|
||||
not call tz.tzset . That is an issue for the programmer, though,
|
||||
since changing the timezone is worthless without that call.
|
||||
|
||||
"""
|
||||
self.lang = _getlang()
|
||||
self.__calc_weekday()
|
||||
self.__calc_month()
|
||||
self.__calc_am_pm()
|
||||
self.__calc_timezone()
|
||||
self.__calc_date_time()
|
||||
if _getlang() != self.lang:
|
||||
raise ValueError("locale changed during initialization")
|
||||
if time.tzname != self.tzname or time.daylight != self.daylight:
|
||||
raise ValueError("timezone changed during initialization")
|
||||
|
||||
def __calc_weekday(self):
|
||||
# Set self.a_weekday and self.f_weekday using the calendar
|
||||
# module.
|
||||
a_weekday = [calendar.day_abbr[i].lower() for i in range(7)]
|
||||
f_weekday = [calendar.day_name[i].lower() for i in range(7)]
|
||||
self.a_weekday = a_weekday
|
||||
self.f_weekday = f_weekday
|
||||
|
||||
def __calc_month(self):
|
||||
# Set self.f_month and self.a_month using the calendar module.
|
||||
a_month = [calendar.month_abbr[i].lower() for i in range(13)]
|
||||
f_month = [calendar.month_name[i].lower() for i in range(13)]
|
||||
self.a_month = a_month
|
||||
self.f_month = f_month
|
||||
|
||||
def __calc_am_pm(self):
|
||||
# Set self.am_pm by using time.strftime().
|
||||
|
||||
# The magic date (1999,3,17,hour,44,55,2,76,0) is not really that
|
||||
# magical; just happened to have used it everywhere else where a
|
||||
# static date was needed.
|
||||
am_pm = []
|
||||
for hour in (1, 22):
|
||||
time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0))
|
||||
am_pm.append(time.strftime("%p", time_tuple).lower())
|
||||
self.am_pm = am_pm
|
||||
|
||||
def __calc_date_time(self):
|
||||
# Set self.date_time, self.date, & self.time by using
|
||||
# time.strftime().
|
||||
|
||||
# Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of
|
||||
# overloaded numbers is minimized. The order in which searches for
|
||||
# values within the format string is very important; it eliminates
|
||||
# possible ambiguity for what something represents.
|
||||
time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
|
||||
date_time = [None, None, None]
|
||||
date_time[0] = time.strftime("%c", time_tuple).lower()
|
||||
date_time[1] = time.strftime("%x", time_tuple).lower()
|
||||
date_time[2] = time.strftime("%X", time_tuple).lower()
|
||||
replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'),
|
||||
(self.f_month[3], '%B'), (self.a_weekday[2], '%a'),
|
||||
(self.a_month[3], '%b'), (self.am_pm[1], '%p'),
|
||||
('1999', '%Y'), ('99', '%y'), ('22', '%H'),
|
||||
('44', '%M'), ('55', '%S'), ('76', '%j'),
|
||||
('17', '%d'), ('03', '%m'), ('3', '%m'),
|
||||
# '3' needed for when no leading zero.
|
||||
('2', '%w'), ('10', '%I')]
|
||||
replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone
|
||||
for tz in tz_values])
|
||||
for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')):
|
||||
current_format = date_time[offset]
|
||||
for old, new in replacement_pairs:
|
||||
# Must deal with possible lack of locale info
|
||||
# manifesting itself as the empty string (e.g., Swedish's
|
||||
# lack of AM/PM info) or a platform returning a tuple of empty
|
||||
# strings (e.g., MacOS 9 having timezone as ('','')).
|
||||
if old:
|
||||
current_format = current_format.replace(old, new)
|
||||
# If %W is used, then Sunday, 2005-01-03 will fall on week 0 since
|
||||
# 2005-01-03 occurs before the first Monday of the year. Otherwise
|
||||
# %U is used.
|
||||
time_tuple = time.struct_time((1999,1,3,1,1,1,6,3,0))
|
||||
if '00' in time.strftime(directive, time_tuple):
|
||||
U_W = '%W'
|
||||
else:
|
||||
U_W = '%U'
|
||||
date_time[offset] = current_format.replace('11', U_W)
|
||||
self.LC_date_time = date_time[0]
|
||||
self.LC_date = date_time[1]
|
||||
self.LC_time = date_time[2]
|
||||
|
||||
def __calc_timezone(self):
|
||||
# Set self.timezone by using time.tzname.
|
||||
# Do not worry about possibility of time.tzname[0] == time.tzname[1]
|
||||
# and time.daylight; handle that in strptime.
|
||||
try:
|
||||
time.tzset()
|
||||
except AttributeError:
|
||||
pass
|
||||
self.tzname = time.tzname
|
||||
self.daylight = time.daylight
|
||||
no_saving = frozenset({"utc", "gmt", self.tzname[0].lower()})
|
||||
if self.daylight:
|
||||
has_saving = frozenset({self.tzname[1].lower()})
|
||||
else:
|
||||
has_saving = frozenset()
|
||||
self.timezone = (no_saving, has_saving)
|
||||
|
||||
|
||||
class TimeRE(dict):
|
||||
"""Handle conversion from format directives to regexes."""
|
||||
|
||||
def __init__(self, locale_time=None):
|
||||
"""Create keys/values.
|
||||
|
||||
Order of execution is important for dependency reasons.
|
||||
|
||||
"""
|
||||
if locale_time:
|
||||
self.locale_time = locale_time
|
||||
else:
|
||||
self.locale_time = LocaleTime()
|
||||
base = super()
|
||||
base.__init__({
|
||||
# The " [1-9]" part of the regex is to make %c from ANSI C work
|
||||
'd': r"(?P<d>3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
|
||||
'f': r"(?P<f>[0-9]{1,6})",
|
||||
'H': r"(?P<H>2[0-3]|[0-1]\d|\d)",
|
||||
'I': r"(?P<I>1[0-2]|0[1-9]|[1-9])",
|
||||
'G': r"(?P<G>\d\d\d\d)",
|
||||
'j': r"(?P<j>36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])",
|
||||
'm': r"(?P<m>1[0-2]|0[1-9]|[1-9])",
|
||||
'M': r"(?P<M>[0-5]\d|\d)",
|
||||
'S': r"(?P<S>6[0-1]|[0-5]\d|\d)",
|
||||
'U': r"(?P<U>5[0-3]|[0-4]\d|\d)",
|
||||
'w': r"(?P<w>[0-6])",
|
||||
'u': r"(?P<u>[1-7])",
|
||||
'V': r"(?P<V>5[0-3]|0[1-9]|[1-4]\d|\d)",
|
||||
# W is set below by using 'U'
|
||||
'y': r"(?P<y>\d\d)",
|
||||
#XXX: Does 'Y' need to worry about having less or more than
|
||||
# 4 digits?
|
||||
'Y': r"(?P<Y>\d\d\d\d)",
|
||||
'z': r"(?P<z>[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))",
|
||||
'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
|
||||
'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
|
||||
'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
|
||||
'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'),
|
||||
'p': self.__seqToRE(self.locale_time.am_pm, 'p'),
|
||||
'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone
|
||||
for tz in tz_names),
|
||||
'Z'),
|
||||
'%': '%'})
|
||||
base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
|
||||
base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
|
||||
base.__setitem__('x', self.pattern(self.locale_time.LC_date))
|
||||
base.__setitem__('X', self.pattern(self.locale_time.LC_time))
|
||||
|
||||
def __seqToRE(self, to_convert, directive):
|
||||
"""Convert a list to a regex string for matching a directive.
|
||||
|
||||
Want possible matching values to be from longest to shortest. This
|
||||
prevents the possibility of a match occurring for a value that also
|
||||
a substring of a larger value that should have matched (e.g., 'abc'
|
||||
matching when 'abcdef' should have been the match).
|
||||
|
||||
"""
|
||||
to_convert = sorted(to_convert, key=len, reverse=True)
|
||||
for value in to_convert:
|
||||
if value != '':
|
||||
break
|
||||
else:
|
||||
return ''
|
||||
regex = '|'.join(re_escape(stuff) for stuff in to_convert)
|
||||
regex = '(?P<%s>%s' % (directive, regex)
|
||||
return '%s)' % regex
|
||||
|
||||
def pattern(self, format):
|
||||
"""Return regex pattern for the format string.
|
||||
|
||||
Need to make sure that any characters that might be interpreted as
|
||||
regex syntax are escaped.
|
||||
|
||||
"""
|
||||
processed_format = ''
|
||||
# The sub() call escapes all characters that might be misconstrued
|
||||
# as regex syntax. Cannot use re.escape since we have to deal with
|
||||
# format directives (%m, etc.).
|
||||
regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
|
||||
format = regex_chars.sub(r"\\\1", format)
|
||||
whitespace_replacement = re_compile(r'\s+')
|
||||
format = whitespace_replacement.sub(r'\\s+', format)
|
||||
while '%' in format:
|
||||
directive_index = format.index('%')+1
|
||||
processed_format = "%s%s%s" % (processed_format,
|
||||
format[:directive_index-1],
|
||||
self[format[directive_index]])
|
||||
format = format[directive_index+1:]
|
||||
return "%s%s" % (processed_format, format)
|
||||
|
||||
def compile(self, format):
|
||||
"""Return a compiled re object for the format string."""
|
||||
return re_compile(self.pattern(format), IGNORECASE)
|
||||
|
||||
_cache_lock = _thread_allocate_lock()
|
||||
# DO NOT modify _TimeRE_cache or _regex_cache without acquiring the cache lock
|
||||
# first!
|
||||
_TimeRE_cache = TimeRE()
|
||||
_CACHE_MAX_SIZE = 5 # Max number of regexes stored in _regex_cache
|
||||
_regex_cache = {}
|
||||
|
||||
def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon):
|
||||
"""Calculate the Julian day based on the year, week of the year, and day of
|
||||
the week, with week_start_day representing whether the week of the year
|
||||
assumes the week starts on Sunday or Monday (6 or 0)."""
|
||||
first_weekday = datetime_date(year, 1, 1).weekday()
|
||||
# If we are dealing with the %U directive (week starts on Sunday), it's
|
||||
# easier to just shift the view to Sunday being the first day of the
|
||||
# week.
|
||||
if not week_starts_Mon:
|
||||
first_weekday = (first_weekday + 1) % 7
|
||||
day_of_week = (day_of_week + 1) % 7
|
||||
# Need to watch out for a week 0 (when the first day of the year is not
|
||||
# the same as that specified by %U or %W).
|
||||
week_0_length = (7 - first_weekday) % 7
|
||||
if week_of_year == 0:
|
||||
return 1 + day_of_week - first_weekday
|
||||
else:
|
||||
days_to_week = week_0_length + (7 * (week_of_year - 1))
|
||||
return 1 + days_to_week + day_of_week
|
||||
|
||||
|
||||
def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
|
||||
"""Return a 2-tuple consisting of a time struct and an int containing
|
||||
the number of microseconds based on the input string and the
|
||||
format string."""
|
||||
|
||||
for index, arg in enumerate([data_string, format]):
|
||||
if not isinstance(arg, str):
|
||||
msg = "strptime() argument {} must be str, not {}"
|
||||
raise TypeError(msg.format(index, type(arg)))
|
||||
|
||||
global _TimeRE_cache, _regex_cache
|
||||
with _cache_lock:
|
||||
locale_time = _TimeRE_cache.locale_time
|
||||
if (_getlang() != locale_time.lang or
|
||||
time.tzname != locale_time.tzname or
|
||||
time.daylight != locale_time.daylight):
|
||||
_TimeRE_cache = TimeRE()
|
||||
_regex_cache.clear()
|
||||
locale_time = _TimeRE_cache.locale_time
|
||||
if len(_regex_cache) > _CACHE_MAX_SIZE:
|
||||
_regex_cache.clear()
|
||||
format_regex = _regex_cache.get(format)
|
||||
if not format_regex:
|
||||
try:
|
||||
format_regex = _TimeRE_cache.compile(format)
|
||||
# KeyError raised when a bad format is found; can be specified as
|
||||
# \\, in which case it was a stray % but with a space after it
|
||||
except KeyError as err:
|
||||
bad_directive = err.args[0]
|
||||
if bad_directive == "\\":
|
||||
bad_directive = "%"
|
||||
del err
|
||||
raise ValueError("'%s' is a bad directive in format '%s'" %
|
||||
(bad_directive, format)) from None
|
||||
# IndexError only occurs when the format string is "%"
|
||||
except IndexError:
|
||||
raise ValueError("stray %% in format '%s'" % format) from None
|
||||
_regex_cache[format] = format_regex
|
||||
found = format_regex.match(data_string)
|
||||
if not found:
|
||||
raise ValueError("time data %r does not match format %r" %
|
||||
(data_string, format))
|
||||
if len(data_string) != found.end():
|
||||
raise ValueError("unconverted data remains: %s" %
|
||||
data_string[found.end():])
|
||||
|
||||
iso_year = year = None
|
||||
month = day = 1
|
||||
hour = minute = second = fraction = 0
|
||||
tz = -1
|
||||
gmtoff = None
|
||||
gmtoff_fraction = 0
|
||||
iso_week = week_of_year = None
|
||||
week_of_year_start = None
|
||||
# weekday and julian defaulted to None so as to signal need to calculate
|
||||
# values
|
||||
weekday = julian = None
|
||||
found_dict = found.groupdict()
|
||||
for group_key in found_dict.keys():
|
||||
# Directives not explicitly handled below:
|
||||
# c, x, X
|
||||
# handled by making out of other directives
|
||||
# U, W
|
||||
# worthless without day of the week
|
||||
if group_key == 'y':
|
||||
year = int(found_dict['y'])
|
||||
# Open Group specification for strptime() states that a %y
|
||||
#value in the range of [00, 68] is in the century 2000, while
|
||||
#[69,99] is in the century 1900
|
||||
if year <= 68:
|
||||
year += 2000
|
||||
else:
|
||||
year += 1900
|
||||
elif group_key == 'Y':
|
||||
year = int(found_dict['Y'])
|
||||
elif group_key == 'G':
|
||||
iso_year = int(found_dict['G'])
|
||||
elif group_key == 'm':
|
||||
month = int(found_dict['m'])
|
||||
elif group_key == 'B':
|
||||
month = locale_time.f_month.index(found_dict['B'].lower())
|
||||
elif group_key == 'b':
|
||||
month = locale_time.a_month.index(found_dict['b'].lower())
|
||||
elif group_key == 'd':
|
||||
day = int(found_dict['d'])
|
||||
elif group_key == 'H':
|
||||
hour = int(found_dict['H'])
|
||||
elif group_key == 'I':
|
||||
hour = int(found_dict['I'])
|
||||
ampm = found_dict.get('p', '').lower()
|
||||
# If there was no AM/PM indicator, we'll treat this like AM
|
||||
if ampm in ('', locale_time.am_pm[0]):
|
||||
# We're in AM so the hour is correct unless we're
|
||||
# looking at 12 midnight.
|
||||
# 12 midnight == 12 AM == hour 0
|
||||
if hour == 12:
|
||||
hour = 0
|
||||
elif ampm == locale_time.am_pm[1]:
|
||||
# We're in PM so we need to add 12 to the hour unless
|
||||
# we're looking at 12 noon.
|
||||
# 12 noon == 12 PM == hour 12
|
||||
if hour != 12:
|
||||
hour += 12
|
||||
elif group_key == 'M':
|
||||
minute = int(found_dict['M'])
|
||||
elif group_key == 'S':
|
||||
second = int(found_dict['S'])
|
||||
elif group_key == 'f':
|
||||
s = found_dict['f']
|
||||
# Pad to always return microseconds.
|
||||
s += "0" * (6 - len(s))
|
||||
fraction = int(s)
|
||||
elif group_key == 'A':
|
||||
weekday = locale_time.f_weekday.index(found_dict['A'].lower())
|
||||
elif group_key == 'a':
|
||||
weekday = locale_time.a_weekday.index(found_dict['a'].lower())
|
||||
elif group_key == 'w':
|
||||
weekday = int(found_dict['w'])
|
||||
if weekday == 0:
|
||||
weekday = 6
|
||||
else:
|
||||
weekday -= 1
|
||||
elif group_key == 'u':
|
||||
weekday = int(found_dict['u'])
|
||||
weekday -= 1
|
||||
elif group_key == 'j':
|
||||
julian = int(found_dict['j'])
|
||||
elif group_key in ('U', 'W'):
|
||||
week_of_year = int(found_dict[group_key])
|
||||
if group_key == 'U':
|
||||
# U starts week on Sunday.
|
||||
week_of_year_start = 6
|
||||
else:
|
||||
# W starts week on Monday.
|
||||
week_of_year_start = 0
|
||||
elif group_key == 'V':
|
||||
iso_week = int(found_dict['V'])
|
||||
elif group_key == 'z':
|
||||
z = found_dict['z']
|
||||
if z == 'Z':
|
||||
gmtoff = 0
|
||||
else:
|
||||
if z[3] == ':':
|
||||
z = z[:3] + z[4:]
|
||||
if len(z) > 5:
|
||||
if z[5] != ':':
|
||||
msg = f"Inconsistent use of : in {found_dict['z']}"
|
||||
raise ValueError(msg)
|
||||
z = z[:5] + z[6:]
|
||||
hours = int(z[1:3])
|
||||
minutes = int(z[3:5])
|
||||
seconds = int(z[5:7] or 0)
|
||||
gmtoff = (hours * 60 * 60) + (minutes * 60) + seconds
|
||||
gmtoff_remainder = z[8:]
|
||||
# Pad to always return microseconds.
|
||||
gmtoff_remainder_padding = "0" * (6 - len(gmtoff_remainder))
|
||||
gmtoff_fraction = int(gmtoff_remainder + gmtoff_remainder_padding)
|
||||
if z.startswith("-"):
|
||||
gmtoff = -gmtoff
|
||||
gmtoff_fraction = -gmtoff_fraction
|
||||
elif group_key == 'Z':
|
||||
# Since -1 is default value only need to worry about setting tz if
|
||||
# it can be something other than -1.
|
||||
found_zone = found_dict['Z'].lower()
|
||||
for value, tz_values in enumerate(locale_time.timezone):
|
||||
if found_zone in tz_values:
|
||||
# Deal with bad locale setup where timezone names are the
|
||||
# same and yet time.daylight is true; too ambiguous to
|
||||
# be able to tell what timezone has daylight savings
|
||||
if (time.tzname[0] == time.tzname[1] and
|
||||
time.daylight and found_zone not in ("utc", "gmt")):
|
||||
break
|
||||
else:
|
||||
tz = value
|
||||
break
|
||||
|
||||
# Deal with the cases where ambiguities arise
|
||||
# don't assume default values for ISO week/year
|
||||
if iso_year is not None:
|
||||
if julian is not None:
|
||||
raise ValueError("Day of the year directive '%j' is not "
|
||||
"compatible with ISO year directive '%G'. "
|
||||
"Use '%Y' instead.")
|
||||
elif iso_week is None or weekday is None:
|
||||
raise ValueError("ISO year directive '%G' must be used with "
|
||||
"the ISO week directive '%V' and a weekday "
|
||||
"directive ('%A', '%a', '%w', or '%u').")
|
||||
elif iso_week is not None:
|
||||
if year is None or weekday is None:
|
||||
raise ValueError("ISO week directive '%V' must be used with "
|
||||
"the ISO year directive '%G' and a weekday "
|
||||
"directive ('%A', '%a', '%w', or '%u').")
|
||||
else:
|
||||
raise ValueError("ISO week directive '%V' is incompatible with "
|
||||
"the year directive '%Y'. Use the ISO year '%G' "
|
||||
"instead.")
|
||||
|
||||
leap_year_fix = False
|
||||
if year is None:
|
||||
if month == 2 and day == 29:
|
||||
year = 1904 # 1904 is first leap year of 20th century
|
||||
leap_year_fix = True
|
||||
else:
|
||||
year = 1900
|
||||
|
||||
# If we know the week of the year and what day of that week, we can figure
|
||||
# out the Julian day of the year.
|
||||
if julian is None and weekday is not None:
|
||||
if week_of_year is not None:
|
||||
week_starts_Mon = True if week_of_year_start == 0 else False
|
||||
julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
|
||||
week_starts_Mon)
|
||||
elif iso_year is not None and iso_week is not None:
|
||||
datetime_result = datetime_date.fromisocalendar(iso_year, iso_week, weekday + 1)
|
||||
year = datetime_result.year
|
||||
month = datetime_result.month
|
||||
day = datetime_result.day
|
||||
if julian is not None and julian <= 0:
|
||||
year -= 1
|
||||
yday = 366 if calendar.isleap(year) else 365
|
||||
julian += yday
|
||||
|
||||
if julian is None:
|
||||
# Cannot pre-calculate datetime_date() since can change in Julian
|
||||
# calculation and thus could have different value for the day of
|
||||
# the week calculation.
|
||||
# Need to add 1 to result since first day of the year is 1, not 0.
|
||||
julian = datetime_date(year, month, day).toordinal() - \
|
||||
datetime_date(year, 1, 1).toordinal() + 1
|
||||
else: # Assume that if they bothered to include Julian day (or if it was
|
||||
# calculated above with year/week/weekday) it will be accurate.
|
||||
datetime_result = datetime_date.fromordinal(
|
||||
(julian - 1) +
|
||||
datetime_date(year, 1, 1).toordinal())
|
||||
year = datetime_result.year
|
||||
month = datetime_result.month
|
||||
day = datetime_result.day
|
||||
if weekday is None:
|
||||
weekday = datetime_date(year, month, day).weekday()
|
||||
# Add timezone info
|
||||
tzname = found_dict.get("Z")
|
||||
|
||||
if leap_year_fix:
|
||||
# the caller didn't supply a year but asked for Feb 29th. We couldn't
|
||||
# use the default of 1900 for computations. We set it back to ensure
|
||||
# that February 29th is smaller than March 1st.
|
||||
year = 1900
|
||||
|
||||
return (year, month, day,
|
||||
hour, minute, second,
|
||||
weekday, julian, tz, tzname, gmtoff), fraction, gmtoff_fraction
|
||||
|
||||
def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"):
|
||||
"""Return a time struct based on the input string and the
|
||||
format string."""
|
||||
tt = _strptime(data_string, format)[0]
|
||||
return time.struct_time(tt[:time._STRUCT_TM_ITEMS])
|
||||
|
||||
def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"):
|
||||
"""Return a class cls instance based on the input string and the
|
||||
format string."""
|
||||
tt, fraction, gmtoff_fraction = _strptime(data_string, format)
|
||||
tzname, gmtoff = tt[-2:]
|
||||
args = tt[:6] + (fraction,)
|
||||
if gmtoff is not None:
|
||||
tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction)
|
||||
if tzname:
|
||||
tz = datetime_timezone(tzdelta, tzname)
|
||||
else:
|
||||
tz = datetime_timezone(tzdelta)
|
||||
args += (tz,)
|
||||
|
||||
return cls(*args)
|
||||
13
Lib/_weakrefset.py
vendored
13
Lib/_weakrefset.py
vendored
@@ -3,7 +3,6 @@
|
||||
# by abc.py to load everything else at startup.
|
||||
|
||||
from _weakref import ref
|
||||
from types import GenericAlias
|
||||
|
||||
__all__ = ['WeakSet']
|
||||
|
||||
@@ -51,14 +50,10 @@ class WeakSet:
|
||||
self.update(data)
|
||||
|
||||
def _commit_removals(self):
|
||||
pop = self._pending_removals.pop
|
||||
l = self._pending_removals
|
||||
discard = self.data.discard
|
||||
while True:
|
||||
try:
|
||||
item = pop()
|
||||
except IndexError:
|
||||
return
|
||||
discard(item)
|
||||
while l:
|
||||
discard(l.pop())
|
||||
|
||||
def __iter__(self):
|
||||
with _IterationGuard(self):
|
||||
@@ -202,5 +197,3 @@ class WeakSet:
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.data)
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
94
Lib/abc.py
vendored
94
Lib/abc.py
vendored
@@ -11,14 +11,13 @@ def abstractmethod(funcobj):
|
||||
class that has a metaclass derived from ABCMeta cannot be
|
||||
instantiated unless all of its abstract methods are overridden.
|
||||
The abstract methods can be called using any of the normal
|
||||
'super' call mechanisms. abstractmethod() may be used to declare
|
||||
abstract methods for properties and descriptors.
|
||||
'super' call mechanisms.
|
||||
|
||||
Usage:
|
||||
|
||||
class C(metaclass=ABCMeta):
|
||||
@abstractmethod
|
||||
def my_abstract_method(self, arg1, arg2, argN):
|
||||
def my_abstract_method(self, ...):
|
||||
...
|
||||
"""
|
||||
funcobj.__isabstractmethod__ = True
|
||||
@@ -28,14 +27,17 @@ def abstractmethod(funcobj):
|
||||
class abstractclassmethod(classmethod):
|
||||
"""A decorator indicating abstract classmethods.
|
||||
|
||||
Deprecated, use 'classmethod' with 'abstractmethod' instead:
|
||||
Similar to abstractmethod.
|
||||
|
||||
class C(ABC):
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
Usage:
|
||||
|
||||
class C(metaclass=ABCMeta):
|
||||
@abstractclassmethod
|
||||
def my_abstract_classmethod(cls, ...):
|
||||
...
|
||||
|
||||
'abstractclassmethod' is deprecated. Use 'classmethod' with
|
||||
'abstractmethod' instead.
|
||||
"""
|
||||
|
||||
__isabstractmethod__ = True
|
||||
@@ -48,14 +50,17 @@ class abstractclassmethod(classmethod):
|
||||
class abstractstaticmethod(staticmethod):
|
||||
"""A decorator indicating abstract staticmethods.
|
||||
|
||||
Deprecated, use 'staticmethod' with 'abstractmethod' instead:
|
||||
Similar to abstractmethod.
|
||||
|
||||
class C(ABC):
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
Usage:
|
||||
|
||||
class C(metaclass=ABCMeta):
|
||||
@abstractstaticmethod
|
||||
def my_abstract_staticmethod(...):
|
||||
...
|
||||
|
||||
'abstractstaticmethod' is deprecated. Use 'staticmethod' with
|
||||
'abstractmethod' instead.
|
||||
"""
|
||||
|
||||
__isabstractmethod__ = True
|
||||
@@ -68,14 +73,29 @@ class abstractstaticmethod(staticmethod):
|
||||
class abstractproperty(property):
|
||||
"""A decorator indicating abstract properties.
|
||||
|
||||
Deprecated, use 'property' with 'abstractmethod' instead:
|
||||
Requires that the metaclass is ABCMeta or derived from it. A
|
||||
class that has a metaclass derived from ABCMeta cannot be
|
||||
instantiated unless all of its abstract properties are overridden.
|
||||
The abstract properties can be called using any of the normal
|
||||
'super' call mechanisms.
|
||||
|
||||
class C(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
Usage:
|
||||
|
||||
class C(metaclass=ABCMeta):
|
||||
@abstractproperty
|
||||
def my_abstract_property(self):
|
||||
...
|
||||
|
||||
This defines a read-only property; you can also define a read-write
|
||||
abstract property using the 'long' form of property declaration:
|
||||
|
||||
class C(metaclass=ABCMeta):
|
||||
def getx(self): ...
|
||||
def setx(self, value): ...
|
||||
x = abstractproperty(getx, setx)
|
||||
|
||||
'abstractproperty' is deprecated. Use 'property' with 'abstractmethod'
|
||||
instead.
|
||||
"""
|
||||
|
||||
__isabstractmethod__ = True
|
||||
@@ -85,10 +105,6 @@ try:
|
||||
from _abc import (get_cache_token, _abc_init, _abc_register,
|
||||
_abc_instancecheck, _abc_subclasscheck, _get_dump,
|
||||
_reset_registry, _reset_caches)
|
||||
# TODO: RUSTPYTHON missing _abc module implementation.
|
||||
except ModuleNotFoundError:
|
||||
from _py_abc import ABCMeta, get_cache_token
|
||||
ABCMeta.__module__ = 'abc'
|
||||
except ImportError:
|
||||
from _py_abc import ABCMeta, get_cache_token
|
||||
ABCMeta.__module__ = 'abc'
|
||||
@@ -106,7 +122,7 @@ else:
|
||||
implementations defined by the registering ABC be callable (not
|
||||
even via super()).
|
||||
"""
|
||||
def __new__(mcls, name, bases, namespace, /, **kwargs):
|
||||
def __new__(mcls, name, bases, namespace, **kwargs):
|
||||
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
|
||||
_abc_init(cls)
|
||||
return cls
|
||||
@@ -147,44 +163,6 @@ else:
|
||||
_reset_caches(cls)
|
||||
|
||||
|
||||
def update_abstractmethods(cls):
|
||||
"""Recalculate the set of abstract methods of an abstract class.
|
||||
|
||||
If a class has had one of its abstract methods implemented after the
|
||||
class was created, the method will not be considered implemented until
|
||||
this function is called. Alternatively, if a new abstract method has been
|
||||
added to the class, it will only be considered an abstract method of the
|
||||
class after this function is called.
|
||||
|
||||
This function should be called before any use is made of the class,
|
||||
usually in class decorators that add methods to the subject class.
|
||||
|
||||
Returns cls, to allow usage as a class decorator.
|
||||
|
||||
If cls is not an instance of ABCMeta, does nothing.
|
||||
"""
|
||||
if not hasattr(cls, '__abstractmethods__'):
|
||||
# We check for __abstractmethods__ here because cls might by a C
|
||||
# implementation or a python implementation (especially during
|
||||
# testing), and we want to handle both cases.
|
||||
return cls
|
||||
|
||||
abstracts = set()
|
||||
# Check the existing abstract methods of the parents, keep only the ones
|
||||
# that are not implemented.
|
||||
for scls in cls.__bases__:
|
||||
for name in getattr(scls, '__abstractmethods__', ()):
|
||||
value = getattr(cls, name, None)
|
||||
if getattr(value, "__isabstractmethod__", False):
|
||||
abstracts.add(name)
|
||||
# Also add any other newly added abstract methods.
|
||||
for name, value in cls.__dict__.items():
|
||||
if getattr(value, "__isabstractmethod__", False):
|
||||
abstracts.add(name)
|
||||
cls.__abstractmethods__ = frozenset(abstracts)
|
||||
return cls
|
||||
|
||||
|
||||
class ABC(metaclass=ABCMeta):
|
||||
"""Helper class that provides a standard way to create an ABC using
|
||||
inheritance.
|
||||
|
||||
65
Lib/aifc.py
vendored
65
Lib/aifc.py
vendored
@@ -138,11 +138,7 @@ import struct
|
||||
import builtins
|
||||
import warnings
|
||||
|
||||
__all__ = ["Error", "open"]
|
||||
|
||||
|
||||
warnings._deprecated(__name__, remove=(3, 13))
|
||||
|
||||
__all__ = ["Error", "open", "openfp"]
|
||||
|
||||
class Error(Exception):
|
||||
pass
|
||||
@@ -255,9 +251,7 @@ def _write_float(f, x):
|
||||
_write_ulong(f, himant)
|
||||
_write_ulong(f, lomant)
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
from chunk import Chunk
|
||||
from chunk import Chunk
|
||||
from collections import namedtuple
|
||||
|
||||
_aifc_params = namedtuple('_aifc_params',
|
||||
@@ -453,33 +447,21 @@ class Aifc_read:
|
||||
#
|
||||
|
||||
def _alaw2lin(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
return audioop.alaw2lin(data, 2)
|
||||
|
||||
def _ulaw2lin(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
return audioop.ulaw2lin(data, 2)
|
||||
|
||||
def _adpcm2lin(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
if not hasattr(self, '_adpcmstate'):
|
||||
# first time
|
||||
self._adpcmstate = None
|
||||
data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
|
||||
return data
|
||||
|
||||
def _sowt2lin(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
return audioop.byteswap(data, 2)
|
||||
|
||||
def _read_comm_chunk(self, chunk):
|
||||
self._nchannels = _read_short(chunk)
|
||||
self._nframes = _read_long(chunk)
|
||||
@@ -515,8 +497,6 @@ class Aifc_read:
|
||||
self._convert = self._ulaw2lin
|
||||
elif self._comptype in (b'alaw', b'ALAW'):
|
||||
self._convert = self._alaw2lin
|
||||
elif self._comptype in (b'sowt', b'SOWT'):
|
||||
self._convert = self._sowt2lin
|
||||
else:
|
||||
raise Error('unsupported compression type')
|
||||
self._sampwidth = 2
|
||||
@@ -679,7 +659,7 @@ class Aifc_write:
|
||||
if self._nframeswritten:
|
||||
raise Error('cannot change parameters after starting to write')
|
||||
if comptype not in (b'NONE', b'ulaw', b'ULAW',
|
||||
b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
|
||||
b'alaw', b'ALAW', b'G722'):
|
||||
raise Error('unsupported compression type')
|
||||
self._comptype = comptype
|
||||
self._compname = compname
|
||||
@@ -700,7 +680,7 @@ class Aifc_write:
|
||||
if self._nframeswritten:
|
||||
raise Error('cannot change parameters after starting to write')
|
||||
if comptype not in (b'NONE', b'ulaw', b'ULAW',
|
||||
b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
|
||||
b'alaw', b'ALAW', b'G722'):
|
||||
raise Error('unsupported compression type')
|
||||
self.setnchannels(nchannels)
|
||||
self.setsampwidth(sampwidth)
|
||||
@@ -784,43 +764,28 @@ class Aifc_write:
|
||||
#
|
||||
|
||||
def _lin2alaw(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
return audioop.lin2alaw(data, 2)
|
||||
|
||||
def _lin2ulaw(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
return audioop.lin2ulaw(data, 2)
|
||||
|
||||
def _lin2adpcm(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
import audioop
|
||||
if not hasattr(self, '_adpcmstate'):
|
||||
self._adpcmstate = None
|
||||
data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
|
||||
return data
|
||||
|
||||
def _lin2sowt(self, data):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DeprecationWarning)
|
||||
import audioop
|
||||
return audioop.byteswap(data, 2)
|
||||
|
||||
def _ensure_header_written(self, datasize):
|
||||
if not self._nframeswritten:
|
||||
if self._comptype in (b'ULAW', b'ulaw',
|
||||
b'ALAW', b'alaw', b'G722',
|
||||
b'sowt', b'SOWT'):
|
||||
if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
|
||||
if not self._sampwidth:
|
||||
self._sampwidth = 2
|
||||
if self._sampwidth != 2:
|
||||
raise Error('sample width must be 2 when compressing '
|
||||
'with ulaw/ULAW, alaw/ALAW, sowt/SOWT '
|
||||
'or G7.22 (ADPCM)')
|
||||
'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)')
|
||||
if not self._nchannels:
|
||||
raise Error('# channels not specified')
|
||||
if not self._sampwidth:
|
||||
@@ -836,8 +801,6 @@ class Aifc_write:
|
||||
self._convert = self._lin2ulaw
|
||||
elif self._comptype in (b'alaw', b'ALAW'):
|
||||
self._convert = self._lin2alaw
|
||||
elif self._comptype in (b'sowt', b'SOWT'):
|
||||
self._convert = self._lin2sowt
|
||||
|
||||
def _write_header(self, initlength):
|
||||
if self._aifc and self._comptype != b'NONE':
|
||||
@@ -957,6 +920,10 @@ def open(f, mode=None):
|
||||
else:
|
||||
raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
|
||||
|
||||
def openfp(f, mode=None):
|
||||
warnings.warn("aifc.openfp is deprecated since Python 3.7. "
|
||||
"Use aifc.open instead.", DeprecationWarning, stacklevel=2)
|
||||
return open(f, mode=mode)
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
|
||||
4
Lib/antigravity.py
vendored
4
Lib/antigravity.py
vendored
@@ -11,7 +11,7 @@ def geohash(latitude, longitude, datedow):
|
||||
37.857713 -122.544543
|
||||
|
||||
'''
|
||||
# https://xkcd.com/426/
|
||||
h = hashlib.md5(datedow, usedforsecurity=False).hexdigest()
|
||||
# http://xkcd.com/426/
|
||||
h = hashlib.md5(datedow).hexdigest()
|
||||
p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])]
|
||||
print('%d%s %d%s' % (latitude, p[1:], longitude, q[1:]))
|
||||
|
||||
352
Lib/argparse.py
vendored
352
Lib/argparse.py
vendored
@@ -1,5 +1,4 @@
|
||||
# Author: Steven J. Bethard <steven.bethard@gmail.com>.
|
||||
# New maintainer as of 29 August 2019: Raymond Hettinger <raymond.hettinger@gmail.com>
|
||||
|
||||
"""Command-line parsing library
|
||||
|
||||
@@ -67,7 +66,6 @@ __all__ = [
|
||||
'ArgumentParser',
|
||||
'ArgumentError',
|
||||
'ArgumentTypeError',
|
||||
'BooleanOptionalAction',
|
||||
'FileType',
|
||||
'HelpFormatter',
|
||||
'ArgumentDefaultsHelpFormatter',
|
||||
@@ -89,8 +87,6 @@ import os as _os
|
||||
import re as _re
|
||||
import sys as _sys
|
||||
|
||||
import warnings
|
||||
|
||||
from gettext import gettext as _, ngettext
|
||||
|
||||
SUPPRESS = '==SUPPRESS=='
|
||||
@@ -131,7 +127,7 @@ class _AttributeHolder(object):
|
||||
return '%s(%s)' % (type_name, ', '.join(arg_strings))
|
||||
|
||||
def _get_kwargs(self):
|
||||
return list(self.__dict__.items())
|
||||
return sorted(self.__dict__.items())
|
||||
|
||||
def _get_args(self):
|
||||
return []
|
||||
@@ -153,7 +149,6 @@ def _copy_items(items):
|
||||
# Formatting Help
|
||||
# ===============
|
||||
|
||||
|
||||
class HelpFormatter(object):
|
||||
"""Formatter for generating usage messages and argument help strings.
|
||||
|
||||
@@ -169,12 +164,15 @@ class HelpFormatter(object):
|
||||
|
||||
# default setting for width
|
||||
if width is None:
|
||||
import shutil
|
||||
width = shutil.get_terminal_size().columns
|
||||
try:
|
||||
width = int(_os.environ['COLUMNS'])
|
||||
except (KeyError, ValueError):
|
||||
width = 80
|
||||
width -= 2
|
||||
|
||||
self._prog = prog
|
||||
self._indent_increment = indent_increment
|
||||
self._max_help_position = max_help_position
|
||||
self._max_help_position = min(max_help_position,
|
||||
max(width - 20, indent_increment * 2))
|
||||
self._width = width
|
||||
@@ -267,7 +265,7 @@ class HelpFormatter(object):
|
||||
invocations.append(get_invocation(subaction))
|
||||
|
||||
# update the maximum item length
|
||||
invocation_length = max(map(len, invocations))
|
||||
invocation_length = max([len(s) for s in invocations])
|
||||
action_length = invocation_length + self._current_indent
|
||||
self._action_max_length = max(self._action_max_length,
|
||||
action_length)
|
||||
@@ -345,22 +343,21 @@ class HelpFormatter(object):
|
||||
def get_lines(parts, indent, prefix=None):
|
||||
lines = []
|
||||
line = []
|
||||
indent_length = len(indent)
|
||||
if prefix is not None:
|
||||
line_len = len(prefix) - 1
|
||||
else:
|
||||
line_len = indent_length - 1
|
||||
line_len = len(indent) - 1
|
||||
for part in parts:
|
||||
if line_len + 1 + len(part) > text_width and line:
|
||||
lines.append(indent + ' '.join(line))
|
||||
line = []
|
||||
line_len = indent_length - 1
|
||||
line_len = len(indent) - 1
|
||||
line.append(part)
|
||||
line_len += len(part) + 1
|
||||
if line:
|
||||
lines.append(indent + ' '.join(line))
|
||||
if prefix is not None:
|
||||
lines[0] = lines[0][indent_length:]
|
||||
lines[0] = lines[0][len(indent):]
|
||||
return lines
|
||||
|
||||
# if prog is short, follow it with optionals or positionals
|
||||
@@ -396,44 +393,27 @@ class HelpFormatter(object):
|
||||
group_actions = set()
|
||||
inserts = {}
|
||||
for group in groups:
|
||||
if not group._group_actions:
|
||||
raise ValueError(f'empty group {group}')
|
||||
|
||||
try:
|
||||
start = actions.index(group._group_actions[0])
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
group_action_count = len(group._group_actions)
|
||||
end = start + group_action_count
|
||||
end = start + len(group._group_actions)
|
||||
if actions[start:end] == group._group_actions:
|
||||
|
||||
suppressed_actions_count = 0
|
||||
for action in group._group_actions:
|
||||
group_actions.add(action)
|
||||
if action.help is SUPPRESS:
|
||||
suppressed_actions_count += 1
|
||||
|
||||
exposed_actions_count = group_action_count - suppressed_actions_count
|
||||
|
||||
if not group.required:
|
||||
if start in inserts:
|
||||
inserts[start] += ' ['
|
||||
else:
|
||||
inserts[start] = '['
|
||||
if end in inserts:
|
||||
inserts[end] += ']'
|
||||
else:
|
||||
inserts[end] = ']'
|
||||
elif exposed_actions_count > 1:
|
||||
inserts[end] = ']'
|
||||
else:
|
||||
if start in inserts:
|
||||
inserts[start] += ' ('
|
||||
else:
|
||||
inserts[start] = '('
|
||||
if end in inserts:
|
||||
inserts[end] += ')'
|
||||
else:
|
||||
inserts[end] = ')'
|
||||
inserts[end] = ')'
|
||||
for i in range(start + 1, end):
|
||||
inserts[i] = '|'
|
||||
|
||||
@@ -470,7 +450,7 @@ class HelpFormatter(object):
|
||||
# if the Optional doesn't take a value, format is:
|
||||
# -s or --long
|
||||
if action.nargs == 0:
|
||||
part = action.format_usage()
|
||||
part = '%s' % option_string
|
||||
|
||||
# if the Optional takes a value, format is:
|
||||
# -s ARGS or --long ARGS
|
||||
@@ -499,6 +479,7 @@ class HelpFormatter(object):
|
||||
text = _re.sub(r'(%s) ' % open, r'\1', text)
|
||||
text = _re.sub(r' (%s)' % close, r'\1', text)
|
||||
text = _re.sub(r'%s *%s' % (open, close), r'', text)
|
||||
text = _re.sub(r'\(([^|]*)\)', r'\1', text)
|
||||
text = text.strip()
|
||||
|
||||
# return the text
|
||||
@@ -540,13 +521,12 @@ class HelpFormatter(object):
|
||||
parts = [action_header]
|
||||
|
||||
# if there was help for the action, add lines of help text
|
||||
if action.help and action.help.strip():
|
||||
if action.help:
|
||||
help_text = self._expand_help(action)
|
||||
if help_text:
|
||||
help_lines = self._split_lines(help_text, help_width)
|
||||
parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
|
||||
for line in help_lines[1:]:
|
||||
parts.append('%*s%s\n' % (help_position, '', line))
|
||||
help_lines = self._split_lines(help_text, help_width)
|
||||
parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
|
||||
for line in help_lines[1:]:
|
||||
parts.append('%*s%s\n' % (help_position, '', line))
|
||||
|
||||
# or add a newline if the description doesn't end with one
|
||||
elif not action_header.endswith('\n'):
|
||||
@@ -606,11 +586,7 @@ class HelpFormatter(object):
|
||||
elif action.nargs == OPTIONAL:
|
||||
result = '[%s]' % get_metavar(1)
|
||||
elif action.nargs == ZERO_OR_MORE:
|
||||
metavar = get_metavar(1)
|
||||
if len(metavar) == 2:
|
||||
result = '[%s [%s ...]]' % metavar
|
||||
else:
|
||||
result = '[%s ...]' % metavar
|
||||
result = '[%s [%s ...]]' % get_metavar(2)
|
||||
elif action.nargs == ONE_OR_MORE:
|
||||
result = '%s [%s ...]' % get_metavar(2)
|
||||
elif action.nargs == REMAINDER:
|
||||
@@ -620,10 +596,7 @@ class HelpFormatter(object):
|
||||
elif action.nargs == SUPPRESS:
|
||||
result = ''
|
||||
else:
|
||||
try:
|
||||
formats = ['%s' for _ in range(action.nargs)]
|
||||
except TypeError:
|
||||
raise ValueError("invalid nargs value") from None
|
||||
formats = ['%s' for _ in range(action.nargs)]
|
||||
result = ' '.join(formats) % get_metavar(action.nargs)
|
||||
return result
|
||||
|
||||
@@ -704,19 +677,8 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
|
||||
"""
|
||||
|
||||
def _get_help_string(self, action):
|
||||
"""
|
||||
Add the default value to the option help message.
|
||||
|
||||
ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't
|
||||
already present. This code will do that, detecting cornercases to
|
||||
prevent duplicates or cases where it wouldn't make sense to the end
|
||||
user.
|
||||
"""
|
||||
help = action.help
|
||||
if help is None:
|
||||
help = ''
|
||||
|
||||
if '%(default)' not in help:
|
||||
if '%(default)' not in action.help:
|
||||
if action.default is not SUPPRESS:
|
||||
defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
|
||||
if action.option_strings or action.nargs in defaulting_nargs:
|
||||
@@ -724,7 +686,6 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
|
||||
return help
|
||||
|
||||
|
||||
|
||||
class MetavarTypeHelpFormatter(HelpFormatter):
|
||||
"""Help message formatter which uses the argument 'type' as the default
|
||||
metavar value (instead of the argument 'dest')
|
||||
@@ -740,6 +701,7 @@ class MetavarTypeHelpFormatter(HelpFormatter):
|
||||
return action.type.__name__
|
||||
|
||||
|
||||
|
||||
# =====================
|
||||
# Options and Arguments
|
||||
# =====================
|
||||
@@ -748,13 +710,11 @@ def _get_action_name(argument):
|
||||
if argument is None:
|
||||
return None
|
||||
elif argument.option_strings:
|
||||
return '/'.join(argument.option_strings)
|
||||
return '/'.join(argument.option_strings)
|
||||
elif argument.metavar not in (None, SUPPRESS):
|
||||
return argument.metavar
|
||||
elif argument.dest not in (None, SUPPRESS):
|
||||
return argument.dest
|
||||
elif argument.choices:
|
||||
return '{' + ','.join(argument.choices) + '}'
|
||||
else:
|
||||
return None
|
||||
|
||||
@@ -774,7 +734,7 @@ class ArgumentError(Exception):
|
||||
if self.argument_name is None:
|
||||
format = '%(message)s'
|
||||
else:
|
||||
format = _('argument %(argument_name)s: %(message)s')
|
||||
format = 'argument %(argument_name)s: %(message)s'
|
||||
return format % dict(message=self.message,
|
||||
argument_name=self.argument_name)
|
||||
|
||||
@@ -870,79 +830,15 @@ class Action(_AttributeHolder):
|
||||
'default',
|
||||
'type',
|
||||
'choices',
|
||||
'required',
|
||||
'help',
|
||||
'metavar',
|
||||
]
|
||||
return [(name, getattr(self, name)) for name in names]
|
||||
|
||||
def format_usage(self):
|
||||
return self.option_strings[0]
|
||||
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
raise NotImplementedError(_('.__call__() not defined'))
|
||||
|
||||
|
||||
# FIXME: remove together with `BooleanOptionalAction` deprecated arguments.
|
||||
_deprecated_default = object()
|
||||
|
||||
class BooleanOptionalAction(Action):
|
||||
def __init__(self,
|
||||
option_strings,
|
||||
dest,
|
||||
default=None,
|
||||
type=_deprecated_default,
|
||||
choices=_deprecated_default,
|
||||
required=False,
|
||||
help=None,
|
||||
metavar=_deprecated_default):
|
||||
|
||||
_option_strings = []
|
||||
for option_string in option_strings:
|
||||
_option_strings.append(option_string)
|
||||
|
||||
if option_string.startswith('--'):
|
||||
option_string = '--no-' + option_string[2:]
|
||||
_option_strings.append(option_string)
|
||||
|
||||
# We need `_deprecated` special value to ban explicit arguments that
|
||||
# match default value. Like:
|
||||
# parser.add_argument('-f', action=BooleanOptionalAction, type=int)
|
||||
for field_name in ('type', 'choices', 'metavar'):
|
||||
if locals()[field_name] is not _deprecated_default:
|
||||
warnings._deprecated(
|
||||
field_name,
|
||||
"{name!r} is deprecated as of Python 3.12 and will be "
|
||||
"removed in Python {remove}.",
|
||||
remove=(3, 14))
|
||||
|
||||
if type is _deprecated_default:
|
||||
type = None
|
||||
if choices is _deprecated_default:
|
||||
choices = None
|
||||
if metavar is _deprecated_default:
|
||||
metavar = None
|
||||
|
||||
super().__init__(
|
||||
option_strings=_option_strings,
|
||||
dest=dest,
|
||||
nargs=0,
|
||||
default=default,
|
||||
type=type,
|
||||
choices=choices,
|
||||
required=required,
|
||||
help=help,
|
||||
metavar=metavar)
|
||||
|
||||
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
if option_string in self.option_strings:
|
||||
setattr(namespace, self.dest, not option_string.startswith('--no-'))
|
||||
|
||||
def format_usage(self):
|
||||
return ' | '.join(self.option_strings)
|
||||
|
||||
|
||||
class _StoreAction(Action):
|
||||
|
||||
def __init__(self,
|
||||
@@ -957,7 +853,7 @@ class _StoreAction(Action):
|
||||
help=None,
|
||||
metavar=None):
|
||||
if nargs == 0:
|
||||
raise ValueError('nargs for store actions must be != 0; if you '
|
||||
raise ValueError('nargs for store actions must be > 0; if you '
|
||||
'have nothing to store, actions such as store '
|
||||
'true or store const may be more appropriate')
|
||||
if const is not None and nargs != OPTIONAL:
|
||||
@@ -983,7 +879,7 @@ class _StoreConstAction(Action):
|
||||
def __init__(self,
|
||||
option_strings,
|
||||
dest,
|
||||
const=None,
|
||||
const,
|
||||
default=None,
|
||||
required=False,
|
||||
help=None,
|
||||
@@ -1049,7 +945,7 @@ class _AppendAction(Action):
|
||||
help=None,
|
||||
metavar=None):
|
||||
if nargs == 0:
|
||||
raise ValueError('nargs for append actions must be != 0; if arg '
|
||||
raise ValueError('nargs for append actions must be > 0; if arg '
|
||||
'strings are not supplying the value to append, '
|
||||
'the append const action may be more appropriate')
|
||||
if const is not None and nargs != OPTIONAL:
|
||||
@@ -1078,7 +974,7 @@ class _AppendConstAction(Action):
|
||||
def __init__(self,
|
||||
option_strings,
|
||||
dest,
|
||||
const=None,
|
||||
const,
|
||||
default=None,
|
||||
required=False,
|
||||
help=None,
|
||||
@@ -1210,13 +1106,6 @@ class _SubParsersAction(Action):
|
||||
|
||||
aliases = kwargs.pop('aliases', ())
|
||||
|
||||
if name in self._name_parser_map:
|
||||
raise ArgumentError(self, _('conflicting subparser: %s') % name)
|
||||
for alias in aliases:
|
||||
if alias in self._name_parser_map:
|
||||
raise ArgumentError(
|
||||
self, _('conflicting subparser alias: %s') % alias)
|
||||
|
||||
# create a pseudo-action to hold the choice help
|
||||
if 'help' in kwargs:
|
||||
help = kwargs.pop('help')
|
||||
@@ -1268,12 +1157,6 @@ class _SubParsersAction(Action):
|
||||
vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
|
||||
getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
|
||||
|
||||
class _ExtendAction(_AppendAction):
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
items = getattr(namespace, self.dest, None)
|
||||
items = _copy_items(items)
|
||||
items.extend(values)
|
||||
setattr(namespace, self.dest, items)
|
||||
|
||||
# ==============
|
||||
# Type classes
|
||||
@@ -1306,9 +1189,9 @@ class FileType(object):
|
||||
# the special argument "-" means sys.std{in,out}
|
||||
if string == '-':
|
||||
if 'r' in self._mode:
|
||||
return _sys.stdin.buffer if 'b' in self._mode else _sys.stdin
|
||||
elif any(c in self._mode for c in 'wax'):
|
||||
return _sys.stdout.buffer if 'b' in self._mode else _sys.stdout
|
||||
return _sys.stdin
|
||||
elif 'w' in self._mode:
|
||||
return _sys.stdout
|
||||
else:
|
||||
msg = _('argument "-" with mode %r') % self._mode
|
||||
raise ValueError(msg)
|
||||
@@ -1318,9 +1201,8 @@ class FileType(object):
|
||||
return open(string, self._mode, self._bufsize, self._encoding,
|
||||
self._errors)
|
||||
except OSError as e:
|
||||
args = {'filename': string, 'error': e}
|
||||
message = _("can't open '%(filename)s': %(error)s")
|
||||
raise ArgumentTypeError(message % args)
|
||||
message = _("can't open '%s': %s")
|
||||
raise ArgumentTypeError(message % (string, e))
|
||||
|
||||
def __repr__(self):
|
||||
args = self._mode, self._bufsize
|
||||
@@ -1383,7 +1265,6 @@ class _ActionsContainer(object):
|
||||
self.register('action', 'help', _HelpAction)
|
||||
self.register('action', 'version', _VersionAction)
|
||||
self.register('action', 'parsers', _SubParsersAction)
|
||||
self.register('action', 'extend', _ExtendAction)
|
||||
|
||||
# raise an exception if the conflict handler is invalid
|
||||
self._get_handler()
|
||||
@@ -1476,10 +1357,6 @@ class _ActionsContainer(object):
|
||||
if not callable(type_func):
|
||||
raise ValueError('%r is not callable' % (type_func,))
|
||||
|
||||
if type_func is FileType:
|
||||
raise ValueError('%r is a FileType class object, instance of it'
|
||||
' must be passed' % (type_func,))
|
||||
|
||||
# raise an error if the metavar does not match the type
|
||||
if hasattr(self, "_get_formatter"):
|
||||
try:
|
||||
@@ -1594,8 +1471,10 @@ class _ActionsContainer(object):
|
||||
|
||||
# strings starting with two prefix characters are long options
|
||||
option_strings.append(option_string)
|
||||
if len(option_string) > 1 and option_string[1] in self.prefix_chars:
|
||||
long_option_strings.append(option_string)
|
||||
if option_string[0] in self.prefix_chars:
|
||||
if len(option_string) > 1:
|
||||
if option_string[1] in self.prefix_chars:
|
||||
long_option_strings.append(option_string)
|
||||
|
||||
# infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
|
||||
dest = kwargs.pop('dest', None)
|
||||
@@ -1697,14 +1576,6 @@ class _ArgumentGroup(_ActionsContainer):
|
||||
super(_ArgumentGroup, self)._remove_action(action)
|
||||
self._group_actions.remove(action)
|
||||
|
||||
def add_argument_group(self, *args, **kwargs):
|
||||
warnings.warn(
|
||||
"Nesting argument groups is deprecated.",
|
||||
category=DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
return super().add_argument_group(*args, **kwargs)
|
||||
|
||||
|
||||
class _MutuallyExclusiveGroup(_ArgumentGroup):
|
||||
|
||||
@@ -1725,21 +1596,12 @@ class _MutuallyExclusiveGroup(_ArgumentGroup):
|
||||
self._container._remove_action(action)
|
||||
self._group_actions.remove(action)
|
||||
|
||||
def add_mutually_exclusive_group(self, *args, **kwargs):
|
||||
warnings.warn(
|
||||
"Nesting mutually exclusive groups is deprecated.",
|
||||
category=DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
return super().add_mutually_exclusive_group(*args, **kwargs)
|
||||
|
||||
|
||||
class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
"""Object for parsing command line strings into Python objects.
|
||||
|
||||
Keyword Arguments:
|
||||
- prog -- The name of the program (default:
|
||||
``os.path.basename(sys.argv[0])``)
|
||||
- prog -- The name of the program (default: sys.argv[0])
|
||||
- usage -- A usage message (default: auto-generated from arguments)
|
||||
- description -- A description of what the program does
|
||||
- epilog -- Text following the argument descriptions
|
||||
@@ -1752,8 +1614,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
- conflict_handler -- String indicating how to handle conflicts
|
||||
- add_help -- Add a -h/-help option
|
||||
- allow_abbrev -- Allow long options to be abbreviated unambiguously
|
||||
- exit_on_error -- Determines whether or not ArgumentParser exits with
|
||||
error info when an error occurs
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
@@ -1768,14 +1628,19 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
argument_default=None,
|
||||
conflict_handler='error',
|
||||
add_help=True,
|
||||
allow_abbrev=True,
|
||||
exit_on_error=True):
|
||||
|
||||
superinit = super(ArgumentParser, self).__init__
|
||||
superinit(description=description,
|
||||
prefix_chars=prefix_chars,
|
||||
argument_default=argument_default,
|
||||
conflict_handler=conflict_handler)
|
||||
allow_abbrev=True):
|
||||
_ActionsContainer.__init__(self,
|
||||
description=description,
|
||||
prefix_chars=prefix_chars,
|
||||
argument_default=argument_default,
|
||||
conflict_handler=conflict_handler)
|
||||
# FIXME: get multiple inheritance method resolution right so we can use
|
||||
# what's below instead of the modified version above
|
||||
# superinit = super(ArgumentParser, self).__init__
|
||||
# superinit(description=description,
|
||||
# prefix_chars=prefix_chars,
|
||||
# argument_default=argument_default,
|
||||
# conflict_handler=conflict_handler)
|
||||
|
||||
# default setting for prog
|
||||
if prog is None:
|
||||
@@ -1788,11 +1653,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
self.fromfile_prefix_chars = fromfile_prefix_chars
|
||||
self.add_help = add_help
|
||||
self.allow_abbrev = allow_abbrev
|
||||
self.exit_on_error = exit_on_error
|
||||
|
||||
add_group = self.add_argument_group
|
||||
self._positionals = add_group(_('positional arguments'))
|
||||
self._optionals = add_group(_('options'))
|
||||
self._optionals = add_group(_('optional arguments'))
|
||||
self._subparsers = None
|
||||
|
||||
# register types
|
||||
@@ -1919,18 +1783,15 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
setattr(namespace, dest, self._defaults[dest])
|
||||
|
||||
# parse the arguments and exit if there are any errors
|
||||
if self.exit_on_error:
|
||||
try:
|
||||
namespace, args = self._parse_known_args(args, namespace)
|
||||
except ArgumentError as err:
|
||||
self.error(str(err))
|
||||
else:
|
||||
try:
|
||||
namespace, args = self._parse_known_args(args, namespace)
|
||||
|
||||
if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
|
||||
args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
|
||||
delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
|
||||
return namespace, args
|
||||
if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
|
||||
args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
|
||||
delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
|
||||
return namespace, args
|
||||
except ArgumentError:
|
||||
err = _sys.exc_info()[1]
|
||||
self.error(str(err))
|
||||
|
||||
def _parse_known_args(self, arg_strings, namespace):
|
||||
# replace arg strings that are file references
|
||||
@@ -2026,11 +1887,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
# arguments, try to parse more single-dash options out
|
||||
# of the tail of the option string
|
||||
chars = self.prefix_chars
|
||||
if (
|
||||
arg_count == 0
|
||||
and option_string[1] not in chars
|
||||
and explicit_arg != ''
|
||||
):
|
||||
if arg_count == 0 and option_string[1] not in chars:
|
||||
action_tuples.append((action, [], option_string))
|
||||
char = option_string[0]
|
||||
option_string = char + explicit_arg[0]
|
||||
@@ -2194,16 +2051,15 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
# replace arguments referencing files with the file content
|
||||
else:
|
||||
try:
|
||||
with open(arg_string[1:],
|
||||
encoding=_sys.getfilesystemencoding(),
|
||||
errors=_sys.getfilesystemencodeerrors()) as args_file:
|
||||
with open(arg_string[1:]) as args_file:
|
||||
arg_strings = []
|
||||
for arg_line in args_file.read().splitlines():
|
||||
for arg in self.convert_arg_line_to_args(arg_line):
|
||||
arg_strings.append(arg)
|
||||
arg_strings = self._read_args_from_files(arg_strings)
|
||||
new_arg_strings.extend(arg_strings)
|
||||
except OSError as err:
|
||||
except OSError:
|
||||
err = _sys.exc_info()[1]
|
||||
self.error(str(err))
|
||||
|
||||
# return the modified argument list
|
||||
@@ -2224,11 +2080,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
OPTIONAL: _('expected at most one argument'),
|
||||
ONE_OR_MORE: _('expected at least one argument'),
|
||||
}
|
||||
msg = nargs_errors.get(action.nargs)
|
||||
if msg is None:
|
||||
msg = ngettext('expected %s argument',
|
||||
default = ngettext('expected %s argument',
|
||||
'expected %s arguments',
|
||||
action.nargs) % action.nargs
|
||||
msg = nargs_errors.get(action.nargs, default)
|
||||
raise ArgumentError(action, msg)
|
||||
|
||||
# return the number of arguments matched
|
||||
@@ -2275,23 +2130,24 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
action = self._option_string_actions[option_string]
|
||||
return action, option_string, explicit_arg
|
||||
|
||||
# search through all possible prefixes of the option string
|
||||
# and all actions in the parser for possible interpretations
|
||||
option_tuples = self._get_option_tuples(arg_string)
|
||||
if self.allow_abbrev:
|
||||
# search through all possible prefixes of the option string
|
||||
# and all actions in the parser for possible interpretations
|
||||
option_tuples = self._get_option_tuples(arg_string)
|
||||
|
||||
# if multiple actions match, the option string was ambiguous
|
||||
if len(option_tuples) > 1:
|
||||
options = ', '.join([option_string
|
||||
for action, option_string, explicit_arg in option_tuples])
|
||||
args = {'option': arg_string, 'matches': options}
|
||||
msg = _('ambiguous option: %(option)s could match %(matches)s')
|
||||
self.error(msg % args)
|
||||
# if multiple actions match, the option string was ambiguous
|
||||
if len(option_tuples) > 1:
|
||||
options = ', '.join([option_string
|
||||
for action, option_string, explicit_arg in option_tuples])
|
||||
args = {'option': arg_string, 'matches': options}
|
||||
msg = _('ambiguous option: %(option)s could match %(matches)s')
|
||||
self.error(msg % args)
|
||||
|
||||
# if exactly one action matched, this segmentation is good,
|
||||
# so return the parsed action
|
||||
elif len(option_tuples) == 1:
|
||||
option_tuple, = option_tuples
|
||||
return option_tuple
|
||||
# if exactly one action matched, this segmentation is good,
|
||||
# so return the parsed action
|
||||
elif len(option_tuples) == 1:
|
||||
option_tuple, = option_tuples
|
||||
return option_tuple
|
||||
|
||||
# if it was not found as an option, but it looks like a negative
|
||||
# number, it was meant to be positional
|
||||
@@ -2315,17 +2171,16 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
# split at the '='
|
||||
chars = self.prefix_chars
|
||||
if option_string[0] in chars and option_string[1] in chars:
|
||||
if self.allow_abbrev:
|
||||
if '=' in option_string:
|
||||
option_prefix, explicit_arg = option_string.split('=', 1)
|
||||
else:
|
||||
option_prefix = option_string
|
||||
explicit_arg = None
|
||||
for option_string in self._option_string_actions:
|
||||
if option_string.startswith(option_prefix):
|
||||
action = self._option_string_actions[option_string]
|
||||
tup = action, option_string, explicit_arg
|
||||
result.append(tup)
|
||||
if '=' in option_string:
|
||||
option_prefix, explicit_arg = option_string.split('=', 1)
|
||||
else:
|
||||
option_prefix = option_string
|
||||
explicit_arg = None
|
||||
for option_string in self._option_string_actions:
|
||||
if option_string.startswith(option_prefix):
|
||||
action = self._option_string_actions[option_string]
|
||||
tup = action, option_string, explicit_arg
|
||||
result.append(tup)
|
||||
|
||||
# single character options can be concatenated with their arguments
|
||||
# but multiple character options always have to have their argument
|
||||
@@ -2510,11 +2365,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
not action.option_strings):
|
||||
if action.default is not None:
|
||||
value = action.default
|
||||
self._check_value(action, value)
|
||||
else:
|
||||
# since arg_strings is always [] at this point
|
||||
# there is no need to use self._check_value(action, value)
|
||||
value = arg_strings
|
||||
self._check_value(action, value)
|
||||
|
||||
# single argument or optional argument produces a single value
|
||||
elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
|
||||
@@ -2555,8 +2408,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
result = type_func(arg_string)
|
||||
|
||||
# ArgumentTypeErrors indicate errors
|
||||
except ArgumentTypeError as err:
|
||||
msg = str(err)
|
||||
except ArgumentTypeError:
|
||||
name = getattr(action.type, '__name__', repr(action.type))
|
||||
msg = str(_sys.exc_info()[1])
|
||||
raise ArgumentError(action, msg)
|
||||
|
||||
# TypeErrors or ValueErrors also indicate errors
|
||||
@@ -2627,11 +2481,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
|
||||
def _print_message(self, message, file=None):
|
||||
if message:
|
||||
file = file or _sys.stderr
|
||||
try:
|
||||
file.write(message)
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
if file is None:
|
||||
file = _sys.stderr
|
||||
file.write(message)
|
||||
|
||||
# ===============
|
||||
# Exiting methods
|
||||
|
||||
1604
Lib/ast.py
vendored
1604
Lib/ast.py
vendored
File diff suppressed because it is too large
Load Diff
@@ -1,14 +1,22 @@
|
||||
"""The asyncio package, tracking PEP 3156."""
|
||||
|
||||
# flake8: noqa
|
||||
|
||||
import sys
|
||||
|
||||
import selectors
|
||||
# XXX RustPython TODO: _overlapped
|
||||
if sys.platform == 'win32' and False:
|
||||
# Similar thing for _overlapped.
|
||||
try:
|
||||
from . import _overlapped
|
||||
except ImportError:
|
||||
import _overlapped # Will also be exported.
|
||||
|
||||
|
||||
# This relies on each of the submodules having an __all__ variable.
|
||||
from .base_events import *
|
||||
from .coroutines import *
|
||||
from .events import *
|
||||
from .exceptions import *
|
||||
from .futures import *
|
||||
from .locks import *
|
||||
from .protocols import *
|
||||
@@ -17,15 +25,11 @@ from .queues import *
|
||||
from .streams import *
|
||||
from .subprocess import *
|
||||
from .tasks import *
|
||||
from .taskgroups import *
|
||||
from .timeouts import *
|
||||
from .threads import *
|
||||
from .transports import *
|
||||
|
||||
__all__ = (base_events.__all__ +
|
||||
coroutines.__all__ +
|
||||
events.__all__ +
|
||||
exceptions.__all__ +
|
||||
futures.__all__ +
|
||||
locks.__all__ +
|
||||
protocols.__all__ +
|
||||
@@ -34,9 +38,6 @@ __all__ = (base_events.__all__ +
|
||||
streams.__all__ +
|
||||
subprocess.__all__ +
|
||||
tasks.__all__ +
|
||||
taskgroups.__all__ +
|
||||
threads.__all__ +
|
||||
timeouts.__all__ +
|
||||
transports.__all__)
|
||||
|
||||
if sys.platform == 'win32': # pragma: no cover
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
import ast
|
||||
import asyncio
|
||||
import code
|
||||
import concurrent.futures
|
||||
import inspect
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
import warnings
|
||||
|
||||
from . import futures
|
||||
|
||||
|
||||
class AsyncIOInteractiveConsole(code.InteractiveConsole):
|
||||
|
||||
def __init__(self, locals, loop):
|
||||
super().__init__(locals)
|
||||
self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
|
||||
|
||||
self.loop = loop
|
||||
|
||||
def runcode(self, code):
|
||||
future = concurrent.futures.Future()
|
||||
|
||||
def callback():
|
||||
global repl_future
|
||||
global repl_future_interrupted
|
||||
|
||||
repl_future = None
|
||||
repl_future_interrupted = False
|
||||
|
||||
func = types.FunctionType(code, self.locals)
|
||||
try:
|
||||
coro = func()
|
||||
except SystemExit:
|
||||
raise
|
||||
except KeyboardInterrupt as ex:
|
||||
repl_future_interrupted = True
|
||||
future.set_exception(ex)
|
||||
return
|
||||
except BaseException as ex:
|
||||
future.set_exception(ex)
|
||||
return
|
||||
|
||||
if not inspect.iscoroutine(coro):
|
||||
future.set_result(coro)
|
||||
return
|
||||
|
||||
try:
|
||||
repl_future = self.loop.create_task(coro)
|
||||
futures._chain_future(repl_future, future)
|
||||
except BaseException as exc:
|
||||
future.set_exception(exc)
|
||||
|
||||
loop.call_soon_threadsafe(callback)
|
||||
|
||||
try:
|
||||
return future.result()
|
||||
except SystemExit:
|
||||
raise
|
||||
except BaseException:
|
||||
if repl_future_interrupted:
|
||||
self.write("\nKeyboardInterrupt\n")
|
||||
else:
|
||||
self.showtraceback()
|
||||
|
||||
|
||||
class REPLThread(threading.Thread):
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
banner = (
|
||||
f'asyncio REPL {sys.version} on {sys.platform}\n'
|
||||
f'Use "await" directly instead of "asyncio.run()".\n'
|
||||
f'Type "help", "copyright", "credits" or "license" '
|
||||
f'for more information.\n'
|
||||
f'{getattr(sys, "ps1", ">>> ")}import asyncio'
|
||||
)
|
||||
|
||||
console.interact(
|
||||
banner=banner,
|
||||
exitmsg='exiting asyncio REPL...')
|
||||
finally:
|
||||
warnings.filterwarnings(
|
||||
'ignore',
|
||||
message=r'^coroutine .* was never awaited$',
|
||||
category=RuntimeWarning)
|
||||
|
||||
loop.call_soon_threadsafe(loop.stop)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
repl_locals = {'asyncio': asyncio}
|
||||
for key in {'__name__', '__package__',
|
||||
'__loader__', '__spec__',
|
||||
'__builtins__', '__file__'}:
|
||||
repl_locals[key] = locals()[key]
|
||||
|
||||
console = AsyncIOInteractiveConsole(repl_locals, loop)
|
||||
|
||||
repl_future = None
|
||||
repl_future_interrupted = False
|
||||
|
||||
try:
|
||||
import readline # NoQA
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
repl_thread = REPLThread()
|
||||
repl_thread.daemon = True
|
||||
repl_thread.start()
|
||||
|
||||
while True:
|
||||
try:
|
||||
loop.run_forever()
|
||||
except KeyboardInterrupt:
|
||||
if repl_future and not repl_future.done():
|
||||
repl_future.cancel()
|
||||
repl_future_interrupted = True
|
||||
continue
|
||||
else:
|
||||
break
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,18 @@
|
||||
__all__ = ()
|
||||
__all__ = []
|
||||
|
||||
import concurrent.futures._base
|
||||
import reprlib
|
||||
|
||||
from . import format_helpers
|
||||
from . import events
|
||||
|
||||
Error = concurrent.futures._base.Error
|
||||
CancelledError = concurrent.futures.CancelledError
|
||||
TimeoutError = concurrent.futures.TimeoutError
|
||||
|
||||
|
||||
class InvalidStateError(Error):
|
||||
"""The operation is not allowed in this state."""
|
||||
|
||||
|
||||
# States for Future.
|
||||
_PENDING = 'PENDING'
|
||||
@@ -28,17 +38,17 @@ def _format_callbacks(cb):
|
||||
cb = ''
|
||||
|
||||
def format_cb(callback):
|
||||
return format_helpers._format_callback_source(callback, ())
|
||||
return events._format_callback_source(callback, ())
|
||||
|
||||
if size == 1:
|
||||
cb = format_cb(cb[0][0])
|
||||
cb = format_cb(cb[0])
|
||||
elif size == 2:
|
||||
cb = '{}, {}'.format(format_cb(cb[0][0]), format_cb(cb[1][0]))
|
||||
cb = '{}, {}'.format(format_cb(cb[0]), format_cb(cb[1]))
|
||||
elif size > 2:
|
||||
cb = '{}, <{} more>, {}'.format(format_cb(cb[0][0]),
|
||||
cb = '{}, <{} more>, {}'.format(format_cb(cb[0]),
|
||||
size - 2,
|
||||
format_cb(cb[-1][0]))
|
||||
return f'cb=[{cb}]'
|
||||
format_cb(cb[-1]))
|
||||
return 'cb=[%s]' % cb
|
||||
|
||||
|
||||
def _future_repr_info(future):
|
||||
@@ -47,21 +57,15 @@ def _future_repr_info(future):
|
||||
info = [future._state.lower()]
|
||||
if future._state == _FINISHED:
|
||||
if future._exception is not None:
|
||||
info.append(f'exception={future._exception!r}')
|
||||
info.append('exception={!r}'.format(future._exception))
|
||||
else:
|
||||
# use reprlib to limit the length of the output, especially
|
||||
# for very long strings
|
||||
result = reprlib.repr(future._result)
|
||||
info.append(f'result={result}')
|
||||
info.append('result={}'.format(result))
|
||||
if future._callbacks:
|
||||
info.append(_format_callbacks(future._callbacks))
|
||||
if future._source_traceback:
|
||||
frame = future._source_traceback[-1]
|
||||
info.append(f'created at {frame[0]}:{frame[1]}')
|
||||
info.append('created at %s:%s' % (frame[0], frame[1]))
|
||||
return info
|
||||
|
||||
|
||||
@reprlib.recursive_repr()
|
||||
def _future_repr(future):
|
||||
info = ' '.join(_future_repr_info(future))
|
||||
return f'<{future.__class__.__name__} {info}>'
|
||||
|
||||
@@ -2,8 +2,10 @@ import collections
|
||||
import subprocess
|
||||
import warnings
|
||||
|
||||
from . import compat
|
||||
from . import protocols
|
||||
from . import transports
|
||||
from .coroutines import coroutine
|
||||
from .log import logger
|
||||
|
||||
|
||||
@@ -57,9 +59,9 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
if self._closed:
|
||||
info.append('closed')
|
||||
if self._pid is not None:
|
||||
info.append(f'pid={self._pid}')
|
||||
info.append('pid=%s' % self._pid)
|
||||
if self._returncode is not None:
|
||||
info.append(f'returncode={self._returncode}')
|
||||
info.append('returncode=%s' % self._returncode)
|
||||
elif self._pid is not None:
|
||||
info.append('running')
|
||||
else:
|
||||
@@ -67,19 +69,19 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
|
||||
stdin = self._pipes.get(0)
|
||||
if stdin is not None:
|
||||
info.append(f'stdin={stdin.pipe}')
|
||||
info.append('stdin=%s' % stdin.pipe)
|
||||
|
||||
stdout = self._pipes.get(1)
|
||||
stderr = self._pipes.get(2)
|
||||
if stdout is not None and stderr is stdout:
|
||||
info.append(f'stdout=stderr={stdout.pipe}')
|
||||
info.append('stdout=stderr=%s' % stdout.pipe)
|
||||
else:
|
||||
if stdout is not None:
|
||||
info.append(f'stdout={stdout.pipe}')
|
||||
info.append('stdout=%s' % stdout.pipe)
|
||||
if stderr is not None:
|
||||
info.append(f'stderr={stderr.pipe}')
|
||||
info.append('stderr=%s' % stderr.pipe)
|
||||
|
||||
return '<{}>'.format(' '.join(info))
|
||||
return '<%s>' % ' '.join(info)
|
||||
|
||||
def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
|
||||
raise NotImplementedError
|
||||
@@ -103,13 +105,12 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
continue
|
||||
proto.pipe.close()
|
||||
|
||||
if (self._proc is not None and
|
||||
# has the child process finished?
|
||||
self._returncode is None and
|
||||
# the child process has finished, but the
|
||||
# transport hasn't been notified yet?
|
||||
self._proc.poll() is None):
|
||||
|
||||
if (self._proc is not None
|
||||
# the child process finished?
|
||||
and self._returncode is None
|
||||
# the child process finished but the transport was not notified yet?
|
||||
and self._proc.poll() is None
|
||||
):
|
||||
if self._loop.get_debug():
|
||||
logger.warning('Close running child process: kill %r', self)
|
||||
|
||||
@@ -120,10 +121,15 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
|
||||
# Don't clear the _proc reference yet: _post_init() may still run
|
||||
|
||||
def __del__(self, _warn=warnings.warn):
|
||||
if not self._closed:
|
||||
_warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
|
||||
self.close()
|
||||
# On Python 3.3 and older, objects with a destructor part of a reference
|
||||
# cycle are never destroyed. It's not more the case on Python 3.4 thanks
|
||||
# to the PEP 442.
|
||||
if compat.PY34:
|
||||
def __del__(self):
|
||||
if not self._closed:
|
||||
warnings.warn("unclosed transport %r" % self, ResourceWarning,
|
||||
source=self)
|
||||
self.close()
|
||||
|
||||
def get_pid(self):
|
||||
return self._pid
|
||||
@@ -153,25 +159,26 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
self._check_proc()
|
||||
self._proc.kill()
|
||||
|
||||
async def _connect_pipes(self, waiter):
|
||||
@coroutine
|
||||
def _connect_pipes(self, waiter):
|
||||
try:
|
||||
proc = self._proc
|
||||
loop = self._loop
|
||||
|
||||
if proc.stdin is not None:
|
||||
_, pipe = await loop.connect_write_pipe(
|
||||
_, pipe = yield from loop.connect_write_pipe(
|
||||
lambda: WriteSubprocessPipeProto(self, 0),
|
||||
proc.stdin)
|
||||
self._pipes[0] = pipe
|
||||
|
||||
if proc.stdout is not None:
|
||||
_, pipe = await loop.connect_read_pipe(
|
||||
_, pipe = yield from loop.connect_read_pipe(
|
||||
lambda: ReadSubprocessPipeProto(self, 1),
|
||||
proc.stdout)
|
||||
self._pipes[1] = pipe
|
||||
|
||||
if proc.stderr is not None:
|
||||
_, pipe = await loop.connect_read_pipe(
|
||||
_, pipe = yield from loop.connect_read_pipe(
|
||||
lambda: ReadSubprocessPipeProto(self, 2),
|
||||
proc.stderr)
|
||||
self._pipes[2] = pipe
|
||||
@@ -182,9 +189,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
for callback, data in self._pending_calls:
|
||||
loop.call_soon(callback, *data)
|
||||
self._pending_calls = None
|
||||
except (SystemExit, KeyboardInterrupt):
|
||||
raise
|
||||
except BaseException as exc:
|
||||
except Exception as exc:
|
||||
if waiter is not None and not waiter.cancelled():
|
||||
waiter.set_exception(exc)
|
||||
else:
|
||||
@@ -208,17 +213,24 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
assert returncode is not None, returncode
|
||||
assert self._returncode is None, self._returncode
|
||||
if self._loop.get_debug():
|
||||
logger.info('%r exited with return code %r', self, returncode)
|
||||
logger.info('%r exited with return code %r',
|
||||
self, returncode)
|
||||
self._returncode = returncode
|
||||
if self._proc.returncode is None:
|
||||
# asyncio uses a child watcher: copy the status into the Popen
|
||||
# object. On Python 3.6, it is required to avoid a ResourceWarning.
|
||||
self._proc.returncode = returncode
|
||||
self._call(self._protocol.process_exited)
|
||||
|
||||
self._try_finish()
|
||||
|
||||
async def _wait(self):
|
||||
# wake up futures waiting for wait()
|
||||
for waiter in self._exit_waiters:
|
||||
if not waiter.cancelled():
|
||||
waiter.set_result(returncode)
|
||||
self._exit_waiters = None
|
||||
|
||||
@coroutine
|
||||
def _wait(self):
|
||||
"""Wait until the process exit and return the process return code.
|
||||
|
||||
This method is a coroutine."""
|
||||
@@ -227,7 +239,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
|
||||
waiter = self._loop.create_future()
|
||||
self._exit_waiters.append(waiter)
|
||||
return await waiter
|
||||
return (yield from waiter)
|
||||
|
||||
def _try_finish(self):
|
||||
assert not self._finished
|
||||
@@ -242,11 +254,6 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
try:
|
||||
self._protocol.connection_lost(exc)
|
||||
finally:
|
||||
# wake up futures waiting for wait()
|
||||
for waiter in self._exit_waiters:
|
||||
if not waiter.cancelled():
|
||||
waiter.set_result(self._returncode)
|
||||
self._exit_waiters = None
|
||||
self._loop = None
|
||||
self._proc = None
|
||||
self._protocol = None
|
||||
@@ -264,7 +271,8 @@ class WriteSubprocessPipeProto(protocols.BaseProtocol):
|
||||
self.pipe = transport
|
||||
|
||||
def __repr__(self):
|
||||
return f'<{self.__class__.__name__} fd={self.fd} pipe={self.pipe!r}>'
|
||||
return ('<%s fd=%s pipe=%r>'
|
||||
% (self.__class__.__name__, self.fd, self.pipe))
|
||||
|
||||
def connection_lost(self, exc):
|
||||
self.disconnected = True
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import linecache
|
||||
import reprlib
|
||||
import traceback
|
||||
|
||||
from . import base_futures
|
||||
@@ -9,42 +8,25 @@ from . import coroutines
|
||||
def _task_repr_info(task):
|
||||
info = base_futures._future_repr_info(task)
|
||||
|
||||
if task.cancelling() and not task.done():
|
||||
if task._must_cancel:
|
||||
# replace status
|
||||
info[0] = 'cancelling'
|
||||
|
||||
info.insert(1, 'name=%r' % task.get_name())
|
||||
coro = coroutines._format_coroutine(task._coro)
|
||||
info.insert(1, 'coro=<%s>' % coro)
|
||||
|
||||
if task._fut_waiter is not None:
|
||||
info.insert(2, f'wait_for={task._fut_waiter!r}')
|
||||
|
||||
if task._coro:
|
||||
coro = coroutines._format_coroutine(task._coro)
|
||||
info.insert(2, f'coro=<{coro}>')
|
||||
|
||||
info.insert(2, 'wait_for=%r' % task._fut_waiter)
|
||||
return info
|
||||
|
||||
|
||||
@reprlib.recursive_repr()
|
||||
def _task_repr(task):
|
||||
info = ' '.join(_task_repr_info(task))
|
||||
return f'<{task.__class__.__name__} {info}>'
|
||||
|
||||
|
||||
def _task_get_stack(task, limit):
|
||||
frames = []
|
||||
if hasattr(task._coro, 'cr_frame'):
|
||||
# case 1: 'async def' coroutines
|
||||
try:
|
||||
# 'async def' coroutines
|
||||
f = task._coro.cr_frame
|
||||
elif hasattr(task._coro, 'gi_frame'):
|
||||
# case 2: legacy coroutines
|
||||
except AttributeError:
|
||||
f = task._coro.gi_frame
|
||||
elif hasattr(task._coro, 'ag_frame'):
|
||||
# case 3: async generators
|
||||
f = task._coro.ag_frame
|
||||
else:
|
||||
# case 4: unknown objects
|
||||
f = None
|
||||
if f is not None:
|
||||
while f is not None:
|
||||
if limit is not None:
|
||||
@@ -79,15 +61,15 @@ def _task_print_stack(task, limit, file):
|
||||
linecache.checkcache(filename)
|
||||
line = linecache.getline(filename, lineno, f.f_globals)
|
||||
extracted_list.append((filename, lineno, name, line))
|
||||
|
||||
exc = task._exception
|
||||
if not extracted_list:
|
||||
print(f'No stack for {task!r}', file=file)
|
||||
print('No stack for %r' % task, file=file)
|
||||
elif exc is not None:
|
||||
print(f'Traceback for {task!r} (most recent call last):', file=file)
|
||||
print('Traceback for %r (most recent call last):' % task,
|
||||
file=file)
|
||||
else:
|
||||
print(f'Stack for {task!r} (most recent call last):', file=file)
|
||||
|
||||
print('Stack for %r (most recent call last):' % task,
|
||||
file=file)
|
||||
traceback.print_list(extracted_list, file=file)
|
||||
if exc is not None:
|
||||
for line in traceback.format_exception_only(exc.__class__, exc):
|
||||
|
||||
18
Lib/asyncio/compat.py
Normal file
18
Lib/asyncio/compat.py
Normal file
@@ -0,0 +1,18 @@
|
||||
"""Compatibility helpers for the different Python versions."""
|
||||
|
||||
import sys
|
||||
|
||||
PY34 = sys.version_info >= (3, 4)
|
||||
PY35 = sys.version_info >= (3, 5)
|
||||
PY352 = sys.version_info >= (3, 5, 2)
|
||||
|
||||
|
||||
def flatten_list_bytes(list_of_data):
|
||||
"""Concatenate a sequence of bytes-like objects."""
|
||||
if not PY34:
|
||||
# On Python 3.3 and older, bytes.join() doesn't handle
|
||||
# memoryview.
|
||||
list_of_data = (
|
||||
bytes(data) if isinstance(data, memoryview) else data
|
||||
for data in list_of_data)
|
||||
return b''.join(list_of_data)
|
||||
@@ -1,41 +1,7 @@
|
||||
# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0
|
||||
# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0)
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io
|
||||
|
||||
import enum
|
||||
"""Constants."""
|
||||
|
||||
# After the connection is lost, log warnings after this many write()s.
|
||||
LOG_THRESHOLD_FOR_CONNLOST_WRITES = 5
|
||||
|
||||
# Seconds to wait before retrying accept().
|
||||
ACCEPT_RETRY_DELAY = 1
|
||||
|
||||
# Number of stack entries to capture in debug mode.
|
||||
# The larger the number, the slower the operation in debug mode
|
||||
# (see extract_stack() in format_helpers.py).
|
||||
DEBUG_STACK_DEPTH = 10
|
||||
|
||||
# Number of seconds to wait for SSL handshake to complete
|
||||
# The default timeout matches that of Nginx.
|
||||
SSL_HANDSHAKE_TIMEOUT = 60.0
|
||||
|
||||
# Number of seconds to wait for SSL shutdown to complete
|
||||
# The default timeout mimics lingering_time
|
||||
SSL_SHUTDOWN_TIMEOUT = 30.0
|
||||
|
||||
# Used in sendfile fallback code. We use fallback for platforms
|
||||
# that don't support sendfile, or for TLS connections.
|
||||
SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 256
|
||||
|
||||
FLOW_CONTROL_HIGH_WATER_SSL_READ = 256 # KiB
|
||||
FLOW_CONTROL_HIGH_WATER_SSL_WRITE = 512 # KiB
|
||||
|
||||
# Default timeout for joining the threads in the threadpool
|
||||
THREAD_JOIN_TIMEOUT = 300
|
||||
|
||||
# The enum should be here to break circular dependencies between
|
||||
# base_events and sslproto
|
||||
class _SendfileMode(enum.Enum):
|
||||
UNSUPPORTED = enum.auto()
|
||||
TRY_NATIVE = enum.auto()
|
||||
FALLBACK = enum.auto()
|
||||
|
||||
@@ -1,16 +1,249 @@
|
||||
__all__ = 'iscoroutinefunction', 'iscoroutine'
|
||||
__all__ = ['coroutine',
|
||||
'iscoroutinefunction', 'iscoroutine']
|
||||
|
||||
import collections.abc
|
||||
import functools
|
||||
import inspect
|
||||
import opcode
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
import types
|
||||
|
||||
from . import compat
|
||||
from . import events
|
||||
from . import base_futures
|
||||
from .log import logger
|
||||
|
||||
def _is_debug_mode():
|
||||
# See: https://docs.python.org/3/library/asyncio-dev.html#asyncio-debug-mode.
|
||||
return sys.flags.dev_mode or (not sys.flags.ignore_environment and
|
||||
bool(os.environ.get('PYTHONASYNCIODEBUG')))
|
||||
|
||||
# Opcode of "yield from" instruction
|
||||
_YIELD_FROM = opcode.opmap['YIELD_FROM']
|
||||
|
||||
# If you set _DEBUG to true, @coroutine will wrap the resulting
|
||||
# generator objects in a CoroWrapper instance (defined below). That
|
||||
# instance will log a message when the generator is never iterated
|
||||
# over, which may happen when you forget to use "yield from" with a
|
||||
# coroutine call. Note that the value of the _DEBUG flag is taken
|
||||
# when the decorator is used, so to be of any use it must be set
|
||||
# before you define your coroutines. A downside of using this feature
|
||||
# is that tracebacks show entries for the CoroWrapper.__next__ method
|
||||
# when _DEBUG is true.
|
||||
_DEBUG = (not sys.flags.ignore_environment and
|
||||
bool(os.environ.get('PYTHONASYNCIODEBUG')))
|
||||
|
||||
|
||||
try:
|
||||
_types_coroutine = types.coroutine
|
||||
_types_CoroutineType = types.CoroutineType
|
||||
except AttributeError:
|
||||
# Python 3.4
|
||||
_types_coroutine = None
|
||||
_types_CoroutineType = None
|
||||
|
||||
try:
|
||||
_inspect_iscoroutinefunction = inspect.iscoroutinefunction
|
||||
except AttributeError:
|
||||
# Python 3.4
|
||||
_inspect_iscoroutinefunction = lambda func: False
|
||||
|
||||
try:
|
||||
from collections.abc import Coroutine as _CoroutineABC, \
|
||||
Awaitable as _AwaitableABC
|
||||
except ImportError:
|
||||
_CoroutineABC = _AwaitableABC = None
|
||||
|
||||
|
||||
# Check for CPython issue #21209
|
||||
def has_yield_from_bug():
|
||||
class MyGen:
|
||||
def __init__(self):
|
||||
self.send_args = None
|
||||
def __iter__(self):
|
||||
return self
|
||||
def __next__(self):
|
||||
return 42
|
||||
def send(self, *what):
|
||||
self.send_args = what
|
||||
return None
|
||||
def yield_from_gen(gen):
|
||||
yield from gen
|
||||
value = (1, 2, 3)
|
||||
gen = MyGen()
|
||||
coro = yield_from_gen(gen)
|
||||
next(coro)
|
||||
coro.send(value)
|
||||
return gen.send_args != (value,)
|
||||
_YIELD_FROM_BUG = has_yield_from_bug()
|
||||
del has_yield_from_bug
|
||||
|
||||
|
||||
def debug_wrapper(gen):
|
||||
# This function is called from 'sys.set_coroutine_wrapper'.
|
||||
# We only wrap here coroutines defined via 'async def' syntax.
|
||||
# Generator-based coroutines are wrapped in @coroutine
|
||||
# decorator.
|
||||
return CoroWrapper(gen, None)
|
||||
|
||||
|
||||
class CoroWrapper:
|
||||
# Wrapper for coroutine object in _DEBUG mode.
|
||||
|
||||
def __init__(self, gen, func=None):
|
||||
assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen
|
||||
self.gen = gen
|
||||
self.func = func # Used to unwrap @coroutine decorator
|
||||
self._source_traceback = traceback.extract_stack(sys._getframe(1))
|
||||
self.__name__ = getattr(gen, '__name__', None)
|
||||
self.__qualname__ = getattr(gen, '__qualname__', None)
|
||||
|
||||
def __repr__(self):
|
||||
coro_repr = _format_coroutine(self)
|
||||
if self._source_traceback:
|
||||
frame = self._source_traceback[-1]
|
||||
coro_repr += ', created at %s:%s' % (frame[0], frame[1])
|
||||
return '<%s %s>' % (self.__class__.__name__, coro_repr)
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
return self.gen.send(None)
|
||||
|
||||
if _YIELD_FROM_BUG:
|
||||
# For for CPython issue #21209: using "yield from" and a custom
|
||||
# generator, generator.send(tuple) unpacks the tuple instead of passing
|
||||
# the tuple unchanged. Check if the caller is a generator using "yield
|
||||
# from" to decide if the parameter should be unpacked or not.
|
||||
def send(self, *value):
|
||||
frame = sys._getframe()
|
||||
caller = frame.f_back
|
||||
assert caller.f_lasti >= 0
|
||||
if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM:
|
||||
value = value[0]
|
||||
return self.gen.send(value)
|
||||
else:
|
||||
def send(self, value):
|
||||
return self.gen.send(value)
|
||||
|
||||
def throw(self, type, value=None, traceback=None):
|
||||
return self.gen.throw(type, value, traceback)
|
||||
|
||||
def close(self):
|
||||
return self.gen.close()
|
||||
|
||||
@property
|
||||
def gi_frame(self):
|
||||
return self.gen.gi_frame
|
||||
|
||||
@property
|
||||
def gi_running(self):
|
||||
return self.gen.gi_running
|
||||
|
||||
@property
|
||||
def gi_code(self):
|
||||
return self.gen.gi_code
|
||||
|
||||
if compat.PY35:
|
||||
|
||||
def __await__(self):
|
||||
cr_await = getattr(self.gen, 'cr_await', None)
|
||||
if cr_await is not None:
|
||||
raise RuntimeError(
|
||||
"Cannot await on coroutine {!r} while it's "
|
||||
"awaiting for {!r}".format(self.gen, cr_await))
|
||||
return self
|
||||
|
||||
@property
|
||||
def gi_yieldfrom(self):
|
||||
return self.gen.gi_yieldfrom
|
||||
|
||||
@property
|
||||
def cr_await(self):
|
||||
return self.gen.cr_await
|
||||
|
||||
@property
|
||||
def cr_running(self):
|
||||
return self.gen.cr_running
|
||||
|
||||
@property
|
||||
def cr_code(self):
|
||||
return self.gen.cr_code
|
||||
|
||||
@property
|
||||
def cr_frame(self):
|
||||
return self.gen.cr_frame
|
||||
|
||||
def __del__(self):
|
||||
# Be careful accessing self.gen.frame -- self.gen might not exist.
|
||||
gen = getattr(self, 'gen', None)
|
||||
frame = getattr(gen, 'gi_frame', None)
|
||||
if frame is None:
|
||||
frame = getattr(gen, 'cr_frame', None)
|
||||
if frame is not None and frame.f_lasti == -1:
|
||||
msg = '%r was never yielded from' % self
|
||||
tb = getattr(self, '_source_traceback', ())
|
||||
if tb:
|
||||
tb = ''.join(traceback.format_list(tb))
|
||||
msg += ('\nCoroutine object created at '
|
||||
'(most recent call last):\n')
|
||||
msg += tb.rstrip()
|
||||
logger.error(msg)
|
||||
|
||||
|
||||
def coroutine(func):
|
||||
"""Decorator to mark coroutines.
|
||||
|
||||
If the coroutine is not yielded from before it is destroyed,
|
||||
an error message is logged.
|
||||
"""
|
||||
if _inspect_iscoroutinefunction(func):
|
||||
# In Python 3.5 that's all we need to do for coroutines
|
||||
# defiend with "async def".
|
||||
# Wrapping in CoroWrapper will happen via
|
||||
# 'sys.set_coroutine_wrapper' function.
|
||||
return func
|
||||
|
||||
if inspect.isgeneratorfunction(func):
|
||||
coro = func
|
||||
else:
|
||||
@functools.wraps(func)
|
||||
def coro(*args, **kw):
|
||||
res = func(*args, **kw)
|
||||
if (base_futures.isfuture(res) or inspect.isgenerator(res) or
|
||||
isinstance(res, CoroWrapper)):
|
||||
res = yield from res
|
||||
elif _AwaitableABC is not None:
|
||||
# If 'func' returns an Awaitable (new in 3.5) we
|
||||
# want to run it.
|
||||
try:
|
||||
await_meth = res.__await__
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
if isinstance(res, _AwaitableABC):
|
||||
res = yield from await_meth()
|
||||
return res
|
||||
|
||||
if not _DEBUG:
|
||||
if _types_coroutine is None:
|
||||
wrapper = coro
|
||||
else:
|
||||
wrapper = _types_coroutine(coro)
|
||||
else:
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwds):
|
||||
w = CoroWrapper(coro(*args, **kwds), func=func)
|
||||
if w._source_traceback:
|
||||
del w._source_traceback[-1]
|
||||
# Python < 3.5 does not implement __qualname__
|
||||
# on generator objects, so we set it manually.
|
||||
# We use getattr as some callables (such as
|
||||
# functools.partial may lack __qualname__).
|
||||
w.__name__ = getattr(func, '__name__', None)
|
||||
w.__qualname__ = getattr(func, '__qualname__', None)
|
||||
return w
|
||||
|
||||
wrapper._is_coroutine = _is_coroutine # For iscoroutinefunction().
|
||||
return wrapper
|
||||
|
||||
|
||||
# A marker for iscoroutinefunction.
|
||||
@@ -19,91 +252,93 @@ _is_coroutine = object()
|
||||
|
||||
def iscoroutinefunction(func):
|
||||
"""Return True if func is a decorated coroutine function."""
|
||||
return (inspect.iscoroutinefunction(func) or
|
||||
getattr(func, '_is_coroutine', None) is _is_coroutine)
|
||||
return (getattr(func, '_is_coroutine', None) is _is_coroutine or
|
||||
_inspect_iscoroutinefunction(func))
|
||||
|
||||
|
||||
# Prioritize native coroutine check to speed-up
|
||||
# asyncio.iscoroutine.
|
||||
_COROUTINE_TYPES = (types.CoroutineType, collections.abc.Coroutine)
|
||||
_iscoroutine_typecache = set()
|
||||
_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper)
|
||||
if _CoroutineABC is not None:
|
||||
_COROUTINE_TYPES += (_CoroutineABC,)
|
||||
if _types_CoroutineType is not None:
|
||||
# Prioritize native coroutine check to speed-up
|
||||
# asyncio.iscoroutine.
|
||||
_COROUTINE_TYPES = (_types_CoroutineType,) + _COROUTINE_TYPES
|
||||
|
||||
|
||||
def iscoroutine(obj):
|
||||
"""Return True if obj is a coroutine object."""
|
||||
if type(obj) in _iscoroutine_typecache:
|
||||
return True
|
||||
|
||||
if isinstance(obj, _COROUTINE_TYPES):
|
||||
# Just in case we don't want to cache more than 100
|
||||
# positive types. That shouldn't ever happen, unless
|
||||
# someone stressing the system on purpose.
|
||||
if len(_iscoroutine_typecache) < 100:
|
||||
_iscoroutine_typecache.add(type(obj))
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
return isinstance(obj, _COROUTINE_TYPES)
|
||||
|
||||
|
||||
def _format_coroutine(coro):
|
||||
assert iscoroutine(coro)
|
||||
|
||||
def get_name(coro):
|
||||
# Coroutines compiled with Cython sometimes don't have
|
||||
# proper __qualname__ or __name__. While that is a bug
|
||||
# in Cython, asyncio shouldn't crash with an AttributeError
|
||||
# in its __repr__ functions.
|
||||
if hasattr(coro, '__qualname__') and coro.__qualname__:
|
||||
coro_name = coro.__qualname__
|
||||
elif hasattr(coro, '__name__') and coro.__name__:
|
||||
coro_name = coro.__name__
|
||||
else:
|
||||
# Stop masking Cython bugs, expose them in a friendly way.
|
||||
coro_name = f'<{type(coro).__name__} without __name__>'
|
||||
return f'{coro_name}()'
|
||||
if not hasattr(coro, 'cr_code') and not hasattr(coro, 'gi_code'):
|
||||
# Most likely a built-in type or a Cython coroutine.
|
||||
|
||||
def is_running(coro):
|
||||
# Built-in types might not have __qualname__ or __name__.
|
||||
coro_name = getattr(
|
||||
coro, '__qualname__',
|
||||
getattr(coro, '__name__', type(coro).__name__))
|
||||
coro_name = '{}()'.format(coro_name)
|
||||
|
||||
running = False
|
||||
try:
|
||||
return coro.cr_running
|
||||
running = coro.cr_running
|
||||
except AttributeError:
|
||||
try:
|
||||
return coro.gi_running
|
||||
running = coro.gi_running
|
||||
except AttributeError:
|
||||
return False
|
||||
pass
|
||||
|
||||
coro_code = None
|
||||
if hasattr(coro, 'cr_code') and coro.cr_code:
|
||||
coro_code = coro.cr_code
|
||||
elif hasattr(coro, 'gi_code') and coro.gi_code:
|
||||
coro_code = coro.gi_code
|
||||
|
||||
coro_name = get_name(coro)
|
||||
|
||||
if not coro_code:
|
||||
# Built-in types might not have __qualname__ or __name__.
|
||||
if is_running(coro):
|
||||
return f'{coro_name} running'
|
||||
if running:
|
||||
return '{} running'.format(coro_name)
|
||||
else:
|
||||
return coro_name
|
||||
|
||||
coro_frame = None
|
||||
if hasattr(coro, 'gi_frame') and coro.gi_frame:
|
||||
coro_name = None
|
||||
if isinstance(coro, CoroWrapper):
|
||||
func = coro.func
|
||||
coro_name = coro.__qualname__
|
||||
if coro_name is not None:
|
||||
coro_name = '{}()'.format(coro_name)
|
||||
else:
|
||||
func = coro
|
||||
|
||||
if coro_name is None:
|
||||
coro_name = events._format_callback(func, (), {})
|
||||
|
||||
try:
|
||||
coro_code = coro.gi_code
|
||||
except AttributeError:
|
||||
coro_code = coro.cr_code
|
||||
|
||||
try:
|
||||
coro_frame = coro.gi_frame
|
||||
elif hasattr(coro, 'cr_frame') and coro.cr_frame:
|
||||
except AttributeError:
|
||||
coro_frame = coro.cr_frame
|
||||
|
||||
# If Cython's coroutine has a fake code object without proper
|
||||
# co_filename -- expose that.
|
||||
filename = coro_code.co_filename or '<empty co_filename>'
|
||||
|
||||
filename = coro_code.co_filename
|
||||
lineno = 0
|
||||
|
||||
if coro_frame is not None:
|
||||
if (isinstance(coro, CoroWrapper) and
|
||||
not inspect.isgeneratorfunction(coro.func) and
|
||||
coro.func is not None):
|
||||
source = events._get_function_source(coro.func)
|
||||
if source is not None:
|
||||
filename, lineno = source
|
||||
if coro_frame is None:
|
||||
coro_repr = ('%s done, defined at %s:%s'
|
||||
% (coro_name, filename, lineno))
|
||||
else:
|
||||
coro_repr = ('%s running, defined at %s:%s'
|
||||
% (coro_name, filename, lineno))
|
||||
elif coro_frame is not None:
|
||||
lineno = coro_frame.f_lineno
|
||||
coro_repr = f'{coro_name} running at {filename}:{lineno}'
|
||||
|
||||
coro_repr = ('%s running at %s:%s'
|
||||
% (coro_name, filename, lineno))
|
||||
else:
|
||||
lineno = coro_code.co_firstlineno
|
||||
coro_repr = f'{coro_name} done, defined at {filename}:{lineno}'
|
||||
coro_repr = ('%s done, defined at %s:%s'
|
||||
% (coro_name, filename, lineno))
|
||||
|
||||
return coro_repr
|
||||
|
||||
@@ -1,50 +1,96 @@
|
||||
"""Event loop and event loop policy."""
|
||||
|
||||
# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0
|
||||
# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0)
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io
|
||||
__all__ = ['AbstractEventLoopPolicy',
|
||||
'AbstractEventLoop', 'AbstractServer',
|
||||
'Handle', 'TimerHandle',
|
||||
'get_event_loop_policy', 'set_event_loop_policy',
|
||||
'get_event_loop', 'set_event_loop', 'new_event_loop',
|
||||
'get_child_watcher', 'set_child_watcher',
|
||||
'_set_running_loop', 'get_running_loop',
|
||||
'_get_running_loop',
|
||||
]
|
||||
|
||||
__all__ = (
|
||||
'AbstractEventLoopPolicy',
|
||||
'AbstractEventLoop', 'AbstractServer',
|
||||
'Handle', 'TimerHandle',
|
||||
'get_event_loop_policy', 'set_event_loop_policy',
|
||||
'get_event_loop', 'set_event_loop', 'new_event_loop',
|
||||
'get_child_watcher', 'set_child_watcher',
|
||||
'_set_running_loop', 'get_running_loop',
|
||||
'_get_running_loop',
|
||||
)
|
||||
|
||||
import contextvars
|
||||
import os
|
||||
import signal
|
||||
import functools
|
||||
import inspect
|
||||
import reprlib
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
from . import format_helpers
|
||||
from asyncio import compat
|
||||
|
||||
|
||||
def _get_function_source(func):
|
||||
if compat.PY34:
|
||||
func = inspect.unwrap(func)
|
||||
elif hasattr(func, '__wrapped__'):
|
||||
func = func.__wrapped__
|
||||
if inspect.isfunction(func):
|
||||
code = func.__code__
|
||||
return (code.co_filename, code.co_firstlineno)
|
||||
if isinstance(func, functools.partial):
|
||||
return _get_function_source(func.func)
|
||||
if compat.PY34 and isinstance(func, functools.partialmethod):
|
||||
return _get_function_source(func.func)
|
||||
return None
|
||||
|
||||
|
||||
def _format_args_and_kwargs(args, kwargs):
|
||||
"""Format function arguments and keyword arguments.
|
||||
|
||||
Special case for a single parameter: ('hello',) is formatted as ('hello').
|
||||
"""
|
||||
# use reprlib to limit the length of the output
|
||||
items = []
|
||||
if args:
|
||||
items.extend(reprlib.repr(arg) for arg in args)
|
||||
if kwargs:
|
||||
items.extend('{}={}'.format(k, reprlib.repr(v))
|
||||
for k, v in kwargs.items())
|
||||
return '(' + ', '.join(items) + ')'
|
||||
|
||||
|
||||
def _format_callback(func, args, kwargs, suffix=''):
|
||||
if isinstance(func, functools.partial):
|
||||
suffix = _format_args_and_kwargs(args, kwargs) + suffix
|
||||
return _format_callback(func.func, func.args, func.keywords, suffix)
|
||||
|
||||
if hasattr(func, '__qualname__'):
|
||||
func_repr = getattr(func, '__qualname__')
|
||||
elif hasattr(func, '__name__'):
|
||||
func_repr = getattr(func, '__name__')
|
||||
else:
|
||||
func_repr = repr(func)
|
||||
|
||||
func_repr += _format_args_and_kwargs(args, kwargs)
|
||||
if suffix:
|
||||
func_repr += suffix
|
||||
return func_repr
|
||||
|
||||
def _format_callback_source(func, args):
|
||||
func_repr = _format_callback(func, args, None)
|
||||
source = _get_function_source(func)
|
||||
if source:
|
||||
func_repr += ' at %s:%s' % source
|
||||
return func_repr
|
||||
|
||||
|
||||
class Handle:
|
||||
"""Object returned by callback registration methods."""
|
||||
|
||||
__slots__ = ('_callback', '_args', '_cancelled', '_loop',
|
||||
'_source_traceback', '_repr', '__weakref__',
|
||||
'_context')
|
||||
'_source_traceback', '_repr', '__weakref__')
|
||||
|
||||
def __init__(self, callback, args, loop, context=None):
|
||||
if context is None:
|
||||
context = contextvars.copy_context()
|
||||
self._context = context
|
||||
def __init__(self, callback, args, loop):
|
||||
self._loop = loop
|
||||
self._callback = callback
|
||||
self._args = args
|
||||
self._cancelled = False
|
||||
self._repr = None
|
||||
if self._loop.get_debug():
|
||||
self._source_traceback = format_helpers.extract_stack(
|
||||
sys._getframe(1))
|
||||
self._source_traceback = traceback.extract_stack(sys._getframe(1))
|
||||
else:
|
||||
self._source_traceback = None
|
||||
|
||||
@@ -53,21 +99,17 @@ class Handle:
|
||||
if self._cancelled:
|
||||
info.append('cancelled')
|
||||
if self._callback is not None:
|
||||
info.append(format_helpers._format_callback_source(
|
||||
self._callback, self._args))
|
||||
info.append(_format_callback_source(self._callback, self._args))
|
||||
if self._source_traceback:
|
||||
frame = self._source_traceback[-1]
|
||||
info.append(f'created at {frame[0]}:{frame[1]}')
|
||||
info.append('created at %s:%s' % (frame[0], frame[1]))
|
||||
return info
|
||||
|
||||
def __repr__(self):
|
||||
if self._repr is not None:
|
||||
return self._repr
|
||||
info = self._repr_info()
|
||||
return '<{}>'.format(' '.join(info))
|
||||
|
||||
def get_context(self):
|
||||
return self._context
|
||||
return '<%s>' % ' '.join(info)
|
||||
|
||||
def cancel(self):
|
||||
if not self._cancelled:
|
||||
@@ -80,18 +122,12 @@ class Handle:
|
||||
self._callback = None
|
||||
self._args = None
|
||||
|
||||
def cancelled(self):
|
||||
return self._cancelled
|
||||
|
||||
def _run(self):
|
||||
try:
|
||||
self._context.run(self._callback, *self._args)
|
||||
except (SystemExit, KeyboardInterrupt):
|
||||
raise
|
||||
except BaseException as exc:
|
||||
cb = format_helpers._format_callback_source(
|
||||
self._callback, self._args)
|
||||
msg = f'Exception in callback {cb}'
|
||||
self._callback(*self._args)
|
||||
except Exception as exc:
|
||||
cb = _format_callback_source(self._callback, self._args)
|
||||
msg = 'Exception in callback {}'.format(cb)
|
||||
context = {
|
||||
'message': msg,
|
||||
'exception': exc,
|
||||
@@ -108,8 +144,9 @@ class TimerHandle(Handle):
|
||||
|
||||
__slots__ = ['_scheduled', '_when']
|
||||
|
||||
def __init__(self, when, callback, args, loop, context=None):
|
||||
super().__init__(callback, args, loop, context)
|
||||
def __init__(self, when, callback, args, loop):
|
||||
assert when is not None
|
||||
super().__init__(callback, args, loop)
|
||||
if self._source_traceback:
|
||||
del self._source_traceback[-1]
|
||||
self._when = when
|
||||
@@ -118,31 +155,27 @@ class TimerHandle(Handle):
|
||||
def _repr_info(self):
|
||||
info = super()._repr_info()
|
||||
pos = 2 if self._cancelled else 1
|
||||
info.insert(pos, f'when={self._when}')
|
||||
info.insert(pos, 'when=%s' % self._when)
|
||||
return info
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self._when)
|
||||
|
||||
def __lt__(self, other):
|
||||
if isinstance(other, TimerHandle):
|
||||
return self._when < other._when
|
||||
return NotImplemented
|
||||
return self._when < other._when
|
||||
|
||||
def __le__(self, other):
|
||||
if isinstance(other, TimerHandle):
|
||||
return self._when < other._when or self.__eq__(other)
|
||||
return NotImplemented
|
||||
if self._when < other._when:
|
||||
return True
|
||||
return self.__eq__(other)
|
||||
|
||||
def __gt__(self, other):
|
||||
if isinstance(other, TimerHandle):
|
||||
return self._when > other._when
|
||||
return NotImplemented
|
||||
return self._when > other._when
|
||||
|
||||
def __ge__(self, other):
|
||||
if isinstance(other, TimerHandle):
|
||||
return self._when > other._when or self.__eq__(other)
|
||||
return NotImplemented
|
||||
if self._when > other._when:
|
||||
return True
|
||||
return self.__eq__(other)
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, TimerHandle):
|
||||
@@ -152,60 +185,26 @@ class TimerHandle(Handle):
|
||||
self._cancelled == other._cancelled)
|
||||
return NotImplemented
|
||||
|
||||
def __ne__(self, other):
|
||||
equal = self.__eq__(other)
|
||||
return NotImplemented if equal is NotImplemented else not equal
|
||||
|
||||
def cancel(self):
|
||||
if not self._cancelled:
|
||||
self._loop._timer_handle_cancelled(self)
|
||||
super().cancel()
|
||||
|
||||
def when(self):
|
||||
"""Return a scheduled callback time.
|
||||
|
||||
The time is an absolute timestamp, using the same time
|
||||
reference as loop.time().
|
||||
"""
|
||||
return self._when
|
||||
|
||||
|
||||
class AbstractServer:
|
||||
"""Abstract server returned by create_server()."""
|
||||
|
||||
def close(self):
|
||||
"""Stop serving. This leaves existing connections open."""
|
||||
raise NotImplementedError
|
||||
return NotImplemented
|
||||
|
||||
def get_loop(self):
|
||||
"""Get the event loop the Server object is attached to."""
|
||||
raise NotImplementedError
|
||||
|
||||
def is_serving(self):
|
||||
"""Return True if the server is accepting connections."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def start_serving(self):
|
||||
"""Start accepting connections.
|
||||
|
||||
This method is idempotent, so it can be called when
|
||||
the server is already being serving.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def serve_forever(self):
|
||||
"""Start accepting connections until the coroutine is cancelled.
|
||||
|
||||
The server is closed when the coroutine is cancelled.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def wait_closed(self):
|
||||
def wait_closed(self):
|
||||
"""Coroutine to wait until service is closed."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
self.close()
|
||||
await self.wait_closed()
|
||||
return NotImplemented
|
||||
|
||||
|
||||
class AbstractEventLoop:
|
||||
@@ -251,27 +250,23 @@ class AbstractEventLoop:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def shutdown_asyncgens(self):
|
||||
def shutdown_asyncgens(self):
|
||||
"""Shutdown all active asynchronous generators."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def shutdown_default_executor(self):
|
||||
"""Schedule the shutdown of the default executor."""
|
||||
raise NotImplementedError
|
||||
|
||||
# Methods scheduling callbacks. All these return Handles.
|
||||
|
||||
def _timer_handle_cancelled(self, handle):
|
||||
"""Notification that a TimerHandle has been cancelled."""
|
||||
raise NotImplementedError
|
||||
|
||||
def call_soon(self, callback, *args, context=None):
|
||||
return self.call_later(0, callback, *args, context=context)
|
||||
def call_soon(self, callback, *args):
|
||||
return self.call_later(0, callback, *args)
|
||||
|
||||
def call_later(self, delay, callback, *args, context=None):
|
||||
def call_later(self, delay, callback, *args):
|
||||
raise NotImplementedError
|
||||
|
||||
def call_at(self, when, callback, *args, context=None):
|
||||
def call_at(self, when, callback, *args):
|
||||
raise NotImplementedError
|
||||
|
||||
def time(self):
|
||||
@@ -282,12 +277,12 @@ class AbstractEventLoop:
|
||||
|
||||
# Method scheduling a coroutine object: create a task.
|
||||
|
||||
def create_task(self, coro, *, name=None, context=None):
|
||||
def create_task(self, coro):
|
||||
raise NotImplementedError
|
||||
|
||||
# Methods for interacting with threads.
|
||||
|
||||
def call_soon_threadsafe(self, callback, *args, context=None):
|
||||
def call_soon_threadsafe(self, callback, *args):
|
||||
raise NotImplementedError
|
||||
|
||||
def run_in_executor(self, executor, func, *args):
|
||||
@@ -298,31 +293,21 @@ class AbstractEventLoop:
|
||||
|
||||
# Network I/O methods returning Futures.
|
||||
|
||||
async def getaddrinfo(self, host, port, *,
|
||||
family=0, type=0, proto=0, flags=0):
|
||||
def getaddrinfo(self, host, port, *, family=0, type=0, proto=0, flags=0):
|
||||
raise NotImplementedError
|
||||
|
||||
async def getnameinfo(self, sockaddr, flags=0):
|
||||
def getnameinfo(self, sockaddr, flags=0):
|
||||
raise NotImplementedError
|
||||
|
||||
async def create_connection(
|
||||
self, protocol_factory, host=None, port=None,
|
||||
*, ssl=None, family=0, proto=0,
|
||||
flags=0, sock=None, local_addr=None,
|
||||
server_hostname=None,
|
||||
ssl_handshake_timeout=None,
|
||||
ssl_shutdown_timeout=None,
|
||||
happy_eyeballs_delay=None, interleave=None):
|
||||
def create_connection(self, protocol_factory, host=None, port=None, *,
|
||||
ssl=None, family=0, proto=0, flags=0, sock=None,
|
||||
local_addr=None, server_hostname=None):
|
||||
raise NotImplementedError
|
||||
|
||||
async def create_server(
|
||||
self, protocol_factory, host=None, port=None,
|
||||
*, family=socket.AF_UNSPEC,
|
||||
flags=socket.AI_PASSIVE, sock=None, backlog=100,
|
||||
ssl=None, reuse_address=None, reuse_port=None,
|
||||
ssl_handshake_timeout=None,
|
||||
ssl_shutdown_timeout=None,
|
||||
start_serving=True):
|
||||
def create_server(self, protocol_factory, host=None, port=None, *,
|
||||
family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE,
|
||||
sock=None, backlog=100, ssl=None, reuse_address=None,
|
||||
reuse_port=None):
|
||||
"""A coroutine which creates a TCP server bound to host and port.
|
||||
|
||||
The return value is a Server object which can be used to stop
|
||||
@@ -330,8 +315,8 @@ class AbstractEventLoop:
|
||||
|
||||
If host is an empty string or None all interfaces are assumed
|
||||
and a list of multiple sockets will be returned (most likely
|
||||
one for IPv4 and another one for IPv6). The host parameter can also be
|
||||
a sequence (e.g. list) of hosts to bind to.
|
||||
one for IPv4 and another one for IPv6). The host parameter can also be a
|
||||
sequence (e.g. list) of hosts to bind to.
|
||||
|
||||
family can be set to either AF_INET or AF_INET6 to force the
|
||||
socket to use IPv4 or IPv6. If not set it will be determined
|
||||
@@ -357,62 +342,22 @@ class AbstractEventLoop:
|
||||
the same port as other existing endpoints are bound to, so long as
|
||||
they all set this flag when being created. This option is not
|
||||
supported on Windows.
|
||||
|
||||
ssl_handshake_timeout is the time in seconds that an SSL server
|
||||
will wait for completion of the SSL handshake before aborting the
|
||||
connection. Default is 60s.
|
||||
|
||||
ssl_shutdown_timeout is the time in seconds that an SSL server
|
||||
will wait for completion of the SSL shutdown procedure
|
||||
before aborting the connection. Default is 30s.
|
||||
|
||||
start_serving set to True (default) causes the created server
|
||||
to start accepting connections immediately. When set to False,
|
||||
the user should await Server.start_serving() or Server.serve_forever()
|
||||
to make the server to start accepting connections.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def sendfile(self, transport, file, offset=0, count=None,
|
||||
*, fallback=True):
|
||||
"""Send a file through a transport.
|
||||
|
||||
Return an amount of sent bytes.
|
||||
"""
|
||||
def create_unix_connection(self, protocol_factory, path, *,
|
||||
ssl=None, sock=None,
|
||||
server_hostname=None):
|
||||
raise NotImplementedError
|
||||
|
||||
async def start_tls(self, transport, protocol, sslcontext, *,
|
||||
server_side=False,
|
||||
server_hostname=None,
|
||||
ssl_handshake_timeout=None,
|
||||
ssl_shutdown_timeout=None):
|
||||
"""Upgrade a transport to TLS.
|
||||
|
||||
Return a new transport that *protocol* should start using
|
||||
immediately.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def create_unix_connection(
|
||||
self, protocol_factory, path=None, *,
|
||||
ssl=None, sock=None,
|
||||
server_hostname=None,
|
||||
ssl_handshake_timeout=None,
|
||||
ssl_shutdown_timeout=None):
|
||||
raise NotImplementedError
|
||||
|
||||
async def create_unix_server(
|
||||
self, protocol_factory, path=None, *,
|
||||
sock=None, backlog=100, ssl=None,
|
||||
ssl_handshake_timeout=None,
|
||||
ssl_shutdown_timeout=None,
|
||||
start_serving=True):
|
||||
def create_unix_server(self, protocol_factory, path, *,
|
||||
sock=None, backlog=100, ssl=None):
|
||||
"""A coroutine which creates a UNIX Domain Socket server.
|
||||
|
||||
The return value is a Server object, which can be used to stop
|
||||
the service.
|
||||
|
||||
path is a str, representing a file system path to bind the
|
||||
path is a str, representing a file systsem path to bind the
|
||||
server socket to.
|
||||
|
||||
sock can optionally be specified in order to use a preexisting
|
||||
@@ -423,40 +368,14 @@ class AbstractEventLoop:
|
||||
|
||||
ssl can be set to an SSLContext to enable SSL over the
|
||||
accepted connections.
|
||||
|
||||
ssl_handshake_timeout is the time in seconds that an SSL server
|
||||
will wait for the SSL handshake to complete (defaults to 60s).
|
||||
|
||||
ssl_shutdown_timeout is the time in seconds that an SSL server
|
||||
will wait for the SSL shutdown to finish (defaults to 30s).
|
||||
|
||||
start_serving set to True (default) causes the created server
|
||||
to start accepting connections immediately. When set to False,
|
||||
the user should await Server.start_serving() or Server.serve_forever()
|
||||
to make the server to start accepting connections.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def connect_accepted_socket(
|
||||
self, protocol_factory, sock,
|
||||
*, ssl=None,
|
||||
ssl_handshake_timeout=None,
|
||||
ssl_shutdown_timeout=None):
|
||||
"""Handle an accepted connection.
|
||||
|
||||
This is used by servers that accept connections outside of
|
||||
asyncio, but use asyncio to handle connections.
|
||||
|
||||
This method is a coroutine. When completed, the coroutine
|
||||
returns a (transport, protocol) pair.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def create_datagram_endpoint(self, protocol_factory,
|
||||
local_addr=None, remote_addr=None, *,
|
||||
family=0, proto=0, flags=0,
|
||||
reuse_address=None, reuse_port=None,
|
||||
allow_broadcast=None, sock=None):
|
||||
def create_datagram_endpoint(self, protocol_factory,
|
||||
local_addr=None, remote_addr=None, *,
|
||||
family=0, proto=0, flags=0,
|
||||
reuse_address=None, reuse_port=None,
|
||||
allow_broadcast=None, sock=None):
|
||||
"""A coroutine which creates a datagram endpoint.
|
||||
|
||||
This method will try to establish the endpoint in the background.
|
||||
@@ -464,8 +383,8 @@ class AbstractEventLoop:
|
||||
|
||||
protocol_factory must be a callable returning a protocol instance.
|
||||
|
||||
socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending on
|
||||
host (or family if specified), socket type SOCK_DGRAM.
|
||||
socket family AF_INET or socket.AF_INET6 depending on host (or
|
||||
family if specified), socket type SOCK_DGRAM.
|
||||
|
||||
reuse_address tells the kernel to reuse a local socket in
|
||||
TIME_WAIT state, without waiting for its natural timeout to
|
||||
@@ -489,7 +408,7 @@ class AbstractEventLoop:
|
||||
|
||||
# Pipes and subprocesses.
|
||||
|
||||
async def connect_read_pipe(self, protocol_factory, pipe):
|
||||
def connect_read_pipe(self, protocol_factory, pipe):
|
||||
"""Register read pipe in event loop. Set the pipe to non-blocking mode.
|
||||
|
||||
protocol_factory should instantiate object with Protocol interface.
|
||||
@@ -499,10 +418,10 @@ class AbstractEventLoop:
|
||||
# The reason to accept file-like object instead of just file descriptor
|
||||
# is: we need to own pipe and close it at transport finishing
|
||||
# Can got complicated errors if pass f.fileno(),
|
||||
# close fd in pipe transport then close f and vice versa.
|
||||
# close fd in pipe transport then close f and vise versa.
|
||||
raise NotImplementedError
|
||||
|
||||
async def connect_write_pipe(self, protocol_factory, pipe):
|
||||
def connect_write_pipe(self, protocol_factory, pipe):
|
||||
"""Register write pipe in event loop.
|
||||
|
||||
protocol_factory should instantiate object with BaseProtocol interface.
|
||||
@@ -512,21 +431,17 @@ class AbstractEventLoop:
|
||||
# The reason to accept file-like object instead of just file descriptor
|
||||
# is: we need to own pipe and close it at transport finishing
|
||||
# Can got complicated errors if pass f.fileno(),
|
||||
# close fd in pipe transport then close f and vice versa.
|
||||
# close fd in pipe transport then close f and vise versa.
|
||||
raise NotImplementedError
|
||||
|
||||
async def subprocess_shell(self, protocol_factory, cmd, *,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
**kwargs):
|
||||
def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
**kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
async def subprocess_exec(self, protocol_factory, *args,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
**kwargs):
|
||||
def subprocess_exec(self, protocol_factory, *args, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
**kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
# Ready-based callback registration methods.
|
||||
@@ -548,32 +463,16 @@ class AbstractEventLoop:
|
||||
|
||||
# Completion based I/O methods returning Futures.
|
||||
|
||||
async def sock_recv(self, sock, nbytes):
|
||||
def sock_recv(self, sock, nbytes):
|
||||
raise NotImplementedError
|
||||
|
||||
async def sock_recv_into(self, sock, buf):
|
||||
def sock_sendall(self, sock, data):
|
||||
raise NotImplementedError
|
||||
|
||||
async def sock_recvfrom(self, sock, bufsize):
|
||||
def sock_connect(self, sock, address):
|
||||
raise NotImplementedError
|
||||
|
||||
async def sock_recvfrom_into(self, sock, buf, nbytes=0):
|
||||
raise NotImplementedError
|
||||
|
||||
async def sock_sendall(self, sock, data):
|
||||
raise NotImplementedError
|
||||
|
||||
async def sock_sendto(self, sock, data, address):
|
||||
raise NotImplementedError
|
||||
|
||||
async def sock_connect(self, sock, address):
|
||||
raise NotImplementedError
|
||||
|
||||
async def sock_accept(self, sock):
|
||||
raise NotImplementedError
|
||||
|
||||
async def sock_sendfile(self, sock, file, offset=0, count=None,
|
||||
*, fallback=None):
|
||||
def sock_accept(self, sock):
|
||||
raise NotImplementedError
|
||||
|
||||
# Signal handling.
|
||||
@@ -621,7 +520,7 @@ class AbstractEventLoopPolicy:
|
||||
def get_event_loop(self):
|
||||
"""Get the event loop for the current context.
|
||||
|
||||
Returns an event loop object implementing the AbstractEventLoop interface,
|
||||
Returns an event loop object implementing the BaseEventLoop interface,
|
||||
or raises an exception in case no event loop has been set for the
|
||||
current context and the current policy does not specify to create one.
|
||||
|
||||
@@ -672,43 +571,23 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy):
|
||||
self._local = self._Local()
|
||||
|
||||
def get_event_loop(self):
|
||||
"""Get the event loop for the current context.
|
||||
"""Get the event loop.
|
||||
|
||||
Returns an instance of EventLoop or raises an exception.
|
||||
This may be None or an instance of EventLoop.
|
||||
"""
|
||||
if (self._local._loop is None and
|
||||
not self._local._set_called and
|
||||
threading.current_thread() is threading.main_thread()):
|
||||
stacklevel = 2
|
||||
try:
|
||||
f = sys._getframe(1)
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
# Move up the call stack so that the warning is attached
|
||||
# to the line outside asyncio itself.
|
||||
while f:
|
||||
module = f.f_globals.get('__name__')
|
||||
if not (module == 'asyncio' or module.startswith('asyncio.')):
|
||||
break
|
||||
f = f.f_back
|
||||
stacklevel += 1
|
||||
import warnings
|
||||
warnings.warn('There is no current event loop',
|
||||
DeprecationWarning, stacklevel=stacklevel)
|
||||
not self._local._set_called and
|
||||
isinstance(threading.current_thread(), threading._MainThread)):
|
||||
self.set_event_loop(self.new_event_loop())
|
||||
|
||||
if self._local._loop is None:
|
||||
raise RuntimeError('There is no current event loop in thread %r.'
|
||||
% threading.current_thread().name)
|
||||
|
||||
return self._local._loop
|
||||
|
||||
def set_event_loop(self, loop):
|
||||
"""Set the event loop."""
|
||||
self._local._set_called = True
|
||||
if loop is not None and not isinstance(loop, AbstractEventLoop):
|
||||
raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'")
|
||||
assert loop is None or isinstance(loop, AbstractEventLoop)
|
||||
self._local._loop = loop
|
||||
|
||||
def new_event_loop(self):
|
||||
@@ -732,9 +611,7 @@ _lock = threading.Lock()
|
||||
|
||||
# A TLS for the running event loop, used by _get_running_loop.
|
||||
class _RunningLoop(threading.local):
|
||||
loop_pid = (None, None)
|
||||
|
||||
|
||||
_loop = None
|
||||
_running_loop = _RunningLoop()
|
||||
|
||||
|
||||
@@ -756,10 +633,7 @@ def _get_running_loop():
|
||||
This is a low-level function intended to be used by event loops.
|
||||
This function is thread-specific.
|
||||
"""
|
||||
# NOTE: this function is implemented in C (see _asynciomodule.c)
|
||||
running_loop, pid = _running_loop.loop_pid
|
||||
if running_loop is not None and pid == os.getpid():
|
||||
return running_loop
|
||||
return _running_loop._loop
|
||||
|
||||
|
||||
def _set_running_loop(loop):
|
||||
@@ -768,8 +642,7 @@ def _set_running_loop(loop):
|
||||
This is a low-level function intended to be used by event loops.
|
||||
This function is thread-specific.
|
||||
"""
|
||||
# NOTE: this function is implemented in C (see _asynciomodule.c)
|
||||
_running_loop.loop_pid = (loop, os.getpid())
|
||||
_running_loop._loop = loop
|
||||
|
||||
|
||||
def _init_event_loop_policy():
|
||||
@@ -792,8 +665,7 @@ def set_event_loop_policy(policy):
|
||||
|
||||
If policy is None, the default policy is restored."""
|
||||
global _event_loop_policy
|
||||
if policy is not None and not isinstance(policy, AbstractEventLoopPolicy):
|
||||
raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'")
|
||||
assert policy is None or isinstance(policy, AbstractEventLoopPolicy)
|
||||
_event_loop_policy = policy
|
||||
|
||||
|
||||
@@ -806,7 +678,6 @@ def get_event_loop():
|
||||
If there is no running event loop set, the function will return
|
||||
the result of `get_event_loop_policy().get_event_loop()` call.
|
||||
"""
|
||||
# NOTE: this function is implemented in C (see _asynciomodule.c)
|
||||
current_loop = _get_running_loop()
|
||||
if current_loop is not None:
|
||||
return current_loop
|
||||
@@ -832,37 +703,3 @@ def set_child_watcher(watcher):
|
||||
"""Equivalent to calling
|
||||
get_event_loop_policy().set_child_watcher(watcher)."""
|
||||
return get_event_loop_policy().set_child_watcher(watcher)
|
||||
|
||||
|
||||
# Alias pure-Python implementations for testing purposes.
|
||||
_py__get_running_loop = _get_running_loop
|
||||
_py__set_running_loop = _set_running_loop
|
||||
_py_get_running_loop = get_running_loop
|
||||
_py_get_event_loop = get_event_loop
|
||||
|
||||
|
||||
try:
|
||||
# get_event_loop() is one of the most frequently called
|
||||
# functions in asyncio. Pure Python implementation is
|
||||
# about 4 times slower than C-accelerated.
|
||||
from _asyncio import (_get_running_loop, _set_running_loop,
|
||||
get_running_loop, get_event_loop)
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
# Alias C implementations for testing purposes.
|
||||
_c__get_running_loop = _get_running_loop
|
||||
_c__set_running_loop = _set_running_loop
|
||||
_c_get_running_loop = get_running_loop
|
||||
_c_get_event_loop = get_event_loop
|
||||
|
||||
|
||||
if hasattr(os, 'fork'):
|
||||
def on_fork():
|
||||
# Reset the loop and wakeupfd in the forked child process.
|
||||
if _event_loop_policy is not None:
|
||||
_event_loop_policy._local = BaseDefaultEventLoopPolicy._Local()
|
||||
_set_running_loop(None)
|
||||
signal.set_wakeup_fd(-1)
|
||||
|
||||
os.register_at_fork(after_in_child=on_fork)
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
"""asyncio exceptions."""
|
||||
|
||||
|
||||
__all__ = ('BrokenBarrierError',
|
||||
'CancelledError', 'InvalidStateError', 'TimeoutError',
|
||||
'IncompleteReadError', 'LimitOverrunError',
|
||||
'SendfileNotAvailableError')
|
||||
|
||||
|
||||
class CancelledError(BaseException):
|
||||
"""The Future or Task was cancelled."""
|
||||
|
||||
|
||||
TimeoutError = TimeoutError # make local alias for the standard exception
|
||||
|
||||
|
||||
class InvalidStateError(Exception):
|
||||
"""The operation is not allowed in this state."""
|
||||
|
||||
|
||||
class SendfileNotAvailableError(RuntimeError):
|
||||
"""Sendfile syscall is not available.
|
||||
|
||||
Raised if OS does not support sendfile syscall for given socket or
|
||||
file type.
|
||||
"""
|
||||
|
||||
|
||||
class IncompleteReadError(EOFError):
|
||||
"""
|
||||
Incomplete read error. Attributes:
|
||||
|
||||
- partial: read bytes string before the end of stream was reached
|
||||
- expected: total number of expected bytes (or None if unknown)
|
||||
"""
|
||||
def __init__(self, partial, expected):
|
||||
r_expected = 'undefined' if expected is None else repr(expected)
|
||||
super().__init__(f'{len(partial)} bytes read on a total of '
|
||||
f'{r_expected} expected bytes')
|
||||
self.partial = partial
|
||||
self.expected = expected
|
||||
|
||||
def __reduce__(self):
|
||||
return type(self), (self.partial, self.expected)
|
||||
|
||||
|
||||
class LimitOverrunError(Exception):
|
||||
"""Reached the buffer limit while looking for a separator.
|
||||
|
||||
Attributes:
|
||||
- consumed: total number of to be consumed bytes.
|
||||
"""
|
||||
def __init__(self, message, consumed):
|
||||
super().__init__(message)
|
||||
self.consumed = consumed
|
||||
|
||||
def __reduce__(self):
|
||||
return type(self), (self.args[0], self.consumed)
|
||||
|
||||
|
||||
class BrokenBarrierError(RuntimeError):
|
||||
"""Barrier is broken by barrier.abort() call."""
|
||||
@@ -1,76 +0,0 @@
|
||||
import functools
|
||||
import inspect
|
||||
import reprlib
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from . import constants
|
||||
|
||||
|
||||
def _get_function_source(func):
|
||||
func = inspect.unwrap(func)
|
||||
if inspect.isfunction(func):
|
||||
code = func.__code__
|
||||
return (code.co_filename, code.co_firstlineno)
|
||||
if isinstance(func, functools.partial):
|
||||
return _get_function_source(func.func)
|
||||
if isinstance(func, functools.partialmethod):
|
||||
return _get_function_source(func.func)
|
||||
return None
|
||||
|
||||
|
||||
def _format_callback_source(func, args):
|
||||
func_repr = _format_callback(func, args, None)
|
||||
source = _get_function_source(func)
|
||||
if source:
|
||||
func_repr += f' at {source[0]}:{source[1]}'
|
||||
return func_repr
|
||||
|
||||
|
||||
def _format_args_and_kwargs(args, kwargs):
|
||||
"""Format function arguments and keyword arguments.
|
||||
|
||||
Special case for a single parameter: ('hello',) is formatted as ('hello').
|
||||
"""
|
||||
# use reprlib to limit the length of the output
|
||||
items = []
|
||||
if args:
|
||||
items.extend(reprlib.repr(arg) for arg in args)
|
||||
if kwargs:
|
||||
items.extend(f'{k}={reprlib.repr(v)}' for k, v in kwargs.items())
|
||||
return '({})'.format(', '.join(items))
|
||||
|
||||
|
||||
def _format_callback(func, args, kwargs, suffix=''):
|
||||
if isinstance(func, functools.partial):
|
||||
suffix = _format_args_and_kwargs(args, kwargs) + suffix
|
||||
return _format_callback(func.func, func.args, func.keywords, suffix)
|
||||
|
||||
if hasattr(func, '__qualname__') and func.__qualname__:
|
||||
func_repr = func.__qualname__
|
||||
elif hasattr(func, '__name__') and func.__name__:
|
||||
func_repr = func.__name__
|
||||
else:
|
||||
func_repr = repr(func)
|
||||
|
||||
func_repr += _format_args_and_kwargs(args, kwargs)
|
||||
if suffix:
|
||||
func_repr += suffix
|
||||
return func_repr
|
||||
|
||||
|
||||
def extract_stack(f=None, limit=None):
|
||||
"""Replacement for traceback.extract_stack() that only does the
|
||||
necessary work for asyncio debug mode.
|
||||
"""
|
||||
if f is None:
|
||||
f = sys._getframe().f_back
|
||||
if limit is None:
|
||||
# Limit the amount of work to a reasonable amount, as extract_stack()
|
||||
# can be called for each coroutine and future in debug mode.
|
||||
limit = constants.DEBUG_STACK_DEPTH
|
||||
stack = traceback.StackSummary.extract(traceback.walk_stack(f),
|
||||
limit=limit,
|
||||
lookup_lines=False)
|
||||
stack.reverse()
|
||||
return stack
|
||||
@@ -1,21 +1,21 @@
|
||||
"""A Future class similar to the one in PEP 3148."""
|
||||
|
||||
__all__ = (
|
||||
'Future', 'wrap_future', 'isfuture',
|
||||
)
|
||||
__all__ = ['CancelledError', 'TimeoutError', 'InvalidStateError',
|
||||
'Future', 'wrap_future', 'isfuture']
|
||||
|
||||
import concurrent.futures
|
||||
import contextvars
|
||||
import logging
|
||||
import sys
|
||||
from types import GenericAlias
|
||||
import traceback
|
||||
|
||||
from . import base_futures
|
||||
from . import compat
|
||||
from . import events
|
||||
from . import exceptions
|
||||
from . import format_helpers
|
||||
|
||||
|
||||
CancelledError = base_futures.CancelledError
|
||||
InvalidStateError = base_futures.InvalidStateError
|
||||
TimeoutError = base_futures.TimeoutError
|
||||
isfuture = base_futures.isfuture
|
||||
|
||||
|
||||
@@ -27,18 +27,96 @@ _FINISHED = base_futures._FINISHED
|
||||
STACK_DEBUG = logging.DEBUG - 1 # heavy-duty debugging
|
||||
|
||||
|
||||
class _TracebackLogger:
|
||||
"""Helper to log a traceback upon destruction if not cleared.
|
||||
|
||||
This solves a nasty problem with Futures and Tasks that have an
|
||||
exception set: if nobody asks for the exception, the exception is
|
||||
never logged. This violates the Zen of Python: 'Errors should
|
||||
never pass silently. Unless explicitly silenced.'
|
||||
|
||||
However, we don't want to log the exception as soon as
|
||||
set_exception() is called: if the calling code is written
|
||||
properly, it will get the exception and handle it properly. But
|
||||
we *do* want to log it if result() or exception() was never called
|
||||
-- otherwise developers waste a lot of time wondering why their
|
||||
buggy code fails silently.
|
||||
|
||||
An earlier attempt added a __del__() method to the Future class
|
||||
itself, but this backfired because the presence of __del__()
|
||||
prevents garbage collection from breaking cycles. A way out of
|
||||
this catch-22 is to avoid having a __del__() method on the Future
|
||||
class itself, but instead to have a reference to a helper object
|
||||
with a __del__() method that logs the traceback, where we ensure
|
||||
that the helper object doesn't participate in cycles, and only the
|
||||
Future has a reference to it.
|
||||
|
||||
The helper object is added when set_exception() is called. When
|
||||
the Future is collected, and the helper is present, the helper
|
||||
object is also collected, and its __del__() method will log the
|
||||
traceback. When the Future's result() or exception() method is
|
||||
called (and a helper object is present), it removes the helper
|
||||
object, after calling its clear() method to prevent it from
|
||||
logging.
|
||||
|
||||
One downside is that we do a fair amount of work to extract the
|
||||
traceback from the exception, even when it is never logged. It
|
||||
would seem cheaper to just store the exception object, but that
|
||||
references the traceback, which references stack frames, which may
|
||||
reference the Future, which references the _TracebackLogger, and
|
||||
then the _TracebackLogger would be included in a cycle, which is
|
||||
what we're trying to avoid! As an optimization, we don't
|
||||
immediately format the exception; we only do the work when
|
||||
activate() is called, which call is delayed until after all the
|
||||
Future's callbacks have run. Since usually a Future has at least
|
||||
one callback (typically set by 'yield from') and usually that
|
||||
callback extracts the callback, thereby removing the need to
|
||||
format the exception.
|
||||
|
||||
PS. I don't claim credit for this solution. I first heard of it
|
||||
in a discussion about closing files when they are collected.
|
||||
"""
|
||||
|
||||
__slots__ = ('loop', 'source_traceback', 'exc', 'tb')
|
||||
|
||||
def __init__(self, future, exc):
|
||||
self.loop = future._loop
|
||||
self.source_traceback = future._source_traceback
|
||||
self.exc = exc
|
||||
self.tb = None
|
||||
|
||||
def activate(self):
|
||||
exc = self.exc
|
||||
if exc is not None:
|
||||
self.exc = None
|
||||
self.tb = traceback.format_exception(exc.__class__, exc,
|
||||
exc.__traceback__)
|
||||
|
||||
def clear(self):
|
||||
self.exc = None
|
||||
self.tb = None
|
||||
|
||||
def __del__(self):
|
||||
if self.tb:
|
||||
msg = 'Future/Task exception was never retrieved\n'
|
||||
if self.source_traceback:
|
||||
src = ''.join(traceback.format_list(self.source_traceback))
|
||||
msg += 'Future/Task created at (most recent call last):\n'
|
||||
msg += '%s\n' % src.rstrip()
|
||||
msg += ''.join(self.tb).rstrip()
|
||||
self.loop.call_exception_handler({'message': msg})
|
||||
|
||||
|
||||
class Future:
|
||||
"""This class is *almost* compatible with concurrent.futures.Future.
|
||||
|
||||
Differences:
|
||||
|
||||
- This class is not thread-safe.
|
||||
|
||||
- result() and exception() do not take a timeout argument and
|
||||
raise an exception when the future isn't done yet.
|
||||
|
||||
- Callbacks registered with add_done_callback() are always called
|
||||
via the event loop's call_soon().
|
||||
via the event loop's call_soon_threadsafe().
|
||||
|
||||
- This class is not compatible with the wait() and as_completed()
|
||||
methods in the concurrent.futures package.
|
||||
@@ -52,9 +130,6 @@ class Future:
|
||||
_exception = None
|
||||
_loop = None
|
||||
_source_traceback = None
|
||||
_cancel_message = None
|
||||
# A saved CancelledError for later chaining as an exception context.
|
||||
_cancelled_exc = None
|
||||
|
||||
# This field is used for a dual purpose:
|
||||
# - Its presence is a marker to declare that a class implements
|
||||
@@ -62,12 +137,12 @@ class Future:
|
||||
# The value must also be not-None, to enable a subclass to declare
|
||||
# that it is not compatible by setting this to None.
|
||||
# - It is set by __iter__() below so that Task._step() can tell
|
||||
# the difference between
|
||||
# `await Future()` or`yield from Future()` (correct) vs.
|
||||
# the difference between `yield from Future()` (correct) vs.
|
||||
# `yield Future()` (incorrect).
|
||||
_asyncio_future_blocking = False
|
||||
|
||||
__log_traceback = False
|
||||
_log_traceback = False # Used for Python 3.4 and later
|
||||
_tb_logger = None # Used for Python 3.3 only
|
||||
|
||||
def __init__(self, *, loop=None):
|
||||
"""Initialize the future.
|
||||
@@ -82,83 +157,47 @@ class Future:
|
||||
self._loop = loop
|
||||
self._callbacks = []
|
||||
if self._loop.get_debug():
|
||||
self._source_traceback = format_helpers.extract_stack(
|
||||
sys._getframe(1))
|
||||
self._source_traceback = traceback.extract_stack(sys._getframe(1))
|
||||
|
||||
_repr_info = base_futures._future_repr_info
|
||||
|
||||
def __repr__(self):
|
||||
return base_futures._future_repr(self)
|
||||
return '<%s %s>' % (self.__class__.__name__, ' '.join(self._repr_info()))
|
||||
|
||||
def __del__(self):
|
||||
if not self.__log_traceback:
|
||||
# set_exception() was not called, or result() or exception()
|
||||
# has consumed the exception
|
||||
return
|
||||
exc = self._exception
|
||||
context = {
|
||||
'message':
|
||||
f'{self.__class__.__name__} exception was never retrieved',
|
||||
'exception': exc,
|
||||
'future': self,
|
||||
}
|
||||
if self._source_traceback:
|
||||
context['source_traceback'] = self._source_traceback
|
||||
self._loop.call_exception_handler(context)
|
||||
# On Python 3.3 and older, objects with a destructor part of a reference
|
||||
# cycle are never destroyed. It's not more the case on Python 3.4 thanks
|
||||
# to the PEP 442.
|
||||
if compat.PY34:
|
||||
def __del__(self):
|
||||
if not self._log_traceback:
|
||||
# set_exception() was not called, or result() or exception()
|
||||
# has consumed the exception
|
||||
return
|
||||
exc = self._exception
|
||||
context = {
|
||||
'message': ('%s exception was never retrieved'
|
||||
% self.__class__.__name__),
|
||||
'exception': exc,
|
||||
'future': self,
|
||||
}
|
||||
if self._source_traceback:
|
||||
context['source_traceback'] = self._source_traceback
|
||||
self._loop.call_exception_handler(context)
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
|
||||
@property
|
||||
def _log_traceback(self):
|
||||
return self.__log_traceback
|
||||
|
||||
@_log_traceback.setter
|
||||
def _log_traceback(self, val):
|
||||
if val:
|
||||
raise ValueError('_log_traceback can only be set to False')
|
||||
self.__log_traceback = False
|
||||
|
||||
def get_loop(self):
|
||||
"""Return the event loop the Future is bound to."""
|
||||
loop = self._loop
|
||||
if loop is None:
|
||||
raise RuntimeError("Future object is not initialized.")
|
||||
return loop
|
||||
|
||||
def _make_cancelled_error(self):
|
||||
"""Create the CancelledError to raise if the Future is cancelled.
|
||||
|
||||
This should only be called once when handling a cancellation since
|
||||
it erases the saved context exception value.
|
||||
"""
|
||||
if self._cancelled_exc is not None:
|
||||
exc = self._cancelled_exc
|
||||
self._cancelled_exc = None
|
||||
return exc
|
||||
|
||||
if self._cancel_message is None:
|
||||
exc = exceptions.CancelledError()
|
||||
else:
|
||||
exc = exceptions.CancelledError(self._cancel_message)
|
||||
exc.__context__ = self._cancelled_exc
|
||||
# Remove the reference since we don't need this anymore.
|
||||
self._cancelled_exc = None
|
||||
return exc
|
||||
|
||||
def cancel(self, msg=None):
|
||||
def cancel(self):
|
||||
"""Cancel the future and schedule callbacks.
|
||||
|
||||
If the future is already done or cancelled, return False. Otherwise,
|
||||
change the future's state to cancelled, schedule the callbacks and
|
||||
return True.
|
||||
"""
|
||||
self.__log_traceback = False
|
||||
if self._state != _PENDING:
|
||||
return False
|
||||
self._state = _CANCELLED
|
||||
self._cancel_message = msg
|
||||
self.__schedule_callbacks()
|
||||
self._schedule_callbacks()
|
||||
return True
|
||||
|
||||
def __schedule_callbacks(self):
|
||||
def _schedule_callbacks(self):
|
||||
"""Internal: Ask the event loop to call all callbacks.
|
||||
|
||||
The callbacks are scheduled to be called as soon as possible. Also
|
||||
@@ -169,8 +208,8 @@ class Future:
|
||||
return
|
||||
|
||||
self._callbacks[:] = []
|
||||
for callback, ctx in callbacks:
|
||||
self._loop.call_soon(callback, self, context=ctx)
|
||||
for callback in callbacks:
|
||||
self._loop.call_soon(callback, self)
|
||||
|
||||
def cancelled(self):
|
||||
"""Return True if the future was cancelled."""
|
||||
@@ -194,13 +233,15 @@ class Future:
|
||||
the future is done and has an exception set, this exception is raised.
|
||||
"""
|
||||
if self._state == _CANCELLED:
|
||||
exc = self._make_cancelled_error()
|
||||
raise exc
|
||||
raise CancelledError
|
||||
if self._state != _FINISHED:
|
||||
raise exceptions.InvalidStateError('Result is not ready.')
|
||||
self.__log_traceback = False
|
||||
raise InvalidStateError('Result is not ready.')
|
||||
self._log_traceback = False
|
||||
if self._tb_logger is not None:
|
||||
self._tb_logger.clear()
|
||||
self._tb_logger = None
|
||||
if self._exception is not None:
|
||||
raise self._exception.with_traceback(self._exception_tb)
|
||||
raise self._exception
|
||||
return self._result
|
||||
|
||||
def exception(self):
|
||||
@@ -212,14 +253,16 @@ class Future:
|
||||
InvalidStateError.
|
||||
"""
|
||||
if self._state == _CANCELLED:
|
||||
exc = self._make_cancelled_error()
|
||||
raise exc
|
||||
raise CancelledError
|
||||
if self._state != _FINISHED:
|
||||
raise exceptions.InvalidStateError('Exception is not set.')
|
||||
self.__log_traceback = False
|
||||
raise InvalidStateError('Exception is not set.')
|
||||
self._log_traceback = False
|
||||
if self._tb_logger is not None:
|
||||
self._tb_logger.clear()
|
||||
self._tb_logger = None
|
||||
return self._exception
|
||||
|
||||
def add_done_callback(self, fn, *, context=None):
|
||||
def add_done_callback(self, fn):
|
||||
"""Add a callback to be run when the future becomes done.
|
||||
|
||||
The callback is called with a single argument - the future object. If
|
||||
@@ -227,11 +270,9 @@ class Future:
|
||||
scheduled with call_soon.
|
||||
"""
|
||||
if self._state != _PENDING:
|
||||
self._loop.call_soon(fn, self, context=context)
|
||||
self._loop.call_soon(fn, self)
|
||||
else:
|
||||
if context is None:
|
||||
context = contextvars.copy_context()
|
||||
self._callbacks.append((fn, context))
|
||||
self._callbacks.append(fn)
|
||||
|
||||
# New method not in PEP 3148.
|
||||
|
||||
@@ -240,9 +281,7 @@ class Future:
|
||||
|
||||
Returns the number of callbacks removed.
|
||||
"""
|
||||
filtered_callbacks = [(f, ctx)
|
||||
for (f, ctx) in self._callbacks
|
||||
if f != fn]
|
||||
filtered_callbacks = [f for f in self._callbacks if f != fn]
|
||||
removed_count = len(self._callbacks) - len(filtered_callbacks)
|
||||
if removed_count:
|
||||
self._callbacks[:] = filtered_callbacks
|
||||
@@ -257,10 +296,10 @@ class Future:
|
||||
InvalidStateError.
|
||||
"""
|
||||
if self._state != _PENDING:
|
||||
raise exceptions.InvalidStateError(f'{self._state}: {self!r}')
|
||||
raise InvalidStateError('{}: {!r}'.format(self._state, self))
|
||||
self._result = result
|
||||
self._state = _FINISHED
|
||||
self.__schedule_callbacks()
|
||||
self._schedule_callbacks()
|
||||
|
||||
def set_exception(self, exception):
|
||||
"""Mark the future done and set an exception.
|
||||
@@ -269,45 +308,38 @@ class Future:
|
||||
InvalidStateError.
|
||||
"""
|
||||
if self._state != _PENDING:
|
||||
raise exceptions.InvalidStateError(f'{self._state}: {self!r}')
|
||||
raise InvalidStateError('{}: {!r}'.format(self._state, self))
|
||||
if isinstance(exception, type):
|
||||
exception = exception()
|
||||
if type(exception) is StopIteration:
|
||||
raise TypeError("StopIteration interacts badly with generators "
|
||||
"and cannot be raised into a Future")
|
||||
self._exception = exception
|
||||
self._exception_tb = exception.__traceback__
|
||||
self._state = _FINISHED
|
||||
self.__schedule_callbacks()
|
||||
self.__log_traceback = True
|
||||
self._schedule_callbacks()
|
||||
if compat.PY34:
|
||||
self._log_traceback = True
|
||||
else:
|
||||
self._tb_logger = _TracebackLogger(self, exception)
|
||||
# Arrange for the logger to be activated after all callbacks
|
||||
# have had a chance to call result() or exception().
|
||||
self._loop.call_soon(self._tb_logger.activate)
|
||||
|
||||
def __await__(self):
|
||||
def __iter__(self):
|
||||
if not self.done():
|
||||
self._asyncio_future_blocking = True
|
||||
yield self # This tells Task to wait for completion.
|
||||
if not self.done():
|
||||
raise RuntimeError("await wasn't used with future")
|
||||
assert self.done(), "yield from wasn't used with future"
|
||||
return self.result() # May raise too.
|
||||
|
||||
__iter__ = __await__ # make compatible with 'yield from'.
|
||||
if compat.PY35:
|
||||
__await__ = __iter__ # make compatible with 'await' expression
|
||||
|
||||
|
||||
# Needed for testing purposes.
|
||||
_PyFuture = Future
|
||||
|
||||
|
||||
def _get_loop(fut):
|
||||
# Tries to call Future.get_loop() if it's available.
|
||||
# Otherwise fallbacks to using the old '_loop' property.
|
||||
try:
|
||||
get_loop = fut.get_loop
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
return get_loop()
|
||||
return fut._loop
|
||||
|
||||
|
||||
def _set_result_unless_cancelled(fut, result):
|
||||
"""Helper setting the result only if the future was not cancelled."""
|
||||
if fut.cancelled():
|
||||
@@ -315,18 +347,6 @@ def _set_result_unless_cancelled(fut, result):
|
||||
fut.set_result(result)
|
||||
|
||||
|
||||
def _convert_future_exc(exc):
|
||||
exc_class = type(exc)
|
||||
if exc_class is concurrent.futures.CancelledError:
|
||||
return exceptions.CancelledError(*exc.args)
|
||||
elif exc_class is concurrent.futures.TimeoutError:
|
||||
return exceptions.TimeoutError(*exc.args)
|
||||
elif exc_class is concurrent.futures.InvalidStateError:
|
||||
return exceptions.InvalidStateError(*exc.args)
|
||||
else:
|
||||
return exc
|
||||
|
||||
|
||||
def _set_concurrent_future_state(concurrent, source):
|
||||
"""Copy state from a future to a concurrent.futures.Future."""
|
||||
assert source.done()
|
||||
@@ -336,7 +356,7 @@ def _set_concurrent_future_state(concurrent, source):
|
||||
return
|
||||
exception = source.exception()
|
||||
if exception is not None:
|
||||
concurrent.set_exception(_convert_future_exc(exception))
|
||||
concurrent.set_exception(exception)
|
||||
else:
|
||||
result = source.result()
|
||||
concurrent.set_result(result)
|
||||
@@ -356,7 +376,7 @@ def _copy_future_state(source, dest):
|
||||
else:
|
||||
exception = source.exception()
|
||||
if exception is not None:
|
||||
dest.set_exception(_convert_future_exc(exception))
|
||||
dest.set_exception(exception)
|
||||
else:
|
||||
result = source.result()
|
||||
dest.set_result(result)
|
||||
@@ -375,8 +395,8 @@ def _chain_future(source, destination):
|
||||
if not isfuture(destination) and not isinstance(destination,
|
||||
concurrent.futures.Future):
|
||||
raise TypeError('A future is required for destination argument')
|
||||
source_loop = _get_loop(source) if isfuture(source) else None
|
||||
dest_loop = _get_loop(destination) if isfuture(destination) else None
|
||||
source_loop = source._loop if isfuture(source) else None
|
||||
dest_loop = destination._loop if isfuture(destination) else None
|
||||
|
||||
def _set_state(future, other):
|
||||
if isfuture(future):
|
||||
@@ -392,14 +412,9 @@ def _chain_future(source, destination):
|
||||
source_loop.call_soon_threadsafe(source.cancel)
|
||||
|
||||
def _call_set_state(source):
|
||||
if (destination.cancelled() and
|
||||
dest_loop is not None and dest_loop.is_closed()):
|
||||
return
|
||||
if dest_loop is None or dest_loop is source_loop:
|
||||
_set_state(destination, source)
|
||||
else:
|
||||
if dest_loop.is_closed():
|
||||
return
|
||||
dest_loop.call_soon_threadsafe(_set_state, destination, source)
|
||||
|
||||
destination.add_done_callback(_call_check_cancel)
|
||||
@@ -411,7 +426,7 @@ def wrap_future(future, *, loop=None):
|
||||
if isfuture(future):
|
||||
return future
|
||||
assert isinstance(future, concurrent.futures.Future), \
|
||||
f'concurrent.futures.Future is expected, got {future!r}'
|
||||
'concurrent.futures.Future is expected, got {!r}'.format(future)
|
||||
if loop is None:
|
||||
loop = events.get_event_loop()
|
||||
new_future = loop.create_future()
|
||||
|
||||
@@ -1,26 +1,92 @@
|
||||
"""Synchronization primitives."""
|
||||
|
||||
__all__ = ('Lock', 'Event', 'Condition', 'Semaphore',
|
||||
'BoundedSemaphore', 'Barrier')
|
||||
__all__ = ['Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore']
|
||||
|
||||
import collections
|
||||
import enum
|
||||
|
||||
from . import exceptions
|
||||
from . import mixins
|
||||
from . import compat
|
||||
from . import events
|
||||
from . import futures
|
||||
from .coroutines import coroutine
|
||||
|
||||
class _ContextManagerMixin:
|
||||
async def __aenter__(self):
|
||||
await self.acquire()
|
||||
|
||||
class _ContextManager:
|
||||
"""Context manager.
|
||||
|
||||
This enables the following idiom for acquiring and releasing a
|
||||
lock around a block:
|
||||
|
||||
with (yield from lock):
|
||||
<block>
|
||||
|
||||
while failing loudly when accidentally using:
|
||||
|
||||
with lock:
|
||||
<block>
|
||||
"""
|
||||
|
||||
def __init__(self, lock):
|
||||
self._lock = lock
|
||||
|
||||
def __enter__(self):
|
||||
# We have no use for the "as ..." clause in the with
|
||||
# statement for locks.
|
||||
return None
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
self.release()
|
||||
def __exit__(self, *args):
|
||||
try:
|
||||
self._lock.release()
|
||||
finally:
|
||||
self._lock = None # Crudely prevent reuse.
|
||||
|
||||
|
||||
class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
class _ContextManagerMixin:
|
||||
def __enter__(self):
|
||||
raise RuntimeError(
|
||||
'"yield from" should be used as context manager expression')
|
||||
|
||||
def __exit__(self, *args):
|
||||
# This must exist because __enter__ exists, even though that
|
||||
# always raises; that's how the with-statement works.
|
||||
pass
|
||||
|
||||
@coroutine
|
||||
def __iter__(self):
|
||||
# This is not a coroutine. It is meant to enable the idiom:
|
||||
#
|
||||
# with (yield from lock):
|
||||
# <block>
|
||||
#
|
||||
# as an alternative to:
|
||||
#
|
||||
# yield from lock.acquire()
|
||||
# try:
|
||||
# <block>
|
||||
# finally:
|
||||
# lock.release()
|
||||
yield from self.acquire()
|
||||
return _ContextManager(self)
|
||||
|
||||
if compat.PY35:
|
||||
|
||||
def __await__(self):
|
||||
# To make "with await lock" work.
|
||||
yield from self.acquire()
|
||||
return _ContextManager(self)
|
||||
|
||||
@coroutine
|
||||
def __aenter__(self):
|
||||
yield from self.acquire()
|
||||
# We have no use for the "as ..." clause in the with
|
||||
# statement for locks.
|
||||
return None
|
||||
|
||||
@coroutine
|
||||
def __aexit__(self, exc_type, exc, tb):
|
||||
self.release()
|
||||
|
||||
|
||||
class Lock(_ContextManagerMixin):
|
||||
"""Primitive lock objects.
|
||||
|
||||
A primitive lock is a synchronization primitive that is not owned
|
||||
@@ -42,16 +108,16 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
release() call resets the state to unlocked; first coroutine which
|
||||
is blocked in acquire() is being processed.
|
||||
|
||||
acquire() is a coroutine and should be called with 'await'.
|
||||
acquire() is a coroutine and should be called with 'yield from'.
|
||||
|
||||
Locks also support the asynchronous context management protocol.
|
||||
'async with lock' statement should be used.
|
||||
Locks also support the context management protocol. '(yield from lock)'
|
||||
should be used as the context manager expression.
|
||||
|
||||
Usage:
|
||||
|
||||
lock = Lock()
|
||||
...
|
||||
await lock.acquire()
|
||||
yield from lock
|
||||
try:
|
||||
...
|
||||
finally:
|
||||
@@ -61,65 +127,57 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
|
||||
lock = Lock()
|
||||
...
|
||||
async with lock:
|
||||
with (yield from lock):
|
||||
...
|
||||
|
||||
Lock objects can be tested for locking state:
|
||||
|
||||
if not lock.locked():
|
||||
await lock.acquire()
|
||||
yield from lock
|
||||
else:
|
||||
# lock is acquired
|
||||
...
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._waiters = None
|
||||
def __init__(self, *, loop=None):
|
||||
self._waiters = collections.deque()
|
||||
self._locked = False
|
||||
if loop is not None:
|
||||
self._loop = loop
|
||||
else:
|
||||
self._loop = events.get_event_loop()
|
||||
|
||||
def __repr__(self):
|
||||
res = super().__repr__()
|
||||
extra = 'locked' if self._locked else 'unlocked'
|
||||
if self._waiters:
|
||||
extra = f'{extra}, waiters:{len(self._waiters)}'
|
||||
return f'<{res[1:-1]} [{extra}]>'
|
||||
extra = '{},waiters:{}'.format(extra, len(self._waiters))
|
||||
return '<{} [{}]>'.format(res[1:-1], extra)
|
||||
|
||||
def locked(self):
|
||||
"""Return True if lock is acquired."""
|
||||
return self._locked
|
||||
|
||||
async def acquire(self):
|
||||
@coroutine
|
||||
def acquire(self):
|
||||
"""Acquire a lock.
|
||||
|
||||
This method blocks until the lock is unlocked, then sets it to
|
||||
locked and returns True.
|
||||
"""
|
||||
if (not self._locked and (self._waiters is None or
|
||||
all(w.cancelled() for w in self._waiters))):
|
||||
if not self._locked and all(w.cancelled() for w in self._waiters):
|
||||
self._locked = True
|
||||
return True
|
||||
|
||||
if self._waiters is None:
|
||||
self._waiters = collections.deque()
|
||||
fut = self._get_loop().create_future()
|
||||
fut = self._loop.create_future()
|
||||
self._waiters.append(fut)
|
||||
|
||||
# Finally block should be called before the CancelledError
|
||||
# handling as we don't want CancelledError to call
|
||||
# _wake_up_first() and attempt to wake up itself.
|
||||
try:
|
||||
try:
|
||||
await fut
|
||||
finally:
|
||||
self._waiters.remove(fut)
|
||||
except exceptions.CancelledError:
|
||||
if not self._locked:
|
||||
self._wake_up_first()
|
||||
raise
|
||||
|
||||
self._locked = True
|
||||
return True
|
||||
yield from fut
|
||||
self._locked = True
|
||||
return True
|
||||
finally:
|
||||
self._waiters.remove(fut)
|
||||
|
||||
def release(self):
|
||||
"""Release a lock.
|
||||
@@ -134,27 +192,16 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
"""
|
||||
if self._locked:
|
||||
self._locked = False
|
||||
self._wake_up_first()
|
||||
# Wake up the first waiter who isn't cancelled.
|
||||
for fut in self._waiters:
|
||||
if not fut.done():
|
||||
fut.set_result(True)
|
||||
break
|
||||
else:
|
||||
raise RuntimeError('Lock is not acquired.')
|
||||
|
||||
def _wake_up_first(self):
|
||||
"""Wake up the first waiter if it isn't done."""
|
||||
if not self._waiters:
|
||||
return
|
||||
try:
|
||||
fut = next(iter(self._waiters))
|
||||
except StopIteration:
|
||||
return
|
||||
|
||||
# .done() necessarily means that a waiter will wake up later on and
|
||||
# either take the lock, or, if it was cancelled and lock wasn't
|
||||
# taken already, will hit this again and wake up a new waiter.
|
||||
if not fut.done():
|
||||
fut.set_result(True)
|
||||
|
||||
|
||||
class Event(mixins._LoopBoundMixin):
|
||||
class Event:
|
||||
"""Asynchronous equivalent to threading.Event.
|
||||
|
||||
Class implementing event objects. An event manages a flag that can be set
|
||||
@@ -163,16 +210,20 @@ class Event(mixins._LoopBoundMixin):
|
||||
false.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, *, loop=None):
|
||||
self._waiters = collections.deque()
|
||||
self._value = False
|
||||
if loop is not None:
|
||||
self._loop = loop
|
||||
else:
|
||||
self._loop = events.get_event_loop()
|
||||
|
||||
def __repr__(self):
|
||||
res = super().__repr__()
|
||||
extra = 'set' if self._value else 'unset'
|
||||
if self._waiters:
|
||||
extra = f'{extra}, waiters:{len(self._waiters)}'
|
||||
return f'<{res[1:-1]} [{extra}]>'
|
||||
extra = '{},waiters:{}'.format(extra, len(self._waiters))
|
||||
return '<{} [{}]>'.format(res[1:-1], extra)
|
||||
|
||||
def is_set(self):
|
||||
"""Return True if and only if the internal flag is true."""
|
||||
@@ -196,7 +247,8 @@ class Event(mixins._LoopBoundMixin):
|
||||
to true again."""
|
||||
self._value = False
|
||||
|
||||
async def wait(self):
|
||||
@coroutine
|
||||
def wait(self):
|
||||
"""Block until the internal flag is true.
|
||||
|
||||
If the internal flag is true on entry, return True
|
||||
@@ -206,16 +258,16 @@ class Event(mixins._LoopBoundMixin):
|
||||
if self._value:
|
||||
return True
|
||||
|
||||
fut = self._get_loop().create_future()
|
||||
fut = self._loop.create_future()
|
||||
self._waiters.append(fut)
|
||||
try:
|
||||
await fut
|
||||
yield from fut
|
||||
return True
|
||||
finally:
|
||||
self._waiters.remove(fut)
|
||||
|
||||
|
||||
class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
class Condition(_ContextManagerMixin):
|
||||
"""Asynchronous equivalent to threading.Condition.
|
||||
|
||||
This class implements condition variable objects. A condition variable
|
||||
@@ -225,9 +277,16 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
A new Lock object is created and used as the underlying lock.
|
||||
"""
|
||||
|
||||
def __init__(self, lock=None):
|
||||
def __init__(self, lock=None, *, loop=None):
|
||||
if loop is not None:
|
||||
self._loop = loop
|
||||
else:
|
||||
self._loop = events.get_event_loop()
|
||||
|
||||
if lock is None:
|
||||
lock = Lock()
|
||||
lock = Lock(loop=self._loop)
|
||||
elif lock._loop is not self._loop:
|
||||
raise ValueError("loop argument must agree with lock")
|
||||
|
||||
self._lock = lock
|
||||
# Export the lock's locked(), acquire() and release() methods.
|
||||
@@ -241,10 +300,11 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
res = super().__repr__()
|
||||
extra = 'locked' if self.locked() else 'unlocked'
|
||||
if self._waiters:
|
||||
extra = f'{extra}, waiters:{len(self._waiters)}'
|
||||
return f'<{res[1:-1]} [{extra}]>'
|
||||
extra = '{},waiters:{}'.format(extra, len(self._waiters))
|
||||
return '<{} [{}]>'.format(res[1:-1], extra)
|
||||
|
||||
async def wait(self):
|
||||
@coroutine
|
||||
def wait(self):
|
||||
"""Wait until notified.
|
||||
|
||||
If the calling coroutine has not acquired the lock when this
|
||||
@@ -260,28 +320,25 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
|
||||
self.release()
|
||||
try:
|
||||
fut = self._get_loop().create_future()
|
||||
fut = self._loop.create_future()
|
||||
self._waiters.append(fut)
|
||||
try:
|
||||
await fut
|
||||
yield from fut
|
||||
return True
|
||||
finally:
|
||||
self._waiters.remove(fut)
|
||||
|
||||
finally:
|
||||
# Must reacquire lock even if wait is cancelled
|
||||
cancelled = False
|
||||
while True:
|
||||
try:
|
||||
await self.acquire()
|
||||
yield from self.acquire()
|
||||
break
|
||||
except exceptions.CancelledError:
|
||||
cancelled = True
|
||||
except futures.CancelledError:
|
||||
pass
|
||||
|
||||
if cancelled:
|
||||
raise exceptions.CancelledError
|
||||
|
||||
async def wait_for(self, predicate):
|
||||
@coroutine
|
||||
def wait_for(self, predicate):
|
||||
"""Wait until a predicate becomes true.
|
||||
|
||||
The predicate should be a callable which result will be
|
||||
@@ -290,7 +347,7 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
"""
|
||||
result = predicate()
|
||||
while not result:
|
||||
await self.wait()
|
||||
yield from self.wait()
|
||||
result = predicate()
|
||||
return result
|
||||
|
||||
@@ -327,7 +384,7 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
self.notify(len(self._waiters))
|
||||
|
||||
|
||||
class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
class Semaphore(_ContextManagerMixin):
|
||||
"""A Semaphore implementation.
|
||||
|
||||
A semaphore manages an internal counter which is decremented by each
|
||||
@@ -342,25 +399,37 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
ValueError is raised.
|
||||
"""
|
||||
|
||||
def __init__(self, value=1):
|
||||
def __init__(self, value=1, *, loop=None):
|
||||
if value < 0:
|
||||
raise ValueError("Semaphore initial value must be >= 0")
|
||||
self._waiters = None
|
||||
self._value = value
|
||||
self._waiters = collections.deque()
|
||||
if loop is not None:
|
||||
self._loop = loop
|
||||
else:
|
||||
self._loop = events.get_event_loop()
|
||||
|
||||
def __repr__(self):
|
||||
res = super().__repr__()
|
||||
extra = 'locked' if self.locked() else f'unlocked, value:{self._value}'
|
||||
extra = 'locked' if self.locked() else 'unlocked,value:{}'.format(
|
||||
self._value)
|
||||
if self._waiters:
|
||||
extra = f'{extra}, waiters:{len(self._waiters)}'
|
||||
return f'<{res[1:-1]} [{extra}]>'
|
||||
extra = '{},waiters:{}'.format(extra, len(self._waiters))
|
||||
return '<{} [{}]>'.format(res[1:-1], extra)
|
||||
|
||||
def _wake_up_next(self):
|
||||
while self._waiters:
|
||||
waiter = self._waiters.popleft()
|
||||
if not waiter.done():
|
||||
waiter.set_result(None)
|
||||
return
|
||||
|
||||
def locked(self):
|
||||
"""Returns True if semaphore cannot be acquired immediately."""
|
||||
return self._value == 0 or (
|
||||
any(not w.cancelled() for w in (self._waiters or ())))
|
||||
"""Returns True if semaphore can not be acquired immediately."""
|
||||
return self._value == 0
|
||||
|
||||
async def acquire(self):
|
||||
@coroutine
|
||||
def acquire(self):
|
||||
"""Acquire a semaphore.
|
||||
|
||||
If the internal counter is larger than zero on entry,
|
||||
@@ -369,53 +438,28 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin):
|
||||
called release() to make it larger than 0, and then return
|
||||
True.
|
||||
"""
|
||||
if not self.locked():
|
||||
self._value -= 1
|
||||
return True
|
||||
|
||||
if self._waiters is None:
|
||||
self._waiters = collections.deque()
|
||||
fut = self._get_loop().create_future()
|
||||
self._waiters.append(fut)
|
||||
|
||||
# Finally block should be called before the CancelledError
|
||||
# handling as we don't want CancelledError to call
|
||||
# _wake_up_first() and attempt to wake up itself.
|
||||
try:
|
||||
while self._value <= 0:
|
||||
fut = self._loop.create_future()
|
||||
self._waiters.append(fut)
|
||||
try:
|
||||
await fut
|
||||
finally:
|
||||
self._waiters.remove(fut)
|
||||
except exceptions.CancelledError:
|
||||
if not fut.cancelled():
|
||||
self._value += 1
|
||||
self._wake_up_next()
|
||||
raise
|
||||
|
||||
if self._value > 0:
|
||||
self._wake_up_next()
|
||||
yield from fut
|
||||
except:
|
||||
# See the similar code in Queue.get.
|
||||
fut.cancel()
|
||||
if self._value > 0 and not fut.cancelled():
|
||||
self._wake_up_next()
|
||||
raise
|
||||
self._value -= 1
|
||||
return True
|
||||
|
||||
def release(self):
|
||||
"""Release a semaphore, incrementing the internal counter by one.
|
||||
|
||||
When it was zero on entry and another coroutine is waiting for it to
|
||||
become larger than zero again, wake up that coroutine.
|
||||
"""
|
||||
self._value += 1
|
||||
self._wake_up_next()
|
||||
|
||||
def _wake_up_next(self):
|
||||
"""Wake up the first waiter that isn't done."""
|
||||
if not self._waiters:
|
||||
return
|
||||
|
||||
for fut in self._waiters:
|
||||
if not fut.done():
|
||||
self._value -= 1
|
||||
fut.set_result(True)
|
||||
return
|
||||
|
||||
|
||||
class BoundedSemaphore(Semaphore):
|
||||
"""A bounded semaphore implementation.
|
||||
@@ -424,163 +468,11 @@ class BoundedSemaphore(Semaphore):
|
||||
above the initial value.
|
||||
"""
|
||||
|
||||
def __init__(self, value=1):
|
||||
def __init__(self, value=1, *, loop=None):
|
||||
self._bound_value = value
|
||||
super().__init__(value)
|
||||
super().__init__(value, loop=loop)
|
||||
|
||||
def release(self):
|
||||
if self._value >= self._bound_value:
|
||||
raise ValueError('BoundedSemaphore released too many times')
|
||||
super().release()
|
||||
|
||||
|
||||
|
||||
class _BarrierState(enum.Enum):
|
||||
FILLING = 'filling'
|
||||
DRAINING = 'draining'
|
||||
RESETTING = 'resetting'
|
||||
BROKEN = 'broken'
|
||||
|
||||
|
||||
class Barrier(mixins._LoopBoundMixin):
|
||||
"""Asyncio equivalent to threading.Barrier
|
||||
|
||||
Implements a Barrier primitive.
|
||||
Useful for synchronizing a fixed number of tasks at known synchronization
|
||||
points. Tasks block on 'wait()' and are simultaneously awoken once they
|
||||
have all made their call.
|
||||
"""
|
||||
|
||||
def __init__(self, parties):
|
||||
"""Create a barrier, initialised to 'parties' tasks."""
|
||||
if parties < 1:
|
||||
raise ValueError('parties must be > 0')
|
||||
|
||||
self._cond = Condition() # notify all tasks when state changes
|
||||
|
||||
self._parties = parties
|
||||
self._state = _BarrierState.FILLING
|
||||
self._count = 0 # count tasks in Barrier
|
||||
|
||||
def __repr__(self):
|
||||
res = super().__repr__()
|
||||
extra = f'{self._state.value}'
|
||||
if not self.broken:
|
||||
extra += f', waiters:{self.n_waiting}/{self.parties}'
|
||||
return f'<{res[1:-1]} [{extra}]>'
|
||||
|
||||
async def __aenter__(self):
|
||||
# wait for the barrier reaches the parties number
|
||||
# when start draining release and return index of waited task
|
||||
return await self.wait()
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
pass
|
||||
|
||||
async def wait(self):
|
||||
"""Wait for the barrier.
|
||||
|
||||
When the specified number of tasks have started waiting, they are all
|
||||
simultaneously awoken.
|
||||
Returns an unique and individual index number from 0 to 'parties-1'.
|
||||
"""
|
||||
async with self._cond:
|
||||
await self._block() # Block while the barrier drains or resets.
|
||||
try:
|
||||
index = self._count
|
||||
self._count += 1
|
||||
if index + 1 == self._parties:
|
||||
# We release the barrier
|
||||
await self._release()
|
||||
else:
|
||||
await self._wait()
|
||||
return index
|
||||
finally:
|
||||
self._count -= 1
|
||||
# Wake up any tasks waiting for barrier to drain.
|
||||
self._exit()
|
||||
|
||||
async def _block(self):
|
||||
# Block until the barrier is ready for us,
|
||||
# or raise an exception if it is broken.
|
||||
#
|
||||
# It is draining or resetting, wait until done
|
||||
# unless a CancelledError occurs
|
||||
await self._cond.wait_for(
|
||||
lambda: self._state not in (
|
||||
_BarrierState.DRAINING, _BarrierState.RESETTING
|
||||
)
|
||||
)
|
||||
|
||||
# see if the barrier is in a broken state
|
||||
if self._state is _BarrierState.BROKEN:
|
||||
raise exceptions.BrokenBarrierError("Barrier aborted")
|
||||
|
||||
async def _release(self):
|
||||
# Release the tasks waiting in the barrier.
|
||||
|
||||
# Enter draining state.
|
||||
# Next waiting tasks will be blocked until the end of draining.
|
||||
self._state = _BarrierState.DRAINING
|
||||
self._cond.notify_all()
|
||||
|
||||
async def _wait(self):
|
||||
# Wait in the barrier until we are released. Raise an exception
|
||||
# if the barrier is reset or broken.
|
||||
|
||||
# wait for end of filling
|
||||
# unless a CancelledError occurs
|
||||
await self._cond.wait_for(lambda: self._state is not _BarrierState.FILLING)
|
||||
|
||||
if self._state in (_BarrierState.BROKEN, _BarrierState.RESETTING):
|
||||
raise exceptions.BrokenBarrierError("Abort or reset of barrier")
|
||||
|
||||
def _exit(self):
|
||||
# If we are the last tasks to exit the barrier, signal any tasks
|
||||
# waiting for the barrier to drain.
|
||||
if self._count == 0:
|
||||
if self._state in (_BarrierState.RESETTING, _BarrierState.DRAINING):
|
||||
self._state = _BarrierState.FILLING
|
||||
self._cond.notify_all()
|
||||
|
||||
async def reset(self):
|
||||
"""Reset the barrier to the initial state.
|
||||
|
||||
Any tasks currently waiting will get the BrokenBarrier exception
|
||||
raised.
|
||||
"""
|
||||
async with self._cond:
|
||||
if self._count > 0:
|
||||
if self._state is not _BarrierState.RESETTING:
|
||||
#reset the barrier, waking up tasks
|
||||
self._state = _BarrierState.RESETTING
|
||||
else:
|
||||
self._state = _BarrierState.FILLING
|
||||
self._cond.notify_all()
|
||||
|
||||
async def abort(self):
|
||||
"""Place the barrier into a 'broken' state.
|
||||
|
||||
Useful in case of error. Any currently waiting tasks and tasks
|
||||
attempting to 'wait()' will have BrokenBarrierError raised.
|
||||
"""
|
||||
async with self._cond:
|
||||
self._state = _BarrierState.BROKEN
|
||||
self._cond.notify_all()
|
||||
|
||||
@property
|
||||
def parties(self):
|
||||
"""Return the number of tasks required to trip the barrier."""
|
||||
return self._parties
|
||||
|
||||
@property
|
||||
def n_waiting(self):
|
||||
"""Return the number of tasks currently waiting at the barrier."""
|
||||
if self._state is _BarrierState.FILLING:
|
||||
return self._count
|
||||
return 0
|
||||
|
||||
@property
|
||||
def broken(self):
|
||||
"""Return True if the barrier is in a broken state."""
|
||||
return self._state is _BarrierState.BROKEN
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
"""Event loop mixins."""
|
||||
|
||||
import threading
|
||||
from . import events
|
||||
|
||||
_global_lock = threading.Lock()
|
||||
|
||||
|
||||
class _LoopBoundMixin:
|
||||
_loop = None
|
||||
|
||||
def _get_loop(self):
|
||||
loop = events._get_running_loop()
|
||||
|
||||
if self._loop is None:
|
||||
with _global_lock:
|
||||
if self._loop is None:
|
||||
self._loop = loop
|
||||
if loop is not self._loop:
|
||||
raise RuntimeError(f'{self!r} is bound to a different event loop')
|
||||
return loop
|
||||
@@ -4,45 +4,20 @@ A proactor is a "notify-on-completion" multiplexer. Currently a
|
||||
proactor is only implemented on Windows with IOCP.
|
||||
"""
|
||||
|
||||
__all__ = 'BaseProactorEventLoop',
|
||||
__all__ = ['BaseProactorEventLoop']
|
||||
|
||||
import io
|
||||
import os
|
||||
import socket
|
||||
import warnings
|
||||
import signal
|
||||
import threading
|
||||
import collections
|
||||
|
||||
from . import base_events
|
||||
from . import compat
|
||||
from . import constants
|
||||
from . import futures
|
||||
from . import exceptions
|
||||
from . import protocols
|
||||
from . import sslproto
|
||||
from . import transports
|
||||
from . import trsock
|
||||
from .log import logger
|
||||
|
||||
|
||||
def _set_socket_extra(transport, sock):
|
||||
transport._extra['socket'] = trsock.TransportSocket(sock)
|
||||
|
||||
try:
|
||||
transport._extra['sockname'] = sock.getsockname()
|
||||
except socket.error:
|
||||
if transport._loop.get_debug():
|
||||
logger.warning(
|
||||
"getsockname() failed on %r", sock, exc_info=True)
|
||||
|
||||
if 'peername' not in transport._extra:
|
||||
try:
|
||||
transport._extra['peername'] = sock.getpeername()
|
||||
except socket.error:
|
||||
# UDP sockets may not have a peer name
|
||||
transport._extra['peername'] = None
|
||||
|
||||
|
||||
class _ProactorBasePipeTransport(transports._FlowControlMixin,
|
||||
transports.BaseTransport):
|
||||
"""Base class for pipe and socket transports."""
|
||||
@@ -52,7 +27,7 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
|
||||
super().__init__(extra, loop)
|
||||
self._set_extra(sock)
|
||||
self._sock = sock
|
||||
self.set_protocol(protocol)
|
||||
self._protocol = protocol
|
||||
self._server = server
|
||||
self._buffer = None # None or bytearray.
|
||||
self._read_fut = None
|
||||
@@ -60,7 +35,6 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
|
||||
self._pending_write = 0
|
||||
self._conn_lost = 0
|
||||
self._closing = False # Set when close() called.
|
||||
self._called_connection_lost = False
|
||||
self._eof_written = False
|
||||
if self._server is not None:
|
||||
self._server._attach()
|
||||
@@ -77,16 +51,17 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
|
||||
elif self._closing:
|
||||
info.append('closing')
|
||||
if self._sock is not None:
|
||||
info.append(f'fd={self._sock.fileno()}')
|
||||
info.append('fd=%s' % self._sock.fileno())
|
||||
if self._read_fut is not None:
|
||||
info.append(f'read={self._read_fut!r}')
|
||||
info.append('read=%s' % self._read_fut)
|
||||
if self._write_fut is not None:
|
||||
info.append(f'write={self._write_fut!r}')
|
||||
info.append("write=%r" % self._write_fut)
|
||||
if self._buffer:
|
||||
info.append(f'write_bufsize={len(self._buffer)}')
|
||||
bufsize = len(self._buffer)
|
||||
info.append('write_bufsize=%s' % bufsize)
|
||||
if self._eof_written:
|
||||
info.append('EOF written')
|
||||
return '<{}>'.format(' '.join(info))
|
||||
return '<%s>' % ' '.join(info)
|
||||
|
||||
def _set_extra(self, sock):
|
||||
self._extra['pipe'] = sock
|
||||
@@ -111,33 +86,31 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
|
||||
self._read_fut.cancel()
|
||||
self._read_fut = None
|
||||
|
||||
def __del__(self, _warn=warnings.warn):
|
||||
if self._sock is not None:
|
||||
_warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
|
||||
self._sock.close()
|
||||
# On Python 3.3 and older, objects with a destructor part of a reference
|
||||
# cycle are never destroyed. It's not more the case on Python 3.4 thanks
|
||||
# to the PEP 442.
|
||||
if compat.PY34:
|
||||
def __del__(self):
|
||||
if self._sock is not None:
|
||||
warnings.warn("unclosed transport %r" % self, ResourceWarning,
|
||||
source=self)
|
||||
self.close()
|
||||
|
||||
def _fatal_error(self, exc, message='Fatal error on pipe transport'):
|
||||
try:
|
||||
if isinstance(exc, OSError):
|
||||
if self._loop.get_debug():
|
||||
logger.debug("%r: %s", self, message, exc_info=True)
|
||||
else:
|
||||
self._loop.call_exception_handler({
|
||||
'message': message,
|
||||
'exception': exc,
|
||||
'transport': self,
|
||||
'protocol': self._protocol,
|
||||
})
|
||||
finally:
|
||||
self._force_close(exc)
|
||||
if isinstance(exc, base_events._FATAL_ERROR_IGNORE):
|
||||
if self._loop.get_debug():
|
||||
logger.debug("%r: %s", self, message, exc_info=True)
|
||||
else:
|
||||
self._loop.call_exception_handler({
|
||||
'message': message,
|
||||
'exception': exc,
|
||||
'transport': self,
|
||||
'protocol': self._protocol,
|
||||
})
|
||||
self._force_close(exc)
|
||||
|
||||
def _force_close(self, exc):
|
||||
if self._empty_waiter is not None and not self._empty_waiter.done():
|
||||
if exc is None:
|
||||
self._empty_waiter.set_result(None)
|
||||
else:
|
||||
self._empty_waiter.set_exception(exc)
|
||||
if self._closing and self._called_connection_lost:
|
||||
if self._closing:
|
||||
return
|
||||
self._closing = True
|
||||
self._conn_lost += 1
|
||||
@@ -152,8 +125,6 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
|
||||
self._loop.call_soon(self._call_connection_lost, exc)
|
||||
|
||||
def _call_connection_lost(self, exc):
|
||||
if self._called_connection_lost:
|
||||
return
|
||||
try:
|
||||
self._protocol.connection_lost(exc)
|
||||
finally:
|
||||
@@ -161,7 +132,7 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
|
||||
# end then it may fail with ERROR_NETNAME_DELETED if we
|
||||
# just close our end. First calling shutdown() seems to
|
||||
# cure it, but maybe using DisconnectEx() would be better.
|
||||
if hasattr(self._sock, 'shutdown') and self._sock.fileno() != -1:
|
||||
if hasattr(self._sock, 'shutdown'):
|
||||
self._sock.shutdown(socket.SHUT_RDWR)
|
||||
self._sock.close()
|
||||
self._sock = None
|
||||
@@ -169,7 +140,6 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
|
||||
if server is not None:
|
||||
server._detach()
|
||||
self._server = None
|
||||
self._called_connection_lost = True
|
||||
|
||||
def get_write_buffer_size(self):
|
||||
size = self._pending_write
|
||||
@@ -183,127 +153,53 @@ class _ProactorReadPipeTransport(_ProactorBasePipeTransport,
|
||||
"""Transport for read pipes."""
|
||||
|
||||
def __init__(self, loop, sock, protocol, waiter=None,
|
||||
extra=None, server=None, buffer_size=65536):
|
||||
self._pending_data_length = -1
|
||||
self._paused = True
|
||||
extra=None, server=None):
|
||||
super().__init__(loop, sock, protocol, waiter, extra, server)
|
||||
|
||||
self._data = bytearray(buffer_size)
|
||||
self._loop.call_soon(self._loop_reading)
|
||||
self._paused = False
|
||||
|
||||
def is_reading(self):
|
||||
return not self._paused and not self._closing
|
||||
self._loop.call_soon(self._loop_reading)
|
||||
|
||||
def pause_reading(self):
|
||||
if self._closing or self._paused:
|
||||
return
|
||||
if self._closing:
|
||||
raise RuntimeError('Cannot pause_reading() when closing')
|
||||
if self._paused:
|
||||
raise RuntimeError('Already paused')
|
||||
self._paused = True
|
||||
|
||||
# bpo-33694: Don't cancel self._read_fut because cancelling an
|
||||
# overlapped WSASend() loss silently data with the current proactor
|
||||
# implementation.
|
||||
#
|
||||
# If CancelIoEx() fails with ERROR_NOT_FOUND, it means that WSASend()
|
||||
# completed (even if HasOverlappedIoCompleted() returns 0), but
|
||||
# Overlapped.cancel() currently silently ignores the ERROR_NOT_FOUND
|
||||
# error. Once the overlapped is ignored, the IOCP loop will ignores the
|
||||
# completion I/O event and so not read the result of the overlapped
|
||||
# WSARecv().
|
||||
|
||||
if self._loop.get_debug():
|
||||
logger.debug("%r pauses reading", self)
|
||||
|
||||
def resume_reading(self):
|
||||
if self._closing or not self._paused:
|
||||
return
|
||||
|
||||
if not self._paused:
|
||||
raise RuntimeError('Not paused')
|
||||
self._paused = False
|
||||
if self._read_fut is None:
|
||||
self._loop.call_soon(self._loop_reading, None)
|
||||
|
||||
length = self._pending_data_length
|
||||
self._pending_data_length = -1
|
||||
if length > -1:
|
||||
# Call the protocol method after calling _loop_reading(),
|
||||
# since the protocol can decide to pause reading again.
|
||||
self._loop.call_soon(self._data_received, self._data[:length], length)
|
||||
|
||||
if self._closing:
|
||||
return
|
||||
self._loop.call_soon(self._loop_reading, self._read_fut)
|
||||
if self._loop.get_debug():
|
||||
logger.debug("%r resumes reading", self)
|
||||
|
||||
def _eof_received(self):
|
||||
if self._loop.get_debug():
|
||||
logger.debug("%r received EOF", self)
|
||||
|
||||
try:
|
||||
keep_open = self._protocol.eof_received()
|
||||
except (SystemExit, KeyboardInterrupt):
|
||||
raise
|
||||
except BaseException as exc:
|
||||
self._fatal_error(
|
||||
exc, 'Fatal error: protocol.eof_received() call failed.')
|
||||
return
|
||||
|
||||
if not keep_open:
|
||||
self.close()
|
||||
|
||||
def _data_received(self, data, length):
|
||||
if self._paused:
|
||||
# Don't call any protocol method while reading is paused.
|
||||
# The protocol will be called on resume_reading().
|
||||
assert self._pending_data_length == -1
|
||||
self._pending_data_length = length
|
||||
return
|
||||
|
||||
if length == 0:
|
||||
self._eof_received()
|
||||
return
|
||||
|
||||
if isinstance(self._protocol, protocols.BufferedProtocol):
|
||||
try:
|
||||
protocols._feed_data_to_buffered_proto(self._protocol, data)
|
||||
except (SystemExit, KeyboardInterrupt):
|
||||
raise
|
||||
except BaseException as exc:
|
||||
self._fatal_error(exc,
|
||||
'Fatal error: protocol.buffer_updated() '
|
||||
'call failed.')
|
||||
return
|
||||
else:
|
||||
self._protocol.data_received(data)
|
||||
|
||||
def _loop_reading(self, fut=None):
|
||||
length = -1
|
||||
if self._paused:
|
||||
return
|
||||
data = None
|
||||
|
||||
try:
|
||||
if fut is not None:
|
||||
assert self._read_fut is fut or (self._read_fut is None and
|
||||
self._closing)
|
||||
self._read_fut = None
|
||||
if fut.done():
|
||||
# deliver data later in "finally" clause
|
||||
length = fut.result()
|
||||
if length == 0:
|
||||
# we got end-of-file so no need to reschedule a new read
|
||||
return
|
||||
|
||||
# It's a new slice so make it immutable so protocols upstream don't have problems
|
||||
data = bytes(memoryview(self._data)[:length])
|
||||
else:
|
||||
# the future will be replaced by next proactor.recv call
|
||||
fut.cancel()
|
||||
data = fut.result() # deliver data later in "finally" clause
|
||||
|
||||
if self._closing:
|
||||
# since close() has been called we ignore any read data
|
||||
data = None
|
||||
return
|
||||
|
||||
# bpo-33694: buffer_updated() has currently no fast path because of
|
||||
# a data loss issue caused by overlapped WSASend() cancellation.
|
||||
if data == b'':
|
||||
# we got end-of-file so no need to reschedule a new read
|
||||
return
|
||||
|
||||
if not self._paused:
|
||||
# reschedule a new read
|
||||
self._read_fut = self._loop._proactor.recv_into(self._sock, self._data)
|
||||
# reschedule a new read
|
||||
self._read_fut = self._loop._proactor.recv(self._sock, 4096)
|
||||
except ConnectionAbortedError as exc:
|
||||
if not self._closing:
|
||||
self._fatal_error(exc, 'Fatal read error on pipe transport')
|
||||
@@ -314,36 +210,32 @@ class _ProactorReadPipeTransport(_ProactorBasePipeTransport,
|
||||
self._force_close(exc)
|
||||
except OSError as exc:
|
||||
self._fatal_error(exc, 'Fatal read error on pipe transport')
|
||||
except exceptions.CancelledError:
|
||||
except futures.CancelledError:
|
||||
if not self._closing:
|
||||
raise
|
||||
else:
|
||||
if not self._paused:
|
||||
self._read_fut.add_done_callback(self._loop_reading)
|
||||
self._read_fut.add_done_callback(self._loop_reading)
|
||||
finally:
|
||||
if length > -1:
|
||||
self._data_received(data, length)
|
||||
if data:
|
||||
self._protocol.data_received(data)
|
||||
elif data is not None:
|
||||
if self._loop.get_debug():
|
||||
logger.debug("%r received EOF", self)
|
||||
keep_open = self._protocol.eof_received()
|
||||
if not keep_open:
|
||||
self.close()
|
||||
|
||||
|
||||
class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
|
||||
transports.WriteTransport):
|
||||
"""Transport for write pipes."""
|
||||
|
||||
_start_tls_compatible = True
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
super().__init__(*args, **kw)
|
||||
self._empty_waiter = None
|
||||
|
||||
def write(self, data):
|
||||
if not isinstance(data, (bytes, bytearray, memoryview)):
|
||||
raise TypeError(
|
||||
f"data argument must be a bytes-like object, "
|
||||
f"not {type(data).__name__}")
|
||||
raise TypeError('data argument must be byte-ish (%r)',
|
||||
type(data))
|
||||
if self._eof_written:
|
||||
raise RuntimeError('write_eof() already called')
|
||||
if self._empty_waiter is not None:
|
||||
raise RuntimeError('unable to write; sendfile is in progress')
|
||||
|
||||
if not data:
|
||||
return
|
||||
@@ -375,10 +267,6 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
|
||||
|
||||
def _loop_writing(self, f=None, data=None):
|
||||
try:
|
||||
if f is not None and self._write_fut is None and self._closing:
|
||||
# XXX most likely self._force_close() has been called, and
|
||||
# it has set self._write_fut to None.
|
||||
return
|
||||
assert f is self._write_fut
|
||||
self._write_fut = None
|
||||
self._pending_write = 0
|
||||
@@ -407,8 +295,6 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
|
||||
self._maybe_pause_protocol()
|
||||
else:
|
||||
self._write_fut.add_done_callback(self._loop_writing)
|
||||
if self._empty_waiter is not None and self._write_fut is None:
|
||||
self._empty_waiter.set_result(None)
|
||||
except ConnectionResetError as exc:
|
||||
self._force_close(exc)
|
||||
except OSError as exc:
|
||||
@@ -423,17 +309,6 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
|
||||
def abort(self):
|
||||
self._force_close(None)
|
||||
|
||||
def _make_empty_waiter(self):
|
||||
if self._empty_waiter is not None:
|
||||
raise RuntimeError("Empty waiter is already set")
|
||||
self._empty_waiter = self._loop.create_future()
|
||||
if self._write_fut is None:
|
||||
self._empty_waiter.set_result(None)
|
||||
return self._empty_waiter
|
||||
|
||||
def _reset_empty_waiter(self):
|
||||
self._empty_waiter = None
|
||||
|
||||
|
||||
class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport):
|
||||
def __init__(self, *args, **kw):
|
||||
@@ -457,138 +332,6 @@ class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport):
|
||||
self.close()
|
||||
|
||||
|
||||
class _ProactorDatagramTransport(_ProactorBasePipeTransport,
|
||||
transports.DatagramTransport):
|
||||
max_size = 256 * 1024
|
||||
def __init__(self, loop, sock, protocol, address=None,
|
||||
waiter=None, extra=None):
|
||||
self._address = address
|
||||
self._empty_waiter = None
|
||||
self._buffer_size = 0
|
||||
# We don't need to call _protocol.connection_made() since our base
|
||||
# constructor does it for us.
|
||||
super().__init__(loop, sock, protocol, waiter=waiter, extra=extra)
|
||||
|
||||
# The base constructor sets _buffer = None, so we set it here
|
||||
self._buffer = collections.deque()
|
||||
self._loop.call_soon(self._loop_reading)
|
||||
|
||||
def _set_extra(self, sock):
|
||||
_set_socket_extra(self, sock)
|
||||
|
||||
def get_write_buffer_size(self):
|
||||
return self._buffer_size
|
||||
|
||||
def abort(self):
|
||||
self._force_close(None)
|
||||
|
||||
def sendto(self, data, addr=None):
|
||||
if not isinstance(data, (bytes, bytearray, memoryview)):
|
||||
raise TypeError('data argument must be bytes-like object (%r)',
|
||||
type(data))
|
||||
|
||||
if not data:
|
||||
return
|
||||
|
||||
if self._address is not None and addr not in (None, self._address):
|
||||
raise ValueError(
|
||||
f'Invalid address: must be None or {self._address}')
|
||||
|
||||
if self._conn_lost and self._address:
|
||||
if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
|
||||
logger.warning('socket.sendto() raised exception.')
|
||||
self._conn_lost += 1
|
||||
return
|
||||
|
||||
# Ensure that what we buffer is immutable.
|
||||
self._buffer.append((bytes(data), addr))
|
||||
self._buffer_size += len(data)
|
||||
|
||||
if self._write_fut is None:
|
||||
# No current write operations are active, kick one off
|
||||
self._loop_writing()
|
||||
# else: A write operation is already kicked off
|
||||
|
||||
self._maybe_pause_protocol()
|
||||
|
||||
def _loop_writing(self, fut=None):
|
||||
try:
|
||||
if self._conn_lost:
|
||||
return
|
||||
|
||||
assert fut is self._write_fut
|
||||
self._write_fut = None
|
||||
if fut:
|
||||
# We are in a _loop_writing() done callback, get the result
|
||||
fut.result()
|
||||
|
||||
if not self._buffer or (self._conn_lost and self._address):
|
||||
# The connection has been closed
|
||||
if self._closing:
|
||||
self._loop.call_soon(self._call_connection_lost, None)
|
||||
return
|
||||
|
||||
data, addr = self._buffer.popleft()
|
||||
self._buffer_size -= len(data)
|
||||
if self._address is not None:
|
||||
self._write_fut = self._loop._proactor.send(self._sock,
|
||||
data)
|
||||
else:
|
||||
self._write_fut = self._loop._proactor.sendto(self._sock,
|
||||
data,
|
||||
addr=addr)
|
||||
except OSError as exc:
|
||||
self._protocol.error_received(exc)
|
||||
except Exception as exc:
|
||||
self._fatal_error(exc, 'Fatal write error on datagram transport')
|
||||
else:
|
||||
self._write_fut.add_done_callback(self._loop_writing)
|
||||
self._maybe_resume_protocol()
|
||||
|
||||
def _loop_reading(self, fut=None):
|
||||
data = None
|
||||
try:
|
||||
if self._conn_lost:
|
||||
return
|
||||
|
||||
assert self._read_fut is fut or (self._read_fut is None and
|
||||
self._closing)
|
||||
|
||||
self._read_fut = None
|
||||
if fut is not None:
|
||||
res = fut.result()
|
||||
|
||||
if self._closing:
|
||||
# since close() has been called we ignore any read data
|
||||
data = None
|
||||
return
|
||||
|
||||
if self._address is not None:
|
||||
data, addr = res, self._address
|
||||
else:
|
||||
data, addr = res
|
||||
|
||||
if self._conn_lost:
|
||||
return
|
||||
if self._address is not None:
|
||||
self._read_fut = self._loop._proactor.recv(self._sock,
|
||||
self.max_size)
|
||||
else:
|
||||
self._read_fut = self._loop._proactor.recvfrom(self._sock,
|
||||
self.max_size)
|
||||
except OSError as exc:
|
||||
self._protocol.error_received(exc)
|
||||
except exceptions.CancelledError:
|
||||
if not self._closing:
|
||||
raise
|
||||
else:
|
||||
if self._read_fut is not None:
|
||||
self._read_fut.add_done_callback(self._loop_reading)
|
||||
finally:
|
||||
if data:
|
||||
self._protocol.datagram_received(data, addr)
|
||||
|
||||
|
||||
class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport,
|
||||
_ProactorBaseWritePipeTransport,
|
||||
transports.Transport):
|
||||
@@ -606,15 +349,21 @@ class _ProactorSocketTransport(_ProactorReadPipeTransport,
|
||||
transports.Transport):
|
||||
"""Transport for connected sockets."""
|
||||
|
||||
_sendfile_compatible = constants._SendfileMode.TRY_NATIVE
|
||||
|
||||
def __init__(self, loop, sock, protocol, waiter=None,
|
||||
extra=None, server=None):
|
||||
super().__init__(loop, sock, protocol, waiter, extra, server)
|
||||
base_events._set_nodelay(sock)
|
||||
|
||||
def _set_extra(self, sock):
|
||||
_set_socket_extra(self, sock)
|
||||
self._extra['socket'] = sock
|
||||
try:
|
||||
self._extra['sockname'] = sock.getsockname()
|
||||
except (socket.error, AttributeError):
|
||||
if self._loop.get_debug():
|
||||
logger.warning("getsockname() failed on %r",
|
||||
sock, exc_info=True)
|
||||
if 'peername' not in self._extra:
|
||||
try:
|
||||
self._extra['peername'] = sock.getpeername()
|
||||
except (socket.error, AttributeError):
|
||||
if self._loop.get_debug():
|
||||
logger.warning("getpeername() failed on %r",
|
||||
sock, exc_info=True)
|
||||
|
||||
def can_write_eof(self):
|
||||
return True
|
||||
@@ -638,35 +387,26 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
|
||||
self._accept_futures = {} # socket file descriptor => Future
|
||||
proactor.set_loop(self)
|
||||
self._make_self_pipe()
|
||||
if threading.current_thread() is threading.main_thread():
|
||||
# wakeup fd can only be installed to a file descriptor from the main thread
|
||||
signal.set_wakeup_fd(self._csock.fileno())
|
||||
|
||||
def _make_socket_transport(self, sock, protocol, waiter=None,
|
||||
extra=None, server=None):
|
||||
return _ProactorSocketTransport(self, sock, protocol, waiter,
|
||||
extra, server)
|
||||
|
||||
def _make_ssl_transport(
|
||||
self, rawsock, protocol, sslcontext, waiter=None,
|
||||
*, server_side=False, server_hostname=None,
|
||||
extra=None, server=None,
|
||||
ssl_handshake_timeout=None,
|
||||
ssl_shutdown_timeout=None):
|
||||
ssl_protocol = sslproto.SSLProtocol(
|
||||
self, protocol, sslcontext, waiter,
|
||||
server_side, server_hostname,
|
||||
ssl_handshake_timeout=ssl_handshake_timeout,
|
||||
ssl_shutdown_timeout=ssl_shutdown_timeout)
|
||||
def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None,
|
||||
*, server_side=False, server_hostname=None,
|
||||
extra=None, server=None):
|
||||
if not sslproto._is_sslproto_available():
|
||||
raise NotImplementedError("Proactor event loop requires Python 3.5"
|
||||
" or newer (ssl.MemoryBIO) to support "
|
||||
"SSL")
|
||||
|
||||
ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter,
|
||||
server_side, server_hostname)
|
||||
_ProactorSocketTransport(self, rawsock, ssl_protocol,
|
||||
extra=extra, server=server)
|
||||
return ssl_protocol._app_transport
|
||||
|
||||
def _make_datagram_transport(self, sock, protocol,
|
||||
address=None, waiter=None, extra=None):
|
||||
return _ProactorDatagramTransport(self, sock, protocol, address,
|
||||
waiter, extra)
|
||||
|
||||
def _make_duplex_pipe_transport(self, sock, protocol, waiter=None,
|
||||
extra=None):
|
||||
return _ProactorDuplexPipeTransport(self,
|
||||
@@ -688,8 +428,6 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
|
||||
if self.is_closed():
|
||||
return
|
||||
|
||||
if threading.current_thread() is threading.main_thread():
|
||||
signal.set_wakeup_fd(-1)
|
||||
# Call these methods before closing the event loop (before calling
|
||||
# BaseEventLoop.close), because they can schedule callbacks with
|
||||
# call_soon(), which is forbidden when the event loop is closed.
|
||||
@@ -702,73 +440,20 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
|
||||
# Close the event loop
|
||||
super().close()
|
||||
|
||||
async def sock_recv(self, sock, n):
|
||||
return await self._proactor.recv(sock, n)
|
||||
def sock_recv(self, sock, n):
|
||||
return self._proactor.recv(sock, n)
|
||||
|
||||
async def sock_recv_into(self, sock, buf):
|
||||
return await self._proactor.recv_into(sock, buf)
|
||||
def sock_sendall(self, sock, data):
|
||||
return self._proactor.send(sock, data)
|
||||
|
||||
async def sock_recvfrom(self, sock, bufsize):
|
||||
return await self._proactor.recvfrom(sock, bufsize)
|
||||
def sock_connect(self, sock, address):
|
||||
return self._proactor.connect(sock, address)
|
||||
|
||||
async def sock_recvfrom_into(self, sock, buf, nbytes=0):
|
||||
if not nbytes:
|
||||
nbytes = len(buf)
|
||||
def sock_accept(self, sock):
|
||||
return self._proactor.accept(sock)
|
||||
|
||||
return await self._proactor.recvfrom_into(sock, buf, nbytes)
|
||||
|
||||
async def sock_sendall(self, sock, data):
|
||||
return await self._proactor.send(sock, data)
|
||||
|
||||
async def sock_sendto(self, sock, data, address):
|
||||
return await self._proactor.sendto(sock, data, 0, address)
|
||||
|
||||
async def sock_connect(self, sock, address):
|
||||
return await self._proactor.connect(sock, address)
|
||||
|
||||
async def sock_accept(self, sock):
|
||||
return await self._proactor.accept(sock)
|
||||
|
||||
async def _sock_sendfile_native(self, sock, file, offset, count):
|
||||
try:
|
||||
fileno = file.fileno()
|
||||
except (AttributeError, io.UnsupportedOperation) as err:
|
||||
raise exceptions.SendfileNotAvailableError("not a regular file")
|
||||
try:
|
||||
fsize = os.fstat(fileno).st_size
|
||||
except OSError:
|
||||
raise exceptions.SendfileNotAvailableError("not a regular file")
|
||||
blocksize = count if count else fsize
|
||||
if not blocksize:
|
||||
return 0 # empty file
|
||||
|
||||
blocksize = min(blocksize, 0xffff_ffff)
|
||||
end_pos = min(offset + count, fsize) if count else fsize
|
||||
offset = min(offset, fsize)
|
||||
total_sent = 0
|
||||
try:
|
||||
while True:
|
||||
blocksize = min(end_pos - offset, blocksize)
|
||||
if blocksize <= 0:
|
||||
return total_sent
|
||||
await self._proactor.sendfile(sock, file, offset, blocksize)
|
||||
offset += blocksize
|
||||
total_sent += blocksize
|
||||
finally:
|
||||
if total_sent > 0:
|
||||
file.seek(offset)
|
||||
|
||||
async def _sendfile_native(self, transp, file, offset, count):
|
||||
resume_reading = transp.is_reading()
|
||||
transp.pause_reading()
|
||||
await transp._make_empty_waiter()
|
||||
try:
|
||||
return await self.sock_sendfile(transp._sock, file, offset, count,
|
||||
fallback=False)
|
||||
finally:
|
||||
transp._reset_empty_waiter()
|
||||
if resume_reading:
|
||||
transp.resume_reading()
|
||||
def _socketpair(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def _close_self_pipe(self):
|
||||
if self._self_reading_future is not None:
|
||||
@@ -782,30 +467,21 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
|
||||
|
||||
def _make_self_pipe(self):
|
||||
# A self-socket, really. :-)
|
||||
self._ssock, self._csock = socket.socketpair()
|
||||
self._ssock, self._csock = self._socketpair()
|
||||
self._ssock.setblocking(False)
|
||||
self._csock.setblocking(False)
|
||||
self._internal_fds += 1
|
||||
self.call_soon(self._loop_self_reading)
|
||||
|
||||
def _loop_self_reading(self, f=None):
|
||||
try:
|
||||
if f is not None:
|
||||
f.result() # may raise
|
||||
if self._self_reading_future is not f:
|
||||
# When we scheduled this Future, we assigned it to
|
||||
# _self_reading_future. If it's not there now, something has
|
||||
# tried to cancel the loop while this callback was still in the
|
||||
# queue (see windows_events.ProactorEventLoop.run_forever). In
|
||||
# that case stop here instead of continuing to schedule a new
|
||||
# iteration.
|
||||
return
|
||||
f = self._proactor.recv(self._ssock, 4096)
|
||||
except exceptions.CancelledError:
|
||||
except futures.CancelledError:
|
||||
# _close_self_pipe() has been called, stop waiting for data
|
||||
return
|
||||
except (SystemExit, KeyboardInterrupt):
|
||||
raise
|
||||
except BaseException as exc:
|
||||
except Exception as exc:
|
||||
self.call_exception_handler({
|
||||
'message': 'Error on reading from the event loop self pipe',
|
||||
'exception': exc,
|
||||
@@ -816,27 +492,10 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
|
||||
f.add_done_callback(self._loop_self_reading)
|
||||
|
||||
def _write_to_self(self):
|
||||
# This may be called from a different thread, possibly after
|
||||
# _close_self_pipe() has been called or even while it is
|
||||
# running. Guard for self._csock being None or closed. When
|
||||
# a socket is closed, send() raises OSError (with errno set to
|
||||
# EBADF, but let's not rely on the exact error code).
|
||||
csock = self._csock
|
||||
if csock is None:
|
||||
return
|
||||
|
||||
try:
|
||||
csock.send(b'\0')
|
||||
except OSError:
|
||||
if self._debug:
|
||||
logger.debug("Fail to write a null byte into the "
|
||||
"self-pipe socket",
|
||||
exc_info=True)
|
||||
self._csock.send(b'\0')
|
||||
|
||||
def _start_serving(self, protocol_factory, sock,
|
||||
sslcontext=None, server=None, backlog=100,
|
||||
ssl_handshake_timeout=None,
|
||||
ssl_shutdown_timeout=None):
|
||||
sslcontext=None, server=None, backlog=100):
|
||||
|
||||
def loop(f=None):
|
||||
try:
|
||||
@@ -849,9 +508,7 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
|
||||
if sslcontext is not None:
|
||||
self._make_ssl_transport(
|
||||
conn, protocol, sslcontext, server_side=True,
|
||||
extra={'peername': addr}, server=server,
|
||||
ssl_handshake_timeout=ssl_handshake_timeout,
|
||||
ssl_shutdown_timeout=ssl_shutdown_timeout)
|
||||
extra={'peername': addr}, server=server)
|
||||
else:
|
||||
self._make_socket_transport(
|
||||
conn, protocol,
|
||||
@@ -864,13 +521,13 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
|
||||
self.call_exception_handler({
|
||||
'message': 'Accept failed on a socket',
|
||||
'exception': exc,
|
||||
'socket': trsock.TransportSocket(sock),
|
||||
'socket': sock,
|
||||
})
|
||||
sock.close()
|
||||
elif self._debug:
|
||||
logger.debug("Accept failed on socket %r",
|
||||
sock, exc_info=True)
|
||||
except exceptions.CancelledError:
|
||||
except futures.CancelledError:
|
||||
sock.close()
|
||||
else:
|
||||
self._accept_futures[sock.fileno()] = f
|
||||
@@ -888,8 +545,6 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
|
||||
self._accept_futures.clear()
|
||||
|
||||
def _stop_serving(self, sock):
|
||||
future = self._accept_futures.pop(sock.fileno(), None)
|
||||
if future:
|
||||
future.cancel()
|
||||
self._stop_accept_futures()
|
||||
self._proactor._stop_serving(sock)
|
||||
sock.close()
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
"""Abstract Protocol base classes."""
|
||||
"""Abstract Protocol class."""
|
||||
|
||||
__all__ = (
|
||||
'BaseProtocol', 'Protocol', 'DatagramProtocol',
|
||||
'SubprocessProtocol', 'BufferedProtocol',
|
||||
)
|
||||
__all__ = ['BaseProtocol', 'Protocol', 'DatagramProtocol',
|
||||
'SubprocessProtocol']
|
||||
|
||||
|
||||
class BaseProtocol:
|
||||
@@ -16,8 +14,6 @@ class BaseProtocol:
|
||||
write-only transport like write pipe
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def connection_made(self, transport):
|
||||
"""Called when a connection is made.
|
||||
|
||||
@@ -89,8 +85,6 @@ class Protocol(BaseProtocol):
|
||||
* CL: connection_lost()
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def data_received(self, data):
|
||||
"""Called when some data is received.
|
||||
|
||||
@@ -106,64 +100,9 @@ class Protocol(BaseProtocol):
|
||||
"""
|
||||
|
||||
|
||||
class BufferedProtocol(BaseProtocol):
|
||||
"""Interface for stream protocol with manual buffer control.
|
||||
|
||||
Event methods, such as `create_server` and `create_connection`,
|
||||
accept factories that return protocols that implement this interface.
|
||||
|
||||
The idea of BufferedProtocol is that it allows to manually allocate
|
||||
and control the receive buffer. Event loops can then use the buffer
|
||||
provided by the protocol to avoid unnecessary data copies. This
|
||||
can result in noticeable performance improvement for protocols that
|
||||
receive big amounts of data. Sophisticated protocols can allocate
|
||||
the buffer only once at creation time.
|
||||
|
||||
State machine of calls:
|
||||
|
||||
start -> CM [-> GB [-> BU?]]* [-> ER?] -> CL -> end
|
||||
|
||||
* CM: connection_made()
|
||||
* GB: get_buffer()
|
||||
* BU: buffer_updated()
|
||||
* ER: eof_received()
|
||||
* CL: connection_lost()
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def get_buffer(self, sizehint):
|
||||
"""Called to allocate a new receive buffer.
|
||||
|
||||
*sizehint* is a recommended minimal size for the returned
|
||||
buffer. When set to -1, the buffer size can be arbitrary.
|
||||
|
||||
Must return an object that implements the
|
||||
:ref:`buffer protocol <bufferobjects>`.
|
||||
It is an error to return a zero-sized buffer.
|
||||
"""
|
||||
|
||||
def buffer_updated(self, nbytes):
|
||||
"""Called when the buffer was updated with the received data.
|
||||
|
||||
*nbytes* is the total number of bytes that were written to
|
||||
the buffer.
|
||||
"""
|
||||
|
||||
def eof_received(self):
|
||||
"""Called when the other end calls write_eof() or equivalent.
|
||||
|
||||
If this returns a false value (including None), the transport
|
||||
will close itself. If it returns a true value, closing the
|
||||
transport is up to the protocol.
|
||||
"""
|
||||
|
||||
|
||||
class DatagramProtocol(BaseProtocol):
|
||||
"""Interface for datagram protocol."""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def datagram_received(self, data, addr):
|
||||
"""Called when some datagram is received."""
|
||||
|
||||
@@ -177,8 +116,6 @@ class DatagramProtocol(BaseProtocol):
|
||||
class SubprocessProtocol(BaseProtocol):
|
||||
"""Interface for protocol for subprocess calls."""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def pipe_data_received(self, fd, data):
|
||||
"""Called when the subprocess writes data into stdout/stderr pipe.
|
||||
|
||||
@@ -195,22 +132,3 @@ class SubprocessProtocol(BaseProtocol):
|
||||
|
||||
def process_exited(self):
|
||||
"""Called when subprocess has exited."""
|
||||
|
||||
|
||||
def _feed_data_to_buffered_proto(proto, data):
|
||||
data_len = len(data)
|
||||
while data_len:
|
||||
buf = proto.get_buffer(data_len)
|
||||
buf_len = len(buf)
|
||||
if not buf_len:
|
||||
raise RuntimeError('get_buffer() returned an empty buffer')
|
||||
|
||||
if buf_len >= data_len:
|
||||
buf[:data_len] = data
|
||||
proto.buffer_updated(data_len)
|
||||
return
|
||||
else:
|
||||
buf[:buf_len] = data[:buf_len]
|
||||
proto.buffer_updated(buf_len)
|
||||
data = data[buf_len:]
|
||||
data_len = len(data)
|
||||
|
||||
@@ -1,28 +1,35 @@
|
||||
__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty')
|
||||
"""Queues"""
|
||||
|
||||
__all__ = ['Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty']
|
||||
|
||||
import collections
|
||||
import heapq
|
||||
from types import GenericAlias
|
||||
|
||||
from . import compat
|
||||
from . import events
|
||||
from . import locks
|
||||
from . import mixins
|
||||
from .coroutines import coroutine
|
||||
|
||||
|
||||
class QueueEmpty(Exception):
|
||||
"""Raised when Queue.get_nowait() is called on an empty Queue."""
|
||||
"""Exception raised when Queue.get_nowait() is called on a Queue object
|
||||
which is empty.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class QueueFull(Exception):
|
||||
"""Raised when the Queue.put_nowait() method is called on a full Queue."""
|
||||
"""Exception raised when the Queue.put_nowait() method is called on a Queue
|
||||
object which is full.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class Queue(mixins._LoopBoundMixin):
|
||||
class Queue:
|
||||
"""A queue, useful for coordinating producer and consumer coroutines.
|
||||
|
||||
If maxsize is less than or equal to zero, the queue size is infinite. If it
|
||||
is an integer greater than 0, then "await put()" will block when the
|
||||
is an integer greater than 0, then "yield from put()" will block when the
|
||||
queue reaches maxsize, until an item is removed by get().
|
||||
|
||||
Unlike the standard library Queue, you can reliably know this Queue's size
|
||||
@@ -30,7 +37,11 @@ class Queue(mixins._LoopBoundMixin):
|
||||
interrupted between calling qsize() and doing an operation on the Queue.
|
||||
"""
|
||||
|
||||
def __init__(self, maxsize=0):
|
||||
def __init__(self, maxsize=0, *, loop=None):
|
||||
if loop is None:
|
||||
self._loop = events.get_event_loop()
|
||||
else:
|
||||
self._loop = loop
|
||||
self._maxsize = maxsize
|
||||
|
||||
# Futures.
|
||||
@@ -38,7 +49,7 @@ class Queue(mixins._LoopBoundMixin):
|
||||
# Futures.
|
||||
self._putters = collections.deque()
|
||||
self._unfinished_tasks = 0
|
||||
self._finished = locks.Event()
|
||||
self._finished = locks.Event(loop=self._loop)
|
||||
self._finished.set()
|
||||
self._init(maxsize)
|
||||
|
||||
@@ -64,23 +75,22 @@ class Queue(mixins._LoopBoundMixin):
|
||||
break
|
||||
|
||||
def __repr__(self):
|
||||
return f'<{type(self).__name__} at {id(self):#x} {self._format()}>'
|
||||
return '<{} at {:#x} {}>'.format(
|
||||
type(self).__name__, id(self), self._format())
|
||||
|
||||
def __str__(self):
|
||||
return f'<{type(self).__name__} {self._format()}>'
|
||||
|
||||
__class_getitem__ = classmethod(GenericAlias)
|
||||
return '<{} {}>'.format(type(self).__name__, self._format())
|
||||
|
||||
def _format(self):
|
||||
result = f'maxsize={self._maxsize!r}'
|
||||
result = 'maxsize={!r}'.format(self._maxsize)
|
||||
if getattr(self, '_queue', None):
|
||||
result += f' _queue={list(self._queue)!r}'
|
||||
result += ' _queue={!r}'.format(list(self._queue))
|
||||
if self._getters:
|
||||
result += f' _getters[{len(self._getters)}]'
|
||||
result += ' _getters[{}]'.format(len(self._getters))
|
||||
if self._putters:
|
||||
result += f' _putters[{len(self._putters)}]'
|
||||
result += ' _putters[{}]'.format(len(self._putters))
|
||||
if self._unfinished_tasks:
|
||||
result += f' tasks={self._unfinished_tasks}'
|
||||
result += ' tasks={}'.format(self._unfinished_tasks)
|
||||
return result
|
||||
|
||||
def qsize(self):
|
||||
@@ -107,26 +117,22 @@ class Queue(mixins._LoopBoundMixin):
|
||||
else:
|
||||
return self.qsize() >= self._maxsize
|
||||
|
||||
async def put(self, item):
|
||||
@coroutine
|
||||
def put(self, item):
|
||||
"""Put an item into the queue.
|
||||
|
||||
Put an item into the queue. If the queue is full, wait until a free
|
||||
slot is available before adding item.
|
||||
|
||||
This method is a coroutine.
|
||||
"""
|
||||
while self.full():
|
||||
putter = self._get_loop().create_future()
|
||||
putter = self._loop.create_future()
|
||||
self._putters.append(putter)
|
||||
try:
|
||||
await putter
|
||||
yield from putter
|
||||
except:
|
||||
putter.cancel() # Just in case putter is not done yet.
|
||||
try:
|
||||
# Clean self._putters from canceled putters.
|
||||
self._putters.remove(putter)
|
||||
except ValueError:
|
||||
# The putter could be removed from self._putters by a
|
||||
# previous get_nowait call.
|
||||
pass
|
||||
if not self.full() and not putter.cancelled():
|
||||
# We were woken up by get_nowait(), but can't take
|
||||
# the call. Wake up the next in line.
|
||||
@@ -146,25 +152,21 @@ class Queue(mixins._LoopBoundMixin):
|
||||
self._finished.clear()
|
||||
self._wakeup_next(self._getters)
|
||||
|
||||
async def get(self):
|
||||
@coroutine
|
||||
def get(self):
|
||||
"""Remove and return an item from the queue.
|
||||
|
||||
If queue is empty, wait until an item is available.
|
||||
|
||||
This method is a coroutine.
|
||||
"""
|
||||
while self.empty():
|
||||
getter = self._get_loop().create_future()
|
||||
getter = self._loop.create_future()
|
||||
self._getters.append(getter)
|
||||
try:
|
||||
await getter
|
||||
yield from getter
|
||||
except:
|
||||
getter.cancel() # Just in case getter is not done yet.
|
||||
try:
|
||||
# Clean self._getters from canceled getters.
|
||||
self._getters.remove(getter)
|
||||
except ValueError:
|
||||
# The getter could be removed from self._getters by a
|
||||
# previous put_nowait call.
|
||||
pass
|
||||
if not self.empty() and not getter.cancelled():
|
||||
# We were woken up by put_nowait(), but can't take
|
||||
# the call. Wake up the next in line.
|
||||
@@ -203,7 +205,8 @@ class Queue(mixins._LoopBoundMixin):
|
||||
if self._unfinished_tasks == 0:
|
||||
self._finished.set()
|
||||
|
||||
async def join(self):
|
||||
@coroutine
|
||||
def join(self):
|
||||
"""Block until all items in the queue have been gotten and processed.
|
||||
|
||||
The count of unfinished tasks goes up whenever an item is added to the
|
||||
@@ -212,7 +215,7 @@ class Queue(mixins._LoopBoundMixin):
|
||||
When the count of unfinished tasks drops to zero, join() unblocks.
|
||||
"""
|
||||
if self._unfinished_tasks > 0:
|
||||
await self._finished.wait()
|
||||
yield from self._finished.wait()
|
||||
|
||||
|
||||
class PriorityQueue(Queue):
|
||||
@@ -242,3 +245,9 @@ class LifoQueue(Queue):
|
||||
|
||||
def _get(self):
|
||||
return self._queue.pop()
|
||||
|
||||
|
||||
if not compat.PY35:
|
||||
JoinableQueue = Queue
|
||||
"""Deprecated alias for Queue."""
|
||||
__all__.append('JoinableQueue')
|
||||
|
||||
@@ -1,168 +1,16 @@
|
||||
__all__ = ('Runner', 'run')
|
||||
__all__ = ['run']
|
||||
|
||||
import contextvars
|
||||
import enum
|
||||
import functools
|
||||
import threading
|
||||
import signal
|
||||
from . import coroutines
|
||||
from . import events
|
||||
from . import exceptions
|
||||
from . import tasks
|
||||
from . import constants
|
||||
|
||||
class _State(enum.Enum):
|
||||
CREATED = "created"
|
||||
INITIALIZED = "initialized"
|
||||
CLOSED = "closed"
|
||||
|
||||
|
||||
class Runner:
|
||||
"""A context manager that controls event loop life cycle.
|
||||
|
||||
The context manager always creates a new event loop,
|
||||
allows to run async functions inside it,
|
||||
and properly finalizes the loop at the context manager exit.
|
||||
|
||||
If debug is True, the event loop will be run in debug mode.
|
||||
If loop_factory is passed, it is used for new event loop creation.
|
||||
|
||||
asyncio.run(main(), debug=True)
|
||||
|
||||
is a shortcut for
|
||||
|
||||
with asyncio.Runner(debug=True) as runner:
|
||||
runner.run(main())
|
||||
|
||||
The run() method can be called multiple times within the runner's context.
|
||||
|
||||
This can be useful for interactive console (e.g. IPython),
|
||||
unittest runners, console tools, -- everywhere when async code
|
||||
is called from existing sync framework and where the preferred single
|
||||
asyncio.run() call doesn't work.
|
||||
|
||||
"""
|
||||
|
||||
# Note: the class is final, it is not intended for inheritance.
|
||||
|
||||
def __init__(self, *, debug=None, loop_factory=None):
|
||||
self._state = _State.CREATED
|
||||
self._debug = debug
|
||||
self._loop_factory = loop_factory
|
||||
self._loop = None
|
||||
self._context = None
|
||||
self._interrupt_count = 0
|
||||
self._set_event_loop = False
|
||||
|
||||
def __enter__(self):
|
||||
self._lazy_init()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
"""Shutdown and close event loop."""
|
||||
if self._state is not _State.INITIALIZED:
|
||||
return
|
||||
try:
|
||||
loop = self._loop
|
||||
_cancel_all_tasks(loop)
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
loop.run_until_complete(
|
||||
loop.shutdown_default_executor(constants.THREAD_JOIN_TIMEOUT))
|
||||
finally:
|
||||
if self._set_event_loop:
|
||||
events.set_event_loop(None)
|
||||
loop.close()
|
||||
self._loop = None
|
||||
self._state = _State.CLOSED
|
||||
|
||||
def get_loop(self):
|
||||
"""Return embedded event loop."""
|
||||
self._lazy_init()
|
||||
return self._loop
|
||||
|
||||
def run(self, coro, *, context=None):
|
||||
"""Run a coroutine inside the embedded event loop."""
|
||||
if not coroutines.iscoroutine(coro):
|
||||
raise ValueError("a coroutine was expected, got {!r}".format(coro))
|
||||
|
||||
if events._get_running_loop() is not None:
|
||||
# fail fast with short traceback
|
||||
raise RuntimeError(
|
||||
"Runner.run() cannot be called from a running event loop")
|
||||
|
||||
self._lazy_init()
|
||||
|
||||
if context is None:
|
||||
context = self._context
|
||||
task = self._loop.create_task(coro, context=context)
|
||||
|
||||
if (threading.current_thread() is threading.main_thread()
|
||||
and signal.getsignal(signal.SIGINT) is signal.default_int_handler
|
||||
):
|
||||
sigint_handler = functools.partial(self._on_sigint, main_task=task)
|
||||
try:
|
||||
signal.signal(signal.SIGINT, sigint_handler)
|
||||
except ValueError:
|
||||
# `signal.signal` may throw if `threading.main_thread` does
|
||||
# not support signals (e.g. embedded interpreter with signals
|
||||
# not registered - see gh-91880)
|
||||
sigint_handler = None
|
||||
else:
|
||||
sigint_handler = None
|
||||
|
||||
self._interrupt_count = 0
|
||||
try:
|
||||
return self._loop.run_until_complete(task)
|
||||
except exceptions.CancelledError:
|
||||
if self._interrupt_count > 0:
|
||||
uncancel = getattr(task, "uncancel", None)
|
||||
if uncancel is not None and uncancel() == 0:
|
||||
raise KeyboardInterrupt()
|
||||
raise # CancelledError
|
||||
finally:
|
||||
if (sigint_handler is not None
|
||||
and signal.getsignal(signal.SIGINT) is sigint_handler
|
||||
):
|
||||
signal.signal(signal.SIGINT, signal.default_int_handler)
|
||||
|
||||
def _lazy_init(self):
|
||||
if self._state is _State.CLOSED:
|
||||
raise RuntimeError("Runner is closed")
|
||||
if self._state is _State.INITIALIZED:
|
||||
return
|
||||
if self._loop_factory is None:
|
||||
self._loop = events.new_event_loop()
|
||||
if not self._set_event_loop:
|
||||
# Call set_event_loop only once to avoid calling
|
||||
# attach_loop multiple times on child watchers
|
||||
events.set_event_loop(self._loop)
|
||||
self._set_event_loop = True
|
||||
else:
|
||||
self._loop = self._loop_factory()
|
||||
if self._debug is not None:
|
||||
self._loop.set_debug(self._debug)
|
||||
self._context = contextvars.copy_context()
|
||||
self._state = _State.INITIALIZED
|
||||
|
||||
def _on_sigint(self, signum, frame, main_task):
|
||||
self._interrupt_count += 1
|
||||
if self._interrupt_count == 1 and not main_task.done():
|
||||
main_task.cancel()
|
||||
# wakeup loop if it is blocked by select() with long timeout
|
||||
self._loop.call_soon_threadsafe(lambda: None)
|
||||
return
|
||||
raise KeyboardInterrupt()
|
||||
|
||||
|
||||
def run(main, *, debug=None, loop_factory=None):
|
||||
"""Execute the coroutine and return the result.
|
||||
def run(main, *, debug=False):
|
||||
"""Run a coroutine.
|
||||
|
||||
This function runs the passed coroutine, taking care of
|
||||
managing the asyncio event loop, finalizing asynchronous
|
||||
generators and closing the default executor.
|
||||
managing the asyncio event loop and finalizing asynchronous
|
||||
generators.
|
||||
|
||||
This function cannot be called when another asyncio event loop is
|
||||
running in the same thread.
|
||||
@@ -173,10 +21,6 @@ def run(main, *, debug=None, loop_factory=None):
|
||||
It should be used as a main entry point for asyncio programs, and should
|
||||
ideally only be called once.
|
||||
|
||||
The executor is given a timeout duration of 5 minutes to shutdown.
|
||||
If the executor hasn't finished within that duration, a warning is
|
||||
emitted and the executor is closed.
|
||||
|
||||
Example:
|
||||
|
||||
async def main():
|
||||
@@ -186,12 +30,24 @@ def run(main, *, debug=None, loop_factory=None):
|
||||
asyncio.run(main())
|
||||
"""
|
||||
if events._get_running_loop() is not None:
|
||||
# fail fast with short traceback
|
||||
raise RuntimeError(
|
||||
"asyncio.run() cannot be called from a running event loop")
|
||||
|
||||
with Runner(debug=debug, loop_factory=loop_factory) as runner:
|
||||
return runner.run(main)
|
||||
if not coroutines.iscoroutine(main):
|
||||
raise ValueError("a coroutine was expected, got {!r}".format(main))
|
||||
|
||||
loop = events.new_event_loop()
|
||||
try:
|
||||
events.set_event_loop(loop)
|
||||
loop.set_debug(debug)
|
||||
return loop.run_until_complete(main)
|
||||
finally:
|
||||
try:
|
||||
_cancel_all_tasks(loop)
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
finally:
|
||||
events.set_event_loop(None)
|
||||
loop.close()
|
||||
|
||||
|
||||
def _cancel_all_tasks(loop):
|
||||
@@ -202,7 +58,8 @@ def _cancel_all_tasks(loop):
|
||||
for task in to_cancel:
|
||||
task.cancel()
|
||||
|
||||
loop.run_until_complete(tasks.gather(*to_cancel, return_exceptions=True))
|
||||
loop.run_until_complete(
|
||||
tasks.gather(*to_cancel, loop=loop, return_exceptions=True))
|
||||
|
||||
for task in to_cancel:
|
||||
if task.cancelled():
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,149 +0,0 @@
|
||||
"""Support for running coroutines in parallel with staggered start times."""
|
||||
|
||||
__all__ = 'staggered_race',
|
||||
|
||||
import contextlib
|
||||
import typing
|
||||
|
||||
from . import events
|
||||
from . import exceptions as exceptions_mod
|
||||
from . import locks
|
||||
from . import tasks
|
||||
|
||||
|
||||
async def staggered_race(
|
||||
coro_fns: typing.Iterable[typing.Callable[[], typing.Awaitable]],
|
||||
delay: typing.Optional[float],
|
||||
*,
|
||||
loop: events.AbstractEventLoop = None,
|
||||
) -> typing.Tuple[
|
||||
typing.Any,
|
||||
typing.Optional[int],
|
||||
typing.List[typing.Optional[Exception]]
|
||||
]:
|
||||
"""Run coroutines with staggered start times and take the first to finish.
|
||||
|
||||
This method takes an iterable of coroutine functions. The first one is
|
||||
started immediately. From then on, whenever the immediately preceding one
|
||||
fails (raises an exception), or when *delay* seconds has passed, the next
|
||||
coroutine is started. This continues until one of the coroutines complete
|
||||
successfully, in which case all others are cancelled, or until all
|
||||
coroutines fail.
|
||||
|
||||
The coroutines provided should be well-behaved in the following way:
|
||||
|
||||
* They should only ``return`` if completed successfully.
|
||||
|
||||
* They should always raise an exception if they did not complete
|
||||
successfully. In particular, if they handle cancellation, they should
|
||||
probably reraise, like this::
|
||||
|
||||
try:
|
||||
# do work
|
||||
except asyncio.CancelledError:
|
||||
# undo partially completed work
|
||||
raise
|
||||
|
||||
Args:
|
||||
coro_fns: an iterable of coroutine functions, i.e. callables that
|
||||
return a coroutine object when called. Use ``functools.partial`` or
|
||||
lambdas to pass arguments.
|
||||
|
||||
delay: amount of time, in seconds, between starting coroutines. If
|
||||
``None``, the coroutines will run sequentially.
|
||||
|
||||
loop: the event loop to use.
|
||||
|
||||
Returns:
|
||||
tuple *(winner_result, winner_index, exceptions)* where
|
||||
|
||||
- *winner_result*: the result of the winning coroutine, or ``None``
|
||||
if no coroutines won.
|
||||
|
||||
- *winner_index*: the index of the winning coroutine in
|
||||
``coro_fns``, or ``None`` if no coroutines won. If the winning
|
||||
coroutine may return None on success, *winner_index* can be used
|
||||
to definitively determine whether any coroutine won.
|
||||
|
||||
- *exceptions*: list of exceptions returned by the coroutines.
|
||||
``len(exceptions)`` is equal to the number of coroutines actually
|
||||
started, and the order is the same as in ``coro_fns``. The winning
|
||||
coroutine's entry is ``None``.
|
||||
|
||||
"""
|
||||
# TODO: when we have aiter() and anext(), allow async iterables in coro_fns.
|
||||
loop = loop or events.get_running_loop()
|
||||
enum_coro_fns = enumerate(coro_fns)
|
||||
winner_result = None
|
||||
winner_index = None
|
||||
exceptions = []
|
||||
running_tasks = []
|
||||
|
||||
async def run_one_coro(
|
||||
previous_failed: typing.Optional[locks.Event]) -> None:
|
||||
# Wait for the previous task to finish, or for delay seconds
|
||||
if previous_failed is not None:
|
||||
with contextlib.suppress(exceptions_mod.TimeoutError):
|
||||
# Use asyncio.wait_for() instead of asyncio.wait() here, so
|
||||
# that if we get cancelled at this point, Event.wait() is also
|
||||
# cancelled, otherwise there will be a "Task destroyed but it is
|
||||
# pending" later.
|
||||
await tasks.wait_for(previous_failed.wait(), delay)
|
||||
# Get the next coroutine to run
|
||||
try:
|
||||
this_index, coro_fn = next(enum_coro_fns)
|
||||
except StopIteration:
|
||||
return
|
||||
# Start task that will run the next coroutine
|
||||
this_failed = locks.Event()
|
||||
next_task = loop.create_task(run_one_coro(this_failed))
|
||||
running_tasks.append(next_task)
|
||||
assert len(running_tasks) == this_index + 2
|
||||
# Prepare place to put this coroutine's exceptions if not won
|
||||
exceptions.append(None)
|
||||
assert len(exceptions) == this_index + 1
|
||||
|
||||
try:
|
||||
result = await coro_fn()
|
||||
except (SystemExit, KeyboardInterrupt):
|
||||
raise
|
||||
except BaseException as e:
|
||||
exceptions[this_index] = e
|
||||
this_failed.set() # Kickstart the next coroutine
|
||||
else:
|
||||
# Store winner's results
|
||||
nonlocal winner_index, winner_result
|
||||
assert winner_index is None
|
||||
winner_index = this_index
|
||||
winner_result = result
|
||||
# Cancel all other tasks. We take care to not cancel the current
|
||||
# task as well. If we do so, then since there is no `await` after
|
||||
# here and CancelledError are usually thrown at one, we will
|
||||
# encounter a curious corner case where the current task will end
|
||||
# up as done() == True, cancelled() == False, exception() ==
|
||||
# asyncio.CancelledError. This behavior is specified in
|
||||
# https://bugs.python.org/issue30048
|
||||
for i, t in enumerate(running_tasks):
|
||||
if i != this_index:
|
||||
t.cancel()
|
||||
|
||||
first_task = loop.create_task(run_one_coro(None))
|
||||
running_tasks.append(first_task)
|
||||
try:
|
||||
# Wait for a growing list of tasks to all finish: poor man's version of
|
||||
# curio's TaskGroup or trio's nursery
|
||||
done_count = 0
|
||||
while done_count != len(running_tasks):
|
||||
done, _ = await tasks.wait(running_tasks)
|
||||
done_count = len(done)
|
||||
# If run_one_coro raises an unhandled exception, it's probably a
|
||||
# programming error, and I want to see it.
|
||||
if __debug__:
|
||||
for d in done:
|
||||
if d.done() and not d.cancelled() and d.exception():
|
||||
raise d.exception()
|
||||
return winner_result, winner_index, exceptions
|
||||
finally:
|
||||
# Make sure no tasks are left running if we leave this function
|
||||
for t in running_tasks:
|
||||
t.cancel()
|
||||
@@ -1,30 +1,55 @@
|
||||
__all__ = (
|
||||
'StreamReader', 'StreamWriter', 'StreamReaderProtocol',
|
||||
'open_connection', 'start_server')
|
||||
"""Stream-related things."""
|
||||
|
||||
__all__ = ['StreamReader', 'StreamWriter', 'StreamReaderProtocol',
|
||||
'open_connection', 'start_server',
|
||||
'IncompleteReadError',
|
||||
'LimitOverrunError',
|
||||
]
|
||||
|
||||
import collections
|
||||
import socket
|
||||
import sys
|
||||
import warnings
|
||||
import weakref
|
||||
|
||||
if hasattr(socket, 'AF_UNIX'):
|
||||
__all__ += ('open_unix_connection', 'start_unix_server')
|
||||
__all__.extend(['open_unix_connection', 'start_unix_server'])
|
||||
|
||||
from . import coroutines
|
||||
from . import compat
|
||||
from . import events
|
||||
from . import exceptions
|
||||
from . import format_helpers
|
||||
from . import protocols
|
||||
from .coroutines import coroutine
|
||||
from .log import logger
|
||||
from .tasks import sleep
|
||||
|
||||
|
||||
_DEFAULT_LIMIT = 2 ** 16 # 64 KiB
|
||||
_DEFAULT_LIMIT = 2 ** 16
|
||||
|
||||
|
||||
async def open_connection(host=None, port=None, *,
|
||||
limit=_DEFAULT_LIMIT, **kwds):
|
||||
class IncompleteReadError(EOFError):
|
||||
"""
|
||||
Incomplete read error. Attributes:
|
||||
|
||||
- partial: read bytes string before the end of stream was reached
|
||||
- expected: total number of expected bytes (or None if unknown)
|
||||
"""
|
||||
def __init__(self, partial, expected):
|
||||
super().__init__("%d bytes read on a total of %r expected bytes"
|
||||
% (len(partial), expected))
|
||||
self.partial = partial
|
||||
self.expected = expected
|
||||
|
||||
|
||||
class LimitOverrunError(Exception):
|
||||
"""Reached the buffer limit while looking for a separator.
|
||||
|
||||
Attributes:
|
||||
- consumed: total number of to be consumed bytes.
|
||||
"""
|
||||
def __init__(self, message, consumed):
|
||||
super().__init__(message)
|
||||
self.consumed = consumed
|
||||
|
||||
|
||||
@coroutine
|
||||
def open_connection(host=None, port=None, *,
|
||||
loop=None, limit=_DEFAULT_LIMIT, **kwds):
|
||||
"""A wrapper for create_connection() returning a (reader, writer) pair.
|
||||
|
||||
The reader returned is a StreamReader instance; the writer is a
|
||||
@@ -42,17 +67,19 @@ async def open_connection(host=None, port=None, *,
|
||||
StreamReaderProtocol classes, just copy the code -- there's
|
||||
really nothing special here except some convenience.)
|
||||
"""
|
||||
loop = events.get_running_loop()
|
||||
if loop is None:
|
||||
loop = events.get_event_loop()
|
||||
reader = StreamReader(limit=limit, loop=loop)
|
||||
protocol = StreamReaderProtocol(reader, loop=loop)
|
||||
transport, _ = await loop.create_connection(
|
||||
transport, _ = yield from loop.create_connection(
|
||||
lambda: protocol, host, port, **kwds)
|
||||
writer = StreamWriter(transport, protocol, reader, loop)
|
||||
return reader, writer
|
||||
|
||||
|
||||
async def start_server(client_connected_cb, host=None, port=None, *,
|
||||
limit=_DEFAULT_LIMIT, **kwds):
|
||||
@coroutine
|
||||
def start_server(client_connected_cb, host=None, port=None, *,
|
||||
loop=None, limit=_DEFAULT_LIMIT, **kwds):
|
||||
"""Start a socket server, call back for each client connected.
|
||||
|
||||
The first parameter, `client_connected_cb`, takes two parameters:
|
||||
@@ -67,13 +94,15 @@ async def start_server(client_connected_cb, host=None, port=None, *,
|
||||
positional host and port, with various optional keyword arguments
|
||||
following. The return value is the same as loop.create_server().
|
||||
|
||||
Additional optional keyword argument is limit (to set the buffer
|
||||
limit passed to the StreamReader).
|
||||
Additional optional keyword arguments are loop (to set the event loop
|
||||
instance to use) and limit (to set the buffer limit passed to the
|
||||
StreamReader).
|
||||
|
||||
The return value is the same as loop.create_server(), i.e. a
|
||||
Server object which can be used to stop the service.
|
||||
"""
|
||||
loop = events.get_running_loop()
|
||||
if loop is None:
|
||||
loop = events.get_event_loop()
|
||||
|
||||
def factory():
|
||||
reader = StreamReader(limit=limit, loop=loop)
|
||||
@@ -81,28 +110,31 @@ async def start_server(client_connected_cb, host=None, port=None, *,
|
||||
loop=loop)
|
||||
return protocol
|
||||
|
||||
return await loop.create_server(factory, host, port, **kwds)
|
||||
return (yield from loop.create_server(factory, host, port, **kwds))
|
||||
|
||||
|
||||
if hasattr(socket, 'AF_UNIX'):
|
||||
# UNIX Domain Sockets are supported on this platform
|
||||
|
||||
async def open_unix_connection(path=None, *,
|
||||
limit=_DEFAULT_LIMIT, **kwds):
|
||||
@coroutine
|
||||
def open_unix_connection(path=None, *,
|
||||
loop=None, limit=_DEFAULT_LIMIT, **kwds):
|
||||
"""Similar to `open_connection` but works with UNIX Domain Sockets."""
|
||||
loop = events.get_running_loop()
|
||||
|
||||
if loop is None:
|
||||
loop = events.get_event_loop()
|
||||
reader = StreamReader(limit=limit, loop=loop)
|
||||
protocol = StreamReaderProtocol(reader, loop=loop)
|
||||
transport, _ = await loop.create_unix_connection(
|
||||
transport, _ = yield from loop.create_unix_connection(
|
||||
lambda: protocol, path, **kwds)
|
||||
writer = StreamWriter(transport, protocol, reader, loop)
|
||||
return reader, writer
|
||||
|
||||
async def start_unix_server(client_connected_cb, path=None, *,
|
||||
limit=_DEFAULT_LIMIT, **kwds):
|
||||
@coroutine
|
||||
def start_unix_server(client_connected_cb, path=None, *,
|
||||
loop=None, limit=_DEFAULT_LIMIT, **kwds):
|
||||
"""Similar to `start_server` but works with UNIX Domain Sockets."""
|
||||
loop = events.get_running_loop()
|
||||
if loop is None:
|
||||
loop = events.get_event_loop()
|
||||
|
||||
def factory():
|
||||
reader = StreamReader(limit=limit, loop=loop)
|
||||
@@ -110,14 +142,14 @@ if hasattr(socket, 'AF_UNIX'):
|
||||
loop=loop)
|
||||
return protocol
|
||||
|
||||
return await loop.create_unix_server(factory, path, **kwds)
|
||||
return (yield from loop.create_unix_server(factory, path, **kwds))
|
||||
|
||||
|
||||
class FlowControlMixin(protocols.Protocol):
|
||||
"""Reusable flow control logic for StreamWriter.drain().
|
||||
|
||||
This implements the protocol methods pause_writing(),
|
||||
resume_writing() and connection_lost(). If the subclass overrides
|
||||
resume_reading() and connection_lost(). If the subclass overrides
|
||||
these it must call the super methods.
|
||||
|
||||
StreamWriter.drain() must wait for _drain_helper() coroutine.
|
||||
@@ -129,7 +161,7 @@ class FlowControlMixin(protocols.Protocol):
|
||||
else:
|
||||
self._loop = loop
|
||||
self._paused = False
|
||||
self._drain_waiters = collections.deque()
|
||||
self._drain_waiter = None
|
||||
self._connection_lost = False
|
||||
|
||||
def pause_writing(self):
|
||||
@@ -144,37 +176,39 @@ class FlowControlMixin(protocols.Protocol):
|
||||
if self._loop.get_debug():
|
||||
logger.debug("%r resumes writing", self)
|
||||
|
||||
for waiter in self._drain_waiters:
|
||||
waiter = self._drain_waiter
|
||||
if waiter is not None:
|
||||
self._drain_waiter = None
|
||||
if not waiter.done():
|
||||
waiter.set_result(None)
|
||||
|
||||
def connection_lost(self, exc):
|
||||
self._connection_lost = True
|
||||
# Wake up the writer(s) if currently paused.
|
||||
# Wake up the writer if currently paused.
|
||||
if not self._paused:
|
||||
return
|
||||
waiter = self._drain_waiter
|
||||
if waiter is None:
|
||||
return
|
||||
self._drain_waiter = None
|
||||
if waiter.done():
|
||||
return
|
||||
if exc is None:
|
||||
waiter.set_result(None)
|
||||
else:
|
||||
waiter.set_exception(exc)
|
||||
|
||||
for waiter in self._drain_waiters:
|
||||
if not waiter.done():
|
||||
if exc is None:
|
||||
waiter.set_result(None)
|
||||
else:
|
||||
waiter.set_exception(exc)
|
||||
|
||||
async def _drain_helper(self):
|
||||
@coroutine
|
||||
def _drain_helper(self):
|
||||
if self._connection_lost:
|
||||
raise ConnectionResetError('Connection lost')
|
||||
if not self._paused:
|
||||
return
|
||||
waiter = self._drain_waiter
|
||||
assert waiter is None or waiter.cancelled()
|
||||
waiter = self._loop.create_future()
|
||||
self._drain_waiters.append(waiter)
|
||||
try:
|
||||
await waiter
|
||||
finally:
|
||||
self._drain_waiters.remove(waiter)
|
||||
|
||||
def _get_close_waiter(self, stream):
|
||||
raise NotImplementedError
|
||||
self._drain_waiter = waiter
|
||||
yield from waiter
|
||||
|
||||
|
||||
class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
|
||||
@@ -186,110 +220,40 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
|
||||
call inappropriate methods of the protocol.)
|
||||
"""
|
||||
|
||||
_source_traceback = None
|
||||
|
||||
def __init__(self, stream_reader, client_connected_cb=None, loop=None):
|
||||
super().__init__(loop=loop)
|
||||
if stream_reader is not None:
|
||||
self._stream_reader_wr = weakref.ref(stream_reader)
|
||||
self._source_traceback = stream_reader._source_traceback
|
||||
else:
|
||||
self._stream_reader_wr = None
|
||||
if client_connected_cb is not None:
|
||||
# This is a stream created by the `create_server()` function.
|
||||
# Keep a strong reference to the reader until a connection
|
||||
# is established.
|
||||
self._strong_reader = stream_reader
|
||||
self._reject_connection = False
|
||||
self._stream_reader = stream_reader
|
||||
self._stream_writer = None
|
||||
self._task = None
|
||||
self._transport = None
|
||||
self._client_connected_cb = client_connected_cb
|
||||
self._over_ssl = False
|
||||
self._closed = self._loop.create_future()
|
||||
|
||||
@property
|
||||
def _stream_reader(self):
|
||||
if self._stream_reader_wr is None:
|
||||
return None
|
||||
return self._stream_reader_wr()
|
||||
|
||||
def _replace_writer(self, writer):
|
||||
loop = self._loop
|
||||
transport = writer.transport
|
||||
self._stream_writer = writer
|
||||
self._transport = transport
|
||||
self._over_ssl = transport.get_extra_info('sslcontext') is not None
|
||||
|
||||
def connection_made(self, transport):
|
||||
if self._reject_connection:
|
||||
context = {
|
||||
'message': ('An open stream was garbage collected prior to '
|
||||
'establishing network connection; '
|
||||
'call "stream.close()" explicitly.')
|
||||
}
|
||||
if self._source_traceback:
|
||||
context['source_traceback'] = self._source_traceback
|
||||
self._loop.call_exception_handler(context)
|
||||
transport.abort()
|
||||
return
|
||||
self._transport = transport
|
||||
reader = self._stream_reader
|
||||
if reader is not None:
|
||||
reader.set_transport(transport)
|
||||
self._stream_reader.set_transport(transport)
|
||||
self._over_ssl = transport.get_extra_info('sslcontext') is not None
|
||||
if self._client_connected_cb is not None:
|
||||
self._stream_writer = StreamWriter(transport, self,
|
||||
reader,
|
||||
self._stream_reader,
|
||||
self._loop)
|
||||
res = self._client_connected_cb(reader,
|
||||
res = self._client_connected_cb(self._stream_reader,
|
||||
self._stream_writer)
|
||||
if coroutines.iscoroutine(res):
|
||||
def callback(task):
|
||||
if task.cancelled():
|
||||
transport.close()
|
||||
return
|
||||
exc = task.exception()
|
||||
if exc is not None:
|
||||
self._loop.call_exception_handler({
|
||||
'message': 'Unhandled exception in client_connected_cb',
|
||||
'exception': exc,
|
||||
'transport': transport,
|
||||
})
|
||||
transport.close()
|
||||
|
||||
self._task = self._loop.create_task(res)
|
||||
self._task.add_done_callback(callback)
|
||||
|
||||
self._strong_reader = None
|
||||
self._loop.create_task(res)
|
||||
|
||||
def connection_lost(self, exc):
|
||||
reader = self._stream_reader
|
||||
if reader is not None:
|
||||
if self._stream_reader is not None:
|
||||
if exc is None:
|
||||
reader.feed_eof()
|
||||
self._stream_reader.feed_eof()
|
||||
else:
|
||||
reader.set_exception(exc)
|
||||
if not self._closed.done():
|
||||
if exc is None:
|
||||
self._closed.set_result(None)
|
||||
else:
|
||||
self._closed.set_exception(exc)
|
||||
self._stream_reader.set_exception(exc)
|
||||
super().connection_lost(exc)
|
||||
self._stream_reader_wr = None
|
||||
self._stream_reader = None
|
||||
self._stream_writer = None
|
||||
self._task = None
|
||||
self._transport = None
|
||||
|
||||
def data_received(self, data):
|
||||
reader = self._stream_reader
|
||||
if reader is not None:
|
||||
reader.feed_data(data)
|
||||
self._stream_reader.feed_data(data)
|
||||
|
||||
def eof_received(self):
|
||||
reader = self._stream_reader
|
||||
if reader is not None:
|
||||
reader.feed_eof()
|
||||
self._stream_reader.feed_eof()
|
||||
if self._over_ssl:
|
||||
# Prevent a warning in SSLProtocol.eof_received:
|
||||
# "returning true from eof_received()
|
||||
@@ -297,20 +261,6 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _get_close_waiter(self, stream):
|
||||
return self._closed
|
||||
|
||||
def __del__(self):
|
||||
# Prevent reports about unhandled exceptions.
|
||||
# Better than self._closed._log_traceback = False hack
|
||||
try:
|
||||
closed = self._closed
|
||||
except AttributeError:
|
||||
pass # failed constructor
|
||||
else:
|
||||
if closed.done() and not closed.cancelled():
|
||||
closed.exception()
|
||||
|
||||
|
||||
class StreamWriter:
|
||||
"""Wraps a Transport.
|
||||
@@ -329,14 +279,12 @@ class StreamWriter:
|
||||
assert reader is None or isinstance(reader, StreamReader)
|
||||
self._reader = reader
|
||||
self._loop = loop
|
||||
self._complete_fut = self._loop.create_future()
|
||||
self._complete_fut.set_result(None)
|
||||
|
||||
def __repr__(self):
|
||||
info = [self.__class__.__name__, f'transport={self._transport!r}']
|
||||
info = [self.__class__.__name__, 'transport=%r' % self._transport]
|
||||
if self._reader is not None:
|
||||
info.append(f'reader={self._reader!r}')
|
||||
return '<{}>'.format(' '.join(info))
|
||||
info.append('reader=%r' % self._reader)
|
||||
return '<%s>' % ' '.join(info)
|
||||
|
||||
@property
|
||||
def transport(self):
|
||||
@@ -357,68 +305,36 @@ class StreamWriter:
|
||||
def close(self):
|
||||
return self._transport.close()
|
||||
|
||||
def is_closing(self):
|
||||
return self._transport.is_closing()
|
||||
|
||||
async def wait_closed(self):
|
||||
await self._protocol._get_close_waiter(self)
|
||||
|
||||
def get_extra_info(self, name, default=None):
|
||||
return self._transport.get_extra_info(name, default)
|
||||
|
||||
async def drain(self):
|
||||
@coroutine
|
||||
def drain(self):
|
||||
"""Flush the write buffer.
|
||||
|
||||
The intended use is to write
|
||||
|
||||
w.write(data)
|
||||
await w.drain()
|
||||
yield from w.drain()
|
||||
"""
|
||||
if self._reader is not None:
|
||||
exc = self._reader.exception()
|
||||
if exc is not None:
|
||||
raise exc
|
||||
if self._transport.is_closing():
|
||||
# Wait for protocol.connection_lost() call
|
||||
# Raise connection closing error if any,
|
||||
# ConnectionResetError otherwise
|
||||
# Yield to the event loop so connection_lost() may be
|
||||
# called. Without this, _drain_helper() would return
|
||||
# immediately, and code that calls
|
||||
# write(...); await drain()
|
||||
# in a loop would never call connection_lost(), so it
|
||||
# would not see an error when the socket is closed.
|
||||
await sleep(0)
|
||||
await self._protocol._drain_helper()
|
||||
if self._transport is not None:
|
||||
if self._transport.is_closing():
|
||||
# Yield to the event loop so connection_lost() may be
|
||||
# called. Without this, _drain_helper() would return
|
||||
# immediately, and code that calls
|
||||
# write(...); yield from drain()
|
||||
# in a loop would never call connection_lost(), so it
|
||||
# would not see an error when the socket is closed.
|
||||
yield
|
||||
yield from self._protocol._drain_helper()
|
||||
|
||||
async def start_tls(self, sslcontext, *,
|
||||
server_hostname=None,
|
||||
ssl_handshake_timeout=None,
|
||||
ssl_shutdown_timeout=None):
|
||||
"""Upgrade an existing stream-based connection to TLS."""
|
||||
server_side = self._protocol._client_connected_cb is not None
|
||||
protocol = self._protocol
|
||||
await self.drain()
|
||||
new_transport = await self._loop.start_tls( # type: ignore
|
||||
self._transport, protocol, sslcontext,
|
||||
server_side=server_side, server_hostname=server_hostname,
|
||||
ssl_handshake_timeout=ssl_handshake_timeout,
|
||||
ssl_shutdown_timeout=ssl_shutdown_timeout)
|
||||
self._transport = new_transport
|
||||
protocol._replace_writer(self)
|
||||
|
||||
def __del__(self):
|
||||
if not self._transport.is_closing():
|
||||
if self._loop.is_closed():
|
||||
warnings.warn("loop is closed", ResourceWarning)
|
||||
else:
|
||||
self.close()
|
||||
warnings.warn(f"unclosed {self!r}", ResourceWarning)
|
||||
|
||||
class StreamReader:
|
||||
|
||||
_source_traceback = None
|
||||
|
||||
def __init__(self, limit=_DEFAULT_LIMIT, loop=None):
|
||||
# The line length limit is a security feature;
|
||||
# it also doubles as half the buffer limit.
|
||||
@@ -437,27 +353,24 @@ class StreamReader:
|
||||
self._exception = None
|
||||
self._transport = None
|
||||
self._paused = False
|
||||
if self._loop.get_debug():
|
||||
self._source_traceback = format_helpers.extract_stack(
|
||||
sys._getframe(1))
|
||||
|
||||
def __repr__(self):
|
||||
info = ['StreamReader']
|
||||
if self._buffer:
|
||||
info.append(f'{len(self._buffer)} bytes')
|
||||
info.append('%d bytes' % len(self._buffer))
|
||||
if self._eof:
|
||||
info.append('eof')
|
||||
if self._limit != _DEFAULT_LIMIT:
|
||||
info.append(f'limit={self._limit}')
|
||||
info.append('l=%d' % self._limit)
|
||||
if self._waiter:
|
||||
info.append(f'waiter={self._waiter!r}')
|
||||
info.append('w=%r' % self._waiter)
|
||||
if self._exception:
|
||||
info.append(f'exception={self._exception!r}')
|
||||
info.append('e=%r' % self._exception)
|
||||
if self._transport:
|
||||
info.append(f'transport={self._transport!r}')
|
||||
info.append('t=%r' % self._transport)
|
||||
if self._paused:
|
||||
info.append('paused')
|
||||
return '<{}>'.format(' '.join(info))
|
||||
return '<%s>' % ' '.join(info)
|
||||
|
||||
def exception(self):
|
||||
return self._exception
|
||||
@@ -518,7 +431,8 @@ class StreamReader:
|
||||
else:
|
||||
self._paused = True
|
||||
|
||||
async def _wait_for_data(self, func_name):
|
||||
@coroutine
|
||||
def _wait_for_data(self, func_name):
|
||||
"""Wait until feed_data() or feed_eof() is called.
|
||||
|
||||
If stream was paused, automatically resume it.
|
||||
@@ -528,9 +442,8 @@ class StreamReader:
|
||||
# would have an unexpected behaviour. It would not possible to know
|
||||
# which coroutine would get the next data.
|
||||
if self._waiter is not None:
|
||||
raise RuntimeError(
|
||||
f'{func_name}() called while another coroutine is '
|
||||
f'already waiting for incoming data')
|
||||
raise RuntimeError('%s() called while another coroutine is '
|
||||
'already waiting for incoming data' % func_name)
|
||||
|
||||
assert not self._eof, '_wait_for_data after EOF'
|
||||
|
||||
@@ -542,11 +455,12 @@ class StreamReader:
|
||||
|
||||
self._waiter = self._loop.create_future()
|
||||
try:
|
||||
await self._waiter
|
||||
yield from self._waiter
|
||||
finally:
|
||||
self._waiter = None
|
||||
|
||||
async def readline(self):
|
||||
@coroutine
|
||||
def readline(self):
|
||||
"""Read chunk of data from the stream until newline (b'\n') is found.
|
||||
|
||||
On success, return chunk that ends with newline. If only partial
|
||||
@@ -565,10 +479,10 @@ class StreamReader:
|
||||
sep = b'\n'
|
||||
seplen = len(sep)
|
||||
try:
|
||||
line = await self.readuntil(sep)
|
||||
except exceptions.IncompleteReadError as e:
|
||||
line = yield from self.readuntil(sep)
|
||||
except IncompleteReadError as e:
|
||||
return e.partial
|
||||
except exceptions.LimitOverrunError as e:
|
||||
except LimitOverrunError as e:
|
||||
if self._buffer.startswith(sep, e.consumed):
|
||||
del self._buffer[:e.consumed + seplen]
|
||||
else:
|
||||
@@ -577,7 +491,8 @@ class StreamReader:
|
||||
raise ValueError(e.args[0])
|
||||
return line
|
||||
|
||||
async def readuntil(self, separator=b'\n'):
|
||||
@coroutine
|
||||
def readuntil(self, separator=b'\n'):
|
||||
"""Read data from the stream until ``separator`` is found.
|
||||
|
||||
On success, the data and separator will be removed from the
|
||||
@@ -643,7 +558,7 @@ class StreamReader:
|
||||
# see upper comment for explanation.
|
||||
offset = buflen + 1 - seplen
|
||||
if offset > self._limit:
|
||||
raise exceptions.LimitOverrunError(
|
||||
raise LimitOverrunError(
|
||||
'Separator is not found, and chunk exceed the limit',
|
||||
offset)
|
||||
|
||||
@@ -654,13 +569,13 @@ class StreamReader:
|
||||
if self._eof:
|
||||
chunk = bytes(self._buffer)
|
||||
self._buffer.clear()
|
||||
raise exceptions.IncompleteReadError(chunk, None)
|
||||
raise IncompleteReadError(chunk, None)
|
||||
|
||||
# _wait_for_data() will resume reading if stream was paused.
|
||||
await self._wait_for_data('readuntil')
|
||||
yield from self._wait_for_data('readuntil')
|
||||
|
||||
if isep > self._limit:
|
||||
raise exceptions.LimitOverrunError(
|
||||
raise LimitOverrunError(
|
||||
'Separator is found, but chunk is longer than limit', isep)
|
||||
|
||||
chunk = self._buffer[:isep + seplen]
|
||||
@@ -668,20 +583,20 @@ class StreamReader:
|
||||
self._maybe_resume_transport()
|
||||
return bytes(chunk)
|
||||
|
||||
async def read(self, n=-1):
|
||||
@coroutine
|
||||
def read(self, n=-1):
|
||||
"""Read up to `n` bytes from the stream.
|
||||
|
||||
If `n` is not provided or set to -1,
|
||||
read until EOF, then return all read bytes.
|
||||
If EOF was received and the internal buffer is empty,
|
||||
return an empty bytes object.
|
||||
If n is not provided, or set to -1, read until EOF and return all read
|
||||
bytes. If the EOF was received and the internal buffer is empty, return
|
||||
an empty bytes object.
|
||||
|
||||
If `n` is 0, return an empty bytes object immediately.
|
||||
If n is zero, return empty bytes object immediately.
|
||||
|
||||
If `n` is positive, return at most `n` available bytes
|
||||
as soon as at least 1 byte is available in the internal buffer.
|
||||
If EOF is received before any byte is read, return an empty
|
||||
bytes object.
|
||||
If n is positive, this function try to read `n` bytes, and may return
|
||||
less or equal bytes than requested, but at least one byte. If EOF was
|
||||
received before any byte is read, this function returns empty byte
|
||||
object.
|
||||
|
||||
Returned value is not limited with limit, configured at stream
|
||||
creation.
|
||||
@@ -703,23 +618,24 @@ class StreamReader:
|
||||
# bytes. So just call self.read(self._limit) until EOF.
|
||||
blocks = []
|
||||
while True:
|
||||
block = await self.read(self._limit)
|
||||
block = yield from self.read(self._limit)
|
||||
if not block:
|
||||
break
|
||||
blocks.append(block)
|
||||
return b''.join(blocks)
|
||||
|
||||
if not self._buffer and not self._eof:
|
||||
await self._wait_for_data('read')
|
||||
yield from self._wait_for_data('read')
|
||||
|
||||
# This will work right even if buffer is less than n bytes
|
||||
data = bytes(memoryview(self._buffer)[:n])
|
||||
data = bytes(self._buffer[:n])
|
||||
del self._buffer[:n]
|
||||
|
||||
self._maybe_resume_transport()
|
||||
return data
|
||||
|
||||
async def readexactly(self, n):
|
||||
@coroutine
|
||||
def readexactly(self, n):
|
||||
"""Read exactly `n` bytes.
|
||||
|
||||
Raise an IncompleteReadError if EOF is reached before `n` bytes can be
|
||||
@@ -747,24 +663,33 @@ class StreamReader:
|
||||
if self._eof:
|
||||
incomplete = bytes(self._buffer)
|
||||
self._buffer.clear()
|
||||
raise exceptions.IncompleteReadError(incomplete, n)
|
||||
raise IncompleteReadError(incomplete, n)
|
||||
|
||||
await self._wait_for_data('readexactly')
|
||||
yield from self._wait_for_data('readexactly')
|
||||
|
||||
if len(self._buffer) == n:
|
||||
data = bytes(self._buffer)
|
||||
self._buffer.clear()
|
||||
else:
|
||||
data = bytes(memoryview(self._buffer)[:n])
|
||||
data = bytes(self._buffer[:n])
|
||||
del self._buffer[:n]
|
||||
self._maybe_resume_transport()
|
||||
return data
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
if compat.PY35:
|
||||
@coroutine
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
val = await self.readline()
|
||||
if val == b'':
|
||||
raise StopAsyncIteration
|
||||
return val
|
||||
@coroutine
|
||||
def __anext__(self):
|
||||
val = yield from self.readline()
|
||||
if val == b'':
|
||||
raise StopAsyncIteration
|
||||
return val
|
||||
|
||||
if compat.PY352:
|
||||
# In Python 3.5.2 and greater, __aiter__ should return
|
||||
# the asynchronous iterator directly.
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
__all__ = 'create_subprocess_exec', 'create_subprocess_shell'
|
||||
__all__ = ['create_subprocess_exec', 'create_subprocess_shell']
|
||||
|
||||
import subprocess
|
||||
|
||||
@@ -6,6 +6,7 @@ from . import events
|
||||
from . import protocols
|
||||
from . import streams
|
||||
from . import tasks
|
||||
from .coroutines import coroutine
|
||||
from .log import logger
|
||||
|
||||
|
||||
@@ -23,19 +24,16 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
|
||||
self._limit = limit
|
||||
self.stdin = self.stdout = self.stderr = None
|
||||
self._transport = None
|
||||
self._process_exited = False
|
||||
self._pipe_fds = []
|
||||
self._stdin_closed = self._loop.create_future()
|
||||
|
||||
def __repr__(self):
|
||||
info = [self.__class__.__name__]
|
||||
if self.stdin is not None:
|
||||
info.append(f'stdin={self.stdin!r}')
|
||||
info.append('stdin=%r' % self.stdin)
|
||||
if self.stdout is not None:
|
||||
info.append(f'stdout={self.stdout!r}')
|
||||
info.append('stdout=%r' % self.stdout)
|
||||
if self.stderr is not None:
|
||||
info.append(f'stderr={self.stderr!r}')
|
||||
return '<{}>'.format(' '.join(info))
|
||||
info.append('stderr=%r' % self.stderr)
|
||||
return '<%s>' % ' '.join(info)
|
||||
|
||||
def connection_made(self, transport):
|
||||
self._transport = transport
|
||||
@@ -45,14 +43,12 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
|
||||
self.stdout = streams.StreamReader(limit=self._limit,
|
||||
loop=self._loop)
|
||||
self.stdout.set_transport(stdout_transport)
|
||||
self._pipe_fds.append(1)
|
||||
|
||||
stderr_transport = transport.get_pipe_transport(2)
|
||||
if stderr_transport is not None:
|
||||
self.stderr = streams.StreamReader(limit=self._limit,
|
||||
loop=self._loop)
|
||||
self.stderr.set_transport(stderr_transport)
|
||||
self._pipe_fds.append(2)
|
||||
|
||||
stdin_transport = transport.get_pipe_transport(0)
|
||||
if stdin_transport is not None:
|
||||
@@ -77,13 +73,6 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
|
||||
if pipe is not None:
|
||||
pipe.close()
|
||||
self.connection_lost(exc)
|
||||
if exc is None:
|
||||
self._stdin_closed.set_result(None)
|
||||
else:
|
||||
self._stdin_closed.set_exception(exc)
|
||||
# Since calling `wait_closed()` is not mandatory,
|
||||
# we shouldn't log the traceback if this is not awaited.
|
||||
self._stdin_closed._log_traceback = False
|
||||
return
|
||||
if fd == 1:
|
||||
reader = self.stdout
|
||||
@@ -91,28 +80,15 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
|
||||
reader = self.stderr
|
||||
else:
|
||||
reader = None
|
||||
if reader is not None:
|
||||
if reader != None:
|
||||
if exc is None:
|
||||
reader.feed_eof()
|
||||
else:
|
||||
reader.set_exception(exc)
|
||||
|
||||
if fd in self._pipe_fds:
|
||||
self._pipe_fds.remove(fd)
|
||||
self._maybe_close_transport()
|
||||
|
||||
def process_exited(self):
|
||||
self._process_exited = True
|
||||
self._maybe_close_transport()
|
||||
|
||||
def _maybe_close_transport(self):
|
||||
if len(self._pipe_fds) == 0 and self._process_exited:
|
||||
self._transport.close()
|
||||
self._transport = None
|
||||
|
||||
def _get_close_waiter(self, stream):
|
||||
if stream is self.stdin:
|
||||
return self._stdin_closed
|
||||
self._transport.close()
|
||||
self._transport = None
|
||||
|
||||
|
||||
class Process:
|
||||
@@ -126,15 +102,18 @@ class Process:
|
||||
self.pid = transport.get_pid()
|
||||
|
||||
def __repr__(self):
|
||||
return f'<{self.__class__.__name__} {self.pid}>'
|
||||
return '<%s %s>' % (self.__class__.__name__, self.pid)
|
||||
|
||||
@property
|
||||
def returncode(self):
|
||||
return self._transport.get_returncode()
|
||||
|
||||
async def wait(self):
|
||||
"""Wait until the process exit and return the process return code."""
|
||||
return await self._transport._wait()
|
||||
@coroutine
|
||||
def wait(self):
|
||||
"""Wait until the process exit and return the process return code.
|
||||
|
||||
This method is a coroutine."""
|
||||
return (yield from self._transport._wait())
|
||||
|
||||
def send_signal(self, signal):
|
||||
self._transport.send_signal(signal)
|
||||
@@ -145,19 +124,17 @@ class Process:
|
||||
def kill(self):
|
||||
self._transport.kill()
|
||||
|
||||
async def _feed_stdin(self, input):
|
||||
@coroutine
|
||||
def _feed_stdin(self, input):
|
||||
debug = self._loop.get_debug()
|
||||
self.stdin.write(input)
|
||||
if debug:
|
||||
logger.debug('%r communicate: feed stdin (%s bytes)',
|
||||
self, len(input))
|
||||
try:
|
||||
if input is not None:
|
||||
self.stdin.write(input)
|
||||
if debug:
|
||||
logger.debug(
|
||||
'%r communicate: feed stdin (%s bytes)', self, len(input))
|
||||
|
||||
await self.stdin.drain()
|
||||
yield from self.stdin.drain()
|
||||
except (BrokenPipeError, ConnectionResetError) as exc:
|
||||
# communicate() ignores BrokenPipeError and ConnectionResetError.
|
||||
# write() and drain() can raise these exceptions.
|
||||
# communicate() ignores BrokenPipeError and ConnectionResetError
|
||||
if debug:
|
||||
logger.debug('%r communicate: stdin got %r', self, exc)
|
||||
|
||||
@@ -165,10 +142,12 @@ class Process:
|
||||
logger.debug('%r communicate: close stdin', self)
|
||||
self.stdin.close()
|
||||
|
||||
async def _noop(self):
|
||||
@coroutine
|
||||
def _noop(self):
|
||||
return None
|
||||
|
||||
async def _read_stream(self, fd):
|
||||
@coroutine
|
||||
def _read_stream(self, fd):
|
||||
transport = self._transport.get_pipe_transport(fd)
|
||||
if fd == 2:
|
||||
stream = self.stderr
|
||||
@@ -178,15 +157,16 @@ class Process:
|
||||
if self._loop.get_debug():
|
||||
name = 'stdout' if fd == 1 else 'stderr'
|
||||
logger.debug('%r communicate: read %s', self, name)
|
||||
output = await stream.read()
|
||||
output = yield from stream.read()
|
||||
if self._loop.get_debug():
|
||||
name = 'stdout' if fd == 1 else 'stderr'
|
||||
logger.debug('%r communicate: close %s', self, name)
|
||||
transport.close()
|
||||
return output
|
||||
|
||||
async def communicate(self, input=None):
|
||||
if self.stdin is not None:
|
||||
@coroutine
|
||||
def communicate(self, input=None):
|
||||
if input is not None:
|
||||
stdin = self._feed_stdin(input)
|
||||
else:
|
||||
stdin = self._noop()
|
||||
@@ -198,32 +178,36 @@ class Process:
|
||||
stderr = self._read_stream(2)
|
||||
else:
|
||||
stderr = self._noop()
|
||||
stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr)
|
||||
await self.wait()
|
||||
stdin, stdout, stderr = yield from tasks.gather(stdin, stdout, stderr,
|
||||
loop=self._loop)
|
||||
yield from self.wait()
|
||||
return (stdout, stderr)
|
||||
|
||||
|
||||
async def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None,
|
||||
limit=streams._DEFAULT_LIMIT, **kwds):
|
||||
loop = events.get_running_loop()
|
||||
@coroutine
|
||||
def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None,
|
||||
loop=None, limit=streams._DEFAULT_LIMIT, **kwds):
|
||||
if loop is None:
|
||||
loop = events.get_event_loop()
|
||||
protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
|
||||
loop=loop)
|
||||
transport, protocol = await loop.subprocess_shell(
|
||||
protocol_factory,
|
||||
cmd, stdin=stdin, stdout=stdout,
|
||||
stderr=stderr, **kwds)
|
||||
transport, protocol = yield from loop.subprocess_shell(
|
||||
protocol_factory,
|
||||
cmd, stdin=stdin, stdout=stdout,
|
||||
stderr=stderr, **kwds)
|
||||
return Process(transport, protocol, loop)
|
||||
|
||||
|
||||
async def create_subprocess_exec(program, *args, stdin=None, stdout=None,
|
||||
stderr=None, limit=streams._DEFAULT_LIMIT,
|
||||
**kwds):
|
||||
loop = events.get_running_loop()
|
||||
@coroutine
|
||||
def create_subprocess_exec(program, *args, stdin=None, stdout=None,
|
||||
stderr=None, loop=None,
|
||||
limit=streams._DEFAULT_LIMIT, **kwds):
|
||||
if loop is None:
|
||||
loop = events.get_event_loop()
|
||||
protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
|
||||
loop=loop)
|
||||
transport, protocol = await loop.subprocess_exec(
|
||||
protocol_factory,
|
||||
program, *args,
|
||||
stdin=stdin, stdout=stdout,
|
||||
stderr=stderr, **kwds)
|
||||
transport, protocol = yield from loop.subprocess_exec(
|
||||
protocol_factory,
|
||||
program, *args,
|
||||
stdin=stdin, stdout=stdout,
|
||||
stderr=stderr, **kwds)
|
||||
return Process(transport, protocol, loop)
|
||||
|
||||
@@ -1,240 +0,0 @@
|
||||
# Adapted with permission from the EdgeDB project;
|
||||
# license: PSFL.
|
||||
|
||||
|
||||
__all__ = ("TaskGroup",)
|
||||
|
||||
from . import events
|
||||
from . import exceptions
|
||||
from . import tasks
|
||||
|
||||
|
||||
class TaskGroup:
|
||||
"""Asynchronous context manager for managing groups of tasks.
|
||||
|
||||
Example use:
|
||||
|
||||
async with asyncio.TaskGroup() as group:
|
||||
task1 = group.create_task(some_coroutine(...))
|
||||
task2 = group.create_task(other_coroutine(...))
|
||||
print("Both tasks have completed now.")
|
||||
|
||||
All tasks are awaited when the context manager exits.
|
||||
|
||||
Any exceptions other than `asyncio.CancelledError` raised within
|
||||
a task will cancel all remaining tasks and wait for them to exit.
|
||||
The exceptions are then combined and raised as an `ExceptionGroup`.
|
||||
"""
|
||||
def __init__(self):
|
||||
self._entered = False
|
||||
self._exiting = False
|
||||
self._aborting = False
|
||||
self._loop = None
|
||||
self._parent_task = None
|
||||
self._parent_cancel_requested = False
|
||||
self._tasks = set()
|
||||
self._errors = []
|
||||
self._base_error = None
|
||||
self._on_completed_fut = None
|
||||
|
||||
def __repr__(self):
|
||||
info = ['']
|
||||
if self._tasks:
|
||||
info.append(f'tasks={len(self._tasks)}')
|
||||
if self._errors:
|
||||
info.append(f'errors={len(self._errors)}')
|
||||
if self._aborting:
|
||||
info.append('cancelling')
|
||||
elif self._entered:
|
||||
info.append('entered')
|
||||
|
||||
info_str = ' '.join(info)
|
||||
return f'<TaskGroup{info_str}>'
|
||||
|
||||
async def __aenter__(self):
|
||||
if self._entered:
|
||||
raise RuntimeError(
|
||||
f"TaskGroup {self!r} has already been entered")
|
||||
if self._loop is None:
|
||||
self._loop = events.get_running_loop()
|
||||
self._parent_task = tasks.current_task(self._loop)
|
||||
if self._parent_task is None:
|
||||
raise RuntimeError(
|
||||
f'TaskGroup {self!r} cannot determine the parent task')
|
||||
self._entered = True
|
||||
|
||||
return self
|
||||
|
||||
async def __aexit__(self, et, exc, tb):
|
||||
self._exiting = True
|
||||
|
||||
if (exc is not None and
|
||||
self._is_base_error(exc) and
|
||||
self._base_error is None):
|
||||
self._base_error = exc
|
||||
|
||||
propagate_cancellation_error = \
|
||||
exc if et is exceptions.CancelledError else None
|
||||
if self._parent_cancel_requested:
|
||||
# If this flag is set we *must* call uncancel().
|
||||
if self._parent_task.uncancel() == 0:
|
||||
# If there are no pending cancellations left,
|
||||
# don't propagate CancelledError.
|
||||
propagate_cancellation_error = None
|
||||
|
||||
if et is not None:
|
||||
if not self._aborting:
|
||||
# Our parent task is being cancelled:
|
||||
#
|
||||
# async with TaskGroup() as g:
|
||||
# g.create_task(...)
|
||||
# await ... # <- CancelledError
|
||||
#
|
||||
# or there's an exception in "async with":
|
||||
#
|
||||
# async with TaskGroup() as g:
|
||||
# g.create_task(...)
|
||||
# 1 / 0
|
||||
#
|
||||
self._abort()
|
||||
|
||||
# We use while-loop here because "self._on_completed_fut"
|
||||
# can be cancelled multiple times if our parent task
|
||||
# is being cancelled repeatedly (or even once, when
|
||||
# our own cancellation is already in progress)
|
||||
while self._tasks:
|
||||
if self._on_completed_fut is None:
|
||||
self._on_completed_fut = self._loop.create_future()
|
||||
|
||||
try:
|
||||
await self._on_completed_fut
|
||||
except exceptions.CancelledError as ex:
|
||||
if not self._aborting:
|
||||
# Our parent task is being cancelled:
|
||||
#
|
||||
# async def wrapper():
|
||||
# async with TaskGroup() as g:
|
||||
# g.create_task(foo)
|
||||
#
|
||||
# "wrapper" is being cancelled while "foo" is
|
||||
# still running.
|
||||
propagate_cancellation_error = ex
|
||||
self._abort()
|
||||
|
||||
self._on_completed_fut = None
|
||||
|
||||
assert not self._tasks
|
||||
|
||||
if self._base_error is not None:
|
||||
raise self._base_error
|
||||
|
||||
# Propagate CancelledError if there is one, except if there
|
||||
# are other errors -- those have priority.
|
||||
if propagate_cancellation_error and not self._errors:
|
||||
raise propagate_cancellation_error
|
||||
|
||||
if et is not None and et is not exceptions.CancelledError:
|
||||
self._errors.append(exc)
|
||||
|
||||
if self._errors:
|
||||
# Exceptions are heavy objects that can have object
|
||||
# cycles (bad for GC); let's not keep a reference to
|
||||
# a bunch of them.
|
||||
try:
|
||||
me = BaseExceptionGroup('unhandled errors in a TaskGroup', self._errors)
|
||||
raise me from None
|
||||
finally:
|
||||
self._errors = None
|
||||
|
||||
def create_task(self, coro, *, name=None, context=None):
|
||||
"""Create a new task in this group and return it.
|
||||
|
||||
Similar to `asyncio.create_task`.
|
||||
"""
|
||||
if not self._entered:
|
||||
raise RuntimeError(f"TaskGroup {self!r} has not been entered")
|
||||
if self._exiting and not self._tasks:
|
||||
raise RuntimeError(f"TaskGroup {self!r} is finished")
|
||||
if self._aborting:
|
||||
raise RuntimeError(f"TaskGroup {self!r} is shutting down")
|
||||
if context is None:
|
||||
task = self._loop.create_task(coro)
|
||||
else:
|
||||
task = self._loop.create_task(coro, context=context)
|
||||
tasks._set_task_name(task, name)
|
||||
# optimization: Immediately call the done callback if the task is
|
||||
# already done (e.g. if the coro was able to complete eagerly),
|
||||
# and skip scheduling a done callback
|
||||
if task.done():
|
||||
self._on_task_done(task)
|
||||
else:
|
||||
self._tasks.add(task)
|
||||
task.add_done_callback(self._on_task_done)
|
||||
return task
|
||||
|
||||
# Since Python 3.8 Tasks propagate all exceptions correctly,
|
||||
# except for KeyboardInterrupt and SystemExit which are
|
||||
# still considered special.
|
||||
|
||||
def _is_base_error(self, exc: BaseException) -> bool:
|
||||
assert isinstance(exc, BaseException)
|
||||
return isinstance(exc, (SystemExit, KeyboardInterrupt))
|
||||
|
||||
def _abort(self):
|
||||
self._aborting = True
|
||||
|
||||
for t in self._tasks:
|
||||
if not t.done():
|
||||
t.cancel()
|
||||
|
||||
def _on_task_done(self, task):
|
||||
self._tasks.discard(task)
|
||||
|
||||
if self._on_completed_fut is not None and not self._tasks:
|
||||
if not self._on_completed_fut.done():
|
||||
self._on_completed_fut.set_result(True)
|
||||
|
||||
if task.cancelled():
|
||||
return
|
||||
|
||||
exc = task.exception()
|
||||
if exc is None:
|
||||
return
|
||||
|
||||
self._errors.append(exc)
|
||||
if self._is_base_error(exc) and self._base_error is None:
|
||||
self._base_error = exc
|
||||
|
||||
if self._parent_task.done():
|
||||
# Not sure if this case is possible, but we want to handle
|
||||
# it anyways.
|
||||
self._loop.call_exception_handler({
|
||||
'message': f'Task {task!r} has errored out but its parent '
|
||||
f'task {self._parent_task} is already completed',
|
||||
'exception': exc,
|
||||
'task': task,
|
||||
})
|
||||
return
|
||||
|
||||
if not self._aborting and not self._parent_cancel_requested:
|
||||
# If parent task *is not* being cancelled, it means that we want
|
||||
# to manually cancel it to abort whatever is being run right now
|
||||
# in the TaskGroup. But we want to mark parent task as
|
||||
# "not cancelled" later in __aexit__. Example situation that
|
||||
# we need to handle:
|
||||
#
|
||||
# async def foo():
|
||||
# try:
|
||||
# async with TaskGroup() as g:
|
||||
# g.create_task(crash_soon())
|
||||
# await something # <- this needs to be canceled
|
||||
# # by the TaskGroup, e.g.
|
||||
# # foo() needs to be cancelled
|
||||
# except Exception:
|
||||
# # Ignore any exceptions raised in the TaskGroup
|
||||
# pass
|
||||
# await something_else # this line has to be called
|
||||
# # after TaskGroup is finished.
|
||||
self._abort()
|
||||
self._parent_cancel_requested = True
|
||||
self._parent_task.cancel()
|
||||
File diff suppressed because it is too large
Load Diff
503
Lib/asyncio/test_utils.py
Normal file
503
Lib/asyncio/test_utils.py
Normal file
@@ -0,0 +1,503 @@
|
||||
"""Utilities shared by tests."""
|
||||
|
||||
import collections
|
||||
import contextlib
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import socketserver
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
import weakref
|
||||
|
||||
from unittest import mock
|
||||
|
||||
from http.server import HTTPServer
|
||||
from wsgiref.simple_server import WSGIRequestHandler, WSGIServer
|
||||
|
||||
try:
|
||||
import ssl
|
||||
except ImportError: # pragma: no cover
|
||||
ssl = None
|
||||
|
||||
from . import base_events
|
||||
from . import compat
|
||||
from . import events
|
||||
from . import futures
|
||||
from . import selectors
|
||||
from . import tasks
|
||||
from .coroutines import coroutine
|
||||
from .log import logger
|
||||
|
||||
|
||||
if sys.platform == 'win32': # pragma: no cover
|
||||
from .windows_utils import socketpair
|
||||
else:
|
||||
from socket import socketpair # pragma: no cover
|
||||
|
||||
|
||||
def dummy_ssl_context():
|
||||
if ssl is None:
|
||||
return None
|
||||
else:
|
||||
return ssl.SSLContext(ssl.PROTOCOL_SSLv23)
|
||||
|
||||
|
||||
def run_briefly(loop):
|
||||
@coroutine
|
||||
def once():
|
||||
pass
|
||||
gen = once()
|
||||
t = loop.create_task(gen)
|
||||
# Don't log a warning if the task is not done after run_until_complete().
|
||||
# It occurs if the loop is stopped or if a task raises a BaseException.
|
||||
t._log_destroy_pending = False
|
||||
try:
|
||||
loop.run_until_complete(t)
|
||||
finally:
|
||||
gen.close()
|
||||
|
||||
|
||||
def run_until(loop, pred, timeout=30):
|
||||
deadline = time.time() + timeout
|
||||
while not pred():
|
||||
if timeout is not None:
|
||||
timeout = deadline - time.time()
|
||||
if timeout <= 0:
|
||||
raise futures.TimeoutError()
|
||||
loop.run_until_complete(tasks.sleep(0.001, loop=loop))
|
||||
|
||||
|
||||
def run_once(loop):
|
||||
"""Legacy API to run once through the event loop.
|
||||
|
||||
This is the recommended pattern for test code. It will poll the
|
||||
selector once and run all callbacks scheduled in response to I/O
|
||||
events.
|
||||
"""
|
||||
loop.call_soon(loop.stop)
|
||||
loop.run_forever()
|
||||
|
||||
|
||||
class SilentWSGIRequestHandler(WSGIRequestHandler):
|
||||
|
||||
def get_stderr(self):
|
||||
return io.StringIO()
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
|
||||
class SilentWSGIServer(WSGIServer):
|
||||
|
||||
request_timeout = 2
|
||||
|
||||
def get_request(self):
|
||||
request, client_addr = super().get_request()
|
||||
request.settimeout(self.request_timeout)
|
||||
return request, client_addr
|
||||
|
||||
def handle_error(self, request, client_address):
|
||||
pass
|
||||
|
||||
|
||||
class SSLWSGIServerMixin:
|
||||
|
||||
def finish_request(self, request, client_address):
|
||||
# The relative location of our test directory (which
|
||||
# contains the ssl key and certificate files) differs
|
||||
# between the stdlib and stand-alone asyncio.
|
||||
# Prefer our own if we can find it.
|
||||
here = os.path.join(os.path.dirname(__file__), '..', 'tests')
|
||||
if not os.path.isdir(here):
|
||||
here = os.path.join(os.path.dirname(os.__file__),
|
||||
'test', 'test_asyncio')
|
||||
keyfile = os.path.join(here, 'ssl_key.pem')
|
||||
certfile = os.path.join(here, 'ssl_cert.pem')
|
||||
context = ssl.SSLContext()
|
||||
context.load_cert_chain(certfile, keyfile)
|
||||
|
||||
ssock = context.wrap_socket(request, server_side=True)
|
||||
try:
|
||||
self.RequestHandlerClass(ssock, client_address, self)
|
||||
ssock.close()
|
||||
except OSError:
|
||||
# maybe socket has been closed by peer
|
||||
pass
|
||||
|
||||
|
||||
class SSLWSGIServer(SSLWSGIServerMixin, SilentWSGIServer):
|
||||
pass
|
||||
|
||||
|
||||
def _run_test_server(*, address, use_ssl=False, server_cls, server_ssl_cls):
|
||||
|
||||
def app(environ, start_response):
|
||||
status = '200 OK'
|
||||
headers = [('Content-type', 'text/plain')]
|
||||
start_response(status, headers)
|
||||
return [b'Test message']
|
||||
|
||||
# Run the test WSGI server in a separate thread in order not to
|
||||
# interfere with event handling in the main thread
|
||||
server_class = server_ssl_cls if use_ssl else server_cls
|
||||
httpd = server_class(address, SilentWSGIRequestHandler)
|
||||
httpd.set_app(app)
|
||||
httpd.address = httpd.server_address
|
||||
server_thread = threading.Thread(
|
||||
target=lambda: httpd.serve_forever(poll_interval=0.05))
|
||||
server_thread.start()
|
||||
try:
|
||||
yield httpd
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
server_thread.join()
|
||||
|
||||
|
||||
if hasattr(socket, 'AF_UNIX'):
|
||||
|
||||
class UnixHTTPServer(socketserver.UnixStreamServer, HTTPServer):
|
||||
|
||||
def server_bind(self):
|
||||
socketserver.UnixStreamServer.server_bind(self)
|
||||
self.server_name = '127.0.0.1'
|
||||
self.server_port = 80
|
||||
|
||||
|
||||
class UnixWSGIServer(UnixHTTPServer, WSGIServer):
|
||||
|
||||
request_timeout = 2
|
||||
|
||||
def server_bind(self):
|
||||
UnixHTTPServer.server_bind(self)
|
||||
self.setup_environ()
|
||||
|
||||
def get_request(self):
|
||||
request, client_addr = super().get_request()
|
||||
request.settimeout(self.request_timeout)
|
||||
# Code in the stdlib expects that get_request
|
||||
# will return a socket and a tuple (host, port).
|
||||
# However, this isn't true for UNIX sockets,
|
||||
# as the second return value will be a path;
|
||||
# hence we return some fake data sufficient
|
||||
# to get the tests going
|
||||
return request, ('127.0.0.1', '')
|
||||
|
||||
|
||||
class SilentUnixWSGIServer(UnixWSGIServer):
|
||||
|
||||
def handle_error(self, request, client_address):
|
||||
pass
|
||||
|
||||
|
||||
class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer):
|
||||
pass
|
||||
|
||||
|
||||
def gen_unix_socket_path():
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
return file.name
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def unix_socket_path():
|
||||
path = gen_unix_socket_path()
|
||||
try:
|
||||
yield path
|
||||
finally:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def run_test_unix_server(*, use_ssl=False):
|
||||
with unix_socket_path() as path:
|
||||
yield from _run_test_server(address=path, use_ssl=use_ssl,
|
||||
server_cls=SilentUnixWSGIServer,
|
||||
server_ssl_cls=UnixSSLWSGIServer)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def run_test_server(*, host='127.0.0.1', port=0, use_ssl=False):
|
||||
yield from _run_test_server(address=(host, port), use_ssl=use_ssl,
|
||||
server_cls=SilentWSGIServer,
|
||||
server_ssl_cls=SSLWSGIServer)
|
||||
|
||||
|
||||
def make_test_protocol(base):
|
||||
dct = {}
|
||||
for name in dir(base):
|
||||
if name.startswith('__') and name.endswith('__'):
|
||||
# skip magic names
|
||||
continue
|
||||
dct[name] = MockCallback(return_value=None)
|
||||
return type('TestProtocol', (base,) + base.__bases__, dct)()
|
||||
|
||||
|
||||
class TestSelector(selectors.BaseSelector):
|
||||
|
||||
def __init__(self):
|
||||
self.keys = {}
|
||||
|
||||
def register(self, fileobj, events, data=None):
|
||||
key = selectors.SelectorKey(fileobj, 0, events, data)
|
||||
self.keys[fileobj] = key
|
||||
return key
|
||||
|
||||
def unregister(self, fileobj):
|
||||
return self.keys.pop(fileobj)
|
||||
|
||||
def select(self, timeout):
|
||||
return []
|
||||
|
||||
def get_map(self):
|
||||
return self.keys
|
||||
|
||||
|
||||
class TestLoop(base_events.BaseEventLoop):
|
||||
"""Loop for unittests.
|
||||
|
||||
It manages self time directly.
|
||||
If something scheduled to be executed later then
|
||||
on next loop iteration after all ready handlers done
|
||||
generator passed to __init__ is calling.
|
||||
|
||||
Generator should be like this:
|
||||
|
||||
def gen():
|
||||
...
|
||||
when = yield ...
|
||||
... = yield time_advance
|
||||
|
||||
Value returned by yield is absolute time of next scheduled handler.
|
||||
Value passed to yield is time advance to move loop's time forward.
|
||||
"""
|
||||
|
||||
def __init__(self, gen=None):
|
||||
super().__init__()
|
||||
|
||||
if gen is None:
|
||||
def gen():
|
||||
yield
|
||||
self._check_on_close = False
|
||||
else:
|
||||
self._check_on_close = True
|
||||
|
||||
self._gen = gen()
|
||||
next(self._gen)
|
||||
self._time = 0
|
||||
self._clock_resolution = 1e-9
|
||||
self._timers = []
|
||||
self._selector = TestSelector()
|
||||
|
||||
self.readers = {}
|
||||
self.writers = {}
|
||||
self.reset_counters()
|
||||
|
||||
self._transports = weakref.WeakValueDictionary()
|
||||
|
||||
def time(self):
|
||||
return self._time
|
||||
|
||||
def advance_time(self, advance):
|
||||
"""Move test time forward."""
|
||||
if advance:
|
||||
self._time += advance
|
||||
|
||||
def close(self):
|
||||
super().close()
|
||||
if self._check_on_close:
|
||||
try:
|
||||
self._gen.send(0)
|
||||
except StopIteration:
|
||||
pass
|
||||
else: # pragma: no cover
|
||||
raise AssertionError("Time generator is not finished")
|
||||
|
||||
def _add_reader(self, fd, callback, *args):
|
||||
self.readers[fd] = events.Handle(callback, args, self)
|
||||
|
||||
def _remove_reader(self, fd):
|
||||
self.remove_reader_count[fd] += 1
|
||||
if fd in self.readers:
|
||||
del self.readers[fd]
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def assert_reader(self, fd, callback, *args):
|
||||
assert fd in self.readers, 'fd {} is not registered'.format(fd)
|
||||
handle = self.readers[fd]
|
||||
assert handle._callback == callback, '{!r} != {!r}'.format(
|
||||
handle._callback, callback)
|
||||
assert handle._args == args, '{!r} != {!r}'.format(
|
||||
handle._args, args)
|
||||
|
||||
def _add_writer(self, fd, callback, *args):
|
||||
self.writers[fd] = events.Handle(callback, args, self)
|
||||
|
||||
def _remove_writer(self, fd):
|
||||
self.remove_writer_count[fd] += 1
|
||||
if fd in self.writers:
|
||||
del self.writers[fd]
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def assert_writer(self, fd, callback, *args):
|
||||
assert fd in self.writers, 'fd {} is not registered'.format(fd)
|
||||
handle = self.writers[fd]
|
||||
assert handle._callback == callback, '{!r} != {!r}'.format(
|
||||
handle._callback, callback)
|
||||
assert handle._args == args, '{!r} != {!r}'.format(
|
||||
handle._args, args)
|
||||
|
||||
def _ensure_fd_no_transport(self, fd):
|
||||
try:
|
||||
transport = self._transports[fd]
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
raise RuntimeError(
|
||||
'File descriptor {!r} is used by transport {!r}'.format(
|
||||
fd, transport))
|
||||
|
||||
def add_reader(self, fd, callback, *args):
|
||||
"""Add a reader callback."""
|
||||
self._ensure_fd_no_transport(fd)
|
||||
return self._add_reader(fd, callback, *args)
|
||||
|
||||
def remove_reader(self, fd):
|
||||
"""Remove a reader callback."""
|
||||
self._ensure_fd_no_transport(fd)
|
||||
return self._remove_reader(fd)
|
||||
|
||||
def add_writer(self, fd, callback, *args):
|
||||
"""Add a writer callback.."""
|
||||
self._ensure_fd_no_transport(fd)
|
||||
return self._add_writer(fd, callback, *args)
|
||||
|
||||
def remove_writer(self, fd):
|
||||
"""Remove a writer callback."""
|
||||
self._ensure_fd_no_transport(fd)
|
||||
return self._remove_writer(fd)
|
||||
|
||||
def reset_counters(self):
|
||||
self.remove_reader_count = collections.defaultdict(int)
|
||||
self.remove_writer_count = collections.defaultdict(int)
|
||||
|
||||
def _run_once(self):
|
||||
super()._run_once()
|
||||
for when in self._timers:
|
||||
advance = self._gen.send(when)
|
||||
self.advance_time(advance)
|
||||
self._timers = []
|
||||
|
||||
def call_at(self, when, callback, *args):
|
||||
self._timers.append(when)
|
||||
return super().call_at(when, callback, *args)
|
||||
|
||||
def _process_events(self, event_list):
|
||||
return
|
||||
|
||||
def _write_to_self(self):
|
||||
pass
|
||||
|
||||
|
||||
def MockCallback(**kwargs):
|
||||
return mock.Mock(spec=['__call__'], **kwargs)
|
||||
|
||||
|
||||
class MockPattern(str):
|
||||
"""A regex based str with a fuzzy __eq__.
|
||||
|
||||
Use this helper with 'mock.assert_called_with', or anywhere
|
||||
where a regex comparison between strings is needed.
|
||||
|
||||
For instance:
|
||||
mock_call.assert_called_with(MockPattern('spam.*ham'))
|
||||
"""
|
||||
def __eq__(self, other):
|
||||
return bool(re.search(str(self), other, re.S))
|
||||
|
||||
|
||||
def get_function_source(func):
|
||||
source = events._get_function_source(func)
|
||||
if source is None:
|
||||
raise ValueError("unable to get the source of %r" % (func,))
|
||||
return source
|
||||
|
||||
|
||||
class TestCase(unittest.TestCase):
|
||||
def set_event_loop(self, loop, *, cleanup=True):
|
||||
assert loop is not None
|
||||
# ensure that the event loop is passed explicitly in asyncio
|
||||
events.set_event_loop(None)
|
||||
if cleanup:
|
||||
self.addCleanup(loop.close)
|
||||
|
||||
def new_test_loop(self, gen=None):
|
||||
loop = TestLoop(gen)
|
||||
self.set_event_loop(loop)
|
||||
return loop
|
||||
|
||||
def setUp(self):
|
||||
self._get_running_loop = events._get_running_loop
|
||||
events._get_running_loop = lambda: None
|
||||
|
||||
def tearDown(self):
|
||||
events._get_running_loop = self._get_running_loop
|
||||
|
||||
events.set_event_loop(None)
|
||||
|
||||
# Detect CPython bug #23353: ensure that yield/yield-from is not used
|
||||
# in an except block of a generator
|
||||
self.assertEqual(sys.exc_info(), (None, None, None))
|
||||
|
||||
if not compat.PY34:
|
||||
# Python 3.3 compatibility
|
||||
def subTest(self, *args, **kwargs):
|
||||
class EmptyCM:
|
||||
def __enter__(self):
|
||||
pass
|
||||
def __exit__(self, *exc):
|
||||
pass
|
||||
return EmptyCM()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def disable_logger():
|
||||
"""Context manager to disable asyncio logger.
|
||||
|
||||
For example, it can be used to ignore warnings in debug mode.
|
||||
"""
|
||||
old_level = logger.level
|
||||
try:
|
||||
logger.setLevel(logging.CRITICAL+1)
|
||||
yield
|
||||
finally:
|
||||
logger.setLevel(old_level)
|
||||
|
||||
|
||||
def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM,
|
||||
family=socket.AF_INET):
|
||||
"""Create a mock of a non-blocking socket."""
|
||||
sock = mock.MagicMock(socket.socket)
|
||||
sock.proto = proto
|
||||
sock.type = type
|
||||
sock.family = family
|
||||
sock.gettimeout.return_value = 0.0
|
||||
return sock
|
||||
|
||||
|
||||
def force_legacy_ssl_support():
|
||||
return mock.patch('asyncio.sslproto._is_sslproto_available',
|
||||
return_value=False)
|
||||
@@ -1,25 +0,0 @@
|
||||
"""High-level support for working with threads in asyncio"""
|
||||
|
||||
import functools
|
||||
import contextvars
|
||||
|
||||
from . import events
|
||||
|
||||
|
||||
__all__ = "to_thread",
|
||||
|
||||
|
||||
async def to_thread(func, /, *args, **kwargs):
|
||||
"""Asynchronously run function *func* in a separate thread.
|
||||
|
||||
Any *args and **kwargs supplied for this function are directly passed
|
||||
to *func*. Also, the current :class:`contextvars.Context` is propagated,
|
||||
allowing context variables from the main thread to be accessed in the
|
||||
separate thread.
|
||||
|
||||
Return a coroutine that can be awaited to get the eventual result of *func*.
|
||||
"""
|
||||
loop = events.get_running_loop()
|
||||
ctx = contextvars.copy_context()
|
||||
func_call = functools.partial(ctx.run, func, *args, **kwargs)
|
||||
return await loop.run_in_executor(None, func_call)
|
||||
@@ -1,168 +0,0 @@
|
||||
import enum
|
||||
|
||||
from types import TracebackType
|
||||
from typing import final, Optional, Type
|
||||
|
||||
from . import events
|
||||
from . import exceptions
|
||||
from . import tasks
|
||||
|
||||
|
||||
__all__ = (
|
||||
"Timeout",
|
||||
"timeout",
|
||||
"timeout_at",
|
||||
)
|
||||
|
||||
|
||||
class _State(enum.Enum):
|
||||
CREATED = "created"
|
||||
ENTERED = "active"
|
||||
EXPIRING = "expiring"
|
||||
EXPIRED = "expired"
|
||||
EXITED = "finished"
|
||||
|
||||
|
||||
@final
|
||||
class Timeout:
|
||||
"""Asynchronous context manager for cancelling overdue coroutines.
|
||||
|
||||
Use `timeout()` or `timeout_at()` rather than instantiating this class directly.
|
||||
"""
|
||||
|
||||
def __init__(self, when: Optional[float]) -> None:
|
||||
"""Schedule a timeout that will trigger at a given loop time.
|
||||
|
||||
- If `when` is `None`, the timeout will never trigger.
|
||||
- If `when < loop.time()`, the timeout will trigger on the next
|
||||
iteration of the event loop.
|
||||
"""
|
||||
self._state = _State.CREATED
|
||||
|
||||
self._timeout_handler: Optional[events.TimerHandle] = None
|
||||
self._task: Optional[tasks.Task] = None
|
||||
self._when = when
|
||||
|
||||
def when(self) -> Optional[float]:
|
||||
"""Return the current deadline."""
|
||||
return self._when
|
||||
|
||||
def reschedule(self, when: Optional[float]) -> None:
|
||||
"""Reschedule the timeout."""
|
||||
if self._state is not _State.ENTERED:
|
||||
if self._state is _State.CREATED:
|
||||
raise RuntimeError("Timeout has not been entered")
|
||||
raise RuntimeError(
|
||||
f"Cannot change state of {self._state.value} Timeout",
|
||||
)
|
||||
|
||||
self._when = when
|
||||
|
||||
if self._timeout_handler is not None:
|
||||
self._timeout_handler.cancel()
|
||||
|
||||
if when is None:
|
||||
self._timeout_handler = None
|
||||
else:
|
||||
loop = events.get_running_loop()
|
||||
if when <= loop.time():
|
||||
self._timeout_handler = loop.call_soon(self._on_timeout)
|
||||
else:
|
||||
self._timeout_handler = loop.call_at(when, self._on_timeout)
|
||||
|
||||
def expired(self) -> bool:
|
||||
"""Is timeout expired during execution?"""
|
||||
return self._state in (_State.EXPIRING, _State.EXPIRED)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
info = ['']
|
||||
if self._state is _State.ENTERED:
|
||||
when = round(self._when, 3) if self._when is not None else None
|
||||
info.append(f"when={when}")
|
||||
info_str = ' '.join(info)
|
||||
return f"<Timeout [{self._state.value}]{info_str}>"
|
||||
|
||||
async def __aenter__(self) -> "Timeout":
|
||||
if self._state is not _State.CREATED:
|
||||
raise RuntimeError("Timeout has already been entered")
|
||||
task = tasks.current_task()
|
||||
if task is None:
|
||||
raise RuntimeError("Timeout should be used inside a task")
|
||||
self._state = _State.ENTERED
|
||||
self._task = task
|
||||
self._cancelling = self._task.cancelling()
|
||||
self.reschedule(self._when)
|
||||
return self
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: Optional[Type[BaseException]],
|
||||
exc_val: Optional[BaseException],
|
||||
exc_tb: Optional[TracebackType],
|
||||
) -> Optional[bool]:
|
||||
assert self._state in (_State.ENTERED, _State.EXPIRING)
|
||||
|
||||
if self._timeout_handler is not None:
|
||||
self._timeout_handler.cancel()
|
||||
self._timeout_handler = None
|
||||
|
||||
if self._state is _State.EXPIRING:
|
||||
self._state = _State.EXPIRED
|
||||
|
||||
if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError:
|
||||
# Since there are no new cancel requests, we're
|
||||
# handling this.
|
||||
raise TimeoutError from exc_val
|
||||
elif self._state is _State.ENTERED:
|
||||
self._state = _State.EXITED
|
||||
|
||||
return None
|
||||
|
||||
def _on_timeout(self) -> None:
|
||||
assert self._state is _State.ENTERED
|
||||
self._task.cancel()
|
||||
self._state = _State.EXPIRING
|
||||
# drop the reference early
|
||||
self._timeout_handler = None
|
||||
|
||||
|
||||
def timeout(delay: Optional[float]) -> Timeout:
|
||||
"""Timeout async context manager.
|
||||
|
||||
Useful in cases when you want to apply timeout logic around block
|
||||
of code or in cases when asyncio.wait_for is not suitable. For example:
|
||||
|
||||
>>> async with asyncio.timeout(10): # 10 seconds timeout
|
||||
... await long_running_task()
|
||||
|
||||
|
||||
delay - value in seconds or None to disable timeout logic
|
||||
|
||||
long_running_task() is interrupted by raising asyncio.CancelledError,
|
||||
the top-most affected timeout() context manager converts CancelledError
|
||||
into TimeoutError.
|
||||
"""
|
||||
loop = events.get_running_loop()
|
||||
return Timeout(loop.time() + delay if delay is not None else None)
|
||||
|
||||
|
||||
def timeout_at(when: Optional[float]) -> Timeout:
|
||||
"""Schedule the timeout at absolute time.
|
||||
|
||||
Like timeout() but argument gives absolute time in the same clock system
|
||||
as loop.time().
|
||||
|
||||
Please note: it is not POSIX time but a time with
|
||||
undefined starting base, e.g. the time of the system power on.
|
||||
|
||||
>>> async with asyncio.timeout_at(loop.time() + 10):
|
||||
... await long_running_task()
|
||||
|
||||
|
||||
when - a deadline when timeout occurs or None to disable timeout logic
|
||||
|
||||
long_running_task() is interrupted by raising asyncio.CancelledError,
|
||||
the top-most affected timeout() context manager converts CancelledError
|
||||
into TimeoutError.
|
||||
"""
|
||||
return Timeout(when)
|
||||
@@ -1,16 +1,15 @@
|
||||
"""Abstract Transport class."""
|
||||
|
||||
__all__ = (
|
||||
'BaseTransport', 'ReadTransport', 'WriteTransport',
|
||||
'Transport', 'DatagramTransport', 'SubprocessTransport',
|
||||
)
|
||||
from asyncio import compat
|
||||
|
||||
__all__ = ['BaseTransport', 'ReadTransport', 'WriteTransport',
|
||||
'Transport', 'DatagramTransport', 'SubprocessTransport',
|
||||
]
|
||||
|
||||
|
||||
class BaseTransport:
|
||||
"""Base class for transports."""
|
||||
|
||||
__slots__ = ('_extra',)
|
||||
|
||||
def __init__(self, extra=None):
|
||||
if extra is None:
|
||||
extra = {}
|
||||
@@ -29,8 +28,8 @@ class BaseTransport:
|
||||
|
||||
Buffered data will be flushed asynchronously. No more data
|
||||
will be received. After all buffered data is flushed, the
|
||||
protocol's connection_lost() method will (eventually) be
|
||||
called with None as its argument.
|
||||
protocol's connection_lost() method will (eventually) called
|
||||
with None as its argument.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -46,12 +45,6 @@ class BaseTransport:
|
||||
class ReadTransport(BaseTransport):
|
||||
"""Interface for read-only transports."""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def is_reading(self):
|
||||
"""Return True if the transport is receiving."""
|
||||
raise NotImplementedError
|
||||
|
||||
def pause_reading(self):
|
||||
"""Pause the receiving end.
|
||||
|
||||
@@ -72,8 +65,6 @@ class ReadTransport(BaseTransport):
|
||||
class WriteTransport(BaseTransport):
|
||||
"""Interface for write-only transports."""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def set_write_buffer_limits(self, high=None, low=None):
|
||||
"""Set the high- and low-water limits for write flow control.
|
||||
|
||||
@@ -99,12 +90,6 @@ class WriteTransport(BaseTransport):
|
||||
"""Return the current size of the write buffer."""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_write_buffer_limits(self):
|
||||
"""Get the high and low watermarks for write flow control.
|
||||
Return a tuple (low, high) where low and high are
|
||||
positive number of bytes."""
|
||||
raise NotImplementedError
|
||||
|
||||
def write(self, data):
|
||||
"""Write some data bytes to the transport.
|
||||
|
||||
@@ -119,7 +104,7 @@ class WriteTransport(BaseTransport):
|
||||
The default implementation concatenates the arguments and
|
||||
calls write() on the result.
|
||||
"""
|
||||
data = b''.join(list_of_data)
|
||||
data = compat.flatten_list_bytes(list_of_data)
|
||||
self.write(data)
|
||||
|
||||
def write_eof(self):
|
||||
@@ -166,14 +151,10 @@ class Transport(ReadTransport, WriteTransport):
|
||||
except writelines(), which calls write() in a loop.
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class DatagramTransport(BaseTransport):
|
||||
"""Interface for datagram (UDP) transports."""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def sendto(self, data, addr=None):
|
||||
"""Send data to the transport.
|
||||
|
||||
@@ -196,8 +177,6 @@ class DatagramTransport(BaseTransport):
|
||||
|
||||
class SubprocessTransport(BaseTransport):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def get_pid(self):
|
||||
"""Get subprocess id."""
|
||||
raise NotImplementedError
|
||||
@@ -265,8 +244,6 @@ class _FlowControlMixin(Transport):
|
||||
resume_writing() may be called.
|
||||
"""
|
||||
|
||||
__slots__ = ('_loop', '_protocol_paused', '_high_water', '_low_water')
|
||||
|
||||
def __init__(self, extra=None, loop=None):
|
||||
super().__init__(extra)
|
||||
assert loop is not None
|
||||
@@ -282,9 +259,7 @@ class _FlowControlMixin(Transport):
|
||||
self._protocol_paused = True
|
||||
try:
|
||||
self._protocol.pause_writing()
|
||||
except (SystemExit, KeyboardInterrupt):
|
||||
raise
|
||||
except BaseException as exc:
|
||||
except Exception as exc:
|
||||
self._loop.call_exception_handler({
|
||||
'message': 'protocol.pause_writing() failed',
|
||||
'exception': exc,
|
||||
@@ -294,13 +269,11 @@ class _FlowControlMixin(Transport):
|
||||
|
||||
def _maybe_resume_protocol(self):
|
||||
if (self._protocol_paused and
|
||||
self.get_write_buffer_size() <= self._low_water):
|
||||
self.get_write_buffer_size() <= self._low_water):
|
||||
self._protocol_paused = False
|
||||
try:
|
||||
self._protocol.resume_writing()
|
||||
except (SystemExit, KeyboardInterrupt):
|
||||
raise
|
||||
except BaseException as exc:
|
||||
except Exception as exc:
|
||||
self._loop.call_exception_handler({
|
||||
'message': 'protocol.resume_writing() failed',
|
||||
'exception': exc,
|
||||
@@ -314,16 +287,14 @@ class _FlowControlMixin(Transport):
|
||||
def _set_write_buffer_limits(self, high=None, low=None):
|
||||
if high is None:
|
||||
if low is None:
|
||||
high = 64 * 1024
|
||||
high = 64*1024
|
||||
else:
|
||||
high = 4 * low
|
||||
high = 4*low
|
||||
if low is None:
|
||||
low = high // 4
|
||||
|
||||
if not high >= low >= 0:
|
||||
raise ValueError(
|
||||
f'high ({high!r}) must be >= low ({low!r}) must be >= 0')
|
||||
|
||||
raise ValueError('high (%r) must be >= low (%r) must be >= 0' %
|
||||
(high, low))
|
||||
self._high_water = high
|
||||
self._low_water = low
|
||||
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
import socket
|
||||
|
||||
|
||||
class TransportSocket:
|
||||
|
||||
"""A socket-like wrapper for exposing real transport sockets.
|
||||
|
||||
These objects can be safely returned by APIs like
|
||||
`transport.get_extra_info('socket')`. All potentially disruptive
|
||||
operations (like "socket.close()") are banned.
|
||||
"""
|
||||
|
||||
__slots__ = ('_sock',)
|
||||
|
||||
def __init__(self, sock: socket.socket):
|
||||
self._sock = sock
|
||||
|
||||
@property
|
||||
def family(self):
|
||||
return self._sock.family
|
||||
|
||||
@property
|
||||
def type(self):
|
||||
return self._sock.type
|
||||
|
||||
@property
|
||||
def proto(self):
|
||||
return self._sock.proto
|
||||
|
||||
def __repr__(self):
|
||||
s = (
|
||||
f"<asyncio.TransportSocket fd={self.fileno()}, "
|
||||
f"family={self.family!s}, type={self.type!s}, "
|
||||
f"proto={self.proto}"
|
||||
)
|
||||
|
||||
if self.fileno() != -1:
|
||||
try:
|
||||
laddr = self.getsockname()
|
||||
if laddr:
|
||||
s = f"{s}, laddr={laddr}"
|
||||
except socket.error:
|
||||
pass
|
||||
try:
|
||||
raddr = self.getpeername()
|
||||
if raddr:
|
||||
s = f"{s}, raddr={raddr}"
|
||||
except socket.error:
|
||||
pass
|
||||
|
||||
return f"{s}>"
|
||||
|
||||
def __getstate__(self):
|
||||
raise TypeError("Cannot serialize asyncio.TransportSocket object")
|
||||
|
||||
def fileno(self):
|
||||
return self._sock.fileno()
|
||||
|
||||
def dup(self):
|
||||
return self._sock.dup()
|
||||
|
||||
def get_inheritable(self):
|
||||
return self._sock.get_inheritable()
|
||||
|
||||
def shutdown(self, how):
|
||||
# asyncio doesn't currently provide a high-level transport API
|
||||
# to shutdown the connection.
|
||||
self._sock.shutdown(how)
|
||||
|
||||
def getsockopt(self, *args, **kwargs):
|
||||
return self._sock.getsockopt(*args, **kwargs)
|
||||
|
||||
def setsockopt(self, *args, **kwargs):
|
||||
self._sock.setsockopt(*args, **kwargs)
|
||||
|
||||
def getpeername(self):
|
||||
return self._sock.getpeername()
|
||||
|
||||
def getsockname(self):
|
||||
return self._sock.getsockname()
|
||||
|
||||
def getsockbyname(self):
|
||||
return self._sock.getsockbyname()
|
||||
|
||||
def settimeout(self, value):
|
||||
if value == 0:
|
||||
return
|
||||
raise ValueError(
|
||||
'settimeout(): only 0 timeout is allowed on transport sockets')
|
||||
|
||||
def gettimeout(self):
|
||||
return 0
|
||||
|
||||
def setblocking(self, flag):
|
||||
if not flag:
|
||||
return
|
||||
raise ValueError(
|
||||
'setblocking(): transport sockets cannot be blocking')
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,41 +1,32 @@
|
||||
"""Selector and proactor event loops for Windows."""
|
||||
|
||||
import sys
|
||||
|
||||
if sys.platform != 'win32': # pragma: no cover
|
||||
raise ImportError('win32 only')
|
||||
|
||||
import _overlapped
|
||||
import _winapi
|
||||
import errno
|
||||
from functools import partial
|
||||
import math
|
||||
import msvcrt
|
||||
import socket
|
||||
import struct
|
||||
import time
|
||||
import weakref
|
||||
|
||||
from . import events
|
||||
from . import base_subprocess
|
||||
from . import futures
|
||||
from . import exceptions
|
||||
from . import proactor_events
|
||||
from . import selector_events
|
||||
from . import tasks
|
||||
from . import windows_utils
|
||||
# XXX RustPython TODO: _overlapped
|
||||
# from . import _overlapped
|
||||
from .coroutines import coroutine
|
||||
from .log import logger
|
||||
|
||||
|
||||
__all__ = (
|
||||
'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor',
|
||||
'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy',
|
||||
'WindowsProactorEventLoopPolicy',
|
||||
)
|
||||
__all__ = ['SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor',
|
||||
'DefaultEventLoopPolicy',
|
||||
]
|
||||
|
||||
|
||||
NULL = _winapi.NULL
|
||||
INFINITE = _winapi.INFINITE
|
||||
NULL = 0
|
||||
INFINITE = 0xffffffff
|
||||
ERROR_CONNECTION_REFUSED = 1225
|
||||
ERROR_CONNECTION_ABORTED = 1236
|
||||
|
||||
@@ -62,7 +53,7 @@ class _OverlappedFuture(futures.Future):
|
||||
info = super()._repr_info()
|
||||
if self._ov is not None:
|
||||
state = 'pending' if self._ov.pending else 'completed'
|
||||
info.insert(1, f'overlapped=<{state}, {self._ov.address:#x}>')
|
||||
info.insert(1, 'overlapped=<%s, %#x>' % (state, self._ov.address))
|
||||
return info
|
||||
|
||||
def _cancel_overlapped(self):
|
||||
@@ -81,9 +72,9 @@ class _OverlappedFuture(futures.Future):
|
||||
self._loop.call_exception_handler(context)
|
||||
self._ov = None
|
||||
|
||||
def cancel(self, msg=None):
|
||||
def cancel(self):
|
||||
self._cancel_overlapped()
|
||||
return super().cancel(msg=msg)
|
||||
return super().cancel()
|
||||
|
||||
def set_exception(self, exception):
|
||||
super().set_exception(exception)
|
||||
@@ -118,12 +109,12 @@ class _BaseWaitHandleFuture(futures.Future):
|
||||
|
||||
def _repr_info(self):
|
||||
info = super()._repr_info()
|
||||
info.append(f'handle={self._handle:#x}')
|
||||
info.append('handle=%#x' % self._handle)
|
||||
if self._handle is not None:
|
||||
state = 'signaled' if self._poll() else 'waiting'
|
||||
info.append(state)
|
||||
if self._wait_handle is not None:
|
||||
info.append(f'wait_handle={self._wait_handle:#x}')
|
||||
info.append('wait_handle=%#x' % self._wait_handle)
|
||||
return info
|
||||
|
||||
def _unregister_wait_cb(self, fut):
|
||||
@@ -155,9 +146,9 @@ class _BaseWaitHandleFuture(futures.Future):
|
||||
|
||||
self._unregister_wait_cb(None)
|
||||
|
||||
def cancel(self, msg=None):
|
||||
def cancel(self):
|
||||
self._unregister_wait()
|
||||
return super().cancel(msg=msg)
|
||||
return super().cancel()
|
||||
|
||||
def set_exception(self, exception):
|
||||
self._unregister_wait()
|
||||
@@ -306,6 +297,9 @@ class PipeServer(object):
|
||||
class _WindowsSelectorEventLoop(selector_events.BaseSelectorEventLoop):
|
||||
"""Windows version of selector event loop."""
|
||||
|
||||
def _socketpair(self):
|
||||
return windows_utils.socketpair()
|
||||
|
||||
|
||||
class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
|
||||
"""Windows version of proactor event loop using IOCP."""
|
||||
@@ -315,34 +309,20 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
|
||||
proactor = IocpProactor()
|
||||
super().__init__(proactor)
|
||||
|
||||
def run_forever(self):
|
||||
try:
|
||||
assert self._self_reading_future is None
|
||||
self.call_soon(self._loop_self_reading)
|
||||
super().run_forever()
|
||||
finally:
|
||||
if self._self_reading_future is not None:
|
||||
ov = self._self_reading_future._ov
|
||||
self._self_reading_future.cancel()
|
||||
# self_reading_future always uses IOCP, so even though it's
|
||||
# been cancelled, we need to make sure that the IOCP message
|
||||
# is received so that the kernel is not holding on to the
|
||||
# memory, possibly causing memory corruption later. Only
|
||||
# unregister it if IO is complete in all respects. Otherwise
|
||||
# we need another _poll() later to complete the IO.
|
||||
if ov is not None and not ov.pending:
|
||||
self._proactor._unregister(ov)
|
||||
self._self_reading_future = None
|
||||
def _socketpair(self):
|
||||
return windows_utils.socketpair()
|
||||
|
||||
async def create_pipe_connection(self, protocol_factory, address):
|
||||
@coroutine
|
||||
def create_pipe_connection(self, protocol_factory, address):
|
||||
f = self._proactor.connect_pipe(address)
|
||||
pipe = await f
|
||||
pipe = yield from f
|
||||
protocol = protocol_factory()
|
||||
trans = self._make_duplex_pipe_transport(pipe, protocol,
|
||||
extra={'addr': address})
|
||||
return trans, protocol
|
||||
|
||||
async def start_serving_pipe(self, protocol_factory, address):
|
||||
@coroutine
|
||||
def start_serving_pipe(self, protocol_factory, address):
|
||||
server = PipeServer(address)
|
||||
|
||||
def loop_accept_pipe(f=None):
|
||||
@@ -367,10 +347,6 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
|
||||
return
|
||||
|
||||
f = self._proactor.accept_pipe(pipe)
|
||||
except BrokenPipeError:
|
||||
if pipe and pipe.fileno() != -1:
|
||||
pipe.close()
|
||||
self.call_soon(loop_accept_pipe)
|
||||
except OSError as exc:
|
||||
if pipe and pipe.fileno() != -1:
|
||||
self.call_exception_handler({
|
||||
@@ -382,8 +358,7 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
|
||||
elif self._debug:
|
||||
logger.warning("Accept pipe failed on pipe %r",
|
||||
pipe, exc_info=True)
|
||||
self.call_soon(loop_accept_pipe)
|
||||
except exceptions.CancelledError:
|
||||
except futures.CancelledError:
|
||||
if pipe:
|
||||
pipe.close()
|
||||
else:
|
||||
@@ -393,22 +368,28 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
|
||||
self.call_soon(loop_accept_pipe)
|
||||
return [server]
|
||||
|
||||
async def _make_subprocess_transport(self, protocol, args, shell,
|
||||
stdin, stdout, stderr, bufsize,
|
||||
extra=None, **kwargs):
|
||||
@coroutine
|
||||
def _make_subprocess_transport(self, protocol, args, shell,
|
||||
stdin, stdout, stderr, bufsize,
|
||||
extra=None, **kwargs):
|
||||
waiter = self.create_future()
|
||||
transp = _WindowsSubprocessTransport(self, protocol, args, shell,
|
||||
stdin, stdout, stderr, bufsize,
|
||||
waiter=waiter, extra=extra,
|
||||
**kwargs)
|
||||
try:
|
||||
await waiter
|
||||
except (SystemExit, KeyboardInterrupt):
|
||||
raise
|
||||
except BaseException:
|
||||
yield from waiter
|
||||
except Exception as exc:
|
||||
# Workaround CPython bug #23353: using yield/yield-from in an
|
||||
# except block of a generator doesn't clear properly sys.exc_info()
|
||||
err = exc
|
||||
else:
|
||||
err = None
|
||||
|
||||
if err is not None:
|
||||
transp.close()
|
||||
await transp._wait()
|
||||
raise
|
||||
yield from transp._wait()
|
||||
raise err
|
||||
|
||||
return transp
|
||||
|
||||
@@ -416,7 +397,7 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
|
||||
class IocpProactor:
|
||||
"""Proactor implementation using IOCP."""
|
||||
|
||||
def __init__(self, concurrency=INFINITE):
|
||||
def __init__(self, concurrency=0xffffffff):
|
||||
self._loop = None
|
||||
self._results = []
|
||||
self._iocp = _overlapped.CreateIoCompletionPort(
|
||||
@@ -426,16 +407,10 @@ class IocpProactor:
|
||||
self._unregistered = []
|
||||
self._stopped_serving = weakref.WeakSet()
|
||||
|
||||
def _check_closed(self):
|
||||
if self._iocp is None:
|
||||
raise RuntimeError('IocpProactor is closed')
|
||||
|
||||
def __repr__(self):
|
||||
info = ['overlapped#=%s' % len(self._cache),
|
||||
'result#=%s' % len(self._results)]
|
||||
if self._iocp is None:
|
||||
info.append('closed')
|
||||
return '<%s %s>' % (self.__class__.__name__, " ".join(info))
|
||||
return ('<%s overlapped#=%s result#=%s>'
|
||||
% (self.__class__.__name__, len(self._cache),
|
||||
len(self._results)))
|
||||
|
||||
def set_loop(self, loop):
|
||||
self._loop = loop
|
||||
@@ -445,40 +420,13 @@ class IocpProactor:
|
||||
self._poll(timeout)
|
||||
tmp = self._results
|
||||
self._results = []
|
||||
try:
|
||||
return tmp
|
||||
finally:
|
||||
# Needed to break cycles when an exception occurs.
|
||||
tmp = None
|
||||
return tmp
|
||||
|
||||
def _result(self, value):
|
||||
fut = self._loop.create_future()
|
||||
fut.set_result(value)
|
||||
return fut
|
||||
|
||||
@staticmethod
|
||||
def finish_socket_func(trans, key, ov):
|
||||
try:
|
||||
return ov.getresult()
|
||||
except OSError as exc:
|
||||
if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED,
|
||||
_overlapped.ERROR_OPERATION_ABORTED):
|
||||
raise ConnectionResetError(*exc.args)
|
||||
else:
|
||||
raise
|
||||
|
||||
@classmethod
|
||||
def _finish_recvfrom(cls, trans, key, ov, *, empty_result):
|
||||
try:
|
||||
return cls.finish_socket_func(trans, key, ov)
|
||||
except OSError as exc:
|
||||
# WSARecvFrom will report ERROR_PORT_UNREACHABLE when the same
|
||||
# socket is used to send to an address that is not listening.
|
||||
if exc.winerror == _overlapped.ERROR_PORT_UNREACHABLE:
|
||||
return empty_result, None
|
||||
else:
|
||||
raise
|
||||
|
||||
def recv(self, conn, nbytes, flags=0):
|
||||
self._register_with_iocp(conn)
|
||||
ov = _overlapped.Overlapped(NULL)
|
||||
@@ -490,50 +438,16 @@ class IocpProactor:
|
||||
except BrokenPipeError:
|
||||
return self._result(b'')
|
||||
|
||||
return self._register(ov, conn, self.finish_socket_func)
|
||||
def finish_recv(trans, key, ov):
|
||||
try:
|
||||
return ov.getresult()
|
||||
except OSError as exc:
|
||||
if exc.winerror == _overlapped.ERROR_NETNAME_DELETED:
|
||||
raise ConnectionResetError(*exc.args)
|
||||
else:
|
||||
raise
|
||||
|
||||
def recv_into(self, conn, buf, flags=0):
|
||||
self._register_with_iocp(conn)
|
||||
ov = _overlapped.Overlapped(NULL)
|
||||
try:
|
||||
if isinstance(conn, socket.socket):
|
||||
ov.WSARecvInto(conn.fileno(), buf, flags)
|
||||
else:
|
||||
ov.ReadFileInto(conn.fileno(), buf)
|
||||
except BrokenPipeError:
|
||||
return self._result(0)
|
||||
|
||||
return self._register(ov, conn, self.finish_socket_func)
|
||||
|
||||
def recvfrom(self, conn, nbytes, flags=0):
|
||||
self._register_with_iocp(conn)
|
||||
ov = _overlapped.Overlapped(NULL)
|
||||
try:
|
||||
ov.WSARecvFrom(conn.fileno(), nbytes, flags)
|
||||
except BrokenPipeError:
|
||||
return self._result((b'', None))
|
||||
|
||||
return self._register(ov, conn, partial(self._finish_recvfrom,
|
||||
empty_result=b''))
|
||||
|
||||
def recvfrom_into(self, conn, buf, flags=0):
|
||||
self._register_with_iocp(conn)
|
||||
ov = _overlapped.Overlapped(NULL)
|
||||
try:
|
||||
ov.WSARecvFromInto(conn.fileno(), buf, flags)
|
||||
except BrokenPipeError:
|
||||
return self._result((0, None))
|
||||
|
||||
return self._register(ov, conn, partial(self._finish_recvfrom,
|
||||
empty_result=0))
|
||||
|
||||
def sendto(self, conn, buf, flags=0, addr=None):
|
||||
self._register_with_iocp(conn)
|
||||
ov = _overlapped.Overlapped(NULL)
|
||||
|
||||
ov.WSASendTo(conn.fileno(), buf, flags, addr)
|
||||
|
||||
return self._register(ov, conn, self.finish_socket_func)
|
||||
return self._register(ov, conn, finish_recv)
|
||||
|
||||
def send(self, conn, buf, flags=0):
|
||||
self._register_with_iocp(conn)
|
||||
@@ -543,7 +457,16 @@ class IocpProactor:
|
||||
else:
|
||||
ov.WriteFile(conn.fileno(), buf)
|
||||
|
||||
return self._register(ov, conn, self.finish_socket_func)
|
||||
def finish_send(trans, key, ov):
|
||||
try:
|
||||
return ov.getresult()
|
||||
except OSError as exc:
|
||||
if exc.winerror == _overlapped.ERROR_NETNAME_DELETED:
|
||||
raise ConnectionResetError(*exc.args)
|
||||
else:
|
||||
raise
|
||||
|
||||
return self._register(ov, conn, finish_send)
|
||||
|
||||
def accept(self, listener):
|
||||
self._register_with_iocp(listener)
|
||||
@@ -560,11 +483,12 @@ class IocpProactor:
|
||||
conn.settimeout(listener.gettimeout())
|
||||
return conn, conn.getpeername()
|
||||
|
||||
async def accept_coro(future, conn):
|
||||
@coroutine
|
||||
def accept_coro(future, conn):
|
||||
# Coroutine closing the accept socket if the future is cancelled
|
||||
try:
|
||||
await future
|
||||
except exceptions.CancelledError:
|
||||
yield from future
|
||||
except futures.CancelledError:
|
||||
conn.close()
|
||||
raise
|
||||
|
||||
@@ -574,14 +498,6 @@ class IocpProactor:
|
||||
return future
|
||||
|
||||
def connect(self, conn, address):
|
||||
if conn.type == socket.SOCK_DGRAM:
|
||||
# WSAConnect will complete immediately for UDP sockets so we don't
|
||||
# need to register any IOCP operation
|
||||
_overlapped.WSAConnect(conn.fileno(), address)
|
||||
fut = self._loop.create_future()
|
||||
fut.set_result(None)
|
||||
return fut
|
||||
|
||||
self._register_with_iocp(conn)
|
||||
# The socket needs to be locally bound before we call ConnectEx().
|
||||
try:
|
||||
@@ -604,18 +520,6 @@ class IocpProactor:
|
||||
|
||||
return self._register(ov, conn, finish_connect)
|
||||
|
||||
def sendfile(self, sock, file, offset, count):
|
||||
self._register_with_iocp(sock)
|
||||
ov = _overlapped.Overlapped(NULL)
|
||||
offset_low = offset & 0xffff_ffff
|
||||
offset_high = (offset >> 32) & 0xffff_ffff
|
||||
ov.TransmitFile(sock.fileno(),
|
||||
msvcrt.get_osfhandle(file.fileno()),
|
||||
offset_low, offset_high,
|
||||
count, 0, 0)
|
||||
|
||||
return self._register(ov, sock, self.finish_socket_func)
|
||||
|
||||
def accept_pipe(self, pipe):
|
||||
self._register_with_iocp(pipe)
|
||||
ov = _overlapped.Overlapped(NULL)
|
||||
@@ -633,12 +537,13 @@ class IocpProactor:
|
||||
|
||||
return self._register(ov, pipe, finish_accept_pipe)
|
||||
|
||||
async def connect_pipe(self, address):
|
||||
@coroutine
|
||||
def connect_pipe(self, address):
|
||||
delay = CONNECT_PIPE_INIT_DELAY
|
||||
while True:
|
||||
# Unfortunately there is no way to do an overlapped connect to
|
||||
# a pipe. Call CreateFile() in a loop until it doesn't fail with
|
||||
# ERROR_PIPE_BUSY.
|
||||
# Unfortunately there is no way to do an overlapped connect to a pipe.
|
||||
# Call CreateFile() in a loop until it doesn't fail with
|
||||
# ERROR_PIPE_BUSY
|
||||
try:
|
||||
handle = _overlapped.ConnectPipe(address)
|
||||
break
|
||||
@@ -648,7 +553,7 @@ class IocpProactor:
|
||||
|
||||
# ConnectPipe() failed with ERROR_PIPE_BUSY: retry later
|
||||
delay = min(delay * 2, CONNECT_PIPE_MAX_DELAY)
|
||||
await tasks.sleep(delay)
|
||||
yield from tasks.sleep(delay, loop=self._loop)
|
||||
|
||||
return windows_utils.PipeHandle(handle)
|
||||
|
||||
@@ -668,8 +573,6 @@ class IocpProactor:
|
||||
return fut
|
||||
|
||||
def _wait_for_handle(self, handle, timeout, _is_cancel):
|
||||
self._check_closed()
|
||||
|
||||
if timeout is None:
|
||||
ms = _winapi.INFINITE
|
||||
else:
|
||||
@@ -712,8 +615,6 @@ class IocpProactor:
|
||||
# that succeed immediately.
|
||||
|
||||
def _register(self, ov, obj, callback):
|
||||
self._check_closed()
|
||||
|
||||
# Return a future which will be set with the result of the
|
||||
# operation when it completes. The future's value is actually
|
||||
# the value returned by callback().
|
||||
@@ -750,7 +651,6 @@ class IocpProactor:
|
||||
already be signalled (pending in the proactor event queue). It is also
|
||||
safe if the event is never signalled (because it was cancelled).
|
||||
"""
|
||||
self._check_closed()
|
||||
self._unregistered.append(ov)
|
||||
|
||||
def _get_accept_socket(self, family):
|
||||
@@ -807,10 +707,8 @@ class IocpProactor:
|
||||
else:
|
||||
f.set_result(value)
|
||||
self._results.append(f)
|
||||
finally:
|
||||
f = None
|
||||
|
||||
# Remove unregistered futures
|
||||
# Remove unregisted futures
|
||||
for ov in self._unregistered:
|
||||
self._cache.pop(ov.address, None)
|
||||
self._unregistered.clear()
|
||||
@@ -822,12 +720,8 @@ class IocpProactor:
|
||||
self._stopped_serving.add(obj)
|
||||
|
||||
def close(self):
|
||||
if self._iocp is None:
|
||||
# already closed
|
||||
return
|
||||
|
||||
# Cancel remaining registered operations.
|
||||
for fut, ov, obj, callback in list(self._cache.values()):
|
||||
for address, (fut, ov, obj, callback) in list(self._cache.items()):
|
||||
if fut.cancelled():
|
||||
# Nothing to do with cancelled futures
|
||||
pass
|
||||
@@ -848,25 +742,14 @@ class IocpProactor:
|
||||
context['source_traceback'] = fut._source_traceback
|
||||
self._loop.call_exception_handler(context)
|
||||
|
||||
# Wait until all cancelled overlapped complete: don't exit with running
|
||||
# overlapped to prevent a crash. Display progress every second if the
|
||||
# loop is still running.
|
||||
msg_update = 1.0
|
||||
start_time = time.monotonic()
|
||||
next_msg = start_time + msg_update
|
||||
while self._cache:
|
||||
if next_msg <= time.monotonic():
|
||||
logger.debug('%r is running after closing for %.1f seconds',
|
||||
self, time.monotonic() - start_time)
|
||||
next_msg = time.monotonic() + msg_update
|
||||
|
||||
# handle a few events, or timeout
|
||||
self._poll(msg_update)
|
||||
if not self._poll(1):
|
||||
logger.debug('taking long time to close proactor')
|
||||
|
||||
self._results = []
|
||||
|
||||
_winapi.CloseHandle(self._iocp)
|
||||
self._iocp = None
|
||||
if self._iocp is not None:
|
||||
_winapi.CloseHandle(self._iocp)
|
||||
self._iocp = None
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
@@ -890,12 +773,8 @@ class _WindowsSubprocessTransport(base_subprocess.BaseSubprocessTransport):
|
||||
SelectorEventLoop = _WindowsSelectorEventLoop
|
||||
|
||||
|
||||
class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
|
||||
class _WindowsDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
|
||||
_loop_factory = SelectorEventLoop
|
||||
|
||||
|
||||
class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
|
||||
_loop_factory = ProactorEventLoop
|
||||
|
||||
|
||||
DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy
|
||||
DefaultEventLoopPolicy = _WindowsDefaultEventLoopPolicy
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
"""Various Windows specific bits and pieces."""
|
||||
"""
|
||||
Various Windows specific bits and pieces
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
@@ -9,12 +11,13 @@ import _winapi
|
||||
import itertools
|
||||
import msvcrt
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import warnings
|
||||
|
||||
|
||||
__all__ = 'pipe', 'Popen', 'PIPE', 'PipeHandle'
|
||||
__all__ = ['socketpair', 'pipe', 'Popen', 'PIPE', 'PipeHandle']
|
||||
|
||||
|
||||
# Constants/globals
|
||||
@@ -26,14 +29,61 @@ STDOUT = subprocess.STDOUT
|
||||
_mmap_counter = itertools.count()
|
||||
|
||||
|
||||
if hasattr(socket, 'socketpair'):
|
||||
# Since Python 3.5, socket.socketpair() is now also available on Windows
|
||||
socketpair = socket.socketpair
|
||||
else:
|
||||
# Replacement for socket.socketpair()
|
||||
def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
|
||||
"""A socket pair usable as a self-pipe, for Windows.
|
||||
|
||||
Origin: https://gist.github.com/4325783, by Geert Jansen.
|
||||
Public domain.
|
||||
"""
|
||||
if family == socket.AF_INET:
|
||||
host = '127.0.0.1'
|
||||
elif family == socket.AF_INET6:
|
||||
host = '::1'
|
||||
else:
|
||||
raise ValueError("Only AF_INET and AF_INET6 socket address "
|
||||
"families are supported")
|
||||
if type != socket.SOCK_STREAM:
|
||||
raise ValueError("Only SOCK_STREAM socket type is supported")
|
||||
if proto != 0:
|
||||
raise ValueError("Only protocol zero is supported")
|
||||
|
||||
# We create a connected TCP socket. Note the trick with setblocking(0)
|
||||
# that prevents us from having to create a thread.
|
||||
lsock = socket.socket(family, type, proto)
|
||||
try:
|
||||
lsock.bind((host, 0))
|
||||
lsock.listen(1)
|
||||
# On IPv6, ignore flow_info and scope_id
|
||||
addr, port = lsock.getsockname()[:2]
|
||||
csock = socket.socket(family, type, proto)
|
||||
try:
|
||||
csock.setblocking(False)
|
||||
try:
|
||||
csock.connect((addr, port))
|
||||
except (BlockingIOError, InterruptedError):
|
||||
pass
|
||||
csock.setblocking(True)
|
||||
ssock, _ = lsock.accept()
|
||||
except:
|
||||
csock.close()
|
||||
raise
|
||||
finally:
|
||||
lsock.close()
|
||||
return (ssock, csock)
|
||||
|
||||
|
||||
# Replacement for os.pipe() using handles instead of fds
|
||||
|
||||
|
||||
def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE):
|
||||
"""Like os.pipe() but with overlapped support and using handles not fds."""
|
||||
address = tempfile.mktemp(
|
||||
prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format(
|
||||
os.getpid(), next(_mmap_counter)))
|
||||
address = tempfile.mktemp(prefix=r'\\.\pipe\python-pipe-%d-%d-' %
|
||||
(os.getpid(), next(_mmap_counter)))
|
||||
|
||||
if duplex:
|
||||
openmode = _winapi.PIPE_ACCESS_DUPLEX
|
||||
@@ -88,10 +138,10 @@ class PipeHandle:
|
||||
|
||||
def __repr__(self):
|
||||
if self._handle is not None:
|
||||
handle = f'handle={self._handle!r}'
|
||||
handle = 'handle=%r' % self._handle
|
||||
else:
|
||||
handle = 'closed'
|
||||
return f'<{self.__class__.__name__} {handle}>'
|
||||
return '<%s %s>' % (self.__class__.__name__, handle)
|
||||
|
||||
@property
|
||||
def handle(self):
|
||||
@@ -99,7 +149,7 @@ class PipeHandle:
|
||||
|
||||
def fileno(self):
|
||||
if self._handle is None:
|
||||
raise ValueError("I/O operation on closed pipe")
|
||||
raise ValueError("I/O operatioon on closed pipe")
|
||||
return self._handle
|
||||
|
||||
def close(self, *, CloseHandle=_winapi.CloseHandle):
|
||||
@@ -107,9 +157,10 @@ class PipeHandle:
|
||||
CloseHandle(self._handle)
|
||||
self._handle = None
|
||||
|
||||
def __del__(self, _warn=warnings.warn):
|
||||
def __del__(self):
|
||||
if self._handle is not None:
|
||||
_warn(f"unclosed {self!r}", ResourceWarning, source=self)
|
||||
warnings.warn("unclosed %r" % self, ResourceWarning,
|
||||
source=self)
|
||||
self.close()
|
||||
|
||||
def __enter__(self):
|
||||
|
||||
159
Lib/base64.py
vendored
159
Lib/base64.py
vendored
@@ -1,4 +1,4 @@
|
||||
#! /usr/bin/env python3
|
||||
#! /usr/bin/python3.6
|
||||
|
||||
"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings"""
|
||||
|
||||
@@ -16,7 +16,7 @@ __all__ = [
|
||||
'encode', 'decode', 'encodebytes', 'decodebytes',
|
||||
# Generalized interface for other encodings
|
||||
'b64encode', 'b64decode', 'b32encode', 'b32decode',
|
||||
'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode',
|
||||
'b16encode', 'b16decode',
|
||||
# Base85 and Ascii85 encodings
|
||||
'b85encode', 'b85decode', 'a85encode', 'a85decode',
|
||||
# Standard Base64 encoding
|
||||
@@ -76,16 +76,15 @@ def b64decode(s, altchars=None, validate=False):
|
||||
normal base-64 alphabet nor the alternative alphabet are discarded prior
|
||||
to the padding check. If validate is True, these non-alphabet characters
|
||||
in the input result in a binascii.Error.
|
||||
For more information about the strict base64 check, see:
|
||||
|
||||
https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64
|
||||
"""
|
||||
s = _bytes_from_decode_data(s)
|
||||
if altchars is not None:
|
||||
altchars = _bytes_from_decode_data(altchars)
|
||||
assert len(altchars) == 2, repr(altchars)
|
||||
s = s.translate(bytes.maketrans(altchars, b'+/'))
|
||||
return binascii.a2b_base64(s, strict_mode=validate)
|
||||
if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s):
|
||||
raise binascii.Error('Non-base64 digit found')
|
||||
return binascii.a2b_base64(s)
|
||||
|
||||
|
||||
def standard_b64encode(s):
|
||||
@@ -136,40 +135,19 @@ def urlsafe_b64decode(s):
|
||||
|
||||
|
||||
# Base32 encoding/decoding must be done in Python
|
||||
_B32_ENCODE_DOCSTRING = '''
|
||||
Encode the bytes-like objects using {encoding} and return a bytes object.
|
||||
'''
|
||||
_B32_DECODE_DOCSTRING = '''
|
||||
Decode the {encoding} encoded bytes-like object or ASCII string s.
|
||||
|
||||
Optional casefold is a flag specifying whether a lowercase alphabet is
|
||||
acceptable as input. For security purposes, the default is False.
|
||||
{extra_args}
|
||||
The result is returned as a bytes object. A binascii.Error is raised if
|
||||
the input is incorrectly padded or if there are non-alphabet
|
||||
characters present in the input.
|
||||
'''
|
||||
_B32_DECODE_MAP01_DOCSTRING = '''
|
||||
RFC 3548 allows for optional mapping of the digit 0 (zero) to the
|
||||
letter O (oh), and for optional mapping of the digit 1 (one) to
|
||||
either the letter I (eye) or letter L (el). The optional argument
|
||||
map01 when not None, specifies which letter the digit 1 should be
|
||||
mapped to (when map01 is not None, the digit 0 is always mapped to
|
||||
the letter O). For security purposes the default is None, so that
|
||||
0 and 1 are not allowed in the input.
|
||||
'''
|
||||
_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
|
||||
_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV'
|
||||
_b32tab2 = {}
|
||||
_b32rev = {}
|
||||
_b32tab2 = None
|
||||
_b32rev = None
|
||||
|
||||
def _b32encode(alphabet, s):
|
||||
def b32encode(s):
|
||||
"""Encode the bytes-like object s using Base32 and return a bytes object.
|
||||
"""
|
||||
global _b32tab2
|
||||
# Delay the initialization of the table to not waste memory
|
||||
# if the function is never called
|
||||
if alphabet not in _b32tab2:
|
||||
b32tab = [bytes((i,)) for i in alphabet]
|
||||
_b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab]
|
||||
if _b32tab2 is None:
|
||||
b32tab = [bytes((i,)) for i in _b32alphabet]
|
||||
_b32tab2 = [a + b for a in b32tab for b in b32tab]
|
||||
b32tab = None
|
||||
|
||||
if not isinstance(s, bytes_types):
|
||||
@@ -180,9 +158,9 @@ def _b32encode(alphabet, s):
|
||||
s = s + b'\0' * (5 - leftover) # Don't use += !
|
||||
encoded = bytearray()
|
||||
from_bytes = int.from_bytes
|
||||
b32tab2 = _b32tab2[alphabet]
|
||||
b32tab2 = _b32tab2
|
||||
for i in range(0, len(s), 5):
|
||||
c = from_bytes(s[i: i + 5]) # big endian
|
||||
c = from_bytes(s[i: i + 5], 'big')
|
||||
encoded += (b32tab2[c >> 30] + # bits 1 - 10
|
||||
b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20
|
||||
b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30
|
||||
@@ -199,12 +177,29 @@ def _b32encode(alphabet, s):
|
||||
encoded[-1:] = b'='
|
||||
return bytes(encoded)
|
||||
|
||||
def _b32decode(alphabet, s, casefold=False, map01=None):
|
||||
def b32decode(s, casefold=False, map01=None):
|
||||
"""Decode the Base32 encoded bytes-like object or ASCII string s.
|
||||
|
||||
Optional casefold is a flag specifying whether a lowercase alphabet is
|
||||
acceptable as input. For security purposes, the default is False.
|
||||
|
||||
RFC 3548 allows for optional mapping of the digit 0 (zero) to the
|
||||
letter O (oh), and for optional mapping of the digit 1 (one) to
|
||||
either the letter I (eye) or letter L (el). The optional argument
|
||||
map01 when not None, specifies which letter the digit 1 should be
|
||||
mapped to (when map01 is not None, the digit 0 is always mapped to
|
||||
the letter O). For security purposes the default is None, so that
|
||||
0 and 1 are not allowed in the input.
|
||||
|
||||
The result is returned as a bytes object. A binascii.Error is raised if
|
||||
the input is incorrectly padded or if there are non-alphabet
|
||||
characters present in the input.
|
||||
"""
|
||||
global _b32rev
|
||||
# Delay the initialization of the table to not waste memory
|
||||
# if the function is never called
|
||||
if alphabet not in _b32rev:
|
||||
_b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)}
|
||||
if _b32rev is None:
|
||||
_b32rev = {v: k for k, v in enumerate(_b32alphabet)}
|
||||
s = _bytes_from_decode_data(s)
|
||||
if len(s) % 8:
|
||||
raise binascii.Error('Incorrect padding')
|
||||
@@ -225,7 +220,7 @@ def _b32decode(alphabet, s, casefold=False, map01=None):
|
||||
padchars = l - len(s)
|
||||
# Now decode the full quanta
|
||||
decoded = bytearray()
|
||||
b32rev = _b32rev[alphabet]
|
||||
b32rev = _b32rev
|
||||
for i in range(0, len(s), 8):
|
||||
quanta = s[i: i + 8]
|
||||
acc = 0
|
||||
@@ -234,38 +229,18 @@ def _b32decode(alphabet, s, casefold=False, map01=None):
|
||||
acc = (acc << 5) + b32rev[c]
|
||||
except KeyError:
|
||||
raise binascii.Error('Non-base32 digit found') from None
|
||||
decoded += acc.to_bytes(5) # big endian
|
||||
decoded += acc.to_bytes(5, 'big')
|
||||
# Process the last, partial quanta
|
||||
if l % 8 or padchars not in {0, 1, 3, 4, 6}:
|
||||
raise binascii.Error('Incorrect padding')
|
||||
if padchars and decoded:
|
||||
acc <<= 5 * padchars
|
||||
last = acc.to_bytes(5) # big endian
|
||||
last = acc.to_bytes(5, 'big')
|
||||
leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1
|
||||
decoded[-5:] = last[:leftover]
|
||||
return bytes(decoded)
|
||||
|
||||
|
||||
def b32encode(s):
|
||||
return _b32encode(_b32alphabet, s)
|
||||
b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32')
|
||||
|
||||
def b32decode(s, casefold=False, map01=None):
|
||||
return _b32decode(_b32alphabet, s, casefold, map01)
|
||||
b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32',
|
||||
extra_args=_B32_DECODE_MAP01_DOCSTRING)
|
||||
|
||||
def b32hexencode(s):
|
||||
return _b32encode(_b32hexalphabet, s)
|
||||
b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex')
|
||||
|
||||
def b32hexdecode(s, casefold=False):
|
||||
# base32hex does not have the 01 mapping
|
||||
return _b32decode(_b32hexalphabet, s, casefold)
|
||||
b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex',
|
||||
extra_args='')
|
||||
|
||||
|
||||
# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
|
||||
# lowercase. The RFC also recommends against accepting input case
|
||||
# insensitively.
|
||||
@@ -345,7 +320,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
|
||||
global _a85chars, _a85chars2
|
||||
# Delay the initialization of tables to not waste memory
|
||||
# if the function is never called
|
||||
if _a85chars2 is None:
|
||||
if _a85chars is None:
|
||||
_a85chars = [bytes((i,)) for i in range(33, 118)]
|
||||
_a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]
|
||||
|
||||
@@ -453,7 +428,7 @@ def b85encode(b, pad=False):
|
||||
global _b85chars, _b85chars2
|
||||
# Delay the initialization of tables to not waste memory
|
||||
# if the function is never called
|
||||
if _b85chars2 is None:
|
||||
if _b85chars is None:
|
||||
_b85chars = [bytes((i,)) for i in _b85alphabet]
|
||||
_b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
|
||||
return _85encode(b, _b85chars, _b85chars2, pad)
|
||||
@@ -508,8 +483,14 @@ MAXBINSIZE = (MAXLINESIZE//4)*3
|
||||
|
||||
def encode(input, output):
|
||||
"""Encode a file; input and output are binary files."""
|
||||
while s := input.read(MAXBINSIZE):
|
||||
while len(s) < MAXBINSIZE and (ns := input.read(MAXBINSIZE-len(s))):
|
||||
while True:
|
||||
s = input.read(MAXBINSIZE)
|
||||
if not s:
|
||||
break
|
||||
while len(s) < MAXBINSIZE:
|
||||
ns = input.read(MAXBINSIZE-len(s))
|
||||
if not ns:
|
||||
break
|
||||
s += ns
|
||||
line = binascii.b2a_base64(s)
|
||||
output.write(line)
|
||||
@@ -517,7 +498,10 @@ def encode(input, output):
|
||||
|
||||
def decode(input, output):
|
||||
"""Decode a file; input and output are binary files."""
|
||||
while line := input.readline():
|
||||
while True:
|
||||
line = input.readline()
|
||||
if not line:
|
||||
break
|
||||
s = binascii.a2b_base64(line)
|
||||
output.write(s)
|
||||
|
||||
@@ -547,34 +531,49 @@ def encodebytes(s):
|
||||
pieces.append(binascii.b2a_base64(chunk))
|
||||
return b"".join(pieces)
|
||||
|
||||
def encodestring(s):
|
||||
"""Legacy alias of encodebytes()."""
|
||||
import warnings
|
||||
warnings.warn("encodestring() is a deprecated alias since 3.1, "
|
||||
"use encodebytes()",
|
||||
DeprecationWarning, 2)
|
||||
return encodebytes(s)
|
||||
|
||||
|
||||
def decodebytes(s):
|
||||
"""Decode a bytestring of base-64 data into a bytes object."""
|
||||
_input_type_check(s)
|
||||
return binascii.a2b_base64(s)
|
||||
|
||||
def decodestring(s):
|
||||
"""Legacy alias of decodebytes()."""
|
||||
import warnings
|
||||
warnings.warn("decodestring() is a deprecated alias since Python 3.1, "
|
||||
"use decodebytes()",
|
||||
DeprecationWarning, 2)
|
||||
return decodebytes(s)
|
||||
|
||||
|
||||
# Usable as a script...
|
||||
def main():
|
||||
"""Small main program"""
|
||||
import sys, getopt
|
||||
usage = f"""usage: {sys.argv[0]} [-h|-d|-e|-u] [file|-]
|
||||
-h: print this help message and exit
|
||||
-d, -u: decode
|
||||
-e: encode (default)"""
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'hdeu')
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'deut')
|
||||
except getopt.error as msg:
|
||||
sys.stdout = sys.stderr
|
||||
print(msg)
|
||||
print(usage)
|
||||
print("""usage: %s [-d|-e|-u|-t] [file|-]
|
||||
-d, -u: decode
|
||||
-e: encode (default)
|
||||
-t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0])
|
||||
sys.exit(2)
|
||||
func = encode
|
||||
for o, a in opts:
|
||||
if o == '-e': func = encode
|
||||
if o == '-d': func = decode
|
||||
if o == '-u': func = decode
|
||||
if o == '-h': print(usage); return
|
||||
if o == '-t': test(); return
|
||||
if args and args[0] != '-':
|
||||
with open(args[0], 'rb') as f:
|
||||
func(f, sys.stdout.buffer)
|
||||
@@ -582,5 +581,15 @@ def main():
|
||||
func(sys.stdin.buffer, sys.stdout.buffer)
|
||||
|
||||
|
||||
def test():
|
||||
s0 = b"Aladdin:open sesame"
|
||||
print(repr(s0))
|
||||
s1 = encodebytes(s0)
|
||||
print(repr(s1))
|
||||
s2 = decodebytes(s1)
|
||||
print(repr(s2))
|
||||
assert s0 == s2
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
75
Lib/bdb.py
vendored
75
Lib/bdb.py
vendored
@@ -34,8 +34,6 @@ class Bdb:
|
||||
self.fncache = {}
|
||||
self.frame_returning = None
|
||||
|
||||
self._load_breaks()
|
||||
|
||||
def canonic(self, filename):
|
||||
"""Return canonical form of filename.
|
||||
|
||||
@@ -119,7 +117,7 @@ class Bdb:
|
||||
"""Invoke user function and return trace function for call event.
|
||||
|
||||
If the debugger stops on this function call, invoke
|
||||
self.user_call(). Raise BdbQuit if self.quitting is set.
|
||||
self.user_call(). Raise BbdQuit if self.quitting is set.
|
||||
Return self.trace_dispatch to continue tracing in this scope.
|
||||
"""
|
||||
# XXX 'arg' is no longer used
|
||||
@@ -367,12 +365,6 @@ class Bdb:
|
||||
# Call self.get_*break*() to see the breakpoints or better
|
||||
# for bp in Breakpoint.bpbynumber: if bp: bp.bpprint().
|
||||
|
||||
def _add_to_breaks(self, filename, lineno):
|
||||
"""Add breakpoint to breaks, if not already there."""
|
||||
bp_linenos = self.breaks.setdefault(filename, [])
|
||||
if lineno not in bp_linenos:
|
||||
bp_linenos.append(lineno)
|
||||
|
||||
def set_break(self, filename, lineno, temporary=False, cond=None,
|
||||
funcname=None):
|
||||
"""Set a new breakpoint for filename:lineno.
|
||||
@@ -385,21 +377,12 @@ class Bdb:
|
||||
line = linecache.getline(filename, lineno)
|
||||
if not line:
|
||||
return 'Line %s:%d does not exist' % (filename, lineno)
|
||||
self._add_to_breaks(filename, lineno)
|
||||
list = self.breaks.setdefault(filename, [])
|
||||
if lineno not in list:
|
||||
list.append(lineno)
|
||||
bp = Breakpoint(filename, lineno, temporary, cond, funcname)
|
||||
return None
|
||||
|
||||
def _load_breaks(self):
|
||||
"""Apply all breakpoints (set in other instances) to this one.
|
||||
|
||||
Populates this instance's breaks list from the Breakpoint class's
|
||||
list, which can have breakpoints set by another Bdb instance. This
|
||||
is necessary for interactive sessions to keep the breakpoints
|
||||
active across multiple calls to run().
|
||||
"""
|
||||
for (filename, lineno) in Breakpoint.bplist.keys():
|
||||
self._add_to_breaks(filename, lineno)
|
||||
|
||||
def _prune_breaks(self, filename, lineno):
|
||||
"""Prune breakpoints for filename:lineno.
|
||||
|
||||
@@ -570,12 +553,9 @@ class Bdb:
|
||||
rv = frame.f_locals['__return__']
|
||||
s += '->'
|
||||
s += reprlib.repr(rv)
|
||||
if lineno is not None:
|
||||
line = linecache.getline(filename, lineno, frame.f_globals)
|
||||
if line:
|
||||
s += lprefix + line.strip()
|
||||
else:
|
||||
s += f'{lprefix}Warning: lineno is None'
|
||||
line = linecache.getline(filename, lineno, frame.f_globals)
|
||||
if line:
|
||||
s += lprefix + line.strip()
|
||||
return s
|
||||
|
||||
# The following methods can be called by clients to use
|
||||
@@ -631,11 +611,26 @@ class Bdb:
|
||||
|
||||
# This method is more useful to debug a single function call.
|
||||
|
||||
def runcall(self, func, /, *args, **kwds):
|
||||
def runcall(*args, **kwds):
|
||||
"""Debug a single function call.
|
||||
|
||||
Return the result of the function call.
|
||||
"""
|
||||
if len(args) >= 2:
|
||||
self, func, *args = args
|
||||
elif not args:
|
||||
raise TypeError("descriptor 'runcall' of 'Bdb' object "
|
||||
"needs an argument")
|
||||
elif 'func' in kwds:
|
||||
func = kwds.pop('func')
|
||||
self, *args = args
|
||||
import warnings
|
||||
warnings.warn("Passing 'func' as keyword argument is deprecated",
|
||||
DeprecationWarning, stacklevel=2)
|
||||
else:
|
||||
raise TypeError('runcall expected at least 1 positional argument, '
|
||||
'got %d' % (len(args)-1))
|
||||
|
||||
self.reset()
|
||||
sys.settrace(self.trace_dispatch)
|
||||
res = None
|
||||
@@ -647,6 +642,7 @@ class Bdb:
|
||||
self.quitting = True
|
||||
sys.settrace(None)
|
||||
return res
|
||||
runcall.__text_signature__ = '($self, func, /, *args, **kwds)'
|
||||
|
||||
|
||||
def set_trace():
|
||||
@@ -701,12 +697,6 @@ class Breakpoint:
|
||||
else:
|
||||
self.bplist[file, line] = [self]
|
||||
|
||||
@staticmethod
|
||||
def clearBreakpoints():
|
||||
Breakpoint.next = 1
|
||||
Breakpoint.bplist = {}
|
||||
Breakpoint.bpbynumber = [None]
|
||||
|
||||
def deleteMe(self):
|
||||
"""Delete the breakpoint from the list associated to a file:line.
|
||||
|
||||
@@ -808,18 +798,15 @@ def checkfuncname(b, frame):
|
||||
return True
|
||||
|
||||
|
||||
# Determines if there is an effective (active) breakpoint at this
|
||||
# line of code. Returns breakpoint number or 0 if none
|
||||
def effective(file, line, frame):
|
||||
"""Return (active breakpoint, delete temporary flag) or (None, None) as
|
||||
breakpoint to act upon.
|
||||
"""Determine which breakpoint for this file:line is to be acted upon.
|
||||
|
||||
The "active breakpoint" is the first entry in bplist[line, file] (which
|
||||
must exist) that is enabled, for which checkfuncname is True, and that
|
||||
has neither a False condition nor a positive ignore count. The flag,
|
||||
meaning that a temporary breakpoint should be deleted, is False only
|
||||
when the condiion cannot be evaluated (in which case, ignore count is
|
||||
ignored).
|
||||
|
||||
If no such entry exists, then (None, None) is returned.
|
||||
Called only if we know there is a breakpoint at this location. Return
|
||||
the breakpoint that was triggered and a boolean that indicates if it is
|
||||
ok to delete a temporary breakpoint. Return (None, None) if there is no
|
||||
matching breakpoint.
|
||||
"""
|
||||
possibles = Breakpoint.bplist[file, line]
|
||||
for b in possibles:
|
||||
|
||||
138
Lib/bisect.py
vendored
138
Lib/bisect.py
vendored
@@ -1,118 +1,92 @@
|
||||
"""Bisection algorithms."""
|
||||
|
||||
|
||||
def insort_right(a, x, lo=0, hi=None, *, key=None):
|
||||
def insort_right(a, x, lo=0, hi=None):
|
||||
"""Insert item x in list a, and keep it sorted assuming a is sorted.
|
||||
|
||||
If x is already in a, insert it to the right of the rightmost x.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
|
||||
A custom key function can be supplied to customize the sort order.
|
||||
"""
|
||||
if key is None:
|
||||
lo = bisect_right(a, x, lo, hi)
|
||||
else:
|
||||
lo = bisect_right(a, key(x), lo, hi, key=key)
|
||||
a.insert(lo, x)
|
||||
|
||||
|
||||
def bisect_right(a, x, lo=0, hi=None, *, key=None):
|
||||
"""Return the index where to insert item x in list a, assuming a is sorted.
|
||||
|
||||
The return value i is such that all e in a[:i] have e <= x, and all e in
|
||||
a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will
|
||||
insert just after the rightmost x already there.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
|
||||
A custom key function can be supplied to customize the sort order.
|
||||
"""
|
||||
|
||||
if lo < 0:
|
||||
raise ValueError('lo must be non-negative')
|
||||
if hi is None:
|
||||
hi = len(a)
|
||||
# Note, the comparison uses "<" to match the
|
||||
# __lt__() logic in list.sort() and in heapq.
|
||||
if key is None:
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
if x < a[mid]:
|
||||
hi = mid
|
||||
else:
|
||||
lo = mid + 1
|
||||
else:
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
if x < key(a[mid]):
|
||||
hi = mid
|
||||
else:
|
||||
lo = mid + 1
|
||||
while lo < hi:
|
||||
mid = (lo+hi)//2
|
||||
if x < a[mid]: hi = mid
|
||||
else: lo = mid+1
|
||||
a.insert(lo, x)
|
||||
|
||||
insort = insort_right # backward compatibility
|
||||
|
||||
def bisect_right(a, x, lo=0, hi=None):
|
||||
"""Return the index where to insert item x in list a, assuming a is sorted.
|
||||
|
||||
The return value i is such that all e in a[:i] have e <= x, and all e in
|
||||
a[i:] have e > x. So if x already appears in the list, a.insert(x) will
|
||||
insert just after the rightmost x already there.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
"""
|
||||
|
||||
if lo < 0:
|
||||
raise ValueError('lo must be non-negative')
|
||||
if hi is None:
|
||||
hi = len(a)
|
||||
while lo < hi:
|
||||
mid = (lo+hi)//2
|
||||
if x < a[mid]: hi = mid
|
||||
else: lo = mid+1
|
||||
return lo
|
||||
|
||||
bisect = bisect_right # backward compatibility
|
||||
|
||||
def insort_left(a, x, lo=0, hi=None, *, key=None):
|
||||
def insort_left(a, x, lo=0, hi=None):
|
||||
"""Insert item x in list a, and keep it sorted assuming a is sorted.
|
||||
|
||||
If x is already in a, insert it to the left of the leftmost x.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
|
||||
A custom key function can be supplied to customize the sort order.
|
||||
"""
|
||||
|
||||
if key is None:
|
||||
lo = bisect_left(a, x, lo, hi)
|
||||
else:
|
||||
lo = bisect_left(a, key(x), lo, hi, key=key)
|
||||
a.insert(lo, x)
|
||||
|
||||
def bisect_left(a, x, lo=0, hi=None, *, key=None):
|
||||
"""Return the index where to insert item x in list a, assuming a is sorted.
|
||||
|
||||
The return value i is such that all e in a[:i] have e < x, and all e in
|
||||
a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will
|
||||
insert just before the leftmost x already there.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
|
||||
A custom key function can be supplied to customize the sort order.
|
||||
"""
|
||||
|
||||
if lo < 0:
|
||||
raise ValueError('lo must be non-negative')
|
||||
if hi is None:
|
||||
hi = len(a)
|
||||
# Note, the comparison uses "<" to match the
|
||||
# __lt__() logic in list.sort() and in heapq.
|
||||
if key is None:
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
if a[mid] < x:
|
||||
lo = mid + 1
|
||||
else:
|
||||
hi = mid
|
||||
else:
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
if key(a[mid]) < x:
|
||||
lo = mid + 1
|
||||
else:
|
||||
hi = mid
|
||||
return lo
|
||||
while lo < hi:
|
||||
mid = (lo+hi)//2
|
||||
if a[mid] < x: lo = mid+1
|
||||
else: hi = mid
|
||||
a.insert(lo, x)
|
||||
|
||||
|
||||
def bisect_left(a, x, lo=0, hi=None):
|
||||
"""Return the index where to insert item x in list a, assuming a is sorted.
|
||||
|
||||
The return value i is such that all e in a[:i] have e < x, and all e in
|
||||
a[i:] have e >= x. So if x already appears in the list, a.insert(x) will
|
||||
insert just before the leftmost x already there.
|
||||
|
||||
Optional args lo (default 0) and hi (default len(a)) bound the
|
||||
slice of a to be searched.
|
||||
"""
|
||||
|
||||
if lo < 0:
|
||||
raise ValueError('lo must be non-negative')
|
||||
if hi is None:
|
||||
hi = len(a)
|
||||
while lo < hi:
|
||||
mid = (lo+hi)//2
|
||||
if a[mid] < x: lo = mid+1
|
||||
else: hi = mid
|
||||
return lo
|
||||
|
||||
# Overwrite above definitions with a fast C implementation
|
||||
try:
|
||||
from _bisect import *
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Create aliases
|
||||
bisect = bisect_right
|
||||
insort = insort_right
|
||||
|
||||
344
Lib/bz2.py
vendored
344
Lib/bz2.py
vendored
@@ -1,344 +0,0 @@
|
||||
"""Interface to the libbzip2 compression library.
|
||||
|
||||
This module provides a file interface, classes for incremental
|
||||
(de)compression, and functions for one-shot (de)compression.
|
||||
"""
|
||||
|
||||
__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor",
|
||||
"open", "compress", "decompress"]
|
||||
|
||||
__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
|
||||
|
||||
from builtins import open as _builtin_open
|
||||
import io
|
||||
import os
|
||||
import _compression
|
||||
|
||||
from _bz2 import BZ2Compressor, BZ2Decompressor
|
||||
|
||||
|
||||
_MODE_CLOSED = 0
|
||||
_MODE_READ = 1
|
||||
# Value 2 no longer used
|
||||
_MODE_WRITE = 3
|
||||
|
||||
|
||||
class BZ2File(_compression.BaseStream):
|
||||
|
||||
"""A file object providing transparent bzip2 (de)compression.
|
||||
|
||||
A BZ2File can act as a wrapper for an existing file object, or refer
|
||||
directly to a named file on disk.
|
||||
|
||||
Note that BZ2File provides a *binary* file interface - data read is
|
||||
returned as bytes, and data to be written should be given as bytes.
|
||||
"""
|
||||
|
||||
def __init__(self, filename, mode="r", *, compresslevel=9):
|
||||
"""Open a bzip2-compressed file.
|
||||
|
||||
If filename is a str, bytes, or PathLike object, it gives the
|
||||
name of the file to be opened. Otherwise, it should be a file
|
||||
object, which will be used to read or write the compressed data.
|
||||
|
||||
mode can be 'r' for reading (default), 'w' for (over)writing,
|
||||
'x' for creating exclusively, or 'a' for appending. These can
|
||||
equivalently be given as 'rb', 'wb', 'xb', and 'ab'.
|
||||
|
||||
If mode is 'w', 'x' or 'a', compresslevel can be a number between 1
|
||||
and 9 specifying the level of compression: 1 produces the least
|
||||
compression, and 9 (default) produces the most compression.
|
||||
|
||||
If mode is 'r', the input file may be the concatenation of
|
||||
multiple compressed streams.
|
||||
"""
|
||||
self._fp = None
|
||||
self._closefp = False
|
||||
self._mode = _MODE_CLOSED
|
||||
|
||||
if not (1 <= compresslevel <= 9):
|
||||
raise ValueError("compresslevel must be between 1 and 9")
|
||||
|
||||
if mode in ("", "r", "rb"):
|
||||
mode = "rb"
|
||||
mode_code = _MODE_READ
|
||||
elif mode in ("w", "wb"):
|
||||
mode = "wb"
|
||||
mode_code = _MODE_WRITE
|
||||
self._compressor = BZ2Compressor(compresslevel)
|
||||
elif mode in ("x", "xb"):
|
||||
mode = "xb"
|
||||
mode_code = _MODE_WRITE
|
||||
self._compressor = BZ2Compressor(compresslevel)
|
||||
elif mode in ("a", "ab"):
|
||||
mode = "ab"
|
||||
mode_code = _MODE_WRITE
|
||||
self._compressor = BZ2Compressor(compresslevel)
|
||||
else:
|
||||
raise ValueError("Invalid mode: %r" % (mode,))
|
||||
|
||||
if isinstance(filename, (str, bytes, os.PathLike)):
|
||||
self._fp = _builtin_open(filename, mode)
|
||||
self._closefp = True
|
||||
self._mode = mode_code
|
||||
elif hasattr(filename, "read") or hasattr(filename, "write"):
|
||||
self._fp = filename
|
||||
self._mode = mode_code
|
||||
else:
|
||||
raise TypeError("filename must be a str, bytes, file or PathLike object")
|
||||
|
||||
if self._mode == _MODE_READ:
|
||||
raw = _compression.DecompressReader(self._fp,
|
||||
BZ2Decompressor, trailing_error=OSError)
|
||||
self._buffer = io.BufferedReader(raw)
|
||||
else:
|
||||
self._pos = 0
|
||||
|
||||
def close(self):
|
||||
"""Flush and close the file.
|
||||
|
||||
May be called more than once without error. Once the file is
|
||||
closed, any other operation on it will raise a ValueError.
|
||||
"""
|
||||
if self._mode == _MODE_CLOSED:
|
||||
return
|
||||
try:
|
||||
if self._mode == _MODE_READ:
|
||||
self._buffer.close()
|
||||
elif self._mode == _MODE_WRITE:
|
||||
self._fp.write(self._compressor.flush())
|
||||
self._compressor = None
|
||||
finally:
|
||||
try:
|
||||
if self._closefp:
|
||||
self._fp.close()
|
||||
finally:
|
||||
self._fp = None
|
||||
self._closefp = False
|
||||
self._mode = _MODE_CLOSED
|
||||
self._buffer = None
|
||||
|
||||
@property
|
||||
def closed(self):
|
||||
"""True if this file is closed."""
|
||||
return self._mode == _MODE_CLOSED
|
||||
|
||||
def fileno(self):
|
||||
"""Return the file descriptor for the underlying file."""
|
||||
self._check_not_closed()
|
||||
return self._fp.fileno()
|
||||
|
||||
def seekable(self):
|
||||
"""Return whether the file supports seeking."""
|
||||
return self.readable() and self._buffer.seekable()
|
||||
|
||||
def readable(self):
|
||||
"""Return whether the file was opened for reading."""
|
||||
self._check_not_closed()
|
||||
return self._mode == _MODE_READ
|
||||
|
||||
def writable(self):
|
||||
"""Return whether the file was opened for writing."""
|
||||
self._check_not_closed()
|
||||
return self._mode == _MODE_WRITE
|
||||
|
||||
def peek(self, n=0):
|
||||
"""Return buffered data without advancing the file position.
|
||||
|
||||
Always returns at least one byte of data, unless at EOF.
|
||||
The exact number of bytes returned is unspecified.
|
||||
"""
|
||||
self._check_can_read()
|
||||
# Relies on the undocumented fact that BufferedReader.peek()
|
||||
# always returns at least one byte (except at EOF), independent
|
||||
# of the value of n
|
||||
return self._buffer.peek(n)
|
||||
|
||||
def read(self, size=-1):
|
||||
"""Read up to size uncompressed bytes from the file.
|
||||
|
||||
If size is negative or omitted, read until EOF is reached.
|
||||
Returns b'' if the file is already at EOF.
|
||||
"""
|
||||
self._check_can_read()
|
||||
return self._buffer.read(size)
|
||||
|
||||
def read1(self, size=-1):
|
||||
"""Read up to size uncompressed bytes, while trying to avoid
|
||||
making multiple reads from the underlying stream. Reads up to a
|
||||
buffer's worth of data if size is negative.
|
||||
|
||||
Returns b'' if the file is at EOF.
|
||||
"""
|
||||
self._check_can_read()
|
||||
if size < 0:
|
||||
size = io.DEFAULT_BUFFER_SIZE
|
||||
return self._buffer.read1(size)
|
||||
|
||||
def readinto(self, b):
|
||||
"""Read bytes into b.
|
||||
|
||||
Returns the number of bytes read (0 for EOF).
|
||||
"""
|
||||
self._check_can_read()
|
||||
return self._buffer.readinto(b)
|
||||
|
||||
def readline(self, size=-1):
|
||||
"""Read a line of uncompressed bytes from the file.
|
||||
|
||||
The terminating newline (if present) is retained. If size is
|
||||
non-negative, no more than size bytes will be read (in which
|
||||
case the line may be incomplete). Returns b'' if already at EOF.
|
||||
"""
|
||||
if not isinstance(size, int):
|
||||
if not hasattr(size, "__index__"):
|
||||
raise TypeError("Integer argument expected")
|
||||
size = size.__index__()
|
||||
self._check_can_read()
|
||||
return self._buffer.readline(size)
|
||||
|
||||
def readlines(self, size=-1):
|
||||
"""Read a list of lines of uncompressed bytes from the file.
|
||||
|
||||
size can be specified to control the number of lines read: no
|
||||
further lines will be read once the total size of the lines read
|
||||
so far equals or exceeds size.
|
||||
"""
|
||||
if not isinstance(size, int):
|
||||
if not hasattr(size, "__index__"):
|
||||
raise TypeError("Integer argument expected")
|
||||
size = size.__index__()
|
||||
self._check_can_read()
|
||||
return self._buffer.readlines(size)
|
||||
|
||||
def write(self, data):
|
||||
"""Write a byte string to the file.
|
||||
|
||||
Returns the number of uncompressed bytes written, which is
|
||||
always the length of data in bytes. Note that due to buffering,
|
||||
the file on disk may not reflect the data written until close()
|
||||
is called.
|
||||
"""
|
||||
self._check_can_write()
|
||||
if isinstance(data, (bytes, bytearray)):
|
||||
length = len(data)
|
||||
else:
|
||||
# accept any data that supports the buffer protocol
|
||||
data = memoryview(data)
|
||||
length = data.nbytes
|
||||
|
||||
compressed = self._compressor.compress(data)
|
||||
self._fp.write(compressed)
|
||||
self._pos += length
|
||||
return length
|
||||
|
||||
def writelines(self, seq):
|
||||
"""Write a sequence of byte strings to the file.
|
||||
|
||||
Returns the number of uncompressed bytes written.
|
||||
seq can be any iterable yielding byte strings.
|
||||
|
||||
Line separators are not added between the written byte strings.
|
||||
"""
|
||||
return _compression.BaseStream.writelines(self, seq)
|
||||
|
||||
def seek(self, offset, whence=io.SEEK_SET):
|
||||
"""Change the file position.
|
||||
|
||||
The new position is specified by offset, relative to the
|
||||
position indicated by whence. Values for whence are:
|
||||
|
||||
0: start of stream (default); offset must not be negative
|
||||
1: current stream position
|
||||
2: end of stream; offset must not be positive
|
||||
|
||||
Returns the new file position.
|
||||
|
||||
Note that seeking is emulated, so depending on the parameters,
|
||||
this operation may be extremely slow.
|
||||
"""
|
||||
self._check_can_seek()
|
||||
return self._buffer.seek(offset, whence)
|
||||
|
||||
def tell(self):
|
||||
"""Return the current file position."""
|
||||
self._check_not_closed()
|
||||
if self._mode == _MODE_READ:
|
||||
return self._buffer.tell()
|
||||
return self._pos
|
||||
|
||||
|
||||
def open(filename, mode="rb", compresslevel=9,
|
||||
encoding=None, errors=None, newline=None):
|
||||
"""Open a bzip2-compressed file in binary or text mode.
|
||||
|
||||
The filename argument can be an actual filename (a str, bytes, or
|
||||
PathLike object), or an existing file object to read from or write
|
||||
to.
|
||||
|
||||
The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or
|
||||
"ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode.
|
||||
The default mode is "rb", and the default compresslevel is 9.
|
||||
|
||||
For binary mode, this function is equivalent to the BZ2File
|
||||
constructor: BZ2File(filename, mode, compresslevel). In this case,
|
||||
the encoding, errors and newline arguments must not be provided.
|
||||
|
||||
For text mode, a BZ2File object is created, and wrapped in an
|
||||
io.TextIOWrapper instance with the specified encoding, error
|
||||
handling behavior, and line ending(s).
|
||||
|
||||
"""
|
||||
if "t" in mode:
|
||||
if "b" in mode:
|
||||
raise ValueError("Invalid mode: %r" % (mode,))
|
||||
else:
|
||||
if encoding is not None:
|
||||
raise ValueError("Argument 'encoding' not supported in binary mode")
|
||||
if errors is not None:
|
||||
raise ValueError("Argument 'errors' not supported in binary mode")
|
||||
if newline is not None:
|
||||
raise ValueError("Argument 'newline' not supported in binary mode")
|
||||
|
||||
bz_mode = mode.replace("t", "")
|
||||
binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel)
|
||||
|
||||
if "t" in mode:
|
||||
encoding = io.text_encoding(encoding)
|
||||
return io.TextIOWrapper(binary_file, encoding, errors, newline)
|
||||
else:
|
||||
return binary_file
|
||||
|
||||
|
||||
def compress(data, compresslevel=9):
|
||||
"""Compress a block of data.
|
||||
|
||||
compresslevel, if given, must be a number between 1 and 9.
|
||||
|
||||
For incremental compression, use a BZ2Compressor object instead.
|
||||
"""
|
||||
comp = BZ2Compressor(compresslevel)
|
||||
return comp.compress(data) + comp.flush()
|
||||
|
||||
|
||||
def decompress(data):
|
||||
"""Decompress a block of data.
|
||||
|
||||
For incremental decompression, use a BZ2Decompressor object instead.
|
||||
"""
|
||||
results = []
|
||||
while data:
|
||||
decomp = BZ2Decompressor()
|
||||
try:
|
||||
res = decomp.decompress(data)
|
||||
except OSError:
|
||||
if results:
|
||||
break # Leftover data is not a valid bzip2 stream; ignore it.
|
||||
else:
|
||||
raise # Error on the first iteration; bail out.
|
||||
results.append(res)
|
||||
if not decomp.eof:
|
||||
raise ValueError("Compressed data ended before the "
|
||||
"end-of-stream marker was reached")
|
||||
data = decomp.unused_data
|
||||
return b"".join(results)
|
||||
128
Lib/calendar.py
vendored
128
Lib/calendar.py
vendored
@@ -7,22 +7,15 @@ set the first day of the week (0=Monday, 6=Sunday)."""
|
||||
|
||||
import sys
|
||||
import datetime
|
||||
from enum import IntEnum, global_enum
|
||||
import locale as _locale
|
||||
from itertools import repeat
|
||||
import warnings
|
||||
|
||||
__all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
|
||||
"firstweekday", "isleap", "leapdays", "weekday", "monthrange",
|
||||
"monthcalendar", "prmonth", "month", "prcal", "calendar",
|
||||
"timegm", "month_name", "month_abbr", "day_name", "day_abbr",
|
||||
"Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar",
|
||||
"LocaleHTMLCalendar", "weekheader",
|
||||
"Day", "Month", "JANUARY", "FEBRUARY", "MARCH",
|
||||
"APRIL", "MAY", "JUNE", "JULY",
|
||||
"AUGUST", "SEPTEMBER", "OCTOBER", "NOVEMBER", "DECEMBER",
|
||||
"MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY",
|
||||
"SATURDAY", "SUNDAY"]
|
||||
"LocaleHTMLCalendar", "weekheader"]
|
||||
|
||||
# Exception raised for bad input (with string parameter for details)
|
||||
error = ValueError
|
||||
@@ -42,46 +35,9 @@ class IllegalWeekdayError(ValueError):
|
||||
return "bad weekday number %r; must be 0 (Monday) to 6 (Sunday)" % self.weekday
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
if name in ('January', 'February'):
|
||||
warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead",
|
||||
DeprecationWarning, stacklevel=2)
|
||||
if name == 'January':
|
||||
return 1
|
||||
else:
|
||||
return 2
|
||||
|
||||
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
||||
|
||||
|
||||
# Constants for months
|
||||
@global_enum
|
||||
class Month(IntEnum):
|
||||
JANUARY = 1
|
||||
FEBRUARY = 2
|
||||
MARCH = 3
|
||||
APRIL = 4
|
||||
MAY = 5
|
||||
JUNE = 6
|
||||
JULY = 7
|
||||
AUGUST = 8
|
||||
SEPTEMBER = 9
|
||||
OCTOBER = 10
|
||||
NOVEMBER = 11
|
||||
DECEMBER = 12
|
||||
|
||||
|
||||
# Constants for days
|
||||
@global_enum
|
||||
class Day(IntEnum):
|
||||
MONDAY = 0
|
||||
TUESDAY = 1
|
||||
WEDNESDAY = 2
|
||||
THURSDAY = 3
|
||||
FRIDAY = 4
|
||||
SATURDAY = 5
|
||||
SUNDAY = 6
|
||||
|
||||
# Constants for months referenced later
|
||||
January = 1
|
||||
February = 2
|
||||
|
||||
# Number of days per month (except for February in leap years)
|
||||
mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
|
||||
@@ -137,6 +93,9 @@ day_abbr = _localized_day('%a')
|
||||
month_name = _localized_month('%B')
|
||||
month_abbr = _localized_month('%b')
|
||||
|
||||
# Constants for weekdays
|
||||
(MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY) = range(7)
|
||||
|
||||
|
||||
def isleap(year):
|
||||
"""Return True for leap years, False for non-leap years."""
|
||||
@@ -155,7 +114,7 @@ def weekday(year, month, day):
|
||||
"""Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31)."""
|
||||
if not datetime.MINYEAR <= year <= datetime.MAXYEAR:
|
||||
year = 2000 + year % 400
|
||||
return Day(datetime.date(year, month, day).weekday())
|
||||
return datetime.date(year, month, day).weekday()
|
||||
|
||||
|
||||
def monthrange(year, month):
|
||||
@@ -164,12 +123,12 @@ def monthrange(year, month):
|
||||
if not 1 <= month <= 12:
|
||||
raise IllegalMonthError(month)
|
||||
day1 = weekday(year, month, 1)
|
||||
ndays = mdays[month] + (month == FEBRUARY and isleap(year))
|
||||
ndays = mdays[month] + (month == February and isleap(year))
|
||||
return day1, ndays
|
||||
|
||||
|
||||
def _monthlen(year, month):
|
||||
return mdays[month] + (month == FEBRUARY and isleap(year))
|
||||
return mdays[month] + (month == February and isleap(year))
|
||||
|
||||
|
||||
def _prevmonth(year, month):
|
||||
@@ -299,7 +258,10 @@ class Calendar(object):
|
||||
Each month contains between 4 and 6 weeks and each week contains 1-7
|
||||
days. Days are datetime.date objects.
|
||||
"""
|
||||
months = [self.monthdatescalendar(year, m) for m in Month]
|
||||
months = [
|
||||
self.monthdatescalendar(year, i)
|
||||
for i in range(January, January+12)
|
||||
]
|
||||
return [months[i:i+width] for i in range(0, len(months), width) ]
|
||||
|
||||
def yeardays2calendar(self, year, width=3):
|
||||
@@ -309,7 +271,10 @@ class Calendar(object):
|
||||
(day number, weekday number) tuples. Day numbers outside this month are
|
||||
zero.
|
||||
"""
|
||||
months = [self.monthdays2calendar(year, m) for m in Month]
|
||||
months = [
|
||||
self.monthdays2calendar(year, i)
|
||||
for i in range(January, January+12)
|
||||
]
|
||||
return [months[i:i+width] for i in range(0, len(months), width) ]
|
||||
|
||||
def yeardayscalendar(self, year, width=3):
|
||||
@@ -318,7 +283,10 @@ class Calendar(object):
|
||||
yeardatescalendar()). Entries in the week lists are day numbers.
|
||||
Day numbers outside this month are zero.
|
||||
"""
|
||||
months = [self.monthdayscalendar(year, m) for m in Month]
|
||||
months = [
|
||||
self.monthdayscalendar(year, i)
|
||||
for i in range(January, January+12)
|
||||
]
|
||||
return [months[i:i+width] for i in range(0, len(months), width) ]
|
||||
|
||||
|
||||
@@ -539,7 +507,7 @@ class HTMLCalendar(Calendar):
|
||||
a('\n')
|
||||
a('<tr><th colspan="%d" class="%s">%s</th></tr>' % (
|
||||
width, self.cssclass_year_head, theyear))
|
||||
for i in range(JANUARY, JANUARY+12, width):
|
||||
for i in range(January, January+12, width):
|
||||
# months in this row
|
||||
months = range(i, min(i+width, 13))
|
||||
a('<tr>')
|
||||
@@ -578,67 +546,71 @@ class HTMLCalendar(Calendar):
|
||||
class different_locale:
|
||||
def __init__(self, locale):
|
||||
self.locale = locale
|
||||
self.oldlocale = None
|
||||
|
||||
def __enter__(self):
|
||||
self.oldlocale = _locale.setlocale(_locale.LC_TIME, None)
|
||||
self.oldlocale = _locale.getlocale(_locale.LC_TIME)
|
||||
_locale.setlocale(_locale.LC_TIME, self.locale)
|
||||
|
||||
def __exit__(self, *args):
|
||||
if self.oldlocale is None:
|
||||
return
|
||||
_locale.setlocale(_locale.LC_TIME, self.oldlocale)
|
||||
|
||||
|
||||
def _get_default_locale():
|
||||
locale = _locale.setlocale(_locale.LC_TIME, None)
|
||||
if locale == "C":
|
||||
with different_locale(""):
|
||||
# The LC_TIME locale does not seem to be configured:
|
||||
# get the user preferred locale.
|
||||
locale = _locale.setlocale(_locale.LC_TIME, None)
|
||||
return locale
|
||||
|
||||
|
||||
class LocaleTextCalendar(TextCalendar):
|
||||
"""
|
||||
This class can be passed a locale name in the constructor and will return
|
||||
month and weekday names in the specified locale.
|
||||
month and weekday names in the specified locale. If this locale includes
|
||||
an encoding all strings containing month and weekday names will be returned
|
||||
as unicode.
|
||||
"""
|
||||
|
||||
def __init__(self, firstweekday=0, locale=None):
|
||||
TextCalendar.__init__(self, firstweekday)
|
||||
if locale is None:
|
||||
locale = _get_default_locale()
|
||||
locale = _locale.getdefaultlocale()
|
||||
self.locale = locale
|
||||
|
||||
def formatweekday(self, day, width):
|
||||
with different_locale(self.locale):
|
||||
return super().formatweekday(day, width)
|
||||
if width >= 9:
|
||||
names = day_name
|
||||
else:
|
||||
names = day_abbr
|
||||
name = names[day]
|
||||
return name[:width].center(width)
|
||||
|
||||
def formatmonthname(self, theyear, themonth, width, withyear=True):
|
||||
with different_locale(self.locale):
|
||||
return super().formatmonthname(theyear, themonth, width, withyear)
|
||||
s = month_name[themonth]
|
||||
if withyear:
|
||||
s = "%s %r" % (s, theyear)
|
||||
return s.center(width)
|
||||
|
||||
|
||||
class LocaleHTMLCalendar(HTMLCalendar):
|
||||
"""
|
||||
This class can be passed a locale name in the constructor and will return
|
||||
month and weekday names in the specified locale.
|
||||
month and weekday names in the specified locale. If this locale includes
|
||||
an encoding all strings containing month and weekday names will be returned
|
||||
as unicode.
|
||||
"""
|
||||
def __init__(self, firstweekday=0, locale=None):
|
||||
HTMLCalendar.__init__(self, firstweekday)
|
||||
if locale is None:
|
||||
locale = _get_default_locale()
|
||||
locale = _locale.getdefaultlocale()
|
||||
self.locale = locale
|
||||
|
||||
def formatweekday(self, day):
|
||||
with different_locale(self.locale):
|
||||
return super().formatweekday(day)
|
||||
s = day_abbr[day]
|
||||
return '<th class="%s">%s</th>' % (self.cssclasses[day], s)
|
||||
|
||||
def formatmonthname(self, theyear, themonth, withyear=True):
|
||||
with different_locale(self.locale):
|
||||
return super().formatmonthname(theyear, themonth, withyear)
|
||||
s = month_name[themonth]
|
||||
if withyear:
|
||||
s = '%s %s' % (s, theyear)
|
||||
return '<tr><th colspan="7" class="month">%s</th></tr>' % s
|
||||
|
||||
|
||||
# Support for old module level interface
|
||||
c = TextCalendar()
|
||||
@@ -723,7 +695,7 @@ def main(args):
|
||||
parser.add_argument(
|
||||
"-L", "--locale",
|
||||
default=None,
|
||||
help="locale to use for month and weekday names"
|
||||
help="locale to be used from month and weekday names"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-e", "--encoding",
|
||||
|
||||
46
Lib/cgi.py
vendored
46
Lib/cgi.py
vendored
@@ -13,11 +13,6 @@
|
||||
|
||||
This module defines a number of utilities for use by CGI scripts
|
||||
written in Python.
|
||||
|
||||
The global variable maxlen can be set to an integer indicating the maximum size
|
||||
of a POST request. POST requests larger than this size will result in a
|
||||
ValueError being raised during parsing. The default value of this variable is 0,
|
||||
meaning the request size is unlimited.
|
||||
"""
|
||||
|
||||
# History
|
||||
@@ -46,16 +41,12 @@ from email.message import Message
|
||||
import html
|
||||
import locale
|
||||
import tempfile
|
||||
import warnings
|
||||
|
||||
__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart",
|
||||
"parse_header", "test", "print_exception", "print_environ",
|
||||
"print_form", "print_directory", "print_arguments",
|
||||
"print_environ_usage"]
|
||||
|
||||
|
||||
warnings._deprecated(__name__, remove=(3,13))
|
||||
|
||||
# Logging support
|
||||
# ===============
|
||||
|
||||
@@ -86,11 +77,9 @@ def initlog(*allargs):
|
||||
|
||||
"""
|
||||
global log, logfile, logfp
|
||||
warnings.warn("cgi.log() is deprecated as of 3.10. Use logging instead",
|
||||
DeprecationWarning, stacklevel=2)
|
||||
if logfile and not logfp:
|
||||
try:
|
||||
logfp = open(logfile, "a", encoding="locale")
|
||||
logfp = open(logfile, "a")
|
||||
except OSError:
|
||||
pass
|
||||
if not logfp:
|
||||
@@ -126,8 +115,7 @@ log = initlog # The current logging function
|
||||
# 0 ==> unlimited input
|
||||
maxlen = 0
|
||||
|
||||
def parse(fp=None, environ=os.environ, keep_blank_values=0,
|
||||
strict_parsing=0, separator='&'):
|
||||
def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
"""Parse a query in the environment or from a file (default stdin)
|
||||
|
||||
Arguments, all optional:
|
||||
@@ -146,9 +134,6 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0,
|
||||
strict_parsing: flag indicating what to do with parsing errors.
|
||||
If false (the default), errors are silently ignored.
|
||||
If true, errors raise a ValueError exception.
|
||||
|
||||
separator: str. The symbol to use for separating the query arguments.
|
||||
Defaults to &.
|
||||
"""
|
||||
if fp is None:
|
||||
fp = sys.stdin
|
||||
@@ -169,7 +154,7 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0,
|
||||
if environ['REQUEST_METHOD'] == 'POST':
|
||||
ctype, pdict = parse_header(environ['CONTENT_TYPE'])
|
||||
if ctype == 'multipart/form-data':
|
||||
return parse_multipart(fp, pdict, separator=separator)
|
||||
return parse_multipart(fp, pdict)
|
||||
elif ctype == 'application/x-www-form-urlencoded':
|
||||
clength = int(environ['CONTENT_LENGTH'])
|
||||
if maxlen and clength > maxlen:
|
||||
@@ -193,10 +178,10 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0,
|
||||
qs = ""
|
||||
environ['QUERY_STRING'] = qs # XXX Shouldn't, really
|
||||
return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
|
||||
encoding=encoding, separator=separator)
|
||||
encoding=encoding)
|
||||
|
||||
|
||||
def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'):
|
||||
def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"):
|
||||
"""Parse multipart input.
|
||||
|
||||
Arguments:
|
||||
@@ -209,18 +194,15 @@ def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'
|
||||
value is a list of values for that field. For non-file fields, the value
|
||||
is a list of strings.
|
||||
"""
|
||||
# RFC 2046, Section 5.1 : The "multipart" boundary delimiters are always
|
||||
# RFC 2026, Section 5.1 : The "multipart" boundary delimiters are always
|
||||
# represented as 7bit US-ASCII.
|
||||
boundary = pdict['boundary'].decode('ascii')
|
||||
ctype = "multipart/form-data; boundary={}".format(boundary)
|
||||
headers = Message()
|
||||
headers.set_type(ctype)
|
||||
try:
|
||||
headers['Content-Length'] = pdict['CONTENT-LENGTH']
|
||||
except KeyError:
|
||||
pass
|
||||
headers['Content-Length'] = pdict['CONTENT-LENGTH']
|
||||
fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors,
|
||||
environ={'REQUEST_METHOD': 'POST'}, separator=separator)
|
||||
environ={'REQUEST_METHOD': 'POST'})
|
||||
return {k: fs.getlist(k) for k in fs}
|
||||
|
||||
def _parseparam(s):
|
||||
@@ -330,7 +312,7 @@ class FieldStorage:
|
||||
def __init__(self, fp=None, headers=None, outerboundary=b'',
|
||||
environ=os.environ, keep_blank_values=0, strict_parsing=0,
|
||||
limit=None, encoding='utf-8', errors='replace',
|
||||
max_num_fields=None, separator='&'):
|
||||
max_num_fields=None):
|
||||
"""Constructor. Read multipart/* until last part.
|
||||
|
||||
Arguments, all optional:
|
||||
@@ -378,7 +360,6 @@ class FieldStorage:
|
||||
self.keep_blank_values = keep_blank_values
|
||||
self.strict_parsing = strict_parsing
|
||||
self.max_num_fields = max_num_fields
|
||||
self.separator = separator
|
||||
if 'REQUEST_METHOD' in environ:
|
||||
method = environ['REQUEST_METHOD'].upper()
|
||||
self.qs_on_post = None
|
||||
@@ -605,7 +586,7 @@ class FieldStorage:
|
||||
query = urllib.parse.parse_qsl(
|
||||
qs, self.keep_blank_values, self.strict_parsing,
|
||||
encoding=self.encoding, errors=self.errors,
|
||||
max_num_fields=self.max_num_fields, separator=self.separator)
|
||||
max_num_fields=self.max_num_fields)
|
||||
self.list = [MiniFieldStorage(key, value) for key, value in query]
|
||||
self.skip_lines()
|
||||
|
||||
@@ -621,7 +602,7 @@ class FieldStorage:
|
||||
query = urllib.parse.parse_qsl(
|
||||
self.qs_on_post, self.keep_blank_values, self.strict_parsing,
|
||||
encoding=self.encoding, errors=self.errors,
|
||||
max_num_fields=self.max_num_fields, separator=self.separator)
|
||||
max_num_fields=self.max_num_fields)
|
||||
self.list.extend(MiniFieldStorage(key, value) for key, value in query)
|
||||
|
||||
klass = self.FieldStorageClass or self.__class__
|
||||
@@ -665,7 +646,7 @@ class FieldStorage:
|
||||
else self.limit - self.bytes_read
|
||||
part = klass(self.fp, headers, ib, environ, keep_blank_values,
|
||||
strict_parsing, limit,
|
||||
self.encoding, self.errors, max_num_fields, self.separator)
|
||||
self.encoding, self.errors, max_num_fields)
|
||||
|
||||
if max_num_fields is not None:
|
||||
max_num_fields -= 1
|
||||
@@ -755,8 +736,7 @@ class FieldStorage:
|
||||
last_line_lfend = True
|
||||
_read = 0
|
||||
while 1:
|
||||
|
||||
if self.limit is not None and 0 <= self.limit <= _read:
|
||||
if self.limit is not None and _read >= self.limit:
|
||||
break
|
||||
line = self.fp.readline(1<<16) # bytes
|
||||
self.bytes_read += len(line)
|
||||
|
||||
332
Lib/cgitb.py
vendored
332
Lib/cgitb.py
vendored
@@ -1,332 +0,0 @@
|
||||
"""More comprehensive traceback formatting for Python scripts.
|
||||
|
||||
To enable this module, do:
|
||||
|
||||
import cgitb; cgitb.enable()
|
||||
|
||||
at the top of your script. The optional arguments to enable() are:
|
||||
|
||||
display - if true, tracebacks are displayed in the web browser
|
||||
logdir - if set, tracebacks are written to files in this directory
|
||||
context - number of lines of source code to show for each stack frame
|
||||
format - 'text' or 'html' controls the output format
|
||||
|
||||
By default, tracebacks are displayed but not saved, the context is 5 lines
|
||||
and the output format is 'html' (for backwards compatibility with the
|
||||
original use of this module)
|
||||
|
||||
Alternatively, if you have caught an exception and want cgitb to display it
|
||||
for you, call cgitb.handler(). The optional argument to handler() is a
|
||||
3-item tuple (etype, evalue, etb) just like the value of sys.exc_info().
|
||||
The default handler displays output as HTML.
|
||||
|
||||
"""
|
||||
import inspect
|
||||
import keyword
|
||||
import linecache
|
||||
import os
|
||||
import pydoc
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import tokenize
|
||||
import traceback
|
||||
import warnings
|
||||
from html import escape as html_escape
|
||||
|
||||
warnings._deprecated(__name__, remove=(3, 13))
|
||||
|
||||
|
||||
def reset():
|
||||
"""Return a string that resets the CGI and browser to a known state."""
|
||||
return '''<!--: spam
|
||||
Content-Type: text/html
|
||||
|
||||
<body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> -->
|
||||
<body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> --> -->
|
||||
</font> </font> </font> </script> </object> </blockquote> </pre>
|
||||
</table> </table> </table> </table> </table> </font> </font> </font>'''
|
||||
|
||||
__UNDEF__ = [] # a special sentinel object
|
||||
def small(text):
|
||||
if text:
|
||||
return '<small>' + text + '</small>'
|
||||
else:
|
||||
return ''
|
||||
|
||||
def strong(text):
|
||||
if text:
|
||||
return '<strong>' + text + '</strong>'
|
||||
else:
|
||||
return ''
|
||||
|
||||
def grey(text):
|
||||
if text:
|
||||
return '<font color="#909090">' + text + '</font>'
|
||||
else:
|
||||
return ''
|
||||
|
||||
def lookup(name, frame, locals):
|
||||
"""Find the value for a given name in the given environment."""
|
||||
if name in locals:
|
||||
return 'local', locals[name]
|
||||
if name in frame.f_globals:
|
||||
return 'global', frame.f_globals[name]
|
||||
if '__builtins__' in frame.f_globals:
|
||||
builtins = frame.f_globals['__builtins__']
|
||||
if isinstance(builtins, dict):
|
||||
if name in builtins:
|
||||
return 'builtin', builtins[name]
|
||||
else:
|
||||
if hasattr(builtins, name):
|
||||
return 'builtin', getattr(builtins, name)
|
||||
return None, __UNDEF__
|
||||
|
||||
def scanvars(reader, frame, locals):
|
||||
"""Scan one logical line of Python and look up values of variables used."""
|
||||
vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__
|
||||
for ttype, token, start, end, line in tokenize.generate_tokens(reader):
|
||||
if ttype == tokenize.NEWLINE: break
|
||||
if ttype == tokenize.NAME and token not in keyword.kwlist:
|
||||
if lasttoken == '.':
|
||||
if parent is not __UNDEF__:
|
||||
value = getattr(parent, token, __UNDEF__)
|
||||
vars.append((prefix + token, prefix, value))
|
||||
else:
|
||||
where, value = lookup(token, frame, locals)
|
||||
vars.append((token, where, value))
|
||||
elif token == '.':
|
||||
prefix += lasttoken + '.'
|
||||
parent = value
|
||||
else:
|
||||
parent, prefix = None, ''
|
||||
lasttoken = token
|
||||
return vars
|
||||
|
||||
def html(einfo, context=5):
|
||||
"""Return a nice HTML document describing a given traceback."""
|
||||
etype, evalue, etb = einfo
|
||||
if isinstance(etype, type):
|
||||
etype = etype.__name__
|
||||
pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
|
||||
date = time.ctime(time.time())
|
||||
head = f'''
|
||||
<body bgcolor="#f0f0f8">
|
||||
<table width="100%" cellspacing=0 cellpadding=2 border=0 summary="heading">
|
||||
<tr bgcolor="#6622aa">
|
||||
<td valign=bottom> <br>
|
||||
<font color="#ffffff" face="helvetica, arial"> <br>
|
||||
<big><big><strong>{html_escape(str(etype))}</strong></big></big></font></td>
|
||||
<td align=right valign=bottom>
|
||||
<font color="#ffffff" face="helvetica, arial">{pyver}<br>{date}</font></td>
|
||||
</tr></table>
|
||||
<p>A problem occurred in a Python script. Here is the sequence of
|
||||
function calls leading up to the error, in the order they occurred.</p>'''
|
||||
|
||||
indent = '<tt>' + small(' ' * 5) + ' </tt>'
|
||||
frames = []
|
||||
records = inspect.getinnerframes(etb, context)
|
||||
for frame, file, lnum, func, lines, index in records:
|
||||
if file:
|
||||
file = os.path.abspath(file)
|
||||
link = '<a href="file://%s">%s</a>' % (file, pydoc.html.escape(file))
|
||||
else:
|
||||
file = link = '?'
|
||||
args, varargs, varkw, locals = inspect.getargvalues(frame)
|
||||
call = ''
|
||||
if func != '?':
|
||||
call = 'in ' + strong(pydoc.html.escape(func))
|
||||
if func != "<module>":
|
||||
call += inspect.formatargvalues(args, varargs, varkw, locals,
|
||||
formatvalue=lambda value: '=' + pydoc.html.repr(value))
|
||||
|
||||
highlight = {}
|
||||
def reader(lnum=[lnum]):
|
||||
highlight[lnum[0]] = 1
|
||||
try: return linecache.getline(file, lnum[0])
|
||||
finally: lnum[0] += 1
|
||||
vars = scanvars(reader, frame, locals)
|
||||
|
||||
rows = ['<tr><td bgcolor="#d8bbff">%s%s %s</td></tr>' %
|
||||
('<big> </big>', link, call)]
|
||||
if index is not None:
|
||||
i = lnum - index
|
||||
for line in lines:
|
||||
num = small(' ' * (5-len(str(i))) + str(i)) + ' '
|
||||
if i in highlight:
|
||||
line = '<tt>=>%s%s</tt>' % (num, pydoc.html.preformat(line))
|
||||
rows.append('<tr><td bgcolor="#ffccee">%s</td></tr>' % line)
|
||||
else:
|
||||
line = '<tt> %s%s</tt>' % (num, pydoc.html.preformat(line))
|
||||
rows.append('<tr><td>%s</td></tr>' % grey(line))
|
||||
i += 1
|
||||
|
||||
done, dump = {}, []
|
||||
for name, where, value in vars:
|
||||
if name in done: continue
|
||||
done[name] = 1
|
||||
if value is not __UNDEF__:
|
||||
if where in ('global', 'builtin'):
|
||||
name = ('<em>%s</em> ' % where) + strong(name)
|
||||
elif where == 'local':
|
||||
name = strong(name)
|
||||
else:
|
||||
name = where + strong(name.split('.')[-1])
|
||||
dump.append('%s = %s' % (name, pydoc.html.repr(value)))
|
||||
else:
|
||||
dump.append(name + ' <em>undefined</em>')
|
||||
|
||||
rows.append('<tr><td>%s</td></tr>' % small(grey(', '.join(dump))))
|
||||
frames.append('''
|
||||
<table width="100%%" cellspacing=0 cellpadding=0 border=0>
|
||||
%s</table>''' % '\n'.join(rows))
|
||||
|
||||
exception = ['<p>%s: %s' % (strong(pydoc.html.escape(str(etype))),
|
||||
pydoc.html.escape(str(evalue)))]
|
||||
for name in dir(evalue):
|
||||
if name[:1] == '_': continue
|
||||
value = pydoc.html.repr(getattr(evalue, name))
|
||||
exception.append('\n<br>%s%s =\n%s' % (indent, name, value))
|
||||
|
||||
return head + ''.join(frames) + ''.join(exception) + '''
|
||||
|
||||
|
||||
<!-- The above is a description of an error in a Python program, formatted
|
||||
for a web browser because the 'cgitb' module was enabled. In case you
|
||||
are not reading this in a web browser, here is the original traceback:
|
||||
|
||||
%s
|
||||
-->
|
||||
''' % pydoc.html.escape(
|
||||
''.join(traceback.format_exception(etype, evalue, etb)))
|
||||
|
||||
def text(einfo, context=5):
|
||||
"""Return a plain text document describing a given traceback."""
|
||||
etype, evalue, etb = einfo
|
||||
if isinstance(etype, type):
|
||||
etype = etype.__name__
|
||||
pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
|
||||
date = time.ctime(time.time())
|
||||
head = "%s\n%s\n%s\n" % (str(etype), pyver, date) + '''
|
||||
A problem occurred in a Python script. Here is the sequence of
|
||||
function calls leading up to the error, in the order they occurred.
|
||||
'''
|
||||
|
||||
frames = []
|
||||
records = inspect.getinnerframes(etb, context)
|
||||
for frame, file, lnum, func, lines, index in records:
|
||||
file = file and os.path.abspath(file) or '?'
|
||||
args, varargs, varkw, locals = inspect.getargvalues(frame)
|
||||
call = ''
|
||||
if func != '?':
|
||||
call = 'in ' + func
|
||||
if func != "<module>":
|
||||
call += inspect.formatargvalues(args, varargs, varkw, locals,
|
||||
formatvalue=lambda value: '=' + pydoc.text.repr(value))
|
||||
|
||||
highlight = {}
|
||||
def reader(lnum=[lnum]):
|
||||
highlight[lnum[0]] = 1
|
||||
try: return linecache.getline(file, lnum[0])
|
||||
finally: lnum[0] += 1
|
||||
vars = scanvars(reader, frame, locals)
|
||||
|
||||
rows = [' %s %s' % (file, call)]
|
||||
if index is not None:
|
||||
i = lnum - index
|
||||
for line in lines:
|
||||
num = '%5d ' % i
|
||||
rows.append(num+line.rstrip())
|
||||
i += 1
|
||||
|
||||
done, dump = {}, []
|
||||
for name, where, value in vars:
|
||||
if name in done: continue
|
||||
done[name] = 1
|
||||
if value is not __UNDEF__:
|
||||
if where == 'global': name = 'global ' + name
|
||||
elif where != 'local': name = where + name.split('.')[-1]
|
||||
dump.append('%s = %s' % (name, pydoc.text.repr(value)))
|
||||
else:
|
||||
dump.append(name + ' undefined')
|
||||
|
||||
rows.append('\n'.join(dump))
|
||||
frames.append('\n%s\n' % '\n'.join(rows))
|
||||
|
||||
exception = ['%s: %s' % (str(etype), str(evalue))]
|
||||
for name in dir(evalue):
|
||||
value = pydoc.text.repr(getattr(evalue, name))
|
||||
exception.append('\n%s%s = %s' % (" "*4, name, value))
|
||||
|
||||
return head + ''.join(frames) + ''.join(exception) + '''
|
||||
|
||||
The above is a description of an error in a Python program. Here is
|
||||
the original traceback:
|
||||
|
||||
%s
|
||||
''' % ''.join(traceback.format_exception(etype, evalue, etb))
|
||||
|
||||
class Hook:
|
||||
"""A hook to replace sys.excepthook that shows tracebacks in HTML."""
|
||||
|
||||
def __init__(self, display=1, logdir=None, context=5, file=None,
|
||||
format="html"):
|
||||
self.display = display # send tracebacks to browser if true
|
||||
self.logdir = logdir # log tracebacks to files if not None
|
||||
self.context = context # number of source code lines per frame
|
||||
self.file = file or sys.stdout # place to send the output
|
||||
self.format = format
|
||||
|
||||
def __call__(self, etype, evalue, etb):
|
||||
self.handle((etype, evalue, etb))
|
||||
|
||||
def handle(self, info=None):
|
||||
info = info or sys.exc_info()
|
||||
if self.format == "html":
|
||||
self.file.write(reset())
|
||||
|
||||
formatter = (self.format=="html") and html or text
|
||||
plain = False
|
||||
try:
|
||||
doc = formatter(info, self.context)
|
||||
except: # just in case something goes wrong
|
||||
doc = ''.join(traceback.format_exception(*info))
|
||||
plain = True
|
||||
|
||||
if self.display:
|
||||
if plain:
|
||||
doc = pydoc.html.escape(doc)
|
||||
self.file.write('<pre>' + doc + '</pre>\n')
|
||||
else:
|
||||
self.file.write(doc + '\n')
|
||||
else:
|
||||
self.file.write('<p>A problem occurred in a Python script.\n')
|
||||
|
||||
if self.logdir is not None:
|
||||
suffix = ['.txt', '.html'][self.format=="html"]
|
||||
(fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir)
|
||||
|
||||
try:
|
||||
with os.fdopen(fd, 'w') as file:
|
||||
file.write(doc)
|
||||
msg = '%s contains the description of this error.' % path
|
||||
except:
|
||||
msg = 'Tried to save traceback to %s, but failed.' % path
|
||||
|
||||
if self.format == 'html':
|
||||
self.file.write('<p>%s</p>\n' % msg)
|
||||
else:
|
||||
self.file.write(msg + '\n')
|
||||
try:
|
||||
self.file.flush()
|
||||
except: pass
|
||||
|
||||
handler = Hook().handle
|
||||
def enable(display=1, logdir=None, context=5, format="html"):
|
||||
"""Install an exception handler that formats tracebacks as HTML.
|
||||
|
||||
The optional argument 'display' can be set to 0 to suppress sending the
|
||||
traceback to the browser, and 'logdir' can be set to a directory to cause
|
||||
tracebacks to be written to files there."""
|
||||
sys.excepthook = Hook(display=display, logdir=logdir,
|
||||
context=context, format=format)
|
||||
6
Lib/chunk.py
vendored
6
Lib/chunk.py
vendored
@@ -48,10 +48,6 @@ specifies whether or not chunks are aligned on 2-byte boundaries. The
|
||||
default is 1, i.e. aligned.
|
||||
"""
|
||||
|
||||
import warnings
|
||||
|
||||
warnings._deprecated(__name__, remove=(3, 13))
|
||||
|
||||
class Chunk:
|
||||
def __init__(self, file, align=True, bigendian=True, inclheader=False):
|
||||
import struct
|
||||
@@ -68,7 +64,7 @@ class Chunk:
|
||||
try:
|
||||
self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0]
|
||||
except struct.error:
|
||||
raise EOFError from None
|
||||
raise EOFError
|
||||
if inclheader:
|
||||
self.chunksize = self.chunksize - 8 # subtract header
|
||||
self.size_read = 0
|
||||
|
||||
10
Lib/cmd.py
vendored
10
Lib/cmd.py
vendored
@@ -310,10 +310,10 @@ class Cmd:
|
||||
names = self.get_names()
|
||||
cmds_doc = []
|
||||
cmds_undoc = []
|
||||
topics = set()
|
||||
help = {}
|
||||
for name in names:
|
||||
if name[:5] == 'help_':
|
||||
topics.add(name[5:])
|
||||
help[name[5:]]=1
|
||||
names.sort()
|
||||
# There can be duplicates if routines overridden
|
||||
prevname = ''
|
||||
@@ -323,16 +323,16 @@ class Cmd:
|
||||
continue
|
||||
prevname = name
|
||||
cmd=name[3:]
|
||||
if cmd in topics:
|
||||
if cmd in help:
|
||||
cmds_doc.append(cmd)
|
||||
topics.remove(cmd)
|
||||
del help[cmd]
|
||||
elif getattr(self, name).__doc__:
|
||||
cmds_doc.append(cmd)
|
||||
else:
|
||||
cmds_undoc.append(cmd)
|
||||
self.stdout.write("%s\n"%str(self.doc_leader))
|
||||
self.print_topics(self.doc_header, cmds_doc, 15,80)
|
||||
self.print_topics(self.misc_header, sorted(topics),15,80)
|
||||
self.print_topics(self.misc_header, list(help.keys()),15,80)
|
||||
self.print_topics(self.undoc_header, cmds_undoc, 15,80)
|
||||
|
||||
def print_topics(self, header, cmds, cmdlen, maxcol):
|
||||
|
||||
9
Lib/code.py
vendored
9
Lib/code.py
vendored
@@ -7,6 +7,7 @@
|
||||
|
||||
import sys
|
||||
import traceback
|
||||
import argparse
|
||||
from codeop import CommandCompiler, compile_command
|
||||
|
||||
__all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact",
|
||||
@@ -40,7 +41,7 @@ class InteractiveInterpreter:
|
||||
|
||||
Arguments are as for compile_command().
|
||||
|
||||
One of several things can happen:
|
||||
One several things can happen:
|
||||
|
||||
1) The input is incorrect; compile_command() raised an
|
||||
exception (SyntaxError or OverflowError). A syntax traceback
|
||||
@@ -106,7 +107,6 @@ class InteractiveInterpreter:
|
||||
|
||||
"""
|
||||
type, value, tb = sys.exc_info()
|
||||
sys.last_exc = value
|
||||
sys.last_type = type
|
||||
sys.last_value = value
|
||||
sys.last_traceback = tb
|
||||
@@ -120,7 +120,7 @@ class InteractiveInterpreter:
|
||||
else:
|
||||
# Stuff in the right filename
|
||||
value = SyntaxError(msg, (filename, lineno, offset, line))
|
||||
sys.last_exc = sys.last_value = value
|
||||
sys.last_value = value
|
||||
if sys.excepthook is sys.__excepthook__:
|
||||
lines = traceback.format_exception_only(type, value)
|
||||
self.write(''.join(lines))
|
||||
@@ -139,7 +139,6 @@ class InteractiveInterpreter:
|
||||
"""
|
||||
sys.last_type, sys.last_value, last_tb = ei = sys.exc_info()
|
||||
sys.last_traceback = last_tb
|
||||
sys.last_exc = ei[1]
|
||||
try:
|
||||
lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next)
|
||||
if sys.excepthook is sys.__excepthook__:
|
||||
@@ -304,8 +303,6 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-q', action='store_true',
|
||||
help="don't print version and copyright messages")
|
||||
|
||||
59
Lib/codecs.py
vendored
59
Lib/codecs.py
vendored
@@ -83,7 +83,7 @@ BOM64_BE = BOM_UTF32_BE
|
||||
class CodecInfo(tuple):
|
||||
"""Codec details when looking up the codec registry"""
|
||||
|
||||
# Private API to allow Python 3.4 to denylist the known non-Unicode
|
||||
# Private API to allow Python 3.4 to blacklist the known non-Unicode
|
||||
# codecs in the standard library. A more general mechanism to
|
||||
# reliably distinguish test encodings from other codecs will hopefully
|
||||
# be defined for Python 3.5
|
||||
@@ -386,7 +386,7 @@ class StreamWriter(Codec):
|
||||
|
||||
def reset(self):
|
||||
|
||||
""" Resets the codec buffers used for keeping internal state.
|
||||
""" Flushes and resets the codec buffers used for keeping state.
|
||||
|
||||
Calling this method should ensure that the data on the
|
||||
output is put into a clean state, that allows appending
|
||||
@@ -414,9 +414,6 @@ class StreamWriter(Codec):
|
||||
def __exit__(self, type, value, tb):
|
||||
self.stream.close()
|
||||
|
||||
def __reduce_ex__(self, proto):
|
||||
raise TypeError("can't serialize %s" % self.__class__.__name__)
|
||||
|
||||
###
|
||||
|
||||
class StreamReader(Codec):
|
||||
@@ -623,7 +620,7 @@ class StreamReader(Codec):
|
||||
|
||||
def reset(self):
|
||||
|
||||
""" Resets the codec buffers used for keeping internal state.
|
||||
""" Resets the codec buffers used for keeping state.
|
||||
|
||||
Note that no stream repositioning should take place.
|
||||
This method is primarily intended to be able to recover
|
||||
@@ -666,9 +663,6 @@ class StreamReader(Codec):
|
||||
def __exit__(self, type, value, tb):
|
||||
self.stream.close()
|
||||
|
||||
def __reduce_ex__(self, proto):
|
||||
raise TypeError("can't serialize %s" % self.__class__.__name__)
|
||||
|
||||
###
|
||||
|
||||
class StreamReaderWriter:
|
||||
@@ -756,9 +750,6 @@ class StreamReaderWriter:
|
||||
def __exit__(self, type, value, tb):
|
||||
self.stream.close()
|
||||
|
||||
def __reduce_ex__(self, proto):
|
||||
raise TypeError("can't serialize %s" % self.__class__.__name__)
|
||||
|
||||
###
|
||||
|
||||
class StreamRecoder:
|
||||
@@ -847,7 +838,7 @@ class StreamRecoder:
|
||||
|
||||
def writelines(self, list):
|
||||
|
||||
data = b''.join(list)
|
||||
data = ''.join(list)
|
||||
data, bytesdecoded = self.decode(data, self.errors)
|
||||
return self.writer.write(data)
|
||||
|
||||
@@ -856,12 +847,6 @@ class StreamRecoder:
|
||||
self.reader.reset()
|
||||
self.writer.reset()
|
||||
|
||||
def seek(self, offset, whence=0):
|
||||
# Seeks must be propagated to both the readers and writers
|
||||
# as they might need to reset their internal buffers.
|
||||
self.reader.seek(offset, whence)
|
||||
self.writer.seek(offset, whence)
|
||||
|
||||
def __getattr__(self, name,
|
||||
getattr=getattr):
|
||||
|
||||
@@ -875,12 +860,9 @@ class StreamRecoder:
|
||||
def __exit__(self, type, value, tb):
|
||||
self.stream.close()
|
||||
|
||||
def __reduce_ex__(self, proto):
|
||||
raise TypeError("can't serialize %s" % self.__class__.__name__)
|
||||
|
||||
### Shortcuts
|
||||
|
||||
def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
|
||||
def open(filename, mode='r', encoding=None, errors='strict', buffering=1):
|
||||
|
||||
""" Open an encoded file using the given mode and return
|
||||
a wrapped version providing transparent encoding/decoding.
|
||||
@@ -890,8 +872,7 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
|
||||
codecs. Output is also codec dependent and will usually be
|
||||
Unicode as well.
|
||||
|
||||
If encoding is not None, then the
|
||||
underlying encoded files are always opened in binary mode.
|
||||
Underlying encoded files are always opened in binary mode.
|
||||
The default file mode is 'r', meaning to open the file in read mode.
|
||||
|
||||
encoding specifies the encoding which is to be used for the
|
||||
@@ -902,8 +883,7 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
|
||||
encoding error occurs.
|
||||
|
||||
buffering has the same meaning as for the builtin open() API.
|
||||
It defaults to -1 which means that the default buffer size will
|
||||
be used.
|
||||
It defaults to line buffered.
|
||||
|
||||
The returned wrapped file object provides an extra attribute
|
||||
.encoding which allows querying the used encoding. This
|
||||
@@ -918,16 +898,11 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
|
||||
file = builtins.open(filename, mode, buffering)
|
||||
if encoding is None:
|
||||
return file
|
||||
|
||||
try:
|
||||
info = lookup(encoding)
|
||||
srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
|
||||
# Add attributes to simplify introspection
|
||||
srw.encoding = encoding
|
||||
return srw
|
||||
except:
|
||||
file.close()
|
||||
raise
|
||||
info = lookup(encoding)
|
||||
srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
|
||||
# Add attributes to simplify introspection
|
||||
srw.encoding = encoding
|
||||
return srw
|
||||
|
||||
def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'):
|
||||
|
||||
@@ -1127,3 +1102,13 @@ except LookupError:
|
||||
_false = 0
|
||||
if _false:
|
||||
import encodings
|
||||
|
||||
### Tests
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# Make stdout translate Latin-1 output into UTF-8 output
|
||||
sys.stdout = EncodedFile(sys.stdout, 'latin-1', 'utf-8')
|
||||
|
||||
# Have stdin translate Latin-1 input into UTF-8 input
|
||||
sys.stdin = EncodedFile(sys.stdin, 'utf-8', 'latin-1')
|
||||
|
||||
97
Lib/codeop.py
vendored
97
Lib/codeop.py
vendored
@@ -10,6 +10,30 @@ and:
|
||||
syntax error (OverflowError and ValueError can be produced by
|
||||
malformed literals).
|
||||
|
||||
Approach:
|
||||
|
||||
First, check if the source consists entirely of blank lines and
|
||||
comments; if so, replace it with 'pass', because the built-in
|
||||
parser doesn't always do the right thing for these.
|
||||
|
||||
Compile three times: as is, with \n, and with \n\n appended. If it
|
||||
compiles as is, it's complete. If it compiles with one \n appended,
|
||||
we expect more. If it doesn't compile either way, we compare the
|
||||
error we get when compiling with \n or \n\n appended. If the errors
|
||||
are the same, the code is broken. But if the errors are different, we
|
||||
expect more. Not intuitive; not even guaranteed to hold in future
|
||||
releases; but this matches the compiler's behavior from Python 1.4
|
||||
through 2.2, at least.
|
||||
|
||||
Caveat:
|
||||
|
||||
It is possible (but not likely) that the parser stops parsing with a
|
||||
successful outcome before reaching the end of the source; in this
|
||||
case, trailing symbols may be ignored instead of causing an error.
|
||||
For example, a backslash followed by two newlines may be followed by
|
||||
arbitrary garbage. This will be fixed once the API for the parser is
|
||||
better.
|
||||
|
||||
The two interfaces are:
|
||||
|
||||
compile_command(source, filename, symbol):
|
||||
@@ -33,61 +57,49 @@ Compile():
|
||||
"""
|
||||
|
||||
import __future__
|
||||
import warnings
|
||||
|
||||
_features = [getattr(__future__, fname)
|
||||
for fname in __future__.all_feature_names]
|
||||
|
||||
__all__ = ["compile_command", "Compile", "CommandCompiler"]
|
||||
|
||||
# The following flags match the values from Include/cpython/compile.h
|
||||
# Caveat emptor: These flags are undocumented on purpose and depending
|
||||
# on their effect outside the standard library is **unsupported**.
|
||||
PyCF_DONT_IMPLY_DEDENT = 0x200
|
||||
PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
|
||||
PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h
|
||||
|
||||
def _maybe_compile(compiler, source, filename, symbol):
|
||||
# Check for source consisting of only blank lines and comments.
|
||||
# Check for source consisting of only blank lines and comments
|
||||
for line in source.split("\n"):
|
||||
line = line.strip()
|
||||
if line and line[0] != '#':
|
||||
break # Leave it alone.
|
||||
break # Leave it alone
|
||||
else:
|
||||
if symbol != "eval":
|
||||
source = "pass" # Replace it with a 'pass' statement
|
||||
|
||||
# Disable compiler warnings when checking for incomplete input.
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
|
||||
try:
|
||||
compiler(source, filename, symbol)
|
||||
except SyntaxError: # Let other compile() errors propagate.
|
||||
try:
|
||||
compiler(source + "\n", filename, symbol)
|
||||
return None
|
||||
except SyntaxError as e:
|
||||
if "incomplete input" in str(e):
|
||||
return None
|
||||
# fallthrough
|
||||
err = err1 = err2 = None
|
||||
code = code1 = code2 = None
|
||||
|
||||
return compiler(source, filename, symbol, incomplete_input=False)
|
||||
try:
|
||||
code = compiler(source, filename, symbol)
|
||||
except SyntaxError as err:
|
||||
pass
|
||||
|
||||
def _is_syntax_error(err1, err2):
|
||||
rep1 = repr(err1)
|
||||
rep2 = repr(err2)
|
||||
if "was never closed" in rep1 and "was never closed" in rep2:
|
||||
return False
|
||||
if rep1 == rep2:
|
||||
return True
|
||||
return False
|
||||
try:
|
||||
code1 = compiler(source + "\n", filename, symbol)
|
||||
except SyntaxError as e:
|
||||
err1 = e
|
||||
|
||||
def _compile(source, filename, symbol, incomplete_input=True):
|
||||
flags = 0
|
||||
if incomplete_input:
|
||||
flags |= PyCF_ALLOW_INCOMPLETE_INPUT
|
||||
flags |= PyCF_DONT_IMPLY_DEDENT
|
||||
return compile(source, filename, symbol, flags)
|
||||
try:
|
||||
code2 = compiler(source + "\n\n", filename, symbol)
|
||||
except SyntaxError as e:
|
||||
err2 = e
|
||||
|
||||
if code:
|
||||
return code
|
||||
if not code1 and repr(err1) == repr(err2):
|
||||
raise err1
|
||||
|
||||
def _compile(source, filename, symbol):
|
||||
return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT)
|
||||
|
||||
def compile_command(source, filename="<input>", symbol="single"):
|
||||
r"""Compile a command and determine whether it is incomplete.
|
||||
@@ -97,8 +109,7 @@ def compile_command(source, filename="<input>", symbol="single"):
|
||||
source -- the source string; may contain \n characters
|
||||
filename -- optional filename from which source was read; default
|
||||
"<input>"
|
||||
symbol -- optional grammar start symbol; "single" (default), "exec"
|
||||
or "eval"
|
||||
symbol -- optional grammar start symbol; "single" (default) or "eval"
|
||||
|
||||
Return value / exceptions raised:
|
||||
|
||||
@@ -116,14 +127,10 @@ class Compile:
|
||||
statement, it "remembers" and compiles all subsequent program texts
|
||||
with the statement in force."""
|
||||
def __init__(self):
|
||||
self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
|
||||
self.flags = PyCF_DONT_IMPLY_DEDENT
|
||||
|
||||
def __call__(self, source, filename, symbol, **kwargs):
|
||||
flags = self.flags
|
||||
if kwargs.get('incomplete_input', True) is False:
|
||||
flags &= ~PyCF_DONT_IMPLY_DEDENT
|
||||
flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT
|
||||
codeob = compile(source, filename, symbol, flags, True)
|
||||
def __call__(self, source, filename, symbol):
|
||||
codeob = compile(source, filename, symbol, self.flags, 1)
|
||||
for feature in _features:
|
||||
if codeob.co_flags & feature.compiler_flag:
|
||||
self.flags |= feature.compiler_flag
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,11 +1,7 @@
|
||||
from reprlib import recursive_repr as _recursive_repr
|
||||
|
||||
class defaultdict(dict):
|
||||
def __init__(self, *args, **kwargs):
|
||||
if len(args) >= 1:
|
||||
default_factory = args[0]
|
||||
if default_factory is not None and not callable(default_factory):
|
||||
raise TypeError("first argument must be callable or None")
|
||||
args = args[1:]
|
||||
else:
|
||||
default_factory = None
|
||||
@@ -13,46 +9,11 @@ class defaultdict(dict):
|
||||
self.default_factory = default_factory
|
||||
|
||||
def __missing__(self, key):
|
||||
if self.default_factory is not None:
|
||||
val = self.default_factory()
|
||||
if self.default_factory:
|
||||
return self.default_factory()
|
||||
else:
|
||||
raise KeyError(key)
|
||||
self[key] = val
|
||||
return val
|
||||
|
||||
@_recursive_repr()
|
||||
def __repr_factory(factory):
|
||||
return repr(factory)
|
||||
|
||||
def __repr__(self):
|
||||
return f"{type(self).__name__}({defaultdict.__repr_factory(self.default_factory)}, {dict.__repr__(self)})"
|
||||
return f"defaultdict({self.default_factory}, {dict.__repr__(self)})"
|
||||
|
||||
def copy(self):
|
||||
return type(self)(self.default_factory, self)
|
||||
|
||||
__copy__ = copy
|
||||
|
||||
def __reduce__(self):
|
||||
if self.default_factory is not None:
|
||||
args = self.default_factory,
|
||||
else:
|
||||
args = ()
|
||||
return type(self), args, None, None, iter(self.items())
|
||||
|
||||
def __or__(self, other):
|
||||
if not isinstance(other, dict):
|
||||
return NotImplemented
|
||||
|
||||
new = defaultdict(self.default_factory, self)
|
||||
new.update(other)
|
||||
return new
|
||||
|
||||
def __ror__(self, other):
|
||||
if not isinstance(other, dict):
|
||||
return NotImplemented
|
||||
|
||||
new = defaultdict(self.default_factory, other)
|
||||
new.update(self)
|
||||
return new
|
||||
|
||||
defaultdict.__module__ = 'collections'
|
||||
|
||||
@@ -1,3 +1,2 @@
|
||||
from _collections_abc import *
|
||||
from _collections_abc import __all__
|
||||
from _collections_abc import _CallableGenericAlias
|
||||
|
||||
28
Lib/colorsys.py
vendored
28
Lib/colorsys.py
vendored
@@ -1,14 +1,10 @@
|
||||
"""Conversion functions between RGB and other color systems.
|
||||
|
||||
This modules provides two functions for each color system ABC:
|
||||
|
||||
rgb_to_abc(r, g, b) --> a, b, c
|
||||
abc_to_rgb(a, b, c) --> r, g, b
|
||||
|
||||
All inputs and outputs are triples of floats in the range [0.0...1.0]
|
||||
(with the exception of I and Q, which covers a slightly larger range).
|
||||
Inputs outside the valid range may cause exceptions or invalid outputs.
|
||||
|
||||
Supported color systems:
|
||||
RGB: Red, Green, Blue components
|
||||
YIQ: Luminance, Chrominance (used by composite video signals)
|
||||
@@ -75,18 +71,17 @@ def yiq_to_rgb(y, i, q):
|
||||
def rgb_to_hls(r, g, b):
|
||||
maxc = max(r, g, b)
|
||||
minc = min(r, g, b)
|
||||
sumc = (maxc+minc)
|
||||
rangec = (maxc-minc)
|
||||
l = sumc/2.0
|
||||
# XXX Can optimize (maxc+minc) and (maxc-minc)
|
||||
l = (minc+maxc)/2.0
|
||||
if minc == maxc:
|
||||
return 0.0, l, 0.0
|
||||
if l <= 0.5:
|
||||
s = rangec / sumc
|
||||
s = (maxc-minc) / (maxc+minc)
|
||||
else:
|
||||
s = rangec / (2.0-maxc-minc) # Not always 2.0-sumc: gh-106498.
|
||||
rc = (maxc-r) / rangec
|
||||
gc = (maxc-g) / rangec
|
||||
bc = (maxc-b) / rangec
|
||||
s = (maxc-minc) / (2.0-maxc-minc)
|
||||
rc = (maxc-r) / (maxc-minc)
|
||||
gc = (maxc-g) / (maxc-minc)
|
||||
bc = (maxc-b) / (maxc-minc)
|
||||
if r == maxc:
|
||||
h = bc-gc
|
||||
elif g == maxc:
|
||||
@@ -125,14 +120,13 @@ def _v(m1, m2, hue):
|
||||
def rgb_to_hsv(r, g, b):
|
||||
maxc = max(r, g, b)
|
||||
minc = min(r, g, b)
|
||||
rangec = (maxc-minc)
|
||||
v = maxc
|
||||
if minc == maxc:
|
||||
return 0.0, 0.0, v
|
||||
s = rangec / maxc
|
||||
rc = (maxc-r) / rangec
|
||||
gc = (maxc-g) / rangec
|
||||
bc = (maxc-b) / rangec
|
||||
s = (maxc-minc) / maxc
|
||||
rc = (maxc-r) / (maxc-minc)
|
||||
gc = (maxc-g) / (maxc-minc)
|
||||
bc = (maxc-b) / (maxc-minc)
|
||||
if r == maxc:
|
||||
h = bc-gc
|
||||
elif g == maxc:
|
||||
|
||||
286
Lib/compileall.py
vendored
286
Lib/compileall.py
vendored
@@ -4,7 +4,7 @@ When called as a script with arguments, this compiles the directories
|
||||
given as arguments recursively; the -l option prevents it from
|
||||
recursing into directories.
|
||||
|
||||
Without arguments, it compiles all modules on sys.path, without
|
||||
Without arguments, if compiles all modules on sys.path, without
|
||||
recursing into subdirectories. (Even though it should do so for
|
||||
packages -- for now, you'll have to deal with packages separately.)
|
||||
|
||||
@@ -15,14 +15,16 @@ import sys
|
||||
import importlib.util
|
||||
import py_compile
|
||||
import struct
|
||||
import filecmp
|
||||
|
||||
try:
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
except ImportError:
|
||||
ProcessPoolExecutor = None
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
__all__ = ["compile_dir","compile_file","compile_path"]
|
||||
|
||||
def _walk_dir(dir, maxlevels, quiet=0):
|
||||
def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
|
||||
if quiet < 2 and isinstance(dir, os.PathLike):
|
||||
dir = os.fspath(dir)
|
||||
if not quiet:
|
||||
@@ -38,94 +40,59 @@ def _walk_dir(dir, maxlevels, quiet=0):
|
||||
if name == '__pycache__':
|
||||
continue
|
||||
fullname = os.path.join(dir, name)
|
||||
if ddir is not None:
|
||||
dfile = os.path.join(ddir, name)
|
||||
else:
|
||||
dfile = None
|
||||
if not os.path.isdir(fullname):
|
||||
yield fullname
|
||||
elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
|
||||
os.path.isdir(fullname) and not os.path.islink(fullname)):
|
||||
yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
|
||||
quiet=quiet)
|
||||
yield from _walk_dir(fullname, ddir=dfile,
|
||||
maxlevels=maxlevels - 1, quiet=quiet)
|
||||
|
||||
def compile_dir(dir, maxlevels=None, ddir=None, force=False,
|
||||
rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
|
||||
invalidation_mode=None, *, stripdir=None,
|
||||
prependdir=None, limit_sl_dest=None, hardlink_dupes=False):
|
||||
def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
|
||||
quiet=0, legacy=False, optimize=-1, workers=1):
|
||||
"""Byte-compile all modules in the given directory tree.
|
||||
|
||||
Arguments (only dir is required):
|
||||
|
||||
dir: the directory to byte-compile
|
||||
maxlevels: maximum recursion level (default `sys.getrecursionlimit()`)
|
||||
maxlevels: maximum recursion level (default 10)
|
||||
ddir: the directory that will be prepended to the path to the
|
||||
file as it is compiled into each byte-code file.
|
||||
force: if True, force compilation, even if timestamps are up-to-date
|
||||
quiet: full output with False or 0, errors only with 1,
|
||||
no output with 2
|
||||
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
|
||||
optimize: int or list of optimization levels or -1 for level of
|
||||
the interpreter. Multiple levels leads to multiple compiled
|
||||
files each with one optimization level.
|
||||
optimize: optimization level or -1 for level of the interpreter
|
||||
workers: maximum number of parallel workers
|
||||
invalidation_mode: how the up-to-dateness of the pyc will be checked
|
||||
stripdir: part of path to left-strip from source file path
|
||||
prependdir: path to prepend to beginning of original file path, applied
|
||||
after stripdir
|
||||
limit_sl_dest: ignore symlinks if they are pointing outside of
|
||||
the defined path
|
||||
hardlink_dupes: hardlink duplicated pyc files
|
||||
"""
|
||||
ProcessPoolExecutor = None
|
||||
if ddir is not None and (stripdir is not None or prependdir is not None):
|
||||
raise ValueError(("Destination dir (ddir) cannot be used "
|
||||
"in combination with stripdir or prependdir"))
|
||||
if ddir is not None:
|
||||
stripdir = dir
|
||||
prependdir = ddir
|
||||
ddir = None
|
||||
if workers < 0:
|
||||
if workers is not None and workers < 0:
|
||||
raise ValueError('workers must be greater or equal to 0')
|
||||
if workers != 1:
|
||||
# Check if this is a system where ProcessPoolExecutor can function.
|
||||
from concurrent.futures.process import _check_system_limits
|
||||
try:
|
||||
_check_system_limits()
|
||||
except NotImplementedError:
|
||||
workers = 1
|
||||
else:
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
if maxlevels is None:
|
||||
maxlevels = sys.getrecursionlimit()
|
||||
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
|
||||
|
||||
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
|
||||
ddir=ddir)
|
||||
success = True
|
||||
if workers != 1 and ProcessPoolExecutor is not None:
|
||||
# If workers == 0, let ProcessPoolExecutor choose
|
||||
if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
|
||||
workers = workers or None
|
||||
with ProcessPoolExecutor(max_workers=workers) as executor:
|
||||
results = executor.map(partial(compile_file,
|
||||
ddir=ddir, force=force,
|
||||
rx=rx, quiet=quiet,
|
||||
legacy=legacy,
|
||||
optimize=optimize,
|
||||
invalidation_mode=invalidation_mode,
|
||||
stripdir=stripdir,
|
||||
prependdir=prependdir,
|
||||
limit_sl_dest=limit_sl_dest,
|
||||
hardlink_dupes=hardlink_dupes),
|
||||
optimize=optimize),
|
||||
files)
|
||||
success = min(results, default=True)
|
||||
else:
|
||||
for file in files:
|
||||
if not compile_file(file, ddir, force, rx, quiet,
|
||||
legacy, optimize, invalidation_mode,
|
||||
stripdir=stripdir, prependdir=prependdir,
|
||||
limit_sl_dest=limit_sl_dest,
|
||||
hardlink_dupes=hardlink_dupes):
|
||||
legacy, optimize):
|
||||
success = False
|
||||
return success
|
||||
|
||||
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
||||
legacy=False, optimize=-1,
|
||||
invalidation_mode=None, *, stripdir=None, prependdir=None,
|
||||
limit_sl_dest=None, hardlink_dupes=False):
|
||||
legacy=False, optimize=-1):
|
||||
"""Byte-compile one file.
|
||||
|
||||
Arguments (only fullname is required):
|
||||
@@ -137,114 +104,49 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
||||
quiet: full output with False or 0, errors only with 1,
|
||||
no output with 2
|
||||
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
|
||||
optimize: int or list of optimization levels or -1 for level of
|
||||
the interpreter. Multiple levels leads to multiple compiled
|
||||
files each with one optimization level.
|
||||
invalidation_mode: how the up-to-dateness of the pyc will be checked
|
||||
stripdir: part of path to left-strip from source file path
|
||||
prependdir: path to prepend to beginning of original file path, applied
|
||||
after stripdir
|
||||
limit_sl_dest: ignore symlinks if they are pointing outside of
|
||||
the defined path.
|
||||
hardlink_dupes: hardlink duplicated pyc files
|
||||
optimize: optimization level or -1 for level of the interpreter
|
||||
"""
|
||||
|
||||
if ddir is not None and (stripdir is not None or prependdir is not None):
|
||||
raise ValueError(("Destination dir (ddir) cannot be used "
|
||||
"in combination with stripdir or prependdir"))
|
||||
|
||||
success = True
|
||||
fullname = os.fspath(fullname)
|
||||
stripdir = os.fspath(stripdir) if stripdir is not None else None
|
||||
if quiet < 2 and isinstance(fullname, os.PathLike):
|
||||
fullname = os.fspath(fullname)
|
||||
name = os.path.basename(fullname)
|
||||
|
||||
dfile = None
|
||||
|
||||
if ddir is not None:
|
||||
dfile = os.path.join(ddir, name)
|
||||
|
||||
if stripdir is not None:
|
||||
fullname_parts = fullname.split(os.path.sep)
|
||||
stripdir_parts = stripdir.split(os.path.sep)
|
||||
ddir_parts = list(fullname_parts)
|
||||
|
||||
for spart, opart in zip(stripdir_parts, fullname_parts):
|
||||
if spart == opart:
|
||||
ddir_parts.remove(spart)
|
||||
|
||||
dfile = os.path.join(*ddir_parts)
|
||||
|
||||
if prependdir is not None:
|
||||
if dfile is None:
|
||||
dfile = os.path.join(prependdir, fullname)
|
||||
else:
|
||||
dfile = os.path.join(prependdir, dfile)
|
||||
|
||||
if isinstance(optimize, int):
|
||||
optimize = [optimize]
|
||||
|
||||
# Use set() to remove duplicates.
|
||||
# Use sorted() to create pyc files in a deterministic order.
|
||||
optimize = sorted(set(optimize))
|
||||
|
||||
if hardlink_dupes and len(optimize) < 2:
|
||||
raise ValueError("Hardlinking of duplicated bytecode makes sense "
|
||||
"only for more than one optimization level")
|
||||
|
||||
else:
|
||||
dfile = None
|
||||
if rx is not None:
|
||||
mo = rx.search(fullname)
|
||||
if mo:
|
||||
return success
|
||||
|
||||
if limit_sl_dest is not None and os.path.islink(fullname):
|
||||
if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents:
|
||||
return success
|
||||
|
||||
opt_cfiles = {}
|
||||
|
||||
if os.path.isfile(fullname):
|
||||
for opt_level in optimize:
|
||||
if legacy:
|
||||
opt_cfiles[opt_level] = fullname + 'c'
|
||||
if legacy:
|
||||
cfile = fullname + 'c'
|
||||
else:
|
||||
if optimize >= 0:
|
||||
opt = optimize if optimize >= 1 else ''
|
||||
cfile = importlib.util.cache_from_source(
|
||||
fullname, optimization=opt)
|
||||
else:
|
||||
if opt_level >= 0:
|
||||
opt = opt_level if opt_level >= 1 else ''
|
||||
cfile = (importlib.util.cache_from_source(
|
||||
fullname, optimization=opt))
|
||||
opt_cfiles[opt_level] = cfile
|
||||
else:
|
||||
cfile = importlib.util.cache_from_source(fullname)
|
||||
opt_cfiles[opt_level] = cfile
|
||||
|
||||
cfile = importlib.util.cache_from_source(fullname)
|
||||
cache_dir = os.path.dirname(cfile)
|
||||
head, tail = name[:-3], name[-3:]
|
||||
if tail == '.py':
|
||||
if not force:
|
||||
try:
|
||||
mtime = int(os.stat(fullname).st_mtime)
|
||||
expect = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER,
|
||||
0, mtime & 0xFFFF_FFFF)
|
||||
for cfile in opt_cfiles.values():
|
||||
with open(cfile, 'rb') as chandle:
|
||||
actual = chandle.read(12)
|
||||
if expect != actual:
|
||||
break
|
||||
else:
|
||||
expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER,
|
||||
mtime)
|
||||
with open(cfile, 'rb') as chandle:
|
||||
actual = chandle.read(8)
|
||||
if expect == actual:
|
||||
return success
|
||||
except OSError:
|
||||
pass
|
||||
if not quiet:
|
||||
print('Compiling {!r}...'.format(fullname))
|
||||
try:
|
||||
for index, opt_level in enumerate(optimize):
|
||||
cfile = opt_cfiles[opt_level]
|
||||
ok = py_compile.compile(fullname, cfile, dfile, True,
|
||||
optimize=opt_level,
|
||||
invalidation_mode=invalidation_mode)
|
||||
if index > 0 and hardlink_dupes:
|
||||
previous_cfile = opt_cfiles[optimize[index - 1]]
|
||||
if filecmp.cmp(cfile, previous_cfile, shallow=False):
|
||||
os.unlink(cfile)
|
||||
os.link(previous_cfile, cfile)
|
||||
ok = py_compile.compile(fullname, cfile, dfile, True,
|
||||
optimize=optimize)
|
||||
except py_compile.PyCompileError as err:
|
||||
success = False
|
||||
if quiet >= 2:
|
||||
@@ -254,8 +156,9 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
||||
else:
|
||||
print('*** ', end='')
|
||||
# escape non-printable characters in msg
|
||||
encoding = sys.stdout.encoding or sys.getdefaultencoding()
|
||||
msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding)
|
||||
msg = err.msg.encode(sys.stdout.encoding,
|
||||
errors='backslashreplace')
|
||||
msg = msg.decode(sys.stdout.encoding)
|
||||
print(msg)
|
||||
except (SyntaxError, UnicodeError, OSError) as e:
|
||||
success = False
|
||||
@@ -272,8 +175,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
||||
return success
|
||||
|
||||
def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
|
||||
legacy=False, optimize=-1,
|
||||
invalidation_mode=None):
|
||||
legacy=False, optimize=-1):
|
||||
"""Byte-compile all module on sys.path.
|
||||
|
||||
Arguments (all optional):
|
||||
@@ -284,7 +186,6 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
|
||||
quiet: as for compile_dir() (default 0)
|
||||
legacy: as for compile_dir() (default False)
|
||||
optimize: as for compile_dir() (default -1)
|
||||
invalidation_mode: as for compiler_dir()
|
||||
"""
|
||||
success = True
|
||||
for dir in sys.path:
|
||||
@@ -292,16 +193,9 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
|
||||
if quiet < 2:
|
||||
print('Skipping current directory')
|
||||
else:
|
||||
success = success and compile_dir(
|
||||
dir,
|
||||
maxlevels,
|
||||
None,
|
||||
force,
|
||||
quiet=quiet,
|
||||
legacy=legacy,
|
||||
optimize=optimize,
|
||||
invalidation_mode=invalidation_mode,
|
||||
)
|
||||
success = success and compile_dir(dir, maxlevels, None,
|
||||
force, quiet=quiet,
|
||||
legacy=legacy, optimize=optimize)
|
||||
return success
|
||||
|
||||
|
||||
@@ -312,7 +206,7 @@ def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Utilities to support installing Python libraries.')
|
||||
parser.add_argument('-l', action='store_const', const=0,
|
||||
default=None, dest='maxlevels',
|
||||
default=10, dest='maxlevels',
|
||||
help="don't recurse into subdirectories")
|
||||
parser.add_argument('-r', type=int, dest='recursion',
|
||||
help=('control the maximum recursion level. '
|
||||
@@ -330,20 +224,6 @@ def main():
|
||||
'compile-time tracebacks and in runtime '
|
||||
'tracebacks in cases where the source file is '
|
||||
'unavailable'))
|
||||
parser.add_argument('-s', metavar='STRIPDIR', dest='stripdir',
|
||||
default=None,
|
||||
help=('part of path to left-strip from path '
|
||||
'to source file - for example buildroot. '
|
||||
'`-d` and `-s` options cannot be '
|
||||
'specified together.'))
|
||||
parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir',
|
||||
default=None,
|
||||
help=('path to add as prefix to path '
|
||||
'to source file - for example / to make '
|
||||
'it absolute when some part is removed '
|
||||
'by `-s` option. '
|
||||
'`-d` and `-p` options cannot be '
|
||||
'specified together.'))
|
||||
parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
|
||||
help=('skip files matching the regular expression; '
|
||||
'the regexp is searched for in the full path '
|
||||
@@ -358,23 +238,6 @@ def main():
|
||||
'to the equivalent of -l sys.path'))
|
||||
parser.add_argument('-j', '--workers', default=1,
|
||||
type=int, help='Run compileall concurrently')
|
||||
invalidation_modes = [mode.name.lower().replace('_', '-')
|
||||
for mode in py_compile.PycInvalidationMode]
|
||||
parser.add_argument('--invalidation-mode',
|
||||
choices=sorted(invalidation_modes),
|
||||
help=('set .pyc invalidation mode; defaults to '
|
||||
'"checked-hash" if the SOURCE_DATE_EPOCH '
|
||||
'environment variable is set, and '
|
||||
'"timestamp" otherwise.'))
|
||||
parser.add_argument('-o', action='append', type=int, dest='opt_levels',
|
||||
help=('Optimization levels to run compilation with. '
|
||||
'Default is -1 which uses the optimization level '
|
||||
'of the Python interpreter itself (see -O).'))
|
||||
parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',
|
||||
help='Ignore symlinks pointing outsite of the DIR')
|
||||
parser.add_argument('--hardlink-dupes', action='store_true',
|
||||
dest='hardlink_dupes',
|
||||
help='Hardlink duplicated pyc files')
|
||||
|
||||
args = parser.parse_args()
|
||||
compile_dests = args.compile_dest
|
||||
@@ -383,31 +246,16 @@ def main():
|
||||
import re
|
||||
args.rx = re.compile(args.rx)
|
||||
|
||||
if args.limit_sl_dest == "":
|
||||
args.limit_sl_dest = None
|
||||
|
||||
if args.recursion is not None:
|
||||
maxlevels = args.recursion
|
||||
else:
|
||||
maxlevels = args.maxlevels
|
||||
|
||||
if args.opt_levels is None:
|
||||
args.opt_levels = [-1]
|
||||
|
||||
if len(args.opt_levels) == 1 and args.hardlink_dupes:
|
||||
parser.error(("Hardlinking of duplicated bytecode makes sense "
|
||||
"only for more than one optimization level."))
|
||||
|
||||
if args.ddir is not None and (
|
||||
args.stripdir is not None or args.prependdir is not None
|
||||
):
|
||||
parser.error("-d cannot be used in combination with -s or -p")
|
||||
|
||||
# if flist is provided then load it
|
||||
if args.flist:
|
||||
try:
|
||||
with (sys.stdin if args.flist=='-' else
|
||||
open(args.flist, encoding="utf-8")) as f:
|
||||
with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
|
||||
for line in f:
|
||||
compile_dests.append(line.strip())
|
||||
except OSError:
|
||||
@@ -415,11 +263,8 @@ def main():
|
||||
print("Error reading file list {}".format(args.flist))
|
||||
return False
|
||||
|
||||
if args.invalidation_mode:
|
||||
ivl_mode = args.invalidation_mode.replace('-', '_').upper()
|
||||
invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]
|
||||
else:
|
||||
invalidation_mode = None
|
||||
if args.workers is not None:
|
||||
args.workers = args.workers or None
|
||||
|
||||
success = True
|
||||
try:
|
||||
@@ -427,30 +272,17 @@ def main():
|
||||
for dest in compile_dests:
|
||||
if os.path.isfile(dest):
|
||||
if not compile_file(dest, args.ddir, args.force, args.rx,
|
||||
args.quiet, args.legacy,
|
||||
invalidation_mode=invalidation_mode,
|
||||
stripdir=args.stripdir,
|
||||
prependdir=args.prependdir,
|
||||
optimize=args.opt_levels,
|
||||
limit_sl_dest=args.limit_sl_dest,
|
||||
hardlink_dupes=args.hardlink_dupes):
|
||||
args.quiet, args.legacy):
|
||||
success = False
|
||||
else:
|
||||
if not compile_dir(dest, maxlevels, args.ddir,
|
||||
args.force, args.rx, args.quiet,
|
||||
args.legacy, workers=args.workers,
|
||||
invalidation_mode=invalidation_mode,
|
||||
stripdir=args.stripdir,
|
||||
prependdir=args.prependdir,
|
||||
optimize=args.opt_levels,
|
||||
limit_sl_dest=args.limit_sl_dest,
|
||||
hardlink_dupes=args.hardlink_dupes):
|
||||
args.legacy, workers=args.workers):
|
||||
success = False
|
||||
return success
|
||||
else:
|
||||
return compile_path(legacy=args.legacy, force=args.force,
|
||||
quiet=args.quiet,
|
||||
invalidation_mode=invalidation_mode)
|
||||
quiet=args.quiet)
|
||||
except KeyboardInterrupt:
|
||||
if args.quiet < 2:
|
||||
print("\n[interrupted]")
|
||||
|
||||
@@ -10,44 +10,9 @@ from concurrent.futures._base import (FIRST_COMPLETED,
|
||||
ALL_COMPLETED,
|
||||
CancelledError,
|
||||
TimeoutError,
|
||||
InvalidStateError,
|
||||
BrokenExecutor,
|
||||
Future,
|
||||
Executor,
|
||||
wait,
|
||||
as_completed)
|
||||
|
||||
__all__ = (
|
||||
'FIRST_COMPLETED',
|
||||
'FIRST_EXCEPTION',
|
||||
'ALL_COMPLETED',
|
||||
'CancelledError',
|
||||
'TimeoutError',
|
||||
'BrokenExecutor',
|
||||
'Future',
|
||||
'Executor',
|
||||
'wait',
|
||||
'as_completed',
|
||||
'ProcessPoolExecutor',
|
||||
'ThreadPoolExecutor',
|
||||
)
|
||||
|
||||
|
||||
def __dir__():
|
||||
return __all__ + ('__author__', '__doc__')
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
global ProcessPoolExecutor, ThreadPoolExecutor
|
||||
|
||||
if name == 'ProcessPoolExecutor':
|
||||
from .process import ProcessPoolExecutor as pe
|
||||
ProcessPoolExecutor = pe
|
||||
return pe
|
||||
|
||||
if name == 'ThreadPoolExecutor':
|
||||
from .thread import ThreadPoolExecutor as te
|
||||
ThreadPoolExecutor = te
|
||||
return te
|
||||
|
||||
raise AttributeError(f"module {__name__} has no attribute {name}")
|
||||
from concurrent.futures.process import ProcessPoolExecutor
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
@@ -7,7 +7,6 @@ import collections
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
|
||||
FIRST_COMPLETED = 'FIRST_COMPLETED'
|
||||
FIRST_EXCEPTION = 'FIRST_EXCEPTION'
|
||||
@@ -54,10 +53,6 @@ class TimeoutError(Error):
|
||||
"""The operation exceeded the given deadline."""
|
||||
pass
|
||||
|
||||
class InvalidStateError(Error):
|
||||
"""The operation is not allowed in this state."""
|
||||
pass
|
||||
|
||||
class _Waiter(object):
|
||||
"""Provides the event that wait() and as_completed() block on."""
|
||||
def __init__(self):
|
||||
@@ -175,29 +170,6 @@ def _create_and_install_waiters(fs, return_when):
|
||||
|
||||
return waiter
|
||||
|
||||
|
||||
def _yield_finished_futures(fs, waiter, ref_collect):
|
||||
"""
|
||||
Iterate on the list *fs*, yielding finished futures one by one in
|
||||
reverse order.
|
||||
Before yielding a future, *waiter* is removed from its waiters
|
||||
and the future is removed from each set in the collection of sets
|
||||
*ref_collect*.
|
||||
|
||||
The aim of this function is to avoid keeping stale references after
|
||||
the future is yielded and before the iterator resumes.
|
||||
"""
|
||||
while fs:
|
||||
f = fs[-1]
|
||||
for futures_set in ref_collect:
|
||||
futures_set.remove(f)
|
||||
with f._condition:
|
||||
f._waiters.remove(waiter)
|
||||
del f
|
||||
# Careful not to keep a reference to the popped value
|
||||
yield fs.pop()
|
||||
|
||||
|
||||
def as_completed(fs, timeout=None):
|
||||
"""An iterator over the given futures that yields each as it completes.
|
||||
|
||||
@@ -217,30 +189,28 @@ def as_completed(fs, timeout=None):
|
||||
before the given timeout.
|
||||
"""
|
||||
if timeout is not None:
|
||||
end_time = timeout + time.monotonic()
|
||||
end_time = timeout + time.time()
|
||||
|
||||
fs = set(fs)
|
||||
total_futures = len(fs)
|
||||
with _AcquireFutures(fs):
|
||||
finished = set(
|
||||
f for f in fs
|
||||
if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
|
||||
pending = fs - finished
|
||||
waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
|
||||
finished = list(finished)
|
||||
|
||||
try:
|
||||
yield from _yield_finished_futures(finished, waiter,
|
||||
ref_collect=(fs,))
|
||||
yield from finished
|
||||
|
||||
while pending:
|
||||
if timeout is None:
|
||||
wait_timeout = None
|
||||
else:
|
||||
wait_timeout = end_time - time.monotonic()
|
||||
wait_timeout = end_time - time.time()
|
||||
if wait_timeout < 0:
|
||||
raise TimeoutError(
|
||||
'%d (of %d) futures unfinished' % (
|
||||
len(pending), total_futures))
|
||||
len(pending), len(fs)))
|
||||
|
||||
waiter.event.wait(wait_timeout)
|
||||
|
||||
@@ -249,13 +219,11 @@ def as_completed(fs, timeout=None):
|
||||
waiter.finished_futures = []
|
||||
waiter.event.clear()
|
||||
|
||||
# reverse to keep finishing order
|
||||
finished.reverse()
|
||||
yield from _yield_finished_futures(finished, waiter,
|
||||
ref_collect=(fs, pending))
|
||||
for future in finished:
|
||||
yield future
|
||||
pending.remove(future)
|
||||
|
||||
finally:
|
||||
# Remove waiter from unfinished futures
|
||||
for f in fs:
|
||||
with f._condition:
|
||||
f._waiters.remove(waiter)
|
||||
@@ -284,14 +252,13 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
|
||||
A named 2-tuple of sets. The first set, named 'done', contains the
|
||||
futures that completed (is finished or cancelled) before the wait
|
||||
completed. The second set, named 'not_done', contains uncompleted
|
||||
futures. Duplicate futures given to *fs* are removed and will be
|
||||
returned only once.
|
||||
futures.
|
||||
"""
|
||||
fs = set(fs)
|
||||
with _AcquireFutures(fs):
|
||||
done = {f for f in fs
|
||||
if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]}
|
||||
not_done = fs - done
|
||||
done = set(f for f in fs
|
||||
if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
|
||||
not_done = set(fs) - done
|
||||
|
||||
if (return_when == FIRST_COMPLETED) and done:
|
||||
return DoneAndNotDoneFutures(done, not_done)
|
||||
elif (return_when == FIRST_EXCEPTION) and done:
|
||||
@@ -310,7 +277,7 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
|
||||
f._waiters.remove(waiter)
|
||||
|
||||
done.update(waiter.finished_futures)
|
||||
return DoneAndNotDoneFutures(done, fs - done)
|
||||
return DoneAndNotDoneFutures(done, set(fs) - done)
|
||||
|
||||
class Future(object):
|
||||
"""Represents the result of an asynchronous computation."""
|
||||
@@ -381,17 +348,13 @@ class Future(object):
|
||||
return self._state == RUNNING
|
||||
|
||||
def done(self):
|
||||
"""Return True if the future was cancelled or finished executing."""
|
||||
"""Return True of the future was cancelled or finished executing."""
|
||||
with self._condition:
|
||||
return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]
|
||||
|
||||
def __get_result(self):
|
||||
if self._exception:
|
||||
try:
|
||||
raise self._exception
|
||||
finally:
|
||||
# Break a reference cycle with the exception in self._exception
|
||||
self = None
|
||||
raise self._exception
|
||||
else:
|
||||
return self._result
|
||||
|
||||
@@ -410,10 +373,7 @@ class Future(object):
|
||||
if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]:
|
||||
self._done_callbacks.append(fn)
|
||||
return
|
||||
try:
|
||||
fn(self)
|
||||
except Exception:
|
||||
LOGGER.exception('exception calling callback for %r', self)
|
||||
fn(self)
|
||||
|
||||
def result(self, timeout=None):
|
||||
"""Return the result of the call that the future represents.
|
||||
@@ -431,24 +391,20 @@ class Future(object):
|
||||
timeout.
|
||||
Exception: If the call raised then that exception will be raised.
|
||||
"""
|
||||
try:
|
||||
with self._condition:
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self.__get_result()
|
||||
with self._condition:
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self.__get_result()
|
||||
|
||||
self._condition.wait(timeout)
|
||||
self._condition.wait(timeout)
|
||||
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self.__get_result()
|
||||
else:
|
||||
raise TimeoutError()
|
||||
finally:
|
||||
# Break a reference cycle with the exception in self._exception
|
||||
self = None
|
||||
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
|
||||
raise CancelledError()
|
||||
elif self._state == FINISHED:
|
||||
return self.__get_result()
|
||||
else:
|
||||
raise TimeoutError()
|
||||
|
||||
def exception(self, timeout=None):
|
||||
"""Return the exception raised by the call that the future represents.
|
||||
@@ -530,8 +486,6 @@ class Future(object):
|
||||
Should only be used by Executor implementations and unit tests.
|
||||
"""
|
||||
with self._condition:
|
||||
if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
|
||||
raise InvalidStateError('{}: {!r}'.format(self._state, self))
|
||||
self._result = result
|
||||
self._state = FINISHED
|
||||
for waiter in self._waiters:
|
||||
@@ -545,8 +499,6 @@ class Future(object):
|
||||
Should only be used by Executor implementations and unit tests.
|
||||
"""
|
||||
with self._condition:
|
||||
if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
|
||||
raise InvalidStateError('{}: {!r}'.format(self._state, self))
|
||||
self._exception = exception
|
||||
self._state = FINISHED
|
||||
for waiter in self._waiters:
|
||||
@@ -554,12 +506,10 @@ class Future(object):
|
||||
self._condition.notify_all()
|
||||
self._invoke_callbacks()
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
class Executor(object):
|
||||
"""This is an abstract base class for concrete asynchronous executors."""
|
||||
|
||||
def submit(self, fn, /, *args, **kwargs):
|
||||
def submit(self, fn, *args, **kwargs):
|
||||
"""Submits a callable to be executed with the given arguments.
|
||||
|
||||
Schedules the callable to be executed as fn(*args, **kwargs) and returns
|
||||
@@ -593,7 +543,7 @@ class Executor(object):
|
||||
Exception: If fn(*args) raises for any values.
|
||||
"""
|
||||
if timeout is not None:
|
||||
end_time = timeout + time.monotonic()
|
||||
end_time = timeout + time.time()
|
||||
|
||||
fs = [self.submit(fn, *args) for args in zip(*iterables)]
|
||||
|
||||
@@ -601,20 +551,17 @@ class Executor(object):
|
||||
# before the first iterator value is required.
|
||||
def result_iterator():
|
||||
try:
|
||||
# reverse to keep finishing order
|
||||
fs.reverse()
|
||||
while fs:
|
||||
# Careful not to keep a reference to the popped future
|
||||
for future in fs:
|
||||
if timeout is None:
|
||||
yield fs.pop().result()
|
||||
yield future.result()
|
||||
else:
|
||||
yield fs.pop().result(end_time - time.monotonic())
|
||||
yield future.result(end_time - time.time())
|
||||
finally:
|
||||
for future in fs:
|
||||
future.cancel()
|
||||
return result_iterator()
|
||||
|
||||
def shutdown(self, wait=True, *, cancel_futures=False):
|
||||
def shutdown(self, wait=True):
|
||||
"""Clean-up the resources associated with the Executor.
|
||||
|
||||
It is safe to call this method several times. Otherwise, no other
|
||||
@@ -624,9 +571,6 @@ class Executor(object):
|
||||
wait: If True then shutdown will not return until all running
|
||||
futures have finished executing and the resources used by the
|
||||
executor have been reclaimed.
|
||||
cancel_futures: If True then shutdown will cancel all pending
|
||||
futures. Futures that are completed or running will not be
|
||||
cancelled.
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -636,9 +580,3 @@ class Executor(object):
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.shutdown(wait=True)
|
||||
return False
|
||||
|
||||
|
||||
class BrokenExecutor(RuntimeError):
|
||||
"""
|
||||
Raised when a executor has become non-functional after a severe failure.
|
||||
"""
|
||||
|
||||
@@ -3,15 +3,15 @@
|
||||
|
||||
"""Implements ProcessPoolExecutor.
|
||||
|
||||
The following diagram and text describe the data-flow through the system:
|
||||
The follow diagram and text describe the data-flow through the system:
|
||||
|
||||
|======================= In-process =====================|== Out-of-process ==|
|
||||
|
||||
+----------+ +----------+ +--------+ +-----------+ +---------+
|
||||
| | => | Work Ids | | | | Call Q | | Process |
|
||||
| | +----------+ | | +-----------+ | Pool |
|
||||
| | | ... | | | | ... | +---------+
|
||||
| | | 6 | => | | => | 5, call() | => | |
|
||||
| | => | Work Ids | => | | => | Call Q | => | |
|
||||
| | +----------+ | | +-----------+ | |
|
||||
| | | ... | | | | ... | | |
|
||||
| | | 6 | | | | 5, call() | | |
|
||||
| | | 7 | | | | ... | | |
|
||||
| Process | | ... | | Local | +-----------+ | Process |
|
||||
| Pool | +----------+ | Worker | | #1..n |
|
||||
@@ -45,74 +45,52 @@ Process #1..n:
|
||||
|
||||
__author__ = 'Brian Quinlan (brian@sweetapp.com)'
|
||||
|
||||
import atexit
|
||||
import os
|
||||
from concurrent.futures import _base
|
||||
import queue
|
||||
import multiprocessing as mp
|
||||
import multiprocessing.connection
|
||||
from multiprocessing.queues import Queue
|
||||
from queue import Full
|
||||
import multiprocessing
|
||||
from multiprocessing import SimpleQueue
|
||||
from multiprocessing.connection import wait
|
||||
import threading
|
||||
import weakref
|
||||
from functools import partial
|
||||
import itertools
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
# Workers are created as daemon threads and processes. This is done to allow the
|
||||
# interpreter to exit when there are still idle processes in a
|
||||
# ProcessPoolExecutor's process pool (i.e. shutdown() was not called). However,
|
||||
# allowing workers to die with the interpreter has two undesirable properties:
|
||||
# - The workers would still be running during interpreter shutdown,
|
||||
# meaning that they would fail in unpredictable ways.
|
||||
# - The workers could be killed while evaluating a work item, which could
|
||||
# be bad if the callable being evaluated has external side-effects e.g.
|
||||
# writing to a file.
|
||||
#
|
||||
# To work around this problem, an exit handler is installed which tells the
|
||||
# workers to exit when their work queues are empty and then waits until the
|
||||
# threads/processes finish.
|
||||
|
||||
_threads_wakeups = weakref.WeakKeyDictionary()
|
||||
_global_shutdown = False
|
||||
|
||||
|
||||
class _ThreadWakeup:
|
||||
def __init__(self):
|
||||
self._closed = False
|
||||
self._reader, self._writer = mp.Pipe(duplex=False)
|
||||
|
||||
def close(self):
|
||||
if not self._closed:
|
||||
self._closed = True
|
||||
self._writer.close()
|
||||
self._reader.close()
|
||||
|
||||
def wakeup(self):
|
||||
if not self._closed:
|
||||
self._writer.send_bytes(b"")
|
||||
|
||||
def clear(self):
|
||||
if not self._closed:
|
||||
while self._reader.poll():
|
||||
self._reader.recv_bytes()
|
||||
|
||||
_threads_queues = weakref.WeakKeyDictionary()
|
||||
_shutdown = False
|
||||
|
||||
def _python_exit():
|
||||
global _global_shutdown
|
||||
_global_shutdown = True
|
||||
items = list(_threads_wakeups.items())
|
||||
for _, thread_wakeup in items:
|
||||
# call not protected by ProcessPoolExecutor._shutdown_lock
|
||||
thread_wakeup.wakeup()
|
||||
for t, _ in items:
|
||||
global _shutdown
|
||||
_shutdown = True
|
||||
items = list(_threads_queues.items())
|
||||
for t, q in items:
|
||||
q.put(None)
|
||||
for t, q in items:
|
||||
t.join()
|
||||
|
||||
# Register for `_python_exit()` to be called just before joining all
|
||||
# non-daemon threads. This is used instead of `atexit.register()` for
|
||||
# compatibility with subinterpreters, which no longer support daemon threads.
|
||||
# See bpo-39812 for context.
|
||||
threading._register_atexit(_python_exit)
|
||||
|
||||
# Controls how many more calls than processes will be queued in the call queue.
|
||||
# A smaller number will mean that processes spend more time idle waiting for
|
||||
# work while a larger number will make Future.cancel() succeed less frequently
|
||||
# (Futures in the call queue cannot be cancelled).
|
||||
EXTRA_QUEUED_CALLS = 1
|
||||
|
||||
|
||||
# On Windows, WaitForMultipleObjects is used to wait for processes to finish.
|
||||
# It can wait on, at most, 63 objects. There is an overhead of two objects:
|
||||
# - the result queue reader
|
||||
# - the thread wakeup reader
|
||||
_MAX_WINDOWS_WORKERS = 63 - 2
|
||||
|
||||
# Hack to embed stringification of remote traceback in local traceback
|
||||
|
||||
class _RemoteTraceback(Exception):
|
||||
@@ -126,9 +104,6 @@ class _ExceptionWithTraceback:
|
||||
tb = traceback.format_exception(type(exc), exc, tb)
|
||||
tb = ''.join(tb)
|
||||
self.exc = exc
|
||||
# Traceback object needs to be garbage-collected as its frames
|
||||
# contain references to all the objects in the exception scope
|
||||
self.exc.__traceback__ = None
|
||||
self.tb = '\n"""\n%s"""' % tb
|
||||
def __reduce__(self):
|
||||
return _rebuild_exc, (self.exc, self.tb)
|
||||
@@ -157,32 +132,6 @@ class _CallItem(object):
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
|
||||
|
||||
class _SafeQueue(Queue):
|
||||
"""Safe Queue set exception to the future object linked to a job"""
|
||||
def __init__(self, max_size=0, *, ctx, pending_work_items, shutdown_lock,
|
||||
thread_wakeup):
|
||||
self.pending_work_items = pending_work_items
|
||||
self.shutdown_lock = shutdown_lock
|
||||
self.thread_wakeup = thread_wakeup
|
||||
super().__init__(max_size, ctx=ctx)
|
||||
|
||||
def _on_queue_feeder_error(self, e, obj):
|
||||
if isinstance(obj, _CallItem):
|
||||
tb = traceback.format_exception(type(e), e, e.__traceback__)
|
||||
e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb)))
|
||||
work_item = self.pending_work_items.pop(obj.work_id, None)
|
||||
with self.shutdown_lock:
|
||||
self.thread_wakeup.wakeup()
|
||||
# work_item can be None if another process terminated. In this
|
||||
# case, the executor_manager_thread fails all work_items
|
||||
# with BrokenProcessPool
|
||||
if work_item is not None:
|
||||
work_item.future.set_exception(e)
|
||||
else:
|
||||
super()._on_queue_feeder_error(e, obj)
|
||||
|
||||
|
||||
def _get_chunks(*iterables, chunksize):
|
||||
""" Iterates over zip()ed iterables in chunks. """
|
||||
it = zip(*iterables)
|
||||
@@ -192,7 +141,6 @@ def _get_chunks(*iterables, chunksize):
|
||||
return
|
||||
yield chunk
|
||||
|
||||
|
||||
def _process_chunk(fn, chunk):
|
||||
""" Processes a chunk of an iterable passed to map.
|
||||
|
||||
@@ -204,38 +152,19 @@ def _process_chunk(fn, chunk):
|
||||
"""
|
||||
return [fn(*args) for args in chunk]
|
||||
|
||||
|
||||
def _sendback_result(result_queue, work_id, result=None, exception=None):
|
||||
"""Safely send back the given result or exception"""
|
||||
try:
|
||||
result_queue.put(_ResultItem(work_id, result=result,
|
||||
exception=exception))
|
||||
except BaseException as e:
|
||||
exc = _ExceptionWithTraceback(e, e.__traceback__)
|
||||
result_queue.put(_ResultItem(work_id, exception=exc))
|
||||
|
||||
|
||||
def _process_worker(call_queue, result_queue, initializer, initargs):
|
||||
def _process_worker(call_queue, result_queue):
|
||||
"""Evaluates calls from call_queue and places the results in result_queue.
|
||||
|
||||
This worker is run in a separate process.
|
||||
|
||||
Args:
|
||||
call_queue: A ctx.Queue of _CallItems that will be read and
|
||||
call_queue: A multiprocessing.Queue of _CallItems that will be read and
|
||||
evaluated by the worker.
|
||||
result_queue: A ctx.Queue of _ResultItems that will written
|
||||
result_queue: A multiprocessing.Queue of _ResultItems that will written
|
||||
to by the worker.
|
||||
initializer: A callable initializer, or None
|
||||
initargs: A tuple of args for the initializer
|
||||
shutdown: A multiprocessing.Event that will be set as a signal to the
|
||||
worker that it should exit when call_queue is empty.
|
||||
"""
|
||||
if initializer is not None:
|
||||
try:
|
||||
initializer(*initargs)
|
||||
except BaseException:
|
||||
_base.LOGGER.critical('Exception in initializer:', exc_info=True)
|
||||
# The parent will notice that the process stopped and
|
||||
# mark the pool broken
|
||||
return
|
||||
while True:
|
||||
call_item = call_queue.get(block=True)
|
||||
if call_item is None:
|
||||
@@ -246,303 +175,171 @@ def _process_worker(call_queue, result_queue, initializer, initargs):
|
||||
r = call_item.fn(*call_item.args, **call_item.kwargs)
|
||||
except BaseException as e:
|
||||
exc = _ExceptionWithTraceback(e, e.__traceback__)
|
||||
_sendback_result(result_queue, call_item.work_id, exception=exc)
|
||||
result_queue.put(_ResultItem(call_item.work_id, exception=exc))
|
||||
else:
|
||||
_sendback_result(result_queue, call_item.work_id, result=r)
|
||||
del r
|
||||
result_queue.put(_ResultItem(call_item.work_id,
|
||||
result=r))
|
||||
|
||||
# Liberate the resource as soon as possible, to avoid holding onto
|
||||
# open files or shared memory that is not needed anymore
|
||||
del call_item
|
||||
def _add_call_item_to_queue(pending_work_items,
|
||||
work_ids,
|
||||
call_queue):
|
||||
"""Fills call_queue with _WorkItems from pending_work_items.
|
||||
|
||||
|
||||
class _ExecutorManagerThread(threading.Thread):
|
||||
"""Manages the communication between this process and the worker processes.
|
||||
|
||||
The manager is run in a local thread.
|
||||
This function never blocks.
|
||||
|
||||
Args:
|
||||
executor: A reference to the ProcessPoolExecutor that owns
|
||||
this thread. A weakref will be own by the manager as well as
|
||||
references to internal objects used to introspect the state of
|
||||
the executor.
|
||||
pending_work_items: A dict mapping work ids to _WorkItems e.g.
|
||||
{5: <_WorkItem...>, 6: <_WorkItem...>, ...}
|
||||
work_ids: A queue.Queue of work ids e.g. Queue([5, 6, ...]). Work ids
|
||||
are consumed and the corresponding _WorkItems from
|
||||
pending_work_items are transformed into _CallItems and put in
|
||||
call_queue.
|
||||
call_queue: A multiprocessing.Queue that will be filled with _CallItems
|
||||
derived from _WorkItems.
|
||||
"""
|
||||
while True:
|
||||
if call_queue.full():
|
||||
return
|
||||
try:
|
||||
work_id = work_ids.get(block=False)
|
||||
except queue.Empty:
|
||||
return
|
||||
else:
|
||||
work_item = pending_work_items[work_id]
|
||||
|
||||
def __init__(self, executor):
|
||||
# Store references to necessary internals of the executor.
|
||||
|
||||
# A _ThreadWakeup to allow waking up the queue_manager_thread from the
|
||||
# main Thread and avoid deadlocks caused by permanently locked queues.
|
||||
self.thread_wakeup = executor._executor_manager_thread_wakeup
|
||||
self.shutdown_lock = executor._shutdown_lock
|
||||
|
||||
# A weakref.ref to the ProcessPoolExecutor that owns this thread. Used
|
||||
# to determine if the ProcessPoolExecutor has been garbage collected
|
||||
# and that the manager can exit.
|
||||
# When the executor gets garbage collected, the weakref callback
|
||||
# will wake up the queue management thread so that it can terminate
|
||||
# if there is no pending work item.
|
||||
def weakref_cb(_,
|
||||
thread_wakeup=self.thread_wakeup,
|
||||
shutdown_lock=self.shutdown_lock):
|
||||
mp.util.debug('Executor collected: triggering callback for'
|
||||
' QueueManager wakeup')
|
||||
with shutdown_lock:
|
||||
thread_wakeup.wakeup()
|
||||
|
||||
self.executor_reference = weakref.ref(executor, weakref_cb)
|
||||
|
||||
# A list of the ctx.Process instances used as workers.
|
||||
self.processes = executor._processes
|
||||
|
||||
# A ctx.Queue that will be filled with _CallItems derived from
|
||||
# _WorkItems for processing by the process workers.
|
||||
self.call_queue = executor._call_queue
|
||||
|
||||
# A ctx.SimpleQueue of _ResultItems generated by the process workers.
|
||||
self.result_queue = executor._result_queue
|
||||
|
||||
# A queue.Queue of work ids e.g. Queue([5, 6, ...]).
|
||||
self.work_ids_queue = executor._work_ids
|
||||
|
||||
# A dict mapping work ids to _WorkItems e.g.
|
||||
# {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
|
||||
self.pending_work_items = executor._pending_work_items
|
||||
|
||||
super().__init__()
|
||||
|
||||
def run(self):
|
||||
# Main loop for the executor manager thread.
|
||||
|
||||
while True:
|
||||
self.add_call_item_to_queue()
|
||||
|
||||
result_item, is_broken, cause = self.wait_result_broken_or_wakeup()
|
||||
|
||||
if is_broken:
|
||||
self.terminate_broken(cause)
|
||||
return
|
||||
if result_item is not None:
|
||||
self.process_result_item(result_item)
|
||||
# Delete reference to result_item to avoid keeping references
|
||||
# while waiting on new results.
|
||||
del result_item
|
||||
|
||||
# attempt to increment idle process count
|
||||
executor = self.executor_reference()
|
||||
if executor is not None:
|
||||
executor._idle_worker_semaphore.release()
|
||||
del executor
|
||||
|
||||
if self.is_shutting_down():
|
||||
self.flag_executor_shutting_down()
|
||||
|
||||
# Since no new work items can be added, it is safe to shutdown
|
||||
# this thread if there are no pending work items.
|
||||
if not self.pending_work_items:
|
||||
self.join_executor_internals()
|
||||
return
|
||||
|
||||
def add_call_item_to_queue(self):
|
||||
# Fills call_queue with _WorkItems from pending_work_items.
|
||||
# This function never blocks.
|
||||
while True:
|
||||
if self.call_queue.full():
|
||||
return
|
||||
try:
|
||||
work_id = self.work_ids_queue.get(block=False)
|
||||
except queue.Empty:
|
||||
return
|
||||
if work_item.future.set_running_or_notify_cancel():
|
||||
call_queue.put(_CallItem(work_id,
|
||||
work_item.fn,
|
||||
work_item.args,
|
||||
work_item.kwargs),
|
||||
block=True)
|
||||
else:
|
||||
work_item = self.pending_work_items[work_id]
|
||||
del pending_work_items[work_id]
|
||||
continue
|
||||
|
||||
if work_item.future.set_running_or_notify_cancel():
|
||||
self.call_queue.put(_CallItem(work_id,
|
||||
work_item.fn,
|
||||
work_item.args,
|
||||
work_item.kwargs),
|
||||
block=True)
|
||||
else:
|
||||
del self.pending_work_items[work_id]
|
||||
continue
|
||||
def _queue_management_worker(executor_reference,
|
||||
processes,
|
||||
pending_work_items,
|
||||
work_ids_queue,
|
||||
call_queue,
|
||||
result_queue):
|
||||
"""Manages the communication between this process and the worker processes.
|
||||
|
||||
def wait_result_broken_or_wakeup(self):
|
||||
# Wait for a result to be ready in the result_queue while checking
|
||||
# that all worker processes are still running, or for a wake up
|
||||
# signal send. The wake up signals come either from new tasks being
|
||||
# submitted, from the executor being shutdown/gc-ed, or from the
|
||||
# shutdown of the python interpreter.
|
||||
result_reader = self.result_queue._reader
|
||||
assert not self.thread_wakeup._closed
|
||||
wakeup_reader = self.thread_wakeup._reader
|
||||
readers = [result_reader, wakeup_reader]
|
||||
worker_sentinels = [p.sentinel for p in list(self.processes.values())]
|
||||
ready = mp.connection.wait(readers + worker_sentinels)
|
||||
This function is run in a local thread.
|
||||
|
||||
cause = None
|
||||
is_broken = True
|
||||
result_item = None
|
||||
if result_reader in ready:
|
||||
try:
|
||||
result_item = result_reader.recv()
|
||||
is_broken = False
|
||||
except BaseException as e:
|
||||
cause = traceback.format_exception(type(e), e, e.__traceback__)
|
||||
Args:
|
||||
executor_reference: A weakref.ref to the ProcessPoolExecutor that owns
|
||||
this thread. Used to determine if the ProcessPoolExecutor has been
|
||||
garbage collected and that this function can exit.
|
||||
process: A list of the multiprocessing.Process instances used as
|
||||
workers.
|
||||
pending_work_items: A dict mapping work ids to _WorkItems e.g.
|
||||
{5: <_WorkItem...>, 6: <_WorkItem...>, ...}
|
||||
work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]).
|
||||
call_queue: A multiprocessing.Queue that will be filled with _CallItems
|
||||
derived from _WorkItems for processing by the process workers.
|
||||
result_queue: A multiprocessing.Queue of _ResultItems generated by the
|
||||
process workers.
|
||||
"""
|
||||
executor = None
|
||||
|
||||
elif wakeup_reader in ready:
|
||||
is_broken = False
|
||||
def shutting_down():
|
||||
return _shutdown or executor is None or executor._shutdown_thread
|
||||
|
||||
with self.shutdown_lock:
|
||||
self.thread_wakeup.clear()
|
||||
def shutdown_worker():
|
||||
# This is an upper bound
|
||||
nb_children_alive = sum(p.is_alive() for p in processes.values())
|
||||
for i in range(0, nb_children_alive):
|
||||
call_queue.put_nowait(None)
|
||||
# Release the queue's resources as soon as possible.
|
||||
call_queue.close()
|
||||
# If .join() is not called on the created processes then
|
||||
# some multiprocessing.Queue methods may deadlock on Mac OS X.
|
||||
for p in processes.values():
|
||||
p.join()
|
||||
|
||||
return result_item, is_broken, cause
|
||||
reader = result_queue._reader
|
||||
|
||||
def process_result_item(self, result_item):
|
||||
# Process the received a result_item. This can be either the PID of a
|
||||
# worker that exited gracefully or a _ResultItem
|
||||
while True:
|
||||
_add_call_item_to_queue(pending_work_items,
|
||||
work_ids_queue,
|
||||
call_queue)
|
||||
|
||||
sentinels = [p.sentinel for p in processes.values()]
|
||||
assert sentinels
|
||||
ready = wait([reader] + sentinels)
|
||||
if reader in ready:
|
||||
result_item = reader.recv()
|
||||
else:
|
||||
# Mark the process pool broken so that submits fail right now.
|
||||
executor = executor_reference()
|
||||
if executor is not None:
|
||||
executor._broken = True
|
||||
executor._shutdown_thread = True
|
||||
executor = None
|
||||
# All futures in flight must be marked failed
|
||||
for work_id, work_item in pending_work_items.items():
|
||||
work_item.future.set_exception(
|
||||
BrokenProcessPool(
|
||||
"A process in the process pool was "
|
||||
"terminated abruptly while the future was "
|
||||
"running or pending."
|
||||
))
|
||||
# Delete references to object. See issue16284
|
||||
del work_item
|
||||
pending_work_items.clear()
|
||||
# Terminate remaining workers forcibly: the queues or their
|
||||
# locks may be in a dirty state and block forever.
|
||||
for p in processes.values():
|
||||
p.terminate()
|
||||
shutdown_worker()
|
||||
return
|
||||
if isinstance(result_item, int):
|
||||
# Clean shutdown of a worker using its PID
|
||||
# (avoids marking the executor broken)
|
||||
assert self.is_shutting_down()
|
||||
p = self.processes.pop(result_item)
|
||||
assert shutting_down()
|
||||
p = processes.pop(result_item)
|
||||
p.join()
|
||||
if not self.processes:
|
||||
self.join_executor_internals()
|
||||
if not processes:
|
||||
shutdown_worker()
|
||||
return
|
||||
else:
|
||||
# Received a _ResultItem so mark the future as completed.
|
||||
work_item = self.pending_work_items.pop(result_item.work_id, None)
|
||||
elif result_item is not None:
|
||||
work_item = pending_work_items.pop(result_item.work_id, None)
|
||||
# work_item can be None if another process terminated (see above)
|
||||
if work_item is not None:
|
||||
if result_item.exception:
|
||||
work_item.future.set_exception(result_item.exception)
|
||||
else:
|
||||
work_item.future.set_result(result_item.result)
|
||||
|
||||
def is_shutting_down(self):
|
||||
# Check whether we should start shutting down the executor.
|
||||
executor = self.executor_reference()
|
||||
# Delete references to object. See issue16284
|
||||
del work_item
|
||||
# Check whether we should start shutting down.
|
||||
executor = executor_reference()
|
||||
# No more work items can be added if:
|
||||
# - The interpreter is shutting down OR
|
||||
# - The executor that owns this worker has been collected OR
|
||||
# - The executor that owns this worker has been shutdown.
|
||||
return (_global_shutdown or executor is None
|
||||
or executor._shutdown_thread)
|
||||
|
||||
def terminate_broken(self, cause):
|
||||
# Terminate the executor because it is in a broken state. The cause
|
||||
# argument can be used to display more information on the error that
|
||||
# lead the executor into becoming broken.
|
||||
|
||||
# Mark the process pool broken so that submits fail right now.
|
||||
executor = self.executor_reference()
|
||||
if executor is not None:
|
||||
executor._broken = ('A child process terminated '
|
||||
'abruptly, the process pool is not '
|
||||
'usable anymore')
|
||||
executor._shutdown_thread = True
|
||||
executor = None
|
||||
|
||||
# All pending tasks are to be marked failed with the following
|
||||
# BrokenProcessPool error
|
||||
bpe = BrokenProcessPool("A process in the process pool was "
|
||||
"terminated abruptly while the future was "
|
||||
"running or pending.")
|
||||
if cause is not None:
|
||||
bpe.__cause__ = _RemoteTraceback(
|
||||
f"\n'''\n{''.join(cause)}'''")
|
||||
|
||||
# Mark pending tasks as failed.
|
||||
for work_id, work_item in self.pending_work_items.items():
|
||||
work_item.future.set_exception(bpe)
|
||||
# Delete references to object. See issue16284
|
||||
del work_item
|
||||
self.pending_work_items.clear()
|
||||
|
||||
# Terminate remaining workers forcibly: the queues or their
|
||||
# locks may be in a dirty state and block forever.
|
||||
for p in self.processes.values():
|
||||
p.terminate()
|
||||
|
||||
# clean up resources
|
||||
self.join_executor_internals()
|
||||
|
||||
def flag_executor_shutting_down(self):
|
||||
# Flag the executor as shutting down and cancel remaining tasks if
|
||||
# requested as early as possible if it is not gc-ed yet.
|
||||
executor = self.executor_reference()
|
||||
if executor is not None:
|
||||
executor._shutdown_thread = True
|
||||
# Cancel pending work items if requested.
|
||||
if executor._cancel_pending_futures:
|
||||
# Cancel all pending futures and update pending_work_items
|
||||
# to only have futures that are currently running.
|
||||
new_pending_work_items = {}
|
||||
for work_id, work_item in self.pending_work_items.items():
|
||||
if not work_item.future.cancel():
|
||||
new_pending_work_items[work_id] = work_item
|
||||
self.pending_work_items = new_pending_work_items
|
||||
# Drain work_ids_queue since we no longer need to
|
||||
# add items to the call queue.
|
||||
while True:
|
||||
try:
|
||||
self.work_ids_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
# Make sure we do this only once to not waste time looping
|
||||
# on running processes over and over.
|
||||
executor._cancel_pending_futures = False
|
||||
|
||||
def shutdown_workers(self):
|
||||
n_children_to_stop = self.get_n_children_alive()
|
||||
n_sentinels_sent = 0
|
||||
# Send the right number of sentinels, to make sure all children are
|
||||
# properly terminated.
|
||||
while (n_sentinels_sent < n_children_to_stop
|
||||
and self.get_n_children_alive() > 0):
|
||||
for i in range(n_children_to_stop - n_sentinels_sent):
|
||||
try:
|
||||
self.call_queue.put_nowait(None)
|
||||
n_sentinels_sent += 1
|
||||
except queue.Full:
|
||||
break
|
||||
|
||||
def join_executor_internals(self):
|
||||
self.shutdown_workers()
|
||||
# Release the queue's resources as soon as possible.
|
||||
self.call_queue.close()
|
||||
self.call_queue.join_thread()
|
||||
with self.shutdown_lock:
|
||||
self.thread_wakeup.close()
|
||||
# If .join() is not called on the created processes then
|
||||
# some ctx.Queue methods may deadlock on Mac OS X.
|
||||
for p in self.processes.values():
|
||||
p.join()
|
||||
|
||||
def get_n_children_alive(self):
|
||||
# This is an upper bound on the number of children alive.
|
||||
return sum(p.is_alive() for p in self.processes.values())
|
||||
|
||||
if shutting_down():
|
||||
try:
|
||||
# Since no new work items can be added, it is safe to shutdown
|
||||
# this thread if there are no pending work items.
|
||||
if not pending_work_items:
|
||||
shutdown_worker()
|
||||
return
|
||||
except Full:
|
||||
# This is not a problem: we will eventually be woken up (in
|
||||
# result_queue.get()) and be able to send a sentinel again.
|
||||
pass
|
||||
executor = None
|
||||
|
||||
_system_limits_checked = False
|
||||
_system_limited = None
|
||||
|
||||
|
||||
def _check_system_limits():
|
||||
global _system_limits_checked, _system_limited
|
||||
if _system_limits_checked:
|
||||
if _system_limited:
|
||||
raise NotImplementedError(_system_limited)
|
||||
_system_limits_checked = True
|
||||
try:
|
||||
import multiprocessing.synchronize
|
||||
except ImportError:
|
||||
_system_limited = (
|
||||
"This Python build lacks multiprocessing.synchronize, usually due "
|
||||
"to named semaphores being unavailable on this platform."
|
||||
)
|
||||
raise NotImplementedError(_system_limited)
|
||||
try:
|
||||
nsems_max = os.sysconf("SC_SEM_NSEMS_MAX")
|
||||
except (AttributeError, ValueError):
|
||||
@@ -556,24 +353,11 @@ def _check_system_limits():
|
||||
# minimum number of semaphores available
|
||||
# according to POSIX
|
||||
return
|
||||
_system_limited = ("system provides too few semaphores (%d"
|
||||
" available, 256 necessary)" % nsems_max)
|
||||
_system_limited = "system provides too few semaphores (%d available, 256 necessary)" % nsems_max
|
||||
raise NotImplementedError(_system_limited)
|
||||
|
||||
|
||||
def _chain_from_iterable_of_lists(iterable):
|
||||
"""
|
||||
Specialized implementation of itertools.chain.from_iterable.
|
||||
Each item in *iterable* should be a list. This function is
|
||||
careful not to keep references to yielded objects.
|
||||
"""
|
||||
for element in iterable:
|
||||
element.reverse()
|
||||
while element:
|
||||
yield element.pop()
|
||||
|
||||
|
||||
class BrokenProcessPool(_base.BrokenExecutor):
|
||||
class BrokenProcessPool(RuntimeError):
|
||||
"""
|
||||
Raised when a process in a ProcessPoolExecutor terminated abruptly
|
||||
while a future was in the running state.
|
||||
@@ -581,143 +365,82 @@ class BrokenProcessPool(_base.BrokenExecutor):
|
||||
|
||||
|
||||
class ProcessPoolExecutor(_base.Executor):
|
||||
def __init__(self, max_workers=None, mp_context=None,
|
||||
initializer=None, initargs=()):
|
||||
def __init__(self, max_workers=None):
|
||||
"""Initializes a new ProcessPoolExecutor instance.
|
||||
|
||||
Args:
|
||||
max_workers: The maximum number of processes that can be used to
|
||||
execute the given calls. If None or not given then as many
|
||||
worker processes will be created as the machine has processors.
|
||||
mp_context: A multiprocessing context to launch the workers. This
|
||||
object should provide SimpleQueue, Queue and Process.
|
||||
initializer: A callable used to initialize worker processes.
|
||||
initargs: A tuple of arguments to pass to the initializer.
|
||||
"""
|
||||
_check_system_limits()
|
||||
|
||||
if max_workers is None:
|
||||
self._max_workers = os.cpu_count() or 1
|
||||
if sys.platform == 'win32':
|
||||
self._max_workers = min(_MAX_WINDOWS_WORKERS,
|
||||
self._max_workers)
|
||||
else:
|
||||
if max_workers <= 0:
|
||||
raise ValueError("max_workers must be greater than 0")
|
||||
elif (sys.platform == 'win32' and
|
||||
max_workers > _MAX_WINDOWS_WORKERS):
|
||||
raise ValueError(
|
||||
f"max_workers must be <= {_MAX_WINDOWS_WORKERS}")
|
||||
|
||||
self._max_workers = max_workers
|
||||
|
||||
if mp_context is None:
|
||||
mp_context = mp.get_context()
|
||||
self._mp_context = mp_context
|
||||
|
||||
# https://github.com/python/cpython/issues/90622
|
||||
self._safe_to_dynamically_spawn_children = (
|
||||
self._mp_context.get_start_method(allow_none=False) != "fork")
|
||||
|
||||
if initializer is not None and not callable(initializer):
|
||||
raise TypeError("initializer must be a callable")
|
||||
self._initializer = initializer
|
||||
self._initargs = initargs
|
||||
|
||||
# Management thread
|
||||
self._executor_manager_thread = None
|
||||
|
||||
# Make the call queue slightly larger than the number of processes to
|
||||
# prevent the worker processes from idling. But don't make it too big
|
||||
# because futures in the call queue cannot be cancelled.
|
||||
self._call_queue = multiprocessing.Queue(self._max_workers +
|
||||
EXTRA_QUEUED_CALLS)
|
||||
# Killed worker processes can produce spurious "broken pipe"
|
||||
# tracebacks in the queue's own worker thread. But we detect killed
|
||||
# processes anyway, so silence the tracebacks.
|
||||
self._call_queue._ignore_epipe = True
|
||||
self._result_queue = SimpleQueue()
|
||||
self._work_ids = queue.Queue()
|
||||
self._queue_management_thread = None
|
||||
# Map of pids to processes
|
||||
self._processes = {}
|
||||
|
||||
# Shutdown is a two-step process.
|
||||
self._shutdown_thread = False
|
||||
self._shutdown_lock = threading.Lock()
|
||||
self._idle_worker_semaphore = threading.Semaphore(0)
|
||||
self._broken = False
|
||||
self._queue_count = 0
|
||||
self._pending_work_items = {}
|
||||
self._cancel_pending_futures = False
|
||||
|
||||
# _ThreadWakeup is a communication channel used to interrupt the wait
|
||||
# of the main loop of executor_manager_thread from another thread (e.g.
|
||||
# when calling executor.submit or executor.shutdown). We do not use the
|
||||
# _result_queue to send wakeup signals to the executor_manager_thread
|
||||
# as it could result in a deadlock if a worker process dies with the
|
||||
# _result_queue write lock still acquired.
|
||||
#
|
||||
# _shutdown_lock must be locked to access _ThreadWakeup.
|
||||
self._executor_manager_thread_wakeup = _ThreadWakeup()
|
||||
|
||||
# Create communication channels for the executor
|
||||
# Make the call queue slightly larger than the number of processes to
|
||||
# prevent the worker processes from idling. But don't make it too big
|
||||
# because futures in the call queue cannot be cancelled.
|
||||
queue_size = self._max_workers + EXTRA_QUEUED_CALLS
|
||||
self._call_queue = _SafeQueue(
|
||||
max_size=queue_size, ctx=self._mp_context,
|
||||
pending_work_items=self._pending_work_items,
|
||||
shutdown_lock=self._shutdown_lock,
|
||||
thread_wakeup=self._executor_manager_thread_wakeup)
|
||||
# Killed worker processes can produce spurious "broken pipe"
|
||||
# tracebacks in the queue's own worker thread. But we detect killed
|
||||
# processes anyway, so silence the tracebacks.
|
||||
self._call_queue._ignore_epipe = True
|
||||
self._result_queue = mp_context.SimpleQueue()
|
||||
self._work_ids = queue.Queue()
|
||||
|
||||
def _start_executor_manager_thread(self):
|
||||
if self._executor_manager_thread is None:
|
||||
def _start_queue_management_thread(self):
|
||||
# When the executor gets lost, the weakref callback will wake up
|
||||
# the queue management thread.
|
||||
def weakref_cb(_, q=self._result_queue):
|
||||
q.put(None)
|
||||
if self._queue_management_thread is None:
|
||||
# Start the processes so that their sentinels are known.
|
||||
if not self._safe_to_dynamically_spawn_children: # ie, using fork.
|
||||
self._launch_processes()
|
||||
self._executor_manager_thread = _ExecutorManagerThread(self)
|
||||
self._executor_manager_thread.start()
|
||||
_threads_wakeups[self._executor_manager_thread] = \
|
||||
self._executor_manager_thread_wakeup
|
||||
self._adjust_process_count()
|
||||
self._queue_management_thread = threading.Thread(
|
||||
target=_queue_management_worker,
|
||||
args=(weakref.ref(self, weakref_cb),
|
||||
self._processes,
|
||||
self._pending_work_items,
|
||||
self._work_ids,
|
||||
self._call_queue,
|
||||
self._result_queue))
|
||||
self._queue_management_thread.daemon = True
|
||||
self._queue_management_thread.start()
|
||||
_threads_queues[self._queue_management_thread] = self._result_queue
|
||||
|
||||
def _adjust_process_count(self):
|
||||
# if there's an idle process, we don't need to spawn a new one.
|
||||
if self._idle_worker_semaphore.acquire(blocking=False):
|
||||
return
|
||||
|
||||
process_count = len(self._processes)
|
||||
if process_count < self._max_workers:
|
||||
# Assertion disabled as this codepath is also used to replace a
|
||||
# worker that unexpectedly dies, even when using the 'fork' start
|
||||
# method. That means there is still a potential deadlock bug. If a
|
||||
# 'fork' mp_context worker dies, we'll be forking a new one when
|
||||
# we know a thread is running (self._executor_manager_thread).
|
||||
#assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622'
|
||||
self._spawn_process()
|
||||
|
||||
def _launch_processes(self):
|
||||
# https://github.com/python/cpython/issues/90622
|
||||
assert not self._executor_manager_thread, (
|
||||
'Processes cannot be fork()ed after the thread has started, '
|
||||
'deadlock in the child processes could result.')
|
||||
for _ in range(len(self._processes), self._max_workers):
|
||||
self._spawn_process()
|
||||
p = multiprocessing.Process(
|
||||
target=_process_worker,
|
||||
args=(self._call_queue,
|
||||
self._result_queue))
|
||||
p.start()
|
||||
self._processes[p.pid] = p
|
||||
|
||||
def _spawn_process(self):
|
||||
p = self._mp_context.Process(
|
||||
target=_process_worker,
|
||||
args=(self._call_queue,
|
||||
self._result_queue,
|
||||
self._initializer,
|
||||
self._initargs))
|
||||
p.start()
|
||||
self._processes[p.pid] = p
|
||||
|
||||
def submit(self, fn, /, *args, **kwargs):
|
||||
def submit(self, fn, *args, **kwargs):
|
||||
with self._shutdown_lock:
|
||||
if self._broken:
|
||||
raise BrokenProcessPool(self._broken)
|
||||
raise BrokenProcessPool('A child process terminated '
|
||||
'abruptly, the process pool is not usable anymore')
|
||||
if self._shutdown_thread:
|
||||
raise RuntimeError('cannot schedule new futures after shutdown')
|
||||
if _global_shutdown:
|
||||
raise RuntimeError('cannot schedule new futures after '
|
||||
'interpreter shutdown')
|
||||
|
||||
f = _base.Future()
|
||||
w = _WorkItem(f, fn, args, kwargs)
|
||||
@@ -726,11 +449,9 @@ class ProcessPoolExecutor(_base.Executor):
|
||||
self._work_ids.put(self._queue_count)
|
||||
self._queue_count += 1
|
||||
# Wake up queue management thread
|
||||
self._executor_manager_thread_wakeup.wakeup()
|
||||
self._result_queue.put(None)
|
||||
|
||||
if self._safe_to_dynamically_spawn_children:
|
||||
self._adjust_process_count()
|
||||
self._start_executor_manager_thread()
|
||||
self._start_queue_management_thread()
|
||||
return f
|
||||
submit.__doc__ = _base.Executor.submit.__doc__
|
||||
|
||||
@@ -761,26 +482,22 @@ class ProcessPoolExecutor(_base.Executor):
|
||||
results = super().map(partial(_process_chunk, fn),
|
||||
_get_chunks(*iterables, chunksize=chunksize),
|
||||
timeout=timeout)
|
||||
return _chain_from_iterable_of_lists(results)
|
||||
return itertools.chain.from_iterable(results)
|
||||
|
||||
def shutdown(self, wait=True, *, cancel_futures=False):
|
||||
def shutdown(self, wait=True):
|
||||
with self._shutdown_lock:
|
||||
self._cancel_pending_futures = cancel_futures
|
||||
self._shutdown_thread = True
|
||||
if self._executor_manager_thread_wakeup is not None:
|
||||
# Wake up queue management thread
|
||||
self._executor_manager_thread_wakeup.wakeup()
|
||||
|
||||
if self._executor_manager_thread is not None and wait:
|
||||
self._executor_manager_thread.join()
|
||||
if self._queue_management_thread:
|
||||
# Wake up queue management thread
|
||||
self._result_queue.put(None)
|
||||
if wait:
|
||||
self._queue_management_thread.join()
|
||||
# To reduce the risk of opening too many files, remove references to
|
||||
# objects that use file descriptors.
|
||||
self._executor_manager_thread = None
|
||||
self._queue_management_thread = None
|
||||
self._call_queue = None
|
||||
if self._result_queue is not None and wait:
|
||||
self._result_queue.close()
|
||||
self._result_queue = None
|
||||
self._processes = None
|
||||
self._executor_manager_thread_wakeup = None
|
||||
|
||||
shutdown.__doc__ = _base.Executor.shutdown.__doc__
|
||||
|
||||
atexit.register(_python_exit)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user