Rust-related/RustPython
1
0
Fork 1
mirror of https://github.com/RustPython/RustPython.git synced 2026-06-02 19:39:49 +09:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: Rust-related:2025-03-10-main-20
Branches Tags
Rust-related:main Rust-related:copilot/fix-isalnum-function-behavior Rust-related:dependabot/cargo/random-dc9d2ff19f Rust-related:dependabot/cargo/crypto-493bd7afb0 Rust-related:copilot/add-shared-unicode-crate Rust-related:copilot/fix-str-subclass-type-issue Rust-related:typelock Rust-related:specialization Rust-related:release mschoi:dev mschoi:impl_del_object_dict mschoi:impl_del_object_set_dict mschoi:main mschoi:update_test_float_from_CPython_v3.12.7 mschoi:update_fstring_from_v3.12.7 mschoi:redox-release mschoi:no_std mschoi:release mschoi:function_attributes mschoi:wasm-output mschoi:framestack
Rust-related:0.5.0 Rust-related:2025-11-10-main-55 Rust-related:2025-11-03-main-54 Rust-related:2025-10-27-main-53 Rust-related:2025-10-13-main-51 Rust-related:2025-10-06-main-50 Rust-related:2025-09-29-main-49 Rust-related:2025-09-22-main-48 Rust-related:2025-09-15-main-47 Rust-related:2025-09-08-main-46 Rust-related:2025-09-01-main-45 Rust-related:2025-08-25-main-44 Rust-related:2025-08-18-main-43 Rust-related:2025-08-11-main-42 Rust-related:2025-08-04-main-41 Rust-related:2025-07-28-main-40 Rust-related:2025-07-21-main-39 Rust-related:2025-07-14-main-38 Rust-related:2025-07-07-main-37 Rust-related:2025-06-30-main-36 Rust-related:2025-06-23-main-35 Rust-related:2025-06-16-main-34 Rust-related:2025-06-09-main-33 Rust-related:2025-06-02-main-32 Rust-related:2025-05-19-main-30 Rust-related:2025-05-12-main-29 Rust-related:2025-05-05-main-28 Rust-related:2025-04-28-main-27 Rust-related:2025-04-21-main-26 Rust-related:2025-04-14-main-25 Rust-related:2025-03-17-main-21 Rust-related:2025-03-10-main-20 Rust-related:2025-03-03-main-14 Rust-related:2025-02-24-main-13 Rust-related:2025-02-17-main-12 Rust-related:2025-02-10-main-11 Rust-related:2025-02-03-main-10 Rust-related:2025-01-27-main-9 Rust-related:2025-01-20-main-8 Rust-related:2025-01-13-main-7 Rust-related:2025-01-06-main-6 Rust-related:2025-01-03-main-5 Rust-related:2024-12-30-main-4 Rust-related:0.4.0 Rust-related:0.3.1 Rust-related:0.3.0 Rust-related:v0.2.0 Rust-related:0.1.0 mschoi:0.4.0 mschoi:0.3.1 mschoi:0.3.0 mschoi:v0.2.0 mschoi:v0.1.2 mschoi:0.1.0
..
compare: mschoi:v0.1.2
Branches Tags
mschoi:dev mschoi:impl_del_object_dict mschoi:impl_del_object_set_dict mschoi:main mschoi:update_test_float_from_CPython_v3.12.7 mschoi:update_fstring_from_v3.12.7 mschoi:redox-release mschoi:no_std mschoi:release mschoi:function_attributes mschoi:wasm-output mschoi:framestack Rust-related:main Rust-related:copilot/fix-isalnum-function-behavior Rust-related:dependabot/cargo/random-dc9d2ff19f Rust-related:dependabot/cargo/crypto-493bd7afb0 Rust-related:copilot/add-shared-unicode-crate Rust-related:copilot/fix-str-subclass-type-issue Rust-related:typelock Rust-related:specialization Rust-related:release
mschoi:0.4.0 mschoi:0.3.1 mschoi:0.3.0 mschoi:v0.2.0 mschoi:v0.1.2 mschoi:0.1.0 Rust-related:0.5.0 Rust-related:2025-11-10-main-55 Rust-related:2025-11-03-main-54 Rust-related:2025-10-27-main-53 Rust-related:2025-10-13-main-51 Rust-related:2025-10-06-main-50 Rust-related:2025-09-29-main-49 Rust-related:2025-09-22-main-48 Rust-related:2025-09-15-main-47 Rust-related:2025-09-08-main-46 Rust-related:2025-09-01-main-45 Rust-related:2025-08-25-main-44 Rust-related:2025-08-18-main-43 Rust-related:2025-08-11-main-42 Rust-related:2025-08-04-main-41 Rust-related:2025-07-28-main-40 Rust-related:2025-07-21-main-39 Rust-related:2025-07-14-main-38 Rust-related:2025-07-07-main-37 Rust-related:2025-06-30-main-36 Rust-related:2025-06-23-main-35 Rust-related:2025-06-16-main-34 Rust-related:2025-06-09-main-33 Rust-related:2025-06-02-main-32 Rust-related:2025-05-19-main-30 Rust-related:2025-05-12-main-29 Rust-related:2025-05-05-main-28 Rust-related:2025-04-28-main-27 Rust-related:2025-04-21-main-26 Rust-related:2025-04-14-main-25 Rust-related:2025-03-17-main-21 Rust-related:2025-03-10-main-20 Rust-related:2025-03-03-main-14 Rust-related:2025-02-24-main-13 Rust-related:2025-02-17-main-12 Rust-related:2025-02-10-main-11 Rust-related:2025-02-03-main-10 Rust-related:2025-01-27-main-9 Rust-related:2025-01-20-main-8 Rust-related:2025-01-13-main-7 Rust-related:2025-01-06-main-6 Rust-related:2025-01-03-main-5 Rust-related:2024-12-30-main-4 Rust-related:0.4.0 Rust-related:0.3.1 Rust-related:0.3.0 Rust-related:v0.2.0 Rust-related:0.1.0

1 Commits
2025-03-10 ... v0.1.2

Author SHA1 Message Date
Noah
5343c0bd0b Release 0.1.2 
rustpython@0.1.2
rustpython-bytecode@0.1.2
rustpython-compiler@0.1.2
rustpython-derive@0.1.2
rustpython-parser@0.1.2
rustpython-vm@0.1.2
rustpython_freeze@0.1.2
rustpython_wasm@0.1.2

Generated by cargo-workspaces
 2020-06-21 18:50:01 -05:00
2159 changed files with 86550 additions and 624138 deletions
Expand all files Collapse all files

5
.cargo/config.toml
View File

@@ -1,5 +0,0 @@
[target.'cfg(target_env = "msvc")']
rustflags = "-C link-arg=/STACK:8000000"
[target.'cfg(all(target_os = "windows", not(target_env = "msvc")))']
rustflags = "-C link-args=-Wl,--stack,8000000"

299
.cspell.json
View File

@@ -1,299 +0,0 @@
// See: https://github.com/streetsidesoftware/cspell/tree/master/packages/cspell
{
"version": "0.2",
// language - current active spelling language
"language": "en",
// dictionaries - list of the names of the dictionaries to use
"dictionaries": [
"en_US",
"softwareTerms",
"c",
"cpp",
"python",
"python-custom",
"rust",
"unix",
"posix",
"winapi"
],
// dictionaryDefinitions - this list defines any custom dictionaries to use
"dictionaryDefinitions": [],
"ignorePaths": [
"**/__pycache__/**",
"Lib/**"
],
// words - list of words to be always considered correct
"words": [
// Rust
"ahash",
"bidi",
"biguint",
"bindgen",
"bitflags",
"bstr",
"byteorder",
"chrono",
"consts",
"cstring",
"flate2",
"fract",
"hasher",
"idents",
"indexmap",
"insta",
"keccak",
"lalrpop",
"libc",
"libz",
"longlong",
"Manually",
"maplit",
"memmap",
"metas",
"modpow",
"nanos",
"objclass",
"peekable",
"powc",
"powf",
"prepended",
"punct",
"replacen",
"rsplitn",
"rustc",
"rustfmt",
"seekfrom",
"splitn",
"subsec",
"timsort",
"trai",
"ulonglong",
"unic",
"unistd",
"winapi",
"winsock",
// Python
"abstractmethods",
"aiter",
"anext",
"arrayiterator",
"arraytype",
"asend",
"athrow",
"basicsize",
"cformat",
"classcell",
"closesocket",
"codepoint",
"codepoints",
"cpython",
"decompressor",
"defaultaction",
"descr",
"dictcomp",
"dictitems",
"dictkeys",
"dictview",
"docstring",
"docstrings",
"dunder",
"eventmask",
"fdel",
"fget",
"fileencoding",
"fillchar",
"finallyhandler",
"frombytes",
"fromhex",
"fromunicode",
"fset",
"fspath",
"fstring",
"fstrings",
"genexpr",
"getattro",
"getformat",
"getnewargs",
"getweakrefcount",
"getweakrefs",
"hostnames",
"idiv",
"impls",
"infj",
"instancecheck",
"instanceof",
"isabstractmethod",
"itemiterator",
"itemsize",
"iternext",
"keyiterator",
"kwarg",
"kwargs",
"linearization",
"linearize",
"listcomp",
"mappingproxy",
"maxsplit",
"memoryview",
"memoryviewiterator",
"metaclass",
"metaclasses",
"metatype",
"mro",
"mros",
"nanj",
"ndigits",
"ndim",
"nonbytes",
"origname",
"posixsubprocess",
"pyexpat",
"PYTHONDEBUG",
"PYTHONHOME",
"PYTHONINSPECT",
"PYTHONOPTIMIZE",
"PYTHONPATH",
"PYTHONPATH",
"PYTHONVERBOSE",
"PYTHONWARNINGS",
"qualname",
"radd",
"rdiv",
"rdivmod",
"reconstructor",
"reversevalueiterator",
"rfloordiv",
"rlshift",
"rmod",
"rpow",
"rrshift",
"rsub",
"rtruediv",
"scproxy",
"setattro",
"setcomp",
"showwarnmsg",
"warnmsg",
"stacklevel",
"subclasscheck",
"subclasshook",
"unionable",
"unraisablehook",
"valueiterator",
"vararg",
"varargs",
"varnames",
"warningregistry",
"warnopts",
"weakproxy",
"xopts",
// RustPython
"baseclass",
"Bytecode",
"cfgs",
"codegen",
"dedentations",
"dedents",
"deduped",
"downcasted",
"dumpable",
"GetSet",
"internable",
"makeunicodedata",
"miri",
"notrace",
"pyarg",
"pyarg",
"pyargs",
"PyAttr",
"pyc",
"PyClass",
"PyClassMethod",
"PyException",
"PyFunction",
"pygetset",
"pyimpl",
"pymember",
"PyMethod",
"PyModule",
"pyname",
"pyobj",
"PyObject",
"pypayload",
"PyProperty",
"pyref",
"PyResult",
"pyslot",
"PyStaticMethod",
"pystr",
"pystruct",
"pystructseq",
"pytrace",
"reducelib",
"richcompare",
"RustPython",
"struc",
"tracebacks",
"typealiases",
"Unconstructible",
"unhashable",
"uninit",
"unraisable",
"wasi",
"zelf",
// cpython
"argtypes",
"asdl",
"asname",
"augassign",
"badsyntax",
"basetype",
"boolop",
"bxor",
"cellarg",
"cellvar",
"cellvars",
"cmpop",
"dictoffset",
"elts",
"excepthandler",
"finalbody",
"freevar",
"freevars",
"fromlist",
"heaptype",
"IMMUTABLETYPE",
"kwonlyarg",
"kwonlyargs",
"linearise",
"maxdepth",
"mult",
"nkwargs",
"orelse",
"patma",
"posonlyarg",
"posonlyargs",
"prec",
"stackdepth",
"unaryop",
"unparse",
"unparser",
"VARKEYWORDS",
"varkwarg",
"wbits",
"withitem",
"withs"
],
// flagWords - list of words to be always considered incorrect
"flagWords": [
],
"ignoreRegExpList": [
],
// languageSettings - allow for per programming language configuration settings.
"languageSettings": [
{
"languageId": "python",
"locale": "en"
}
]
}

4
.devcontainer/devcontainer.json
View File

@@ -1,4 +0,0 @@
{
"image": "mcr.microsoft.com/devcontainers/base:jammy",
"onCreateCommand": "curl https://sh.rustup.rs -sSf | sh -s -- -y"
}

2
.dockerignore
View File

@@ -8,7 +8,7 @@
.vscode
wasm-pack.log
.idea/
extra_tests/snippets/resources
tests/snippets/resources
flame-graph.html
flame.txt

3
.flake8
View File

@@ -1,3 +0,0 @@
[flake8]
# black's line length
max-line-length = 88

8
.gitattributes vendored
View File

@@ -1,7 +1 @@
Lib/** linguist-vendored
Cargo.lock linguist-generated -merge
*.snap linguist-generated -merge
vm/src/stdlib/ast/gen.rs linguist-generated -merge
Lib/*.py text working-tree-encoding=UTF-8 eol=LF
**/*.rs text working-tree-encoding=UTF-8 eol=LF
*.pck binary
Lib/* linguist-vendored

16
.github/ISSUE_TEMPLATE/empty.md vendored
View File

@@ -1,16 +0,0 @@
---
name: Generic issue template
about: which is not covered by other templates
title: ''
labels:
assignees: ''
---
## Summary
<!-- Short description of the issue. -->
## Details
<!-- Whatever you want to share -->

16
.github/ISSUE_TEMPLATE/feature-request.md vendored
View File

@@ -1,16 +0,0 @@
---
name: Feature request
about: Request a feature to use RustPython (as a Rust library)
title: ''
labels: C-enhancement
assignees: 'youknowone'
---
## Summary
<!-- Short description of the request. Please use incompatibility form to report missing features as Python interpreter -->
## Expected use case
<!-- By sharing detailed use case, we can understand the requirements better! If it will be used by open source projects, please also share the project URL. -->

24
.github/ISSUE_TEMPLATE/report-bug.md vendored
View File

@@ -1,24 +0,0 @@
---
name: Report bugs
about: Report a bug not related to CPython compatibility
title: ''
labels: C-bug
assignees: ''
---
## Summary
<!-- Short description of the bug -->
## Expected
<!-- What's the expected result? Using ``` ``` block is preferred for text. -->
## Actual
<!-- What's the actual result? Using ``` ``` block is preferred for text. -->
## Python Documentation
<!-- If applicable. -->

4
.github/ISSUE_TEMPLATE/report-incompatibility.md vendored
View File

@@ -2,7 +2,7 @@
name: Report incompatibility
about: Report an incompatibility between RustPython and CPython
title: ''
labels: C-compat
labels: feat
assignees: ''
---
@@ -11,6 +11,6 @@ assignees: ''
<!-- What Python feature is missing from RustPython? Give a short description of the feature and how you ran into its absence. -->
## Python Documentation or reference to CPython source code
## Python Documentation
<!-- Give a link to the feature in the CPython documentation (https://docs.python.org/3/) in order to assist in its implementation. -->

13
.github/dependabot.yml vendored
View File

@@ -1,13 +0,0 @@
# Keep GitHub Actions up to date with GitHub's Dependabot...
# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
version: 2
updates:
- package-ecosystem: github-actions
directory: /
groups:
github-actions:
patterns:
- "*" # Group all Actions updates into a single larger pull request
schedule:
interval: weekly

448
.github/workflows/ci.yaml vendored
View File

@@ -1,121 +1,15 @@
on:
push:
branches: [main, release]
pull_request:
types: [unlabeled, opened, synchronize, reopened]
merge_group:
workflow_dispatch:
branches: [master, release]
pull_request:
name: CI
# Cancel previous workflows if they are the same workflow on same ref (branch/tags)
# with the same event (push/pull_request) even they are in progress.
# This setting will help reduce the number of duplicated workflows.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true
env:
CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl
# Skip additional tests on Windows. They are checked on Linux and MacOS.
# test_glob: many failing tests
# test_io: many failing tests
# test_os: many failing tests
# test_pathlib: support.rmtree() failing
# test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)')
# test_venv: couple of failing tests
WINDOWS_SKIPS: >-
test_glob
test_io
test_os
test_rlcompleter
test_pathlib
test_posixpath
test_venv
# configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417
# socketserver: seems related to configparser crash.
MACOS_SKIPS: >-
test_configparser
test_socketserver
# PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently
# only run on Linux to speed up the CI.
PLATFORM_INDEPENDENT_TESTS: >-
test__colorize
test_array
test_asyncgen
test_binop
test_bisect
test_bool
test_bytes
test_call
test_class
test_cmath
test_collections
test_complex
test_contains
test_copy
test_dataclasses
test_decimal
test_decorators
test_defaultdict
test_deque
test_dict
test_dictcomps
test_dictviews
test_dis
test_enumerate
test_exception_variations
test_float
test_format
test_fractions
test_genericalias
test_genericclass
test_grammar
test_range
test_index
test_int
test_int_literal
test_isinstance
test_iter
test_iterlen
test_itertools
test_json
test_keyword
test_keywordonlyarg
test_list
test_long
test_longexp
test_math
test_operator
test_ordered_dict
test_pow
test_raise
test_richcmp
test_scope
test_set
test_slice
test_sort
test_string
test_string_literals
test_strtod
test_structseq
test_subclassinit
test_super
test_syntax
test_tuple
test_types
test_unary
test_unpack
test_weakref
test_yield_from
# Python version targeted by the CI.
PYTHON_VERSION: "3.13.1"
CARGO_ARGS: --all --features ssl
jobs:
rust_tests:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
env:
RUST_BACKTRACE: full
name: Run rust tests
runs-on: ${{ matrix.os }}
strategy:
@@ -123,122 +17,25 @@ jobs:
os: [macos-latest, ubuntu-latest, windows-latest]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
components: clippy
- uses: Swatinem/rust-cache@v2
- uses: actions/checkout@master
- name: Set up the Windows environment
shell: bash
run: |
cargo install --target-dir=target -v cargo-vcpkg
cargo vcpkg -v build
powershell.exe scripts/symlinks-to-hardlinks.ps1
if: runner.os == 'Windows'
- name: Set up the Mac environment
run: brew install autoconf automake libtool
if: runner.os == 'macOS'
- name: run clippy
run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --exclude rustpython_wasm -- -Dwarnings
- name: Cache cargo dependencies
uses: actions/cache@v1
with:
key: ${{ runner.os }}-rust_tests-${{ hashFiles('Cargo.lock') }}
path: target
restore-keys: |
${{ runner.os }}-rust_tests-
- name: run rust tests
run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }}
if: runner.os != 'macOS'
- name: run rust tests
run: cargo test --workspace --exclude rustpython_wasm --exclude rustpython-jit --verbose --features threading ${{ env.CARGO_ARGS }}
if: runner.os == 'macOS'
- name: check compilation without threading
run: cargo check ${{ env.CARGO_ARGS }}
- name: Test example projects
run:
cargo run --manifest-path example_projects/barebone/Cargo.toml
cargo run --manifest-path example_projects/frozen_stdlib/Cargo.toml
if: runner.os == 'Linux'
- name: prepare AppleSilicon build
uses: dtolnay/rust-toolchain@stable
uses: actions-rs/cargo@v1
with:
target: aarch64-apple-darwin
if: runner.os == 'macOS'
- name: Check compilation for Apple Silicon
run: cargo check --target aarch64-apple-darwin
if: runner.os == 'macOS'
- name: prepare iOS build
uses: dtolnay/rust-toolchain@stable
with:
target: aarch64-apple-ios
if: runner.os == 'macOS'
- name: Check compilation for iOS
run: cargo check --target aarch64-apple-ios
if: runner.os == 'macOS'
exotic_targets:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
name: Ensure compilation on various targets
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
target: i686-unknown-linux-gnu
- name: Install gcc-multilib and musl-tools
run: sudo apt-get update && sudo apt-get install gcc-multilib musl-tools
- name: Check compilation for x86 32bit
run: cargo check --target i686-unknown-linux-gnu
- uses: dtolnay/rust-toolchain@stable
with:
target: aarch64-linux-android
- name: Check compilation for android
run: cargo check --target aarch64-linux-android
- uses: dtolnay/rust-toolchain@stable
with:
target: aarch64-unknown-linux-gnu
- name: Install gcc-aarch64-linux-gnu
run: sudo apt install gcc-aarch64-linux-gnu
- name: Check compilation for aarch64 linux gnu
run: cargo check --target aarch64-unknown-linux-gnu
- uses: dtolnay/rust-toolchain@stable
with:
target: i686-unknown-linux-musl
- name: Check compilation for musl
run: cargo check --target i686-unknown-linux-musl
- uses: dtolnay/rust-toolchain@stable
with:
target: x86_64-unknown-freebsd
- name: Check compilation for freebsd
run: cargo check --target x86_64-unknown-freebsd
- uses: dtolnay/rust-toolchain@stable
with:
target: x86_64-unknown-freebsd
- name: Check compilation for freeBSD
run: cargo check --target x86_64-unknown-freebsd
- name: Prepare repository for redox compilation
run: bash scripts/redox/uncomment-cargo.sh
- name: Check compilation for Redox
uses: coolreader18/redoxer-action@v1
with:
command: check
args: --ignore-rust-version
command: test
args: --verbose ${{ env.CARGO_ARGS }}
snippets_cpython:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
env:
RUST_BACKTRACE: full
name: Run snippets and cpython tests
runs-on: ${{ matrix.os }}
strategy:
@@ -246,186 +43,119 @@ jobs:
os: [macos-latest, ubuntu-latest, windows-latest]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- uses: actions/checkout@master
- name: Set up the Windows environment
shell: bash
run: |
cargo install cargo-vcpkg
cargo vcpkg build
powershell.exe scripts/symlinks-to-hardlinks.ps1
if: runner.os == 'Windows'
- name: Set up the Mac environment
run: brew install autoconf automake libtool openssl@3
if: runner.os == 'macOS'
- name: build rustpython
run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }}
if: runner.os == 'macOS'
- name: build rustpython
run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit
if: runner.os != 'macOS'
- uses: actions/setup-python@v5
- name: Cache cargo dependencies
uses: actions/cache@v1
with:
python-version: ${{ env.PYTHON_VERSION }}
key: ${{ runner.os }}-snippets-${{ hashFiles('Cargo.lock') }}
path: target
restore-keys: |
${{ runner.os }}-snippets-
- name: build rustpython
uses: actions-rs/cargo@v1
with:
command: build
args: --release --verbose ${{ env.CARGO_ARGS }}
- uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Install pipenv
run: |
python -V
python -m pip install --upgrade pip
python -m pip install pipenv
- run: pipenv install
working-directory: ./tests
- name: run snippets
run: python -m pip install -r requirements.txt && pytest -v
working-directory: ./extra_tests
- if: runner.os == 'Linux'
name: run cpython platform-independent tests
run:
target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }}
- if: runner.os == 'Linux'
name: run cpython platform-dependent tests (Linux)
run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }}
- if: runner.os == 'macOS'
name: run cpython platform-dependent tests (MacOS)
run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.MACOS_SKIPS }}
- if: runner.os == 'Windows'
name: run cpython platform-dependent tests (windows partial - fixme)
run:
target/release/rustpython -m test -j 1 --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }}
- if: runner.os != 'Windows'
name: check that --install-pip succeeds
run: |
mkdir site-packages
target/release/rustpython --install-pip ensurepip --user
target/release/rustpython -m pip install six
- if: runner.os != 'Windows'
name: Check that ensurepip succeeds.
run: |
target/release/rustpython -m ensurepip
target/release/rustpython -c "import pip"
- if: runner.os != 'Windows'
name: Check if pip inside venv is functional
run: |
target/release/rustpython -m venv testvenv
testvenv/bin/rustpython -m pip install wheel
- name: Check whats_left is not broken
run: python -I whats_left.py
run: pipenv run pytest -v
working-directory: ./tests
- name: run cpython tests
run: target/release/rustpython -m test -v
env:
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
if: runner.os != 'Windows'
lint:
format:
name: Check Rust code with rustfmt and clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: actions/checkout@master
- uses: actions-rs/toolchain@v1
with:
components: rustfmt, clippy
profile: minimal
toolchain: stable
components: rustfmt
override: true
- name: run rustfmt
run: cargo fmt --check
- name: run clippy on wasm
run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
- uses: actions/setup-python@v5
uses: actions-rs/cargo@v1
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: install ruff
run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778
- name: run python lint
run: ruff extra_tests wasm examples --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source
- name: install prettier
run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH
- name: check wasm code with prettier
# prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506
run: cd wasm && git ls-files -z | xargs -0 prettier --check -u
command: fmt
args: --all -- --check
- name: run clippy
uses: actions-rs/cargo@v1
with:
command: clippy
args: ${{ env.CARGO_ARGS }} -- -Dwarnings
miri:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
name: Run tests under miri
lint:
name: Lint Python code with flake8
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@master
- uses: actions/checkout@master
- uses: actions/setup-python@v1
with:
toolchain: nightly
components: miri
- uses: Swatinem/rust-cache@v2
- name: Run tests under miri
# miri-ignore-leaks because the type-object circular reference means that there will always be
# a memory leak, at least until we have proper cyclic gc
run: MIRIFLAGS='-Zmiri-ignore-leaks' cargo +nightly miri test -p rustpython-vm -- miri_test
python-version: 3.8
- name: install flake8
run: python -m pip install flake8
- name: run lint
run: flake8 . --count --exclude=./.*,./Lib,./vm/Lib --select=E9,F63,F7,F82 --show-source --statistics
wasm:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
name: Check the WASM package and demo
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- uses: actions/checkout@master
- name: Cache cargo dependencies
uses: actions/cache@v1
with:
key: ${{ runner.os }}-wasm-${{ hashFiles('**/Cargo.lock') }}
path: target
restore-keys: |
${{ runner.os }}-wasm-
- name: install wasm-pack
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
- name: install geckodriver
run: |
wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz
wget https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckodriver-v0.24.0-linux32.tar.gz
mkdir geckodriver
tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver
- uses: actions/setup-python@v5
tar -xzf geckodriver-v0.24.0-linux32.tar.gz -C geckodriver
- uses: actions/setup-python@v1
with:
python-version: ${{ env.PYTHON_VERSION }}
- run: python -m pip install -r requirements.txt
python-version: 3.8
- name: Install pipenv
run: |
python -V
python -m pip install --upgrade pip
python -m pip install pipenv
- run: pipenv install
working-directory: ./wasm/tests
- uses: actions/setup-node@v4
with:
cache: "npm"
cache-dependency-path: "wasm/demo/package-lock.json"
- uses: actions/setup-node@v1
- name: run test
run: |
export PATH=$PATH:`pwd`/../../geckodriver
npm install
npm run test
env:
NODE_OPTIONS: "--openssl-legacy-provider"
working-directory: ./wasm/demo
- uses: mwilliamson/setup-wabt-action@v3
with: { wabt-version: "1.0.36" }
- name: check wasm32-unknown without js
run: |
cd wasm/wasm-unknown-test
cargo build --release --verbose
if wasm-objdump -xj Import target/wasm32-unknown-unknown/release/wasm_unknown_test.wasm; then
echo "ERROR: wasm32-unknown module expects imports from the host environment" >2
fi
- name: build notebook demo
if: github.ref == 'refs/heads/release'
run: |
npm install
npm run dist
mv dist ../demo/dist/notebook
env:
NODE_OPTIONS: "--openssl-legacy-provider"
working-directory: ./wasm/notebook
- name: Deploy demo to Github Pages
if: success() && github.ref == 'refs/heads/release'
uses: peaceiris/actions-gh-pages@v4
uses: peaceiris/actions-gh-pages@v2
env:
ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
PUBLISH_DIR: ./wasm/demo/dist
EXTERNAL_REPOSITORY: RustPython/demo
PUBLISH_BRANCH: master
wasm-wasi:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
name: Run snippets and cpython tests on wasm-wasi
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
target: wasm32-wasip1
- uses: Swatinem/rust-cache@v2
- name: Setup Wasmer
uses: wasmerio/setup-wasmer@v3
- name: Install clang
run: sudo apt-get update && sudo apt-get install clang -y
- name: build rustpython
run: cargo build --release --target wasm32-wasip1 --features freeze-stdlib,stdlib --verbose
- name: run snippets
run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py
- name: run cpython unittest
run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py

178
.github/workflows/cron-ci.yaml vendored
View File

@@ -1,56 +1,74 @@
on:
schedule:
- cron: '0 0 * * 6'
workflow_dispatch:
push:
paths:
- .github/workflows/cron-ci.yaml
name: Periodic checks/tasks
env:
CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl,jit
PYTHON_VERSION: "3.13.1"
jobs:
# codecov collects code coverage data from the rust tests, python snippets and python test suite.
# This is done using cargo-llvm-cov, which is a wrapper around llvm-cov.
redox:
name: Check compilation on Redox
runs-on: ubuntu-latest
container:
image: redoxos/redoxer:latest
steps:
- uses: actions/checkout@master
- name: prepare repository for redoxer compilation
run: bash scripts/redox/uncomment-cargo.sh
- name: compile for redox
run: redoxer build --verbose
codecov:
name: Collect code coverage data
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: taiki-e/install-action@cargo-llvm-cov
- uses: actions/setup-python@v5
- uses: actions/checkout@master
- uses: actions-rs/toolchain@v1
with:
python-version: ${{ env.PYTHON_VERSION }}
- run: sudo apt-get update && sudo apt-get -y install lcov
- name: Run cargo-llvm-cov with Rust tests.
run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,importlib,encodings,ssl,jit
- name: Run cargo-llvm-cov with Python snippets.
run: python scripts/cargo-llvm-cov.py
continue-on-error: true
- name: Run cargo-llvm-cov with Python test suite.
run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed
continue-on-error: true
- name: Prepare code coverage data
run: cargo llvm-cov report --lcov --output-path='codecov.lcov'
- name: Upload to Codecov
uses: codecov/codecov-action@v5
toolchain: nightly
override: true
- uses: actions-rs/cargo@v1
with:
file: ./codecov.lcov
command: build
args: --verbose
env:
CARGO_INCREMENTAL: '0'
RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests' # -Cpanic=abort
- uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Install pipenv
run: |
python -V
python -m pip install --upgrade pip
python -m pip install pipenv
- run: pipenv install
working-directory: ./tests
- name: run snippets
run: pipenv run pytest -v
working-directory: ./tests
env:
RUSTPYTHON_DEBUG: 'true'
- name: run cpython tests
run: cargo run -- -m test -v
env:
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
- uses: actions-rs/grcov@v0.1
id: coverage
- name: upload to Codecov
uses: codecov/codecov-action@v1
with:
file: ${{ steps.coverage.outputs.report }}
testdata:
name: Collect regression test data
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: actions/checkout@master
- name: build rustpython
run: cargo build --release --verbose
uses: actions-rs/cargo@v1
with:
command: build
args: --release --verbose --all
- name: collect tests data
run: cargo run --release extra_tests/jsontests.py
run: cargo run --release tests/jsontests.py
env:
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
- name: upload tests data to the website
@@ -64,91 +82,7 @@ jobs:
git clone git@github.com:RustPython/rustpython.github.io.git website
cd website
cp ../extra_tests/cpython_tests_results.json ./_data/regrtests_results.json
cp ../tests/cpython_tests_results.json ./_data/regrtests_results.json
git add ./_data/regrtests_results.json
if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results" --author="$GITHUB_ACTOR"; then
git push
fi
whatsleft:
name: Collect what is left data
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: build rustpython
run: cargo build --release --verbose
- name: Collect what is left data
run: |
chmod +x ./whats_left.py
./whats_left.py > whats_left.temp
env:
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
- name: Upload data to the website
env:
SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
GITHUB_ACTOR: ${{ github.actor }}
run: |
echo "$SSHKEY" >~/github_key
chmod 600 ~/github_key
export GIT_SSH_COMMAND="ssh -i ~/github_key"
git clone git@github.com:RustPython/rustpython.github.io.git website
cd website
[ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp
cp ../whats_left.temp ./_data/whats_left.temp
rm ./_data/whats_left/modules.csv
echo -e "modules" > ./_data/whats_left/modules.csv
cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv
git add -A
if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then
git push
fi
benchmark:
name: Collect benchmark data
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: actions/setup-python@v5
with:
python-version: 3.9
- run: cargo install cargo-criterion
- name: build benchmarks
run: cargo build --release --benches
- name: collect execution benchmark data
run: cargo criterion --bench execution
- name: collect microbenchmarks data
run: cargo criterion --bench microbenchmarks
- name: restructure generated files
run: |
cd ./target/criterion/reports
find -type d -name cpython | xargs rm -rf
find -type d -name rustpython | xargs rm -rf
find -mindepth 2 -maxdepth 2 -name violin.svg | xargs rm -rf
find -type f -not -name violin.svg | xargs rm -rf
for file in $(find -type f -name violin.svg); do mv $file $(echo $file | sed -E "s_\./([^/]+)/([^/]+)/violin\.svg_./\1/\2.svg_"); done
find -mindepth 2 -maxdepth 2 -type d | xargs rm -rf
cd ..
mv reports/* .
rmdir reports
- name: upload benchmark data to the website
env:
SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
run: |
echo "$SSHKEY" >~/github_key
chmod 600 ~/github_key
export GIT_SSH_COMMAND="ssh -i ~/github_key"
git clone git@github.com:RustPython/rustpython.github.io.git website
cd website
rm -rf ./assets/criterion
cp -r ../target/criterion ./assets/criterion
git add ./assets/criterion
if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then
git push
fi
git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results" --author="$GITHUB_ACTOR"
git push

171
.github/workflows/release.yml vendored
View File

@@ -1,171 +0,0 @@
name: Release
on:
schedule:
# 9 AM UTC on every Monday
- cron: "0 9 * * Mon"
workflow_dispatch:
inputs:
pre-release:
type: boolean
description: Mark "Pre-Release"
required: false
default: true
permissions:
contents: write
env:
CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl
jobs:
build:
runs-on: ${{ matrix.platform.runner }}
strategy:
matrix:
platform:
- runner: ubuntu-latest
target: x86_64-unknown-linux-gnu
# - runner: ubuntu-latest
# target: i686-unknown-linux-gnu
# - runner: ubuntu-latest
# target: aarch64-unknown-linux-gnu
# - runner: ubuntu-latest
# target: armv7-unknown-linux-gnueabi
# - runner: ubuntu-latest
# target: s390x-unknown-linux-gnu
# - runner: ubuntu-latest
# target: powerpc64le-unknown-linux-gnu
- runner: macos-latest
target: aarch64-apple-darwin
# - runner: macos-latest
# target: x86_64-apple-darwin
- runner: windows-latest
target: x86_64-pc-windows-msvc
# - runner: windows-latest
# target: i686-pc-windows-msvc
# - runner: windows-latest
# target: aarch64-pc-windows-msvc
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- name: Set up Environment
shell: bash
run: rustup target add ${{ matrix.platform.target }}
- name: Set up Windows Environment
shell: bash
run: |
cargo install --target-dir=target -v cargo-vcpkg
cargo vcpkg -v build
if: runner.os == 'Windows'
- name: Set up MacOS Environment
run: brew install autoconf automake libtool
if: runner.os == 'macOS'
- name: Build RustPython
run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }}
if: runner.os == 'macOS'
- name: Build RustPython
run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }},jit
if: runner.os != 'macOS'
- name: Rename Binary
run: cp target/${{ matrix.platform.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}
if: runner.os != 'Windows'
- name: Rename Binary
run: cp target/${{ matrix.platform.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}.exe
if: runner.os == 'Windows'
- name: Upload Binary Artifacts
uses: actions/upload-artifact@v4
with:
name: rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}
path: target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}*
build-wasm:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
targets: wasm32-wasip1
- name: Build RustPython
run: cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release
- name: Rename Binary
run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm
- name: Upload Binary Artifacts
uses: actions/upload-artifact@v4
with:
name: rustpython-release-wasm32-wasip1
path: target/rustpython-release-wasm32-wasip1.wasm
- name: install wasm-pack
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
- uses: actions/setup-node@v4
- uses: mwilliamson/setup-wabt-action@v3
with: { wabt-version: "1.0.30" }
- name: build demo
run: |
npm install
npm run dist
env:
NODE_OPTIONS: "--openssl-legacy-provider"
working-directory: ./wasm/demo
- name: build notebook demo
run: |
npm install
npm run dist
mv dist ../demo/dist/notebook
env:
NODE_OPTIONS: "--openssl-legacy-provider"
working-directory: ./wasm/notebook
- name: Deploy demo to Github Pages
uses: peaceiris/actions-gh-pages@v4
with:
deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
publish_dir: ./wasm/demo/dist
external_repository: RustPython/demo
publish_branch: master
release:
runs-on: ubuntu-latest
needs: [build, build-wasm]
steps:
- name: Download Binary Artifacts
uses: actions/download-artifact@v4
with:
path: bin
pattern: rustpython-release-*
merge-multiple: true
- name: List Binaries
run: |
ls -lah bin/
file bin/*
- name: Create Release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
tag: ${{ github.ref_name }}
run: ${{ github.run_number }}
run: |
if [[ "${{ github.event.inputs.pre-release }}" == "false" ]]; then
RELEASE_TYPE_NAME=Release
PRERELEASE_ARG=
else
RELEASE_TYPE_NAME=Pre-Release
PRERELEASE_ARG=--prerelease
fi
today=$(date '+%Y-%m-%d')
gh release create "$today-$tag-$run" \
--repo="$GITHUB_REPOSITORY" \
--title="RustPython $RELEASE_TYPE_NAME $today-$tag #$run" \
--target="$tag" \
--generate-notes \
$PRERELEASE_ARG \
bin/rustpython-release-*

7
.gitignore vendored
View File

@@ -9,15 +9,10 @@ __pycache__
.vscode
wasm-pack.log
.idea/
.envrc
.python-version
tests/snippets/resources
flame-graph.html
flame.txt
flamescope.json
/wapm.lock
/wapm_packages
/.cargo/config
extra_tests/snippets/resources
extra_tests/not_impl.py

7
.gitpod.Dockerfile vendored
View File

@@ -11,11 +11,4 @@ RUN rm -rf ~/.rustup && \
curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && \
rustup target add wasm32-unknown-unknown
RUN sudo apt-get -q update \
&& sudo apt-get install -yq \
libpython3.6 \
rust-lldb \
&& sudo rm -rf /var/lib/apt/lists/*
ENV RUST_LLDB=/usr/bin/lldb-8
USER root

4
.gitpod.yml
View File

@@ -1,6 +1,2 @@
image:
file: .gitpod.Dockerfile
vscode:
extensions:
- vadimcn.vscode-lldb@1.5.3:vTh/rWhvJ5nQpeAVsD20QA==

8
.mailmap
View File

@@ -1,8 +0,0 @@
#
# This list is used by git-shortlog to aggregate contributions. It is
# necessary when either the author's full name is not always written
# the same way, and/or the same author contributes from different
# email addresses.
#
Noa <coolreader18@gmail.com> <33094578+coolreader18@users.noreply.github.com>

16
.theia/launch.json
View File

@@ -1,16 +0,0 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
"version": "0.2.0",
"configurations": [
{
"type": "lldb",
"request": "launch",
"name": "Debug Rust Code",
//"preLaunchTask": "cargo",
"program": "${workspaceFolder}/target/debug/rustpython",
"cwd": "${workspaceFolder}",
//"valuesFormatting": "parseText"
}
]
}

8
.theia/settings.json
View File

@@ -1,8 +0,0 @@
{
"cpp.buildConfigurations": [
{
"name": "",
"directory": ""
},
]
}

298
.vscode/launch.json vendored
View File

@@ -1,298 +0,0 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "lldb",
"request": "launch",
"name": "Debug executable 'rustpython'",
"preLaunchTask": "Build RustPython Debug",
"program": "target/debug/rustpython",
"args": [],
"env": {
"RUST_BACKTRACE": "1"
},
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug executable 'rustpython' without SSL",
"preLaunchTask": "Build RustPython Debug without SSL",
"program": "target/debug/rustpython",
"args": [],
"env": {
"RUST_BACKTRACE": "1"
},
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython"
],
"filter": {
"name": "rustpython",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug benchmark 'execution'",
"cargo": {
"args": [
"test",
"--no-run",
"--bench=execution",
"--package=rustpython"
],
"filter": {
"name": "execution",
"kind": "bench"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug benchmark 'microbenchmarks'",
"cargo": {
"args": [
"test",
"--no-run",
"--bench=microbenchmarks",
"--package=rustpython"
],
"filter": {
"name": "microbenchmarks",
"kind": "bench"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython-pylib'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython-pylib"
],
"filter": {
"name": "rustpython-pylib",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython-bytecode'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython-bytecode"
],
"filter": {
"name": "rustpython-bytecode",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython-compiler'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython-compiler"
],
"filter": {
"name": "rustpython-compiler",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython-compiler-core'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython-compiler-core"
],
"filter": {
"name": "rustpython-compiler-core",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython-ast'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython-ast"
],
"filter": {
"name": "rustpython-ast",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython-parser'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython-parser"
],
"filter": {
"name": "rustpython-parser",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython-vm'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython-vm"
],
"filter": {
"name": "rustpython-vm",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython-common'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython-common"
],
"filter": {
"name": "rustpython-common",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython-jit'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython-jit"
],
"filter": {
"name": "rustpython-jit",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug integration test 'integration'",
"cargo": {
"args": [
"test",
"--no-run",
"--test=integration",
"--package=rustpython-jit"
],
"filter": {
"name": "integration",
"kind": "test"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'rustpython_wasm'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=rustpython_wasm"
],
"filter": {
"name": "rustpython_wasm",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
}
]
}

36
.vscode/tasks.json vendored
View File

@@ -1,36 +0,0 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "Build RustPython Debug without SSL",
"type": "shell",
"command": "cargo",
"args": [
"build",
],
"problemMatcher": [
"$rustc",
],
"group": {
"kind": "build",
"isDefault": true,
},
},
{
"label": "Build RustPython Debug",
"type": "shell",
"command": "cargo",
"args": [
"build",
"--features=ssl"
],
"problemMatcher": [
"$rustc",
],
"group": {
"kind": "build",
"isDefault": true,
},
},
],
}

3546
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

215
Cargo.toml
View File

@@ -1,199 +1,54 @@
[package]
name = "rustpython"
version = "0.1.2"
authors = ["RustPython Team"]
edition = "2018"
description = "A python interpreter written in rust."
include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
version.workspace = true
authors.workspace = true
edition.workspace = true
rust-version.workspace = true
repository.workspace = true
license.workspace = true
repository = "https://github.com/RustPython/RustPython"
license = "MIT"
[workspace]
members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "examples/freeze"]
[[bench]]
name = "bench"
path = "./benchmarks/bench.rs"
[features]
default = ["threading", "stdlib", "importlib"]
importlib = ["rustpython-vm/importlib"]
encodings = ["rustpython-vm/encodings"]
stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"]
flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
jit = ["rustpython-vm/jit"]
threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
bz2 = ["stdlib", "rustpython-stdlib/bz2"]
sqlite = ["rustpython-stdlib/sqlite"]
ssl = ["rustpython-stdlib/ssl"]
ssl-vendor = ["ssl", "rustpython-stdlib/ssl-vendor"]
freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
ssl = ["rustpython-vm/ssl"]
[dependencies]
rustpython-compiler = { workspace = true }
rustpython-pylib = { workspace = true, optional = true }
rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
rustpython-vm = { workspace = true, features = ["compiler"] }
rustpython-parser = { workspace = true }
log = "0.4"
env_logger = "0.7"
clap = "2.33"
rustpython-compiler = {path = "compiler", version = "0.1.1"}
rustpython-parser = {path = "parser", version = "0.1.1"}
rustpython-vm = {path = "vm", version = "0.1.1"}
dirs = { package = "dirs-next", version = "1.0" }
num-traits = "0.2.8"
cfg-if = "0.1"
cfg-if = { workspace = true }
log = { workspace = true }
flame = { workspace = true, optional = true }
flame = { version = "0.2", optional = true }
flamescope = { version = "0.1", optional = true }
clap = "2.34"
dirs = { package = "dirs-next", version = "2.0.0" }
env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
flamescope = { version = "0.1.2", optional = true }
[target.'cfg(not(target_os = "wasi"))'.dependencies]
rustyline = "6.0"
[target.'cfg(windows)'.dependencies]
libc = { workspace = true }
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
rustyline = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
pyo3 = { version = "0.22", features = ["auto-initialize"] }
[[bench]]
name = "execution"
harness = false
[[bench]]
name = "microbenchmarks"
harness = false
[dev-dependencies.cpython]
version = "0.2"
[[bin]]
name = "rustpython"
path = "src/main.rs"
[profile.dev.package."*"]
opt-level = 3
[profile.test]
opt-level = 3
# https://github.com/rust-lang/rust/issues/92869
# lto = "thin"
[profile.bench]
lto = "thin"
codegen-units = 1
opt-level = 3
[profile.release]
lto = "thin"
[patch.crates-io]
# REDOX START, Uncomment when you want to compile/check with redoxer
# REDOX START, Uncommment when you want to compile/check with redoxer
# # following patches are just waiting on a new version to be released to crates.io
# nix = { git = "https://github.com/nix-rust/nix" }
# crossbeam-utils = { git = "https://github.com/crossbeam-rs/crossbeam" }
# socket2 = { git = "https://github.com/alexcrichton/socket2-rs" }
# REDOX END
# Used only on Windows to build the vcpkg dependencies
[package.metadata.vcpkg]
git = "https://github.com/microsoft/vcpkg"
# The revision of the vcpkg repository to use
# https://github.com/microsoft/vcpkg/tags
rev = "2024.02.14"
[package.metadata.vcpkg.target]
x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] }
[workspace]
resolver = "2"
members = [
"compiler", "compiler/core", "compiler/codegen",
".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl",
"wasm/lib",
]
[workspace.package]
version = "0.4.0"
authors = ["RustPython Team"]
edition = "2024"
rust-version = "1.85.0"
repository = "https://github.com/RustPython/RustPython"
license = "MIT"
[workspace.dependencies]
rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
rustpython-compiler = { path = "compiler", version = "0.4.0" }
rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }
rustpython-common = { path = "common", version = "0.4.0" }
rustpython-derive = { path = "derive", version = "0.4.0" }
rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" }
rustpython-jit = { path = "jit", version = "0.4.0" }
rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" }
rustpython-pylib = { path = "pylib", version = "0.4.0" }
rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" }
rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
# rustpython-literal = { version = "0.4.0" }
# rustpython-parser-core = { version = "0.4.0" }
# rustpython-parser = { version = "0.4.0" }
# rustpython-ast = { version = "0.4.0" }
# rustpython-format= { version = "0.4.0" }
rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
# rustpython-literal = { path = "../RustPython-parser/literal" }
# rustpython-parser-core = { path = "../RustPython-parser/core" }
# rustpython-parser = { path = "../RustPython-parser/parser" }
# rustpython-ast = { path = "../RustPython-parser/ast" }
# rustpython-format = { path = "../RustPython-parser/format" }
ahash = "0.8.11"
ascii = "1.1"
bitflags = "2.4.2"
bstr = "1"
cfg-if = "1.0"
chrono = "0.4.39"
criterion = { version = "0.3.5", features = ["html_reports"] }
crossbeam-utils = "0.8.21"
flame = "0.2.2"
getrandom = "0.3"
glob = "0.3"
hex = "0.4.3"
indexmap = { version = "2.2.6", features = ["std"] }
insta = "1.38.0"
itertools = "0.14.0"
is-macro = "0.3.7"
junction = "1.2.0"
libc = "0.2.169"
log = "0.4.25"
nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
malachite-bigint = "0.2.3"
malachite-q = "0.4.22"
malachite-base = "0.4.22"
memchr = "2.7.4"
num-complex = "0.4.6"
num-integer = "0.1.46"
num-traits = "0.2"
num_enum = { version = "0.7", default-features = false }
once_cell = "1.20.3"
parking_lot = "0.12.3"
paste = "1.0.15"
rand = "0.9"
rustix = { version = "0.38", features = ["event"] }
rustyline = "15.0.0"
serde = { version = "1.0.133", default-features = false }
schannel = "0.1.27"
static_assertions = "1.1"
strum = "0.27"
strum_macros = "0.27"
syn = "2"
thiserror = "2.0"
thread_local = "1.1.8"
unicode_names2 = "1.3.0"
widestring = "1.1.0"
windows-sys = "0.59.0"
wasm-bindgen = "0.2.100"
# Lints
[workspace.lints.rust]
unsafe_code = "allow"
unsafe_op_in_unsafe_fn = "deny"
elided_lifetimes_in_paths = "warn"
[workspace.lints.clippy]
perf = "warn"
style = "warn"
complexity = "warn"
suspicious = "warn"
correctness = "warn"

109
DEVELOPMENT.md
View File

@@ -19,21 +19,17 @@ The contents of the Development Guide include:
RustPython requires the following:
- Rust latest stable version (e.g 1.69.0 as of Apr 20 2023)
- Rust latest stable version (e.g 1.38.0 at Oct 1st 2019)
- To check Rust version: `rustc --version`
- If you have `rustup` on your system, enter to update to the latest
stable version: `rustup update stable`
- If you do not have Rust installed, use [rustup](https://rustup.rs/) to
do so.
- CPython version 3.13 or higher
- CPython version 3.7.4 or higher
- CPython can be installed by your operating system's package manager,
from the [Python website](https://www.python.org/downloads/), or
using a third-party distribution, such as
[Anaconda](https://www.anaconda.com/distribution/).
- [macOS] In case of libffi-sys compilation error, make sure autoconf, automake,
libtool are installed
- To install with [Homebrew](https://brew.sh), enter
`brew install autoconf automake libtool`
- [Optional] The Python package, `pytest`, is used for testing Python code
snippets. To install, enter `python3 -m pip install pytest`.
@@ -41,58 +37,27 @@ RustPython requires the following:
The Rust code style used is the default
[rustfmt](https://github.com/rust-lang/rustfmt) codestyle. Please format your
code accordingly, or run `cargo fmt` to autoformat it. We also use
[clippy](https://github.com/rust-lang/rust-clippy) to lint Rust code, which
you can check yourself with `cargo clippy`.
code accordingly. We also use [clippy](https://github.com/rust-lang/rust-clippy)
to detect rust code issues.
Custom Python code (i.e. code not copied from CPython's standard library) should
follow the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style. We also use
[ruff](https://beta.ruff.rs/docs/) to check Python code style.
In addition to language specific tools, [cspell](https://github.com/streetsidesoftware/cspell),
a code spell checker, is used in order to ensure correct spellings for code.
Python code should follow the
[PEP 8](https://www.python.org/dev/peps/pep-0008/) style. We also use
[flake8](http://flake8.pycqa.org/en/latest/) to check Python code style.
## Testing
To test RustPython's functionality, a collection of Python snippets is located
in the `extra_tests/snippets` directory and can be run using `pytest`:
in the `tests/snippets` directory and can be run using `pytest`:
```shell
$ cd extra_tests
$ cd tests
$ pytest -v
```
Rust unit tests can be run with `cargo`:
```shell
$ cargo test --workspace --exclude rustpython_wasm
```
Python unit tests can be run by compiling RustPython and running the test module:
```shell
$ cargo run --release -- -m test
```
There are a few test options that are especially useful:
- `-j <n>` enables parallel testing (which is a lot faster), where `<n>` is the
number of threads to be used, ideally the same as number of cores on your CPU.
If you don't know, `-j 4` or `-j 8` are good options.
- `-v` enables verbose mode, adding additional information about the tests being
run.
- `<test_name>` specifies a single test to run instead of running all tests.
For example, to run all tests in parallel:
```shell
$ cargo run --release -- -m test -j 4
```
To run only `test_cmath` (located at `Lib/test/test_cmath`) verbosely:
```shell
$ cargo run --release -- -m test test_cmath -v
$ cargo test --all
```
## Profiling
@@ -118,29 +83,34 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i
Understanding a new codebase takes time. Here's a brief view of the
repository's structure:
- `bytecode/src`: python bytecode representation in rust structures
- `compiler/src`: python compilation to bytecode
- `core/src`: python bytecode representation in rust structures
- `parser/src`: python lexing, parsing and ast
- `derive/src`: Rust language extensions and macros specific to rustpython
- `parser/src`: python lexing, parsing and ast
- `Lib`: Carefully selected / copied files from CPython sourcecode. This is
the python side of the standard library.
- `test`: CPython test suite
- `vm/src`: python virtual machine
- `builtins`: Builtin functions and types
- `builtins.rs`: Builtin functions
- `compile.rs`: the python compiler from ast to bytecode
- `obj`: python builtin types
- `stdlib`: Standard library parts implemented in rust.
- `src`: using the other subcrates to bring rustpython to life.
- `docs`: documentation (work in progress)
- `py_code_object`: CPython bytecode to rustpython bytecode converter (work in
progress)
- `wasm`: Binary crate and resources for WebAssembly build
- `extra_tests`: extra integration test snippets as a supplement to `Lib/test`
- `tests`: integration test snippets
## Understanding Internals
The RustPython workspace includes the `rustpython` top-level crate. The `Cargo.toml`
file in the root of the repo provide configuration of the crate and the
implementation is found in the `src` directory (specifically, `src/lib.rs`).
implementation is found in the `src` directory (specifically,
`src/main.rs`).
The top-level `rustpython` binary depends on several lower-level crates including:
- `rustpython-parser` (implementation in `compiler/parser/src`)
- `rustpython-parser` (implementation in `parser/src`)
- `rustpython-compiler` (implementation in `compiler/src`)
- `rustpython-vm` (implementation in `vm/src`)
@@ -158,26 +128,25 @@ enable a line of code to go through a series of steps:
This crate contains the lexer and parser to convert a line of code to
an Abstract Syntax Tree (AST):
- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens
- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses
- Lexer: `parser/lexer.rs` converts Python source code into tokens
- Parser: `parser/parser.rs` takes the tokens generated by the lexer and parses
the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax
tree are Rust structs and enums.
- The Parser relies on `LALRPOP`, a Rust parser generator framework. The
LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`.
- The Parser relies on `LALRPOP`, a Rust parser generator framework.
- More information on parsers and a tutorial can be found in the
[LALRPOP book](https://lalrpop.github.io/lalrpop/).
- AST: `compiler/ast/` implements in Rust the Python types and expressions
[LALRPOP book](https://lalrpop.github.io/lalrpop/README.html).
- AST: `parser/ast.rs` implements in Rust the Python types and expressions
represented by the AST nodes.
### rustpython-compiler
The `rustpython-compiler` crate's purpose is to transform the AST (Abstract Syntax
Tree) to bytecode. The implementation of the compiler is found in the
`compiler/src` directory. The compiler implements Python's symbol table,
ast->bytecode compiler, and bytecode optimizer in Rust.
`compiler/src` directory. The compiler implements Python's peephole optimizer
implementation, Symbol table, and streams in Rust.
Implementation of bytecode structure in Rust is found in the `compiler/core/src`
directory. `compiler/core/src/bytecode.rs` contains the representation of
Implementation of bytecode structure in Rust is found in the `bytecode/src`
directory. The `bytecode/src/bytecode.rs` contains the representation of
instructions and operations in Rust. Further information about Python's
bytecode instructions can be found in the
[Python documentation](https://docs.python.org/3/library/dis.html#bytecodes).
@@ -189,20 +158,10 @@ executes Python's instructions. The `vm/src` directory contains code to
implement the read and evaluation loop that fetches and dispatches
instructions. This directory also contains the implementation of the
Python Standard Library modules in Rust (`vm/src/stdlib`). In Python
everything can be represented as an object. The `vm/src/builtins` directory holds
the Rust code used to represent different Python objects and their methods. The
core implementation of what a Python object is can be found in
`vm/src/object/core.rs`.
### Code generation
There are some code generations involved in building RustPython:
- some part of the AST code is generated from `vm/src/stdlib/ast/gen.rs` to `compiler/ast/src/ast_gen.rs`.
- the `__doc__` attributes are generated by the
[__doc__](https://github.com/RustPython/__doc__) project which is then included as the `rustpython-doc` crate.
everything can be represented as an Object. `vm/src/obj` directory holds
the Rust code used to represent a Python Object and its methods.
## Questions
Have you tried these steps and have a question, please chat with us on
[Discord](https://discord.gg/vru8NypEhv).
[gitter](https://gitter.im/rustpython/Lobby).

2
LICENSE
View File

@@ -1,6 +1,6 @@
MIT License
Copyright (c) 2025 RustPython Team
Copyright (c) 2020 RustPython Team
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

49
Lib/__future__.py vendored
View File

@@ -33,7 +33,7 @@ in releases at or after that, modules no longer need
to use the feature in question, but may continue to use such imports.
MandatoryRelease may also be None, meaning that a planned feature got
dropped or that the release version is undetermined.
dropped.
Instances of class _Feature have two corresponding methods,
.getOptionalRelease() and .getMandatoryRelease().
@@ -42,7 +42,7 @@ CompilerFlag is the (bitfield) flag that should be passed in the fourth
argument to the builtin function compile() to enable the feature in
dynamically compiled code. This flag is stored in the .compiler_flag
attribute on _Future instances. These values must match the appropriate
#defines of CO_xxx flags in Include/cpython/compile.h.
#defines of CO_xxx flags in Include/compile.h.
No feature line is ever to be deleted from this file.
"""
@@ -57,29 +57,25 @@ all_feature_names = [
"unicode_literals",
"barry_as_FLUFL",
"generator_stop",
"annotations",
]
__all__ = ["all_feature_names"] + all_feature_names
# The CO_xxx symbols are defined here under the same names defined in
# code.h and used by compile.h, so that an editor search will find them here.
# However, they're not exported in __all__, because they don't really belong to
# The CO_xxx symbols are defined here under the same names used by
# compile.h, so that an editor search will find them here. However,
# they're not exported in __all__, because they don't really belong to
# this module.
CO_NESTED = 0x0010 # nested_scopes
CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000)
CO_FUTURE_DIVISION = 0x20000 # division
CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default
CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement
CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function
CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals
CO_FUTURE_BARRY_AS_BDFL = 0x400000
CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators
CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime
CO_NESTED = 0x0010 # nested_scopes
CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000)
CO_FUTURE_DIVISION = 0x2000 # division
CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement
CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function
CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
CO_FUTURE_BARRY_AS_BDFL = 0x40000
CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators
class _Feature:
def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
self.optional = optionalRelease
self.mandatory = mandatoryRelease
@@ -90,14 +86,16 @@ class _Feature:
This is a 5-tuple, of the same form as sys.version_info.
"""
return self.optional
def getMandatoryRelease(self):
"""Return release in which this feature will become mandatory.
This is a 5-tuple, of the same form as sys.version_info, or, if
the feature was dropped, or the release date is undetermined, is None.
the feature was dropped, is None.
"""
return self.mandatory
def __repr__(self):
@@ -105,7 +103,6 @@ class _Feature:
self.mandatory,
self.compiler_flag))
nested_scopes = _Feature((2, 1, 0, "beta", 1),
(2, 2, 0, "alpha", 0),
CO_NESTED)
@@ -135,13 +132,9 @@ unicode_literals = _Feature((2, 6, 0, "alpha", 2),
CO_FUTURE_UNICODE_LITERALS)
barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2),
(4, 0, 0, "alpha", 0),
CO_FUTURE_BARRY_AS_BDFL)
(3, 9, 0, "alpha", 0),
CO_FUTURE_BARRY_AS_BDFL)
generator_stop = _Feature((3, 5, 0, "beta", 1),
(3, 7, 0, "alpha", 0),
CO_FUTURE_GENERATOR_STOP)
annotations = _Feature((3, 7, 0, "beta", 1),
None,
CO_FUTURE_ANNOTATIONS)
(3, 7, 0, "alpha", 0),
CO_FUTURE_GENERATOR_STOP)

16
Lib/__hello__.py vendored
View File

@@ -1,16 +0,0 @@
initialized = True
class TestFrozenUtf8_1:
"""\u00b6"""
class TestFrozenUtf8_2:
"""\u03c0"""
class TestFrozenUtf8_4:
"""\U0001f600"""
def main():
print("Hello world!")
if __name__ == '__main__':
main()

7
Lib/__phello__/__init__.py
View File

@@ -1,7 +0,0 @@
initialized = True
def main():
print("Hello world!")
if __name__ == '__main__':
main()

0
Lib/__phello__/ham/__init__.py
View File

0
Lib/__phello__/ham/eggs.py
View File

7
Lib/__phello__/spam.py
View File

@@ -1,7 +0,0 @@
initialized = True
def main():
print("Hello world!")
if __name__ == '__main__':
main()

181
Lib/_android_support.py vendored
View File

@@ -1,181 +0,0 @@
import io
import sys
from threading import RLock
from time import sleep, time
# The maximum length of a log message in bytes, including the level marker and
# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at
# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71.
# Messages longer than this will be truncated by logcat. This limit has already
# been reduced at least once in the history of Android (from 4076 to 4068 between
# API level 23 and 26), so leave some headroom.
MAX_BYTES_PER_WRITE = 4000
# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this
# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE.
# However, if the actual number of bytes per character is smaller than that,
# then we may still join multiple consecutive text writes into binary
# writes containing a larger number of characters.
MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4
# When embedded in an app on current versions of Android, there's no easy way to
# monitor the C-level stdout and stderr. The testbed comes with a .c file to
# redirect them to the system log using a pipe, but that wouldn't be convenient
# or appropriate for all apps. So we redirect at the Python level instead.
def init_streams(android_log_write, stdout_prio, stderr_prio):
if sys.executable:
return # Not embedded in an app.
global logcat
logcat = Logcat(android_log_write)
sys.stdout = TextLogStream(
stdout_prio, "python.stdout", sys.stdout.fileno())
sys.stderr = TextLogStream(
stderr_prio, "python.stderr", sys.stderr.fileno())
class TextLogStream(io.TextIOWrapper):
def __init__(self, prio, tag, fileno=None, **kwargs):
# The default is surrogateescape for stdout and backslashreplace for
# stderr, but in the context of an Android log, readability is more
# important than reversibility.
kwargs.setdefault("encoding", "UTF-8")
kwargs.setdefault("errors", "backslashreplace")
super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs)
self._lock = RLock()
self._pending_bytes = []
self._pending_bytes_count = 0
def __repr__(self):
return f"<TextLogStream {self.buffer.tag!r}>"
def write(self, s):
if not isinstance(s, str):
raise TypeError(
f"write() argument must be str, not {type(s).__name__}")
# In case `s` is a str subclass that writes itself to stdout or stderr
# when we call its methods, convert it to an actual str.
s = str.__str__(s)
# We want to emit one log message per line wherever possible, so split
# the string into lines first. Note that "".splitlines() == [], so
# nothing will be logged for an empty string.
with self._lock:
for line in s.splitlines(keepends=True):
while line:
chunk = line[:MAX_CHARS_PER_WRITE]
line = line[MAX_CHARS_PER_WRITE:]
self._write_chunk(chunk)
return len(s)
# The size and behavior of TextIOWrapper's buffer is not part of its public
# API, so we handle buffering ourselves to avoid truncation.
def _write_chunk(self, s):
b = s.encode(self.encoding, self.errors)
if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE:
self.flush()
self._pending_bytes.append(b)
self._pending_bytes_count += len(b)
if (
self.write_through
or b.endswith(b"\n")
or self._pending_bytes_count > MAX_BYTES_PER_WRITE
):
self.flush()
def flush(self):
with self._lock:
self.buffer.write(b"".join(self._pending_bytes))
self._pending_bytes.clear()
self._pending_bytes_count = 0
# Since this is a line-based logging system, line buffering cannot be turned
# off, i.e. a newline always causes a flush.
@property
def line_buffering(self):
return True
class BinaryLogStream(io.RawIOBase):
def __init__(self, prio, tag, fileno=None):
self.prio = prio
self.tag = tag
self._fileno = fileno
def __repr__(self):
return f"<BinaryLogStream {self.tag!r}>"
def writable(self):
return True
def write(self, b):
if type(b) is not bytes:
try:
b = bytes(memoryview(b))
except TypeError:
raise TypeError(
f"write() argument must be bytes-like, not {type(b).__name__}"
) from None
# Writing an empty string to the stream should have no effect.
if b:
logcat.write(self.prio, self.tag, b)
return len(b)
# This is needed by the test suite --timeout option, which uses faulthandler.
def fileno(self):
if self._fileno is None:
raise io.UnsupportedOperation("fileno")
return self._fileno
# When a large volume of data is written to logcat at once, e.g. when a test
# module fails in --verbose3 mode, there's a risk of overflowing logcat's own
# buffer and losing messages. We avoid this by imposing a rate limit using the
# token bucket algorithm, based on a conservative estimate of how fast `adb
# logcat` can consume data.
MAX_BYTES_PER_SECOND = 1024 * 1024
# The logcat buffer size of a device can be determined by running `logcat -g`.
# We set the token bucket size to half of the buffer size of our current minimum
# API level, because other things on the system will be producing messages as
# well.
BUCKET_SIZE = 128 * 1024
# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39
PER_MESSAGE_OVERHEAD = 28
class Logcat:
def __init__(self, android_log_write):
self.android_log_write = android_log_write
self._lock = RLock()
self._bucket_level = 0
self._prev_write_time = time()
def write(self, prio, tag, message):
# Encode null bytes using "modified UTF-8" to avoid them truncating the
# message.
message = message.replace(b"\x00", b"\xc0\x80")
with self._lock:
now = time()
self._bucket_level += (
(now - self._prev_write_time) * MAX_BYTES_PER_SECOND)
# If the bucket level is still below zero, the clock must have gone
# backwards, so reset it to zero and continue.
self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE))
self._prev_write_time = now
self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message)
if self._bucket_level < 0:
sleep(-self._bucket_level / MAX_BYTES_PER_SECOND)
self.android_log_write(prio, tag, message)

66
Lib/_apple_support.py vendored
View File

@@ -1,66 +0,0 @@
import io
import sys
def init_streams(log_write, stdout_level, stderr_level):
# Redirect stdout and stderr to the Apple system log. This method is
# invoked by init_apple_streams() (initconfig.c) if config->use_system_logger
# is enabled.
sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors)
sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors)
class SystemLog(io.TextIOWrapper):
def __init__(self, log_write, level, **kwargs):
kwargs.setdefault("encoding", "UTF-8")
kwargs.setdefault("line_buffering", True)
super().__init__(LogStream(log_write, level), **kwargs)
def __repr__(self):
return f"<SystemLog (level {self.buffer.level})>"
def write(self, s):
if not isinstance(s, str):
raise TypeError(
f"write() argument must be str, not {type(s).__name__}")
# In case `s` is a str subclass that writes itself to stdout or stderr
# when we call its methods, convert it to an actual str.
s = str.__str__(s)
# We want to emit one log message per line, so split
# the string before sending it to the superclass.
for line in s.splitlines(keepends=True):
super().write(line)
return len(s)
class LogStream(io.RawIOBase):
def __init__(self, log_write, level):
self.log_write = log_write
self.level = level
def __repr__(self):
return f"<LogStream (level {self.level!r})>"
def writable(self):
return True
def write(self, b):
if type(b) is not bytes:
try:
b = bytes(memoryview(b))
except TypeError:
raise TypeError(
f"write() argument must be bytes-like, not {type(b).__name__}"
) from None
# Writing an empty string to the stream should have no effect.
if b:
# Encode null bytes using "modified UTF-8" to avoid truncating the
# message. This should not affect the return value, as the caller
# may be expecting it to match the length of the input.
self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80"))
return len(b)

806
Lib/_pycodecs.py → Lib/_codecs.py vendored
View File

File diff suppressed because it is too large Load Diff

264
Lib/_collections_abc.py vendored
View File

@@ -6,41 +6,9 @@
Unit tests are in test_collections.
"""
############ Maintenance notes #########################################
#
# ABCs are different from other standard library modules in that they
# specify compliance tests. In general, once an ABC has been published,
# new methods (either abstract or concrete) cannot be added.
#
# Though classes that inherit from an ABC would automatically receive a
# new mixin method, registered classes would become non-compliant and
# violate the contract promised by ``isinstance(someobj, SomeABC)``.
#
# Though irritating, the correct procedure for adding new abstract or
# mixin methods is to create a new ABC as a subclass of the previous
# ABC. For example, union(), intersection(), and difference() cannot
# be added to Set but could go into a new ABC that extends Set.
#
# Because they are so hard to change, new ABCs should have their APIs
# carefully thought through prior to publication.
#
# Since ABCMeta only checks for the presence of methods, it is possible
# to alter the signature of a method by adding optional arguments
# or changing parameters names. This is still a bit dubious but at
# least it won't cause isinstance() to return an incorrect result.
#
#
#######################################################################
from abc import ABCMeta, abstractmethod
import sys
GenericAlias = type(list[int])
EllipsisType = type(...)
def _f(): pass
FunctionType = type(_f)
del _f
__all__ = ["Awaitable", "Coroutine",
"AsyncIterable", "AsyncIterator", "AsyncGenerator",
"Hashable", "Iterable", "Iterator", "Generator", "Reversible",
@@ -49,7 +17,7 @@ __all__ = ["Awaitable", "Coroutine",
"Mapping", "MutableMapping",
"MappingView", "KeysView", "ItemsView", "ValuesView",
"Sequence", "MutableSequence",
"ByteString", "Buffer",
"ByteString",
]
# This module has been renamed from collections.abc to _collections_abc to
@@ -92,14 +60,15 @@ _coro = _coro()
coroutine = type(_coro)
_coro.close() # Prevent ResourceWarning
del _coro
## asynchronous generator ##
async def _ag(): yield
_ag = _ag()
async_generator = type(_ag)
del _ag
# XXX RustPython TODO: async generators
# ## asynchronous generator ##
# async def _ag(): yield
# _ag = _ag()
# async_generator = type(_ag)
# del _ag
### ONE-TRICK PONIES ###
# ## ONE-TRICK PONIES ###
def _check_methods(C, *methods):
mro = C.__mro__
@@ -142,8 +111,6 @@ class Awaitable(metaclass=ABCMeta):
return _check_methods(C, "__await__")
return NotImplemented
__class_getitem__ = classmethod(GenericAlias)
class Coroutine(Awaitable):
@@ -203,8 +170,6 @@ class AsyncIterable(metaclass=ABCMeta):
return _check_methods(C, "__aiter__")
return NotImplemented
__class_getitem__ = classmethod(GenericAlias)
class AsyncIterator(AsyncIterable):
@@ -273,7 +238,7 @@ class AsyncGenerator(AsyncIterator):
return NotImplemented
AsyncGenerator.register(async_generator)
# AsyncGenerator.register(async_generator)
class Iterable(metaclass=ABCMeta):
@@ -291,8 +256,6 @@ class Iterable(metaclass=ABCMeta):
return _check_methods(C, "__iter__")
return NotImplemented
__class_getitem__ = classmethod(GenericAlias)
class Iterator(Iterable):
@@ -312,10 +275,9 @@ class Iterator(Iterable):
return _check_methods(C, '__iter__', '__next__')
return NotImplemented
Iterator.register(bytes_iterator)
Iterator.register(bytearray_iterator)
#Iterator.register(callable_iterator)
# Iterator.register(callable_iterator)
Iterator.register(dict_keyiterator)
Iterator.register(dict_valueiterator)
Iterator.register(dict_itemiterator)
@@ -392,10 +354,8 @@ class Generator(Iterator):
'send', 'throw', 'close')
return NotImplemented
Generator.register(generator)
class Sized(metaclass=ABCMeta):
__slots__ = ()
@@ -425,9 +385,6 @@ class Container(metaclass=ABCMeta):
return _check_methods(C, "__contains__")
return NotImplemented
__class_getitem__ = classmethod(GenericAlias)
class Collection(Sized, Iterable, Container):
__slots__ = ()
@@ -438,106 +395,6 @@ class Collection(Sized, Iterable, Container):
return _check_methods(C, "__len__", "__iter__", "__contains__")
return NotImplemented
class Buffer(metaclass=ABCMeta):
__slots__ = ()
@abstractmethod
def __buffer__(self, flags: int, /) -> memoryview:
raise NotImplementedError
@classmethod
def __subclasshook__(cls, C):
if cls is Buffer:
return _check_methods(C, "__buffer__")
return NotImplemented
class _CallableGenericAlias(GenericAlias):
""" Represent `Callable[argtypes, resulttype]`.
This sets ``__args__`` to a tuple containing the flattened ``argtypes``
followed by ``resulttype``.
Example: ``Callable[[int, str], float]`` sets ``__args__`` to
``(int, str, float)``.
"""
__slots__ = ()
def __new__(cls, origin, args):
if not (isinstance(args, tuple) and len(args) == 2):
raise TypeError(
"Callable must be used as Callable[[arg, ...], result].")
t_args, t_result = args
if isinstance(t_args, (tuple, list)):
args = (*t_args, t_result)
elif not _is_param_expr(t_args):
raise TypeError(f"Expected a list of types, an ellipsis, "
f"ParamSpec, or Concatenate. Got {t_args}")
return super().__new__(cls, origin, args)
def __repr__(self):
if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]):
return super().__repr__()
return (f'collections.abc.Callable'
f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], '
f'{_type_repr(self.__args__[-1])}]')
def __reduce__(self):
args = self.__args__
if not (len(args) == 2 and _is_param_expr(args[0])):
args = list(args[:-1]), args[-1]
return _CallableGenericAlias, (Callable, args)
def __getitem__(self, item):
# Called during TypeVar substitution, returns the custom subclass
# rather than the default types.GenericAlias object. Most of the
# code is copied from typing's _GenericAlias and the builtin
# types.GenericAlias.
if not isinstance(item, tuple):
item = (item,)
new_args = super().__getitem__(item).__args__
# args[0] occurs due to things like Z[[int, str, bool]] from PEP 612
if not isinstance(new_args[0], (tuple, list)):
t_result = new_args[-1]
t_args = new_args[:-1]
new_args = (t_args, t_result)
return _CallableGenericAlias(Callable, tuple(new_args))
def _is_param_expr(obj):
"""Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or
``_ConcatenateGenericAlias`` from typing.py
"""
if obj is Ellipsis:
return True
if isinstance(obj, list):
return True
obj = type(obj)
names = ('ParamSpec', '_ConcatenateGenericAlias')
return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names)
def _type_repr(obj):
"""Return the repr() of an object, special-casing types (internal helper).
Copied from :mod:`typing` since collections.abc
shouldn't depend on that module.
(Keep this roughly in sync with the typing version.)
"""
if isinstance(obj, type):
if obj.__module__ == 'builtins':
return obj.__qualname__
return f'{obj.__module__}.{obj.__qualname__}'
if obj is Ellipsis:
return '...'
if isinstance(obj, FunctionType):
return obj.__name__
return repr(obj)
class Callable(metaclass=ABCMeta):
__slots__ = ()
@@ -552,13 +409,12 @@ class Callable(metaclass=ABCMeta):
return _check_methods(C, "__call__")
return NotImplemented
__class_getitem__ = classmethod(_CallableGenericAlias)
### SETS ###
class Set(Collection):
"""A set is a finite, iterable container.
This class provides concrete generic implementations of all
@@ -686,7 +542,6 @@ class Set(Collection):
hx = hash(x)
h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167
h &= MASK
h ^= (h >> 11) ^ (h >> 25)
h = h * 69069 + 907133923
h &= MASK
if h > MAX:
@@ -695,7 +550,6 @@ class Set(Collection):
h = 590923713
return h
Set.register(frozenset)
@@ -778,25 +632,24 @@ class MutableSet(Set):
self.discard(value)
return self
MutableSet.register(set)
### MAPPINGS ###
class Mapping(Collection):
__slots__ = ()
"""A Mapping is a generic container for associating key/value
pairs.
This class provides concrete generic implementations of all
methods except for __getitem__, __iter__, and __len__.
"""
__slots__ = ()
# Tell ABCMeta.__new__ that this class should have TPFLAGS_MAPPING set.
__abc_tpflags__ = 1 << 6 # Py_TPFLAGS_MAPPING
@abstractmethod
def __getitem__(self, key):
raise KeyError
@@ -851,15 +704,13 @@ class MappingView(Sized):
def __repr__(self):
return '{0.__class__.__name__}({0._mapping!r})'.format(self)
__class_getitem__ = classmethod(GenericAlias)
class KeysView(MappingView, Set):
__slots__ = ()
@classmethod
def _from_iterable(cls, it):
def _from_iterable(self, it):
return set(it)
def __contains__(self, key):
@@ -868,7 +719,6 @@ class KeysView(MappingView, Set):
def __iter__(self):
yield from self._mapping
KeysView.register(dict_keys)
@@ -877,7 +727,7 @@ class ItemsView(MappingView, Set):
__slots__ = ()
@classmethod
def _from_iterable(cls, it):
def _from_iterable(self, it):
return set(it)
def __contains__(self, item):
@@ -893,7 +743,6 @@ class ItemsView(MappingView, Set):
for key in self._mapping:
yield (key, self._mapping[key])
ItemsView.register(dict_items)
@@ -912,20 +761,21 @@ class ValuesView(MappingView, Collection):
for key in self._mapping:
yield self._mapping[key]
ValuesView.register(dict_values)
class MutableMapping(Mapping):
__slots__ = ()
"""A MutableMapping is a generic container for associating
key/value pairs.
This class provides concrete generic implementations of all
methods except for __getitem__, __setitem__, __delitem__,
__iter__, and __len__.
"""
__slots__ = ()
"""
@abstractmethod
def __setitem__(self, key, value):
@@ -971,21 +821,34 @@ class MutableMapping(Mapping):
except KeyError:
pass
def update(self, other=(), /, **kwds):
def update(*args, **kwds):
''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.
If E present and has a .keys() method, does: for k in E: D[k] = E[k]
If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v
In either case, this is followed by: for k, v in F.items(): D[k] = v
'''
if isinstance(other, Mapping):
for key in other:
self[key] = other[key]
elif hasattr(other, "keys"):
for key in other.keys():
self[key] = other[key]
else:
for key, value in other:
self[key] = value
if not args:
raise TypeError("descriptor 'update' of 'MutableMapping' object "
"needs an argument")
self, *args = args
if len(args) > 1:
raise TypeError('update expected at most 1 arguments, got %d' %
len(args))
if args:
other = args[0]
try:
mapping_inst = isinstance(other, Mapping)
except TypeError:
mapping_inst = False
if mapping_inst:
for key in other:
self[key] = other[key]
elif hasattr(other, "keys"):
for key in other.keys():
self[key] = other[key]
else:
for key, value in other:
self[key] = value
for key, value in kwds.items():
self[key] = value
@@ -997,13 +860,14 @@ class MutableMapping(Mapping):
self[key] = default
return default
MutableMapping.register(dict)
### SEQUENCES ###
class Sequence(Reversible, Collection):
"""All the operations on a read-only sequence.
Concrete subclasses must override __new__ or __init__,
@@ -1012,9 +876,6 @@ class Sequence(Reversible, Collection):
__slots__ = ()
# Tell ABCMeta.__new__ that this class should have TPFLAGS_SEQUENCE set.
__abc_tpflags__ = 1 << 5 # Py_TPFLAGS_SEQUENCE
@abstractmethod
def __getitem__(self, index):
raise IndexError
@@ -1055,10 +916,10 @@ class Sequence(Reversible, Collection):
while stop is None or i < stop:
try:
v = self[i]
if v is value or v == value:
return i
except IndexError:
break
if v is value or v == value:
return i
i += 1
raise ValueError
@@ -1071,27 +932,9 @@ Sequence.register(str)
Sequence.register(range)
Sequence.register(memoryview)
class _DeprecateByteStringMeta(ABCMeta):
def __new__(cls, name, bases, namespace, **kwargs):
if name != "ByteString":
import warnings
warnings._deprecated(
"collections.abc.ByteString",
remove=(3, 14),
)
return super().__new__(cls, name, bases, namespace, **kwargs)
class ByteString(Sequence):
def __instancecheck__(cls, instance):
import warnings
warnings._deprecated(
"collections.abc.ByteString",
remove=(3, 14),
)
return super().__instancecheck__(instance)
class ByteString(Sequence, metaclass=_DeprecateByteStringMeta):
"""This unifies bytes and bytearray.
XXX Should add all their methods.
@@ -1104,13 +947,15 @@ ByteString.register(bytearray)
class MutableSequence(Sequence):
__slots__ = ()
"""All the operations on a read-write sequence.
Concrete subclasses must provide __new__ or __init__,
__getitem__, __setitem__, __delitem__, __len__, and insert().
"""
__slots__ = ()
"""
@abstractmethod
def __setitem__(self, index, value):
@@ -1168,6 +1013,5 @@ class MutableSequence(Sequence):
self.extend(values)
return self
MutableSequence.register(list)
MutableSequence.register(bytearray) # Multiply inheriting, see ByteString

67
Lib/_colorize.py vendored
View File

@@ -1,67 +0,0 @@
import io
import os
import sys
COLORIZE = True
class ANSIColors:
BOLD_GREEN = "\x1b[1;32m"
BOLD_MAGENTA = "\x1b[1;35m"
BOLD_RED = "\x1b[1;31m"
GREEN = "\x1b[32m"
GREY = "\x1b[90m"
MAGENTA = "\x1b[35m"
RED = "\x1b[31m"
RESET = "\x1b[0m"
YELLOW = "\x1b[33m"
NoColors = ANSIColors()
for attr in dir(NoColors):
if not attr.startswith("__"):
setattr(NoColors, attr, "")
def get_colors(colorize: bool = False, *, file=None) -> ANSIColors:
if colorize or can_colorize(file=file):
return ANSIColors()
else:
return NoColors
def can_colorize(*, file=None) -> bool:
if file is None:
file = sys.stdout
if not sys.flags.ignore_environment:
if os.environ.get("PYTHON_COLORS") == "0":
return False
if os.environ.get("PYTHON_COLORS") == "1":
return True
if os.environ.get("NO_COLOR"):
return False
if not COLORIZE:
return False
if os.environ.get("FORCE_COLOR"):
return True
if os.environ.get("TERM") == "dumb":
return False
if not hasattr(file, "fileno"):
return False
if sys.platform == "win32":
try:
import nt
if not nt._supports_virtual_terminal():
return False
except (ImportError, AttributeError):
return False
try:
return os.isatty(file.fileno())
except io.UnsupportedOperation:
return file.isatty()

8
Lib/_compat_pickle.py vendored
View File

@@ -148,14 +148,6 @@ except NameError:
else:
PYTHON2_EXCEPTIONS += ("WindowsError",)
# NOTE: RUSTPYTHON exceptions
try:
JitError
except NameError:
pass
else:
PYTHON2_EXCEPTIONS += ("JitError",)
for excname in PYTHON2_EXCEPTIONS:
NAME_MAPPING[("exceptions", excname)] = ("builtins", excname)

162
Lib/_compression.py vendored
View File

@@ -1,162 +0,0 @@
"""Internal classes used by the gzip, lzma and bz2 modules"""
import io
import sys
BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size
class BaseStream(io.BufferedIOBase):
"""Mode-checking helper functions."""
def _check_not_closed(self):
if self.closed:
raise ValueError("I/O operation on closed file")
def _check_can_read(self):
if not self.readable():
raise io.UnsupportedOperation("File not open for reading")
def _check_can_write(self):
if not self.writable():
raise io.UnsupportedOperation("File not open for writing")
def _check_can_seek(self):
if not self.readable():
raise io.UnsupportedOperation("Seeking is only supported "
"on files open for reading")
if not self.seekable():
raise io.UnsupportedOperation("The underlying file object "
"does not support seeking")
class DecompressReader(io.RawIOBase):
"""Adapts the decompressor API to a RawIOBase reader API"""
def readable(self):
return True
def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
self._fp = fp
self._eof = False
self._pos = 0 # Current offset in decompressed stream
# Set to size of decompressed stream once it is known, for SEEK_END
self._size = -1
# Save the decompressor factory and arguments.
# If the file contains multiple compressed streams, each
# stream will need a separate decompressor object. A new decompressor
# object is also needed when implementing a backwards seek().
self._decomp_factory = decomp_factory
self._decomp_args = decomp_args
self._decompressor = self._decomp_factory(**self._decomp_args)
# Exception class to catch from decompressor signifying invalid
# trailing data to ignore
self._trailing_error = trailing_error
def close(self):
self._decompressor = None
return super().close()
def seekable(self):
return self._fp.seekable()
def readinto(self, b):
with memoryview(b) as view, view.cast("B") as byte_view:
data = self.read(len(byte_view))
byte_view[:len(data)] = data
return len(data)
def read(self, size=-1):
if size < 0:
return self.readall()
if not size or self._eof:
return b""
data = None # Default if EOF is encountered
# Depending on the input data, our call to the decompressor may not
# return any data. In this case, try again after reading another block.
while True:
if self._decompressor.eof:
rawblock = (self._decompressor.unused_data or
self._fp.read(BUFFER_SIZE))
if not rawblock:
break
# Continue to next stream.
self._decompressor = self._decomp_factory(
**self._decomp_args)
try:
data = self._decompressor.decompress(rawblock, size)
except self._trailing_error:
# Trailing data isn't a valid compressed stream; ignore it.
break
else:
if self._decompressor.needs_input:
rawblock = self._fp.read(BUFFER_SIZE)
if not rawblock:
raise EOFError("Compressed file ended before the "
"end-of-stream marker was reached")
else:
rawblock = b""
data = self._decompressor.decompress(rawblock, size)
if data:
break
if not data:
self._eof = True
self._size = self._pos
return b""
self._pos += len(data)
return data
def readall(self):
chunks = []
# sys.maxsize means the max length of output buffer is unlimited,
# so that the whole input buffer can be decompressed within one
# .decompress() call.
while data := self.read(sys.maxsize):
chunks.append(data)
return b"".join(chunks)
# Rewind the file to the beginning of the data stream.
def _rewind(self):
self._fp.seek(0)
self._eof = False
self._pos = 0
self._decompressor = self._decomp_factory(**self._decomp_args)
def seek(self, offset, whence=io.SEEK_SET):
# Recalculate offset as an absolute file position.
if whence == io.SEEK_SET:
pass
elif whence == io.SEEK_CUR:
offset = self._pos + offset
elif whence == io.SEEK_END:
# Seeking relative to EOF - we need to know the file's size.
if self._size < 0:
while self.read(io.DEFAULT_BUFFER_SIZE):
pass
offset = self._size + offset
else:
raise ValueError("Invalid value for whence: {}".format(whence))
# Make it so that offset is the number of bytes to skip forward.
if offset < self._pos:
self._rewind()
else:
offset -= self._pos
# Read and discard data until we reach the desired position.
while offset > 0:
data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset))
if not data:
break
offset -= len(data)
return self._pos
def tell(self):
"""Return the current file position."""
return self._pos

74
Lib/_dummy_os.py vendored
View File

@@ -1,74 +0,0 @@
"""
A shim of the os module containing only simple path-related utilities
"""
try:
from os import *
except ImportError:
import abc, sys
def __getattr__(name):
if name in {"_path_normpath", "__path__"}:
raise AttributeError(name)
if name.isupper():
return 0
def dummy(*args, **kwargs):
import io
return io.UnsupportedOperation(f"{name}: no os specific module found")
dummy.__name__ = f"dummy_{name}"
return dummy
sys.modules['os'] = sys.modules['posix'] = sys.modules[__name__]
import posixpath as path
sys.modules['os.path'] = path
del sys
sep = path.sep
supports_dir_fd = set()
supports_effective_ids = set()
supports_fd = set()
supports_follow_symlinks = set()
def fspath(path):
"""Return the path representation of a path-like object.
If str or bytes is passed in, it is returned unchanged. Otherwise the
os.PathLike interface is used to get the path representation. If the
path representation is not str or bytes, TypeError is raised. If the
provided path is not str, bytes, or os.PathLike, TypeError is raised.
"""
if isinstance(path, (str, bytes)):
return path
# Work from the object's type to match method resolution of other magic
# methods.
path_type = type(path)
try:
path_repr = path_type.__fspath__(path)
except AttributeError:
if hasattr(path_type, '__fspath__'):
raise
else:
raise TypeError("expected str, bytes or os.PathLike object, "
"not " + path_type.__name__)
if isinstance(path_repr, (str, bytes)):
return path_repr
else:
raise TypeError("expected {}.__fspath__() to return str or bytes, "
"not {}".format(path_type.__name__,
type(path_repr).__name__))
class PathLike(abc.ABC):
"""Abstract base class for implementing the file system path protocol."""
@abc.abstractmethod
def __fspath__(self):
"""Return the file system path representation of the object."""
raise NotImplementedError
@classmethod
def __subclasshook__(cls, subclass):
return hasattr(subclass, '__fspath__')

42
Lib/_dummy_thread.py vendored
View File

@@ -14,8 +14,7 @@ Suggested usage is::
# Exports only things specified by thread documentation;
# skipping obsolete synonyms allocate(), start_new(), exit_thread().
__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
'interrupt_main', 'LockType', 'RLock',
'_count']
'interrupt_main', 'LockType']
# A dummy value
TIMEOUT_MAX = 2**31
@@ -86,10 +85,6 @@ def _set_sentinel():
"""Dummy implementation of _thread._set_sentinel()."""
return LockType()
def _count():
"""Dummy implementation of _thread._count()."""
return 0
class LockType(object):
"""Class implementing dummy implementation of _thread.LockType.
@@ -145,9 +140,6 @@ class LockType(object):
def locked(self):
return self.locked_status
def _at_fork_reinit(self):
self.locked_status = False
def __repr__(self):
return "<%s %s.%s object at %s>" % (
"locked" if self.locked_status else "unlocked",
@@ -169,35 +161,3 @@ def interrupt_main():
else:
global _interrupt
_interrupt = True
class RLock:
def __init__(self):
self.locked_count = 0
def acquire(self, waitflag=None, timeout=-1):
self.locked_count += 1
return True
__enter__ = acquire
def __exit__(self, typ, val, tb):
self.release()
def release(self):
if not self.locked_count:
raise error
self.locked_count -= 1
return True
def locked(self):
return self.locked_status != 0
def __repr__(self):
return "<%s %s.%s object owner=%s count=%s at %s>" % (
"locked" if self.locked_count else "unlocked",
self.__class__.__module__,
self.__class__.__qualname__,
get_ident() if self.locked_count else 0,
self.locked_count,
hex(id(self))
)

71
Lib/_ios_support.py vendored
View File

@@ -1,71 +0,0 @@
import sys
try:
from ctypes import cdll, c_void_p, c_char_p, util
except ImportError:
# ctypes is an optional module. If it's not present, we're limited in what
# we can tell about the system, but we don't want to prevent the module
# from working.
print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr)
objc = None
else:
# ctypes is available. Load the ObjC library, and wrap the objc_getClass,
# sel_registerName methods
lib = util.find_library("objc")
if lib is None:
# Failed to load the objc library
raise ImportError("ObjC runtime library couldn't be loaded")
objc = cdll.LoadLibrary(lib)
objc.objc_getClass.restype = c_void_p
objc.objc_getClass.argtypes = [c_char_p]
objc.sel_registerName.restype = c_void_p
objc.sel_registerName.argtypes = [c_char_p]
def get_platform_ios():
# Determine if this is a simulator using the multiarch value
is_simulator = sys.implementation._multiarch.endswith("simulator")
# We can't use ctypes; abort
if not objc:
return None
# Most of the methods return ObjC objects
objc.objc_msgSend.restype = c_void_p
# All the methods used have no arguments.
objc.objc_msgSend.argtypes = [c_void_p, c_void_p]
# Equivalent of:
# device = [UIDevice currentDevice]
UIDevice = objc.objc_getClass(b"UIDevice")
SEL_currentDevice = objc.sel_registerName(b"currentDevice")
device = objc.objc_msgSend(UIDevice, SEL_currentDevice)
# Equivalent of:
# device_systemVersion = [device systemVersion]
SEL_systemVersion = objc.sel_registerName(b"systemVersion")
device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion)
# Equivalent of:
# device_systemName = [device systemName]
SEL_systemName = objc.sel_registerName(b"systemName")
device_systemName = objc.objc_msgSend(device, SEL_systemName)
# Equivalent of:
# device_model = [device model]
SEL_model = objc.sel_registerName(b"model")
device_model = objc.objc_msgSend(device, SEL_model)
# UTF8String returns a const char*;
SEL_UTF8String = objc.sel_registerName(b"UTF8String")
objc.objc_msgSend.restype = c_char_p
# Equivalent of:
# system = [device_systemName UTF8String]
# release = [device_systemVersion UTF8String]
# model = [device_model UTF8String]
system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode()
release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode()
model = objc.objc_msgSend(device_model, SEL_UTF8String).decode()
return system, release, model, is_simulator

35
Lib/_markupbase.py vendored
View File

@@ -29,6 +29,10 @@ class ParserBase:
raise RuntimeError(
"_markupbase.ParserBase must be subclassed")
def error(self, message):
raise NotImplementedError(
"subclasses of ParserBase must override error()")
def reset(self):
self.lineno = 1
self.offset = 0
@@ -127,11 +131,12 @@ class ParserBase:
# also in data attribute specifications of attlist declaration
# also link type declaration subsets in linktype declarations
# also link attribute specification lists in link declarations
raise AssertionError("unsupported '[' char in %s declaration" % decltype)
self.error("unsupported '[' char in %s declaration" % decltype)
else:
raise AssertionError("unexpected '[' char in declaration")
self.error("unexpected '[' char in declaration")
else:
raise AssertionError("unexpected %r char in declaration" % rawdata[j])
self.error(
"unexpected %r char in declaration" % rawdata[j])
if j < 0:
return j
return -1 # incomplete
@@ -151,9 +156,7 @@ class ParserBase:
# look for MS Office ]> ending
match= _msmarkedsectionclose.search(rawdata, i+3)
else:
raise AssertionError(
'unknown status keyword %r in marked section' % rawdata[i+3:j]
)
self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
if not match:
return -1
if report:
@@ -165,7 +168,7 @@ class ParserBase:
def parse_comment(self, i, report=1):
rawdata = self.rawdata
if rawdata[i:i+4] != '<!--':
raise AssertionError('unexpected call to parse_comment()')
self.error('unexpected call to parse_comment()')
match = _commentclose.search(rawdata, i+4)
if not match:
return -1
@@ -189,9 +192,7 @@ class ParserBase:
return -1
if s != "<!":
self.updatepos(declstartpos, j + 1)
raise AssertionError(
"unexpected char in internal subset (in %r)" % s
)
self.error("unexpected char in internal subset (in %r)" % s)
if (j + 2) == n:
# end of buffer; incomplete
return -1
@@ -208,9 +209,8 @@ class ParserBase:
return -1
if name not in {"attlist", "element", "entity", "notation"}:
self.updatepos(declstartpos, j + 2)
raise AssertionError(
"unknown declaration %r in internal subset" % name
)
self.error(
"unknown declaration %r in internal subset" % name)
# handle the individual names
meth = getattr(self, "_parse_doctype_" + name)
j = meth(j, declstartpos)
@@ -234,14 +234,14 @@ class ParserBase:
if rawdata[j] == ">":
return j
self.updatepos(declstartpos, j)
raise AssertionError("unexpected char after internal subset")
self.error("unexpected char after internal subset")
else:
return -1
elif c.isspace():
j = j + 1
else:
self.updatepos(declstartpos, j)
raise AssertionError("unexpected char %r in internal subset" % c)
self.error("unexpected char %r in internal subset" % c)
# end of buffer reached
return -1
@@ -387,9 +387,8 @@ class ParserBase:
return name.lower(), m.end()
else:
self.updatepos(declstartpos, i)
raise AssertionError(
"expected name token at %r" % rawdata[declstartpos:declstartpos+20]
)
self.error("expected name token at %r"
% rawdata[declstartpos:declstartpos+20])
# To be overridden -- handlers for unknown objects
def unknown_decl(self, data):

574
Lib/_osx_support.py vendored
View File

@@ -1,574 +0,0 @@
"""Shared OS X support functions."""
import os
import re
import sys
__all__ = [
'compiler_fixup',
'customize_config_vars',
'customize_compiler',
'get_platform_osx',
]
# configuration variables that may contain universal build flags,
# like "-arch" or "-isdkroot", that may need customization for
# the user environment
_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS',
'BLDSHARED', 'LDSHARED', 'CC', 'CXX',
'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS',
'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS')
# configuration variables that may contain compiler calls
_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX')
# prefix added to original configuration variable names
_INITPRE = '_OSX_SUPPORT_INITIAL_'
def _find_executable(executable, path=None):
"""Tries to find 'executable' in the directories listed in 'path'.
A string listing directories separated by 'os.pathsep'; defaults to
os.environ['PATH']. Returns the complete filename or None if not found.
"""
if path is None:
path = os.environ['PATH']
paths = path.split(os.pathsep)
base, ext = os.path.splitext(executable)
if (sys.platform == 'win32') and (ext != '.exe'):
executable = executable + '.exe'
if not os.path.isfile(executable):
for p in paths:
f = os.path.join(p, executable)
if os.path.isfile(f):
# the file exists, we have a shot at spawn working
return f
return None
else:
return executable
def _read_output(commandstring, capture_stderr=False):
"""Output from successful command execution or None"""
# Similar to os.popen(commandstring, "r").read(),
# but without actually using os.popen because that
# function is not usable during python bootstrap.
# tempfile is also not available then.
import contextlib
try:
import tempfile
fp = tempfile.NamedTemporaryFile()
except ImportError:
fp = open("/tmp/_osx_support.%s"%(
os.getpid(),), "w+b")
with contextlib.closing(fp) as fp:
if capture_stderr:
cmd = "%s >'%s' 2>&1" % (commandstring, fp.name)
else:
cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
return fp.read().decode('utf-8').strip() if not os.system(cmd) else None
def _find_build_tool(toolname):
"""Find a build tool on current path or using xcrun"""
return (_find_executable(toolname)
or _read_output("/usr/bin/xcrun -find %s" % (toolname,))
or ''
)
_SYSTEM_VERSION = None
def _get_system_version():
"""Return the OS X system version as a string"""
# Reading this plist is a documented way to get the system
# version (see the documentation for the Gestalt Manager)
# We avoid using platform.mac_ver to avoid possible bootstrap issues during
# the build of Python itself (distutils is used to build standard library
# extensions).
global _SYSTEM_VERSION
if _SYSTEM_VERSION is None:
_SYSTEM_VERSION = ''
try:
f = open('/System/Library/CoreServices/SystemVersion.plist', encoding="utf-8")
except OSError:
# We're on a plain darwin box, fall back to the default
# behaviour.
pass
else:
try:
m = re.search(r'<key>ProductUserVisibleVersion</key>\s*'
r'<string>(.*?)</string>', f.read())
finally:
f.close()
if m is not None:
_SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2])
# else: fall back to the default behaviour
return _SYSTEM_VERSION
_SYSTEM_VERSION_TUPLE = None
def _get_system_version_tuple():
"""
Return the macOS system version as a tuple
The return value is safe to use to compare
two version numbers.
"""
global _SYSTEM_VERSION_TUPLE
if _SYSTEM_VERSION_TUPLE is None:
osx_version = _get_system_version()
if osx_version:
try:
_SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.'))
except ValueError:
_SYSTEM_VERSION_TUPLE = ()
return _SYSTEM_VERSION_TUPLE
def _remove_original_values(_config_vars):
"""Remove original unmodified values for testing"""
# This is needed for higher-level cross-platform tests of get_platform.
for k in list(_config_vars):
if k.startswith(_INITPRE):
del _config_vars[k]
def _save_modified_value(_config_vars, cv, newvalue):
"""Save modified and original unmodified value of configuration var"""
oldvalue = _config_vars.get(cv, '')
if (oldvalue != newvalue) and (_INITPRE + cv not in _config_vars):
_config_vars[_INITPRE + cv] = oldvalue
_config_vars[cv] = newvalue
_cache_default_sysroot = None
def _default_sysroot(cc):
""" Returns the root of the default SDK for this system, or '/' """
global _cache_default_sysroot
if _cache_default_sysroot is not None:
return _cache_default_sysroot
contents = _read_output('%s -c -E -v - </dev/null' % (cc,), True)
in_incdirs = False
for line in contents.splitlines():
if line.startswith("#include <...>"):
in_incdirs = True
elif line.startswith("End of search list"):
in_incdirs = False
elif in_incdirs:
line = line.strip()
if line == '/usr/include':
_cache_default_sysroot = '/'
elif line.endswith(".sdk/usr/include"):
_cache_default_sysroot = line[:-12]
if _cache_default_sysroot is None:
_cache_default_sysroot = '/'
return _cache_default_sysroot
def _supports_universal_builds():
"""Returns True if universal builds are supported on this system"""
# As an approximation, we assume that if we are running on 10.4 or above,
# then we are running with an Xcode environment that supports universal
# builds, in particular -isysroot and -arch arguments to the compiler. This
# is in support of allowing 10.4 universal builds to run on 10.3.x systems.
osx_version = _get_system_version_tuple()
return bool(osx_version >= (10, 4)) if osx_version else False
def _supports_arm64_builds():
"""Returns True if arm64 builds are supported on this system"""
# There are two sets of systems supporting macOS/arm64 builds:
# 1. macOS 11 and later, unconditionally
# 2. macOS 10.15 with Xcode 12.2 or later
# For now the second category is ignored.
osx_version = _get_system_version_tuple()
return osx_version >= (11, 0) if osx_version else False
def _find_appropriate_compiler(_config_vars):
"""Find appropriate C compiler for extension module builds"""
# Issue #13590:
# The OSX location for the compiler varies between OSX
# (or rather Xcode) releases. With older releases (up-to 10.5)
# the compiler is in /usr/bin, with newer releases the compiler
# can only be found inside Xcode.app if the "Command Line Tools"
# are not installed.
#
# Furthermore, the compiler that can be used varies between
# Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2'
# as the compiler, after that 'clang' should be used because
# gcc-4.2 is either not present, or a copy of 'llvm-gcc' that
# miscompiles Python.
# skip checks if the compiler was overridden with a CC env variable
if 'CC' in os.environ:
return _config_vars
# The CC config var might contain additional arguments.
# Ignore them while searching.
cc = oldcc = _config_vars['CC'].split()[0]
if not _find_executable(cc):
# Compiler is not found on the shell search PATH.
# Now search for clang, first on PATH (if the Command LIne
# Tools have been installed in / or if the user has provided
# another location via CC). If not found, try using xcrun
# to find an uninstalled clang (within a selected Xcode).
# NOTE: Cannot use subprocess here because of bootstrap
# issues when building Python itself (and os.popen is
# implemented on top of subprocess and is therefore not
# usable as well)
cc = _find_build_tool('clang')
elif os.path.basename(cc).startswith('gcc'):
# Compiler is GCC, check if it is LLVM-GCC
data = _read_output("'%s' --version"
% (cc.replace("'", "'\"'\"'"),))
if data and 'llvm-gcc' in data:
# Found LLVM-GCC, fall back to clang
cc = _find_build_tool('clang')
if not cc:
raise SystemError(
"Cannot locate working compiler")
if cc != oldcc:
# Found a replacement compiler.
# Modify config vars using new compiler, if not already explicitly
# overridden by an env variable, preserving additional arguments.
for cv in _COMPILER_CONFIG_VARS:
if cv in _config_vars and cv not in os.environ:
cv_split = _config_vars[cv].split()
cv_split[0] = cc if cv != 'CXX' else cc + '++'
_save_modified_value(_config_vars, cv, ' '.join(cv_split))
return _config_vars
def _remove_universal_flags(_config_vars):
"""Remove all universal build arguments from config vars"""
for cv in _UNIVERSAL_CONFIG_VARS:
# Do not alter a config var explicitly overridden by env var
if cv in _config_vars and cv not in os.environ:
flags = _config_vars[cv]
flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII)
flags = re.sub(r'-isysroot\s*\S+', ' ', flags)
_save_modified_value(_config_vars, cv, flags)
return _config_vars
def _remove_unsupported_archs(_config_vars):
"""Remove any unsupported archs from config vars"""
# Different Xcode releases support different sets for '-arch'
# flags. In particular, Xcode 4.x no longer supports the
# PPC architectures.
#
# This code automatically removes '-arch ppc' and '-arch ppc64'
# when these are not supported. That makes it possible to
# build extensions on OSX 10.7 and later with the prebuilt
# 32-bit installer on the python.org website.
# skip checks if the compiler was overridden with a CC env variable
if 'CC' in os.environ:
return _config_vars
if re.search(r'-arch\s+ppc', _config_vars['CFLAGS']) is not None:
# NOTE: Cannot use subprocess here because of bootstrap
# issues when building Python itself
status = os.system(
"""echo 'int main{};' | """
"""'%s' -c -arch ppc -x c -o /dev/null /dev/null 2>/dev/null"""
%(_config_vars['CC'].replace("'", "'\"'\"'"),))
if status:
# The compile failed for some reason. Because of differences
# across Xcode and compiler versions, there is no reliable way
# to be sure why it failed. Assume here it was due to lack of
# PPC support and remove the related '-arch' flags from each
# config variables not explicitly overridden by an environment
# variable. If the error was for some other reason, we hope the
# failure will show up again when trying to compile an extension
# module.
for cv in _UNIVERSAL_CONFIG_VARS:
if cv in _config_vars and cv not in os.environ:
flags = _config_vars[cv]
flags = re.sub(r'-arch\s+ppc\w*\s', ' ', flags)
_save_modified_value(_config_vars, cv, flags)
return _config_vars
def _override_all_archs(_config_vars):
"""Allow override of all archs with ARCHFLAGS env var"""
# NOTE: This name was introduced by Apple in OSX 10.5 and
# is used by several scripting languages distributed with
# that OS release.
if 'ARCHFLAGS' in os.environ:
arch = os.environ['ARCHFLAGS']
for cv in _UNIVERSAL_CONFIG_VARS:
if cv in _config_vars and '-arch' in _config_vars[cv]:
flags = _config_vars[cv]
flags = re.sub(r'-arch\s+\w+\s', ' ', flags)
flags = flags + ' ' + arch
_save_modified_value(_config_vars, cv, flags)
return _config_vars
def _check_for_unavailable_sdk(_config_vars):
"""Remove references to any SDKs not available"""
# If we're on OSX 10.5 or later and the user tries to
# compile an extension using an SDK that is not present
# on the current machine it is better to not use an SDK
# than to fail. This is particularly important with
# the standalone Command Line Tools alternative to a
# full-blown Xcode install since the CLT packages do not
# provide SDKs. If the SDK is not present, it is assumed
# that the header files and dev libs have been installed
# to /usr and /System/Library by either a standalone CLT
# package or the CLT component within Xcode.
cflags = _config_vars.get('CFLAGS', '')
m = re.search(r'-isysroot\s*(\S+)', cflags)
if m is not None:
sdk = m.group(1)
if not os.path.exists(sdk):
for cv in _UNIVERSAL_CONFIG_VARS:
# Do not alter a config var explicitly overridden by env var
if cv in _config_vars and cv not in os.environ:
flags = _config_vars[cv]
flags = re.sub(r'-isysroot\s*\S+(?:\s|$)', ' ', flags)
_save_modified_value(_config_vars, cv, flags)
return _config_vars
def compiler_fixup(compiler_so, cc_args):
"""
This function will strip '-isysroot PATH' and '-arch ARCH' from the
compile flags if the user has specified one them in extra_compile_flags.
This is needed because '-arch ARCH' adds another architecture to the
build, without a way to remove an architecture. Furthermore GCC will
barf if multiple '-isysroot' arguments are present.
"""
stripArch = stripSysroot = False
compiler_so = list(compiler_so)
if not _supports_universal_builds():
# OSX before 10.4.0, these don't support -arch and -isysroot at
# all.
stripArch = stripSysroot = True
else:
stripArch = '-arch' in cc_args
stripSysroot = any(arg for arg in cc_args if arg.startswith('-isysroot'))
if stripArch or 'ARCHFLAGS' in os.environ:
while True:
try:
index = compiler_so.index('-arch')
# Strip this argument and the next one:
del compiler_so[index:index+2]
except ValueError:
break
elif not _supports_arm64_builds():
# Look for "-arch arm64" and drop that
for idx in reversed(range(len(compiler_so))):
if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64":
del compiler_so[idx:idx+2]
if 'ARCHFLAGS' in os.environ and not stripArch:
# User specified different -arch flags in the environ,
# see also distutils.sysconfig
compiler_so = compiler_so + os.environ['ARCHFLAGS'].split()
if stripSysroot:
while True:
indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
if not indices:
break
index = indices[0]
if compiler_so[index] == '-isysroot':
# Strip this argument and the next one:
del compiler_so[index:index+2]
else:
# It's '-isysroot/some/path' in one arg
del compiler_so[index:index+1]
# Check if the SDK that is used during compilation actually exists,
# the universal build requires the usage of a universal SDK and not all
# users have that installed by default.
sysroot = None
argvar = cc_args
indices = [i for i,x in enumerate(cc_args) if x.startswith('-isysroot')]
if not indices:
argvar = compiler_so
indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
for idx in indices:
if argvar[idx] == '-isysroot':
sysroot = argvar[idx+1]
break
else:
sysroot = argvar[idx][len('-isysroot'):]
break
if sysroot and not os.path.isdir(sysroot):
sys.stderr.write(f"Compiling with an SDK that doesn't seem to exist: {sysroot}\n")
sys.stderr.write("Please check your Xcode installation\n")
sys.stderr.flush()
return compiler_so
def customize_config_vars(_config_vars):
"""Customize Python build configuration variables.
Called internally from sysconfig with a mutable mapping
containing name/value pairs parsed from the configured
makefile used to build this interpreter. Returns
the mapping updated as needed to reflect the environment
in which the interpreter is running; in the case of
a Python from a binary installer, the installed
environment may be very different from the build
environment, i.e. different OS levels, different
built tools, different available CPU architectures.
This customization is performed whenever
distutils.sysconfig.get_config_vars() is first
called. It may be used in environments where no
compilers are present, i.e. when installing pure
Python dists. Customization of compiler paths
and detection of unavailable archs is deferred
until the first extension module build is
requested (in distutils.sysconfig.customize_compiler).
Currently called from distutils.sysconfig
"""
if not _supports_universal_builds():
# On Mac OS X before 10.4, check if -arch and -isysroot
# are in CFLAGS or LDFLAGS and remove them if they are.
# This is needed when building extensions on a 10.3 system
# using a universal build of python.
_remove_universal_flags(_config_vars)
# Allow user to override all archs with ARCHFLAGS env var
_override_all_archs(_config_vars)
# Remove references to sdks that are not found
_check_for_unavailable_sdk(_config_vars)
return _config_vars
def customize_compiler(_config_vars):
"""Customize compiler path and configuration variables.
This customization is performed when the first
extension module build is requested
in distutils.sysconfig.customize_compiler.
"""
# Find a compiler to use for extension module builds
_find_appropriate_compiler(_config_vars)
# Remove ppc arch flags if not supported here
_remove_unsupported_archs(_config_vars)
# Allow user to override all archs with ARCHFLAGS env var
_override_all_archs(_config_vars)
return _config_vars
def get_platform_osx(_config_vars, osname, release, machine):
"""Filter values for get_platform()"""
# called from get_platform() in sysconfig and distutils.util
#
# For our purposes, we'll assume that the system version from
# distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set
# to. This makes the compatibility story a bit more sane because the
# machine is going to compile and link as if it were
# MACOSX_DEPLOYMENT_TARGET.
macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '')
macrelease = _get_system_version() or macver
macver = macver or macrelease
if macver:
release = macver
osname = "macosx"
# Use the original CFLAGS value, if available, so that we
# return the same machine type for the platform string.
# Otherwise, distutils may consider this a cross-compiling
# case and disallow installs.
cflags = _config_vars.get(_INITPRE+'CFLAGS',
_config_vars.get('CFLAGS', ''))
if macrelease:
try:
macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
except ValueError:
macrelease = (10, 3)
else:
# assume no universal support
macrelease = (10, 3)
if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
# The universal build will build fat binaries, but not on
# systems before 10.4
machine = 'fat'
archs = re.findall(r'-arch\s+(\S+)', cflags)
archs = tuple(sorted(set(archs)))
if len(archs) == 1:
machine = archs[0]
elif archs == ('arm64', 'x86_64'):
machine = 'universal2'
elif archs == ('i386', 'ppc'):
machine = 'fat'
elif archs == ('i386', 'x86_64'):
machine = 'intel'
elif archs == ('i386', 'ppc', 'x86_64'):
machine = 'fat3'
elif archs == ('ppc64', 'x86_64'):
machine = 'fat64'
elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'):
machine = 'universal'
else:
raise ValueError(
"Don't know machine value for archs=%r" % (archs,))
elif machine == 'i386':
# On OSX the machine type returned by uname is always the
# 32-bit variant, even if the executable architecture is
# the 64-bit variant
if sys.maxsize >= 2**32:
machine = 'x86_64'
elif machine in ('PowerPC', 'Power_Macintosh'):
# Pick a sane name for the PPC architecture.
# See 'i386' case
if sys.maxsize >= 2**32:
machine = 'ppc64'
else:
machine = 'ppc'
return (osname, release, machine)

6
Lib/_py_abc.py vendored
View File

@@ -32,8 +32,8 @@ class ABCMeta(type):
# external code.
_abc_invalidation_counter = 0
def __new__(mcls, name, bases, namespace, /, **kwargs):
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
def __new__(mcls, name, bases, namespace, **kwargs):
cls = type.__new__(mcls, name, bases, namespace, **kwargs)
# Compute set of abstract method names
abstracts = {name
for name, value in namespace.items()
@@ -43,7 +43,7 @@ class ABCMeta(type):
value = getattr(cls, name, None)
if getattr(value, "__isabstractmethod__", False):
abstracts.add(name)
cls.__abstractmethods__ = frozenset(abstracts)
cls.__abstractmethods__ = set(abstracts)
# Set up inheritance registry
cls._abc_registry = WeakSet()
cls._abc_cache = WeakSet()

2643
Lib/_pydatetime.py vendored
View File

File diff suppressed because it is too large Load Diff

27
Lib/_pydecimal.py vendored
View File

@@ -734,23 +734,18 @@ class Decimal(object):
"""
if isinstance(f, int): # handle integer inputs
sign = 0 if f >= 0 else 1
k = 0
coeff = str(abs(f))
elif isinstance(f, float):
if _math.isinf(f) or _math.isnan(f):
return cls(repr(f))
if _math.copysign(1.0, f) == 1.0:
sign = 0
else:
sign = 1
n, d = abs(f).as_integer_ratio()
k = d.bit_length() - 1
coeff = str(n*5**k)
else:
return cls(f)
if not isinstance(f, float):
raise TypeError("argument must be int or float.")
result = _dec_from_triple(sign, coeff, -k)
if _math.isinf(f) or _math.isnan(f):
return cls(repr(f))
if _math.copysign(1.0, f) == 1.0:
sign = 0
else:
sign = 1
n, d = abs(f).as_integer_ratio()
k = d.bit_length() - 1
result = _dec_from_triple(sign, str(n*5**k), -k)
if cls is Decimal:
return result
else:

2701
Lib/_pyio.py vendored
View File

File diff suppressed because it is too large Load Diff

3
Lib/_sitebuiltins.py vendored
View File

@@ -10,6 +10,7 @@ The objects used by the site module to add custom builtins.
import sys
class Quitter(object):
def __init__(self, name, eof):
self.name = name
@@ -47,7 +48,7 @@ class _Printer(object):
data = None
for filename in self.__filenames:
try:
with open(filename, encoding='utf-8') as fp:
with open(filename, "r") as fp:
data = fp.read()
break
except OSError:

1297
Lib/_sre.py vendored Normal file
View File

File diff suppressed because it is too large Load Diff

565
Lib/_strptime.py vendored
View File

@@ -1,565 +0,0 @@
"""Strptime-related classes and functions.
CLASSES:
LocaleTime -- Discovers and stores locale-specific time information
TimeRE -- Creates regexes for pattern matching a string of text containing
time information
FUNCTIONS:
_getlang -- Figure out what language is being used for the locale
strptime -- Calculates the time struct represented by the passed-in string
"""
import time
import locale
import calendar
from re import compile as re_compile
from re import IGNORECASE
from re import escape as re_escape
from datetime import (date as datetime_date,
timedelta as datetime_timedelta,
timezone as datetime_timezone)
from _thread import allocate_lock as _thread_allocate_lock
__all__ = []
def _getlang():
# Figure out what the current language is set to.
return locale.getlocale(locale.LC_TIME)
class LocaleTime(object):
"""Stores and handles locale-specific information related to time.
ATTRIBUTES:
f_weekday -- full weekday names (7-item list)
a_weekday -- abbreviated weekday names (7-item list)
f_month -- full month names (13-item list; dummy value in [0], which
is added by code)
a_month -- abbreviated month names (13-item list, dummy value in
[0], which is added by code)
am_pm -- AM/PM representation (2-item list)
LC_date_time -- format string for date/time representation (string)
LC_date -- format string for date representation (string)
LC_time -- format string for time representation (string)
timezone -- daylight- and non-daylight-savings timezone representation
(2-item list of sets)
lang -- Language used by instance (2-item tuple)
"""
def __init__(self):
"""Set all attributes.
Order of methods called matters for dependency reasons.
The locale language is set at the offset and then checked again before
exiting. This is to make sure that the attributes were not set with a
mix of information from more than one locale. This would most likely
happen when using threads where one thread calls a locale-dependent
function while another thread changes the locale while the function in
the other thread is still running. Proper coding would call for
locks to prevent changing the locale while locale-dependent code is
running. The check here is done in case someone does not think about
doing this.
Only other possible issue is if someone changed the timezone and did
not call tz.tzset . That is an issue for the programmer, though,
since changing the timezone is worthless without that call.
"""
self.lang = _getlang()
self.__calc_weekday()
self.__calc_month()
self.__calc_am_pm()
self.__calc_timezone()
self.__calc_date_time()
if _getlang() != self.lang:
raise ValueError("locale changed during initialization")
if time.tzname != self.tzname or time.daylight != self.daylight:
raise ValueError("timezone changed during initialization")
def __calc_weekday(self):
# Set self.a_weekday and self.f_weekday using the calendar
# module.
a_weekday = [calendar.day_abbr[i].lower() for i in range(7)]
f_weekday = [calendar.day_name[i].lower() for i in range(7)]
self.a_weekday = a_weekday
self.f_weekday = f_weekday
def __calc_month(self):
# Set self.f_month and self.a_month using the calendar module.
a_month = [calendar.month_abbr[i].lower() for i in range(13)]
f_month = [calendar.month_name[i].lower() for i in range(13)]
self.a_month = a_month
self.f_month = f_month
def __calc_am_pm(self):
# Set self.am_pm by using time.strftime().
# The magic date (1999,3,17,hour,44,55,2,76,0) is not really that
# magical; just happened to have used it everywhere else where a
# static date was needed.
am_pm = []
for hour in (1, 22):
time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0))
am_pm.append(time.strftime("%p", time_tuple).lower())
self.am_pm = am_pm
def __calc_date_time(self):
# Set self.date_time, self.date, & self.time by using
# time.strftime().
# Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of
# overloaded numbers is minimized. The order in which searches for
# values within the format string is very important; it eliminates
# possible ambiguity for what something represents.
time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
date_time = [None, None, None]
date_time[0] = time.strftime("%c", time_tuple).lower()
date_time[1] = time.strftime("%x", time_tuple).lower()
date_time[2] = time.strftime("%X", time_tuple).lower()
replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'),
(self.f_month[3], '%B'), (self.a_weekday[2], '%a'),
(self.a_month[3], '%b'), (self.am_pm[1], '%p'),
('1999', '%Y'), ('99', '%y'), ('22', '%H'),
('44', '%M'), ('55', '%S'), ('76', '%j'),
('17', '%d'), ('03', '%m'), ('3', '%m'),
# '3' needed for when no leading zero.
('2', '%w'), ('10', '%I')]
replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone
for tz in tz_values])
for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')):
current_format = date_time[offset]
for old, new in replacement_pairs:
# Must deal with possible lack of locale info
# manifesting itself as the empty string (e.g., Swedish's
# lack of AM/PM info) or a platform returning a tuple of empty
# strings (e.g., MacOS 9 having timezone as ('','')).
if old:
current_format = current_format.replace(old, new)
# If %W is used, then Sunday, 2005-01-03 will fall on week 0 since
# 2005-01-03 occurs before the first Monday of the year. Otherwise
# %U is used.
time_tuple = time.struct_time((1999,1,3,1,1,1,6,3,0))
if '00' in time.strftime(directive, time_tuple):
U_W = '%W'
else:
U_W = '%U'
date_time[offset] = current_format.replace('11', U_W)
self.LC_date_time = date_time[0]
self.LC_date = date_time[1]
self.LC_time = date_time[2]
def __calc_timezone(self):
# Set self.timezone by using time.tzname.
# Do not worry about possibility of time.tzname[0] == time.tzname[1]
# and time.daylight; handle that in strptime.
try:
time.tzset()
except AttributeError:
pass
self.tzname = time.tzname
self.daylight = time.daylight
no_saving = frozenset({"utc", "gmt", self.tzname[0].lower()})
if self.daylight:
has_saving = frozenset({self.tzname[1].lower()})
else:
has_saving = frozenset()
self.timezone = (no_saving, has_saving)
class TimeRE(dict):
"""Handle conversion from format directives to regexes."""
def __init__(self, locale_time=None):
"""Create keys/values.
Order of execution is important for dependency reasons.
"""
if locale_time:
self.locale_time = locale_time
else:
self.locale_time = LocaleTime()
base = super()
base.__init__({
# The " [1-9]" part of the regex is to make %c from ANSI C work
'd': r"(?P<d>3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
'f': r"(?P<f>[0-9]{1,6})",
'H': r"(?P<H>2[0-3]|[0-1]\d|\d)",
'I': r"(?P<I>1[0-2]|0[1-9]|[1-9])",
'G': r"(?P<G>\d\d\d\d)",
'j': r"(?P<j>36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])",
'm': r"(?P<m>1[0-2]|0[1-9]|[1-9])",
'M': r"(?P<M>[0-5]\d|\d)",
'S': r"(?P<S>6[0-1]|[0-5]\d|\d)",
'U': r"(?P<U>5[0-3]|[0-4]\d|\d)",
'w': r"(?P<w>[0-6])",
'u': r"(?P<u>[1-7])",
'V': r"(?P<V>5[0-3]|0[1-9]|[1-4]\d|\d)",
# W is set below by using 'U'
'y': r"(?P<y>\d\d)",
#XXX: Does 'Y' need to worry about having less or more than
# 4 digits?
'Y': r"(?P<Y>\d\d\d\d)",
'z': r"(?P<z>[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))",
'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'),
'p': self.__seqToRE(self.locale_time.am_pm, 'p'),
'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone
for tz in tz_names),
'Z'),
'%': '%'})
base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
base.__setitem__('x', self.pattern(self.locale_time.LC_date))
base.__setitem__('X', self.pattern(self.locale_time.LC_time))
def __seqToRE(self, to_convert, directive):
"""Convert a list to a regex string for matching a directive.
Want possible matching values to be from longest to shortest. This
prevents the possibility of a match occurring for a value that also
a substring of a larger value that should have matched (e.g., 'abc'
matching when 'abcdef' should have been the match).
"""
to_convert = sorted(to_convert, key=len, reverse=True)
for value in to_convert:
if value != '':
break
else:
return ''
regex = '|'.join(re_escape(stuff) for stuff in to_convert)
regex = '(?P<%s>%s' % (directive, regex)
return '%s)' % regex
def pattern(self, format):
"""Return regex pattern for the format string.
Need to make sure that any characters that might be interpreted as
regex syntax are escaped.
"""
processed_format = ''
# The sub() call escapes all characters that might be misconstrued
# as regex syntax. Cannot use re.escape since we have to deal with
# format directives (%m, etc.).
regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
format = regex_chars.sub(r"\\\1", format)
whitespace_replacement = re_compile(r'\s+')
format = whitespace_replacement.sub(r'\\s+', format)
while '%' in format:
directive_index = format.index('%')+1
processed_format = "%s%s%s" % (processed_format,
format[:directive_index-1],
self[format[directive_index]])
format = format[directive_index+1:]
return "%s%s" % (processed_format, format)
def compile(self, format):
"""Return a compiled re object for the format string."""
return re_compile(self.pattern(format), IGNORECASE)
_cache_lock = _thread_allocate_lock()
# DO NOT modify _TimeRE_cache or _regex_cache without acquiring the cache lock
# first!
_TimeRE_cache = TimeRE()
_CACHE_MAX_SIZE = 5 # Max number of regexes stored in _regex_cache
_regex_cache = {}
def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon):
"""Calculate the Julian day based on the year, week of the year, and day of
the week, with week_start_day representing whether the week of the year
assumes the week starts on Sunday or Monday (6 or 0)."""
first_weekday = datetime_date(year, 1, 1).weekday()
# If we are dealing with the %U directive (week starts on Sunday), it's
# easier to just shift the view to Sunday being the first day of the
# week.
if not week_starts_Mon:
first_weekday = (first_weekday + 1) % 7
day_of_week = (day_of_week + 1) % 7
# Need to watch out for a week 0 (when the first day of the year is not
# the same as that specified by %U or %W).
week_0_length = (7 - first_weekday) % 7
if week_of_year == 0:
return 1 + day_of_week - first_weekday
else:
days_to_week = week_0_length + (7 * (week_of_year - 1))
return 1 + days_to_week + day_of_week
def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
"""Return a 2-tuple consisting of a time struct and an int containing
the number of microseconds based on the input string and the
format string."""
for index, arg in enumerate([data_string, format]):
if not isinstance(arg, str):
msg = "strptime() argument {} must be str, not {}"
raise TypeError(msg.format(index, type(arg)))
global _TimeRE_cache, _regex_cache
with _cache_lock:
locale_time = _TimeRE_cache.locale_time
if (_getlang() != locale_time.lang or
time.tzname != locale_time.tzname or
time.daylight != locale_time.daylight):
_TimeRE_cache = TimeRE()
_regex_cache.clear()
locale_time = _TimeRE_cache.locale_time
if len(_regex_cache) > _CACHE_MAX_SIZE:
_regex_cache.clear()
format_regex = _regex_cache.get(format)
if not format_regex:
try:
format_regex = _TimeRE_cache.compile(format)
# KeyError raised when a bad format is found; can be specified as
# \\, in which case it was a stray % but with a space after it
except KeyError as err:
bad_directive = err.args[0]
if bad_directive == "\\":
bad_directive = "%"
del err
raise ValueError("'%s' is a bad directive in format '%s'" %
(bad_directive, format)) from None
# IndexError only occurs when the format string is "%"
except IndexError:
raise ValueError("stray %% in format '%s'" % format) from None
_regex_cache[format] = format_regex
found = format_regex.match(data_string)
if not found:
raise ValueError("time data %r does not match format %r" %
(data_string, format))
if len(data_string) != found.end():
raise ValueError("unconverted data remains: %s" %
data_string[found.end():])
iso_year = year = None
month = day = 1
hour = minute = second = fraction = 0
tz = -1
gmtoff = None
gmtoff_fraction = 0
iso_week = week_of_year = None
week_of_year_start = None
# weekday and julian defaulted to None so as to signal need to calculate
# values
weekday = julian = None
found_dict = found.groupdict()
for group_key in found_dict.keys():
# Directives not explicitly handled below:
# c, x, X
# handled by making out of other directives
# U, W
# worthless without day of the week
if group_key == 'y':
year = int(found_dict['y'])
# Open Group specification for strptime() states that a %y
#value in the range of [00, 68] is in the century 2000, while
#[69,99] is in the century 1900
if year <= 68:
year += 2000
else:
year += 1900
elif group_key == 'Y':
year = int(found_dict['Y'])
elif group_key == 'G':
iso_year = int(found_dict['G'])
elif group_key == 'm':
month = int(found_dict['m'])
elif group_key == 'B':
month = locale_time.f_month.index(found_dict['B'].lower())
elif group_key == 'b':
month = locale_time.a_month.index(found_dict['b'].lower())
elif group_key == 'd':
day = int(found_dict['d'])
elif group_key == 'H':
hour = int(found_dict['H'])
elif group_key == 'I':
hour = int(found_dict['I'])
ampm = found_dict.get('p', '').lower()
# If there was no AM/PM indicator, we'll treat this like AM
if ampm in ('', locale_time.am_pm[0]):
# We're in AM so the hour is correct unless we're
# looking at 12 midnight.
# 12 midnight == 12 AM == hour 0
if hour == 12:
hour = 0
elif ampm == locale_time.am_pm[1]:
# We're in PM so we need to add 12 to the hour unless
# we're looking at 12 noon.
# 12 noon == 12 PM == hour 12
if hour != 12:
hour += 12
elif group_key == 'M':
minute = int(found_dict['M'])
elif group_key == 'S':
second = int(found_dict['S'])
elif group_key == 'f':
s = found_dict['f']
# Pad to always return microseconds.
s += "0" * (6 - len(s))
fraction = int(s)
elif group_key == 'A':
weekday = locale_time.f_weekday.index(found_dict['A'].lower())
elif group_key == 'a':
weekday = locale_time.a_weekday.index(found_dict['a'].lower())
elif group_key == 'w':
weekday = int(found_dict['w'])
if weekday == 0:
weekday = 6
else:
weekday -= 1
elif group_key == 'u':
weekday = int(found_dict['u'])
weekday -= 1
elif group_key == 'j':
julian = int(found_dict['j'])
elif group_key in ('U', 'W'):
week_of_year = int(found_dict[group_key])
if group_key == 'U':
# U starts week on Sunday.
week_of_year_start = 6
else:
# W starts week on Monday.
week_of_year_start = 0
elif group_key == 'V':
iso_week = int(found_dict['V'])
elif group_key == 'z':
z = found_dict['z']
if z == 'Z':
gmtoff = 0
else:
if z[3] == ':':
z = z[:3] + z[4:]
if len(z) > 5:
if z[5] != ':':
msg = f"Inconsistent use of : in {found_dict['z']}"
raise ValueError(msg)
z = z[:5] + z[6:]
hours = int(z[1:3])
minutes = int(z[3:5])
seconds = int(z[5:7] or 0)
gmtoff = (hours * 60 * 60) + (minutes * 60) + seconds
gmtoff_remainder = z[8:]
# Pad to always return microseconds.
gmtoff_remainder_padding = "0" * (6 - len(gmtoff_remainder))
gmtoff_fraction = int(gmtoff_remainder + gmtoff_remainder_padding)
if z.startswith("-"):
gmtoff = -gmtoff
gmtoff_fraction = -gmtoff_fraction
elif group_key == 'Z':
# Since -1 is default value only need to worry about setting tz if
# it can be something other than -1.
found_zone = found_dict['Z'].lower()
for value, tz_values in enumerate(locale_time.timezone):
if found_zone in tz_values:
# Deal with bad locale setup where timezone names are the
# same and yet time.daylight is true; too ambiguous to
# be able to tell what timezone has daylight savings
if (time.tzname[0] == time.tzname[1] and
time.daylight and found_zone not in ("utc", "gmt")):
break
else:
tz = value
break
# Deal with the cases where ambiguities arise
# don't assume default values for ISO week/year
if iso_year is not None:
if julian is not None:
raise ValueError("Day of the year directive '%j' is not "
"compatible with ISO year directive '%G'. "
"Use '%Y' instead.")
elif iso_week is None or weekday is None:
raise ValueError("ISO year directive '%G' must be used with "
"the ISO week directive '%V' and a weekday "
"directive ('%A', '%a', '%w', or '%u').")
elif iso_week is not None:
if year is None or weekday is None:
raise ValueError("ISO week directive '%V' must be used with "
"the ISO year directive '%G' and a weekday "
"directive ('%A', '%a', '%w', or '%u').")
else:
raise ValueError("ISO week directive '%V' is incompatible with "
"the year directive '%Y'. Use the ISO year '%G' "
"instead.")
leap_year_fix = False
if year is None:
if month == 2 and day == 29:
year = 1904 # 1904 is first leap year of 20th century
leap_year_fix = True
else:
year = 1900
# If we know the week of the year and what day of that week, we can figure
# out the Julian day of the year.
if julian is None and weekday is not None:
if week_of_year is not None:
week_starts_Mon = True if week_of_year_start == 0 else False
julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
week_starts_Mon)
elif iso_year is not None and iso_week is not None:
datetime_result = datetime_date.fromisocalendar(iso_year, iso_week, weekday + 1)
year = datetime_result.year
month = datetime_result.month
day = datetime_result.day
if julian is not None and julian <= 0:
year -= 1
yday = 366 if calendar.isleap(year) else 365
julian += yday
if julian is None:
# Cannot pre-calculate datetime_date() since can change in Julian
# calculation and thus could have different value for the day of
# the week calculation.
# Need to add 1 to result since first day of the year is 1, not 0.
julian = datetime_date(year, month, day).toordinal() - \
datetime_date(year, 1, 1).toordinal() + 1
else: # Assume that if they bothered to include Julian day (or if it was
# calculated above with year/week/weekday) it will be accurate.
datetime_result = datetime_date.fromordinal(
(julian - 1) +
datetime_date(year, 1, 1).toordinal())
year = datetime_result.year
month = datetime_result.month
day = datetime_result.day
if weekday is None:
weekday = datetime_date(year, month, day).weekday()
# Add timezone info
tzname = found_dict.get("Z")
if leap_year_fix:
# the caller didn't supply a year but asked for Feb 29th. We couldn't
# use the default of 1900 for computations. We set it back to ensure
# that February 29th is smaller than March 1st.
year = 1900
return (year, month, day,
hour, minute, second,
weekday, julian, tz, tzname, gmtoff), fraction, gmtoff_fraction
def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"):
"""Return a time struct based on the input string and the
format string."""
tt = _strptime(data_string, format)[0]
return time.struct_time(tt[:time._STRUCT_TM_ITEMS])
def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"):
"""Return a class cls instance based on the input string and the
format string."""
tt, fraction, gmtoff_fraction = _strptime(data_string, format)
tzname, gmtoff = tt[-2:]
args = tt[:6] + (fraction,)
if gmtoff is not None:
tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction)
if tzname:
tz = datetime_timezone(tzdelta, tzname)
else:
tz = datetime_timezone(tzdelta)
args += (tz,)
return cls(*args)

13
Lib/_weakrefset.py vendored
View File

@@ -3,7 +3,6 @@
# by abc.py to load everything else at startup.
from _weakref import ref
from types import GenericAlias
__all__ = ['WeakSet']
@@ -51,14 +50,10 @@ class WeakSet:
self.update(data)
def _commit_removals(self):
pop = self._pending_removals.pop
l = self._pending_removals
discard = self.data.discard
while True:
try:
item = pop()
except IndexError:
return
discard(item)
while l:
discard(l.pop())
def __iter__(self):
with _IterationGuard(self):
@@ -202,5 +197,3 @@ class WeakSet:
def __repr__(self):
return repr(self.data)
__class_getitem__ = classmethod(GenericAlias)

94
Lib/abc.py vendored
View File

@@ -11,14 +11,13 @@ def abstractmethod(funcobj):
class that has a metaclass derived from ABCMeta cannot be
instantiated unless all of its abstract methods are overridden.
The abstract methods can be called using any of the normal
'super' call mechanisms. abstractmethod() may be used to declare
abstract methods for properties and descriptors.
'super' call mechanisms.
Usage:
class C(metaclass=ABCMeta):
@abstractmethod
def my_abstract_method(self, arg1, arg2, argN):
def my_abstract_method(self, ...):
...
"""
funcobj.__isabstractmethod__ = True
@@ -28,14 +27,17 @@ def abstractmethod(funcobj):
class abstractclassmethod(classmethod):
"""A decorator indicating abstract classmethods.
Deprecated, use 'classmethod' with 'abstractmethod' instead:
Similar to abstractmethod.
class C(ABC):
@classmethod
@abstractmethod
Usage:
class C(metaclass=ABCMeta):
@abstractclassmethod
def my_abstract_classmethod(cls, ...):
...
'abstractclassmethod' is deprecated. Use 'classmethod' with
'abstractmethod' instead.
"""
__isabstractmethod__ = True
@@ -48,14 +50,17 @@ class abstractclassmethod(classmethod):
class abstractstaticmethod(staticmethod):
"""A decorator indicating abstract staticmethods.
Deprecated, use 'staticmethod' with 'abstractmethod' instead:
Similar to abstractmethod.
class C(ABC):
@staticmethod
@abstractmethod
Usage:
class C(metaclass=ABCMeta):
@abstractstaticmethod
def my_abstract_staticmethod(...):
...
'abstractstaticmethod' is deprecated. Use 'staticmethod' with
'abstractmethod' instead.
"""
__isabstractmethod__ = True
@@ -68,14 +73,29 @@ class abstractstaticmethod(staticmethod):
class abstractproperty(property):
"""A decorator indicating abstract properties.
Deprecated, use 'property' with 'abstractmethod' instead:
Requires that the metaclass is ABCMeta or derived from it. A
class that has a metaclass derived from ABCMeta cannot be
instantiated unless all of its abstract properties are overridden.
The abstract properties can be called using any of the normal
'super' call mechanisms.
class C(ABC):
@property
@abstractmethod
Usage:
class C(metaclass=ABCMeta):
@abstractproperty
def my_abstract_property(self):
...
This defines a read-only property; you can also define a read-write
abstract property using the 'long' form of property declaration:
class C(metaclass=ABCMeta):
def getx(self): ...
def setx(self, value): ...
x = abstractproperty(getx, setx)
'abstractproperty' is deprecated. Use 'property' with 'abstractmethod'
instead.
"""
__isabstractmethod__ = True
@@ -85,10 +105,6 @@ try:
from _abc import (get_cache_token, _abc_init, _abc_register,
_abc_instancecheck, _abc_subclasscheck, _get_dump,
_reset_registry, _reset_caches)
# TODO: RUSTPYTHON missing _abc module implementation.
except ModuleNotFoundError:
from _py_abc import ABCMeta, get_cache_token
ABCMeta.__module__ = 'abc'
except ImportError:
from _py_abc import ABCMeta, get_cache_token
ABCMeta.__module__ = 'abc'
@@ -106,7 +122,7 @@ else:
implementations defined by the registering ABC be callable (not
even via super()).
"""
def __new__(mcls, name, bases, namespace, /, **kwargs):
def __new__(mcls, name, bases, namespace, **kwargs):
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
_abc_init(cls)
return cls
@@ -147,44 +163,6 @@ else:
_reset_caches(cls)
def update_abstractmethods(cls):
"""Recalculate the set of abstract methods of an abstract class.
If a class has had one of its abstract methods implemented after the
class was created, the method will not be considered implemented until
this function is called. Alternatively, if a new abstract method has been
added to the class, it will only be considered an abstract method of the
class after this function is called.
This function should be called before any use is made of the class,
usually in class decorators that add methods to the subject class.
Returns cls, to allow usage as a class decorator.
If cls is not an instance of ABCMeta, does nothing.
"""
if not hasattr(cls, '__abstractmethods__'):
# We check for __abstractmethods__ here because cls might by a C
# implementation or a python implementation (especially during
# testing), and we want to handle both cases.
return cls
abstracts = set()
# Check the existing abstract methods of the parents, keep only the ones
# that are not implemented.
for scls in cls.__bases__:
for name in getattr(scls, '__abstractmethods__', ()):
value = getattr(cls, name, None)
if getattr(value, "__isabstractmethod__", False):
abstracts.add(name)
# Also add any other newly added abstract methods.
for name, value in cls.__dict__.items():
if getattr(value, "__isabstractmethod__", False):
abstracts.add(name)
cls.__abstractmethods__ = frozenset(abstracts)
return cls
class ABC(metaclass=ABCMeta):
"""Helper class that provides a standard way to create an ABC using
inheritance.

65
Lib/aifc.py vendored
View File

@@ -138,11 +138,7 @@ import struct
import builtins
import warnings
__all__ = ["Error", "open"]
warnings._deprecated(__name__, remove=(3, 13))
__all__ = ["Error", "open", "openfp"]
class Error(Exception):
pass
@@ -255,9 +251,7 @@ def _write_float(f, x):
_write_ulong(f, himant)
_write_ulong(f, lomant)
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
from chunk import Chunk
from chunk import Chunk
from collections import namedtuple
_aifc_params = namedtuple('_aifc_params',
@@ -453,33 +447,21 @@ class Aifc_read:
#
def _alaw2lin(self, data):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning)
import audioop
import audioop
return audioop.alaw2lin(data, 2)
def _ulaw2lin(self, data):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning)
import audioop
import audioop
return audioop.ulaw2lin(data, 2)
def _adpcm2lin(self, data):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning)
import audioop
import audioop
if not hasattr(self, '_adpcmstate'):
# first time
self._adpcmstate = None
data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
return data
def _sowt2lin(self, data):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning)
import audioop
return audioop.byteswap(data, 2)
def _read_comm_chunk(self, chunk):
self._nchannels = _read_short(chunk)
self._nframes = _read_long(chunk)
@@ -515,8 +497,6 @@ class Aifc_read:
self._convert = self._ulaw2lin
elif self._comptype in (b'alaw', b'ALAW'):
self._convert = self._alaw2lin
elif self._comptype in (b'sowt', b'SOWT'):
self._convert = self._sowt2lin
else:
raise Error('unsupported compression type')
self._sampwidth = 2
@@ -679,7 +659,7 @@ class Aifc_write:
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if comptype not in (b'NONE', b'ulaw', b'ULAW',
b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
b'alaw', b'ALAW', b'G722'):
raise Error('unsupported compression type')
self._comptype = comptype
self._compname = compname
@@ -700,7 +680,7 @@ class Aifc_write:
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if comptype not in (b'NONE', b'ulaw', b'ULAW',
b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
b'alaw', b'ALAW', b'G722'):
raise Error('unsupported compression type')
self.setnchannels(nchannels)
self.setsampwidth(sampwidth)
@@ -784,43 +764,28 @@ class Aifc_write:
#
def _lin2alaw(self, data):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning)
import audioop
import audioop
return audioop.lin2alaw(data, 2)
def _lin2ulaw(self, data):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning)
import audioop
import audioop
return audioop.lin2ulaw(data, 2)
def _lin2adpcm(self, data):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning)
import audioop
import audioop
if not hasattr(self, '_adpcmstate'):
self._adpcmstate = None
data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
return data
def _lin2sowt(self, data):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning)
import audioop
return audioop.byteswap(data, 2)
def _ensure_header_written(self, datasize):
if not self._nframeswritten:
if self._comptype in (b'ULAW', b'ulaw',
b'ALAW', b'alaw', b'G722',
b'sowt', b'SOWT'):
if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
if not self._sampwidth:
self._sampwidth = 2
if self._sampwidth != 2:
raise Error('sample width must be 2 when compressing '
'with ulaw/ULAW, alaw/ALAW, sowt/SOWT '
'or G7.22 (ADPCM)')
'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)')
if not self._nchannels:
raise Error('# channels not specified')
if not self._sampwidth:
@@ -836,8 +801,6 @@ class Aifc_write:
self._convert = self._lin2ulaw
elif self._comptype in (b'alaw', b'ALAW'):
self._convert = self._lin2alaw
elif self._comptype in (b'sowt', b'SOWT'):
self._convert = self._lin2sowt
def _write_header(self, initlength):
if self._aifc and self._comptype != b'NONE':
@@ -957,6 +920,10 @@ def open(f, mode=None):
else:
raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
def openfp(f, mode=None):
warnings.warn("aifc.openfp is deprecated since Python 3.7. "
"Use aifc.open instead.", DeprecationWarning, stacklevel=2)
return open(f, mode=mode)
if __name__ == '__main__':
import sys

4
Lib/antigravity.py vendored
View File

@@ -11,7 +11,7 @@ def geohash(latitude, longitude, datedow):
37.857713 -122.544543
'''
# https://xkcd.com/426/
h = hashlib.md5(datedow, usedforsecurity=False).hexdigest()
# http://xkcd.com/426/
h = hashlib.md5(datedow).hexdigest()
p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])]
print('%d%s %d%s' % (latitude, p[1:], longitude, q[1:]))

352
Lib/argparse.py vendored
View File

@@ -1,5 +1,4 @@
# Author: Steven J. Bethard <steven.bethard@gmail.com>.
# New maintainer as of 29 August 2019: Raymond Hettinger <raymond.hettinger@gmail.com>
"""Command-line parsing library
@@ -67,7 +66,6 @@ __all__ = [
'ArgumentParser',
'ArgumentError',
'ArgumentTypeError',
'BooleanOptionalAction',
'FileType',
'HelpFormatter',
'ArgumentDefaultsHelpFormatter',
@@ -89,8 +87,6 @@ import os as _os
import re as _re
import sys as _sys
import warnings
from gettext import gettext as _, ngettext
SUPPRESS = '==SUPPRESS=='
@@ -131,7 +127,7 @@ class _AttributeHolder(object):
return '%s(%s)' % (type_name, ', '.join(arg_strings))
def _get_kwargs(self):
return list(self.__dict__.items())
return sorted(self.__dict__.items())
def _get_args(self):
return []
@@ -153,7 +149,6 @@ def _copy_items(items):
# Formatting Help
# ===============
class HelpFormatter(object):
"""Formatter for generating usage messages and argument help strings.
@@ -169,12 +164,15 @@ class HelpFormatter(object):
# default setting for width
if width is None:
import shutil
width = shutil.get_terminal_size().columns
try:
width = int(_os.environ['COLUMNS'])
except (KeyError, ValueError):
width = 80
width -= 2
self._prog = prog
self._indent_increment = indent_increment
self._max_help_position = max_help_position
self._max_help_position = min(max_help_position,
max(width - 20, indent_increment * 2))
self._width = width
@@ -267,7 +265,7 @@ class HelpFormatter(object):
invocations.append(get_invocation(subaction))
# update the maximum item length
invocation_length = max(map(len, invocations))
invocation_length = max([len(s) for s in invocations])
action_length = invocation_length + self._current_indent
self._action_max_length = max(self._action_max_length,
action_length)
@@ -345,22 +343,21 @@ class HelpFormatter(object):
def get_lines(parts, indent, prefix=None):
lines = []
line = []
indent_length = len(indent)
if prefix is not None:
line_len = len(prefix) - 1
else:
line_len = indent_length - 1
line_len = len(indent) - 1
for part in parts:
if line_len + 1 + len(part) > text_width and line:
lines.append(indent + ' '.join(line))
line = []
line_len = indent_length - 1
line_len = len(indent) - 1
line.append(part)
line_len += len(part) + 1
if line:
lines.append(indent + ' '.join(line))
if prefix is not None:
lines[0] = lines[0][indent_length:]
lines[0] = lines[0][len(indent):]
return lines
# if prog is short, follow it with optionals or positionals
@@ -396,44 +393,27 @@ class HelpFormatter(object):
group_actions = set()
inserts = {}
for group in groups:
if not group._group_actions:
raise ValueError(f'empty group {group}')
try:
start = actions.index(group._group_actions[0])
except ValueError:
continue
else:
group_action_count = len(group._group_actions)
end = start + group_action_count
end = start + len(group._group_actions)
if actions[start:end] == group._group_actions:
suppressed_actions_count = 0
for action in group._group_actions:
group_actions.add(action)
if action.help is SUPPRESS:
suppressed_actions_count += 1
exposed_actions_count = group_action_count - suppressed_actions_count
if not group.required:
if start in inserts:
inserts[start] += ' ['
else:
inserts[start] = '['
if end in inserts:
inserts[end] += ']'
else:
inserts[end] = ']'
elif exposed_actions_count > 1:
inserts[end] = ']'
else:
if start in inserts:
inserts[start] += ' ('
else:
inserts[start] = '('
if end in inserts:
inserts[end] += ')'
else:
inserts[end] = ')'
inserts[end] = ')'
for i in range(start + 1, end):
inserts[i] = '|'
@@ -470,7 +450,7 @@ class HelpFormatter(object):
# if the Optional doesn't take a value, format is:
# -s or --long
if action.nargs == 0:
part = action.format_usage()
part = '%s' % option_string
# if the Optional takes a value, format is:
# -s ARGS or --long ARGS
@@ -499,6 +479,7 @@ class HelpFormatter(object):
text = _re.sub(r'(%s) ' % open, r'\1', text)
text = _re.sub(r' (%s)' % close, r'\1', text)
text = _re.sub(r'%s *%s' % (open, close), r'', text)
text = _re.sub(r'\(([^|]*)\)', r'\1', text)
text = text.strip()
# return the text
@@ -540,13 +521,12 @@ class HelpFormatter(object):
parts = [action_header]
# if there was help for the action, add lines of help text
if action.help and action.help.strip():
if action.help:
help_text = self._expand_help(action)
if help_text:
help_lines = self._split_lines(help_text, help_width)
parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
for line in help_lines[1:]:
parts.append('%*s%s\n' % (help_position, '', line))
help_lines = self._split_lines(help_text, help_width)
parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
for line in help_lines[1:]:
parts.append('%*s%s\n' % (help_position, '', line))
# or add a newline if the description doesn't end with one
elif not action_header.endswith('\n'):
@@ -606,11 +586,7 @@ class HelpFormatter(object):
elif action.nargs == OPTIONAL:
result = '[%s]' % get_metavar(1)
elif action.nargs == ZERO_OR_MORE:
metavar = get_metavar(1)
if len(metavar) == 2:
result = '[%s [%s ...]]' % metavar
else:
result = '[%s ...]' % metavar
result = '[%s [%s ...]]' % get_metavar(2)
elif action.nargs == ONE_OR_MORE:
result = '%s [%s ...]' % get_metavar(2)
elif action.nargs == REMAINDER:
@@ -620,10 +596,7 @@ class HelpFormatter(object):
elif action.nargs == SUPPRESS:
result = ''
else:
try:
formats = ['%s' for _ in range(action.nargs)]
except TypeError:
raise ValueError("invalid nargs value") from None
formats = ['%s' for _ in range(action.nargs)]
result = ' '.join(formats) % get_metavar(action.nargs)
return result
@@ -704,19 +677,8 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
"""
def _get_help_string(self, action):
"""
Add the default value to the option help message.
ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't
already present. This code will do that, detecting cornercases to
prevent duplicates or cases where it wouldn't make sense to the end
user.
"""
help = action.help
if help is None:
help = ''
if '%(default)' not in help:
if '%(default)' not in action.help:
if action.default is not SUPPRESS:
defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
if action.option_strings or action.nargs in defaulting_nargs:
@@ -724,7 +686,6 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
return help
class MetavarTypeHelpFormatter(HelpFormatter):
"""Help message formatter which uses the argument 'type' as the default
metavar value (instead of the argument 'dest')
@@ -740,6 +701,7 @@ class MetavarTypeHelpFormatter(HelpFormatter):
return action.type.__name__
# =====================
# Options and Arguments
# =====================
@@ -748,13 +710,11 @@ def _get_action_name(argument):
if argument is None:
return None
elif argument.option_strings:
return '/'.join(argument.option_strings)
return '/'.join(argument.option_strings)
elif argument.metavar not in (None, SUPPRESS):
return argument.metavar
elif argument.dest not in (None, SUPPRESS):
return argument.dest
elif argument.choices:
return '{' + ','.join(argument.choices) + '}'
else:
return None
@@ -774,7 +734,7 @@ class ArgumentError(Exception):
if self.argument_name is None:
format = '%(message)s'
else:
format = _('argument %(argument_name)s: %(message)s')
format = 'argument %(argument_name)s: %(message)s'
return format % dict(message=self.message,
argument_name=self.argument_name)
@@ -870,79 +830,15 @@ class Action(_AttributeHolder):
'default',
'type',
'choices',
'required',
'help',
'metavar',
]
return [(name, getattr(self, name)) for name in names]
def format_usage(self):
return self.option_strings[0]
def __call__(self, parser, namespace, values, option_string=None):
raise NotImplementedError(_('.__call__() not defined'))
# FIXME: remove together with `BooleanOptionalAction` deprecated arguments.
_deprecated_default = object()
class BooleanOptionalAction(Action):
def __init__(self,
option_strings,
dest,
default=None,
type=_deprecated_default,
choices=_deprecated_default,
required=False,
help=None,
metavar=_deprecated_default):
_option_strings = []
for option_string in option_strings:
_option_strings.append(option_string)
if option_string.startswith('--'):
option_string = '--no-' + option_string[2:]
_option_strings.append(option_string)
# We need `_deprecated` special value to ban explicit arguments that
# match default value. Like:
# parser.add_argument('-f', action=BooleanOptionalAction, type=int)
for field_name in ('type', 'choices', 'metavar'):
if locals()[field_name] is not _deprecated_default:
warnings._deprecated(
field_name,
"{name!r} is deprecated as of Python 3.12 and will be "
"removed in Python {remove}.",
remove=(3, 14))
if type is _deprecated_default:
type = None
if choices is _deprecated_default:
choices = None
if metavar is _deprecated_default:
metavar = None
super().__init__(
option_strings=_option_strings,
dest=dest,
nargs=0,
default=default,
type=type,
choices=choices,
required=required,
help=help,
metavar=metavar)
def __call__(self, parser, namespace, values, option_string=None):
if option_string in self.option_strings:
setattr(namespace, self.dest, not option_string.startswith('--no-'))
def format_usage(self):
return ' | '.join(self.option_strings)
class _StoreAction(Action):
def __init__(self,
@@ -957,7 +853,7 @@ class _StoreAction(Action):
help=None,
metavar=None):
if nargs == 0:
raise ValueError('nargs for store actions must be != 0; if you '
raise ValueError('nargs for store actions must be > 0; if you '
'have nothing to store, actions such as store '
'true or store const may be more appropriate')
if const is not None and nargs != OPTIONAL:
@@ -983,7 +879,7 @@ class _StoreConstAction(Action):
def __init__(self,
option_strings,
dest,
const=None,
const,
default=None,
required=False,
help=None,
@@ -1049,7 +945,7 @@ class _AppendAction(Action):
help=None,
metavar=None):
if nargs == 0:
raise ValueError('nargs for append actions must be != 0; if arg '
raise ValueError('nargs for append actions must be > 0; if arg '
'strings are not supplying the value to append, '
'the append const action may be more appropriate')
if const is not None and nargs != OPTIONAL:
@@ -1078,7 +974,7 @@ class _AppendConstAction(Action):
def __init__(self,
option_strings,
dest,
const=None,
const,
default=None,
required=False,
help=None,
@@ -1210,13 +1106,6 @@ class _SubParsersAction(Action):
aliases = kwargs.pop('aliases', ())
if name in self._name_parser_map:
raise ArgumentError(self, _('conflicting subparser: %s') % name)
for alias in aliases:
if alias in self._name_parser_map:
raise ArgumentError(
self, _('conflicting subparser alias: %s') % alias)
# create a pseudo-action to hold the choice help
if 'help' in kwargs:
help = kwargs.pop('help')
@@ -1268,12 +1157,6 @@ class _SubParsersAction(Action):
vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
class _ExtendAction(_AppendAction):
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest, None)
items = _copy_items(items)
items.extend(values)
setattr(namespace, self.dest, items)
# ==============
# Type classes
@@ -1306,9 +1189,9 @@ class FileType(object):
# the special argument "-" means sys.std{in,out}
if string == '-':
if 'r' in self._mode:
return _sys.stdin.buffer if 'b' in self._mode else _sys.stdin
elif any(c in self._mode for c in 'wax'):
return _sys.stdout.buffer if 'b' in self._mode else _sys.stdout
return _sys.stdin
elif 'w' in self._mode:
return _sys.stdout
else:
msg = _('argument "-" with mode %r') % self._mode
raise ValueError(msg)
@@ -1318,9 +1201,8 @@ class FileType(object):
return open(string, self._mode, self._bufsize, self._encoding,
self._errors)
except OSError as e:
args = {'filename': string, 'error': e}
message = _("can't open '%(filename)s': %(error)s")
raise ArgumentTypeError(message % args)
message = _("can't open '%s': %s")
raise ArgumentTypeError(message % (string, e))
def __repr__(self):
args = self._mode, self._bufsize
@@ -1383,7 +1265,6 @@ class _ActionsContainer(object):
self.register('action', 'help', _HelpAction)
self.register('action', 'version', _VersionAction)
self.register('action', 'parsers', _SubParsersAction)
self.register('action', 'extend', _ExtendAction)
# raise an exception if the conflict handler is invalid
self._get_handler()
@@ -1476,10 +1357,6 @@ class _ActionsContainer(object):
if not callable(type_func):
raise ValueError('%r is not callable' % (type_func,))
if type_func is FileType:
raise ValueError('%r is a FileType class object, instance of it'
' must be passed' % (type_func,))
# raise an error if the metavar does not match the type
if hasattr(self, "_get_formatter"):
try:
@@ -1594,8 +1471,10 @@ class _ActionsContainer(object):
# strings starting with two prefix characters are long options
option_strings.append(option_string)
if len(option_string) > 1 and option_string[1] in self.prefix_chars:
long_option_strings.append(option_string)
if option_string[0] in self.prefix_chars:
if len(option_string) > 1:
if option_string[1] in self.prefix_chars:
long_option_strings.append(option_string)
# infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
dest = kwargs.pop('dest', None)
@@ -1697,14 +1576,6 @@ class _ArgumentGroup(_ActionsContainer):
super(_ArgumentGroup, self)._remove_action(action)
self._group_actions.remove(action)
def add_argument_group(self, *args, **kwargs):
warnings.warn(
"Nesting argument groups is deprecated.",
category=DeprecationWarning,
stacklevel=2
)
return super().add_argument_group(*args, **kwargs)
class _MutuallyExclusiveGroup(_ArgumentGroup):
@@ -1725,21 +1596,12 @@ class _MutuallyExclusiveGroup(_ArgumentGroup):
self._container._remove_action(action)
self._group_actions.remove(action)
def add_mutually_exclusive_group(self, *args, **kwargs):
warnings.warn(
"Nesting mutually exclusive groups is deprecated.",
category=DeprecationWarning,
stacklevel=2
)
return super().add_mutually_exclusive_group(*args, **kwargs)
class ArgumentParser(_AttributeHolder, _ActionsContainer):
"""Object for parsing command line strings into Python objects.
Keyword Arguments:
- prog -- The name of the program (default:
``os.path.basename(sys.argv[0])``)
- prog -- The name of the program (default: sys.argv[0])
- usage -- A usage message (default: auto-generated from arguments)
- description -- A description of what the program does
- epilog -- Text following the argument descriptions
@@ -1752,8 +1614,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
- conflict_handler -- String indicating how to handle conflicts
- add_help -- Add a -h/-help option
- allow_abbrev -- Allow long options to be abbreviated unambiguously
- exit_on_error -- Determines whether or not ArgumentParser exits with
error info when an error occurs
"""
def __init__(self,
@@ -1768,14 +1628,19 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
argument_default=None,
conflict_handler='error',
add_help=True,
allow_abbrev=True,
exit_on_error=True):
superinit = super(ArgumentParser, self).__init__
superinit(description=description,
prefix_chars=prefix_chars,
argument_default=argument_default,
conflict_handler=conflict_handler)
allow_abbrev=True):
_ActionsContainer.__init__(self,
description=description,
prefix_chars=prefix_chars,
argument_default=argument_default,
conflict_handler=conflict_handler)
# FIXME: get multiple inheritance method resolution right so we can use
# what's below instead of the modified version above
# superinit = super(ArgumentParser, self).__init__
# superinit(description=description,
# prefix_chars=prefix_chars,
# argument_default=argument_default,
# conflict_handler=conflict_handler)
# default setting for prog
if prog is None:
@@ -1788,11 +1653,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
self.fromfile_prefix_chars = fromfile_prefix_chars
self.add_help = add_help
self.allow_abbrev = allow_abbrev
self.exit_on_error = exit_on_error
add_group = self.add_argument_group
self._positionals = add_group(_('positional arguments'))
self._optionals = add_group(_('options'))
self._optionals = add_group(_('optional arguments'))
self._subparsers = None
# register types
@@ -1919,18 +1783,15 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
setattr(namespace, dest, self._defaults[dest])
# parse the arguments and exit if there are any errors
if self.exit_on_error:
try:
namespace, args = self._parse_known_args(args, namespace)
except ArgumentError as err:
self.error(str(err))
else:
try:
namespace, args = self._parse_known_args(args, namespace)
if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
return namespace, args
if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
return namespace, args
except ArgumentError:
err = _sys.exc_info()[1]
self.error(str(err))
def _parse_known_args(self, arg_strings, namespace):
# replace arg strings that are file references
@@ -2026,11 +1887,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
# arguments, try to parse more single-dash options out
# of the tail of the option string
chars = self.prefix_chars
if (
arg_count == 0
and option_string[1] not in chars
and explicit_arg != ''
):
if arg_count == 0 and option_string[1] not in chars:
action_tuples.append((action, [], option_string))
char = option_string[0]
option_string = char + explicit_arg[0]
@@ -2194,16 +2051,15 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
# replace arguments referencing files with the file content
else:
try:
with open(arg_string[1:],
encoding=_sys.getfilesystemencoding(),
errors=_sys.getfilesystemencodeerrors()) as args_file:
with open(arg_string[1:]) as args_file:
arg_strings = []
for arg_line in args_file.read().splitlines():
for arg in self.convert_arg_line_to_args(arg_line):
arg_strings.append(arg)
arg_strings = self._read_args_from_files(arg_strings)
new_arg_strings.extend(arg_strings)
except OSError as err:
except OSError:
err = _sys.exc_info()[1]
self.error(str(err))
# return the modified argument list
@@ -2224,11 +2080,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
OPTIONAL: _('expected at most one argument'),
ONE_OR_MORE: _('expected at least one argument'),
}
msg = nargs_errors.get(action.nargs)
if msg is None:
msg = ngettext('expected %s argument',
default = ngettext('expected %s argument',
'expected %s arguments',
action.nargs) % action.nargs
msg = nargs_errors.get(action.nargs, default)
raise ArgumentError(action, msg)
# return the number of arguments matched
@@ -2275,23 +2130,24 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
action = self._option_string_actions[option_string]
return action, option_string, explicit_arg
# search through all possible prefixes of the option string
# and all actions in the parser for possible interpretations
option_tuples = self._get_option_tuples(arg_string)
if self.allow_abbrev:
# search through all possible prefixes of the option string
# and all actions in the parser for possible interpretations
option_tuples = self._get_option_tuples(arg_string)
# if multiple actions match, the option string was ambiguous
if len(option_tuples) > 1:
options = ', '.join([option_string
for action, option_string, explicit_arg in option_tuples])
args = {'option': arg_string, 'matches': options}
msg = _('ambiguous option: %(option)s could match %(matches)s')
self.error(msg % args)
# if multiple actions match, the option string was ambiguous
if len(option_tuples) > 1:
options = ', '.join([option_string
for action, option_string, explicit_arg in option_tuples])
args = {'option': arg_string, 'matches': options}
msg = _('ambiguous option: %(option)s could match %(matches)s')
self.error(msg % args)
# if exactly one action matched, this segmentation is good,
# so return the parsed action
elif len(option_tuples) == 1:
option_tuple, = option_tuples
return option_tuple
# if exactly one action matched, this segmentation is good,
# so return the parsed action
elif len(option_tuples) == 1:
option_tuple, = option_tuples
return option_tuple
# if it was not found as an option, but it looks like a negative
# number, it was meant to be positional
@@ -2315,17 +2171,16 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
# split at the '='
chars = self.prefix_chars
if option_string[0] in chars and option_string[1] in chars:
if self.allow_abbrev:
if '=' in option_string:
option_prefix, explicit_arg = option_string.split('=', 1)
else:
option_prefix = option_string
explicit_arg = None
for option_string in self._option_string_actions:
if option_string.startswith(option_prefix):
action = self._option_string_actions[option_string]
tup = action, option_string, explicit_arg
result.append(tup)
if '=' in option_string:
option_prefix, explicit_arg = option_string.split('=', 1)
else:
option_prefix = option_string
explicit_arg = None
for option_string in self._option_string_actions:
if option_string.startswith(option_prefix):
action = self._option_string_actions[option_string]
tup = action, option_string, explicit_arg
result.append(tup)
# single character options can be concatenated with their arguments
# but multiple character options always have to have their argument
@@ -2510,11 +2365,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
not action.option_strings):
if action.default is not None:
value = action.default
self._check_value(action, value)
else:
# since arg_strings is always [] at this point
# there is no need to use self._check_value(action, value)
value = arg_strings
self._check_value(action, value)
# single argument or optional argument produces a single value
elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
@@ -2555,8 +2408,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
result = type_func(arg_string)
# ArgumentTypeErrors indicate errors
except ArgumentTypeError as err:
msg = str(err)
except ArgumentTypeError:
name = getattr(action.type, '__name__', repr(action.type))
msg = str(_sys.exc_info()[1])
raise ArgumentError(action, msg)
# TypeErrors or ValueErrors also indicate errors
@@ -2627,11 +2481,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
def _print_message(self, message, file=None):
if message:
file = file or _sys.stderr
try:
file.write(message)
except (AttributeError, OSError):
pass
if file is None:
file = _sys.stderr
file.write(message)
# ===============
# Exiting methods

1604
Lib/ast.py vendored
View File

File diff suppressed because it is too large Load Diff

307
Lib/asynchat.py vendored
View File

@@ -1,307 +0,0 @@
# -*- Mode: Python; tab-width: 4 -*-
# Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp
# Author: Sam Rushing <rushing@nightmare.com>
# ======================================================================
# Copyright 1996 by Sam Rushing
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Sam
# Rushing not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# ======================================================================
r"""A class supporting chat-style (command/response) protocols.
This class adds support for 'chat' style protocols - where one side
sends a 'command', and the other sends a response (examples would be
the common internet protocols - smtp, nntp, ftp, etc..).
The handle_read() method looks at the input stream for the current
'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n'
for multi-line output), calling self.found_terminator() on its
receipt.
for example:
Say you build an async nntp client using this class. At the start
of the connection, you'll have self.terminator set to '\r\n', in
order to process the single-line greeting. Just before issuing a
'LIST' command you'll set it to '\r\n.\r\n'. The output of the LIST
command will be accumulated (using your own 'collect_incoming_data'
method) up to the terminator, and then control will be returned to
you - by calling your self.found_terminator() method.
"""
import asyncore
from collections import deque
class async_chat(asyncore.dispatcher):
"""This is an abstract class. You must derive from this class, and add
the two methods collect_incoming_data() and found_terminator()"""
# these are overridable defaults
ac_in_buffer_size = 65536
ac_out_buffer_size = 65536
# we don't want to enable the use of encoding by default, because that is a
# sign of an application bug that we don't want to pass silently
use_encoding = 0
encoding = 'latin-1'
def __init__(self, sock=None, map=None):
# for string terminator matching
self.ac_in_buffer = b''
# we use a list here rather than io.BytesIO for a few reasons...
# del lst[:] is faster than bio.truncate(0)
# lst = [] is faster than bio.truncate(0)
self.incoming = []
# we toss the use of the "simple producer" and replace it with
# a pure deque, which the original fifo was a wrapping of
self.producer_fifo = deque()
asyncore.dispatcher.__init__(self, sock, map)
def collect_incoming_data(self, data):
raise NotImplementedError("must be implemented in subclass")
def _collect_incoming_data(self, data):
self.incoming.append(data)
def _get_data(self):
d = b''.join(self.incoming)
del self.incoming[:]
return d
def found_terminator(self):
raise NotImplementedError("must be implemented in subclass")
def set_terminator(self, term):
"""Set the input delimiter.
Can be a fixed string of any length, an integer, or None.
"""
if isinstance(term, str) and self.use_encoding:
term = bytes(term, self.encoding)
elif isinstance(term, int) and term < 0:
raise ValueError('the number of received bytes must be positive')
self.terminator = term
def get_terminator(self):
return self.terminator
# grab some more data from the socket,
# throw it to the collector method,
# check for the terminator,
# if found, transition to the next state.
def handle_read(self):
try:
data = self.recv(self.ac_in_buffer_size)
except BlockingIOError:
return
except OSError as why:
self.handle_error()
return
if isinstance(data, str) and self.use_encoding:
data = bytes(str, self.encoding)
self.ac_in_buffer = self.ac_in_buffer + data
# Continue to search for self.terminator in self.ac_in_buffer,
# while calling self.collect_incoming_data. The while loop
# is necessary because we might read several data+terminator
# combos with a single recv(4096).
while self.ac_in_buffer:
lb = len(self.ac_in_buffer)
terminator = self.get_terminator()
if not terminator:
# no terminator, collect it all
self.collect_incoming_data(self.ac_in_buffer)
self.ac_in_buffer = b''
elif isinstance(terminator, int):
# numeric terminator
n = terminator
if lb < n:
self.collect_incoming_data(self.ac_in_buffer)
self.ac_in_buffer = b''
self.terminator = self.terminator - lb
else:
self.collect_incoming_data(self.ac_in_buffer[:n])
self.ac_in_buffer = self.ac_in_buffer[n:]
self.terminator = 0
self.found_terminator()
else:
# 3 cases:
# 1) end of buffer matches terminator exactly:
# collect data, transition
# 2) end of buffer matches some prefix:
# collect data to the prefix
# 3) end of buffer does not match any prefix:
# collect data
terminator_len = len(terminator)
index = self.ac_in_buffer.find(terminator)
if index != -1:
# we found the terminator
if index > 0:
# don't bother reporting the empty string
# (source of subtle bugs)
self.collect_incoming_data(self.ac_in_buffer[:index])
self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
# This does the Right Thing if the terminator
# is changed here.
self.found_terminator()
else:
# check for a prefix of the terminator
index = find_prefix_at_end(self.ac_in_buffer, terminator)
if index:
if index != lb:
# we found a prefix, collect up to the prefix
self.collect_incoming_data(self.ac_in_buffer[:-index])
self.ac_in_buffer = self.ac_in_buffer[-index:]
break
else:
# no prefix, collect it all
self.collect_incoming_data(self.ac_in_buffer)
self.ac_in_buffer = b''
def handle_write(self):
self.initiate_send()
def handle_close(self):
self.close()
def push(self, data):
if not isinstance(data, (bytes, bytearray, memoryview)):
raise TypeError('data argument must be byte-ish (%r)',
type(data))
sabs = self.ac_out_buffer_size
if len(data) > sabs:
for i in range(0, len(data), sabs):
self.producer_fifo.append(data[i:i+sabs])
else:
self.producer_fifo.append(data)
self.initiate_send()
def push_with_producer(self, producer):
self.producer_fifo.append(producer)
self.initiate_send()
def readable(self):
"predicate for inclusion in the readable for select()"
# cannot use the old predicate, it violates the claim of the
# set_terminator method.
# return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
return 1
def writable(self):
"predicate for inclusion in the writable for select()"
return self.producer_fifo or (not self.connected)
def close_when_done(self):
"automatically close this channel once the outgoing queue is empty"
self.producer_fifo.append(None)
def initiate_send(self):
while self.producer_fifo and self.connected:
first = self.producer_fifo[0]
# handle empty string/buffer or None entry
if not first:
del self.producer_fifo[0]
if first is None:
self.handle_close()
return
# handle classic producer behavior
obs = self.ac_out_buffer_size
try:
data = first[:obs]
except TypeError:
data = first.more()
if data:
self.producer_fifo.appendleft(data)
else:
del self.producer_fifo[0]
continue
if isinstance(data, str) and self.use_encoding:
data = bytes(data, self.encoding)
# send the data
try:
num_sent = self.send(data)
except OSError:
self.handle_error()
return
if num_sent:
if num_sent < len(data) or obs < len(first):
self.producer_fifo[0] = first[num_sent:]
else:
del self.producer_fifo[0]
# we tried to send some actual data
return
def discard_buffers(self):
# Emergencies only!
self.ac_in_buffer = b''
del self.incoming[:]
self.producer_fifo.clear()
class simple_producer:
def __init__(self, data, buffer_size=512):
self.data = data
self.buffer_size = buffer_size
def more(self):
if len(self.data) > self.buffer_size:
result = self.data[:self.buffer_size]
self.data = self.data[self.buffer_size:]
return result
else:
result = self.data
self.data = b''
return result
# Given 'haystack', see if any prefix of 'needle' is at its end. This
# assumes an exact match has already been checked. Return the number of
# characters matched.
# for example:
# f_p_a_e("qwerty\r", "\r\n") => 1
# f_p_a_e("qwertydkjf", "\r\n") => 0
# f_p_a_e("qwerty\r\n", "\r\n") => <undefined>
# this could maybe be made faster with a computed regex?
# [answer: no; circa Python-2.0, Jan 2001]
# new python: 28961/s
# old python: 18307/s
# re: 12820/s
# regex: 14035/s
def find_prefix_at_end(haystack, needle):
l = len(needle) - 1
while l and not haystack.endswith(needle[:l]):
l -= 1
return l

19
Lib/asyncio/__init__.py
View File

@@ -1,14 +1,22 @@
"""The asyncio package, tracking PEP 3156."""
# flake8: noqa
import sys
import selectors
# XXX RustPython TODO: _overlapped
if sys.platform == 'win32' and False:
# Similar thing for _overlapped.
try:
from . import _overlapped
except ImportError:
import _overlapped # Will also be exported.
# This relies on each of the submodules having an __all__ variable.
from .base_events import *
from .coroutines import *
from .events import *
from .exceptions import *
from .futures import *
from .locks import *
from .protocols import *
@@ -17,15 +25,11 @@ from .queues import *
from .streams import *
from .subprocess import *
from .tasks import *
from .taskgroups import *
from .timeouts import *
from .threads import *
from .transports import *
__all__ = (base_events.__all__ +
coroutines.__all__ +
events.__all__ +
exceptions.__all__ +
futures.__all__ +
locks.__all__ +
protocols.__all__ +
@@ -34,9 +38,6 @@ __all__ = (base_events.__all__ +
streams.__all__ +
subprocess.__all__ +
tasks.__all__ +
taskgroups.__all__ +
threads.__all__ +
timeouts.__all__ +
transports.__all__)
if sys.platform == 'win32': # pragma: no cover

125
Lib/asyncio/__main__.py
View File

@@ -1,125 +0,0 @@
import ast
import asyncio
import code
import concurrent.futures
import inspect
import sys
import threading
import types
import warnings
from . import futures
class AsyncIOInteractiveConsole(code.InteractiveConsole):
def __init__(self, locals, loop):
super().__init__(locals)
self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
self.loop = loop
def runcode(self, code):
future = concurrent.futures.Future()
def callback():
global repl_future
global repl_future_interrupted
repl_future = None
repl_future_interrupted = False
func = types.FunctionType(code, self.locals)
try:
coro = func()
except SystemExit:
raise
except KeyboardInterrupt as ex:
repl_future_interrupted = True
future.set_exception(ex)
return
except BaseException as ex:
future.set_exception(ex)
return
if not inspect.iscoroutine(coro):
future.set_result(coro)
return
try:
repl_future = self.loop.create_task(coro)
futures._chain_future(repl_future, future)
except BaseException as exc:
future.set_exception(exc)
loop.call_soon_threadsafe(callback)
try:
return future.result()
except SystemExit:
raise
except BaseException:
if repl_future_interrupted:
self.write("\nKeyboardInterrupt\n")
else:
self.showtraceback()
class REPLThread(threading.Thread):
def run(self):
try:
banner = (
f'asyncio REPL {sys.version} on {sys.platform}\n'
f'Use "await" directly instead of "asyncio.run()".\n'
f'Type "help", "copyright", "credits" or "license" '
f'for more information.\n'
f'{getattr(sys, "ps1", ">>> ")}import asyncio'
)
console.interact(
banner=banner,
exitmsg='exiting asyncio REPL...')
finally:
warnings.filterwarnings(
'ignore',
message=r'^coroutine .* was never awaited$',
category=RuntimeWarning)
loop.call_soon_threadsafe(loop.stop)
if __name__ == '__main__':
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
repl_locals = {'asyncio': asyncio}
for key in {'__name__', '__package__',
'__loader__', '__spec__',
'__builtins__', '__file__'}:
repl_locals[key] = locals()[key]
console = AsyncIOInteractiveConsole(repl_locals, loop)
repl_future = None
repl_future_interrupted = False
try:
import readline # NoQA
except ImportError:
pass
repl_thread = REPLThread()
repl_thread.daemon = True
repl_thread.start()
while True:
try:
loop.run_forever()
except KeyboardInterrupt:
if repl_future and not repl_future.done():
repl_future.cancel()
repl_future_interrupted = True
continue
else:
break

1358
Lib/asyncio/base_events.py
View File

File diff suppressed because it is too large Load Diff

38
Lib/asyncio/base_futures.py
View File

@@ -1,8 +1,18 @@
__all__ = ()
__all__ = []
import concurrent.futures._base
import reprlib
from . import format_helpers
from . import events
Error = concurrent.futures._base.Error
CancelledError = concurrent.futures.CancelledError
TimeoutError = concurrent.futures.TimeoutError
class InvalidStateError(Error):
"""The operation is not allowed in this state."""
# States for Future.
_PENDING = 'PENDING'
@@ -28,17 +38,17 @@ def _format_callbacks(cb):
cb = ''
def format_cb(callback):
return format_helpers._format_callback_source(callback, ())
return events._format_callback_source(callback, ())
if size == 1:
cb = format_cb(cb[0][0])
cb = format_cb(cb[0])
elif size == 2:
cb = '{}, {}'.format(format_cb(cb[0][0]), format_cb(cb[1][0]))
cb = '{}, {}'.format(format_cb(cb[0]), format_cb(cb[1]))
elif size > 2:
cb = '{}, <{} more>, {}'.format(format_cb(cb[0][0]),
cb = '{}, <{} more>, {}'.format(format_cb(cb[0]),
size - 2,
format_cb(cb[-1][0]))
return f'cb=[{cb}]'
format_cb(cb[-1]))
return 'cb=[%s]' % cb
def _future_repr_info(future):
@@ -47,21 +57,15 @@ def _future_repr_info(future):
info = [future._state.lower()]
if future._state == _FINISHED:
if future._exception is not None:
info.append(f'exception={future._exception!r}')
info.append('exception={!r}'.format(future._exception))
else:
# use reprlib to limit the length of the output, especially
# for very long strings
result = reprlib.repr(future._result)
info.append(f'result={result}')
info.append('result={}'.format(result))
if future._callbacks:
info.append(_format_callbacks(future._callbacks))
if future._source_traceback:
frame = future._source_traceback[-1]
info.append(f'created at {frame[0]}:{frame[1]}')
info.append('created at %s:%s' % (frame[0], frame[1]))
return info
@reprlib.recursive_repr()
def _future_repr(future):
info = ' '.join(_future_repr_info(future))
return f'<{future.__class__.__name__} {info}>'

78
Lib/asyncio/base_subprocess.py
View File

@@ -2,8 +2,10 @@ import collections
import subprocess
import warnings
from . import compat
from . import protocols
from . import transports
from .coroutines import coroutine
from .log import logger
@@ -57,9 +59,9 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
if self._closed:
info.append('closed')
if self._pid is not None:
info.append(f'pid={self._pid}')
info.append('pid=%s' % self._pid)
if self._returncode is not None:
info.append(f'returncode={self._returncode}')
info.append('returncode=%s' % self._returncode)
elif self._pid is not None:
info.append('running')
else:
@@ -67,19 +69,19 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
stdin = self._pipes.get(0)
if stdin is not None:
info.append(f'stdin={stdin.pipe}')
info.append('stdin=%s' % stdin.pipe)
stdout = self._pipes.get(1)
stderr = self._pipes.get(2)
if stdout is not None and stderr is stdout:
info.append(f'stdout=stderr={stdout.pipe}')
info.append('stdout=stderr=%s' % stdout.pipe)
else:
if stdout is not None:
info.append(f'stdout={stdout.pipe}')
info.append('stdout=%s' % stdout.pipe)
if stderr is not None:
info.append(f'stderr={stderr.pipe}')
info.append('stderr=%s' % stderr.pipe)
return '<{}>'.format(' '.join(info))
return '<%s>' % ' '.join(info)
def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
raise NotImplementedError
@@ -103,13 +105,12 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
continue
proto.pipe.close()
if (self._proc is not None and
# has the child process finished?
self._returncode is None and
# the child process has finished, but the
# transport hasn't been notified yet?
self._proc.poll() is None):
if (self._proc is not None
# the child process finished?
and self._returncode is None
# the child process finished but the transport was not notified yet?
and self._proc.poll() is None
):
if self._loop.get_debug():
logger.warning('Close running child process: kill %r', self)
@@ -120,10 +121,15 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
# Don't clear the _proc reference yet: _post_init() may still run
def __del__(self, _warn=warnings.warn):
if not self._closed:
_warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
self.close()
# On Python 3.3 and older, objects with a destructor part of a reference
# cycle are never destroyed. It's not more the case on Python 3.4 thanks
# to the PEP 442.
if compat.PY34:
def __del__(self):
if not self._closed:
warnings.warn("unclosed transport %r" % self, ResourceWarning,
source=self)
self.close()
def get_pid(self):
return self._pid
@@ -153,25 +159,26 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
self._check_proc()
self._proc.kill()
async def _connect_pipes(self, waiter):
@coroutine
def _connect_pipes(self, waiter):
try:
proc = self._proc
loop = self._loop
if proc.stdin is not None:
_, pipe = await loop.connect_write_pipe(
_, pipe = yield from loop.connect_write_pipe(
lambda: WriteSubprocessPipeProto(self, 0),
proc.stdin)
self._pipes[0] = pipe
if proc.stdout is not None:
_, pipe = await loop.connect_read_pipe(
_, pipe = yield from loop.connect_read_pipe(
lambda: ReadSubprocessPipeProto(self, 1),
proc.stdout)
self._pipes[1] = pipe
if proc.stderr is not None:
_, pipe = await loop.connect_read_pipe(
_, pipe = yield from loop.connect_read_pipe(
lambda: ReadSubprocessPipeProto(self, 2),
proc.stderr)
self._pipes[2] = pipe
@@ -182,9 +189,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
for callback, data in self._pending_calls:
loop.call_soon(callback, *data)
self._pending_calls = None
except (SystemExit, KeyboardInterrupt):
raise
except BaseException as exc:
except Exception as exc:
if waiter is not None and not waiter.cancelled():
waiter.set_exception(exc)
else:
@@ -208,17 +213,24 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
assert returncode is not None, returncode
assert self._returncode is None, self._returncode
if self._loop.get_debug():
logger.info('%r exited with return code %r', self, returncode)
logger.info('%r exited with return code %r',
self, returncode)
self._returncode = returncode
if self._proc.returncode is None:
# asyncio uses a child watcher: copy the status into the Popen
# object. On Python 3.6, it is required to avoid a ResourceWarning.
self._proc.returncode = returncode
self._call(self._protocol.process_exited)
self._try_finish()
async def _wait(self):
# wake up futures waiting for wait()
for waiter in self._exit_waiters:
if not waiter.cancelled():
waiter.set_result(returncode)
self._exit_waiters = None
@coroutine
def _wait(self):
"""Wait until the process exit and return the process return code.
This method is a coroutine."""
@@ -227,7 +239,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
waiter = self._loop.create_future()
self._exit_waiters.append(waiter)
return await waiter
return (yield from waiter)
def _try_finish(self):
assert not self._finished
@@ -242,11 +254,6 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
try:
self._protocol.connection_lost(exc)
finally:
# wake up futures waiting for wait()
for waiter in self._exit_waiters:
if not waiter.cancelled():
waiter.set_result(self._returncode)
self._exit_waiters = None
self._loop = None
self._proc = None
self._protocol = None
@@ -264,7 +271,8 @@ class WriteSubprocessPipeProto(protocols.BaseProtocol):
self.pipe = transport
def __repr__(self):
return f'<{self.__class__.__name__} fd={self.fd} pipe={self.pipe!r}>'
return ('<%s fd=%s pipe=%r>'
% (self.__class__.__name__, self.fd, self.pipe))
def connection_lost(self, exc):
self.disconnected = True

42
Lib/asyncio/base_tasks.py
View File

@@ -1,5 +1,4 @@
import linecache
import reprlib
import traceback
from . import base_futures
@@ -9,42 +8,25 @@ from . import coroutines
def _task_repr_info(task):
info = base_futures._future_repr_info(task)
if task.cancelling() and not task.done():
if task._must_cancel:
# replace status
info[0] = 'cancelling'
info.insert(1, 'name=%r' % task.get_name())
coro = coroutines._format_coroutine(task._coro)
info.insert(1, 'coro=<%s>' % coro)
if task._fut_waiter is not None:
info.insert(2, f'wait_for={task._fut_waiter!r}')
if task._coro:
coro = coroutines._format_coroutine(task._coro)
info.insert(2, f'coro=<{coro}>')
info.insert(2, 'wait_for=%r' % task._fut_waiter)
return info
@reprlib.recursive_repr()
def _task_repr(task):
info = ' '.join(_task_repr_info(task))
return f'<{task.__class__.__name__} {info}>'
def _task_get_stack(task, limit):
frames = []
if hasattr(task._coro, 'cr_frame'):
# case 1: 'async def' coroutines
try:
# 'async def' coroutines
f = task._coro.cr_frame
elif hasattr(task._coro, 'gi_frame'):
# case 2: legacy coroutines
except AttributeError:
f = task._coro.gi_frame
elif hasattr(task._coro, 'ag_frame'):
# case 3: async generators
f = task._coro.ag_frame
else:
# case 4: unknown objects
f = None
if f is not None:
while f is not None:
if limit is not None:
@@ -79,15 +61,15 @@ def _task_print_stack(task, limit, file):
linecache.checkcache(filename)
line = linecache.getline(filename, lineno, f.f_globals)
extracted_list.append((filename, lineno, name, line))
exc = task._exception
if not extracted_list:
print(f'No stack for {task!r}', file=file)
print('No stack for %r' % task, file=file)
elif exc is not None:
print(f'Traceback for {task!r} (most recent call last):', file=file)
print('Traceback for %r (most recent call last):' % task,
file=file)
else:
print(f'Stack for {task!r} (most recent call last):', file=file)
print('Stack for %r (most recent call last):' % task,
file=file)
traceback.print_list(extracted_list, file=file)
if exc is not None:
for line in traceback.format_exception_only(exc.__class__, exc):

18
Lib/asyncio/compat.py Normal file
View File

@@ -0,0 +1,18 @@
"""Compatibility helpers for the different Python versions."""
import sys
PY34 = sys.version_info >= (3, 4)
PY35 = sys.version_info >= (3, 5)
PY352 = sys.version_info >= (3, 5, 2)
def flatten_list_bytes(list_of_data):
"""Concatenate a sequence of bytes-like objects."""
if not PY34:
# On Python 3.3 and older, bytes.join() doesn't handle
# memoryview.
list_of_data = (
bytes(data) if isinstance(data, memoryview) else data
for data in list_of_data)
return b''.join(list_of_data)

36
Lib/asyncio/constants.py
View File

@@ -1,41 +1,7 @@
# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0
# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0)
# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io
import enum
"""Constants."""
# After the connection is lost, log warnings after this many write()s.
LOG_THRESHOLD_FOR_CONNLOST_WRITES = 5
# Seconds to wait before retrying accept().
ACCEPT_RETRY_DELAY = 1
# Number of stack entries to capture in debug mode.
# The larger the number, the slower the operation in debug mode
# (see extract_stack() in format_helpers.py).
DEBUG_STACK_DEPTH = 10
# Number of seconds to wait for SSL handshake to complete
# The default timeout matches that of Nginx.
SSL_HANDSHAKE_TIMEOUT = 60.0
# Number of seconds to wait for SSL shutdown to complete
# The default timeout mimics lingering_time
SSL_SHUTDOWN_TIMEOUT = 30.0
# Used in sendfile fallback code. We use fallback for platforms
# that don't support sendfile, or for TLS connections.
SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 256
FLOW_CONTROL_HIGH_WATER_SSL_READ = 256 # KiB
FLOW_CONTROL_HIGH_WATER_SSL_WRITE = 512 # KiB
# Default timeout for joining the threads in the threadpool
THREAD_JOIN_TIMEOUT = 300
# The enum should be here to break circular dependencies between
# base_events and sslproto
class _SendfileMode(enum.Enum):
UNSUPPORTED = enum.auto()
TRY_NATIVE = enum.auto()
FALLBACK = enum.auto()

365
Lib/asyncio/coroutines.py
View File

@@ -1,16 +1,249 @@
__all__ = 'iscoroutinefunction', 'iscoroutine'
__all__ = ['coroutine',
'iscoroutinefunction', 'iscoroutine']
import collections.abc
import functools
import inspect
import opcode
import os
import sys
import traceback
import types
from . import compat
from . import events
from . import base_futures
from .log import logger
def _is_debug_mode():
# See: https://docs.python.org/3/library/asyncio-dev.html#asyncio-debug-mode.
return sys.flags.dev_mode or (not sys.flags.ignore_environment and
bool(os.environ.get('PYTHONASYNCIODEBUG')))
# Opcode of "yield from" instruction
_YIELD_FROM = opcode.opmap['YIELD_FROM']
# If you set _DEBUG to true, @coroutine will wrap the resulting
# generator objects in a CoroWrapper instance (defined below). That
# instance will log a message when the generator is never iterated
# over, which may happen when you forget to use "yield from" with a
# coroutine call. Note that the value of the _DEBUG flag is taken
# when the decorator is used, so to be of any use it must be set
# before you define your coroutines. A downside of using this feature
# is that tracebacks show entries for the CoroWrapper.__next__ method
# when _DEBUG is true.
_DEBUG = (not sys.flags.ignore_environment and
bool(os.environ.get('PYTHONASYNCIODEBUG')))
try:
_types_coroutine = types.coroutine
_types_CoroutineType = types.CoroutineType
except AttributeError:
# Python 3.4
_types_coroutine = None
_types_CoroutineType = None
try:
_inspect_iscoroutinefunction = inspect.iscoroutinefunction
except AttributeError:
# Python 3.4
_inspect_iscoroutinefunction = lambda func: False
try:
from collections.abc import Coroutine as _CoroutineABC, \
Awaitable as _AwaitableABC
except ImportError:
_CoroutineABC = _AwaitableABC = None
# Check for CPython issue #21209
def has_yield_from_bug():
class MyGen:
def __init__(self):
self.send_args = None
def __iter__(self):
return self
def __next__(self):
return 42
def send(self, *what):
self.send_args = what
return None
def yield_from_gen(gen):
yield from gen
value = (1, 2, 3)
gen = MyGen()
coro = yield_from_gen(gen)
next(coro)
coro.send(value)
return gen.send_args != (value,)
_YIELD_FROM_BUG = has_yield_from_bug()
del has_yield_from_bug
def debug_wrapper(gen):
# This function is called from 'sys.set_coroutine_wrapper'.
# We only wrap here coroutines defined via 'async def' syntax.
# Generator-based coroutines are wrapped in @coroutine
# decorator.
return CoroWrapper(gen, None)
class CoroWrapper:
# Wrapper for coroutine object in _DEBUG mode.
def __init__(self, gen, func=None):
assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen
self.gen = gen
self.func = func # Used to unwrap @coroutine decorator
self._source_traceback = traceback.extract_stack(sys._getframe(1))
self.__name__ = getattr(gen, '__name__', None)
self.__qualname__ = getattr(gen, '__qualname__', None)
def __repr__(self):
coro_repr = _format_coroutine(self)
if self._source_traceback:
frame = self._source_traceback[-1]
coro_repr += ', created at %s:%s' % (frame[0], frame[1])
return '<%s %s>' % (self.__class__.__name__, coro_repr)
def __iter__(self):
return self
def __next__(self):
return self.gen.send(None)
if _YIELD_FROM_BUG:
# For for CPython issue #21209: using "yield from" and a custom
# generator, generator.send(tuple) unpacks the tuple instead of passing
# the tuple unchanged. Check if the caller is a generator using "yield
# from" to decide if the parameter should be unpacked or not.
def send(self, *value):
frame = sys._getframe()
caller = frame.f_back
assert caller.f_lasti >= 0
if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM:
value = value[0]
return self.gen.send(value)
else:
def send(self, value):
return self.gen.send(value)
def throw(self, type, value=None, traceback=None):
return self.gen.throw(type, value, traceback)
def close(self):
return self.gen.close()
@property
def gi_frame(self):
return self.gen.gi_frame
@property
def gi_running(self):
return self.gen.gi_running
@property
def gi_code(self):
return self.gen.gi_code
if compat.PY35:
def __await__(self):
cr_await = getattr(self.gen, 'cr_await', None)
if cr_await is not None:
raise RuntimeError(
"Cannot await on coroutine {!r} while it's "
"awaiting for {!r}".format(self.gen, cr_await))
return self
@property
def gi_yieldfrom(self):
return self.gen.gi_yieldfrom
@property
def cr_await(self):
return self.gen.cr_await
@property
def cr_running(self):
return self.gen.cr_running
@property
def cr_code(self):
return self.gen.cr_code
@property
def cr_frame(self):
return self.gen.cr_frame
def __del__(self):
# Be careful accessing self.gen.frame -- self.gen might not exist.
gen = getattr(self, 'gen', None)
frame = getattr(gen, 'gi_frame', None)
if frame is None:
frame = getattr(gen, 'cr_frame', None)
if frame is not None and frame.f_lasti == -1:
msg = '%r was never yielded from' % self
tb = getattr(self, '_source_traceback', ())
if tb:
tb = ''.join(traceback.format_list(tb))
msg += ('\nCoroutine object created at '
'(most recent call last):\n')
msg += tb.rstrip()
logger.error(msg)
def coroutine(func):
"""Decorator to mark coroutines.
If the coroutine is not yielded from before it is destroyed,
an error message is logged.
"""
if _inspect_iscoroutinefunction(func):
# In Python 3.5 that's all we need to do for coroutines
# defiend with "async def".
# Wrapping in CoroWrapper will happen via
# 'sys.set_coroutine_wrapper' function.
return func
if inspect.isgeneratorfunction(func):
coro = func
else:
@functools.wraps(func)
def coro(*args, **kw):
res = func(*args, **kw)
if (base_futures.isfuture(res) or inspect.isgenerator(res) or
isinstance(res, CoroWrapper)):
res = yield from res
elif _AwaitableABC is not None:
# If 'func' returns an Awaitable (new in 3.5) we
# want to run it.
try:
await_meth = res.__await__
except AttributeError:
pass
else:
if isinstance(res, _AwaitableABC):
res = yield from await_meth()
return res
if not _DEBUG:
if _types_coroutine is None:
wrapper = coro
else:
wrapper = _types_coroutine(coro)
else:
@functools.wraps(func)
def wrapper(*args, **kwds):
w = CoroWrapper(coro(*args, **kwds), func=func)
if w._source_traceback:
del w._source_traceback[-1]
# Python < 3.5 does not implement __qualname__
# on generator objects, so we set it manually.
# We use getattr as some callables (such as
# functools.partial may lack __qualname__).
w.__name__ = getattr(func, '__name__', None)
w.__qualname__ = getattr(func, '__qualname__', None)
return w
wrapper._is_coroutine = _is_coroutine # For iscoroutinefunction().
return wrapper
# A marker for iscoroutinefunction.
@@ -19,91 +252,93 @@ _is_coroutine = object()
def iscoroutinefunction(func):
"""Return True if func is a decorated coroutine function."""
return (inspect.iscoroutinefunction(func) or
getattr(func, '_is_coroutine', None) is _is_coroutine)
return (getattr(func, '_is_coroutine', None) is _is_coroutine or
_inspect_iscoroutinefunction(func))
# Prioritize native coroutine check to speed-up
# asyncio.iscoroutine.
_COROUTINE_TYPES = (types.CoroutineType, collections.abc.Coroutine)
_iscoroutine_typecache = set()
_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper)
if _CoroutineABC is not None:
_COROUTINE_TYPES += (_CoroutineABC,)
if _types_CoroutineType is not None:
# Prioritize native coroutine check to speed-up
# asyncio.iscoroutine.
_COROUTINE_TYPES = (_types_CoroutineType,) + _COROUTINE_TYPES
def iscoroutine(obj):
"""Return True if obj is a coroutine object."""
if type(obj) in _iscoroutine_typecache:
return True
if isinstance(obj, _COROUTINE_TYPES):
# Just in case we don't want to cache more than 100
# positive types. That shouldn't ever happen, unless
# someone stressing the system on purpose.
if len(_iscoroutine_typecache) < 100:
_iscoroutine_typecache.add(type(obj))
return True
else:
return False
return isinstance(obj, _COROUTINE_TYPES)
def _format_coroutine(coro):
assert iscoroutine(coro)
def get_name(coro):
# Coroutines compiled with Cython sometimes don't have
# proper __qualname__ or __name__. While that is a bug
# in Cython, asyncio shouldn't crash with an AttributeError
# in its __repr__ functions.
if hasattr(coro, '__qualname__') and coro.__qualname__:
coro_name = coro.__qualname__
elif hasattr(coro, '__name__') and coro.__name__:
coro_name = coro.__name__
else:
# Stop masking Cython bugs, expose them in a friendly way.
coro_name = f'<{type(coro).__name__} without __name__>'
return f'{coro_name}()'
if not hasattr(coro, 'cr_code') and not hasattr(coro, 'gi_code'):
# Most likely a built-in type or a Cython coroutine.
def is_running(coro):
# Built-in types might not have __qualname__ or __name__.
coro_name = getattr(
coro, '__qualname__',
getattr(coro, '__name__', type(coro).__name__))
coro_name = '{}()'.format(coro_name)
running = False
try:
return coro.cr_running
running = coro.cr_running
except AttributeError:
try:
return coro.gi_running
running = coro.gi_running
except AttributeError:
return False
pass
coro_code = None
if hasattr(coro, 'cr_code') and coro.cr_code:
coro_code = coro.cr_code
elif hasattr(coro, 'gi_code') and coro.gi_code:
coro_code = coro.gi_code
coro_name = get_name(coro)
if not coro_code:
# Built-in types might not have __qualname__ or __name__.
if is_running(coro):
return f'{coro_name} running'
if running:
return '{} running'.format(coro_name)
else:
return coro_name
coro_frame = None
if hasattr(coro, 'gi_frame') and coro.gi_frame:
coro_name = None
if isinstance(coro, CoroWrapper):
func = coro.func
coro_name = coro.__qualname__
if coro_name is not None:
coro_name = '{}()'.format(coro_name)
else:
func = coro
if coro_name is None:
coro_name = events._format_callback(func, (), {})
try:
coro_code = coro.gi_code
except AttributeError:
coro_code = coro.cr_code
try:
coro_frame = coro.gi_frame
elif hasattr(coro, 'cr_frame') and coro.cr_frame:
except AttributeError:
coro_frame = coro.cr_frame
# If Cython's coroutine has a fake code object without proper
# co_filename -- expose that.
filename = coro_code.co_filename or '<empty co_filename>'
filename = coro_code.co_filename
lineno = 0
if coro_frame is not None:
if (isinstance(coro, CoroWrapper) and
not inspect.isgeneratorfunction(coro.func) and
coro.func is not None):
source = events._get_function_source(coro.func)
if source is not None:
filename, lineno = source
if coro_frame is None:
coro_repr = ('%s done, defined at %s:%s'
% (coro_name, filename, lineno))
else:
coro_repr = ('%s running, defined at %s:%s'
% (coro_name, filename, lineno))
elif coro_frame is not None:
lineno = coro_frame.f_lineno
coro_repr = f'{coro_name} running at {filename}:{lineno}'
coro_repr = ('%s running at %s:%s'
% (coro_name, filename, lineno))
else:
lineno = coro_code.co_firstlineno
coro_repr = f'{coro_name} done, defined at {filename}:{lineno}'
coro_repr = ('%s done, defined at %s:%s'
% (coro_name, filename, lineno))
return coro_repr

469
Lib/asyncio/events.py
View File

@@ -1,50 +1,96 @@
"""Event loop and event loop policy."""
# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0
# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0)
# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io
__all__ = ['AbstractEventLoopPolicy',
'AbstractEventLoop', 'AbstractServer',
'Handle', 'TimerHandle',
'get_event_loop_policy', 'set_event_loop_policy',
'get_event_loop', 'set_event_loop', 'new_event_loop',
'get_child_watcher', 'set_child_watcher',
'_set_running_loop', 'get_running_loop',
'_get_running_loop',
]
__all__ = (
'AbstractEventLoopPolicy',
'AbstractEventLoop', 'AbstractServer',
'Handle', 'TimerHandle',
'get_event_loop_policy', 'set_event_loop_policy',
'get_event_loop', 'set_event_loop', 'new_event_loop',
'get_child_watcher', 'set_child_watcher',
'_set_running_loop', 'get_running_loop',
'_get_running_loop',
)
import contextvars
import os
import signal
import functools
import inspect
import reprlib
import socket
import subprocess
import sys
import threading
import traceback
from . import format_helpers
from asyncio import compat
def _get_function_source(func):
if compat.PY34:
func = inspect.unwrap(func)
elif hasattr(func, '__wrapped__'):
func = func.__wrapped__
if inspect.isfunction(func):
code = func.__code__
return (code.co_filename, code.co_firstlineno)
if isinstance(func, functools.partial):
return _get_function_source(func.func)
if compat.PY34 and isinstance(func, functools.partialmethod):
return _get_function_source(func.func)
return None
def _format_args_and_kwargs(args, kwargs):
"""Format function arguments and keyword arguments.
Special case for a single parameter: ('hello',) is formatted as ('hello').
"""
# use reprlib to limit the length of the output
items = []
if args:
items.extend(reprlib.repr(arg) for arg in args)
if kwargs:
items.extend('{}={}'.format(k, reprlib.repr(v))
for k, v in kwargs.items())
return '(' + ', '.join(items) + ')'
def _format_callback(func, args, kwargs, suffix=''):
if isinstance(func, functools.partial):
suffix = _format_args_and_kwargs(args, kwargs) + suffix
return _format_callback(func.func, func.args, func.keywords, suffix)
if hasattr(func, '__qualname__'):
func_repr = getattr(func, '__qualname__')
elif hasattr(func, '__name__'):
func_repr = getattr(func, '__name__')
else:
func_repr = repr(func)
func_repr += _format_args_and_kwargs(args, kwargs)
if suffix:
func_repr += suffix
return func_repr
def _format_callback_source(func, args):
func_repr = _format_callback(func, args, None)
source = _get_function_source(func)
if source:
func_repr += ' at %s:%s' % source
return func_repr
class Handle:
"""Object returned by callback registration methods."""
__slots__ = ('_callback', '_args', '_cancelled', '_loop',
'_source_traceback', '_repr', '__weakref__',
'_context')
'_source_traceback', '_repr', '__weakref__')
def __init__(self, callback, args, loop, context=None):
if context is None:
context = contextvars.copy_context()
self._context = context
def __init__(self, callback, args, loop):
self._loop = loop
self._callback = callback
self._args = args
self._cancelled = False
self._repr = None
if self._loop.get_debug():
self._source_traceback = format_helpers.extract_stack(
sys._getframe(1))
self._source_traceback = traceback.extract_stack(sys._getframe(1))
else:
self._source_traceback = None
@@ -53,21 +99,17 @@ class Handle:
if self._cancelled:
info.append('cancelled')
if self._callback is not None:
info.append(format_helpers._format_callback_source(
self._callback, self._args))
info.append(_format_callback_source(self._callback, self._args))
if self._source_traceback:
frame = self._source_traceback[-1]
info.append(f'created at {frame[0]}:{frame[1]}')
info.append('created at %s:%s' % (frame[0], frame[1]))
return info
def __repr__(self):
if self._repr is not None:
return self._repr
info = self._repr_info()
return '<{}>'.format(' '.join(info))
def get_context(self):
return self._context
return '<%s>' % ' '.join(info)
def cancel(self):
if not self._cancelled:
@@ -80,18 +122,12 @@ class Handle:
self._callback = None
self._args = None
def cancelled(self):
return self._cancelled
def _run(self):
try:
self._context.run(self._callback, *self._args)
except (SystemExit, KeyboardInterrupt):
raise
except BaseException as exc:
cb = format_helpers._format_callback_source(
self._callback, self._args)
msg = f'Exception in callback {cb}'
self._callback(*self._args)
except Exception as exc:
cb = _format_callback_source(self._callback, self._args)
msg = 'Exception in callback {}'.format(cb)
context = {
'message': msg,
'exception': exc,
@@ -108,8 +144,9 @@ class TimerHandle(Handle):
__slots__ = ['_scheduled', '_when']
def __init__(self, when, callback, args, loop, context=None):
super().__init__(callback, args, loop, context)
def __init__(self, when, callback, args, loop):
assert when is not None
super().__init__(callback, args, loop)
if self._source_traceback:
del self._source_traceback[-1]
self._when = when
@@ -118,31 +155,27 @@ class TimerHandle(Handle):
def _repr_info(self):
info = super()._repr_info()
pos = 2 if self._cancelled else 1
info.insert(pos, f'when={self._when}')
info.insert(pos, 'when=%s' % self._when)
return info
def __hash__(self):
return hash(self._when)
def __lt__(self, other):
if isinstance(other, TimerHandle):
return self._when < other._when
return NotImplemented
return self._when < other._when
def __le__(self, other):
if isinstance(other, TimerHandle):
return self._when < other._when or self.__eq__(other)
return NotImplemented
if self._when < other._when:
return True
return self.__eq__(other)
def __gt__(self, other):
if isinstance(other, TimerHandle):
return self._when > other._when
return NotImplemented
return self._when > other._when
def __ge__(self, other):
if isinstance(other, TimerHandle):
return self._when > other._when or self.__eq__(other)
return NotImplemented
if self._when > other._when:
return True
return self.__eq__(other)
def __eq__(self, other):
if isinstance(other, TimerHandle):
@@ -152,60 +185,26 @@ class TimerHandle(Handle):
self._cancelled == other._cancelled)
return NotImplemented
def __ne__(self, other):
equal = self.__eq__(other)
return NotImplemented if equal is NotImplemented else not equal
def cancel(self):
if not self._cancelled:
self._loop._timer_handle_cancelled(self)
super().cancel()
def when(self):
"""Return a scheduled callback time.
The time is an absolute timestamp, using the same time
reference as loop.time().
"""
return self._when
class AbstractServer:
"""Abstract server returned by create_server()."""
def close(self):
"""Stop serving. This leaves existing connections open."""
raise NotImplementedError
return NotImplemented
def get_loop(self):
"""Get the event loop the Server object is attached to."""
raise NotImplementedError
def is_serving(self):
"""Return True if the server is accepting connections."""
raise NotImplementedError
async def start_serving(self):
"""Start accepting connections.
This method is idempotent, so it can be called when
the server is already being serving.
"""
raise NotImplementedError
async def serve_forever(self):
"""Start accepting connections until the coroutine is cancelled.
The server is closed when the coroutine is cancelled.
"""
raise NotImplementedError
async def wait_closed(self):
def wait_closed(self):
"""Coroutine to wait until service is closed."""
raise NotImplementedError
async def __aenter__(self):
return self
async def __aexit__(self, *exc):
self.close()
await self.wait_closed()
return NotImplemented
class AbstractEventLoop:
@@ -251,27 +250,23 @@ class AbstractEventLoop:
"""
raise NotImplementedError
async def shutdown_asyncgens(self):
def shutdown_asyncgens(self):
"""Shutdown all active asynchronous generators."""
raise NotImplementedError
async def shutdown_default_executor(self):
"""Schedule the shutdown of the default executor."""
raise NotImplementedError
# Methods scheduling callbacks. All these return Handles.
def _timer_handle_cancelled(self, handle):
"""Notification that a TimerHandle has been cancelled."""
raise NotImplementedError
def call_soon(self, callback, *args, context=None):
return self.call_later(0, callback, *args, context=context)
def call_soon(self, callback, *args):
return self.call_later(0, callback, *args)
def call_later(self, delay, callback, *args, context=None):
def call_later(self, delay, callback, *args):
raise NotImplementedError
def call_at(self, when, callback, *args, context=None):
def call_at(self, when, callback, *args):
raise NotImplementedError
def time(self):
@@ -282,12 +277,12 @@ class AbstractEventLoop:
# Method scheduling a coroutine object: create a task.
def create_task(self, coro, *, name=None, context=None):
def create_task(self, coro):
raise NotImplementedError
# Methods for interacting with threads.
def call_soon_threadsafe(self, callback, *args, context=None):
def call_soon_threadsafe(self, callback, *args):
raise NotImplementedError
def run_in_executor(self, executor, func, *args):
@@ -298,31 +293,21 @@ class AbstractEventLoop:
# Network I/O methods returning Futures.
async def getaddrinfo(self, host, port, *,
family=0, type=0, proto=0, flags=0):
def getaddrinfo(self, host, port, *, family=0, type=0, proto=0, flags=0):
raise NotImplementedError
async def getnameinfo(self, sockaddr, flags=0):
def getnameinfo(self, sockaddr, flags=0):
raise NotImplementedError
async def create_connection(
self, protocol_factory, host=None, port=None,
*, ssl=None, family=0, proto=0,
flags=0, sock=None, local_addr=None,
server_hostname=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None,
happy_eyeballs_delay=None, interleave=None):
def create_connection(self, protocol_factory, host=None, port=None, *,
ssl=None, family=0, proto=0, flags=0, sock=None,
local_addr=None, server_hostname=None):
raise NotImplementedError
async def create_server(
self, protocol_factory, host=None, port=None,
*, family=socket.AF_UNSPEC,
flags=socket.AI_PASSIVE, sock=None, backlog=100,
ssl=None, reuse_address=None, reuse_port=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None,
start_serving=True):
def create_server(self, protocol_factory, host=None, port=None, *,
family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE,
sock=None, backlog=100, ssl=None, reuse_address=None,
reuse_port=None):
"""A coroutine which creates a TCP server bound to host and port.
The return value is a Server object which can be used to stop
@@ -330,8 +315,8 @@ class AbstractEventLoop:
If host is an empty string or None all interfaces are assumed
and a list of multiple sockets will be returned (most likely
one for IPv4 and another one for IPv6). The host parameter can also be
a sequence (e.g. list) of hosts to bind to.
one for IPv4 and another one for IPv6). The host parameter can also be a
sequence (e.g. list) of hosts to bind to.
family can be set to either AF_INET or AF_INET6 to force the
socket to use IPv4 or IPv6. If not set it will be determined
@@ -357,62 +342,22 @@ class AbstractEventLoop:
the same port as other existing endpoints are bound to, so long as
they all set this flag when being created. This option is not
supported on Windows.
ssl_handshake_timeout is the time in seconds that an SSL server
will wait for completion of the SSL handshake before aborting the
connection. Default is 60s.
ssl_shutdown_timeout is the time in seconds that an SSL server
will wait for completion of the SSL shutdown procedure
before aborting the connection. Default is 30s.
start_serving set to True (default) causes the created server
to start accepting connections immediately. When set to False,
the user should await Server.start_serving() or Server.serve_forever()
to make the server to start accepting connections.
"""
raise NotImplementedError
async def sendfile(self, transport, file, offset=0, count=None,
*, fallback=True):
"""Send a file through a transport.
Return an amount of sent bytes.
"""
def create_unix_connection(self, protocol_factory, path, *,
ssl=None, sock=None,
server_hostname=None):
raise NotImplementedError
async def start_tls(self, transport, protocol, sslcontext, *,
server_side=False,
server_hostname=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None):
"""Upgrade a transport to TLS.
Return a new transport that *protocol* should start using
immediately.
"""
raise NotImplementedError
async def create_unix_connection(
self, protocol_factory, path=None, *,
ssl=None, sock=None,
server_hostname=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None):
raise NotImplementedError
async def create_unix_server(
self, protocol_factory, path=None, *,
sock=None, backlog=100, ssl=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None,
start_serving=True):
def create_unix_server(self, protocol_factory, path, *,
sock=None, backlog=100, ssl=None):
"""A coroutine which creates a UNIX Domain Socket server.
The return value is a Server object, which can be used to stop
the service.
path is a str, representing a file system path to bind the
path is a str, representing a file systsem path to bind the
server socket to.
sock can optionally be specified in order to use a preexisting
@@ -423,40 +368,14 @@ class AbstractEventLoop:
ssl can be set to an SSLContext to enable SSL over the
accepted connections.
ssl_handshake_timeout is the time in seconds that an SSL server
will wait for the SSL handshake to complete (defaults to 60s).
ssl_shutdown_timeout is the time in seconds that an SSL server
will wait for the SSL shutdown to finish (defaults to 30s).
start_serving set to True (default) causes the created server
to start accepting connections immediately. When set to False,
the user should await Server.start_serving() or Server.serve_forever()
to make the server to start accepting connections.
"""
raise NotImplementedError
async def connect_accepted_socket(
self, protocol_factory, sock,
*, ssl=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None):
"""Handle an accepted connection.
This is used by servers that accept connections outside of
asyncio, but use asyncio to handle connections.
This method is a coroutine. When completed, the coroutine
returns a (transport, protocol) pair.
"""
raise NotImplementedError
async def create_datagram_endpoint(self, protocol_factory,
local_addr=None, remote_addr=None, *,
family=0, proto=0, flags=0,
reuse_address=None, reuse_port=None,
allow_broadcast=None, sock=None):
def create_datagram_endpoint(self, protocol_factory,
local_addr=None, remote_addr=None, *,
family=0, proto=0, flags=0,
reuse_address=None, reuse_port=None,
allow_broadcast=None, sock=None):
"""A coroutine which creates a datagram endpoint.
This method will try to establish the endpoint in the background.
@@ -464,8 +383,8 @@ class AbstractEventLoop:
protocol_factory must be a callable returning a protocol instance.
socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending on
host (or family if specified), socket type SOCK_DGRAM.
socket family AF_INET or socket.AF_INET6 depending on host (or
family if specified), socket type SOCK_DGRAM.
reuse_address tells the kernel to reuse a local socket in
TIME_WAIT state, without waiting for its natural timeout to
@@ -489,7 +408,7 @@ class AbstractEventLoop:
# Pipes and subprocesses.
async def connect_read_pipe(self, protocol_factory, pipe):
def connect_read_pipe(self, protocol_factory, pipe):
"""Register read pipe in event loop. Set the pipe to non-blocking mode.
protocol_factory should instantiate object with Protocol interface.
@@ -499,10 +418,10 @@ class AbstractEventLoop:
# The reason to accept file-like object instead of just file descriptor
# is: we need to own pipe and close it at transport finishing
# Can got complicated errors if pass f.fileno(),
# close fd in pipe transport then close f and vice versa.
# close fd in pipe transport then close f and vise versa.
raise NotImplementedError
async def connect_write_pipe(self, protocol_factory, pipe):
def connect_write_pipe(self, protocol_factory, pipe):
"""Register write pipe in event loop.
protocol_factory should instantiate object with BaseProtocol interface.
@@ -512,21 +431,17 @@ class AbstractEventLoop:
# The reason to accept file-like object instead of just file descriptor
# is: we need to own pipe and close it at transport finishing
# Can got complicated errors if pass f.fileno(),
# close fd in pipe transport then close f and vice versa.
# close fd in pipe transport then close f and vise versa.
raise NotImplementedError
async def subprocess_shell(self, protocol_factory, cmd, *,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**kwargs):
def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
**kwargs):
raise NotImplementedError
async def subprocess_exec(self, protocol_factory, *args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**kwargs):
def subprocess_exec(self, protocol_factory, *args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
**kwargs):
raise NotImplementedError
# Ready-based callback registration methods.
@@ -548,32 +463,16 @@ class AbstractEventLoop:
# Completion based I/O methods returning Futures.
async def sock_recv(self, sock, nbytes):
def sock_recv(self, sock, nbytes):
raise NotImplementedError
async def sock_recv_into(self, sock, buf):
def sock_sendall(self, sock, data):
raise NotImplementedError
async def sock_recvfrom(self, sock, bufsize):
def sock_connect(self, sock, address):
raise NotImplementedError
async def sock_recvfrom_into(self, sock, buf, nbytes=0):
raise NotImplementedError
async def sock_sendall(self, sock, data):
raise NotImplementedError
async def sock_sendto(self, sock, data, address):
raise NotImplementedError
async def sock_connect(self, sock, address):
raise NotImplementedError
async def sock_accept(self, sock):
raise NotImplementedError
async def sock_sendfile(self, sock, file, offset=0, count=None,
*, fallback=None):
def sock_accept(self, sock):
raise NotImplementedError
# Signal handling.
@@ -621,7 +520,7 @@ class AbstractEventLoopPolicy:
def get_event_loop(self):
"""Get the event loop for the current context.
Returns an event loop object implementing the AbstractEventLoop interface,
Returns an event loop object implementing the BaseEventLoop interface,
or raises an exception in case no event loop has been set for the
current context and the current policy does not specify to create one.
@@ -672,43 +571,23 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy):
self._local = self._Local()
def get_event_loop(self):
"""Get the event loop for the current context.
"""Get the event loop.
Returns an instance of EventLoop or raises an exception.
This may be None or an instance of EventLoop.
"""
if (self._local._loop is None and
not self._local._set_called and
threading.current_thread() is threading.main_thread()):
stacklevel = 2
try:
f = sys._getframe(1)
except AttributeError:
pass
else:
# Move up the call stack so that the warning is attached
# to the line outside asyncio itself.
while f:
module = f.f_globals.get('__name__')
if not (module == 'asyncio' or module.startswith('asyncio.')):
break
f = f.f_back
stacklevel += 1
import warnings
warnings.warn('There is no current event loop',
DeprecationWarning, stacklevel=stacklevel)
not self._local._set_called and
isinstance(threading.current_thread(), threading._MainThread)):
self.set_event_loop(self.new_event_loop())
if self._local._loop is None:
raise RuntimeError('There is no current event loop in thread %r.'
% threading.current_thread().name)
return self._local._loop
def set_event_loop(self, loop):
"""Set the event loop."""
self._local._set_called = True
if loop is not None and not isinstance(loop, AbstractEventLoop):
raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'")
assert loop is None or isinstance(loop, AbstractEventLoop)
self._local._loop = loop
def new_event_loop(self):
@@ -732,9 +611,7 @@ _lock = threading.Lock()
# A TLS for the running event loop, used by _get_running_loop.
class _RunningLoop(threading.local):
loop_pid = (None, None)
_loop = None
_running_loop = _RunningLoop()
@@ -756,10 +633,7 @@ def _get_running_loop():
This is a low-level function intended to be used by event loops.
This function is thread-specific.
"""
# NOTE: this function is implemented in C (see _asynciomodule.c)
running_loop, pid = _running_loop.loop_pid
if running_loop is not None and pid == os.getpid():
return running_loop
return _running_loop._loop
def _set_running_loop(loop):
@@ -768,8 +642,7 @@ def _set_running_loop(loop):
This is a low-level function intended to be used by event loops.
This function is thread-specific.
"""
# NOTE: this function is implemented in C (see _asynciomodule.c)
_running_loop.loop_pid = (loop, os.getpid())
_running_loop._loop = loop
def _init_event_loop_policy():
@@ -792,8 +665,7 @@ def set_event_loop_policy(policy):
If policy is None, the default policy is restored."""
global _event_loop_policy
if policy is not None and not isinstance(policy, AbstractEventLoopPolicy):
raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'")
assert policy is None or isinstance(policy, AbstractEventLoopPolicy)
_event_loop_policy = policy
@@ -806,7 +678,6 @@ def get_event_loop():
If there is no running event loop set, the function will return
the result of `get_event_loop_policy().get_event_loop()` call.
"""
# NOTE: this function is implemented in C (see _asynciomodule.c)
current_loop = _get_running_loop()
if current_loop is not None:
return current_loop
@@ -832,37 +703,3 @@ def set_child_watcher(watcher):
"""Equivalent to calling
get_event_loop_policy().set_child_watcher(watcher)."""
return get_event_loop_policy().set_child_watcher(watcher)
# Alias pure-Python implementations for testing purposes.
_py__get_running_loop = _get_running_loop
_py__set_running_loop = _set_running_loop
_py_get_running_loop = get_running_loop
_py_get_event_loop = get_event_loop
try:
# get_event_loop() is one of the most frequently called
# functions in asyncio. Pure Python implementation is
# about 4 times slower than C-accelerated.
from _asyncio import (_get_running_loop, _set_running_loop,
get_running_loop, get_event_loop)
except ImportError:
pass
else:
# Alias C implementations for testing purposes.
_c__get_running_loop = _get_running_loop
_c__set_running_loop = _set_running_loop
_c_get_running_loop = get_running_loop
_c_get_event_loop = get_event_loop
if hasattr(os, 'fork'):
def on_fork():
# Reset the loop and wakeupfd in the forked child process.
if _event_loop_policy is not None:
_event_loop_policy._local = BaseDefaultEventLoopPolicy._Local()
_set_running_loop(None)
signal.set_wakeup_fd(-1)
os.register_at_fork(after_in_child=on_fork)

62
Lib/asyncio/exceptions.py
View File

@@ -1,62 +0,0 @@
"""asyncio exceptions."""
__all__ = ('BrokenBarrierError',
'CancelledError', 'InvalidStateError', 'TimeoutError',
'IncompleteReadError', 'LimitOverrunError',
'SendfileNotAvailableError')
class CancelledError(BaseException):
"""The Future or Task was cancelled."""
TimeoutError = TimeoutError # make local alias for the standard exception
class InvalidStateError(Exception):
"""The operation is not allowed in this state."""
class SendfileNotAvailableError(RuntimeError):
"""Sendfile syscall is not available.
Raised if OS does not support sendfile syscall for given socket or
file type.
"""
class IncompleteReadError(EOFError):
"""
Incomplete read error. Attributes:
- partial: read bytes string before the end of stream was reached
- expected: total number of expected bytes (or None if unknown)
"""
def __init__(self, partial, expected):
r_expected = 'undefined' if expected is None else repr(expected)
super().__init__(f'{len(partial)} bytes read on a total of '
f'{r_expected} expected bytes')
self.partial = partial
self.expected = expected
def __reduce__(self):
return type(self), (self.partial, self.expected)
class LimitOverrunError(Exception):
"""Reached the buffer limit while looking for a separator.
Attributes:
- consumed: total number of to be consumed bytes.
"""
def __init__(self, message, consumed):
super().__init__(message)
self.consumed = consumed
def __reduce__(self):
return type(self), (self.args[0], self.consumed)
class BrokenBarrierError(RuntimeError):
"""Barrier is broken by barrier.abort() call."""

76
Lib/asyncio/format_helpers.py
View File

@@ -1,76 +0,0 @@
import functools
import inspect
import reprlib
import sys
import traceback
from . import constants
def _get_function_source(func):
func = inspect.unwrap(func)
if inspect.isfunction(func):
code = func.__code__
return (code.co_filename, code.co_firstlineno)
if isinstance(func, functools.partial):
return _get_function_source(func.func)
if isinstance(func, functools.partialmethod):
return _get_function_source(func.func)
return None
def _format_callback_source(func, args):
func_repr = _format_callback(func, args, None)
source = _get_function_source(func)
if source:
func_repr += f' at {source[0]}:{source[1]}'
return func_repr
def _format_args_and_kwargs(args, kwargs):
"""Format function arguments and keyword arguments.
Special case for a single parameter: ('hello',) is formatted as ('hello').
"""
# use reprlib to limit the length of the output
items = []
if args:
items.extend(reprlib.repr(arg) for arg in args)
if kwargs:
items.extend(f'{k}={reprlib.repr(v)}' for k, v in kwargs.items())
return '({})'.format(', '.join(items))
def _format_callback(func, args, kwargs, suffix=''):
if isinstance(func, functools.partial):
suffix = _format_args_and_kwargs(args, kwargs) + suffix
return _format_callback(func.func, func.args, func.keywords, suffix)
if hasattr(func, '__qualname__') and func.__qualname__:
func_repr = func.__qualname__
elif hasattr(func, '__name__') and func.__name__:
func_repr = func.__name__
else:
func_repr = repr(func)
func_repr += _format_args_and_kwargs(args, kwargs)
if suffix:
func_repr += suffix
return func_repr
def extract_stack(f=None, limit=None):
"""Replacement for traceback.extract_stack() that only does the
necessary work for asyncio debug mode.
"""
if f is None:
f = sys._getframe().f_back
if limit is None:
# Limit the amount of work to a reasonable amount, as extract_stack()
# can be called for each coroutine and future in debug mode.
limit = constants.DEBUG_STACK_DEPTH
stack = traceback.StackSummary.extract(traceback.walk_stack(f),
limit=limit,
lookup_lines=False)
stack.reverse()
return stack

297
Lib/asyncio/futures.py
View File

@@ -1,21 +1,21 @@
"""A Future class similar to the one in PEP 3148."""
__all__ = (
'Future', 'wrap_future', 'isfuture',
)
__all__ = ['CancelledError', 'TimeoutError', 'InvalidStateError',
'Future', 'wrap_future', 'isfuture']
import concurrent.futures
import contextvars
import logging
import sys
from types import GenericAlias
import traceback
from . import base_futures
from . import compat
from . import events
from . import exceptions
from . import format_helpers
CancelledError = base_futures.CancelledError
InvalidStateError = base_futures.InvalidStateError
TimeoutError = base_futures.TimeoutError
isfuture = base_futures.isfuture
@@ -27,18 +27,96 @@ _FINISHED = base_futures._FINISHED
STACK_DEBUG = logging.DEBUG - 1 # heavy-duty debugging
class _TracebackLogger:
"""Helper to log a traceback upon destruction if not cleared.
This solves a nasty problem with Futures and Tasks that have an
exception set: if nobody asks for the exception, the exception is
never logged. This violates the Zen of Python: 'Errors should
never pass silently. Unless explicitly silenced.'
However, we don't want to log the exception as soon as
set_exception() is called: if the calling code is written
properly, it will get the exception and handle it properly. But
we *do* want to log it if result() or exception() was never called
-- otherwise developers waste a lot of time wondering why their
buggy code fails silently.
An earlier attempt added a __del__() method to the Future class
itself, but this backfired because the presence of __del__()
prevents garbage collection from breaking cycles. A way out of
this catch-22 is to avoid having a __del__() method on the Future
class itself, but instead to have a reference to a helper object
with a __del__() method that logs the traceback, where we ensure
that the helper object doesn't participate in cycles, and only the
Future has a reference to it.
The helper object is added when set_exception() is called. When
the Future is collected, and the helper is present, the helper
object is also collected, and its __del__() method will log the
traceback. When the Future's result() or exception() method is
called (and a helper object is present), it removes the helper
object, after calling its clear() method to prevent it from
logging.
One downside is that we do a fair amount of work to extract the
traceback from the exception, even when it is never logged. It
would seem cheaper to just store the exception object, but that
references the traceback, which references stack frames, which may
reference the Future, which references the _TracebackLogger, and
then the _TracebackLogger would be included in a cycle, which is
what we're trying to avoid! As an optimization, we don't
immediately format the exception; we only do the work when
activate() is called, which call is delayed until after all the
Future's callbacks have run. Since usually a Future has at least
one callback (typically set by 'yield from') and usually that
callback extracts the callback, thereby removing the need to
format the exception.
PS. I don't claim credit for this solution. I first heard of it
in a discussion about closing files when they are collected.
"""
__slots__ = ('loop', 'source_traceback', 'exc', 'tb')
def __init__(self, future, exc):
self.loop = future._loop
self.source_traceback = future._source_traceback
self.exc = exc
self.tb = None
def activate(self):
exc = self.exc
if exc is not None:
self.exc = None
self.tb = traceback.format_exception(exc.__class__, exc,
exc.__traceback__)
def clear(self):
self.exc = None
self.tb = None
def __del__(self):
if self.tb:
msg = 'Future/Task exception was never retrieved\n'
if self.source_traceback:
src = ''.join(traceback.format_list(self.source_traceback))
msg += 'Future/Task created at (most recent call last):\n'
msg += '%s\n' % src.rstrip()
msg += ''.join(self.tb).rstrip()
self.loop.call_exception_handler({'message': msg})
class Future:
"""This class is *almost* compatible with concurrent.futures.Future.
Differences:
- This class is not thread-safe.
- result() and exception() do not take a timeout argument and
raise an exception when the future isn't done yet.
- Callbacks registered with add_done_callback() are always called
via the event loop's call_soon().
via the event loop's call_soon_threadsafe().
- This class is not compatible with the wait() and as_completed()
methods in the concurrent.futures package.
@@ -52,9 +130,6 @@ class Future:
_exception = None
_loop = None
_source_traceback = None
_cancel_message = None
# A saved CancelledError for later chaining as an exception context.
_cancelled_exc = None
# This field is used for a dual purpose:
# - Its presence is a marker to declare that a class implements
@@ -62,12 +137,12 @@ class Future:
# The value must also be not-None, to enable a subclass to declare
# that it is not compatible by setting this to None.
# - It is set by __iter__() below so that Task._step() can tell
# the difference between
# `await Future()` or`yield from Future()` (correct) vs.
# the difference between `yield from Future()` (correct) vs.
# `yield Future()` (incorrect).
_asyncio_future_blocking = False
__log_traceback = False
_log_traceback = False # Used for Python 3.4 and later
_tb_logger = None # Used for Python 3.3 only
def __init__(self, *, loop=None):
"""Initialize the future.
@@ -82,83 +157,47 @@ class Future:
self._loop = loop
self._callbacks = []
if self._loop.get_debug():
self._source_traceback = format_helpers.extract_stack(
sys._getframe(1))
self._source_traceback = traceback.extract_stack(sys._getframe(1))
_repr_info = base_futures._future_repr_info
def __repr__(self):
return base_futures._future_repr(self)
return '<%s %s>' % (self.__class__.__name__, ' '.join(self._repr_info()))
def __del__(self):
if not self.__log_traceback:
# set_exception() was not called, or result() or exception()
# has consumed the exception
return
exc = self._exception
context = {
'message':
f'{self.__class__.__name__} exception was never retrieved',
'exception': exc,
'future': self,
}
if self._source_traceback:
context['source_traceback'] = self._source_traceback
self._loop.call_exception_handler(context)
# On Python 3.3 and older, objects with a destructor part of a reference
# cycle are never destroyed. It's not more the case on Python 3.4 thanks
# to the PEP 442.
if compat.PY34:
def __del__(self):
if not self._log_traceback:
# set_exception() was not called, or result() or exception()
# has consumed the exception
return
exc = self._exception
context = {
'message': ('%s exception was never retrieved'
% self.__class__.__name__),
'exception': exc,
'future': self,
}
if self._source_traceback:
context['source_traceback'] = self._source_traceback
self._loop.call_exception_handler(context)
__class_getitem__ = classmethod(GenericAlias)
@property
def _log_traceback(self):
return self.__log_traceback
@_log_traceback.setter
def _log_traceback(self, val):
if val:
raise ValueError('_log_traceback can only be set to False')
self.__log_traceback = False
def get_loop(self):
"""Return the event loop the Future is bound to."""
loop = self._loop
if loop is None:
raise RuntimeError("Future object is not initialized.")
return loop
def _make_cancelled_error(self):
"""Create the CancelledError to raise if the Future is cancelled.
This should only be called once when handling a cancellation since
it erases the saved context exception value.
"""
if self._cancelled_exc is not None:
exc = self._cancelled_exc
self._cancelled_exc = None
return exc
if self._cancel_message is None:
exc = exceptions.CancelledError()
else:
exc = exceptions.CancelledError(self._cancel_message)
exc.__context__ = self._cancelled_exc
# Remove the reference since we don't need this anymore.
self._cancelled_exc = None
return exc
def cancel(self, msg=None):
def cancel(self):
"""Cancel the future and schedule callbacks.
If the future is already done or cancelled, return False. Otherwise,
change the future's state to cancelled, schedule the callbacks and
return True.
"""
self.__log_traceback = False
if self._state != _PENDING:
return False
self._state = _CANCELLED
self._cancel_message = msg
self.__schedule_callbacks()
self._schedule_callbacks()
return True
def __schedule_callbacks(self):
def _schedule_callbacks(self):
"""Internal: Ask the event loop to call all callbacks.
The callbacks are scheduled to be called as soon as possible. Also
@@ -169,8 +208,8 @@ class Future:
return
self._callbacks[:] = []
for callback, ctx in callbacks:
self._loop.call_soon(callback, self, context=ctx)
for callback in callbacks:
self._loop.call_soon(callback, self)
def cancelled(self):
"""Return True if the future was cancelled."""
@@ -194,13 +233,15 @@ class Future:
the future is done and has an exception set, this exception is raised.
"""
if self._state == _CANCELLED:
exc = self._make_cancelled_error()
raise exc
raise CancelledError
if self._state != _FINISHED:
raise exceptions.InvalidStateError('Result is not ready.')
self.__log_traceback = False
raise InvalidStateError('Result is not ready.')
self._log_traceback = False
if self._tb_logger is not None:
self._tb_logger.clear()
self._tb_logger = None
if self._exception is not None:
raise self._exception.with_traceback(self._exception_tb)
raise self._exception
return self._result
def exception(self):
@@ -212,14 +253,16 @@ class Future:
InvalidStateError.
"""
if self._state == _CANCELLED:
exc = self._make_cancelled_error()
raise exc
raise CancelledError
if self._state != _FINISHED:
raise exceptions.InvalidStateError('Exception is not set.')
self.__log_traceback = False
raise InvalidStateError('Exception is not set.')
self._log_traceback = False
if self._tb_logger is not None:
self._tb_logger.clear()
self._tb_logger = None
return self._exception
def add_done_callback(self, fn, *, context=None):
def add_done_callback(self, fn):
"""Add a callback to be run when the future becomes done.
The callback is called with a single argument - the future object. If
@@ -227,11 +270,9 @@ class Future:
scheduled with call_soon.
"""
if self._state != _PENDING:
self._loop.call_soon(fn, self, context=context)
self._loop.call_soon(fn, self)
else:
if context is None:
context = contextvars.copy_context()
self._callbacks.append((fn, context))
self._callbacks.append(fn)
# New method not in PEP 3148.
@@ -240,9 +281,7 @@ class Future:
Returns the number of callbacks removed.
"""
filtered_callbacks = [(f, ctx)
for (f, ctx) in self._callbacks
if f != fn]
filtered_callbacks = [f for f in self._callbacks if f != fn]
removed_count = len(self._callbacks) - len(filtered_callbacks)
if removed_count:
self._callbacks[:] = filtered_callbacks
@@ -257,10 +296,10 @@ class Future:
InvalidStateError.
"""
if self._state != _PENDING:
raise exceptions.InvalidStateError(f'{self._state}: {self!r}')
raise InvalidStateError('{}: {!r}'.format(self._state, self))
self._result = result
self._state = _FINISHED
self.__schedule_callbacks()
self._schedule_callbacks()
def set_exception(self, exception):
"""Mark the future done and set an exception.
@@ -269,45 +308,38 @@ class Future:
InvalidStateError.
"""
if self._state != _PENDING:
raise exceptions.InvalidStateError(f'{self._state}: {self!r}')
raise InvalidStateError('{}: {!r}'.format(self._state, self))
if isinstance(exception, type):
exception = exception()
if type(exception) is StopIteration:
raise TypeError("StopIteration interacts badly with generators "
"and cannot be raised into a Future")
self._exception = exception
self._exception_tb = exception.__traceback__
self._state = _FINISHED
self.__schedule_callbacks()
self.__log_traceback = True
self._schedule_callbacks()
if compat.PY34:
self._log_traceback = True
else:
self._tb_logger = _TracebackLogger(self, exception)
# Arrange for the logger to be activated after all callbacks
# have had a chance to call result() or exception().
self._loop.call_soon(self._tb_logger.activate)
def __await__(self):
def __iter__(self):
if not self.done():
self._asyncio_future_blocking = True
yield self # This tells Task to wait for completion.
if not self.done():
raise RuntimeError("await wasn't used with future")
assert self.done(), "yield from wasn't used with future"
return self.result() # May raise too.
__iter__ = __await__ # make compatible with 'yield from'.
if compat.PY35:
__await__ = __iter__ # make compatible with 'await' expression
# Needed for testing purposes.
_PyFuture = Future
def _get_loop(fut):
# Tries to call Future.get_loop() if it's available.
# Otherwise fallbacks to using the old '_loop' property.
try:
get_loop = fut.get_loop
except AttributeError:
pass
else:
return get_loop()
return fut._loop
def _set_result_unless_cancelled(fut, result):
"""Helper setting the result only if the future was not cancelled."""
if fut.cancelled():
@@ -315,18 +347,6 @@ def _set_result_unless_cancelled(fut, result):
fut.set_result(result)
def _convert_future_exc(exc):
exc_class = type(exc)
if exc_class is concurrent.futures.CancelledError:
return exceptions.CancelledError(*exc.args)
elif exc_class is concurrent.futures.TimeoutError:
return exceptions.TimeoutError(*exc.args)
elif exc_class is concurrent.futures.InvalidStateError:
return exceptions.InvalidStateError(*exc.args)
else:
return exc
def _set_concurrent_future_state(concurrent, source):
"""Copy state from a future to a concurrent.futures.Future."""
assert source.done()
@@ -336,7 +356,7 @@ def _set_concurrent_future_state(concurrent, source):
return
exception = source.exception()
if exception is not None:
concurrent.set_exception(_convert_future_exc(exception))
concurrent.set_exception(exception)
else:
result = source.result()
concurrent.set_result(result)
@@ -356,7 +376,7 @@ def _copy_future_state(source, dest):
else:
exception = source.exception()
if exception is not None:
dest.set_exception(_convert_future_exc(exception))
dest.set_exception(exception)
else:
result = source.result()
dest.set_result(result)
@@ -375,8 +395,8 @@ def _chain_future(source, destination):
if not isfuture(destination) and not isinstance(destination,
concurrent.futures.Future):
raise TypeError('A future is required for destination argument')
source_loop = _get_loop(source) if isfuture(source) else None
dest_loop = _get_loop(destination) if isfuture(destination) else None
source_loop = source._loop if isfuture(source) else None
dest_loop = destination._loop if isfuture(destination) else None
def _set_state(future, other):
if isfuture(future):
@@ -392,14 +412,9 @@ def _chain_future(source, destination):
source_loop.call_soon_threadsafe(source.cancel)
def _call_set_state(source):
if (destination.cancelled() and
dest_loop is not None and dest_loop.is_closed()):
return
if dest_loop is None or dest_loop is source_loop:
_set_state(destination, source)
else:
if dest_loop.is_closed():
return
dest_loop.call_soon_threadsafe(_set_state, destination, source)
destination.add_done_callback(_call_check_cancel)
@@ -411,7 +426,7 @@ def wrap_future(future, *, loop=None):
if isfuture(future):
return future
assert isinstance(future, concurrent.futures.Future), \
f'concurrent.futures.Future is expected, got {future!r}'
'concurrent.futures.Future is expected, got {!r}'.format(future)
if loop is None:
loop = events.get_event_loop()
new_future = loop.create_future()

456
Lib/asyncio/locks.py
View File

@@ -1,26 +1,92 @@
"""Synchronization primitives."""
__all__ = ('Lock', 'Event', 'Condition', 'Semaphore',
'BoundedSemaphore', 'Barrier')
__all__ = ['Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore']
import collections
import enum
from . import exceptions
from . import mixins
from . import compat
from . import events
from . import futures
from .coroutines import coroutine
class _ContextManagerMixin:
async def __aenter__(self):
await self.acquire()
class _ContextManager:
"""Context manager.
This enables the following idiom for acquiring and releasing a
lock around a block:
with (yield from lock):
<block>
while failing loudly when accidentally using:
with lock:
<block>
"""
def __init__(self, lock):
self._lock = lock
def __enter__(self):
# We have no use for the "as ..." clause in the with
# statement for locks.
return None
async def __aexit__(self, exc_type, exc, tb):
self.release()
def __exit__(self, *args):
try:
self._lock.release()
finally:
self._lock = None # Crudely prevent reuse.
class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
class _ContextManagerMixin:
def __enter__(self):
raise RuntimeError(
'"yield from" should be used as context manager expression')
def __exit__(self, *args):
# This must exist because __enter__ exists, even though that
# always raises; that's how the with-statement works.
pass
@coroutine
def __iter__(self):
# This is not a coroutine. It is meant to enable the idiom:
#
# with (yield from lock):
# <block>
#
# as an alternative to:
#
# yield from lock.acquire()
# try:
# <block>
# finally:
# lock.release()
yield from self.acquire()
return _ContextManager(self)
if compat.PY35:
def __await__(self):
# To make "with await lock" work.
yield from self.acquire()
return _ContextManager(self)
@coroutine
def __aenter__(self):
yield from self.acquire()
# We have no use for the "as ..." clause in the with
# statement for locks.
return None
@coroutine
def __aexit__(self, exc_type, exc, tb):
self.release()
class Lock(_ContextManagerMixin):
"""Primitive lock objects.
A primitive lock is a synchronization primitive that is not owned
@@ -42,16 +108,16 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
release() call resets the state to unlocked; first coroutine which
is blocked in acquire() is being processed.
acquire() is a coroutine and should be called with 'await'.
acquire() is a coroutine and should be called with 'yield from'.
Locks also support the asynchronous context management protocol.
'async with lock' statement should be used.
Locks also support the context management protocol. '(yield from lock)'
should be used as the context manager expression.
Usage:
lock = Lock()
...
await lock.acquire()
yield from lock
try:
...
finally:
@@ -61,65 +127,57 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
lock = Lock()
...
async with lock:
with (yield from lock):
...
Lock objects can be tested for locking state:
if not lock.locked():
await lock.acquire()
yield from lock
else:
# lock is acquired
...
"""
def __init__(self):
self._waiters = None
def __init__(self, *, loop=None):
self._waiters = collections.deque()
self._locked = False
if loop is not None:
self._loop = loop
else:
self._loop = events.get_event_loop()
def __repr__(self):
res = super().__repr__()
extra = 'locked' if self._locked else 'unlocked'
if self._waiters:
extra = f'{extra}, waiters:{len(self._waiters)}'
return f'<{res[1:-1]} [{extra}]>'
extra = '{},waiters:{}'.format(extra, len(self._waiters))
return '<{} [{}]>'.format(res[1:-1], extra)
def locked(self):
"""Return True if lock is acquired."""
return self._locked
async def acquire(self):
@coroutine
def acquire(self):
"""Acquire a lock.
This method blocks until the lock is unlocked, then sets it to
locked and returns True.
"""
if (not self._locked and (self._waiters is None or
all(w.cancelled() for w in self._waiters))):
if not self._locked and all(w.cancelled() for w in self._waiters):
self._locked = True
return True
if self._waiters is None:
self._waiters = collections.deque()
fut = self._get_loop().create_future()
fut = self._loop.create_future()
self._waiters.append(fut)
# Finally block should be called before the CancelledError
# handling as we don't want CancelledError to call
# _wake_up_first() and attempt to wake up itself.
try:
try:
await fut
finally:
self._waiters.remove(fut)
except exceptions.CancelledError:
if not self._locked:
self._wake_up_first()
raise
self._locked = True
return True
yield from fut
self._locked = True
return True
finally:
self._waiters.remove(fut)
def release(self):
"""Release a lock.
@@ -134,27 +192,16 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
"""
if self._locked:
self._locked = False
self._wake_up_first()
# Wake up the first waiter who isn't cancelled.
for fut in self._waiters:
if not fut.done():
fut.set_result(True)
break
else:
raise RuntimeError('Lock is not acquired.')
def _wake_up_first(self):
"""Wake up the first waiter if it isn't done."""
if not self._waiters:
return
try:
fut = next(iter(self._waiters))
except StopIteration:
return
# .done() necessarily means that a waiter will wake up later on and
# either take the lock, or, if it was cancelled and lock wasn't
# taken already, will hit this again and wake up a new waiter.
if not fut.done():
fut.set_result(True)
class Event(mixins._LoopBoundMixin):
class Event:
"""Asynchronous equivalent to threading.Event.
Class implementing event objects. An event manages a flag that can be set
@@ -163,16 +210,20 @@ class Event(mixins._LoopBoundMixin):
false.
"""
def __init__(self):
def __init__(self, *, loop=None):
self._waiters = collections.deque()
self._value = False
if loop is not None:
self._loop = loop
else:
self._loop = events.get_event_loop()
def __repr__(self):
res = super().__repr__()
extra = 'set' if self._value else 'unset'
if self._waiters:
extra = f'{extra}, waiters:{len(self._waiters)}'
return f'<{res[1:-1]} [{extra}]>'
extra = '{},waiters:{}'.format(extra, len(self._waiters))
return '<{} [{}]>'.format(res[1:-1], extra)
def is_set(self):
"""Return True if and only if the internal flag is true."""
@@ -196,7 +247,8 @@ class Event(mixins._LoopBoundMixin):
to true again."""
self._value = False
async def wait(self):
@coroutine
def wait(self):
"""Block until the internal flag is true.
If the internal flag is true on entry, return True
@@ -206,16 +258,16 @@ class Event(mixins._LoopBoundMixin):
if self._value:
return True
fut = self._get_loop().create_future()
fut = self._loop.create_future()
self._waiters.append(fut)
try:
await fut
yield from fut
return True
finally:
self._waiters.remove(fut)
class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
class Condition(_ContextManagerMixin):
"""Asynchronous equivalent to threading.Condition.
This class implements condition variable objects. A condition variable
@@ -225,9 +277,16 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
A new Lock object is created and used as the underlying lock.
"""
def __init__(self, lock=None):
def __init__(self, lock=None, *, loop=None):
if loop is not None:
self._loop = loop
else:
self._loop = events.get_event_loop()
if lock is None:
lock = Lock()
lock = Lock(loop=self._loop)
elif lock._loop is not self._loop:
raise ValueError("loop argument must agree with lock")
self._lock = lock
# Export the lock's locked(), acquire() and release() methods.
@@ -241,10 +300,11 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
res = super().__repr__()
extra = 'locked' if self.locked() else 'unlocked'
if self._waiters:
extra = f'{extra}, waiters:{len(self._waiters)}'
return f'<{res[1:-1]} [{extra}]>'
extra = '{},waiters:{}'.format(extra, len(self._waiters))
return '<{} [{}]>'.format(res[1:-1], extra)
async def wait(self):
@coroutine
def wait(self):
"""Wait until notified.
If the calling coroutine has not acquired the lock when this
@@ -260,28 +320,25 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
self.release()
try:
fut = self._get_loop().create_future()
fut = self._loop.create_future()
self._waiters.append(fut)
try:
await fut
yield from fut
return True
finally:
self._waiters.remove(fut)
finally:
# Must reacquire lock even if wait is cancelled
cancelled = False
while True:
try:
await self.acquire()
yield from self.acquire()
break
except exceptions.CancelledError:
cancelled = True
except futures.CancelledError:
pass
if cancelled:
raise exceptions.CancelledError
async def wait_for(self, predicate):
@coroutine
def wait_for(self, predicate):
"""Wait until a predicate becomes true.
The predicate should be a callable which result will be
@@ -290,7 +347,7 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
"""
result = predicate()
while not result:
await self.wait()
yield from self.wait()
result = predicate()
return result
@@ -327,7 +384,7 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
self.notify(len(self._waiters))
class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin):
class Semaphore(_ContextManagerMixin):
"""A Semaphore implementation.
A semaphore manages an internal counter which is decremented by each
@@ -342,25 +399,37 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin):
ValueError is raised.
"""
def __init__(self, value=1):
def __init__(self, value=1, *, loop=None):
if value < 0:
raise ValueError("Semaphore initial value must be >= 0")
self._waiters = None
self._value = value
self._waiters = collections.deque()
if loop is not None:
self._loop = loop
else:
self._loop = events.get_event_loop()
def __repr__(self):
res = super().__repr__()
extra = 'locked' if self.locked() else f'unlocked, value:{self._value}'
extra = 'locked' if self.locked() else 'unlocked,value:{}'.format(
self._value)
if self._waiters:
extra = f'{extra}, waiters:{len(self._waiters)}'
return f'<{res[1:-1]} [{extra}]>'
extra = '{},waiters:{}'.format(extra, len(self._waiters))
return '<{} [{}]>'.format(res[1:-1], extra)
def _wake_up_next(self):
while self._waiters:
waiter = self._waiters.popleft()
if not waiter.done():
waiter.set_result(None)
return
def locked(self):
"""Returns True if semaphore cannot be acquired immediately."""
return self._value == 0 or (
any(not w.cancelled() for w in (self._waiters or ())))
"""Returns True if semaphore can not be acquired immediately."""
return self._value == 0
async def acquire(self):
@coroutine
def acquire(self):
"""Acquire a semaphore.
If the internal counter is larger than zero on entry,
@@ -369,53 +438,28 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin):
called release() to make it larger than 0, and then return
True.
"""
if not self.locked():
self._value -= 1
return True
if self._waiters is None:
self._waiters = collections.deque()
fut = self._get_loop().create_future()
self._waiters.append(fut)
# Finally block should be called before the CancelledError
# handling as we don't want CancelledError to call
# _wake_up_first() and attempt to wake up itself.
try:
while self._value <= 0:
fut = self._loop.create_future()
self._waiters.append(fut)
try:
await fut
finally:
self._waiters.remove(fut)
except exceptions.CancelledError:
if not fut.cancelled():
self._value += 1
self._wake_up_next()
raise
if self._value > 0:
self._wake_up_next()
yield from fut
except:
# See the similar code in Queue.get.
fut.cancel()
if self._value > 0 and not fut.cancelled():
self._wake_up_next()
raise
self._value -= 1
return True
def release(self):
"""Release a semaphore, incrementing the internal counter by one.
When it was zero on entry and another coroutine is waiting for it to
become larger than zero again, wake up that coroutine.
"""
self._value += 1
self._wake_up_next()
def _wake_up_next(self):
"""Wake up the first waiter that isn't done."""
if not self._waiters:
return
for fut in self._waiters:
if not fut.done():
self._value -= 1
fut.set_result(True)
return
class BoundedSemaphore(Semaphore):
"""A bounded semaphore implementation.
@@ -424,163 +468,11 @@ class BoundedSemaphore(Semaphore):
above the initial value.
"""
def __init__(self, value=1):
def __init__(self, value=1, *, loop=None):
self._bound_value = value
super().__init__(value)
super().__init__(value, loop=loop)
def release(self):
if self._value >= self._bound_value:
raise ValueError('BoundedSemaphore released too many times')
super().release()
class _BarrierState(enum.Enum):
FILLING = 'filling'
DRAINING = 'draining'
RESETTING = 'resetting'
BROKEN = 'broken'
class Barrier(mixins._LoopBoundMixin):
"""Asyncio equivalent to threading.Barrier
Implements a Barrier primitive.
Useful for synchronizing a fixed number of tasks at known synchronization
points. Tasks block on 'wait()' and are simultaneously awoken once they
have all made their call.
"""
def __init__(self, parties):
"""Create a barrier, initialised to 'parties' tasks."""
if parties < 1:
raise ValueError('parties must be > 0')
self._cond = Condition() # notify all tasks when state changes
self._parties = parties
self._state = _BarrierState.FILLING
self._count = 0 # count tasks in Barrier
def __repr__(self):
res = super().__repr__()
extra = f'{self._state.value}'
if not self.broken:
extra += f', waiters:{self.n_waiting}/{self.parties}'
return f'<{res[1:-1]} [{extra}]>'
async def __aenter__(self):
# wait for the barrier reaches the parties number
# when start draining release and return index of waited task
return await self.wait()
async def __aexit__(self, *args):
pass
async def wait(self):
"""Wait for the barrier.
When the specified number of tasks have started waiting, they are all
simultaneously awoken.
Returns an unique and individual index number from 0 to 'parties-1'.
"""
async with self._cond:
await self._block() # Block while the barrier drains or resets.
try:
index = self._count
self._count += 1
if index + 1 == self._parties:
# We release the barrier
await self._release()
else:
await self._wait()
return index
finally:
self._count -= 1
# Wake up any tasks waiting for barrier to drain.
self._exit()
async def _block(self):
# Block until the barrier is ready for us,
# or raise an exception if it is broken.
#
# It is draining or resetting, wait until done
# unless a CancelledError occurs
await self._cond.wait_for(
lambda: self._state not in (
_BarrierState.DRAINING, _BarrierState.RESETTING
)
)
# see if the barrier is in a broken state
if self._state is _BarrierState.BROKEN:
raise exceptions.BrokenBarrierError("Barrier aborted")
async def _release(self):
# Release the tasks waiting in the barrier.
# Enter draining state.
# Next waiting tasks will be blocked until the end of draining.
self._state = _BarrierState.DRAINING
self._cond.notify_all()
async def _wait(self):
# Wait in the barrier until we are released. Raise an exception
# if the barrier is reset or broken.
# wait for end of filling
# unless a CancelledError occurs
await self._cond.wait_for(lambda: self._state is not _BarrierState.FILLING)
if self._state in (_BarrierState.BROKEN, _BarrierState.RESETTING):
raise exceptions.BrokenBarrierError("Abort or reset of barrier")
def _exit(self):
# If we are the last tasks to exit the barrier, signal any tasks
# waiting for the barrier to drain.
if self._count == 0:
if self._state in (_BarrierState.RESETTING, _BarrierState.DRAINING):
self._state = _BarrierState.FILLING
self._cond.notify_all()
async def reset(self):
"""Reset the barrier to the initial state.
Any tasks currently waiting will get the BrokenBarrier exception
raised.
"""
async with self._cond:
if self._count > 0:
if self._state is not _BarrierState.RESETTING:
#reset the barrier, waking up tasks
self._state = _BarrierState.RESETTING
else:
self._state = _BarrierState.FILLING
self._cond.notify_all()
async def abort(self):
"""Place the barrier into a 'broken' state.
Useful in case of error. Any currently waiting tasks and tasks
attempting to 'wait()' will have BrokenBarrierError raised.
"""
async with self._cond:
self._state = _BarrierState.BROKEN
self._cond.notify_all()
@property
def parties(self):
"""Return the number of tasks required to trip the barrier."""
return self._parties
@property
def n_waiting(self):
"""Return the number of tasks currently waiting at the barrier."""
if self._state is _BarrierState.FILLING:
return self._count
return 0
@property
def broken(self):
"""Return True if the barrier is in a broken state."""
return self._state is _BarrierState.BROKEN

21
Lib/asyncio/mixins.py
View File

@@ -1,21 +0,0 @@
"""Event loop mixins."""
import threading
from . import events
_global_lock = threading.Lock()
class _LoopBoundMixin:
_loop = None
def _get_loop(self):
loop = events._get_running_loop()
if self._loop is None:
with _global_lock:
if self._loop is None:
self._loop = loop
if loop is not self._loop:
raise RuntimeError(f'{self!r} is bound to a different event loop')
return loop

561
Lib/asyncio/proactor_events.py
View File

@@ -4,45 +4,20 @@ A proactor is a "notify-on-completion" multiplexer. Currently a
proactor is only implemented on Windows with IOCP.
"""
__all__ = 'BaseProactorEventLoop',
__all__ = ['BaseProactorEventLoop']
import io
import os
import socket
import warnings
import signal
import threading
import collections
from . import base_events
from . import compat
from . import constants
from . import futures
from . import exceptions
from . import protocols
from . import sslproto
from . import transports
from . import trsock
from .log import logger
def _set_socket_extra(transport, sock):
transport._extra['socket'] = trsock.TransportSocket(sock)
try:
transport._extra['sockname'] = sock.getsockname()
except socket.error:
if transport._loop.get_debug():
logger.warning(
"getsockname() failed on %r", sock, exc_info=True)
if 'peername' not in transport._extra:
try:
transport._extra['peername'] = sock.getpeername()
except socket.error:
# UDP sockets may not have a peer name
transport._extra['peername'] = None
class _ProactorBasePipeTransport(transports._FlowControlMixin,
transports.BaseTransport):
"""Base class for pipe and socket transports."""
@@ -52,7 +27,7 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
super().__init__(extra, loop)
self._set_extra(sock)
self._sock = sock
self.set_protocol(protocol)
self._protocol = protocol
self._server = server
self._buffer = None # None or bytearray.
self._read_fut = None
@@ -60,7 +35,6 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
self._pending_write = 0
self._conn_lost = 0
self._closing = False # Set when close() called.
self._called_connection_lost = False
self._eof_written = False
if self._server is not None:
self._server._attach()
@@ -77,16 +51,17 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
elif self._closing:
info.append('closing')
if self._sock is not None:
info.append(f'fd={self._sock.fileno()}')
info.append('fd=%s' % self._sock.fileno())
if self._read_fut is not None:
info.append(f'read={self._read_fut!r}')
info.append('read=%s' % self._read_fut)
if self._write_fut is not None:
info.append(f'write={self._write_fut!r}')
info.append("write=%r" % self._write_fut)
if self._buffer:
info.append(f'write_bufsize={len(self._buffer)}')
bufsize = len(self._buffer)
info.append('write_bufsize=%s' % bufsize)
if self._eof_written:
info.append('EOF written')
return '<{}>'.format(' '.join(info))
return '<%s>' % ' '.join(info)
def _set_extra(self, sock):
self._extra['pipe'] = sock
@@ -111,33 +86,31 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
self._read_fut.cancel()
self._read_fut = None
def __del__(self, _warn=warnings.warn):
if self._sock is not None:
_warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
self._sock.close()
# On Python 3.3 and older, objects with a destructor part of a reference
# cycle are never destroyed. It's not more the case on Python 3.4 thanks
# to the PEP 442.
if compat.PY34:
def __del__(self):
if self._sock is not None:
warnings.warn("unclosed transport %r" % self, ResourceWarning,
source=self)
self.close()
def _fatal_error(self, exc, message='Fatal error on pipe transport'):
try:
if isinstance(exc, OSError):
if self._loop.get_debug():
logger.debug("%r: %s", self, message, exc_info=True)
else:
self._loop.call_exception_handler({
'message': message,
'exception': exc,
'transport': self,
'protocol': self._protocol,
})
finally:
self._force_close(exc)
if isinstance(exc, base_events._FATAL_ERROR_IGNORE):
if self._loop.get_debug():
logger.debug("%r: %s", self, message, exc_info=True)
else:
self._loop.call_exception_handler({
'message': message,
'exception': exc,
'transport': self,
'protocol': self._protocol,
})
self._force_close(exc)
def _force_close(self, exc):
if self._empty_waiter is not None and not self._empty_waiter.done():
if exc is None:
self._empty_waiter.set_result(None)
else:
self._empty_waiter.set_exception(exc)
if self._closing and self._called_connection_lost:
if self._closing:
return
self._closing = True
self._conn_lost += 1
@@ -152,8 +125,6 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
self._loop.call_soon(self._call_connection_lost, exc)
def _call_connection_lost(self, exc):
if self._called_connection_lost:
return
try:
self._protocol.connection_lost(exc)
finally:
@@ -161,7 +132,7 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
# end then it may fail with ERROR_NETNAME_DELETED if we
# just close our end. First calling shutdown() seems to
# cure it, but maybe using DisconnectEx() would be better.
if hasattr(self._sock, 'shutdown') and self._sock.fileno() != -1:
if hasattr(self._sock, 'shutdown'):
self._sock.shutdown(socket.SHUT_RDWR)
self._sock.close()
self._sock = None
@@ -169,7 +140,6 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
if server is not None:
server._detach()
self._server = None
self._called_connection_lost = True
def get_write_buffer_size(self):
size = self._pending_write
@@ -183,127 +153,53 @@ class _ProactorReadPipeTransport(_ProactorBasePipeTransport,
"""Transport for read pipes."""
def __init__(self, loop, sock, protocol, waiter=None,
extra=None, server=None, buffer_size=65536):
self._pending_data_length = -1
self._paused = True
extra=None, server=None):
super().__init__(loop, sock, protocol, waiter, extra, server)
self._data = bytearray(buffer_size)
self._loop.call_soon(self._loop_reading)
self._paused = False
def is_reading(self):
return not self._paused and not self._closing
self._loop.call_soon(self._loop_reading)
def pause_reading(self):
if self._closing or self._paused:
return
if self._closing:
raise RuntimeError('Cannot pause_reading() when closing')
if self._paused:
raise RuntimeError('Already paused')
self._paused = True
# bpo-33694: Don't cancel self._read_fut because cancelling an
# overlapped WSASend() loss silently data with the current proactor
# implementation.
#
# If CancelIoEx() fails with ERROR_NOT_FOUND, it means that WSASend()
# completed (even if HasOverlappedIoCompleted() returns 0), but
# Overlapped.cancel() currently silently ignores the ERROR_NOT_FOUND
# error. Once the overlapped is ignored, the IOCP loop will ignores the
# completion I/O event and so not read the result of the overlapped
# WSARecv().
if self._loop.get_debug():
logger.debug("%r pauses reading", self)
def resume_reading(self):
if self._closing or not self._paused:
return
if not self._paused:
raise RuntimeError('Not paused')
self._paused = False
if self._read_fut is None:
self._loop.call_soon(self._loop_reading, None)
length = self._pending_data_length
self._pending_data_length = -1
if length > -1:
# Call the protocol method after calling _loop_reading(),
# since the protocol can decide to pause reading again.
self._loop.call_soon(self._data_received, self._data[:length], length)
if self._closing:
return
self._loop.call_soon(self._loop_reading, self._read_fut)
if self._loop.get_debug():
logger.debug("%r resumes reading", self)
def _eof_received(self):
if self._loop.get_debug():
logger.debug("%r received EOF", self)
try:
keep_open = self._protocol.eof_received()
except (SystemExit, KeyboardInterrupt):
raise
except BaseException as exc:
self._fatal_error(
exc, 'Fatal error: protocol.eof_received() call failed.')
return
if not keep_open:
self.close()
def _data_received(self, data, length):
if self._paused:
# Don't call any protocol method while reading is paused.
# The protocol will be called on resume_reading().
assert self._pending_data_length == -1
self._pending_data_length = length
return
if length == 0:
self._eof_received()
return
if isinstance(self._protocol, protocols.BufferedProtocol):
try:
protocols._feed_data_to_buffered_proto(self._protocol, data)
except (SystemExit, KeyboardInterrupt):
raise
except BaseException as exc:
self._fatal_error(exc,
'Fatal error: protocol.buffer_updated() '
'call failed.')
return
else:
self._protocol.data_received(data)
def _loop_reading(self, fut=None):
length = -1
if self._paused:
return
data = None
try:
if fut is not None:
assert self._read_fut is fut or (self._read_fut is None and
self._closing)
self._read_fut = None
if fut.done():
# deliver data later in "finally" clause
length = fut.result()
if length == 0:
# we got end-of-file so no need to reschedule a new read
return
# It's a new slice so make it immutable so protocols upstream don't have problems
data = bytes(memoryview(self._data)[:length])
else:
# the future will be replaced by next proactor.recv call
fut.cancel()
data = fut.result() # deliver data later in "finally" clause
if self._closing:
# since close() has been called we ignore any read data
data = None
return
# bpo-33694: buffer_updated() has currently no fast path because of
# a data loss issue caused by overlapped WSASend() cancellation.
if data == b'':
# we got end-of-file so no need to reschedule a new read
return
if not self._paused:
# reschedule a new read
self._read_fut = self._loop._proactor.recv_into(self._sock, self._data)
# reschedule a new read
self._read_fut = self._loop._proactor.recv(self._sock, 4096)
except ConnectionAbortedError as exc:
if not self._closing:
self._fatal_error(exc, 'Fatal read error on pipe transport')
@@ -314,36 +210,32 @@ class _ProactorReadPipeTransport(_ProactorBasePipeTransport,
self._force_close(exc)
except OSError as exc:
self._fatal_error(exc, 'Fatal read error on pipe transport')
except exceptions.CancelledError:
except futures.CancelledError:
if not self._closing:
raise
else:
if not self._paused:
self._read_fut.add_done_callback(self._loop_reading)
self._read_fut.add_done_callback(self._loop_reading)
finally:
if length > -1:
self._data_received(data, length)
if data:
self._protocol.data_received(data)
elif data is not None:
if self._loop.get_debug():
logger.debug("%r received EOF", self)
keep_open = self._protocol.eof_received()
if not keep_open:
self.close()
class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
transports.WriteTransport):
"""Transport for write pipes."""
_start_tls_compatible = True
def __init__(self, *args, **kw):
super().__init__(*args, **kw)
self._empty_waiter = None
def write(self, data):
if not isinstance(data, (bytes, bytearray, memoryview)):
raise TypeError(
f"data argument must be a bytes-like object, "
f"not {type(data).__name__}")
raise TypeError('data argument must be byte-ish (%r)',
type(data))
if self._eof_written:
raise RuntimeError('write_eof() already called')
if self._empty_waiter is not None:
raise RuntimeError('unable to write; sendfile is in progress')
if not data:
return
@@ -375,10 +267,6 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
def _loop_writing(self, f=None, data=None):
try:
if f is not None and self._write_fut is None and self._closing:
# XXX most likely self._force_close() has been called, and
# it has set self._write_fut to None.
return
assert f is self._write_fut
self._write_fut = None
self._pending_write = 0
@@ -407,8 +295,6 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
self._maybe_pause_protocol()
else:
self._write_fut.add_done_callback(self._loop_writing)
if self._empty_waiter is not None and self._write_fut is None:
self._empty_waiter.set_result(None)
except ConnectionResetError as exc:
self._force_close(exc)
except OSError as exc:
@@ -423,17 +309,6 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
def abort(self):
self._force_close(None)
def _make_empty_waiter(self):
if self._empty_waiter is not None:
raise RuntimeError("Empty waiter is already set")
self._empty_waiter = self._loop.create_future()
if self._write_fut is None:
self._empty_waiter.set_result(None)
return self._empty_waiter
def _reset_empty_waiter(self):
self._empty_waiter = None
class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport):
def __init__(self, *args, **kw):
@@ -457,138 +332,6 @@ class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport):
self.close()
class _ProactorDatagramTransport(_ProactorBasePipeTransport,
transports.DatagramTransport):
max_size = 256 * 1024
def __init__(self, loop, sock, protocol, address=None,
waiter=None, extra=None):
self._address = address
self._empty_waiter = None
self._buffer_size = 0
# We don't need to call _protocol.connection_made() since our base
# constructor does it for us.
super().__init__(loop, sock, protocol, waiter=waiter, extra=extra)
# The base constructor sets _buffer = None, so we set it here
self._buffer = collections.deque()
self._loop.call_soon(self._loop_reading)
def _set_extra(self, sock):
_set_socket_extra(self, sock)
def get_write_buffer_size(self):
return self._buffer_size
def abort(self):
self._force_close(None)
def sendto(self, data, addr=None):
if not isinstance(data, (bytes, bytearray, memoryview)):
raise TypeError('data argument must be bytes-like object (%r)',
type(data))
if not data:
return
if self._address is not None and addr not in (None, self._address):
raise ValueError(
f'Invalid address: must be None or {self._address}')
if self._conn_lost and self._address:
if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
logger.warning('socket.sendto() raised exception.')
self._conn_lost += 1
return
# Ensure that what we buffer is immutable.
self._buffer.append((bytes(data), addr))
self._buffer_size += len(data)
if self._write_fut is None:
# No current write operations are active, kick one off
self._loop_writing()
# else: A write operation is already kicked off
self._maybe_pause_protocol()
def _loop_writing(self, fut=None):
try:
if self._conn_lost:
return
assert fut is self._write_fut
self._write_fut = None
if fut:
# We are in a _loop_writing() done callback, get the result
fut.result()
if not self._buffer or (self._conn_lost and self._address):
# The connection has been closed
if self._closing:
self._loop.call_soon(self._call_connection_lost, None)
return
data, addr = self._buffer.popleft()
self._buffer_size -= len(data)
if self._address is not None:
self._write_fut = self._loop._proactor.send(self._sock,
data)
else:
self._write_fut = self._loop._proactor.sendto(self._sock,
data,
addr=addr)
except OSError as exc:
self._protocol.error_received(exc)
except Exception as exc:
self._fatal_error(exc, 'Fatal write error on datagram transport')
else:
self._write_fut.add_done_callback(self._loop_writing)
self._maybe_resume_protocol()
def _loop_reading(self, fut=None):
data = None
try:
if self._conn_lost:
return
assert self._read_fut is fut or (self._read_fut is None and
self._closing)
self._read_fut = None
if fut is not None:
res = fut.result()
if self._closing:
# since close() has been called we ignore any read data
data = None
return
if self._address is not None:
data, addr = res, self._address
else:
data, addr = res
if self._conn_lost:
return
if self._address is not None:
self._read_fut = self._loop._proactor.recv(self._sock,
self.max_size)
else:
self._read_fut = self._loop._proactor.recvfrom(self._sock,
self.max_size)
except OSError as exc:
self._protocol.error_received(exc)
except exceptions.CancelledError:
if not self._closing:
raise
else:
if self._read_fut is not None:
self._read_fut.add_done_callback(self._loop_reading)
finally:
if data:
self._protocol.datagram_received(data, addr)
class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport,
_ProactorBaseWritePipeTransport,
transports.Transport):
@@ -606,15 +349,21 @@ class _ProactorSocketTransport(_ProactorReadPipeTransport,
transports.Transport):
"""Transport for connected sockets."""
_sendfile_compatible = constants._SendfileMode.TRY_NATIVE
def __init__(self, loop, sock, protocol, waiter=None,
extra=None, server=None):
super().__init__(loop, sock, protocol, waiter, extra, server)
base_events._set_nodelay(sock)
def _set_extra(self, sock):
_set_socket_extra(self, sock)
self._extra['socket'] = sock
try:
self._extra['sockname'] = sock.getsockname()
except (socket.error, AttributeError):
if self._loop.get_debug():
logger.warning("getsockname() failed on %r",
sock, exc_info=True)
if 'peername' not in self._extra:
try:
self._extra['peername'] = sock.getpeername()
except (socket.error, AttributeError):
if self._loop.get_debug():
logger.warning("getpeername() failed on %r",
sock, exc_info=True)
def can_write_eof(self):
return True
@@ -638,35 +387,26 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
self._accept_futures = {} # socket file descriptor => Future
proactor.set_loop(self)
self._make_self_pipe()
if threading.current_thread() is threading.main_thread():
# wakeup fd can only be installed to a file descriptor from the main thread
signal.set_wakeup_fd(self._csock.fileno())
def _make_socket_transport(self, sock, protocol, waiter=None,
extra=None, server=None):
return _ProactorSocketTransport(self, sock, protocol, waiter,
extra, server)
def _make_ssl_transport(
self, rawsock, protocol, sslcontext, waiter=None,
*, server_side=False, server_hostname=None,
extra=None, server=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None):
ssl_protocol = sslproto.SSLProtocol(
self, protocol, sslcontext, waiter,
server_side, server_hostname,
ssl_handshake_timeout=ssl_handshake_timeout,
ssl_shutdown_timeout=ssl_shutdown_timeout)
def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None,
*, server_side=False, server_hostname=None,
extra=None, server=None):
if not sslproto._is_sslproto_available():
raise NotImplementedError("Proactor event loop requires Python 3.5"
" or newer (ssl.MemoryBIO) to support "
"SSL")
ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter,
server_side, server_hostname)
_ProactorSocketTransport(self, rawsock, ssl_protocol,
extra=extra, server=server)
return ssl_protocol._app_transport
def _make_datagram_transport(self, sock, protocol,
address=None, waiter=None, extra=None):
return _ProactorDatagramTransport(self, sock, protocol, address,
waiter, extra)
def _make_duplex_pipe_transport(self, sock, protocol, waiter=None,
extra=None):
return _ProactorDuplexPipeTransport(self,
@@ -688,8 +428,6 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
if self.is_closed():
return
if threading.current_thread() is threading.main_thread():
signal.set_wakeup_fd(-1)
# Call these methods before closing the event loop (before calling
# BaseEventLoop.close), because they can schedule callbacks with
# call_soon(), which is forbidden when the event loop is closed.
@@ -702,73 +440,20 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
# Close the event loop
super().close()
async def sock_recv(self, sock, n):
return await self._proactor.recv(sock, n)
def sock_recv(self, sock, n):
return self._proactor.recv(sock, n)
async def sock_recv_into(self, sock, buf):
return await self._proactor.recv_into(sock, buf)
def sock_sendall(self, sock, data):
return self._proactor.send(sock, data)
async def sock_recvfrom(self, sock, bufsize):
return await self._proactor.recvfrom(sock, bufsize)
def sock_connect(self, sock, address):
return self._proactor.connect(sock, address)
async def sock_recvfrom_into(self, sock, buf, nbytes=0):
if not nbytes:
nbytes = len(buf)
def sock_accept(self, sock):
return self._proactor.accept(sock)
return await self._proactor.recvfrom_into(sock, buf, nbytes)
async def sock_sendall(self, sock, data):
return await self._proactor.send(sock, data)
async def sock_sendto(self, sock, data, address):
return await self._proactor.sendto(sock, data, 0, address)
async def sock_connect(self, sock, address):
return await self._proactor.connect(sock, address)
async def sock_accept(self, sock):
return await self._proactor.accept(sock)
async def _sock_sendfile_native(self, sock, file, offset, count):
try:
fileno = file.fileno()
except (AttributeError, io.UnsupportedOperation) as err:
raise exceptions.SendfileNotAvailableError("not a regular file")
try:
fsize = os.fstat(fileno).st_size
except OSError:
raise exceptions.SendfileNotAvailableError("not a regular file")
blocksize = count if count else fsize
if not blocksize:
return 0 # empty file
blocksize = min(blocksize, 0xffff_ffff)
end_pos = min(offset + count, fsize) if count else fsize
offset = min(offset, fsize)
total_sent = 0
try:
while True:
blocksize = min(end_pos - offset, blocksize)
if blocksize <= 0:
return total_sent
await self._proactor.sendfile(sock, file, offset, blocksize)
offset += blocksize
total_sent += blocksize
finally:
if total_sent > 0:
file.seek(offset)
async def _sendfile_native(self, transp, file, offset, count):
resume_reading = transp.is_reading()
transp.pause_reading()
await transp._make_empty_waiter()
try:
return await self.sock_sendfile(transp._sock, file, offset, count,
fallback=False)
finally:
transp._reset_empty_waiter()
if resume_reading:
transp.resume_reading()
def _socketpair(self):
raise NotImplementedError
def _close_self_pipe(self):
if self._self_reading_future is not None:
@@ -782,30 +467,21 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
def _make_self_pipe(self):
# A self-socket, really. :-)
self._ssock, self._csock = socket.socketpair()
self._ssock, self._csock = self._socketpair()
self._ssock.setblocking(False)
self._csock.setblocking(False)
self._internal_fds += 1
self.call_soon(self._loop_self_reading)
def _loop_self_reading(self, f=None):
try:
if f is not None:
f.result() # may raise
if self._self_reading_future is not f:
# When we scheduled this Future, we assigned it to
# _self_reading_future. If it's not there now, something has
# tried to cancel the loop while this callback was still in the
# queue (see windows_events.ProactorEventLoop.run_forever). In
# that case stop here instead of continuing to schedule a new
# iteration.
return
f = self._proactor.recv(self._ssock, 4096)
except exceptions.CancelledError:
except futures.CancelledError:
# _close_self_pipe() has been called, stop waiting for data
return
except (SystemExit, KeyboardInterrupt):
raise
except BaseException as exc:
except Exception as exc:
self.call_exception_handler({
'message': 'Error on reading from the event loop self pipe',
'exception': exc,
@@ -816,27 +492,10 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
f.add_done_callback(self._loop_self_reading)
def _write_to_self(self):
# This may be called from a different thread, possibly after
# _close_self_pipe() has been called or even while it is
# running. Guard for self._csock being None or closed. When
# a socket is closed, send() raises OSError (with errno set to
# EBADF, but let's not rely on the exact error code).
csock = self._csock
if csock is None:
return
try:
csock.send(b'\0')
except OSError:
if self._debug:
logger.debug("Fail to write a null byte into the "
"self-pipe socket",
exc_info=True)
self._csock.send(b'\0')
def _start_serving(self, protocol_factory, sock,
sslcontext=None, server=None, backlog=100,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None):
sslcontext=None, server=None, backlog=100):
def loop(f=None):
try:
@@ -849,9 +508,7 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
if sslcontext is not None:
self._make_ssl_transport(
conn, protocol, sslcontext, server_side=True,
extra={'peername': addr}, server=server,
ssl_handshake_timeout=ssl_handshake_timeout,
ssl_shutdown_timeout=ssl_shutdown_timeout)
extra={'peername': addr}, server=server)
else:
self._make_socket_transport(
conn, protocol,
@@ -864,13 +521,13 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
self.call_exception_handler({
'message': 'Accept failed on a socket',
'exception': exc,
'socket': trsock.TransportSocket(sock),
'socket': sock,
})
sock.close()
elif self._debug:
logger.debug("Accept failed on socket %r",
sock, exc_info=True)
except exceptions.CancelledError:
except futures.CancelledError:
sock.close()
else:
self._accept_futures[sock.fileno()] = f
@@ -888,8 +545,6 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
self._accept_futures.clear()
def _stop_serving(self, sock):
future = self._accept_futures.pop(sock.fileno(), None)
if future:
future.cancel()
self._stop_accept_futures()
self._proactor._stop_serving(sock)
sock.close()

88
Lib/asyncio/protocols.py
View File

@@ -1,9 +1,7 @@
"""Abstract Protocol base classes."""
"""Abstract Protocol class."""
__all__ = (
'BaseProtocol', 'Protocol', 'DatagramProtocol',
'SubprocessProtocol', 'BufferedProtocol',
)
__all__ = ['BaseProtocol', 'Protocol', 'DatagramProtocol',
'SubprocessProtocol']
class BaseProtocol:
@@ -16,8 +14,6 @@ class BaseProtocol:
write-only transport like write pipe
"""
__slots__ = ()
def connection_made(self, transport):
"""Called when a connection is made.
@@ -89,8 +85,6 @@ class Protocol(BaseProtocol):
* CL: connection_lost()
"""
__slots__ = ()
def data_received(self, data):
"""Called when some data is received.
@@ -106,64 +100,9 @@ class Protocol(BaseProtocol):
"""
class BufferedProtocol(BaseProtocol):
"""Interface for stream protocol with manual buffer control.
Event methods, such as `create_server` and `create_connection`,
accept factories that return protocols that implement this interface.
The idea of BufferedProtocol is that it allows to manually allocate
and control the receive buffer. Event loops can then use the buffer
provided by the protocol to avoid unnecessary data copies. This
can result in noticeable performance improvement for protocols that
receive big amounts of data. Sophisticated protocols can allocate
the buffer only once at creation time.
State machine of calls:
start -> CM [-> GB [-> BU?]]* [-> ER?] -> CL -> end
* CM: connection_made()
* GB: get_buffer()
* BU: buffer_updated()
* ER: eof_received()
* CL: connection_lost()
"""
__slots__ = ()
def get_buffer(self, sizehint):
"""Called to allocate a new receive buffer.
*sizehint* is a recommended minimal size for the returned
buffer. When set to -1, the buffer size can be arbitrary.
Must return an object that implements the
:ref:`buffer protocol <bufferobjects>`.
It is an error to return a zero-sized buffer.
"""
def buffer_updated(self, nbytes):
"""Called when the buffer was updated with the received data.
*nbytes* is the total number of bytes that were written to
the buffer.
"""
def eof_received(self):
"""Called when the other end calls write_eof() or equivalent.
If this returns a false value (including None), the transport
will close itself. If it returns a true value, closing the
transport is up to the protocol.
"""
class DatagramProtocol(BaseProtocol):
"""Interface for datagram protocol."""
__slots__ = ()
def datagram_received(self, data, addr):
"""Called when some datagram is received."""
@@ -177,8 +116,6 @@ class DatagramProtocol(BaseProtocol):
class SubprocessProtocol(BaseProtocol):
"""Interface for protocol for subprocess calls."""
__slots__ = ()
def pipe_data_received(self, fd, data):
"""Called when the subprocess writes data into stdout/stderr pipe.
@@ -195,22 +132,3 @@ class SubprocessProtocol(BaseProtocol):
def process_exited(self):
"""Called when subprocess has exited."""
def _feed_data_to_buffered_proto(proto, data):
data_len = len(data)
while data_len:
buf = proto.get_buffer(data_len)
buf_len = len(buf)
if not buf_len:
raise RuntimeError('get_buffer() returned an empty buffer')
if buf_len >= data_len:
buf[:data_len] = data
proto.buffer_updated(data_len)
return
else:
buf[:buf_len] = data[:buf_len]
proto.buffer_updated(buf_len)
data = data[buf_len:]
data_len = len(data)

89
Lib/asyncio/queues.py
View File

@@ -1,28 +1,35 @@
__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty')
"""Queues"""
__all__ = ['Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty']
import collections
import heapq
from types import GenericAlias
from . import compat
from . import events
from . import locks
from . import mixins
from .coroutines import coroutine
class QueueEmpty(Exception):
"""Raised when Queue.get_nowait() is called on an empty Queue."""
"""Exception raised when Queue.get_nowait() is called on a Queue object
which is empty.
"""
pass
class QueueFull(Exception):
"""Raised when the Queue.put_nowait() method is called on a full Queue."""
"""Exception raised when the Queue.put_nowait() method is called on a Queue
object which is full.
"""
pass
class Queue(mixins._LoopBoundMixin):
class Queue:
"""A queue, useful for coordinating producer and consumer coroutines.
If maxsize is less than or equal to zero, the queue size is infinite. If it
is an integer greater than 0, then "await put()" will block when the
is an integer greater than 0, then "yield from put()" will block when the
queue reaches maxsize, until an item is removed by get().
Unlike the standard library Queue, you can reliably know this Queue's size
@@ -30,7 +37,11 @@ class Queue(mixins._LoopBoundMixin):
interrupted between calling qsize() and doing an operation on the Queue.
"""
def __init__(self, maxsize=0):
def __init__(self, maxsize=0, *, loop=None):
if loop is None:
self._loop = events.get_event_loop()
else:
self._loop = loop
self._maxsize = maxsize
# Futures.
@@ -38,7 +49,7 @@ class Queue(mixins._LoopBoundMixin):
# Futures.
self._putters = collections.deque()
self._unfinished_tasks = 0
self._finished = locks.Event()
self._finished = locks.Event(loop=self._loop)
self._finished.set()
self._init(maxsize)
@@ -64,23 +75,22 @@ class Queue(mixins._LoopBoundMixin):
break
def __repr__(self):
return f'<{type(self).__name__} at {id(self):#x} {self._format()}>'
return '<{} at {:#x} {}>'.format(
type(self).__name__, id(self), self._format())
def __str__(self):
return f'<{type(self).__name__} {self._format()}>'
__class_getitem__ = classmethod(GenericAlias)
return '<{} {}>'.format(type(self).__name__, self._format())
def _format(self):
result = f'maxsize={self._maxsize!r}'
result = 'maxsize={!r}'.format(self._maxsize)
if getattr(self, '_queue', None):
result += f' _queue={list(self._queue)!r}'
result += ' _queue={!r}'.format(list(self._queue))
if self._getters:
result += f' _getters[{len(self._getters)}]'
result += ' _getters[{}]'.format(len(self._getters))
if self._putters:
result += f' _putters[{len(self._putters)}]'
result += ' _putters[{}]'.format(len(self._putters))
if self._unfinished_tasks:
result += f' tasks={self._unfinished_tasks}'
result += ' tasks={}'.format(self._unfinished_tasks)
return result
def qsize(self):
@@ -107,26 +117,22 @@ class Queue(mixins._LoopBoundMixin):
else:
return self.qsize() >= self._maxsize
async def put(self, item):
@coroutine
def put(self, item):
"""Put an item into the queue.
Put an item into the queue. If the queue is full, wait until a free
slot is available before adding item.
This method is a coroutine.
"""
while self.full():
putter = self._get_loop().create_future()
putter = self._loop.create_future()
self._putters.append(putter)
try:
await putter
yield from putter
except:
putter.cancel() # Just in case putter is not done yet.
try:
# Clean self._putters from canceled putters.
self._putters.remove(putter)
except ValueError:
# The putter could be removed from self._putters by a
# previous get_nowait call.
pass
if not self.full() and not putter.cancelled():
# We were woken up by get_nowait(), but can't take
# the call. Wake up the next in line.
@@ -146,25 +152,21 @@ class Queue(mixins._LoopBoundMixin):
self._finished.clear()
self._wakeup_next(self._getters)
async def get(self):
@coroutine
def get(self):
"""Remove and return an item from the queue.
If queue is empty, wait until an item is available.
This method is a coroutine.
"""
while self.empty():
getter = self._get_loop().create_future()
getter = self._loop.create_future()
self._getters.append(getter)
try:
await getter
yield from getter
except:
getter.cancel() # Just in case getter is not done yet.
try:
# Clean self._getters from canceled getters.
self._getters.remove(getter)
except ValueError:
# The getter could be removed from self._getters by a
# previous put_nowait call.
pass
if not self.empty() and not getter.cancelled():
# We were woken up by put_nowait(), but can't take
# the call. Wake up the next in line.
@@ -203,7 +205,8 @@ class Queue(mixins._LoopBoundMixin):
if self._unfinished_tasks == 0:
self._finished.set()
async def join(self):
@coroutine
def join(self):
"""Block until all items in the queue have been gotten and processed.
The count of unfinished tasks goes up whenever an item is added to the
@@ -212,7 +215,7 @@ class Queue(mixins._LoopBoundMixin):
When the count of unfinished tasks drops to zero, join() unblocks.
"""
if self._unfinished_tasks > 0:
await self._finished.wait()
yield from self._finished.wait()
class PriorityQueue(Queue):
@@ -242,3 +245,9 @@ class LifoQueue(Queue):
def _get(self):
return self._queue.pop()
if not compat.PY35:
JoinableQueue = Queue
"""Deprecated alias for Queue."""
__all__.append('JoinableQueue')

187
Lib/asyncio/runners.py
View File

@@ -1,168 +1,16 @@
__all__ = ('Runner', 'run')
__all__ = ['run']
import contextvars
import enum
import functools
import threading
import signal
from . import coroutines
from . import events
from . import exceptions
from . import tasks
from . import constants
class _State(enum.Enum):
CREATED = "created"
INITIALIZED = "initialized"
CLOSED = "closed"
class Runner:
"""A context manager that controls event loop life cycle.
The context manager always creates a new event loop,
allows to run async functions inside it,
and properly finalizes the loop at the context manager exit.
If debug is True, the event loop will be run in debug mode.
If loop_factory is passed, it is used for new event loop creation.
asyncio.run(main(), debug=True)
is a shortcut for
with asyncio.Runner(debug=True) as runner:
runner.run(main())
The run() method can be called multiple times within the runner's context.
This can be useful for interactive console (e.g. IPython),
unittest runners, console tools, -- everywhere when async code
is called from existing sync framework and where the preferred single
asyncio.run() call doesn't work.
"""
# Note: the class is final, it is not intended for inheritance.
def __init__(self, *, debug=None, loop_factory=None):
self._state = _State.CREATED
self._debug = debug
self._loop_factory = loop_factory
self._loop = None
self._context = None
self._interrupt_count = 0
self._set_event_loop = False
def __enter__(self):
self._lazy_init()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self):
"""Shutdown and close event loop."""
if self._state is not _State.INITIALIZED:
return
try:
loop = self._loop
_cancel_all_tasks(loop)
loop.run_until_complete(loop.shutdown_asyncgens())
loop.run_until_complete(
loop.shutdown_default_executor(constants.THREAD_JOIN_TIMEOUT))
finally:
if self._set_event_loop:
events.set_event_loop(None)
loop.close()
self._loop = None
self._state = _State.CLOSED
def get_loop(self):
"""Return embedded event loop."""
self._lazy_init()
return self._loop
def run(self, coro, *, context=None):
"""Run a coroutine inside the embedded event loop."""
if not coroutines.iscoroutine(coro):
raise ValueError("a coroutine was expected, got {!r}".format(coro))
if events._get_running_loop() is not None:
# fail fast with short traceback
raise RuntimeError(
"Runner.run() cannot be called from a running event loop")
self._lazy_init()
if context is None:
context = self._context
task = self._loop.create_task(coro, context=context)
if (threading.current_thread() is threading.main_thread()
and signal.getsignal(signal.SIGINT) is signal.default_int_handler
):
sigint_handler = functools.partial(self._on_sigint, main_task=task)
try:
signal.signal(signal.SIGINT, sigint_handler)
except ValueError:
# `signal.signal` may throw if `threading.main_thread` does
# not support signals (e.g. embedded interpreter with signals
# not registered - see gh-91880)
sigint_handler = None
else:
sigint_handler = None
self._interrupt_count = 0
try:
return self._loop.run_until_complete(task)
except exceptions.CancelledError:
if self._interrupt_count > 0:
uncancel = getattr(task, "uncancel", None)
if uncancel is not None and uncancel() == 0:
raise KeyboardInterrupt()
raise # CancelledError
finally:
if (sigint_handler is not None
and signal.getsignal(signal.SIGINT) is sigint_handler
):
signal.signal(signal.SIGINT, signal.default_int_handler)
def _lazy_init(self):
if self._state is _State.CLOSED:
raise RuntimeError("Runner is closed")
if self._state is _State.INITIALIZED:
return
if self._loop_factory is None:
self._loop = events.new_event_loop()
if not self._set_event_loop:
# Call set_event_loop only once to avoid calling
# attach_loop multiple times on child watchers
events.set_event_loop(self._loop)
self._set_event_loop = True
else:
self._loop = self._loop_factory()
if self._debug is not None:
self._loop.set_debug(self._debug)
self._context = contextvars.copy_context()
self._state = _State.INITIALIZED
def _on_sigint(self, signum, frame, main_task):
self._interrupt_count += 1
if self._interrupt_count == 1 and not main_task.done():
main_task.cancel()
# wakeup loop if it is blocked by select() with long timeout
self._loop.call_soon_threadsafe(lambda: None)
return
raise KeyboardInterrupt()
def run(main, *, debug=None, loop_factory=None):
"""Execute the coroutine and return the result.
def run(main, *, debug=False):
"""Run a coroutine.
This function runs the passed coroutine, taking care of
managing the asyncio event loop, finalizing asynchronous
generators and closing the default executor.
managing the asyncio event loop and finalizing asynchronous
generators.
This function cannot be called when another asyncio event loop is
running in the same thread.
@@ -173,10 +21,6 @@ def run(main, *, debug=None, loop_factory=None):
It should be used as a main entry point for asyncio programs, and should
ideally only be called once.
The executor is given a timeout duration of 5 minutes to shutdown.
If the executor hasn't finished within that duration, a warning is
emitted and the executor is closed.
Example:
async def main():
@@ -186,12 +30,24 @@ def run(main, *, debug=None, loop_factory=None):
asyncio.run(main())
"""
if events._get_running_loop() is not None:
# fail fast with short traceback
raise RuntimeError(
"asyncio.run() cannot be called from a running event loop")
with Runner(debug=debug, loop_factory=loop_factory) as runner:
return runner.run(main)
if not coroutines.iscoroutine(main):
raise ValueError("a coroutine was expected, got {!r}".format(main))
loop = events.new_event_loop()
try:
events.set_event_loop(loop)
loop.set_debug(debug)
return loop.run_until_complete(main)
finally:
try:
_cancel_all_tasks(loop)
loop.run_until_complete(loop.shutdown_asyncgens())
finally:
events.set_event_loop(None)
loop.close()
def _cancel_all_tasks(loop):
@@ -202,7 +58,8 @@ def _cancel_all_tasks(loop):
for task in to_cancel:
task.cancel()
loop.run_until_complete(tasks.gather(*to_cancel, return_exceptions=True))
loop.run_until_complete(
tasks.gather(*to_cancel, loop=loop, return_exceptions=True))
for task in to_cancel:
if task.cancelled():

1093
Lib/asyncio/selector_events.py
View File

File diff suppressed because it is too large Load Diff

1136
Lib/asyncio/sslproto.py
View File

File diff suppressed because it is too large Load Diff

149
Lib/asyncio/staggered.py
View File

@@ -1,149 +0,0 @@
"""Support for running coroutines in parallel with staggered start times."""
__all__ = 'staggered_race',
import contextlib
import typing
from . import events
from . import exceptions as exceptions_mod
from . import locks
from . import tasks
async def staggered_race(
coro_fns: typing.Iterable[typing.Callable[[], typing.Awaitable]],
delay: typing.Optional[float],
*,
loop: events.AbstractEventLoop = None,
) -> typing.Tuple[
typing.Any,
typing.Optional[int],
typing.List[typing.Optional[Exception]]
]:
"""Run coroutines with staggered start times and take the first to finish.
This method takes an iterable of coroutine functions. The first one is
started immediately. From then on, whenever the immediately preceding one
fails (raises an exception), or when *delay* seconds has passed, the next
coroutine is started. This continues until one of the coroutines complete
successfully, in which case all others are cancelled, or until all
coroutines fail.
The coroutines provided should be well-behaved in the following way:
* They should only ``return`` if completed successfully.
* They should always raise an exception if they did not complete
successfully. In particular, if they handle cancellation, they should
probably reraise, like this::
try:
# do work
except asyncio.CancelledError:
# undo partially completed work
raise
Args:
coro_fns: an iterable of coroutine functions, i.e. callables that
return a coroutine object when called. Use ``functools.partial`` or
lambdas to pass arguments.
delay: amount of time, in seconds, between starting coroutines. If
``None``, the coroutines will run sequentially.
loop: the event loop to use.
Returns:
tuple *(winner_result, winner_index, exceptions)* where
- *winner_result*: the result of the winning coroutine, or ``None``
if no coroutines won.
- *winner_index*: the index of the winning coroutine in
``coro_fns``, or ``None`` if no coroutines won. If the winning
coroutine may return None on success, *winner_index* can be used
to definitively determine whether any coroutine won.
- *exceptions*: list of exceptions returned by the coroutines.
``len(exceptions)`` is equal to the number of coroutines actually
started, and the order is the same as in ``coro_fns``. The winning
coroutine's entry is ``None``.
"""
# TODO: when we have aiter() and anext(), allow async iterables in coro_fns.
loop = loop or events.get_running_loop()
enum_coro_fns = enumerate(coro_fns)
winner_result = None
winner_index = None
exceptions = []
running_tasks = []
async def run_one_coro(
previous_failed: typing.Optional[locks.Event]) -> None:
# Wait for the previous task to finish, or for delay seconds
if previous_failed is not None:
with contextlib.suppress(exceptions_mod.TimeoutError):
# Use asyncio.wait_for() instead of asyncio.wait() here, so
# that if we get cancelled at this point, Event.wait() is also
# cancelled, otherwise there will be a "Task destroyed but it is
# pending" later.
await tasks.wait_for(previous_failed.wait(), delay)
# Get the next coroutine to run
try:
this_index, coro_fn = next(enum_coro_fns)
except StopIteration:
return
# Start task that will run the next coroutine
this_failed = locks.Event()
next_task = loop.create_task(run_one_coro(this_failed))
running_tasks.append(next_task)
assert len(running_tasks) == this_index + 2
# Prepare place to put this coroutine's exceptions if not won
exceptions.append(None)
assert len(exceptions) == this_index + 1
try:
result = await coro_fn()
except (SystemExit, KeyboardInterrupt):
raise
except BaseException as e:
exceptions[this_index] = e
this_failed.set() # Kickstart the next coroutine
else:
# Store winner's results
nonlocal winner_index, winner_result
assert winner_index is None
winner_index = this_index
winner_result = result
# Cancel all other tasks. We take care to not cancel the current
# task as well. If we do so, then since there is no `await` after
# here and CancelledError are usually thrown at one, we will
# encounter a curious corner case where the current task will end
# up as done() == True, cancelled() == False, exception() ==
# asyncio.CancelledError. This behavior is specified in
# https://bugs.python.org/issue30048
for i, t in enumerate(running_tasks):
if i != this_index:
t.cancel()
first_task = loop.create_task(run_one_coro(None))
running_tasks.append(first_task)
try:
# Wait for a growing list of tasks to all finish: poor man's version of
# curio's TaskGroup or trio's nursery
done_count = 0
while done_count != len(running_tasks):
done, _ = await tasks.wait(running_tasks)
done_count = len(done)
# If run_one_coro raises an unhandled exception, it's probably a
# programming error, and I want to see it.
if __debug__:
for d in done:
if d.done() and not d.cancelled() and d.exception():
raise d.exception()
return winner_result, winner_index, exceptions
finally:
# Make sure no tasks are left running if we leave this function
for t in running_tasks:
t.cancel()

411
Lib/asyncio/streams.py
View File

@@ -1,30 +1,55 @@
__all__ = (
'StreamReader', 'StreamWriter', 'StreamReaderProtocol',
'open_connection', 'start_server')
"""Stream-related things."""
__all__ = ['StreamReader', 'StreamWriter', 'StreamReaderProtocol',
'open_connection', 'start_server',
'IncompleteReadError',
'LimitOverrunError',
]
import collections
import socket
import sys
import warnings
import weakref
if hasattr(socket, 'AF_UNIX'):
__all__ += ('open_unix_connection', 'start_unix_server')
__all__.extend(['open_unix_connection', 'start_unix_server'])
from . import coroutines
from . import compat
from . import events
from . import exceptions
from . import format_helpers
from . import protocols
from .coroutines import coroutine
from .log import logger
from .tasks import sleep
_DEFAULT_LIMIT = 2 ** 16 # 64 KiB
_DEFAULT_LIMIT = 2 ** 16
async def open_connection(host=None, port=None, *,
limit=_DEFAULT_LIMIT, **kwds):
class IncompleteReadError(EOFError):
"""
Incomplete read error. Attributes:
- partial: read bytes string before the end of stream was reached
- expected: total number of expected bytes (or None if unknown)
"""
def __init__(self, partial, expected):
super().__init__("%d bytes read on a total of %r expected bytes"
% (len(partial), expected))
self.partial = partial
self.expected = expected
class LimitOverrunError(Exception):
"""Reached the buffer limit while looking for a separator.
Attributes:
- consumed: total number of to be consumed bytes.
"""
def __init__(self, message, consumed):
super().__init__(message)
self.consumed = consumed
@coroutine
def open_connection(host=None, port=None, *,
loop=None, limit=_DEFAULT_LIMIT, **kwds):
"""A wrapper for create_connection() returning a (reader, writer) pair.
The reader returned is a StreamReader instance; the writer is a
@@ -42,17 +67,19 @@ async def open_connection(host=None, port=None, *,
StreamReaderProtocol classes, just copy the code -- there's
really nothing special here except some convenience.)
"""
loop = events.get_running_loop()
if loop is None:
loop = events.get_event_loop()
reader = StreamReader(limit=limit, loop=loop)
protocol = StreamReaderProtocol(reader, loop=loop)
transport, _ = await loop.create_connection(
transport, _ = yield from loop.create_connection(
lambda: protocol, host, port, **kwds)
writer = StreamWriter(transport, protocol, reader, loop)
return reader, writer
async def start_server(client_connected_cb, host=None, port=None, *,
limit=_DEFAULT_LIMIT, **kwds):
@coroutine
def start_server(client_connected_cb, host=None, port=None, *,
loop=None, limit=_DEFAULT_LIMIT, **kwds):
"""Start a socket server, call back for each client connected.
The first parameter, `client_connected_cb`, takes two parameters:
@@ -67,13 +94,15 @@ async def start_server(client_connected_cb, host=None, port=None, *,
positional host and port, with various optional keyword arguments
following. The return value is the same as loop.create_server().
Additional optional keyword argument is limit (to set the buffer
limit passed to the StreamReader).
Additional optional keyword arguments are loop (to set the event loop
instance to use) and limit (to set the buffer limit passed to the
StreamReader).
The return value is the same as loop.create_server(), i.e. a
Server object which can be used to stop the service.
"""
loop = events.get_running_loop()
if loop is None:
loop = events.get_event_loop()
def factory():
reader = StreamReader(limit=limit, loop=loop)
@@ -81,28 +110,31 @@ async def start_server(client_connected_cb, host=None, port=None, *,
loop=loop)
return protocol
return await loop.create_server(factory, host, port, **kwds)
return (yield from loop.create_server(factory, host, port, **kwds))
if hasattr(socket, 'AF_UNIX'):
# UNIX Domain Sockets are supported on this platform
async def open_unix_connection(path=None, *,
limit=_DEFAULT_LIMIT, **kwds):
@coroutine
def open_unix_connection(path=None, *,
loop=None, limit=_DEFAULT_LIMIT, **kwds):
"""Similar to `open_connection` but works with UNIX Domain Sockets."""
loop = events.get_running_loop()
if loop is None:
loop = events.get_event_loop()
reader = StreamReader(limit=limit, loop=loop)
protocol = StreamReaderProtocol(reader, loop=loop)
transport, _ = await loop.create_unix_connection(
transport, _ = yield from loop.create_unix_connection(
lambda: protocol, path, **kwds)
writer = StreamWriter(transport, protocol, reader, loop)
return reader, writer
async def start_unix_server(client_connected_cb, path=None, *,
limit=_DEFAULT_LIMIT, **kwds):
@coroutine
def start_unix_server(client_connected_cb, path=None, *,
loop=None, limit=_DEFAULT_LIMIT, **kwds):
"""Similar to `start_server` but works with UNIX Domain Sockets."""
loop = events.get_running_loop()
if loop is None:
loop = events.get_event_loop()
def factory():
reader = StreamReader(limit=limit, loop=loop)
@@ -110,14 +142,14 @@ if hasattr(socket, 'AF_UNIX'):
loop=loop)
return protocol
return await loop.create_unix_server(factory, path, **kwds)
return (yield from loop.create_unix_server(factory, path, **kwds))
class FlowControlMixin(protocols.Protocol):
"""Reusable flow control logic for StreamWriter.drain().
This implements the protocol methods pause_writing(),
resume_writing() and connection_lost(). If the subclass overrides
resume_reading() and connection_lost(). If the subclass overrides
these it must call the super methods.
StreamWriter.drain() must wait for _drain_helper() coroutine.
@@ -129,7 +161,7 @@ class FlowControlMixin(protocols.Protocol):
else:
self._loop = loop
self._paused = False
self._drain_waiters = collections.deque()
self._drain_waiter = None
self._connection_lost = False
def pause_writing(self):
@@ -144,37 +176,39 @@ class FlowControlMixin(protocols.Protocol):
if self._loop.get_debug():
logger.debug("%r resumes writing", self)
for waiter in self._drain_waiters:
waiter = self._drain_waiter
if waiter is not None:
self._drain_waiter = None
if not waiter.done():
waiter.set_result(None)
def connection_lost(self, exc):
self._connection_lost = True
# Wake up the writer(s) if currently paused.
# Wake up the writer if currently paused.
if not self._paused:
return
waiter = self._drain_waiter
if waiter is None:
return
self._drain_waiter = None
if waiter.done():
return
if exc is None:
waiter.set_result(None)
else:
waiter.set_exception(exc)
for waiter in self._drain_waiters:
if not waiter.done():
if exc is None:
waiter.set_result(None)
else:
waiter.set_exception(exc)
async def _drain_helper(self):
@coroutine
def _drain_helper(self):
if self._connection_lost:
raise ConnectionResetError('Connection lost')
if not self._paused:
return
waiter = self._drain_waiter
assert waiter is None or waiter.cancelled()
waiter = self._loop.create_future()
self._drain_waiters.append(waiter)
try:
await waiter
finally:
self._drain_waiters.remove(waiter)
def _get_close_waiter(self, stream):
raise NotImplementedError
self._drain_waiter = waiter
yield from waiter
class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
@@ -186,110 +220,40 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
call inappropriate methods of the protocol.)
"""
_source_traceback = None
def __init__(self, stream_reader, client_connected_cb=None, loop=None):
super().__init__(loop=loop)
if stream_reader is not None:
self._stream_reader_wr = weakref.ref(stream_reader)
self._source_traceback = stream_reader._source_traceback
else:
self._stream_reader_wr = None
if client_connected_cb is not None:
# This is a stream created by the `create_server()` function.
# Keep a strong reference to the reader until a connection
# is established.
self._strong_reader = stream_reader
self._reject_connection = False
self._stream_reader = stream_reader
self._stream_writer = None
self._task = None
self._transport = None
self._client_connected_cb = client_connected_cb
self._over_ssl = False
self._closed = self._loop.create_future()
@property
def _stream_reader(self):
if self._stream_reader_wr is None:
return None
return self._stream_reader_wr()
def _replace_writer(self, writer):
loop = self._loop
transport = writer.transport
self._stream_writer = writer
self._transport = transport
self._over_ssl = transport.get_extra_info('sslcontext') is not None
def connection_made(self, transport):
if self._reject_connection:
context = {
'message': ('An open stream was garbage collected prior to '
'establishing network connection; '
'call "stream.close()" explicitly.')
}
if self._source_traceback:
context['source_traceback'] = self._source_traceback
self._loop.call_exception_handler(context)
transport.abort()
return
self._transport = transport
reader = self._stream_reader
if reader is not None:
reader.set_transport(transport)
self._stream_reader.set_transport(transport)
self._over_ssl = transport.get_extra_info('sslcontext') is not None
if self._client_connected_cb is not None:
self._stream_writer = StreamWriter(transport, self,
reader,
self._stream_reader,
self._loop)
res = self._client_connected_cb(reader,
res = self._client_connected_cb(self._stream_reader,
self._stream_writer)
if coroutines.iscoroutine(res):
def callback(task):
if task.cancelled():
transport.close()
return
exc = task.exception()
if exc is not None:
self._loop.call_exception_handler({
'message': 'Unhandled exception in client_connected_cb',
'exception': exc,
'transport': transport,
})
transport.close()
self._task = self._loop.create_task(res)
self._task.add_done_callback(callback)
self._strong_reader = None
self._loop.create_task(res)
def connection_lost(self, exc):
reader = self._stream_reader
if reader is not None:
if self._stream_reader is not None:
if exc is None:
reader.feed_eof()
self._stream_reader.feed_eof()
else:
reader.set_exception(exc)
if not self._closed.done():
if exc is None:
self._closed.set_result(None)
else:
self._closed.set_exception(exc)
self._stream_reader.set_exception(exc)
super().connection_lost(exc)
self._stream_reader_wr = None
self._stream_reader = None
self._stream_writer = None
self._task = None
self._transport = None
def data_received(self, data):
reader = self._stream_reader
if reader is not None:
reader.feed_data(data)
self._stream_reader.feed_data(data)
def eof_received(self):
reader = self._stream_reader
if reader is not None:
reader.feed_eof()
self._stream_reader.feed_eof()
if self._over_ssl:
# Prevent a warning in SSLProtocol.eof_received:
# "returning true from eof_received()
@@ -297,20 +261,6 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
return False
return True
def _get_close_waiter(self, stream):
return self._closed
def __del__(self):
# Prevent reports about unhandled exceptions.
# Better than self._closed._log_traceback = False hack
try:
closed = self._closed
except AttributeError:
pass # failed constructor
else:
if closed.done() and not closed.cancelled():
closed.exception()
class StreamWriter:
"""Wraps a Transport.
@@ -329,14 +279,12 @@ class StreamWriter:
assert reader is None or isinstance(reader, StreamReader)
self._reader = reader
self._loop = loop
self._complete_fut = self._loop.create_future()
self._complete_fut.set_result(None)
def __repr__(self):
info = [self.__class__.__name__, f'transport={self._transport!r}']
info = [self.__class__.__name__, 'transport=%r' % self._transport]
if self._reader is not None:
info.append(f'reader={self._reader!r}')
return '<{}>'.format(' '.join(info))
info.append('reader=%r' % self._reader)
return '<%s>' % ' '.join(info)
@property
def transport(self):
@@ -357,68 +305,36 @@ class StreamWriter:
def close(self):
return self._transport.close()
def is_closing(self):
return self._transport.is_closing()
async def wait_closed(self):
await self._protocol._get_close_waiter(self)
def get_extra_info(self, name, default=None):
return self._transport.get_extra_info(name, default)
async def drain(self):
@coroutine
def drain(self):
"""Flush the write buffer.
The intended use is to write
w.write(data)
await w.drain()
yield from w.drain()
"""
if self._reader is not None:
exc = self._reader.exception()
if exc is not None:
raise exc
if self._transport.is_closing():
# Wait for protocol.connection_lost() call
# Raise connection closing error if any,
# ConnectionResetError otherwise
# Yield to the event loop so connection_lost() may be
# called. Without this, _drain_helper() would return
# immediately, and code that calls
# write(...); await drain()
# in a loop would never call connection_lost(), so it
# would not see an error when the socket is closed.
await sleep(0)
await self._protocol._drain_helper()
if self._transport is not None:
if self._transport.is_closing():
# Yield to the event loop so connection_lost() may be
# called. Without this, _drain_helper() would return
# immediately, and code that calls
# write(...); yield from drain()
# in a loop would never call connection_lost(), so it
# would not see an error when the socket is closed.
yield
yield from self._protocol._drain_helper()
async def start_tls(self, sslcontext, *,
server_hostname=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None):
"""Upgrade an existing stream-based connection to TLS."""
server_side = self._protocol._client_connected_cb is not None
protocol = self._protocol
await self.drain()
new_transport = await self._loop.start_tls( # type: ignore
self._transport, protocol, sslcontext,
server_side=server_side, server_hostname=server_hostname,
ssl_handshake_timeout=ssl_handshake_timeout,
ssl_shutdown_timeout=ssl_shutdown_timeout)
self._transport = new_transport
protocol._replace_writer(self)
def __del__(self):
if not self._transport.is_closing():
if self._loop.is_closed():
warnings.warn("loop is closed", ResourceWarning)
else:
self.close()
warnings.warn(f"unclosed {self!r}", ResourceWarning)
class StreamReader:
_source_traceback = None
def __init__(self, limit=_DEFAULT_LIMIT, loop=None):
# The line length limit is a security feature;
# it also doubles as half the buffer limit.
@@ -437,27 +353,24 @@ class StreamReader:
self._exception = None
self._transport = None
self._paused = False
if self._loop.get_debug():
self._source_traceback = format_helpers.extract_stack(
sys._getframe(1))
def __repr__(self):
info = ['StreamReader']
if self._buffer:
info.append(f'{len(self._buffer)} bytes')
info.append('%d bytes' % len(self._buffer))
if self._eof:
info.append('eof')
if self._limit != _DEFAULT_LIMIT:
info.append(f'limit={self._limit}')
info.append('l=%d' % self._limit)
if self._waiter:
info.append(f'waiter={self._waiter!r}')
info.append('w=%r' % self._waiter)
if self._exception:
info.append(f'exception={self._exception!r}')
info.append('e=%r' % self._exception)
if self._transport:
info.append(f'transport={self._transport!r}')
info.append('t=%r' % self._transport)
if self._paused:
info.append('paused')
return '<{}>'.format(' '.join(info))
return '<%s>' % ' '.join(info)
def exception(self):
return self._exception
@@ -518,7 +431,8 @@ class StreamReader:
else:
self._paused = True
async def _wait_for_data(self, func_name):
@coroutine
def _wait_for_data(self, func_name):
"""Wait until feed_data() or feed_eof() is called.
If stream was paused, automatically resume it.
@@ -528,9 +442,8 @@ class StreamReader:
# would have an unexpected behaviour. It would not possible to know
# which coroutine would get the next data.
if self._waiter is not None:
raise RuntimeError(
f'{func_name}() called while another coroutine is '
f'already waiting for incoming data')
raise RuntimeError('%s() called while another coroutine is '
'already waiting for incoming data' % func_name)
assert not self._eof, '_wait_for_data after EOF'
@@ -542,11 +455,12 @@ class StreamReader:
self._waiter = self._loop.create_future()
try:
await self._waiter
yield from self._waiter
finally:
self._waiter = None
async def readline(self):
@coroutine
def readline(self):
"""Read chunk of data from the stream until newline (b'\n') is found.
On success, return chunk that ends with newline. If only partial
@@ -565,10 +479,10 @@ class StreamReader:
sep = b'\n'
seplen = len(sep)
try:
line = await self.readuntil(sep)
except exceptions.IncompleteReadError as e:
line = yield from self.readuntil(sep)
except IncompleteReadError as e:
return e.partial
except exceptions.LimitOverrunError as e:
except LimitOverrunError as e:
if self._buffer.startswith(sep, e.consumed):
del self._buffer[:e.consumed + seplen]
else:
@@ -577,7 +491,8 @@ class StreamReader:
raise ValueError(e.args[0])
return line
async def readuntil(self, separator=b'\n'):
@coroutine
def readuntil(self, separator=b'\n'):
"""Read data from the stream until ``separator`` is found.
On success, the data and separator will be removed from the
@@ -643,7 +558,7 @@ class StreamReader:
# see upper comment for explanation.
offset = buflen + 1 - seplen
if offset > self._limit:
raise exceptions.LimitOverrunError(
raise LimitOverrunError(
'Separator is not found, and chunk exceed the limit',
offset)
@@ -654,13 +569,13 @@ class StreamReader:
if self._eof:
chunk = bytes(self._buffer)
self._buffer.clear()
raise exceptions.IncompleteReadError(chunk, None)
raise IncompleteReadError(chunk, None)
# _wait_for_data() will resume reading if stream was paused.
await self._wait_for_data('readuntil')
yield from self._wait_for_data('readuntil')
if isep > self._limit:
raise exceptions.LimitOverrunError(
raise LimitOverrunError(
'Separator is found, but chunk is longer than limit', isep)
chunk = self._buffer[:isep + seplen]
@@ -668,20 +583,20 @@ class StreamReader:
self._maybe_resume_transport()
return bytes(chunk)
async def read(self, n=-1):
@coroutine
def read(self, n=-1):
"""Read up to `n` bytes from the stream.
If `n` is not provided or set to -1,
read until EOF, then return all read bytes.
If EOF was received and the internal buffer is empty,
return an empty bytes object.
If n is not provided, or set to -1, read until EOF and return all read
bytes. If the EOF was received and the internal buffer is empty, return
an empty bytes object.
If `n` is 0, return an empty bytes object immediately.
If n is zero, return empty bytes object immediately.
If `n` is positive, return at most `n` available bytes
as soon as at least 1 byte is available in the internal buffer.
If EOF is received before any byte is read, return an empty
bytes object.
If n is positive, this function try to read `n` bytes, and may return
less or equal bytes than requested, but at least one byte. If EOF was
received before any byte is read, this function returns empty byte
object.
Returned value is not limited with limit, configured at stream
creation.
@@ -703,23 +618,24 @@ class StreamReader:
# bytes. So just call self.read(self._limit) until EOF.
blocks = []
while True:
block = await self.read(self._limit)
block = yield from self.read(self._limit)
if not block:
break
blocks.append(block)
return b''.join(blocks)
if not self._buffer and not self._eof:
await self._wait_for_data('read')
yield from self._wait_for_data('read')
# This will work right even if buffer is less than n bytes
data = bytes(memoryview(self._buffer)[:n])
data = bytes(self._buffer[:n])
del self._buffer[:n]
self._maybe_resume_transport()
return data
async def readexactly(self, n):
@coroutine
def readexactly(self, n):
"""Read exactly `n` bytes.
Raise an IncompleteReadError if EOF is reached before `n` bytes can be
@@ -747,24 +663,33 @@ class StreamReader:
if self._eof:
incomplete = bytes(self._buffer)
self._buffer.clear()
raise exceptions.IncompleteReadError(incomplete, n)
raise IncompleteReadError(incomplete, n)
await self._wait_for_data('readexactly')
yield from self._wait_for_data('readexactly')
if len(self._buffer) == n:
data = bytes(self._buffer)
self._buffer.clear()
else:
data = bytes(memoryview(self._buffer)[:n])
data = bytes(self._buffer[:n])
del self._buffer[:n]
self._maybe_resume_transport()
return data
def __aiter__(self):
return self
if compat.PY35:
@coroutine
def __aiter__(self):
return self
async def __anext__(self):
val = await self.readline()
if val == b'':
raise StopAsyncIteration
return val
@coroutine
def __anext__(self):
val = yield from self.readline()
if val == b'':
raise StopAsyncIteration
return val
if compat.PY352:
# In Python 3.5.2 and greater, __aiter__ should return
# the asynchronous iterator directly.
def __aiter__(self):
return self

126
Lib/asyncio/subprocess.py
View File

@@ -1,4 +1,4 @@
__all__ = 'create_subprocess_exec', 'create_subprocess_shell'
__all__ = ['create_subprocess_exec', 'create_subprocess_shell']
import subprocess
@@ -6,6 +6,7 @@ from . import events
from . import protocols
from . import streams
from . import tasks
from .coroutines import coroutine
from .log import logger
@@ -23,19 +24,16 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
self._limit = limit
self.stdin = self.stdout = self.stderr = None
self._transport = None
self._process_exited = False
self._pipe_fds = []
self._stdin_closed = self._loop.create_future()
def __repr__(self):
info = [self.__class__.__name__]
if self.stdin is not None:
info.append(f'stdin={self.stdin!r}')
info.append('stdin=%r' % self.stdin)
if self.stdout is not None:
info.append(f'stdout={self.stdout!r}')
info.append('stdout=%r' % self.stdout)
if self.stderr is not None:
info.append(f'stderr={self.stderr!r}')
return '<{}>'.format(' '.join(info))
info.append('stderr=%r' % self.stderr)
return '<%s>' % ' '.join(info)
def connection_made(self, transport):
self._transport = transport
@@ -45,14 +43,12 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
self.stdout = streams.StreamReader(limit=self._limit,
loop=self._loop)
self.stdout.set_transport(stdout_transport)
self._pipe_fds.append(1)
stderr_transport = transport.get_pipe_transport(2)
if stderr_transport is not None:
self.stderr = streams.StreamReader(limit=self._limit,
loop=self._loop)
self.stderr.set_transport(stderr_transport)
self._pipe_fds.append(2)
stdin_transport = transport.get_pipe_transport(0)
if stdin_transport is not None:
@@ -77,13 +73,6 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
if pipe is not None:
pipe.close()
self.connection_lost(exc)
if exc is None:
self._stdin_closed.set_result(None)
else:
self._stdin_closed.set_exception(exc)
# Since calling `wait_closed()` is not mandatory,
# we shouldn't log the traceback if this is not awaited.
self._stdin_closed._log_traceback = False
return
if fd == 1:
reader = self.stdout
@@ -91,28 +80,15 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
reader = self.stderr
else:
reader = None
if reader is not None:
if reader != None:
if exc is None:
reader.feed_eof()
else:
reader.set_exception(exc)
if fd in self._pipe_fds:
self._pipe_fds.remove(fd)
self._maybe_close_transport()
def process_exited(self):
self._process_exited = True
self._maybe_close_transport()
def _maybe_close_transport(self):
if len(self._pipe_fds) == 0 and self._process_exited:
self._transport.close()
self._transport = None
def _get_close_waiter(self, stream):
if stream is self.stdin:
return self._stdin_closed
self._transport.close()
self._transport = None
class Process:
@@ -126,15 +102,18 @@ class Process:
self.pid = transport.get_pid()
def __repr__(self):
return f'<{self.__class__.__name__} {self.pid}>'
return '<%s %s>' % (self.__class__.__name__, self.pid)
@property
def returncode(self):
return self._transport.get_returncode()
async def wait(self):
"""Wait until the process exit and return the process return code."""
return await self._transport._wait()
@coroutine
def wait(self):
"""Wait until the process exit and return the process return code.
This method is a coroutine."""
return (yield from self._transport._wait())
def send_signal(self, signal):
self._transport.send_signal(signal)
@@ -145,19 +124,17 @@ class Process:
def kill(self):
self._transport.kill()
async def _feed_stdin(self, input):
@coroutine
def _feed_stdin(self, input):
debug = self._loop.get_debug()
self.stdin.write(input)
if debug:
logger.debug('%r communicate: feed stdin (%s bytes)',
self, len(input))
try:
if input is not None:
self.stdin.write(input)
if debug:
logger.debug(
'%r communicate: feed stdin (%s bytes)', self, len(input))
await self.stdin.drain()
yield from self.stdin.drain()
except (BrokenPipeError, ConnectionResetError) as exc:
# communicate() ignores BrokenPipeError and ConnectionResetError.
# write() and drain() can raise these exceptions.
# communicate() ignores BrokenPipeError and ConnectionResetError
if debug:
logger.debug('%r communicate: stdin got %r', self, exc)
@@ -165,10 +142,12 @@ class Process:
logger.debug('%r communicate: close stdin', self)
self.stdin.close()
async def _noop(self):
@coroutine
def _noop(self):
return None
async def _read_stream(self, fd):
@coroutine
def _read_stream(self, fd):
transport = self._transport.get_pipe_transport(fd)
if fd == 2:
stream = self.stderr
@@ -178,15 +157,16 @@ class Process:
if self._loop.get_debug():
name = 'stdout' if fd == 1 else 'stderr'
logger.debug('%r communicate: read %s', self, name)
output = await stream.read()
output = yield from stream.read()
if self._loop.get_debug():
name = 'stdout' if fd == 1 else 'stderr'
logger.debug('%r communicate: close %s', self, name)
transport.close()
return output
async def communicate(self, input=None):
if self.stdin is not None:
@coroutine
def communicate(self, input=None):
if input is not None:
stdin = self._feed_stdin(input)
else:
stdin = self._noop()
@@ -198,32 +178,36 @@ class Process:
stderr = self._read_stream(2)
else:
stderr = self._noop()
stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr)
await self.wait()
stdin, stdout, stderr = yield from tasks.gather(stdin, stdout, stderr,
loop=self._loop)
yield from self.wait()
return (stdout, stderr)
async def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None,
limit=streams._DEFAULT_LIMIT, **kwds):
loop = events.get_running_loop()
@coroutine
def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None,
loop=None, limit=streams._DEFAULT_LIMIT, **kwds):
if loop is None:
loop = events.get_event_loop()
protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
loop=loop)
transport, protocol = await loop.subprocess_shell(
protocol_factory,
cmd, stdin=stdin, stdout=stdout,
stderr=stderr, **kwds)
transport, protocol = yield from loop.subprocess_shell(
protocol_factory,
cmd, stdin=stdin, stdout=stdout,
stderr=stderr, **kwds)
return Process(transport, protocol, loop)
async def create_subprocess_exec(program, *args, stdin=None, stdout=None,
stderr=None, limit=streams._DEFAULT_LIMIT,
**kwds):
loop = events.get_running_loop()
@coroutine
def create_subprocess_exec(program, *args, stdin=None, stdout=None,
stderr=None, loop=None,
limit=streams._DEFAULT_LIMIT, **kwds):
if loop is None:
loop = events.get_event_loop()
protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
loop=loop)
transport, protocol = await loop.subprocess_exec(
protocol_factory,
program, *args,
stdin=stdin, stdout=stdout,
stderr=stderr, **kwds)
transport, protocol = yield from loop.subprocess_exec(
protocol_factory,
program, *args,
stdin=stdin, stdout=stdout,
stderr=stderr, **kwds)
return Process(transport, protocol, loop)

240
Lib/asyncio/taskgroups.py
View File

@@ -1,240 +0,0 @@
# Adapted with permission from the EdgeDB project;
# license: PSFL.
__all__ = ("TaskGroup",)
from . import events
from . import exceptions
from . import tasks
class TaskGroup:
"""Asynchronous context manager for managing groups of tasks.
Example use:
async with asyncio.TaskGroup() as group:
task1 = group.create_task(some_coroutine(...))
task2 = group.create_task(other_coroutine(...))
print("Both tasks have completed now.")
All tasks are awaited when the context manager exits.
Any exceptions other than `asyncio.CancelledError` raised within
a task will cancel all remaining tasks and wait for them to exit.
The exceptions are then combined and raised as an `ExceptionGroup`.
"""
def __init__(self):
self._entered = False
self._exiting = False
self._aborting = False
self._loop = None
self._parent_task = None
self._parent_cancel_requested = False
self._tasks = set()
self._errors = []
self._base_error = None
self._on_completed_fut = None
def __repr__(self):
info = ['']
if self._tasks:
info.append(f'tasks={len(self._tasks)}')
if self._errors:
info.append(f'errors={len(self._errors)}')
if self._aborting:
info.append('cancelling')
elif self._entered:
info.append('entered')
info_str = ' '.join(info)
return f'<TaskGroup{info_str}>'
async def __aenter__(self):
if self._entered:
raise RuntimeError(
f"TaskGroup {self!r} has already been entered")
if self._loop is None:
self._loop = events.get_running_loop()
self._parent_task = tasks.current_task(self._loop)
if self._parent_task is None:
raise RuntimeError(
f'TaskGroup {self!r} cannot determine the parent task')
self._entered = True
return self
async def __aexit__(self, et, exc, tb):
self._exiting = True
if (exc is not None and
self._is_base_error(exc) and
self._base_error is None):
self._base_error = exc
propagate_cancellation_error = \
exc if et is exceptions.CancelledError else None
if self._parent_cancel_requested:
# If this flag is set we *must* call uncancel().
if self._parent_task.uncancel() == 0:
# If there are no pending cancellations left,
# don't propagate CancelledError.
propagate_cancellation_error = None
if et is not None:
if not self._aborting:
# Our parent task is being cancelled:
#
# async with TaskGroup() as g:
# g.create_task(...)
# await ... # <- CancelledError
#
# or there's an exception in "async with":
#
# async with TaskGroup() as g:
# g.create_task(...)
# 1 / 0
#
self._abort()
# We use while-loop here because "self._on_completed_fut"
# can be cancelled multiple times if our parent task
# is being cancelled repeatedly (or even once, when
# our own cancellation is already in progress)
while self._tasks:
if self._on_completed_fut is None:
self._on_completed_fut = self._loop.create_future()
try:
await self._on_completed_fut
except exceptions.CancelledError as ex:
if not self._aborting:
# Our parent task is being cancelled:
#
# async def wrapper():
# async with TaskGroup() as g:
# g.create_task(foo)
#
# "wrapper" is being cancelled while "foo" is
# still running.
propagate_cancellation_error = ex
self._abort()
self._on_completed_fut = None
assert not self._tasks
if self._base_error is not None:
raise self._base_error
# Propagate CancelledError if there is one, except if there
# are other errors -- those have priority.
if propagate_cancellation_error and not self._errors:
raise propagate_cancellation_error
if et is not None and et is not exceptions.CancelledError:
self._errors.append(exc)
if self._errors:
# Exceptions are heavy objects that can have object
# cycles (bad for GC); let's not keep a reference to
# a bunch of them.
try:
me = BaseExceptionGroup('unhandled errors in a TaskGroup', self._errors)
raise me from None
finally:
self._errors = None
def create_task(self, coro, *, name=None, context=None):
"""Create a new task in this group and return it.
Similar to `asyncio.create_task`.
"""
if not self._entered:
raise RuntimeError(f"TaskGroup {self!r} has not been entered")
if self._exiting and not self._tasks:
raise RuntimeError(f"TaskGroup {self!r} is finished")
if self._aborting:
raise RuntimeError(f"TaskGroup {self!r} is shutting down")
if context is None:
task = self._loop.create_task(coro)
else:
task = self._loop.create_task(coro, context=context)
tasks._set_task_name(task, name)
# optimization: Immediately call the done callback if the task is
# already done (e.g. if the coro was able to complete eagerly),
# and skip scheduling a done callback
if task.done():
self._on_task_done(task)
else:
self._tasks.add(task)
task.add_done_callback(self._on_task_done)
return task
# Since Python 3.8 Tasks propagate all exceptions correctly,
# except for KeyboardInterrupt and SystemExit which are
# still considered special.
def _is_base_error(self, exc: BaseException) -> bool:
assert isinstance(exc, BaseException)
return isinstance(exc, (SystemExit, KeyboardInterrupt))
def _abort(self):
self._aborting = True
for t in self._tasks:
if not t.done():
t.cancel()
def _on_task_done(self, task):
self._tasks.discard(task)
if self._on_completed_fut is not None and not self._tasks:
if not self._on_completed_fut.done():
self._on_completed_fut.set_result(True)
if task.cancelled():
return
exc = task.exception()
if exc is None:
return
self._errors.append(exc)
if self._is_base_error(exc) and self._base_error is None:
self._base_error = exc
if self._parent_task.done():
# Not sure if this case is possible, but we want to handle
# it anyways.
self._loop.call_exception_handler({
'message': f'Task {task!r} has errored out but its parent '
f'task {self._parent_task} is already completed',
'exception': exc,
'task': task,
})
return
if not self._aborting and not self._parent_cancel_requested:
# If parent task *is not* being cancelled, it means that we want
# to manually cancel it to abort whatever is being run right now
# in the TaskGroup. But we want to mark parent task as
# "not cancelled" later in __aexit__. Example situation that
# we need to handle:
#
# async def foo():
# try:
# async with TaskGroup() as g:
# g.create_task(crash_soon())
# await something # <- this needs to be canceled
# # by the TaskGroup, e.g.
# # foo() needs to be cancelled
# except Exception:
# # Ignore any exceptions raised in the TaskGroup
# pass
# await something_else # this line has to be called
# # after TaskGroup is finished.
self._abort()
self._parent_cancel_requested = True
self._parent_task.cancel()

850
Lib/asyncio/tasks.py
View File

File diff suppressed because it is too large Load Diff

503
Lib/asyncio/test_utils.py Normal file
View File

@@ -0,0 +1,503 @@
"""Utilities shared by tests."""
import collections
import contextlib
import io
import logging
import os
import re
import socket
import socketserver
import sys
import tempfile
import threading
import time
import unittest
import weakref
from unittest import mock
from http.server import HTTPServer
from wsgiref.simple_server import WSGIRequestHandler, WSGIServer
try:
import ssl
except ImportError: # pragma: no cover
ssl = None
from . import base_events
from . import compat
from . import events
from . import futures
from . import selectors
from . import tasks
from .coroutines import coroutine
from .log import logger
if sys.platform == 'win32': # pragma: no cover
from .windows_utils import socketpair
else:
from socket import socketpair # pragma: no cover
def dummy_ssl_context():
if ssl is None:
return None
else:
return ssl.SSLContext(ssl.PROTOCOL_SSLv23)
def run_briefly(loop):
@coroutine
def once():
pass
gen = once()
t = loop.create_task(gen)
# Don't log a warning if the task is not done after run_until_complete().
# It occurs if the loop is stopped or if a task raises a BaseException.
t._log_destroy_pending = False
try:
loop.run_until_complete(t)
finally:
gen.close()
def run_until(loop, pred, timeout=30):
deadline = time.time() + timeout
while not pred():
if timeout is not None:
timeout = deadline - time.time()
if timeout <= 0:
raise futures.TimeoutError()
loop.run_until_complete(tasks.sleep(0.001, loop=loop))
def run_once(loop):
"""Legacy API to run once through the event loop.
This is the recommended pattern for test code. It will poll the
selector once and run all callbacks scheduled in response to I/O
events.
"""
loop.call_soon(loop.stop)
loop.run_forever()
class SilentWSGIRequestHandler(WSGIRequestHandler):
def get_stderr(self):
return io.StringIO()
def log_message(self, format, *args):
pass
class SilentWSGIServer(WSGIServer):
request_timeout = 2
def get_request(self):
request, client_addr = super().get_request()
request.settimeout(self.request_timeout)
return request, client_addr
def handle_error(self, request, client_address):
pass
class SSLWSGIServerMixin:
def finish_request(self, request, client_address):
# The relative location of our test directory (which
# contains the ssl key and certificate files) differs
# between the stdlib and stand-alone asyncio.
# Prefer our own if we can find it.
here = os.path.join(os.path.dirname(__file__), '..', 'tests')
if not os.path.isdir(here):
here = os.path.join(os.path.dirname(os.__file__),
'test', 'test_asyncio')
keyfile = os.path.join(here, 'ssl_key.pem')
certfile = os.path.join(here, 'ssl_cert.pem')
context = ssl.SSLContext()
context.load_cert_chain(certfile, keyfile)
ssock = context.wrap_socket(request, server_side=True)
try:
self.RequestHandlerClass(ssock, client_address, self)
ssock.close()
except OSError:
# maybe socket has been closed by peer
pass
class SSLWSGIServer(SSLWSGIServerMixin, SilentWSGIServer):
pass
def _run_test_server(*, address, use_ssl=False, server_cls, server_ssl_cls):
def app(environ, start_response):
status = '200 OK'
headers = [('Content-type', 'text/plain')]
start_response(status, headers)
return [b'Test message']
# Run the test WSGI server in a separate thread in order not to
# interfere with event handling in the main thread
server_class = server_ssl_cls if use_ssl else server_cls
httpd = server_class(address, SilentWSGIRequestHandler)
httpd.set_app(app)
httpd.address = httpd.server_address
server_thread = threading.Thread(
target=lambda: httpd.serve_forever(poll_interval=0.05))
server_thread.start()
try:
yield httpd
finally:
httpd.shutdown()
httpd.server_close()
server_thread.join()
if hasattr(socket, 'AF_UNIX'):
class UnixHTTPServer(socketserver.UnixStreamServer, HTTPServer):
def server_bind(self):
socketserver.UnixStreamServer.server_bind(self)
self.server_name = '127.0.0.1'
self.server_port = 80
class UnixWSGIServer(UnixHTTPServer, WSGIServer):
request_timeout = 2
def server_bind(self):
UnixHTTPServer.server_bind(self)
self.setup_environ()
def get_request(self):
request, client_addr = super().get_request()
request.settimeout(self.request_timeout)
# Code in the stdlib expects that get_request
# will return a socket and a tuple (host, port).
# However, this isn't true for UNIX sockets,
# as the second return value will be a path;
# hence we return some fake data sufficient
# to get the tests going
return request, ('127.0.0.1', '')
class SilentUnixWSGIServer(UnixWSGIServer):
def handle_error(self, request, client_address):
pass
class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer):
pass
def gen_unix_socket_path():
with tempfile.NamedTemporaryFile() as file:
return file.name
@contextlib.contextmanager
def unix_socket_path():
path = gen_unix_socket_path()
try:
yield path
finally:
try:
os.unlink(path)
except OSError:
pass
@contextlib.contextmanager
def run_test_unix_server(*, use_ssl=False):
with unix_socket_path() as path:
yield from _run_test_server(address=path, use_ssl=use_ssl,
server_cls=SilentUnixWSGIServer,
server_ssl_cls=UnixSSLWSGIServer)
@contextlib.contextmanager
def run_test_server(*, host='127.0.0.1', port=0, use_ssl=False):
yield from _run_test_server(address=(host, port), use_ssl=use_ssl,
server_cls=SilentWSGIServer,
server_ssl_cls=SSLWSGIServer)
def make_test_protocol(base):
dct = {}
for name in dir(base):
if name.startswith('__') and name.endswith('__'):
# skip magic names
continue
dct[name] = MockCallback(return_value=None)
return type('TestProtocol', (base,) + base.__bases__, dct)()
class TestSelector(selectors.BaseSelector):
def __init__(self):
self.keys = {}
def register(self, fileobj, events, data=None):
key = selectors.SelectorKey(fileobj, 0, events, data)
self.keys[fileobj] = key
return key
def unregister(self, fileobj):
return self.keys.pop(fileobj)
def select(self, timeout):
return []
def get_map(self):
return self.keys
class TestLoop(base_events.BaseEventLoop):
"""Loop for unittests.
It manages self time directly.
If something scheduled to be executed later then
on next loop iteration after all ready handlers done
generator passed to __init__ is calling.
Generator should be like this:
def gen():
...
when = yield ...
... = yield time_advance
Value returned by yield is absolute time of next scheduled handler.
Value passed to yield is time advance to move loop's time forward.
"""
def __init__(self, gen=None):
super().__init__()
if gen is None:
def gen():
yield
self._check_on_close = False
else:
self._check_on_close = True
self._gen = gen()
next(self._gen)
self._time = 0
self._clock_resolution = 1e-9
self._timers = []
self._selector = TestSelector()
self.readers = {}
self.writers = {}
self.reset_counters()
self._transports = weakref.WeakValueDictionary()
def time(self):
return self._time
def advance_time(self, advance):
"""Move test time forward."""
if advance:
self._time += advance
def close(self):
super().close()
if self._check_on_close:
try:
self._gen.send(0)
except StopIteration:
pass
else: # pragma: no cover
raise AssertionError("Time generator is not finished")
def _add_reader(self, fd, callback, *args):
self.readers[fd] = events.Handle(callback, args, self)
def _remove_reader(self, fd):
self.remove_reader_count[fd] += 1
if fd in self.readers:
del self.readers[fd]
return True
else:
return False
def assert_reader(self, fd, callback, *args):
assert fd in self.readers, 'fd {} is not registered'.format(fd)
handle = self.readers[fd]
assert handle._callback == callback, '{!r} != {!r}'.format(
handle._callback, callback)
assert handle._args == args, '{!r} != {!r}'.format(
handle._args, args)
def _add_writer(self, fd, callback, *args):
self.writers[fd] = events.Handle(callback, args, self)
def _remove_writer(self, fd):
self.remove_writer_count[fd] += 1
if fd in self.writers:
del self.writers[fd]
return True
else:
return False
def assert_writer(self, fd, callback, *args):
assert fd in self.writers, 'fd {} is not registered'.format(fd)
handle = self.writers[fd]
assert handle._callback == callback, '{!r} != {!r}'.format(
handle._callback, callback)
assert handle._args == args, '{!r} != {!r}'.format(
handle._args, args)
def _ensure_fd_no_transport(self, fd):
try:
transport = self._transports[fd]
except KeyError:
pass
else:
raise RuntimeError(
'File descriptor {!r} is used by transport {!r}'.format(
fd, transport))
def add_reader(self, fd, callback, *args):
"""Add a reader callback."""
self._ensure_fd_no_transport(fd)
return self._add_reader(fd, callback, *args)
def remove_reader(self, fd):
"""Remove a reader callback."""
self._ensure_fd_no_transport(fd)
return self._remove_reader(fd)
def add_writer(self, fd, callback, *args):
"""Add a writer callback.."""
self._ensure_fd_no_transport(fd)
return self._add_writer(fd, callback, *args)
def remove_writer(self, fd):
"""Remove a writer callback."""
self._ensure_fd_no_transport(fd)
return self._remove_writer(fd)
def reset_counters(self):
self.remove_reader_count = collections.defaultdict(int)
self.remove_writer_count = collections.defaultdict(int)
def _run_once(self):
super()._run_once()
for when in self._timers:
advance = self._gen.send(when)
self.advance_time(advance)
self._timers = []
def call_at(self, when, callback, *args):
self._timers.append(when)
return super().call_at(when, callback, *args)
def _process_events(self, event_list):
return
def _write_to_self(self):
pass
def MockCallback(**kwargs):
return mock.Mock(spec=['__call__'], **kwargs)
class MockPattern(str):
"""A regex based str with a fuzzy __eq__.
Use this helper with 'mock.assert_called_with', or anywhere
where a regex comparison between strings is needed.
For instance:
mock_call.assert_called_with(MockPattern('spam.*ham'))
"""
def __eq__(self, other):
return bool(re.search(str(self), other, re.S))
def get_function_source(func):
source = events._get_function_source(func)
if source is None:
raise ValueError("unable to get the source of %r" % (func,))
return source
class TestCase(unittest.TestCase):
def set_event_loop(self, loop, *, cleanup=True):
assert loop is not None
# ensure that the event loop is passed explicitly in asyncio
events.set_event_loop(None)
if cleanup:
self.addCleanup(loop.close)
def new_test_loop(self, gen=None):
loop = TestLoop(gen)
self.set_event_loop(loop)
return loop
def setUp(self):
self._get_running_loop = events._get_running_loop
events._get_running_loop = lambda: None
def tearDown(self):
events._get_running_loop = self._get_running_loop
events.set_event_loop(None)
# Detect CPython bug #23353: ensure that yield/yield-from is not used
# in an except block of a generator
self.assertEqual(sys.exc_info(), (None, None, None))
if not compat.PY34:
# Python 3.3 compatibility
def subTest(self, *args, **kwargs):
class EmptyCM:
def __enter__(self):
pass
def __exit__(self, *exc):
pass
return EmptyCM()
@contextlib.contextmanager
def disable_logger():
"""Context manager to disable asyncio logger.
For example, it can be used to ignore warnings in debug mode.
"""
old_level = logger.level
try:
logger.setLevel(logging.CRITICAL+1)
yield
finally:
logger.setLevel(old_level)
def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM,
family=socket.AF_INET):
"""Create a mock of a non-blocking socket."""
sock = mock.MagicMock(socket.socket)
sock.proto = proto
sock.type = type
sock.family = family
sock.gettimeout.return_value = 0.0
return sock
def force_legacy_ssl_support():
return mock.patch('asyncio.sslproto._is_sslproto_available',
return_value=False)

25
Lib/asyncio/threads.py
View File

@@ -1,25 +0,0 @@
"""High-level support for working with threads in asyncio"""
import functools
import contextvars
from . import events
__all__ = "to_thread",
async def to_thread(func, /, *args, **kwargs):
"""Asynchronously run function *func* in a separate thread.
Any *args and **kwargs supplied for this function are directly passed
to *func*. Also, the current :class:`contextvars.Context` is propagated,
allowing context variables from the main thread to be accessed in the
separate thread.
Return a coroutine that can be awaited to get the eventual result of *func*.
"""
loop = events.get_running_loop()
ctx = contextvars.copy_context()
func_call = functools.partial(ctx.run, func, *args, **kwargs)
return await loop.run_in_executor(None, func_call)

168
Lib/asyncio/timeouts.py
View File

@@ -1,168 +0,0 @@
import enum
from types import TracebackType
from typing import final, Optional, Type
from . import events
from . import exceptions
from . import tasks
__all__ = (
"Timeout",
"timeout",
"timeout_at",
)
class _State(enum.Enum):
CREATED = "created"
ENTERED = "active"
EXPIRING = "expiring"
EXPIRED = "expired"
EXITED = "finished"
@final
class Timeout:
"""Asynchronous context manager for cancelling overdue coroutines.
Use `timeout()` or `timeout_at()` rather than instantiating this class directly.
"""
def __init__(self, when: Optional[float]) -> None:
"""Schedule a timeout that will trigger at a given loop time.
- If `when` is `None`, the timeout will never trigger.
- If `when < loop.time()`, the timeout will trigger on the next
iteration of the event loop.
"""
self._state = _State.CREATED
self._timeout_handler: Optional[events.TimerHandle] = None
self._task: Optional[tasks.Task] = None
self._when = when
def when(self) -> Optional[float]:
"""Return the current deadline."""
return self._when
def reschedule(self, when: Optional[float]) -> None:
"""Reschedule the timeout."""
if self._state is not _State.ENTERED:
if self._state is _State.CREATED:
raise RuntimeError("Timeout has not been entered")
raise RuntimeError(
f"Cannot change state of {self._state.value} Timeout",
)
self._when = when
if self._timeout_handler is not None:
self._timeout_handler.cancel()
if when is None:
self._timeout_handler = None
else:
loop = events.get_running_loop()
if when <= loop.time():
self._timeout_handler = loop.call_soon(self._on_timeout)
else:
self._timeout_handler = loop.call_at(when, self._on_timeout)
def expired(self) -> bool:
"""Is timeout expired during execution?"""
return self._state in (_State.EXPIRING, _State.EXPIRED)
def __repr__(self) -> str:
info = ['']
if self._state is _State.ENTERED:
when = round(self._when, 3) if self._when is not None else None
info.append(f"when={when}")
info_str = ' '.join(info)
return f"<Timeout [{self._state.value}]{info_str}>"
async def __aenter__(self) -> "Timeout":
if self._state is not _State.CREATED:
raise RuntimeError("Timeout has already been entered")
task = tasks.current_task()
if task is None:
raise RuntimeError("Timeout should be used inside a task")
self._state = _State.ENTERED
self._task = task
self._cancelling = self._task.cancelling()
self.reschedule(self._when)
return self
async def __aexit__(
self,
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType],
) -> Optional[bool]:
assert self._state in (_State.ENTERED, _State.EXPIRING)
if self._timeout_handler is not None:
self._timeout_handler.cancel()
self._timeout_handler = None
if self._state is _State.EXPIRING:
self._state = _State.EXPIRED
if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError:
# Since there are no new cancel requests, we're
# handling this.
raise TimeoutError from exc_val
elif self._state is _State.ENTERED:
self._state = _State.EXITED
return None
def _on_timeout(self) -> None:
assert self._state is _State.ENTERED
self._task.cancel()
self._state = _State.EXPIRING
# drop the reference early
self._timeout_handler = None
def timeout(delay: Optional[float]) -> Timeout:
"""Timeout async context manager.
Useful in cases when you want to apply timeout logic around block
of code or in cases when asyncio.wait_for is not suitable. For example:
>>> async with asyncio.timeout(10): # 10 seconds timeout
... await long_running_task()
delay - value in seconds or None to disable timeout logic
long_running_task() is interrupted by raising asyncio.CancelledError,
the top-most affected timeout() context manager converts CancelledError
into TimeoutError.
"""
loop = events.get_running_loop()
return Timeout(loop.time() + delay if delay is not None else None)
def timeout_at(when: Optional[float]) -> Timeout:
"""Schedule the timeout at absolute time.
Like timeout() but argument gives absolute time in the same clock system
as loop.time().
Please note: it is not POSIX time but a time with
undefined starting base, e.g. the time of the system power on.
>>> async with asyncio.timeout_at(loop.time() + 10):
... await long_running_task()
when - a deadline when timeout occurs or None to disable timeout logic
long_running_task() is interrupted by raising asyncio.CancelledError,
the top-most affected timeout() context manager converts CancelledError
into TimeoutError.
"""
return Timeout(when)

59
Lib/asyncio/transports.py
View File

@@ -1,16 +1,15 @@
"""Abstract Transport class."""
__all__ = (
'BaseTransport', 'ReadTransport', 'WriteTransport',
'Transport', 'DatagramTransport', 'SubprocessTransport',
)
from asyncio import compat
__all__ = ['BaseTransport', 'ReadTransport', 'WriteTransport',
'Transport', 'DatagramTransport', 'SubprocessTransport',
]
class BaseTransport:
"""Base class for transports."""
__slots__ = ('_extra',)
def __init__(self, extra=None):
if extra is None:
extra = {}
@@ -29,8 +28,8 @@ class BaseTransport:
Buffered data will be flushed asynchronously. No more data
will be received. After all buffered data is flushed, the
protocol's connection_lost() method will (eventually) be
called with None as its argument.
protocol's connection_lost() method will (eventually) called
with None as its argument.
"""
raise NotImplementedError
@@ -46,12 +45,6 @@ class BaseTransport:
class ReadTransport(BaseTransport):
"""Interface for read-only transports."""
__slots__ = ()
def is_reading(self):
"""Return True if the transport is receiving."""
raise NotImplementedError
def pause_reading(self):
"""Pause the receiving end.
@@ -72,8 +65,6 @@ class ReadTransport(BaseTransport):
class WriteTransport(BaseTransport):
"""Interface for write-only transports."""
__slots__ = ()
def set_write_buffer_limits(self, high=None, low=None):
"""Set the high- and low-water limits for write flow control.
@@ -99,12 +90,6 @@ class WriteTransport(BaseTransport):
"""Return the current size of the write buffer."""
raise NotImplementedError
def get_write_buffer_limits(self):
"""Get the high and low watermarks for write flow control.
Return a tuple (low, high) where low and high are
positive number of bytes."""
raise NotImplementedError
def write(self, data):
"""Write some data bytes to the transport.
@@ -119,7 +104,7 @@ class WriteTransport(BaseTransport):
The default implementation concatenates the arguments and
calls write() on the result.
"""
data = b''.join(list_of_data)
data = compat.flatten_list_bytes(list_of_data)
self.write(data)
def write_eof(self):
@@ -166,14 +151,10 @@ class Transport(ReadTransport, WriteTransport):
except writelines(), which calls write() in a loop.
"""
__slots__ = ()
class DatagramTransport(BaseTransport):
"""Interface for datagram (UDP) transports."""
__slots__ = ()
def sendto(self, data, addr=None):
"""Send data to the transport.
@@ -196,8 +177,6 @@ class DatagramTransport(BaseTransport):
class SubprocessTransport(BaseTransport):
__slots__ = ()
def get_pid(self):
"""Get subprocess id."""
raise NotImplementedError
@@ -265,8 +244,6 @@ class _FlowControlMixin(Transport):
resume_writing() may be called.
"""
__slots__ = ('_loop', '_protocol_paused', '_high_water', '_low_water')
def __init__(self, extra=None, loop=None):
super().__init__(extra)
assert loop is not None
@@ -282,9 +259,7 @@ class _FlowControlMixin(Transport):
self._protocol_paused = True
try:
self._protocol.pause_writing()
except (SystemExit, KeyboardInterrupt):
raise
except BaseException as exc:
except Exception as exc:
self._loop.call_exception_handler({
'message': 'protocol.pause_writing() failed',
'exception': exc,
@@ -294,13 +269,11 @@ class _FlowControlMixin(Transport):
def _maybe_resume_protocol(self):
if (self._protocol_paused and
self.get_write_buffer_size() <= self._low_water):
self.get_write_buffer_size() <= self._low_water):
self._protocol_paused = False
try:
self._protocol.resume_writing()
except (SystemExit, KeyboardInterrupt):
raise
except BaseException as exc:
except Exception as exc:
self._loop.call_exception_handler({
'message': 'protocol.resume_writing() failed',
'exception': exc,
@@ -314,16 +287,14 @@ class _FlowControlMixin(Transport):
def _set_write_buffer_limits(self, high=None, low=None):
if high is None:
if low is None:
high = 64 * 1024
high = 64*1024
else:
high = 4 * low
high = 4*low
if low is None:
low = high // 4
if not high >= low >= 0:
raise ValueError(
f'high ({high!r}) must be >= low ({low!r}) must be >= 0')
raise ValueError('high (%r) must be >= low (%r) must be >= 0' %
(high, low))
self._high_water = high
self._low_water = low

98
Lib/asyncio/trsock.py
View File

@@ -1,98 +0,0 @@
import socket
class TransportSocket:
"""A socket-like wrapper for exposing real transport sockets.
These objects can be safely returned by APIs like
`transport.get_extra_info('socket')`. All potentially disruptive
operations (like "socket.close()") are banned.
"""
__slots__ = ('_sock',)
def __init__(self, sock: socket.socket):
self._sock = sock
@property
def family(self):
return self._sock.family
@property
def type(self):
return self._sock.type
@property
def proto(self):
return self._sock.proto
def __repr__(self):
s = (
f"<asyncio.TransportSocket fd={self.fileno()}, "
f"family={self.family!s}, type={self.type!s}, "
f"proto={self.proto}"
)
if self.fileno() != -1:
try:
laddr = self.getsockname()
if laddr:
s = f"{s}, laddr={laddr}"
except socket.error:
pass
try:
raddr = self.getpeername()
if raddr:
s = f"{s}, raddr={raddr}"
except socket.error:
pass
return f"{s}>"
def __getstate__(self):
raise TypeError("Cannot serialize asyncio.TransportSocket object")
def fileno(self):
return self._sock.fileno()
def dup(self):
return self._sock.dup()
def get_inheritable(self):
return self._sock.get_inheritable()
def shutdown(self, how):
# asyncio doesn't currently provide a high-level transport API
# to shutdown the connection.
self._sock.shutdown(how)
def getsockopt(self, *args, **kwargs):
return self._sock.getsockopt(*args, **kwargs)
def setsockopt(self, *args, **kwargs):
self._sock.setsockopt(*args, **kwargs)
def getpeername(self):
return self._sock.getpeername()
def getsockname(self):
return self._sock.getsockname()
def getsockbyname(self):
return self._sock.getsockbyname()
def settimeout(self, value):
if value == 0:
return
raise ValueError(
'settimeout(): only 0 timeout is allowed on transport sockets')
def gettimeout(self):
return 0
def setblocking(self, flag):
if not flag:
return
raise ValueError(
'setblocking(): transport sockets cannot be blocking')

786
Lib/asyncio/unix_events.py
View File

File diff suppressed because it is too large Load Diff

289
Lib/asyncio/windows_events.py
View File

@@ -1,41 +1,32 @@
"""Selector and proactor event loops for Windows."""
import sys
if sys.platform != 'win32': # pragma: no cover
raise ImportError('win32 only')
import _overlapped
import _winapi
import errno
from functools import partial
import math
import msvcrt
import socket
import struct
import time
import weakref
from . import events
from . import base_subprocess
from . import futures
from . import exceptions
from . import proactor_events
from . import selector_events
from . import tasks
from . import windows_utils
# XXX RustPython TODO: _overlapped
# from . import _overlapped
from .coroutines import coroutine
from .log import logger
__all__ = (
'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor',
'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy',
'WindowsProactorEventLoopPolicy',
)
__all__ = ['SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor',
'DefaultEventLoopPolicy',
]
NULL = _winapi.NULL
INFINITE = _winapi.INFINITE
NULL = 0
INFINITE = 0xffffffff
ERROR_CONNECTION_REFUSED = 1225
ERROR_CONNECTION_ABORTED = 1236
@@ -62,7 +53,7 @@ class _OverlappedFuture(futures.Future):
info = super()._repr_info()
if self._ov is not None:
state = 'pending' if self._ov.pending else 'completed'
info.insert(1, f'overlapped=<{state}, {self._ov.address:#x}>')
info.insert(1, 'overlapped=<%s, %#x>' % (state, self._ov.address))
return info
def _cancel_overlapped(self):
@@ -81,9 +72,9 @@ class _OverlappedFuture(futures.Future):
self._loop.call_exception_handler(context)
self._ov = None
def cancel(self, msg=None):
def cancel(self):
self._cancel_overlapped()
return super().cancel(msg=msg)
return super().cancel()
def set_exception(self, exception):
super().set_exception(exception)
@@ -118,12 +109,12 @@ class _BaseWaitHandleFuture(futures.Future):
def _repr_info(self):
info = super()._repr_info()
info.append(f'handle={self._handle:#x}')
info.append('handle=%#x' % self._handle)
if self._handle is not None:
state = 'signaled' if self._poll() else 'waiting'
info.append(state)
if self._wait_handle is not None:
info.append(f'wait_handle={self._wait_handle:#x}')
info.append('wait_handle=%#x' % self._wait_handle)
return info
def _unregister_wait_cb(self, fut):
@@ -155,9 +146,9 @@ class _BaseWaitHandleFuture(futures.Future):
self._unregister_wait_cb(None)
def cancel(self, msg=None):
def cancel(self):
self._unregister_wait()
return super().cancel(msg=msg)
return super().cancel()
def set_exception(self, exception):
self._unregister_wait()
@@ -306,6 +297,9 @@ class PipeServer(object):
class _WindowsSelectorEventLoop(selector_events.BaseSelectorEventLoop):
"""Windows version of selector event loop."""
def _socketpair(self):
return windows_utils.socketpair()
class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
"""Windows version of proactor event loop using IOCP."""
@@ -315,34 +309,20 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
proactor = IocpProactor()
super().__init__(proactor)
def run_forever(self):
try:
assert self._self_reading_future is None
self.call_soon(self._loop_self_reading)
super().run_forever()
finally:
if self._self_reading_future is not None:
ov = self._self_reading_future._ov
self._self_reading_future.cancel()
# self_reading_future always uses IOCP, so even though it's
# been cancelled, we need to make sure that the IOCP message
# is received so that the kernel is not holding on to the
# memory, possibly causing memory corruption later. Only
# unregister it if IO is complete in all respects. Otherwise
# we need another _poll() later to complete the IO.
if ov is not None and not ov.pending:
self._proactor._unregister(ov)
self._self_reading_future = None
def _socketpair(self):
return windows_utils.socketpair()
async def create_pipe_connection(self, protocol_factory, address):
@coroutine
def create_pipe_connection(self, protocol_factory, address):
f = self._proactor.connect_pipe(address)
pipe = await f
pipe = yield from f
protocol = protocol_factory()
trans = self._make_duplex_pipe_transport(pipe, protocol,
extra={'addr': address})
return trans, protocol
async def start_serving_pipe(self, protocol_factory, address):
@coroutine
def start_serving_pipe(self, protocol_factory, address):
server = PipeServer(address)
def loop_accept_pipe(f=None):
@@ -367,10 +347,6 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
return
f = self._proactor.accept_pipe(pipe)
except BrokenPipeError:
if pipe and pipe.fileno() != -1:
pipe.close()
self.call_soon(loop_accept_pipe)
except OSError as exc:
if pipe and pipe.fileno() != -1:
self.call_exception_handler({
@@ -382,8 +358,7 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
elif self._debug:
logger.warning("Accept pipe failed on pipe %r",
pipe, exc_info=True)
self.call_soon(loop_accept_pipe)
except exceptions.CancelledError:
except futures.CancelledError:
if pipe:
pipe.close()
else:
@@ -393,22 +368,28 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
self.call_soon(loop_accept_pipe)
return [server]
async def _make_subprocess_transport(self, protocol, args, shell,
stdin, stdout, stderr, bufsize,
extra=None, **kwargs):
@coroutine
def _make_subprocess_transport(self, protocol, args, shell,
stdin, stdout, stderr, bufsize,
extra=None, **kwargs):
waiter = self.create_future()
transp = _WindowsSubprocessTransport(self, protocol, args, shell,
stdin, stdout, stderr, bufsize,
waiter=waiter, extra=extra,
**kwargs)
try:
await waiter
except (SystemExit, KeyboardInterrupt):
raise
except BaseException:
yield from waiter
except Exception as exc:
# Workaround CPython bug #23353: using yield/yield-from in an
# except block of a generator doesn't clear properly sys.exc_info()
err = exc
else:
err = None
if err is not None:
transp.close()
await transp._wait()
raise
yield from transp._wait()
raise err
return transp
@@ -416,7 +397,7 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
class IocpProactor:
"""Proactor implementation using IOCP."""
def __init__(self, concurrency=INFINITE):
def __init__(self, concurrency=0xffffffff):
self._loop = None
self._results = []
self._iocp = _overlapped.CreateIoCompletionPort(
@@ -426,16 +407,10 @@ class IocpProactor:
self._unregistered = []
self._stopped_serving = weakref.WeakSet()
def _check_closed(self):
if self._iocp is None:
raise RuntimeError('IocpProactor is closed')
def __repr__(self):
info = ['overlapped#=%s' % len(self._cache),
'result#=%s' % len(self._results)]
if self._iocp is None:
info.append('closed')
return '<%s %s>' % (self.__class__.__name__, " ".join(info))
return ('<%s overlapped#=%s result#=%s>'
% (self.__class__.__name__, len(self._cache),
len(self._results)))
def set_loop(self, loop):
self._loop = loop
@@ -445,40 +420,13 @@ class IocpProactor:
self._poll(timeout)
tmp = self._results
self._results = []
try:
return tmp
finally:
# Needed to break cycles when an exception occurs.
tmp = None
return tmp
def _result(self, value):
fut = self._loop.create_future()
fut.set_result(value)
return fut
@staticmethod
def finish_socket_func(trans, key, ov):
try:
return ov.getresult()
except OSError as exc:
if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED,
_overlapped.ERROR_OPERATION_ABORTED):
raise ConnectionResetError(*exc.args)
else:
raise
@classmethod
def _finish_recvfrom(cls, trans, key, ov, *, empty_result):
try:
return cls.finish_socket_func(trans, key, ov)
except OSError as exc:
# WSARecvFrom will report ERROR_PORT_UNREACHABLE when the same
# socket is used to send to an address that is not listening.
if exc.winerror == _overlapped.ERROR_PORT_UNREACHABLE:
return empty_result, None
else:
raise
def recv(self, conn, nbytes, flags=0):
self._register_with_iocp(conn)
ov = _overlapped.Overlapped(NULL)
@@ -490,50 +438,16 @@ class IocpProactor:
except BrokenPipeError:
return self._result(b'')
return self._register(ov, conn, self.finish_socket_func)
def finish_recv(trans, key, ov):
try:
return ov.getresult()
except OSError as exc:
if exc.winerror == _overlapped.ERROR_NETNAME_DELETED:
raise ConnectionResetError(*exc.args)
else:
raise
def recv_into(self, conn, buf, flags=0):
self._register_with_iocp(conn)
ov = _overlapped.Overlapped(NULL)
try:
if isinstance(conn, socket.socket):
ov.WSARecvInto(conn.fileno(), buf, flags)
else:
ov.ReadFileInto(conn.fileno(), buf)
except BrokenPipeError:
return self._result(0)
return self._register(ov, conn, self.finish_socket_func)
def recvfrom(self, conn, nbytes, flags=0):
self._register_with_iocp(conn)
ov = _overlapped.Overlapped(NULL)
try:
ov.WSARecvFrom(conn.fileno(), nbytes, flags)
except BrokenPipeError:
return self._result((b'', None))
return self._register(ov, conn, partial(self._finish_recvfrom,
empty_result=b''))
def recvfrom_into(self, conn, buf, flags=0):
self._register_with_iocp(conn)
ov = _overlapped.Overlapped(NULL)
try:
ov.WSARecvFromInto(conn.fileno(), buf, flags)
except BrokenPipeError:
return self._result((0, None))
return self._register(ov, conn, partial(self._finish_recvfrom,
empty_result=0))
def sendto(self, conn, buf, flags=0, addr=None):
self._register_with_iocp(conn)
ov = _overlapped.Overlapped(NULL)
ov.WSASendTo(conn.fileno(), buf, flags, addr)
return self._register(ov, conn, self.finish_socket_func)
return self._register(ov, conn, finish_recv)
def send(self, conn, buf, flags=0):
self._register_with_iocp(conn)
@@ -543,7 +457,16 @@ class IocpProactor:
else:
ov.WriteFile(conn.fileno(), buf)
return self._register(ov, conn, self.finish_socket_func)
def finish_send(trans, key, ov):
try:
return ov.getresult()
except OSError as exc:
if exc.winerror == _overlapped.ERROR_NETNAME_DELETED:
raise ConnectionResetError(*exc.args)
else:
raise
return self._register(ov, conn, finish_send)
def accept(self, listener):
self._register_with_iocp(listener)
@@ -560,11 +483,12 @@ class IocpProactor:
conn.settimeout(listener.gettimeout())
return conn, conn.getpeername()
async def accept_coro(future, conn):
@coroutine
def accept_coro(future, conn):
# Coroutine closing the accept socket if the future is cancelled
try:
await future
except exceptions.CancelledError:
yield from future
except futures.CancelledError:
conn.close()
raise
@@ -574,14 +498,6 @@ class IocpProactor:
return future
def connect(self, conn, address):
if conn.type == socket.SOCK_DGRAM:
# WSAConnect will complete immediately for UDP sockets so we don't
# need to register any IOCP operation
_overlapped.WSAConnect(conn.fileno(), address)
fut = self._loop.create_future()
fut.set_result(None)
return fut
self._register_with_iocp(conn)
# The socket needs to be locally bound before we call ConnectEx().
try:
@@ -604,18 +520,6 @@ class IocpProactor:
return self._register(ov, conn, finish_connect)
def sendfile(self, sock, file, offset, count):
self._register_with_iocp(sock)
ov = _overlapped.Overlapped(NULL)
offset_low = offset & 0xffff_ffff
offset_high = (offset >> 32) & 0xffff_ffff
ov.TransmitFile(sock.fileno(),
msvcrt.get_osfhandle(file.fileno()),
offset_low, offset_high,
count, 0, 0)
return self._register(ov, sock, self.finish_socket_func)
def accept_pipe(self, pipe):
self._register_with_iocp(pipe)
ov = _overlapped.Overlapped(NULL)
@@ -633,12 +537,13 @@ class IocpProactor:
return self._register(ov, pipe, finish_accept_pipe)
async def connect_pipe(self, address):
@coroutine
def connect_pipe(self, address):
delay = CONNECT_PIPE_INIT_DELAY
while True:
# Unfortunately there is no way to do an overlapped connect to
# a pipe. Call CreateFile() in a loop until it doesn't fail with
# ERROR_PIPE_BUSY.
# Unfortunately there is no way to do an overlapped connect to a pipe.
# Call CreateFile() in a loop until it doesn't fail with
# ERROR_PIPE_BUSY
try:
handle = _overlapped.ConnectPipe(address)
break
@@ -648,7 +553,7 @@ class IocpProactor:
# ConnectPipe() failed with ERROR_PIPE_BUSY: retry later
delay = min(delay * 2, CONNECT_PIPE_MAX_DELAY)
await tasks.sleep(delay)
yield from tasks.sleep(delay, loop=self._loop)
return windows_utils.PipeHandle(handle)
@@ -668,8 +573,6 @@ class IocpProactor:
return fut
def _wait_for_handle(self, handle, timeout, _is_cancel):
self._check_closed()
if timeout is None:
ms = _winapi.INFINITE
else:
@@ -712,8 +615,6 @@ class IocpProactor:
# that succeed immediately.
def _register(self, ov, obj, callback):
self._check_closed()
# Return a future which will be set with the result of the
# operation when it completes. The future's value is actually
# the value returned by callback().
@@ -750,7 +651,6 @@ class IocpProactor:
already be signalled (pending in the proactor event queue). It is also
safe if the event is never signalled (because it was cancelled).
"""
self._check_closed()
self._unregistered.append(ov)
def _get_accept_socket(self, family):
@@ -807,10 +707,8 @@ class IocpProactor:
else:
f.set_result(value)
self._results.append(f)
finally:
f = None
# Remove unregistered futures
# Remove unregisted futures
for ov in self._unregistered:
self._cache.pop(ov.address, None)
self._unregistered.clear()
@@ -822,12 +720,8 @@ class IocpProactor:
self._stopped_serving.add(obj)
def close(self):
if self._iocp is None:
# already closed
return
# Cancel remaining registered operations.
for fut, ov, obj, callback in list(self._cache.values()):
for address, (fut, ov, obj, callback) in list(self._cache.items()):
if fut.cancelled():
# Nothing to do with cancelled futures
pass
@@ -848,25 +742,14 @@ class IocpProactor:
context['source_traceback'] = fut._source_traceback
self._loop.call_exception_handler(context)
# Wait until all cancelled overlapped complete: don't exit with running
# overlapped to prevent a crash. Display progress every second if the
# loop is still running.
msg_update = 1.0
start_time = time.monotonic()
next_msg = start_time + msg_update
while self._cache:
if next_msg <= time.monotonic():
logger.debug('%r is running after closing for %.1f seconds',
self, time.monotonic() - start_time)
next_msg = time.monotonic() + msg_update
# handle a few events, or timeout
self._poll(msg_update)
if not self._poll(1):
logger.debug('taking long time to close proactor')
self._results = []
_winapi.CloseHandle(self._iocp)
self._iocp = None
if self._iocp is not None:
_winapi.CloseHandle(self._iocp)
self._iocp = None
def __del__(self):
self.close()
@@ -890,12 +773,8 @@ class _WindowsSubprocessTransport(base_subprocess.BaseSubprocessTransport):
SelectorEventLoop = _WindowsSelectorEventLoop
class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
class _WindowsDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
_loop_factory = SelectorEventLoop
class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
_loop_factory = ProactorEventLoop
DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy
DefaultEventLoopPolicy = _WindowsDefaultEventLoopPolicy

74
Lib/asyncio/windows_utils.py
View File

@@ -1,4 +1,6 @@
"""Various Windows specific bits and pieces."""
"""
Various Windows specific bits and pieces
"""
import sys
@@ -7,14 +9,16 @@ if sys.platform != 'win32': # pragma: no cover
import _winapi
import itertools
import msvcrt
# XXX RustPython TODO: msvcrt
# import msvcrt
import os
import socket
import subprocess
import tempfile
import warnings
__all__ = 'pipe', 'Popen', 'PIPE', 'PipeHandle'
__all__ = ['socketpair', 'pipe', 'Popen', 'PIPE', 'PipeHandle']
# Constants/globals
@@ -26,14 +30,61 @@ STDOUT = subprocess.STDOUT
_mmap_counter = itertools.count()
if hasattr(socket, 'socketpair'):
# Since Python 3.5, socket.socketpair() is now also available on Windows
socketpair = socket.socketpair
else:
# Replacement for socket.socketpair()
def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
"""A socket pair usable as a self-pipe, for Windows.
Origin: https://gist.github.com/4325783, by Geert Jansen.
Public domain.
"""
if family == socket.AF_INET:
host = '127.0.0.1'
elif family == socket.AF_INET6:
host = '::1'
else:
raise ValueError("Only AF_INET and AF_INET6 socket address "
"families are supported")
if type != socket.SOCK_STREAM:
raise ValueError("Only SOCK_STREAM socket type is supported")
if proto != 0:
raise ValueError("Only protocol zero is supported")
# We create a connected TCP socket. Note the trick with setblocking(0)
# that prevents us from having to create a thread.
lsock = socket.socket(family, type, proto)
try:
lsock.bind((host, 0))
lsock.listen(1)
# On IPv6, ignore flow_info and scope_id
addr, port = lsock.getsockname()[:2]
csock = socket.socket(family, type, proto)
try:
csock.setblocking(False)
try:
csock.connect((addr, port))
except (BlockingIOError, InterruptedError):
pass
csock.setblocking(True)
ssock, _ = lsock.accept()
except:
csock.close()
raise
finally:
lsock.close()
return (ssock, csock)
# Replacement for os.pipe() using handles instead of fds
def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE):
"""Like os.pipe() but with overlapped support and using handles not fds."""
address = tempfile.mktemp(
prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format(
os.getpid(), next(_mmap_counter)))
address = tempfile.mktemp(prefix=r'\\.\pipe\python-pipe-%d-%d-' %
(os.getpid(), next(_mmap_counter)))
if duplex:
openmode = _winapi.PIPE_ACCESS_DUPLEX
@@ -88,10 +139,10 @@ class PipeHandle:
def __repr__(self):
if self._handle is not None:
handle = f'handle={self._handle!r}'
handle = 'handle=%r' % self._handle
else:
handle = 'closed'
return f'<{self.__class__.__name__} {handle}>'
return '<%s %s>' % (self.__class__.__name__, handle)
@property
def handle(self):
@@ -99,7 +150,7 @@ class PipeHandle:
def fileno(self):
if self._handle is None:
raise ValueError("I/O operation on closed pipe")
raise ValueError("I/O operatioon on closed pipe")
return self._handle
def close(self, *, CloseHandle=_winapi.CloseHandle):
@@ -107,9 +158,10 @@ class PipeHandle:
CloseHandle(self._handle)
self._handle = None
def __del__(self, _warn=warnings.warn):
def __del__(self):
if self._handle is not None:
_warn(f"unclosed {self!r}", ResourceWarning, source=self)
warnings.warn("unclosed %r" % self, ResourceWarning,
source=self)
self.close()
def __enter__(self):

642
Lib/asyncore.py vendored
View File

@@ -1,642 +0,0 @@
# -*- Mode: Python -*-
# Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp
# Author: Sam Rushing <rushing@nightmare.com>
# ======================================================================
# Copyright 1996 by Sam Rushing
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Sam
# Rushing not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# ======================================================================
"""Basic infrastructure for asynchronous socket service clients and servers.
There are only two ways to have a program on a single processor do "more
than one thing at a time". Multi-threaded programming is the simplest and
most popular way to do it, but there is another very different technique,
that lets you have nearly all the advantages of multi-threading, without
actually using multiple threads. it's really only practical if your program
is largely I/O bound. If your program is CPU bound, then pre-emptive
scheduled threads are probably what you really need. Network servers are
rarely CPU-bound, however.
If your operating system supports the select() system call in its I/O
library (and nearly all do), then you can use it to juggle multiple
communication channels at once; doing other work while your I/O is taking
place in the "background." Although this strategy can seem strange and
complex, especially at first, it is in many ways easier to understand and
control than multi-threaded programming. The module documented here solves
many of the difficult problems for you, making the task of building
sophisticated high-performance network servers and clients a snap.
"""
import select
import socket
import sys
import time
import warnings
import os
from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
errorcode
_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
EBADF})
try:
socket_map
except NameError:
socket_map = {}
def _strerror(err):
try:
return os.strerror(err)
except (ValueError, OverflowError, NameError):
if err in errorcode:
return errorcode[err]
return "Unknown error %s" %err
class ExitNow(Exception):
pass
_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit)
def read(obj):
try:
obj.handle_read_event()
except _reraised_exceptions:
raise
except:
obj.handle_error()
def write(obj):
try:
obj.handle_write_event()
except _reraised_exceptions:
raise
except:
obj.handle_error()
def _exception(obj):
try:
obj.handle_expt_event()
except _reraised_exceptions:
raise
except:
obj.handle_error()
def readwrite(obj, flags):
try:
if flags & select.POLLIN:
obj.handle_read_event()
if flags & select.POLLOUT:
obj.handle_write_event()
if flags & select.POLLPRI:
obj.handle_expt_event()
if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
obj.handle_close()
except OSError as e:
if e.args[0] not in _DISCONNECTED:
obj.handle_error()
else:
obj.handle_close()
except _reraised_exceptions:
raise
except:
obj.handle_error()
def poll(timeout=0.0, map=None):
if map is None:
map = socket_map
if map:
r = []; w = []; e = []
for fd, obj in list(map.items()):
is_r = obj.readable()
is_w = obj.writable()
if is_r:
r.append(fd)
# accepting sockets should not be writable
if is_w and not obj.accepting:
w.append(fd)
if is_r or is_w:
e.append(fd)
if [] == r == w == e:
time.sleep(timeout)
return
r, w, e = select.select(r, w, e, timeout)
for fd in r:
obj = map.get(fd)
if obj is None:
continue
read(obj)
for fd in w:
obj = map.get(fd)
if obj is None:
continue
write(obj)
for fd in e:
obj = map.get(fd)
if obj is None:
continue
_exception(obj)
def poll2(timeout=0.0, map=None):
# Use the poll() support added to the select module in Python 2.0
if map is None:
map = socket_map
if timeout is not None:
# timeout is in milliseconds
timeout = int(timeout*1000)
pollster = select.poll()
if map:
for fd, obj in list(map.items()):
flags = 0
if obj.readable():
flags |= select.POLLIN | select.POLLPRI
# accepting sockets should not be writable
if obj.writable() and not obj.accepting:
flags |= select.POLLOUT
if flags:
pollster.register(fd, flags)
r = pollster.poll(timeout)
for fd, flags in r:
obj = map.get(fd)
if obj is None:
continue
readwrite(obj, flags)
poll3 = poll2 # Alias for backward compatibility
def loop(timeout=30.0, use_poll=False, map=None, count=None):
if map is None:
map = socket_map
if use_poll and hasattr(select, 'poll'):
poll_fun = poll2
else:
poll_fun = poll
if count is None:
while map:
poll_fun(timeout, map)
else:
while map and count > 0:
poll_fun(timeout, map)
count = count - 1
class dispatcher:
debug = False
connected = False
accepting = False
connecting = False
closing = False
addr = None
ignore_log_types = frozenset({'warning'})
def __init__(self, sock=None, map=None):
if map is None:
self._map = socket_map
else:
self._map = map
self._fileno = None
if sock:
# Set to nonblocking just to make sure for cases where we
# get a socket from a blocking source.
sock.setblocking(0)
self.set_socket(sock, map)
self.connected = True
# The constructor no longer requires that the socket
# passed be connected.
try:
self.addr = sock.getpeername()
except OSError as err:
if err.args[0] in (ENOTCONN, EINVAL):
# To handle the case where we got an unconnected
# socket.
self.connected = False
else:
# The socket is broken in some unknown way, alert
# the user and remove it from the map (to prevent
# polling of broken sockets).
self.del_channel(map)
raise
else:
self.socket = None
def __repr__(self):
status = [self.__class__.__module__+"."+self.__class__.__qualname__]
if self.accepting and self.addr:
status.append('listening')
elif self.connected:
status.append('connected')
if self.addr is not None:
try:
status.append('%s:%d' % self.addr)
except TypeError:
status.append(repr(self.addr))
return '<%s at %#x>' % (' '.join(status), id(self))
def add_channel(self, map=None):
#self.log_info('adding channel %s' % self)
if map is None:
map = self._map
map[self._fileno] = self
def del_channel(self, map=None):
fd = self._fileno
if map is None:
map = self._map
if fd in map:
#self.log_info('closing channel %d:%s' % (fd, self))
del map[fd]
self._fileno = None
def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM):
self.family_and_type = family, type
sock = socket.socket(family, type)
sock.setblocking(0)
self.set_socket(sock)
def set_socket(self, sock, map=None):
self.socket = sock
self._fileno = sock.fileno()
self.add_channel(map)
def set_reuse_addr(self):
# try to re-use a server port if possible
try:
self.socket.setsockopt(
socket.SOL_SOCKET, socket.SO_REUSEADDR,
self.socket.getsockopt(socket.SOL_SOCKET,
socket.SO_REUSEADDR) | 1
)
except OSError:
pass
# ==================================================
# predicates for select()
# these are used as filters for the lists of sockets
# to pass to select().
# ==================================================
def readable(self):
return True
def writable(self):
return True
# ==================================================
# socket object methods.
# ==================================================
def listen(self, num):
self.accepting = True
if os.name == 'nt' and num > 5:
num = 5
return self.socket.listen(num)
def bind(self, addr):
self.addr = addr
return self.socket.bind(addr)
def connect(self, address):
self.connected = False
self.connecting = True
err = self.socket.connect_ex(address)
if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \
or err == EINVAL and os.name == 'nt':
self.addr = address
return
if err in (0, EISCONN):
self.addr = address
self.handle_connect_event()
else:
raise OSError(err, errorcode[err])
def accept(self):
# XXX can return either an address pair or None
try:
conn, addr = self.socket.accept()
except TypeError:
return None
except OSError as why:
if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
return None
else:
raise
else:
return conn, addr
def send(self, data):
try:
result = self.socket.send(data)
return result
except OSError as why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
def recv(self, buffer_size):
try:
data = self.socket.recv(buffer_size)
if not data:
# a closed connection is indicated by signaling
# a read condition, and having recv() return 0.
self.handle_close()
return b''
else:
return data
except OSError as why:
# winsock sometimes raises ENOTCONN
if why.args[0] in _DISCONNECTED:
self.handle_close()
return b''
else:
raise
def close(self):
self.connected = False
self.accepting = False
self.connecting = False
self.del_channel()
if self.socket is not None:
try:
self.socket.close()
except OSError as why:
if why.args[0] not in (ENOTCONN, EBADF):
raise
# log and log_info may be overridden to provide more sophisticated
# logging and warning methods. In general, log is for 'hit' logging
# and 'log_info' is for informational, warning and error logging.
def log(self, message):
sys.stderr.write('log: %s\n' % str(message))
def log_info(self, message, type='info'):
if type not in self.ignore_log_types:
print('%s: %s' % (type, message))
def handle_read_event(self):
if self.accepting:
# accepting sockets are never connected, they "spawn" new
# sockets that are connected
self.handle_accept()
elif not self.connected:
if self.connecting:
self.handle_connect_event()
self.handle_read()
else:
self.handle_read()
def handle_connect_event(self):
err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0:
raise OSError(err, _strerror(err))
self.handle_connect()
self.connected = True
self.connecting = False
def handle_write_event(self):
if self.accepting:
# Accepting sockets shouldn't get a write event.
# We will pretend it didn't happen.
return
if not self.connected:
if self.connecting:
self.handle_connect_event()
self.handle_write()
def handle_expt_event(self):
# handle_expt_event() is called if there might be an error on the
# socket, or if there is OOB data
# check for the error condition first
err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0:
# we can get here when select.select() says that there is an
# exceptional condition on the socket
# since there is an error, we'll go ahead and close the socket
# like we would in a subclassed handle_read() that received no
# data
self.handle_close()
else:
self.handle_expt()
def handle_error(self):
nil, t, v, tbinfo = compact_traceback()
# sometimes a user repr method will crash.
try:
self_repr = repr(self)
except:
self_repr = '<__repr__(self) failed for object at %0x>' % id(self)
self.log_info(
'uncaptured python exception, closing channel %s (%s:%s %s)' % (
self_repr,
t,
v,
tbinfo
),
'error'
)
self.handle_close()
def handle_expt(self):
self.log_info('unhandled incoming priority event', 'warning')
def handle_read(self):
self.log_info('unhandled read event', 'warning')
def handle_write(self):
self.log_info('unhandled write event', 'warning')
def handle_connect(self):
self.log_info('unhandled connect event', 'warning')
def handle_accept(self):
pair = self.accept()
if pair is not None:
self.handle_accepted(*pair)
def handle_accepted(self, sock, addr):
sock.close()
self.log_info('unhandled accepted event', 'warning')
def handle_close(self):
self.log_info('unhandled close event', 'warning')
self.close()
# ---------------------------------------------------------------------------
# adds simple buffered output capability, useful for simple clients.
# [for more sophisticated usage use asynchat.async_chat]
# ---------------------------------------------------------------------------
class dispatcher_with_send(dispatcher):
def __init__(self, sock=None, map=None):
dispatcher.__init__(self, sock, map)
self.out_buffer = b''
def initiate_send(self):
num_sent = 0
num_sent = dispatcher.send(self, self.out_buffer[:65536])
self.out_buffer = self.out_buffer[num_sent:]
def handle_write(self):
self.initiate_send()
def writable(self):
return (not self.connected) or len(self.out_buffer)
def send(self, data):
if self.debug:
self.log_info('sending %s' % repr(data))
self.out_buffer = self.out_buffer + data
self.initiate_send()
# ---------------------------------------------------------------------------
# used for debugging.
# ---------------------------------------------------------------------------
def compact_traceback():
t, v, tb = sys.exc_info()
tbinfo = []
if not tb: # Must have a traceback
raise AssertionError("traceback does not exist")
while tb:
tbinfo.append((
tb.tb_frame.f_code.co_filename,
tb.tb_frame.f_code.co_name,
str(tb.tb_lineno)
))
tb = tb.tb_next
# just to be safe
del tb
file, function, line = tbinfo[-1]
info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])
return (file, function, line), t, v, info
def close_all(map=None, ignore_all=False):
if map is None:
map = socket_map
for x in list(map.values()):
try:
x.close()
except OSError as x:
if x.args[0] == EBADF:
pass
elif not ignore_all:
raise
except _reraised_exceptions:
raise
except:
if not ignore_all:
raise
map.clear()
# Asynchronous File I/O:
#
# After a little research (reading man pages on various unixen, and
# digging through the linux kernel), I've determined that select()
# isn't meant for doing asynchronous file i/o.
# Heartening, though - reading linux/mm/filemap.c shows that linux
# supports asynchronous read-ahead. So _MOST_ of the time, the data
# will be sitting in memory for us already when we go to read it.
#
# What other OS's (besides NT) support async file i/o? [VMS?]
#
# Regardless, this is useful for pipes, and stdin/stdout...
if os.name == 'posix':
class file_wrapper:
# Here we override just enough to make a file
# look like a socket for the purposes of asyncore.
# The passed fd is automatically os.dup()'d
def __init__(self, fd):
self.fd = os.dup(fd)
def __del__(self):
if self.fd >= 0:
warnings.warn("unclosed file %r" % self, ResourceWarning,
source=self)
self.close()
def recv(self, *args):
return os.read(self.fd, *args)
def send(self, *args):
return os.write(self.fd, *args)
def getsockopt(self, level, optname, buflen=None):
if (level == socket.SOL_SOCKET and
optname == socket.SO_ERROR and
not buflen):
return 0
raise NotImplementedError("Only asyncore specific behaviour "
"implemented.")
read = recv
write = send
def close(self):
if self.fd < 0:
return
fd = self.fd
self.fd = -1
os.close(fd)
def fileno(self):
return self.fd
class file_dispatcher(dispatcher):
def __init__(self, fd, map=None):
dispatcher.__init__(self, None, map)
self.connected = True
try:
fd = fd.fileno()
except AttributeError:
pass
self.set_file(fd)
# set it to non-blocking mode
os.set_blocking(fd, False)
def set_file(self, fd):
self.socket = file_wrapper(fd)
self._fileno = self.socket.fileno()
self.add_channel()

9
Lib/atexit.py vendored Normal file
View File

@@ -0,0 +1,9 @@
# Dummy implementation of atexit
def register(func, *args, **kwargs):
return func
def unregister(func):
pass

159
Lib/base64.py vendored
View File

@@ -1,4 +1,4 @@
#! /usr/bin/env python3
#! /usr/bin/python3.6
"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings"""
@@ -16,7 +16,7 @@ __all__ = [
'encode', 'decode', 'encodebytes', 'decodebytes',
# Generalized interface for other encodings
'b64encode', 'b64decode', 'b32encode', 'b32decode',
'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode',
'b16encode', 'b16decode',
# Base85 and Ascii85 encodings
'b85encode', 'b85decode', 'a85encode', 'a85decode',
# Standard Base64 encoding
@@ -76,16 +76,15 @@ def b64decode(s, altchars=None, validate=False):
normal base-64 alphabet nor the alternative alphabet are discarded prior
to the padding check. If validate is True, these non-alphabet characters
in the input result in a binascii.Error.
For more information about the strict base64 check, see:
https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64
"""
s = _bytes_from_decode_data(s)
if altchars is not None:
altchars = _bytes_from_decode_data(altchars)
assert len(altchars) == 2, repr(altchars)
s = s.translate(bytes.maketrans(altchars, b'+/'))
return binascii.a2b_base64(s, strict_mode=validate)
if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s):
raise binascii.Error('Non-base64 digit found')
return binascii.a2b_base64(s)
def standard_b64encode(s):
@@ -136,40 +135,19 @@ def urlsafe_b64decode(s):
# Base32 encoding/decoding must be done in Python
_B32_ENCODE_DOCSTRING = '''
Encode the bytes-like objects using {encoding} and return a bytes object.
'''
_B32_DECODE_DOCSTRING = '''
Decode the {encoding} encoded bytes-like object or ASCII string s.
Optional casefold is a flag specifying whether a lowercase alphabet is
acceptable as input. For security purposes, the default is False.
{extra_args}
The result is returned as a bytes object. A binascii.Error is raised if
the input is incorrectly padded or if there are non-alphabet
characters present in the input.
'''
_B32_DECODE_MAP01_DOCSTRING = '''
RFC 3548 allows for optional mapping of the digit 0 (zero) to the
letter O (oh), and for optional mapping of the digit 1 (one) to
either the letter I (eye) or letter L (el). The optional argument
map01 when not None, specifies which letter the digit 1 should be
mapped to (when map01 is not None, the digit 0 is always mapped to
the letter O). For security purposes the default is None, so that
0 and 1 are not allowed in the input.
'''
_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV'
_b32tab2 = {}
_b32rev = {}
_b32tab2 = None
_b32rev = None
def _b32encode(alphabet, s):
def b32encode(s):
"""Encode the bytes-like object s using Base32 and return a bytes object.
"""
global _b32tab2
# Delay the initialization of the table to not waste memory
# if the function is never called
if alphabet not in _b32tab2:
b32tab = [bytes((i,)) for i in alphabet]
_b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab]
if _b32tab2 is None:
b32tab = [bytes((i,)) for i in _b32alphabet]
_b32tab2 = [a + b for a in b32tab for b in b32tab]
b32tab = None
if not isinstance(s, bytes_types):
@@ -180,9 +158,9 @@ def _b32encode(alphabet, s):
s = s + b'\0' * (5 - leftover) # Don't use += !
encoded = bytearray()
from_bytes = int.from_bytes
b32tab2 = _b32tab2[alphabet]
b32tab2 = _b32tab2
for i in range(0, len(s), 5):
c = from_bytes(s[i: i + 5]) # big endian
c = from_bytes(s[i: i + 5], 'big')
encoded += (b32tab2[c >> 30] + # bits 1 - 10
b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20
b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30
@@ -199,12 +177,29 @@ def _b32encode(alphabet, s):
encoded[-1:] = b'='
return bytes(encoded)
def _b32decode(alphabet, s, casefold=False, map01=None):
def b32decode(s, casefold=False, map01=None):
"""Decode the Base32 encoded bytes-like object or ASCII string s.
Optional casefold is a flag specifying whether a lowercase alphabet is
acceptable as input. For security purposes, the default is False.
RFC 3548 allows for optional mapping of the digit 0 (zero) to the
letter O (oh), and for optional mapping of the digit 1 (one) to
either the letter I (eye) or letter L (el). The optional argument
map01 when not None, specifies which letter the digit 1 should be
mapped to (when map01 is not None, the digit 0 is always mapped to
the letter O). For security purposes the default is None, so that
0 and 1 are not allowed in the input.
The result is returned as a bytes object. A binascii.Error is raised if
the input is incorrectly padded or if there are non-alphabet
characters present in the input.
"""
global _b32rev
# Delay the initialization of the table to not waste memory
# if the function is never called
if alphabet not in _b32rev:
_b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)}
if _b32rev is None:
_b32rev = {v: k for k, v in enumerate(_b32alphabet)}
s = _bytes_from_decode_data(s)
if len(s) % 8:
raise binascii.Error('Incorrect padding')
@@ -225,7 +220,7 @@ def _b32decode(alphabet, s, casefold=False, map01=None):
padchars = l - len(s)
# Now decode the full quanta
decoded = bytearray()
b32rev = _b32rev[alphabet]
b32rev = _b32rev
for i in range(0, len(s), 8):
quanta = s[i: i + 8]
acc = 0
@@ -234,38 +229,18 @@ def _b32decode(alphabet, s, casefold=False, map01=None):
acc = (acc << 5) + b32rev[c]
except KeyError:
raise binascii.Error('Non-base32 digit found') from None
decoded += acc.to_bytes(5) # big endian
decoded += acc.to_bytes(5, 'big')
# Process the last, partial quanta
if l % 8 or padchars not in {0, 1, 3, 4, 6}:
raise binascii.Error('Incorrect padding')
if padchars and decoded:
acc <<= 5 * padchars
last = acc.to_bytes(5) # big endian
last = acc.to_bytes(5, 'big')
leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1
decoded[-5:] = last[:leftover]
return bytes(decoded)
def b32encode(s):
return _b32encode(_b32alphabet, s)
b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32')
def b32decode(s, casefold=False, map01=None):
return _b32decode(_b32alphabet, s, casefold, map01)
b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32',
extra_args=_B32_DECODE_MAP01_DOCSTRING)
def b32hexencode(s):
return _b32encode(_b32hexalphabet, s)
b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex')
def b32hexdecode(s, casefold=False):
# base32hex does not have the 01 mapping
return _b32decode(_b32hexalphabet, s, casefold)
b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex',
extra_args='')
# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
# lowercase. The RFC also recommends against accepting input case
# insensitively.
@@ -345,7 +320,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
global _a85chars, _a85chars2
# Delay the initialization of tables to not waste memory
# if the function is never called
if _a85chars2 is None:
if _a85chars is None:
_a85chars = [bytes((i,)) for i in range(33, 118)]
_a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]
@@ -453,7 +428,7 @@ def b85encode(b, pad=False):
global _b85chars, _b85chars2
# Delay the initialization of tables to not waste memory
# if the function is never called
if _b85chars2 is None:
if _b85chars is None:
_b85chars = [bytes((i,)) for i in _b85alphabet]
_b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
return _85encode(b, _b85chars, _b85chars2, pad)
@@ -508,8 +483,14 @@ MAXBINSIZE = (MAXLINESIZE//4)*3
def encode(input, output):
"""Encode a file; input and output are binary files."""
while s := input.read(MAXBINSIZE):
while len(s) < MAXBINSIZE and (ns := input.read(MAXBINSIZE-len(s))):
while True:
s = input.read(MAXBINSIZE)
if not s:
break
while len(s) < MAXBINSIZE:
ns = input.read(MAXBINSIZE-len(s))
if not ns:
break
s += ns
line = binascii.b2a_base64(s)
output.write(line)
@@ -517,7 +498,10 @@ def encode(input, output):
def decode(input, output):
"""Decode a file; input and output are binary files."""
while line := input.readline():
while True:
line = input.readline()
if not line:
break
s = binascii.a2b_base64(line)
output.write(s)
@@ -547,34 +531,49 @@ def encodebytes(s):
pieces.append(binascii.b2a_base64(chunk))
return b"".join(pieces)
def encodestring(s):
"""Legacy alias of encodebytes()."""
import warnings
warnings.warn("encodestring() is a deprecated alias since 3.1, "
"use encodebytes()",
DeprecationWarning, 2)
return encodebytes(s)
def decodebytes(s):
"""Decode a bytestring of base-64 data into a bytes object."""
_input_type_check(s)
return binascii.a2b_base64(s)
def decodestring(s):
"""Legacy alias of decodebytes()."""
import warnings
warnings.warn("decodestring() is a deprecated alias since Python 3.1, "
"use decodebytes()",
DeprecationWarning, 2)
return decodebytes(s)
# Usable as a script...
def main():
"""Small main program"""
import sys, getopt
usage = f"""usage: {sys.argv[0]} [-h|-d|-e|-u] [file|-]
-h: print this help message and exit
-d, -u: decode
-e: encode (default)"""
try:
opts, args = getopt.getopt(sys.argv[1:], 'hdeu')
opts, args = getopt.getopt(sys.argv[1:], 'deut')
except getopt.error as msg:
sys.stdout = sys.stderr
print(msg)
print(usage)
print("""usage: %s [-d|-e|-u|-t] [file|-]
-d, -u: decode
-e: encode (default)
-t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0])
sys.exit(2)
func = encode
for o, a in opts:
if o == '-e': func = encode
if o == '-d': func = decode
if o == '-u': func = decode
if o == '-h': print(usage); return
if o == '-t': test(); return
if args and args[0] != '-':
with open(args[0], 'rb') as f:
func(f, sys.stdout.buffer)
@@ -582,5 +581,15 @@ def main():
func(sys.stdin.buffer, sys.stdout.buffer)
def test():
s0 = b"Aladdin:open sesame"
print(repr(s0))
s1 = encodebytes(s0)
print(repr(s1))
s2 = decodebytes(s1)
print(repr(s2))
assert s0 == s2
if __name__ == '__main__':
main()

75
Lib/bdb.py vendored
View File

@@ -34,8 +34,6 @@ class Bdb:
self.fncache = {}
self.frame_returning = None
self._load_breaks()
def canonic(self, filename):
"""Return canonical form of filename.
@@ -119,7 +117,7 @@ class Bdb:
"""Invoke user function and return trace function for call event.
If the debugger stops on this function call, invoke
self.user_call(). Raise BdbQuit if self.quitting is set.
self.user_call(). Raise BbdQuit if self.quitting is set.
Return self.trace_dispatch to continue tracing in this scope.
"""
# XXX 'arg' is no longer used
@@ -367,12 +365,6 @@ class Bdb:
# Call self.get_*break*() to see the breakpoints or better
# for bp in Breakpoint.bpbynumber: if bp: bp.bpprint().
def _add_to_breaks(self, filename, lineno):
"""Add breakpoint to breaks, if not already there."""
bp_linenos = self.breaks.setdefault(filename, [])
if lineno not in bp_linenos:
bp_linenos.append(lineno)
def set_break(self, filename, lineno, temporary=False, cond=None,
funcname=None):
"""Set a new breakpoint for filename:lineno.
@@ -385,21 +377,12 @@ class Bdb:
line = linecache.getline(filename, lineno)
if not line:
return 'Line %s:%d does not exist' % (filename, lineno)
self._add_to_breaks(filename, lineno)
list = self.breaks.setdefault(filename, [])
if lineno not in list:
list.append(lineno)
bp = Breakpoint(filename, lineno, temporary, cond, funcname)
return None
def _load_breaks(self):
"""Apply all breakpoints (set in other instances) to this one.
Populates this instance's breaks list from the Breakpoint class's
list, which can have breakpoints set by another Bdb instance. This
is necessary for interactive sessions to keep the breakpoints
active across multiple calls to run().
"""
for (filename, lineno) in Breakpoint.bplist.keys():
self._add_to_breaks(filename, lineno)
def _prune_breaks(self, filename, lineno):
"""Prune breakpoints for filename:lineno.
@@ -570,12 +553,9 @@ class Bdb:
rv = frame.f_locals['__return__']
s += '->'
s += reprlib.repr(rv)
if lineno is not None:
line = linecache.getline(filename, lineno, frame.f_globals)
if line:
s += lprefix + line.strip()
else:
s += f'{lprefix}Warning: lineno is None'
line = linecache.getline(filename, lineno, frame.f_globals)
if line:
s += lprefix + line.strip()
return s
# The following methods can be called by clients to use
@@ -631,11 +611,26 @@ class Bdb:
# This method is more useful to debug a single function call.
def runcall(self, func, /, *args, **kwds):
def runcall(*args, **kwds):
"""Debug a single function call.
Return the result of the function call.
"""
if len(args) >= 2:
self, func, *args = args
elif not args:
raise TypeError("descriptor 'runcall' of 'Bdb' object "
"needs an argument")
elif 'func' in kwds:
func = kwds.pop('func')
self, *args = args
import warnings
warnings.warn("Passing 'func' as keyword argument is deprecated",
DeprecationWarning, stacklevel=2)
else:
raise TypeError('runcall expected at least 1 positional argument, '
'got %d' % (len(args)-1))
self.reset()
sys.settrace(self.trace_dispatch)
res = None
@@ -647,6 +642,7 @@ class Bdb:
self.quitting = True
sys.settrace(None)
return res
runcall.__text_signature__ = '($self, func, /, *args, **kwds)'
def set_trace():
@@ -701,12 +697,6 @@ class Breakpoint:
else:
self.bplist[file, line] = [self]
@staticmethod
def clearBreakpoints():
Breakpoint.next = 1
Breakpoint.bplist = {}
Breakpoint.bpbynumber = [None]
def deleteMe(self):
"""Delete the breakpoint from the list associated to a file:line.
@@ -808,18 +798,15 @@ def checkfuncname(b, frame):
return True
# Determines if there is an effective (active) breakpoint at this
# line of code. Returns breakpoint number or 0 if none
def effective(file, line, frame):
"""Return (active breakpoint, delete temporary flag) or (None, None) as
breakpoint to act upon.
"""Determine which breakpoint for this file:line is to be acted upon.
The "active breakpoint" is the first entry in bplist[line, file] (which
must exist) that is enabled, for which checkfuncname is True, and that
has neither a False condition nor a positive ignore count. The flag,
meaning that a temporary breakpoint should be deleted, is False only
when the condiion cannot be evaluated (in which case, ignore count is
ignored).
If no such entry exists, then (None, None) is returned.
Called only if we know there is a breakpoint at this location. Return
the breakpoint that was triggered and a boolean that indicates if it is
ok to delete a temporary breakpoint. Return (None, None) if there is no
matching breakpoint.
"""
possibles = Breakpoint.bplist[file, line]
for b in possibles:

138
Lib/bisect.py vendored
View File

@@ -1,118 +1,92 @@
"""Bisection algorithms."""
def insort_right(a, x, lo=0, hi=None, *, key=None):
def insort_right(a, x, lo=0, hi=None):
"""Insert item x in list a, and keep it sorted assuming a is sorted.
If x is already in a, insert it to the right of the rightmost x.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
A custom key function can be supplied to customize the sort order.
"""
if key is None:
lo = bisect_right(a, x, lo, hi)
else:
lo = bisect_right(a, key(x), lo, hi, key=key)
a.insert(lo, x)
def bisect_right(a, x, lo=0, hi=None, *, key=None):
"""Return the index where to insert item x in list a, assuming a is sorted.
The return value i is such that all e in a[:i] have e <= x, and all e in
a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will
insert just after the rightmost x already there.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
A custom key function can be supplied to customize the sort order.
"""
if lo < 0:
raise ValueError('lo must be non-negative')
if hi is None:
hi = len(a)
# Note, the comparison uses "<" to match the
# __lt__() logic in list.sort() and in heapq.
if key is None:
while lo < hi:
mid = (lo + hi) // 2
if x < a[mid]:
hi = mid
else:
lo = mid + 1
else:
while lo < hi:
mid = (lo + hi) // 2
if x < key(a[mid]):
hi = mid
else:
lo = mid + 1
while lo < hi:
mid = (lo+hi)//2
if x < a[mid]: hi = mid
else: lo = mid+1
a.insert(lo, x)
insort = insort_right # backward compatibility
def bisect_right(a, x, lo=0, hi=None):
"""Return the index where to insert item x in list a, assuming a is sorted.
The return value i is such that all e in a[:i] have e <= x, and all e in
a[i:] have e > x. So if x already appears in the list, a.insert(x) will
insert just after the rightmost x already there.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
"""
if lo < 0:
raise ValueError('lo must be non-negative')
if hi is None:
hi = len(a)
while lo < hi:
mid = (lo+hi)//2
if x < a[mid]: hi = mid
else: lo = mid+1
return lo
bisect = bisect_right # backward compatibility
def insort_left(a, x, lo=0, hi=None, *, key=None):
def insort_left(a, x, lo=0, hi=None):
"""Insert item x in list a, and keep it sorted assuming a is sorted.
If x is already in a, insert it to the left of the leftmost x.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
A custom key function can be supplied to customize the sort order.
"""
if key is None:
lo = bisect_left(a, x, lo, hi)
else:
lo = bisect_left(a, key(x), lo, hi, key=key)
a.insert(lo, x)
def bisect_left(a, x, lo=0, hi=None, *, key=None):
"""Return the index where to insert item x in list a, assuming a is sorted.
The return value i is such that all e in a[:i] have e < x, and all e in
a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will
insert just before the leftmost x already there.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
A custom key function can be supplied to customize the sort order.
"""
if lo < 0:
raise ValueError('lo must be non-negative')
if hi is None:
hi = len(a)
# Note, the comparison uses "<" to match the
# __lt__() logic in list.sort() and in heapq.
if key is None:
while lo < hi:
mid = (lo + hi) // 2
if a[mid] < x:
lo = mid + 1
else:
hi = mid
else:
while lo < hi:
mid = (lo + hi) // 2
if key(a[mid]) < x:
lo = mid + 1
else:
hi = mid
return lo
while lo < hi:
mid = (lo+hi)//2
if a[mid] < x: lo = mid+1
else: hi = mid
a.insert(lo, x)
def bisect_left(a, x, lo=0, hi=None):
"""Return the index where to insert item x in list a, assuming a is sorted.
The return value i is such that all e in a[:i] have e < x, and all e in
a[i:] have e >= x. So if x already appears in the list, a.insert(x) will
insert just before the leftmost x already there.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
"""
if lo < 0:
raise ValueError('lo must be non-negative')
if hi is None:
hi = len(a)
while lo < hi:
mid = (lo+hi)//2
if a[mid] < x: lo = mid+1
else: hi = mid
return lo
# Overwrite above definitions with a fast C implementation
try:
from _bisect import *
except ImportError:
pass
# Create aliases
bisect = bisect_right
insort = insort_right

344
Lib/bz2.py vendored
View File

@@ -1,344 +0,0 @@
"""Interface to the libbzip2 compression library.
This module provides a file interface, classes for incremental
(de)compression, and functions for one-shot (de)compression.
"""
__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor",
"open", "compress", "decompress"]
__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
from builtins import open as _builtin_open
import io
import os
import _compression
from _bz2 import BZ2Compressor, BZ2Decompressor
_MODE_CLOSED = 0
_MODE_READ = 1
# Value 2 no longer used
_MODE_WRITE = 3
class BZ2File(_compression.BaseStream):
"""A file object providing transparent bzip2 (de)compression.
A BZ2File can act as a wrapper for an existing file object, or refer
directly to a named file on disk.
Note that BZ2File provides a *binary* file interface - data read is
returned as bytes, and data to be written should be given as bytes.
"""
def __init__(self, filename, mode="r", *, compresslevel=9):
"""Open a bzip2-compressed file.
If filename is a str, bytes, or PathLike object, it gives the
name of the file to be opened. Otherwise, it should be a file
object, which will be used to read or write the compressed data.
mode can be 'r' for reading (default), 'w' for (over)writing,
'x' for creating exclusively, or 'a' for appending. These can
equivalently be given as 'rb', 'wb', 'xb', and 'ab'.
If mode is 'w', 'x' or 'a', compresslevel can be a number between 1
and 9 specifying the level of compression: 1 produces the least
compression, and 9 (default) produces the most compression.
If mode is 'r', the input file may be the concatenation of
multiple compressed streams.
"""
self._fp = None
self._closefp = False
self._mode = _MODE_CLOSED
if not (1 <= compresslevel <= 9):
raise ValueError("compresslevel must be between 1 and 9")
if mode in ("", "r", "rb"):
mode = "rb"
mode_code = _MODE_READ
elif mode in ("w", "wb"):
mode = "wb"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
elif mode in ("x", "xb"):
mode = "xb"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
elif mode in ("a", "ab"):
mode = "ab"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
else:
raise ValueError("Invalid mode: %r" % (mode,))
if isinstance(filename, (str, bytes, os.PathLike)):
self._fp = _builtin_open(filename, mode)
self._closefp = True
self._mode = mode_code
elif hasattr(filename, "read") or hasattr(filename, "write"):
self._fp = filename
self._mode = mode_code
else:
raise TypeError("filename must be a str, bytes, file or PathLike object")
if self._mode == _MODE_READ:
raw = _compression.DecompressReader(self._fp,
BZ2Decompressor, trailing_error=OSError)
self._buffer = io.BufferedReader(raw)
else:
self._pos = 0
def close(self):
"""Flush and close the file.
May be called more than once without error. Once the file is
closed, any other operation on it will raise a ValueError.
"""
if self._mode == _MODE_CLOSED:
return
try:
if self._mode == _MODE_READ:
self._buffer.close()
elif self._mode == _MODE_WRITE:
self._fp.write(self._compressor.flush())
self._compressor = None
finally:
try:
if self._closefp:
self._fp.close()
finally:
self._fp = None
self._closefp = False
self._mode = _MODE_CLOSED
self._buffer = None
@property
def closed(self):
"""True if this file is closed."""
return self._mode == _MODE_CLOSED
def fileno(self):
"""Return the file descriptor for the underlying file."""
self._check_not_closed()
return self._fp.fileno()
def seekable(self):
"""Return whether the file supports seeking."""
return self.readable() and self._buffer.seekable()
def readable(self):
"""Return whether the file was opened for reading."""
self._check_not_closed()
return self._mode == _MODE_READ
def writable(self):
"""Return whether the file was opened for writing."""
self._check_not_closed()
return self._mode == _MODE_WRITE
def peek(self, n=0):
"""Return buffered data without advancing the file position.
Always returns at least one byte of data, unless at EOF.
The exact number of bytes returned is unspecified.
"""
self._check_can_read()
# Relies on the undocumented fact that BufferedReader.peek()
# always returns at least one byte (except at EOF), independent
# of the value of n
return self._buffer.peek(n)
def read(self, size=-1):
"""Read up to size uncompressed bytes from the file.
If size is negative or omitted, read until EOF is reached.
Returns b'' if the file is already at EOF.
"""
self._check_can_read()
return self._buffer.read(size)
def read1(self, size=-1):
"""Read up to size uncompressed bytes, while trying to avoid
making multiple reads from the underlying stream. Reads up to a
buffer's worth of data if size is negative.
Returns b'' if the file is at EOF.
"""
self._check_can_read()
if size < 0:
size = io.DEFAULT_BUFFER_SIZE
return self._buffer.read1(size)
def readinto(self, b):
"""Read bytes into b.
Returns the number of bytes read (0 for EOF).
"""
self._check_can_read()
return self._buffer.readinto(b)
def readline(self, size=-1):
"""Read a line of uncompressed bytes from the file.
The terminating newline (if present) is retained. If size is
non-negative, no more than size bytes will be read (in which
case the line may be incomplete). Returns b'' if already at EOF.
"""
if not isinstance(size, int):
if not hasattr(size, "__index__"):
raise TypeError("Integer argument expected")
size = size.__index__()
self._check_can_read()
return self._buffer.readline(size)
def readlines(self, size=-1):
"""Read a list of lines of uncompressed bytes from the file.
size can be specified to control the number of lines read: no
further lines will be read once the total size of the lines read
so far equals or exceeds size.
"""
if not isinstance(size, int):
if not hasattr(size, "__index__"):
raise TypeError("Integer argument expected")
size = size.__index__()
self._check_can_read()
return self._buffer.readlines(size)
def write(self, data):
"""Write a byte string to the file.
Returns the number of uncompressed bytes written, which is
always the length of data in bytes. Note that due to buffering,
the file on disk may not reflect the data written until close()
is called.
"""
self._check_can_write()
if isinstance(data, (bytes, bytearray)):
length = len(data)
else:
# accept any data that supports the buffer protocol
data = memoryview(data)
length = data.nbytes
compressed = self._compressor.compress(data)
self._fp.write(compressed)
self._pos += length
return length
def writelines(self, seq):
"""Write a sequence of byte strings to the file.
Returns the number of uncompressed bytes written.
seq can be any iterable yielding byte strings.
Line separators are not added between the written byte strings.
"""
return _compression.BaseStream.writelines(self, seq)
def seek(self, offset, whence=io.SEEK_SET):
"""Change the file position.
The new position is specified by offset, relative to the
position indicated by whence. Values for whence are:
0: start of stream (default); offset must not be negative
1: current stream position
2: end of stream; offset must not be positive
Returns the new file position.
Note that seeking is emulated, so depending on the parameters,
this operation may be extremely slow.
"""
self._check_can_seek()
return self._buffer.seek(offset, whence)
def tell(self):
"""Return the current file position."""
self._check_not_closed()
if self._mode == _MODE_READ:
return self._buffer.tell()
return self._pos
def open(filename, mode="rb", compresslevel=9,
encoding=None, errors=None, newline=None):
"""Open a bzip2-compressed file in binary or text mode.
The filename argument can be an actual filename (a str, bytes, or
PathLike object), or an existing file object to read from or write
to.
The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or
"ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode.
The default mode is "rb", and the default compresslevel is 9.
For binary mode, this function is equivalent to the BZ2File
constructor: BZ2File(filename, mode, compresslevel). In this case,
the encoding, errors and newline arguments must not be provided.
For text mode, a BZ2File object is created, and wrapped in an
io.TextIOWrapper instance with the specified encoding, error
handling behavior, and line ending(s).
"""
if "t" in mode:
if "b" in mode:
raise ValueError("Invalid mode: %r" % (mode,))
else:
if encoding is not None:
raise ValueError("Argument 'encoding' not supported in binary mode")
if errors is not None:
raise ValueError("Argument 'errors' not supported in binary mode")
if newline is not None:
raise ValueError("Argument 'newline' not supported in binary mode")
bz_mode = mode.replace("t", "")
binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel)
if "t" in mode:
encoding = io.text_encoding(encoding)
return io.TextIOWrapper(binary_file, encoding, errors, newline)
else:
return binary_file
def compress(data, compresslevel=9):
"""Compress a block of data.
compresslevel, if given, must be a number between 1 and 9.
For incremental compression, use a BZ2Compressor object instead.
"""
comp = BZ2Compressor(compresslevel)
return comp.compress(data) + comp.flush()
def decompress(data):
"""Decompress a block of data.
For incremental decompression, use a BZ2Decompressor object instead.
"""
results = []
while data:
decomp = BZ2Decompressor()
try:
res = decomp.decompress(data)
except OSError:
if results:
break # Leftover data is not a valid bzip2 stream; ignore it.
else:
raise # Error on the first iteration; bail out.
results.append(res)
if not decomp.eof:
raise ValueError("Compressed data ended before the "
"end-of-stream marker was reached")
data = decomp.unused_data
return b"".join(results)

247
Lib/calendar.py vendored
View File

@@ -7,22 +7,15 @@ set the first day of the week (0=Monday, 6=Sunday)."""
import sys
import datetime
from enum import IntEnum, global_enum
import locale as _locale
from itertools import repeat
import warnings
__all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
"firstweekday", "isleap", "leapdays", "weekday", "monthrange",
"monthcalendar", "prmonth", "month", "prcal", "calendar",
"timegm", "month_name", "month_abbr", "day_name", "day_abbr",
"Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar",
"LocaleHTMLCalendar", "weekheader",
"Day", "Month", "JANUARY", "FEBRUARY", "MARCH",
"APRIL", "MAY", "JUNE", "JULY",
"AUGUST", "SEPTEMBER", "OCTOBER", "NOVEMBER", "DECEMBER",
"MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY",
"SATURDAY", "SUNDAY"]
"LocaleHTMLCalendar", "weekheader"]
# Exception raised for bad input (with string parameter for details)
error = ValueError
@@ -42,46 +35,9 @@ class IllegalWeekdayError(ValueError):
return "bad weekday number %r; must be 0 (Monday) to 6 (Sunday)" % self.weekday
def __getattr__(name):
if name in ('January', 'February'):
warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead",
DeprecationWarning, stacklevel=2)
if name == 'January':
return 1
else:
return 2
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
# Constants for months
@global_enum
class Month(IntEnum):
JANUARY = 1
FEBRUARY = 2
MARCH = 3
APRIL = 4
MAY = 5
JUNE = 6
JULY = 7
AUGUST = 8
SEPTEMBER = 9
OCTOBER = 10
NOVEMBER = 11
DECEMBER = 12
# Constants for days
@global_enum
class Day(IntEnum):
MONDAY = 0
TUESDAY = 1
WEDNESDAY = 2
THURSDAY = 3
FRIDAY = 4
SATURDAY = 5
SUNDAY = 6
# Constants for months referenced later
January = 1
February = 2
# Number of days per month (except for February in leap years)
mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
@@ -137,6 +93,9 @@ day_abbr = _localized_day('%a')
month_name = _localized_month('%B')
month_abbr = _localized_month('%b')
# Constants for weekdays
(MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY) = range(7)
def isleap(year):
"""Return True for leap years, False for non-leap years."""
@@ -152,10 +111,9 @@ def leapdays(y1, y2):
def weekday(year, month, day):
"""Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31)."""
if not datetime.MINYEAR <= year <= datetime.MAXYEAR:
year = 2000 + year % 400
return Day(datetime.date(year, month, day).weekday())
"""Return weekday (0-6 ~ Mon-Sun) for year (1970-...), month (1-12),
day (1-31)."""
return datetime.date(year, month, day).weekday()
def monthrange(year, month):
@@ -164,28 +122,10 @@ def monthrange(year, month):
if not 1 <= month <= 12:
raise IllegalMonthError(month)
day1 = weekday(year, month, 1)
ndays = mdays[month] + (month == FEBRUARY and isleap(year))
ndays = mdays[month] + (month == February and isleap(year))
return day1, ndays
def _monthlen(year, month):
return mdays[month] + (month == FEBRUARY and isleap(year))
def _prevmonth(year, month):
if month == 1:
return year-1, 12
else:
return year, month-1
def _nextmonth(year, month):
if month == 12:
return year+1, 1
else:
return year, month+1
class Calendar(object):
"""
Base calendar class. This class doesn't do any formatting. It simply
@@ -217,8 +157,28 @@ class Calendar(object):
values and will always iterate through complete weeks, so it will yield
dates outside the specified month.
"""
for y, m, d in self.itermonthdays3(year, month):
yield datetime.date(y, m, d)
date = datetime.date(year, month, 1)
# Go back to the beginning of the week
days = (date.weekday() - self.firstweekday) % 7
date -= datetime.timedelta(days=days)
oneday = datetime.timedelta(days=1)
while True:
yield date
try:
date += oneday
except OverflowError:
# Adding one day could fail after datetime.MAXYEAR
break
if date.month != month and date.weekday() == self.firstweekday:
break
def itermonthdays2(self, year, month):
"""
Like itermonthdates(), but will yield (day number, weekday number)
tuples. For days outside the specified month the day number is 0.
"""
for i, d in enumerate(self.itermonthdays(year, month), self.firstweekday):
yield d, i % 7
def itermonthdays(self, year, month):
"""
@@ -232,40 +192,6 @@ class Calendar(object):
days_after = (self.firstweekday - day1 - ndays) % 7
yield from repeat(0, days_after)
def itermonthdays2(self, year, month):
"""
Like itermonthdates(), but will yield (day number, weekday number)
tuples. For days outside the specified month the day number is 0.
"""
for i, d in enumerate(self.itermonthdays(year, month), self.firstweekday):
yield d, i % 7
def itermonthdays3(self, year, month):
"""
Like itermonthdates(), but will yield (year, month, day) tuples. Can be
used for dates outside of datetime.date range.
"""
day1, ndays = monthrange(year, month)
days_before = (day1 - self.firstweekday) % 7
days_after = (self.firstweekday - day1 - ndays) % 7
y, m = _prevmonth(year, month)
end = _monthlen(y, m) + 1
for d in range(end-days_before, end):
yield y, m, d
for d in range(1, ndays + 1):
yield year, month, d
y, m = _nextmonth(year, month)
for d in range(1, days_after + 1):
yield y, m, d
def itermonthdays4(self, year, month):
"""
Like itermonthdates(), but will yield (year, month, day, day_of_week) tuples.
Can be used for dates outside of datetime.date range.
"""
for i, (y, m, d) in enumerate(self.itermonthdays3(year, month)):
yield y, m, d, (self.firstweekday + i) % 7
def monthdatescalendar(self, year, month):
"""
Return a matrix (list of lists) representing a month's calendar.
@@ -299,7 +225,10 @@ class Calendar(object):
Each month contains between 4 and 6 weeks and each week contains 1-7
days. Days are datetime.date objects.
"""
months = [self.monthdatescalendar(year, m) for m in Month]
months = [
self.monthdatescalendar(year, i)
for i in range(January, January+12)
]
return [months[i:i+width] for i in range(0, len(months), width) ]
def yeardays2calendar(self, year, width=3):
@@ -309,7 +238,10 @@ class Calendar(object):
(day number, weekday number) tuples. Day numbers outside this month are
zero.
"""
months = [self.monthdays2calendar(year, m) for m in Month]
months = [
self.monthdays2calendar(year, i)
for i in range(January, January+12)
]
return [months[i:i+width] for i in range(0, len(months), width) ]
def yeardayscalendar(self, year, width=3):
@@ -318,7 +250,10 @@ class Calendar(object):
yeardatescalendar()). Entries in the week lists are day numbers.
Day numbers outside this month are zero.
"""
months = [self.monthdayscalendar(year, m) for m in Month]
months = [
self.monthdayscalendar(year, i)
for i in range(January, January+12)
]
return [months[i:i+width] for i in range(0, len(months), width) ]
@@ -332,7 +267,7 @@ class TextCalendar(Calendar):
"""
Print a single week (no newline).
"""
print(self.formatweek(theweek, width), end='')
print(self.formatweek(theweek, width), end=' ')
def formatday(self, day, weekday, width):
"""
@@ -436,7 +371,7 @@ class TextCalendar(Calendar):
def pryear(self, theyear, w=0, l=0, c=6, m=3):
"""Print a year's calendar."""
print(self.formatyear(theyear, w, l, c, m), end='')
print(self.formatyear(theyear, w, l, c, m))
class HTMLCalendar(Calendar):
@@ -447,31 +382,12 @@ class HTMLCalendar(Calendar):
# CSS classes for the day <td>s
cssclasses = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"]
# CSS classes for the day <th>s
cssclasses_weekday_head = cssclasses
# CSS class for the days before and after current month
cssclass_noday = "noday"
# CSS class for the month's head
cssclass_month_head = "month"
# CSS class for the month
cssclass_month = "month"
# CSS class for the year's table head
cssclass_year_head = "year"
# CSS class for the whole year table
cssclass_year = "year"
def formatday(self, day, weekday):
"""
Return a day as a table cell.
"""
if day == 0:
# day outside month
return '<td class="%s">&nbsp;</td>' % self.cssclass_noday
return '<td class="noday">&nbsp;</td>' # day outside month
else:
return '<td class="%s">%d</td>' % (self.cssclasses[weekday], day)
@@ -486,8 +402,7 @@ class HTMLCalendar(Calendar):
"""
Return a weekday name as a table header.
"""
return '<th class="%s">%s</th>' % (
self.cssclasses_weekday_head[day], day_abbr[day])
return '<th class="%s">%s</th>' % (self.cssclasses[day], day_abbr[day])
def formatweekheader(self):
"""
@@ -504,8 +419,7 @@ class HTMLCalendar(Calendar):
s = '%s %s' % (month_name[themonth], theyear)
else:
s = '%s' % month_name[themonth]
return '<tr><th colspan="7" class="%s">%s</th></tr>' % (
self.cssclass_month_head, s)
return '<tr><th colspan="7" class="month">%s</th></tr>' % s
def formatmonth(self, theyear, themonth, withyear=True):
"""
@@ -513,8 +427,7 @@ class HTMLCalendar(Calendar):
"""
v = []
a = v.append
a('<table border="0" cellpadding="0" cellspacing="0" class="%s">' % (
self.cssclass_month))
a('<table border="0" cellpadding="0" cellspacing="0" class="month">')
a('\n')
a(self.formatmonthname(theyear, themonth, withyear=withyear))
a('\n')
@@ -534,12 +447,10 @@ class HTMLCalendar(Calendar):
v = []
a = v.append
width = max(width, 1)
a('<table border="0" cellpadding="0" cellspacing="0" class="%s">' %
self.cssclass_year)
a('<table border="0" cellpadding="0" cellspacing="0" class="year">')
a('\n')
a('<tr><th colspan="%d" class="%s">%s</th></tr>' % (
width, self.cssclass_year_head, theyear))
for i in range(JANUARY, JANUARY+12, width):
a('<tr><th colspan="%d" class="year">%s</th></tr>' % (width, theyear))
for i in range(January, January+12, width):
# months in this row
months = range(i, min(i+width, 13))
a('<tr>')
@@ -578,67 +489,71 @@ class HTMLCalendar(Calendar):
class different_locale:
def __init__(self, locale):
self.locale = locale
self.oldlocale = None
def __enter__(self):
self.oldlocale = _locale.setlocale(_locale.LC_TIME, None)
self.oldlocale = _locale.getlocale(_locale.LC_TIME)
_locale.setlocale(_locale.LC_TIME, self.locale)
def __exit__(self, *args):
if self.oldlocale is None:
return
_locale.setlocale(_locale.LC_TIME, self.oldlocale)
def _get_default_locale():
locale = _locale.setlocale(_locale.LC_TIME, None)
if locale == "C":
with different_locale(""):
# The LC_TIME locale does not seem to be configured:
# get the user preferred locale.
locale = _locale.setlocale(_locale.LC_TIME, None)
return locale
class LocaleTextCalendar(TextCalendar):
"""
This class can be passed a locale name in the constructor and will return
month and weekday names in the specified locale.
month and weekday names in the specified locale. If this locale includes
an encoding all strings containing month and weekday names will be returned
as unicode.
"""
def __init__(self, firstweekday=0, locale=None):
TextCalendar.__init__(self, firstweekday)
if locale is None:
locale = _get_default_locale()
locale = _locale.getdefaultlocale()
self.locale = locale
def formatweekday(self, day, width):
with different_locale(self.locale):
return super().formatweekday(day, width)
if width >= 9:
names = day_name
else:
names = day_abbr
name = names[day]
return name[:width].center(width)
def formatmonthname(self, theyear, themonth, width, withyear=True):
with different_locale(self.locale):
return super().formatmonthname(theyear, themonth, width, withyear)
s = month_name[themonth]
if withyear:
s = "%s %r" % (s, theyear)
return s.center(width)
class LocaleHTMLCalendar(HTMLCalendar):
"""
This class can be passed a locale name in the constructor and will return
month and weekday names in the specified locale.
month and weekday names in the specified locale. If this locale includes
an encoding all strings containing month and weekday names will be returned
as unicode.
"""
def __init__(self, firstweekday=0, locale=None):
HTMLCalendar.__init__(self, firstweekday)
if locale is None:
locale = _get_default_locale()
locale = _locale.getdefaultlocale()
self.locale = locale
def formatweekday(self, day):
with different_locale(self.locale):
return super().formatweekday(day)
s = day_abbr[day]
return '<th class="%s">%s</th>' % (self.cssclasses[day], s)
def formatmonthname(self, theyear, themonth, withyear=True):
with different_locale(self.locale):
return super().formatmonthname(theyear, themonth, withyear)
s = month_name[themonth]
if withyear:
s = '%s %s' % (s, theyear)
return '<tr><th colspan="7" class="month">%s</th></tr>' % s
# Support for old module level interface
c = TextCalendar()
@@ -723,7 +638,7 @@ def main(args):
parser.add_argument(
"-L", "--locale",
default=None,
help="locale to use for month and weekday names"
help="locale to be used from month and weekday names"
)
parser.add_argument(
"-e", "--encoding",

1012
Lib/cgi.py vendored
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More