change ci

* Remove uu.py and test_uu.py

* patch email.message

.cspell.json

@@ -52,6 +52,7 @@
     "metas",
     "modpow",
     "nanos",
+    "objclass",
     "peekable",
     "powc",
     "powf",

.devcontainer/devcontainer.json

@@ -1,6 +1,4 @@
 {
-  "image": "mcr.microsoft.com/devcontainers/universal:2",
+  "image": "mcr.microsoft.com/devcontainers/base:jammy",
-  "features": {
+  "onCreateCommand": "curl https://sh.rustup.rs -sSf | sh -s -- -y"
-    "ghcr.io/devcontainers/features/rust:1": {}
+}
-  }
-}

.gitattributes

@@ -4,3 +4,4 @@ Cargo.lock  linguist-generated -merge
 vm/src/stdlib/ast/gen.rs    linguist-generated -merge
 Lib/*.py    text working-tree-encoding=UTF-8 eol=LF
 **/*.rs     text working-tree-encoding=UTF-8 eol=LF
+*.pck       binary

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Keep GitHub Actions up to date with GitHub's Dependabot...
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: weekly

.github/scripts/code_review.py

@@ -0,0 +1,304 @@
+"""Code Reviewer for Gitea."""
+
+import asyncio
+import fnmatch
+import json
+import os
+import re
+from typing import Any
+
+import requests
+import aiohttp
+from model import Model
+
+ACCESS_TOKEN = os.getenv("ACCESS_TOKEN", "")
+HEADERS = {"Authorization": f"token {ACCESS_TOKEN}"}
+
+GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH")
+try:
+    with open(GITHUB_EVENT_PATH, "r") as f:
+        EVENT_DATA = json.load(f)
+except FileNotFoundError:
+    print("Failed to load event data.")
+    exit(1)
+
+FULL_CONTEXT_MODEL_NAME = os.getenv("FULL_CONTEXT_MODEL", "")
+SINGLE_CHUNK_MODEL_NAME = os.getenv("SINGLE_CHUNK_MODEL", "")
+FULL_CONTEXT_API_KEY = os.getenv("FULL_CONTEXT_API_KEY", "")
+SINGLE_CHUNK_API_KEY = os.getenv("SINGLE_CHUNK_API_KEY", "")
+
+EXCLUDE_PATTERNS = os.getenv("EXCLUDE", "").split(",")
+
+
+def get_diff() -> str | None:
+    """Get code difference between base and head from Gitea.
+
+    Returns:
+        str | None: code difference between base and head, or None if failed to get diff
+    """
+    url = EVENT_DATA["pull_request"]["diff_url"]
+    try:
+        response = requests.get(url, headers=HEADERS)
+        response.raise_for_status()
+        return response.text
+    except requests.RequestException as e:
+        print(f"Failed to get diff: {e}")
+        return None
+
+
+def parse_diff(diff: str) -> list[dict[str, Any]]:
+    """Parse diff into list of dicts.
+
+    Args:
+        diff: str, code difference between base and head
+
+    Returns:
+        list[dict[str, Any]]: list of dicts, each dict represents a code chunks
+    """
+    file_pattern = re.compile(
+        r"(?s)diff --git a/(.+?) b/(.*?)\r?\n(.*?)(?=diff --git a/|$)", re.S
+    )
+    old_new_pattern = re.compile(r"(?m)^(---|\+\+\+)\s+(.*)$")
+    chunk_range_pattern = re.compile(
+        r"@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*?)?(?=@@|\Z)",
+        re.MULTILINE | re.DOTALL,
+    )
+    list_diff = []
+    for match in file_pattern.finditer(diff):
+        diff_text = match.group(3)
+
+        old_new_match = list(old_new_pattern.finditer(diff_text))
+        if len(old_new_match) != 2:
+            continue
+
+        old_file = old_new_match[0].group(2)
+        old_file = old_file.lstrip("a/") if old_file.startswith("a/") else old_file
+
+        new_file = old_new_match[1].group(2)
+        if new_file == "/dev/null":
+            print("Neglict deleted file")
+            continue
+        new_file = new_file.lstrip("b/")
+        if any(fnmatch.fnmatch(new_file, pattern) for pattern in EXCLUDE_PATTERNS):
+            print(f"Exclude file {new_file}")
+            continue
+
+        output_diff_text = []
+        for chunk_range_match in chunk_range_pattern.finditer(diff_text):
+            old_idx = int(chunk_range_match.group(1))
+            new_idx = int(chunk_range_match.group(3))
+            for line in chunk_range_match.group(5).splitlines():
+                if line.startswith("-"):
+                    output_diff_text.append(f"{old_idx} None {line}")
+                    old_idx += 1
+                elif line.startswith("+"):
+                    output_diff_text.append(f"None {new_idx} {line}")
+                    new_idx += 1
+                else:
+                    output_diff_text.append(f"{old_idx} {new_idx} {line}")
+                    old_idx += 1
+                    new_idx += 1
+
+        output_diff_text = "\n".join(output_diff_text)
+        list_diff.append(
+            {
+                "file": new_file,
+                "chunk": output_diff_text,
+            }
+        )
+    return list_diff
+
+
+def create_comment(
+    file: str, ai_response: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    """Create comments for single chunk review.
+
+    Args:
+        file: str, file name
+        ai_response: list[dict[str, Any]], AI response for single chunk review
+
+    Returns:
+        list[dict[str, Any]]: comments for single chunk review
+    """
+    comments = []
+    for ai_response in ai_response:
+        comments.append(
+            {
+                "body": f"[REVIEW] {ai_response['reviewComment']}",
+                "path": file,
+                "new_position": int(ai_response["lineNumber"]),
+            }
+        )
+    return comments
+
+
+async def analyze_single_chunks(
+    single_chunk_model: Model, parsed_diff: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    """Analyze single chunks and create comments.
+
+    Args:
+        single_chunk_model: AI Session for single chunk analysis
+        parsed_diff: list[dict[str, Any]], parsed diff
+
+    Returns:
+        list[dict[str, Any]]: comments for single chunk review
+    """
+
+    async def process_single_chunk(diff: dict[str, Any]):
+        file = diff["file"]
+        chunk = diff["chunk"]
+        response = await single_chunk_model.get_response_single_chunk(
+            file, title, description, chunk
+        )
+        response = response.strip("`").lstrip("json").strip() or "[]"
+
+        try:
+            response_json = json.loads(response)
+            return create_comment(file, response_json)
+        except json.JSONDecodeError:
+            print(f"Failed to parse response: {response}")
+            return []
+
+    title = EVENT_DATA["pull_request"]["title"]
+    description = EVENT_DATA["pull_request"]["body"]
+    tasks = [process_single_chunk(diff) for diff in parsed_diff]
+    results = await asyncio.gather(*tasks)
+
+    # Flatten the list of comments
+    comments = [comment for result in results for comment in result]
+    return comments
+
+
+async def get_file_content(file: str) -> str | None:
+    """Get file content from Gitea.
+
+    Args:
+        file: str, file name
+
+    Returns:
+        str | None: file content, or None if failed to get file content
+    """
+    repo_url = EVENT_DATA["pull_request"]["head"]["repo"]["url"]
+    branch = EVENT_DATA["pull_request"]["head"]["ref"]
+
+    replaced_file = file.replace("/", "%2F")
+    url = f"{repo_url}/raw/{branch}%2F{replaced_file}?ref={branch}"
+
+    try:
+        async with aiohttp.ClientSession(headers=HEADERS) as session:
+            async with session.get(url) as response:
+                response.raise_for_status()
+                return await response.text()
+    except aiohttp.ClientError as e:  # More specific exception handling
+        print(f"Network error fetching {file}: {e}")
+    except asyncio.TimeoutError:
+        print(f"Timeout fetching {file}")
+    return None
+
+
+async def analyze_full_context(
+    full_context_model: Model, parsed_diff: list[dict[str, Any]]
+) -> str:
+    """Analyze full context and create review.
+
+    Args:
+        full_context_model: AI Session for full context analysis
+        parsed_diff: list[dict[str, Any]], parsed diff
+
+    Returns:
+        str: review for full context
+    """
+
+    async def get_file_data(diff: dict[str, Any]):
+        file = diff["file"]
+        chunk = diff["chunk"]
+        content = get_file_content(file)
+        if content is None:
+            return None
+        return f"File: {file}\n{content}\nDiff: {chunk}"
+
+    tasks = [get_file_data(diff) for diff in parsed_diff]
+    file_contents_list = await asyncio.gather(*tasks)
+
+    file_contents = [item for item in file_contents_list if item is not None]
+
+    if not file_contents:
+        return ""
+
+    title = EVENT_DATA["pull_request"]["title"]
+    description = EVENT_DATA["pull_request"]["body"]
+    response = await full_context_model.get_response_full_context(
+        title, description, file_contents
+    )
+    response = response.strip("`").lstrip("markdown").strip()
+    return response
+
+
+def post_review(
+    full_context_review: str, single_chunk_comments: list[dict[str, Any]]
+) -> None:
+    """Post review to Gitea.
+
+    Args:
+        full_context_review: str, review for full context
+        single_chunk_comments: list[dict[str, Any]], comments for single chunk review
+    """
+    repo_url = EVENT_DATA["pull_request"]["head"]["repo"]["url"]
+    pull_number = EVENT_DATA["number"]
+    commit_id = EVENT_DATA["pull_request"]["head"]["sha"]
+    url = f"{repo_url}/pulls/{pull_number}/reviews"
+    data = {
+        "body": full_context_review,
+        "event": "COMMENT",
+        "comments": single_chunk_comments,
+        "commit_id": commit_id,
+    }
+    response = requests.post(url, headers=HEADERS, json=data)
+    response.raise_for_status()
+
+
+async def main() -> None:
+    """Code Reviewer for Gitea: Asynchronous version."""
+    if EVENT_DATA["action"] not in ["opened", "synchronized"]:
+        print("Unsupported event.")
+        return
+
+    diff = get_diff()
+    if diff is None:
+        return
+    elif not diff:
+        print("No diff found.")
+        return
+
+    full_context_model = Model(
+        model=FULL_CONTEXT_MODEL_NAME,
+        api_key=FULL_CONTEXT_API_KEY,
+        is_full_context=True,
+    )
+    single_chunk_model = Model(
+        model=SINGLE_CHUNK_MODEL_NAME,
+        api_key=SINGLE_CHUNK_API_KEY,
+        is_full_context=False,
+    )
+
+    parsed_diff = parse_diff(diff)
+    comments_task = asyncio.create_task(
+        analyze_single_chunks(single_chunk_model, parsed_diff)
+    )
+
+    if EVENT_DATA["action"] == "opened":
+        full_context_response_task = asyncio.create_task(
+            analyze_full_context(full_context_model, parsed_diff)
+        )
+        full_context_response = await full_context_response_task
+    else:
+        full_context_response = ""
+
+    comments = await comments_task
+    post_review(full_context_response, comments)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

.github/scripts/model.py

@@ -0,0 +1,261 @@
+"""Model for code review."""
+
+from enum import Enum
+from typing import Any
+
+import google.generativeai as genai
+import typing_extensions as typing
+from anthropic import AsyncAnthropic
+from openai import AsyncOpenAI
+
+
+class GoogleResponse(typing.TypedDict):
+    """The response from Google model."""
+
+    lineNumber: int
+    reviewComment: str
+
+
+class ModelProvider(Enum):
+    """The model provider."""
+
+    OPENAI = "openai"
+    ANTHROPIC = "anthropic"
+    GOOGLE = "google"
+    DEEPSEEK = "deepseek"
+
+    @classmethod
+    def from_model(cls, model: str) -> "ModelProvider":
+        """Get the model provider from the model name.
+
+        Args:
+            model (str): The model name.
+
+        Returns:
+            ModelProvider: The model provider.
+        """
+        for prefix, provider in PREFIX_TO_MODEL.items():
+            if model.startswith(prefix):
+                return provider
+        raise ValueError(f"Unknown model: {model}")
+
+
+PREFIX_TO_MODEL = {
+    "gpt": ModelProvider.OPENAI,
+    "o1": ModelProvider.OPENAI,
+    "o3": ModelProvider.OPENAI,
+    "claude": ModelProvider.ANTHROPIC,
+    "gemini": ModelProvider.GOOGLE,
+    "deepseek": ModelProvider.DEEPSEEK,
+}
+
+
+class Model:
+    """The model class.
+
+    Attributes:
+        model (str): The model name.
+        api_key (str): The API key.
+        system_prompt (str): The system prompt.
+        max_tokens (int): The maximum tokens.
+    """
+
+    def __init__(  # noqa: D107
+        self,
+        model: str,
+        api_key: str,
+        is_full_context: bool,
+        max_tokens: int = 4196,
+    ):
+        self.model = model
+        self.system_prompt = (
+            FULL_CONTEXT_SYSTEM_PROMPT
+            if is_full_context
+            else SINGLE_CHUNK_SYSTEM_PROMPT
+        )
+        self.max_tokens = max_tokens
+        self.provider = ModelProvider.from_model(model)
+        self.session = self.create_session(api_key)
+
+    def create_session(self, api_key: str) -> Any:
+        """Create a session for the model.
+
+        Args:
+            api_key (str): The API key.
+
+        Returns:
+            Any: The session.
+        """
+        match self.provider:
+            case ModelProvider.OPENAI:
+                return AsyncOpenAI(api_key=api_key)
+            case ModelProvider.ANTHROPIC:
+                return AsyncAnthropic(api_key=api_key)
+            case ModelProvider.GOOGLE:
+                genai.configure(api_key=api_key)
+                return genai.GenerativeModel(
+                    model_name=self.model, system_instruction=self.system_prompt
+                )
+            case ModelProvider.DEEPSEEK:
+                return AsyncOpenAI(api_key=api_key, base_url="https://api.deepseek.com")
+
+    async def request(self, prompt: str) -> str:
+        """Request the model to generate a response.
+
+        Args:
+            prompt (str): The prompt to generate a response for.
+
+        Returns:
+            str: The generated response.
+        """
+        match self.provider:
+            case ModelProvider.OPENAI | ModelProvider.DEEPSEEK:
+                response = await self.session.chat.completions.create(
+                    model=self.model,
+                    messages=[
+                        {"role": "system", "content": self.system_prompt},
+                        {"role": "user", "content": prompt},
+                    ],
+                    temperature=0.2,
+                    max_tokens=self.max_tokens,
+                    top_p=1,
+                    frequency_penalty=0,
+                    presence_penalty=0,
+                )
+                return response.choices[0].message.content.strip()
+            case ModelProvider.ANTHROPIC:
+                response = await self.session.messages.create(
+                    model=self.model,
+                    messages=[{"role": "user", "content": prompt}],
+                    system=[
+                        {
+                            "type": "text",
+                            "text": self.system_prompt,
+                            "cache_control": {"type": "ephemeral"},
+                        }
+                    ],
+                    temperature=0.2,
+                    max_tokens=self.max_tokens,
+                )
+                return response.content[0].text.strip()
+            case ModelProvider.GOOGLE:
+                response = await self.session.generate_content_async(
+                    prompt,
+                    generation_config=genai.GenerationConfig(
+                        response_mime_type="application/json",
+                        response_schema=list[GoogleResponse],
+                    ),
+                )
+                return response.text.strip()
+
+    async def get_response_single_chunk(
+        self, file: str, title: str, description: str, chunk: str
+    ) -> str:
+        """Get the response for a single chunk.
+
+        Args:
+            file (str): The file name.
+            title (str): The pull request title.
+            description (str): The pull request description.
+            chunk (str): The diff chunk.
+
+        Returns:
+            str: The response.
+        """
+        prompt = SINGLE_CHUNK_USER_PROMPT.format(file, title, description, chunk)
+        return await self.request(prompt)
+
+    async def get_response_full_context(
+        self, title: str, description: str, file_contents: list[str]
+    ) -> str:
+        """Get the response for full context.
+
+        Args:
+            title (str): The pull request title.
+            description (str): The pull request description.
+            file_contents (list[str]): The file contents, diffs.
+
+        Returns:
+            str: The response.
+        """
+        try:
+            prompt = FULL_CONTEXT_USER_PROMPT.format(
+                title, description, "\n".join(file_contents)
+            )
+            return await self.request(prompt)
+        except Exception as e:
+            print(f"Error during full context response: {e}")
+            print(prompt)
+            return None
+
+
+SINGLE_CHUNK_SYSTEM_PROMPT = (
+    "Your task is to review pull requests. Instructions:\n"
+    "- Provide the response in the following JSON format:  "
+    """[{{"lineNumber": int, "reviewComment": str}}] \n"""
+    "- lineNumber is about the line number of the code that in new file. \n"
+    "- lineNumber can be found at the front of each line. \n"
+    "- At the first number is old line number, the second number is new line number. \n"
+    "- If the line starts with `+`, it means the line is added. \n"
+    "- If the line starts with `-`, it means the line is deleted. \n"
+    "- Evaluate whether the code changes and additions are appropriate "
+    "and if the new code structure is suitable. \n"
+    "- Do not give positive comments or compliments. \n"
+    "- Provide comments and suggestions ONLY if there is something to improve"
+    "otherwise return an empty array. \n"
+    "- Write the comment in GitHub Markdown format. \n"
+    "- Use the given description only for the overall context "
+    "and only comment the code. \n"
+    "- Do not suggest type hint or naming convention. \n"
+    "- IMPORTANT: NEVER suggest adding comments to the code. \n"
+)
+SINGLE_CHUNK_USER_PROMPT = (
+    "Review the following code diff in the file "
+    "{} and take the pull request title and description into account "
+    "when writing the response. \n"
+    "Pull request title: {} \n"
+    "Pull request description: \n"
+    "--- \n"
+    "{} \n"
+    "--- \n"
+    "Git diff to review: \n"
+    "```diff \n"
+    "{} \n"
+    "```"
+)
+
+FULL_CONTEXT_SYSTEM_PROMPT = (
+    "You are an experienced software engineer specializing in reviewing pull "
+    "requests. Your task is to provide an overall code review summary for a PR. "
+    "Focus on assessing the following aspects:\n"
+    "1. **Code Structure & Architecture:** "
+    "Evaluate whether the code is well-organized, modular, "
+    "and adheres to clean code principles. Suggest improvements if needed.\n"
+    "2. **Refactoring Opportunities:** "
+    "Identify areas where the code can be optimized or simplified without changing "
+    "its behavior.\n"
+    "3. **Potential Future Problems:** "
+    "Highlight possible scalability, maintainability, or dependency issues that might "
+    "arise in the future based on the current implementation.\n"
+    "Be constructive and clear in your feedback. Avoid commenting on trivial issues "
+    "or syntax errors—focus on high-level feedback.\n"
+    "Precise instructions:\n"
+    "- Do not give positive comments or compliments.\n"
+    "- Provide comments and suggestions ONLY if there is something to improve, "
+    "otherwise return an empty string.\n"
+    "- Write the comment in GitHub Markdown format.\n"
+    "- Do not start with 'markdown' or '```markdown'.\n"
+    "- IMPORTANT: Give example code block or pseudo code if you can.\n"
+)
+
+FULL_CONTEXT_USER_PROMPT = (
+    "Review the following code and take the pull request title "
+    "and description into account when writing the response. \n"
+    "Pull request title: {} \n"
+    "Pull request description: \n"
+    "--- \n"
+    "{} \n"
+    "--- \n"
+    "Code to review: \n"
+    "{}"
+)

.github/workflows/ci.yaml

@@ -1,40 +1,18 @@
 on:
-  push:
-    branches: [main, release]
   pull_request:
+    branches: [dev]
     types: [unlabeled, opened, synchronize, reopened]
   merge_group:
+  workflow_dispatch:
 
-name: CI
+name: Dev-CI
-
-# Cancel previous workflows if they are the same workflow on same ref (branch/tags)
-# with the same event (push/pull_request) even they are in progress.
-# This setting will help reduce the number of duplicated workflows.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
-  cancel-in-progress: true
 
 env:
-  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
+  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl
-  # Skip additional tests on Windows. They are checked on Linux and MacOS.
-  WINDOWS_SKIPS: >-
-    test_glob
-    test_importlib
-    test_io
-    test_os
-    test_pathlib
-    test_posixpath
-    test_shutil
-    test_venv
-  # configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417
-  # socketserver: seems related to configparser crash.
-  MACOS_SKIPS: >-
-    test_configparser
-    test_socketserver
   # PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently
   # only run on Linux to speed up the CI.
   PLATFORM_INDEPENDENT_TESTS: >-
-    test_argparse
+    test__colorize
     test_array
     test_asyncgen
     test_binop
@@ -59,7 +37,6 @@ env:
     test_dis
     test_enumerate
     test_exception_variations
-    test_exceptions
     test_float
     test_format
     test_fractions
@@ -100,12 +77,11 @@ env:
     test_tuple
     test_types
     test_unary
-    test_unicode
     test_unpack
     test_weakref
     test_yield_from
   # Python version targeted by the CI.
-  PYTHON_VERSION: "3.12.0"
+  PYTHON_VERSION: "3.13.1"
 
 jobs:
   rust_tests:
@@ -113,65 +89,34 @@ jobs:
     env:
       RUST_BACKTRACE: full
     name: Run rust tests
-    runs-on:  ${{ matrix.os }}
+    runs-on:  ubuntu-latest
-    strategy:
-      matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
-      fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
       - uses: Swatinem/rust-cache@v2
 
-      - name: Set up the Windows environment
-        shell: bash
-        run: |
-          cargo install --target-dir=target -v cargo-vcpkg
-          cargo vcpkg -v build
-        if: runner.os == 'Windows'
-      - name: Set up the Mac environment
-        run: brew install autoconf automake libtool
-        if: runner.os == 'macOS'
-
       - name: run clippy
         run: cargo clippy ${{ env.CARGO_ARGS }} --workspace  --exclude rustpython_wasm -- -Dwarnings
 
       - name: run rust tests
         run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }}
-        if: runner.os != 'macOS'  
+      
-      # temp skip ssl linking for Mac to avoid CI failure
-      - name: run rust tests (MacOS no ssl)
-        run: cargo test --workspace --exclude rustpython_wasm --verbose --no-default-features --features threading,stdlib,zlib,importlib,encodings,jit
-        if: runner.os == 'macOS'
-
       - name: check compilation without threading
         run: cargo check ${{ env.CARGO_ARGS }}
 
-      - name: prepare AppleSilicon build
+      - name: Test example projects
-        uses: dtolnay/rust-toolchain@stable
+        run:
-        with:
+          cargo run --manifest-path example_projects/barebone/Cargo.toml
-          target: aarch64-apple-darwin
+          cargo run --manifest-path example_projects/frozen_stdlib/Cargo.toml
-        if: runner.os == 'macOS'
-      - name: Check compilation for Apple Silicon
-        run: cargo check --target aarch64-apple-darwin
-        if: runner.os == 'macOS'
-      - name: prepare iOS build
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          target: aarch64-apple-ios
-        if: runner.os == 'macOS'
-      - name: Check compilation for iOS
-        run: cargo check --target aarch64-apple-ios
-        if: runner.os == 'macOS'
 
   exotic_targets:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     name: Ensure compilation on various targets
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: i686-unknown-linux-gnu
@@ -211,13 +156,6 @@ jobs:
       - name: Check compilation for freebsd
         run: cargo check --target x86_64-unknown-freebsd
 
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          target: wasm32-unknown-unknown
-
-      - name: Check compilation for wasm32
-        run: cargo check --target wasm32-unknown-unknown --no-default-features
-
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: x86_64-unknown-freebsd
@@ -231,68 +169,44 @@ jobs:
         uses: coolreader18/redoxer-action@v1
         with:
           command: check
+          args: --ignore-rust-version
 
   snippets_cpython:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     env:
       RUST_BACKTRACE: full
     name: Run snippets and cpython tests
-    runs-on:  ${{ matrix.os }}
+    runs-on:  ubuntu-latest
-    strategy:
-      matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
-      fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-      - name: Set up the Windows environment
-        shell: bash
-        run: |
-          cargo install cargo-vcpkg
-          cargo vcpkg build
-        if: runner.os == 'Windows'
-      - name: Set up the Mac environment
-        run: brew install autoconf automake libtool openssl@3
-        if: runner.os == 'macOS'
-
       - name: build rustpython
-        run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }}
+        run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: run snippets
         run: python -m pip install -r requirements.txt && pytest -v
         working-directory: ./extra_tests
-      - if: runner.os == 'Linux'
+      - name: run cpython platform-independent tests
-        name: run cpython platform-independent tests
         run:
           target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }}
-      - if: runner.os == 'Linux'
+      - name: run cpython platform-dependent tests (Linux)
-        name: run cpython platform-dependent tests (Linux)
         run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }}
-      - if: runner.os == 'macOS'
+      - name: check that --install-pip succeeds
-        name: run cpython platform-dependent tests (MacOS)
-        run: target/release/rustpython -m test -j 1 all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.MACOS_SKIPS }}
-      - if: runner.os == 'Windows'
-        name: run cpython platform-dependent tests (windows partial - fixme)
-        run:
-          target/release/rustpython -m test -j 1 all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }}
-      - if: runner.os != 'Windows'
-        name: check that --install-pip succeeds
         run: |
           mkdir site-packages
           target/release/rustpython --install-pip ensurepip --user
-      - if: runner.os != 'Windows'
+          target/release/rustpython -m pip install six
-        name: Check that ensurepip succeeds.
+      - name: Check that ensurepip succeeds.
         run: |
           target/release/rustpython -m ensurepip
           target/release/rustpython -c "import pip"
-      - if: runner.os != 'Windows'
+      - name: Check if pip inside venv is functional
-        name: Check if pip inside venv is functional
         run: |
           target/release/rustpython -m venv testvenv
           testvenv/bin/rustpython -m pip install wheel
@@ -303,7 +217,7 @@ jobs:
     name: Check Rust code with rustfmt and clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
             components: rustfmt, clippy
@@ -311,7 +225,7 @@ jobs:
         run: cargo fmt --check
       - name: run clippy on wasm
         run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: install ruff
@@ -329,7 +243,7 @@ jobs:
     name: Run tests under miri
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@master
         with:
             toolchain: nightly
@@ -341,70 +255,3 @@ jobs:
         # a memory leak, at least until we have proper cyclic gc
         run: MIRIFLAGS='-Zmiri-ignore-leaks' cargo +nightly miri test -p rustpython-vm -- miri_test
 
-  wasm:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    name: Check the WASM package and demo
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@stable
-
-      - uses: Swatinem/rust-cache@v2
-      - name: install wasm-pack
-        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
-      - name: install geckodriver
-        run: |
-          wget https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz
-          mkdir geckodriver
-          tar -xzf geckodriver-v0.30.0-linux64.tar.gz -C geckodriver
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-      - run: python -m pip install -r requirements.txt
-        working-directory: ./wasm/tests
-      - uses: actions/setup-node@v3
-      - name: run test
-        run: |
-          export PATH=$PATH:`pwd`/../../geckodriver
-          npm install
-          npm run test
-        env:
-          NODE_OPTIONS: "--openssl-legacy-provider"
-        working-directory: ./wasm/demo
-      - name: build notebook demo
-        if: github.ref == 'refs/heads/release'
-        run: |
-          npm install
-          npm run dist
-          mv dist ../demo/dist/notebook
-        env:
-          NODE_OPTIONS: "--openssl-legacy-provider"
-        working-directory: ./wasm/notebook
-      - name: Deploy demo to Github Pages
-        if: success() && github.ref == 'refs/heads/release'
-        uses: peaceiris/actions-gh-pages@v2
-        env:
-          ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
-          PUBLISH_DIR: ./wasm/demo/dist
-          EXTERNAL_REPOSITORY: RustPython/demo
-          PUBLISH_BRANCH: master
-
-  wasm-wasi:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    name: Run snippets and cpython tests on wasm-wasi
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          target: wasm32-wasi
-
-      - uses: Swatinem/rust-cache@v2
-      - name: Setup Wasmer
-        uses: wasmerio/setup-wasmer@v2
-      - name: Install clang
-        run: sudo apt-get update && sudo apt-get install clang -y
-      - name: build rustpython
-        run: cargo build --release --target wasm32-wasi --features freeze-stdlib,stdlib --verbose
-      - name: run snippets
-        run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py

.github/workflows/code-review.yml

@@ -0,0 +1,36 @@
+name: Code Review
+
+on:
+  pull_request:
+    branches: [dev]
+    types: [opened, synchronize]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  review:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install aiohttp requests py-gitea openai anthropic google-generativeai
+
+      - name: Run Code Review
+        env:
+          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+          FULL_CONTEXT_MODEL: o3-mini
+          FULL_CONTEXT_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          SINGLE_CHUNK_MODEL: gemini-2.0-flash-exp
+          SINGLE_CHUNK_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          EXCLUDE: "*.yml,*.yaml"
+        run: python .gitea/scripts/code_review.py
+

.github/workflows/cron-ci.yaml

@@ -2,12 +2,15 @@ on:
   schedule:
     - cron: '0 0 * * 6'
   workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/cron-ci.yaml
 
 name: Periodic checks/tasks
 
 env:
-  CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl,jit
-  PYTHON_VERSION: "3.12.0"
+  PYTHON_VERSION: "3.13.1"
 
 jobs:
   # codecov collects code coverage data from the rust tests, python snippets and python test suite.
@@ -16,15 +19,15 @@ jobs:
     name: Collect code coverage data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: taiki-e/install-action@cargo-llvm-cov
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - run: sudo apt-get update && sudo apt-get -y install lcov
       - name: Run cargo-llvm-cov with Rust tests.
-        run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
+        run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,importlib,encodings,ssl,jit
       - name: Run cargo-llvm-cov with Python snippets.
         run: python scripts/cargo-llvm-cov.py
         continue-on-error: true
@@ -34,7 +37,7 @@ jobs:
       - name: Prepare code coverage data
         run: cargo llvm-cov report --lcov --output-path='codecov.lcov'
       - name: Upload to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v5
         with:
           file: ./codecov.lcov
 
@@ -42,7 +45,7 @@ jobs:
     name: Collect regression test data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - name: build rustpython
         run: cargo build --release --verbose
@@ -71,9 +74,9 @@ jobs:
     name: Collect what is left data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: build rustpython
@@ -97,6 +100,9 @@ jobs:
           cd website
           [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp
           cp ../whats_left.temp ./_data/whats_left.temp
+          rm ./_data/whats_left/modules.csv
+          echo -e "module" > ./_data/whats_left/modules.csv
+          cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv
           git add -A
           if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then
             git push
@@ -106,9 +112,9 @@ jobs:
     name: Collect benchmark data
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - run: cargo install cargo-criterion

.github/workflows/release.yml

@@ -0,0 +1,171 @@
+name: Release
+
+on:
+  schedule:
+    # 9 AM UTC on every Monday
+    - cron: "0 9 * * Mon"
+  workflow_dispatch:
+    inputs:
+      pre-release:
+        type: boolean
+        description: Mark "Pre-Release"
+        required: false
+        default: true
+
+permissions:
+  contents: write
+
+env:
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl
+
+jobs:
+  build:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: i686-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: aarch64-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: armv7-unknown-linux-gnueabi
+#          - runner: ubuntu-latest
+#            target: s390x-unknown-linux-gnu
+#          - runner: ubuntu-latest
+#            target: powerpc64le-unknown-linux-gnu
+          - runner: macos-latest
+            target: aarch64-apple-darwin
+#          - runner: macos-latest
+#            target: x86_64-apple-darwin
+          - runner: windows-latest
+            target: x86_64-pc-windows-msvc
+#          - runner: windows-latest
+#            target: i686-pc-windows-msvc
+#          - runner: windows-latest
+#            target: aarch64-pc-windows-msvc
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Set up Environment
+        shell: bash
+        run: rustup target add ${{ matrix.platform.target }}
+      - name: Set up Windows Environment
+        shell: bash
+        run: |
+          cargo install --target-dir=target -v cargo-vcpkg
+          cargo vcpkg -v build
+        if: runner.os == 'Windows'
+      - name: Set up MacOS Environment
+        run: brew install autoconf automake libtool
+        if: runner.os == 'macOS'
+
+      - name: Build RustPython
+        run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }}
+        if: runner.os == 'macOS'
+      - name: Build RustPython
+        run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }},jit
+        if: runner.os != 'macOS'
+
+      - name: Rename Binary
+        run: cp target/${{ matrix.platform.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}
+        if: runner.os != 'Windows'
+      - name: Rename Binary
+        run: cp target/${{ matrix.platform.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}.exe
+        if: runner.os == 'Windows'
+
+      - name: Upload Binary Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}
+          path: target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}*
+
+  build-wasm:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: wasm32-wasip1
+
+      - name: Build RustPython
+        run: cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release
+
+      - name: Rename Binary
+        run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm
+
+      - name: Upload Binary Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rustpython-release-wasm32-wasip1
+          path: target/rustpython-release-wasm32-wasip1.wasm
+
+      - name: install wasm-pack
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+      - uses: actions/setup-node@v4
+      - uses: mwilliamson/setup-wabt-action@v3
+        with: { wabt-version: "1.0.30" }
+      - name: build demo
+        run: |
+          npm install
+          npm run dist
+        env:
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        working-directory: ./wasm/demo
+      - name: build notebook demo
+        run: |
+          npm install
+          npm run dist
+          mv dist ../demo/dist/notebook
+        env:
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        working-directory: ./wasm/notebook
+      - name: Deploy demo to Github Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
+          publish_dir: ./wasm/demo/dist
+          external_repository: RustPython/demo
+          publish_branch: master
+
+  release:
+    runs-on: ubuntu-latest
+    needs: [build, build-wasm]
+    steps:
+      - name: Download Binary Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: bin
+          pattern: rustpython-release-*
+          merge-multiple: true
+
+      - name: List Binaries
+        run: |
+          ls -lah bin/
+          file bin/*
+      - name: Create Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref_name }}
+          run: ${{ github.run_number }}
+        run: |
+          if [[ "${{ github.event.inputs.pre-release }}" == "false" ]]; then
+            RELEASE_TYPE_NAME=Release
+            PRERELEASE_ARG=
+          else
+            RELEASE_TYPE_NAME=Pre-Release
+            PRERELEASE_ARG=--prerelease
+          fi
+          
+          today=$(date '+%Y-%m-%d')
+          gh release create "$today-$tag-$run" \
+              --repo="$GITHUB_REPOSITORY" \
+              --title="RustPython $RELEASE_TYPE_NAME $today-$tag #$run" \
+              --target="$tag" \
+              --generate-notes \
+              $PRERELEASE_ARG \
+              bin/rustpython-release-*

.gitignore

@@ -9,6 +9,8 @@ __pycache__
 .vscode
 wasm-pack.log
 .idea/
+.envrc
+.python-version
 
 flame-graph.html
 flame.txt

Cargo.toml

@@ -1,108 +1,35 @@
 [package]
 name = "rustpython"
-version = "0.3.1"
-authors = ["RustPython Team"]
-edition = "2021"
-rust-version = "1.67.1"
 description = "A python interpreter written in rust."
-repository = "https://github.com/RustPython/RustPython"
-license = "MIT"
 include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
-
+version.workspace = true
-[workspace]
+authors.workspace = true
-resolver = "2"
+edition.workspace = true
-members = [
+rust-version.workspace = true
-    "compiler", "compiler/core", "compiler/codegen",
+repository.workspace = true
-    ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "wasm/lib", "derive-impl",
+license.workspace = true
-]
-
-[workspace.dependencies]
-rustpython-compiler-core = { path = "compiler/core", version = "0.3.1" }
-rustpython-compiler = { path = "compiler", version = "0.3.1" }
-rustpython-codegen = { path = "compiler/codegen", version = "0.3.1" }
-rustpython-common = { path = "common", version = "0.3.1" }
-rustpython-derive = { path = "derive", version = "0.3.1" }
-rustpython-derive-impl = { path = "derive-impl", version = "0.3.1" }
-rustpython-jit = { path = "jit", version = "0.3.1" }
-rustpython-vm = { path = "vm", default-features = false, version = "0.3.1" }
-rustpython-pylib = { path = "pylib", version = "0.3.1" }
-rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.3.1" }
-rustpython-sre_engine = { path = "vm/sre_engine", version = "0.3.1" }
-rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
-
-rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
-rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
-rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
-rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
-rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.3.1", rev = "a95045bc627b2fbf84caf4f010e521846be7b37f" }
-# rustpython-literal = { path = "../RustPython-parser/literal" }
-# rustpython-parser-core = { path = "../RustPython-parser/core" }
-# rustpython-parser = { path = "../RustPython-parser/parser" }
-# rustpython-ast = { path = "../RustPython-parser/ast" }
-# rustpython-format = { path = "../RustPython-parser/format" }
-
-ahash = "0.8.11"
-ascii = "1.0"
-atty = "0.2.14"
-bitflags = "2.4.1"
-bstr = "0.2.17"
-cfg-if = "1.0"
-chrono = "0.4.37"
-crossbeam-utils = "0.8.19"
-flame = "0.2.2"
-glob = "0.3"
-hex = "0.4.3"
-indexmap = { version = "2.2.6", features = ["std"] }
-insta = "1.38.0"
-itertools = "0.11.0"
-is-macro = "0.3.0"
-libc = "0.2.153"
-log = "0.4.16"
-nix = { version = "0.27", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
-malachite-bigint = "0.2.0"
-malachite-q = "0.4.4"
-malachite-base = "0.4.4"
-memchr = "2.7.2"
-num-complex = "0.4.0"
-num-integer = "0.1.44"
-num-traits = "0.2"
-num_enum = "0.7"
-once_cell = "1.19.0"
-parking_lot = "0.12.1"
-paste = "1.0.7"
-rand = "0.8.5"
-rustyline = "14.0.0"
-serde = { version = "1.0.133", default-features = false }
-schannel = "0.1.22"
-static_assertions = "1.1"
-syn = "1.0.109"
-thiserror = "1.0"
-thread_local = "1.1.4"
-unicode_names2 = "1.1.0"
-widestring = "1.1.0"
 
 [features]
-default = ["threading", "stdlib", "zlib", "importlib"]
+default = ["threading", "stdlib", "importlib"]
 importlib = ["rustpython-vm/importlib"]
 encodings = ["rustpython-vm/encodings"]
 stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"]
 flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
-freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
+freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
 threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
-zlib = ["stdlib", "rustpython-stdlib/zlib"]
 bz2 = ["stdlib", "rustpython-stdlib/bz2"]
+sqlite = ["rustpython-stdlib/sqlite"]
 ssl = ["rustpython-stdlib/ssl"]
-ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
+ssl-vendor = ["ssl", "rustpython-stdlib/ssl-vendor"]
 
 [dependencies]
 rustpython-compiler = { workspace = true }
 rustpython-pylib = { workspace = true, optional = true }
-rustpython-stdlib = { workspace = true, optional = true }
+rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
 rustpython-vm = { workspace = true, features = ["compiler"] }
 rustpython-parser = { workspace = true }
 
-atty = { workspace = true }
 cfg-if = { workspace = true }
 log = { workspace = true }
 flame = { workspace = true, optional = true }
@@ -119,8 +46,8 @@ libc = { workspace = true }
 rustyline = { workspace = true }
 
 [dev-dependencies]
-criterion = { version = "0.3.5", features = ["html_reports"] }
+criterion = { workspace = true }
-pyo3 = { version = "0.20.2", features = ["auto-initialize"] }
+pyo3 = { version = "0.22", features = ["auto-initialize"] }
 
 [[bench]]
 name = "execution"
@@ -163,3 +90,110 @@ rev = "2024.02.14"
 
 [package.metadata.vcpkg.target]
 x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] }
+
+[workspace]
+resolver = "2"
+members = [
+    "compiler", "compiler/core", "compiler/codegen",
+    ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl",
+    "wasm/lib",
+]
+
+[workspace.package]
+version = "0.4.0"
+authors = ["RustPython Team"]
+edition = "2024"
+rust-version = "1.85.0"
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+
+[workspace.dependencies]
+rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
+rustpython-compiler = { path = "compiler", version = "0.4.0" }
+rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }
+rustpython-common = { path = "common", version = "0.4.0" }
+rustpython-derive = { path = "derive", version = "0.4.0" }
+rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" }
+rustpython-jit = { path = "jit", version = "0.4.0" }
+rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" }
+rustpython-pylib = { path = "pylib", version = "0.4.0" }
+rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" }
+rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
+rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
+
+# rustpython-literal = { version = "0.4.0" }
+# rustpython-parser-core = { version = "0.4.0" }
+# rustpython-parser = { version = "0.4.0" }
+# rustpython-ast = { version = "0.4.0" }
+# rustpython-format= { version = "0.4.0" }
+rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
+rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
+rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
+rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
+rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
+# rustpython-literal = { path = "../RustPython-parser/literal" }
+# rustpython-parser-core = { path = "../RustPython-parser/core" }
+# rustpython-parser = { path = "../RustPython-parser/parser" }
+# rustpython-ast = { path = "../RustPython-parser/ast" }
+# rustpython-format = { path = "../RustPython-parser/format" }
+
+ahash = "0.8.11"
+ascii = "1.1"
+bitflags = "2.4.2"
+bstr = "1"
+cfg-if = "1.0"
+chrono = "0.4.39"
+criterion = { version = "0.3.5", features = ["html_reports"] }
+crossbeam-utils = "0.8.21"
+flame = "0.2.2"
+getrandom = "0.3"
+glob = "0.3"
+hex = "0.4.3"
+indexmap = { version = "2.2.6", features = ["std"] }
+insta = "1.38.0"
+itertools = "0.14.0"
+is-macro = "0.3.7"
+junction = "1.2.0"
+libc = "0.2.169"
+log = "0.4.25"
+nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
+malachite-bigint = "0.2.3"
+malachite-q = "0.4.22"
+malachite-base = "0.4.22"
+memchr = "2.7.4"
+num-complex = "0.4.6"
+num-integer = "0.1.46"
+num-traits = "0.2"
+num_enum = { version = "0.7", default-features = false }
+once_cell = "1.20.3"
+parking_lot = "0.12.3"
+paste = "1.0.15"
+rand = "0.9"
+rustix = { version = "0.38", features = ["event"] }
+rustyline = "15.0.0"
+serde = { version = "1.0.133", default-features = false }
+schannel = "0.1.27"
+static_assertions = "1.1"
+strum = "0.27"
+strum_macros = "0.27"
+syn = "2"
+thiserror = "2.0"
+thread_local = "1.1.8"
+unicode_names2 = "1.3.0"
+widestring = "1.1.0"
+windows-sys = "0.59.0"
+wasm-bindgen = "0.2.100"
+
+# Lints
+
+[workspace.lints.rust]
+unsafe_code = "allow"
+unsafe_op_in_unsafe_fn = "deny"
+elided_lifetimes_in_paths = "warn"
+
+[workspace.lints.clippy]
+perf = "warn"
+style = "warn"
+complexity = "warn"
+suspicious = "warn"
+correctness = "warn"

DEVELOPMENT.md

@@ -25,7 +25,7 @@ RustPython requires the following:
       stable version: `rustup update stable`
     - If you do not have Rust installed, use [rustup](https://rustup.rs/) to
       do so.
-- CPython version 3.12 or higher
+- CPython version 3.13 or higher
     - CPython can be installed by your operating system's package manager,
       from the [Python website](https://www.python.org/downloads/), or
       using a third-party distribution, such as 

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 RustPython Team
+Copyright (c) 2025 RustPython Team
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Lib/__future__.py

@@ -33,7 +33,7 @@ in releases at or after that, modules no longer need
 to use the feature in question, but may continue to use such imports.
 
 MandatoryRelease may also be None, meaning that a planned feature got
-dropped.
+dropped or that the release version is undetermined.
 
 Instances of class _Feature have two corresponding methods,
 .getOptionalRelease() and .getMandatoryRelease().
@@ -96,7 +96,7 @@ class _Feature:
         """Return release in which this feature will become mandatory.
 
         This is a 5-tuple, of the same form as sys.version_info, or, if
-        the feature was dropped, is None.
+        the feature was dropped, or the release date is undetermined, is None.
         """
         return self.mandatory
 
@@ -143,5 +143,5 @@ generator_stop = _Feature((3, 5, 0, "beta", 1),
                           CO_FUTURE_GENERATOR_STOP)
 
 annotations = _Feature((3, 7, 0, "beta", 1),
-                       (3, 11, 0, "alpha", 0),
+                       None,
                        CO_FUTURE_ANNOTATIONS)

Lib/_android_support.py

@@ -0,0 +1,181 @@
+import io
+import sys
+from threading import RLock
+from time import sleep, time
+
+# The maximum length of a log message in bytes, including the level marker and
+# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71.
+# Messages longer than this will be truncated by logcat. This limit has already
+# been reduced at least once in the history of Android (from 4076 to 4068 between
+# API level 23 and 26), so leave some headroom.
+MAX_BYTES_PER_WRITE = 4000
+
+# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this
+# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE.
+# However, if the actual number of bytes per character is smaller than that,
+# then we may still join multiple consecutive text writes into binary
+# writes containing a larger number of characters.
+MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4
+
+
+# When embedded in an app on current versions of Android, there's no easy way to
+# monitor the C-level stdout and stderr. The testbed comes with a .c file to
+# redirect them to the system log using a pipe, but that wouldn't be convenient
+# or appropriate for all apps. So we redirect at the Python level instead.
+def init_streams(android_log_write, stdout_prio, stderr_prio):
+    if sys.executable:
+        return  # Not embedded in an app.
+
+    global logcat
+    logcat = Logcat(android_log_write)
+
+    sys.stdout = TextLogStream(
+        stdout_prio, "python.stdout", sys.stdout.fileno())
+    sys.stderr = TextLogStream(
+        stderr_prio, "python.stderr", sys.stderr.fileno())
+
+
+class TextLogStream(io.TextIOWrapper):
+    def __init__(self, prio, tag, fileno=None, **kwargs):
+        # The default is surrogateescape for stdout and backslashreplace for
+        # stderr, but in the context of an Android log, readability is more
+        # important than reversibility.
+        kwargs.setdefault("encoding", "UTF-8")
+        kwargs.setdefault("errors", "backslashreplace")
+
+        super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs)
+        self._lock = RLock()
+        self._pending_bytes = []
+        self._pending_bytes_count = 0
+
+    def __repr__(self):
+        return f"<TextLogStream {self.buffer.tag!r}>"
+
+    def write(self, s):
+        if not isinstance(s, str):
+            raise TypeError(
+                f"write() argument must be str, not {type(s).__name__}")
+
+        # In case `s` is a str subclass that writes itself to stdout or stderr
+        # when we call its methods, convert it to an actual str.
+        s = str.__str__(s)
+
+        # We want to emit one log message per line wherever possible, so split
+        # the string into lines first. Note that "".splitlines() == [], so
+        # nothing will be logged for an empty string.
+        with self._lock:
+            for line in s.splitlines(keepends=True):
+                while line:
+                    chunk = line[:MAX_CHARS_PER_WRITE]
+                    line = line[MAX_CHARS_PER_WRITE:]
+                    self._write_chunk(chunk)
+
+        return len(s)
+
+    # The size and behavior of TextIOWrapper's buffer is not part of its public
+    # API, so we handle buffering ourselves to avoid truncation.
+    def _write_chunk(self, s):
+        b = s.encode(self.encoding, self.errors)
+        if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE:
+            self.flush()
+
+        self._pending_bytes.append(b)
+        self._pending_bytes_count += len(b)
+        if (
+            self.write_through
+            or b.endswith(b"\n")
+            or self._pending_bytes_count > MAX_BYTES_PER_WRITE
+        ):
+            self.flush()
+
+    def flush(self):
+        with self._lock:
+            self.buffer.write(b"".join(self._pending_bytes))
+            self._pending_bytes.clear()
+            self._pending_bytes_count = 0
+
+    # Since this is a line-based logging system, line buffering cannot be turned
+    # off, i.e. a newline always causes a flush.
+    @property
+    def line_buffering(self):
+        return True
+
+
+class BinaryLogStream(io.RawIOBase):
+    def __init__(self, prio, tag, fileno=None):
+        self.prio = prio
+        self.tag = tag
+        self._fileno = fileno
+
+    def __repr__(self):
+        return f"<BinaryLogStream {self.tag!r}>"
+
+    def writable(self):
+        return True
+
+    def write(self, b):
+        if type(b) is not bytes:
+            try:
+                b = bytes(memoryview(b))
+            except TypeError:
+                raise TypeError(
+                    f"write() argument must be bytes-like, not {type(b).__name__}"
+                ) from None
+
+        # Writing an empty string to the stream should have no effect.
+        if b:
+            logcat.write(self.prio, self.tag, b)
+        return len(b)
+
+    # This is needed by the test suite --timeout option, which uses faulthandler.
+    def fileno(self):
+        if self._fileno is None:
+            raise io.UnsupportedOperation("fileno")
+        return self._fileno
+
+
+# When a large volume of data is written to logcat at once, e.g. when a test
+# module fails in --verbose3 mode, there's a risk of overflowing logcat's own
+# buffer and losing messages. We avoid this by imposing a rate limit using the
+# token bucket algorithm, based on a conservative estimate of how fast `adb
+# logcat` can consume data.
+MAX_BYTES_PER_SECOND = 1024 * 1024
+
+# The logcat buffer size of a device can be determined by running `logcat -g`.
+# We set the token bucket size to half of the buffer size of our current minimum
+# API level, because other things on the system will be producing messages as
+# well.
+BUCKET_SIZE = 128 * 1024
+
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39
+PER_MESSAGE_OVERHEAD = 28
+
+
+class Logcat:
+    def __init__(self, android_log_write):
+        self.android_log_write = android_log_write
+        self._lock = RLock()
+        self._bucket_level = 0
+        self._prev_write_time = time()
+
+    def write(self, prio, tag, message):
+        # Encode null bytes using "modified UTF-8" to avoid them truncating the
+        # message.
+        message = message.replace(b"\x00", b"\xc0\x80")
+
+        with self._lock:
+            now = time()
+            self._bucket_level += (
+                (now - self._prev_write_time) * MAX_BYTES_PER_SECOND)
+
+            # If the bucket level is still below zero, the clock must have gone
+            # backwards, so reset it to zero and continue.
+            self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE))
+            self._prev_write_time = now
+
+            self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message)
+            if self._bucket_level < 0:
+                sleep(-self._bucket_level / MAX_BYTES_PER_SECOND)
+
+            self.android_log_write(prio, tag, message)

Lib/_apple_support.py

@@ -0,0 +1,66 @@
+import io
+import sys
+
+
+def init_streams(log_write, stdout_level, stderr_level):
+    # Redirect stdout and stderr to the Apple system log. This method is
+    # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger
+    # is enabled.
+    sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors)
+    sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors)
+
+
+class SystemLog(io.TextIOWrapper):
+    def __init__(self, log_write, level, **kwargs):
+        kwargs.setdefault("encoding", "UTF-8")
+        kwargs.setdefault("line_buffering", True)
+        super().__init__(LogStream(log_write, level), **kwargs)
+
+    def __repr__(self):
+        return f"<SystemLog (level {self.buffer.level})>"
+
+    def write(self, s):
+        if not isinstance(s, str):
+            raise TypeError(
+                f"write() argument must be str, not {type(s).__name__}")
+
+        # In case `s` is a str subclass that writes itself to stdout or stderr
+        # when we call its methods, convert it to an actual str.
+        s = str.__str__(s)
+
+        # We want to emit one log message per line, so split
+        # the string before sending it to the superclass.
+        for line in s.splitlines(keepends=True):
+            super().write(line)
+
+        return len(s)
+
+
+class LogStream(io.RawIOBase):
+    def __init__(self, log_write, level):
+        self.log_write = log_write
+        self.level = level
+
+    def __repr__(self):
+        return f"<LogStream (level {self.level!r})>"
+
+    def writable(self):
+        return True
+
+    def write(self, b):
+        if type(b) is not bytes:
+            try:
+                b = bytes(memoryview(b))
+            except TypeError:
+                raise TypeError(
+                    f"write() argument must be bytes-like, not {type(b).__name__}"
+                ) from None
+
+        # Writing an empty string to the stream should have no effect.
+        if b:
+            # Encode null bytes using "modified UTF-8" to avoid truncating the
+            # message. This should not affect the return value, as the caller
+            # may be expecting it to match the length of the input.
+            self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80"))
+
+        return len(b)

Lib/_colorize.py

@@ -0,0 +1,67 @@
+import io
+import os
+import sys
+
+COLORIZE = True
+
+
+class ANSIColors:
+    BOLD_GREEN = "\x1b[1;32m"
+    BOLD_MAGENTA = "\x1b[1;35m"
+    BOLD_RED = "\x1b[1;31m"
+    GREEN = "\x1b[32m"
+    GREY = "\x1b[90m"
+    MAGENTA = "\x1b[35m"
+    RED = "\x1b[31m"
+    RESET = "\x1b[0m"
+    YELLOW = "\x1b[33m"
+
+
+NoColors = ANSIColors()
+
+for attr in dir(NoColors):
+    if not attr.startswith("__"):
+        setattr(NoColors, attr, "")
+
+
+def get_colors(colorize: bool = False, *, file=None) -> ANSIColors:
+    if colorize or can_colorize(file=file):
+        return ANSIColors()
+    else:
+        return NoColors
+
+
+def can_colorize(*, file=None) -> bool:
+    if file is None:
+        file = sys.stdout
+
+    if not sys.flags.ignore_environment:
+        if os.environ.get("PYTHON_COLORS") == "0":
+            return False
+        if os.environ.get("PYTHON_COLORS") == "1":
+            return True
+    if os.environ.get("NO_COLOR"):
+        return False
+    if not COLORIZE:
+        return False
+    if os.environ.get("FORCE_COLOR"):
+        return True
+    if os.environ.get("TERM") == "dumb":
+        return False
+
+    if not hasattr(file, "fileno"):
+        return False
+
+    if sys.platform == "win32":
+        try:
+            import nt
+
+            if not nt._supports_virtual_terminal():
+                return False
+        except (ImportError, AttributeError):
+            return False
+
+    try:
+        return os.isatty(file.fileno())
+    except io.UnsupportedOperation:
+        return file.isatty()

Lib/_dummy_os.py

@@ -5,22 +5,30 @@ A shim of the os module containing only simple path-related utilities
 try:
     from os import *
 except ImportError:
-    import abc
+    import abc, sys
 
     def __getattr__(name):
-        raise OSError("no os specific module found")
+        if name in {"_path_normpath", "__path__"}:
+            raise AttributeError(name)
+        if name.isupper():
+            return 0
+        def dummy(*args, **kwargs):
+            import io
+            return io.UnsupportedOperation(f"{name}: no os specific module found")
+        dummy.__name__ = f"dummy_{name}"
+        return dummy
 
-    def _shim():
+    sys.modules['os'] = sys.modules['posix'] = sys.modules[__name__]
-        import _dummy_os, sys
-        sys.modules['os'] = _dummy_os
-        sys.modules['os.path'] = _dummy_os.path
 
     import posixpath as path
-    import sys
     sys.modules['os.path'] = path
     del sys
 
     sep = path.sep
+    supports_dir_fd = set()
+    supports_effective_ids = set()
+    supports_fd = set()
+    supports_follow_symlinks = set()
 
 
     def fspath(path):

Lib/_ios_support.py

@@ -0,0 +1,71 @@
+import sys
+try:
+    from ctypes import cdll, c_void_p, c_char_p, util
+except ImportError:
+    # ctypes is an optional module. If it's not present, we're limited in what
+    # we can tell about the system, but we don't want to prevent the module
+    # from working.
+    print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr)
+    objc = None
+else:
+    # ctypes is available. Load the ObjC library, and wrap the objc_getClass,
+    # sel_registerName methods
+    lib = util.find_library("objc")
+    if lib is None:
+        # Failed to load the objc library
+        raise ImportError("ObjC runtime library couldn't be loaded")
+
+    objc = cdll.LoadLibrary(lib)
+    objc.objc_getClass.restype = c_void_p
+    objc.objc_getClass.argtypes = [c_char_p]
+    objc.sel_registerName.restype = c_void_p
+    objc.sel_registerName.argtypes = [c_char_p]
+
+
+def get_platform_ios():
+    # Determine if this is a simulator using the multiarch value
+    is_simulator = sys.implementation._multiarch.endswith("simulator")
+
+    # We can't use ctypes; abort
+    if not objc:
+        return None
+
+    # Most of the methods return ObjC objects
+    objc.objc_msgSend.restype = c_void_p
+    # All the methods used have no arguments.
+    objc.objc_msgSend.argtypes = [c_void_p, c_void_p]
+
+    # Equivalent of:
+    #   device = [UIDevice currentDevice]
+    UIDevice = objc.objc_getClass(b"UIDevice")
+    SEL_currentDevice = objc.sel_registerName(b"currentDevice")
+    device = objc.objc_msgSend(UIDevice, SEL_currentDevice)
+
+    # Equivalent of:
+    #   device_systemVersion = [device systemVersion]
+    SEL_systemVersion = objc.sel_registerName(b"systemVersion")
+    device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion)
+
+    # Equivalent of:
+    #   device_systemName = [device systemName]
+    SEL_systemName = objc.sel_registerName(b"systemName")
+    device_systemName = objc.objc_msgSend(device, SEL_systemName)
+
+    # Equivalent of:
+    #   device_model = [device model]
+    SEL_model = objc.sel_registerName(b"model")
+    device_model = objc.objc_msgSend(device, SEL_model)
+
+    # UTF8String returns a const char*;
+    SEL_UTF8String = objc.sel_registerName(b"UTF8String")
+    objc.objc_msgSend.restype = c_char_p
+
+    # Equivalent of:
+    #   system = [device_systemName UTF8String]
+    #   release = [device_systemVersion UTF8String]
+    #   model = [device_model UTF8String]
+    system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode()
+    release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode()
+    model = objc.objc_msgSend(device_model, SEL_UTF8String).decode()
+
+    return system, release, model, is_simulator

Lib/_osx_support.py

@@ -507,6 +507,11 @@ def get_platform_osx(_config_vars, osname, release, machine):
     # MACOSX_DEPLOYMENT_TARGET.
 
     macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '')
+    if macver and '.' not in macver:
+        # Ensure that the version includes at least a major
+        # and minor version, even if MACOSX_DEPLOYMENT_TARGET
+        # is set to a single-label version like "14".
+        macver += '.0'
     macrelease = _get_system_version() or macver
     macver = macver or macrelease
 

Lib/_pydecimal.py

@@ -734,18 +734,23 @@ class Decimal(object):
 
         """
         if isinstance(f, int):                # handle integer inputs
-            return cls(f)
+            sign = 0 if f >= 0 else 1
-        if not isinstance(f, float):
+            k = 0
-            raise TypeError("argument must be int or float.")
+            coeff = str(abs(f))
-        if _math.isinf(f) or _math.isnan(f):
+        elif isinstance(f, float):
-            return cls(repr(f))
+            if _math.isinf(f) or _math.isnan(f):
-        if _math.copysign(1.0, f) == 1.0:
+                return cls(repr(f))
-            sign = 0
+            if _math.copysign(1.0, f) == 1.0:
+                sign = 0
+            else:
+                sign = 1
+            n, d = abs(f).as_integer_ratio()
+            k = d.bit_length() - 1
+            coeff = str(n*5**k)
         else:
-            sign = 1
+            raise TypeError("argument must be int or float.")
-        n, d = abs(f).as_integer_ratio()
+
-        k = d.bit_length() - 1
+        result = _dec_from_triple(sign, coeff, -k)
-        result = _dec_from_triple(sign, str(n*5**k), -k)
         if cls is Decimal:
             return result
         else:

Lib/_pyio.py

@@ -44,8 +44,9 @@ def text_encoding(encoding, stacklevel=2):
     """
     A helper function to choose the text encoding.
 
-    When encoding is not None, just return it.
+    When encoding is not None, this function returns it.
-    Otherwise, return the default text encoding (i.e. "locale").
+    Otherwise, this function returns the default text encoding
+    (i.e. "locale" or "utf-8" depends on UTF-8 mode).
 
     This function emits an EncodingWarning if *encoding* is None and
     sys.flags.warn_default_encoding is true.
@@ -55,7 +56,10 @@ def text_encoding(encoding, stacklevel=2):
     However, please consider using encoding="utf-8" for new APIs.
     """
     if encoding is None:
-        encoding = "locale"
+        if sys.flags.utf8_mode:
+            encoding = "utf-8"
+        else:
+            encoding = "locale"
         if sys.flags.warn_default_encoding:
             import warnings
             warnings.warn("'encoding' argument not specified.",
@@ -101,7 +105,6 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     'b'       binary mode
     't'       text mode (default)
     '+'       open a disk file for updating (reading and writing)
-    'U'       universal newline mode (deprecated)
     ========= ===============================================================
 
     The default mode is 'rt' (open for reading text). For binary random
@@ -117,10 +120,6 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     returned as strings, the bytes having been first decoded using a
     platform-dependent encoding or using the specified encoding if given.
 
-    'U' mode is deprecated and will raise an exception in future versions
-    of Python.  It has no effect in Python 3.  Use newline to control
-    universal newlines mode.
-
     buffering is an optional integer used to set the buffering policy.
     Pass 0 to switch buffering off (only allowed in binary mode), 1 to select
     line buffering (only usable in text mode), and an integer > 1 to indicate
@@ -206,7 +205,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     if errors is not None and not isinstance(errors, str):
         raise TypeError("invalid errors: %r" % errors)
     modes = set(mode)
-    if modes - set("axrwb+tU") or len(mode) > len(modes):
+    if modes - set("axrwb+t") or len(mode) > len(modes):
         raise ValueError("invalid mode: %r" % mode)
     creating = "x" in modes
     reading = "r" in modes
@@ -215,13 +214,6 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     updating = "+" in modes
     text = "t" in modes
     binary = "b" in modes
-    if "U" in modes:
-        if creating or writing or appending or updating:
-            raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'")
-        import warnings
-        warnings.warn("'U' mode is deprecated",
-                      DeprecationWarning, 2)
-        reading = True
     if text and binary:
         raise ValueError("can't have text and binary mode at once")
     if creating + reading + writing + appending > 1:
@@ -311,22 +303,6 @@ except AttributeError:
     open_code = _open_code_with_warning
 
 
-def __getattr__(name):
-    if name == "OpenWrapper":
-        # bpo-43680: Until Python 3.9, _pyio.open was not a static method and
-        # builtins.open was set to OpenWrapper to not become a bound method
-        # when set to a class variable. _io.open is a built-in function whereas
-        # _pyio.open is a Python function. In Python 3.10, _pyio.open() is now
-        # a static method, and builtins.open() is now io.open().
-        import warnings
-        warnings.warn('OpenWrapper is deprecated, use open instead',
-                      DeprecationWarning, stacklevel=2)
-        global OpenWrapper
-        OpenWrapper = open
-        return OpenWrapper
-    raise AttributeError(name)
-
-
 # In normal operation, both `UnsupportedOperation`s should be bound to the
 # same object.
 try:
@@ -338,8 +314,7 @@ except AttributeError:
 
 class IOBase(metaclass=abc.ABCMeta):
 
-    """The abstract base class for all I/O classes, acting on streams of
+    """The abstract base class for all I/O classes.
-    bytes. There is no public constructor.
 
     This class provides dummy implementations for many methods that
     derived classes can override selectively; the default implementations
@@ -1154,6 +1129,7 @@ class BufferedReader(_BufferedIOMixin):
         do at most one raw read to satisfy it.  We never return more
         than self.buffer_size.
         """
+        self._checkClosed("peek of closed file")
         with self._read_lock:
             return self._peek_unlocked(size)
 
@@ -1172,6 +1148,7 @@ class BufferedReader(_BufferedIOMixin):
         """Reads up to size bytes, with at most one read() system call."""
         # Returns up to size bytes.  If at least one byte is buffered, we
         # only return buffered bytes.  Otherwise, we do one raw read.
+        self._checkClosed("read of closed file")
         if size < 0:
             size = self.buffer_size
         if size == 0:
@@ -1189,6 +1166,8 @@ class BufferedReader(_BufferedIOMixin):
     def _readinto(self, buf, read1):
         """Read data into *buf* with at most one system call."""
 
+        self._checkClosed("readinto of closed file")
+
         # Need to create a memoryview object of type 'b', otherwise
         # we may not be able to assign bytes to it, and slicing it
         # would create a new object.
@@ -1233,11 +1212,13 @@ class BufferedReader(_BufferedIOMixin):
         return written
 
     def tell(self):
-        return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
+        # GH-95782: Keep return value non-negative
+        return max(_BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos, 0)
 
     def seek(self, pos, whence=0):
         if whence not in valid_seek_flags:
             raise ValueError("invalid whence value")
+        self._checkClosed("seek of closed file")
         with self._read_lock:
             if whence == 1:
                 pos -= len(self._read_buf) - self._read_pos
@@ -1845,7 +1826,7 @@ class TextIOBase(IOBase):
     """Base class for text I/O.
 
     This class provides a character and line based interface to stream
-    I/O. There is no public constructor.
+    I/O.
     """
 
     def read(self, size=-1):
@@ -1997,7 +1978,7 @@ class TextIOWrapper(TextIOBase):
     r"""Character and line based layer over a BufferedIOBase object, buffer.
 
     encoding gives the name of the encoding that the stream will be
-    decoded or encoded with. It defaults to locale.getpreferredencoding(False).
+    decoded or encoded with. It defaults to locale.getencoding().
 
     errors determines the strictness of encoding and decoding (see the
     codecs.register) and defaults to "strict".
@@ -2031,19 +2012,7 @@ class TextIOWrapper(TextIOBase):
         encoding = text_encoding(encoding)
 
         if encoding == "locale":
-            try:
+            encoding = self._get_locale_encoding()
-                encoding = os.device_encoding(buffer.fileno()) or "locale"
-            except (AttributeError, UnsupportedOperation):
-                pass
-
-        if encoding == "locale":
-            try:
-                import locale
-            except ImportError:
-                # Importing locale may fail if Python is being built
-                encoding = "utf-8"
-            else:
-                encoding = locale.getpreferredencoding(False)
 
         if not isinstance(encoding, str):
             raise ValueError("invalid encoding: %r" % encoding)
@@ -2176,6 +2145,8 @@ class TextIOWrapper(TextIOBase):
         else:
             if not isinstance(encoding, str):
                 raise TypeError("invalid encoding: %r" % encoding)
+            if encoding == "locale":
+                encoding = self._get_locale_encoding()
 
         if newline is Ellipsis:
             newline = self._readnl
@@ -2243,8 +2214,9 @@ class TextIOWrapper(TextIOBase):
         self.buffer.write(b)
         if self._line_buffering and (haslf or "\r" in s):
             self.flush()
-        self._set_decoded_chars('')
+        if self._snapshot is not None:
-        self._snapshot = None
+            self._set_decoded_chars('')
+            self._snapshot = None
         if self._decoder:
             self._decoder.reset()
         return length
@@ -2280,6 +2252,15 @@ class TextIOWrapper(TextIOBase):
         self._decoded_chars_used += len(chars)
         return chars
 
+    def _get_locale_encoding(self):
+        try:
+            import locale
+        except ImportError:
+            # Importing locale may fail if Python is being built
+            return "utf-8"
+        else:
+            return locale.getencoding()
+
     def _rewind_decoded_chars(self, n):
         """Rewind the _decoded_chars buffer."""
         if self._decoded_chars_used < n:
@@ -2549,8 +2530,9 @@ class TextIOWrapper(TextIOBase):
             # Read everything.
             result = (self._get_decoded_chars() +
                       decoder.decode(self.buffer.read(), final=True))
-            self._set_decoded_chars('')
+            if self._snapshot is not None:
-            self._snapshot = None
+                self._set_decoded_chars('')
+                self._snapshot = None
             return result
         else:
             # Keep reading chunks until we have size characters to return.

Lib/_sitebuiltins.py

@@ -10,7 +10,6 @@ The objects used by the site module to add custom builtins.
 
 import sys
 
-
 class Quitter(object):
     def __init__(self, name, eof):
         self.name = name
@@ -48,7 +47,7 @@ class _Printer(object):
         data = None
         for filename in self.__filenames:
             try:
-                with open(filename, "r") as fp:
+                with open(filename, encoding='utf-8') as fp:
                     data = fp.read()
                 break
             except OSError:

Lib/_strptime.py

@@ -0,0 +1,565 @@
+"""Strptime-related classes and functions.
+
+CLASSES:
+    LocaleTime -- Discovers and stores locale-specific time information
+    TimeRE -- Creates regexes for pattern matching a string of text containing
+                time information
+
+FUNCTIONS:
+    _getlang -- Figure out what language is being used for the locale
+    strptime -- Calculates the time struct represented by the passed-in string
+
+"""
+import time
+import locale
+import calendar
+from re import compile as re_compile
+from re import IGNORECASE
+from re import escape as re_escape
+from datetime import (date as datetime_date,
+                      timedelta as datetime_timedelta,
+                      timezone as datetime_timezone)
+from _thread import allocate_lock as _thread_allocate_lock
+
+__all__ = []
+
+def _getlang():
+    # Figure out what the current language is set to.
+    return locale.getlocale(locale.LC_TIME)
+
+class LocaleTime(object):
+    """Stores and handles locale-specific information related to time.
+
+    ATTRIBUTES:
+        f_weekday -- full weekday names (7-item list)
+        a_weekday -- abbreviated weekday names (7-item list)
+        f_month -- full month names (13-item list; dummy value in [0], which
+                    is added by code)
+        a_month -- abbreviated month names (13-item list, dummy value in
+                    [0], which is added by code)
+        am_pm -- AM/PM representation (2-item list)
+        LC_date_time -- format string for date/time representation (string)
+        LC_date -- format string for date representation (string)
+        LC_time -- format string for time representation (string)
+        timezone -- daylight- and non-daylight-savings timezone representation
+                    (2-item list of sets)
+        lang -- Language used by instance (2-item tuple)
+    """
+
+    def __init__(self):
+        """Set all attributes.
+
+        Order of methods called matters for dependency reasons.
+
+        The locale language is set at the offset and then checked again before
+        exiting.  This is to make sure that the attributes were not set with a
+        mix of information from more than one locale.  This would most likely
+        happen when using threads where one thread calls a locale-dependent
+        function while another thread changes the locale while the function in
+        the other thread is still running.  Proper coding would call for
+        locks to prevent changing the locale while locale-dependent code is
+        running.  The check here is done in case someone does not think about
+        doing this.
+
+        Only other possible issue is if someone changed the timezone and did
+        not call tz.tzset .  That is an issue for the programmer, though,
+        since changing the timezone is worthless without that call.
+
+        """
+        self.lang = _getlang()
+        self.__calc_weekday()
+        self.__calc_month()
+        self.__calc_am_pm()
+        self.__calc_timezone()
+        self.__calc_date_time()
+        if _getlang() != self.lang:
+            raise ValueError("locale changed during initialization")
+        if time.tzname != self.tzname or time.daylight != self.daylight:
+            raise ValueError("timezone changed during initialization")
+
+    def __calc_weekday(self):
+        # Set self.a_weekday and self.f_weekday using the calendar
+        # module.
+        a_weekday = [calendar.day_abbr[i].lower() for i in range(7)]
+        f_weekday = [calendar.day_name[i].lower() for i in range(7)]
+        self.a_weekday = a_weekday
+        self.f_weekday = f_weekday
+
+    def __calc_month(self):
+        # Set self.f_month and self.a_month using the calendar module.
+        a_month = [calendar.month_abbr[i].lower() for i in range(13)]
+        f_month = [calendar.month_name[i].lower() for i in range(13)]
+        self.a_month = a_month
+        self.f_month = f_month
+
+    def __calc_am_pm(self):
+        # Set self.am_pm by using time.strftime().
+
+        # The magic date (1999,3,17,hour,44,55,2,76,0) is not really that
+        # magical; just happened to have used it everywhere else where a
+        # static date was needed.
+        am_pm = []
+        for hour in (1, 22):
+            time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0))
+            am_pm.append(time.strftime("%p", time_tuple).lower())
+        self.am_pm = am_pm
+
+    def __calc_date_time(self):
+        # Set self.date_time, self.date, & self.time by using
+        # time.strftime().
+
+        # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of
+        # overloaded numbers is minimized.  The order in which searches for
+        # values within the format string is very important; it eliminates
+        # possible ambiguity for what something represents.
+        time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
+        date_time = [None, None, None]
+        date_time[0] = time.strftime("%c", time_tuple).lower()
+        date_time[1] = time.strftime("%x", time_tuple).lower()
+        date_time[2] = time.strftime("%X", time_tuple).lower()
+        replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'),
+                    (self.f_month[3], '%B'), (self.a_weekday[2], '%a'),
+                    (self.a_month[3], '%b'), (self.am_pm[1], '%p'),
+                    ('1999', '%Y'), ('99', '%y'), ('22', '%H'),
+                    ('44', '%M'), ('55', '%S'), ('76', '%j'),
+                    ('17', '%d'), ('03', '%m'), ('3', '%m'),
+                    # '3' needed for when no leading zero.
+                    ('2', '%w'), ('10', '%I')]
+        replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone
+                                                for tz in tz_values])
+        for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')):
+            current_format = date_time[offset]
+            for old, new in replacement_pairs:
+                # Must deal with possible lack of locale info
+                # manifesting itself as the empty string (e.g., Swedish's
+                # lack of AM/PM info) or a platform returning a tuple of empty
+                # strings (e.g., MacOS 9 having timezone as ('','')).
+                if old:
+                    current_format = current_format.replace(old, new)
+            # If %W is used, then Sunday, 2005-01-03 will fall on week 0 since
+            # 2005-01-03 occurs before the first Monday of the year.  Otherwise
+            # %U is used.
+            time_tuple = time.struct_time((1999,1,3,1,1,1,6,3,0))
+            if '00' in time.strftime(directive, time_tuple):
+                U_W = '%W'
+            else:
+                U_W = '%U'
+            date_time[offset] = current_format.replace('11', U_W)
+        self.LC_date_time = date_time[0]
+        self.LC_date = date_time[1]
+        self.LC_time = date_time[2]
+
+    def __calc_timezone(self):
+        # Set self.timezone by using time.tzname.
+        # Do not worry about possibility of time.tzname[0] == time.tzname[1]
+        # and time.daylight; handle that in strptime.
+        try:
+            time.tzset()
+        except AttributeError:
+            pass
+        self.tzname = time.tzname
+        self.daylight = time.daylight
+        no_saving = frozenset({"utc", "gmt", self.tzname[0].lower()})
+        if self.daylight:
+            has_saving = frozenset({self.tzname[1].lower()})
+        else:
+            has_saving = frozenset()
+        self.timezone = (no_saving, has_saving)
+
+
+class TimeRE(dict):
+    """Handle conversion from format directives to regexes."""
+
+    def __init__(self, locale_time=None):
+        """Create keys/values.
+
+        Order of execution is important for dependency reasons.
+
+        """
+        if locale_time:
+            self.locale_time = locale_time
+        else:
+            self.locale_time = LocaleTime()
+        base = super()
+        base.__init__({
+            # The " [1-9]" part of the regex is to make %c from ANSI C work
+            'd': r"(?P<d>3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
+            'f': r"(?P<f>[0-9]{1,6})",
+            'H': r"(?P<H>2[0-3]|[0-1]\d|\d)",
+            'I': r"(?P<I>1[0-2]|0[1-9]|[1-9])",
+            'G': r"(?P<G>\d\d\d\d)",
+            'j': r"(?P<j>36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])",
+            'm': r"(?P<m>1[0-2]|0[1-9]|[1-9])",
+            'M': r"(?P<M>[0-5]\d|\d)",
+            'S': r"(?P<S>6[0-1]|[0-5]\d|\d)",
+            'U': r"(?P<U>5[0-3]|[0-4]\d|\d)",
+            'w': r"(?P<w>[0-6])",
+            'u': r"(?P<u>[1-7])",
+            'V': r"(?P<V>5[0-3]|0[1-9]|[1-4]\d|\d)",
+            # W is set below by using 'U'
+            'y': r"(?P<y>\d\d)",
+            #XXX: Does 'Y' need to worry about having less or more than
+            #     4 digits?
+            'Y': r"(?P<Y>\d\d\d\d)",
+            'z': r"(?P<z>[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))",
+            'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
+            'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
+            'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
+            'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'),
+            'p': self.__seqToRE(self.locale_time.am_pm, 'p'),
+            'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone
+                                        for tz in tz_names),
+                                'Z'),
+            '%': '%'})
+        base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
+        base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
+        base.__setitem__('x', self.pattern(self.locale_time.LC_date))
+        base.__setitem__('X', self.pattern(self.locale_time.LC_time))
+
+    def __seqToRE(self, to_convert, directive):
+        """Convert a list to a regex string for matching a directive.
+
+        Want possible matching values to be from longest to shortest.  This
+        prevents the possibility of a match occurring for a value that also
+        a substring of a larger value that should have matched (e.g., 'abc'
+        matching when 'abcdef' should have been the match).
+
+        """
+        to_convert = sorted(to_convert, key=len, reverse=True)
+        for value in to_convert:
+            if value != '':
+                break
+        else:
+            return ''
+        regex = '|'.join(re_escape(stuff) for stuff in to_convert)
+        regex = '(?P<%s>%s' % (directive, regex)
+        return '%s)' % regex
+
+    def pattern(self, format):
+        """Return regex pattern for the format string.
+
+        Need to make sure that any characters that might be interpreted as
+        regex syntax are escaped.
+
+        """
+        processed_format = ''
+        # The sub() call escapes all characters that might be misconstrued
+        # as regex syntax.  Cannot use re.escape since we have to deal with
+        # format directives (%m, etc.).
+        regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
+        format = regex_chars.sub(r"\\\1", format)
+        whitespace_replacement = re_compile(r'\s+')
+        format = whitespace_replacement.sub(r'\\s+', format)
+        while '%' in format:
+            directive_index = format.index('%')+1
+            processed_format = "%s%s%s" % (processed_format,
+                                           format[:directive_index-1],
+                                           self[format[directive_index]])
+            format = format[directive_index+1:]
+        return "%s%s" % (processed_format, format)
+
+    def compile(self, format):
+        """Return a compiled re object for the format string."""
+        return re_compile(self.pattern(format), IGNORECASE)
+
+_cache_lock = _thread_allocate_lock()
+# DO NOT modify _TimeRE_cache or _regex_cache without acquiring the cache lock
+# first!
+_TimeRE_cache = TimeRE()
+_CACHE_MAX_SIZE = 5 # Max number of regexes stored in _regex_cache
+_regex_cache = {}
+
+def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon):
+    """Calculate the Julian day based on the year, week of the year, and day of
+    the week, with week_start_day representing whether the week of the year
+    assumes the week starts on Sunday or Monday (6 or 0)."""
+    first_weekday = datetime_date(year, 1, 1).weekday()
+    # If we are dealing with the %U directive (week starts on Sunday), it's
+    # easier to just shift the view to Sunday being the first day of the
+    # week.
+    if not week_starts_Mon:
+        first_weekday = (first_weekday + 1) % 7
+        day_of_week = (day_of_week + 1) % 7
+    # Need to watch out for a week 0 (when the first day of the year is not
+    # the same as that specified by %U or %W).
+    week_0_length = (7 - first_weekday) % 7
+    if week_of_year == 0:
+        return 1 + day_of_week - first_weekday
+    else:
+        days_to_week = week_0_length + (7 * (week_of_year - 1))
+        return 1 + days_to_week + day_of_week
+
+
+def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a 2-tuple consisting of a time struct and an int containing
+    the number of microseconds based on the input string and the
+    format string."""
+
+    for index, arg in enumerate([data_string, format]):
+        if not isinstance(arg, str):
+            msg = "strptime() argument {} must be str, not {}"
+            raise TypeError(msg.format(index, type(arg)))
+
+    global _TimeRE_cache, _regex_cache
+    with _cache_lock:
+        locale_time = _TimeRE_cache.locale_time
+        if (_getlang() != locale_time.lang or
+            time.tzname != locale_time.tzname or
+            time.daylight != locale_time.daylight):
+            _TimeRE_cache = TimeRE()
+            _regex_cache.clear()
+            locale_time = _TimeRE_cache.locale_time
+        if len(_regex_cache) > _CACHE_MAX_SIZE:
+            _regex_cache.clear()
+        format_regex = _regex_cache.get(format)
+        if not format_regex:
+            try:
+                format_regex = _TimeRE_cache.compile(format)
+            # KeyError raised when a bad format is found; can be specified as
+            # \\, in which case it was a stray % but with a space after it
+            except KeyError as err:
+                bad_directive = err.args[0]
+                if bad_directive == "\\":
+                    bad_directive = "%"
+                del err
+                raise ValueError("'%s' is a bad directive in format '%s'" %
+                                    (bad_directive, format)) from None
+            # IndexError only occurs when the format string is "%"
+            except IndexError:
+                raise ValueError("stray %% in format '%s'" % format) from None
+            _regex_cache[format] = format_regex
+    found = format_regex.match(data_string)
+    if not found:
+        raise ValueError("time data %r does not match format %r" %
+                         (data_string, format))
+    if len(data_string) != found.end():
+        raise ValueError("unconverted data remains: %s" %
+                          data_string[found.end():])
+
+    iso_year = year = None
+    month = day = 1
+    hour = minute = second = fraction = 0
+    tz = -1
+    gmtoff = None
+    gmtoff_fraction = 0
+    iso_week = week_of_year = None
+    week_of_year_start = None
+    # weekday and julian defaulted to None so as to signal need to calculate
+    # values
+    weekday = julian = None
+    found_dict = found.groupdict()
+    for group_key in found_dict.keys():
+        # Directives not explicitly handled below:
+        #   c, x, X
+        #      handled by making out of other directives
+        #   U, W
+        #      worthless without day of the week
+        if group_key == 'y':
+            year = int(found_dict['y'])
+            # Open Group specification for strptime() states that a %y
+            #value in the range of [00, 68] is in the century 2000, while
+            #[69,99] is in the century 1900
+            if year <= 68:
+                year += 2000
+            else:
+                year += 1900
+        elif group_key == 'Y':
+            year = int(found_dict['Y'])
+        elif group_key == 'G':
+            iso_year = int(found_dict['G'])
+        elif group_key == 'm':
+            month = int(found_dict['m'])
+        elif group_key == 'B':
+            month = locale_time.f_month.index(found_dict['B'].lower())
+        elif group_key == 'b':
+            month = locale_time.a_month.index(found_dict['b'].lower())
+        elif group_key == 'd':
+            day = int(found_dict['d'])
+        elif group_key == 'H':
+            hour = int(found_dict['H'])
+        elif group_key == 'I':
+            hour = int(found_dict['I'])
+            ampm = found_dict.get('p', '').lower()
+            # If there was no AM/PM indicator, we'll treat this like AM
+            if ampm in ('', locale_time.am_pm[0]):
+                # We're in AM so the hour is correct unless we're
+                # looking at 12 midnight.
+                # 12 midnight == 12 AM == hour 0
+                if hour == 12:
+                    hour = 0
+            elif ampm == locale_time.am_pm[1]:
+                # We're in PM so we need to add 12 to the hour unless
+                # we're looking at 12 noon.
+                # 12 noon == 12 PM == hour 12
+                if hour != 12:
+                    hour += 12
+        elif group_key == 'M':
+            minute = int(found_dict['M'])
+        elif group_key == 'S':
+            second = int(found_dict['S'])
+        elif group_key == 'f':
+            s = found_dict['f']
+            # Pad to always return microseconds.
+            s += "0" * (6 - len(s))
+            fraction = int(s)
+        elif group_key == 'A':
+            weekday = locale_time.f_weekday.index(found_dict['A'].lower())
+        elif group_key == 'a':
+            weekday = locale_time.a_weekday.index(found_dict['a'].lower())
+        elif group_key == 'w':
+            weekday = int(found_dict['w'])
+            if weekday == 0:
+                weekday = 6
+            else:
+                weekday -= 1
+        elif group_key == 'u':
+            weekday = int(found_dict['u'])
+            weekday -= 1
+        elif group_key == 'j':
+            julian = int(found_dict['j'])
+        elif group_key in ('U', 'W'):
+            week_of_year = int(found_dict[group_key])
+            if group_key == 'U':
+                # U starts week on Sunday.
+                week_of_year_start = 6
+            else:
+                # W starts week on Monday.
+                week_of_year_start = 0
+        elif group_key == 'V':
+            iso_week = int(found_dict['V'])
+        elif group_key == 'z':
+            z = found_dict['z']
+            if z == 'Z':
+                gmtoff = 0
+            else:
+                if z[3] == ':':
+                    z = z[:3] + z[4:]
+                    if len(z) > 5:
+                        if z[5] != ':':
+                            msg = f"Inconsistent use of : in {found_dict['z']}"
+                            raise ValueError(msg)
+                        z = z[:5] + z[6:]
+                hours = int(z[1:3])
+                minutes = int(z[3:5])
+                seconds = int(z[5:7] or 0)
+                gmtoff = (hours * 60 * 60) + (minutes * 60) + seconds
+                gmtoff_remainder = z[8:]
+                # Pad to always return microseconds.
+                gmtoff_remainder_padding = "0" * (6 - len(gmtoff_remainder))
+                gmtoff_fraction = int(gmtoff_remainder + gmtoff_remainder_padding)
+                if z.startswith("-"):
+                    gmtoff = -gmtoff
+                    gmtoff_fraction = -gmtoff_fraction
+        elif group_key == 'Z':
+            # Since -1 is default value only need to worry about setting tz if
+            # it can be something other than -1.
+            found_zone = found_dict['Z'].lower()
+            for value, tz_values in enumerate(locale_time.timezone):
+                if found_zone in tz_values:
+                    # Deal with bad locale setup where timezone names are the
+                    # same and yet time.daylight is true; too ambiguous to
+                    # be able to tell what timezone has daylight savings
+                    if (time.tzname[0] == time.tzname[1] and
+                       time.daylight and found_zone not in ("utc", "gmt")):
+                        break
+                    else:
+                        tz = value
+                        break
+
+    # Deal with the cases where ambiguities arise
+    # don't assume default values for ISO week/year
+    if iso_year is not None:
+        if julian is not None:
+            raise ValueError("Day of the year directive '%j' is not "
+                             "compatible with ISO year directive '%G'. "
+                             "Use '%Y' instead.")
+        elif iso_week is None or weekday is None:
+            raise ValueError("ISO year directive '%G' must be used with "
+                             "the ISO week directive '%V' and a weekday "
+                             "directive ('%A', '%a', '%w', or '%u').")
+    elif iso_week is not None:
+        if year is None or weekday is None:
+            raise ValueError("ISO week directive '%V' must be used with "
+                             "the ISO year directive '%G' and a weekday "
+                             "directive ('%A', '%a', '%w', or '%u').")
+        else:
+            raise ValueError("ISO week directive '%V' is incompatible with "
+                             "the year directive '%Y'. Use the ISO year '%G' "
+                             "instead.")
+
+    leap_year_fix = False
+    if year is None:
+        if month == 2 and day == 29:
+            year = 1904  # 1904 is first leap year of 20th century
+            leap_year_fix = True
+        else:
+            year = 1900
+
+    # If we know the week of the year and what day of that week, we can figure
+    # out the Julian day of the year.
+    if julian is None and weekday is not None:
+        if week_of_year is not None:
+            week_starts_Mon = True if week_of_year_start == 0 else False
+            julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
+                                                week_starts_Mon)
+        elif iso_year is not None and iso_week is not None:
+            datetime_result = datetime_date.fromisocalendar(iso_year, iso_week, weekday + 1)
+            year = datetime_result.year
+            month = datetime_result.month
+            day = datetime_result.day
+        if julian is not None and julian <= 0:
+            year -= 1
+            yday = 366 if calendar.isleap(year) else 365
+            julian += yday
+
+    if julian is None:
+        # Cannot pre-calculate datetime_date() since can change in Julian
+        # calculation and thus could have different value for the day of
+        # the week calculation.
+        # Need to add 1 to result since first day of the year is 1, not 0.
+        julian = datetime_date(year, month, day).toordinal() - \
+                  datetime_date(year, 1, 1).toordinal() + 1
+    else:  # Assume that if they bothered to include Julian day (or if it was
+           # calculated above with year/week/weekday) it will be accurate.
+        datetime_result = datetime_date.fromordinal(
+                            (julian - 1) +
+                            datetime_date(year, 1, 1).toordinal())
+        year = datetime_result.year
+        month = datetime_result.month
+        day = datetime_result.day
+    if weekday is None:
+        weekday = datetime_date(year, month, day).weekday()
+    # Add timezone info
+    tzname = found_dict.get("Z")
+
+    if leap_year_fix:
+        # the caller didn't supply a year but asked for Feb 29th. We couldn't
+        # use the default of 1900 for computations. We set it back to ensure
+        # that February 29th is smaller than March 1st.
+        year = 1900
+
+    return (year, month, day,
+            hour, minute, second,
+            weekday, julian, tz, tzname, gmtoff), fraction, gmtoff_fraction
+
+def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a time struct based on the input string and the
+    format string."""
+    tt = _strptime(data_string, format)[0]
+    return time.struct_time(tt[:time._STRUCT_TM_ITEMS])
+
+def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a class cls instance based on the input string and the
+    format string."""
+    tt, fraction, gmtoff_fraction = _strptime(data_string, format)
+    tzname, gmtoff = tt[-2:]
+    args = tt[:6] + (fraction,)
+    if gmtoff is not None:
+        tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction)
+        if tzname:
+            tz = datetime_timezone(tzdelta, tzname)
+        else:
+            tz = datetime_timezone(tzdelta)
+        args += (tz,)
+
+    return cls(*args)

Lib/asyncio/__init__.py

@@ -1,22 +1,14 @@
 """The asyncio package, tracking PEP 3156."""
 
 # flake8: noqa
+
 import sys
 
-import selectors
-# XXX RustPython TODO: _overlapped
-if sys.platform == 'win32' and False:
-    # Similar thing for _overlapped.
-    try:
-        from . import _overlapped
-    except ImportError:
-        import _overlapped  # Will also be exported.
-
-
 # This relies on each of the submodules having an __all__ variable.
 from .base_events import *
 from .coroutines import *
 from .events import *
+from .exceptions import *
 from .futures import *
 from .locks import *
 from .protocols import *
@@ -25,11 +17,15 @@ from .queues import *
 from .streams import *
 from .subprocess import *
 from .tasks import *
+from .taskgroups import *
+from .timeouts import *
+from .threads import *
 from .transports import *
 
 __all__ = (base_events.__all__ +
            coroutines.__all__ +
            events.__all__ +
+           exceptions.__all__ +
            futures.__all__ +
            locks.__all__ +
            protocols.__all__ +
@@ -38,6 +34,9 @@ __all__ = (base_events.__all__ +
            streams.__all__ +
            subprocess.__all__ +
            tasks.__all__ +
+           taskgroups.__all__ +
+           threads.__all__ +
+           timeouts.__all__ +
            transports.__all__)
 
 if sys.platform == 'win32':  # pragma: no cover

Lib/asyncio/__main__.py

@@ -0,0 +1,125 @@
+import ast
+import asyncio
+import code
+import concurrent.futures
+import inspect
+import sys
+import threading
+import types
+import warnings
+
+from . import futures
+
+
+class AsyncIOInteractiveConsole(code.InteractiveConsole):
+
+    def __init__(self, locals, loop):
+        super().__init__(locals)
+        self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
+
+        self.loop = loop
+
+    def runcode(self, code):
+        future = concurrent.futures.Future()
+
+        def callback():
+            global repl_future
+            global repl_future_interrupted
+
+            repl_future = None
+            repl_future_interrupted = False
+
+            func = types.FunctionType(code, self.locals)
+            try:
+                coro = func()
+            except SystemExit:
+                raise
+            except KeyboardInterrupt as ex:
+                repl_future_interrupted = True
+                future.set_exception(ex)
+                return
+            except BaseException as ex:
+                future.set_exception(ex)
+                return
+
+            if not inspect.iscoroutine(coro):
+                future.set_result(coro)
+                return
+
+            try:
+                repl_future = self.loop.create_task(coro)
+                futures._chain_future(repl_future, future)
+            except BaseException as exc:
+                future.set_exception(exc)
+
+        loop.call_soon_threadsafe(callback)
+
+        try:
+            return future.result()
+        except SystemExit:
+            raise
+        except BaseException:
+            if repl_future_interrupted:
+                self.write("\nKeyboardInterrupt\n")
+            else:
+                self.showtraceback()
+
+
+class REPLThread(threading.Thread):
+
+    def run(self):
+        try:
+            banner = (
+                f'asyncio REPL {sys.version} on {sys.platform}\n'
+                f'Use "await" directly instead of "asyncio.run()".\n'
+                f'Type "help", "copyright", "credits" or "license" '
+                f'for more information.\n'
+                f'{getattr(sys, "ps1", ">>> ")}import asyncio'
+            )
+
+            console.interact(
+                banner=banner,
+                exitmsg='exiting asyncio REPL...')
+        finally:
+            warnings.filterwarnings(
+                'ignore',
+                message=r'^coroutine .* was never awaited$',
+                category=RuntimeWarning)
+
+            loop.call_soon_threadsafe(loop.stop)
+
+
+if __name__ == '__main__':
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+
+    repl_locals = {'asyncio': asyncio}
+    for key in {'__name__', '__package__',
+                '__loader__', '__spec__',
+                '__builtins__', '__file__'}:
+        repl_locals[key] = locals()[key]
+
+    console = AsyncIOInteractiveConsole(repl_locals, loop)
+
+    repl_future = None
+    repl_future_interrupted = False
+
+    try:
+        import readline  # NoQA
+    except ImportError:
+        pass
+
+    repl_thread = REPLThread()
+    repl_thread.daemon = True
+    repl_thread.start()
+
+    while True:
+        try:
+            loop.run_forever()
+        except KeyboardInterrupt:
+            if repl_future and not repl_future.done():
+                repl_future.cancel()
+                repl_future_interrupted = True
+            continue
+        else:
+            break

Lib/asyncio/base_futures.py

@@ -1,18 +1,8 @@
-__all__ = []
+__all__ = ()
 
-import concurrent.futures._base
 import reprlib
 
-from . import events
+from . import format_helpers
-
-Error = concurrent.futures._base.Error
-CancelledError = concurrent.futures.CancelledError
-TimeoutError = concurrent.futures.TimeoutError
-
-
-class InvalidStateError(Error):
-    """The operation is not allowed in this state."""
-
 
 # States for Future.
 _PENDING = 'PENDING'
@@ -38,17 +28,17 @@ def _format_callbacks(cb):
         cb = ''
 
     def format_cb(callback):
-        return events._format_callback_source(callback, ())
+        return format_helpers._format_callback_source(callback, ())
 
     if size == 1:
-        cb = format_cb(cb[0])
+        cb = format_cb(cb[0][0])
     elif size == 2:
-        cb = '{}, {}'.format(format_cb(cb[0]), format_cb(cb[1]))
+        cb = '{}, {}'.format(format_cb(cb[0][0]), format_cb(cb[1][0]))
     elif size > 2:
-        cb = '{}, <{} more>, {}'.format(format_cb(cb[0]),
+        cb = '{}, <{} more>, {}'.format(format_cb(cb[0][0]),
                                         size - 2,
-                                        format_cb(cb[-1]))
+                                        format_cb(cb[-1][0]))
-    return 'cb=[%s]' % cb
+    return f'cb=[{cb}]'
 
 
 def _future_repr_info(future):
@@ -57,15 +47,21 @@ def _future_repr_info(future):
     info = [future._state.lower()]
     if future._state == _FINISHED:
         if future._exception is not None:
-            info.append('exception={!r}'.format(future._exception))
+            info.append(f'exception={future._exception!r}')
         else:
             # use reprlib to limit the length of the output, especially
             # for very long strings
             result = reprlib.repr(future._result)
-            info.append('result={}'.format(result))
+            info.append(f'result={result}')
     if future._callbacks:
         info.append(_format_callbacks(future._callbacks))
     if future._source_traceback:
         frame = future._source_traceback[-1]
-        info.append('created at %s:%s' % (frame[0], frame[1]))
+        info.append(f'created at {frame[0]}:{frame[1]}')
     return info
+
+
+@reprlib.recursive_repr()
+def _future_repr(future):
+    info = ' '.join(_future_repr_info(future))
+    return f'<{future.__class__.__name__} {info}>'

Lib/asyncio/base_subprocess.py

@@ -2,10 +2,8 @@ import collections
 import subprocess
 import warnings
 
-from . import compat
 from . import protocols
 from . import transports
-from .coroutines import coroutine
 from .log import logger
 
 
@@ -59,9 +57,9 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
         if self._closed:
             info.append('closed')
         if self._pid is not None:
-            info.append('pid=%s' % self._pid)
+            info.append(f'pid={self._pid}')
         if self._returncode is not None:
-            info.append('returncode=%s' % self._returncode)
+            info.append(f'returncode={self._returncode}')
         elif self._pid is not None:
             info.append('running')
         else:
@@ -69,19 +67,19 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
 
         stdin = self._pipes.get(0)
         if stdin is not None:
-            info.append('stdin=%s' % stdin.pipe)
+            info.append(f'stdin={stdin.pipe}')
 
         stdout = self._pipes.get(1)
         stderr = self._pipes.get(2)
         if stdout is not None and stderr is stdout:
-            info.append('stdout=stderr=%s' % stdout.pipe)
+            info.append(f'stdout=stderr={stdout.pipe}')
         else:
             if stdout is not None:
-                info.append('stdout=%s' % stdout.pipe)
+                info.append(f'stdout={stdout.pipe}')
             if stderr is not None:
-                info.append('stderr=%s' % stderr.pipe)
+                info.append(f'stderr={stderr.pipe}')
 
-        return '<%s>' % ' '.join(info)
+        return '<{}>'.format(' '.join(info))
 
     def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
         raise NotImplementedError
@@ -105,12 +103,13 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
                 continue
             proto.pipe.close()
 
-        if (self._proc is not None
+        if (self._proc is not None and
-        # the child process finished?
+                # has the child process finished?
-        and self._returncode is None
+                self._returncode is None and
-        # the child process finished but the transport was not notified yet?
+                # the child process has finished, but the
-        and self._proc.poll() is None
+                # transport hasn't been notified yet?
-        ):
+                self._proc.poll() is None):
+
             if self._loop.get_debug():
                 logger.warning('Close running child process: kill %r', self)
 
@@ -121,15 +120,10 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
 
             # Don't clear the _proc reference yet: _post_init() may still run
 
-    # On Python 3.3 and older, objects with a destructor part of a reference
+    def __del__(self, _warn=warnings.warn):
-    # cycle are never destroyed. It's not more the case on Python 3.4 thanks
+        if not self._closed:
-    # to the PEP 442.
+            _warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
-    if compat.PY34:
+            self.close()
-        def __del__(self):
-            if not self._closed:
-                warnings.warn("unclosed transport %r" % self, ResourceWarning,
-                              source=self)
-                self.close()
 
     def get_pid(self):
         return self._pid
@@ -159,26 +153,25 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
         self._check_proc()
         self._proc.kill()
 
-    @coroutine
+    async def _connect_pipes(self, waiter):
-    def _connect_pipes(self, waiter):
         try:
             proc = self._proc
             loop = self._loop
 
             if proc.stdin is not None:
-                _, pipe = yield from loop.connect_write_pipe(
+                _, pipe = await loop.connect_write_pipe(
                     lambda: WriteSubprocessPipeProto(self, 0),
                     proc.stdin)
                 self._pipes[0] = pipe
 
             if proc.stdout is not None:
-                _, pipe = yield from loop.connect_read_pipe(
+                _, pipe = await loop.connect_read_pipe(
                     lambda: ReadSubprocessPipeProto(self, 1),
                     proc.stdout)
                 self._pipes[1] = pipe
 
             if proc.stderr is not None:
-                _, pipe = yield from loop.connect_read_pipe(
+                _, pipe = await loop.connect_read_pipe(
                     lambda: ReadSubprocessPipeProto(self, 2),
                     proc.stderr)
                 self._pipes[2] = pipe
@@ -189,7 +182,9 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
             for callback, data in self._pending_calls:
                 loop.call_soon(callback, *data)
             self._pending_calls = None
-        except Exception as exc:
+        except (SystemExit, KeyboardInterrupt):
+            raise
+        except BaseException as exc:
             if waiter is not None and not waiter.cancelled():
                 waiter.set_exception(exc)
         else:
@@ -213,24 +208,17 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
         assert returncode is not None, returncode
         assert self._returncode is None, self._returncode
         if self._loop.get_debug():
-            logger.info('%r exited with return code %r',
+            logger.info('%r exited with return code %r', self, returncode)
-                        self, returncode)
         self._returncode = returncode
         if self._proc.returncode is None:
             # asyncio uses a child watcher: copy the status into the Popen
             # object. On Python 3.6, it is required to avoid a ResourceWarning.
             self._proc.returncode = returncode
         self._call(self._protocol.process_exited)
+
         self._try_finish()
 
-        # wake up futures waiting for wait()
+    async def _wait(self):
-        for waiter in self._exit_waiters:
-            if not waiter.cancelled():
-                waiter.set_result(returncode)
-        self._exit_waiters = None
-
-    @coroutine
-    def _wait(self):
         """Wait until the process exit and return the process return code.
 
         This method is a coroutine."""
@@ -239,7 +227,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
 
         waiter = self._loop.create_future()
         self._exit_waiters.append(waiter)
-        return (yield from waiter)
+        return await waiter
 
     def _try_finish(self):
         assert not self._finished
@@ -254,6 +242,11 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
         try:
             self._protocol.connection_lost(exc)
         finally:
+            # wake up futures waiting for wait()
+            for waiter in self._exit_waiters:
+                if not waiter.cancelled():
+                    waiter.set_result(self._returncode)
+            self._exit_waiters = None
             self._loop = None
             self._proc = None
             self._protocol = None
@@ -271,8 +264,7 @@ class WriteSubprocessPipeProto(protocols.BaseProtocol):
         self.pipe = transport
 
     def __repr__(self):
-        return ('<%s fd=%s pipe=%r>'
+        return f'<{self.__class__.__name__} fd={self.fd} pipe={self.pipe!r}>'
-                % (self.__class__.__name__, self.fd, self.pipe))
 
     def connection_lost(self, exc):
         self.disconnected = True

Lib/asyncio/base_tasks.py

@@ -1,4 +1,5 @@
 import linecache
+import reprlib
 import traceback
 
 from . import base_futures
@@ -8,25 +9,42 @@ from . import coroutines
 def _task_repr_info(task):
     info = base_futures._future_repr_info(task)
 
-    if task._must_cancel:
+    if task.cancelling() and not task.done():
         # replace status
         info[0] = 'cancelling'
 
-    coro = coroutines._format_coroutine(task._coro)
+    info.insert(1, 'name=%r' % task.get_name())
-    info.insert(1, 'coro=<%s>' % coro)
 
     if task._fut_waiter is not None:
-        info.insert(2, 'wait_for=%r' % task._fut_waiter)
+        info.insert(2, f'wait_for={task._fut_waiter!r}')
+
+    if task._coro:
+        coro = coroutines._format_coroutine(task._coro)
+        info.insert(2, f'coro=<{coro}>')
+
     return info
 
 
+@reprlib.recursive_repr()
+def _task_repr(task):
+    info = ' '.join(_task_repr_info(task))
+    return f'<{task.__class__.__name__} {info}>'
+
+
 def _task_get_stack(task, limit):
     frames = []
-    try:
+    if hasattr(task._coro, 'cr_frame'):
-        # 'async def' coroutines
+        # case 1: 'async def' coroutines
         f = task._coro.cr_frame
-    except AttributeError:
+    elif hasattr(task._coro, 'gi_frame'):
+        # case 2: legacy coroutines
         f = task._coro.gi_frame
+    elif hasattr(task._coro, 'ag_frame'):
+        # case 3: async generators
+        f = task._coro.ag_frame
+    else:
+        # case 4: unknown objects
+        f = None
     if f is not None:
         while f is not None:
             if limit is not None:
@@ -61,15 +79,15 @@ def _task_print_stack(task, limit, file):
             linecache.checkcache(filename)
         line = linecache.getline(filename, lineno, f.f_globals)
         extracted_list.append((filename, lineno, name, line))
+
     exc = task._exception
     if not extracted_list:
-        print('No stack for %r' % task, file=file)
+        print(f'No stack for {task!r}', file=file)
     elif exc is not None:
-        print('Traceback for %r (most recent call last):' % task,
+        print(f'Traceback for {task!r} (most recent call last):', file=file)
-              file=file)
     else:
-        print('Stack for %r (most recent call last):' % task,
+        print(f'Stack for {task!r} (most recent call last):', file=file)
-              file=file)
+
     traceback.print_list(extracted_list, file=file)
     if exc is not None:
         for line in traceback.format_exception_only(exc.__class__, exc):

Lib/asyncio/compat.py

@@ -1,18 +0,0 @@
-"""Compatibility helpers for the different Python versions."""
-
-import sys
-
-PY34 = sys.version_info >= (3, 4)
-PY35 = sys.version_info >= (3, 5)
-PY352 = sys.version_info >= (3, 5, 2)
-
-
-def flatten_list_bytes(list_of_data):
-    """Concatenate a sequence of bytes-like objects."""
-    if not PY34:
-        # On Python 3.3 and older, bytes.join() doesn't handle
-        # memoryview.
-        list_of_data = (
-            bytes(data) if isinstance(data, memoryview) else data
-            for data in list_of_data)
-    return b''.join(list_of_data)

Lib/asyncio/constants.py

@@ -1,7 +1,41 @@
-"""Constants."""
+# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0
+# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0)
+# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc.  http://magic.io
+
+import enum
 
 # After the connection is lost, log warnings after this many write()s.
 LOG_THRESHOLD_FOR_CONNLOST_WRITES = 5
 
 # Seconds to wait before retrying accept().
 ACCEPT_RETRY_DELAY = 1
+
+# Number of stack entries to capture in debug mode.
+# The larger the number, the slower the operation in debug mode
+# (see extract_stack() in format_helpers.py).
+DEBUG_STACK_DEPTH = 10
+
+# Number of seconds to wait for SSL handshake to complete
+# The default timeout matches that of Nginx.
+SSL_HANDSHAKE_TIMEOUT = 60.0
+
+# Number of seconds to wait for SSL shutdown to complete
+# The default timeout mimics lingering_time
+SSL_SHUTDOWN_TIMEOUT = 30.0
+
+# Used in sendfile fallback code.  We use fallback for platforms
+# that don't support sendfile, or for TLS connections.
+SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 256
+
+FLOW_CONTROL_HIGH_WATER_SSL_READ = 256  # KiB
+FLOW_CONTROL_HIGH_WATER_SSL_WRITE = 512  # KiB
+
+# Default timeout for joining the threads in the threadpool
+THREAD_JOIN_TIMEOUT = 300
+
+# The enum should be here to break circular dependencies between
+# base_events and sslproto
+class _SendfileMode(enum.Enum):
+    UNSUPPORTED = enum.auto()
+    TRY_NATIVE = enum.auto()
+    FALLBACK = enum.auto()

Lib/asyncio/coroutines.py

@@ -1,249 +1,16 @@
-__all__ = ['coroutine',
+__all__ = 'iscoroutinefunction', 'iscoroutine'
-           'iscoroutinefunction', 'iscoroutine']
 
-import functools
+import collections.abc
 import inspect
-import opcode
 import os
 import sys
-import traceback
 import types
 
-from . import compat
-from . import events
-from . import base_futures
-from .log import logger
 
-
+def _is_debug_mode():
-# Opcode of "yield from" instruction
+    # See: https://docs.python.org/3/library/asyncio-dev.html#asyncio-debug-mode.
-_YIELD_FROM = opcode.opmap['YIELD_FROM']
+    return sys.flags.dev_mode or (not sys.flags.ignore_environment and
-
+                                  bool(os.environ.get('PYTHONASYNCIODEBUG')))
-# If you set _DEBUG to true, @coroutine will wrap the resulting
-# generator objects in a CoroWrapper instance (defined below).  That
-# instance will log a message when the generator is never iterated
-# over, which may happen when you forget to use "yield from" with a
-# coroutine call.  Note that the value of the _DEBUG flag is taken
-# when the decorator is used, so to be of any use it must be set
-# before you define your coroutines.  A downside of using this feature
-# is that tracebacks show entries for the CoroWrapper.__next__ method
-# when _DEBUG is true.
-_DEBUG = (not sys.flags.ignore_environment and
-          bool(os.environ.get('PYTHONASYNCIODEBUG')))
-
-
-try:
-    _types_coroutine = types.coroutine
-    _types_CoroutineType = types.CoroutineType
-except AttributeError:
-    # Python 3.4
-    _types_coroutine = None
-    _types_CoroutineType = None
-
-try:
-    _inspect_iscoroutinefunction = inspect.iscoroutinefunction
-except AttributeError:
-    # Python 3.4
-    _inspect_iscoroutinefunction = lambda func: False
-
-try:
-    from collections.abc import Coroutine as _CoroutineABC, \
-                                Awaitable as _AwaitableABC
-except ImportError:
-    _CoroutineABC = _AwaitableABC = None
-
-
-# Check for CPython issue #21209
-def has_yield_from_bug():
-    class MyGen:
-        def __init__(self):
-            self.send_args = None
-        def __iter__(self):
-            return self
-        def __next__(self):
-            return 42
-        def send(self, *what):
-            self.send_args = what
-            return None
-    def yield_from_gen(gen):
-        yield from gen
-    value = (1, 2, 3)
-    gen = MyGen()
-    coro = yield_from_gen(gen)
-    next(coro)
-    coro.send(value)
-    return gen.send_args != (value,)
-_YIELD_FROM_BUG = has_yield_from_bug()
-del has_yield_from_bug
-
-
-def debug_wrapper(gen):
-    # This function is called from 'sys.set_coroutine_wrapper'.
-    # We only wrap here coroutines defined via 'async def' syntax.
-    # Generator-based coroutines are wrapped in @coroutine
-    # decorator.
-    return CoroWrapper(gen, None)
-
-
-class CoroWrapper:
-    # Wrapper for coroutine object in _DEBUG mode.
-
-    def __init__(self, gen, func=None):
-        assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen
-        self.gen = gen
-        self.func = func  # Used to unwrap @coroutine decorator
-        self._source_traceback = traceback.extract_stack(sys._getframe(1))
-        self.__name__ = getattr(gen, '__name__', None)
-        self.__qualname__ = getattr(gen, '__qualname__', None)
-
-    def __repr__(self):
-        coro_repr = _format_coroutine(self)
-        if self._source_traceback:
-            frame = self._source_traceback[-1]
-            coro_repr += ', created at %s:%s' % (frame[0], frame[1])
-        return '<%s %s>' % (self.__class__.__name__, coro_repr)
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        return self.gen.send(None)
-
-    if _YIELD_FROM_BUG:
-        # For for CPython issue #21209: using "yield from" and a custom
-        # generator, generator.send(tuple) unpacks the tuple instead of passing
-        # the tuple unchanged. Check if the caller is a generator using "yield
-        # from" to decide if the parameter should be unpacked or not.
-        def send(self, *value):
-            frame = sys._getframe()
-            caller = frame.f_back
-            assert caller.f_lasti >= 0
-            if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM:
-                value = value[0]
-            return self.gen.send(value)
-    else:
-        def send(self, value):
-            return self.gen.send(value)
-
-    def throw(self, type, value=None, traceback=None):
-        return self.gen.throw(type, value, traceback)
-
-    def close(self):
-        return self.gen.close()
-
-    @property
-    def gi_frame(self):
-        return self.gen.gi_frame
-
-    @property
-    def gi_running(self):
-        return self.gen.gi_running
-
-    @property
-    def gi_code(self):
-        return self.gen.gi_code
-
-    if compat.PY35:
-
-        def __await__(self):
-            cr_await = getattr(self.gen, 'cr_await', None)
-            if cr_await is not None:
-                raise RuntimeError(
-                    "Cannot await on coroutine {!r} while it's "
-                    "awaiting for {!r}".format(self.gen, cr_await))
-            return self
-
-        @property
-        def gi_yieldfrom(self):
-            return self.gen.gi_yieldfrom
-
-        @property
-        def cr_await(self):
-            return self.gen.cr_await
-
-        @property
-        def cr_running(self):
-            return self.gen.cr_running
-
-        @property
-        def cr_code(self):
-            return self.gen.cr_code
-
-        @property
-        def cr_frame(self):
-            return self.gen.cr_frame
-
-    def __del__(self):
-        # Be careful accessing self.gen.frame -- self.gen might not exist.
-        gen = getattr(self, 'gen', None)
-        frame = getattr(gen, 'gi_frame', None)
-        if frame is None:
-            frame = getattr(gen, 'cr_frame', None)
-        if frame is not None and frame.f_lasti == -1:
-            msg = '%r was never yielded from' % self
-            tb = getattr(self, '_source_traceback', ())
-            if tb:
-                tb = ''.join(traceback.format_list(tb))
-                msg += ('\nCoroutine object created at '
-                        '(most recent call last):\n')
-                msg += tb.rstrip()
-            logger.error(msg)
-
-
-def coroutine(func):
-    """Decorator to mark coroutines.
-
-    If the coroutine is not yielded from before it is destroyed,
-    an error message is logged.
-    """
-    if _inspect_iscoroutinefunction(func):
-        # In Python 3.5 that's all we need to do for coroutines
-        # defiend with "async def".
-        # Wrapping in CoroWrapper will happen via
-        # 'sys.set_coroutine_wrapper' function.
-        return func
-
-    if inspect.isgeneratorfunction(func):
-        coro = func
-    else:
-        @functools.wraps(func)
-        def coro(*args, **kw):
-            res = func(*args, **kw)
-            if (base_futures.isfuture(res) or inspect.isgenerator(res) or
-                isinstance(res, CoroWrapper)):
-                res = yield from res
-            elif _AwaitableABC is not None:
-                # If 'func' returns an Awaitable (new in 3.5) we
-                # want to run it.
-                try:
-                    await_meth = res.__await__
-                except AttributeError:
-                    pass
-                else:
-                    if isinstance(res, _AwaitableABC):
-                        res = yield from await_meth()
-            return res
-
-    if not _DEBUG:
-        if _types_coroutine is None:
-            wrapper = coro
-        else:
-            wrapper = _types_coroutine(coro)
-    else:
-        @functools.wraps(func)
-        def wrapper(*args, **kwds):
-            w = CoroWrapper(coro(*args, **kwds), func=func)
-            if w._source_traceback:
-                del w._source_traceback[-1]
-            # Python < 3.5 does not implement __qualname__
-            # on generator objects, so we set it manually.
-            # We use getattr as some callables (such as
-            # functools.partial may lack __qualname__).
-            w.__name__ = getattr(func, '__name__', None)
-            w.__qualname__ = getattr(func, '__qualname__', None)
-            return w
-
-    wrapper._is_coroutine = _is_coroutine  # For iscoroutinefunction().
-    return wrapper
 
 
 # A marker for iscoroutinefunction.
@@ -252,93 +19,91 @@ _is_coroutine = object()
 
 def iscoroutinefunction(func):
     """Return True if func is a decorated coroutine function."""
-    return (getattr(func, '_is_coroutine', None) is _is_coroutine or
+    return (inspect.iscoroutinefunction(func) or
-            _inspect_iscoroutinefunction(func))
+            getattr(func, '_is_coroutine', None) is _is_coroutine)
 
 
-_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper)
+# Prioritize native coroutine check to speed-up
-if _CoroutineABC is not None:
+# asyncio.iscoroutine.
-    _COROUTINE_TYPES += (_CoroutineABC,)
+_COROUTINE_TYPES = (types.CoroutineType, collections.abc.Coroutine)
-if _types_CoroutineType is not None:
+_iscoroutine_typecache = set()
-    # Prioritize native coroutine check to speed-up
-    # asyncio.iscoroutine.
-    _COROUTINE_TYPES = (_types_CoroutineType,) + _COROUTINE_TYPES
 
 
 def iscoroutine(obj):
     """Return True if obj is a coroutine object."""
-    return isinstance(obj, _COROUTINE_TYPES)
+    if type(obj) in _iscoroutine_typecache:
+        return True
+
+    if isinstance(obj, _COROUTINE_TYPES):
+        # Just in case we don't want to cache more than 100
+        # positive types.  That shouldn't ever happen, unless
+        # someone stressing the system on purpose.
+        if len(_iscoroutine_typecache) < 100:
+            _iscoroutine_typecache.add(type(obj))
+        return True
+    else:
+        return False
 
 
 def _format_coroutine(coro):
     assert iscoroutine(coro)
 
-    if not hasattr(coro, 'cr_code') and not hasattr(coro, 'gi_code'):
+    def get_name(coro):
-        # Most likely a built-in type or a Cython coroutine.
+        # Coroutines compiled with Cython sometimes don't have
+        # proper __qualname__ or __name__.  While that is a bug
+        # in Cython, asyncio shouldn't crash with an AttributeError
+        # in its __repr__ functions.
+        if hasattr(coro, '__qualname__') and coro.__qualname__:
+            coro_name = coro.__qualname__
+        elif hasattr(coro, '__name__') and coro.__name__:
+            coro_name = coro.__name__
+        else:
+            # Stop masking Cython bugs, expose them in a friendly way.
+            coro_name = f'<{type(coro).__name__} without __name__>'
+        return f'{coro_name}()'
 
-        # Built-in types might not have __qualname__ or __name__.
+    def is_running(coro):
-        coro_name = getattr(
-            coro, '__qualname__',
-            getattr(coro, '__name__', type(coro).__name__))
-        coro_name = '{}()'.format(coro_name)
-
-        running = False
         try:
-            running = coro.cr_running
+            return coro.cr_running
         except AttributeError:
             try:
-                running = coro.gi_running
+                return coro.gi_running
             except AttributeError:
-                pass
+                return False
 
-        if running:
+    coro_code = None
-            return '{} running'.format(coro_name)
+    if hasattr(coro, 'cr_code') and coro.cr_code:
+        coro_code = coro.cr_code
+    elif hasattr(coro, 'gi_code') and coro.gi_code:
+        coro_code = coro.gi_code
+
+    coro_name = get_name(coro)
+
+    if not coro_code:
+        # Built-in types might not have __qualname__ or __name__.
+        if is_running(coro):
+            return f'{coro_name} running'
         else:
             return coro_name
 
-    coro_name = None
+    coro_frame = None
-    if isinstance(coro, CoroWrapper):
+    if hasattr(coro, 'gi_frame') and coro.gi_frame:
-        func = coro.func
-        coro_name = coro.__qualname__
-        if coro_name is not None:
-            coro_name = '{}()'.format(coro_name)
-    else:
-        func = coro
-
-    if coro_name is None:
-        coro_name = events._format_callback(func, (), {})
-
-    try:
-        coro_code = coro.gi_code
-    except AttributeError:
-        coro_code = coro.cr_code
-
-    try:
         coro_frame = coro.gi_frame
-    except AttributeError:
+    elif hasattr(coro, 'cr_frame') and coro.cr_frame:
         coro_frame = coro.cr_frame
 
-    filename = coro_code.co_filename
+    # If Cython's coroutine has a fake code object without proper
+    # co_filename -- expose that.
+    filename = coro_code.co_filename or '<empty co_filename>'
+
     lineno = 0
-    if (isinstance(coro, CoroWrapper) and
+
-            not inspect.isgeneratorfunction(coro.func) and
+    if coro_frame is not None:
-            coro.func is not None):
-        source = events._get_function_source(coro.func)
-        if source is not None:
-            filename, lineno = source
-        if coro_frame is None:
-            coro_repr = ('%s done, defined at %s:%s'
-                         % (coro_name, filename, lineno))
-        else:
-            coro_repr = ('%s running, defined at %s:%s'
-                         % (coro_name, filename, lineno))
-    elif coro_frame is not None:
         lineno = coro_frame.f_lineno
-        coro_repr = ('%s running at %s:%s'
+        coro_repr = f'{coro_name} running at {filename}:{lineno}'
-                     % (coro_name, filename, lineno))
+
     else:
         lineno = coro_code.co_firstlineno
-        coro_repr = ('%s done, defined at %s:%s'
+        coro_repr = f'{coro_name} done, defined at {filename}:{lineno}'
-                     % (coro_name, filename, lineno))
 
     return coro_repr

Lib/asyncio/events.py

@@ -1,96 +1,50 @@
 """Event loop and event loop policy."""
 
-__all__ = ['AbstractEventLoopPolicy',
+# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0
-           'AbstractEventLoop', 'AbstractServer',
+# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0)
-           'Handle', 'TimerHandle',
+# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc.  http://magic.io
-           'get_event_loop_policy', 'set_event_loop_policy',
-           'get_event_loop', 'set_event_loop', 'new_event_loop',
-           'get_child_watcher', 'set_child_watcher',
-           '_set_running_loop', 'get_running_loop',
-           '_get_running_loop',
-           ]
 
-import functools
+__all__ = (
-import inspect
+    'AbstractEventLoopPolicy',
-import reprlib
+    'AbstractEventLoop', 'AbstractServer',
+    'Handle', 'TimerHandle',
+    'get_event_loop_policy', 'set_event_loop_policy',
+    'get_event_loop', 'set_event_loop', 'new_event_loop',
+    'get_child_watcher', 'set_child_watcher',
+    '_set_running_loop', 'get_running_loop',
+    '_get_running_loop',
+)
+
+import contextvars
+import os
+import signal
 import socket
 import subprocess
 import sys
 import threading
-import traceback
 
-from asyncio import compat
+from . import format_helpers
-
-
-def _get_function_source(func):
-    if compat.PY34:
-        func = inspect.unwrap(func)
-    elif hasattr(func, '__wrapped__'):
-        func = func.__wrapped__
-    if inspect.isfunction(func):
-        code = func.__code__
-        return (code.co_filename, code.co_firstlineno)
-    if isinstance(func, functools.partial):
-        return _get_function_source(func.func)
-    if compat.PY34 and isinstance(func, functools.partialmethod):
-        return _get_function_source(func.func)
-    return None
-
-
-def _format_args_and_kwargs(args, kwargs):
-    """Format function arguments and keyword arguments.
-
-    Special case for a single parameter: ('hello',) is formatted as ('hello').
-    """
-    # use reprlib to limit the length of the output
-    items = []
-    if args:
-        items.extend(reprlib.repr(arg) for arg in args)
-    if kwargs:
-        items.extend('{}={}'.format(k, reprlib.repr(v))
-                     for k, v in kwargs.items())
-    return '(' + ', '.join(items) + ')'
-
-
-def _format_callback(func, args, kwargs, suffix=''):
-    if isinstance(func, functools.partial):
-        suffix = _format_args_and_kwargs(args, kwargs) + suffix
-        return _format_callback(func.func, func.args, func.keywords, suffix)
-
-    if hasattr(func, '__qualname__'):
-        func_repr = getattr(func, '__qualname__')
-    elif hasattr(func, '__name__'):
-        func_repr = getattr(func, '__name__')
-    else:
-        func_repr = repr(func)
-
-    func_repr += _format_args_and_kwargs(args, kwargs)
-    if suffix:
-        func_repr += suffix
-    return func_repr
-
-def _format_callback_source(func, args):
-    func_repr = _format_callback(func, args, None)
-    source = _get_function_source(func)
-    if source:
-        func_repr += ' at %s:%s' % source
-    return func_repr
 
 
 class Handle:
     """Object returned by callback registration methods."""
 
     __slots__ = ('_callback', '_args', '_cancelled', '_loop',
-                 '_source_traceback', '_repr', '__weakref__')
+                 '_source_traceback', '_repr', '__weakref__',
+                 '_context')
 
-    def __init__(self, callback, args, loop):
+    def __init__(self, callback, args, loop, context=None):
+        if context is None:
+            context = contextvars.copy_context()
+        self._context = context
         self._loop = loop
         self._callback = callback
         self._args = args
         self._cancelled = False
         self._repr = None
         if self._loop.get_debug():
-            self._source_traceback = traceback.extract_stack(sys._getframe(1))
+            self._source_traceback = format_helpers.extract_stack(
+                sys._getframe(1))
         else:
             self._source_traceback = None
 
@@ -99,17 +53,21 @@ class Handle:
         if self._cancelled:
             info.append('cancelled')
         if self._callback is not None:
-            info.append(_format_callback_source(self._callback, self._args))
+            info.append(format_helpers._format_callback_source(
+                self._callback, self._args))
         if self._source_traceback:
             frame = self._source_traceback[-1]
-            info.append('created at %s:%s' % (frame[0], frame[1]))
+            info.append(f'created at {frame[0]}:{frame[1]}')
         return info
 
     def __repr__(self):
         if self._repr is not None:
             return self._repr
         info = self._repr_info()
-        return '<%s>' % ' '.join(info)
+        return '<{}>'.format(' '.join(info))
+
+    def get_context(self):
+        return self._context
 
     def cancel(self):
         if not self._cancelled:
@@ -122,12 +80,18 @@ class Handle:
             self._callback = None
             self._args = None
 
+    def cancelled(self):
+        return self._cancelled
+
     def _run(self):
         try:
-            self._callback(*self._args)
+            self._context.run(self._callback, *self._args)
-        except Exception as exc:
+        except (SystemExit, KeyboardInterrupt):
-            cb = _format_callback_source(self._callback, self._args)
+            raise
-            msg = 'Exception in callback {}'.format(cb)
+        except BaseException as exc:
+            cb = format_helpers._format_callback_source(
+                self._callback, self._args)
+            msg = f'Exception in callback {cb}'
             context = {
                 'message': msg,
                 'exception': exc,
@@ -144,9 +108,8 @@ class TimerHandle(Handle):
 
     __slots__ = ['_scheduled', '_when']
 
-    def __init__(self, when, callback, args, loop):
+    def __init__(self, when, callback, args, loop, context=None):
-        assert when is not None
+        super().__init__(callback, args, loop, context)
-        super().__init__(callback, args, loop)
         if self._source_traceback:
             del self._source_traceback[-1]
         self._when = when
@@ -155,27 +118,31 @@ class TimerHandle(Handle):
     def _repr_info(self):
         info = super()._repr_info()
         pos = 2 if self._cancelled else 1
-        info.insert(pos, 'when=%s' % self._when)
+        info.insert(pos, f'when={self._when}')
         return info
 
     def __hash__(self):
         return hash(self._when)
 
     def __lt__(self, other):
-        return self._when < other._when
+        if isinstance(other, TimerHandle):
+            return self._when < other._when
+        return NotImplemented
 
     def __le__(self, other):
-        if self._when < other._when:
+        if isinstance(other, TimerHandle):
-            return True
+            return self._when < other._when or self.__eq__(other)
-        return self.__eq__(other)
+        return NotImplemented
 
     def __gt__(self, other):
-        return self._when > other._when
+        if isinstance(other, TimerHandle):
+            return self._when > other._when
+        return NotImplemented
 
     def __ge__(self, other):
-        if self._when > other._when:
+        if isinstance(other, TimerHandle):
-            return True
+            return self._when > other._when or self.__eq__(other)
-        return self.__eq__(other)
+        return NotImplemented
 
     def __eq__(self, other):
         if isinstance(other, TimerHandle):
@@ -185,26 +152,60 @@ class TimerHandle(Handle):
                     self._cancelled == other._cancelled)
         return NotImplemented
 
-    def __ne__(self, other):
-        equal = self.__eq__(other)
-        return NotImplemented if equal is NotImplemented else not equal
-
     def cancel(self):
         if not self._cancelled:
             self._loop._timer_handle_cancelled(self)
         super().cancel()
 
+    def when(self):
+        """Return a scheduled callback time.
+
+        The time is an absolute timestamp, using the same time
+        reference as loop.time().
+        """
+        return self._when
+
 
 class AbstractServer:
     """Abstract server returned by create_server()."""
 
     def close(self):
         """Stop serving.  This leaves existing connections open."""
-        return NotImplemented
+        raise NotImplementedError
 
-    def wait_closed(self):
+    def get_loop(self):
+        """Get the event loop the Server object is attached to."""
+        raise NotImplementedError
+
+    def is_serving(self):
+        """Return True if the server is accepting connections."""
+        raise NotImplementedError
+
+    async def start_serving(self):
+        """Start accepting connections.
+
+        This method is idempotent, so it can be called when
+        the server is already being serving.
+        """
+        raise NotImplementedError
+
+    async def serve_forever(self):
+        """Start accepting connections until the coroutine is cancelled.
+
+        The server is closed when the coroutine is cancelled.
+        """
+        raise NotImplementedError
+
+    async def wait_closed(self):
         """Coroutine to wait until service is closed."""
-        return NotImplemented
+        raise NotImplementedError
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        self.close()
+        await self.wait_closed()
 
 
 class AbstractEventLoop:
@@ -250,23 +251,27 @@ class AbstractEventLoop:
         """
         raise NotImplementedError
 
-    def shutdown_asyncgens(self):
+    async def shutdown_asyncgens(self):
         """Shutdown all active asynchronous generators."""
         raise NotImplementedError
 
+    async def shutdown_default_executor(self):
+        """Schedule the shutdown of the default executor."""
+        raise NotImplementedError
+
     # Methods scheduling callbacks.  All these return Handles.
 
     def _timer_handle_cancelled(self, handle):
         """Notification that a TimerHandle has been cancelled."""
         raise NotImplementedError
 
-    def call_soon(self, callback, *args):
+    def call_soon(self, callback, *args, context=None):
-        return self.call_later(0, callback, *args)
+        return self.call_later(0, callback, *args, context=context)
 
-    def call_later(self, delay, callback, *args):
+    def call_later(self, delay, callback, *args, context=None):
         raise NotImplementedError
 
-    def call_at(self, when, callback, *args):
+    def call_at(self, when, callback, *args, context=None):
         raise NotImplementedError
 
     def time(self):
@@ -277,12 +282,12 @@ class AbstractEventLoop:
 
     # Method scheduling a coroutine object: create a task.
 
-    def create_task(self, coro):
+    def create_task(self, coro, *, name=None, context=None):
         raise NotImplementedError
 
     # Methods for interacting with threads.
 
-    def call_soon_threadsafe(self, callback, *args):
+    def call_soon_threadsafe(self, callback, *args, context=None):
         raise NotImplementedError
 
     def run_in_executor(self, executor, func, *args):
@@ -293,21 +298,31 @@ class AbstractEventLoop:
 
     # Network I/O methods returning Futures.
 
-    def getaddrinfo(self, host, port, *, family=0, type=0, proto=0, flags=0):
+    async def getaddrinfo(self, host, port, *,
+                          family=0, type=0, proto=0, flags=0):
         raise NotImplementedError
 
-    def getnameinfo(self, sockaddr, flags=0):
+    async def getnameinfo(self, sockaddr, flags=0):
         raise NotImplementedError
 
-    def create_connection(self, protocol_factory, host=None, port=None, *,
+    async def create_connection(
-                          ssl=None, family=0, proto=0, flags=0, sock=None,
+            self, protocol_factory, host=None, port=None,
-                          local_addr=None, server_hostname=None):
+            *, ssl=None, family=0, proto=0,
+            flags=0, sock=None, local_addr=None,
+            server_hostname=None,
+            ssl_handshake_timeout=None,
+            ssl_shutdown_timeout=None,
+            happy_eyeballs_delay=None, interleave=None):
         raise NotImplementedError
 
-    def create_server(self, protocol_factory, host=None, port=None, *,
+    async def create_server(
-                      family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE,
+            self, protocol_factory, host=None, port=None,
-                      sock=None, backlog=100, ssl=None, reuse_address=None,
+            *, family=socket.AF_UNSPEC,
-                      reuse_port=None):
+            flags=socket.AI_PASSIVE, sock=None, backlog=100,
+            ssl=None, reuse_address=None, reuse_port=None,
+            ssl_handshake_timeout=None,
+            ssl_shutdown_timeout=None,
+            start_serving=True):
         """A coroutine which creates a TCP server bound to host and port.
 
         The return value is a Server object which can be used to stop
@@ -315,8 +330,8 @@ class AbstractEventLoop:
 
         If host is an empty string or None all interfaces are assumed
         and a list of multiple sockets will be returned (most likely
-        one for IPv4 and another one for IPv6). The host parameter can also be a
+        one for IPv4 and another one for IPv6). The host parameter can also be
-        sequence (e.g. list) of hosts to bind to.
+        a sequence (e.g. list) of hosts to bind to.
 
         family can be set to either AF_INET or AF_INET6 to force the
         socket to use IPv4 or IPv6. If not set it will be determined
@@ -342,22 +357,62 @@ class AbstractEventLoop:
         the same port as other existing endpoints are bound to, so long as
         they all set this flag when being created. This option is not
         supported on Windows.
+
+        ssl_handshake_timeout is the time in seconds that an SSL server
+        will wait for completion of the SSL handshake before aborting the
+        connection. Default is 60s.
+
+        ssl_shutdown_timeout is the time in seconds that an SSL server
+        will wait for completion of the SSL shutdown procedure
+        before aborting the connection. Default is 30s.
+
+        start_serving set to True (default) causes the created server
+        to start accepting connections immediately.  When set to False,
+        the user should await Server.start_serving() or Server.serve_forever()
+        to make the server to start accepting connections.
         """
         raise NotImplementedError
 
-    def create_unix_connection(self, protocol_factory, path, *,
+    async def sendfile(self, transport, file, offset=0, count=None,
-                               ssl=None, sock=None,
+                       *, fallback=True):
-                               server_hostname=None):
+        """Send a file through a transport.
+
+        Return an amount of sent bytes.
+        """
         raise NotImplementedError
 
-    def create_unix_server(self, protocol_factory, path, *,
+    async def start_tls(self, transport, protocol, sslcontext, *,
-                           sock=None, backlog=100, ssl=None):
+                        server_side=False,
+                        server_hostname=None,
+                        ssl_handshake_timeout=None,
+                        ssl_shutdown_timeout=None):
+        """Upgrade a transport to TLS.
+
+        Return a new transport that *protocol* should start using
+        immediately.
+        """
+        raise NotImplementedError
+
+    async def create_unix_connection(
+            self, protocol_factory, path=None, *,
+            ssl=None, sock=None,
+            server_hostname=None,
+            ssl_handshake_timeout=None,
+            ssl_shutdown_timeout=None):
+        raise NotImplementedError
+
+    async def create_unix_server(
+            self, protocol_factory, path=None, *,
+            sock=None, backlog=100, ssl=None,
+            ssl_handshake_timeout=None,
+            ssl_shutdown_timeout=None,
+            start_serving=True):
         """A coroutine which creates a UNIX Domain Socket server.
 
         The return value is a Server object, which can be used to stop
         the service.
 
-        path is a str, representing a file systsem path to bind the
+        path is a str, representing a file system path to bind the
         server socket to.
 
         sock can optionally be specified in order to use a preexisting
@@ -368,14 +423,40 @@ class AbstractEventLoop:
 
         ssl can be set to an SSLContext to enable SSL over the
         accepted connections.
+
+        ssl_handshake_timeout is the time in seconds that an SSL server
+        will wait for the SSL handshake to complete (defaults to 60s).
+
+        ssl_shutdown_timeout is the time in seconds that an SSL server
+        will wait for the SSL shutdown to finish (defaults to 30s).
+
+        start_serving set to True (default) causes the created server
+        to start accepting connections immediately.  When set to False,
+        the user should await Server.start_serving() or Server.serve_forever()
+        to make the server to start accepting connections.
         """
         raise NotImplementedError
 
-    def create_datagram_endpoint(self, protocol_factory,
+    async def connect_accepted_socket(
-                                 local_addr=None, remote_addr=None, *,
+            self, protocol_factory, sock,
-                                 family=0, proto=0, flags=0,
+            *, ssl=None,
-                                 reuse_address=None, reuse_port=None,
+            ssl_handshake_timeout=None,
-                                 allow_broadcast=None, sock=None):
+            ssl_shutdown_timeout=None):
+        """Handle an accepted connection.
+
+        This is used by servers that accept connections outside of
+        asyncio, but use asyncio to handle connections.
+
+        This method is a coroutine.  When completed, the coroutine
+        returns a (transport, protocol) pair.
+        """
+        raise NotImplementedError
+
+    async def create_datagram_endpoint(self, protocol_factory,
+                                       local_addr=None, remote_addr=None, *,
+                                       family=0, proto=0, flags=0,
+                                       reuse_address=None, reuse_port=None,
+                                       allow_broadcast=None, sock=None):
         """A coroutine which creates a datagram endpoint.
 
         This method will try to establish the endpoint in the background.
@@ -383,8 +464,8 @@ class AbstractEventLoop:
 
         protocol_factory must be a callable returning a protocol instance.
 
-        socket family AF_INET or socket.AF_INET6 depending on host (or
+        socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending on
-        family if specified), socket type SOCK_DGRAM.
+        host (or family if specified), socket type SOCK_DGRAM.
 
         reuse_address tells the kernel to reuse a local socket in
         TIME_WAIT state, without waiting for its natural timeout to
@@ -408,7 +489,7 @@ class AbstractEventLoop:
 
     # Pipes and subprocesses.
 
-    def connect_read_pipe(self, protocol_factory, pipe):
+    async def connect_read_pipe(self, protocol_factory, pipe):
         """Register read pipe in event loop. Set the pipe to non-blocking mode.
 
         protocol_factory should instantiate object with Protocol interface.
@@ -418,10 +499,10 @@ class AbstractEventLoop:
         # The reason to accept file-like object instead of just file descriptor
         # is: we need to own pipe and close it at transport finishing
         # Can got complicated errors if pass f.fileno(),
-        # close fd in pipe transport then close f and vise versa.
+        # close fd in pipe transport then close f and vice versa.
         raise NotImplementedError
 
-    def connect_write_pipe(self, protocol_factory, pipe):
+    async def connect_write_pipe(self, protocol_factory, pipe):
         """Register write pipe in event loop.
 
         protocol_factory should instantiate object with BaseProtocol interface.
@@ -431,17 +512,21 @@ class AbstractEventLoop:
         # The reason to accept file-like object instead of just file descriptor
         # is: we need to own pipe and close it at transport finishing
         # Can got complicated errors if pass f.fileno(),
-        # close fd in pipe transport then close f and vise versa.
+        # close fd in pipe transport then close f and vice versa.
         raise NotImplementedError
 
-    def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
+    async def subprocess_shell(self, protocol_factory, cmd, *,
-                         stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                               stdin=subprocess.PIPE,
-                         **kwargs):
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE,
+                               **kwargs):
         raise NotImplementedError
 
-    def subprocess_exec(self, protocol_factory, *args, stdin=subprocess.PIPE,
+    async def subprocess_exec(self, protocol_factory, *args,
-                        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                              stdin=subprocess.PIPE,
-                        **kwargs):
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE,
+                              **kwargs):
         raise NotImplementedError
 
     # Ready-based callback registration methods.
@@ -463,16 +548,32 @@ class AbstractEventLoop:
 
     # Completion based I/O methods returning Futures.
 
-    def sock_recv(self, sock, nbytes):
+    async def sock_recv(self, sock, nbytes):
         raise NotImplementedError
 
-    def sock_sendall(self, sock, data):
+    async def sock_recv_into(self, sock, buf):
         raise NotImplementedError
 
-    def sock_connect(self, sock, address):
+    async def sock_recvfrom(self, sock, bufsize):
         raise NotImplementedError
 
-    def sock_accept(self, sock):
+    async def sock_recvfrom_into(self, sock, buf, nbytes=0):
+        raise NotImplementedError
+
+    async def sock_sendall(self, sock, data):
+        raise NotImplementedError
+
+    async def sock_sendto(self, sock, data, address):
+        raise NotImplementedError
+
+    async def sock_connect(self, sock, address):
+        raise NotImplementedError
+
+    async def sock_accept(self, sock):
+        raise NotImplementedError
+
+    async def sock_sendfile(self, sock, file, offset=0, count=None,
+                            *, fallback=None):
         raise NotImplementedError
 
     # Signal handling.
@@ -520,7 +621,7 @@ class AbstractEventLoopPolicy:
     def get_event_loop(self):
         """Get the event loop for the current context.
 
-        Returns an event loop object implementing the BaseEventLoop interface,
+        Returns an event loop object implementing the AbstractEventLoop interface,
         or raises an exception in case no event loop has been set for the
         current context and the current policy does not specify to create one.
 
@@ -571,23 +672,43 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy):
         self._local = self._Local()
 
     def get_event_loop(self):
-        """Get the event loop.
+        """Get the event loop for the current context.
 
-        This may be None or an instance of EventLoop.
+        Returns an instance of EventLoop or raises an exception.
         """
         if (self._local._loop is None and
-            not self._local._set_called and
+                not self._local._set_called and
-            isinstance(threading.current_thread(), threading._MainThread)):
+                threading.current_thread() is threading.main_thread()):
+            stacklevel = 2
+            try:
+                f = sys._getframe(1)
+            except AttributeError:
+                pass
+            else:
+                # Move up the call stack so that the warning is attached
+                # to the line outside asyncio itself.
+                while f:
+                    module = f.f_globals.get('__name__')
+                    if not (module == 'asyncio' or module.startswith('asyncio.')):
+                        break
+                    f = f.f_back
+                    stacklevel += 1
+            import warnings
+            warnings.warn('There is no current event loop',
+                          DeprecationWarning, stacklevel=stacklevel)
             self.set_event_loop(self.new_event_loop())
+
         if self._local._loop is None:
             raise RuntimeError('There is no current event loop in thread %r.'
                                % threading.current_thread().name)
+
         return self._local._loop
 
     def set_event_loop(self, loop):
         """Set the event loop."""
         self._local._set_called = True
-        assert loop is None or isinstance(loop, AbstractEventLoop)
+        if loop is not None and not isinstance(loop, AbstractEventLoop):
+            raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'")
         self._local._loop = loop
 
     def new_event_loop(self):
@@ -611,7 +732,9 @@ _lock = threading.Lock()
 
 # A TLS for the running event loop, used by _get_running_loop.
 class _RunningLoop(threading.local):
-    _loop = None
+    loop_pid = (None, None)
+
+
 _running_loop = _RunningLoop()
 
 
@@ -633,7 +756,10 @@ def _get_running_loop():
     This is a low-level function intended to be used by event loops.
     This function is thread-specific.
     """
-    return _running_loop._loop
+    # NOTE: this function is implemented in C (see _asynciomodule.c)
+    running_loop, pid = _running_loop.loop_pid
+    if running_loop is not None and pid == os.getpid():
+        return running_loop
 
 
 def _set_running_loop(loop):
@@ -642,7 +768,8 @@ def _set_running_loop(loop):
     This is a low-level function intended to be used by event loops.
     This function is thread-specific.
     """
-    _running_loop._loop = loop
+    # NOTE: this function is implemented in C (see _asynciomodule.c)
+    _running_loop.loop_pid = (loop, os.getpid())
 
 
 def _init_event_loop_policy():
@@ -665,7 +792,8 @@ def set_event_loop_policy(policy):
 
     If policy is None, the default policy is restored."""
     global _event_loop_policy
-    assert policy is None or isinstance(policy, AbstractEventLoopPolicy)
+    if policy is not None and not isinstance(policy, AbstractEventLoopPolicy):
+        raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'")
     _event_loop_policy = policy
 
 
@@ -678,6 +806,7 @@ def get_event_loop():
     If there is no running event loop set, the function will return
     the result of `get_event_loop_policy().get_event_loop()` call.
     """
+    # NOTE: this function is implemented in C (see _asynciomodule.c)
     current_loop = _get_running_loop()
     if current_loop is not None:
         return current_loop
@@ -703,3 +832,37 @@ def set_child_watcher(watcher):
     """Equivalent to calling
     get_event_loop_policy().set_child_watcher(watcher)."""
     return get_event_loop_policy().set_child_watcher(watcher)
+
+
+# Alias pure-Python implementations for testing purposes.
+_py__get_running_loop = _get_running_loop
+_py__set_running_loop = _set_running_loop
+_py_get_running_loop = get_running_loop
+_py_get_event_loop = get_event_loop
+
+
+try:
+    # get_event_loop() is one of the most frequently called
+    # functions in asyncio.  Pure Python implementation is
+    # about 4 times slower than C-accelerated.
+    from _asyncio import (_get_running_loop, _set_running_loop,
+                          get_running_loop, get_event_loop)
+except ImportError:
+    pass
+else:
+    # Alias C implementations for testing purposes.
+    _c__get_running_loop = _get_running_loop
+    _c__set_running_loop = _set_running_loop
+    _c_get_running_loop = get_running_loop
+    _c_get_event_loop = get_event_loop
+
+
+if hasattr(os, 'fork'):
+    def on_fork():
+        # Reset the loop and wakeupfd in the forked child process.
+        if _event_loop_policy is not None:
+            _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local()
+        _set_running_loop(None)
+        signal.set_wakeup_fd(-1)
+
+    os.register_at_fork(after_in_child=on_fork)

Lib/asyncio/exceptions.py

@@ -0,0 +1,62 @@
+"""asyncio exceptions."""
+
+
+__all__ = ('BrokenBarrierError',
+           'CancelledError', 'InvalidStateError', 'TimeoutError',
+           'IncompleteReadError', 'LimitOverrunError',
+           'SendfileNotAvailableError')
+
+
+class CancelledError(BaseException):
+    """The Future or Task was cancelled."""
+
+
+TimeoutError = TimeoutError  # make local alias for the standard exception
+
+
+class InvalidStateError(Exception):
+    """The operation is not allowed in this state."""
+
+
+class SendfileNotAvailableError(RuntimeError):
+    """Sendfile syscall is not available.
+
+    Raised if OS does not support sendfile syscall for given socket or
+    file type.
+    """
+
+
+class IncompleteReadError(EOFError):
+    """
+    Incomplete read error. Attributes:
+
+    - partial: read bytes string before the end of stream was reached
+    - expected: total number of expected bytes (or None if unknown)
+    """
+    def __init__(self, partial, expected):
+        r_expected = 'undefined' if expected is None else repr(expected)
+        super().__init__(f'{len(partial)} bytes read on a total of '
+                         f'{r_expected} expected bytes')
+        self.partial = partial
+        self.expected = expected
+
+    def __reduce__(self):
+        return type(self), (self.partial, self.expected)
+
+
+class LimitOverrunError(Exception):
+    """Reached the buffer limit while looking for a separator.
+
+    Attributes:
+    - consumed: total number of to be consumed bytes.
+    """
+    def __init__(self, message, consumed):
+        super().__init__(message)
+        self.consumed = consumed
+
+    def __reduce__(self):
+        return type(self), (self.args[0], self.consumed)
+
+
+class BrokenBarrierError(RuntimeError):
+    """Barrier is broken by barrier.abort() call."""

Lib/asyncio/format_helpers.py

@@ -0,0 +1,76 @@
+import functools
+import inspect
+import reprlib
+import sys
+import traceback
+
+from . import constants
+
+
+def _get_function_source(func):
+    func = inspect.unwrap(func)
+    if inspect.isfunction(func):
+        code = func.__code__
+        return (code.co_filename, code.co_firstlineno)
+    if isinstance(func, functools.partial):
+        return _get_function_source(func.func)
+    if isinstance(func, functools.partialmethod):
+        return _get_function_source(func.func)
+    return None
+
+
+def _format_callback_source(func, args):
+    func_repr = _format_callback(func, args, None)
+    source = _get_function_source(func)
+    if source:
+        func_repr += f' at {source[0]}:{source[1]}'
+    return func_repr
+
+
+def _format_args_and_kwargs(args, kwargs):
+    """Format function arguments and keyword arguments.
+
+    Special case for a single parameter: ('hello',) is formatted as ('hello').
+    """
+    # use reprlib to limit the length of the output
+    items = []
+    if args:
+        items.extend(reprlib.repr(arg) for arg in args)
+    if kwargs:
+        items.extend(f'{k}={reprlib.repr(v)}' for k, v in kwargs.items())
+    return '({})'.format(', '.join(items))
+
+
+def _format_callback(func, args, kwargs, suffix=''):
+    if isinstance(func, functools.partial):
+        suffix = _format_args_and_kwargs(args, kwargs) + suffix
+        return _format_callback(func.func, func.args, func.keywords, suffix)
+
+    if hasattr(func, '__qualname__') and func.__qualname__:
+        func_repr = func.__qualname__
+    elif hasattr(func, '__name__') and func.__name__:
+        func_repr = func.__name__
+    else:
+        func_repr = repr(func)
+
+    func_repr += _format_args_and_kwargs(args, kwargs)
+    if suffix:
+        func_repr += suffix
+    return func_repr
+
+
+def extract_stack(f=None, limit=None):
+    """Replacement for traceback.extract_stack() that only does the
+    necessary work for asyncio debug mode.
+    """
+    if f is None:
+        f = sys._getframe().f_back
+    if limit is None:
+        # Limit the amount of work to a reasonable amount, as extract_stack()
+        # can be called for each coroutine and future in debug mode.
+        limit = constants.DEBUG_STACK_DEPTH
+    stack = traceback.StackSummary.extract(traceback.walk_stack(f),
+                                           limit=limit,
+                                           lookup_lines=False)
+    stack.reverse()
+    return stack

Lib/asyncio/futures.py

@@ -1,21 +1,21 @@
 """A Future class similar to the one in PEP 3148."""
 
-__all__ = ['CancelledError', 'TimeoutError', 'InvalidStateError',
+__all__ = (
-           'Future', 'wrap_future', 'isfuture']
+    'Future', 'wrap_future', 'isfuture',
+)
 
 import concurrent.futures
+import contextvars
 import logging
 import sys
-import traceback
+from types import GenericAlias
 
 from . import base_futures
-from . import compat
 from . import events
+from . import exceptions
+from . import format_helpers
 
 
-CancelledError = base_futures.CancelledError
-InvalidStateError = base_futures.InvalidStateError
-TimeoutError = base_futures.TimeoutError
 isfuture = base_futures.isfuture
 
 
@@ -27,96 +27,18 @@ _FINISHED = base_futures._FINISHED
 STACK_DEBUG = logging.DEBUG - 1  # heavy-duty debugging
 
 
-class _TracebackLogger:
-    """Helper to log a traceback upon destruction if not cleared.
-
-    This solves a nasty problem with Futures and Tasks that have an
-    exception set: if nobody asks for the exception, the exception is
-    never logged.  This violates the Zen of Python: 'Errors should
-    never pass silently.  Unless explicitly silenced.'
-
-    However, we don't want to log the exception as soon as
-    set_exception() is called: if the calling code is written
-    properly, it will get the exception and handle it properly.  But
-    we *do* want to log it if result() or exception() was never called
-    -- otherwise developers waste a lot of time wondering why their
-    buggy code fails silently.
-
-    An earlier attempt added a __del__() method to the Future class
-    itself, but this backfired because the presence of __del__()
-    prevents garbage collection from breaking cycles.  A way out of
-    this catch-22 is to avoid having a __del__() method on the Future
-    class itself, but instead to have a reference to a helper object
-    with a __del__() method that logs the traceback, where we ensure
-    that the helper object doesn't participate in cycles, and only the
-    Future has a reference to it.
-
-    The helper object is added when set_exception() is called.  When
-    the Future is collected, and the helper is present, the helper
-    object is also collected, and its __del__() method will log the
-    traceback.  When the Future's result() or exception() method is
-    called (and a helper object is present), it removes the helper
-    object, after calling its clear() method to prevent it from
-    logging.
-
-    One downside is that we do a fair amount of work to extract the
-    traceback from the exception, even when it is never logged.  It
-    would seem cheaper to just store the exception object, but that
-    references the traceback, which references stack frames, which may
-    reference the Future, which references the _TracebackLogger, and
-    then the _TracebackLogger would be included in a cycle, which is
-    what we're trying to avoid!  As an optimization, we don't
-    immediately format the exception; we only do the work when
-    activate() is called, which call is delayed until after all the
-    Future's callbacks have run.  Since usually a Future has at least
-    one callback (typically set by 'yield from') and usually that
-    callback extracts the callback, thereby removing the need to
-    format the exception.
-
-    PS. I don't claim credit for this solution.  I first heard of it
-    in a discussion about closing files when they are collected.
-    """
-
-    __slots__ = ('loop', 'source_traceback', 'exc', 'tb')
-
-    def __init__(self, future, exc):
-        self.loop = future._loop
-        self.source_traceback = future._source_traceback
-        self.exc = exc
-        self.tb = None
-
-    def activate(self):
-        exc = self.exc
-        if exc is not None:
-            self.exc = None
-            self.tb = traceback.format_exception(exc.__class__, exc,
-                                                 exc.__traceback__)
-
-    def clear(self):
-        self.exc = None
-        self.tb = None
-
-    def __del__(self):
-        if self.tb:
-            msg = 'Future/Task exception was never retrieved\n'
-            if self.source_traceback:
-                src = ''.join(traceback.format_list(self.source_traceback))
-                msg += 'Future/Task created at (most recent call last):\n'
-                msg += '%s\n' % src.rstrip()
-            msg += ''.join(self.tb).rstrip()
-            self.loop.call_exception_handler({'message': msg})
-
-
 class Future:
     """This class is *almost* compatible with concurrent.futures.Future.
 
     Differences:
 
+    - This class is not thread-safe.
+
     - result() and exception() do not take a timeout argument and
       raise an exception when the future isn't done yet.
 
     - Callbacks registered with add_done_callback() are always called
-      via the event loop's call_soon_threadsafe().
+      via the event loop's call_soon().
 
     - This class is not compatible with the wait() and as_completed()
       methods in the concurrent.futures package.
@@ -130,6 +52,9 @@ class Future:
     _exception = None
     _loop = None
     _source_traceback = None
+    _cancel_message = None
+    # A saved CancelledError for later chaining as an exception context.
+    _cancelled_exc = None
 
     # This field is used for a dual purpose:
     # - Its presence is a marker to declare that a class implements
@@ -137,12 +62,12 @@ class Future:
     #   The value must also be not-None, to enable a subclass to declare
     #   that it is not compatible by setting this to None.
     # - It is set by __iter__() below so that Task._step() can tell
-    #   the difference between `yield from Future()` (correct) vs.
+    #   the difference between
+    #   `await Future()` or`yield from Future()` (correct) vs.
     #   `yield Future()` (incorrect).
     _asyncio_future_blocking = False
 
-    _log_traceback = False   # Used for Python 3.4 and later
+    __log_traceback = False
-    _tb_logger = None        # Used for Python 3.3 only
 
     def __init__(self, *, loop=None):
         """Initialize the future.
@@ -157,50 +82,83 @@ class Future:
             self._loop = loop
         self._callbacks = []
         if self._loop.get_debug():
-            self._source_traceback = traceback.extract_stack(sys._getframe(1))
+            self._source_traceback = format_helpers.extract_stack(
-
+                sys._getframe(1))
-    _repr_info = base_futures._future_repr_info
 
     def __repr__(self):
-        return '<%s %s>' % (self.__class__.__name__, ' '.join(self._repr_info()))
+        return base_futures._future_repr(self)
 
-    # On Python 3.3 and older, objects with a destructor part of a reference
+    def __del__(self):
-    # cycle are never destroyed. It's not more the case on Python 3.4 thanks
+        if not self.__log_traceback:
-    # to the PEP 442.
+            # set_exception() was not called, or result() or exception()
-    if compat.PY34:
+            # has consumed the exception
-        def __del__(self):
+            return
-            if not self._log_traceback:
+        exc = self._exception
-                # set_exception() was not called, or result() or exception()
+        context = {
-                # has consumed the exception
+            'message':
-                return
+                f'{self.__class__.__name__} exception was never retrieved',
-            exc = self._exception
+            'exception': exc,
-            context = {
+            'future': self,
-                'message': ('%s exception was never retrieved'
+        }
-                            % self.__class__.__name__),
+        if self._source_traceback:
-                'exception': exc,
+            context['source_traceback'] = self._source_traceback
-                'future': self,
+        self._loop.call_exception_handler(context)
-            }
-            if self._source_traceback:
-                context['source_traceback'] = self._source_traceback
-            self._loop.call_exception_handler(context)
 
-    def __class_getitem__(cls, type):
+    __class_getitem__ = classmethod(GenericAlias)
-        return cls
 
-    def cancel(self):
+    @property
+    def _log_traceback(self):
+        return self.__log_traceback
+
+    @_log_traceback.setter
+    def _log_traceback(self, val):
+        if val:
+            raise ValueError('_log_traceback can only be set to False')
+        self.__log_traceback = False
+
+    def get_loop(self):
+        """Return the event loop the Future is bound to."""
+        loop = self._loop
+        if loop is None:
+            raise RuntimeError("Future object is not initialized.")
+        return loop
+
+    def _make_cancelled_error(self):
+        """Create the CancelledError to raise if the Future is cancelled.
+
+        This should only be called once when handling a cancellation since
+        it erases the saved context exception value.
+        """
+        if self._cancelled_exc is not None:
+            exc = self._cancelled_exc
+            self._cancelled_exc = None
+            return exc
+
+        if self._cancel_message is None:
+            exc = exceptions.CancelledError()
+        else:
+            exc = exceptions.CancelledError(self._cancel_message)
+        exc.__context__ = self._cancelled_exc
+        # Remove the reference since we don't need this anymore.
+        self._cancelled_exc = None
+        return exc
+
+    def cancel(self, msg=None):
         """Cancel the future and schedule callbacks.
 
         If the future is already done or cancelled, return False.  Otherwise,
         change the future's state to cancelled, schedule the callbacks and
         return True.
         """
+        self.__log_traceback = False
         if self._state != _PENDING:
             return False
         self._state = _CANCELLED
-        self._schedule_callbacks()
+        self._cancel_message = msg
+        self.__schedule_callbacks()
         return True
 
-    def _schedule_callbacks(self):
+    def __schedule_callbacks(self):
         """Internal: Ask the event loop to call all callbacks.
 
         The callbacks are scheduled to be called as soon as possible. Also
@@ -211,8 +169,8 @@ class Future:
             return
 
         self._callbacks[:] = []
-        for callback in callbacks:
+        for callback, ctx in callbacks:
-            self._loop.call_soon(callback, self)
+            self._loop.call_soon(callback, self, context=ctx)
 
     def cancelled(self):
         """Return True if the future was cancelled."""
@@ -236,15 +194,13 @@ class Future:
         the future is done and has an exception set, this exception is raised.
         """
         if self._state == _CANCELLED:
-            raise CancelledError
+            exc = self._make_cancelled_error()
+            raise exc
         if self._state != _FINISHED:
-            raise InvalidStateError('Result is not ready.')
+            raise exceptions.InvalidStateError('Result is not ready.')
-        self._log_traceback = False
+        self.__log_traceback = False
-        if self._tb_logger is not None:
-            self._tb_logger.clear()
-            self._tb_logger = None
         if self._exception is not None:
-            raise self._exception
+            raise self._exception.with_traceback(self._exception_tb)
         return self._result
 
     def exception(self):
@@ -256,16 +212,14 @@ class Future:
         InvalidStateError.
         """
         if self._state == _CANCELLED:
-            raise CancelledError
+            exc = self._make_cancelled_error()
+            raise exc
         if self._state != _FINISHED:
-            raise InvalidStateError('Exception is not set.')
+            raise exceptions.InvalidStateError('Exception is not set.')
-        self._log_traceback = False
+        self.__log_traceback = False
-        if self._tb_logger is not None:
-            self._tb_logger.clear()
-            self._tb_logger = None
         return self._exception
 
-    def add_done_callback(self, fn):
+    def add_done_callback(self, fn, *, context=None):
         """Add a callback to be run when the future becomes done.
 
         The callback is called with a single argument - the future object. If
@@ -273,9 +227,11 @@ class Future:
         scheduled with call_soon.
         """
         if self._state != _PENDING:
-            self._loop.call_soon(fn, self)
+            self._loop.call_soon(fn, self, context=context)
         else:
-            self._callbacks.append(fn)
+            if context is None:
+                context = contextvars.copy_context()
+            self._callbacks.append((fn, context))
 
     # New method not in PEP 3148.
 
@@ -284,7 +240,9 @@ class Future:
 
         Returns the number of callbacks removed.
         """
-        filtered_callbacks = [f for f in self._callbacks if f != fn]
+        filtered_callbacks = [(f, ctx)
+                              for (f, ctx) in self._callbacks
+                              if f != fn]
         removed_count = len(self._callbacks) - len(filtered_callbacks)
         if removed_count:
             self._callbacks[:] = filtered_callbacks
@@ -299,10 +257,10 @@ class Future:
         InvalidStateError.
         """
         if self._state != _PENDING:
-            raise InvalidStateError('{}: {!r}'.format(self._state, self))
+            raise exceptions.InvalidStateError(f'{self._state}: {self!r}')
         self._result = result
         self._state = _FINISHED
-        self._schedule_callbacks()
+        self.__schedule_callbacks()
 
     def set_exception(self, exception):
         """Mark the future done and set an exception.
@@ -311,38 +269,45 @@ class Future:
         InvalidStateError.
         """
         if self._state != _PENDING:
-            raise InvalidStateError('{}: {!r}'.format(self._state, self))
+            raise exceptions.InvalidStateError(f'{self._state}: {self!r}')
         if isinstance(exception, type):
             exception = exception()
         if type(exception) is StopIteration:
             raise TypeError("StopIteration interacts badly with generators "
                             "and cannot be raised into a Future")
         self._exception = exception
+        self._exception_tb = exception.__traceback__
         self._state = _FINISHED
-        self._schedule_callbacks()
+        self.__schedule_callbacks()
-        if compat.PY34:
+        self.__log_traceback = True
-            self._log_traceback = True
-        else:
-            self._tb_logger = _TracebackLogger(self, exception)
-            # Arrange for the logger to be activated after all callbacks
-            # have had a chance to call result() or exception().
-            self._loop.call_soon(self._tb_logger.activate)
 
-    def __iter__(self):
+    def __await__(self):
         if not self.done():
             self._asyncio_future_blocking = True
             yield self  # This tells Task to wait for completion.
-        assert self.done(), "yield from wasn't used with future"
+        if not self.done():
+            raise RuntimeError("await wasn't used with future")
         return self.result()  # May raise too.
 
-    if compat.PY35:
+    __iter__ = __await__  # make compatible with 'yield from'.
-        __await__ = __iter__ # make compatible with 'await' expression
 
 
 # Needed for testing purposes.
 _PyFuture = Future
 
 
+def _get_loop(fut):
+    # Tries to call Future.get_loop() if it's available.
+    # Otherwise fallbacks to using the old '_loop' property.
+    try:
+        get_loop = fut.get_loop
+    except AttributeError:
+        pass
+    else:
+        return get_loop()
+    return fut._loop
+
+
 def _set_result_unless_cancelled(fut, result):
     """Helper setting the result only if the future was not cancelled."""
     if fut.cancelled():
@@ -350,6 +315,18 @@ def _set_result_unless_cancelled(fut, result):
     fut.set_result(result)
 
 
+def _convert_future_exc(exc):
+    exc_class = type(exc)
+    if exc_class is concurrent.futures.CancelledError:
+        return exceptions.CancelledError(*exc.args)
+    elif exc_class is concurrent.futures.TimeoutError:
+        return exceptions.TimeoutError(*exc.args)
+    elif exc_class is concurrent.futures.InvalidStateError:
+        return exceptions.InvalidStateError(*exc.args)
+    else:
+        return exc
+
+
 def _set_concurrent_future_state(concurrent, source):
     """Copy state from a future to a concurrent.futures.Future."""
     assert source.done()
@@ -359,7 +336,7 @@ def _set_concurrent_future_state(concurrent, source):
         return
     exception = source.exception()
     if exception is not None:
-        concurrent.set_exception(exception)
+        concurrent.set_exception(_convert_future_exc(exception))
     else:
         result = source.result()
         concurrent.set_result(result)
@@ -379,7 +356,7 @@ def _copy_future_state(source, dest):
     else:
         exception = source.exception()
         if exception is not None:
-            dest.set_exception(exception)
+            dest.set_exception(_convert_future_exc(exception))
         else:
             result = source.result()
             dest.set_result(result)
@@ -398,8 +375,8 @@ def _chain_future(source, destination):
     if not isfuture(destination) and not isinstance(destination,
                                                     concurrent.futures.Future):
         raise TypeError('A future is required for destination argument')
-    source_loop = source._loop if isfuture(source) else None
+    source_loop = _get_loop(source) if isfuture(source) else None
-    dest_loop = destination._loop if isfuture(destination) else None
+    dest_loop = _get_loop(destination) if isfuture(destination) else None
 
     def _set_state(future, other):
         if isfuture(future):
@@ -415,9 +392,14 @@ def _chain_future(source, destination):
                 source_loop.call_soon_threadsafe(source.cancel)
 
     def _call_set_state(source):
+        if (destination.cancelled() and
+                dest_loop is not None and dest_loop.is_closed()):
+            return
         if dest_loop is None or dest_loop is source_loop:
             _set_state(destination, source)
         else:
+            if dest_loop.is_closed():
+                return
             dest_loop.call_soon_threadsafe(_set_state, destination, source)
 
     destination.add_done_callback(_call_check_cancel)
@@ -429,7 +411,7 @@ def wrap_future(future, *, loop=None):
     if isfuture(future):
         return future
     assert isinstance(future, concurrent.futures.Future), \
-        'concurrent.futures.Future is expected, got {!r}'.format(future)
+        f'concurrent.futures.Future is expected, got {future!r}'
     if loop is None:
         loop = events.get_event_loop()
     new_future = loop.create_future()

Lib/asyncio/locks.py

@@ -1,92 +1,26 @@
 """Synchronization primitives."""
 
-__all__ = ['Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore']
+__all__ = ('Lock', 'Event', 'Condition', 'Semaphore',
+           'BoundedSemaphore', 'Barrier')
 
 import collections
+import enum
 
-from . import compat
+from . import exceptions
-from . import events
+from . import mixins
-from . import futures
-from .coroutines import coroutine
 
-
+class _ContextManagerMixin:
-class _ContextManager:
+    async def __aenter__(self):
-    """Context manager.
+        await self.acquire()
-
-    This enables the following idiom for acquiring and releasing a
-    lock around a block:
-
-        with (yield from lock):
-            <block>
-
-    while failing loudly when accidentally using:
-
-        with lock:
-            <block>
-    """
-
-    def __init__(self, lock):
-        self._lock = lock
-
-    def __enter__(self):
         # We have no use for the "as ..."  clause in the with
         # statement for locks.
         return None
 
-    def __exit__(self, *args):
+    async def __aexit__(self, exc_type, exc, tb):
-        try:
+        self.release()
-            self._lock.release()
-        finally:
-            self._lock = None  # Crudely prevent reuse.
 
 
-class _ContextManagerMixin:
+class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
-    def __enter__(self):
-        raise RuntimeError(
-            '"yield from" should be used as context manager expression')
-
-    def __exit__(self, *args):
-        # This must exist because __enter__ exists, even though that
-        # always raises; that's how the with-statement works.
-        pass
-
-    @coroutine
-    def __iter__(self):
-        # This is not a coroutine.  It is meant to enable the idiom:
-        #
-        #     with (yield from lock):
-        #         <block>
-        #
-        # as an alternative to:
-        #
-        #     yield from lock.acquire()
-        #     try:
-        #         <block>
-        #     finally:
-        #         lock.release()
-        yield from self.acquire()
-        return _ContextManager(self)
-
-    if compat.PY35:
-
-        def __await__(self):
-            # To make "with await lock" work.
-            yield from self.acquire()
-            return _ContextManager(self)
-
-        @coroutine
-        def __aenter__(self):
-            yield from self.acquire()
-            # We have no use for the "as ..."  clause in the with
-            # statement for locks.
-            return None
-
-        @coroutine
-        def __aexit__(self, exc_type, exc, tb):
-            self.release()
-
-
-class Lock(_ContextManagerMixin):
     """Primitive lock objects.
 
     A primitive lock is a synchronization primitive that is not owned
@@ -108,16 +42,16 @@ class Lock(_ContextManagerMixin):
     release() call resets the state to unlocked; first coroutine which
     is blocked in acquire() is being processed.
 
-    acquire() is a coroutine and should be called with 'yield from'.
+    acquire() is a coroutine and should be called with 'await'.
 
-    Locks also support the context management protocol.  '(yield from lock)'
+    Locks also support the asynchronous context management protocol.
-    should be used as the context manager expression.
+    'async with lock' statement should be used.
 
     Usage:
 
         lock = Lock()
         ...
-        yield from lock
+        await lock.acquire()
         try:
             ...
         finally:
@@ -127,57 +61,65 @@ class Lock(_ContextManagerMixin):
 
         lock = Lock()
         ...
-        with (yield from lock):
+        async with lock:
              ...
 
     Lock objects can be tested for locking state:
 
         if not lock.locked():
-           yield from lock
+           await lock.acquire()
         else:
            # lock is acquired
            ...
 
     """
 
-    def __init__(self, *, loop=None):
+    def __init__(self):
-        self._waiters = collections.deque()
+        self._waiters = None
         self._locked = False
-        if loop is not None:
-            self._loop = loop
-        else:
-            self._loop = events.get_event_loop()
 
     def __repr__(self):
         res = super().__repr__()
         extra = 'locked' if self._locked else 'unlocked'
         if self._waiters:
-            extra = '{},waiters:{}'.format(extra, len(self._waiters))
+            extra = f'{extra}, waiters:{len(self._waiters)}'
-        return '<{} [{}]>'.format(res[1:-1], extra)
+        return f'<{res[1:-1]} [{extra}]>'
 
     def locked(self):
         """Return True if lock is acquired."""
         return self._locked
 
-    @coroutine
+    async def acquire(self):
-    def acquire(self):
         """Acquire a lock.
 
         This method blocks until the lock is unlocked, then sets it to
         locked and returns True.
         """
-        if not self._locked and all(w.cancelled() for w in self._waiters):
+        if (not self._locked and (self._waiters is None or
+                all(w.cancelled() for w in self._waiters))):
             self._locked = True
             return True
 
-        fut = self._loop.create_future()
+        if self._waiters is None:
+            self._waiters = collections.deque()
+        fut = self._get_loop().create_future()
         self._waiters.append(fut)
+
+        # Finally block should be called before the CancelledError
+        # handling as we don't want CancelledError to call
+        # _wake_up_first() and attempt to wake up itself.
         try:
-            yield from fut
+            try:
-            self._locked = True
+                await fut
-            return True
+            finally:
-        finally:
+                self._waiters.remove(fut)
-            self._waiters.remove(fut)
+        except exceptions.CancelledError:
+            if not self._locked:
+                self._wake_up_first()
+            raise
+
+        self._locked = True
+        return True
 
     def release(self):
         """Release a lock.
@@ -192,16 +134,27 @@ class Lock(_ContextManagerMixin):
         """
         if self._locked:
             self._locked = False
-            # Wake up the first waiter who isn't cancelled.
+            self._wake_up_first()
-            for fut in self._waiters:
-                if not fut.done():
-                    fut.set_result(True)
-                    break
         else:
             raise RuntimeError('Lock is not acquired.')
 
+    def _wake_up_first(self):
+        """Wake up the first waiter if it isn't done."""
+        if not self._waiters:
+            return
+        try:
+            fut = next(iter(self._waiters))
+        except StopIteration:
+            return
 
-class Event:
+        # .done() necessarily means that a waiter will wake up later on and
+        # either take the lock, or, if it was cancelled and lock wasn't
+        # taken already, will hit this again and wake up a new waiter.
+        if not fut.done():
+            fut.set_result(True)
+
+
+class Event(mixins._LoopBoundMixin):
     """Asynchronous equivalent to threading.Event.
 
     Class implementing event objects. An event manages a flag that can be set
@@ -210,20 +163,16 @@ class Event:
     false.
     """
 
-    def __init__(self, *, loop=None):
+    def __init__(self):
         self._waiters = collections.deque()
         self._value = False
-        if loop is not None:
-            self._loop = loop
-        else:
-            self._loop = events.get_event_loop()
 
     def __repr__(self):
         res = super().__repr__()
         extra = 'set' if self._value else 'unset'
         if self._waiters:
-            extra = '{},waiters:{}'.format(extra, len(self._waiters))
+            extra = f'{extra}, waiters:{len(self._waiters)}'
-        return '<{} [{}]>'.format(res[1:-1], extra)
+        return f'<{res[1:-1]} [{extra}]>'
 
     def is_set(self):
         """Return True if and only if the internal flag is true."""
@@ -247,8 +196,7 @@ class Event:
         to true again."""
         self._value = False
 
-    @coroutine
+    async def wait(self):
-    def wait(self):
         """Block until the internal flag is true.
 
         If the internal flag is true on entry, return True
@@ -258,16 +206,16 @@ class Event:
         if self._value:
             return True
 
-        fut = self._loop.create_future()
+        fut = self._get_loop().create_future()
         self._waiters.append(fut)
         try:
-            yield from fut
+            await fut
             return True
         finally:
             self._waiters.remove(fut)
 
 
-class Condition(_ContextManagerMixin):
+class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
     """Asynchronous equivalent to threading.Condition.
 
     This class implements condition variable objects. A condition variable
@@ -277,16 +225,9 @@ class Condition(_ContextManagerMixin):
     A new Lock object is created and used as the underlying lock.
     """
 
-    def __init__(self, lock=None, *, loop=None):
+    def __init__(self, lock=None):
-        if loop is not None:
-            self._loop = loop
-        else:
-            self._loop = events.get_event_loop()
-
         if lock is None:
-            lock = Lock(loop=self._loop)
+            lock = Lock()
-        elif lock._loop is not self._loop:
-            raise ValueError("loop argument must agree with lock")
 
         self._lock = lock
         # Export the lock's locked(), acquire() and release() methods.
@@ -300,11 +241,10 @@ class Condition(_ContextManagerMixin):
         res = super().__repr__()
         extra = 'locked' if self.locked() else 'unlocked'
         if self._waiters:
-            extra = '{},waiters:{}'.format(extra, len(self._waiters))
+            extra = f'{extra}, waiters:{len(self._waiters)}'
-        return '<{} [{}]>'.format(res[1:-1], extra)
+        return f'<{res[1:-1]} [{extra}]>'
 
-    @coroutine
+    async def wait(self):
-    def wait(self):
         """Wait until notified.
 
         If the calling coroutine has not acquired the lock when this
@@ -320,25 +260,28 @@ class Condition(_ContextManagerMixin):
 
         self.release()
         try:
-            fut = self._loop.create_future()
+            fut = self._get_loop().create_future()
             self._waiters.append(fut)
             try:
-                yield from fut
+                await fut
                 return True
             finally:
                 self._waiters.remove(fut)
 
         finally:
             # Must reacquire lock even if wait is cancelled
+            cancelled = False
             while True:
                 try:
-                    yield from self.acquire()
+                    await self.acquire()
                     break
-                except futures.CancelledError:
+                except exceptions.CancelledError:
-                    pass
+                    cancelled = True
 
-    @coroutine
+            if cancelled:
-    def wait_for(self, predicate):
+                raise exceptions.CancelledError
+
+    async def wait_for(self, predicate):
         """Wait until a predicate becomes true.
 
         The predicate should be a callable which result will be
@@ -347,7 +290,7 @@ class Condition(_ContextManagerMixin):
         """
         result = predicate()
         while not result:
-            yield from self.wait()
+            await self.wait()
             result = predicate()
         return result
 
@@ -384,7 +327,7 @@ class Condition(_ContextManagerMixin):
         self.notify(len(self._waiters))
 
 
-class Semaphore(_ContextManagerMixin):
+class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin):
     """A Semaphore implementation.
 
     A semaphore manages an internal counter which is decremented by each
@@ -399,37 +342,25 @@ class Semaphore(_ContextManagerMixin):
     ValueError is raised.
     """
 
-    def __init__(self, value=1, *, loop=None):
+    def __init__(self, value=1):
         if value < 0:
             raise ValueError("Semaphore initial value must be >= 0")
+        self._waiters = None
         self._value = value
-        self._waiters = collections.deque()
-        if loop is not None:
-            self._loop = loop
-        else:
-            self._loop = events.get_event_loop()
 
     def __repr__(self):
         res = super().__repr__()
-        extra = 'locked' if self.locked() else 'unlocked,value:{}'.format(
+        extra = 'locked' if self.locked() else f'unlocked, value:{self._value}'
-            self._value)
         if self._waiters:
-            extra = '{},waiters:{}'.format(extra, len(self._waiters))
+            extra = f'{extra}, waiters:{len(self._waiters)}'
-        return '<{} [{}]>'.format(res[1:-1], extra)
+        return f'<{res[1:-1]} [{extra}]>'
-
-    def _wake_up_next(self):
-        while self._waiters:
-            waiter = self._waiters.popleft()
-            if not waiter.done():
-                waiter.set_result(None)
-                return
 
     def locked(self):
-        """Returns True if semaphore can not be acquired immediately."""
+        """Returns True if semaphore cannot be acquired immediately."""
-        return self._value == 0
+        return self._value == 0 or (
+            any(not w.cancelled() for w in (self._waiters or ())))
 
-    @coroutine
+    async def acquire(self):
-    def acquire(self):
         """Acquire a semaphore.
 
         If the internal counter is larger than zero on entry,
@@ -438,28 +369,53 @@ class Semaphore(_ContextManagerMixin):
         called release() to make it larger than 0, and then return
         True.
         """
-        while self._value <= 0:
+        if not self.locked():
-            fut = self._loop.create_future()
+            self._value -= 1
-            self._waiters.append(fut)
+            return True
+
+        if self._waiters is None:
+            self._waiters = collections.deque()
+        fut = self._get_loop().create_future()
+        self._waiters.append(fut)
+
+        # Finally block should be called before the CancelledError
+        # handling as we don't want CancelledError to call
+        # _wake_up_first() and attempt to wake up itself.
+        try:
             try:
-                yield from fut
+                await fut
-            except:
+            finally:
-                # See the similar code in Queue.get.
+                self._waiters.remove(fut)
-                fut.cancel()
+        except exceptions.CancelledError:
-                if self._value > 0 and not fut.cancelled():
+            if not fut.cancelled():
-                    self._wake_up_next()
+                self._value += 1
-                raise
+                self._wake_up_next()
-        self._value -= 1
+            raise
+
+        if self._value > 0:
+            self._wake_up_next()
         return True
 
     def release(self):
         """Release a semaphore, incrementing the internal counter by one.
+
         When it was zero on entry and another coroutine is waiting for it to
         become larger than zero again, wake up that coroutine.
         """
         self._value += 1
         self._wake_up_next()
 
+    def _wake_up_next(self):
+        """Wake up the first waiter that isn't done."""
+        if not self._waiters:
+            return
+
+        for fut in self._waiters:
+            if not fut.done():
+                self._value -= 1
+                fut.set_result(True)
+                return
+
 
 class BoundedSemaphore(Semaphore):
     """A bounded semaphore implementation.
@@ -468,11 +424,163 @@ class BoundedSemaphore(Semaphore):
     above the initial value.
     """
 
-    def __init__(self, value=1, *, loop=None):
+    def __init__(self, value=1):
         self._bound_value = value
-        super().__init__(value, loop=loop)
+        super().__init__(value)
 
     def release(self):
         if self._value >= self._bound_value:
             raise ValueError('BoundedSemaphore released too many times')
         super().release()
+
+
+
+class _BarrierState(enum.Enum):
+    FILLING = 'filling'
+    DRAINING = 'draining'
+    RESETTING = 'resetting'
+    BROKEN = 'broken'
+
+
+class Barrier(mixins._LoopBoundMixin):
+    """Asyncio equivalent to threading.Barrier
+
+    Implements a Barrier primitive.
+    Useful for synchronizing a fixed number of tasks at known synchronization
+    points. Tasks block on 'wait()' and are simultaneously awoken once they
+    have all made their call.
+    """
+
+    def __init__(self, parties):
+        """Create a barrier, initialised to 'parties' tasks."""
+        if parties < 1:
+            raise ValueError('parties must be > 0')
+
+        self._cond = Condition() # notify all tasks when state changes
+
+        self._parties = parties
+        self._state = _BarrierState.FILLING
+        self._count = 0       # count tasks in Barrier
+
+    def __repr__(self):
+        res = super().__repr__()
+        extra = f'{self._state.value}'
+        if not self.broken:
+            extra += f', waiters:{self.n_waiting}/{self.parties}'
+        return f'<{res[1:-1]} [{extra}]>'
+
+    async def __aenter__(self):
+        # wait for the barrier reaches the parties number
+        # when start draining release and return index of waited task
+        return await self.wait()
+
+    async def __aexit__(self, *args):
+        pass
+
+    async def wait(self):
+        """Wait for the barrier.
+
+        When the specified number of tasks have started waiting, they are all
+        simultaneously awoken.
+        Returns an unique and individual index number from 0 to 'parties-1'.
+        """
+        async with self._cond:
+            await self._block() # Block while the barrier drains or resets.
+            try:
+                index = self._count
+                self._count += 1
+                if index + 1 == self._parties:
+                    # We release the barrier
+                    await self._release()
+                else:
+                    await self._wait()
+                return index
+            finally:
+                self._count -= 1
+                # Wake up any tasks waiting for barrier to drain.
+                self._exit()
+
+    async def _block(self):
+        # Block until the barrier is ready for us,
+        # or raise an exception if it is broken.
+        #
+        # It is draining or resetting, wait until done
+        # unless a CancelledError occurs
+        await self._cond.wait_for(
+            lambda: self._state not in (
+                _BarrierState.DRAINING, _BarrierState.RESETTING
+            )
+        )
+
+        # see if the barrier is in a broken state
+        if self._state is _BarrierState.BROKEN:
+            raise exceptions.BrokenBarrierError("Barrier aborted")
+
+    async def _release(self):
+        # Release the tasks waiting in the barrier.
+
+        # Enter draining state.
+        # Next waiting tasks will be blocked until the end of draining.
+        self._state = _BarrierState.DRAINING
+        self._cond.notify_all()
+
+    async def _wait(self):
+        # Wait in the barrier until we are released. Raise an exception
+        # if the barrier is reset or broken.
+
+        # wait for end of filling
+        # unless a CancelledError occurs
+        await self._cond.wait_for(lambda: self._state is not _BarrierState.FILLING)
+
+        if self._state in (_BarrierState.BROKEN, _BarrierState.RESETTING):
+            raise exceptions.BrokenBarrierError("Abort or reset of barrier")
+
+    def _exit(self):
+        # If we are the last tasks to exit the barrier, signal any tasks
+        # waiting for the barrier to drain.
+        if self._count == 0:
+            if self._state in (_BarrierState.RESETTING, _BarrierState.DRAINING):
+                self._state = _BarrierState.FILLING
+            self._cond.notify_all()
+
+    async def reset(self):
+        """Reset the barrier to the initial state.
+
+        Any tasks currently waiting will get the BrokenBarrier exception
+        raised.
+        """
+        async with self._cond:
+            if self._count > 0:
+                if self._state is not _BarrierState.RESETTING:
+                    #reset the barrier, waking up tasks
+                    self._state = _BarrierState.RESETTING
+            else:
+                self._state = _BarrierState.FILLING
+            self._cond.notify_all()
+
+    async def abort(self):
+        """Place the barrier into a 'broken' state.
+
+        Useful in case of error.  Any currently waiting tasks and tasks
+        attempting to 'wait()' will have BrokenBarrierError raised.
+        """
+        async with self._cond:
+            self._state = _BarrierState.BROKEN
+            self._cond.notify_all()
+
+    @property
+    def parties(self):
+        """Return the number of tasks required to trip the barrier."""
+        return self._parties
+
+    @property
+    def n_waiting(self):
+        """Return the number of tasks currently waiting at the barrier."""
+        if self._state is _BarrierState.FILLING:
+            return self._count
+        return 0
+
+    @property
+    def broken(self):
+        """Return True if the barrier is in a broken state."""
+        return self._state is _BarrierState.BROKEN

Lib/asyncio/mixins.py

@@ -0,0 +1,21 @@
+"""Event loop mixins."""
+
+import threading
+from . import events
+
+_global_lock = threading.Lock()
+
+
+class _LoopBoundMixin:
+    _loop = None
+
+    def _get_loop(self):
+        loop = events._get_running_loop()
+
+        if self._loop is None:
+            with _global_lock:
+                if self._loop is None:
+                    self._loop = loop
+        if loop is not self._loop:
+            raise RuntimeError(f'{self!r} is bound to a different event loop')
+        return loop

Lib/asyncio/proactor_events.py

@@ -4,20 +4,45 @@ A proactor is a "notify-on-completion" multiplexer.  Currently a
 proactor is only implemented on Windows with IOCP.
 """
 
-__all__ = ['BaseProactorEventLoop']
+__all__ = 'BaseProactorEventLoop',
 
+import io
+import os
 import socket
 import warnings
+import signal
+import threading
+import collections
 
 from . import base_events
-from . import compat
 from . import constants
 from . import futures
+from . import exceptions
+from . import protocols
 from . import sslproto
 from . import transports
+from . import trsock
 from .log import logger
 
 
+def _set_socket_extra(transport, sock):
+    transport._extra['socket'] = trsock.TransportSocket(sock)
+
+    try:
+        transport._extra['sockname'] = sock.getsockname()
+    except socket.error:
+        if transport._loop.get_debug():
+            logger.warning(
+                "getsockname() failed on %r", sock, exc_info=True)
+
+    if 'peername' not in transport._extra:
+        try:
+            transport._extra['peername'] = sock.getpeername()
+        except socket.error:
+            # UDP sockets may not have a peer name
+            transport._extra['peername'] = None
+
+
 class _ProactorBasePipeTransport(transports._FlowControlMixin,
                                  transports.BaseTransport):
     """Base class for pipe and socket transports."""
@@ -27,7 +52,7 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
         super().__init__(extra, loop)
         self._set_extra(sock)
         self._sock = sock
-        self._protocol = protocol
+        self.set_protocol(protocol)
         self._server = server
         self._buffer = None  # None or bytearray.
         self._read_fut = None
@@ -35,6 +60,7 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
         self._pending_write = 0
         self._conn_lost = 0
         self._closing = False  # Set when close() called.
+        self._called_connection_lost = False
         self._eof_written = False
         if self._server is not None:
             self._server._attach()
@@ -51,17 +77,16 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
         elif self._closing:
             info.append('closing')
         if self._sock is not None:
-            info.append('fd=%s' % self._sock.fileno())
+            info.append(f'fd={self._sock.fileno()}')
         if self._read_fut is not None:
-            info.append('read=%s' % self._read_fut)
+            info.append(f'read={self._read_fut!r}')
         if self._write_fut is not None:
-            info.append("write=%r" % self._write_fut)
+            info.append(f'write={self._write_fut!r}')
         if self._buffer:
-            bufsize = len(self._buffer)
+            info.append(f'write_bufsize={len(self._buffer)}')
-            info.append('write_bufsize=%s' % bufsize)
         if self._eof_written:
             info.append('EOF written')
-        return '<%s>' % ' '.join(info)
+        return '<{}>'.format(' '.join(info))
 
     def _set_extra(self, sock):
         self._extra['pipe'] = sock
@@ -86,31 +111,33 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
             self._read_fut.cancel()
             self._read_fut = None
 
-    # On Python 3.3 and older, objects with a destructor part of a reference
+    def __del__(self, _warn=warnings.warn):
-    # cycle are never destroyed. It's not more the case on Python 3.4 thanks
+        if self._sock is not None:
-    # to the PEP 442.
+            _warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
-    if compat.PY34:
+            self._sock.close()
-        def __del__(self):
-            if self._sock is not None:
-                warnings.warn("unclosed transport %r" % self, ResourceWarning,
-                              source=self)
-                self.close()
 
     def _fatal_error(self, exc, message='Fatal error on pipe transport'):
-        if isinstance(exc, base_events._FATAL_ERROR_IGNORE):
+        try:
-            if self._loop.get_debug():
+            if isinstance(exc, OSError):
-                logger.debug("%r: %s", self, message, exc_info=True)
+                if self._loop.get_debug():
-        else:
+                    logger.debug("%r: %s", self, message, exc_info=True)
-            self._loop.call_exception_handler({
+            else:
-                'message': message,
+                self._loop.call_exception_handler({
-                'exception': exc,
+                    'message': message,
-                'transport': self,
+                    'exception': exc,
-                'protocol': self._protocol,
+                    'transport': self,
-            })
+                    'protocol': self._protocol,
-        self._force_close(exc)
+                })
+        finally:
+            self._force_close(exc)
 
     def _force_close(self, exc):
-        if self._closing:
+        if self._empty_waiter is not None and not self._empty_waiter.done():
+            if exc is None:
+                self._empty_waiter.set_result(None)
+            else:
+                self._empty_waiter.set_exception(exc)
+        if self._closing and self._called_connection_lost:
             return
         self._closing = True
         self._conn_lost += 1
@@ -125,6 +152,8 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
         self._loop.call_soon(self._call_connection_lost, exc)
 
     def _call_connection_lost(self, exc):
+        if self._called_connection_lost:
+            return
         try:
             self._protocol.connection_lost(exc)
         finally:
@@ -132,7 +161,7 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
             # end then it may fail with ERROR_NETNAME_DELETED if we
             # just close our end.  First calling shutdown() seems to
             # cure it, but maybe using DisconnectEx() would be better.
-            if hasattr(self._sock, 'shutdown'):
+            if hasattr(self._sock, 'shutdown') and self._sock.fileno() != -1:
                 self._sock.shutdown(socket.SHUT_RDWR)
             self._sock.close()
             self._sock = None
@@ -140,6 +169,7 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin,
             if server is not None:
                 server._detach()
                 self._server = None
+            self._called_connection_lost = True
 
     def get_write_buffer_size(self):
         size = self._pending_write
@@ -153,53 +183,127 @@ class _ProactorReadPipeTransport(_ProactorBasePipeTransport,
     """Transport for read pipes."""
 
     def __init__(self, loop, sock, protocol, waiter=None,
-                 extra=None, server=None):
+                 extra=None, server=None, buffer_size=65536):
+        self._pending_data_length = -1
+        self._paused = True
         super().__init__(loop, sock, protocol, waiter, extra, server)
-        self._paused = False
+
+        self._data = bytearray(buffer_size)
         self._loop.call_soon(self._loop_reading)
+        self._paused = False
+
+    def is_reading(self):
+        return not self._paused and not self._closing
 
     def pause_reading(self):
-        if self._closing:
+        if self._closing or self._paused:
-            raise RuntimeError('Cannot pause_reading() when closing')
+            return
-        if self._paused:
-            raise RuntimeError('Already paused')
         self._paused = True
+
+        # bpo-33694: Don't cancel self._read_fut because cancelling an
+        # overlapped WSASend() loss silently data with the current proactor
+        # implementation.
+        #
+        # If CancelIoEx() fails with ERROR_NOT_FOUND, it means that WSASend()
+        # completed (even if HasOverlappedIoCompleted() returns 0), but
+        # Overlapped.cancel() currently silently ignores the ERROR_NOT_FOUND
+        # error. Once the overlapped is ignored, the IOCP loop will ignores the
+        # completion I/O event and so not read the result of the overlapped
+        # WSARecv().
+
         if self._loop.get_debug():
             logger.debug("%r pauses reading", self)
 
     def resume_reading(self):
-        if not self._paused:
+        if self._closing or not self._paused:
-            raise RuntimeError('Not paused')
-        self._paused = False
-        if self._closing:
             return
-        self._loop.call_soon(self._loop_reading, self._read_fut)
+
+        self._paused = False
+        if self._read_fut is None:
+            self._loop.call_soon(self._loop_reading, None)
+
+        length = self._pending_data_length
+        self._pending_data_length = -1
+        if length > -1:
+            # Call the protocol method after calling _loop_reading(),
+            # since the protocol can decide to pause reading again.
+            self._loop.call_soon(self._data_received, self._data[:length], length)
+
         if self._loop.get_debug():
             logger.debug("%r resumes reading", self)
 
-    def _loop_reading(self, fut=None):
+    def _eof_received(self):
-        if self._paused:
+        if self._loop.get_debug():
-            return
+            logger.debug("%r received EOF", self)
-        data = None
 
+        try:
+            keep_open = self._protocol.eof_received()
+        except (SystemExit, KeyboardInterrupt):
+            raise
+        except BaseException as exc:
+            self._fatal_error(
+                exc, 'Fatal error: protocol.eof_received() call failed.')
+            return
+
+        if not keep_open:
+            self.close()
+
+    def _data_received(self, data, length):
+        if self._paused:
+            # Don't call any protocol method while reading is paused.
+            # The protocol will be called on resume_reading().
+            assert self._pending_data_length == -1
+            self._pending_data_length = length
+            return
+
+        if length == 0:
+            self._eof_received()
+            return
+
+        if isinstance(self._protocol, protocols.BufferedProtocol):
+            try:
+                protocols._feed_data_to_buffered_proto(self._protocol, data)
+            except (SystemExit, KeyboardInterrupt):
+                raise
+            except BaseException as exc:
+                self._fatal_error(exc,
+                                  'Fatal error: protocol.buffer_updated() '
+                                  'call failed.')
+                return
+        else:
+            self._protocol.data_received(data)
+
+    def _loop_reading(self, fut=None):
+        length = -1
+        data = None
         try:
             if fut is not None:
                 assert self._read_fut is fut or (self._read_fut is None and
                                                  self._closing)
                 self._read_fut = None
-                data = fut.result()  # deliver data later in "finally" clause
+                if fut.done():
+                    # deliver data later in "finally" clause
+                    length = fut.result()
+                    if length == 0:
+                        # we got end-of-file so no need to reschedule a new read
+                        return
+
+                    # It's a new slice so make it immutable so protocols upstream don't have problems
+                    data = bytes(memoryview(self._data)[:length])
+                else:
+                    # the future will be replaced by next proactor.recv call
+                    fut.cancel()
 
             if self._closing:
                 # since close() has been called we ignore any read data
-                data = None
                 return
 
-            if data == b'':
+            # bpo-33694: buffer_updated() has currently no fast path because of
-                # we got end-of-file so no need to reschedule a new read
+            # a data loss issue caused by overlapped WSASend() cancellation.
-                return
 
-            # reschedule a new read
+            if not self._paused:
-            self._read_fut = self._loop._proactor.recv(self._sock, 4096)
+                # reschedule a new read
+                self._read_fut = self._loop._proactor.recv_into(self._sock, self._data)
         except ConnectionAbortedError as exc:
             if not self._closing:
                 self._fatal_error(exc, 'Fatal read error on pipe transport')
@@ -210,32 +314,36 @@ class _ProactorReadPipeTransport(_ProactorBasePipeTransport,
             self._force_close(exc)
         except OSError as exc:
             self._fatal_error(exc, 'Fatal read error on pipe transport')
-        except futures.CancelledError:
+        except exceptions.CancelledError:
             if not self._closing:
                 raise
         else:
-            self._read_fut.add_done_callback(self._loop_reading)
+            if not self._paused:
+                self._read_fut.add_done_callback(self._loop_reading)
         finally:
-            if data:
+            if length > -1:
-                self._protocol.data_received(data)
+                self._data_received(data, length)
-            elif data is not None:
-                if self._loop.get_debug():
-                    logger.debug("%r received EOF", self)
-                keep_open = self._protocol.eof_received()
-                if not keep_open:
-                    self.close()
 
 
 class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
                                       transports.WriteTransport):
     """Transport for write pipes."""
 
+    _start_tls_compatible = True
+
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        self._empty_waiter = None
+
     def write(self, data):
         if not isinstance(data, (bytes, bytearray, memoryview)):
-            raise TypeError('data argument must be byte-ish (%r)',
+            raise TypeError(
-                            type(data))
+                f"data argument must be a bytes-like object, "
+                f"not {type(data).__name__}")
         if self._eof_written:
             raise RuntimeError('write_eof() already called')
+        if self._empty_waiter is not None:
+            raise RuntimeError('unable to write; sendfile is in progress')
 
         if not data:
             return
@@ -267,6 +375,10 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
 
     def _loop_writing(self, f=None, data=None):
         try:
+            if f is not None and self._write_fut is None and self._closing:
+                # XXX most likely self._force_close() has been called, and
+                # it has set self._write_fut to None.
+                return
             assert f is self._write_fut
             self._write_fut = None
             self._pending_write = 0
@@ -295,6 +407,8 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
                     self._maybe_pause_protocol()
                 else:
                     self._write_fut.add_done_callback(self._loop_writing)
+            if self._empty_waiter is not None and self._write_fut is None:
+                self._empty_waiter.set_result(None)
         except ConnectionResetError as exc:
             self._force_close(exc)
         except OSError as exc:
@@ -309,6 +423,17 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
     def abort(self):
         self._force_close(None)
 
+    def _make_empty_waiter(self):
+        if self._empty_waiter is not None:
+            raise RuntimeError("Empty waiter is already set")
+        self._empty_waiter = self._loop.create_future()
+        if self._write_fut is None:
+            self._empty_waiter.set_result(None)
+        return self._empty_waiter
+
+    def _reset_empty_waiter(self):
+        self._empty_waiter = None
+
 
 class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport):
     def __init__(self, *args, **kw):
@@ -332,6 +457,138 @@ class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport):
             self.close()
 
 
+class _ProactorDatagramTransport(_ProactorBasePipeTransport,
+                                 transports.DatagramTransport):
+    max_size = 256 * 1024
+    def __init__(self, loop, sock, protocol, address=None,
+                 waiter=None, extra=None):
+        self._address = address
+        self._empty_waiter = None
+        self._buffer_size = 0
+        # We don't need to call _protocol.connection_made() since our base
+        # constructor does it for us.
+        super().__init__(loop, sock, protocol, waiter=waiter, extra=extra)
+
+        # The base constructor sets _buffer = None, so we set it here
+        self._buffer = collections.deque()
+        self._loop.call_soon(self._loop_reading)
+
+    def _set_extra(self, sock):
+        _set_socket_extra(self, sock)
+
+    def get_write_buffer_size(self):
+        return self._buffer_size
+
+    def abort(self):
+        self._force_close(None)
+
+    def sendto(self, data, addr=None):
+        if not isinstance(data, (bytes, bytearray, memoryview)):
+            raise TypeError('data argument must be bytes-like object (%r)',
+                            type(data))
+
+        if not data:
+            return
+
+        if self._address is not None and addr not in (None, self._address):
+            raise ValueError(
+                f'Invalid address: must be None or {self._address}')
+
+        if self._conn_lost and self._address:
+            if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
+                logger.warning('socket.sendto() raised exception.')
+            self._conn_lost += 1
+            return
+
+        # Ensure that what we buffer is immutable.
+        self._buffer.append((bytes(data), addr))
+        self._buffer_size += len(data)
+
+        if self._write_fut is None:
+            # No current write operations are active, kick one off
+            self._loop_writing()
+        # else: A write operation is already kicked off
+
+        self._maybe_pause_protocol()
+
+    def _loop_writing(self, fut=None):
+        try:
+            if self._conn_lost:
+                return
+
+            assert fut is self._write_fut
+            self._write_fut = None
+            if fut:
+                # We are in a _loop_writing() done callback, get the result
+                fut.result()
+
+            if not self._buffer or (self._conn_lost and self._address):
+                # The connection has been closed
+                if self._closing:
+                    self._loop.call_soon(self._call_connection_lost, None)
+                return
+
+            data, addr = self._buffer.popleft()
+            self._buffer_size -= len(data)
+            if self._address is not None:
+                self._write_fut = self._loop._proactor.send(self._sock,
+                                                            data)
+            else:
+                self._write_fut = self._loop._proactor.sendto(self._sock,
+                                                              data,
+                                                              addr=addr)
+        except OSError as exc:
+            self._protocol.error_received(exc)
+        except Exception as exc:
+            self._fatal_error(exc, 'Fatal write error on datagram transport')
+        else:
+            self._write_fut.add_done_callback(self._loop_writing)
+            self._maybe_resume_protocol()
+
+    def _loop_reading(self, fut=None):
+        data = None
+        try:
+            if self._conn_lost:
+                return
+
+            assert self._read_fut is fut or (self._read_fut is None and
+                                             self._closing)
+
+            self._read_fut = None
+            if fut is not None:
+                res = fut.result()
+
+                if self._closing:
+                    # since close() has been called we ignore any read data
+                    data = None
+                    return
+
+                if self._address is not None:
+                    data, addr = res, self._address
+                else:
+                    data, addr = res
+
+            if self._conn_lost:
+                return
+            if self._address is not None:
+                self._read_fut = self._loop._proactor.recv(self._sock,
+                                                           self.max_size)
+            else:
+                self._read_fut = self._loop._proactor.recvfrom(self._sock,
+                                                               self.max_size)
+        except OSError as exc:
+            self._protocol.error_received(exc)
+        except exceptions.CancelledError:
+            if not self._closing:
+                raise
+        else:
+            if self._read_fut is not None:
+                self._read_fut.add_done_callback(self._loop_reading)
+        finally:
+            if data:
+                self._protocol.datagram_received(data, addr)
+
+
 class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport,
                                    _ProactorBaseWritePipeTransport,
                                    transports.Transport):
@@ -349,21 +606,15 @@ class _ProactorSocketTransport(_ProactorReadPipeTransport,
                                transports.Transport):
     """Transport for connected sockets."""
 
+    _sendfile_compatible = constants._SendfileMode.TRY_NATIVE
+
+    def __init__(self, loop, sock, protocol, waiter=None,
+                 extra=None, server=None):
+        super().__init__(loop, sock, protocol, waiter, extra, server)
+        base_events._set_nodelay(sock)
+
     def _set_extra(self, sock):
-        self._extra['socket'] = sock
+        _set_socket_extra(self, sock)
-        try:
-            self._extra['sockname'] = sock.getsockname()
-        except (socket.error, AttributeError):
-            if self._loop.get_debug():
-                logger.warning("getsockname() failed on %r",
-                             sock, exc_info=True)
-        if 'peername' not in self._extra:
-            try:
-                self._extra['peername'] = sock.getpeername()
-            except (socket.error, AttributeError):
-                if self._loop.get_debug():
-                    logger.warning("getpeername() failed on %r",
-                                   sock, exc_info=True)
 
     def can_write_eof(self):
         return True
@@ -387,26 +638,35 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
         self._accept_futures = {}   # socket file descriptor => Future
         proactor.set_loop(self)
         self._make_self_pipe()
+        if threading.current_thread() is threading.main_thread():
+            # wakeup fd can only be installed to a file descriptor from the main thread
+            signal.set_wakeup_fd(self._csock.fileno())
 
     def _make_socket_transport(self, sock, protocol, waiter=None,
                                extra=None, server=None):
         return _ProactorSocketTransport(self, sock, protocol, waiter,
                                         extra, server)
 
-    def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None,
+    def _make_ssl_transport(
-                            *, server_side=False, server_hostname=None,
+            self, rawsock, protocol, sslcontext, waiter=None,
-                            extra=None, server=None):
+            *, server_side=False, server_hostname=None,
-        if not sslproto._is_sslproto_available():
+            extra=None, server=None,
-            raise NotImplementedError("Proactor event loop requires Python 3.5"
+            ssl_handshake_timeout=None,
-                                      " or newer (ssl.MemoryBIO) to support "
+            ssl_shutdown_timeout=None):
-                                      "SSL")
+        ssl_protocol = sslproto.SSLProtocol(
-
+                self, protocol, sslcontext, waiter,
-        ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter,
+                server_side, server_hostname,
-                                            server_side, server_hostname)
+                ssl_handshake_timeout=ssl_handshake_timeout,
+                ssl_shutdown_timeout=ssl_shutdown_timeout)
         _ProactorSocketTransport(self, rawsock, ssl_protocol,
                                  extra=extra, server=server)
         return ssl_protocol._app_transport
 
+    def _make_datagram_transport(self, sock, protocol,
+                                 address=None, waiter=None, extra=None):
+        return _ProactorDatagramTransport(self, sock, protocol, address,
+                                          waiter, extra)
+
     def _make_duplex_pipe_transport(self, sock, protocol, waiter=None,
                                     extra=None):
         return _ProactorDuplexPipeTransport(self,
@@ -428,6 +688,8 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
         if self.is_closed():
             return
 
+        if threading.current_thread() is threading.main_thread():
+            signal.set_wakeup_fd(-1)
         # Call these methods before closing the event loop (before calling
         # BaseEventLoop.close), because they can schedule callbacks with
         # call_soon(), which is forbidden when the event loop is closed.
@@ -440,20 +702,73 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
         # Close the event loop
         super().close()
 
-    def sock_recv(self, sock, n):
+    async def sock_recv(self, sock, n):
-        return self._proactor.recv(sock, n)
+        return await self._proactor.recv(sock, n)
 
-    def sock_sendall(self, sock, data):
+    async def sock_recv_into(self, sock, buf):
-        return self._proactor.send(sock, data)
+        return await self._proactor.recv_into(sock, buf)
 
-    def sock_connect(self, sock, address):
+    async def sock_recvfrom(self, sock, bufsize):
-        return self._proactor.connect(sock, address)
+        return await self._proactor.recvfrom(sock, bufsize)
 
-    def sock_accept(self, sock):
+    async def sock_recvfrom_into(self, sock, buf, nbytes=0):
-        return self._proactor.accept(sock)
+        if not nbytes:
+            nbytes = len(buf)
 
-    def _socketpair(self):
+        return await self._proactor.recvfrom_into(sock, buf, nbytes)
-        raise NotImplementedError
+
+    async def sock_sendall(self, sock, data):
+        return await self._proactor.send(sock, data)
+
+    async def sock_sendto(self, sock, data, address):
+        return await self._proactor.sendto(sock, data, 0, address)
+
+    async def sock_connect(self, sock, address):
+        return await self._proactor.connect(sock, address)
+
+    async def sock_accept(self, sock):
+        return await self._proactor.accept(sock)
+
+    async def _sock_sendfile_native(self, sock, file, offset, count):
+        try:
+            fileno = file.fileno()
+        except (AttributeError, io.UnsupportedOperation) as err:
+            raise exceptions.SendfileNotAvailableError("not a regular file")
+        try:
+            fsize = os.fstat(fileno).st_size
+        except OSError:
+            raise exceptions.SendfileNotAvailableError("not a regular file")
+        blocksize = count if count else fsize
+        if not blocksize:
+            return 0  # empty file
+
+        blocksize = min(blocksize, 0xffff_ffff)
+        end_pos = min(offset + count, fsize) if count else fsize
+        offset = min(offset, fsize)
+        total_sent = 0
+        try:
+            while True:
+                blocksize = min(end_pos - offset, blocksize)
+                if blocksize <= 0:
+                    return total_sent
+                await self._proactor.sendfile(sock, file, offset, blocksize)
+                offset += blocksize
+                total_sent += blocksize
+        finally:
+            if total_sent > 0:
+                file.seek(offset)
+
+    async def _sendfile_native(self, transp, file, offset, count):
+        resume_reading = transp.is_reading()
+        transp.pause_reading()
+        await transp._make_empty_waiter()
+        try:
+            return await self.sock_sendfile(transp._sock, file, offset, count,
+                                            fallback=False)
+        finally:
+            transp._reset_empty_waiter()
+            if resume_reading:
+                transp.resume_reading()
 
     def _close_self_pipe(self):
         if self._self_reading_future is not None:
@@ -467,21 +782,30 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
 
     def _make_self_pipe(self):
         # A self-socket, really. :-)
-        self._ssock, self._csock = self._socketpair()
+        self._ssock, self._csock = socket.socketpair()
         self._ssock.setblocking(False)
         self._csock.setblocking(False)
         self._internal_fds += 1
-        self.call_soon(self._loop_self_reading)
 
     def _loop_self_reading(self, f=None):
         try:
             if f is not None:
                 f.result()  # may raise
+            if self._self_reading_future is not f:
+                # When we scheduled this Future, we assigned it to
+                # _self_reading_future. If it's not there now, something has
+                # tried to cancel the loop while this callback was still in the
+                # queue (see windows_events.ProactorEventLoop.run_forever). In
+                # that case stop here instead of continuing to schedule a new
+                # iteration.
+                return
             f = self._proactor.recv(self._ssock, 4096)
-        except futures.CancelledError:
+        except exceptions.CancelledError:
             # _close_self_pipe() has been called, stop waiting for data
             return
-        except Exception as exc:
+        except (SystemExit, KeyboardInterrupt):
+            raise
+        except BaseException as exc:
             self.call_exception_handler({
                 'message': 'Error on reading from the event loop self pipe',
                 'exception': exc,
@@ -492,10 +816,27 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
             f.add_done_callback(self._loop_self_reading)
 
     def _write_to_self(self):
-        self._csock.send(b'\0')
+        # This may be called from a different thread, possibly after
+        # _close_self_pipe() has been called or even while it is
+        # running.  Guard for self._csock being None or closed.  When
+        # a socket is closed, send() raises OSError (with errno set to
+        # EBADF, but let's not rely on the exact error code).
+        csock = self._csock
+        if csock is None:
+            return
+
+        try:
+            csock.send(b'\0')
+        except OSError:
+            if self._debug:
+                logger.debug("Fail to write a null byte into the "
+                             "self-pipe socket",
+                             exc_info=True)
 
     def _start_serving(self, protocol_factory, sock,
-                       sslcontext=None, server=None, backlog=100):
+                       sslcontext=None, server=None, backlog=100,
+                       ssl_handshake_timeout=None,
+                       ssl_shutdown_timeout=None):
 
         def loop(f=None):
             try:
@@ -508,7 +849,9 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
                     if sslcontext is not None:
                         self._make_ssl_transport(
                             conn, protocol, sslcontext, server_side=True,
-                            extra={'peername': addr}, server=server)
+                            extra={'peername': addr}, server=server,
+                            ssl_handshake_timeout=ssl_handshake_timeout,
+                            ssl_shutdown_timeout=ssl_shutdown_timeout)
                     else:
                         self._make_socket_transport(
                             conn, protocol,
@@ -521,13 +864,13 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
                     self.call_exception_handler({
                         'message': 'Accept failed on a socket',
                         'exception': exc,
-                        'socket': sock,
+                        'socket': trsock.TransportSocket(sock),
                     })
                     sock.close()
                 elif self._debug:
                     logger.debug("Accept failed on socket %r",
                                  sock, exc_info=True)
-            except futures.CancelledError:
+            except exceptions.CancelledError:
                 sock.close()
             else:
                 self._accept_futures[sock.fileno()] = f
@@ -545,6 +888,8 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
         self._accept_futures.clear()
 
     def _stop_serving(self, sock):
-        self._stop_accept_futures()
+        future = self._accept_futures.pop(sock.fileno(), None)
+        if future:
+            future.cancel()
         self._proactor._stop_serving(sock)
         sock.close()

Lib/asyncio/protocols.py

@@ -1,7 +1,9 @@
-"""Abstract Protocol class."""
+"""Abstract Protocol base classes."""
 
-__all__ = ['BaseProtocol', 'Protocol', 'DatagramProtocol',
+__all__ = (
-           'SubprocessProtocol']
+    'BaseProtocol', 'Protocol', 'DatagramProtocol',
+    'SubprocessProtocol', 'BufferedProtocol',
+)
 
 
 class BaseProtocol:
@@ -14,6 +16,8 @@ class BaseProtocol:
     write-only transport like write pipe
     """
 
+    __slots__ = ()
+
     def connection_made(self, transport):
         """Called when a connection is made.
 
@@ -85,6 +89,8 @@ class Protocol(BaseProtocol):
     * CL: connection_lost()
     """
 
+    __slots__ = ()
+
     def data_received(self, data):
         """Called when some data is received.
 
@@ -100,9 +106,64 @@ class Protocol(BaseProtocol):
         """
 
 
+class BufferedProtocol(BaseProtocol):
+    """Interface for stream protocol with manual buffer control.
+
+    Event methods, such as `create_server` and `create_connection`,
+    accept factories that return protocols that implement this interface.
+
+    The idea of BufferedProtocol is that it allows to manually allocate
+    and control the receive buffer.  Event loops can then use the buffer
+    provided by the protocol to avoid unnecessary data copies.  This
+    can result in noticeable performance improvement for protocols that
+    receive big amounts of data.  Sophisticated protocols can allocate
+    the buffer only once at creation time.
+
+    State machine of calls:
+
+      start -> CM [-> GB [-> BU?]]* [-> ER?] -> CL -> end
+
+    * CM: connection_made()
+    * GB: get_buffer()
+    * BU: buffer_updated()
+    * ER: eof_received()
+    * CL: connection_lost()
+    """
+
+    __slots__ = ()
+
+    def get_buffer(self, sizehint):
+        """Called to allocate a new receive buffer.
+
+        *sizehint* is a recommended minimal size for the returned
+        buffer.  When set to -1, the buffer size can be arbitrary.
+
+        Must return an object that implements the
+        :ref:`buffer protocol <bufferobjects>`.
+        It is an error to return a zero-sized buffer.
+        """
+
+    def buffer_updated(self, nbytes):
+        """Called when the buffer was updated with the received data.
+
+        *nbytes* is the total number of bytes that were written to
+        the buffer.
+        """
+
+    def eof_received(self):
+        """Called when the other end calls write_eof() or equivalent.
+
+        If this returns a false value (including None), the transport
+        will close itself.  If it returns a true value, closing the
+        transport is up to the protocol.
+        """
+
+
 class DatagramProtocol(BaseProtocol):
     """Interface for datagram protocol."""
 
+    __slots__ = ()
+
     def datagram_received(self, data, addr):
         """Called when some datagram is received."""
 
@@ -116,6 +177,8 @@ class DatagramProtocol(BaseProtocol):
 class SubprocessProtocol(BaseProtocol):
     """Interface for protocol for subprocess calls."""
 
+    __slots__ = ()
+
     def pipe_data_received(self, fd, data):
         """Called when the subprocess writes data into stdout/stderr pipe.
 
@@ -132,3 +195,22 @@ class SubprocessProtocol(BaseProtocol):
 
     def process_exited(self):
         """Called when subprocess has exited."""
+
+
+def _feed_data_to_buffered_proto(proto, data):
+    data_len = len(data)
+    while data_len:
+        buf = proto.get_buffer(data_len)
+        buf_len = len(buf)
+        if not buf_len:
+            raise RuntimeError('get_buffer() returned an empty buffer')
+
+        if buf_len >= data_len:
+            buf[:data_len] = data
+            proto.buffer_updated(data_len)
+            return
+        else:
+            buf[:buf_len] = data[:buf_len]
+            proto.buffer_updated(buf_len)
+            data = data[buf_len:]
+            data_len = len(data)

Lib/asyncio/queues.py

@@ -1,35 +1,28 @@
-"""Queues"""
+__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty')
-
-__all__ = ['Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty']
 
 import collections
 import heapq
+from types import GenericAlias
 
-from . import compat
-from . import events
 from . import locks
-from .coroutines import coroutine
+from . import mixins
 
 
 class QueueEmpty(Exception):
-    """Exception raised when Queue.get_nowait() is called on a Queue object
+    """Raised when Queue.get_nowait() is called on an empty Queue."""
-    which is empty.
-    """
     pass
 
 
 class QueueFull(Exception):
-    """Exception raised when the Queue.put_nowait() method is called on a Queue
+    """Raised when the Queue.put_nowait() method is called on a full Queue."""
-    object which is full.
-    """
     pass
 
 
-class Queue:
+class Queue(mixins._LoopBoundMixin):
     """A queue, useful for coordinating producer and consumer coroutines.
 
     If maxsize is less than or equal to zero, the queue size is infinite. If it
-    is an integer greater than 0, then "yield from put()" will block when the
+    is an integer greater than 0, then "await put()" will block when the
     queue reaches maxsize, until an item is removed by get().
 
     Unlike the standard library Queue, you can reliably know this Queue's size
@@ -37,11 +30,7 @@ class Queue:
     interrupted between calling qsize() and doing an operation on the Queue.
     """
 
-    def __init__(self, maxsize=0, *, loop=None):
+    def __init__(self, maxsize=0):
-        if loop is None:
-            self._loop = events.get_event_loop()
-        else:
-            self._loop = loop
         self._maxsize = maxsize
 
         # Futures.
@@ -49,7 +38,7 @@ class Queue:
         # Futures.
         self._putters = collections.deque()
         self._unfinished_tasks = 0
-        self._finished = locks.Event(loop=self._loop)
+        self._finished = locks.Event()
         self._finished.set()
         self._init(maxsize)
 
@@ -75,25 +64,23 @@ class Queue:
                 break
 
     def __repr__(self):
-        return '<{} at {:#x} {}>'.format(
+        return f'<{type(self).__name__} at {id(self):#x} {self._format()}>'
-            type(self).__name__, id(self), self._format())
 
     def __str__(self):
-        return '<{} {}>'.format(type(self).__name__, self._format())
+        return f'<{type(self).__name__} {self._format()}>'
 
-    def __class_getitem__(cls, type):
+    __class_getitem__ = classmethod(GenericAlias)
-        return cls
 
     def _format(self):
-        result = 'maxsize={!r}'.format(self._maxsize)
+        result = f'maxsize={self._maxsize!r}'
         if getattr(self, '_queue', None):
-            result += ' _queue={!r}'.format(list(self._queue))
+            result += f' _queue={list(self._queue)!r}'
         if self._getters:
-            result += ' _getters[{}]'.format(len(self._getters))
+            result += f' _getters[{len(self._getters)}]'
         if self._putters:
-            result += ' _putters[{}]'.format(len(self._putters))
+            result += f' _putters[{len(self._putters)}]'
         if self._unfinished_tasks:
-            result += ' tasks={}'.format(self._unfinished_tasks)
+            result += f' tasks={self._unfinished_tasks}'
         return result
 
     def qsize(self):
@@ -120,22 +107,26 @@ class Queue:
         else:
             return self.qsize() >= self._maxsize
 
-    @coroutine
+    async def put(self, item):
-    def put(self, item):
         """Put an item into the queue.
 
         Put an item into the queue. If the queue is full, wait until a free
         slot is available before adding item.
-
-        This method is a coroutine.
         """
         while self.full():
-            putter = self._loop.create_future()
+            putter = self._get_loop().create_future()
             self._putters.append(putter)
             try:
-                yield from putter
+                await putter
             except:
                 putter.cancel()  # Just in case putter is not done yet.
+                try:
+                    # Clean self._putters from canceled putters.
+                    self._putters.remove(putter)
+                except ValueError:
+                    # The putter could be removed from self._putters by a
+                    # previous get_nowait call.
+                    pass
                 if not self.full() and not putter.cancelled():
                     # We were woken up by get_nowait(), but can't take
                     # the call.  Wake up the next in line.
@@ -155,21 +146,25 @@ class Queue:
         self._finished.clear()
         self._wakeup_next(self._getters)
 
-    @coroutine
+    async def get(self):
-    def get(self):
         """Remove and return an item from the queue.
 
         If queue is empty, wait until an item is available.
-
-        This method is a coroutine.
         """
         while self.empty():
-            getter = self._loop.create_future()
+            getter = self._get_loop().create_future()
             self._getters.append(getter)
             try:
-                yield from getter
+                await getter
             except:
                 getter.cancel()  # Just in case getter is not done yet.
+                try:
+                    # Clean self._getters from canceled getters.
+                    self._getters.remove(getter)
+                except ValueError:
+                    # The getter could be removed from self._getters by a
+                    # previous put_nowait call.
+                    pass
                 if not self.empty() and not getter.cancelled():
                     # We were woken up by put_nowait(), but can't take
                     # the call.  Wake up the next in line.
@@ -208,8 +203,7 @@ class Queue:
         if self._unfinished_tasks == 0:
             self._finished.set()
 
-    @coroutine
+    async def join(self):
-    def join(self):
         """Block until all items in the queue have been gotten and processed.
 
         The count of unfinished tasks goes up whenever an item is added to the
@@ -218,7 +212,7 @@ class Queue:
         When the count of unfinished tasks drops to zero, join() unblocks.
         """
         if self._unfinished_tasks > 0:
-            yield from self._finished.wait()
+            await self._finished.wait()
 
 
 class PriorityQueue(Queue):
@@ -248,9 +242,3 @@ class LifoQueue(Queue):
 
     def _get(self):
         return self._queue.pop()
-
-
-if not compat.PY35:
-    JoinableQueue = Queue
-    """Deprecated alias for Queue."""
-    __all__.append('JoinableQueue')

Lib/asyncio/runners.py

@@ -1,16 +1,168 @@
-__all__ = ['run']
+__all__ = ('Runner', 'run')
 
+import contextvars
+import enum
+import functools
+import threading
+import signal
 from . import coroutines
 from . import events
+from . import exceptions
 from . import tasks
+from . import constants
+
+class _State(enum.Enum):
+    CREATED = "created"
+    INITIALIZED = "initialized"
+    CLOSED = "closed"
 
 
-def run(main, *, debug=False):
+class Runner:
-    """Run a coroutine.
+    """A context manager that controls event loop life cycle.
+
+    The context manager always creates a new event loop,
+    allows to run async functions inside it,
+    and properly finalizes the loop at the context manager exit.
+
+    If debug is True, the event loop will be run in debug mode.
+    If loop_factory is passed, it is used for new event loop creation.
+
+    asyncio.run(main(), debug=True)
+
+    is a shortcut for
+
+    with asyncio.Runner(debug=True) as runner:
+        runner.run(main())
+
+    The run() method can be called multiple times within the runner's context.
+
+    This can be useful for interactive console (e.g. IPython),
+    unittest runners, console tools, -- everywhere when async code
+    is called from existing sync framework and where the preferred single
+    asyncio.run() call doesn't work.
+
+    """
+
+    # Note: the class is final, it is not intended for inheritance.
+
+    def __init__(self, *, debug=None, loop_factory=None):
+        self._state = _State.CREATED
+        self._debug = debug
+        self._loop_factory = loop_factory
+        self._loop = None
+        self._context = None
+        self._interrupt_count = 0
+        self._set_event_loop = False
+
+    def __enter__(self):
+        self._lazy_init()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def close(self):
+        """Shutdown and close event loop."""
+        if self._state is not _State.INITIALIZED:
+            return
+        try:
+            loop = self._loop
+            _cancel_all_tasks(loop)
+            loop.run_until_complete(loop.shutdown_asyncgens())
+            loop.run_until_complete(
+                loop.shutdown_default_executor(constants.THREAD_JOIN_TIMEOUT))
+        finally:
+            if self._set_event_loop:
+                events.set_event_loop(None)
+            loop.close()
+            self._loop = None
+            self._state = _State.CLOSED
+
+    def get_loop(self):
+        """Return embedded event loop."""
+        self._lazy_init()
+        return self._loop
+
+    def run(self, coro, *, context=None):
+        """Run a coroutine inside the embedded event loop."""
+        if not coroutines.iscoroutine(coro):
+            raise ValueError("a coroutine was expected, got {!r}".format(coro))
+
+        if events._get_running_loop() is not None:
+            # fail fast with short traceback
+            raise RuntimeError(
+                "Runner.run() cannot be called from a running event loop")
+
+        self._lazy_init()
+
+        if context is None:
+            context = self._context
+        task = self._loop.create_task(coro, context=context)
+
+        if (threading.current_thread() is threading.main_thread()
+            and signal.getsignal(signal.SIGINT) is signal.default_int_handler
+        ):
+            sigint_handler = functools.partial(self._on_sigint, main_task=task)
+            try:
+                signal.signal(signal.SIGINT, sigint_handler)
+            except ValueError:
+                # `signal.signal` may throw if `threading.main_thread` does
+                # not support signals (e.g. embedded interpreter with signals
+                # not registered - see gh-91880)
+                sigint_handler = None
+        else:
+            sigint_handler = None
+
+        self._interrupt_count = 0
+        try:
+            return self._loop.run_until_complete(task)
+        except exceptions.CancelledError:
+            if self._interrupt_count > 0:
+                uncancel = getattr(task, "uncancel", None)
+                if uncancel is not None and uncancel() == 0:
+                    raise KeyboardInterrupt()
+            raise  # CancelledError
+        finally:
+            if (sigint_handler is not None
+                and signal.getsignal(signal.SIGINT) is sigint_handler
+            ):
+                signal.signal(signal.SIGINT, signal.default_int_handler)
+
+    def _lazy_init(self):
+        if self._state is _State.CLOSED:
+            raise RuntimeError("Runner is closed")
+        if self._state is _State.INITIALIZED:
+            return
+        if self._loop_factory is None:
+            self._loop = events.new_event_loop()
+            if not self._set_event_loop:
+                # Call set_event_loop only once to avoid calling
+                # attach_loop multiple times on child watchers
+                events.set_event_loop(self._loop)
+                self._set_event_loop = True
+        else:
+            self._loop = self._loop_factory()
+        if self._debug is not None:
+            self._loop.set_debug(self._debug)
+        self._context = contextvars.copy_context()
+        self._state = _State.INITIALIZED
+
+    def _on_sigint(self, signum, frame, main_task):
+        self._interrupt_count += 1
+        if self._interrupt_count == 1 and not main_task.done():
+            main_task.cancel()
+            # wakeup loop if it is blocked by select() with long timeout
+            self._loop.call_soon_threadsafe(lambda: None)
+            return
+        raise KeyboardInterrupt()
+
+
+def run(main, *, debug=None, loop_factory=None):
+    """Execute the coroutine and return the result.
 
     This function runs the passed coroutine, taking care of
-    managing the asyncio event loop and finalizing asynchronous
+    managing the asyncio event loop, finalizing asynchronous
-    generators.
+    generators and closing the default executor.
 
     This function cannot be called when another asyncio event loop is
     running in the same thread.
@@ -21,6 +173,10 @@ def run(main, *, debug=False):
     It should be used as a main entry point for asyncio programs, and should
     ideally only be called once.
 
+    The executor is given a timeout duration of 5 minutes to shutdown.
+    If the executor hasn't finished within that duration, a warning is
+    emitted and the executor is closed.
+
     Example:
 
         async def main():
@@ -30,24 +186,12 @@ def run(main, *, debug=False):
         asyncio.run(main())
     """
     if events._get_running_loop() is not None:
+        # fail fast with short traceback
         raise RuntimeError(
             "asyncio.run() cannot be called from a running event loop")
 
-    if not coroutines.iscoroutine(main):
+    with Runner(debug=debug, loop_factory=loop_factory) as runner:
-        raise ValueError("a coroutine was expected, got {!r}".format(main))
+        return runner.run(main)
-
-    loop = events.new_event_loop()
-    try:
-        events.set_event_loop(loop)
-        loop.set_debug(debug)
-        return loop.run_until_complete(main)
-    finally:
-        try:
-            _cancel_all_tasks(loop)
-            loop.run_until_complete(loop.shutdown_asyncgens())
-        finally:
-            events.set_event_loop(None)
-            loop.close()
 
 
 def _cancel_all_tasks(loop):
@@ -58,8 +202,7 @@ def _cancel_all_tasks(loop):
     for task in to_cancel:
         task.cancel()
 
-    loop.run_until_complete(
+    loop.run_until_complete(tasks.gather(*to_cancel, return_exceptions=True))
-        tasks.gather(*to_cancel, loop=loop, return_exceptions=True))
 
     for task in to_cancel:
         if task.cancelled():

Lib/asyncio/staggered.py

@@ -0,0 +1,149 @@
+"""Support for running coroutines in parallel with staggered start times."""
+
+__all__ = 'staggered_race',
+
+import contextlib
+import typing
+
+from . import events
+from . import exceptions as exceptions_mod
+from . import locks
+from . import tasks
+
+
+async def staggered_race(
+        coro_fns: typing.Iterable[typing.Callable[[], typing.Awaitable]],
+        delay: typing.Optional[float],
+        *,
+        loop: events.AbstractEventLoop = None,
+) -> typing.Tuple[
+    typing.Any,
+    typing.Optional[int],
+    typing.List[typing.Optional[Exception]]
+]:
+    """Run coroutines with staggered start times and take the first to finish.
+
+    This method takes an iterable of coroutine functions. The first one is
+    started immediately. From then on, whenever the immediately preceding one
+    fails (raises an exception), or when *delay* seconds has passed, the next
+    coroutine is started. This continues until one of the coroutines complete
+    successfully, in which case all others are cancelled, or until all
+    coroutines fail.
+
+    The coroutines provided should be well-behaved in the following way:
+
+    * They should only ``return`` if completed successfully.
+
+    * They should always raise an exception if they did not complete
+      successfully. In particular, if they handle cancellation, they should
+      probably reraise, like this::
+
+        try:
+            # do work
+        except asyncio.CancelledError:
+            # undo partially completed work
+            raise
+
+    Args:
+        coro_fns: an iterable of coroutine functions, i.e. callables that
+            return a coroutine object when called. Use ``functools.partial`` or
+            lambdas to pass arguments.
+
+        delay: amount of time, in seconds, between starting coroutines. If
+            ``None``, the coroutines will run sequentially.
+
+        loop: the event loop to use.
+
+    Returns:
+        tuple *(winner_result, winner_index, exceptions)* where
+
+        - *winner_result*: the result of the winning coroutine, or ``None``
+          if no coroutines won.
+
+        - *winner_index*: the index of the winning coroutine in
+          ``coro_fns``, or ``None`` if no coroutines won. If the winning
+          coroutine may return None on success, *winner_index* can be used
+          to definitively determine whether any coroutine won.
+
+        - *exceptions*: list of exceptions returned by the coroutines.
+          ``len(exceptions)`` is equal to the number of coroutines actually
+          started, and the order is the same as in ``coro_fns``. The winning
+          coroutine's entry is ``None``.
+
+    """
+    # TODO: when we have aiter() and anext(), allow async iterables in coro_fns.
+    loop = loop or events.get_running_loop()
+    enum_coro_fns = enumerate(coro_fns)
+    winner_result = None
+    winner_index = None
+    exceptions = []
+    running_tasks = []
+
+    async def run_one_coro(
+            previous_failed: typing.Optional[locks.Event]) -> None:
+        # Wait for the previous task to finish, or for delay seconds
+        if previous_failed is not None:
+            with contextlib.suppress(exceptions_mod.TimeoutError):
+                # Use asyncio.wait_for() instead of asyncio.wait() here, so
+                # that if we get cancelled at this point, Event.wait() is also
+                # cancelled, otherwise there will be a "Task destroyed but it is
+                # pending" later.
+                await tasks.wait_for(previous_failed.wait(), delay)
+        # Get the next coroutine to run
+        try:
+            this_index, coro_fn = next(enum_coro_fns)
+        except StopIteration:
+            return
+        # Start task that will run the next coroutine
+        this_failed = locks.Event()
+        next_task = loop.create_task(run_one_coro(this_failed))
+        running_tasks.append(next_task)
+        assert len(running_tasks) == this_index + 2
+        # Prepare place to put this coroutine's exceptions if not won
+        exceptions.append(None)
+        assert len(exceptions) == this_index + 1
+
+        try:
+            result = await coro_fn()
+        except (SystemExit, KeyboardInterrupt):
+            raise
+        except BaseException as e:
+            exceptions[this_index] = e
+            this_failed.set()  # Kickstart the next coroutine
+        else:
+            # Store winner's results
+            nonlocal winner_index, winner_result
+            assert winner_index is None
+            winner_index = this_index
+            winner_result = result
+            # Cancel all other tasks. We take care to not cancel the current
+            # task as well. If we do so, then since there is no `await` after
+            # here and CancelledError are usually thrown at one, we will
+            # encounter a curious corner case where the current task will end
+            # up as done() == True, cancelled() == False, exception() ==
+            # asyncio.CancelledError. This behavior is specified in
+            # https://bugs.python.org/issue30048
+            for i, t in enumerate(running_tasks):
+                if i != this_index:
+                    t.cancel()
+
+    first_task = loop.create_task(run_one_coro(None))
+    running_tasks.append(first_task)
+    try:
+        # Wait for a growing list of tasks to all finish: poor man's version of
+        # curio's TaskGroup or trio's nursery
+        done_count = 0
+        while done_count != len(running_tasks):
+            done, _ = await tasks.wait(running_tasks)
+            done_count = len(done)
+            # If run_one_coro raises an unhandled exception, it's probably a
+            # programming error, and I want to see it.
+            if __debug__:
+                for d in done:
+                    if d.done() and not d.cancelled() and d.exception():
+                        raise d.exception()
+        return winner_result, winner_index, exceptions
+    finally:
+        # Make sure no tasks are left running if we leave this function
+        for t in running_tasks:
+            t.cancel()

Lib/asyncio/streams.py

@@ -1,55 +1,30 @@
-"""Stream-related things."""
+__all__ = (
-
+    'StreamReader', 'StreamWriter', 'StreamReaderProtocol',
-__all__ = ['StreamReader', 'StreamWriter', 'StreamReaderProtocol',
+    'open_connection', 'start_server')
-           'open_connection', 'start_server',
-           'IncompleteReadError',
-           'LimitOverrunError',
-           ]
 
+import collections
 import socket
+import sys
+import warnings
+import weakref
 
 if hasattr(socket, 'AF_UNIX'):
-    __all__.extend(['open_unix_connection', 'start_unix_server'])
+    __all__ += ('open_unix_connection', 'start_unix_server')
 
 from . import coroutines
-from . import compat
 from . import events
+from . import exceptions
+from . import format_helpers
 from . import protocols
-from .coroutines import coroutine
 from .log import logger
+from .tasks import sleep
 
 
-_DEFAULT_LIMIT = 2 ** 16
+_DEFAULT_LIMIT = 2 ** 16  # 64 KiB
 
 
-class IncompleteReadError(EOFError):
+async def open_connection(host=None, port=None, *,
-    """
+                          limit=_DEFAULT_LIMIT, **kwds):
-    Incomplete read error. Attributes:
-
-    - partial: read bytes string before the end of stream was reached
-    - expected: total number of expected bytes (or None if unknown)
-    """
-    def __init__(self, partial, expected):
-        super().__init__("%d bytes read on a total of %r expected bytes"
-                         % (len(partial), expected))
-        self.partial = partial
-        self.expected = expected
-
-
-class LimitOverrunError(Exception):
-    """Reached the buffer limit while looking for a separator.
-
-    Attributes:
-    - consumed: total number of to be consumed bytes.
-    """
-    def __init__(self, message, consumed):
-        super().__init__(message)
-        self.consumed = consumed
-
-
-@coroutine
-def open_connection(host=None, port=None, *,
-                    loop=None, limit=_DEFAULT_LIMIT, **kwds):
     """A wrapper for create_connection() returning a (reader, writer) pair.
 
     The reader returned is a StreamReader instance; the writer is a
@@ -67,19 +42,17 @@ def open_connection(host=None, port=None, *,
     StreamReaderProtocol classes, just copy the code -- there's
     really nothing special here except some convenience.)
     """
-    if loop is None:
+    loop = events.get_running_loop()
-        loop = events.get_event_loop()
     reader = StreamReader(limit=limit, loop=loop)
     protocol = StreamReaderProtocol(reader, loop=loop)
-    transport, _ = yield from loop.create_connection(
+    transport, _ = await loop.create_connection(
         lambda: protocol, host, port, **kwds)
     writer = StreamWriter(transport, protocol, reader, loop)
     return reader, writer
 
 
-@coroutine
+async def start_server(client_connected_cb, host=None, port=None, *,
-def start_server(client_connected_cb, host=None, port=None, *,
+                       limit=_DEFAULT_LIMIT, **kwds):
-                 loop=None, limit=_DEFAULT_LIMIT, **kwds):
     """Start a socket server, call back for each client connected.
 
     The first parameter, `client_connected_cb`, takes two parameters:
@@ -94,15 +67,13 @@ def start_server(client_connected_cb, host=None, port=None, *,
     positional host and port, with various optional keyword arguments
     following.  The return value is the same as loop.create_server().
 
-    Additional optional keyword arguments are loop (to set the event loop
+    Additional optional keyword argument is limit (to set the buffer
-    instance to use) and limit (to set the buffer limit passed to the
+    limit passed to the StreamReader).
-    StreamReader).
 
     The return value is the same as loop.create_server(), i.e. a
     Server object which can be used to stop the service.
     """
-    if loop is None:
+    loop = events.get_running_loop()
-        loop = events.get_event_loop()
 
     def factory():
         reader = StreamReader(limit=limit, loop=loop)
@@ -110,31 +81,28 @@ def start_server(client_connected_cb, host=None, port=None, *,
                                         loop=loop)
         return protocol
 
-    return (yield from loop.create_server(factory, host, port, **kwds))
+    return await loop.create_server(factory, host, port, **kwds)
 
 
 if hasattr(socket, 'AF_UNIX'):
     # UNIX Domain Sockets are supported on this platform
 
-    @coroutine
+    async def open_unix_connection(path=None, *,
-    def open_unix_connection(path=None, *,
+                                   limit=_DEFAULT_LIMIT, **kwds):
-                             loop=None, limit=_DEFAULT_LIMIT, **kwds):
         """Similar to `open_connection` but works with UNIX Domain Sockets."""
-        if loop is None:
+        loop = events.get_running_loop()
-            loop = events.get_event_loop()
+
         reader = StreamReader(limit=limit, loop=loop)
         protocol = StreamReaderProtocol(reader, loop=loop)
-        transport, _ = yield from loop.create_unix_connection(
+        transport, _ = await loop.create_unix_connection(
             lambda: protocol, path, **kwds)
         writer = StreamWriter(transport, protocol, reader, loop)
         return reader, writer
 
-    @coroutine
+    async def start_unix_server(client_connected_cb, path=None, *,
-    def start_unix_server(client_connected_cb, path=None, *,
+                                limit=_DEFAULT_LIMIT, **kwds):
-                          loop=None, limit=_DEFAULT_LIMIT, **kwds):
         """Similar to `start_server` but works with UNIX Domain Sockets."""
-        if loop is None:
+        loop = events.get_running_loop()
-            loop = events.get_event_loop()
 
         def factory():
             reader = StreamReader(limit=limit, loop=loop)
@@ -142,14 +110,14 @@ if hasattr(socket, 'AF_UNIX'):
                                             loop=loop)
             return protocol
 
-        return (yield from loop.create_unix_server(factory, path, **kwds))
+        return await loop.create_unix_server(factory, path, **kwds)
 
 
 class FlowControlMixin(protocols.Protocol):
     """Reusable flow control logic for StreamWriter.drain().
 
     This implements the protocol methods pause_writing(),
-    resume_reading() and connection_lost().  If the subclass overrides
+    resume_writing() and connection_lost().  If the subclass overrides
     these it must call the super methods.
 
     StreamWriter.drain() must wait for _drain_helper() coroutine.
@@ -161,7 +129,7 @@ class FlowControlMixin(protocols.Protocol):
         else:
             self._loop = loop
         self._paused = False
-        self._drain_waiter = None
+        self._drain_waiters = collections.deque()
         self._connection_lost = False
 
     def pause_writing(self):
@@ -176,39 +144,37 @@ class FlowControlMixin(protocols.Protocol):
         if self._loop.get_debug():
             logger.debug("%r resumes writing", self)
 
-        waiter = self._drain_waiter
+        for waiter in self._drain_waiters:
-        if waiter is not None:
-            self._drain_waiter = None
             if not waiter.done():
                 waiter.set_result(None)
 
     def connection_lost(self, exc):
         self._connection_lost = True
-        # Wake up the writer if currently paused.
+        # Wake up the writer(s) if currently paused.
         if not self._paused:
             return
-        waiter = self._drain_waiter
-        if waiter is None:
-            return
-        self._drain_waiter = None
-        if waiter.done():
-            return
-        if exc is None:
-            waiter.set_result(None)
-        else:
-            waiter.set_exception(exc)
 
-    @coroutine
+        for waiter in self._drain_waiters:
-    def _drain_helper(self):
+            if not waiter.done():
+                if exc is None:
+                    waiter.set_result(None)
+                else:
+                    waiter.set_exception(exc)
+
+    async def _drain_helper(self):
         if self._connection_lost:
             raise ConnectionResetError('Connection lost')
         if not self._paused:
             return
-        waiter = self._drain_waiter
-        assert waiter is None or waiter.cancelled()
         waiter = self._loop.create_future()
-        self._drain_waiter = waiter
+        self._drain_waiters.append(waiter)
-        yield from waiter
+        try:
+            await waiter
+        finally:
+            self._drain_waiters.remove(waiter)
+
+    def _get_close_waiter(self, stream):
+        raise NotImplementedError
 
 
 class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
@@ -220,40 +186,110 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
     call inappropriate methods of the protocol.)
     """
 
+    _source_traceback = None
+
     def __init__(self, stream_reader, client_connected_cb=None, loop=None):
         super().__init__(loop=loop)
-        self._stream_reader = stream_reader
+        if stream_reader is not None:
+            self._stream_reader_wr = weakref.ref(stream_reader)
+            self._source_traceback = stream_reader._source_traceback
+        else:
+            self._stream_reader_wr = None
+        if client_connected_cb is not None:
+            # This is a stream created by the `create_server()` function.
+            # Keep a strong reference to the reader until a connection
+            # is established.
+            self._strong_reader = stream_reader
+        self._reject_connection = False
         self._stream_writer = None
+        self._task = None
+        self._transport = None
         self._client_connected_cb = client_connected_cb
         self._over_ssl = False
+        self._closed = self._loop.create_future()
+
+    @property
+    def _stream_reader(self):
+        if self._stream_reader_wr is None:
+            return None
+        return self._stream_reader_wr()
+
+    def _replace_writer(self, writer):
+        loop = self._loop
+        transport = writer.transport
+        self._stream_writer = writer
+        self._transport = transport
+        self._over_ssl = transport.get_extra_info('sslcontext') is not None
 
     def connection_made(self, transport):
-        self._stream_reader.set_transport(transport)
+        if self._reject_connection:
+            context = {
+                'message': ('An open stream was garbage collected prior to '
+                            'establishing network connection; '
+                            'call "stream.close()" explicitly.')
+            }
+            if self._source_traceback:
+                context['source_traceback'] = self._source_traceback
+            self._loop.call_exception_handler(context)
+            transport.abort()
+            return
+        self._transport = transport
+        reader = self._stream_reader
+        if reader is not None:
+            reader.set_transport(transport)
         self._over_ssl = transport.get_extra_info('sslcontext') is not None
         if self._client_connected_cb is not None:
             self._stream_writer = StreamWriter(transport, self,
-                                               self._stream_reader,
+                                               reader,
                                                self._loop)
-            res = self._client_connected_cb(self._stream_reader,
+            res = self._client_connected_cb(reader,
                                             self._stream_writer)
             if coroutines.iscoroutine(res):
-                self._loop.create_task(res)
+                def callback(task):
+                    if task.cancelled():
+                        transport.close()
+                        return
+                    exc = task.exception()
+                    if exc is not None:
+                        self._loop.call_exception_handler({
+                            'message': 'Unhandled exception in client_connected_cb',
+                            'exception': exc,
+                            'transport': transport,
+                        })
+                        transport.close()
+
+                self._task = self._loop.create_task(res)
+                self._task.add_done_callback(callback)
+
+            self._strong_reader = None
 
     def connection_lost(self, exc):
-        if self._stream_reader is not None:
+        reader = self._stream_reader
+        if reader is not None:
             if exc is None:
-                self._stream_reader.feed_eof()
+                reader.feed_eof()
             else:
-                self._stream_reader.set_exception(exc)
+                reader.set_exception(exc)
+        if not self._closed.done():
+            if exc is None:
+                self._closed.set_result(None)
+            else:
+                self._closed.set_exception(exc)
         super().connection_lost(exc)
-        self._stream_reader = None
+        self._stream_reader_wr = None
         self._stream_writer = None
+        self._task = None
+        self._transport = None
 
     def data_received(self, data):
-        self._stream_reader.feed_data(data)
+        reader = self._stream_reader
+        if reader is not None:
+            reader.feed_data(data)
 
     def eof_received(self):
-        self._stream_reader.feed_eof()
+        reader = self._stream_reader
+        if reader is not None:
+            reader.feed_eof()
         if self._over_ssl:
             # Prevent a warning in SSLProtocol.eof_received:
             # "returning true from eof_received()
@@ -261,6 +297,20 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
             return False
         return True
 
+    def _get_close_waiter(self, stream):
+        return self._closed
+
+    def __del__(self):
+        # Prevent reports about unhandled exceptions.
+        # Better than self._closed._log_traceback = False hack
+        try:
+            closed = self._closed
+        except AttributeError:
+            pass  # failed constructor
+        else:
+            if closed.done() and not closed.cancelled():
+                closed.exception()
+
 
 class StreamWriter:
     """Wraps a Transport.
@@ -279,12 +329,14 @@ class StreamWriter:
         assert reader is None or isinstance(reader, StreamReader)
         self._reader = reader
         self._loop = loop
+        self._complete_fut = self._loop.create_future()
+        self._complete_fut.set_result(None)
 
     def __repr__(self):
-        info = [self.__class__.__name__, 'transport=%r' % self._transport]
+        info = [self.__class__.__name__, f'transport={self._transport!r}']
         if self._reader is not None:
-            info.append('reader=%r' % self._reader)
+            info.append(f'reader={self._reader!r}')
-        return '<%s>' % ' '.join(info)
+        return '<{}>'.format(' '.join(info))
 
     @property
     def transport(self):
@@ -305,36 +357,68 @@ class StreamWriter:
     def close(self):
         return self._transport.close()
 
+    def is_closing(self):
+        return self._transport.is_closing()
+
+    async def wait_closed(self):
+        await self._protocol._get_close_waiter(self)
+
     def get_extra_info(self, name, default=None):
         return self._transport.get_extra_info(name, default)
 
-    @coroutine
+    async def drain(self):
-    def drain(self):
         """Flush the write buffer.
 
         The intended use is to write
 
           w.write(data)
-          yield from w.drain()
+          await w.drain()
         """
         if self._reader is not None:
             exc = self._reader.exception()
             if exc is not None:
                 raise exc
-        if self._transport is not None:
+        if self._transport.is_closing():
-            if self._transport.is_closing():
+            # Wait for protocol.connection_lost() call
-                # Yield to the event loop so connection_lost() may be
+            # Raise connection closing error if any,
-                # called.  Without this, _drain_helper() would return
+            # ConnectionResetError otherwise
-                # immediately, and code that calls
+            # Yield to the event loop so connection_lost() may be
-                #     write(...); yield from drain()
+            # called.  Without this, _drain_helper() would return
-                # in a loop would never call connection_lost(), so it
+            # immediately, and code that calls
-                # would not see an error when the socket is closed.
+            #     write(...); await drain()
-                yield
+            # in a loop would never call connection_lost(), so it
-        yield from self._protocol._drain_helper()
+            # would not see an error when the socket is closed.
+            await sleep(0)
+        await self._protocol._drain_helper()
 
+    async def start_tls(self, sslcontext, *,
+                        server_hostname=None,
+                        ssl_handshake_timeout=None,
+                        ssl_shutdown_timeout=None):
+        """Upgrade an existing stream-based connection to TLS."""
+        server_side = self._protocol._client_connected_cb is not None
+        protocol = self._protocol
+        await self.drain()
+        new_transport = await self._loop.start_tls(  # type: ignore
+            self._transport, protocol, sslcontext,
+            server_side=server_side, server_hostname=server_hostname,
+            ssl_handshake_timeout=ssl_handshake_timeout,
+            ssl_shutdown_timeout=ssl_shutdown_timeout)
+        self._transport = new_transport
+        protocol._replace_writer(self)
+
+    def __del__(self):
+        if not self._transport.is_closing():
+            if self._loop.is_closed():
+                warnings.warn("loop is closed", ResourceWarning)
+            else:
+                self.close()
+                warnings.warn(f"unclosed {self!r}", ResourceWarning)
 
 class StreamReader:
 
+    _source_traceback = None
+
     def __init__(self, limit=_DEFAULT_LIMIT, loop=None):
         # The line length limit is  a security feature;
         # it also doubles as half the buffer limit.
@@ -353,24 +437,27 @@ class StreamReader:
         self._exception = None
         self._transport = None
         self._paused = False
+        if self._loop.get_debug():
+            self._source_traceback = format_helpers.extract_stack(
+                sys._getframe(1))
 
     def __repr__(self):
         info = ['StreamReader']
         if self._buffer:
-            info.append('%d bytes' % len(self._buffer))
+            info.append(f'{len(self._buffer)} bytes')
         if self._eof:
             info.append('eof')
         if self._limit != _DEFAULT_LIMIT:
-            info.append('l=%d' % self._limit)
+            info.append(f'limit={self._limit}')
         if self._waiter:
-            info.append('w=%r' % self._waiter)
+            info.append(f'waiter={self._waiter!r}')
         if self._exception:
-            info.append('e=%r' % self._exception)
+            info.append(f'exception={self._exception!r}')
         if self._transport:
-            info.append('t=%r' % self._transport)
+            info.append(f'transport={self._transport!r}')
         if self._paused:
             info.append('paused')
-        return '<%s>' % ' '.join(info)
+        return '<{}>'.format(' '.join(info))
 
     def exception(self):
         return self._exception
@@ -431,8 +518,7 @@ class StreamReader:
             else:
                 self._paused = True
 
-    @coroutine
+    async def _wait_for_data(self, func_name):
-    def _wait_for_data(self, func_name):
         """Wait until feed_data() or feed_eof() is called.
 
         If stream was paused, automatically resume it.
@@ -442,8 +528,9 @@ class StreamReader:
         # would have an unexpected behaviour. It would not possible to know
         # which coroutine would get the next data.
         if self._waiter is not None:
-            raise RuntimeError('%s() called while another coroutine is '
+            raise RuntimeError(
-                               'already waiting for incoming data' % func_name)
+                f'{func_name}() called while another coroutine is '
+                f'already waiting for incoming data')
 
         assert not self._eof, '_wait_for_data after EOF'
 
@@ -455,12 +542,11 @@ class StreamReader:
 
         self._waiter = self._loop.create_future()
         try:
-            yield from self._waiter
+            await self._waiter
         finally:
             self._waiter = None
 
-    @coroutine
+    async def readline(self):
-    def readline(self):
         """Read chunk of data from the stream until newline (b'\n') is found.
 
         On success, return chunk that ends with newline. If only partial
@@ -479,10 +565,10 @@ class StreamReader:
         sep = b'\n'
         seplen = len(sep)
         try:
-            line = yield from self.readuntil(sep)
+            line = await self.readuntil(sep)
-        except IncompleteReadError as e:
+        except exceptions.IncompleteReadError as e:
             return e.partial
-        except LimitOverrunError as e:
+        except exceptions.LimitOverrunError as e:
             if self._buffer.startswith(sep, e.consumed):
                 del self._buffer[:e.consumed + seplen]
             else:
@@ -491,8 +577,7 @@ class StreamReader:
             raise ValueError(e.args[0])
         return line
 
-    @coroutine
+    async def readuntil(self, separator=b'\n'):
-    def readuntil(self, separator=b'\n'):
         """Read data from the stream until ``separator`` is found.
 
         On success, the data and separator will be removed from the
@@ -558,7 +643,7 @@ class StreamReader:
                 # see upper comment for explanation.
                 offset = buflen + 1 - seplen
                 if offset > self._limit:
-                    raise LimitOverrunError(
+                    raise exceptions.LimitOverrunError(
                         'Separator is not found, and chunk exceed the limit',
                         offset)
 
@@ -569,13 +654,13 @@ class StreamReader:
             if self._eof:
                 chunk = bytes(self._buffer)
                 self._buffer.clear()
-                raise IncompleteReadError(chunk, None)
+                raise exceptions.IncompleteReadError(chunk, None)
 
             # _wait_for_data() will resume reading if stream was paused.
-            yield from self._wait_for_data('readuntil')
+            await self._wait_for_data('readuntil')
 
         if isep > self._limit:
-            raise LimitOverrunError(
+            raise exceptions.LimitOverrunError(
                 'Separator is found, but chunk is longer than limit', isep)
 
         chunk = self._buffer[:isep + seplen]
@@ -583,20 +668,20 @@ class StreamReader:
         self._maybe_resume_transport()
         return bytes(chunk)
 
-    @coroutine
+    async def read(self, n=-1):
-    def read(self, n=-1):
         """Read up to `n` bytes from the stream.
 
-        If n is not provided, or set to -1, read until EOF and return all read
+        If `n` is not provided or set to -1,
-        bytes. If the EOF was received and the internal buffer is empty, return
+        read until EOF, then return all read bytes.
-        an empty bytes object.
+        If EOF was received and the internal buffer is empty,
+        return an empty bytes object.
 
-        If n is zero, return empty bytes object immediately.
+        If `n` is 0, return an empty bytes object immediately.
 
-        If n is positive, this function try to read `n` bytes, and may return
+        If `n` is positive, return at most `n` available bytes
-        less or equal bytes than requested, but at least one byte. If EOF was
+        as soon as at least 1 byte is available in the internal buffer.
-        received before any byte is read, this function returns empty byte
+        If EOF is received before any byte is read, return an empty
-        object.
+        bytes object.
 
         Returned value is not limited with limit, configured at stream
         creation.
@@ -618,24 +703,23 @@ class StreamReader:
             # bytes.  So just call self.read(self._limit) until EOF.
             blocks = []
             while True:
-                block = yield from self.read(self._limit)
+                block = await self.read(self._limit)
                 if not block:
                     break
                 blocks.append(block)
             return b''.join(blocks)
 
         if not self._buffer and not self._eof:
-            yield from self._wait_for_data('read')
+            await self._wait_for_data('read')
 
         # This will work right even if buffer is less than n bytes
-        data = bytes(self._buffer[:n])
+        data = bytes(memoryview(self._buffer)[:n])
         del self._buffer[:n]
 
         self._maybe_resume_transport()
         return data
 
-    @coroutine
+    async def readexactly(self, n):
-    def readexactly(self, n):
         """Read exactly `n` bytes.
 
         Raise an IncompleteReadError if EOF is reached before `n` bytes can be
@@ -663,33 +747,24 @@ class StreamReader:
             if self._eof:
                 incomplete = bytes(self._buffer)
                 self._buffer.clear()
-                raise IncompleteReadError(incomplete, n)
+                raise exceptions.IncompleteReadError(incomplete, n)
 
-            yield from self._wait_for_data('readexactly')
+            await self._wait_for_data('readexactly')
 
         if len(self._buffer) == n:
             data = bytes(self._buffer)
             self._buffer.clear()
         else:
-            data = bytes(self._buffer[:n])
+            data = bytes(memoryview(self._buffer)[:n])
             del self._buffer[:n]
         self._maybe_resume_transport()
         return data
 
-    if compat.PY35:
+    def __aiter__(self):
-        @coroutine
+        return self
-        def __aiter__(self):
-            return self
 
-        @coroutine
+    async def __anext__(self):
-        def __anext__(self):
+        val = await self.readline()
-            val = yield from self.readline()
+        if val == b'':
-            if val == b'':
+            raise StopAsyncIteration
-                raise StopAsyncIteration
+        return val
-            return val
-
-    if compat.PY352:
-        # In Python 3.5.2 and greater, __aiter__ should return
-        # the asynchronous iterator directly.
-        def __aiter__(self):
-            return self

Lib/asyncio/subprocess.py

@@ -1,4 +1,4 @@
-__all__ = ['create_subprocess_exec', 'create_subprocess_shell']
+__all__ = 'create_subprocess_exec', 'create_subprocess_shell'
 
 import subprocess
 
@@ -6,7 +6,6 @@ from . import events
 from . import protocols
 from . import streams
 from . import tasks
-from .coroutines import coroutine
 from .log import logger
 
 
@@ -24,16 +23,19 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
         self._limit = limit
         self.stdin = self.stdout = self.stderr = None
         self._transport = None
+        self._process_exited = False
+        self._pipe_fds = []
+        self._stdin_closed = self._loop.create_future()
 
     def __repr__(self):
         info = [self.__class__.__name__]
         if self.stdin is not None:
-            info.append('stdin=%r' % self.stdin)
+            info.append(f'stdin={self.stdin!r}')
         if self.stdout is not None:
-            info.append('stdout=%r' % self.stdout)
+            info.append(f'stdout={self.stdout!r}')
         if self.stderr is not None:
-            info.append('stderr=%r' % self.stderr)
+            info.append(f'stderr={self.stderr!r}')
-        return '<%s>' % ' '.join(info)
+        return '<{}>'.format(' '.join(info))
 
     def connection_made(self, transport):
         self._transport = transport
@@ -43,12 +45,14 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
             self.stdout = streams.StreamReader(limit=self._limit,
                                                loop=self._loop)
             self.stdout.set_transport(stdout_transport)
+            self._pipe_fds.append(1)
 
         stderr_transport = transport.get_pipe_transport(2)
         if stderr_transport is not None:
             self.stderr = streams.StreamReader(limit=self._limit,
                                                loop=self._loop)
             self.stderr.set_transport(stderr_transport)
+            self._pipe_fds.append(2)
 
         stdin_transport = transport.get_pipe_transport(0)
         if stdin_transport is not None:
@@ -73,6 +77,13 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
             if pipe is not None:
                 pipe.close()
             self.connection_lost(exc)
+            if exc is None:
+                self._stdin_closed.set_result(None)
+            else:
+                self._stdin_closed.set_exception(exc)
+                # Since calling `wait_closed()` is not mandatory,
+                # we shouldn't log the traceback if this is not awaited.
+                self._stdin_closed._log_traceback = False
             return
         if fd == 1:
             reader = self.stdout
@@ -80,15 +91,28 @@ class SubprocessStreamProtocol(streams.FlowControlMixin,
             reader = self.stderr
         else:
             reader = None
-        if reader != None:
+        if reader is not None:
             if exc is None:
                 reader.feed_eof()
             else:
                 reader.set_exception(exc)
 
+        if fd in self._pipe_fds:
+            self._pipe_fds.remove(fd)
+        self._maybe_close_transport()
+
     def process_exited(self):
-        self._transport.close()
+        self._process_exited = True
-        self._transport = None
+        self._maybe_close_transport()
+
+    def _maybe_close_transport(self):
+        if len(self._pipe_fds) == 0 and self._process_exited:
+            self._transport.close()
+            self._transport = None
+
+    def _get_close_waiter(self, stream):
+        if stream is self.stdin:
+            return self._stdin_closed
 
 
 class Process:
@@ -102,18 +126,15 @@ class Process:
         self.pid = transport.get_pid()
 
     def __repr__(self):
-        return '<%s %s>' % (self.__class__.__name__, self.pid)
+        return f'<{self.__class__.__name__} {self.pid}>'
 
     @property
     def returncode(self):
         return self._transport.get_returncode()
 
-    @coroutine
+    async def wait(self):
-    def wait(self):
+        """Wait until the process exit and return the process return code."""
-        """Wait until the process exit and return the process return code.
+        return await self._transport._wait()
-
-        This method is a coroutine."""
-        return (yield from self._transport._wait())
 
     def send_signal(self, signal):
         self._transport.send_signal(signal)
@@ -124,17 +145,19 @@ class Process:
     def kill(self):
         self._transport.kill()
 
-    @coroutine
+    async def _feed_stdin(self, input):
-    def _feed_stdin(self, input):
         debug = self._loop.get_debug()
-        self.stdin.write(input)
-        if debug:
-            logger.debug('%r communicate: feed stdin (%s bytes)',
-                        self, len(input))
         try:
-            yield from self.stdin.drain()
+            if input is not None:
+                self.stdin.write(input)
+                if debug:
+                    logger.debug(
+                        '%r communicate: feed stdin (%s bytes)', self, len(input))
+
+            await self.stdin.drain()
         except (BrokenPipeError, ConnectionResetError) as exc:
-            # communicate() ignores BrokenPipeError and ConnectionResetError
+            # communicate() ignores BrokenPipeError and ConnectionResetError.
+            # write() and drain() can raise these exceptions.
             if debug:
                 logger.debug('%r communicate: stdin got %r', self, exc)
 
@@ -142,12 +165,10 @@ class Process:
             logger.debug('%r communicate: close stdin', self)
         self.stdin.close()
 
-    @coroutine
+    async def _noop(self):
-    def _noop(self):
         return None
 
-    @coroutine
+    async def _read_stream(self, fd):
-    def _read_stream(self, fd):
         transport = self._transport.get_pipe_transport(fd)
         if fd == 2:
             stream = self.stderr
@@ -157,16 +178,15 @@ class Process:
         if self._loop.get_debug():
             name = 'stdout' if fd == 1 else 'stderr'
             logger.debug('%r communicate: read %s', self, name)
-        output = yield from stream.read()
+        output = await stream.read()
         if self._loop.get_debug():
             name = 'stdout' if fd == 1 else 'stderr'
             logger.debug('%r communicate: close %s', self, name)
         transport.close()
         return output
 
-    @coroutine
+    async def communicate(self, input=None):
-    def communicate(self, input=None):
+        if self.stdin is not None:
-        if input is not None:
             stdin = self._feed_stdin(input)
         else:
             stdin = self._noop()
@@ -178,36 +198,32 @@ class Process:
             stderr = self._read_stream(2)
         else:
             stderr = self._noop()
-        stdin, stdout, stderr = yield from tasks.gather(stdin, stdout, stderr,
+        stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr)
-                                                        loop=self._loop)
+        await self.wait()
-        yield from self.wait()
         return (stdout, stderr)
 
 
-@coroutine
+async def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None,
-def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None,
+                                  limit=streams._DEFAULT_LIMIT, **kwds):
-                            loop=None, limit=streams._DEFAULT_LIMIT, **kwds):
+    loop = events.get_running_loop()
-    if loop is None:
-        loop = events.get_event_loop()
     protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
                                                         loop=loop)
-    transport, protocol = yield from loop.subprocess_shell(
+    transport, protocol = await loop.subprocess_shell(
-                                            protocol_factory,
+        protocol_factory,
-                                            cmd, stdin=stdin, stdout=stdout,
+        cmd, stdin=stdin, stdout=stdout,
-                                            stderr=stderr, **kwds)
+        stderr=stderr, **kwds)
     return Process(transport, protocol, loop)
 
-@coroutine
+
-def create_subprocess_exec(program, *args, stdin=None, stdout=None,
+async def create_subprocess_exec(program, *args, stdin=None, stdout=None,
-                           stderr=None, loop=None,
+                                 stderr=None, limit=streams._DEFAULT_LIMIT,
-                           limit=streams._DEFAULT_LIMIT, **kwds):
+                                 **kwds):
-    if loop is None:
+    loop = events.get_running_loop()
-        loop = events.get_event_loop()
     protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
                                                         loop=loop)
-    transport, protocol = yield from loop.subprocess_exec(
+    transport, protocol = await loop.subprocess_exec(
-                                            protocol_factory,
+        protocol_factory,
-                                            program, *args,
+        program, *args,
-                                            stdin=stdin, stdout=stdout,
+        stdin=stdin, stdout=stdout,
-                                            stderr=stderr, **kwds)
+        stderr=stderr, **kwds)
     return Process(transport, protocol, loop)

Lib/asyncio/taskgroups.py

@@ -0,0 +1,240 @@
+# Adapted with permission from the EdgeDB project;
+# license: PSFL.
+
+
+__all__ = ("TaskGroup",)
+
+from . import events
+from . import exceptions
+from . import tasks
+
+
+class TaskGroup:
+    """Asynchronous context manager for managing groups of tasks.
+
+    Example use:
+
+        async with asyncio.TaskGroup() as group:
+            task1 = group.create_task(some_coroutine(...))
+            task2 = group.create_task(other_coroutine(...))
+        print("Both tasks have completed now.")
+
+    All tasks are awaited when the context manager exits.
+
+    Any exceptions other than `asyncio.CancelledError` raised within
+    a task will cancel all remaining tasks and wait for them to exit.
+    The exceptions are then combined and raised as an `ExceptionGroup`.
+    """
+    def __init__(self):
+        self._entered = False
+        self._exiting = False
+        self._aborting = False
+        self._loop = None
+        self._parent_task = None
+        self._parent_cancel_requested = False
+        self._tasks = set()
+        self._errors = []
+        self._base_error = None
+        self._on_completed_fut = None
+
+    def __repr__(self):
+        info = ['']
+        if self._tasks:
+            info.append(f'tasks={len(self._tasks)}')
+        if self._errors:
+            info.append(f'errors={len(self._errors)}')
+        if self._aborting:
+            info.append('cancelling')
+        elif self._entered:
+            info.append('entered')
+
+        info_str = ' '.join(info)
+        return f'<TaskGroup{info_str}>'
+
+    async def __aenter__(self):
+        if self._entered:
+            raise RuntimeError(
+                f"TaskGroup {self!r} has already been entered")
+        if self._loop is None:
+            self._loop = events.get_running_loop()
+        self._parent_task = tasks.current_task(self._loop)
+        if self._parent_task is None:
+            raise RuntimeError(
+                f'TaskGroup {self!r} cannot determine the parent task')
+        self._entered = True
+
+        return self
+
+    async def __aexit__(self, et, exc, tb):
+        self._exiting = True
+
+        if (exc is not None and
+                self._is_base_error(exc) and
+                self._base_error is None):
+            self._base_error = exc
+
+        propagate_cancellation_error = \
+            exc if et is exceptions.CancelledError else None
+        if self._parent_cancel_requested:
+            # If this flag is set we *must* call uncancel().
+            if self._parent_task.uncancel() == 0:
+                # If there are no pending cancellations left,
+                # don't propagate CancelledError.
+                propagate_cancellation_error = None
+
+        if et is not None:
+            if not self._aborting:
+                # Our parent task is being cancelled:
+                #
+                #    async with TaskGroup() as g:
+                #        g.create_task(...)
+                #        await ...  # <- CancelledError
+                #
+                # or there's an exception in "async with":
+                #
+                #    async with TaskGroup() as g:
+                #        g.create_task(...)
+                #        1 / 0
+                #
+                self._abort()
+
+        # We use while-loop here because "self._on_completed_fut"
+        # can be cancelled multiple times if our parent task
+        # is being cancelled repeatedly (or even once, when
+        # our own cancellation is already in progress)
+        while self._tasks:
+            if self._on_completed_fut is None:
+                self._on_completed_fut = self._loop.create_future()
+
+            try:
+                await self._on_completed_fut
+            except exceptions.CancelledError as ex:
+                if not self._aborting:
+                    # Our parent task is being cancelled:
+                    #
+                    #    async def wrapper():
+                    #        async with TaskGroup() as g:
+                    #            g.create_task(foo)
+                    #
+                    # "wrapper" is being cancelled while "foo" is
+                    # still running.
+                    propagate_cancellation_error = ex
+                    self._abort()
+
+            self._on_completed_fut = None
+
+        assert not self._tasks
+
+        if self._base_error is not None:
+            raise self._base_error
+
+        # Propagate CancelledError if there is one, except if there
+        # are other errors -- those have priority.
+        if propagate_cancellation_error and not self._errors:
+            raise propagate_cancellation_error
+
+        if et is not None and et is not exceptions.CancelledError:
+            self._errors.append(exc)
+
+        if self._errors:
+            # Exceptions are heavy objects that can have object
+            # cycles (bad for GC); let's not keep a reference to
+            # a bunch of them.
+            try:
+                me = BaseExceptionGroup('unhandled errors in a TaskGroup', self._errors)
+                raise me from None
+            finally:
+                self._errors = None
+
+    def create_task(self, coro, *, name=None, context=None):
+        """Create a new task in this group and return it.
+
+        Similar to `asyncio.create_task`.
+        """
+        if not self._entered:
+            raise RuntimeError(f"TaskGroup {self!r} has not been entered")
+        if self._exiting and not self._tasks:
+            raise RuntimeError(f"TaskGroup {self!r} is finished")
+        if self._aborting:
+            raise RuntimeError(f"TaskGroup {self!r} is shutting down")
+        if context is None:
+            task = self._loop.create_task(coro)
+        else:
+            task = self._loop.create_task(coro, context=context)
+        tasks._set_task_name(task, name)
+        # optimization: Immediately call the done callback if the task is
+        # already done (e.g. if the coro was able to complete eagerly),
+        # and skip scheduling a done callback
+        if task.done():
+            self._on_task_done(task)
+        else:
+            self._tasks.add(task)
+            task.add_done_callback(self._on_task_done)
+        return task
+
+    # Since Python 3.8 Tasks propagate all exceptions correctly,
+    # except for KeyboardInterrupt and SystemExit which are
+    # still considered special.
+
+    def _is_base_error(self, exc: BaseException) -> bool:
+        assert isinstance(exc, BaseException)
+        return isinstance(exc, (SystemExit, KeyboardInterrupt))
+
+    def _abort(self):
+        self._aborting = True
+
+        for t in self._tasks:
+            if not t.done():
+                t.cancel()
+
+    def _on_task_done(self, task):
+        self._tasks.discard(task)
+
+        if self._on_completed_fut is not None and not self._tasks:
+            if not self._on_completed_fut.done():
+                self._on_completed_fut.set_result(True)
+
+        if task.cancelled():
+            return
+
+        exc = task.exception()
+        if exc is None:
+            return
+
+        self._errors.append(exc)
+        if self._is_base_error(exc) and self._base_error is None:
+            self._base_error = exc
+
+        if self._parent_task.done():
+            # Not sure if this case is possible, but we want to handle
+            # it anyways.
+            self._loop.call_exception_handler({
+                'message': f'Task {task!r} has errored out but its parent '
+                           f'task {self._parent_task} is already completed',
+                'exception': exc,
+                'task': task,
+            })
+            return
+
+        if not self._aborting and not self._parent_cancel_requested:
+            # If parent task *is not* being cancelled, it means that we want
+            # to manually cancel it to abort whatever is being run right now
+            # in the TaskGroup.  But we want to mark parent task as
+            # "not cancelled" later in __aexit__.  Example situation that
+            # we need to handle:
+            #
+            #    async def foo():
+            #        try:
+            #            async with TaskGroup() as g:
+            #                g.create_task(crash_soon())
+            #                await something  # <- this needs to be canceled
+            #                                 #    by the TaskGroup, e.g.
+            #                                 #    foo() needs to be cancelled
+            #        except Exception:
+            #            # Ignore any exceptions raised in the TaskGroup
+            #            pass
+            #        await something_else     # this line has to be called
+            #                                 # after TaskGroup is finished.
+            self._abort()
+            self._parent_cancel_requested = True
+            self._parent_task.cancel()

Lib/asyncio/test_utils.py

@@ -1,503 +0,0 @@
-"""Utilities shared by tests."""
-
-import collections
-import contextlib
-import io
-import logging
-import os
-import re
-import socket
-import socketserver
-import sys
-import tempfile
-import threading
-import time
-import unittest
-import weakref
-
-from unittest import mock
-
-from http.server import HTTPServer
-from wsgiref.simple_server import WSGIRequestHandler, WSGIServer
-
-try:
-    import ssl
-except ImportError:  # pragma: no cover
-    ssl = None
-
-from . import base_events
-from . import compat
-from . import events
-from . import futures
-from . import selectors
-from . import tasks
-from .coroutines import coroutine
-from .log import logger
-
-
-if sys.platform == 'win32':  # pragma: no cover
-    from .windows_utils import socketpair
-else:
-    from socket import socketpair  # pragma: no cover
-
-
-def dummy_ssl_context():
-    if ssl is None:
-        return None
-    else:
-        return ssl.SSLContext(ssl.PROTOCOL_SSLv23)
-
-
-def run_briefly(loop):
-    @coroutine
-    def once():
-        pass
-    gen = once()
-    t = loop.create_task(gen)
-    # Don't log a warning if the task is not done after run_until_complete().
-    # It occurs if the loop is stopped or if a task raises a BaseException.
-    t._log_destroy_pending = False
-    try:
-        loop.run_until_complete(t)
-    finally:
-        gen.close()
-
-
-def run_until(loop, pred, timeout=30):
-    deadline = time.time() + timeout
-    while not pred():
-        if timeout is not None:
-            timeout = deadline - time.time()
-            if timeout <= 0:
-                raise futures.TimeoutError()
-        loop.run_until_complete(tasks.sleep(0.001, loop=loop))
-
-
-def run_once(loop):
-    """Legacy API to run once through the event loop.
-
-    This is the recommended pattern for test code.  It will poll the
-    selector once and run all callbacks scheduled in response to I/O
-    events.
-    """
-    loop.call_soon(loop.stop)
-    loop.run_forever()
-
-
-class SilentWSGIRequestHandler(WSGIRequestHandler):
-
-    def get_stderr(self):
-        return io.StringIO()
-
-    def log_message(self, format, *args):
-        pass
-
-
-class SilentWSGIServer(WSGIServer):
-
-    request_timeout = 2
-
-    def get_request(self):
-        request, client_addr = super().get_request()
-        request.settimeout(self.request_timeout)
-        return request, client_addr
-
-    def handle_error(self, request, client_address):
-        pass
-
-
-class SSLWSGIServerMixin:
-
-    def finish_request(self, request, client_address):
-        # The relative location of our test directory (which
-        # contains the ssl key and certificate files) differs
-        # between the stdlib and stand-alone asyncio.
-        # Prefer our own if we can find it.
-        here = os.path.join(os.path.dirname(__file__), '..', 'tests')
-        if not os.path.isdir(here):
-            here = os.path.join(os.path.dirname(os.__file__),
-                                'test', 'test_asyncio')
-        keyfile = os.path.join(here, 'ssl_key.pem')
-        certfile = os.path.join(here, 'ssl_cert.pem')
-        context = ssl.SSLContext()
-        context.load_cert_chain(certfile, keyfile)
-
-        ssock = context.wrap_socket(request, server_side=True)
-        try:
-            self.RequestHandlerClass(ssock, client_address, self)
-            ssock.close()
-        except OSError:
-            # maybe socket has been closed by peer
-            pass
-
-
-class SSLWSGIServer(SSLWSGIServerMixin, SilentWSGIServer):
-    pass
-
-
-def _run_test_server(*, address, use_ssl=False, server_cls, server_ssl_cls):
-
-    def app(environ, start_response):
-        status = '200 OK'
-        headers = [('Content-type', 'text/plain')]
-        start_response(status, headers)
-        return [b'Test message']
-
-    # Run the test WSGI server in a separate thread in order not to
-    # interfere with event handling in the main thread
-    server_class = server_ssl_cls if use_ssl else server_cls
-    httpd = server_class(address, SilentWSGIRequestHandler)
-    httpd.set_app(app)
-    httpd.address = httpd.server_address
-    server_thread = threading.Thread(
-        target=lambda: httpd.serve_forever(poll_interval=0.05))
-    server_thread.start()
-    try:
-        yield httpd
-    finally:
-        httpd.shutdown()
-        httpd.server_close()
-        server_thread.join()
-
-
-if hasattr(socket, 'AF_UNIX'):
-
-    class UnixHTTPServer(socketserver.UnixStreamServer, HTTPServer):
-
-        def server_bind(self):
-            socketserver.UnixStreamServer.server_bind(self)
-            self.server_name = '127.0.0.1'
-            self.server_port = 80
-
-
-    class UnixWSGIServer(UnixHTTPServer, WSGIServer):
-
-        request_timeout = 2
-
-        def server_bind(self):
-            UnixHTTPServer.server_bind(self)
-            self.setup_environ()
-
-        def get_request(self):
-            request, client_addr = super().get_request()
-            request.settimeout(self.request_timeout)
-            # Code in the stdlib expects that get_request
-            # will return a socket and a tuple (host, port).
-            # However, this isn't true for UNIX sockets,
-            # as the second return value will be a path;
-            # hence we return some fake data sufficient
-            # to get the tests going
-            return request, ('127.0.0.1', '')
-
-
-    class SilentUnixWSGIServer(UnixWSGIServer):
-
-        def handle_error(self, request, client_address):
-            pass
-
-
-    class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer):
-        pass
-
-
-    def gen_unix_socket_path():
-        with tempfile.NamedTemporaryFile() as file:
-            return file.name
-
-
-    @contextlib.contextmanager
-    def unix_socket_path():
-        path = gen_unix_socket_path()
-        try:
-            yield path
-        finally:
-            try:
-                os.unlink(path)
-            except OSError:
-                pass
-
-
-    @contextlib.contextmanager
-    def run_test_unix_server(*, use_ssl=False):
-        with unix_socket_path() as path:
-            yield from _run_test_server(address=path, use_ssl=use_ssl,
-                                        server_cls=SilentUnixWSGIServer,
-                                        server_ssl_cls=UnixSSLWSGIServer)
-
-
-@contextlib.contextmanager
-def run_test_server(*, host='127.0.0.1', port=0, use_ssl=False):
-    yield from _run_test_server(address=(host, port), use_ssl=use_ssl,
-                                server_cls=SilentWSGIServer,
-                                server_ssl_cls=SSLWSGIServer)
-
-
-def make_test_protocol(base):
-    dct = {}
-    for name in dir(base):
-        if name.startswith('__') and name.endswith('__'):
-            # skip magic names
-            continue
-        dct[name] = MockCallback(return_value=None)
-    return type('TestProtocol', (base,) + base.__bases__, dct)()
-
-
-class TestSelector(selectors.BaseSelector):
-
-    def __init__(self):
-        self.keys = {}
-
-    def register(self, fileobj, events, data=None):
-        key = selectors.SelectorKey(fileobj, 0, events, data)
-        self.keys[fileobj] = key
-        return key
-
-    def unregister(self, fileobj):
-        return self.keys.pop(fileobj)
-
-    def select(self, timeout):
-        return []
-
-    def get_map(self):
-        return self.keys
-
-
-class TestLoop(base_events.BaseEventLoop):
-    """Loop for unittests.
-
-    It manages self time directly.
-    If something scheduled to be executed later then
-    on next loop iteration after all ready handlers done
-    generator passed to __init__ is calling.
-
-    Generator should be like this:
-
-        def gen():
-            ...
-            when = yield ...
-            ... = yield time_advance
-
-    Value returned by yield is absolute time of next scheduled handler.
-    Value passed to yield is time advance to move loop's time forward.
-    """
-
-    def __init__(self, gen=None):
-        super().__init__()
-
-        if gen is None:
-            def gen():
-                yield
-            self._check_on_close = False
-        else:
-            self._check_on_close = True
-
-        self._gen = gen()
-        next(self._gen)
-        self._time = 0
-        self._clock_resolution = 1e-9
-        self._timers = []
-        self._selector = TestSelector()
-
-        self.readers = {}
-        self.writers = {}
-        self.reset_counters()
-
-        self._transports = weakref.WeakValueDictionary()
-
-    def time(self):
-        return self._time
-
-    def advance_time(self, advance):
-        """Move test time forward."""
-        if advance:
-            self._time += advance
-
-    def close(self):
-        super().close()
-        if self._check_on_close:
-            try:
-                self._gen.send(0)
-            except StopIteration:
-                pass
-            else:  # pragma: no cover
-                raise AssertionError("Time generator is not finished")
-
-    def _add_reader(self, fd, callback, *args):
-        self.readers[fd] = events.Handle(callback, args, self)
-
-    def _remove_reader(self, fd):
-        self.remove_reader_count[fd] += 1
-        if fd in self.readers:
-            del self.readers[fd]
-            return True
-        else:
-            return False
-
-    def assert_reader(self, fd, callback, *args):
-        assert fd in self.readers, 'fd {} is not registered'.format(fd)
-        handle = self.readers[fd]
-        assert handle._callback == callback, '{!r} != {!r}'.format(
-            handle._callback, callback)
-        assert handle._args == args, '{!r} != {!r}'.format(
-            handle._args, args)
-
-    def _add_writer(self, fd, callback, *args):
-        self.writers[fd] = events.Handle(callback, args, self)
-
-    def _remove_writer(self, fd):
-        self.remove_writer_count[fd] += 1
-        if fd in self.writers:
-            del self.writers[fd]
-            return True
-        else:
-            return False
-
-    def assert_writer(self, fd, callback, *args):
-        assert fd in self.writers, 'fd {} is not registered'.format(fd)
-        handle = self.writers[fd]
-        assert handle._callback == callback, '{!r} != {!r}'.format(
-            handle._callback, callback)
-        assert handle._args == args, '{!r} != {!r}'.format(
-            handle._args, args)
-
-    def _ensure_fd_no_transport(self, fd):
-        try:
-            transport = self._transports[fd]
-        except KeyError:
-            pass
-        else:
-            raise RuntimeError(
-                'File descriptor {!r} is used by transport {!r}'.format(
-                    fd, transport))
-
-    def add_reader(self, fd, callback, *args):
-        """Add a reader callback."""
-        self._ensure_fd_no_transport(fd)
-        return self._add_reader(fd, callback, *args)
-
-    def remove_reader(self, fd):
-        """Remove a reader callback."""
-        self._ensure_fd_no_transport(fd)
-        return self._remove_reader(fd)
-
-    def add_writer(self, fd, callback, *args):
-        """Add a writer callback.."""
-        self._ensure_fd_no_transport(fd)
-        return self._add_writer(fd, callback, *args)
-
-    def remove_writer(self, fd):
-        """Remove a writer callback."""
-        self._ensure_fd_no_transport(fd)
-        return self._remove_writer(fd)
-
-    def reset_counters(self):
-        self.remove_reader_count = collections.defaultdict(int)
-        self.remove_writer_count = collections.defaultdict(int)
-
-    def _run_once(self):
-        super()._run_once()
-        for when in self._timers:
-            advance = self._gen.send(when)
-            self.advance_time(advance)
-        self._timers = []
-
-    def call_at(self, when, callback, *args):
-        self._timers.append(when)
-        return super().call_at(when, callback, *args)
-
-    def _process_events(self, event_list):
-        return
-
-    def _write_to_self(self):
-        pass
-
-
-def MockCallback(**kwargs):
-    return mock.Mock(spec=['__call__'], **kwargs)
-
-
-class MockPattern(str):
-    """A regex based str with a fuzzy __eq__.
-
-    Use this helper with 'mock.assert_called_with', or anywhere
-    where a regex comparison between strings is needed.
-
-    For instance:
-       mock_call.assert_called_with(MockPattern('spam.*ham'))
-    """
-    def __eq__(self, other):
-        return bool(re.search(str(self), other, re.S))
-
-
-def get_function_source(func):
-    source = events._get_function_source(func)
-    if source is None:
-        raise ValueError("unable to get the source of %r" % (func,))
-    return source
-
-
-class TestCase(unittest.TestCase):
-    def set_event_loop(self, loop, *, cleanup=True):
-        assert loop is not None
-        # ensure that the event loop is passed explicitly in asyncio
-        events.set_event_loop(None)
-        if cleanup:
-            self.addCleanup(loop.close)
-
-    def new_test_loop(self, gen=None):
-        loop = TestLoop(gen)
-        self.set_event_loop(loop)
-        return loop
-
-    def setUp(self):
-        self._get_running_loop = events._get_running_loop
-        events._get_running_loop = lambda: None
-
-    def tearDown(self):
-        events._get_running_loop = self._get_running_loop
-
-        events.set_event_loop(None)
-
-        # Detect CPython bug #23353: ensure that yield/yield-from is not used
-        # in an except block of a generator
-        self.assertEqual(sys.exc_info(), (None, None, None))
-
-    if not compat.PY34:
-        # Python 3.3 compatibility
-        def subTest(self, *args, **kwargs):
-            class EmptyCM:
-                def __enter__(self):
-                    pass
-                def __exit__(self, *exc):
-                    pass
-            return EmptyCM()
-
-
-@contextlib.contextmanager
-def disable_logger():
-    """Context manager to disable asyncio logger.
-
-    For example, it can be used to ignore warnings in debug mode.
-    """
-    old_level = logger.level
-    try:
-        logger.setLevel(logging.CRITICAL+1)
-        yield
-    finally:
-        logger.setLevel(old_level)
-
-
-def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM,
-                            family=socket.AF_INET):
-    """Create a mock of a non-blocking socket."""
-    sock = mock.MagicMock(socket.socket)
-    sock.proto = proto
-    sock.type = type
-    sock.family = family
-    sock.gettimeout.return_value = 0.0
-    return sock
-
-
-def force_legacy_ssl_support():
-    return mock.patch('asyncio.sslproto._is_sslproto_available',
-                      return_value=False)

Lib/asyncio/threads.py

@@ -0,0 +1,25 @@
+"""High-level support for working with threads in asyncio"""
+
+import functools
+import contextvars
+
+from . import events
+
+
+__all__ = "to_thread",
+
+
+async def to_thread(func, /, *args, **kwargs):
+    """Asynchronously run function *func* in a separate thread.
+
+    Any *args and **kwargs supplied for this function are directly passed
+    to *func*. Also, the current :class:`contextvars.Context` is propagated,
+    allowing context variables from the main thread to be accessed in the
+    separate thread.
+
+    Return a coroutine that can be awaited to get the eventual result of *func*.
+    """
+    loop = events.get_running_loop()
+    ctx = contextvars.copy_context()
+    func_call = functools.partial(ctx.run, func, *args, **kwargs)
+    return await loop.run_in_executor(None, func_call)

Lib/asyncio/timeouts.py

@@ -0,0 +1,168 @@
+import enum
+
+from types import TracebackType
+from typing import final, Optional, Type
+
+from . import events
+from . import exceptions
+from . import tasks
+
+
+__all__ = (
+    "Timeout",
+    "timeout",
+    "timeout_at",
+)
+
+
+class _State(enum.Enum):
+    CREATED = "created"
+    ENTERED = "active"
+    EXPIRING = "expiring"
+    EXPIRED = "expired"
+    EXITED = "finished"
+
+
+@final
+class Timeout:
+    """Asynchronous context manager for cancelling overdue coroutines.
+
+    Use `timeout()` or `timeout_at()` rather than instantiating this class directly.
+    """
+
+    def __init__(self, when: Optional[float]) -> None:
+        """Schedule a timeout that will trigger at a given loop time.
+
+        - If `when` is `None`, the timeout will never trigger.
+        - If `when < loop.time()`, the timeout will trigger on the next
+          iteration of the event loop.
+        """
+        self._state = _State.CREATED
+
+        self._timeout_handler: Optional[events.TimerHandle] = None
+        self._task: Optional[tasks.Task] = None
+        self._when = when
+
+    def when(self) -> Optional[float]:
+        """Return the current deadline."""
+        return self._when
+
+    def reschedule(self, when: Optional[float]) -> None:
+        """Reschedule the timeout."""
+        if self._state is not _State.ENTERED:
+            if self._state is _State.CREATED:
+                raise RuntimeError("Timeout has not been entered")
+            raise RuntimeError(
+                f"Cannot change state of {self._state.value} Timeout",
+            )
+
+        self._when = when
+
+        if self._timeout_handler is not None:
+            self._timeout_handler.cancel()
+
+        if when is None:
+            self._timeout_handler = None
+        else:
+            loop = events.get_running_loop()
+            if when <= loop.time():
+                self._timeout_handler = loop.call_soon(self._on_timeout)
+            else:
+                self._timeout_handler = loop.call_at(when, self._on_timeout)
+
+    def expired(self) -> bool:
+        """Is timeout expired during execution?"""
+        return self._state in (_State.EXPIRING, _State.EXPIRED)
+
+    def __repr__(self) -> str:
+        info = ['']
+        if self._state is _State.ENTERED:
+            when = round(self._when, 3) if self._when is not None else None
+            info.append(f"when={when}")
+        info_str = ' '.join(info)
+        return f"<Timeout [{self._state.value}]{info_str}>"
+
+    async def __aenter__(self) -> "Timeout":
+        if self._state is not _State.CREATED:
+            raise RuntimeError("Timeout has already been entered")
+        task = tasks.current_task()
+        if task is None:
+            raise RuntimeError("Timeout should be used inside a task")
+        self._state = _State.ENTERED
+        self._task = task
+        self._cancelling = self._task.cancelling()
+        self.reschedule(self._when)
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> Optional[bool]:
+        assert self._state in (_State.ENTERED, _State.EXPIRING)
+
+        if self._timeout_handler is not None:
+            self._timeout_handler.cancel()
+            self._timeout_handler = None
+
+        if self._state is _State.EXPIRING:
+            self._state = _State.EXPIRED
+
+            if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError:
+                # Since there are no new cancel requests, we're
+                # handling this.
+                raise TimeoutError from exc_val
+        elif self._state is _State.ENTERED:
+            self._state = _State.EXITED
+
+        return None
+
+    def _on_timeout(self) -> None:
+        assert self._state is _State.ENTERED
+        self._task.cancel()
+        self._state = _State.EXPIRING
+        # drop the reference early
+        self._timeout_handler = None
+
+
+def timeout(delay: Optional[float]) -> Timeout:
+    """Timeout async context manager.
+
+    Useful in cases when you want to apply timeout logic around block
+    of code or in cases when asyncio.wait_for is not suitable. For example:
+
+    >>> async with asyncio.timeout(10):  # 10 seconds timeout
+    ...     await long_running_task()
+
+
+    delay - value in seconds or None to disable timeout logic
+
+    long_running_task() is interrupted by raising asyncio.CancelledError,
+    the top-most affected timeout() context manager converts CancelledError
+    into TimeoutError.
+    """
+    loop = events.get_running_loop()
+    return Timeout(loop.time() + delay if delay is not None else None)
+
+
+def timeout_at(when: Optional[float]) -> Timeout:
+    """Schedule the timeout at absolute time.
+
+    Like timeout() but argument gives absolute time in the same clock system
+    as loop.time().
+
+    Please note: it is not POSIX time but a time with
+    undefined starting base, e.g. the time of the system power on.
+
+    >>> async with asyncio.timeout_at(loop.time() + 10):
+    ...     await long_running_task()
+
+
+    when - a deadline when timeout occurs or None to disable timeout logic
+
+    long_running_task() is interrupted by raising asyncio.CancelledError,
+    the top-most affected timeout() context manager converts CancelledError
+    into TimeoutError.
+    """
+    return Timeout(when)

Lib/asyncio/transports.py

@@ -1,15 +1,16 @@
 """Abstract Transport class."""
 
-from asyncio import compat
+__all__ = (
-
+    'BaseTransport', 'ReadTransport', 'WriteTransport',
-__all__ = ['BaseTransport', 'ReadTransport', 'WriteTransport',
+    'Transport', 'DatagramTransport', 'SubprocessTransport',
-           'Transport', 'DatagramTransport', 'SubprocessTransport',
+)
-           ]
 
 
 class BaseTransport:
     """Base class for transports."""
 
+    __slots__ = ('_extra',)
+
     def __init__(self, extra=None):
         if extra is None:
             extra = {}
@@ -28,8 +29,8 @@ class BaseTransport:
 
         Buffered data will be flushed asynchronously.  No more data
         will be received.  After all buffered data is flushed, the
-        protocol's connection_lost() method will (eventually) called
+        protocol's connection_lost() method will (eventually) be
-        with None as its argument.
+        called with None as its argument.
         """
         raise NotImplementedError
 
@@ -45,6 +46,12 @@ class BaseTransport:
 class ReadTransport(BaseTransport):
     """Interface for read-only transports."""
 
+    __slots__ = ()
+
+    def is_reading(self):
+        """Return True if the transport is receiving."""
+        raise NotImplementedError
+
     def pause_reading(self):
         """Pause the receiving end.
 
@@ -65,6 +72,8 @@ class ReadTransport(BaseTransport):
 class WriteTransport(BaseTransport):
     """Interface for write-only transports."""
 
+    __slots__ = ()
+
     def set_write_buffer_limits(self, high=None, low=None):
         """Set the high- and low-water limits for write flow control.
 
@@ -90,6 +99,12 @@ class WriteTransport(BaseTransport):
         """Return the current size of the write buffer."""
         raise NotImplementedError
 
+    def get_write_buffer_limits(self):
+        """Get the high and low watermarks for write flow control.
+        Return a tuple (low, high) where low and high are
+        positive number of bytes."""
+        raise NotImplementedError
+
     def write(self, data):
         """Write some data bytes to the transport.
 
@@ -104,7 +119,7 @@ class WriteTransport(BaseTransport):
         The default implementation concatenates the arguments and
         calls write() on the result.
         """
-        data = compat.flatten_list_bytes(list_of_data)
+        data = b''.join(list_of_data)
         self.write(data)
 
     def write_eof(self):
@@ -151,10 +166,14 @@ class Transport(ReadTransport, WriteTransport):
     except writelines(), which calls write() in a loop.
     """
 
+    __slots__ = ()
+
 
 class DatagramTransport(BaseTransport):
     """Interface for datagram (UDP) transports."""
 
+    __slots__ = ()
+
     def sendto(self, data, addr=None):
         """Send data to the transport.
 
@@ -177,6 +196,8 @@ class DatagramTransport(BaseTransport):
 
 class SubprocessTransport(BaseTransport):
 
+    __slots__ = ()
+
     def get_pid(self):
         """Get subprocess id."""
         raise NotImplementedError
@@ -244,6 +265,8 @@ class _FlowControlMixin(Transport):
     resume_writing() may be called.
     """
 
+    __slots__ = ('_loop', '_protocol_paused', '_high_water', '_low_water')
+
     def __init__(self, extra=None, loop=None):
         super().__init__(extra)
         assert loop is not None
@@ -259,7 +282,9 @@ class _FlowControlMixin(Transport):
             self._protocol_paused = True
             try:
                 self._protocol.pause_writing()
-            except Exception as exc:
+            except (SystemExit, KeyboardInterrupt):
+                raise
+            except BaseException as exc:
                 self._loop.call_exception_handler({
                     'message': 'protocol.pause_writing() failed',
                     'exception': exc,
@@ -269,11 +294,13 @@ class _FlowControlMixin(Transport):
 
     def _maybe_resume_protocol(self):
         if (self._protocol_paused and
-            self.get_write_buffer_size() <= self._low_water):
+                self.get_write_buffer_size() <= self._low_water):
             self._protocol_paused = False
             try:
                 self._protocol.resume_writing()
-            except Exception as exc:
+            except (SystemExit, KeyboardInterrupt):
+                raise
+            except BaseException as exc:
                 self._loop.call_exception_handler({
                     'message': 'protocol.resume_writing() failed',
                     'exception': exc,
@@ -287,14 +314,16 @@ class _FlowControlMixin(Transport):
     def _set_write_buffer_limits(self, high=None, low=None):
         if high is None:
             if low is None:
-                high = 64*1024
+                high = 64 * 1024
             else:
-                high = 4*low
+                high = 4 * low
         if low is None:
             low = high // 4
+
         if not high >= low >= 0:
-            raise ValueError('high (%r) must be >= low (%r) must be >= 0' %
+            raise ValueError(
-                             (high, low))
+                f'high ({high!r}) must be >= low ({low!r}) must be >= 0')
+
         self._high_water = high
         self._low_water = low
 

Lib/asyncio/trsock.py

@@ -0,0 +1,98 @@
+import socket
+
+
+class TransportSocket:
+
+    """A socket-like wrapper for exposing real transport sockets.
+
+    These objects can be safely returned by APIs like
+    `transport.get_extra_info('socket')`.  All potentially disruptive
+    operations (like "socket.close()") are banned.
+    """
+
+    __slots__ = ('_sock',)
+
+    def __init__(self, sock: socket.socket):
+        self._sock = sock
+
+    @property
+    def family(self):
+        return self._sock.family
+
+    @property
+    def type(self):
+        return self._sock.type
+
+    @property
+    def proto(self):
+        return self._sock.proto
+
+    def __repr__(self):
+        s = (
+            f"<asyncio.TransportSocket fd={self.fileno()}, "
+            f"family={self.family!s}, type={self.type!s}, "
+            f"proto={self.proto}"
+        )
+
+        if self.fileno() != -1:
+            try:
+                laddr = self.getsockname()
+                if laddr:
+                    s = f"{s}, laddr={laddr}"
+            except socket.error:
+                pass
+            try:
+                raddr = self.getpeername()
+                if raddr:
+                    s = f"{s}, raddr={raddr}"
+            except socket.error:
+                pass
+
+        return f"{s}>"
+
+    def __getstate__(self):
+        raise TypeError("Cannot serialize asyncio.TransportSocket object")
+
+    def fileno(self):
+        return self._sock.fileno()
+
+    def dup(self):
+        return self._sock.dup()
+
+    def get_inheritable(self):
+        return self._sock.get_inheritable()
+
+    def shutdown(self, how):
+        # asyncio doesn't currently provide a high-level transport API
+        # to shutdown the connection.
+        self._sock.shutdown(how)
+
+    def getsockopt(self, *args, **kwargs):
+        return self._sock.getsockopt(*args, **kwargs)
+
+    def setsockopt(self, *args, **kwargs):
+        self._sock.setsockopt(*args, **kwargs)
+
+    def getpeername(self):
+        return self._sock.getpeername()
+
+    def getsockname(self):
+        return self._sock.getsockname()
+
+    def getsockbyname(self):
+        return self._sock.getsockbyname()
+
+    def settimeout(self, value):
+        if value == 0:
+            return
+        raise ValueError(
+            'settimeout(): only 0 timeout is allowed on transport sockets')
+
+    def gettimeout(self):
+        return 0
+
+    def setblocking(self, flag):
+        if not flag:
+            return
+        raise ValueError(
+            'setblocking(): transport sockets cannot be blocking')

Lib/asyncio/windows_events.py

@@ -1,32 +1,41 @@
 """Selector and proactor event loops for Windows."""
 
+import sys
+
+if sys.platform != 'win32':  # pragma: no cover
+    raise ImportError('win32 only')
+
+import _overlapped
 import _winapi
 import errno
+from functools import partial
 import math
+import msvcrt
 import socket
 import struct
+import time
 import weakref
 
 from . import events
 from . import base_subprocess
 from . import futures
+from . import exceptions
 from . import proactor_events
 from . import selector_events
 from . import tasks
 from . import windows_utils
-# XXX RustPython TODO: _overlapped
-# from . import _overlapped
-from .coroutines import coroutine
 from .log import logger
 
 
-__all__ = ['SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor',
+__all__ = (
-           'DefaultEventLoopPolicy',
+    'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor',
-           ]
+    'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy',
+    'WindowsProactorEventLoopPolicy',
+)
 
 
-NULL = 0
+NULL = _winapi.NULL
-INFINITE = 0xffffffff
+INFINITE = _winapi.INFINITE
 ERROR_CONNECTION_REFUSED = 1225
 ERROR_CONNECTION_ABORTED = 1236
 
@@ -53,7 +62,7 @@ class _OverlappedFuture(futures.Future):
         info = super()._repr_info()
         if self._ov is not None:
             state = 'pending' if self._ov.pending else 'completed'
-            info.insert(1, 'overlapped=<%s, %#x>' % (state, self._ov.address))
+            info.insert(1, f'overlapped=<{state}, {self._ov.address:#x}>')
         return info
 
     def _cancel_overlapped(self):
@@ -72,9 +81,9 @@ class _OverlappedFuture(futures.Future):
             self._loop.call_exception_handler(context)
         self._ov = None
 
-    def cancel(self):
+    def cancel(self, msg=None):
         self._cancel_overlapped()
-        return super().cancel()
+        return super().cancel(msg=msg)
 
     def set_exception(self, exception):
         super().set_exception(exception)
@@ -109,12 +118,12 @@ class _BaseWaitHandleFuture(futures.Future):
 
     def _repr_info(self):
         info = super()._repr_info()
-        info.append('handle=%#x' % self._handle)
+        info.append(f'handle={self._handle:#x}')
         if self._handle is not None:
             state = 'signaled' if self._poll() else 'waiting'
             info.append(state)
         if self._wait_handle is not None:
-            info.append('wait_handle=%#x' % self._wait_handle)
+            info.append(f'wait_handle={self._wait_handle:#x}')
         return info
 
     def _unregister_wait_cb(self, fut):
@@ -146,9 +155,9 @@ class _BaseWaitHandleFuture(futures.Future):
 
         self._unregister_wait_cb(None)
 
-    def cancel(self):
+    def cancel(self, msg=None):
         self._unregister_wait()
-        return super().cancel()
+        return super().cancel(msg=msg)
 
     def set_exception(self, exception):
         self._unregister_wait()
@@ -297,9 +306,6 @@ class PipeServer(object):
 class _WindowsSelectorEventLoop(selector_events.BaseSelectorEventLoop):
     """Windows version of selector event loop."""
 
-    def _socketpair(self):
-        return windows_utils.socketpair()
-
 
 class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
     """Windows version of proactor event loop using IOCP."""
@@ -309,20 +315,34 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
             proactor = IocpProactor()
         super().__init__(proactor)
 
-    def _socketpair(self):
+    def run_forever(self):
-        return windows_utils.socketpair()
+        try:
+            assert self._self_reading_future is None
+            self.call_soon(self._loop_self_reading)
+            super().run_forever()
+        finally:
+            if self._self_reading_future is not None:
+                ov = self._self_reading_future._ov
+                self._self_reading_future.cancel()
+                # self_reading_future always uses IOCP, so even though it's
+                # been cancelled, we need to make sure that the IOCP message
+                # is received so that the kernel is not holding on to the
+                # memory, possibly causing memory corruption later. Only
+                # unregister it if IO is complete in all respects. Otherwise
+                # we need another _poll() later to complete the IO.
+                if ov is not None and not ov.pending:
+                    self._proactor._unregister(ov)
+                self._self_reading_future = None
 
-    @coroutine
+    async def create_pipe_connection(self, protocol_factory, address):
-    def create_pipe_connection(self, protocol_factory, address):
         f = self._proactor.connect_pipe(address)
-        pipe = yield from f
+        pipe = await f
         protocol = protocol_factory()
         trans = self._make_duplex_pipe_transport(pipe, protocol,
                                                  extra={'addr': address})
         return trans, protocol
 
-    @coroutine
+    async def start_serving_pipe(self, protocol_factory, address):
-    def start_serving_pipe(self, protocol_factory, address):
         server = PipeServer(address)
 
         def loop_accept_pipe(f=None):
@@ -347,6 +367,10 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
                     return
 
                 f = self._proactor.accept_pipe(pipe)
+            except BrokenPipeError:
+                if pipe and pipe.fileno() != -1:
+                    pipe.close()
+                self.call_soon(loop_accept_pipe)
             except OSError as exc:
                 if pipe and pipe.fileno() != -1:
                     self.call_exception_handler({
@@ -358,7 +382,8 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
                 elif self._debug:
                     logger.warning("Accept pipe failed on pipe %r",
                                    pipe, exc_info=True)
-            except futures.CancelledError:
+                self.call_soon(loop_accept_pipe)
+            except exceptions.CancelledError:
                 if pipe:
                     pipe.close()
             else:
@@ -368,28 +393,22 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
         self.call_soon(loop_accept_pipe)
         return [server]
 
-    @coroutine
+    async def _make_subprocess_transport(self, protocol, args, shell,
-    def _make_subprocess_transport(self, protocol, args, shell,
+                                         stdin, stdout, stderr, bufsize,
-                                   stdin, stdout, stderr, bufsize,
+                                         extra=None, **kwargs):
-                                   extra=None, **kwargs):
         waiter = self.create_future()
         transp = _WindowsSubprocessTransport(self, protocol, args, shell,
                                              stdin, stdout, stderr, bufsize,
                                              waiter=waiter, extra=extra,
                                              **kwargs)
         try:
-            yield from waiter
+            await waiter
-        except Exception as exc:
+        except (SystemExit, KeyboardInterrupt):
-            # Workaround CPython bug #23353: using yield/yield-from in an
+            raise
-            # except block of a generator doesn't clear properly sys.exc_info()
+        except BaseException:
-            err = exc
-        else:
-            err = None
-
-        if err is not None:
             transp.close()
-            yield from transp._wait()
+            await transp._wait()
-            raise err
+            raise
 
         return transp
 
@@ -397,7 +416,7 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
 class IocpProactor:
     """Proactor implementation using IOCP."""
 
-    def __init__(self, concurrency=0xffffffff):
+    def __init__(self, concurrency=INFINITE):
         self._loop = None
         self._results = []
         self._iocp = _overlapped.CreateIoCompletionPort(
@@ -407,10 +426,16 @@ class IocpProactor:
         self._unregistered = []
         self._stopped_serving = weakref.WeakSet()
 
+    def _check_closed(self):
+        if self._iocp is None:
+            raise RuntimeError('IocpProactor is closed')
+
     def __repr__(self):
-        return ('<%s overlapped#=%s result#=%s>'
+        info = ['overlapped#=%s' % len(self._cache),
-                % (self.__class__.__name__, len(self._cache),
+                'result#=%s' % len(self._results)]
-                   len(self._results)))
+        if self._iocp is None:
+            info.append('closed')
+        return '<%s %s>' % (self.__class__.__name__, " ".join(info))
 
     def set_loop(self, loop):
         self._loop = loop
@@ -420,13 +445,40 @@ class IocpProactor:
             self._poll(timeout)
         tmp = self._results
         self._results = []
-        return tmp
+        try:
+            return tmp
+        finally:
+            # Needed to break cycles when an exception occurs.
+            tmp = None
 
     def _result(self, value):
         fut = self._loop.create_future()
         fut.set_result(value)
         return fut
 
+    @staticmethod
+    def finish_socket_func(trans, key, ov):
+        try:
+            return ov.getresult()
+        except OSError as exc:
+            if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED,
+                                _overlapped.ERROR_OPERATION_ABORTED):
+                raise ConnectionResetError(*exc.args)
+            else:
+                raise
+
+    @classmethod
+    def _finish_recvfrom(cls, trans, key, ov, *, empty_result):
+        try:
+            return cls.finish_socket_func(trans, key, ov)
+        except OSError as exc:
+            # WSARecvFrom will report ERROR_PORT_UNREACHABLE when the same
+            # socket is used to send to an address that is not listening.
+            if exc.winerror == _overlapped.ERROR_PORT_UNREACHABLE:
+                return empty_result, None
+            else:
+                raise
+
     def recv(self, conn, nbytes, flags=0):
         self._register_with_iocp(conn)
         ov = _overlapped.Overlapped(NULL)
@@ -438,16 +490,50 @@ class IocpProactor:
         except BrokenPipeError:
             return self._result(b'')
 
-        def finish_recv(trans, key, ov):
+        return self._register(ov, conn, self.finish_socket_func)
-            try:
-                return ov.getresult()
-            except OSError as exc:
-                if exc.winerror == _overlapped.ERROR_NETNAME_DELETED:
-                    raise ConnectionResetError(*exc.args)
-                else:
-                    raise
 
-        return self._register(ov, conn, finish_recv)
+    def recv_into(self, conn, buf, flags=0):
+        self._register_with_iocp(conn)
+        ov = _overlapped.Overlapped(NULL)
+        try:
+            if isinstance(conn, socket.socket):
+                ov.WSARecvInto(conn.fileno(), buf, flags)
+            else:
+                ov.ReadFileInto(conn.fileno(), buf)
+        except BrokenPipeError:
+            return self._result(0)
+
+        return self._register(ov, conn, self.finish_socket_func)
+
+    def recvfrom(self, conn, nbytes, flags=0):
+        self._register_with_iocp(conn)
+        ov = _overlapped.Overlapped(NULL)
+        try:
+            ov.WSARecvFrom(conn.fileno(), nbytes, flags)
+        except BrokenPipeError:
+            return self._result((b'', None))
+
+        return self._register(ov, conn, partial(self._finish_recvfrom,
+                                                empty_result=b''))
+
+    def recvfrom_into(self, conn, buf, flags=0):
+        self._register_with_iocp(conn)
+        ov = _overlapped.Overlapped(NULL)
+        try:
+            ov.WSARecvFromInto(conn.fileno(), buf, flags)
+        except BrokenPipeError:
+            return self._result((0, None))
+
+        return self._register(ov, conn, partial(self._finish_recvfrom,
+                                                empty_result=0))
+
+    def sendto(self, conn, buf, flags=0, addr=None):
+        self._register_with_iocp(conn)
+        ov = _overlapped.Overlapped(NULL)
+
+        ov.WSASendTo(conn.fileno(), buf, flags, addr)
+
+        return self._register(ov, conn, self.finish_socket_func)
 
     def send(self, conn, buf, flags=0):
         self._register_with_iocp(conn)
@@ -457,16 +543,7 @@ class IocpProactor:
         else:
             ov.WriteFile(conn.fileno(), buf)
 
-        def finish_send(trans, key, ov):
+        return self._register(ov, conn, self.finish_socket_func)
-            try:
-                return ov.getresult()
-            except OSError as exc:
-                if exc.winerror == _overlapped.ERROR_NETNAME_DELETED:
-                    raise ConnectionResetError(*exc.args)
-                else:
-                    raise
-
-        return self._register(ov, conn, finish_send)
 
     def accept(self, listener):
         self._register_with_iocp(listener)
@@ -483,12 +560,11 @@ class IocpProactor:
             conn.settimeout(listener.gettimeout())
             return conn, conn.getpeername()
 
-        @coroutine
+        async def accept_coro(future, conn):
-        def accept_coro(future, conn):
             # Coroutine closing the accept socket if the future is cancelled
             try:
-                yield from future
+                await future
-            except futures.CancelledError:
+            except exceptions.CancelledError:
                 conn.close()
                 raise
 
@@ -498,6 +574,14 @@ class IocpProactor:
         return future
 
     def connect(self, conn, address):
+        if conn.type == socket.SOCK_DGRAM:
+            # WSAConnect will complete immediately for UDP sockets so we don't
+            # need to register any IOCP operation
+            _overlapped.WSAConnect(conn.fileno(), address)
+            fut = self._loop.create_future()
+            fut.set_result(None)
+            return fut
+
         self._register_with_iocp(conn)
         # The socket needs to be locally bound before we call ConnectEx().
         try:
@@ -520,6 +604,18 @@ class IocpProactor:
 
         return self._register(ov, conn, finish_connect)
 
+    def sendfile(self, sock, file, offset, count):
+        self._register_with_iocp(sock)
+        ov = _overlapped.Overlapped(NULL)
+        offset_low = offset & 0xffff_ffff
+        offset_high = (offset >> 32) & 0xffff_ffff
+        ov.TransmitFile(sock.fileno(),
+                        msvcrt.get_osfhandle(file.fileno()),
+                        offset_low, offset_high,
+                        count, 0, 0)
+
+        return self._register(ov, sock, self.finish_socket_func)
+
     def accept_pipe(self, pipe):
         self._register_with_iocp(pipe)
         ov = _overlapped.Overlapped(NULL)
@@ -537,13 +633,12 @@ class IocpProactor:
 
         return self._register(ov, pipe, finish_accept_pipe)
 
-    @coroutine
+    async def connect_pipe(self, address):
-    def connect_pipe(self, address):
         delay = CONNECT_PIPE_INIT_DELAY
         while True:
-            # Unfortunately there is no way to do an overlapped connect to a pipe.
+            # Unfortunately there is no way to do an overlapped connect to
-            # Call CreateFile() in a loop until it doesn't fail with
+            # a pipe.  Call CreateFile() in a loop until it doesn't fail with
-            # ERROR_PIPE_BUSY
+            # ERROR_PIPE_BUSY.
             try:
                 handle = _overlapped.ConnectPipe(address)
                 break
@@ -553,7 +648,7 @@ class IocpProactor:
 
             # ConnectPipe() failed with ERROR_PIPE_BUSY: retry later
             delay = min(delay * 2, CONNECT_PIPE_MAX_DELAY)
-            yield from tasks.sleep(delay, loop=self._loop)
+            await tasks.sleep(delay)
 
         return windows_utils.PipeHandle(handle)
 
@@ -573,6 +668,8 @@ class IocpProactor:
         return fut
 
     def _wait_for_handle(self, handle, timeout, _is_cancel):
+        self._check_closed()
+
         if timeout is None:
             ms = _winapi.INFINITE
         else:
@@ -615,6 +712,8 @@ class IocpProactor:
             # that succeed immediately.
 
     def _register(self, ov, obj, callback):
+        self._check_closed()
+
         # Return a future which will be set with the result of the
         # operation when it completes.  The future's value is actually
         # the value returned by callback().
@@ -651,6 +750,7 @@ class IocpProactor:
         already be signalled (pending in the proactor event queue). It is also
         safe if the event is never signalled (because it was cancelled).
         """
+        self._check_closed()
         self._unregistered.append(ov)
 
     def _get_accept_socket(self, family):
@@ -707,8 +807,10 @@ class IocpProactor:
                 else:
                     f.set_result(value)
                     self._results.append(f)
+                finally:
+                    f = None
 
-        # Remove unregisted futures
+        # Remove unregistered futures
         for ov in self._unregistered:
             self._cache.pop(ov.address, None)
         self._unregistered.clear()
@@ -720,8 +822,12 @@ class IocpProactor:
         self._stopped_serving.add(obj)
 
     def close(self):
+        if self._iocp is None:
+            # already closed
+            return
+
         # Cancel remaining registered operations.
-        for address, (fut, ov, obj, callback) in list(self._cache.items()):
+        for fut, ov, obj, callback in list(self._cache.values()):
             if fut.cancelled():
                 # Nothing to do with cancelled futures
                 pass
@@ -742,14 +848,25 @@ class IocpProactor:
                             context['source_traceback'] = fut._source_traceback
                         self._loop.call_exception_handler(context)
 
+        # Wait until all cancelled overlapped complete: don't exit with running
+        # overlapped to prevent a crash. Display progress every second if the
+        # loop is still running.
+        msg_update = 1.0
+        start_time = time.monotonic()
+        next_msg = start_time + msg_update
         while self._cache:
-            if not self._poll(1):
+            if next_msg <= time.monotonic():
-                logger.debug('taking long time to close proactor')
+                logger.debug('%r is running after closing for %.1f seconds',
+                             self, time.monotonic() - start_time)
+                next_msg = time.monotonic() + msg_update
+
+            # handle a few events, or timeout
+            self._poll(msg_update)
 
         self._results = []
-        if self._iocp is not None:
+
-            _winapi.CloseHandle(self._iocp)
+        _winapi.CloseHandle(self._iocp)
-            self._iocp = None
+        self._iocp = None
 
     def __del__(self):
         self.close()
@@ -773,8 +890,12 @@ class _WindowsSubprocessTransport(base_subprocess.BaseSubprocessTransport):
 SelectorEventLoop = _WindowsSelectorEventLoop
 
 
-class _WindowsDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
+class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
     _loop_factory = SelectorEventLoop
 
 
-DefaultEventLoopPolicy = _WindowsDefaultEventLoopPolicy
+class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
+    _loop_factory = ProactorEventLoop
+
+
+DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy

Lib/asyncio/windows_utils.py

@@ -1,6 +1,4 @@
-"""
+"""Various Windows specific bits and pieces."""
-Various Windows specific bits and pieces
-"""
 
 import sys
 
@@ -11,13 +9,12 @@ import _winapi
 import itertools
 import msvcrt
 import os
-import socket
 import subprocess
 import tempfile
 import warnings
 
 
-__all__ = ['socketpair', 'pipe', 'Popen', 'PIPE', 'PipeHandle']
+__all__ = 'pipe', 'Popen', 'PIPE', 'PipeHandle'
 
 
 # Constants/globals
@@ -29,61 +26,14 @@ STDOUT = subprocess.STDOUT
 _mmap_counter = itertools.count()
 
 
-if hasattr(socket, 'socketpair'):
-    # Since Python 3.5, socket.socketpair() is now also available on Windows
-    socketpair = socket.socketpair
-else:
-    # Replacement for socket.socketpair()
-    def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
-        """A socket pair usable as a self-pipe, for Windows.
-
-        Origin: https://gist.github.com/4325783, by Geert Jansen.
-        Public domain.
-        """
-        if family == socket.AF_INET:
-            host = '127.0.0.1'
-        elif family == socket.AF_INET6:
-            host = '::1'
-        else:
-            raise ValueError("Only AF_INET and AF_INET6 socket address "
-                             "families are supported")
-        if type != socket.SOCK_STREAM:
-            raise ValueError("Only SOCK_STREAM socket type is supported")
-        if proto != 0:
-            raise ValueError("Only protocol zero is supported")
-
-        # We create a connected TCP socket. Note the trick with setblocking(0)
-        # that prevents us from having to create a thread.
-        lsock = socket.socket(family, type, proto)
-        try:
-            lsock.bind((host, 0))
-            lsock.listen(1)
-            # On IPv6, ignore flow_info and scope_id
-            addr, port = lsock.getsockname()[:2]
-            csock = socket.socket(family, type, proto)
-            try:
-                csock.setblocking(False)
-                try:
-                    csock.connect((addr, port))
-                except (BlockingIOError, InterruptedError):
-                    pass
-                csock.setblocking(True)
-                ssock, _ = lsock.accept()
-            except:
-                csock.close()
-                raise
-        finally:
-            lsock.close()
-        return (ssock, csock)
-
-
 # Replacement for os.pipe() using handles instead of fds
 
 
 def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE):
     """Like os.pipe() but with overlapped support and using handles not fds."""
-    address = tempfile.mktemp(prefix=r'\\.\pipe\python-pipe-%d-%d-' %
+    address = tempfile.mktemp(
-                              (os.getpid(), next(_mmap_counter)))
+        prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format(
+            os.getpid(), next(_mmap_counter)))
 
     if duplex:
         openmode = _winapi.PIPE_ACCESS_DUPLEX
@@ -138,10 +88,10 @@ class PipeHandle:
 
     def __repr__(self):
         if self._handle is not None:
-            handle = 'handle=%r' % self._handle
+            handle = f'handle={self._handle!r}'
         else:
             handle = 'closed'
-        return '<%s %s>' % (self.__class__.__name__, handle)
+        return f'<{self.__class__.__name__} {handle}>'
 
     @property
     def handle(self):
@@ -149,7 +99,7 @@ class PipeHandle:
 
     def fileno(self):
         if self._handle is None:
-            raise ValueError("I/O operatioon on closed pipe")
+            raise ValueError("I/O operation on closed pipe")
         return self._handle
 
     def close(self, *, CloseHandle=_winapi.CloseHandle):
@@ -157,10 +107,9 @@ class PipeHandle:
             CloseHandle(self._handle)
             self._handle = None
 
-    def __del__(self):
+    def __del__(self, _warn=warnings.warn):
         if self._handle is not None:
-            warnings.warn("unclosed %r" % self, ResourceWarning,
+            _warn(f"unclosed {self!r}", ResourceWarning, source=self)
-                          source=self)
             self.close()
 
     def __enter__(self):

Lib/codecs.py

@@ -414,6 +414,9 @@ class StreamWriter(Codec):
     def __exit__(self, type, value, tb):
         self.stream.close()
 
+    def __reduce_ex__(self, proto):
+        raise TypeError("can't serialize %s" % self.__class__.__name__)
+
 ###
 
 class StreamReader(Codec):
@@ -663,6 +666,9 @@ class StreamReader(Codec):
     def __exit__(self, type, value, tb):
         self.stream.close()
 
+    def __reduce_ex__(self, proto):
+        raise TypeError("can't serialize %s" % self.__class__.__name__)
+
 ###
 
 class StreamReaderWriter:
@@ -750,6 +756,9 @@ class StreamReaderWriter:
     def __exit__(self, type, value, tb):
         self.stream.close()
 
+    def __reduce_ex__(self, proto):
+        raise TypeError("can't serialize %s" % self.__class__.__name__)
+
 ###
 
 class StreamRecoder:
@@ -866,6 +875,9 @@ class StreamRecoder:
     def __exit__(self, type, value, tb):
         self.stream.close()
 
+    def __reduce_ex__(self, proto):
+        raise TypeError("can't serialize %s" % self.__class__.__name__)
+
 ### Shortcuts
 
 def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
@@ -878,7 +890,8 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
         codecs. Output is also codec dependent and will usually be
         Unicode as well.
 
-        Underlying encoded files are always opened in binary mode.
+        If encoding is not None, then the
+        underlying encoded files are always opened in binary mode.
         The default file mode is 'r', meaning to open the file in read mode.
 
         encoding specifies the encoding which is to be used for the
@@ -1114,13 +1127,3 @@ except LookupError:
 _false = 0
 if _false:
     import encodings
-
-### Tests
-
-if __name__ == '__main__':
-
-    # Make stdout translate Latin-1 output into UTF-8 output
-    sys.stdout = EncodedFile(sys.stdout, 'latin-1', 'utf-8')
-
-    # Have stdin translate Latin-1 input into UTF-8 input
-    sys.stdin = EncodedFile(sys.stdin, 'utf-8', 'latin-1')

Lib/contextlib.py

@@ -145,14 +145,17 @@ class _GeneratorContextManager(
             except StopIteration:
                 return False
             else:
-                raise RuntimeError("generator didn't stop")
+                try:
+                    raise RuntimeError("generator didn't stop")
+                finally:
+                    self.gen.close()
         else:
             if value is None:
                 # Need to force instantiation so we can reliably
                 # tell if we get the same exception back
                 value = typ()
             try:
-                self.gen.throw(typ, value, traceback)
+                self.gen.throw(value)
             except StopIteration as exc:
                 # Suppress StopIteration *unless* it's the same exception that
                 # was passed to throw().  This prevents a StopIteration
@@ -187,7 +190,10 @@ class _GeneratorContextManager(
                     raise
                 exc.__traceback__ = traceback
                 return False
-            raise RuntimeError("generator didn't stop after throw()")
+            try:
+                raise RuntimeError("generator didn't stop after throw()")
+            finally:
+                self.gen.close()
 
 class _AsyncGeneratorContextManager(
     _GeneratorContextManagerBase,
@@ -212,14 +218,17 @@ class _AsyncGeneratorContextManager(
             except StopAsyncIteration:
                 return False
             else:
-                raise RuntimeError("generator didn't stop")
+                try:
+                    raise RuntimeError("generator didn't stop")
+                finally:
+                    await self.gen.aclose()
         else:
             if value is None:
                 # Need to force instantiation so we can reliably
                 # tell if we get the same exception back
                 value = typ()
             try:
-                await self.gen.athrow(typ, value, traceback)
+                await self.gen.athrow(value)
             except StopAsyncIteration as exc:
                 # Suppress StopIteration *unless* it's the same exception that
                 # was passed to throw().  This prevents a StopIteration
@@ -254,7 +263,10 @@ class _AsyncGeneratorContextManager(
                     raise
                 exc.__traceback__ = traceback
                 return False
-            raise RuntimeError("generator didn't stop after athrow()")
+            try:
+                raise RuntimeError("generator didn't stop after athrow()")
+            finally:
+                await self.gen.aclose()
 
 
 def contextmanager(func):
@@ -441,7 +453,16 @@ class suppress(AbstractContextManager):
         # exactly reproduce the limitations of the CPython interpreter.
         #
         # See http://bugs.python.org/issue12029 for more details
-        return exctype is not None and issubclass(exctype, self._exceptions)
+        if exctype is None:
+            return
+        if issubclass(exctype, self._exceptions):
+            return True
+        if issubclass(exctype, BaseExceptionGroup):
+            match, rest = excinst.split(self._exceptions)
+            if rest is None:
+                return True
+            raise rest
+        return False
 
 
 class _BaseExitStack:

Lib/copyreg.py

@@ -25,16 +25,16 @@ def constructor(object):
 
 # Example: provide pickling support for complex numbers.
 
-try:
+def pickle_complex(c):
-    complex
+    return complex, (c.real, c.imag)
-except NameError:
-    pass
-else:
 
-    def pickle_complex(c):
+pickle(complex, pickle_complex, complex)
-        return complex, (c.real, c.imag)
 
-    pickle(complex, pickle_complex, complex)
+def pickle_union(obj):
+    import functools, operator
+    return functools.reduce, (operator.or_, obj.__args__)
+
+pickle(type(int | str), pickle_union)
 
 # Support for pickling new-style objects
 
@@ -48,6 +48,7 @@ def _reconstructor(cls, base, state):
     return obj
 
 _HEAPTYPE = 1<<9
+_new_type = type(int.__new__)
 
 # Python code for object.__reduce_ex__ for protocols 0 and 1
 
@@ -57,6 +58,9 @@ def _reduce_ex(self, proto):
     for base in cls.__mro__:
         if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE:
             break
+        new = base.__new__
+        if isinstance(new, _new_type) and new.__self__ is base:
+            break
     else:
         base = object # not really reachable
     if base is object:
@@ -79,6 +83,10 @@ def _reduce_ex(self, proto):
         except AttributeError:
             dict = None
     else:
+        if (type(self).__getstate__ is object.__getstate__ and
+            getattr(self, "__slots__", None)):
+            raise TypeError("a class that defines __slots__ without "
+                            "defining __getstate__ cannot be pickled")
         dict = getstate()
     if dict:
         return _reconstructor, args, dict

Lib/ctypes/test/test_numbers.py

@@ -82,14 +82,6 @@ class NumberTestCase(unittest.TestCase):
             self.assertRaises(TypeError, t, "")
             self.assertRaises(TypeError, t, None)
 
-    @unittest.skip('test disabled')
-    def test_valid_ranges(self):
-        # invalid values of the correct type
-        # raise ValueError (not OverflowError)
-        for t, (l, h) in zip(unsigned_types, unsigned_ranges):
-            self.assertRaises(ValueError, t, l-1)
-            self.assertRaises(ValueError, t, h+1)
-
     def test_from_param(self):
         # the from_param class method attribute always
         # returns PyCArgObject instances
@@ -106,7 +98,7 @@ class NumberTestCase(unittest.TestCase):
     def test_floats(self):
         # c_float and c_double can be created from
         # Python int and float
-        class FloatLike(object):
+        class FloatLike:
             def __float__(self):
                 return 2.0
         f = FloatLike()
@@ -117,15 +109,15 @@ class NumberTestCase(unittest.TestCase):
             self.assertEqual(t(f).value, 2.0)
 
     def test_integers(self):
-        class FloatLike(object):
+        class FloatLike:
             def __float__(self):
                 return 2.0
         f = FloatLike()
-        class IntLike(object):
+        class IntLike:
             def __int__(self):
                 return 2
         d = IntLike()
-        class IndexLike(object):
+        class IndexLike:
             def __index__(self):
                 return 2
         i = IndexLike()
@@ -155,10 +147,10 @@ class NumberTestCase(unittest.TestCase):
 
             # alignment of the type...
             self.assertEqual((code, alignment(t)),
-                                 (code, align))
+                             (code, align))
             # and alignment of an instance
             self.assertEqual((code, alignment(t())),
-                                 (code, align))
+                             (code, align))
 
     def test_int_from_address(self):
         from array import array
@@ -205,19 +197,6 @@ class NumberTestCase(unittest.TestCase):
         a[0] = ord('?')
         self.assertEqual(v.value, b'?')
 
-    # array does not support c_bool / 't'
-    @unittest.skip('test disabled')
-    def test_bool_from_address(self):
-        from ctypes import c_bool
-        from array import array
-        a = array(c_bool._type_, [True])
-        v = t.from_address(a.buffer_info()[0])
-        self.assertEqual(v.value, a[0])
-        self.assertEqual(type(v) is t)
-        a[0] = False
-        self.assertEqual(v.value, a[0])
-        self.assertEqual(type(v) is t)
-
     def test_init(self):
         # c_int() can be initialized from Python's int, and c_int.
         # Not from c_long or so, which seems strange, abc should
@@ -234,62 +213,6 @@ class NumberTestCase(unittest.TestCase):
             if (hasattr(t, "__ctype_le__")):
                 self.assertRaises(OverflowError, t.__ctype_le__, big_int)
 
-    @unittest.skip('test disabled')
-    def test_perf(self):
-        check_perf()
-
-from ctypes import _SimpleCData
-class c_int_S(_SimpleCData):
-    _type_ = "i"
-    __slots__ = []
-
-def run_test(rep, msg, func, arg=None):
-##    items = [None] * rep
-    items = range(rep)
-    from time import perf_counter as clock
-    if arg is not None:
-        start = clock()
-        for i in items:
-            func(arg); func(arg); func(arg); func(arg); func(arg)
-        stop = clock()
-    else:
-        start = clock()
-        for i in items:
-            func(); func(); func(); func(); func()
-        stop = clock()
-    print("%15s: %.2f us" % (msg, ((stop-start)*1e6/5/rep)))
-
-def check_perf():
-    # Construct 5 objects
-    from ctypes import c_int
-
-    REP = 200000
-
-    run_test(REP, "int()", int)
-    run_test(REP, "int(999)", int)
-    run_test(REP, "c_int()", c_int)
-    run_test(REP, "c_int(999)", c_int)
-    run_test(REP, "c_int_S()", c_int_S)
-    run_test(REP, "c_int_S(999)", c_int_S)
-
-# Python 2.3 -OO, win2k, P4 700 MHz:
-#
-#          int(): 0.87 us
-#       int(999): 0.87 us
-#        c_int(): 3.35 us
-#     c_int(999): 3.34 us
-#      c_int_S(): 3.23 us
-#   c_int_S(999): 3.24 us
-
-# Python 2.2 -OO, win2k, P4 700 MHz:
-#
-#          int(): 0.89 us
-#       int(999): 0.89 us
-#        c_int(): 9.99 us
-#     c_int(999): 10.02 us
-#      c_int_S(): 9.87 us
-#   c_int_S(999): 9.85 us
 
 if __name__ == '__main__':
-##    check_perf()
     unittest.main()

Lib/difflib.py

@@ -1200,25 +1200,6 @@ def context_diff(a, b, fromfile='', tofile='',
     strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'.
     The modification times are normally expressed in the ISO 8601 format.
     If not specified, the strings default to blanks.
-
-    Example:
-
-    >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True),
-    ...       'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')),
-    ...       end="")
-    *** Original
-    --- Current
-    ***************
-    *** 1,4 ****
-      one
-    ! two
-    ! three
-      four
-    --- 1,4 ----
-    + zero
-      one
-    ! tree
-      four
     """
 
     _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm)

Lib/email/message.py

@@ -7,7 +7,6 @@
 __all__ = ['Message', 'EmailMessage']
 
 import re
-import uu
 import quopri
 from io import BytesIO, StringIO
 
@@ -101,6 +100,35 @@ def _unquotevalue(value):
         return utils.unquote(value)
 
 
+def _decode_uu(encoded):
+    """Decode uuencoded data."""
+    decoded_lines = []
+    encoded_lines_iter = iter(encoded.splitlines())
+    for line in encoded_lines_iter:
+        if line.startswith(b"begin "):
+            mode, _, path = line.removeprefix(b"begin ").partition(b" ")
+            try:
+                int(mode, base=8)
+            except ValueError:
+                continue
+            else:
+                break
+    else:
+        raise ValueError("`begin` line not found")
+    for line in encoded_lines_iter:
+        if not line:
+            raise ValueError("Truncated input")
+        elif line.strip(b' \t\r\n\f') == b'end':
+            break
+        try:
+            decoded_line = binascii.a2b_uu(line)
+        except binascii.Error:
+            # Workaround for broken uuencoders by /Fredrik Lundh
+            nbytes = (((line[0]-32) & 63) * 4 + 5) // 3
+            decoded_line = binascii.a2b_uu(line[:nbytes])
+        decoded_lines.append(decoded_line)
+
+    return b''.join(decoded_lines)
 
 class Message:
     """Basic message object.
@@ -288,13 +316,10 @@ class Message:
                 self.policy.handle_defect(self, defect)
             return value
         elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'):
-            in_file = BytesIO(bpayload)
-            out_file = BytesIO()
             try:
-                uu.decode(in_file, out_file, quiet=True)
+                return _decode_uu(bpayload)
-                return out_file.getvalue()
+            except ValueError:
-            except uu.Error:
+                # Some decoding problem.
-                # Some decoding problem
                 return bpayload
         if isinstance(payload, str):
             return bpayload

Lib/encodings/__init__.py

@@ -156,6 +156,10 @@ def search_function(encoding):
 codecs.register(search_function)
 
 if sys.platform == 'win32':
+    # bpo-671666, bpo-46668: If Python does not implement a codec for current
+    # Windows ANSI code page, use the "mbcs" codec instead:
+    # WideCharToMultiByte() and MultiByteToWideChar() functions with CP_ACP.
+    # Python does not support custom code pages.
     def _alias_mbcs(encoding):
         try:
             import _winapi

Lib/encodings/cp65001.py

@@ -1,43 +0,0 @@
-"""
-Code page 65001: Windows UTF-8 (CP_UTF8).
-"""
-
-import codecs
-import functools
-
-if not hasattr(codecs, 'code_page_encode'):
-    raise LookupError("cp65001 encoding is only available on Windows")
-
-### Codec APIs
-
-encode = functools.partial(codecs.code_page_encode, 65001)
-_decode = functools.partial(codecs.code_page_decode, 65001)
-
-def decode(input, errors='strict'):
-    return codecs.code_page_decode(65001, input, errors, True)
-
-class IncrementalEncoder(codecs.IncrementalEncoder):
-    def encode(self, input, final=False):
-        return encode(input, self.errors)[0]
-
-class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
-    _buffer_decode = _decode
-
-class StreamWriter(codecs.StreamWriter):
-    encode = encode
-
-class StreamReader(codecs.StreamReader):
-    decode = _decode
-
-### encodings module API
-
-def getregentry():
-    return codecs.CodecInfo(
-        name='cp65001',
-        encode=encode,
-        decode=decode,
-        incrementalencoder=IncrementalEncoder,
-        incrementaldecoder=IncrementalDecoder,
-        streamreader=StreamReader,
-        streamwriter=StreamWriter,
-    )

Lib/encodings/idna.py

@@ -39,23 +39,21 @@ def nameprep(label):
 
     # Check bidi
     RandAL = [stringprep.in_table_d1(x) for x in label]
-    for c in RandAL:
+    if any(RandAL):
-        if c:
+        # There is a RandAL char in the string. Must perform further
-            # There is a RandAL char in the string. Must perform further
+        # tests:
-            # tests:
+        # 1) The characters in section 5.8 MUST be prohibited.
-            # 1) The characters in section 5.8 MUST be prohibited.
+        # This is table C.8, which was already checked
-            # This is table C.8, which was already checked
+        # 2) If a string contains any RandALCat character, the string
-            # 2) If a string contains any RandALCat character, the string
+        # MUST NOT contain any LCat character.
-            # MUST NOT contain any LCat character.
+        if any(stringprep.in_table_d2(x) for x in label):
-            if any(stringprep.in_table_d2(x) for x in label):
+            raise UnicodeError("Violation of BIDI requirement 2")
-                raise UnicodeError("Violation of BIDI requirement 2")
+        # 3) If a string contains any RandALCat character, a
-
+        # RandALCat character MUST be the first character of the
-            # 3) If a string contains any RandALCat character, a
+        # string, and a RandALCat character MUST be the last
-            # RandALCat character MUST be the first character of the
+        # character of the string.
-            # string, and a RandALCat character MUST be the last
+        if not RandAL[0] or not RandAL[-1]:
-            # character of the string.
+            raise UnicodeError("Violation of BIDI requirement 3")
-            if not RandAL[0] or not RandAL[-1]:
-                raise UnicodeError("Violation of BIDI requirement 3")
 
     return label
 
@@ -103,6 +101,16 @@ def ToASCII(label):
     raise UnicodeError("label empty or too long")
 
 def ToUnicode(label):
+    if len(label) > 1024:
+        # Protection from https://github.com/python/cpython/issues/98433.
+        # https://datatracker.ietf.org/doc/html/rfc5894#section-6
+        # doesn't specify a label size limit prior to NAMEPREP. But having
+        # one makes practical sense.
+        # This leaves ample room for nameprep() to remove Nothing characters
+        # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still
+        # preventing us from wasting time decoding a big thing that'll just
+        # hit the actual <= 63 length limit in Step 6.
+        raise UnicodeError("label way too long")
     # Step 1: Check for ASCII
     if isinstance(label, bytes):
         pure_ascii = True

Lib/encodings/mac_centeuro.py

@@ -1,307 +0,0 @@
-""" Python Character Mapping Codec mac_centeuro generated from 'MAPPINGS/VENDORS/APPLE/CENTEURO.TXT' with gencodec.py.
-
-"""#"
-
-import codecs
-
-### Codec APIs
-
-class Codec(codecs.Codec):
-
-    def encode(self,input,errors='strict'):
-        return codecs.charmap_encode(input,errors,encoding_table)
-
-    def decode(self,input,errors='strict'):
-        return codecs.charmap_decode(input,errors,decoding_table)
-
-class IncrementalEncoder(codecs.IncrementalEncoder):
-    def encode(self, input, final=False):
-        return codecs.charmap_encode(input,self.errors,encoding_table)[0]
-
-class IncrementalDecoder(codecs.IncrementalDecoder):
-    def decode(self, input, final=False):
-        return codecs.charmap_decode(input,self.errors,decoding_table)[0]
-
-class StreamWriter(Codec,codecs.StreamWriter):
-    pass
-
-class StreamReader(Codec,codecs.StreamReader):
-    pass
-
-### encodings module API
-
-def getregentry():
-    return codecs.CodecInfo(
-        name='mac-centeuro',
-        encode=Codec().encode,
-        decode=Codec().decode,
-        incrementalencoder=IncrementalEncoder,
-        incrementaldecoder=IncrementalDecoder,
-        streamreader=StreamReader,
-        streamwriter=StreamWriter,
-    )
-
-
-### Decoding Table
-
-decoding_table = (
-    '\x00'     #  0x00 -> CONTROL CHARACTER
-    '\x01'     #  0x01 -> CONTROL CHARACTER
-    '\x02'     #  0x02 -> CONTROL CHARACTER
-    '\x03'     #  0x03 -> CONTROL CHARACTER
-    '\x04'     #  0x04 -> CONTROL CHARACTER
-    '\x05'     #  0x05 -> CONTROL CHARACTER
-    '\x06'     #  0x06 -> CONTROL CHARACTER
-    '\x07'     #  0x07 -> CONTROL CHARACTER
-    '\x08'     #  0x08 -> CONTROL CHARACTER
-    '\t'       #  0x09 -> CONTROL CHARACTER
-    '\n'       #  0x0A -> CONTROL CHARACTER
-    '\x0b'     #  0x0B -> CONTROL CHARACTER
-    '\x0c'     #  0x0C -> CONTROL CHARACTER
-    '\r'       #  0x0D -> CONTROL CHARACTER
-    '\x0e'     #  0x0E -> CONTROL CHARACTER
-    '\x0f'     #  0x0F -> CONTROL CHARACTER
-    '\x10'     #  0x10 -> CONTROL CHARACTER
-    '\x11'     #  0x11 -> CONTROL CHARACTER
-    '\x12'     #  0x12 -> CONTROL CHARACTER
-    '\x13'     #  0x13 -> CONTROL CHARACTER
-    '\x14'     #  0x14 -> CONTROL CHARACTER
-    '\x15'     #  0x15 -> CONTROL CHARACTER
-    '\x16'     #  0x16 -> CONTROL CHARACTER
-    '\x17'     #  0x17 -> CONTROL CHARACTER
-    '\x18'     #  0x18 -> CONTROL CHARACTER
-    '\x19'     #  0x19 -> CONTROL CHARACTER
-    '\x1a'     #  0x1A -> CONTROL CHARACTER
-    '\x1b'     #  0x1B -> CONTROL CHARACTER
-    '\x1c'     #  0x1C -> CONTROL CHARACTER
-    '\x1d'     #  0x1D -> CONTROL CHARACTER
-    '\x1e'     #  0x1E -> CONTROL CHARACTER
-    '\x1f'     #  0x1F -> CONTROL CHARACTER
-    ' '        #  0x20 -> SPACE
-    '!'        #  0x21 -> EXCLAMATION MARK
-    '"'        #  0x22 -> QUOTATION MARK
-    '#'        #  0x23 -> NUMBER SIGN
-    '$'        #  0x24 -> DOLLAR SIGN
-    '%'        #  0x25 -> PERCENT SIGN
-    '&'        #  0x26 -> AMPERSAND
-    "'"        #  0x27 -> APOSTROPHE
-    '('        #  0x28 -> LEFT PARENTHESIS
-    ')'        #  0x29 -> RIGHT PARENTHESIS
-    '*'        #  0x2A -> ASTERISK
-    '+'        #  0x2B -> PLUS SIGN
-    ','        #  0x2C -> COMMA
-    '-'        #  0x2D -> HYPHEN-MINUS
-    '.'        #  0x2E -> FULL STOP
-    '/'        #  0x2F -> SOLIDUS
-    '0'        #  0x30 -> DIGIT ZERO
-    '1'        #  0x31 -> DIGIT ONE
-    '2'        #  0x32 -> DIGIT TWO
-    '3'        #  0x33 -> DIGIT THREE
-    '4'        #  0x34 -> DIGIT FOUR
-    '5'        #  0x35 -> DIGIT FIVE
-    '6'        #  0x36 -> DIGIT SIX
-    '7'        #  0x37 -> DIGIT SEVEN
-    '8'        #  0x38 -> DIGIT EIGHT
-    '9'        #  0x39 -> DIGIT NINE
-    ':'        #  0x3A -> COLON
-    ';'        #  0x3B -> SEMICOLON
-    '<'        #  0x3C -> LESS-THAN SIGN
-    '='        #  0x3D -> EQUALS SIGN
-    '>'        #  0x3E -> GREATER-THAN SIGN
-    '?'        #  0x3F -> QUESTION MARK
-    '@'        #  0x40 -> COMMERCIAL AT
-    'A'        #  0x41 -> LATIN CAPITAL LETTER A
-    'B'        #  0x42 -> LATIN CAPITAL LETTER B
-    'C'        #  0x43 -> LATIN CAPITAL LETTER C
-    'D'        #  0x44 -> LATIN CAPITAL LETTER D
-    'E'        #  0x45 -> LATIN CAPITAL LETTER E
-    'F'        #  0x46 -> LATIN CAPITAL LETTER F
-    'G'        #  0x47 -> LATIN CAPITAL LETTER G
-    'H'        #  0x48 -> LATIN CAPITAL LETTER H
-    'I'        #  0x49 -> LATIN CAPITAL LETTER I
-    'J'        #  0x4A -> LATIN CAPITAL LETTER J
-    'K'        #  0x4B -> LATIN CAPITAL LETTER K
-    'L'        #  0x4C -> LATIN CAPITAL LETTER L
-    'M'        #  0x4D -> LATIN CAPITAL LETTER M
-    'N'        #  0x4E -> LATIN CAPITAL LETTER N
-    'O'        #  0x4F -> LATIN CAPITAL LETTER O
-    'P'        #  0x50 -> LATIN CAPITAL LETTER P
-    'Q'        #  0x51 -> LATIN CAPITAL LETTER Q
-    'R'        #  0x52 -> LATIN CAPITAL LETTER R
-    'S'        #  0x53 -> LATIN CAPITAL LETTER S
-    'T'        #  0x54 -> LATIN CAPITAL LETTER T
-    'U'        #  0x55 -> LATIN CAPITAL LETTER U
-    'V'        #  0x56 -> LATIN CAPITAL LETTER V
-    'W'        #  0x57 -> LATIN CAPITAL LETTER W
-    'X'        #  0x58 -> LATIN CAPITAL LETTER X
-    'Y'        #  0x59 -> LATIN CAPITAL LETTER Y
-    'Z'        #  0x5A -> LATIN CAPITAL LETTER Z
-    '['        #  0x5B -> LEFT SQUARE BRACKET
-    '\\'       #  0x5C -> REVERSE SOLIDUS
-    ']'        #  0x5D -> RIGHT SQUARE BRACKET
-    '^'        #  0x5E -> CIRCUMFLEX ACCENT
-    '_'        #  0x5F -> LOW LINE
-    '`'        #  0x60 -> GRAVE ACCENT
-    'a'        #  0x61 -> LATIN SMALL LETTER A
-    'b'        #  0x62 -> LATIN SMALL LETTER B
-    'c'        #  0x63 -> LATIN SMALL LETTER C
-    'd'        #  0x64 -> LATIN SMALL LETTER D
-    'e'        #  0x65 -> LATIN SMALL LETTER E
-    'f'        #  0x66 -> LATIN SMALL LETTER F
-    'g'        #  0x67 -> LATIN SMALL LETTER G
-    'h'        #  0x68 -> LATIN SMALL LETTER H
-    'i'        #  0x69 -> LATIN SMALL LETTER I
-    'j'        #  0x6A -> LATIN SMALL LETTER J
-    'k'        #  0x6B -> LATIN SMALL LETTER K
-    'l'        #  0x6C -> LATIN SMALL LETTER L
-    'm'        #  0x6D -> LATIN SMALL LETTER M
-    'n'        #  0x6E -> LATIN SMALL LETTER N
-    'o'        #  0x6F -> LATIN SMALL LETTER O
-    'p'        #  0x70 -> LATIN SMALL LETTER P
-    'q'        #  0x71 -> LATIN SMALL LETTER Q
-    'r'        #  0x72 -> LATIN SMALL LETTER R
-    's'        #  0x73 -> LATIN SMALL LETTER S
-    't'        #  0x74 -> LATIN SMALL LETTER T
-    'u'        #  0x75 -> LATIN SMALL LETTER U
-    'v'        #  0x76 -> LATIN SMALL LETTER V
-    'w'        #  0x77 -> LATIN SMALL LETTER W
-    'x'        #  0x78 -> LATIN SMALL LETTER X
-    'y'        #  0x79 -> LATIN SMALL LETTER Y
-    'z'        #  0x7A -> LATIN SMALL LETTER Z
-    '{'        #  0x7B -> LEFT CURLY BRACKET
-    '|'        #  0x7C -> VERTICAL LINE
-    '}'        #  0x7D -> RIGHT CURLY BRACKET
-    '~'        #  0x7E -> TILDE
-    '\x7f'     #  0x7F -> CONTROL CHARACTER
-    '\xc4'     #  0x80 -> LATIN CAPITAL LETTER A WITH DIAERESIS
-    '\u0100'   #  0x81 -> LATIN CAPITAL LETTER A WITH MACRON
-    '\u0101'   #  0x82 -> LATIN SMALL LETTER A WITH MACRON
-    '\xc9'     #  0x83 -> LATIN CAPITAL LETTER E WITH ACUTE
-    '\u0104'   #  0x84 -> LATIN CAPITAL LETTER A WITH OGONEK
-    '\xd6'     #  0x85 -> LATIN CAPITAL LETTER O WITH DIAERESIS
-    '\xdc'     #  0x86 -> LATIN CAPITAL LETTER U WITH DIAERESIS
-    '\xe1'     #  0x87 -> LATIN SMALL LETTER A WITH ACUTE
-    '\u0105'   #  0x88 -> LATIN SMALL LETTER A WITH OGONEK
-    '\u010c'   #  0x89 -> LATIN CAPITAL LETTER C WITH CARON
-    '\xe4'     #  0x8A -> LATIN SMALL LETTER A WITH DIAERESIS
-    '\u010d'   #  0x8B -> LATIN SMALL LETTER C WITH CARON
-    '\u0106'   #  0x8C -> LATIN CAPITAL LETTER C WITH ACUTE
-    '\u0107'   #  0x8D -> LATIN SMALL LETTER C WITH ACUTE
-    '\xe9'     #  0x8E -> LATIN SMALL LETTER E WITH ACUTE
-    '\u0179'   #  0x8F -> LATIN CAPITAL LETTER Z WITH ACUTE
-    '\u017a'   #  0x90 -> LATIN SMALL LETTER Z WITH ACUTE
-    '\u010e'   #  0x91 -> LATIN CAPITAL LETTER D WITH CARON
-    '\xed'     #  0x92 -> LATIN SMALL LETTER I WITH ACUTE
-    '\u010f'   #  0x93 -> LATIN SMALL LETTER D WITH CARON
-    '\u0112'   #  0x94 -> LATIN CAPITAL LETTER E WITH MACRON
-    '\u0113'   #  0x95 -> LATIN SMALL LETTER E WITH MACRON
-    '\u0116'   #  0x96 -> LATIN CAPITAL LETTER E WITH DOT ABOVE
-    '\xf3'     #  0x97 -> LATIN SMALL LETTER O WITH ACUTE
-    '\u0117'   #  0x98 -> LATIN SMALL LETTER E WITH DOT ABOVE
-    '\xf4'     #  0x99 -> LATIN SMALL LETTER O WITH CIRCUMFLEX
-    '\xf6'     #  0x9A -> LATIN SMALL LETTER O WITH DIAERESIS
-    '\xf5'     #  0x9B -> LATIN SMALL LETTER O WITH TILDE
-    '\xfa'     #  0x9C -> LATIN SMALL LETTER U WITH ACUTE
-    '\u011a'   #  0x9D -> LATIN CAPITAL LETTER E WITH CARON
-    '\u011b'   #  0x9E -> LATIN SMALL LETTER E WITH CARON
-    '\xfc'     #  0x9F -> LATIN SMALL LETTER U WITH DIAERESIS
-    '\u2020'   #  0xA0 -> DAGGER
-    '\xb0'     #  0xA1 -> DEGREE SIGN
-    '\u0118'   #  0xA2 -> LATIN CAPITAL LETTER E WITH OGONEK
-    '\xa3'     #  0xA3 -> POUND SIGN
-    '\xa7'     #  0xA4 -> SECTION SIGN
-    '\u2022'   #  0xA5 -> BULLET
-    '\xb6'     #  0xA6 -> PILCROW SIGN
-    '\xdf'     #  0xA7 -> LATIN SMALL LETTER SHARP S
-    '\xae'     #  0xA8 -> REGISTERED SIGN
-    '\xa9'     #  0xA9 -> COPYRIGHT SIGN
-    '\u2122'   #  0xAA -> TRADE MARK SIGN
-    '\u0119'   #  0xAB -> LATIN SMALL LETTER E WITH OGONEK
-    '\xa8'     #  0xAC -> DIAERESIS
-    '\u2260'   #  0xAD -> NOT EQUAL TO
-    '\u0123'   #  0xAE -> LATIN SMALL LETTER G WITH CEDILLA
-    '\u012e'   #  0xAF -> LATIN CAPITAL LETTER I WITH OGONEK
-    '\u012f'   #  0xB0 -> LATIN SMALL LETTER I WITH OGONEK
-    '\u012a'   #  0xB1 -> LATIN CAPITAL LETTER I WITH MACRON
-    '\u2264'   #  0xB2 -> LESS-THAN OR EQUAL TO
-    '\u2265'   #  0xB3 -> GREATER-THAN OR EQUAL TO
-    '\u012b'   #  0xB4 -> LATIN SMALL LETTER I WITH MACRON
-    '\u0136'   #  0xB5 -> LATIN CAPITAL LETTER K WITH CEDILLA
-    '\u2202'   #  0xB6 -> PARTIAL DIFFERENTIAL
-    '\u2211'   #  0xB7 -> N-ARY SUMMATION
-    '\u0142'   #  0xB8 -> LATIN SMALL LETTER L WITH STROKE
-    '\u013b'   #  0xB9 -> LATIN CAPITAL LETTER L WITH CEDILLA
-    '\u013c'   #  0xBA -> LATIN SMALL LETTER L WITH CEDILLA
-    '\u013d'   #  0xBB -> LATIN CAPITAL LETTER L WITH CARON
-    '\u013e'   #  0xBC -> LATIN SMALL LETTER L WITH CARON
-    '\u0139'   #  0xBD -> LATIN CAPITAL LETTER L WITH ACUTE
-    '\u013a'   #  0xBE -> LATIN SMALL LETTER L WITH ACUTE
-    '\u0145'   #  0xBF -> LATIN CAPITAL LETTER N WITH CEDILLA
-    '\u0146'   #  0xC0 -> LATIN SMALL LETTER N WITH CEDILLA
-    '\u0143'   #  0xC1 -> LATIN CAPITAL LETTER N WITH ACUTE
-    '\xac'     #  0xC2 -> NOT SIGN
-    '\u221a'   #  0xC3 -> SQUARE ROOT
-    '\u0144'   #  0xC4 -> LATIN SMALL LETTER N WITH ACUTE
-    '\u0147'   #  0xC5 -> LATIN CAPITAL LETTER N WITH CARON
-    '\u2206'   #  0xC6 -> INCREMENT
-    '\xab'     #  0xC7 -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
-    '\xbb'     #  0xC8 -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
-    '\u2026'   #  0xC9 -> HORIZONTAL ELLIPSIS
-    '\xa0'     #  0xCA -> NO-BREAK SPACE
-    '\u0148'   #  0xCB -> LATIN SMALL LETTER N WITH CARON
-    '\u0150'   #  0xCC -> LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
-    '\xd5'     #  0xCD -> LATIN CAPITAL LETTER O WITH TILDE
-    '\u0151'   #  0xCE -> LATIN SMALL LETTER O WITH DOUBLE ACUTE
-    '\u014c'   #  0xCF -> LATIN CAPITAL LETTER O WITH MACRON
-    '\u2013'   #  0xD0 -> EN DASH
-    '\u2014'   #  0xD1 -> EM DASH
-    '\u201c'   #  0xD2 -> LEFT DOUBLE QUOTATION MARK
-    '\u201d'   #  0xD3 -> RIGHT DOUBLE QUOTATION MARK
-    '\u2018'   #  0xD4 -> LEFT SINGLE QUOTATION MARK
-    '\u2019'   #  0xD5 -> RIGHT SINGLE QUOTATION MARK
-    '\xf7'     #  0xD6 -> DIVISION SIGN
-    '\u25ca'   #  0xD7 -> LOZENGE
-    '\u014d'   #  0xD8 -> LATIN SMALL LETTER O WITH MACRON
-    '\u0154'   #  0xD9 -> LATIN CAPITAL LETTER R WITH ACUTE
-    '\u0155'   #  0xDA -> LATIN SMALL LETTER R WITH ACUTE
-    '\u0158'   #  0xDB -> LATIN CAPITAL LETTER R WITH CARON
-    '\u2039'   #  0xDC -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK
-    '\u203a'   #  0xDD -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
-    '\u0159'   #  0xDE -> LATIN SMALL LETTER R WITH CARON
-    '\u0156'   #  0xDF -> LATIN CAPITAL LETTER R WITH CEDILLA
-    '\u0157'   #  0xE0 -> LATIN SMALL LETTER R WITH CEDILLA
-    '\u0160'   #  0xE1 -> LATIN CAPITAL LETTER S WITH CARON
-    '\u201a'   #  0xE2 -> SINGLE LOW-9 QUOTATION MARK
-    '\u201e'   #  0xE3 -> DOUBLE LOW-9 QUOTATION MARK
-    '\u0161'   #  0xE4 -> LATIN SMALL LETTER S WITH CARON
-    '\u015a'   #  0xE5 -> LATIN CAPITAL LETTER S WITH ACUTE
-    '\u015b'   #  0xE6 -> LATIN SMALL LETTER S WITH ACUTE
-    '\xc1'     #  0xE7 -> LATIN CAPITAL LETTER A WITH ACUTE
-    '\u0164'   #  0xE8 -> LATIN CAPITAL LETTER T WITH CARON
-    '\u0165'   #  0xE9 -> LATIN SMALL LETTER T WITH CARON
-    '\xcd'     #  0xEA -> LATIN CAPITAL LETTER I WITH ACUTE
-    '\u017d'   #  0xEB -> LATIN CAPITAL LETTER Z WITH CARON
-    '\u017e'   #  0xEC -> LATIN SMALL LETTER Z WITH CARON
-    '\u016a'   #  0xED -> LATIN CAPITAL LETTER U WITH MACRON
-    '\xd3'     #  0xEE -> LATIN CAPITAL LETTER O WITH ACUTE
-    '\xd4'     #  0xEF -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX
-    '\u016b'   #  0xF0 -> LATIN SMALL LETTER U WITH MACRON
-    '\u016e'   #  0xF1 -> LATIN CAPITAL LETTER U WITH RING ABOVE
-    '\xda'     #  0xF2 -> LATIN CAPITAL LETTER U WITH ACUTE
-    '\u016f'   #  0xF3 -> LATIN SMALL LETTER U WITH RING ABOVE
-    '\u0170'   #  0xF4 -> LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
-    '\u0171'   #  0xF5 -> LATIN SMALL LETTER U WITH DOUBLE ACUTE
-    '\u0172'   #  0xF6 -> LATIN CAPITAL LETTER U WITH OGONEK
-    '\u0173'   #  0xF7 -> LATIN SMALL LETTER U WITH OGONEK
-    '\xdd'     #  0xF8 -> LATIN CAPITAL LETTER Y WITH ACUTE
-    '\xfd'     #  0xF9 -> LATIN SMALL LETTER Y WITH ACUTE
-    '\u0137'   #  0xFA -> LATIN SMALL LETTER K WITH CEDILLA
-    '\u017b'   #  0xFB -> LATIN CAPITAL LETTER Z WITH DOT ABOVE
-    '\u0141'   #  0xFC -> LATIN CAPITAL LETTER L WITH STROKE
-    '\u017c'   #  0xFD -> LATIN SMALL LETTER Z WITH DOT ABOVE
-    '\u0122'   #  0xFE -> LATIN CAPITAL LETTER G WITH CEDILLA
-    '\u02c7'   #  0xFF -> CARON
-)
-
-### Encoding table
-encoding_table=codecs.charmap_build(decoding_table)

Lib/encodings/unicode_internal.py

@@ -1,45 +0,0 @@
-""" Python 'unicode-internal' Codec
-
-
-Written by Marc-Andre Lemburg (mal@lemburg.com).
-
-(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
-
-"""
-import codecs
-
-### Codec APIs
-
-class Codec(codecs.Codec):
-
-    # Note: Binding these as C functions will result in the class not
-    # converting them to methods. This is intended.
-    encode = codecs.unicode_internal_encode
-    decode = codecs.unicode_internal_decode
-
-class IncrementalEncoder(codecs.IncrementalEncoder):
-    def encode(self, input, final=False):
-        return codecs.unicode_internal_encode(input, self.errors)[0]
-
-class IncrementalDecoder(codecs.IncrementalDecoder):
-    def decode(self, input, final=False):
-        return codecs.unicode_internal_decode(input, self.errors)[0]
-
-class StreamWriter(Codec,codecs.StreamWriter):
-    pass
-
-class StreamReader(Codec,codecs.StreamReader):
-    pass
-
-### encodings module API
-
-def getregentry():
-    return codecs.CodecInfo(
-        name='unicode-internal',
-        encode=Codec.encode,
-        decode=Codec.decode,
-        incrementalencoder=IncrementalEncoder,
-        incrementaldecoder=IncrementalDecoder,
-        streamwriter=StreamWriter,
-        streamreader=StreamReader,
-    )

Lib/ensurepip/__init__.py

@@ -9,11 +9,9 @@ from importlib import resources
 
 
 __all__ = ["version", "bootstrap"]
-_PACKAGE_NAMES = ('setuptools', 'pip')
+_PACKAGE_NAMES = ('pip',)
-_SETUPTOOLS_VERSION = "65.5.0"
+_PIP_VERSION = "23.2.1"
-_PIP_VERSION = "22.3.1"
 _PROJECTS = [
-    ("setuptools", _SETUPTOOLS_VERSION, "py3"),
     ("pip", _PIP_VERSION, "py3"),
 ]
 
@@ -153,17 +151,17 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
 
     _disable_pip_configuration_settings()
 
-    # By default, installing pip and setuptools installs all of the
+    # By default, installing pip installs all of the
     # following scripts (X.Y == running Python version):
     #
-    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
+    #   pip, pipX, pipX.Y
     #
     # pip 1.5+ allows ensurepip to request that some of those be left out
     if altinstall:
-        # omit pip, pipX and easy_install
+        # omit pip, pipX
         os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
     elif not default_pip:
-        # omit pip and easy_install
+        # omit pip
         os.environ["ENSUREPIP_OPTIONS"] = "install"
 
     with tempfile.TemporaryDirectory() as tmpdir:
@@ -271,14 +269,14 @@ def _main(argv=None):
         action="store_true",
         default=False,
         help=("Make an alternate install, installing only the X.Y versioned "
-              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)."),
+              "scripts (Default: pipX, pipX.Y)."),
     )
     parser.add_argument(
         "--default-pip",
         action="store_true",
         default=False,
         help=("Make a default pip install, installing the unqualified pip "
-              "and easy_install in addition to the versioned scripts."),
+              "in addition to the versioned scripts."),
     )
 
     args = parser.parse_args(argv)

Lib/filecmp.py

@@ -10,10 +10,7 @@ Functions:
 
 """
 
-try:
+import os
-    import os
-except ImportError:
-    import _dummy_os as os
 import stat
 from itertools import filterfalse
 from types import GenericAlias
@@ -160,17 +157,17 @@ class dircmp:
             a_path = os.path.join(self.left, x)
             b_path = os.path.join(self.right, x)
 
-            ok = 1
+            ok = True
             try:
                 a_stat = os.stat(a_path)
             except OSError:
                 # print('Can\'t stat', a_path, ':', why.args[1])
-                ok = 0
+                ok = False
             try:
                 b_stat = os.stat(b_path)
             except OSError:
                 # print('Can\'t stat', b_path, ':', why.args[1])
-                ok = 0
+                ok = False
 
             if ok:
                 a_type = stat.S_IFMT(a_stat.st_mode)
@@ -245,7 +242,7 @@ class dircmp:
 
     methodmap = dict(subdirs=phase4,
                      same_files=phase3, diff_files=phase3, funny_files=phase3,
-                     common_dirs = phase2, common_files=phase2, common_funny=phase2,
+                     common_dirs=phase2, common_files=phase2, common_funny=phase2,
                      common=phase1, left_only=phase1, right_only=phase1,
                      left_list=phase0, right_list=phase0)
 

Lib/fractions.py

@@ -4,6 +4,7 @@
 """Fraction, infinite-precision, rational numbers."""
 
 from decimal import Decimal
+import functools
 import math
 import numbers
 import operator
@@ -20,21 +21,144 @@ _PyHASH_MODULUS = sys.hash_info.modulus
 # _PyHASH_MODULUS.
 _PyHASH_INF = sys.hash_info.inf
 
+@functools.lru_cache(maxsize = 1 << 14)
+def _hash_algorithm(numerator, denominator):
+
+    # To make sure that the hash of a Fraction agrees with the hash
+    # of a numerically equal integer, float or Decimal instance, we
+    # follow the rules for numeric hashes outlined in the
+    # documentation.  (See library docs, 'Built-in Types').
+
+    try:
+        dinv = pow(denominator, -1, _PyHASH_MODULUS)
+    except ValueError:
+        # ValueError means there is no modular inverse.
+        hash_ = _PyHASH_INF
+    else:
+        # The general algorithm now specifies that the absolute value of
+        # the hash is
+        #    (|N| * dinv) % P
+        # where N is self._numerator and P is _PyHASH_MODULUS.  That's
+        # optimized here in two ways:  first, for a non-negative int i,
+        # hash(i) == i % P, but the int hash implementation doesn't need
+        # to divide, and is faster than doing % P explicitly.  So we do
+        #    hash(|N| * dinv)
+        # instead.  Second, N is unbounded, so its product with dinv may
+        # be arbitrarily expensive to compute.  The final answer is the
+        # same if we use the bounded |N| % P instead, which can again
+        # be done with an int hash() call.  If 0 <= i < P, hash(i) == i,
+        # so this nested hash() call wastes a bit of time making a
+        # redundant copy when |N| < P, but can save an arbitrarily large
+        # amount of computation for large |N|.
+        hash_ = hash(hash(abs(numerator)) * dinv)
+    result = hash_ if numerator >= 0 else -hash_
+    return -2 if result == -1 else result
+
 _RATIONAL_FORMAT = re.compile(r"""
-    \A\s*                                 # optional whitespace at the start,
+    \A\s*                                  # optional whitespace at the start,
-    (?P<sign>[-+]?)                       # an optional sign, then
+    (?P<sign>[-+]?)                        # an optional sign, then
-    (?=\d|\.\d)                           # lookahead for digit or .digit
+    (?=\d|\.\d)                            # lookahead for digit or .digit
-    (?P<num>\d*|\d+(_\d+)*)               # numerator (possibly empty)
+    (?P<num>\d*|\d+(_\d+)*)                # numerator (possibly empty)
-    (?:                                   # followed by
+    (?:                                    # followed by
-       (?:/(?P<denom>\d+(_\d+)*))?        # an optional denominator
+       (?:\s*/\s*(?P<denom>\d+(_\d+)*))?   # an optional denominator
-    |                                     # or
+    |                                      # or
-       (?:\.(?P<decimal>d*|\d+(_\d+)*))?  # an optional fractional part
+       (?:\.(?P<decimal>\d*|\d+(_\d+)*))?  # an optional fractional part
-       (?:E(?P<exp>[-+]?\d+(_\d+)*))?     # and optional exponent
+       (?:E(?P<exp>[-+]?\d+(_\d+)*))?      # and optional exponent
     )
-    \s*\Z                                 # and optional whitespace to finish
+    \s*\Z                                  # and optional whitespace to finish
 """, re.VERBOSE | re.IGNORECASE)
 
 
+# Helpers for formatting
+
+def _round_to_exponent(n, d, exponent, no_neg_zero=False):
+    """Round a rational number to the nearest multiple of a given power of 10.
+
+    Rounds the rational number n/d to the nearest integer multiple of
+    10**exponent, rounding to the nearest even integer multiple in the case of
+    a tie. Returns a pair (sign: bool, significand: int) representing the
+    rounded value (-1)**sign * significand * 10**exponent.
+
+    If no_neg_zero is true, then the returned sign will always be False when
+    the significand is zero. Otherwise, the sign reflects the sign of the
+    input.
+
+    d must be positive, but n and d need not be relatively prime.
+    """
+    if exponent >= 0:
+        d *= 10**exponent
+    else:
+        n *= 10**-exponent
+
+    # The divmod quotient is correct for round-ties-towards-positive-infinity;
+    # In the case of a tie, we zero out the least significant bit of q.
+    q, r = divmod(n + (d >> 1), d)
+    if r == 0 and d & 1 == 0:
+        q &= -2
+
+    sign = q < 0 if no_neg_zero else n < 0
+    return sign, abs(q)
+
+
+def _round_to_figures(n, d, figures):
+    """Round a rational number to a given number of significant figures.
+
+    Rounds the rational number n/d to the given number of significant figures
+    using the round-ties-to-even rule, and returns a triple
+    (sign: bool, significand: int, exponent: int) representing the rounded
+    value (-1)**sign * significand * 10**exponent.
+
+    In the special case where n = 0, returns a significand of zero and
+    an exponent of 1 - figures, for compatibility with formatting.
+    Otherwise, the returned significand satisfies
+    10**(figures - 1) <= significand < 10**figures.
+
+    d must be positive, but n and d need not be relatively prime.
+    figures must be positive.
+    """
+    # Special case for n == 0.
+    if n == 0:
+        return False, 0, 1 - figures
+
+    # Find integer m satisfying 10**(m - 1) <= abs(n)/d <= 10**m. (If abs(n)/d
+    # is a power of 10, either of the two possible values for m is fine.)
+    str_n, str_d = str(abs(n)), str(d)
+    m = len(str_n) - len(str_d) + (str_d <= str_n)
+
+    # Round to a multiple of 10**(m - figures). The significand we get
+    # satisfies 10**(figures - 1) <= significand <= 10**figures.
+    exponent = m - figures
+    sign, significand = _round_to_exponent(n, d, exponent)
+
+    # Adjust in the case where significand == 10**figures, to ensure that
+    # 10**(figures - 1) <= significand < 10**figures.
+    if len(str(significand)) == figures + 1:
+        significand //= 10
+        exponent += 1
+
+    return sign, significand, exponent
+
+
+# Pattern for matching float-style format specifications;
+# supports 'e', 'E', 'f', 'F', 'g', 'G' and '%' presentation types.
+_FLOAT_FORMAT_SPECIFICATION_MATCHER = re.compile(r"""
+    (?:
+        (?P<fill>.)?
+        (?P<align>[<>=^])
+    )?
+    (?P<sign>[-+ ]?)
+    (?P<no_neg_zero>z)?
+    (?P<alt>\#)?
+    # A '0' that's *not* followed by another digit is parsed as a minimum width
+    # rather than a zeropad flag.
+    (?P<zeropad>0(?=[0-9]))?
+    (?P<minimumwidth>0|[1-9][0-9]*)?
+    (?P<thousands_sep>[,_])?
+    (?:\.(?P<precision>0|[1-9][0-9]*))?
+    (?P<presentation_type>[eEfFgG%])
+""", re.DOTALL | re.VERBOSE).fullmatch
+
+
 class Fraction(numbers.Rational):
     """This class implements rational numbers.
 
@@ -59,7 +183,7 @@ class Fraction(numbers.Rational):
     __slots__ = ('_numerator', '_denominator')
 
     # We're immutable, so use __new__ not __init__
-    def __new__(cls, numerator=0, denominator=None, *, _normalize=True):
+    def __new__(cls, numerator=0, denominator=None):
         """Constructs a Rational.
 
         Takes a string like '3/2' or '1.5', another Rational instance, a
@@ -155,12 +279,11 @@ class Fraction(numbers.Rational):
 
         if denominator == 0:
             raise ZeroDivisionError('Fraction(%s, 0)' % numerator)
-        if _normalize:
+        g = math.gcd(numerator, denominator)
-            g = math.gcd(numerator, denominator)
+        if denominator < 0:
-            if denominator < 0:
+            g = -g
-                g = -g
+        numerator //= g
-            numerator //= g
+        denominator //= g
-            denominator //= g
         self._numerator = numerator
         self._denominator = denominator
         return self
@@ -177,7 +300,7 @@ class Fraction(numbers.Rational):
         elif not isinstance(f, float):
             raise TypeError("%s.from_float() only takes floats, not %r (%s)" %
                             (cls.__name__, f, type(f).__name__))
-        return cls(*f.as_integer_ratio())
+        return cls._from_coprime_ints(*f.as_integer_ratio())
 
     @classmethod
     def from_decimal(cls, dec):
@@ -189,13 +312,28 @@ class Fraction(numbers.Rational):
             raise TypeError(
                 "%s.from_decimal() only takes Decimals, not %r (%s)" %
                 (cls.__name__, dec, type(dec).__name__))
-        return cls(*dec.as_integer_ratio())
+        return cls._from_coprime_ints(*dec.as_integer_ratio())
+
+    @classmethod
+    def _from_coprime_ints(cls, numerator, denominator, /):
+        """Convert a pair of ints to a rational number, for internal use.
+
+        The ratio of integers should be in lowest terms and the denominator
+        should be positive.
+        """
+        obj = super(Fraction, cls).__new__(cls)
+        obj._numerator = numerator
+        obj._denominator = denominator
+        return obj
+
+    def is_integer(self):
+        """Return True if the Fraction is an integer."""
+        return self._denominator == 1
 
     def as_integer_ratio(self):
-        """Return the integer ratio as a tuple.
+        """Return a pair of integers, whose ratio is equal to the original Fraction.
 
-        Return a tuple of two integers, whose ratio is equal to the
+        The ratio is in lowest terms and has a positive denominator.
-        Fraction and with a positive denominator.
         """
         return (self._numerator, self._denominator)
 
@@ -245,14 +383,16 @@ class Fraction(numbers.Rational):
                 break
             p0, q0, p1, q1 = p1, q1, p0+a*p1, q2
             n, d = d, n-a*d
-
         k = (max_denominator-q0)//q1
-        bound1 = Fraction(p0+k*p1, q0+k*q1)
+
-        bound2 = Fraction(p1, q1)
+        # Determine which of the candidates (p0+k*p1)/(q0+k*q1) and p1/q1 is
-        if abs(bound2 - self) <= abs(bound1-self):
+        # closer to self. The distance between them is 1/(q1*(q0+k*q1)), while
-            return bound2
+        # the distance from p1/q1 to self is d/(q1*self._denominator). So we
+        # need to compare 2*(q0+k*q1) with self._denominator/d.
+        if 2*d*(q0+k*q1) <= self._denominator:
+            return Fraction._from_coprime_ints(p1, q1)
         else:
-            return bound1
+            return Fraction._from_coprime_ints(p0+k*p1, q0+k*q1)
 
     @property
     def numerator(a):
@@ -274,6 +414,122 @@ class Fraction(numbers.Rational):
         else:
             return '%s/%s' % (self._numerator, self._denominator)
 
+    def __format__(self, format_spec, /):
+        """Format this fraction according to the given format specification."""
+
+        # Backwards compatiblility with existing formatting.
+        if not format_spec:
+            return str(self)
+
+        # Validate and parse the format specifier.
+        match = _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec)
+        if match is None:
+            raise ValueError(
+                f"Invalid format specifier {format_spec!r} "
+                f"for object of type {type(self).__name__!r}"
+            )
+        elif match["align"] is not None and match["zeropad"] is not None:
+            # Avoid the temptation to guess.
+            raise ValueError(
+                f"Invalid format specifier {format_spec!r} "
+                f"for object of type {type(self).__name__!r}; "
+                "can't use explicit alignment when zero-padding"
+            )
+        fill = match["fill"] or " "
+        align = match["align"] or ">"
+        pos_sign = "" if match["sign"] == "-" else match["sign"]
+        no_neg_zero = bool(match["no_neg_zero"])
+        alternate_form = bool(match["alt"])
+        zeropad = bool(match["zeropad"])
+        minimumwidth = int(match["minimumwidth"] or "0")
+        thousands_sep = match["thousands_sep"]
+        precision = int(match["precision"] or "6")
+        presentation_type = match["presentation_type"]
+        trim_zeros = presentation_type in "gG" and not alternate_form
+        trim_point = not alternate_form
+        exponent_indicator = "E" if presentation_type in "EFG" else "e"
+
+        # Round to get the digits we need, figure out where to place the point,
+        # and decide whether to use scientific notation. 'point_pos' is the
+        # relative to the _end_ of the digit string: that is, it's the number
+        # of digits that should follow the point.
+        if presentation_type in "fF%":
+            exponent = -precision
+            if presentation_type == "%":
+                exponent -= 2
+            negative, significand = _round_to_exponent(
+                self._numerator, self._denominator, exponent, no_neg_zero)
+            scientific = False
+            point_pos = precision
+        else:  # presentation_type in "eEgG"
+            figures = (
+                max(precision, 1)
+                if presentation_type in "gG"
+                else precision + 1
+            )
+            negative, significand, exponent = _round_to_figures(
+                self._numerator, self._denominator, figures)
+            scientific = (
+                presentation_type in "eE"
+                or exponent > 0
+                or exponent + figures <= -4
+            )
+            point_pos = figures - 1 if scientific else -exponent
+
+        # Get the suffix - the part following the digits, if any.
+        if presentation_type == "%":
+            suffix = "%"
+        elif scientific:
+            suffix = f"{exponent_indicator}{exponent + point_pos:+03d}"
+        else:
+            suffix = ""
+
+        # String of output digits, padded sufficiently with zeros on the left
+        # so that we'll have at least one digit before the decimal point.
+        digits = f"{significand:0{point_pos + 1}d}"
+
+        # Before padding, the output has the form f"{sign}{leading}{trailing}",
+        # where `leading` includes thousands separators if necessary and
+        # `trailing` includes the decimal separator where appropriate.
+        sign = "-" if negative else pos_sign
+        leading = digits[: len(digits) - point_pos]
+        frac_part = digits[len(digits) - point_pos :]
+        if trim_zeros:
+            frac_part = frac_part.rstrip("0")
+        separator = "" if trim_point and not frac_part else "."
+        trailing = separator + frac_part + suffix
+
+        # Do zero padding if required.
+        if zeropad:
+            min_leading = minimumwidth - len(sign) - len(trailing)
+            # When adding thousands separators, they'll be added to the
+            # zero-padded portion too, so we need to compensate.
+            leading = leading.zfill(
+                3 * min_leading // 4 + 1 if thousands_sep else min_leading
+            )
+
+        # Insert thousands separators if required.
+        if thousands_sep:
+            first_pos = 1 + (len(leading) - 1) % 3
+            leading = leading[:first_pos] + "".join(
+                thousands_sep + leading[pos : pos + 3]
+                for pos in range(first_pos, len(leading), 3)
+            )
+
+        # We now have a sign and a body. Pad with fill character if necessary
+        # and return.
+        body = leading + trailing
+        padding = fill * (minimumwidth - len(sign) - len(body))
+        if align == ">":
+            return padding + sign + body
+        elif align == "<":
+            return sign + body + padding
+        elif align == "^":
+            half = len(padding) // 2
+            return padding[:half] + sign + body + padding[half:]
+        else:  # align == "="
+            return sign + padding + body
+
     def _operator_fallbacks(monomorphic_operator, fallback_operator):
         """Generates forward and reverse operators given a purely-rational
         operator and a function from the operator module.
@@ -355,8 +611,10 @@ class Fraction(numbers.Rational):
 
         """
         def forward(a, b):
-            if isinstance(b, (int, Fraction)):
+            if isinstance(b, Fraction):
                 return monomorphic_operator(a, b)
+            elif isinstance(b, int):
+                return monomorphic_operator(a, Fraction(b))
             elif isinstance(b, float):
                 return fallback_operator(float(a), b)
             elif isinstance(b, complex):
@@ -369,7 +627,7 @@ class Fraction(numbers.Rational):
         def reverse(b, a):
             if isinstance(a, numbers.Rational):
                 # Includes ints.
-                return monomorphic_operator(a, b)
+                return monomorphic_operator(Fraction(a), b)
             elif isinstance(a, numbers.Real):
                 return fallback_operator(float(a), float(b))
             elif isinstance(a, numbers.Complex):
@@ -451,40 +709,40 @@ class Fraction(numbers.Rational):
 
     def _add(a, b):
         """a + b"""
-        na, da = a.numerator, a.denominator
+        na, da = a._numerator, a._denominator
-        nb, db = b.numerator, b.denominator
+        nb, db = b._numerator, b._denominator
         g = math.gcd(da, db)
         if g == 1:
-            return Fraction(na * db + da * nb, da * db, _normalize=False)
+            return Fraction._from_coprime_ints(na * db + da * nb, da * db)
         s = da // g
         t = na * (db // g) + nb * s
         g2 = math.gcd(t, g)
         if g2 == 1:
-            return Fraction(t, s * db, _normalize=False)
+            return Fraction._from_coprime_ints(t, s * db)
-        return Fraction(t // g2, s * (db // g2), _normalize=False)
+        return Fraction._from_coprime_ints(t // g2, s * (db // g2))
 
     __add__, __radd__ = _operator_fallbacks(_add, operator.add)
 
     def _sub(a, b):
         """a - b"""
-        na, da = a.numerator, a.denominator
+        na, da = a._numerator, a._denominator
-        nb, db = b.numerator, b.denominator
+        nb, db = b._numerator, b._denominator
         g = math.gcd(da, db)
         if g == 1:
-            return Fraction(na * db - da * nb, da * db, _normalize=False)
+            return Fraction._from_coprime_ints(na * db - da * nb, da * db)
         s = da // g
         t = na * (db // g) - nb * s
         g2 = math.gcd(t, g)
         if g2 == 1:
-            return Fraction(t, s * db, _normalize=False)
+            return Fraction._from_coprime_ints(t, s * db)
-        return Fraction(t // g2, s * (db // g2), _normalize=False)
+        return Fraction._from_coprime_ints(t // g2, s * (db // g2))
 
     __sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub)
 
     def _mul(a, b):
         """a * b"""
-        na, da = a.numerator, a.denominator
+        na, da = a._numerator, a._denominator
-        nb, db = b.numerator, b.denominator
+        nb, db = b._numerator, b._denominator
         g1 = math.gcd(na, db)
         if g1 > 1:
             na //= g1
@@ -493,15 +751,17 @@ class Fraction(numbers.Rational):
         if g2 > 1:
             nb //= g2
             da //= g2
-        return Fraction(na * nb, db * da, _normalize=False)
+        return Fraction._from_coprime_ints(na * nb, db * da)
 
     __mul__, __rmul__ = _operator_fallbacks(_mul, operator.mul)
 
     def _div(a, b):
         """a / b"""
         # Same as _mul(), with inversed b.
-        na, da = a.numerator, a.denominator
+        nb, db = b._numerator, b._denominator
-        nb, db = b.numerator, b.denominator
+        if nb == 0:
+            raise ZeroDivisionError('Fraction(%s, 0)' % db)
+        na, da = a._numerator, a._denominator
         g1 = math.gcd(na, nb)
         if g1 > 1:
             na //= g1
@@ -513,7 +773,7 @@ class Fraction(numbers.Rational):
         n, d = na * db, nb * da
         if d < 0:
             n, d = -n, -d
-        return Fraction(n, d, _normalize=False)
+        return Fraction._from_coprime_ints(n, d)
 
     __truediv__, __rtruediv__ = _operator_fallbacks(_div, operator.truediv)
 
@@ -550,17 +810,17 @@ class Fraction(numbers.Rational):
             if b.denominator == 1:
                 power = b.numerator
                 if power >= 0:
-                    return Fraction(a._numerator ** power,
+                    return Fraction._from_coprime_ints(a._numerator ** power,
-                                    a._denominator ** power,
+                                                       a._denominator ** power)
-                                    _normalize=False)
+                elif a._numerator > 0:
-                elif a._numerator >= 0:
+                    return Fraction._from_coprime_ints(a._denominator ** -power,
-                    return Fraction(a._denominator ** -power,
+                                                       a._numerator ** -power)
-                                    a._numerator ** -power,
+                elif a._numerator == 0:
-                                    _normalize=False)
+                    raise ZeroDivisionError('Fraction(%s, 0)' %
+                                            a._denominator ** -power)
                 else:
-                    return Fraction((-a._denominator) ** -power,
+                    return Fraction._from_coprime_ints((-a._denominator) ** -power,
-                                    (-a._numerator) ** -power,
+                                                       (-a._numerator) ** -power)
-                                    _normalize=False)
             else:
                 # A fractional power will generally produce an
                 # irrational number.
@@ -584,15 +844,15 @@ class Fraction(numbers.Rational):
 
     def __pos__(a):
         """+a: Coerces a subclass instance to Fraction"""
-        return Fraction(a._numerator, a._denominator, _normalize=False)
+        return Fraction._from_coprime_ints(a._numerator, a._denominator)
 
     def __neg__(a):
         """-a"""
-        return Fraction(-a._numerator, a._denominator, _normalize=False)
+        return Fraction._from_coprime_ints(-a._numerator, a._denominator)
 
     def __abs__(a):
         """abs(a)"""
-        return Fraction(abs(a._numerator), a._denominator, _normalize=False)
+        return Fraction._from_coprime_ints(abs(a._numerator), a._denominator)
 
     def __int__(a, _index=operator.index):
         """int(a)"""
@@ -610,12 +870,12 @@ class Fraction(numbers.Rational):
 
     def __floor__(a):
         """math.floor(a)"""
-        return a.numerator // a.denominator
+        return a._numerator // a._denominator
 
     def __ceil__(a):
         """math.ceil(a)"""
         # The negations cleverly convince floordiv to return the ceiling.
-        return -(-a.numerator // a.denominator)
+        return -(-a._numerator // a._denominator)
 
     def __round__(self, ndigits=None):
         """round(self, ndigits)
@@ -623,10 +883,11 @@ class Fraction(numbers.Rational):
         Rounds half toward even.
         """
         if ndigits is None:
-            floor, remainder = divmod(self.numerator, self.denominator)
+            d = self._denominator
-            if remainder * 2 < self.denominator:
+            floor, remainder = divmod(self._numerator, d)
+            if remainder * 2 < d:
                 return floor
-            elif remainder * 2 > self.denominator:
+            elif remainder * 2 > d:
                 return floor + 1
             # Deal with the half case:
             elif floor % 2 == 0:
@@ -644,36 +905,7 @@ class Fraction(numbers.Rational):
 
     def __hash__(self):
         """hash(self)"""
-
+        return _hash_algorithm(self._numerator, self._denominator)
-        # To make sure that the hash of a Fraction agrees with the hash
-        # of a numerically equal integer, float or Decimal instance, we
-        # follow the rules for numeric hashes outlined in the
-        # documentation.  (See library docs, 'Built-in Types').
-
-        try:
-            dinv = pow(self._denominator, -1, _PyHASH_MODULUS)
-        except ValueError:
-            # ValueError means there is no modular inverse.
-            hash_ = _PyHASH_INF
-        else:
-            # The general algorithm now specifies that the absolute value of
-            # the hash is
-            #    (|N| * dinv) % P
-            # where N is self._numerator and P is _PyHASH_MODULUS.  That's
-            # optimized here in two ways:  first, for a non-negative int i,
-            # hash(i) == i % P, but the int hash implementation doesn't need
-            # to divide, and is faster than doing % P explicitly.  So we do
-            #    hash(|N| * dinv)
-            # instead.  Second, N is unbounded, so its product with dinv may
-            # be arbitrarily expensive to compute.  The final answer is the
-            # same if we use the bounded |N| % P instead, which can again
-            # be done with an int hash() call.  If 0 <= i < P, hash(i) == i,
-            # so this nested hash() call wastes a bit of time making a
-            # redundant copy when |N| < P, but can save an arbitrarily large
-            # amount of computation for large |N|.
-            hash_ = hash(hash(abs(self._numerator)) * dinv)
-        result = hash_ if self._numerator >= 0 else -hash_
-        return -2 if result == -1 else result
 
     def __eq__(a, b):
         """a == b"""

Lib/genericpath.py

@@ -3,14 +3,11 @@ Path operations common to more than one OS
 Do not use directly.  The OS specific modules import the appropriate
 functions from this module themselves.
 """
-try:
+import os
-    import os
-except ImportError:
-    import _dummy_os as os
 import stat
 
 __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime',
-           'getsize', 'isdir', 'isfile', 'samefile', 'sameopenfile',
+           'getsize', 'isdir', 'isfile', 'islink', 'samefile', 'sameopenfile',
            'samestat']
 
 
@@ -48,6 +45,18 @@ def isdir(s):
     return stat.S_ISDIR(st.st_mode)
 
 
+# Is a path a symbolic link?
+# This will always return false on systems where os.lstat doesn't exist.
+
+def islink(path):
+    """Test whether a path is a symbolic link"""
+    try:
+        st = os.lstat(path)
+    except (OSError, ValueError, AttributeError):
+        return False
+    return stat.S_ISLNK(st.st_mode)
+
+
 def getsize(filename):
     """Return the size of a file, reported by os.stat()."""
     return os.stat(filename).st_size

Lib/getopt.py

@@ -81,7 +81,7 @@ def getopt(args, shortopts, longopts = []):
     """
 
     opts = []
-    if type(longopts) == type(""):
+    if isinstance(longopts, str):
         longopts = [longopts]
     else:
         longopts = list(longopts)

Lib/gettext.py

@@ -46,17 +46,16 @@ internationalized, to the local language and cultural habits.
 #   find this format documented anywhere.
 
 
-import locale
 import os
 import re
 import sys
 
 
 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
-           'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
+           'bindtextdomain', 'find', 'translation', 'install',
-           'bind_textdomain_codeset',
+           'textdomain', 'dgettext', 'dngettext', 'gettext',
-           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
+           'ngettext', 'pgettext', 'dpgettext', 'npgettext',
-           'ldngettext', 'lngettext', 'ngettext',
+           'dnpgettext'
            ]
 
 _default_localedir = os.path.join(sys.base_prefix, 'share', 'locale')
@@ -83,6 +82,7 @@ _token_pattern = re.compile(r"""
         (?P<INVALID>\w+|.)                           # invalid token
     """, re.VERBOSE|re.DOTALL)
 
+
 def _tokenize(plural):
     for mo in re.finditer(_token_pattern, plural):
         kind = mo.lastgroup
@@ -94,12 +94,14 @@ def _tokenize(plural):
         yield value
     yield ''
 
+
 def _error(value):
     if value:
         return ValueError('unexpected token in plural form: %s' % value)
     else:
         return ValueError('unexpected end of plural form')
 
+
 _binary_ops = (
     ('||',),
     ('&&',),
@@ -111,6 +113,7 @@ _binary_ops = (
 _binary_ops = {op: i for i, ops in enumerate(_binary_ops, 1) for op in ops}
 _c2py_ops = {'||': 'or', '&&': 'and', '/': '//'}
 
+
 def _parse(tokens, priority=-1):
     result = ''
     nexttok = next(tokens)
@@ -160,6 +163,7 @@ def _parse(tokens, priority=-1):
 
     return result, nexttok
 
+
 def _as_int(n):
     try:
         i = round(n)
@@ -172,6 +176,7 @@ def _as_int(n):
                   DeprecationWarning, 4)
     return n
 
+
 def c2py(plural):
     """Gets a C expression as used in PO files for plural forms and returns a
     Python function that implements an equivalent expression.
@@ -209,6 +214,7 @@ def c2py(plural):
 
 
 def _expand_lang(loc):
+    import locale
     loc = locale.normalize(loc)
     COMPONENT_CODESET   = 1 << 0
     COMPONENT_TERRITORY = 1 << 1
@@ -249,12 +255,10 @@ def _expand_lang(loc):
     return ret
 
 
-
 class NullTranslations:
     def __init__(self, fp=None):
         self._info = {}
         self._charset = None
-        self._output_charset = None
         self._fallback = None
         if fp is not None:
             self._parse(fp)
@@ -273,13 +277,6 @@ class NullTranslations:
             return self._fallback.gettext(message)
         return message
 
-    def lgettext(self, message):
-        if self._fallback:
-            return self._fallback.lgettext(message)
-        if self._output_charset:
-            return message.encode(self._output_charset)
-        return message.encode(locale.getpreferredencoding())
-
     def ngettext(self, msgid1, msgid2, n):
         if self._fallback:
             return self._fallback.ngettext(msgid1, msgid2, n)
@@ -288,16 +285,18 @@ class NullTranslations:
         else:
             return msgid2
 
-    def lngettext(self, msgid1, msgid2, n):
+    def pgettext(self, context, message):
         if self._fallback:
-            return self._fallback.lngettext(msgid1, msgid2, n)
+            return self._fallback.pgettext(context, message)
+        return message
+
+    def npgettext(self, context, msgid1, msgid2, n):
+        if self._fallback:
+            return self._fallback.npgettext(context, msgid1, msgid2, n)
         if n == 1:
-            tmsg = msgid1
+            return msgid1
         else:
-            tmsg = msgid2
+            return msgid2
-        if self._output_charset:
-            return tmsg.encode(self._output_charset)
-        return tmsg.encode(locale.getpreferredencoding())
 
     def info(self):
         return self._info
@@ -305,24 +304,13 @@ class NullTranslations:
     def charset(self):
         return self._charset
 
-    def output_charset(self):
-        return self._output_charset
-
-    def set_output_charset(self, charset):
-        self._output_charset = charset
-
     def install(self, names=None):
         import builtins
         builtins.__dict__['_'] = self.gettext
-        if hasattr(names, "__contains__"):
+        if names is not None:
-            if "gettext" in names:
+            allowed = {'gettext', 'ngettext', 'npgettext', 'pgettext'}
-                builtins.__dict__['gettext'] = builtins.__dict__['_']
+            for name in allowed & set(names):
-            if "ngettext" in names:
+                builtins.__dict__[name] = getattr(self, name)
-                builtins.__dict__['ngettext'] = self.ngettext
-            if "lgettext" in names:
-                builtins.__dict__['lgettext'] = self.lgettext
-            if "lngettext" in names:
-                builtins.__dict__['lngettext'] = self.lngettext
 
 
 class GNUTranslations(NullTranslations):
@@ -330,6 +318,10 @@ class GNUTranslations(NullTranslations):
     LE_MAGIC = 0x950412de
     BE_MAGIC = 0xde120495
 
+    # The encoding of a msgctxt and a msgid in a .mo file is
+    # msgctxt + "\x04" + msgid (gettext version >= 0.15)
+    CONTEXT = "%s\x04%s"
+
     # Acceptable .mo versions
     VERSIONS = (0, 1)
 
@@ -385,6 +377,9 @@ class GNUTranslations(NullTranslations):
                     item = b_item.decode().strip()
                     if not item:
                         continue
+                    # Skip over comment lines:
+                    if item.startswith('#-#-#-#-#') and item.endswith('#-#-#-#-#'):
+                        continue
                     k = v = None
                     if ':' in item:
                         k, v = item.split(':', 1)
@@ -423,39 +418,16 @@ class GNUTranslations(NullTranslations):
             masteridx += 8
             transidx += 8
 
-    def lgettext(self, message):
-        missing = object()
-        tmsg = self._catalog.get(message, missing)
-        if tmsg is missing:
-            if self._fallback:
-                return self._fallback.lgettext(message)
-            tmsg = message
-        if self._output_charset:
-            return tmsg.encode(self._output_charset)
-        return tmsg.encode(locale.getpreferredencoding())
-
-    def lngettext(self, msgid1, msgid2, n):
-        try:
-            tmsg = self._catalog[(msgid1, self.plural(n))]
-        except KeyError:
-            if self._fallback:
-                return self._fallback.lngettext(msgid1, msgid2, n)
-            if n == 1:
-                tmsg = msgid1
-            else:
-                tmsg = msgid2
-        if self._output_charset:
-            return tmsg.encode(self._output_charset)
-        return tmsg.encode(locale.getpreferredencoding())
-
     def gettext(self, message):
         missing = object()
         tmsg = self._catalog.get(message, missing)
         if tmsg is missing:
-            if self._fallback:
+            tmsg = self._catalog.get((message, self.plural(1)), missing)
-                return self._fallback.gettext(message)
+        if tmsg is not missing:
-            return message
+            return tmsg
-        return tmsg
+        if self._fallback:
+            return self._fallback.gettext(message)
+        return message
 
     def ngettext(self, msgid1, msgid2, n):
         try:
@@ -469,6 +441,31 @@ class GNUTranslations(NullTranslations):
                 tmsg = msgid2
         return tmsg
 
+    def pgettext(self, context, message):
+        ctxt_msg_id = self.CONTEXT % (context, message)
+        missing = object()
+        tmsg = self._catalog.get(ctxt_msg_id, missing)
+        if tmsg is missing:
+            tmsg = self._catalog.get((ctxt_msg_id, self.plural(1)), missing)
+        if tmsg is not missing:
+            return tmsg
+        if self._fallback:
+            return self._fallback.pgettext(context, message)
+        return message
+
+    def npgettext(self, context, msgid1, msgid2, n):
+        ctxt_msg_id = self.CONTEXT % (context, msgid1)
+        try:
+            tmsg = self._catalog[ctxt_msg_id, self.plural(n)]
+        except KeyError:
+            if self._fallback:
+                return self._fallback.npgettext(context, msgid1, msgid2, n)
+            if n == 1:
+                tmsg = msgid1
+            else:
+                tmsg = msgid2
+        return tmsg
+
 
 # Locate a .mo file using the gettext strategy
 def find(domain, localedir=None, languages=None, all=False):
@@ -507,12 +504,12 @@ def find(domain, localedir=None, languages=None, all=False):
     return result
 
 
-
 # a mapping between absolute .mo file path and Translation object
 _translations = {}
 
+
 def translation(domain, localedir=None, languages=None,
-                class_=None, fallback=False, codeset=None):
+                class_=None, fallback=False):
     if class_ is None:
         class_ = GNUTranslations
     mofiles = find(domain, localedir, languages, all=True)
@@ -538,8 +535,6 @@ def translation(domain, localedir=None, languages=None,
         # are not used.
         import copy
         t = copy.copy(t)
-        if codeset:
-            t.set_output_charset(codeset)
         if result is None:
             result = t
         else:
@@ -547,16 +542,13 @@ def translation(domain, localedir=None, languages=None,
     return result
 
 
-def install(domain, localedir=None, codeset=None, names=None):
+def install(domain, localedir=None, *, names=None):
-    t = translation(domain, localedir, fallback=True, codeset=codeset)
+    t = translation(domain, localedir, fallback=True)
     t.install(names)
 
 
-
 # a mapping b/w domains and locale directories
 _localedirs = {}
-# a mapping b/w domains and codesets
-_localecodesets = {}
 # current global domain, `messages' used for compatibility w/ GNU gettext
 _current_domain = 'messages'
 
@@ -575,33 +567,17 @@ def bindtextdomain(domain, localedir=None):
     return _localedirs.get(domain, _default_localedir)
 
 
-def bind_textdomain_codeset(domain, codeset=None):
-    global _localecodesets
-    if codeset is not None:
-        _localecodesets[domain] = codeset
-    return _localecodesets.get(domain)
-
-
 def dgettext(domain, message):
     try:
-        t = translation(domain, _localedirs.get(domain, None),
+        t = translation(domain, _localedirs.get(domain, None))
-                        codeset=_localecodesets.get(domain))
     except OSError:
         return message
     return t.gettext(message)
 
-def ldgettext(domain, message):
-    codeset = _localecodesets.get(domain)
-    try:
-        t = translation(domain, _localedirs.get(domain, None), codeset=codeset)
-    except OSError:
-        return message.encode(codeset or locale.getpreferredencoding())
-    return t.lgettext(message)
 
 def dngettext(domain, msgid1, msgid2, n):
     try:
-        t = translation(domain, _localedirs.get(domain, None),
+        t = translation(domain, _localedirs.get(domain, None))
-                        codeset=_localecodesets.get(domain))
     except OSError:
         if n == 1:
             return msgid1
@@ -609,29 +585,41 @@ def dngettext(domain, msgid1, msgid2, n):
             return msgid2
     return t.ngettext(msgid1, msgid2, n)
 
-def ldngettext(domain, msgid1, msgid2, n):
+
-    codeset = _localecodesets.get(domain)
+def dpgettext(domain, context, message):
     try:
-        t = translation(domain, _localedirs.get(domain, None), codeset=codeset)
+        t = translation(domain, _localedirs.get(domain, None))
+    except OSError:
+        return message
+    return t.pgettext(context, message)
+
+
+def dnpgettext(domain, context, msgid1, msgid2, n):
+    try:
+        t = translation(domain, _localedirs.get(domain, None))
     except OSError:
         if n == 1:
-            tmsg = msgid1
+            return msgid1
         else:
-            tmsg = msgid2
+            return msgid2
-        return tmsg.encode(codeset or locale.getpreferredencoding())
+    return t.npgettext(context, msgid1, msgid2, n)
-    return t.lngettext(msgid1, msgid2, n)
+
 
 def gettext(message):
     return dgettext(_current_domain, message)
 
-def lgettext(message):
-    return ldgettext(_current_domain, message)
 
 def ngettext(msgid1, msgid2, n):
     return dngettext(_current_domain, msgid1, msgid2, n)
 
-def lngettext(msgid1, msgid2, n):
+
-    return ldngettext(_current_domain, msgid1, msgid2, n)
+def pgettext(context, message):
+    return dpgettext(_current_domain, context, message)
+
+
+def npgettext(context, msgid1, msgid2, n):
+    return dnpgettext(_current_domain, context, msgid1, msgid2, n)
+
 
 # dcgettext() has been deemed unnecessary and is not implemented.
 

Lib/glob.py

@@ -132,7 +132,8 @@ def glob1(dirname, pattern):
 
 def _glob2(dirname, pattern, dir_fd, dironly, include_hidden=False):
     assert _isrecursive(pattern)
-    yield pattern[:0]
+    if not dirname or _isdir(dirname, dir_fd):
+        yield pattern[:0]
     yield from _rlistdir(dirname, dir_fd, dironly,
                          include_hidden=include_hidden)
 

Lib/gzip.py

@@ -15,12 +15,16 @@ __all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"]
 
 FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
 
-READ, WRITE = 1, 2
+READ = 'rb'
+WRITE = 'wb'
 
 _COMPRESS_LEVEL_FAST = 1
 _COMPRESS_LEVEL_TRADEOFF = 6
 _COMPRESS_LEVEL_BEST = 9
 
+READ_BUFFER_SIZE = 128 * 1024
+_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE
+
 
 def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST,
          encoding=None, errors=None, newline=None):
@@ -118,6 +122,21 @@ class BadGzipFile(OSError):
     """Exception raised in some cases for invalid gzip files."""
 
 
+class _WriteBufferStream(io.RawIOBase):
+    """Minimal object to pass WriteBuffer flushes into GzipFile"""
+    def __init__(self, gzip_file):
+        self.gzip_file = gzip_file
+
+    def write(self, data):
+        return self.gzip_file._write_raw(data)
+
+    def seekable(self):
+        return False
+
+    def writable(self):
+        return True
+
+
 class GzipFile(_compression.BaseStream):
     """The GzipFile class simulates most of the methods of a file object with
     the exception of the truncate() method.
@@ -160,9 +179,10 @@ class GzipFile(_compression.BaseStream):
         and 9 is slowest and produces the most compression. 0 is no compression
         at all. The default is 9.
 
-        The mtime argument is an optional numeric timestamp to be written
+        The optional mtime argument is the timestamp requested by gzip. The time
-        to the last modification time field in the stream when compressing.
+        is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970.
-        If omitted or None, the current time is used.
+        If mtime is omitted or None, the current time is used. Use mtime = 0
+        to generate a compressed stream that does not depend on creation time.
 
         """
 
@@ -182,6 +202,7 @@ class GzipFile(_compression.BaseStream):
         if mode is None:
             mode = getattr(fileobj, 'mode', 'rb')
 
+
         if mode.startswith('r'):
             self.mode = READ
             raw = _GzipReader(fileobj)
@@ -204,6 +225,9 @@ class GzipFile(_compression.BaseStream):
                                              zlib.DEF_MEM_LEVEL,
                                              0)
             self._write_mtime = mtime
+            self._buffer_size = _WRITE_BUFFER_SIZE
+            self._buffer = io.BufferedWriter(_WriteBufferStream(self),
+                                             buffer_size=self._buffer_size)
         else:
             raise ValueError("Invalid mode: {!r}".format(mode))
 
@@ -212,14 +236,6 @@ class GzipFile(_compression.BaseStream):
         if self.mode == WRITE:
             self._write_gzip_header(compresslevel)
 
-    @property
-    def filename(self):
-        import warnings
-        warnings.warn("use the name attribute", DeprecationWarning, 2)
-        if self.mode == WRITE and self.name[-3:] != ".gz":
-            return self.name + ".gz"
-        return self.name
-
     @property
     def mtime(self):
         """Last modification time read from stream, or None"""
@@ -237,6 +253,11 @@ class GzipFile(_compression.BaseStream):
         self.bufsize = 0
         self.offset = 0  # Current file offset for seek(), tell(), etc
 
+    def tell(self):
+        self._check_not_closed()
+        self._buffer.flush()
+        return super().tell()
+
     def _write_gzip_header(self, compresslevel):
         self.fileobj.write(b'\037\213')             # magic header
         self.fileobj.write(b'\010')                 # compression method
@@ -278,6 +299,10 @@ class GzipFile(_compression.BaseStream):
         if self.fileobj is None:
             raise ValueError("write() on closed GzipFile object")
 
+        return self._buffer.write(data)
+
+    def _write_raw(self, data):
+        # Called by our self._buffer underlying WriteBufferStream.
         if isinstance(data, (bytes, bytearray)):
             length = len(data)
         else:
@@ -326,11 +351,11 @@ class GzipFile(_compression.BaseStream):
 
     def close(self):
         fileobj = self.fileobj
-        if fileobj is None:
+        if fileobj is None or self._buffer.closed:
             return
-        self.fileobj = None
         try:
             if self.mode == WRITE:
+                self._buffer.flush()
                 fileobj.write(self.compress.flush())
                 write32u(fileobj, self.crc)
                 # self.size may exceed 2 GiB, or even 4 GiB
@@ -338,6 +363,7 @@ class GzipFile(_compression.BaseStream):
             elif self.mode == READ:
                 self._buffer.close()
         finally:
+            self.fileobj = None
             myfileobj = self.myfileobj
             if myfileobj:
                 self.myfileobj = None
@@ -346,6 +372,7 @@ class GzipFile(_compression.BaseStream):
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_not_closed()
         if self.mode == WRITE:
+            self._buffer.flush()
             # Ensure the compressor's buffer is flushed
             self.fileobj.write(self.compress.flush(zlib_mode))
             self.fileobj.flush()
@@ -376,6 +403,9 @@ class GzipFile(_compression.BaseStream):
 
     def seek(self, offset, whence=io.SEEK_SET):
         if self.mode == WRITE:
+            self._check_not_closed()
+            # Flush buffer to ensure validity of self.offset
+            self._buffer.flush()
             if whence != io.SEEK_SET:
                 if whence == io.SEEK_CUR:
                     offset = self.offset + offset
@@ -384,10 +414,10 @@ class GzipFile(_compression.BaseStream):
             if offset < self.offset:
                 raise OSError('Negative seek in write mode')
             count = offset - self.offset
-            chunk = b'\0' * 1024
+            chunk = b'\0' * self._buffer_size
-            for i in range(count // 1024):
+            for i in range(count // self._buffer_size):
                 self.write(chunk)
-            self.write(b'\0' * (count % 1024))
+            self.write(b'\0' * (count % self._buffer_size))
         elif self.mode == READ:
             self._check_not_closed()
             return self._buffer.seek(offset, whence)
@@ -454,7 +484,7 @@ def _read_gzip_header(fp):
 
 class _GzipReader(_compression.DecompressReader):
     def __init__(self, fp):
-        super().__init__(_PaddedFile(fp), zlib.decompressobj,
+        super().__init__(_PaddedFile(fp), zlib._ZlibDecompressor,
                          wbits=-zlib.MAX_WBITS)
         # Set flag indicating start of a new member
         self._new_member = True
@@ -502,12 +532,13 @@ class _GzipReader(_compression.DecompressReader):
                 self._new_member = False
 
             # Read a chunk of data from the file
-            buf = self._fp.read(io.DEFAULT_BUFFER_SIZE)
+            if self._decompressor.needs_input:
+                buf = self._fp.read(READ_BUFFER_SIZE)
+                uncompress = self._decompressor.decompress(buf, size)
+            else:
+                uncompress = self._decompressor.decompress(b"", size)
 
-            uncompress = self._decompressor.decompress(buf, size)
+            if self._decompressor.unused_data != b"":
-            if self._decompressor.unconsumed_tail != b"":
-                self._fp.prepend(self._decompressor.unconsumed_tail)
-            elif self._decompressor.unused_data != b"":
                 # Prepend the already read bytes to the fileobj so they can
                 # be seen by _read_eof() and _read_gzip_header()
                 self._fp.prepend(self._decompressor.unused_data)
@@ -518,14 +549,11 @@ class _GzipReader(_compression.DecompressReader):
                 raise EOFError("Compressed file ended before the "
                                "end-of-stream marker was reached")
 
-        self._add_read_data( uncompress )
+        self._crc = zlib.crc32(uncompress, self._crc)
+        self._stream_size += len(uncompress)
         self._pos += len(uncompress)
         return uncompress
 
-    def _add_read_data(self, data):
-        self._crc = zlib.crc32(data, self._crc)
-        self._stream_size = self._stream_size + len(data)
-
     def _read_eof(self):
         # We've read to the end of the file
         # We check that the computed CRC and size of the
@@ -552,43 +580,21 @@ class _GzipReader(_compression.DecompressReader):
         self._new_member = True
 
 
-def _create_simple_gzip_header(compresslevel: int,
+def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=0):
-                               mtime = None) -> bytes:
-    """
-    Write a simple gzip header with no extra fields.
-    :param compresslevel: Compresslevel used to determine the xfl bytes.
-    :param mtime: The mtime (must support conversion to a 32-bit integer).
-    :return: A bytes object representing the gzip header.
-    """
-    if mtime is None:
-        mtime = time.time()
-    if compresslevel == _COMPRESS_LEVEL_BEST:
-        xfl = 2
-    elif compresslevel == _COMPRESS_LEVEL_FAST:
-        xfl = 4
-    else:
-        xfl = 0
-    # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra
-    # fields added to header), mtime, xfl and os (255 for unknown OS).
-    return struct.pack("<BBBBLBB", 0x1f, 0x8b, 8, 0, int(mtime), xfl, 255)
-
-
-def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
     """Compress data in one shot and return the compressed string.
 
     compresslevel sets the compression level in range of 0-9.
-    mtime can be used to set the modification time. The modification time is
+    mtime can be used to set the modification time.
-    set to the current time by default.
+    The modification time is set to 0 by default, for reproducibility.
     """
-    if mtime == 0:
+    # Wbits=31 automatically includes a gzip header and trailer.
-        # Use zlib as it creates the header with 0 mtime by default.
+    gzip_data = zlib.compress(data, level=compresslevel, wbits=31)
-        # This is faster and with less overhead.
+    if mtime is None:
-        return zlib.compress(data, level=compresslevel, wbits=31)
+        mtime = time.time()
-    header = _create_simple_gzip_header(compresslevel, mtime)
+    # Reuse gzip header created by zlib, replace mtime and OS byte for
-    trailer = struct.pack("<LL", zlib.crc32(data), (len(data) & 0xffffffff))
+    # consistency.
-    # Wbits=-15 creates a raw deflate block.
+    header = struct.pack("<4sLBB", gzip_data, int(mtime), gzip_data[8], 255)
-    return (header + zlib.compress(data, level=compresslevel, wbits=-15) +
+    return header + gzip_data[10:]
-            trailer)
 
 
 def decompress(data):
@@ -655,7 +661,7 @@ def main():
                 f = builtins.open(arg, "rb")
                 g = open(arg + ".gz", "wb")
         while True:
-            chunk = f.read(io.DEFAULT_BUFFER_SIZE)
+            chunk = f.read(READ_BUFFER_SIZE)
             if not chunk:
                 break
             g.write(chunk)

Lib/io.py

@@ -45,7 +45,10 @@ __all__ = ["BlockingIOError", "open", "open_code", "IOBase", "RawIOBase",
            "FileIO", "BytesIO", "StringIO", "BufferedIOBase",
            "BufferedReader", "BufferedWriter", "BufferedRWPair",
            "BufferedRandom", "TextIOBase", "TextIOWrapper",
-           "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END"]
+           "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END",
+           "DEFAULT_BUFFER_SIZE", "text_encoding",
+           "IncrementalNewlineDecoder"
+           ]
 
 
 import _io
@@ -54,31 +57,13 @@ import abc
 from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
                  open, open_code, BytesIO, StringIO, BufferedReader,
                  BufferedWriter, BufferedRWPair, BufferedRandom,
-                 # XXX RUSTPYTHON TODO: IncrementalNewlineDecoder
+                 IncrementalNewlineDecoder, text_encoding, TextIOWrapper)
-                 # IncrementalNewlineDecoder, text_encoding, TextIOWrapper)
-                 text_encoding, TextIOWrapper)
 
 try:
     from _io import FileIO
 except ImportError:
     pass
 
-def __getattr__(name):
-    if name == "OpenWrapper":
-        # bpo-43680: Until Python 3.9, _pyio.open was not a static method and
-        # builtins.open was set to OpenWrapper to not become a bound method
-        # when set to a class variable. _io.open is a built-in function whereas
-        # _pyio.open is a Python function. In Python 3.10, _pyio.open() is now
-        # a static method, and builtins.open() is now io.open().
-        import warnings
-        warnings.warn('OpenWrapper is deprecated, use open instead',
-                      DeprecationWarning, stacklevel=2)
-        global OpenWrapper
-        OpenWrapper = open
-        return OpenWrapper
-    raise AttributeError(name)
-
-
 # Pretend this exception was created here.
 UnsupportedOperation.__module__ = "io"
 

Lib/ipaddress.py

@@ -1821,9 +1821,6 @@ class _BaseV6:
     def _explode_shorthand_ip_string(self):
         """Expand a shortened IPv6 address.
 
-        Args:
-            ip_str: A string, the IPv6 address.
-
         Returns:
             A string, the expanded IPv6 address.
 
@@ -1941,6 +1938,9 @@ class IPv6Address(_BaseV6, _BaseAddress):
             return False
         return self._scope_id == getattr(other, '_scope_id', None)
 
+    def __reduce__(self):
+        return (self.__class__, (str(self),))
+
     @property
     def scope_id(self):
         """Identifier of a particular zone of the address's scope.

Lib/keyword.py

@@ -56,7 +56,8 @@ kwlist = [
 softkwlist = [
     '_',
     'case',
-    'match'
+    'match',
+    'type'
 ]
 
 iskeyword = frozenset(kwlist).__contains__

Lib/logging/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -18,13 +18,14 @@
 Logging package for Python. Based on PEP 282 and comments thereto in
 comp.lang.python.
 
-Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
 
 import sys, os, time, io, re, traceback, warnings, weakref, collections.abc
 
+from types import GenericAlias
 from string import Template
 from string import Formatter as StrFormatter
 
@@ -37,7 +38,8 @@ __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR',
            'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass',
            'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown',
            'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory',
-           'lastResort', 'raiseExceptions']
+           'lastResort', 'raiseExceptions', 'getLevelNamesMapping',
+           'getHandlerByName', 'getHandlerNames']
 
 import threading
 
@@ -63,20 +65,25 @@ _startTime = time.time()
 raiseExceptions = True
 
 #
-# If you don't want threading information in the log, set this to zero
+# If you don't want threading information in the log, set this to False
 #
 logThreads = True
 
 #
-# If you don't want multiprocessing information in the log, set this to zero
+# If you don't want multiprocessing information in the log, set this to False
 #
 logMultiprocessing = True
 
 #
-# If you don't want process information in the log, set this to zero
+# If you don't want process information in the log, set this to False
 #
 logProcesses = True
 
+#
+# If you don't want asyncio task information in the log, set this to False
+#
+logAsyncioTasks = True
+
 #---------------------------------------------------------------------------
 #   Level related stuff
 #---------------------------------------------------------------------------
@@ -116,6 +123,9 @@ _nameToLevel = {
     'NOTSET': NOTSET,
 }
 
+def getLevelNamesMapping():
+    return _nameToLevel.copy()
+
 def getLevelName(level):
     """
     Return the textual or numeric representation of logging level 'level'.
@@ -156,15 +166,15 @@ def addLevelName(level, levelName):
     finally:
         _releaseLock()
 
-if hasattr(sys, '_getframe'):
+if hasattr(sys, "_getframe"):
-    currentframe = lambda: sys._getframe(3)
+    currentframe = lambda: sys._getframe(1)
 else: #pragma: no cover
     def currentframe():
         """Return the frame object for the caller's stack frame."""
         try:
             raise Exception
-        except Exception:
+        except Exception as exc:
-            return sys.exc_info()[2].tb_frame.f_back
+            return exc.__traceback__.tb_frame.f_back
 
 #
 # _srcfile is used when walking the stack to check when we've got the first
@@ -181,13 +191,18 @@ else: #pragma: no cover
 _srcfile = os.path.normcase(addLevelName.__code__.co_filename)
 
 # _srcfile is only used in conjunction with sys._getframe().
-# To provide compatibility with older versions of Python, set _srcfile
+# Setting _srcfile to None will prevent findCaller() from being called. This
-# to None if _getframe() is not available; this value will prevent
+# way, you can avoid the overhead of fetching caller information.
-# findCaller() from being called. You can also do this if you want to avoid
+
-# the overhead of fetching caller information, even when _getframe() is
+# The following is based on warnings._is_internal_frame. It makes sure that
-# available.
+# frames of the import mechanism are skipped when logging at module level and
-#if not hasattr(sys, '_getframe'):
+# using a stacklevel value greater than one.
-#    _srcfile = None
+def _is_internal_frame(frame):
+    """Signal whether the frame is a CPython or logging module internal."""
+    filename = os.path.normcase(frame.f_code.co_filename)
+    return filename == _srcfile or (
+            "importlib" in filename and "_bootstrap" in filename
+    )
 
 
 def _checkLevel(level):
@@ -307,7 +322,7 @@ class LogRecord(object):
         # Thus, while not removing the isinstance check, it does now look
         # for collections.abc.Mapping rather than, as before, dict.
         if (args and len(args) == 1 and isinstance(args[0], collections.abc.Mapping)
-            and args[0]):
+                and args[0]):
             args = args[0]
         self.args = args
         self.levelname = getLevelName(level)
@@ -325,7 +340,7 @@ class LogRecord(object):
         self.lineno = lineno
         self.funcName = func
         self.created = ct
-        self.msecs = (ct - int(ct)) * 1000
+        self.msecs = int((ct - int(ct)) * 1000) + 0.0  # see gh-89047
         self.relativeCreated = (self.created - _startTime) * 1000
         if logThreads:
             self.thread = threading.get_ident()
@@ -352,9 +367,18 @@ class LogRecord(object):
         else:
             self.process = None
 
+        self.taskName = None
+        if logAsyncioTasks:
+            asyncio = sys.modules.get('asyncio')
+            if asyncio:
+                try:
+                    self.taskName = asyncio.current_task().get_name()
+                except Exception:
+                    pass
+
     def __repr__(self):
         return '<LogRecord: %s, %s, %s, %s, "%s">'%(self.name, self.levelno,
-            self.pathname, self.lineno, self.msg)
+                                                    self.pathname, self.lineno, self.msg)
 
     def getMessage(self):
         """
@@ -487,7 +511,7 @@ class StringTemplateStyle(PercentStyle):
 
     def usesTime(self):
         fmt = self._fmt
-        return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0
+        return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_search) >= 0
 
     def validate(self):
         pattern = Template.pattern
@@ -557,6 +581,7 @@ class Formatter(object):
                         (typically at application startup time)
     %(thread)d          Thread ID (if available)
     %(threadName)s      Thread name (if available)
+    %(taskName)s        Task name (if available)
     %(process)d         Process ID (if available)
     %(message)s         The result of record.getMessage(), computed just as
                         the record is emitted
@@ -583,7 +608,7 @@ class Formatter(object):
         """
         if style not in _STYLES:
             raise ValueError('Style must be one of: %s' % ','.join(
-                             _STYLES.keys()))
+                _STYLES.keys()))
         self._style = _STYLES[style][0](fmt, defaults=defaults)
         if validate:
             self._style.validate()
@@ -808,23 +833,36 @@ class Filterer(object):
         Determine if a record is loggable by consulting all the filters.
 
         The default is to allow the record to be logged; any filter can veto
-        this and the record is then dropped. Returns a zero value if a record
+        this by returning a false value.
-        is to be dropped, else non-zero.
+        If a filter attached to a handler returns a log record instance,
+        then that instance is used in place of the original log record in
+        any further processing of the event by that handler.
+        If a filter returns any other true value, the original log record
+        is used in any further processing of the event by that handler.
+
+        If none of the filters return false values, this method returns
+        a log record.
+        If any of the filters return a false value, this method returns
+        a false value.
 
         .. versionchanged:: 3.2
 
            Allow filters to be just callables.
+
+        .. versionchanged:: 3.12
+           Allow filters to return a LogRecord instead of
+           modifying it in place.
         """
-        rv = True
         for f in self.filters:
             if hasattr(f, 'filter'):
                 result = f.filter(record)
             else:
                 result = f(record) # assume callable - will raise if not
             if not result:
-                rv = False
+                return False
-                break
+            if isinstance(result, LogRecord):
-        return rv
+                record = result
+        return record
 
 #---------------------------------------------------------------------------
 #   Handler classes and functions
@@ -845,8 +883,9 @@ def _removeHandlerRef(wr):
     if acquire and release and handlers:
         acquire()
         try:
-            if wr in handlers:
+            handlers.remove(wr)
-                handlers.remove(wr)
+        except ValueError:
+            pass
         finally:
             release()
 
@@ -860,6 +899,23 @@ def _addHandlerRef(handler):
     finally:
         _releaseLock()
 
+
+def getHandlerByName(name):
+    """
+    Get a handler with the specified *name*, or None if there isn't one with
+    that name.
+    """
+    return _handlers.get(name)
+
+
+def getHandlerNames():
+    """
+    Return all known handler names as an immutable set.
+    """
+    result = set(_handlers.keys())
+    return frozenset(result)
+
+
 class Handler(Filterer):
     """
     Handler instances dispatch logging events to specific destinations.
@@ -958,10 +1014,14 @@ class Handler(Filterer):
 
         Emission depends on filters which may have been added to the handler.
         Wrap the actual emission of the record with acquisition/release of
-        the I/O thread lock. Returns whether the filter passed the record for
+        the I/O thread lock.
-        emission.
+
+        Returns an instance of the log record that was emitted
+        if it passed all filters, otherwise a false value is returned.
         """
         rv = self.filter(record)
+        if isinstance(rv, LogRecord):
+            record = rv
         if rv:
             self.acquire()
             try:
@@ -1032,7 +1092,7 @@ class Handler(Filterer):
                 else:
                     # couldn't find the right stack frame, for some reason
                     sys.stderr.write('Logged from file %s, line %s\n' % (
-                                     record.filename, record.lineno))
+                        record.filename, record.lineno))
                 # Issue 18671: output logging message and arguments
                 try:
                     sys.stderr.write('Message: %r\n'
@@ -1044,7 +1104,7 @@ class Handler(Filterer):
                     sys.stderr.write('Unable to print the message and arguments'
                                      ' - possible formatting error.\nUse the'
                                      ' traceback above to help find the error.\n'
-                                    )
+                                     )
             except OSError: #pragma: no cover
                 pass    # see issue 5971
             finally:
@@ -1136,6 +1196,8 @@ class StreamHandler(Handler):
             name += ' '
         return '<%s %s(%s)>' % (self.__class__.__name__, name, level)
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 class FileHandler(StreamHandler):
     """
@@ -1459,7 +1521,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.debug("Houston, we have a %s", "thorny problem", exc_info=1)
+        logger.debug("Houston, we have a %s", "thorny problem", exc_info=True)
         """
         if self.isEnabledFor(DEBUG):
             self._log(DEBUG, msg, args, **kwargs)
@@ -1471,7 +1533,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.info("Houston, we have a %s", "interesting problem", exc_info=1)
+        logger.info("Houston, we have a %s", "notable problem", exc_info=True)
         """
         if self.isEnabledFor(INFO):
             self._log(INFO, msg, args, **kwargs)
@@ -1483,14 +1545,14 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1)
+        logger.warning("Houston, we have a %s", "bit of a problem", exc_info=True)
         """
         if self.isEnabledFor(WARNING):
             self._log(WARNING, msg, args, **kwargs)
 
     def warn(self, msg, *args, **kwargs):
         warnings.warn("The 'warn' method is deprecated, "
-            "use 'warning' instead", DeprecationWarning, 2)
+                      "use 'warning' instead", DeprecationWarning, 2)
         self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
@@ -1500,7 +1562,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.error("Houston, we have a %s", "major problem", exc_info=1)
+        logger.error("Houston, we have a %s", "major problem", exc_info=True)
         """
         if self.isEnabledFor(ERROR):
             self._log(ERROR, msg, args, **kwargs)
@@ -1518,7 +1580,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.critical("Houston, we have a %s", "major disaster", exc_info=1)
+        logger.critical("Houston, we have a %s", "major disaster", exc_info=True)
         """
         if self.isEnabledFor(CRITICAL):
             self._log(CRITICAL, msg, args, **kwargs)
@@ -1536,7 +1598,7 @@ class Logger(Filterer):
         To pass exception information, use the keyword argument exc_info with
         a true value, e.g.
 
-        logger.log(level, "We have a %s", "mysterious problem", exc_info=1)
+        logger.log(level, "We have a %s", "mysterious problem", exc_info=True)
         """
         if not isinstance(level, int):
             if raiseExceptions:
@@ -1554,33 +1616,31 @@ class Logger(Filterer):
         f = currentframe()
         #On some versions of IronPython, currentframe() returns None if
         #IronPython isn't run with -X:Frames.
-        if f is not None:
+        if f is None:
-            f = f.f_back
+            return "(unknown file)", 0, "(unknown function)", None
-        orig_f = f
+        while stacklevel > 0:
-        while f and stacklevel > 1:
+            next_f = f.f_back
-            f = f.f_back
+            if next_f is None:
-            stacklevel -= 1
+                ## We've got options here.
-        if not f:
+                ## If we want to use the last (deepest) frame:
-            f = orig_f
+                break
-        rv = "(unknown file)", 0, "(unknown function)", None
+                ## If we want to mimic the warnings module:
-        while hasattr(f, "f_code"):
+                #return ("sys", 1, "(unknown function)", None)
-            co = f.f_code
+                ## If we want to be pedantic:
-            filename = os.path.normcase(co.co_filename)
+                #raise ValueError("call stack is not deep enough")
-            if filename == _srcfile:
+            f = next_f
-                f = f.f_back
+            if not _is_internal_frame(f):
-                continue
+                stacklevel -= 1
-            sinfo = None
+        co = f.f_code
-            if stack_info:
+        sinfo = None
-                sio = io.StringIO()
+        if stack_info:
-                sio.write('Stack (most recent call last):\n')
+            with io.StringIO() as sio:
+                sio.write("Stack (most recent call last):\n")
                 traceback.print_stack(f, file=sio)
                 sinfo = sio.getvalue()
                 if sinfo[-1] == '\n':
                     sinfo = sinfo[:-1]
-                sio.close()
+        return co.co_filename, f.f_lineno, co.co_name, sinfo
-            rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
-            break
-        return rv
 
     def makeRecord(self, name, level, fn, lno, msg, args, exc_info,
                    func=None, extra=None, sinfo=None):
@@ -1589,7 +1649,7 @@ class Logger(Filterer):
         specialized LogRecords.
         """
         rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func,
-                             sinfo)
+                               sinfo)
         if extra is not None:
             for key in extra:
                 if (key in ["message", "asctime"]) or (key in rv.__dict__):
@@ -1630,8 +1690,14 @@ class Logger(Filterer):
         This method is used for unpickled records received from a socket, as
         well as those created locally. Logger-level filtering is applied.
         """
-        if (not self.disabled) and self.filter(record):
+        if self.disabled:
-            self.callHandlers(record)
+            return
+        maybe_record = self.filter(record)
+        if not maybe_record:
+            return
+        if isinstance(maybe_record, LogRecord):
+            record = maybe_record
+        self.callHandlers(record)
 
     def addHandler(self, hdlr):
         """
@@ -1737,7 +1803,7 @@ class Logger(Filterer):
                     is_enabled = self._cache[level] = False
                 else:
                     is_enabled = self._cache[level] = (
-                        level >= self.getEffectiveLevel()
+                            level >= self.getEffectiveLevel()
                     )
             finally:
                 _releaseLock()
@@ -1762,13 +1828,30 @@ class Logger(Filterer):
             suffix = '.'.join((self.name, suffix))
         return self.manager.getLogger(suffix)
 
+    def getChildren(self):
+
+        def _hierlevel(logger):
+            if logger is logger.manager.root:
+                return 0
+            return 1 + logger.name.count('.')
+
+        d = self.manager.loggerDict
+        _acquireLock()
+        try:
+            # exclude PlaceHolders - the last check is to ensure that lower-level
+            # descendants aren't returned - if there are placeholders, a logger's
+            # parent field might point to a grandparent or ancestor thereof.
+            return set(item for item in d.values()
+                       if isinstance(item, Logger) and item.parent is self and
+                       _hierlevel(item) == 1 + _hierlevel(item.parent))
+        finally:
+            _releaseLock()
+
     def __repr__(self):
         level = getLevelName(self.getEffectiveLevel())
         return '<%s %s (%s)>' % (self.__class__.__name__, self.name, level)
 
     def __reduce__(self):
-        # In general, only the root logger will not be accessible via its name.
-        # However, the root logger's class has its own __reduce__ method.
         if getLogger(self.name) is not self:
             import pickle
             raise pickle.PicklingError('logger cannot be pickled')
@@ -1848,7 +1931,7 @@ class LoggerAdapter(object):
 
     def warn(self, msg, *args, **kwargs):
         warnings.warn("The 'warn' method is deprecated, "
-            "use 'warning' instead", DeprecationWarning, 2)
+                      "use 'warning' instead", DeprecationWarning, 2)
         self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
@@ -1902,18 +1985,11 @@ class LoggerAdapter(object):
         """
         return self.logger.hasHandlers()
 
-    def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False):
+    def _log(self, level, msg, args, **kwargs):
         """
         Low-level log implementation, proxied to allow nested logger adapters.
         """
-        return self.logger._log(
+        return self.logger._log(level, msg, args, **kwargs)
-            level,
-            msg,
-            args,
-            exc_info=exc_info,
-            extra=extra,
-            stack_info=stack_info,
-        )
 
     @property
     def manager(self):
@@ -1932,6 +2008,8 @@ class LoggerAdapter(object):
         level = getLevelName(logger.getEffectiveLevel())
         return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level)
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 root = RootLogger(WARNING)
 Logger.root = root
 Logger.manager = Manager(Logger.root)
@@ -1971,7 +2049,7 @@ def basicConfig(**kwargs):
               that this argument is incompatible with 'filename' - if both
               are present, 'stream' is ignored.
     handlers  If specified, this should be an iterable of already created
-              handlers, which will be added to the root handler. Any handler
+              handlers, which will be added to the root logger. Any handler
               in the list which does not have a formatter assigned will be
               assigned the formatter created in this function.
     force     If this keyword  is specified as true, any existing handlers
@@ -2047,7 +2125,7 @@ def basicConfig(**kwargs):
             style = kwargs.pop("style", '%')
             if style not in _STYLES:
                 raise ValueError('Style must be one of: %s' % ','.join(
-                                 _STYLES.keys()))
+                    _STYLES.keys()))
             fs = kwargs.pop("format", _STYLES[style][1])
             fmt = Formatter(fs, dfs, style)
             for h in handlers:
@@ -2124,7 +2202,7 @@ def warning(msg, *args, **kwargs):
 
 def warn(msg, *args, **kwargs):
     warnings.warn("The 'warn' function is deprecated, "
-        "use 'warning' instead", DeprecationWarning, 2)
+                  "use 'warning' instead", DeprecationWarning, 2)
     warning(msg, *args, **kwargs)
 
 def info(msg, *args, **kwargs):
@@ -2179,7 +2257,11 @@ def shutdown(handlerList=_handlerList):
             if h:
                 try:
                     h.acquire()
-                    h.flush()
+                    # MemoryHandlers might not want to be flushed on close,
+                    # but circular imports prevent us scoping this to just
+                    # those handlers.  hence the default to True.
+                    if getattr(h, 'flushOnClose', True):
+                        h.flush()
                     h.close()
                 except (OSError, ValueError):
                     # Ignore errors which might be caused
@@ -2242,7 +2324,9 @@ def _showwarning(message, category, filename, lineno, file=None, line=None):
         logger = getLogger("py.warnings")
         if not logger.handlers:
             logger.addHandler(NullHandler())
-        logger.warning("%s", s)
+        # bpo-46557: Log str(s) as msg instead of logger.warning("%s", s)
+        # since some log aggregation tools group logs by the msg arg
+        logger.warning(str(s))
 
 def captureWarnings(capture):
     """

Lib/logging/config.py

@@ -1,4 +1,4 @@
-# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2023 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -19,18 +19,20 @@ Configuration functions for the logging package for Python. The core package
 is based on PEP 282 and comments thereto in comp.lang.python, and influenced
 by Apache's log4j system.
 
-Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
 
 import errno
+import functools
 import io
 import logging
 import logging.handlers
+import os
+import queue
 import re
 import struct
-import sys
 import threading
 import traceback
 
@@ -59,15 +61,24 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=Non
     """
     import configparser
 
+    if isinstance(fname, str):
+        if not os.path.exists(fname):
+            raise FileNotFoundError(f"{fname} doesn't exist")
+        elif not os.path.getsize(fname):
+            raise RuntimeError(f'{fname} is an empty file')
+
     if isinstance(fname, configparser.RawConfigParser):
         cp = fname
     else:
-        cp = configparser.ConfigParser(defaults)
+        try:
-        if hasattr(fname, 'readline'):
+            cp = configparser.ConfigParser(defaults)
-            cp.read_file(fname)
+            if hasattr(fname, 'readline'):
-        else:
+                cp.read_file(fname)
-            encoding = io.text_encoding(encoding)
+            else:
-            cp.read(fname, encoding=encoding)
+                encoding = io.text_encoding(encoding)
+                cp.read(fname, encoding=encoding)
+        except configparser.ParsingError as e:
+            raise RuntimeError(f'{fname} is invalid: {e}')
 
     formatters = _create_formatters(cp)
 
@@ -113,11 +124,18 @@ def _create_formatters(cp):
         fs = cp.get(sectname, "format", raw=True, fallback=None)
         dfs = cp.get(sectname, "datefmt", raw=True, fallback=None)
         stl = cp.get(sectname, "style", raw=True, fallback='%')
+        defaults = cp.get(sectname, "defaults", raw=True, fallback=None)
+
         c = logging.Formatter
         class_name = cp[sectname].get("class")
         if class_name:
             c = _resolve(class_name)
-        f = c(fs, dfs, stl)
+
+        if defaults is not None:
+            defaults = eval(defaults, vars(logging))
+            f = c(fs, dfs, stl, defaults=defaults)
+        else:
+            f = c(fs, dfs, stl)
         formatters[form] = f
     return formatters
 
@@ -296,7 +314,7 @@ class ConvertingMixin(object):
             if replace:
                 self[key] = result
             if type(result) in (ConvertingDict, ConvertingList,
-                               ConvertingTuple):
+                                ConvertingTuple):
                 result.parent = self
                 result.key = key
         return result
@@ -305,7 +323,7 @@ class ConvertingMixin(object):
         result = self.configurator.convert(value)
         if value is not result:
             if type(result) in (ConvertingDict, ConvertingList,
-                               ConvertingTuple):
+                                ConvertingTuple):
                 result.parent = self
         return result
 
@@ -392,11 +410,9 @@ class BaseConfigurator(object):
                     self.importer(used)
                     found = getattr(found, frag)
             return found
-        except ImportError:
+        except ImportError as e:
-            e, tb = sys.exc_info()[1:]
             v = ValueError('Cannot resolve %r: %s' % (s, e))
-            v.__cause__, v.__traceback__ = e, tb
+            raise v from e
-            raise v
 
     def ext_convert(self, value):
         """Default converter for the ext:// protocol."""
@@ -448,8 +464,8 @@ class BaseConfigurator(object):
         elif not isinstance(value, ConvertingList) and isinstance(value, list):
             value = ConvertingList(value)
             value.configurator = self
-        elif not isinstance(value, ConvertingTuple) and\
+        elif not isinstance(value, ConvertingTuple) and \
-                 isinstance(value, tuple) and not hasattr(value, '_fields'):
+                isinstance(value, tuple) and not hasattr(value, '_fields'):
             value = ConvertingTuple(value)
             value.configurator = self
         elif isinstance(value, str): # str for py3k
@@ -469,10 +485,10 @@ class BaseConfigurator(object):
         c = config.pop('()')
         if not callable(c):
             c = self.resolve(c)
-        props = config.pop('.', None)
         # Check for valid identifiers
-        kwargs = {k: config[k] for k in config if valid_ident(k)}
+        kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))}
         result = c(**kwargs)
+        props = config.pop('.', None)
         if props:
             for name, value in props.items():
                 setattr(result, name, value)
@@ -484,6 +500,33 @@ class BaseConfigurator(object):
             value = tuple(value)
         return value
 
+def _is_queue_like_object(obj):
+    """Check that *obj* implements the Queue API."""
+    if isinstance(obj, (queue.Queue, queue.SimpleQueue)):
+        return True
+    # defer importing multiprocessing as much as possible
+    from multiprocessing.queues import Queue as MPQueue
+    if isinstance(obj, MPQueue):
+        return True
+    # Depending on the multiprocessing start context, we cannot create
+    # a multiprocessing.managers.BaseManager instance 'mm' to get the
+    # runtime type of mm.Queue() or mm.JoinableQueue() (see gh-119819).
+    #
+    # Since we only need an object implementing the Queue API, we only
+    # do a protocol check, but we do not use typing.runtime_checkable()
+    # and typing.Protocol to reduce import time (see gh-121723).
+    #
+    # Ideally, we would have wanted to simply use strict type checking
+    # instead of a protocol-based type checking since the latter does
+    # not check the method signatures.
+    #
+    # Note that only 'put_nowait' and 'get' are required by the logging
+    # queue handler and queue listener (see gh-124653) and that other
+    # methods are either optional or unused.
+    minimal_queue_interface = ['put_nowait', 'get']
+    return all(callable(getattr(obj, method, None))
+               for method in minimal_queue_interface)
+
 class DictConfigurator(BaseConfigurator):
     """
     Configure logging using a dictionary-like object to describe the
@@ -542,7 +585,7 @@ class DictConfigurator(BaseConfigurator):
                 for name in formatters:
                     try:
                         formatters[name] = self.configure_formatter(
-                                                            formatters[name])
+                            formatters[name])
                     except Exception as e:
                         raise ValueError('Unable to configure '
                                          'formatter %r' % name) from e
@@ -566,7 +609,7 @@ class DictConfigurator(BaseConfigurator):
                         handler.name = name
                         handlers[name] = handler
                     except Exception as e:
-                        if 'target not configured yet' in str(e.__cause__):
+                        if ' not configured yet' in str(e.__cause__):
                             deferred.append(name)
                         else:
                             raise ValueError('Unable to configure handler '
@@ -669,18 +712,27 @@ class DictConfigurator(BaseConfigurator):
             dfmt = config.get('datefmt', None)
             style = config.get('style', '%')
             cname = config.get('class', None)
+            defaults = config.get('defaults', None)
 
             if not cname:
                 c = logging.Formatter
             else:
                 c = _resolve(cname)
 
+            kwargs  = {}
+
+            # Add defaults only if it exists.
+            # Prevents TypeError in custom formatter callables that do not
+            # accept it.
+            if defaults is not None:
+                kwargs['defaults'] = defaults
+
             # A TypeError would be raised if "validate" key is passed in with a formatter callable
             # that does not accept "validate" as a parameter
             if 'validate' in config:  # if user hasn't mentioned it, the default will be fine
-                result = c(fmt, dfmt, style, config['validate'])
+                result = c(fmt, dfmt, style, config['validate'], **kwargs)
             else:
-                result = c(fmt, dfmt, style)
+                result = c(fmt, dfmt, style, **kwargs)
 
         return result
 
@@ -697,10 +749,29 @@ class DictConfigurator(BaseConfigurator):
         """Add filters to a filterer from a list of names."""
         for f in filters:
             try:
-                filterer.addFilter(self.config['filters'][f])
+                if callable(f) or callable(getattr(f, 'filter', None)):
+                    filter_ = f
+                else:
+                    filter_ = self.config['filters'][f]
+                filterer.addFilter(filter_)
             except Exception as e:
                 raise ValueError('Unable to add filter %r' % f) from e
 
+    def _configure_queue_handler(self, klass, **kwargs):
+        if 'queue' in kwargs:
+            q = kwargs.pop('queue')
+        else:
+            q = queue.Queue()  # unbounded
+
+        rhl = kwargs.pop('respect_handler_level', False)
+        lklass = kwargs.pop('listener', logging.handlers.QueueListener)
+        handlers = kwargs.pop('handlers', [])
+
+        listener = lklass(q, *handlers, respect_handler_level=rhl)
+        handler = klass(q, **kwargs)
+        handler.listener = listener
+        return handler
+
     def configure_handler(self, config):
         """Configure a handler from a dictionary."""
         config_copy = dict(config)  # for restoring in case of error
@@ -720,28 +791,87 @@ class DictConfigurator(BaseConfigurator):
             factory = c
         else:
             cname = config.pop('class')
-            klass = self.resolve(cname)
+            if callable(cname):
-            #Special case for handler which refers to another handler
+                klass = cname
-            if issubclass(klass, logging.handlers.MemoryHandler) and\
+            else:
-                'target' in config:
+                klass = self.resolve(cname)
-                try:
+            if issubclass(klass, logging.handlers.MemoryHandler):
-                    th = self.config['handlers'][config['target']]
+                if 'flushLevel' in config:
-                    if not isinstance(th, logging.Handler):
+                    config['flushLevel'] = logging._checkLevel(config['flushLevel'])
-                        config.update(config_copy)  # restore for deferred cfg
+                if 'target' in config:
-                        raise TypeError('target not configured yet')
+                    # Special case for handler which refers to another handler
-                    config['target'] = th
+                    try:
-                except Exception as e:
+                        tn = config['target']
-                    raise ValueError('Unable to set target handler '
+                        th = self.config['handlers'][tn]
-                                     '%r' % config['target']) from e
+                        if not isinstance(th, logging.Handler):
-            elif issubclass(klass, logging.handlers.SMTPHandler) and\
+                            config.update(config_copy)  # restore for deferred cfg
-                'mailhost' in config:
+                            raise TypeError('target not configured yet')
+                        config['target'] = th
+                    except Exception as e:
+                        raise ValueError('Unable to set target handler %r' % tn) from e
+            elif issubclass(klass, logging.handlers.QueueHandler):
+                # Another special case for handler which refers to other handlers
+                # if 'handlers' not in config:
+                # raise ValueError('No handlers specified for a QueueHandler')
+                if 'queue' in config:
+                    qspec = config['queue']
+
+                    if isinstance(qspec, str):
+                        q = self.resolve(qspec)
+                        if not callable(q):
+                            raise TypeError('Invalid queue specifier %r' % qspec)
+                        config['queue'] = q()
+                    elif isinstance(qspec, dict):
+                        if '()' not in qspec:
+                            raise TypeError('Invalid queue specifier %r' % qspec)
+                        config['queue'] = self.configure_custom(dict(qspec))
+                    elif not _is_queue_like_object(qspec):
+                        raise TypeError('Invalid queue specifier %r' % qspec)
+
+                if 'listener' in config:
+                    lspec = config['listener']
+                    if isinstance(lspec, type):
+                        if not issubclass(lspec, logging.handlers.QueueListener):
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                    else:
+                        if isinstance(lspec, str):
+                            listener = self.resolve(lspec)
+                            if isinstance(listener, type) and \
+                                    not issubclass(listener, logging.handlers.QueueListener):
+                                raise TypeError('Invalid listener specifier %r' % lspec)
+                        elif isinstance(lspec, dict):
+                            if '()' not in lspec:
+                                raise TypeError('Invalid listener specifier %r' % lspec)
+                            listener = self.configure_custom(dict(lspec))
+                        else:
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                        if not callable(listener):
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                        config['listener'] = listener
+                if 'handlers' in config:
+                    hlist = []
+                    try:
+                        for hn in config['handlers']:
+                            h = self.config['handlers'][hn]
+                            if not isinstance(h, logging.Handler):
+                                config.update(config_copy)  # restore for deferred cfg
+                                raise TypeError('Required handler %r '
+                                                'is not configured yet' % hn)
+                            hlist.append(h)
+                    except Exception as e:
+                        raise ValueError('Unable to set required handler %r' % hn) from e
+                    config['handlers'] = hlist
+            elif issubclass(klass, logging.handlers.SMTPHandler) and \
+                    'mailhost' in config:
                 config['mailhost'] = self.as_tuple(config['mailhost'])
-            elif issubclass(klass, logging.handlers.SysLogHandler) and\
+            elif issubclass(klass, logging.handlers.SysLogHandler) and \
-                'address' in config:
+                    'address' in config:
                 config['address'] = self.as_tuple(config['address'])
-            factory = klass
+            if issubclass(klass, logging.handlers.QueueHandler):
-        props = config.pop('.', None)
+                factory = functools.partial(self._configure_queue_handler, klass)
-        kwargs = {k: config[k] for k in config if valid_ident(k)}
+            else:
+                factory = klass
+        kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))}
         try:
             result = factory(**kwargs)
         except TypeError as te:
@@ -759,6 +889,7 @@ class DictConfigurator(BaseConfigurator):
             result.setLevel(logging._checkLevel(level))
         if filters:
             self.add_filters(result, filters)
+        props = config.pop('.', None)
         if props:
             for name, value in props.items():
                 setattr(result, name, value)
@@ -794,6 +925,7 @@ class DictConfigurator(BaseConfigurator):
         """Configure a non-root logger from a dictionary."""
         logger = logging.getLogger(name)
         self.common_logger_config(logger, config, incremental)
+        logger.disabled = False
         propagate = config.get('propagate', None)
         if propagate is not None:
             logger.propagate = propagate

Lib/logging/handlers.py

@@ -187,15 +187,18 @@ class RotatingFileHandler(BaseRotatingHandler):
         Basically, see if the supplied record would cause the file to exceed
         the size limit we have.
         """
-        # See bpo-45401: Never rollover anything other than regular files
-        if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
-            return False
         if self.stream is None:                 # delay was set...
             self.stream = self._open()
         if self.maxBytes > 0:                   # are we rolling over?
+            pos = self.stream.tell()
+            if not pos:
+                # gh-116263: Never rollover an empty file
+                return False
             msg = "%s\n" % self.format(record)
-            self.stream.seek(0, 2)  #due to non-posix-compliant Windows feature
+            if pos + len(msg) >= self.maxBytes:
-            if self.stream.tell() + len(msg) >= self.maxBytes:
+                # See bpo-45401: Never rollover anything other than regular files
+                if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
+                    return False
                 return True
         return False
 
@@ -232,19 +235,19 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         if self.when == 'S':
             self.interval = 1 # one second
             self.suffix = "%Y-%m-%d_%H-%M-%S"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(?!\d)"
         elif self.when == 'M':
             self.interval = 60 # one minute
             self.suffix = "%Y-%m-%d_%H-%M"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(?!\d)"
         elif self.when == 'H':
             self.interval = 60 * 60 # one hour
             self.suffix = "%Y-%m-%d_%H"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}_\d{2}(?!\d)"
         elif self.when == 'D' or self.when == 'MIDNIGHT':
             self.interval = 60 * 60 * 24 # one day
             self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}(?!\d)"
         elif self.when.startswith('W'):
             self.interval = 60 * 60 * 24 * 7 # one week
             if len(self.when) != 2:
@@ -253,11 +256,17 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                 raise ValueError("Invalid day specified for weekly rollover: %s" % self.when)
             self.dayOfWeek = int(self.when[1])
             self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
+            extMatch = r"(?<!\d)\d{4}-\d{2}-\d{2}(?!\d)"
         else:
             raise ValueError("Invalid rollover interval specified: %s" % self.when)
 
-        self.extMatch = re.compile(self.extMatch, re.ASCII)
+        # extMatch is a pattern for matching a datetime suffix in a file name.
+        # After custom naming, it is no longer guaranteed to be separated by
+        # periods from other parts of the filename.  The lookup statements
+        # (?<!\d) and (?!\d) ensure that the datetime suffix (which itself
+        # starts and ends with digits) is not preceded or followed by digits.
+        # This reduces the number of false matches and improves performance.
+        self.extMatch = re.compile(extMatch, re.ASCII)
         self.interval = self.interval * interval # multiply by units requested
         # The following line added because the filename passed in could be a
         # path object (see Issue #27493), but self.baseFilename will be a string
@@ -295,11 +304,11 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                 rotate_ts = _MIDNIGHT
             else:
                 rotate_ts = ((self.atTime.hour * 60 + self.atTime.minute)*60 +
-                    self.atTime.second)
+                             self.atTime.second)
 
             r = rotate_ts - ((currentHour * 60 + currentMinute) * 60 +
-                currentSecond)
+                             currentSecond)
-            if r < 0:
+            if r <= 0:
                 # Rotate time is before the current time (for example when
                 # self.rotateAt is 13:45 and it now 14:15), rotation is
                 # tomorrow.
@@ -328,17 +337,21 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                         daysToWait = self.dayOfWeek - day
                     else:
                         daysToWait = 6 - day + self.dayOfWeek + 1
-                    newRolloverAt = result + (daysToWait * (60 * 60 * 24))
+                    result += daysToWait * _MIDNIGHT
-                    if not self.utc:
+                result += self.interval - _MIDNIGHT * 7
-                        dstNow = t[-1]
+            else:
-                        dstAtRollover = time.localtime(newRolloverAt)[-1]
+                result += self.interval - _MIDNIGHT
-                        if dstNow != dstAtRollover:
+            if not self.utc:
-                            if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
+                dstNow = t[-1]
-                                addend = -3600
+                dstAtRollover = time.localtime(result)[-1]
-                            else:           # DST bows out before next rollover, so we need to add an hour
+                if dstNow != dstAtRollover:
-                                addend = 3600
+                    if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
-                            newRolloverAt += addend
+                        addend = -3600
-                    result = newRolloverAt
+                        if not time.localtime(result-3600)[-1]:
+                            addend = 0
+                    else:           # DST bows out before next rollover, so we need to add an hour
+                        addend = 3600
+                    result += addend
         return result
 
     def shouldRollover(self, record):
@@ -348,11 +361,15 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         record is not used, as we are just comparing times, but it is needed so
         the method signatures are the same
         """
-        # See bpo-45401: Never rollover anything other than regular files
-        if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
-            return False
         t = int(time.time())
         if t >= self.rolloverAt:
+            # See #89564: Never rollover anything other than regular files
+            if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
+                # The file is not a regular file, so do not rollover, but do
+                # set the next rollover time to avoid repeated checks.
+                self.rolloverAt = self.computeRollover(t)
+                return False
+
             return True
         return False
 
@@ -365,32 +382,28 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         dirName, baseName = os.path.split(self.baseFilename)
         fileNames = os.listdir(dirName)
         result = []
-        # See bpo-44753: Don't use the extension when computing the prefix.
+        if self.namer is None:
-        n, e = os.path.splitext(baseName)
+            prefix = baseName + '.'
-        prefix = n + '.'
+            plen = len(prefix)
-        plen = len(prefix)
+            for fileName in fileNames:
-        for fileName in fileNames:
+                if fileName[:plen] == prefix:
-            if self.namer is None:
+                    suffix = fileName[plen:]
-                # Our files will always start with baseName
+                    if self.extMatch.fullmatch(suffix):
-                if not fileName.startswith(baseName):
+                        result.append(os.path.join(dirName, fileName))
-                    continue
+        else:
-            else:
+            for fileName in fileNames:
-                # Our files could be just about anything after custom naming, but
+                # Our files could be just about anything after custom naming,
-                # likely candidates are of the form
+                # but they should contain the datetime suffix.
-                # foo.log.DATETIME_SUFFIX or foo.DATETIME_SUFFIX.log
+                # Try to find the datetime suffix in the file name and verify
-                if (not fileName.startswith(baseName) and fileName.endswith(e) and
+                # that the file name can be generated by this handler.
-                    len(fileName) > (plen + 1) and not fileName[plen+1].isdigit()):
+                m = self.extMatch.search(fileName)
-                    continue
+                while m:
-
+                    dfn = self.namer(self.baseFilename + "." + m[0])
-            if fileName[:plen] == prefix:
+                    if os.path.basename(dfn) == fileName:
-                suffix = fileName[plen:]
-                # See bpo-45628: The date/time suffix could be anywhere in the
-                # filename
-                parts = suffix.split('.')
-                for part in parts:
-                    if self.extMatch.match(part):
                         result.append(os.path.join(dirName, fileName))
                         break
+                    m = self.extMatch.search(fileName, m.start() + 1)
+
         if len(result) < self.backupCount:
             result = []
         else:
@@ -406,17 +419,14 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         then we have to get a list of matching filenames, sort them and remove
         the one with the oldest suffix.
         """
-        if self.stream:
-            self.stream.close()
-            self.stream = None
         # get the time that this sequence started at and make it a TimeTuple
         currentTime = int(time.time())
-        dstNow = time.localtime(currentTime)[-1]
         t = self.rolloverAt - self.interval
         if self.utc:
             timeTuple = time.gmtime(t)
         else:
             timeTuple = time.localtime(t)
+            dstNow = time.localtime(currentTime)[-1]
             dstThen = timeTuple[-1]
             if dstNow != dstThen:
                 if dstNow:
@@ -427,26 +437,19 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         dfn = self.rotation_filename(self.baseFilename + "." +
                                      time.strftime(self.suffix, timeTuple))
         if os.path.exists(dfn):
-            os.remove(dfn)
+            # Already rolled over.
+            return
+
+        if self.stream:
+            self.stream.close()
+            self.stream = None
         self.rotate(self.baseFilename, dfn)
         if self.backupCount > 0:
             for s in self.getFilesToDelete():
                 os.remove(s)
         if not self.delay:
             self.stream = self._open()
-        newRolloverAt = self.computeRollover(currentTime)
+        self.rolloverAt = self.computeRollover(currentTime)
-        while newRolloverAt <= currentTime:
-            newRolloverAt = newRolloverAt + self.interval
-        #If DST changes and midnight or weekly rollover, adjust for this.
-        if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc:
-            dstAtRollover = time.localtime(newRolloverAt)[-1]
-            if dstNow != dstAtRollover:
-                if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
-                    addend = -3600
-                else:           # DST bows out before next rollover, so we need to add an hour
-                    addend = 3600
-                newRolloverAt += addend
-        self.rolloverAt = newRolloverAt
 
 class WatchedFileHandler(logging.FileHandler):
     """
@@ -800,7 +803,7 @@ class SysLogHandler(logging.Handler):
         "panic":    LOG_EMERG,      #  DEPRECATED
         "warn":     LOG_WARNING,    #  DEPRECATED
         "warning":  LOG_WARNING,
-        }
+    }
 
     facility_names = {
         "auth":         LOG_AUTH,
@@ -827,12 +830,10 @@ class SysLogHandler(logging.Handler):
         "local5":       LOG_LOCAL5,
         "local6":       LOG_LOCAL6,
         "local7":       LOG_LOCAL7,
-        }
+    }
 
-    #The map below appears to be trivially lowercasing the key. However,
+    # Originally added to work around GH-43683. Unnecessary since GH-50043 but kept
-    #there's more to it than meets the eye - in some locales, lowercasing
+    # for backwards compatibility.
-    #gives unexpected results. See SF #1524081: in the Turkish locale,
-    #"INFO".lower() != "info"
     priority_map = {
         "DEBUG" : "debug",
         "INFO" : "info",
@@ -859,12 +860,49 @@ class SysLogHandler(logging.Handler):
         self.address = address
         self.facility = facility
         self.socktype = socktype
+        self.socket = None
+        self.createSocket()
+
+    def _connect_unixsocket(self, address):
+        use_socktype = self.socktype
+        if use_socktype is None:
+            use_socktype = socket.SOCK_DGRAM
+        self.socket = socket.socket(socket.AF_UNIX, use_socktype)
+        try:
+            self.socket.connect(address)
+            # it worked, so set self.socktype to the used type
+            self.socktype = use_socktype
+        except OSError:
+            self.socket.close()
+            if self.socktype is not None:
+                # user didn't specify falling back, so fail
+                raise
+            use_socktype = socket.SOCK_STREAM
+            self.socket = socket.socket(socket.AF_UNIX, use_socktype)
+            try:
+                self.socket.connect(address)
+                # it worked, so set self.socktype to the used type
+                self.socktype = use_socktype
+            except OSError:
+                self.socket.close()
+                raise
+
+    def createSocket(self):
+        """
+        Try to create a socket and, if it's not a datagram socket, connect it
+        to the other end. This method is called during handler initialization,
+        but it's not regarded as an error if the other end isn't listening yet
+        --- the method will be called again when emitting an event,
+        if there is no socket at that point.
+        """
+        address = self.address
+        socktype = self.socktype
 
         if isinstance(address, str):
             self.unixsocket = True
             # Syslog server may be unavailable during handler initialisation.
             # C's openlog() function also ignores connection errors.
-            # Moreover, we ignore these errors while logging, so it not worse
+            # Moreover, we ignore these errors while logging, so it's not worse
             # to ignore it also here.
             try:
                 self._connect_unixsocket(address)
@@ -895,30 +933,6 @@ class SysLogHandler(logging.Handler):
             self.socket = sock
             self.socktype = socktype
 
-    def _connect_unixsocket(self, address):
-        use_socktype = self.socktype
-        if use_socktype is None:
-            use_socktype = socket.SOCK_DGRAM
-        self.socket = socket.socket(socket.AF_UNIX, use_socktype)
-        try:
-            self.socket.connect(address)
-            # it worked, so set self.socktype to the used type
-            self.socktype = use_socktype
-        except OSError:
-            self.socket.close()
-            if self.socktype is not None:
-                # user didn't specify falling back, so fail
-                raise
-            use_socktype = socket.SOCK_STREAM
-            self.socket = socket.socket(socket.AF_UNIX, use_socktype)
-            try:
-                self.socket.connect(address)
-                # it worked, so set self.socktype to the used type
-                self.socktype = use_socktype
-            except OSError:
-                self.socket.close()
-                raise
-
     def encodePriority(self, facility, priority):
         """
         Encode the facility and priority. You can pass in strings or
@@ -938,7 +952,10 @@ class SysLogHandler(logging.Handler):
         """
         self.acquire()
         try:
-            self.socket.close()
+            sock = self.socket
+            if sock:
+                self.socket = None
+                sock.close()
             logging.Handler.close(self)
         finally:
             self.release()
@@ -978,6 +995,10 @@ class SysLogHandler(logging.Handler):
             # Message is a string. Convert to bytes as required by RFC 5424
             msg = msg.encode('utf-8')
             msg = prio + msg
+
+            if not self.socket:
+                self.createSocket()
+
             if self.unixsocket:
                 try:
                     self.socket.send(msg)
@@ -1094,7 +1115,16 @@ class NTEventLogHandler(logging.Handler):
                 dllname = os.path.join(dllname[0], r'win32service.pyd')
             self.dllname = dllname
             self.logtype = logtype
-            self._welu.AddSourceToRegistry(appname, dllname, logtype)
+            # Administrative privileges are required to add a source to the registry.
+            # This may not be available for a user that just wants to add to an
+            # existing source - handle this specific case.
+            try:
+                self._welu.AddSourceToRegistry(appname, dllname, logtype)
+            except Exception as e:
+                # This will probably be a pywintypes.error. Only raise if it's not
+                # an "access denied" error, else let it pass
+                if getattr(e, 'winerror', None) != 5:  # not access denied
+                    raise
             self.deftype = win32evtlog.EVENTLOG_ERROR_TYPE
             self.typemap = {
                 logging.DEBUG   : win32evtlog.EVENTLOG_INFORMATION_TYPE,
@@ -1102,10 +1132,10 @@ class NTEventLogHandler(logging.Handler):
                 logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE,
                 logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
                 logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
-         }
+            }
         except ImportError:
-            print("The Python Win32 extensions for NT (service, event "\
+            print("The Python Win32 extensions for NT (service, event " \
-                        "logging) appear not to be available.")
+                  "logging) appear not to be available.")
             self._welu = None
 
     def getMessageID(self, record):
@@ -1348,7 +1378,7 @@ class MemoryHandler(BufferingHandler):
         Check for buffer full or a record at the flushLevel or higher.
         """
         return (len(self.buffer) >= self.capacity) or \
-                (record.levelno >= self.flushLevel)
+            (record.levelno >= self.flushLevel)
 
     def setTarget(self, target):
         """
@@ -1366,7 +1396,7 @@ class MemoryHandler(BufferingHandler):
         records to the target, if there is one. Override if you want
         different behaviour.
 
-        The record buffer is also cleared by this operation.
+        The record buffer is only cleared if a target has been set.
         """
         self.acquire()
         try:
@@ -1411,6 +1441,7 @@ class QueueHandler(logging.Handler):
         """
         logging.Handler.__init__(self)
         self.queue = queue
+        self.listener = None  # will be set to listener if configured via dictConfig()
 
     def enqueue(self, record):
         """
@@ -1424,12 +1455,15 @@ class QueueHandler(logging.Handler):
 
     def prepare(self, record):
         """
-        Prepares a record for queuing. The object returned by this method is
+        Prepare a record for queuing. The object returned by this method is
         enqueued.
 
-        The base implementation formats the record to merge the message
+        The base implementation formats the record to merge the message and
-        and arguments, and removes unpickleable items from the record
+        arguments, and removes unpickleable items from the record in-place.
-        in-place.
+        Specifically, it overwrites the record's `msg` and
+        `message` attributes with the merged message (obtained by
+        calling the handler's `format` method), and sets the `args`,
+        `exc_info` and `exc_text` attributes to None.
 
         You might want to override this method if you want to convert
         the record to a dict or JSON string, or send a modified copy
@@ -1439,7 +1473,7 @@ class QueueHandler(logging.Handler):
         # (if there's exception data), and also returns the formatted
         # message. We can then use this to replace the original
         # msg + args, as these might be unpickleable. We also zap the
-        # exc_info and exc_text attributes, as they are no longer
+        # exc_info, exc_text and stack_info attributes, as they are no longer
         # needed and, if not None, will typically not be pickleable.
         msg = self.format(record)
         # bpo-35726: make copy of record to avoid affecting other handlers in the chain.
@@ -1449,6 +1483,7 @@ class QueueHandler(logging.Handler):
         record.args = None
         record.exc_info = None
         record.exc_text = None
+        record.stack_info = None
         return record
 
     def emit(self, record):

Lib/multiprocessing/connection.py

@@ -9,6 +9,7 @@
 
 __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
 
+import errno
 import io
 import os
 import sys
@@ -73,11 +74,6 @@ def arbitrary_address(family):
     if family == 'AF_INET':
         return ('localhost', 0)
     elif family == 'AF_UNIX':
-        # Prefer abstract sockets if possible to avoid problems with the address
-        # size.  When coding portable applications, some implementations have
-        # sun_path as short as 92 bytes in the sockaddr_un struct.
-        if util.abstract_sockets_supported:
-            return f"\0listener-{os.getpid()}-{next(_mmap_counter)}"
         return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir())
     elif family == 'AF_PIPE':
         return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
@@ -188,10 +184,9 @@ class _ConnectionBase:
         self._check_closed()
         self._check_writable()
         m = memoryview(buf)
-        # HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
         if m.itemsize > 1:
-            m = memoryview(bytes(m))
+            m = m.cast('B')
-        n = len(m)
+        n = m.nbytes
         if offset < 0:
             raise ValueError("offset is negative")
         if n < offset:
@@ -277,12 +272,22 @@ if _winapi:
         with FILE_FLAG_OVERLAPPED.
         """
         _got_empty_message = False
+        _send_ov = None
 
         def _close(self, _CloseHandle=_winapi.CloseHandle):
+            ov = self._send_ov
+            if ov is not None:
+                # Interrupt WaitForMultipleObjects() in _send_bytes()
+                ov.cancel()
             _CloseHandle(self._handle)
 
         def _send_bytes(self, buf):
+            if self._send_ov is not None:
+                # A connection should only be used by a single thread
+                raise ValueError("concurrent send_bytes() calls "
+                                 "are not supported")
             ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
+            self._send_ov = ov
             try:
                 if err == _winapi.ERROR_IO_PENDING:
                     waitres = _winapi.WaitForMultipleObjects(
@@ -292,7 +297,13 @@ if _winapi:
                 ov.cancel()
                 raise
             finally:
+                self._send_ov = None
                 nwritten, err = ov.GetOverlappedResult(True)
+            if err == _winapi.ERROR_OPERATION_ABORTED:
+                # close() was called by another thread while
+                # WaitForMultipleObjects() was waiting for the overlapped
+                # operation.
+                raise OSError(errno.EPIPE, "handle is closed")
             assert err == 0
             assert nwritten == len(buf)
 
@@ -465,8 +476,9 @@ class Listener(object):
         '''
         if self._listener is None:
             raise OSError('listener is closed')
+
         c = self._listener.accept()
-        if self._authkey:
+        if self._authkey is not None:
             deliver_challenge(c, self._authkey)
             answer_challenge(c, self._authkey)
         return c
@@ -728,39 +740,227 @@ if sys.platform == 'win32':
 # Authentication stuff
 #
 
-MESSAGE_LENGTH = 20
+MESSAGE_LENGTH = 40  # MUST be > 20
 
-CHALLENGE = b'#CHALLENGE#'
+_CHALLENGE = b'#CHALLENGE#'
-WELCOME = b'#WELCOME#'
+_WELCOME = b'#WELCOME#'
-FAILURE = b'#FAILURE#'
+_FAILURE = b'#FAILURE#'
 
-def deliver_challenge(connection, authkey):
+# multiprocessing.connection Authentication Handshake Protocol Description
+# (as documented for reference after reading the existing code)
+# =============================================================================
+#
+# On Windows: native pipes with "overlapped IO" are used to send the bytes,
+# instead of the length prefix SIZE scheme described below. (ie: the OS deals
+# with message sizes for us)
+#
+# Protocol error behaviors:
+#
+# On POSIX, any failure to receive the length prefix into SIZE, for SIZE greater
+# than the requested maxsize to receive, or receiving fewer than SIZE bytes
+# results in the connection being closed and auth to fail.
+#
+# On Windows, receiving too few bytes is never a low level _recv_bytes read
+# error, receiving too many will trigger an error only if receive maxsize
+# value was larger than 128 OR the if the data arrived in smaller pieces.
+#
+#      Serving side                           Client side
+#     ------------------------------  ---------------------------------------
+# 0.                                  Open a connection on the pipe.
+# 1.  Accept connection.
+# 2.  Random 20+ bytes -> MESSAGE
+#     Modern servers always send
+#     more than 20 bytes and include
+#     a {digest} prefix on it with
+#     their preferred HMAC digest.
+#     Legacy ones send ==20 bytes.
+# 3.  send 4 byte length (net order)
+#     prefix followed by:
+#       b'#CHALLENGE#' + MESSAGE
+# 4.                                  Receive 4 bytes, parse as network byte
+#                                     order integer. If it is -1, receive an
+#                                     additional 8 bytes, parse that as network
+#                                     byte order. The result is the length of
+#                                     the data that follows -> SIZE.
+# 5.                                  Receive min(SIZE, 256) bytes -> M1
+# 6.                                  Assert that M1 starts with:
+#                                       b'#CHALLENGE#'
+# 7.                                  Strip that prefix from M1 into -> M2
+# 7.1.                                Parse M2: if it is exactly 20 bytes in
+#                                     length this indicates a legacy server
+#                                     supporting only HMAC-MD5. Otherwise the
+# 7.2.                                preferred digest is looked up from an
+#                                     expected "{digest}" prefix on M2. No prefix
+#                                     or unsupported digest? <- AuthenticationError
+# 7.3.                                Put divined algorithm name in -> D_NAME
+# 8.                                  Compute HMAC-D_NAME of AUTHKEY, M2 -> C_DIGEST
+# 9.                                  Send 4 byte length prefix (net order)
+#                                     followed by C_DIGEST bytes.
+# 10. Receive 4 or 4+8 byte length
+#     prefix (#4 dance) -> SIZE.
+# 11. Receive min(SIZE, 256) -> C_D.
+# 11.1. Parse C_D: legacy servers
+#     accept it as is, "md5" -> D_NAME
+# 11.2. modern servers check the length
+#     of C_D, IF it is 16 bytes?
+# 11.2.1. "md5" -> D_NAME
+#         and skip to step 12.
+# 11.3. longer? expect and parse a "{digest}"
+#     prefix into -> D_NAME.
+#     Strip the prefix and store remaining
+#     bytes in -> C_D.
+# 11.4. Don't like D_NAME? <- AuthenticationError
+# 12. Compute HMAC-D_NAME of AUTHKEY,
+#     MESSAGE into -> M_DIGEST.
+# 13. Compare M_DIGEST == C_D:
+# 14a: Match? Send length prefix &
+#       b'#WELCOME#'
+#    <- RETURN
+# 14b: Mismatch? Send len prefix &
+#       b'#FAILURE#'
+#    <- CLOSE & AuthenticationError
+# 15.                                 Receive 4 or 4+8 byte length prefix (net
+#                                     order) again as in #4 into -> SIZE.
+# 16.                                 Receive min(SIZE, 256) bytes -> M3.
+# 17.                                 Compare M3 == b'#WELCOME#':
+# 17a.                                Match? <- RETURN
+# 17b.                                Mismatch? <- CLOSE & AuthenticationError
+#
+# If this RETURNed, the connection remains open: it has been authenticated.
+#
+# Length prefixes are used consistently. Even on the legacy protocol, this
+# was good fortune and allowed us to evolve the protocol by using the length
+# of the opening challenge or length of the returned digest as a signal as
+# to which protocol the other end supports.
+
+_ALLOWED_DIGESTS = frozenset(
+        {b'md5', b'sha256', b'sha384', b'sha3_256', b'sha3_384'})
+_MAX_DIGEST_LEN = max(len(_) for _ in _ALLOWED_DIGESTS)
+
+# Old hmac-md5 only server versions from Python <=3.11 sent a message of this
+# length. It happens to not match the length of any supported digest so we can
+# use a message of this length to indicate that we should work in backwards
+# compatible md5-only mode without a {digest_name} prefix on our response.
+_MD5ONLY_MESSAGE_LENGTH = 20
+_MD5_DIGEST_LEN = 16
+_LEGACY_LENGTHS = (_MD5ONLY_MESSAGE_LENGTH, _MD5_DIGEST_LEN)
+
+
+def _get_digest_name_and_payload(message: bytes) -> (str, bytes):
+    """Returns a digest name and the payload for a response hash.
+
+    If a legacy protocol is detected based on the message length
+    or contents the digest name returned will be empty to indicate
+    legacy mode where MD5 and no digest prefix should be sent.
+    """
+    # modern message format: b"{digest}payload" longer than 20 bytes
+    # legacy message format: 16 or 20 byte b"payload"
+    if len(message) in _LEGACY_LENGTHS:
+        # Either this was a legacy server challenge, or we're processing
+        # a reply from a legacy client that sent an unprefixed 16-byte
+        # HMAC-MD5 response. All messages using the modern protocol will
+        # be longer than either of these lengths.
+        return '', message
+    if (message.startswith(b'{') and
+        (curly := message.find(b'}', 1, _MAX_DIGEST_LEN+2)) > 0):
+        digest = message[1:curly]
+        if digest in _ALLOWED_DIGESTS:
+            payload = message[curly+1:]
+            return digest.decode('ascii'), payload
+    raise AuthenticationError(
+            'unsupported message length, missing digest prefix, '
+            f'or unsupported digest: {message=}')
+
+
+def _create_response(authkey, message):
+    """Create a MAC based on authkey and message
+
+    The MAC algorithm defaults to HMAC-MD5, unless MD5 is not available or
+    the message has a '{digest_name}' prefix. For legacy HMAC-MD5, the response
+    is the raw MAC, otherwise the response is prefixed with '{digest_name}',
+    e.g. b'{sha256}abcdefg...'
+
+    Note: The MAC protects the entire message including the digest_name prefix.
+    """
     import hmac
+    digest_name = _get_digest_name_and_payload(message)[0]
+    # The MAC protects the entire message: digest header and payload.
+    if not digest_name:
+        # Legacy server without a {digest} prefix on message.
+        # Generate a legacy non-prefixed HMAC-MD5 reply.
+        try:
+            return hmac.new(authkey, message, 'md5').digest()
+        except ValueError:
+            # HMAC-MD5 is not available (FIPS mode?), fall back to
+            # HMAC-SHA2-256 modern protocol. The legacy server probably
+            # doesn't support it and will reject us anyways. :shrug:
+            digest_name = 'sha256'
+    # Modern protocol, indicate the digest used in the reply.
+    response = hmac.new(authkey, message, digest_name).digest()
+    return b'{%s}%s' % (digest_name.encode('ascii'), response)
+
+
+def _verify_challenge(authkey, message, response):
+    """Verify MAC challenge
+
+    If our message did not include a digest_name prefix, the client is allowed
+    to select a stronger digest_name from _ALLOWED_DIGESTS.
+
+    In case our message is prefixed, a client cannot downgrade to a weaker
+    algorithm, because the MAC is calculated over the entire message
+    including the '{digest_name}' prefix.
+    """
+    import hmac
+    response_digest, response_mac = _get_digest_name_and_payload(response)
+    response_digest = response_digest or 'md5'
+    try:
+        expected = hmac.new(authkey, message, response_digest).digest()
+    except ValueError:
+        raise AuthenticationError(f'{response_digest=} unsupported')
+    if len(expected) != len(response_mac):
+        raise AuthenticationError(
+                f'expected {response_digest!r} of length {len(expected)} '
+                f'got {len(response_mac)}')
+    if not hmac.compare_digest(expected, response_mac):
+        raise AuthenticationError('digest received was wrong')
+
+
+def deliver_challenge(connection, authkey: bytes, digest_name='sha256'):
     if not isinstance(authkey, bytes):
         raise ValueError(
             "Authkey must be bytes, not {0!s}".format(type(authkey)))
+    assert MESSAGE_LENGTH > _MD5ONLY_MESSAGE_LENGTH, "protocol constraint"
     message = os.urandom(MESSAGE_LENGTH)
-    connection.send_bytes(CHALLENGE + message)
+    message = b'{%s}%s' % (digest_name.encode('ascii'), message)
-    digest = hmac.new(authkey, message, 'md5').digest()
+    # Even when sending a challenge to a legacy client that does not support
+    # digest prefixes, they'll take the entire thing as a challenge and
+    # respond to it with a raw HMAC-MD5.
+    connection.send_bytes(_CHALLENGE + message)
     response = connection.recv_bytes(256)        # reject large message
-    if response == digest:
+    try:
-        connection.send_bytes(WELCOME)
+        _verify_challenge(authkey, message, response)
+    except AuthenticationError:
+        connection.send_bytes(_FAILURE)
+        raise
     else:
-        connection.send_bytes(FAILURE)
+        connection.send_bytes(_WELCOME)
-        raise AuthenticationError('digest received was wrong')
 
-def answer_challenge(connection, authkey):
+
-    import hmac
+def answer_challenge(connection, authkey: bytes):
     if not isinstance(authkey, bytes):
         raise ValueError(
             "Authkey must be bytes, not {0!s}".format(type(authkey)))
     message = connection.recv_bytes(256)         # reject large message
-    assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
+    if not message.startswith(_CHALLENGE):
-    message = message[len(CHALLENGE):]
+        raise AuthenticationError(
-    digest = hmac.new(authkey, message, 'md5').digest()
+                f'Protocol error, expected challenge: {message=}')
+    message = message[len(_CHALLENGE):]
+    if len(message) < _MD5ONLY_MESSAGE_LENGTH:
+        raise AuthenticationError('challenge too short: {len(message)} bytes')
+    digest = _create_response(authkey, message)
     connection.send_bytes(digest)
     response = connection.recv_bytes(256)        # reject large message
-    if response != WELCOME:
+    if response != _WELCOME:
         raise AuthenticationError('digest sent was rejected')
 
 #
@@ -943,7 +1143,7 @@ else:
                             return ready
 
 #
-# Make connection and socket objects sharable if possible
+# Make connection and socket objects shareable if possible
 #
 
 if sys.platform == 'win32':

Lib/multiprocessing/context.py

@@ -223,6 +223,10 @@ class Process(process.BaseProcess):
     def _Popen(process_obj):
         return _default_context.get_context().Process._Popen(process_obj)
 
+    @staticmethod
+    def _after_fork():
+        return _default_context.get_context().Process._after_fork()
+
 class DefaultContext(BaseContext):
     Process = Process
 
@@ -254,6 +258,7 @@ class DefaultContext(BaseContext):
         return self._actual_context._name
 
     def get_all_start_methods(self):
+        """Returns a list of the supported start methods, default first."""
         if sys.platform == 'win32':
             return ['spawn']
         else:
@@ -283,6 +288,11 @@ if sys.platform != 'win32':
             from .popen_spawn_posix import Popen
             return Popen(process_obj)
 
+        @staticmethod
+        def _after_fork():
+            # process is spawned, nothing to do
+            pass
+
     class ForkServerProcess(process.BaseProcess):
         _start_method = 'forkserver'
         @staticmethod
@@ -326,6 +336,11 @@ else:
             from .popen_spawn_win32 import Popen
             return Popen(process_obj)
 
+        @staticmethod
+        def _after_fork():
+            # process is spawned, nothing to do
+            pass
+
     class SpawnContext(BaseContext):
         _name = 'spawn'
         Process = SpawnProcess

Lib/multiprocessing/forkserver.py

@@ -61,7 +61,7 @@ class ForkServer(object):
 
     def set_forkserver_preload(self, modules_names):
         '''Set list of module names to try to load in forkserver process.'''
-        if not all(type(mod) is str for mod in self._preload_modules):
+        if not all(type(mod) is str for mod in modules_names):
             raise TypeError('module_names must be a list of strings')
         self._preload_modules = modules_names
 

Lib/multiprocessing/managers.py

@@ -49,11 +49,11 @@ def reduce_array(a):
 reduction.register(array.array, reduce_array)
 
 view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
-if view_types[0] is not list:       # only needed in Py3.0
+def rebuild_as_list(obj):
-    def rebuild_as_list(obj):
+    return list, (list(obj),)
-        return list, (list(obj),)
+for view_type in view_types:
-    for view_type in view_types:
+    reduction.register(view_type, rebuild_as_list)
-        reduction.register(view_type, rebuild_as_list)
+del view_type, view_types
 
 #
 # Type for identifying shared objects
@@ -153,7 +153,7 @@ class Server(object):
         Listener, Client = listener_client[serializer]
 
         # do authentication later
-        self.listener = Listener(address=address, backlog=16)
+        self.listener = Listener(address=address, backlog=128)
         self.address = self.listener.address
 
         self.id_to_obj = {'0': (None, ())}
@@ -433,7 +433,6 @@ class Server(object):
                     self.id_to_refcount[ident] = 1
                     self.id_to_obj[ident] = \
                         self.id_to_local_proxy_obj[ident]
-                    obj, exposed, gettypeid = self.id_to_obj[ident]
                     util.debug('Server re-enabled tracking & INCREF %r', ident)
                 else:
                     raise ke
@@ -497,7 +496,7 @@ class BaseManager(object):
     _Server = Server
 
     def __init__(self, address=None, authkey=None, serializer='pickle',
-                 ctx=None):
+                 ctx=None, *, shutdown_timeout=1.0):
         if authkey is None:
             authkey = process.current_process().authkey
         self._address = address     # XXX not final address if eg ('', 0)
@@ -507,6 +506,7 @@ class BaseManager(object):
         self._serializer = serializer
         self._Listener, self._Client = listener_client[serializer]
         self._ctx = ctx or get_context()
+        self._shutdown_timeout = shutdown_timeout
 
     def get_server(self):
         '''
@@ -570,8 +570,8 @@ class BaseManager(object):
         self._state.value = State.STARTED
         self.shutdown = util.Finalize(
             self, type(self)._finalize_manager,
-            args=(self._process, self._address, self._authkey,
+            args=(self._process, self._address, self._authkey, self._state,
-                  self._state, self._Client),
+                  self._Client, self._shutdown_timeout),
             exitpriority=0
             )
 
@@ -656,7 +656,8 @@ class BaseManager(object):
         self.shutdown()
 
     @staticmethod
-    def _finalize_manager(process, address, authkey, state, _Client):
+    def _finalize_manager(process, address, authkey, state, _Client,
+                          shutdown_timeout):
         '''
         Shutdown the manager process; will be registered as a finalizer
         '''
@@ -671,15 +672,17 @@ class BaseManager(object):
             except Exception:
                 pass
 
-            process.join(timeout=1.0)
+            process.join(timeout=shutdown_timeout)
             if process.is_alive():
                 util.info('manager still alive')
                 if hasattr(process, 'terminate'):
                     util.info('trying to `terminate()` manager process')
                     process.terminate()
-                    process.join(timeout=1.0)
+                    process.join(timeout=shutdown_timeout)
                     if process.is_alive():
                         util.info('manager still alive after terminate')
+                        process.kill()
+                        process.join()
 
         state.value = State.SHUTDOWN
         try:
@@ -1338,7 +1341,6 @@ if HAS_SHMEM:
 
         def __del__(self):
             util.debug(f"{self.__class__.__name__}.__del__ by pid {getpid()}")
-            pass
 
         def get_server(self):
             'Better than monkeypatching for now; merge into Server ultimately'

Lib/multiprocessing/pool.py

@@ -203,6 +203,9 @@ class Pool(object):
             processes = os.cpu_count() or 1
         if processes < 1:
             raise ValueError("Number of processes must be at least 1")
+        if maxtasksperchild is not None:
+            if not isinstance(maxtasksperchild, int) or maxtasksperchild <= 0:
+                raise ValueError("maxtasksperchild must be a positive int or None")
 
         if initializer is not None and not callable(initializer):
             raise TypeError('initializer must be a callable')
@@ -693,7 +696,7 @@ class Pool(object):
 
         if (not result_handler.is_alive()) and (len(cache) != 0):
             raise AssertionError(
-                "Cannot have cache with result_hander not alive")
+                "Cannot have cache with result_handler not alive")
 
         result_handler._state = TERMINATE
         change_notifier.put(None)

Lib/multiprocessing/popen_spawn_win32.py

@@ -14,6 +14,7 @@ __all__ = ['Popen']
 #
 #
 
+# Exit code used by Popen.terminate()
 TERMINATE = 0x10000
 WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
 WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
@@ -54,19 +55,20 @@ class Popen(object):
         wfd = msvcrt.open_osfhandle(whandle, 0)
         cmd = spawn.get_command_line(parent_pid=os.getpid(),
                                      pipe_handle=rhandle)
-        cmd = ' '.join('"%s"' % x for x in cmd)
 
         python_exe = spawn.get_executable()
 
         # bpo-35797: When running in a venv, we bypass the redirect
         # executor and launch our base Python.
         if WINENV and _path_eq(python_exe, sys.executable):
-            python_exe = sys._base_executable
+            cmd[0] = python_exe = sys._base_executable
             env = os.environ.copy()
             env["__PYVENV_LAUNCHER__"] = sys.executable
         else:
             env = None
 
+        cmd = ' '.join('"%s"' % x for x in cmd)
+
         with open(wfd, 'wb', closefd=True) as to_child:
             # start process
             try:
@@ -99,18 +101,20 @@ class Popen(object):
         return reduction.duplicate(handle, self.sentinel)
 
     def wait(self, timeout=None):
-        if self.returncode is None:
+        if self.returncode is not None:
-            if timeout is None:
+            return self.returncode
-                msecs = _winapi.INFINITE
-            else:
-                msecs = max(0, int(timeout * 1000 + 0.5))
 
-            res = _winapi.WaitForSingleObject(int(self._handle), msecs)
+        if timeout is None:
-            if res == _winapi.WAIT_OBJECT_0:
+            msecs = _winapi.INFINITE
-                code = _winapi.GetExitCodeProcess(self._handle)
+        else:
-                if code == TERMINATE:
+            msecs = max(0, int(timeout * 1000 + 0.5))
-                    code = -signal.SIGTERM
+
-                self.returncode = code
+        res = _winapi.WaitForSingleObject(int(self._handle), msecs)
+        if res == _winapi.WAIT_OBJECT_0:
+            code = _winapi.GetExitCodeProcess(self._handle)
+            if code == TERMINATE:
+                code = -signal.SIGTERM
+            self.returncode = code
 
         return self.returncode
 
@@ -118,12 +122,22 @@ class Popen(object):
         return self.wait(timeout=0)
 
     def terminate(self):
-        if self.returncode is None:
+        if self.returncode is not None:
-            try:
+            return
-                _winapi.TerminateProcess(int(self._handle), TERMINATE)
+
-            except OSError:
+        try:
-                if self.wait(timeout=1.0) is None:
+            _winapi.TerminateProcess(int(self._handle), TERMINATE)
-                    raise
+        except PermissionError:
+            # ERROR_ACCESS_DENIED (winerror 5) is received when the
+            # process already died.
+            code = _winapi.GetExitCodeProcess(int(self._handle))
+            if code == _winapi.STILL_ACTIVE:
+                raise
+
+        # gh-113009: Don't set self.returncode. Even if GetExitCodeProcess()
+        # returns an exit code different than STILL_ACTIVE, the process can
+        # still be running. Only set self.returncode once WaitForSingleObject()
+        # returns WAIT_OBJECT_0 in wait().
 
     kill = terminate
 

Lib/multiprocessing/process.py

@@ -61,7 +61,7 @@ def parent_process():
 def _cleanup():
     # check for processes which have finished
     for p in list(_children):
-        if p._popen.poll() is not None:
+        if (child_popen := p._popen) and child_popen.poll() is not None:
             _children.discard(p)
 
 #
@@ -304,8 +304,7 @@ class BaseProcess(object):
             if threading._HAVE_THREAD_NATIVE_ID:
                 threading.main_thread()._set_native_id()
             try:
-                util._finalizer_registry.clear()
+                self._after_fork()
-                util._run_after_forkers()
             finally:
                 # delay finalization of the old process object until after
                 # _run_after_forkers() is executed
@@ -336,6 +335,13 @@ class BaseProcess(object):
 
         return exitcode
 
+    @staticmethod
+    def _after_fork():
+        from . import util
+        util._finalizer_registry.clear()
+        util._run_after_forkers()
+
+
 #
 # We subclass bytes to avoid accidental transmission of auth keys over network
 #
@@ -427,6 +433,7 @@ _exitcode_to_name = {}
 for name, signum in list(signal.__dict__.items()):
     if name[:3]=='SIG' and '_' not in name:
         _exitcode_to_name[-signum] = f'-{name}'
+del name, signum
 
 # For debug and leak testing
 _dangling = WeakSet()

Lib/multiprocessing/queues.py

@@ -158,6 +158,20 @@ class Queue(object):
         except AttributeError:
             pass
 
+    def _terminate_broken(self):
+        # Close a Queue on error.
+
+        # gh-94777: Prevent queue writing to a pipe which is no longer read.
+        self._reader.close()
+
+        # gh-107219: Close the connection writer which can unblock
+        # Queue._feed() if it was stuck in send_bytes().
+        if sys.platform == 'win32':
+            self._writer.close()
+
+        self.close()
+        self.join_thread()
+
     def _start_thread(self):
         debug('Queue._start_thread()')
 
@@ -169,13 +183,19 @@ class Queue(object):
                   self._wlock, self._reader.close, self._writer.close,
                   self._ignore_epipe, self._on_queue_feeder_error,
                   self._sem),
-            name='QueueFeederThread'
+            name='QueueFeederThread',
+            daemon=True,
         )
-        self._thread.daemon = True
 
-        debug('doing self._thread.start()')
+        try:
-        self._thread.start()
+            debug('doing self._thread.start()')
-        debug('... done self._thread.start()')
+            self._thread.start()
+            debug('... done self._thread.start()')
+        except:
+            # gh-109047: During Python finalization, creating a thread
+            # can fail with RuntimeError.
+            self._thread = None
+            raise
 
         if not self._joincancelled:
             self._jointhread = Finalize(
@@ -280,6 +300,8 @@ class Queue(object):
         import traceback
         traceback.print_exc()
 
+    __class_getitem__ = classmethod(types.GenericAlias)
+
 
 _sentinel = object()
 

Lib/multiprocessing/resource_sharer.py

@@ -123,7 +123,7 @@ class _ResourceSharer(object):
         from .connection import Listener
         assert self._listener is None, "Already have Listener"
         util.debug('starting listener and thread for sending handles')
-        self._listener = Listener(authkey=process.current_process().authkey)
+        self._listener = Listener(authkey=process.current_process().authkey, backlog=128)
         self._address = self._listener.address
         t = threading.Thread(target=self._serve)
         t.daemon = True

Lib/multiprocessing/resource_tracker.py

@@ -51,15 +51,31 @@ if os.name == 'posix':
     })
 
 
+class ReentrantCallError(RuntimeError):
+    pass
+
+
 class ResourceTracker(object):
 
     def __init__(self):
-        self._lock = threading.Lock()
+        self._lock = threading.RLock()
         self._fd = None
         self._pid = None
 
+    def _reentrant_call_error(self):
+        # gh-109629: this happens if an explicit call to the ResourceTracker
+        # gets interrupted by a garbage collection, invoking a finalizer (*)
+        # that itself calls back into ResourceTracker.
+        #   (*) for example the SemLock finalizer
+        raise ReentrantCallError(
+            "Reentrant call into the multiprocessing resource tracker")
+
     def _stop(self):
         with self._lock:
+            # This should not happen (_stop() isn't called by a finalizer)
+            # but we check for it anyway.
+            if self._lock._recursion_count() > 1:
+                return self._reentrant_call_error()
             if self._fd is None:
                 # not running
                 return
@@ -81,6 +97,9 @@ class ResourceTracker(object):
         This can be run from any process.  Usually a child process will use
         the resource created by its parent.'''
         with self._lock:
+            if self._lock._recursion_count() > 1:
+                # The code below is certainly not reentrant-safe, so bail out
+                return self._reentrant_call_error()
             if self._fd is not None:
                 # resource tracker was launched before, is it still running?
                 if self._check_alive():
@@ -159,12 +178,22 @@ class ResourceTracker(object):
         self._send('UNREGISTER', name, rtype)
 
     def _send(self, cmd, name, rtype):
-        self.ensure_running()
+        try:
+            self.ensure_running()
+        except ReentrantCallError:
+            # The code below might or might not work, depending on whether
+            # the resource tracker was already running and still alive.
+            # Better warn the user.
+            # (XXX is warnings.warn itself reentrant-safe? :-)
+            warnings.warn(
+                f"ResourceTracker called reentrantly for resource cleanup, "
+                f"which is unsupported. "
+                f"The {rtype} object {name!r} might leak.")
         msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii')
-        if len(name) > 512:
+        if len(msg) > 512:
             # posix guarantees that writes to a pipe of less than PIPE_BUF
             # bytes are atomic, and that PIPE_BUF >= 512
-            raise ValueError('name too long')
+            raise ValueError('msg too long')
         nbytes = os.write(self._fd, msg)
         assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format(
             nbytes, len(msg))
@@ -176,6 +205,7 @@ register = _resource_tracker.register
 unregister = _resource_tracker.unregister
 getfd = _resource_tracker.getfd
 
+
 def main(fd):
     '''Run resource tracker.'''
     # protect the process from ^C and "killall python" etc