From 22a4cf0bd32ae70c43320b39a84389e130f65165 Mon Sep 17 00:00:00 2001 From: johnslavik Date: Wed, 10 Jun 2026 13:36:11 +0200 Subject: [PATCH 1/5] Document asyncio introspection tools --- Doc/library/asyncio-graph.rst | 8 +- Doc/library/asyncio-tools.rst | 134 ++++++++++++++++++++++++++++++++++ Doc/library/asyncio.rst | 13 ++++ 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 Doc/library/asyncio-tools.rst diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 5f642a32bf75c24..1a8cdcf6a9e8cba 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -4,7 +4,7 @@ .. _asyncio-graph: ======================== -Call Graph Introspection +Call graph introspection ======================== **Source code:** :source:`Lib/asyncio/graph.py` @@ -17,6 +17,12 @@ a suspended *future*. These utilities and the underlying machinery can be used from within a Python program or by external profilers and debuggers. +.. seealso:: + + :ref:`asyncio-introspection-tools` + Command-line tools for inspecting tasks in another running Python + process. + .. versionadded:: 3.14 diff --git a/Doc/library/asyncio-tools.rst b/Doc/library/asyncio-tools.rst new file mode 100644 index 000000000000000..f38d995591a61d7 --- /dev/null +++ b/Doc/library/asyncio-tools.rst @@ -0,0 +1,134 @@ +.. currentmodule:: asyncio + +.. _asyncio-introspection-tools: + +================================ +Command-line introspection tools +================================ + +**Source code:** :source:`Lib/asyncio/tools.py` + +The :mod:`!asyncio` module can be executed as a script to inspect asyncio +tasks in another Python process: + +.. code-block:: shell-session + + $ python -m asyncio ps [--retries N] PID + $ python -m asyncio pstree [--retries N] PID + +``PID`` is the process ID of the Python process to inspect. The commands use +Python's :ref:`remote debugging support ` to read the target +process state, but do not execute code in the target process. They are only +available on supported platforms and may require permission to inspect another +process. See :ref:`permission-requirements` for details. + +.. seealso:: + + :ref:`asyncio-graph` + Programmatic APIs for inspecting the async call graph of a task or + future in the current process. + +The command examples below use this program, which creates a task hierarchy +suitable for inspection and prints its process ID: + +.. code-block:: python + + import asyncio + import os + + async def play(track): + await asyncio.sleep(3600) + print(f"🎵 Finished: {track}") + + async def album(name, tracks): + async with asyncio.TaskGroup() as tg: + for track in tracks: + tg.create_task(play(track), name=track) + + async def main(): + print(f"PID: {os.getpid()}", flush=True) + async with asyncio.TaskGroup() as tg: + tg.create_task( + album("Sundowning", ["TNDNBTG", "Levitate"]), + name="Sundowning", + ) + tg.create_task( + album("TMBTE", ["DYWTYLM", "Aqua Regia"]), + name="TMBTE", + ) + + asyncio.run(main()) + +Run the program in one terminal and leave it running: + +.. code-block:: shell-session + + $ python example.py + PID: 12345 + +Then pass the printed process ID to the commands from another terminal. +Thread IDs, task IDs, file paths, and line numbers vary between runs and +source layouts. + +.. versionadded:: 3.14 + +Command-line options +==================== + +.. option:: ps PID + + Display a table of pending tasks in the process *PID*. The table includes + the thread ID, task ID, task name, coroutine stack, awaiter chain, awaiter + name, and awaiter ID: + + .. code-block:: shell-session + + $ python -m asyncio ps 12345 + tid task id task name coroutine stack awaiter chain awaiter name awaiter id + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 18445801 0x10a456060 Task-1 TaskGroup._aexit -> TaskGroup.__aexit__ -> main 0x0 + 18445801 0x10a439f60 Sundowning TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a439d70 TMBTE TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a2a3a80 TNDNBTG sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2a38a0 Levitate sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2d7150 DYWTYLM sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + 18445801 0x10a6bdaa0 Aqua Regia sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + +.. option:: pstree PID + + Display the same task and coroutine relationships as a tree: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + └── (T) Task-1 + └── main example.py:12 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) Sundowning + │ └── album example.py:7 + │ └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + │ └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + │ ├── (T) TNDNBTG + │ │ └── play example.py:4 + │ │ └── sleep Lib/asyncio/tasks.py:702 + │ └── (T) Levitate + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) TMBTE + └── album example.py:7 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) DYWTYLM + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) Aqua Regia + └── play example.py:4 + └── sleep Lib/asyncio/tasks.py:702 + +.. option:: --retries N + + Retry failed attempts to inspect the target process up to *N* times. See + :ref:`sampling-efficiency` for details about failed process memory reads. + + .. versionadded:: 3.15 diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 0f72e31dee5f1d1..4fe6f7ee1408f0e 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -47,6 +47,13 @@ asyncio provides a set of **high-level** APIs to: * :ref:`synchronize ` concurrent code; +There are also **introspection** APIs and tools for: + +* inspecting the :ref:`async call graph ` of tasks and futures; + +* inspecting tasks in another running Python process with + :ref:`command-line tools `; + Additionally, there are **low-level** APIs for *library and framework developers* to: @@ -108,7 +115,13 @@ for full functionality and the latest features. asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst + +.. toctree:: + :caption: Introspection APIs + :maxdepth: 1 + asyncio-graph.rst + asyncio-tools.rst .. toctree:: :caption: Low-level APIs From 2a8afdf6123ce2ac72f4fcca2dd683126b5993b0 Mon Sep 17 00:00:00 2001 From: johnslavik Date: Fri, 12 Jun 2026 09:46:23 +0200 Subject: [PATCH 2/5] Add separator line after source code reference --- Doc/library/asyncio-tools.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/asyncio-tools.rst b/Doc/library/asyncio-tools.rst index f38d995591a61d7..56bafd75ee9540a 100644 --- a/Doc/library/asyncio-tools.rst +++ b/Doc/library/asyncio-tools.rst @@ -8,6 +8,8 @@ Command-line introspection tools **Source code:** :source:`Lib/asyncio/tools.py` +------------------------------------- + The :mod:`!asyncio` module can be executed as a script to inspect asyncio tasks in another Python process: From 397ae6c889518f5c51fbe96c30a9f6c039f2109a Mon Sep 17 00:00:00 2001 From: johnslavik Date: Sat, 13 Jun 2026 17:20:41 +0200 Subject: [PATCH 3/5] gh-151213: Improve What's New prose and add cycle detection to asyncio-tools - Rewrite What's New intro to lead with developer problem/value rather than "Added a new command-line interface" - Drop the misleading "remote debugging support" cross-reference from both asyncio-tools.rst and What's New; PYTHON_DISABLE_REMOTE_DEBUG does not affect RemoteUnwinder-based task introspection - Document cycle detection under the pstree option in asyncio-tools.rst (previously only mentioned in What's New) --- Doc/library/asyncio-tools.rst | 16 +++++++++++++--- Doc/whatsnew/3.14.rst | 26 ++++++++++++-------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Doc/library/asyncio-tools.rst b/Doc/library/asyncio-tools.rst index 56bafd75ee9540a..5ce96c951f01b7e 100644 --- a/Doc/library/asyncio-tools.rst +++ b/Doc/library/asyncio-tools.rst @@ -18,9 +18,8 @@ tasks in another Python process: $ python -m asyncio ps [--retries N] PID $ python -m asyncio pstree [--retries N] PID -``PID`` is the process ID of the Python process to inspect. The commands use -Python's :ref:`remote debugging support ` to read the target -process state, but do not execute code in the target process. They are only +``PID`` is the process ID of the Python process to inspect. The commands read +the target process state without executing any code in it. They are only available on supported platforms and may require permission to inspect another process. See :ref:`permission-requirements` for details. @@ -128,6 +127,17 @@ Command-line options └── play example.py:4 └── sleep Lib/asyncio/tasks.py:702 + If the await graph contains a cycle, ``pstree`` reports an error instead + of printing a tree. A cycle in the await graph is unusual and typically + indicates a programming error: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + ERROR: await-graph contains cycles - cannot print a tree! + + cycle: Task-2 → Task-3 → Task-2 + .. option:: --retries N Retry failed attempts to inspect the target process up to *N* times. See diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index cd0d8b7cb006fee..56e27ef88ee60a8 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -691,24 +691,22 @@ Victor Stinner, and Rogdham in :gh:`132983`.) Asyncio introspection capabilities ---------------------------------- -Added a new command-line interface to inspect running Python processes -using asynchronous tasks, available via ``python -m asyncio ps PID`` -or ``python -m asyncio pstree PID``. +Two new subcommands let you inspect the task graph of a running asyncio +application without modifying its source code or restarting it. -The ``ps`` subcommand inspects the given process ID (PID) and displays -information about currently running asyncio tasks. -It outputs a task table: a flat listing of all tasks, their names, -their coroutine stacks, and which tasks are awaiting them. +``python -m asyncio ps PID`` prints a flat table of all pending tasks in the +process: their names, coroutine stacks, and which task (if any) is waiting +for each one. +``python -m asyncio pstree PID`` renders the same information as an async +call tree, showing the full hierarchy of task groups and the coroutine frames +suspended inside each task. When a program hangs or stalls, a glance at the +tree is often enough to identify which branch is blocked and exactly where in +its coroutine stack execution has paused. -The ``pstree`` subcommand fetches the same information, but instead renders a -visual async call tree, showing coroutine relationships in a hierarchical format. -This command is particularly useful for debugging long-running or stuck -asynchronous programs. -It can help developers quickly identify where a program is blocked, -what tasks are pending, and how coroutines are chained together. +Both commands read the target process state without executing any code in it. -For example given this code: +For example, given this code: .. code-block:: python From 33a89765d22630797a275429df6415e06855df5f Mon Sep 17 00:00:00 2001 From: johnslavik Date: Sat, 13 Jun 2026 17:26:09 +0200 Subject: [PATCH 4/5] gh-151213: Revert accidental What's New rewrite The What's New text was already correct; only asyncio-tools.rst needed changes. --- Doc/whatsnew/3.14.rst | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 56e27ef88ee60a8..cd0d8b7cb006fee 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -691,22 +691,24 @@ Victor Stinner, and Rogdham in :gh:`132983`.) Asyncio introspection capabilities ---------------------------------- -Two new subcommands let you inspect the task graph of a running asyncio -application without modifying its source code or restarting it. +Added a new command-line interface to inspect running Python processes +using asynchronous tasks, available via ``python -m asyncio ps PID`` +or ``python -m asyncio pstree PID``. -``python -m asyncio ps PID`` prints a flat table of all pending tasks in the -process: their names, coroutine stacks, and which task (if any) is waiting -for each one. +The ``ps`` subcommand inspects the given process ID (PID) and displays +information about currently running asyncio tasks. +It outputs a task table: a flat listing of all tasks, their names, +their coroutine stacks, and which tasks are awaiting them. -``python -m asyncio pstree PID`` renders the same information as an async -call tree, showing the full hierarchy of task groups and the coroutine frames -suspended inside each task. When a program hangs or stalls, a glance at the -tree is often enough to identify which branch is blocked and exactly where in -its coroutine stack execution has paused. -Both commands read the target process state without executing any code in it. +The ``pstree`` subcommand fetches the same information, but instead renders a +visual async call tree, showing coroutine relationships in a hierarchical format. +This command is particularly useful for debugging long-running or stuck +asynchronous programs. +It can help developers quickly identify where a program is blocked, +what tasks are pending, and how coroutines are chained together. -For example, given this code: +For example given this code: .. code-block:: python From 35ad424c112938c11fea77839f6d6a000d16b775 Mon Sep 17 00:00:00 2001 From: johnslavik Date: Sat, 13 Jun 2026 17:28:45 +0200 Subject: [PATCH 5/5] gh-151213: Improve asyncio-tools reference docs - Add one-sentence value summary to the module intro - Document all seven columns of the ps table output - Note that ps does not perform cycle detection - Add motivation sentences to both ps and pstree option descriptions - Document cycle detection behavior under pstree --- Doc/library/asyncio-tools.rst | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/Doc/library/asyncio-tools.rst b/Doc/library/asyncio-tools.rst index 5ce96c951f01b7e..2712de54309bb1f 100644 --- a/Doc/library/asyncio-tools.rst +++ b/Doc/library/asyncio-tools.rst @@ -10,8 +10,8 @@ Command-line introspection tools ------------------------------------- -The :mod:`!asyncio` module can be executed as a script to inspect asyncio -tasks in another Python process: +The :mod:`!asyncio` module can be executed as a script to inspect the task +graph of another running Python process without modifying it or restarting it: .. code-block:: shell-session @@ -78,9 +78,21 @@ Command-line options .. option:: ps PID - Display a table of pending tasks in the process *PID*. The table includes - the thread ID, task ID, task name, coroutine stack, awaiter chain, awaiter - name, and awaiter ID: + Display a flat table of all pending tasks in the process *PID*. Each row + contains: + + * **tid** — OS thread ID of the thread running the event loop + * **task id** — memory address of the :class:`~asyncio.Task` object + * **task name** — name assigned to the task (see :meth:`~asyncio.Task.get_name`) + * **coroutine stack** — chain of coroutine frame names from outermost to innermost + * **awaiter chain** — coroutine frames of the task that is awaiting this task + * **awaiter name** — name of the awaiting task + * **awaiter id** — memory address of the awaiting task (``0x0`` if none) + + This subcommand prints all tasks regardless of whether the await graph + contains cycles. Use it when you need to filter or process task data + programmatically, or when the task count is large enough that a tree + would be unwieldy: .. code-block:: shell-session @@ -97,7 +109,11 @@ Command-line options .. option:: pstree PID - Display the same task and coroutine relationships as a tree: + Display task and coroutine relationships as a tree. Each task is shown + with its full coroutine stack, nested under the task (if any) that is + awaiting it. This subcommand is useful for quickly identifying which branch + of a task hierarchy is blocked and where in its coroutine stack execution + has paused: .. code-block:: shell-session