From cdd4ad01e428f4e9f334caf2abafe4171abce2cd Mon Sep 17 00:00:00 2001 From: Taegyun Kim Date: Wed, 21 Oct 2026 15:26:02 -0400 Subject: [PATCH] gh-160107: Make AsyncioDebug reachable from _PyRuntime and add a header Debuggers and profilers could only find the asyncio debug offsets by searching loaded binaries for the AsyncioDebug section, and the table had no cookie, version or size to check what they found. _asyncio now stores the address of _Py_AsyncioDebug in _PyRuntime.asyncio_debug_offsets, whose offset is published as _Py_DebugOffsets.runtime_state.asyncio_debug_offsets. The pointer is NULL until _asyncio is initialized. The table now starts with a cookie ("xasyncpy"), PY_VERSION_HEX and its size. The AsyncioDebug section is kept for existing tools. _remote_debugging follows the pointer instead of searching for the section, and rejects a table whose cookie, version or size does not match. --- Include/internal/pycore_debug_offsets.h | 3 + Include/internal/pycore_runtime_structs.h | 5 ++ Lib/test/test_external_inspection.py | 39 ++++++++++- ...-10-21-19-50-13.gh-issue-160107.zc9ciw.rst | 5 ++ Modules/_asynciomodule.c | 12 +++- Modules/_remote_debugging/_remote_debugging.h | 4 +- Modules/_remote_debugging/asyncio.c | 66 ++++++------------- .../debug_offsets_validation.h | 35 ++++++++-- 8 files changed, 117 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-21-19-50-13.gh-issue-160107.zc9ciw.rst diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 6e1eb573c8c2c83..91276531dc71154 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -10,6 +10,7 @@ extern "C" { #define _Py_Debug_Cookie "xdebugpy" +#define _Py_AsyncioDebug_Cookie "xasyncpy" #if defined(__APPLE__) # include @@ -73,6 +74,7 @@ typedef struct _Py_DebugOffsets { uint64_t size; uint64_t finalizing; uint64_t interpreters_head; + uint64_t asyncio_debug_offsets; } runtime_state; // Interpreter state offset; @@ -268,6 +270,7 @@ typedef struct _Py_DebugOffsets { .size = sizeof(_PyRuntimeState), \ .finalizing = offsetof(_PyRuntimeState, _finalizing), \ .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ + .asyncio_debug_offsets = offsetof(_PyRuntimeState, asyncio_debug_offsets), \ }, \ .interpreter_state = { \ .size = sizeof(PyInterpreterState), \ diff --git a/Include/internal/pycore_runtime_structs.h b/Include/internal/pycore_runtime_structs.h index 5470f5858eda087..95e5dc12108d139 100644 --- a/Include/internal/pycore_runtime_structs.h +++ b/Include/internal/pycore_runtime_structs.h @@ -146,6 +146,11 @@ struct pyruntimestate { */ _Py_DebugOffsets debug_offsets; + /* Address of the _asyncio module's debug offsets table, or NULL until the + module is initialized. Debuggers locate this field through + debug_offsets.runtime_state.asyncio_debug_offsets. */ + void *asyncio_debug_offsets; + /* Has been initialized to a safe state. In order to be effective, this must be set to 0 during or right diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index d6243c3fc818eab..cd2ec6e78cd499e 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -19,7 +19,7 @@ requires_remote_subprocess_debugging, ) from test.support.import_helper import import_module -from test.support.script_helper import make_script +from test.support.script_helper import assert_python_ok, make_script from test.support.socket_helper import find_unused_port import subprocess @@ -4747,5 +4747,42 @@ async def main(): ) +class TestAsyncioDebugOffsets(unittest.TestCase): + def test_asyncio_debug_offsets_reachable_from_runtime(self): + import_module("ctypes") + code = textwrap.dedent(""" + import ctypes + import struct + import sys + + runtime = ctypes.addressof( + ctypes.c_char.in_dll(ctypes.pythonapi, "_PyRuntime")) + # _Py_DebugOffsets starts with cookie, version and free_threaded, + # followed by runtime_state: size, finalizing, interpreters_head + # and asyncio_debug_offsets. + cookie, version, _ = struct.unpack( + "=8sQQ", ctypes.string_at(runtime, 24)) + assert cookie == b"xdebugpy", cookie + assert version == sys.hexversion, version + pointer_offset = struct.unpack( + "=Q", ctypes.string_at(runtime + 48, 8))[0] + + def asyncio_table(): + return ctypes.c_void_p.from_address( + runtime + pointer_offset).value + + assert asyncio_table() is None + import _asyncio + address = asyncio_table() + assert address is not None + cookie, table_version, size = struct.unpack( + "=8sQQ", ctypes.string_at(address, 24)) + assert cookie == b"xasyncpy", cookie + assert table_version == version, table_version + assert size == 16 * 8, size + """) + assert_python_ok("-S", "-c", code) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-10-21-19-50-13.gh-issue-160107.zc9ciw.rst b/Misc/NEWS.d/next/Library/2026-10-21-19-50-13.gh-issue-160107.zc9ciw.rst new file mode 100644 index 000000000000000..33e56e2bd4857bd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-21-19-50-13.gh-issue-160107.zc9ciw.rst @@ -0,0 +1,5 @@ +The :mod:`!_asyncio` debug offsets table now starts with a cookie, version +and size, and its address is published in ``_PyRuntime`` so debuggers and +profilers can find it through ``_Py_DebugOffsets`` instead of searching +loaded binaries for the ``AsyncioDebug`` section. :mod:`!_remote_debugging` +now uses this pointer and validates the table header. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 5569f7a3670378b..be66635e634de00 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -99,6 +99,9 @@ typedef struct { || PyObject_TypeCheck(obj, state->TaskType)) typedef struct _Py_AsyncioModuleDebugOffsets { + char cookie[8] _Py_NONSTRING; + uint64_t version; + uint64_t size; struct _asyncio_task_object { uint64_t size; uint64_t task_name; @@ -121,7 +124,10 @@ typedef struct _Py_AsyncioModuleDebugOffsets { } Py_AsyncioModuleDebugOffsets; GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _Py_AsyncioDebug) - = {.asyncio_task_object = { + = {.cookie = _Py_AsyncioDebug_Cookie, + .version = PY_VERSION_HEX, + .size = sizeof(Py_AsyncioModuleDebugOffsets), + .asyncio_task_object = { .size = sizeof(TaskObj), .task_name = offsetof(TaskObj, task_name), .task_awaited_by = offsetof(TaskObj, task_awaited_by), @@ -4315,6 +4321,10 @@ module_init(asyncio_state *state) } state->debug_offsets = &_Py_AsyncioDebug; + // Let debuggers find the table from _PyRuntime. Every interpreter stores + // the same address, and extension modules are never unloaded. + _Py_atomic_store_ptr_release(&_PyRuntime.asyncio_debug_offsets, + &_Py_AsyncioDebug); Py_DECREF(module); return 0; diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 79fa4a92e745bd0..76282ee0c8c5811 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -193,6 +193,9 @@ typedef enum _WIN32_THREADSTATE { * ============================================================================ */ struct _Py_AsyncioModuleDebugOffsets { + char cookie[8] _Py_NONSTRING; + uint64_t version; + uint64_t size; struct _asyncio_task_object { uint64_t size; uint64_t task_name; @@ -712,7 +715,6 @@ iterate_interpreters( * ASYNCIO FUNCTION DECLARATIONS * ============================================================================ */ -extern uintptr_t _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle); extern int read_async_debug(RemoteUnwinderObject *unwinder); extern int ensure_async_debug_offsets(RemoteUnwinderObject *unwinder); diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index 884065b6efdee74..304eb639e35193c 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -12,60 +12,33 @@ * ASYNCIO DEBUG ADDRESS FUNCTIONS * ============================================================================ */ -uintptr_t -_Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) +static uintptr_t +get_async_debug_address(RemoteUnwinderObject *unwinder) { - uintptr_t address; - -#ifdef MS_WINDOWS - // On Windows, search for asyncio debug in executable or DLL - address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio", - NULL); - if (address == 0) { - if (!_Py_RemoteDebug_HasPermissionError()) { - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); - _PyErr_ChainExceptions1(exc); - } - } -#elif defined(__linux__) && HAVE_PROCESS_VM_READV - // On Linux, search for asyncio debug in executable or DLL - address = search_linux_map_for_section(handle, "AsyncioDebug", "python", - NULL); - if (address == 0) { - if (!_Py_RemoteDebug_HasPermissionError()) { - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); - _PyErr_ChainExceptions1(exc); - } - } -#elif defined(__APPLE__) && TARGET_OS_OSX - // On macOS, try libpython first, then fall back to python - address = search_map_for_section(handle, "AsyncioDebug", "libpython", - NULL); - if (address == 0 && !_Py_RemoteDebug_HasPermissionError()) { - PyErr_Clear(); - address = search_map_for_section(handle, "AsyncioDebug", "python", - NULL); + // _asyncio publishes the address of its table in _PyRuntime when it is + // initialized. Read it without the page cache because it changes when the + // target process imports _asyncio. + uintptr_t pointer_addr = unwinder->runtime_start_address + + (uintptr_t)unwinder->debug_offsets.runtime_state.asyncio_debug_offsets; + uintptr_t address = 0; + if (_Py_RemoteDebug_ReadRemoteMemory(&unwinder->handle, pointer_addr, + sizeof(address), &address) < 0) + { + set_exception_cause(unwinder, PyExc_RuntimeError, + "Failed to read AsyncioDebug address from PyRuntime"); + return 0; } if (address == 0) { - if (!_Py_RemoteDebug_HasPermissionError()) { - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); - _PyErr_ChainExceptions1(exc); - } + PyErr_SetString(PyExc_RuntimeError, + "The _asyncio module is not initialized in the remote process"); } -#else - Py_UNREACHABLE(); -#endif - return address; } int read_async_debug(RemoteUnwinderObject *unwinder) { - uintptr_t async_debug_addr = _Py_RemoteDebug_GetAsyncioDebugAddress(&unwinder->handle); + uintptr_t async_debug_addr = get_async_debug_address(unwinder); if (!async_debug_addr) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to get AsyncioDebug address"); return -1; @@ -77,7 +50,10 @@ read_async_debug(RemoteUnwinderObject *unwinder) set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read AsyncioDebug offsets"); return result; } - if (_PyRemoteDebug_ValidateAsyncDebugOffsets(&unwinder->async_debug_offsets) < 0) { + if (_PyRemoteDebug_ValidateAsyncDebugOffsets( + &unwinder->async_debug_offsets, + unwinder->debug_offsets.version) < 0) + { set_exception_cause(unwinder, PyExc_RuntimeError, "Invalid AsyncioDebug offsets"); return PY_REMOTE_DEBUG_INVALID_ASYNC_DEBUG_OFFSETS; } diff --git a/Modules/_remote_debugging/debug_offsets_validation.h b/Modules/_remote_debugging/debug_offsets_validation.h index 66c35e5522a9c7d..3abc0e3d03c387c 100644 --- a/Modules/_remote_debugging/debug_offsets_validation.h +++ b/Modules/_remote_debugging/debug_offsets_validation.h @@ -31,8 +31,8 @@ #define FIELD_SIZE(type, member) sizeof(((type *)0)->member) enum { - PY_REMOTE_DEBUG_OFFSETS_TOTAL_SIZE = 888, - PY_REMOTE_ASYNC_DEBUG_OFFSETS_TOTAL_SIZE = 104, + PY_REMOTE_DEBUG_OFFSETS_TOTAL_SIZE = 896, + PY_REMOTE_ASYNC_DEBUG_OFFSETS_TOTAL_SIZE = 128, }; /* @@ -256,7 +256,8 @@ validate_fixed_field( * the unwinder, so no validation is needed for them. */ #define PY_REMOTE_DEBUG_RUNTIME_STATE_FIELDS(APPLY, buffer_size) \ - APPLY(runtime_state, interpreters_head, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size) + APPLY(runtime_state, interpreters_head, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(runtime_state, asyncio_debug_offsets, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size) /* current_exception also covers the adjacent exc_info pointer. */ #define PY_REMOTE_DEBUG_THREAD_STATE_FIELDS(APPLY, buffer_size) \ @@ -471,8 +472,34 @@ _PyRemoteDebug_ValidateDebugOffsetsLayout(struct _Py_DebugOffsets *debug_offsets /* Called once when AsyncioDebug is loaded, before any task inspection uses it. */ static inline int _PyRemoteDebug_ValidateAsyncDebugOffsets( - struct _Py_AsyncioModuleDebugOffsets *debug_offsets) + struct _Py_AsyncioModuleDebugOffsets *debug_offsets, + uint64_t runtime_version) { + if (memcmp(debug_offsets->cookie, _Py_AsyncioDebug_Cookie, + sizeof(debug_offsets->cookie)) != 0) + { + PyErr_SetString(PyExc_RuntimeError, + "Invalid AsyncioDebug offsets: bad cookie"); + return -1; + } + /* The table must come from the same build as the runtime. */ + if (debug_offsets->version != runtime_version) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid AsyncioDebug offsets: version %#llx does not match " + "runtime version %#llx", + (unsigned long long)debug_offsets->version, + (unsigned long long)runtime_version); + return -1; + } + if (debug_offsets->size < sizeof(*debug_offsets)) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid AsyncioDebug offsets: size %llu is smaller than %zu", + (unsigned long long)debug_offsets->size, + sizeof(*debug_offsets)); + return -1; + } PY_REMOTE_DEBUG_VALIDATE_READ_SECTION(asyncio_task_object, SIZEOF_TASK_OBJ); PY_REMOTE_DEBUG_TASK_OBJECT_FIELDS( PY_REMOTE_DEBUG_VALIDATE_FIELD,