diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 35658aeba705056..4492d6b91b8b010 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1322,6 +1322,31 @@ write:: :class:`threading.local` will *not* survive across different callbacks, even when those calls are made from the same C thread. +Callback functions can return structures and unions by value, in addition to +the simple types: + +.. code-block:: python + + class Point(Structure): + _fields_ = [("x", c_int), ("y", c_int)] + + @CFUNCTYPE(Point) + def get_origin(): + return Point(0, 0) + +.. note:: + + A structure or union returned from a callback is copied *by value*, and + :mod:`!ctypes` does not keep the returned object alive after the callback + returns. If the structure contains a pointer field (such as + :class:`c_char_p` or a :func:`POINTER` type), you must ensure that the + memory it refers to stays valid for as long as the C code uses it -- the + same requirement C itself imposes. Returning a pointer to memory owned by + a temporary Python object will leave the C caller with a dangling pointer. + +.. versionchanged:: next + Callback functions can now return structures and unions. + .. _ctypes-accessing-values-exported-from-dlls: Accessing values exported from dlls diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index c0171596a17a33e..b9e7060fd91ee53 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -4,10 +4,11 @@ import math import sys import unittest +import warnings from _ctypes import CTYPES_MAX_ARGCOUNT -from ctypes import (CDLL, cdll, Structure, CFUNCTYPE, +from ctypes import (CDLL, cdll, Structure, Union, CFUNCTYPE, ArgumentError, POINTER, sizeof, - c_byte, c_ubyte, c_char, + c_byte, c_ubyte, c_char, c_char_p, c_short, c_ushort, c_int, c_uint, c_long, c_longlong, c_ulonglong, c_ulong, c_float, c_double, c_longdouble, py_object) @@ -293,6 +294,159 @@ def callback(check, s): self.assertEqual(s.second, check.second) self.assertEqual(s.third, check.third) + # gh-49960: callbacks may return structures and unions by value. + + def test_callback_return_small_struct(self): + class SmallRet(Structure): + _fields_ = [("a", c_int), ("b", c_int)] + + CALLBACK = CFUNCTYPE(SmallRet) + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_cbk_ret_small_struct + func.argtypes = (CALLBACK,) + func.restype = SmallRet + + result = func(CALLBACK(lambda: SmallRet(17, 42))) + self.assertEqual((result.a, result.b), (17, 42)) + + def test_callback_return_struct_reaches_c(self): + # The C helper sums the fields itself, proving the bytes actually + # reached the C caller rather than only round-tripping in Python. + class SmallRet(Structure): + _fields_ = [("a", c_int), ("b", c_int)] + + CALLBACK = CFUNCTYPE(SmallRet) + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_cbk_ret_small_struct_sum + func.argtypes = (CALLBACK,) + func.restype = c_long + + self.assertEqual(func(CALLBACK(lambda: SmallRet(300, 45))), 345) + + def test_callback_return_large_struct(self): + # Mirrors `Test` in Modules/_ctypes/_ctypes_test.c: >8 bytes, so it + # is returned via a hidden pointer rather than in registers. + class X(Structure): + _fields_ = [("first", c_ulong), + ("second", c_ulong), + ("third", c_ulong)] + + CALLBACK = CFUNCTYPE(X) + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_cbk_ret_large_struct + func.argtypes = (CALLBACK,) + func.restype = X + + result = func(CALLBACK(lambda: X(0xdeadbeef, 0xcafebabe, 0x0bad1dea))) + self.assertEqual(result.first, 0xdeadbeef) + self.assertEqual(result.second, 0xcafebabe) + self.assertEqual(result.third, 0x0bad1dea) + + def test_callback_return_float_struct(self): + # All-float struct: SSE class on x86-64, HFA on AArch64. + class FloatRet(Structure): + _fields_ = [("x", c_double), ("y", c_double)] + + CALLBACK = CFUNCTYPE(FloatRet) + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_cbk_ret_float_struct + func.argtypes = (CALLBACK,) + func.restype = FloatRet + + result = func(CALLBACK(lambda: FloatRet(1.5, -2.25))) + self.assertEqual((result.x, result.y), (1.5, -2.25)) + + def test_callback_return_union(self): + class UnionRet(Union): + _fields_ = [("i", c_int), ("f", c_float)] + + CALLBACK = CFUNCTYPE(UnionRet) + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_cbk_ret_union + func.argtypes = (CALLBACK,) + func.restype = UnionRet + + result = func(CALLBACK(lambda: UnionRet(i=0x41424344))) + self.assertEqual(result.i, 0x41424344) + + def test_callback_return_struct_repeatedly(self): + # Guards against per-call leaks and against regressing into the + # "memory leak in callback function" RuntimeWarning path. + class SmallRet(Structure): + _fields_ = [("a", c_int), ("b", c_int)] + + CALLBACK = CFUNCTYPE(SmallRet) + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_cbk_ret_small_struct_sum + func.argtypes = (CALLBACK,) + func.restype = c_long + + cb = CALLBACK(lambda: SmallRet(1, 2)) + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + for _ in range(100): + self.assertEqual(func(cb), 3) + + def test_callback_return_struct_subclass(self): + # isinstance semantics: a subclass instance is acceptable. + class SmallRet(Structure): + _fields_ = [("a", c_int), ("b", c_int)] + + class SubRet(SmallRet): + pass + + CALLBACK = CFUNCTYPE(SmallRet) + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_cbk_ret_small_struct_sum + func.argtypes = (CALLBACK,) + func.restype = c_long + + self.assertEqual(func(CALLBACK(lambda: SubRet(4, 5))), 9) + + def test_callback_return_struct_wrong_type(self): + class SmallRet(Structure): + _fields_ = [("a", c_int), ("b", c_int)] + + class Other(Structure): + _fields_ = [("q", c_int)] + + CALLBACK = CFUNCTYPE(SmallRet) + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_cbk_ret_small_struct_sum + func.argtypes = (CALLBACK,) + func.restype = c_long + + for bad in (None, 42, Other(1)): + with self.subTest(bad=bad): + def cb(bad=bad): + return bad + with support.catch_unraisable_exception() as cm: + # The buffer is zeroed on failure, so the sum is 0. + self.assertEqual(func(CALLBACK(cb)), 0) + self.assertIsInstance(cm.unraisable.exc_value, TypeError) + self.assertEqual( + cm.unraisable.err_msg, + f"Exception ignored while converting result " + f"of ctypes callback function {cb!r}") + + def test_callback_return_struct_with_pointer(self): + # gh-49960 / bpo-5710 discussion: the struct is copied by value, so + # any pointer it contains must reference memory the caller keeps + # alive. Here `keepalive` does exactly that. + class WithPtr(Structure): + _fields_ = [("s", c_char_p)] + + keepalive = b"hello" + + CALLBACK = CFUNCTYPE(WithPtr) + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_cbk_ret_ptr_struct + func.argtypes = (CALLBACK,) + func.restype = WithPtr + + result = func(CALLBACK(lambda: WithPtr(keepalive))) + self.assertEqual(result.s, b"hello") + def test_callback_too_many_args(self): def func(*args): return len(args) diff --git a/Misc/NEWS.d/next/Library/2026-09-18-00-00-00.gh-issue-49960.MDT3jE.rst b/Misc/NEWS.d/next/Library/2026-09-18-00-00-00.gh-issue-49960.MDT3jE.rst new file mode 100644 index 000000000000000..9793c1143c7d0d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-18-00-00-00.gh-issue-49960.MDT3jE.rst @@ -0,0 +1,3 @@ +:mod:`ctypes` callback functions can now return structures and unions by +value. Previously only "simple" types could be returned, and any other result +type raised :exc:`TypeError`. diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 6a5b20143f55272..93c54af243242b2 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -72,6 +72,79 @@ _testfunc_cbk_large_struct(Test in, void (*func)(Test)) func(in); } +/* + * gh-49960: callbacks returning structs and unions by value. + * + * Each of these exercises a different ABI return class, since struct + * return conventions are highly platform-specific: + * - SmallRet: small integer struct, returned in registers + * - Test: >8 bytes (reused from above), returned via hidden pointer + * - FloatRet: all-float struct, SSE class on x86-64 / HFA on AArch64 + * - UnionRet: union + * - PtrRet: struct containing a pointer + */ + +typedef struct { + int a; + int b; +} SmallRet; + +EXPORT(SmallRet) +_testfunc_cbk_ret_small_struct(SmallRet (*func)(void)) +{ + return func(); +} + +/* Returns a scalar derived from the struct, so a test can prove the bytes + actually reached the C caller rather than only round-tripping in Python. */ +EXPORT(long) +_testfunc_cbk_ret_small_struct_sum(SmallRet (*func)(void)) +{ + SmallRet s = func(); + return (long)s.a + (long)s.b; +} + +EXPORT(Test) +_testfunc_cbk_ret_large_struct(Test (*func)(void)) +{ + return func(); +} + +typedef struct { + double x; + double y; +} FloatRet; + +EXPORT(FloatRet) +_testfunc_cbk_ret_float_struct(FloatRet (*func)(void)) +{ + return func(); +} + +typedef union { + int i; + float f; +} UnionRet; + +EXPORT(UnionRet) +_testfunc_cbk_ret_union(UnionRet (*func)(void)) +{ + return func(); +} + +/* Struct containing a pointer, for the documented pointer-lifetime contract: + the struct is copied by value, so the pointed-to memory must outlive the + callback. */ +typedef struct { + const char *s; +} PtrRet; + +EXPORT(PtrRet) +_testfunc_cbk_ret_ptr_struct(PtrRet (*func)(void)) +{ + return func(); +} + /* * See issue 29565. Update a structure passed by value; * the caller should not see any change. diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index e67817e8f6f4d96..680dd4aa44f46a0 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -127,6 +127,7 @@ is_narrow_int_ffi_type(int type) static void _CallPythonObject(ctypes_state *st, void *mem, ffi_type *restype, + PyObject *restype_obj, SETFUNC setfunc, PyObject *callable, PyObject *converters, @@ -236,11 +237,11 @@ static void _CallPythonObject(ctypes_state *st, Py_XDECREF(error_object); if (restype != &ffi_type_void && result) { - assert(setfunc); /* libffi's closure contract requires integral results narrower than ffi_arg to fill a whole register, sign-extended if signed; - setfunc() only writes restype->size bytes. */ + setfunc() only writes restype->size bytes. Structs and unions + are never narrow integers, so they always write to mem directly. */ union { ffi_arg arg; int8_t s8; @@ -254,64 +255,102 @@ static void _CallPythonObject(ctypes_state *st, is_narrow_int_ffi_type(restype->type); void *resmem = narrow ? (void *)&narrow_res : mem; - /* keep is an object we have to keep alive so that the result - stays valid. If there is no such object, the setfunc will - have returned Py_None. - - If there is such an object, we have no choice than to keep - it alive forever - but a refcount and/or memory leak will - be the result. EXCEPT when restype is py_object - Python - itself knows how to manage the refcount of these objects. - */ - PyObject *keep = setfunc(resmem, result, restype->size); - - if (narrow && keep != NULL) { - ffi_arg widened; - switch (restype->type) { - case FFI_TYPE_SINT8: - widened = (ffi_arg)(ffi_sarg)narrow_res.s8; - break; - case FFI_TYPE_SINT16: - widened = (ffi_arg)(ffi_sarg)narrow_res.s16; - break; - case FFI_TYPE_SINT32: - widened = (ffi_arg)(ffi_sarg)narrow_res.s32; - break; - case FFI_TYPE_UINT8: - widened = narrow_res.u8; - break; - case FFI_TYPE_UINT16: - widened = narrow_res.u16; - break; - case FFI_TYPE_UINT32: - widened = narrow_res.u32; - break; - default: - Py_UNREACHABLE(); + if (setfunc == NULL) { + /* gh-49960: struct/union return. There is no setfunc for these + types, so copy the bytes out of the CData object directly. + The struct is copied by value and no object is kept alive, so + any pointer it contains must reference memory the caller keeps + alive - the same contract C imposes. */ + assert(!narrow); + int ok = 0; + if (CDataObject_Check(st, result)) { + int is_inst = PyObject_IsInstance(result, restype_obj); + if (is_inst < 0) { + /* Discard this failure; the TypeError raised below is the + more useful report and only one can be shown. */ + PyErr_Clear(); + } + else if (is_inst) { + CDataObject *cd = (CDataObject *)result; + Py_BEGIN_CRITICAL_SECTION(cd); + memcpy(mem, cd->b_ptr, restype->size); + Py_END_CRITICAL_SECTION(); + ok = 1; + } + } + if (!ok) { + /* Zero the buffer so the C caller sees deterministic zeros + rather than uninitialised memory. */ + memset(mem, 0, restype->size); + PyErr_Format(PyExc_TypeError, + "ctypes callback function returned unexpected " + "type %T", result); + PyErr_FormatUnraisable( + "Exception ignored while converting result " + "of ctypes callback function %R", + callable); } - memcpy(mem, &widened, sizeof(ffi_arg)); - } - - if (keep == NULL) { - /* Could not convert callback result. */ - PyErr_FormatUnraisable( - "Exception ignored while converting result " - "of ctypes callback function %R", - callable); } - else if (setfunc != _ctypes_get_fielddesc("O")->setfunc) { - if (keep == Py_None) { - /* Nothing to keep */ - Py_DECREF(keep); + else { + /* keep is an object we have to keep alive so that the result + stays valid. If there is no such object, the setfunc will + have returned Py_None. + + If there is such an object, we have no choice than to keep + it alive forever - but a refcount and/or memory leak will + be the result. EXCEPT when restype is py_object - Python + itself knows how to manage the refcount of these objects. + */ + PyObject *keep = setfunc(resmem, result, restype->size); + + if (narrow && keep != NULL) { + ffi_arg widened; + switch (restype->type) { + case FFI_TYPE_SINT8: + widened = (ffi_arg)(ffi_sarg)narrow_res.s8; + break; + case FFI_TYPE_SINT16: + widened = (ffi_arg)(ffi_sarg)narrow_res.s16; + break; + case FFI_TYPE_SINT32: + widened = (ffi_arg)(ffi_sarg)narrow_res.s32; + break; + case FFI_TYPE_UINT8: + widened = narrow_res.u8; + break; + case FFI_TYPE_UINT16: + widened = narrow_res.u16; + break; + case FFI_TYPE_UINT32: + widened = narrow_res.u32; + break; + default: + Py_UNREACHABLE(); + } + memcpy(mem, &widened, sizeof(ffi_arg)); } - else if (PyErr_WarnEx(PyExc_RuntimeWarning, - "memory leak in callback function.", - 1) == -1) { + + if (keep == NULL) { + /* Could not convert callback result. */ PyErr_FormatUnraisable( "Exception ignored while converting result " "of ctypes callback function %R", callable); } + else if (setfunc != _ctypes_get_fielddesc("O")->setfunc) { + if (keep == Py_None) { + /* Nothing to keep */ + Py_DECREF(keep); + } + else if (PyErr_WarnEx(PyExc_RuntimeWarning, + "memory leak in callback function.", + 1) == -1) { + PyErr_FormatUnraisable( + "Exception ignored while converting result " + "of ctypes callback function %R", + callable); + } + } } } @@ -344,6 +383,7 @@ static void closure_fcn(ffi_cif *cif, _CallPythonObject(st, resp, p->ffi_restype, + p->restype, p->setfunc, p->callable, p->converters, @@ -422,10 +462,20 @@ CThunkObject *_ctypes_alloc_callback(ctypes_state *st, goto error; } - if (info == NULL || info->setfunc == NULL) { - PyErr_SetString(PyExc_TypeError, - "invalid result type for callback function"); - goto error; + /* gh-49960: structs and unions have no setfunc (that is reserved for + "simple" types), but can still be returned by value. Leaving + p->setfunc as NULL signals the struct-return path in + _CallPythonObject. */ + if (info == NULL + || ( + info->setfunc == NULL + && !PyCStructTypeObject_Check(st, restype) + && !PyObject_TypeCheck(restype, st->UnionType_Type) + ) + ) { + PyErr_SetString(PyExc_TypeError, + "invalid result type for callback function"); + goto error; } p->setfunc = info->setfunc; p->ffi_restype = &info->ffi_type_pointer;