diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index eedeb180c6b760..4c750ed9920932 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -822,7 +822,39 @@ Object Protocol This function is intended to be used soon after *op* is created, by the code that creates it, such as in the object's :c:member:`~PyTypeObject.tp_new` slot. + It is a fast operation that does not stop other threads. To immortalize an + object that may already be shared, use :c:func:`PyUnstable_Immortalize`. Returns 1 if the object was made immortal and returns 0 if it was not. This function cannot fail. .. versionadded:: 3.15 + +.. c:function:: int PyUnstable_Immortalize(PyObject *op) + + Marks an object *op* :term:`immortal`, even if it is shared across threads. + :class:`str` instances, including subclasses, are not made immortal. + This is intended to reduce reference counting contention in the + :term:`free-threaded build` for objects shared across threads and used in + native code. + + This is a slow operation. In the free-threaded build, it stops all other + threads in the interpreter while immortalizing the object. For newly created, + uniquely referenced objects, prefer :c:func:`PyUnstable_SetImmortal`. + + This is a one-way process: objects can only be made immortal; they cannot be + made mortal once again. Immortal objects do not participate in reference + counting and will never be garbage collected. If the object is GC-tracked, + it is untracked. + + Returns 1 if the object was made immortal and 0 if the object cannot be made + immortal (such as when the object is already immortal). + Returns -1 with an exception set upon failure. + + + .. warning:: + + Immortalizing GC-tracked objects (and other objects on the heap) will + result in them being leaked for the remainder of the process, as immortal + objects are not deallocated at interpreter shutdown. + + .. versionadded:: next diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index b29170b5b0b93d..4771fd2c248d3f 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1245,6 +1245,12 @@ C API changes New features ------------ +* Add :c:func:`PyUnstable_Immortalize` to make already shared objects + :term:`immortal`. In free-threaded builds, the function stops other threads + while immortalizing the object. :class:`str` instances are not supported. + Immortalized objects are not deallocated at interpreter shutdown. + (Contributed by Peter Bierma in :gh:`150865`.) + * Add :c:func:`Py_HashBuffer` to the limited C API. (Contributed by Victor Stinner in :gh:`148233`.) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 12d2e614623e30..0072d4bd6dd232 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -439,6 +439,7 @@ PyAPI_FUNC(void) PyUnstable_EnableTryIncRef(PyObject *); PyAPI_FUNC(int) PyUnstable_Object_IsUniquelyReferenced(PyObject *); PyAPI_FUNC(int) PyUnstable_SetImmortal(PyObject *op); +PyAPI_FUNC(int) PyUnstable_Immortalize(PyObject *op); #if defined(Py_GIL_DISABLED) PyAPI_FUNC(uintptr_t) _Py_GetThreadLocal_Addr(void); diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index a9ad61f458944a..78a9b0cefd9c7e 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -9,6 +9,7 @@ extern "C" { #include "pycore_ast_state.h" // struct ast_state #include "pycore_llist.h" // struct llist_node +#include "pycore_object_stack.h" // _PyObjectStack #include "pycore_opcode_utils.h" // NUM_COMMON_CONSTANTS #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR #include "pycore_structs.h" // PyHamtObject @@ -235,6 +236,9 @@ struct _gc_runtime_state { #endif /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; + // Untracked immortal frames and generators whose references must remain + // visible to GC. Access requires the GIL or a stop-the-world pause. + _PyObjectStack immortal_frames; struct gc_stats *generation_stats; #ifdef Py_GIL_DISABLED /* Protects generation_stats between gc_get_stats_impl() (reader) and diff --git a/Lib/test/test_capi/test_immortal.py b/Lib/test/test_capi/test_immortal.py index 6b882d65a42391..65e836f408097c 100644 --- a/Lib/test/test_capi/test_immortal.py +++ b/Lib/test/test_capi/test_immortal.py @@ -1,5 +1,6 @@ import unittest -from test.support import import_helper +from test.support import Py_DEBUG, Py_GIL_DISABLED, import_helper, threading_helper, skip_if_sanitizer +from test.support.script_helper import assert_python_ok _testcapi = import_helper.import_module('_testcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi') @@ -21,6 +22,472 @@ def test_immortal(self): # CRASHES _testcapi.is_immortal(NULL) + def test_immortalize(self): + script = """if True: + import _testcapi + import gc + import sys + + objects = [object(), [], {"item": []}, set(), bytes([1, 2])] + shared = list(objects) + for obj in objects: + assert not sys._is_immortal(obj) + assert _testcapi.py_immortalize(obj) == 1 + assert sys._is_immortal(obj) + assert not gc.is_tracked(obj) + assert _testcapi.py_immortalize(obj) == 0 + assert all(sys._is_immortal(obj) for obj in shared) + assert _testcapi.py_immortalize(None) == 0 + """ + assert_python_ok("-c", script) + + def test_immortalize_dict_merge(self): + import_helper.import_module('_testlimitedcapi') + script = """if True: + import _testcapi + import _testlimitedcapi + import gc + + mapping = {} + assert _testcapi.py_immortalize(mapping) == 1 + assert not gc.is_tracked(mapping) + # Both no-op and nonempty merges accept an untracked immortal dict. + assert _testlimitedcapi.dict_update(mapping, {}) == 0 + assert _testlimitedcapi.dict_update(mapping, {"item": []}) == 0 + assert _testlimitedcapi.dict_merge(mapping, {"other": 42}, 1) == 0 + assert mapping == {"item": [], "other": 42} + """ + assert_python_ok("-c", script) + + @unittest.skipIf(Py_GIL_DISABLED, "mortal stack reference optimization is GIL-only") + def test_immortalize_updates_live_frames(self): + script = """if True: + import _testcapi + import sys + + def check(obj, clear=False): + assert _testcapi.py_immortalize(obj) == 1 + before = sys.getrefcount(obj) + for _ in range(100): + # Force LOAD_FAST_CHECK, which duplicates the old counted + # reference, rather than borrowing it with LOAD_FAST_BORROW. + if clear: + del obj + refs = [obj] + after = sys.getrefcount(obj) + # BUILD_LIST steals the duplicate, but list destruction skips + # decrefing immortal items. Unbounded increments can overflow. + assert after == before, (before, after) + + check(object()) + """ + assert_python_ok("-c", script) + + @unittest.skipIf(Py_GIL_DISABLED, "mortal stack reference optimization is GIL-only") + @threading_helper.requires_working_threading() + def test_immortalize_other_thread_frames(self): + script = """if True: + import _testcapi + import sys + import threading + from test.support import threading_helper + + ready = threading.Event() + resume = threading.Event() + results = [] + obj = object() + + def worker(local, clear=False): + ready.set() + resume.wait() + before = sys.getrefcount(local) + for _ in range(100): + if clear: + del local + refs = [local] + results.append((before, sys.getrefcount(local))) + + thread = threading.Thread(target=worker, args=(obj,)) + with threading_helper.start_threads([thread]): + ready.wait() + try: + assert _testcapi.py_immortalize(obj) == 1 + finally: + resume.set() + assert len(results) == 1 + before, after = results[0] + assert before == after, results + """ + assert_python_ok("-c", script) + + @unittest.skipIf(Py_GIL_DISABLED, "mortal stack reference optimization is GIL-only") + def test_immortalize_suspended_frames(self): + script = """if True: + import _testcapi + import sys + + def gen(obj, clear=False): + yield + before = sys.getrefcount(obj) + for _ in range(100): + if clear: + del obj + refs = [obj] + yield before, sys.getrefcount(obj) + + for immortal_generator in (False, True): + obj = object() + generator = gen(obj) + next(generator) + if immortal_generator: + _testcapi.py_immortalize(generator) + assert _testcapi.py_immortalize(obj) == 1 + before, after = next(generator) + assert before == after, (before, after) + generator.close() + """ + assert_python_ok("-c", script) + + @unittest.skipUnless(Py_GIL_DISABLED, "deferred refcounting is free-threaded only") + def test_immortalize_keeps_deferred_refcount(self): + script = """if True: + import _testcapi + import _testinternalcapi + import gc + import sys + import types + + def func(): + return 42 + + class Heap: + attr = 42 + + targets = [func, func.__code__, Heap, types.ModuleType("immortal"), []] + _testcapi.pyobject_enable_deferred_refcount(targets[-1]) + for obj in targets: + assert _testinternalcapi.has_deferred_refcount(obj) + assert _testcapi.py_immortalize(obj) == 1 + assert _testinternalcapi.has_deferred_refcount(obj) + assert sys._is_immortal(obj) + assert not gc.is_tracked(obj) + gc.collect() + assert func() == Heap.attr == 42 + """ + assert_python_ok("-c", script) + + @unittest.skipUnless(Py_GIL_DISABLED, "requires free threading") + @threading_helper.requires_working_threading() + def test_immortalize_concurrent(self): + script = """if True: + import _testcapi + import threading + from test.support import threading_helper + + objects = [object() for _ in range(25)] + barrier = threading.Barrier(2) + results = [[], []] + def worker(index): + for obj in objects: + barrier.wait() + results[index].append(_testcapi.py_immortalize(obj)) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(2)] + with threading_helper.start_threads(threads): + pass + assert all(len(result) == len(objects) for result in results) + assert all(sorted(pair) == [0, 1] for pair in zip(*results)), results + """ + assert_python_ok("-c", script) + + @unittest.skipUnless(Py_GIL_DISABLED, "per-thread refcounting is free-threaded only") + @threading_helper.requires_working_threading() + def test_immortalize_flushes_other_thread_refcounts(self): + script = """if True: + import _testcapi + import gc + import threading + import types + import weakref + from test.support import threading_helper + + gc.disable() + code = compile("pass", "", "exec") + ready = threading.Event() + release = threading.Event() + + def worker(): + functions = [types.FunctionType(code, {}) for _ in range(100)] + ready.set() + release.wait() + functions.clear() + + thread = threading.Thread(target=worker) + with threading_helper.start_threads([thread]): + ready.wait() + try: + assert _testcapi.py_immortalize(code) == 1 + # The replacement reuses the id just released by code. + victim = code.replace() + victim_ref = weakref.ref(victim) + gc.collect() + del victim + finally: + release.set() + + gc.collect() + # Pending counts from the worker must not keep the victim alive. + assert victim_ref() is None, "stale per-thread counts reached a reused id" + """ + assert_python_ok("-c", script) + + def test_immortalize_rejects_strings(self): + script = """if True: + import _testcapi + import gc + import sys + + string = "".join(["immortal", "string"]) + if sys.argv[1] == "interned": + string = sys.intern(string) + elif sys.argv[1] == "subclass": + class String(str): + pass + string = String(string) + was_immortal = sys._is_immortal(string) + was_tracked = gc.is_tracked(string) + if sys.argv[1] != "interned": + assert not was_immortal + assert _testcapi.py_immortalize(string) == 0 + assert sys._is_immortal(string) == was_immortal + assert gc.is_tracked(string) == was_tracked + + # Exercise Unicode's consistency checks and both interning orders. + assert string.encode() == b"immortalstring" + if sys.argv[1] != "subclass": + assert sys.intern(string) is string + """ + for state in ("not interned", "interned", "subclass"): + with self.subTest(state=state): + assert_python_ok("-c", script, state) + + @skip_if_sanitizer(thread=True) + def test_arbitrary_immortalization(self): + # This test stresses immortalizing a multitude of objects + import textwrap + + script = """ + import _testcapi + import contextlib + import warnings + import sys + + warnings.simplefilter("ignore", DeprecationWarning) + + # These have weird side effects + blacklisted = {"antigravity", "this"} + + def immortalize(obj): + was_immortal = sys._is_immortal(obj) + result = _testcapi.py_immortalize(obj) + if was_immortal: + assert result == 0 + + if result == 1: + assert sys._is_immortal(obj) + + def immortalize_everything(mod): + _testcapi.py_immortalize(mod) + + for i in dir(mod): + attr = getattr(mod, i) + immortalize(attr) + + for name in dir(attr): + with contextlib.suppress(Exception): + immortalize(getattr(attr, name)) + + for name in sys.stdlib_module_names: + if name in blacklisted: + continue + + with contextlib.suppress(ImportError): + immortalize_everything(__import__(name)) + """ + assert_python_ok("-c", textwrap.dedent(script)) + + @unittest.skipUnless(Py_DEBUG, "requires a debug build") + def test_immortalize_ref_total(self): + # References to immortal objects must be excluded from RefTotal, + # since subsequent decrefs will not be counted. + script = """if True: + import _testcapi + import gc + import sys + + gc.disable() + + def measure(immortalize): + before = sys.gettotalrefcount() + for _ in range(100): + obj = object() + refs = [obj, obj] + if immortalize: + _testcapi.py_immortalize(obj) + del obj, refs + return sys.gettotalrefcount() - before + + # Warm up both paths before comparing their reference totals. + measure(False) + measure(True) + mortal = measure(False) + immortal = measure(True) + assert immortal == mortal, (immortal, mortal) + """ + assert_python_ok("-c", script) + + @unittest.skipUnless(Py_DEBUG and Py_GIL_DISABLED, "requires a free-threaded debug build") + def test_immortalize_per_thread_ref_total(self): + script = """if True: + import _testcapi + import gc + import sys + + gc.disable() + class Heap: + pass + + instances = [Heap() for _ in range(100)] + controls = [object() for _ in range(100)] + assert _testcapi.py_immortalize(Heap) == 1 + + def measure_drop(objects): + before = sys.gettotalrefcount() + objects.clear() + return sys.gettotalrefcount() - before + + measure_drop([]) + immortal_type = measure_drop(instances) + static_type = measure_drop(controls) + assert immortal_type == static_type, (immortal_type, static_type) + """ + assert_python_ok("-c", script) + + @unittest.skipUnless(Py_GIL_DISABLED, "deferred refcounting is free-threaded only") + def test_immortalize_generator_deferred_locals(self): + # Untracking an immortal generator must not hide its deferred locals + # from the GC while they are still needed by the suspended frame. + script = """if True: + import _testcapi + import _testinternalcapi + import gc + import sys + import weakref + + gc.disable() + + def make_generator(): + class Local: + pass + + def gen(local): + yield + yield local + later = yield + yield weakref.ref(later) + yield later + + assert _testinternalcapi.has_deferred_refcount(Local) + generator = gen(Local) + next(generator) + return generator, weakref.ref(Local) + + generator, local_ref = make_generator() + assert local_ref() is not None + assert _testcapi.py_immortalize(generator) == 1 + assert sys._is_immortal(generator) + assert not gc.is_tracked(generator) + + gc.collect() + # Check the weakref before resuming, to avoid using a dangling local. + assert local_ref() is not None, "immortal generator lost its local" + assert next(generator) is local_ref() + next(generator) + if sys.argv[1] == "frozen": + gc.freeze() + class Later: + pass + assert _testinternalcapi.has_deferred_refcount(Later) + # send() stores a new deferred reference after immortalization. + later_ref = generator.send(Later) + del Later + gc.collect() + assert later_ref() is not None, "resumed immortal generator lost its local" + assert next(generator) is later_ref() + generator.close() + """ + for state in ("normal", "frozen"): + with self.subTest(state=state): + assert_python_ok("-c", script, state) + + @unittest.skipUnless(Py_GIL_DISABLED, "deferred refcounting is free-threaded only") + def test_immortalize_frame_deferred_locals(self): + script = """if True: + import _testcapi + import gc + import sys + import weakref + + def make_frame(): + frame = sys._getframe() + assert _testcapi.py_immortalize(frame) == 1 + # LOAD_GLOBAL creates a deferred reference after immortalization. + local = Local + return frame, weakref.ref(local) + + class Local: + pass + frame, local_ref = make_frame() + del Local + gc.collect() + assert local_ref() is not None, "immortal frame lost its local" + assert frame.f_locals["local"] is local_ref() + frame.clear() + """ + assert_python_ok("-c", script) + + def test_immortalize_subinterpreters(self): + # Each interpreter owns and releases its frame registry independently. + script = """if True: + import _testcapi + import gc + + code = '''if True: + import _testcapi + import gc + import sys + + def gen(): + yield object() + + generator = gen() + next(generator) + assert _testcapi.py_immortalize(generator) == 1 + + def make_frame(): + frame = sys._getframe() + assert _testcapi.py_immortalize(frame) == 1 + return frame + + frame = make_frame() + gc.collect() + ''' + for _ in range(3): + assert _testcapi.run_in_subinterp(code) == 0 + gc.collect() + """ + assert_python_ok("-c", script) + class TestInternalCAPI(unittest.TestCase): diff --git a/Misc/NEWS.d/next/C_API/2026-10-21-22-40-30.bpo-0.imrtal.rst b/Misc/NEWS.d/next/C_API/2026-10-21-22-40-30.bpo-0.imrtal.rst new file mode 100644 index 00000000000000..5e71a56b03b07a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-10-21-22-40-30.bpo-0.imrtal.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnstable_Immortalize` to make objects immortal even if they are +already shared across threads. diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 8fcb86e33e55a8..81423b4ff34937 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -246,6 +246,16 @@ test_py_set_immortal(PyObject *self, PyObject *unused) Py_RETURN_NONE; } +static PyObject * +py_immortalize(PyObject *self, PyObject *op) +{ + int result = PyUnstable_Immortalize(op); + if (result < 0) { + return NULL; + } + return PyLong_FromLong(result); +} + #ifdef Py_REF_DEBUG static PyObject * negative_refcount(PyObject *self, PyObject *Py_UNUSED(args)) @@ -360,6 +370,7 @@ static PyMethodDef test_methods[] = { {"pyobject_is_unique_temporary_new_object", pyobject_is_unique_temporary_new_object, METH_NOARGS}, {"test_py_try_inc_ref", test_py_try_inc_ref, METH_NOARGS}, {"test_py_set_immortal", test_py_set_immortal, METH_NOARGS}, + {"py_immortalize", py_immortalize, METH_O}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, {"decref_freed_object", decref_freed_object, METH_NOARGS}, diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b3e6323fb79335..0420f6e27e4238 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4417,7 +4417,8 @@ dict_merge_api(PyObject *a, PyObject *b, int override, PyObject **dupkey) } int res = dict_merge(a, b, override, dupkey); - assert(_PyObject_GC_IS_TRACKED(a)); + // XXX: This can be untracked by PyUnstable_Immortalize() + // assert(_PyObject_GC_IS_TRACKED(a)); return res; } diff --git a/Objects/object.c b/Objects/object.c index f8206e5d3eca32..2f589bbcd604bd 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -10,6 +10,7 @@ #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // _PyObject_MaterializeManagedDict() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() +#include "pycore_frame.h" // PyFrameObject.f_frame #include "pycore_function.h" // _PyClassMethod_GetFunc() #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_genobject.h" // _PyAsyncGenAThrow_Type @@ -24,6 +25,7 @@ #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // export _Py_SwappedOp +#include "pycore_object_stack.h" // _PyObjectStack_Push() #include "pycore_optimizer.h" // _PyUOpExecutor_Type #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() @@ -35,6 +37,7 @@ #include "pycore_typevarobject.h" // _PyTypeAlias_Type #include "pycore_stackref.h" // PyStackRef_FromPyObjectSteal #include "pycore_unionobject.h" // _PyUnion_Type +#include "pycore_uniqueid.h" // _PyObject_MergePerThreadRefcounts() #ifdef Py_LIMITED_API @@ -2911,6 +2914,179 @@ PyUnstable_SetImmortal(PyObject *op) return 1; } +#ifndef Py_GIL_DISABLED +static _PyInterpreterFrame * +get_immortalized_frame(PyObject *op) +{ + if (PyGen_CheckExact(op) || PyCoro_CheckExact(op) || PyAsyncGen_CheckExact(op)) { + PyGenObject *gen = (PyGenObject *)op; + if (gen->gi_frame_state != FRAME_CLEARED) { + return &gen->gi_iframe; + } + } + else if (PyFrame_Check(op)) { + PyFrameObject *frame = (PyFrameObject *)op; + if (frame->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { + return frame->f_frame; + } + } + return NULL; +} + +static void +update_frame_ref(_PyStackRef *ref, PyObject *immortal) +{ +#ifndef Py_STACKREF_DEBUG + if (!PyStackRef_IsNullOrInt(*ref) && + PyStackRef_AsPyObjectBorrow(*ref) == immortal) + { + *ref = PyStackRef_Borrow(*ref); + } +#endif + // Py_STACKREF_DEBUG uses Py_INCREF/Py_DECREF and already supports + // immortalized objects, without rewriting its table-backed references. +} + +static void +update_frame_locals(_PyInterpreterFrame *frame, PyObject *immortal) +{ + if (frame->owner == FRAME_OWNED_BY_INTERPRETER || + frame->stackpointer == NULL) + { + return; + } + update_frame_ref(&frame->f_executable, immortal); + update_frame_ref(&frame->f_funcobj, immortal); + _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); + for (_PyStackRef *ref = locals; ref < frame->stackpointer; ref++) { + update_frame_ref(ref, immortal); + } +} + +static void +update_thread_frames(PyThreadState *tstate, PyObject *immortal) +{ + _PyInterpreterFrame *frame = _PyThreadState_GetFrame(tstate); + while (frame != NULL) { + update_frame_locals(frame, immortal); + frame = frame->previous; + } +} + +static int +update_heap_frames(PyObject *op, void *arg) +{ + _PyInterpreterFrame *frame = get_immortalized_frame(op); + if (frame != NULL) { + update_frame_locals(frame, (PyObject *)arg); + } + return 1; +} + +static void +update_frames(PyInterpreterState *interp, PyObject *immortal) +{ + _Py_FOR_EACH_TSTATE_BEGIN(interp, tstate) { + update_thread_frames(tstate, immortal); + } + _Py_FOR_EACH_TSTATE_END(interp); + PyUnstable_GC_VisitObjects(update_heap_frames, immortal); + for (_PyObjectStackChunk *chunk = interp->gc.immortal_frames.head; + chunk != NULL; chunk = chunk->prev) + { + for (Py_ssize_t i = 0; i < chunk->n; i++) { + update_heap_frames(chunk->objs[i], immortal); + } + } +} +#endif + +#ifdef Py_GIL_DISABLED +static int +has_per_thread_refcounts(PyObject *op) +{ + if (PyType_Check(op) && PyType_HasFeature((PyTypeObject *)op, Py_TPFLAGS_HEAPTYPE)) { + return ((PyHeapTypeObject *)op)->unique_id != _Py_INVALID_UNIQUE_ID; + } + if (PyCode_Check(op)) { + return ((PyCodeObject *)op)->_co_unique_id != _Py_INVALID_UNIQUE_ID; + } + if (PyDict_Check(op)) { + return _PyDict_UniqueId((PyDictObject *)op) != _Py_INVALID_UNIQUE_ID; + } + return 0; +} +#endif + +int +PyUnstable_Immortalize(PyObject *op) +{ + assert(op != NULL); + PyThreadState *tstate = _PyThreadState_GET(); + _Py_EnsureTstateNotNULL(tstate); + + if (_Py_IsImmortal(op) || PyUnicode_Check(op)) { + return 0; + } + + PyInterpreterState *interp = tstate->interp; + // Stop the world so reference counts aren't prone to races. + _PyEval_StopTheWorld(interp); + // Another thread may have immortalized it while we were waiting. + if (_Py_IsImmortal(op)) { + _PyEval_StartTheWorld(interp); + return 0; + } + + if (PyGen_CheckExact(op) || PyCoro_CheckExact(op) || PyAsyncGen_CheckExact(op) || + PyFrame_Check(op)) + { + // Keep deferred locals visible to GC, including those acquired after + // a generator resumes or a frame returns. Register before changing + // the object so allocation failure leaves it untouched. + if (_PyObjectStack_Push(&interp->gc.immortal_frames, op) < 0) { + _PyEval_StartTheWorld(interp); + PyErr_NoMemory(); + return -1; + } + } + +#ifndef Py_GIL_DISABLED + // GIL stack references can bypass the immortality check when duplicated. + update_frames(interp, op); +#endif + + if (PyObject_GC_IsTracked(op)) { + _PyObject_GC_UNTRACK(op); + } + +#ifdef Py_GIL_DISABLED + if (has_per_thread_refcounts(op)) { + // Merge every thread before releasing an id that could be reused. + _Py_FOR_EACH_TSTATE_BEGIN(interp, thread) { + _PyObject_MergePerThreadRefcounts((_PyThreadStateImpl *)thread); + } + _Py_FOR_EACH_TSTATE_END(interp); + _PyObject_DisablePerThreadRefcounting(op); + } +#endif + +#ifdef Py_REF_DEBUG + Py_ssize_t refcnt = Py_REFCNT(op); + if (_PyObject_HasDeferredRefcount(op)) { + refcnt -= _Py_REF_DEFERRED; + } + // Future decrefs of the immortal object will not affect RefTotal. + _Py_AddRefTotal(tstate, -refcnt); +#endif + _Py_SetImmortalUntracked(op); + _PyEval_StartTheWorld(interp); + _PyObject_ASSERT(op, !PyObject_GC_IsTracked(op)); + _PyObject_ASSERT(op, _Py_IsImmortal(op)); + + return 1; +} + void _Py_ResurrectReference(PyObject *op) { diff --git a/Python/gc.c b/Python/gc.c index 5dc79e20191ebc..e5351f5c6a9eba 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -12,6 +12,7 @@ #include "pycore_interpframe.h" // _PyFrame_GetLocalsArray() #include "pycore_object.h" #include "pycore_object_alloc.h" // _PyObject_MallocWithType() +#include "pycore_object_stack.h" // _PyObjectStack_Clear() #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_MaybeUntrack() @@ -1919,6 +1920,8 @@ _PyGC_Fini(PyInterpreterState *interp) finalize_unlink_gc_head(&gcstate->generations[i].head); } finalize_unlink_gc_head(&gcstate->permanent_generation.head); + // Release the registry's storage, leaving the immortal objects alive. + _PyObjectStack_Clear(&gcstate->immortal_frames); } /* for debugging */ diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index e8ed866312c94b..f8308da67cdbae 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1050,6 +1050,21 @@ mark_reachable(PyObject *op) return 0; } +static int +mark_immortal_frames(PyInterpreterState *interp) +{ + for (_PyObjectStackChunk *chunk = interp->gc.immortal_frames.head; + chunk != NULL; chunk = chunk->prev) + { + for (Py_ssize_t i = 0; i < chunk->n; i++) { + if (mark_reachable(chunk->objs[i]) < 0) { + return -1; + } + } + } + return 0; +} + #ifdef GC_DEBUG static bool validate_alive_bits(const mi_heap_t *heap, const mi_heap_area_t *area, @@ -1458,7 +1473,9 @@ deduce_unreachable_heap(PyInterpreterState *interp, // Transitively mark reachable objects by clearing the // _PyGC_BITS_UNREACHABLE flag. - if (gc_visit_heaps(interp, &mark_heap_visitor, &state->base) < 0) { + if (mark_immortal_frames(interp) < 0 || + gc_visit_heaps(interp, &mark_heap_visitor, &state->base) < 0) + { // On out-of-memory, restore the refcounts and bail out. gc_visit_heaps(interp, &restore_refs, &state->base); return -1; @@ -2671,6 +2688,8 @@ _PyGC_Fini(PyInterpreterState *interp) Py_CLEAR(gcstate->callbacks); PyMem_RawFree(gcstate->generation_stats); gcstate->generation_stats = NULL; + // Release the registry's storage, leaving the immortal objects alive. + _PyObjectStack_Clear(&gcstate->immortal_frames); /* We expect that none of this interpreters objects are shared with other interpreters.