From cb594b0ce95af4a7ea19cb9803b33baeaf8df35b Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Fri, 16 Oct 2026 20:24:12 -0700 Subject: [PATCH] [3.14] gh-159136: Avoid stop-the-world when setting function name attributes (gh-159269) Setting __name__, __qualname__, __doc__ or __module__ on a function stopped the world in the free-threaded build since gh-154826 and gh-154851, making functools.wraps() orders of magnitude slower with multiple threads. Use the per-object critical section and _PyObject_XSetRefDelayed() instead, as gen_set_name() already does. Use acquire loads in the readers and hold strong references across error reporting, tracing, and eval-frame recording that may execute Python or detach the thread. The __code__, __defaults__, __kwdefaults__ and __closure__ setters still stop the world to clear func_version. Document synchronization requirements for direct field access and the lifetime of borrowed function attributes. (cherry picked from commit 010bebbf9fd12bba1e3f40bdbddacd51a9c2bf20) Co-authored-by: Neil Schemenauer --- Doc/c-api/function.rst | 9 + Doc/c-api/reflection.rst | 6 + Include/cpython/funcobject.h | 10 + Lib/test/test_capi/test_function.py | 6 + .../test_free_threading/test_functions.py | 117 ++++ Lib/test/test_funcattrs.py | 31 + ...-10-12-20-00-00.gh-issue-159136.fnAttr.rst | 8 + Modules/_testcapi/function.c | 8 + Modules/_testinternalcapi.c | 160 ++++- Objects/funcobject.c | 120 ++-- Objects/genobject.c | 10 +- Python/ceval.c | 57 +- Python/ceval.h | 629 ++++++++++++++++++ 13 files changed, 1105 insertions(+), 66 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-20-00-00.gh-issue-159136.fnAttr.rst create mode 100644 Python/ceval.h diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 7908e4f8561aeb..7ef7ad3746874c 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -14,6 +14,10 @@ There are a few functions specific to Python functions. The C structure used for functions. + In the free-threaded build, direct access to mutable fields requires + synchronization if another thread may modify the function. See + :ref:`freethreading-extensions-howto`. + .. c:var:: PyTypeObject PyFunction_Type @@ -71,6 +75,11 @@ There are a few functions specific to Python functions. This is normally a :class:`string ` containing the module name, but can be set to any other object by Python code. + In the free-threaded build, if the attribute may be modified concurrently, + convert the borrowed reference to a strong reference with :c:func:`Py_XNewRef` + immediately, before calling any other Python API. The same applies to + :c:func:`PyFunction_GET_MODULE`. + .. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 54fd5a064aa2ac..37213dff216b56 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -96,6 +96,12 @@ Reflection Return the name of *func* if it is a function, class or instance object, else the name of *func*\s type. + For a Python function or method, the returned pointer refers to the + function's name and must not be retained across operations that may change + that name. In the free-threaded build, copy the string before detaching + the thread state or executing Python code if the name may be modified + concurrently. + .. c:function:: const char* PyEval_GetFuncDesc(PyObject *func) diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 598cd330bc9ca9..6c1f8a3abdb1cb 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -98,7 +98,12 @@ static inline PyObject* PyFunction_GET_GLOBALS(PyObject *func) { #define PyFunction_GET_GLOBALS(func) PyFunction_GET_GLOBALS(_PyObject_CAST(func)) static inline PyObject* PyFunction_GET_MODULE(PyObject *func) { +#ifdef Py_GIL_DISABLED + return (PyObject *)_Py_atomic_load_ptr_acquire( + &_PyFunction_CAST(func)->func_module); +#else return _PyFunction_CAST(func)->func_module; +#endif } #define PyFunction_GET_MODULE(func) PyFunction_GET_MODULE(_PyObject_CAST(func)) @@ -118,7 +123,12 @@ static inline PyObject* PyFunction_GET_CLOSURE(PyObject *func) { #define PyFunction_GET_CLOSURE(func) PyFunction_GET_CLOSURE(_PyObject_CAST(func)) static inline PyObject* PyFunction_GET_ANNOTATIONS(PyObject *func) { +#ifdef Py_GIL_DISABLED + return (PyObject *)_Py_atomic_load_ptr_acquire( + &_PyFunction_CAST(func)->func_annotations); +#else return _PyFunction_CAST(func)->func_annotations; +#endif } #define PyFunction_GET_ANNOTATIONS(func) PyFunction_GET_ANNOTATIONS(_PyObject_CAST(func)) diff --git a/Lib/test/test_capi/test_function.py b/Lib/test/test_capi/test_function.py index c1a278e5d4da91..9a07e3106f3a22 100644 --- a/Lib/test/test_capi/test_function.py +++ b/Lib/test/test_capi/test_function.py @@ -44,6 +44,12 @@ def some(): with self.assertRaises(SystemError): _testcapi.function_get_module(None) # not a function + self.assertEqual(_testcapi.function_get_module_macro(some), module) + value = object() + some.__module__ = value + self.assertIs(_testcapi.function_get_module(some), value) + self.assertIs(_testcapi.function_get_module_macro(some), value) + def test_function_get_defaults(self): # Test PyFunction_GetDefaults() def some( diff --git a/Lib/test/test_free_threading/test_functions.py b/Lib/test/test_free_threading/test_functions.py index 7a59638d27dc9d..1874922d424db8 100644 --- a/Lib/test/test_free_threading/test_functions.py +++ b/Lib/test/test_free_threading/test_functions.py @@ -81,6 +81,123 @@ def set_annotate(): threading_helper.run_concurrently( [set_annotations, set_annotate] * (NUM_THREADS // 2)) + def test_read_while_replacing_name_attrs(self): + # Readers load __name__, __qualname__, __doc__ and __module__ + # without holding a lock. A writer thread replacing them must not + # free a value a reader thread is about to incref. + def target(): + yield + + def writer(): + for _ in range(ITERS): + target.__name__ = random_string() + target.__qualname__ = random_string() + target.__doc__ = random_string() + target.__module__ = random_string() + + def reader(): + for _ in range(ITERS): + target.__name__ + target.__qualname__ + target.__doc__ + target.__module__ + # make_gen() copies __name__ and __qualname__. + gen = target() + gen.__name__ + gen.__qualname__ + gen.close() + + threading_helper.run_concurrently( + [writer, reader] * (NUM_THREADS // 2)) + + def test_repr_while_replacing_qualname(self): + def target(): + pass + + def writer(): + for _ in range(ITERS): + target.__qualname__ = random_string() + + def reader(): + for _ in range(ITERS): + self.assertRegex(repr(target), r"^$") + + threading_helper.run_concurrently( + [writer, reader] * (NUM_THREADS // 2)) + + def test_c_api_read_while_replacing_module(self): + from test.support import import_helper + _testcapi = import_helper.import_module('_testcapi') + + def target(): + pass + + def writer(): + for _ in range(ITERS): + target.__module__ = random_string() + target.__module__ = object() + target.__module__ = None + + def reader(): + for _ in range(ITERS): + for get_module in (_testcapi.function_get_module, + _testcapi.function_get_module_macro): + value = get_module(target) + self.assertIn(type(value), (str, object, type(None))) + + threading_helper.run_concurrently( + [writer, reader] * (NUM_THREADS // 2)) + + def test_c_api_read_while_replacing_name(self): + from test.support import import_helper + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') + + def target(): + pass + + def writer(): + for _ in range(ITERS): + # Non-ASCII names exercise lazy UTF-8 conversion. + target.__name__ = 'name_\u03bb' + random_string() + + def reader(): + for _ in range(ITERS): + name = _testlimitedcapi.eval_get_func_name(target) + self.assertIsInstance(name, str) + + threading_helper.run_concurrently( + [writer, reader] * (NUM_THREADS // 2)) + + def test_argument_errors_while_replacing_qualname(self): + def target(x, *, y): + pass + + def posonly(x, /): + pass + + calls = ( + lambda: target(), + lambda: target(1), + lambda: target(1, 2), + lambda: target(1, x=2), + lambda: target(1, extra=2), + lambda: posonly(x=1), + ) + + def writer(): + for _ in range(ITERS): + target.__qualname__ = random_string() + posonly.__qualname__ = random_string() + + def reader(): + for _ in range(ITERS): + for call in calls: + with self.assertRaises(TypeError): + call() + + threading_helper.run_concurrently( + [writer, reader] * (NUM_THREADS // 2)) + def test_call_while_replacing_defaults(self): # The eval loop reads __defaults__ and __kwdefaults__ without holding # a lock while pushing a frame. diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py index 526d10d172d9a9..02bdfce25a63fa 100644 --- a/Lib/test/test_funcattrs.py +++ b/Lib/test/test_funcattrs.py @@ -1,8 +1,10 @@ +import gc import textwrap import types import typing import unittest import warnings +import weakref from test import support @@ -50,6 +52,35 @@ class FunctionPropertiesTest(FuncAttrsTest): def test_module(self): self.assertEqual(self.b.__module__, __name__) + def test_qualname_replaced_during_argument_error(self): + class Qualname(str): + pass + + def target(x, /): + pass + + target.__qualname__ = Qualname('original_qualname') + ref = weakref.ref(target.__qualname__) + + class Keyword(str): + __hash__ = str.__hash__ + + def __eq__(self, other): + # Replacing and freeing the qualname while comparisons + # execute Python must not leave a dangling pointer. + target.__qualname__ = 'new_qualname' + gc.collect() + return True + + with self.assertRaisesRegex( + TypeError, + r"new_qualname\(\) got some positional-only arguments " + r"passed as keyword arguments: 'x'", + ): + target(**{Keyword('x'): 1}) + gc.collect() + self.assertIsNone(ref()) + def test_dir_includes_correct_attrs(self): self.b.known_attr = 7 self.assertIn('known_attr', dir(self.b), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-20-00-00.gh-issue-159136.fnAttr.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-20-00-00.gh-issue-159136.fnAttr.rst new file mode 100644 index 00000000000000..21fe270bbc1474 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-20-00-00.gh-issue-159136.fnAttr.rst @@ -0,0 +1,8 @@ +Setting ``__name__``, ``__qualname__``, ``__doc__`` or ``__module__`` on a +function no longer stops all threads in the free-threaded build. The setters +now use the per-object lock and defer freeing the old value with QSBR, the +same approach used for generator ``__name__``. This fixes a large slowdown +in multi-threaded code using :func:`functools.wraps` since 3.14.7. Readers +of these attributes now use atomic loads, and argument error reporting reads +``__qualname__`` after keyword comparisons, which may execute Python code and +replace it. Previously this could use a freed object. diff --git a/Modules/_testcapi/function.c b/Modules/_testcapi/function.c index 40767adbd3f14a..28420ce19f112e 100644 --- a/Modules/_testcapi/function.c +++ b/Modules/_testcapi/function.c @@ -38,6 +38,13 @@ function_get_module(PyObject *self, PyObject *func) } +static PyObject * +function_get_module_macro(PyObject *self, PyObject *func) +{ + return Py_XNewRef(PyFunction_GET_MODULE(func)); +} + + static PyObject * function_get_defaults(PyObject *self, PyObject *func) { @@ -134,6 +141,7 @@ static PyMethodDef test_methods[] = { {"function_get_code", function_get_code, METH_O, NULL}, {"function_get_globals", function_get_globals, METH_O, NULL}, {"function_get_module", function_get_module, METH_O, NULL}, + {"function_get_module_macro", function_get_module_macro, METH_O, NULL}, {"function_get_defaults", function_get_defaults, METH_O, NULL}, {"function_set_defaults", function_set_defaults, METH_VARARGS, NULL}, {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 4a9ef56e3fcd00..56f694ca068a22 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -670,7 +670,8 @@ record_eval(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc) assert(module != NULL); module_state *state = get_module_state(module); Py_DECREF(module); - int res = PyList_Append(state->record_list, func->func_name); + int res = PyList_Append(state->record_list, + FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_name)); if (res < 0) { return NULL; } @@ -692,6 +693,163 @@ set_eval_frame_record(PyObject *self, PyObject *list) Py_RETURN_NONE; } +<<<<<<< HEAD +||||||| parent of 010bebbf9fd (gh-159136: Avoid stop-the-world when setting function name attributes (gh-159269)) +// Defined in interpreter.c +extern PyObject* +Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag); +extern int Test_EvalFrame_Resumes, Test_EvalFrame_Loads; + +static PyObject * +get_eval_frame_stats(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *res = PyDict_New(); + if (res == NULL) { + return NULL; + } + PyObject *resumes = PyLong_FromLong(Test_EvalFrame_Resumes); + if (resumes == NULL || PyDict_SetItemString(res, "resumes", resumes) < 0) { + Py_XDECREF(resumes); + Py_DECREF(res); + return NULL; + } + Py_DECREF(resumes); + PyObject *loads = PyLong_FromLong(Test_EvalFrame_Loads); + if (loads == NULL || PyDict_SetItemString(res, "loads", loads) < 0) { + Py_XDECREF(loads); + Py_DECREF(res); + return NULL; + } + Py_DECREF(loads); + Test_EvalFrame_Resumes = Test_EvalFrame_Loads = 0; + return res; +} + +static PyObject * +record_eval_interp(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc) +{ + if (PyStackRef_FunctionCheck(f->f_funcobj)) { + PyFunctionObject *func = _PyFrame_GetFunction(f); + PyObject *module = _get_current_module(); + assert(module != NULL); + module_state *state = get_module_state(module); + Py_DECREF(module); + int res = PyList_Append(state->record_list, func->func_name); + if (res < 0) { + return NULL; + } + } + + return Test_EvalFrame(tstate, f, exc); +} + +static PyObject * +set_eval_frame_interp(PyObject *self, PyObject *args) +{ + if (PyTuple_GET_SIZE(args) == 1) { + module_state *state = get_module_state(self); + PyObject *list = PyTuple_GET_ITEM(args, 0); + if (!PyList_Check(list)) { + PyErr_SetString(PyExc_TypeError, "argument must be a list"); + return NULL; + } + Py_XSETREF(state->record_list, Py_NewRef(list)); + _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), record_eval_interp); + _PyInterpreterState_SetEvalFrameAllowSpecialization(_PyInterpreterState_GET(), 1); + } else { + _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), Test_EvalFrame); + _PyInterpreterState_SetEvalFrameAllowSpecialization(_PyInterpreterState_GET(), 1); + } + + Py_RETURN_NONE; +} + +static PyObject * +is_specialization_enabled(PyObject *self, PyObject *Py_UNUSED(args)) +{ + return PyBool_FromLong( + _PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())); +} + +======= +// Defined in interpreter.c +extern PyObject* +Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag); +extern int Test_EvalFrame_Resumes, Test_EvalFrame_Loads; + +static PyObject * +get_eval_frame_stats(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *res = PyDict_New(); + if (res == NULL) { + return NULL; + } + PyObject *resumes = PyLong_FromLong(Test_EvalFrame_Resumes); + if (resumes == NULL || PyDict_SetItemString(res, "resumes", resumes) < 0) { + Py_XDECREF(resumes); + Py_DECREF(res); + return NULL; + } + Py_DECREF(resumes); + PyObject *loads = PyLong_FromLong(Test_EvalFrame_Loads); + if (loads == NULL || PyDict_SetItemString(res, "loads", loads) < 0) { + Py_XDECREF(loads); + Py_DECREF(res); + return NULL; + } + Py_DECREF(loads); + Test_EvalFrame_Resumes = Test_EvalFrame_Loads = 0; + return res; +} + +static PyObject * +record_eval_interp(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc) +{ + if (PyStackRef_FunctionCheck(f->f_funcobj)) { + PyFunctionObject *func = _PyFrame_GetFunction(f); + PyObject *module = _get_current_module(); + assert(module != NULL); + module_state *state = get_module_state(module); + Py_DECREF(module); + int res = PyList_Append(state->record_list, + FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_name)); + if (res < 0) { + return NULL; + } + } + + return Test_EvalFrame(tstate, f, exc); +} + +static PyObject * +set_eval_frame_interp(PyObject *self, PyObject *args) +{ + if (PyTuple_GET_SIZE(args) == 1) { + module_state *state = get_module_state(self); + PyObject *list = PyTuple_GET_ITEM(args, 0); + if (!PyList_Check(list)) { + PyErr_SetString(PyExc_TypeError, "argument must be a list"); + return NULL; + } + Py_XSETREF(state->record_list, Py_NewRef(list)); + _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), record_eval_interp); + _PyInterpreterState_SetEvalFrameAllowSpecialization(_PyInterpreterState_GET(), 1); + } else { + _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), Test_EvalFrame); + _PyInterpreterState_SetEvalFrameAllowSpecialization(_PyInterpreterState_GET(), 1); + } + + Py_RETURN_NONE; +} + +static PyObject * +is_specialization_enabled(PyObject *self, PyObject *Py_UNUSED(args)) +{ + return PyBool_FromLong( + _PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())); +} + +>>>>>>> 010bebbf9fd (gh-159136: Avoid stop-the-world when setting function name attributes (gh-159269)) /*[clinic input] _testinternalcapi.compiler_cleandoc -> object diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 412c7ec4d0a66e..9b94e63deb651f 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -40,7 +40,8 @@ notify_func_watchers(PyInterpreterState *interp, PyFunction_WatchEvent event, if (cb(event, func, new_value) < 0) { PyErr_FormatUnraisable( "Exception ignored in %s watcher callback for function %U at %p", - func_event_name(event), func->func_qualname, func); + func_event_name(event), + FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_qualname), func); } } i++; @@ -422,7 +423,7 @@ PyFunction_GetModule(PyObject *op) PyErr_BadInternalCall(); return NULL; } - return ((PyFunctionObject *) op) -> func_module; + return PyFunction_GET_MODULE(op); } PyObject * @@ -552,9 +553,29 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure) return 0; } +static void +func_set_annotations_and_annotate(PyFunctionObject *op, PyObject *annotations, + PyObject *annotate) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + // Care is needed, finalizers can run when references are dropped. Store + // both new values before dropping references to the old. + PyObject *old_annotations = op->func_annotations; + PyObject *old_annotate = op->func_annotate; + FT_ATOMIC_STORE_PTR_RELEASE(op->func_annotations, annotations); + FT_ATOMIC_STORE_PTR_RELEASE(op->func_annotate, annotate); + if (old_annotations != NULL && !_Py_IsImmortal(old_annotations)) { + _PyObject_XDecRefDelayed(old_annotations); + } + if (old_annotate != NULL && !_Py_IsImmortal(old_annotate)) { + _PyObject_XDecRefDelayed(old_annotate); + } +} + static PyObject * func_get_annotation_dict(PyFunctionObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); if (op->func_annotations == NULL) { if (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate)) { Py_RETURN_NONE; @@ -570,7 +591,7 @@ func_get_annotation_dict(PyFunctionObject *op) Py_DECREF(ann_dict); return NULL; } - Py_XSETREF(op->func_annotations, ann_dict); + _PyObject_XSetRefDelayed(&op->func_annotations, ann_dict); return ann_dict; } if (PyTuple_CheckExact(op->func_annotations)) { @@ -592,7 +613,7 @@ func_get_annotation_dict(PyFunctionObject *op) return NULL; } } - Py_SETREF(op->func_annotations, ann_dict); + _PyObject_XSetRefDelayed(&op->func_annotations, ann_dict); } assert(PyDict_Check(op->func_annotations)); return op->func_annotations; @@ -605,7 +626,11 @@ PyFunction_GetAnnotations(PyObject *op) PyErr_BadInternalCall(); return NULL; } - return func_get_annotation_dict((PyFunctionObject *)op); + PyObject *d; + Py_BEGIN_CRITICAL_SECTION(op); + d = func_get_annotation_dict((PyFunctionObject *)op); + Py_END_CRITICAL_SECTION(); + return d; } int @@ -626,15 +651,9 @@ PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) return -1; } PyFunctionObject *func = (PyFunctionObject *)op; - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); - PyObject *old_annotations = func->func_annotations; - func->func_annotations = annotations; - PyObject *old_annotate = func->func_annotate; - func->func_annotate = NULL; - _PyEval_StartTheWorld(interp); - Py_XDECREF(old_annotations); - Py_XDECREF(old_annotate); + Py_BEGIN_CRITICAL_SECTION(func); + func_set_annotations_and_annotate(func, annotations, NULL); + Py_END_CRITICAL_SECTION(); return 0; } @@ -689,11 +708,11 @@ func_set_code(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) Py_ssize_t nclosure = (op->func_closure == NULL ? 0 : PyTuple_GET_SIZE(op->func_closure)); if (nclosure != nfree) { + PyObject *name = FT_ATOMIC_LOAD_PTR_ACQUIRE(op->func_name); PyErr_Format(PyExc_ValueError, "%U() requires a code object with %zd free vars," " not %d", - op->func_name, - nclosure, nfree); + name, nclosure, nfree); return -1; } @@ -725,7 +744,8 @@ static PyObject * func_get_name(PyObject *self, void *Py_UNUSED(ignored)) { PyFunctionObject *op = _PyFunction_CAST(self); - return Py_NewRef(op->func_name); + PyObject *name = FT_ATOMIC_LOAD_PTR_ACQUIRE(op->func_name); + return Py_NewRef(name); } static int @@ -739,12 +759,9 @@ func_set_name(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) "__name__ must be set to a string object"); return -1; } - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); - PyObject *old_name = op->func_name; - op->func_name = Py_NewRef(value); - _PyEval_StartTheWorld(interp); - Py_XDECREF(old_name); + Py_BEGIN_CRITICAL_SECTION(op); + _PyObject_XSetRefDelayed(&op->func_name, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } @@ -752,7 +769,8 @@ static PyObject * func_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { PyFunctionObject *op = _PyFunction_CAST(self); - return Py_NewRef(op->func_qualname); + PyObject *qualname = FT_ATOMIC_LOAD_PTR_ACQUIRE(op->func_qualname); + return Py_NewRef(qualname); } static int @@ -766,12 +784,27 @@ func_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) "__qualname__ must be set to a string object"); return -1; } +<<<<<<< HEAD PyInterpreterState *interp = _PyInterpreterState_GET(); _PyEval_StopTheWorld(interp); PyObject *old_qualname = op->func_qualname; op->func_qualname = Py_NewRef(value); _PyEval_StartTheWorld(interp); Py_XDECREF(old_qualname); +||||||| parent of 010bebbf9fd (gh-159136: Avoid stop-the-world when setting function name attributes (gh-159269)) + handle_func_event(PyFunction_EVENT_MODIFY_QUALNAME, (PyFunctionObject *) op, value); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + PyObject *old_qualname = op->func_qualname; + op->func_qualname = Py_NewRef(value); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_qualname); +======= + handle_func_event(PyFunction_EVENT_MODIFY_QUALNAME, (PyFunctionObject *) op, value); + Py_BEGIN_CRITICAL_SECTION(op); + _PyObject_XSetRefDelayed(&op->func_qualname, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); +>>>>>>> 010bebbf9fd (gh-159136: Avoid stop-the-world when setting function name attributes (gh-159269)) return 0; } @@ -779,7 +812,7 @@ static PyObject * func_get_doc(PyObject *self, void *Py_UNUSED(ignored)) { PyFunctionObject *op = _PyFunction_CAST(self); - PyObject *doc = op->func_doc; + PyObject *doc = FT_ATOMIC_LOAD_PTR_ACQUIRE(op->func_doc); if (doc == NULL) { doc = Py_None; } @@ -791,12 +824,9 @@ func_set_doc(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { /* Legal to del f.__doc__ or to set it to any object. */ PyFunctionObject *op = _PyFunction_CAST(self); - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); - PyObject *old_doc = op->func_doc; - op->func_doc = Py_XNewRef(value); - _PyEval_StartTheWorld(interp); - Py_XDECREF(old_doc); + Py_BEGIN_CRITICAL_SECTION(op); + _PyObject_XSetRefDelayed(&op->func_doc, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } @@ -804,7 +834,7 @@ static PyObject * func_get_module(PyObject *self, void *Py_UNUSED(ignored)) { PyFunctionObject *op = _PyFunction_CAST(self); - PyObject *module = op->func_module; + PyObject *module = FT_ATOMIC_LOAD_PTR_ACQUIRE(op->func_module); if (module == NULL) { module = Py_None; } @@ -816,12 +846,9 @@ func_set_module(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { /* Legal to del f.__module__ or to set it to any object. */ PyFunctionObject *op = _PyFunction_CAST(self); - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); - PyObject *old_module = op->func_module; - op->func_module = Py_XNewRef(value); - _PyEval_StartTheWorld(interp); - Py_XDECREF(old_module); + Py_BEGIN_CRITICAL_SECTION(op); + _PyObject_XSetRefDelayed(&op->func_module, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } @@ -955,12 +982,11 @@ function___annotate___set_impl(PyFunctionObject *self, PyObject *value) return -1; } if (Py_IsNone(value)) { - Py_XSETREF(self->func_annotate, Py_NewRef(value)); + _PyObject_XSetRefDelayed(&self->func_annotate, Py_NewRef(value)); return 0; } else if (PyCallable_Check(value)) { - Py_XSETREF(self->func_annotate, Py_NewRef(value)); - Py_CLEAR(self->func_annotations); + func_set_annotations_and_annotate(self, NULL, Py_NewRef(value)); return 0; } else { @@ -985,9 +1011,10 @@ function___annotations___get_impl(PyFunctionObject *self) PyObject *d = NULL; if (self->func_annotations == NULL && (self->func_annotate == NULL || !PyCallable_Check(self->func_annotate))) { - self->func_annotations = PyDict_New(); - if (self->func_annotations == NULL) + PyObject *ann_dict = PyDict_New(); + if (ann_dict == NULL) return NULL; + FT_ATOMIC_STORE_PTR_RELEASE(self->func_annotations, ann_dict); } d = func_get_annotation_dict(self); return Py_XNewRef(d); @@ -1014,8 +1041,7 @@ function___annotations___set_impl(PyFunctionObject *self, PyObject *value) "__annotations__ must be set to a dict object"); return -1; } - Py_XSETREF(self->func_annotations, Py_XNewRef(value)); - Py_CLEAR(self->func_annotate); + func_set_annotations_and_annotate(self, Py_XNewRef(value), NULL); return 0; } @@ -1252,8 +1278,8 @@ static PyObject* func_repr(PyObject *self) { PyFunctionObject *op = _PyFunction_CAST(self); - return PyUnicode_FromFormat("", - op->func_qualname, op); + PyObject *qualname = FT_ATOMIC_LOAD_PTR_ACQUIRE(op->func_qualname); + return PyUnicode_FromFormat("", qualname, op); } static int diff --git a/Objects/genobject.c b/Objects/genobject.c index bb2b8fdf1598c8..96fed33358be3f 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -926,10 +926,12 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) gen->gi_exc_state.exc_value = NULL; gen->gi_exc_state.previous_item = NULL; gen->gi_iframe.f_executable = PyStackRef_None; - assert(func->func_name != NULL); - gen->gi_name = Py_NewRef(func->func_name); - assert(func->func_qualname != NULL); - gen->gi_qualname = Py_NewRef(func->func_qualname); + PyObject *name = FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_name); + assert(name != NULL); + gen->gi_name = Py_NewRef(name); + PyObject *qualname = FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_qualname); + assert(qualname != NULL); + gen->gi_qualname = Py_NewRef(qualname); _PyObject_GC_TRACK(gen); return (PyObject *)gen; } diff --git a/Python/ceval.c b/Python/ceval.c index eca281e37cc7f6..85232029baf0d3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1551,7 +1551,7 @@ too_many_positional(PyThreadState *tstate, PyCodeObject *co, static int positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, Py_ssize_t kwcount, PyObject* kwnames, - PyObject *qualname) + PyFunctionObject *func) { int posonly_conflicts = 0; PyObject* posonly_names = PyList_New(0); @@ -1598,7 +1598,8 @@ positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, _PyErr_Format(tstate, PyExc_TypeError, "%U() got some positional-only arguments passed" " as keyword arguments: '%U'", - qualname, error_names); + FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_qualname), + error_names); Py_DECREF(error_names); goto fail; } @@ -1757,8 +1758,8 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (keyword == NULL || !PyUnicode_Check(keyword)) { _PyErr_Format(tstate, PyExc_TypeError, - "%U() keywords must be strings", - func->func_qualname); + "%U() keywords must be strings", + FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_qualname)); goto kw_fail; } @@ -1790,7 +1791,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (co->co_posonlyargcount && positional_only_passed_as_keyword(tstate, co, kwcount, kwnames, - func->func_qualname)) + func)) { goto kw_fail; } @@ -1811,15 +1812,16 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, } } + PyObject *qualname = FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_qualname); if (suggestion_keyword) { _PyErr_Format(tstate, PyExc_TypeError, "%U() got an unexpected keyword argument '%S'. Did you mean '%S'?", - func->func_qualname, keyword, suggestion_keyword); + qualname, keyword, suggestion_keyword); Py_DECREF(suggestion_keyword); } else { _PyErr_Format(tstate, PyExc_TypeError, "%U() got an unexpected keyword argument '%S'", - func->func_qualname, keyword); + qualname, keyword); } goto kw_fail; @@ -1840,8 +1842,9 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, kw_found: if (PyStackRef_AsPyObjectBorrow(localsplus[j]) != NULL) { _PyErr_Format(tstate, PyExc_TypeError, - "%U() got multiple values for argument '%S'", - func->func_qualname, keyword); + "%U() got multiple values for argument '%S'", + FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_qualname), + keyword); goto kw_fail; } localsplus[j] = value_stackref; @@ -1851,7 +1854,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, /* Check the number of positional arguments */ if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) { too_many_positional(tstate, co, argcount, func->func_defaults, localsplus, - func->func_qualname); + FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_qualname)); goto fail_post_args; } @@ -1867,7 +1870,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, } if (missing) { missing_arguments(tstate, co, missing, defcount, localsplus, - func->func_qualname); + FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_qualname)); goto fail_post_args; } if (n > m) @@ -1906,7 +1909,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, } if (missing) { missing_arguments(tstate, co, missing, -1, localsplus, - func->func_qualname); + FT_ATOMIC_LOAD_PTR_ACQUIRE(func->func_qualname)); goto fail_post_args; } } @@ -3042,14 +3045,40 @@ PyEval_MergeCompilerFlags(PyCompilerFlags *cf) return result; } +static const char * +functionobject_get_name(PyFunctionObject *f) +{ +#ifdef Py_GIL_DISABLED + for (;;) { + PyObject *name = Py_NewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_name)); + // UTF-8 conversion may detach while acquiring the string's lock. + // This could result in func_name being freed if func_name was + // recently replaced using _PyObject_XSetRefDelayed(). The returned + // char* is only valid until the next quiescent point. + const char *result = PyUnicode_AsUTF8(name); + if (result == NULL) { + Py_DECREF(name); + return NULL; + } + int unchanged = name == FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_name); + Py_DECREF(name); + if (unchanged) { + return result; + } + } +#else + return PyUnicode_AsUTF8(f->func_name); +#endif +} const char * PyEval_GetFuncName(PyObject *func) { if (PyMethod_Check(func)) return PyEval_GetFuncName(PyMethod_GET_FUNCTION(func)); - else if (PyFunction_Check(func)) - return PyUnicode_AsUTF8(((PyFunctionObject*)func)->func_name); + else if (PyFunction_Check(func)) { + return functionobject_get_name((PyFunctionObject *)func); + } else if (PyCFunction_Check(func)) return ((PyCFunctionObject*)func)->m_ml->ml_name; else diff --git a/Python/ceval.h b/Python/ceval.h new file mode 100644 index 00000000000000..00db67265a4f3a --- /dev/null +++ b/Python/ceval.h @@ -0,0 +1,629 @@ +#define _PY_INTERPRETER + +#include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _PySys_Audit() +#include "pycore_backoff.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_ceval.h" // SPECIAL___ENTER__ +#include "pycore_code.h" +#include "pycore_dict.h" +#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS +#include "pycore_floatobject.h" // _PyFloat_ExactDealloc() +#include "pycore_frame.h" +#include "pycore_function.h" +#include "pycore_genobject.h" // _PyCoro_GetAwaitableIter() +#include "pycore_import.h" // _PyImport_IsDefaultImportFunc() +#include "pycore_instruments.h" +#include "pycore_interpframe.h" // _PyFrame_SetStackPointer() +#include "pycore_interpolation.h" // _PyInterpolation_Build() +#include "pycore_intrinsics.h" +#include "pycore_jit.h" +#include "pycore_lazyimportobject.h" +#include "pycore_list.h" // _PyList_GetItemRef() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_opcode_metadata.h" // EXTRA_CASES +#include "pycore_opcode_utils.h" // MAKE_FUNCTION_* +#include "pycore_optimizer.h" // _PyUOpExecutor_Type +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_range.h" // _PyRangeIterObject +#include "pycore_setobject.h" // _PySet_Update() +#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs +#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() +#include "pycore_template.h" // _PyTemplate_Build() +#include "pycore_traceback.h" // _PyTraceBack_FromFrame +#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_uop_ids.h" // Uops + +#include "dictobject.h" +#include "frameobject.h" // _PyInterpreterFrame_GetLine +#include "opcode.h" +#include "pydtrace.h" +#include "setobject.h" +#include "pycore_stackref.h" + +#include // bool + +#if !defined(Py_BUILD_CORE) +# error "ceval.c must be build with Py_BUILD_CORE define for best performance" +#endif + +#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) +// GH-89279: The MSVC compiler does not inline these static inline functions +// in PGO build in _PyEval_EvalFrameDefault(), because this function is over +// the limit of PGO, and that limit cannot be configured. +// Define them as macros to make sure that they are always inlined by the +// preprocessor. + +#undef Py_IS_TYPE +#define Py_IS_TYPE(ob, type) \ + (_PyObject_CAST(ob)->ob_type == (type)) + +#undef Py_XDECREF +#define Py_XDECREF(arg) \ + do { \ + PyObject *xop = _PyObject_CAST(arg); \ + if (xop != NULL) { \ + Py_DECREF(xop); \ + } \ + } while (0) + +#ifndef Py_GIL_DISABLED + +#undef Py_DECREF +#define Py_DECREF(arg) \ + do { \ + PyObject *op = _PyObject_CAST(arg); \ + if (_Py_IsImmortal(op)) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ + if (--op->ob_refcnt == 0) { \ + _PyReftracerTrack(op, PyRefTracer_DESTROY); \ + destructor dealloc = Py_TYPE(op)->tp_dealloc; \ + (*dealloc)(op); \ + } \ + } while (0) + +#undef _Py_DECREF_SPECIALIZED +#define _Py_DECREF_SPECIALIZED(arg, dealloc) \ + do { \ + PyObject *op = _PyObject_CAST(arg); \ + if (_Py_IsImmortal(op)) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ + if (--op->ob_refcnt == 0) { \ + _PyReftracerTrack(op, PyRefTracer_DESTROY); \ + destructor d = (destructor)(dealloc); \ + d(op); \ + } \ + } while (0) + +#else // Py_GIL_DISABLED + +#undef Py_DECREF +#define Py_DECREF(arg) \ + do { \ + PyObject *op = _PyObject_CAST(arg); \ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \ + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ + if (_Py_IsOwnedByCurrentThread(op)) { \ + local--; \ + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \ + if (local == 0) { \ + _Py_MergeZeroLocalRefcount(op); \ + } \ + } \ + else { \ + _Py_DecRefShared(op); \ + } \ + } while (0) + +#undef _Py_DECREF_SPECIALIZED +#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg) + +#endif +#endif + +static void +check_invalid_reentrancy(void) +{ +#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) + // In the free-threaded build, the interpreter must not be re-entered if + // the world-is-stopped. If so, that's a bug somewhere (quite likely in + // the painfully complex typeobject code). + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!interp->stoptheworld.world_stopped); +#endif +} + + +#ifdef Py_DEBUG +static void +dump_item(_PyStackRef item) +{ + if (PyStackRef_IsNull(item)) { + printf(""); + return; + } + if (PyStackRef_IsMalformed(item)) { + printf(""); + return; + } + if (PyStackRef_IsTaggedInt(item)) { + printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item)); + return; + } + PyObject *obj = PyStackRef_AsPyObjectBorrow(item); + if (obj == NULL) { + printf(""); + return; + } + // Don't call __repr__(), it might recurse into the interpreter. + printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); +} + +static void +dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) +{ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame); + _PyStackRef *stack_base = _PyFrame_Stackbase(frame); + PyObject *exc = PyErr_GetRaisedException(); + printf(" locals=["); + for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) { + if (ptr != locals_base) { + printf(", "); + } + dump_item(*ptr); + } + printf("]\n"); + if (stack_pointer < stack_base) { + printf(" stack=%d\n", (int)(stack_pointer-stack_base)); + } + else { + printf(" stack=["); + for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) { + if (ptr != stack_base) { + printf(", "); + } + dump_item(*ptr); + } + printf("]\n"); + } + fflush(stdout); + PyErr_SetRaisedException(exc); + _PyFrame_GetStackPointer(frame); +} + +#if defined(_Py_TIER2) && !defined(_Py_JIT) && defined(Py_DEBUG) +static void +dump_cache_item(_PyStackRef cache, int position, int depth) +{ + if (position < depth) { + dump_item(cache); + } + else { + printf("---"); + } +} +#endif + +static void +lltrace_instruction(_PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, + _Py_CODEUNIT *next_instr, + int opcode, + int oparg) +{ + int offset = 0; + if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + dump_stack(frame, stack_pointer); + offset = (int)(next_instr - _PyFrame_GetBytecode(frame)); + } + const char *opname = _PyOpcode_OpName[opcode]; + assert(opname != NULL); + if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) { + printf("%d: %s %d\n", offset * 2, opname, oparg); + } + else { + printf("%d: %s\n", offset * 2, opname); + } + fflush(stdout); +} + +static void +lltrace_resume_frame(_PyInterpreterFrame *frame) +{ + PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (!PyStackRef_CodeCheck(frame->f_executable) || + fobj == NULL || + !PyFunction_Check(fobj) + ) { + printf("\nResuming frame.\n"); + return; + } + PyFunctionObject *f = (PyFunctionObject *)fobj; + PyObject *exc = PyErr_GetRaisedException(); + PyObject *name = Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_qualname)); + if (name == NULL) { + name = Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_name)); + } + printf("\nResuming frame"); + if (name) { + printf(" for "); + if (PyObject_Print(name, stdout, 0) < 0) { + PyErr_Clear(); + } + Py_DECREF(name); + } + PyObject *module = Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_module)); + if (module) { + printf(" in module "); + if (PyObject_Print(module, stdout, 0) < 0) { + PyErr_Clear(); + } + Py_DECREF(module); + } + printf("\n"); + fflush(stdout); + PyErr_SetRaisedException(exc); +} + +static int +maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) +{ + if (globals == NULL) { + return 0; + } + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + return 0; + } + int r = PyDict_Contains(globals, &_Py_ID(__lltrace__)); + if (r < 0) { + PyErr_Clear(); + return 0; + } + int lltrace = r * 5; // Levels 1-4 only trace uops + if (!lltrace) { + // Can also be controlled by environment variable + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } + } + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } + return lltrace; +} + +#endif + +static void monitor_reraise(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static int monitor_stop_iteration(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, + PyObject *value); +static void monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static int monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc); +static void monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); + +static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); + +#ifdef HAVE_ERRNO_H +#include +#endif + +typedef struct { + _PyInterpreterFrame frame; + _PyStackRef stack[1]; +} _PyEntryFrame; + +static int +do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, int event) +{ + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); + if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { + return 0; + } + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); + if (err == 0) { + PyErr_SetRaisedException(exc); + } + else { + assert(PyErr_Occurred()); + Py_DECREF(exc); + } + return err; +} + +static inline bool +no_tools_for_global_event(PyThreadState *tstate, int event) +{ + return tstate->interp->monitors.tools[event] == 0; +} + +static inline bool +no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) +{ + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); + _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; + if (data) { + return data->active_monitors.tools[event] == 0; + } + else { + return no_tools_for_global_event(tstate, event); + } +} + +static int +monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + return 0; + } + return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); +} + +static void +monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); +} + +static void +monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); +} + +static int +monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *value) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { + return 0; + } + assert(!PyErr_Occurred()); + PyErr_SetObject(PyExc_StopIteration, value); + int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); + if (res < 0) { + return res; + } + PyErr_SetRaisedException(NULL); + return 0; +} + +static void +monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); +} + +static inline unsigned char * +scan_back_to_entry_start(unsigned char *p) { + for (; (p[0]&128) == 0; p--); + return p; +} + +static inline unsigned char * +skip_to_next_entry(unsigned char *p, unsigned char *end) { + while (p < end && ((p[0] & 128) == 0)) { + p++; + } + return p; +} + + +#define MAX_LINEAR_SEARCH 40 + +static Py_NO_INLINE int +get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti) +{ + unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); + unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable); + /* Invariants: + * start_table == end_table OR + * start_table points to a legal entry and end_table points + * beyond the table or to a legal entry that is after index. + */ + if (end - start > MAX_LINEAR_SEARCH) { + int offset; + parse_varint(start, &offset); + if (offset > index) { + return 0; + } + do { + unsigned char * mid = start + ((end-start)>>1); + mid = scan_back_to_entry_start(mid); + parse_varint(mid, &offset); + if (offset > index) { + end = mid; + } + else { + start = mid; + } + + } while (end - start > MAX_LINEAR_SEARCH); + } + unsigned char *scan = start; + while (scan < end) { + int start_offset, size; + scan = parse_varint(scan, &start_offset); + if (start_offset > index) { + break; + } + scan = parse_varint(scan, &size); + if (start_offset + size > index) { + scan = parse_varint(scan, handler); + int depth_and_lasti; + parse_varint(scan, &depth_and_lasti); + *level = depth_and_lasti >> 1; + *lasti = depth_and_lasti & 1; + return 1; + } + scan = skip_to_next_entry(scan, end); + } + return 0; +} + + +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + +/* Logic for the raise statement (too complicated for inlining). + This *consumes* a reference count to each of its arguments. */ +static int +do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) +{ + PyObject *type = NULL, *value = NULL; + + if (exc == NULL) { + /* Reraise */ + _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate); + exc = exc_info->exc_value; + if (Py_IsNone(exc) || exc == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "No active exception to reraise"); + return 0; + } + Py_INCREF(exc); + assert(PyExceptionInstance_Check(exc)); + _PyErr_SetRaisedException(tstate, exc); + return 1; + } + + /* We support the following forms of raise: + raise + raise + raise */ + + if (PyExceptionClass_Check(exc)) { + type = exc; + value = _PyObject_CallNoArgs(exc); + if (value == NULL) + goto raise_error; + if (!PyExceptionInstance_Check(value)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto raise_error; + } + } + else if (PyExceptionInstance_Check(exc)) { + value = exc; + type = PyExceptionInstance_Class(exc); + Py_INCREF(type); + } + else { + /* Not something you can raise. You get an exception + anyway, just not what you specified :-) */ + Py_DECREF(exc); + _PyErr_SetString(tstate, PyExc_TypeError, + "exceptions must derive from BaseException"); + goto raise_error; + } + + assert(type != NULL); + assert(value != NULL); + + if (cause) { + PyObject *fixed_cause; + if (PyExceptionClass_Check(cause)) { + fixed_cause = _PyObject_CallNoArgs(cause); + if (fixed_cause == NULL) + goto raise_error; + if (!PyExceptionInstance_Check(fixed_cause)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + cause, Py_TYPE(fixed_cause)); + Py_DECREF(fixed_cause); + goto raise_error; + } + Py_DECREF(cause); + } + else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + } + else if (Py_IsNone(cause)) { + Py_DECREF(cause); + fixed_cause = NULL; + } + else { + _PyErr_SetString(tstate, PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto raise_error; + } + PyException_SetCause(value, fixed_cause); + } + + _PyErr_SetObject(tstate, type, value); + /* _PyErr_SetObject incref's its arguments */ + Py_DECREF(value); + Py_DECREF(type); + return 0; + +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(cause); + return 0; +} + +/* Disable unused label warnings. They are handy for debugging, even + if computed gotos aren't used. */ + +/* TBD - what about other compilers? */ +#if defined(__GNUC__) || defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-label" +#elif defined(_MSC_VER) /* MS_WINDOWS */ +# pragma warning(push) +# pragma warning(disable:4102) +#endif