diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index b3abe2030a03da..241af60d76c537 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -95,7 +95,7 @@ static inline void _PyGC_SET_FINALIZED(PyObject *op) { /* If we change this, we need to change the default value in the signature of gc.collect. */ -#define NUM_GENERATIONS 3 +#define NUM_GENERATIONS 1 /* NOTE: about untracking of mutable objects. @@ -165,17 +165,25 @@ struct _gc_runtime_state { int enabled; int debug; /* linked lists of container objects */ - struct gc_generation generations[NUM_GENERATIONS]; - PyGC_Head *generation0; + PyGC_Head head; /* a permanent generation which won't be collected */ - struct gc_generation permanent_generation; - struct gc_generation_stats generation_stats[NUM_GENERATIONS]; + struct gc_generation_stats stats; /* true if we are currently running the collector */ int collecting; /* list of uncollectable objects */ PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + /* the number of live GC objects */ + Py_ssize_t gc_live; + /* the threshold at which to trigger a collection */ + Py_ssize_t gc_threshold; + /* The ratio used to compute gc_threshold: + gc_threshold = (1 + gc_scale/100) * gc_live + A value of 100 means to collect every time the number of live + objects doubles. */ + int gc_scale; + /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -192,6 +200,13 @@ struct _gc_runtime_state { extern void _PyGC_InitState(struct _gc_runtime_state *); +static inline int +_PyGC_ShouldCollect(struct _gc_runtime_state *gcstate) +{ + Py_ssize_t live = gcstate->gc_live; + return !gcstate->collecting && gcstate->enabled && live >= gcstate->gc_threshold; +} + extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 8796dfe2f6b8cf..b06cd18578b1d6 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -154,12 +154,12 @@ static inline void _PyObject_GC_TRACK( filename, lineno, __func__); PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = interp->gc.generation0; - PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); + PyGC_Head *head = &interp->gc.head; + PyGC_Head *last = (PyGC_Head*)(head->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - _PyGCHead_SET_NEXT(gc, generation0); - generation0->_gc_prev = (uintptr_t)gc; + _PyGCHead_SET_NEXT(gc, head); + head->_gc_prev = (uintptr_t)gc; } /* Tell the GC to stop tracking this object. diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index cb3fce3732c79b..74558e86475df6 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -119,12 +119,6 @@ extern "C" { }, \ .gc = { \ .enabled = 1, \ - .generations = { \ - /* .head is set in _PyGC_InitState(). */ \ - { .threshold = 700, }, \ - { .threshold = 10, }, \ - { .threshold = 10, }, \ - }, \ }, \ .static_objects = { \ .singletons = { \ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 6630faa6f4471d..34dc68f53fa738 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -125,8 +125,7 @@ gc_decref(PyGC_Head *g) DEBUG_UNCOLLECTABLE | \ DEBUG_SAVEALL -#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) - +static inline void gc_list_init(PyGC_Head *list); static GCState * get_gc_state(void) @@ -139,20 +138,10 @@ get_gc_state(void) void _PyGC_InitState(GCState *gcstate) { -#define INIT_HEAD(GEN) \ - do { \ - GEN.head._gc_next = (uintptr_t)&GEN.head; \ - GEN.head._gc_prev = (uintptr_t)&GEN.head; \ - } while (0) - - for (int i = 0; i < NUM_GENERATIONS; i++) { - assert(gcstate->generations[i].count == 0); - INIT_HEAD(gcstate->generations[i]); - }; - gcstate->generation0 = GEN_HEAD(gcstate, 0); - INIT_HEAD(gcstate->permanent_generation); - -#undef INIT_HEAD + gcstate->enabled = 1; /* automatic collection enabled? */ + gcstate->gc_threshold = 7000; + gcstate->gc_scale = 100; + gc_list_init(&gcstate->head); } @@ -1029,41 +1018,6 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate, } } -/* Clear all free lists - * All free lists are cleared during the collection of the highest generation. - * Allocated items in the free list may keep a pymalloc arena occupied. - * Clearing the free lists may give back memory to the OS earlier. - */ -static void -clear_freelists(PyInterpreterState *interp) -{ - _PyTuple_ClearFreeList(interp); - _PyFloat_ClearFreeList(interp); - _PyList_ClearFreeList(interp); - _PyDict_ClearFreeList(interp); - _PyAsyncGen_ClearFreeLists(interp); - _PyContext_ClearFreeList(interp); -} - -// Show stats for objects in each generations -static void -show_stats_each_generations(GCState *gcstate) -{ - char buf[100]; - size_t pos = 0; - - for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { - pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, - " %zd", - gc_list_size(GEN_HEAD(gcstate, i))); - } - - PySys_FormatStderr( - "gc: objects in each generation:%s\n" - "gc: objects in permanent generation: %zd\n", - buf, gc_list_size(&gcstate->permanent_generation.head)); -} - /* Deduce which objects among "base" are unreachable from outside the list and move them to 'unreachable'. The process consist in the following steps: @@ -1175,6 +1129,17 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, gc_list_merge(resurrected, old_generation); } +static void +update_gc_threshold(GCState *gcstate) +{ + Py_ssize_t live = gcstate->gc_live; + Py_ssize_t threshold = live + (live * gcstate->gc_scale) / 100; + if (threshold < 7000) { + threshold = 7000; + } + gcstate->gc_threshold = threshold; +} + /* This is the main function. Read this to understand how the * collection process works. */ static Py_ssize_t @@ -1182,7 +1147,6 @@ gc_collect_main(PyThreadState *tstate, int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, int nofail) { - int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ PyGC_Head *young; /* the generation we are examining */ @@ -1199,50 +1163,28 @@ gc_collect_main(PyThreadState *tstate, int generation, assert(!_PyErr_Occurred(tstate)); if (gcstate->debug & DEBUG_STATS) { - PySys_WriteStderr("gc: collecting generation %d...\n", generation); - show_stats_each_generations(gcstate); + PySys_WriteStderr("gc: collecting heap...\n"); + PySys_FormatStderr( + "gc: tracked objects: %"PY_FORMAT_SIZE_T"d\n", + gc_list_size(&gcstate->head)); t1 = _PyTime_GetPerfCounter(); } if (PyDTrace_GC_START_ENABLED()) PyDTrace_GC_START(generation); - /* update collection and allocation counters */ - if (generation+1 < NUM_GENERATIONS) - gcstate->generations[generation+1].count += 1; - for (i = 0; i <= generation; i++) - gcstate->generations[i].count = 0; - - /* merge younger generations with one we are currently collecting */ - for (i = 0; i < generation; i++) { - gc_list_merge(GEN_HEAD(gcstate, i), GEN_HEAD(gcstate, generation)); - } - /* handy references */ - young = GEN_HEAD(gcstate, generation); - if (generation < NUM_GENERATIONS-1) - old = GEN_HEAD(gcstate, generation+1); - else - old = young; + young = old = &gcstate->head; validate_list(old, collecting_clear_unreachable_clear); deduce_unreachable(young, &unreachable); untrack_tuples(young); + untrack_dicts(young); + + gcstate->long_lived_pending = 0; + gcstate->long_lived_total = gc_list_size(young); /* Move reachable objects to next generation. */ - if (young != old) { - if (generation == NUM_GENERATIONS - 2) { - gcstate->long_lived_pending += gc_list_size(young); - } - gc_list_merge(young, old); - } - else { - /* We only un-track dicts in full collections, to avoid quadratic - dict build-up. See issue #14775. */ - untrack_dicts(young); - gcstate->long_lived_pending = 0; - gcstate->long_lived_total = gc_list_size(young); - } /* All objects in unreachable are trash, but objects reachable from * legacy finalizers (e.g. tp_del) can't safely be deleted. @@ -1310,12 +1252,6 @@ gc_collect_main(PyThreadState *tstate, int generation, handle_legacy_finalizers(tstate, gcstate, &finalizers, old); validate_list(old, collecting_clear_unreachable_clear); - /* Clear free list only during the collection of the highest - * generation */ - if (generation == NUM_GENERATIONS-1) { - clear_freelists(tstate->interp); - } - if (_PyErr_Occurred(tstate)) { if (nofail) { _PyErr_Clear(tstate); @@ -1333,11 +1269,13 @@ gc_collect_main(PyThreadState *tstate, int generation, *n_uncollectable = n; } - struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; + struct gc_generation_stats *stats = &gcstate->stats; stats->collections++; stats->collected += m; stats->uncollectable += n; + update_gc_threshold(gcstate); + if (PyDTrace_GC_DONE_ENABLED()) { PyDTrace_GC_DONE(n + m); } @@ -1351,8 +1289,7 @@ gc_collect_main(PyThreadState *tstate, int generation, */ static void invoke_gc_callback(PyThreadState *tstate, const char *phase, - int generation, Py_ssize_t collected, - Py_ssize_t uncollectable) + Py_ssize_t collected, Py_ssize_t uncollectable) { assert(!_PyErr_Occurred(tstate)); @@ -1367,7 +1304,7 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase, PyObject *info = NULL; if (PyList_GET_SIZE(gcstate->callbacks) != 0) { info = Py_BuildValue("{sisnsn}", - "generation", generation, + "generation", 0, "collected", collected, "uncollectable", uncollectable); if (info == NULL) { @@ -1399,68 +1336,13 @@ gc_collect_with_callback(PyThreadState *tstate, int generation) { assert(!_PyErr_Occurred(tstate)); Py_ssize_t result, collected, uncollectable; - invoke_gc_callback(tstate, "start", generation, 0, 0); + invoke_gc_callback(tstate, "start", 0, 0); result = gc_collect_main(tstate, generation, &collected, &uncollectable, 0); - invoke_gc_callback(tstate, "stop", generation, collected, uncollectable); + invoke_gc_callback(tstate, "stop", collected, uncollectable); assert(!_PyErr_Occurred(tstate)); return result; } -static Py_ssize_t -gc_collect_generations(PyThreadState *tstate) -{ - GCState *gcstate = &tstate->interp->gc; - /* Find the oldest generation (highest numbered) where the count - * exceeds the threshold. Objects in the that generation and - * generations younger than it will be collected. */ - Py_ssize_t n = 0; - for (int i = NUM_GENERATIONS-1; i >= 0; i--) { - if (gcstate->generations[i].count > gcstate->generations[i].threshold) { - /* Avoid quadratic performance degradation in number - of tracked objects (see also issue #4074): - - To limit the cost of garbage collection, there are two strategies; - - make each collection faster, e.g. by scanning fewer objects - - do less collections - This heuristic is about the latter strategy. - - In addition to the various configurable thresholds, we only trigger a - full collection if the ratio - - long_lived_pending / long_lived_total - - is above a given value (hardwired to 25%). - - The reason is that, while "non-full" collections (i.e., collections of - the young and middle generations) will always examine roughly the same - number of objects -- determined by the aforementioned thresholds --, - the cost of a full collection is proportional to the total number of - long-lived objects, which is virtually unbounded. - - Indeed, it has been remarked that doing a full collection every - of object creations entails a dramatic performance - degradation in workloads which consist in creating and storing lots of - long-lived objects (e.g. building a large list of GC-tracked objects would - show quadratic performance, instead of linear as expected: see issue #4074). - - Using the above ratio, instead, yields amortized linear performance in - the total number of objects (the effect of which can be summarized - thusly: "each full garbage collection is more and more costly as the - number of objects grows, but we do fewer and fewer of them"). - - This heuristic was suggested by Martin von Löwis on python-dev in - June 2008. His original analysis and proposal can be found at: - http://mail.python.org/pipermail/python-dev/2008-June/080579.html - */ - if (i == NUM_GENERATIONS - 1 - && gcstate->long_lived_pending < gcstate->long_lived_total / 4) - continue; - n = gc_collect_with_callback(tstate, i); - break; - } - } - return n; -} #include "clinic/gcmodule.c.h" @@ -1525,7 +1407,7 @@ gc_collect_impl(PyObject *module, int generation) { PyThreadState *tstate = _PyThreadState_GET(); - if (generation < 0 || generation >= NUM_GENERATIONS) { + if (generation < 0 || generation >= 3) { _PyErr_SetString(tstate, PyExc_ValueError, "invalid generation"); return -1; } @@ -1595,15 +1477,14 @@ static PyObject * gc_set_threshold(PyObject *self, PyObject *args) { GCState *gcstate = get_gc_state(); + int threshold0, threshold1, threshold2; if (!PyArg_ParseTuple(args, "i|ii:set_threshold", - &gcstate->generations[0].threshold, - &gcstate->generations[1].threshold, - &gcstate->generations[2].threshold)) + &threshold0, + &threshold1, + &threshold2)) return NULL; - for (int i = 3; i < NUM_GENERATIONS; i++) { - /* generations higher than 2 get the same threshold */ - gcstate->generations[i].threshold = gcstate->generations[2].threshold; - } + // FIXME: does setting threshold0 to zero actually disable collection ??? + gcstate->gc_threshold = threshold0; Py_RETURN_NONE; } @@ -1619,9 +1500,9 @@ gc_get_threshold_impl(PyObject *module) { GCState *gcstate = get_gc_state(); return Py_BuildValue("(iii)", - gcstate->generations[0].threshold, - gcstate->generations[1].threshold, - gcstate->generations[2].threshold); + gcstate->gc_threshold, + 0, + 0); } /*[clinic input] @@ -1635,10 +1516,8 @@ gc_get_count_impl(PyObject *module) /*[clinic end generated code: output=354012e67b16398f input=a392794a08251751]*/ { GCState *gcstate = get_gc_state(); - return Py_BuildValue("(iii)", - gcstate->generations[0].count, - gcstate->generations[1].count, - gcstate->generations[2].count); + int64_t gc_live = gcstate->gc_live; + return Py_BuildValue("(iii)", gc_live, 0, 0); } static int @@ -1687,11 +1566,9 @@ gc_get_referrers(PyObject *self, PyObject *args) } GCState *gcstate = get_gc_state(); - for (int i = 0; i < NUM_GENERATIONS; i++) { - if (!(gc_referrers_for(args, GEN_HEAD(gcstate, i), result))) { - Py_DECREF(result); - return NULL; - } + if (!(gc_referrers_for(args, &gcstate->head, result))) { + Py_DECREF(result); + return NULL; } return result; } @@ -1752,7 +1629,6 @@ gc_get_objects_impl(PyObject *module, Py_ssize_t generation) /*[clinic end generated code: output=48b35fea4ba6cb0e input=ef7da9df9806754c]*/ { PyThreadState *tstate = _PyThreadState_GET(); - int i; PyObject* result; GCState *gcstate = &tstate->interp->gc; @@ -1780,20 +1656,12 @@ gc_get_objects_impl(PyObject *module, Py_ssize_t generation) "generation parameter cannot be negative"); goto error; } - - if (append_objects(result, GEN_HEAD(gcstate, generation))) { - goto error; - } - - return result; } - /* If generation is not passed or None, get all objects from all generations */ - for (i = 0; i < NUM_GENERATIONS; i++) { - if (append_objects(result, GEN_HEAD(gcstate, i))) { - goto error; - } + if (append_objects(result, &gcstate->head)) { + goto error; } + return result; error: @@ -1811,36 +1679,31 @@ static PyObject * gc_get_stats_impl(PyObject *module) /*[clinic end generated code: output=a8ab1d8a5d26f3ab input=1ef4ed9d17b1a470]*/ { - int i; - struct gc_generation_stats stats[NUM_GENERATIONS], *st; + struct gc_generation_stats stats; + PyObject *result, *dict; /* To get consistent values despite allocations while constructing the result list, we use a snapshot of the running stats. */ GCState *gcstate = get_gc_state(); - for (i = 0; i < NUM_GENERATIONS; i++) { - stats[i] = gcstate->generation_stats[i]; - } + stats = gcstate->stats; - PyObject *result = PyList_New(0); + result = PyList_New(0); if (result == NULL) return NULL; - for (i = 0; i < NUM_GENERATIONS; i++) { - PyObject *dict; - st = &stats[i]; - dict = Py_BuildValue("{snsnsn}", - "collections", st->collections, - "collected", st->collected, - "uncollectable", st->uncollectable - ); - if (dict == NULL) - goto error; - if (PyList_Append(result, dict)) { - Py_DECREF(dict); - goto error; - } + dict = Py_BuildValue("{snsnsn}", + "collections", stats.collections, + "collected", stats.collected, + "uncollectable", stats.uncollectable + ); + if (dict == NULL) { + goto error; + } + if (PyList_Append(result, dict)) { Py_DECREF(dict); + goto error; } + Py_DECREF(result); return result; error: @@ -1906,11 +1769,6 @@ static PyObject * gc_freeze_impl(PyObject *module) /*[clinic end generated code: output=502159d9cdc4c139 input=b602b16ac5febbe5]*/ { - GCState *gcstate = get_gc_state(); - for (int i = 0; i < NUM_GENERATIONS; ++i) { - gc_list_merge(GEN_HEAD(gcstate, i), &gcstate->permanent_generation.head); - gcstate->generations[i].count = 0; - } Py_RETURN_NONE; } @@ -1926,9 +1784,6 @@ static PyObject * gc_unfreeze_impl(PyObject *module) /*[clinic end generated code: output=1c15f2043b25e169 input=2dd52b170f4cef6c]*/ { - GCState *gcstate = get_gc_state(); - gc_list_merge(&gcstate->permanent_generation.head, - GEN_HEAD(gcstate, NUM_GENERATIONS-1)); Py_RETURN_NONE; } @@ -1942,8 +1797,7 @@ static Py_ssize_t gc_get_freeze_count_impl(PyObject *module) /*[clinic end generated code: output=61cbd9f43aa032e1 input=45ffbc65cfe2a6ed]*/ { - GCState *gcstate = get_gc_state(); - return gc_list_size(&gcstate->permanent_generation.head); + return 0; } @@ -2152,41 +2006,12 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) } -static void -gc_fini_untrack(PyGC_Head *list) -{ - PyGC_Head *gc; - for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(list)) { - PyObject *op = FROM_GC(gc); - _PyObject_GC_UNTRACK(op); - // gh-92036: If a deallocator function expect the object to be tracked - // by the GC (ex: func_dealloc()), it can crash if called on an object - // which is no longer tracked by the GC. Leak one strong reference on - // purpose so the object is never deleted and its deallocator is not - // called. - Py_INCREF(op); - } -} - - void _PyGC_Fini(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); - - if (!_Py_IsMainInterpreter(interp)) { - // bpo-46070: Explicitly untrack all objects currently tracked by the - // GC. Otherwise, if an object is used later by another interpreter, - // calling PyObject_GC_UnTrack() on the object crashs if the previous - // or the next object of the PyGC_Head structure became a dangling - // pointer. - for (int i = 0; i < NUM_GENERATIONS; i++) { - PyGC_Head *gen = GEN_HEAD(gcstate, i); - gc_fini_untrack(gen); - } - } } /* for debugging */ @@ -2196,6 +2021,22 @@ _PyGC_Dump(PyGC_Head *g) _PyObject_Dump(FROM_GC(g)); } +Py_ssize_t +_PyGC_Collect(PyThreadState *tstate) +{ + Py_ssize_t n; + GCState *gcstate = &tstate->interp->gc; + if (gcstate->collecting) { + return 0; + } + + gcstate->collecting = 1; + n = gc_collect_with_callback(tstate, 0); + gcstate->collecting = 0; + + return n; +} + #ifdef Py_DEBUG static int @@ -2275,13 +2116,8 @@ _PyObject_GC_Link(PyObject *op) GCState *gcstate = &tstate->interp->gc; g->_gc_next = 0; g->_gc_prev = 0; - gcstate->generations[0].count++; /* number of allocated GC objects */ - if (gcstate->generations[0].count > gcstate->generations[0].threshold && - gcstate->enabled && - gcstate->generations[0].threshold && - !gcstate->collecting && - !_PyErr_Occurred(tstate)) - { + gcstate->gc_live++; + if (_PyGC_ShouldCollect(gcstate)) { _Py_ScheduleGC(tstate->interp); } } @@ -2291,7 +2127,7 @@ _Py_RunGC(PyThreadState *tstate) { GCState *gcstate = &tstate->interp->gc; gcstate->collecting = 1; - gc_collect_generations(tstate); + _PyGC_Collect(tstate); gcstate->collecting = 0; } @@ -2340,7 +2176,7 @@ _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) op = (PyVarObject *)gc_alloc(size, presize); if (op == NULL) { return NULL; - } +} _PyObject_InitVar(op, tp, nitems); return op; } @@ -2379,9 +2215,7 @@ PyObject_GC_Del(void *op) gc_list_remove(g); } GCState *gcstate = get_gc_state(); - if (gcstate->generations[0].count > 0) { - gcstate->generations[0].count--; - } + gcstate->gc_live--; PyObject_Free(((char *)op)-presize); }