diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 18227c9e36925d2..11d32588f6f6592 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -57,9 +57,6 @@ PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le); PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le); -PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); - - /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter( diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 64ac3abe00fa01c..f1e116f51eaaf7d 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -12,6 +12,7 @@ extern "C" { #include "pycore_ast_state.h" // struct ast_state #include "pycore_gil.h" // struct _gil_runtime_state #include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_pymem.h" // free lists #include "pycore_warnings.h" // struct _warnings_runtime_state struct _pending_calls { @@ -86,8 +87,8 @@ struct _Py_unicode_state { }; #ifndef WITH_FREELISTS +# define WITH_FREELISTS 0 // without freelists -# define PyFloat_MAXFREELIST 0 // for tuples only store empty tuple singleton # define PyTuple_MAXSAVESIZE 1 # define PyTuple_MAXFREELIST 1 @@ -98,20 +99,6 @@ struct _Py_unicode_state { # define PyContext_MAXFREELIST 0 #endif -#ifndef PyFloat_MAXFREELIST -# define PyFloat_MAXFREELIST 100 -#endif - -struct _Py_float_state { -#if PyFloat_MAXFREELIST > 0 - /* Special free list - free_list is a singly-linked list of available PyFloatObjects, - linked via abuse of their ob_type members. */ - int numfree; - PyFloatObject *free_list; -#endif -}; - /* Speed optimization to avoid frequent malloc/free of small tuples */ #ifndef PyTuple_MAXSAVESIZE // Largest tuple to save on free list @@ -271,6 +258,10 @@ struct _is { int finalizing; +#if WITH_FREELISTS + _PyFreeList small_object_freelist; +#endif + struct _ceval_state ceval; struct _gc_runtime_state gc; @@ -336,7 +327,6 @@ struct _is { PyLongObject* small_ints[_PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS]; struct _Py_bytes_state bytes; struct _Py_unicode_state unicode; - struct _Py_float_state float_state; /* Using a cache is very effective since typically only a single slice is created and then deleted again. */ PySliceObject *slice_cache; @@ -348,7 +338,6 @@ struct _is { struct _Py_async_gen_state async_gen; struct _Py_context_state context; struct _Py_exc_state exc_state; - struct ast_state ast; struct type_cache type_cache; }; diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index d70deee710e30d4..e19077be2906f52 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -100,6 +100,52 @@ void *_PyObject_VirtualAlloc(size_t size); void _PyObject_VirtualFree(void *, size_t size); +#if WITH_FREELISTS +/* Free lists. + * + * Free lists have a pointer to their first entry and + * the amount of space available allowing fast checks + * for emptiness and fullness. + * When empty they are half filled and when full they are + * completely emptied. This helps the underlying allocator + * avoid fragmentation and helps performance. + */ + +typedef struct _freelist { + void *ptr; + uint32_t space; + uint16_t size; + uint16_t capacity; +} _PyFreeList; + +extern void *_PyFreeList_HalfFillAndAllocate(_PyFreeList *list); +extern void _PyFreeList_FreeToFull(_PyFreeList *list, void *ptr); + +static inline void * +_PyFreeList_Alloc(_PyFreeList *list) { + if (list->ptr != NULL) { + void *result = list->ptr; + list->ptr = *((void **)result); + list->space++; + return result; + } + return _PyFreeList_HalfFillAndAllocate(list); +} + +static inline void +_PyFreeList_Free(_PyFreeList *list, void *ptr) { + if (list->space) { + *((void **)ptr) = list->ptr; + list->ptr = ptr; + list->space--; + return; + } + _PyFreeList_FreeToFull(list, ptr); +} + +extern void _PyFreeList_Clear(_PyFreeList *list); +#endif + #ifdef __cplusplus } #endif diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-22-17-23-15.bpo-45575._oOrzd.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-22-17-23-15.bpo-45575._oOrzd.rst new file mode 100644 index 000000000000000..25fc0e777df9f62 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-22-17-23-15.bpo-45575._oOrzd.rst @@ -0,0 +1 @@ +Use a common free list for both ints and floats. diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 7fc192e7201171d..73f04467139d475 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -10,6 +10,7 @@ #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() #include "pycore_pymath.h" // _Py_ADJUST_ERANGE1() +#include "pycore_pymem.h" // Free lists #include "pycore_pystate.h" // _PyInterpreterState_GET() #include @@ -23,21 +24,6 @@ class float "PyObject *" "&PyFloat_Type" #include "clinic/floatobject.c.h" -#ifndef PyFloat_MAXFREELIST -# define PyFloat_MAXFREELIST 100 -#endif - - -#if PyFloat_MAXFREELIST > 0 -static struct _Py_float_state * -get_float_state(void) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->float_state; -} -#endif - - double PyFloat_GetMax(void) { @@ -128,25 +114,14 @@ PyFloat_GetInfo(void) PyObject * PyFloat_FromDouble(double fval) { - PyFloatObject *op; -#if PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = get_float_state(); - op = state->free_list; - if (op != NULL) { -#ifdef Py_DEBUG - // PyFloat_FromDouble() must not be called after _PyFloat_Fini() - assert(state->numfree != -1); -#endif - state->free_list = (PyFloatObject *) Py_TYPE(op); - state->numfree--; - } - else +#if WITH_FREELISTS + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyFloatObject *op = (PyFloatObject *)_PyFreeList_Alloc(&interp->small_object_freelist); +#else + PyFloatObject *op = (PyFloatObject *)PyObject_Malloc(sizeof(PyFloatObject)); #endif - { - op = PyObject_Malloc(sizeof(PyFloatObject)); - if (!op) { - return PyErr_NoMemory(); - } + if (op == NULL) { + return PyErr_NoMemory(); } _PyObject_Init((PyObject*)op, &PyFloat_Type); op->ob_fval = fval; @@ -239,20 +214,10 @@ PyFloat_FromString(PyObject *v) static void float_dealloc(PyFloatObject *op) { -#if PyFloat_MAXFREELIST > 0 +#if WITH_FREELISTS if (PyFloat_CheckExact(op)) { - struct _Py_float_state *state = get_float_state(); -#ifdef Py_DEBUG - // float_dealloc() must not be called after _PyFloat_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree >= PyFloat_MAXFREELIST) { - PyObject_Free(op); - return; - } - state->numfree++; - Py_SET_TYPE(op, (PyTypeObject *)state->free_list); - state->free_list = op; + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyFreeList_Free(&interp->small_object_freelist, op); } else #endif @@ -2045,16 +2010,8 @@ _PyFloat_InitTypes(void) void _PyFloat_ClearFreeList(PyInterpreterState *interp) { -#if PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = &interp->float_state; - PyFloatObject *f = state->free_list; - while (f != NULL) { - PyFloatObject *next = (PyFloatObject*) Py_TYPE(f); - PyObject_Free(f); - f = next; - } - state->free_list = NULL; - state->numfree = 0; +#if WITH_FREELISTS + _PyFreeList_Clear(&interp->small_object_freelist); #endif } @@ -2062,22 +2019,6 @@ void _PyFloat_Fini(PyInterpreterState *interp) { _PyFloat_ClearFreeList(interp); -#if defined(Py_DEBUG) && PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = &interp->float_state; - state->numfree = -1; -#endif -} - -/* Print summary info about the state of the optimized allocator */ -void -_PyFloat_DebugMallocStats(FILE *out) -{ -#if PyFloat_MAXFREELIST > 0 - struct _Py_float_state *state = get_float_state(); - _PyDebugAllocatorStats(out, - "free PyFloatObject", - state->numfree, sizeof(PyFloatObject)); -#endif } diff --git a/Objects/longobject.c b/Objects/longobject.c index 5325d1852bc029f..bf5ea4f58e04d8d 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -7,6 +7,7 @@ #include "pycore_interp.h" // _PY_NSMALLPOSINTS #include "pycore_long.h" // __PyLong_GetSmallInt_internal() #include "pycore_object.h" // _PyObject_InitVar() +#include "pycore_pymem.h" // Free lists #include "pycore_pystate.h" // _Py_IsMainInterpreter() #include @@ -112,6 +113,15 @@ long_normalize(PyLongObject *v) #define MAX_LONG_DIGITS \ ((PY_SSIZE_T_MAX - offsetof(PyLongObject, ob_digit))/sizeof(digit)) +union _small_object { + PyFloatObject f; + PyLongObject l; +}; + +/* Free list is shared between ints and floats as they take the same amount of memory (on most platforms) + */ +#define MAX_DIGITS_FOR_FREELIST ((sizeof(union _small_object) - offsetof(PyLongObject, ob_digit))/sizeof(digit)) + PyLongObject * _PyLong_New(Py_ssize_t size) { @@ -121,16 +131,24 @@ _PyLong_New(Py_ssize_t size) "too many digits in integer"); return NULL; } - /* Fast operations for single digit integers (including zero) - * assume that there is always at least one digit present. */ - Py_ssize_t ndigits = size ? size : 1; - /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + - sizeof(digit)*size. Previous incarnations of this code used - sizeof(PyVarObject) instead of the offsetof, but this risks being - incorrect in the presence of padding between the PyVarObject header - and the digits. */ - result = PyObject_Malloc(offsetof(PyLongObject, ob_digit) + - ndigits*sizeof(digit)); +#if WITH_FREELISTS + if (((size_t)size) <= MAX_DIGITS_FOR_FREELIST) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + result = (PyLongObject *)_PyFreeList_Alloc(&interp->small_object_freelist); +#else + if (size == 0) { + result = PyObject_Malloc(sizeof(PyLongObject)); +#endif + } + else { + /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + + sizeof(digit)*size. Previous incarnations of this code used + sizeof(PyVarObject) instead of the offsetof, but this risks being + incorrect in the presence of padding between the PyVarObject header + and the digits. */ + result = PyObject_Malloc(offsetof(PyLongObject, ob_digit) + + size*sizeof(digit)); + } if (!result) { PyErr_NoMemory(); return NULL; @@ -170,11 +188,14 @@ _PyLong_FromMedium(sdigit x) { assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - /* We could use a freelist here */ - PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject)); +#if WITH_FREELISTS + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyLongObject *v = (PyLongObject *)_PyFreeList_Alloc(&interp->small_object_freelist); +#else + PyLongObject *v = (PyLongObject *)PyObject_Malloc(sizeof(PyLongObject)); +#endif if (v == NULL) { - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } Py_ssize_t sign = x < 0 ? -1: 1; digit abs_x = x < 0 ? -x : x; @@ -5743,12 +5764,28 @@ static PyNumberMethods long_as_number = { long_long, /* nb_index */ }; +static void +int_dealloc(PyLongObject *op) +{ +#if WITH_FREELISTS + if (PyLong_CheckExact(op) && IS_MEDIUM_VALUE(op)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyFreeList_Free(&interp->small_object_freelist, op); + } + else +#endif + { + Py_TYPE(op)->tp_free((PyObject *)op); + } +} + + PyTypeObject PyLong_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "int", /* tp_name */ offsetof(PyLongObject, ob_digit), /* tp_basicsize */ sizeof(digit), /* tp_itemsize */ - 0, /* tp_dealloc */ + (destructor)int_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ diff --git a/Objects/object.c b/Objects/object.c index 25f5a2133d5745c..6b2409f89824350 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2079,7 +2079,6 @@ void _PyObject_DebugTypeStats(FILE *out) { _PyDict_DebugMallocStats(out); - _PyFloat_DebugMallocStats(out); _PyFrame_DebugMallocStats(out); _PyList_DebugMallocStats(out); _PyTuple_DebugMallocStats(out); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 4e17bf44b4e9653..9f1d5e0892271bd 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -733,6 +733,71 @@ PyObject_Free(void *ptr) # define LIKELY(value) (value) #endif +#if WITH_FREELISTS + +void * +_PyFreeList_HalfFillAndAllocate(_PyFreeList *list) +{ + assert(list->ptr == NULL); + if (list->space < 4) { + return PyObject_Malloc(list->size); + } + uint32_t i = 0; + for (; i < list->space>>1; i++) { + void* ptr = PyObject_Malloc(list->size); + if (ptr == NULL) { + break; + } + *((void**)ptr) = list->ptr; + list->ptr = ptr; + } + if (i == 0) { + return NULL; + } + list->space -= (i-1); + void *result = list->ptr; + list->ptr = *((void **)result); + return result; +} + +void +_PyFreeList_Clear(_PyFreeList *list) +{ + int space = 0; + void *head = list->ptr; + while (head) { + void *next = *((void**)head); + PyObject_Free(head); + head = next; + space++; + } + list->ptr = NULL; + list->space += space; +} + +void +_PyFreeList_FreeToFull(_PyFreeList *list, void *ptr) +{ + assert(list->space == 0); + if (list->ptr == NULL) { + PyObject_Free(ptr); + return; + } + int space = 0; + void *head = list->ptr; + while (head) { + void *next = *((void**)head); + PyObject_Free(head); + head = next; + space++; + } + list->ptr = ptr; + *((void **)ptr) = NULL; + list->space = space-1; +} + +#endif + #ifdef WITH_PYMALLOC #ifdef WITH_VALGRIND @@ -2268,7 +2333,6 @@ _PyObject_Free(void *ctx, void *p) } } - /* pymalloc realloc. If nbytes==0, then as the Python docs promise, we do not treat this like diff --git a/Python/pystate.c b/Python/pystate.c index 7804e17a064e154..0382f9ab8944c2d 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -201,6 +201,13 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) return _PyStatus_OK(); } +/* For small object freelist */ +#define SMALL_OBJECT_FREELIST_SIZE 256 +union _small_object { + PyFloatObject f; + PyLongObject l; +}; + PyInterpreterState * PyInterpreterState_New(void) { @@ -270,6 +277,13 @@ PyInterpreterState_New(void) interp->audit_hooks = NULL; +#if WITH_FREELISTS + interp->small_object_freelist.ptr = NULL; + interp->small_object_freelist.space = SMALL_OBJECT_FREELIST_SIZE; + interp->small_object_freelist.size = sizeof(union _small_object); + interp->small_object_freelist.capacity = SMALL_OBJECT_FREELIST_SIZE; +#endif + return interp; out_of_memory: