slab: introduce the flag SLAB_MINIMIZE_WASTE This patch introduces a flag SLAB_MINIMIZE_WASTE for slab and slub. This flag causes allocation of larger slab caches in order to minimize wasted space. Signed-off-by: Mikulas Patocka --- drivers/md/dm-bufio.c | 2 +- include/linux/slab.h | 7 +++++++ mm/slab.c | 4 ++-- mm/slab.h | 7 ++++--- mm/slab_common.c | 2 +- mm/slub.c | 25 ++++++++++++++++++++----- 6 files changed, 35 insertions(+), 12 deletions(-) Index: linux-2.6/include/linux/slab.h =================================================================== --- linux-2.6.orig/include/linux/slab.h 2018-03-14 18:28:23.879999000 +0100 +++ linux-2.6/include/linux/slab.h 2018-03-14 21:00:45.049999000 +0100 @@ -108,6 +108,13 @@ #define SLAB_KASAN 0 #endif +/* + * Use higer order allocations to minimize wasted space. + * Note: the allocation is unreliable if this flag is used, the caller + * must handle allocation failures gracefully. + */ +#define SLAB_MINIMIZE_WASTE ((slab_flags_t __force)0x10000000U) + /* The following flags affect the page allocator grouping pages by mobility */ /* Objects are reclaimable */ #define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0x00020000U) Index: linux-2.6/mm/slab_common.c =================================================================== --- linux-2.6.orig/mm/slab_common.c 2018-03-14 18:28:23.879999000 +0100 +++ linux-2.6/mm/slab_common.c 2018-03-14 18:28:23.869999000 +0100 @@ -52,7 +52,7 @@ static DECLARE_WORK(slab_caches_to_rcu_d SLAB_FAILSLAB | SLAB_KASAN) #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \ - SLAB_ACCOUNT) + SLAB_ACCOUNT | SLAB_MINIMIZE_WASTE) /* * Merge control. If this is set then no merging of slab caches will occur. Index: linux-2.6/mm/slub.c =================================================================== --- linux-2.6.orig/mm/slub.c 2018-03-14 18:28:23.879999000 +0100 +++ linux-2.6/mm/slub.c 2018-03-14 21:01:30.859999000 +0100 @@ -3234,7 +3234,7 @@ static inline int slab_order(int size, i return order; } -static inline int calculate_order(int size, int reserved) +static inline int calculate_order(int size, int reserved, slab_flags_t flags) { int order; int min_objects; @@ -3261,7 +3261,7 @@ static inline int calculate_order(int si order = slab_order(size, min_objects, slub_max_order, fraction, reserved); if (order <= slub_max_order) - return order; + goto ret_order; fraction /= 2; } min_objects--; @@ -3273,15 +3273,30 @@ static inline int calculate_order(int si */ order = slab_order(size, 1, slub_max_order, 1, reserved); if (order <= slub_max_order) - return order; + goto ret_order; /* * Doh this slab cannot be placed using slub_max_order. */ order = slab_order(size, 1, MAX_ORDER, 1, reserved); if (order < MAX_ORDER) - return order; + goto ret_order; return -ENOSYS; + +ret_order: + if (flags & SLAB_MINIMIZE_WASTE) { + /* Increase the order if it decreases waste */ + int test_order; + for (test_order = order + 1; test_order < MAX_ORDER; test_order++) { + unsigned long order_objects = ((PAGE_SIZE << order) - reserved) / size; + unsigned long test_order_objects = ((PAGE_SIZE << test_order) - reserved) / size; + if (test_order_objects >= 64 || test_order_objects > MAX_OBJS_PER_PAGE) + break; + if (test_order_objects > order_objects << (test_order - order)) + order = test_order; + } + } + return order; } static void @@ -3546,7 +3561,7 @@ static int calculate_sizes(struct kmem_c if (forced_order >= 0) order = forced_order; else - order = calculate_order(size, s->reserved); + order = calculate_order(size, s->reserved, flags); if (order < 0) return 0; Index: linux-2.6/drivers/md/dm-bufio.c =================================================================== --- linux-2.6.orig/drivers/md/dm-bufio.c 2018-03-14 18:28:23.879999000 +0100 +++ linux-2.6/drivers/md/dm-bufio.c 2018-03-14 18:28:23.879999000 +0100 @@ -1686,7 +1686,7 @@ struct dm_bufio_client *dm_bufio_client_ if (block_size <= KMALLOC_MAX_SIZE) { char name[26]; snprintf(name, sizeof name, "dm_bufio_cache-%u", c->block_size); - c->slab_cache = kmem_cache_create(name, c->block_size, 0, SLAB_RECLAIM_ACCOUNT, NULL); + c->slab_cache = kmem_cache_create(name, c->block_size, 0, SLAB_RECLAIM_ACCOUNT | SLAB_MINIMIZE_WASTE, NULL); if (!c->slab_cache) { r = -ENOMEM; goto bad; Index: linux-2.6/mm/slab.h =================================================================== --- linux-2.6.orig/mm/slab.h 2018-02-14 20:24:42.218255000 +0100 +++ linux-2.6/mm/slab.h 2018-03-14 20:50:08.939999000 +0100 @@ -142,10 +142,10 @@ static inline slab_flags_t kmem_cache_fl #if defined(CONFIG_SLAB) #define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \ SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | \ - SLAB_ACCOUNT) + SLAB_ACCOUNT | SLAB_MINIMIZE_WASTE) #elif defined(CONFIG_SLUB) #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \ - SLAB_TEMPORARY | SLAB_ACCOUNT) + SLAB_TEMPORARY | SLAB_ACCOUNT | SLAB_MINIMIZE_WASTE) #else #define SLAB_CACHE_FLAGS (0) #endif @@ -164,7 +164,8 @@ static inline slab_flags_t kmem_cache_fl SLAB_NOLEAKTRACE | \ SLAB_RECLAIM_ACCOUNT | \ SLAB_TEMPORARY | \ - SLAB_ACCOUNT) + SLAB_ACCOUNT | \ + SLAB_MINIMIZE_WASTE) int __kmem_cache_shutdown(struct kmem_cache *); void __kmem_cache_release(struct kmem_cache *); Index: linux-2.6/mm/slab.c =================================================================== --- linux-2.6.orig/mm/slab.c 2018-02-14 20:24:42.218255000 +0100 +++ linux-2.6/mm/slab.c 2018-03-14 20:55:14.969999000 +0100 @@ -1789,14 +1789,14 @@ static size_t calculate_slab_order(struc * as GFP_NOFS and we really don't want to have to be allocating * higher-order pages when we are unable to shrink dcache. */ - if (flags & SLAB_RECLAIM_ACCOUNT) + if (flags & SLAB_RECLAIM_ACCOUNT && !(flags & SLAB_MINIMIZE_WASTE)) break; /* * Large number of objects is good, but very large slabs are * currently bad for the gfp()s. */ - if (gfporder >= slab_max_order) + if (gfporder >= slab_max_order && !(flags & SLAB_MINIMIZE_WASTE)) break; /*