dm-bufio: keep some amount of cache regardless of age This patch changes dm-bufio so that it keeps some amount of cache regardless of age. The default amount of memory to keep is the minimum of 1/500 of total memory and 1/50 of vmalloc memory. The value can be changed in the file /sys/module/dm_bufio/parameters/max_kept_cache_size_bytes. Signed-off-by: Mikulas Patocka --- drivers/md/dm-bufio.c | 83 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) Index: linux-3.17/drivers/md/dm-bufio.c =================================================================== --- linux-3.17.orig/drivers/md/dm-bufio.c 2014-10-08 19:06:54.000000000 +0200 +++ linux-3.17/drivers/md/dm-bufio.c 2014-10-08 19:14:58.000000000 +0200 @@ -29,6 +29,8 @@ #define DM_BUFIO_MEMORY_RATIO 50 #define DM_BUFIO_VMALLOC_RATIO 4 +#define DM_BUFIO_KEEP_MEMORY_RATIO 500 +#define DM_BUFIO_KEEP_VMALLOC_RATIO 50 #define DM_BUFIO_WRITEBACK_PERCENT 75 /* @@ -207,15 +209,26 @@ do { \ static unsigned long dm_bufio_default_cache_size; /* + * Default kept cache size: available memory divided by the ratio. + */ +static unsigned long dm_bufio_default_kept_cache_size; + +/* * Total cache size set by the user. */ static unsigned long dm_bufio_cache_size; /* - * A copy of dm_bufio_cache_size because dm_bufio_cache_size can change - * at any time. If it disagrees, the user has changed cache size. + * Total kept cache size set by the user. + */ +static unsigned long dm_bufio_kept_cache_size; + +/* + * A copy of dm_bufio_cache_size and dm_bufio_kept_cache_size because they can + * change at any time. If it disagrees, the user has changed cache size. */ static unsigned long dm_bufio_cache_size_latch; +static unsigned long dm_bufio_kept_cache_size_latch; static DEFINE_SPINLOCK(param_spinlock); @@ -238,6 +251,11 @@ static unsigned long dm_bufio_current_al static unsigned long dm_bufio_cache_size_per_client; /* + * Per-client cache: dm_bufio_kept_cache_size / dm_bufio_client_count + */ +static unsigned long dm_bufio_kept_cache_size_per_client; + +/* * The current number of clients. */ static int dm_bufio_client_count; @@ -296,6 +314,21 @@ static void __cache_size_refresh(void) dm_bufio_cache_size_per_client = dm_bufio_cache_size_latch / (dm_bufio_client_count ? : 1); + + + dm_bufio_kept_cache_size_latch = ACCESS_ONCE(dm_bufio_kept_cache_size); + + /* + * Use default if set to 0 and report the actual cache size used. + */ + if (!dm_bufio_kept_cache_size_latch) { + (void)cmpxchg(&dm_bufio_kept_cache_size, 0, + dm_bufio_default_kept_cache_size); + dm_bufio_kept_cache_size_latch = dm_bufio_default_kept_cache_size; + } + + dm_bufio_kept_cache_size_per_client = dm_bufio_kept_cache_size_latch / + (dm_bufio_client_count ? : 1); } /* @@ -844,7 +877,7 @@ static void __get_memory_limit(struct dm { unsigned long buffers; - if (ACCESS_ONCE(dm_bufio_cache_size) != dm_bufio_cache_size_latch) { + if (unlikely(ACCESS_ONCE(dm_bufio_cache_size) != dm_bufio_cache_size_latch)) { mutex_lock(&dm_bufio_clients_lock); __cache_size_refresh(); mutex_unlock(&dm_bufio_clients_lock); @@ -860,6 +893,19 @@ static void __get_memory_limit(struct dm *threshold_buffers = buffers * DM_BUFIO_WRITEBACK_PERCENT / 100; } +static void __get_kept_memory_limit(struct dm_bufio_client *c, + unsigned long *kept_buffers) +{ + if (unlikely(ACCESS_ONCE(dm_bufio_kept_cache_size) != dm_bufio_kept_cache_size_latch)) { + mutex_lock(&dm_bufio_clients_lock); + __cache_size_refresh(); + mutex_unlock(&dm_bufio_clients_lock); + } + + *kept_buffers = dm_bufio_kept_cache_size_per_client >> + (c->sectors_per_block_bits + SECTOR_SHIFT); +} + /* * Check if we're over watermark. * If we are over threshold_buffers, start freeing buffers. @@ -1696,11 +1742,20 @@ static void cleanup_old_buffers(void) mutex_lock(&dm_bufio_clients_lock); list_for_each_entry(c, &dm_bufio_all_clients, client_list) { + unsigned long kept_buffers; + if (!dm_bufio_trylock(c)) continue; + __get_kept_memory_limit(c, &kept_buffers); + while (!list_empty(&c->lru[LIST_CLEAN])) { struct dm_buffer *b; + + if (c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY] + <= kept_buffers) + break; + b = list_entry(c->lru[LIST_CLEAN].prev, struct dm_buffer, lru_list); if (!__cleanup_old_buffer(b, 0, max_age * HZ)) @@ -1762,6 +1817,25 @@ static int __init dm_bufio_init(void) dm_bufio_default_cache_size = mem; + + mem = (__u64)((totalram_pages - totalhigh_pages) / + DM_BUFIO_KEEP_MEMORY_RATIO) << PAGE_SHIFT; + + if (mem > ULONG_MAX) + mem = ULONG_MAX; + +#ifdef CONFIG_MMU + /* + * Get the size of vmalloc space the same way as VMALLOC_TOTAL + * in fs/proc/internal.h + */ + if (mem > (VMALLOC_END - VMALLOC_START) / DM_BUFIO_KEEP_VMALLOC_RATIO) + mem = (VMALLOC_END - VMALLOC_START) / DM_BUFIO_KEEP_VMALLOC_RATIO; +#endif + + dm_bufio_default_kept_cache_size = mem; + + mutex_lock(&dm_bufio_clients_lock); __cache_size_refresh(); mutex_unlock(&dm_bufio_clients_lock); @@ -1832,6 +1906,9 @@ module_exit(dm_bufio_exit) module_param_named(max_cache_size_bytes, dm_bufio_cache_size, ulong, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache"); +module_param_named(max_kept_cache_size_bytes, dm_bufio_kept_cache_size, ulong, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(max_kept_cache_size_bytes, "Size of metadata cache to keep regardless of age"); + module_param_named(max_age_seconds, dm_bufio_max_age, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(max_age_seconds, "Max age of a buffer in seconds");