From 569daca9c4ec9b3d2b8753718b03ef49b0350e64 Mon Sep 17 00:00:00 2001 Message-Id: <569daca9c4ec9b3d2b8753718b03ef49b0350e64.1678378810.git.heinzm@redhat.com> From: Heinz Mauelshagen Date: Tue, 21 Feb 2023 16:33:16 +0100 Subject: [PATCH] dm buffered: test target buffering all I/O through dm-bufio Test target to impose load on dm-bufio using all its APIs. Signed-off-by: heinzm --- drivers/md/Kconfig | 9 drivers/md/Makefile | 2 drivers/md/dm-buffered-target.c | 807 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 818 insertions(+) create mode 100644 drivers/md/dm-buffered-target.c Index: linux-2.6/drivers/md/Kconfig =================================================================== --- linux-2.6.orig/drivers/md/Kconfig +++ linux-2.6/drivers/md/Kconfig @@ -344,6 +344,15 @@ config DM_WRITECACHE The writecache target doesn't cache reads because reads are supposed to be cached in standard RAM. +config DM_BUFFERED + tristate "Buffered target (EXPERIMENTAL)" + depends on BLK_DEV_DM + select DM_BUFIO + default n + help + dm-buffered is a test target similar to linear, which + redirects all I/O through dm-bufio. + config DM_EBS tristate "Emulated block size target (EXPERIMENTAL)" depends on BLK_DEV_DM && !HIGHMEM Index: linux-2.6/drivers/md/Makefile =================================================================== --- linux-2.6.orig/drivers/md/Makefile +++ linux-2.6/drivers/md/Makefile @@ -21,6 +21,7 @@ dm-thin-pool-y += dm-thin.o dm-thin-meta dm-cache-y += dm-cache-target.o dm-cache-metadata.o dm-cache-policy.o \ dm-cache-background-tracker.o dm-cache-smq-y += dm-cache-policy-smq.o +dm-buffered-y += dm-buffered-target.o dm-ebs-y += dm-ebs-target.o dm-era-y += dm-era-target.o dm-clone-y += dm-clone-target.o dm-clone-metadata.o @@ -76,6 +77,7 @@ obj-$(CONFIG_DM_THIN_PROVISIONING) += dm obj-$(CONFIG_DM_VERITY) += dm-verity.o obj-$(CONFIG_DM_CACHE) += dm-cache.o obj-$(CONFIG_DM_CACHE_SMQ) += dm-cache-smq.o +obj-$(CONFIG_DM_BUFFERED) += dm-buffered.o obj-$(CONFIG_DM_EBS) += dm-ebs.o obj-$(CONFIG_DM_ERA) += dm-era.o obj-$(CONFIG_DM_CLONE) += dm-clone.o Index: linux-2.6/drivers/md/dm-buffered-target.c =================================================================== --- /dev/null +++ linux-2.6/drivers/md/dm-buffered-target.c @@ -0,0 +1,807 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2019,2023 Red Hat GmbH + * + * A test target similar to linear which performs all I/O through dm-bufio + * for the sake of load testing/demonstrating/learning * the dm-bufio API + * (and potentially gaining some performance with direct I/O). + * + * Uses all functions of the dm-bufio API. + * + * This file is released under the GPL. + */ + +#include +#include +#include +#include + +#define DM_MSG_PREFIX "buffered" + +#define DEFAULT_BUFFERED_BLOCK_SIZE (SECTOR_SIZE << 3) /* 4KiB */ +#define MAX_BUFFERED_BLOCK_SIZE (SECTOR_SIZE << 13) /* 4MiB */ + +enum stats { S_BUFFER_SPLITS, S_PREFLUSHS, S_FUA, S_SYNCS, S_READS, + S_PREFETCHED_READS, S_PREFETCHED_WRITES, S_END, }; + +/* buffered target context */ +struct buffered_c { + spinlock_t lock; /* Protect following bio list */ + struct bio_list bios; + struct dm_bufio_client *bufio; + struct workqueue_struct *buffered_wq; + struct work_struct buffered_ws; + struct workqueue_struct *buffered_flush_wq; + struct delayed_work buffered_flush_ws; + struct dm_dev *dev; + sector_t start; + sector_t block_mask; + unsigned int block_shift; + unsigned int async:1; + unsigned int async_set:1; + unsigned int buffer_size; + unsigned int buffer_size_set:1; + unsigned int buffer_size_default:1; + unsigned int discard:1; + unsigned int discard_set:1; + unsigned int discard_passdown:1; + unsigned int discard_passdown_set:1; + unsigned int write_zeroes:1; + unsigned int write_zeroes_set:1; + + /* REMOVEME: stats */ + atomic_t stats[S_END]; +}; + +/* buffer async_memcpy context */ +struct async_c { + struct async_submit_ctl submit; + struct bio *bio; + atomic_t buffer_refcount; +}; + +/* Convert sector to bufio block number */ +static sector_t _to_block(struct buffered_c *bc, sector_t sector) +{ + return sector >> bc->block_shift; +} + +/* Return sector modulo of block size */ +static sector_t _sector_mod(struct buffered_c *bc, sector_t sector) +{ + return sector & bc->block_mask; +} + +/* Use unutilized @bio->bi_next as endio reference counter and take out @ref_count references */ +static void _bio_set_references(struct bio *bio, unsigned int ref_count) +{ + atomic_set((atomic_t *)&bio->bi_next, ref_count); +} + +/* Use unutilized @bio->bi_next as endio reference counter */ +static void _bio_get(struct bio *bio) +{ + atomic_inc((atomic_t *)&bio->bi_next); +} + +/* Perform bio endio completion when unreferenced using bio clone's unutilized bi_next member */ +static void _bio_put(struct bio *bio) +{ + if (atomic_dec_and_test((atomic_t *)&bio->bi_next)) + bio_endio(bio); +} + +/* Set asynchronous memcpy bufio buffer release reference count */ +static void _buffer_get(struct dm_buffer *bp, int ref_count) +{ + struct async_c *ac = dm_bufio_get_aux_data(bp); + + atomic_set(&ac->buffer_refcount, ref_count); +} + +/* Ensure @bp doesn't get released too soon */ +static void _buffer_put(struct dm_buffer *bp) +{ + struct async_c *ac = dm_bufio_get_aux_data(bp); + + if (atomic_dec_and_test(&ac->buffer_refcount)) + dm_bufio_release(bp); +} + +/* Flush any dirty buffers of @bc out */ +static int _buffered_flush(struct buffered_c *bc) +{ + return dm_bufio_write_dirty_buffers(bc->bufio) ?: dm_bufio_issue_flush(bc->bufio); +} + +/* async_memcpy() completion callback */ +static void _complete_memcpy(void *context) +{ + struct dm_buffer *bp = context; + struct async_c *ac = dm_bufio_get_aux_data(bp); + + _bio_put(ac->bio); + _buffer_put(bp); +} + +/* Initialize per bufio buffer async_memcpy() context */ +static struct async_submit_ctl *_init_async_memcpy(struct dm_buffer *bp, struct bio *bio) +{ + struct async_c *ac = dm_bufio_get_aux_data(bp); + + ac->bio = bio; + init_async_submit(&ac->submit, 0, NULL, _complete_memcpy, bp, NULL); + return &ac->submit; +} + +/* Return total number of blocks for @ti */ +static sector_t _buffered_size(struct dm_target *ti) +{ + struct buffered_c *bc = ti->private; + + return _to_block(bc, ti->len - bc->start) + (_sector_mod(bc, ti->len) ? 1 : 0); +} + +static void _memcpy(struct buffered_c *bc, struct dm_buffer *bp, + struct page *dst, struct page *src, + loff_t dst_offset, loff_t src_offset, size_t len, + struct async_submit_ctl *ctl) +{ + if (bc->async) { + async_memcpy(dst, src, dst_offset, src_offset, len, ctl); + } else { + void *d = kmap_local_page(dst); + void *s = kmap_local_page(src); + + memcpy(d + dst_offset, s + src_offset, len); + kunmap_local(d); + kunmap_local(s); + _complete_memcpy(bp); + } +} + +/* + * Process @bvec of @bio optionally doing 2 bufio I/Os + * in case the page of the bio_vec overlaps two buffers. + */ +static void _io(struct buffered_c *bc, struct bio *bio, struct bio_vec *bvec) +{ + bool write = (bio_op(bio) == REQ_OP_WRITE); + unsigned int buffer_offset, bvec_offset = bvec->bv_offset; + unsigned int len, total_len = bvec->bv_len; + unsigned int block_size = dm_bufio_get_block_size(bc->bufio); + sector_t block, sector = bio->bi_iter.bi_sector; + void *buffer; + struct dm_buffer *bp; + + while (total_len) { + block = _to_block(bc, sector); + buffer_offset = to_bytes(_sector_mod(bc, sector)); + len = min(block_size - buffer_offset, total_len); + /* + * Take an additional reference out for the 2nd buffer I/O + * in case the segment is split across 2 buffers. + */ + if (len < total_len) { + _bio_get(bio); + atomic_inc(&bc->stats[S_BUFFER_SPLITS]); + } + + buffer = dm_bufio_get(bc->bufio, block, &bp); + if (!buffer) { + if (write && !buffer_offset && total_len == block_size) { + buffer = dm_bufio_new(bc->bufio, block, &bp); + } else { + buffer = dm_bufio_read(bc->bufio, block, &bp); + atomic_inc(&bc->stats[S_READS]); + } + } else { + atomic_inc(&bc->stats[S_PREFETCHED_READS]); + } + + if (IS_ERR(buffer)) { + /* Memorize first error in bio status. */ + if (!bio->bi_status) + bio->bi_status = PTR_ERR(buffer); + + _bio_put(bio); + /* Continue with any split bio payload */ + + } else if (write) { + /* + * Take out 2 references to be savely handling any single aysnchronous + * write copy to avoid race between copying the data and setting the + * partial buffer dirty otherwise leading to premature buffer releases. + */ + /* (Superfluous) function consistency check example */ + WARN_ON_ONCE(block != dm_bufio_get_block_number(bp)); + /* Superfluous call to cover the API example */ + buffer = dm_bufio_get_block_data(bp); + WARN_ON_ONCE(IS_ERR(buffer)); + _buffer_get(bp, 2); + _memcpy(bc, bp, virt_to_page(buffer), bvec->bv_page, + buffer_offset, bvec_offset, len, _init_async_memcpy(bp, bio)); + + /* Superfluous conditional to show both functions dirtying buffers. */ + if (!buffer_offset && len == block_size) + dm_bufio_mark_buffer_dirty(bp); + else + dm_bufio_mark_partial_buffer_dirty(bp, buffer_offset, + buffer_offset + len); + + /* + * Now release the safeguard reference after + * having dirtied the (partial) buffer. + */ + _buffer_put(bp); + } else { + /* (Superfluous) function consistency check example */ + WARN_ON(block != dm_bufio_get_block_number(bp)); + _buffer_get(bp, 1); + _memcpy(bc, bp, bvec->bv_page, virt_to_page(buffer), + bvec_offset, buffer_offset, len, _init_async_memcpy(bp, bio)); + } + + /* Process any additional buffer even in case of I/O error */ + sector += to_sector(len); + bvec_offset += len; + total_len -= len; + } +} + +/* + * Issue discards to a block range defined by @bio. + * + * Avoid issueing to partial blocks en the edges. + */ +static void _discard_blocks(struct buffered_c *bc, struct bio *bio) +{ + sector_t block, end, start; + + start = _to_block(bc, bio->bi_iter.bi_sector); + end = _to_block(bc, bio_end_sector(bio)); + for (block = start ; block <= end; block++) { + dm_bufio_forget(bc->bufio, block); + cond_resched(); + } + + if (bc->discard_passdown) { + if (_sector_mod(bc, bio->bi_iter.bi_sector)) + start++; + if (_sector_mod(bc, bio_end_sector(bio))) + end--; + if (end >= start) + dm_bufio_issue_discard(bc->bufio, start, end - start + 1); + } +} + +/* Process REQ_OP_WRITE_ZEROES on @bio */ +/* FIXME: very slow, hence ti->num_write_zeroes_bios = 0 in the constructor. */ +static void _write_zeroes(struct buffered_c *bc, struct bio *bio) +{ + loff_t b_offset, e_offset; + sector_t block, mod; + sector_t sector = bio->bi_iter.bi_sector; + sector_t end_sector = bio_end_sector(bio); + sector_t sectors_per_block = to_sector(bc->buffer_size); + void *buffer; + struct dm_buffer *bp; + + while (sector < end_sector) { + block = _to_block(bc, sector); + mod = _sector_mod(bc, sector); + + buffer = dm_bufio_get(bc->bufio, block, &bp); + if (!buffer) + buffer = (mod ? dm_bufio_read : dm_bufio_new)(bc->bufio, block, &bp); + + if (IS_ERR(buffer)) { + bio->bi_status = PTR_ERR(buffer); + break; + } + + b_offset = to_bytes(mod); + if (end_sector - sector < sectors_per_block) + e_offset = b_offset + to_bytes(end_sector - sector); + else + e_offset = bc->buffer_size; + + memset(buffer + b_offset, 0, e_offset - b_offset); + dm_bufio_mark_partial_buffer_dirty(bp, b_offset, e_offset); + dm_bufio_release(bp); + sector += sectors_per_block - mod; + } +} + +/* Process I/O on a single @bio */ +static void __process_bio(struct buffered_c *bc, struct bio *bio) +{ + struct bio_vec bvec; + + switch (bio_op(bio)) { + case REQ_OP_READ: + case REQ_OP_WRITE: + /* + * Take references per segment out and add an + * additional one to prevent an endio race. + */ + _bio_set_references(bio, bio_segments(bio) + 1); + + if (bio->bi_opf & REQ_PREFLUSH) { /* Flush any writes out on request. */ + atomic_inc(&bc->stats[S_PREFLUSHS]); + bio->bi_status = errno_to_blk_status(_buffered_flush(bc)); + if (unlikely(bio->bi_status)) + goto err; + } + + bio_for_each_segment(bvec, bio, bio->bi_iter) + _io(bc, bio, &bvec); + + if (unlikely(bio->bi_status)) { + goto err; + } else if (bio->bi_opf & (REQ_FUA | REQ_SYNC)) { + atomic_inc((bio->bi_opf & REQ_FUA) ? + &bc->stats[S_FUA] : &bc->stats[S_SYNCS]); + bio->bi_status = errno_to_blk_status(_buffered_flush(bc)); + if (unlikely(bio->bi_status)) + goto err; + } + + break; + case REQ_OP_DISCARD: + /* + * Try forgetting buffers on discard (least we can do + * with lag of discard passdown in dm-bufio). + */ + _bio_set_references(bio, 1); + if (bc->discard) + _discard_blocks(bc, bio); + break; + case REQ_OP_WRITE_ZEROES: + /* FIXME: not tested, as none came from upstack back in2 2019. */ + _bio_set_references(bio, 1); + _write_zeroes(bc, bio); + break; + default: + _bio_set_references(bio, 1); + /* Return error for unsupported operation */ + bio->bi_status = errno_to_blk_status(-EOPNOTSUPP); + } + + goto out; +err: + /* On error, reset refecount to 1 for _bio_put() to endio */ + _bio_set_references(bio, 1); +out: + _bio_put(bio); /* Release (additional) reference */ +} + +/* Process I/O on a bio prefetching buffers on a single @bio in a worker. */ +static void _process_bio(struct work_struct *work) +{ + struct buffered_c *bc = container_of(work, struct buffered_c, buffered_ws); + struct bio *bio; + bool write; + sector_t blocks, start, end; + + spin_lock_irq(&bc->lock); + bio = bio_list_pop(&bc->bios); + spin_unlock_irq(&bc->lock); + + if (!bio) + return; + + write = false; + start = _to_block(bc, bio->bi_iter.bi_sector); + + /* Prefetch read and partial write buffers */ + switch (bio_op(bio)) { + case REQ_OP_READ: + blocks = _to_block(bc, bio_end_sector(bio) - bio->bi_iter.bi_sector); + dm_bufio_prefetch(bc->bufio, start, blocks ?: 1); + break; + case REQ_OP_WRITE: + write = true; + /* Beware of partial block updates. */ + if (_sector_mod(bc, bio->bi_iter.bi_sector)) { + dm_bufio_prefetch(bc->bufio, start, 1); + atomic_inc(&bc->stats[S_PREFETCHED_WRITES]); + } + if (_sector_mod(bc, bio_end_sector(bio))) { + end = _to_block(bc, bio_end_sector(bio)); + if (start != end) { + dm_bufio_prefetch(bc->bufio, start, 1); + atomic_inc(&bc->stats[S_PREFETCHED_WRITES]); + } + } + break; + case REQ_OP_WRITE_ZEROES: + write = true; + break; + default: + } + + __process_bio(bc, bio); + + if (!bio_list_empty(&bc->bios)) + queue_work(bc->buffered_wq, &bc->buffered_ws); + + /* Reschedule the flush thread in case of new write(s) */ + if (write) + schedule_delayed_work(&bc->buffered_flush_ws, 2 * HZ); + + cond_resched(); +} + +static void _process_flushs(struct work_struct *work) +{ + struct buffered_c *bc = container_of(to_delayed_work(work), + struct buffered_c, buffered_flush_ws); + _buffered_flush(bc); +} + +static void buffered_dtr(struct dm_target *ti) +{ + struct buffered_c *bc = ti->private; + + if (bc->buffered_wq) + destroy_workqueue(bc->buffered_wq); + if (bc->buffered_flush_wq) + destroy_workqueue(bc->buffered_flush_wq); + if (bc->bufio && !IS_ERR(bc->bufio)) + dm_bufio_client_destroy(bc->bufio); + if (bc->dev) + dm_put_device(ti, bc->dev); + kfree(bc); +} + +/* + * Mapping parameters: + * + * : full pathname of the buffered device + * : offset in sectors into the device + * [] : optional size of bufio buffers in bytes + * [] + * : select to (not) perform asynchronous memory copies + * between bvec pages and buffers + * [] + * : select to (not) perform discards + * [] + * : select to (not) perform passing down discards to the buffered device + * [] + * : select to (not) perform write zeroes + * + */ +static int buffered_ctr(struct dm_target *ti, unsigned int argc, char **argv) +{ + struct buffered_c *bc; + int i, r; + + if (argc < 2 || argc > 7) { + ti->error = "Requires 2 - 7 arguments"; + return -EINVAL; + } + + bc = kzalloc(sizeof(*bc), GFP_KERNEL); + if (!bc) { + ti->error = "Cannot allocate context"; + return -ENOMEM; + } + + ti->private = bc; + + r = (kstrtoull(argv[1], 10, (u64 *)&bc->start) || + bc->start >= (ti->len >> 1)) ? -EINVAL : 0; + if (r) { + ti->error = "Invalid sector offset"; + goto bad; + } + + if (argc > 2) { + if (!strcasecmp(argv[2], "-")) { + bc->buffer_size_default = true; + bc->buffer_size = DEFAULT_BUFFERED_BLOCK_SIZE; + } else { + bc->buffer_size_set = true; + r = (kstrtouint(argv[2], 10, &bc->buffer_size) || + !is_power_of_2(bc->buffer_size) || + bc->buffer_size < SECTOR_SIZE || + bc->buffer_size > MAX_BUFFERED_BLOCK_SIZE) ? -EINVAL : 0; + if (r) { + ti->error = "Invalid block size"; + goto bad; + } + } + } + + if (argc > 3) { + bc->async_set = 1; + + if (!strcasecmp(argv[3], "no_async_memcpy")) { + bc->async = 0; + } else if (!strcasecmp(argv[3], "async_memcpy")) { + bc->async = 1; + } else { + ti->error = "Invalid async memcpy parameter"; + r = -EINVAL; + goto bad; + } + } + + if (argc > 4) { + bc->discard_set = 1; + + if (!strcasecmp(argv[4], "no_discard")) { + bc->discard = 0; + } else if (!strcasecmp(argv[4], "discard")) { + bc->discard = 1; + } else { + ti->error = "Invalid discard parameter"; + r = -EINVAL; + goto bad; + } + } + + if (argc > 5) { + bc->discard_passdown_set = 1; + + if (!strcasecmp(argv[5], "no_discard_passdown")) { + bc->discard_passdown = 0; + } else if (!strcasecmp(argv[5], "discard_passdown")) { + bc->discard_passdown = 1; + } else { + ti->error = "Invalid discard passdown parameter"; + r = -EINVAL; + goto bad; + } + } + + if (argc > 6) { + bc->write_zeroes_set = 1; + + if (!strcasecmp(argv[6], "no_write_zeroes")) { + bc->write_zeroes = 0; + } else if (!strcasecmp(argv[6], "write_zeroes")) { + bc->write_zeroes = 1; + } else { + ti->error = "Invalid write zeroes parameter"; + r = -EINVAL; + goto bad; + } + } + + r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &bc->dev); + if (r) { + ti->error = "Device lookup failed"; + goto bad; + } + + r = -ENOMEM; + bc->buffered_wq = create_workqueue("dm-" DM_MSG_PREFIX "-io"); + if (!bc->buffered_wq) { + ti->error = "Couldn't start dm-" DM_MSG_PREFIX "-io"; + goto bad; + } + + bc->buffered_flush_wq = create_singlethread_workqueue("dm-" DM_MSG_PREFIX "-flush"); + if (!bc->buffered_flush_wq) { + ti->error = "Couldn't start dm-" DM_MSG_PREFIX "-flush"; + goto bad; + } + + bc->buffer_size = bc->buffer_size ?: DEFAULT_BUFFERED_BLOCK_SIZE; + bc->bufio = dm_bufio_client_create(bc->dev->bdev, bc->buffer_size, + 1, sizeof(struct async_c), NULL, NULL, 0); + if (IS_ERR(bc->bufio)) { + ti->error = "Couldn't create bufio client"; + r = PTR_ERR(bc->bufio); + goto bad; + } + + dm_bufio_set_sector_offset(bc->bufio, bc->start); + + bio_list_init(&bc->bios); + spin_lock_init(&bc->lock); + INIT_WORK(&bc->buffered_ws, _process_bio); + INIT_DELAYED_WORK(&bc->buffered_flush_ws, _process_flushs); + + bc->block_shift = __ffs(bc->buffer_size) - SECTOR_SHIFT; + bc->block_mask = ~(~((sector_t)0) << bc->block_shift); + + /* REMOVEME: development statistics */ + for (i = 0; i < ARRAY_SIZE(bc->stats); i++) + atomic_set(&bc->stats[i], 0); + + ti->flush_supported = 1; + ti->num_discard_bios = 1; + ti->discards_supported = 1; + ti->num_write_zeroes_bios = bc->write_zeroes ? 1 : 0; + return 0; +bad: + buffered_dtr(ti); + return r; +} + +static int buffered_map(struct dm_target *ti, struct bio *bio) +{ + struct buffered_c *bc = ti->private; + bool queue; + + spin_lock_irq(&bc->lock); + queue = bio_list_empty(&bc->bios); + bio_list_add(&bc->bios, bio); + spin_unlock_irq(&bc->lock); + + if (queue) + queue_work(bc->buffered_wq, &bc->buffered_ws); + + return DM_MAPIO_SUBMITTED; +} + +static void buffered_postsuspend(struct dm_target *ti) +{ + struct buffered_c *bc = ti->private; + + flush_workqueue(bc->buffered_wq); + cancel_delayed_work_sync(&bc->buffered_flush_ws); + flush_workqueue(bc->buffered_flush_wq); +} + +/* + * Count and/or forget buffers. + * + * Mind count is subject to forget/release/move races. + */ +/* FIXME: count dirty buffers */ +static int __process_buffers(struct dm_target *ti, sector_t *n_buffers, bool forget) +{ + struct buffered_c *bc = ti->private; + sector_t block, end = _buffered_size(ti); + struct dm_buffer *bp; + void *buffer; + + if (!n_buffers && !forget) + return -EINVAL; + + for (block = 0; block < end; block++) { + buffer = dm_bufio_get(bc->bufio, block, &bp); + if (buffer) { + dm_bufio_release(bp); + if (forget) + dm_bufio_forget(bc->bufio, block); + if (n_buffers) + (*n_buffers)++; + } + } + + return 0; +} + +static void buffered_status(struct dm_target *ti, status_type_t type, + unsigned int status_flags, char *result, unsigned int maxlen) +{ + struct buffered_c *bc = ti->private; + int sz = 0; + sector_t n_buffers = 0; + + switch (type) { + case STATUSTYPE_TABLE: + DMEMIT("%s %llu", bc->dev->name, (u64)bc->start); + if (bc->buffer_size_default) + DMEMIT(" -"); + if (bc->buffer_size_set) + DMEMIT(" %u", bc->buffer_size); + if (bc->async_set) + DMEMIT(" %s", bc->async ? "async_memcpy" : "no_async_memcpy"); + if (bc->discard_set) + DMEMIT(" %s", bc->discard ? "discard" : "no_discard"); + if (bc->discard_passdown_set) + DMEMIT(" %s", bc->discard_passdown ? + "discard_passdown" : "no_discard_passdown"); + if (bc->write_zeroes_set) + DMEMIT(" %s", bc->write_zeroes ? "write_zeroes" : "no_write_zeroes"); + break; + case STATUSTYPE_INFO: + DMEMIT("%u %llu %u/%u/%u/%u/%u/%u/%u %llu", + bc->buffer_size, (u64)n_buffers, + atomic_read(&bc->stats[S_BUFFER_SPLITS]), + atomic_read(&bc->stats[S_PREFLUSHS]), + atomic_read(&bc->stats[S_FUA]), + atomic_read(&bc->stats[S_SYNCS]), + atomic_read(&bc->stats[S_READS]), + atomic_read(&bc->stats[S_PREFETCHED_READS]), + atomic_read(&bc->stats[S_PREFETCHED_WRITES]), + (u64)dm_bufio_get_device_size(bc->bufio)); + break; + case STATUSTYPE_IMA: + *result = '\0'; + } +} + +static int buffered_prepare_ioctl(struct dm_target *ti, struct block_device **bdev) +{ + struct buffered_c *bc = ti->private; + + *bdev = bc->dev->bdev; + + /* Only pass down ioctls on precise size matches */ + return !!(bc->start || ti->len != bdev_nr_sectors(bc->dev->bdev)); +} + +static int buffered_iterate_devices(struct dm_target *ti, + iterate_devices_callout_fn fn, void *data) +{ + struct buffered_c *bc = ti->private; + + return fn(ti, bc->dev, bc->start, ti->len, data); +} + +static int buffered_message(struct dm_target *ti, unsigned int argc, char **argv, + char *result, unsigned int maxlen) +{ + struct buffered_c *bc = ti->private; + sector_t n; + + if (!strcasecmp(argv[0], "async_flush")) { + if (argc != 1) + return -EINVAL; + dm_bufio_write_dirty_buffers_async(bc->bufio); + return 0; + } + + if (!strcasecmp(argv[0], "buffers")) { + if (argc != 2) + return -EINVAL; + if (kstrtoull(argv[1], 10, (u64 *)&n) || !n || + n >= _buffered_size(ti) || n > UINT_MAX) + return -EINVAL; + dm_bufio_set_minimum_buffers(bc->bufio, (unsigned int)n); + return 0; + } + + return (argc == 1) ? __process_buffers(ti, NULL, !strcasecmp(argv[0], "forget")) : -EINVAL; +} + +static void buffered_io_hints(struct dm_target *ti, struct queue_limits *limits) +{ + struct buffered_c *bc = ti->private; + + limits->logical_block_size = to_bytes(1); + blk_limits_io_min(limits, limits->logical_block_size); + blk_limits_io_opt(limits, bc->buffer_size); + if (ti->num_write_zeroes_bios) + limits->max_write_zeroes_sectors = min(ti->len, to_sector(MAX_BUFFERED_BLOCK_SIZE)); +} + +static struct target_type buffered_target = { + .name = "buffered", + .version = {1, 0, 0}, + .module = THIS_MODULE, + .ctr = buffered_ctr, + .dtr = buffered_dtr, + .map = buffered_map, + .status = buffered_status, + .postsuspend = buffered_postsuspend, + .iterate_devices = buffered_iterate_devices, + .prepare_ioctl = buffered_prepare_ioctl, + .message = buffered_message, + .io_hints = buffered_io_hints +}; + +static int __init dm_buffered_init(void) +{ + return dm_register_target(&buffered_target); +} + +static void __exit dm_buffered_exit(void) +{ + dm_unregister_target(&buffered_target); +} + +/* Module hooks */ +module_init(dm_buffered_init); +module_exit(dm_buffered_exit); + +MODULE_DESCRIPTION(DM_NAME " buffered test target"); +MODULE_AUTHOR("Heinz Mauelshagen "); +MODULE_LICENSE("GPL");