From a580240c1ae1ee6f7ed39aed921f81935abea95c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 24 May 2013 14:28:30 -0400 Subject: [PATCH 17/18] blk-mq: make flush work again Roll up of new-queue-20130523..new-queue --- block/blk-flush.c | 11 ++++++++--- block/blk-mq.c | 8 ++++---- block/blk-mq.h | 1 + drivers/block/mtip32xx/mtip32xx.c | 2 +- drivers/block/null_blk.c | 2 +- drivers/block/virtio_blk.c | 5 +++-- include/linux/blk-mq.h | 4 ++-- 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/block/blk-flush.c b/block/blk-flush.c index 4c53346..c56c37d 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -126,6 +126,8 @@ static void blk_flush_restore_request(struct request *rq) /* make @rq a normal request */ rq->cmd_flags &= ~REQ_FLUSH_SEQ; rq->end_io = rq->flush.saved_end_io; + + blk_clear_rq_complete(rq); } static void mq_flush_data_run(struct work_struct *work) @@ -228,8 +230,10 @@ static void flush_end_io(struct request *flush_rq, int error) struct request *rq, *n; unsigned long flags = 0; - if (q->mq_ops) + if (q->mq_ops) { + blk_mq_finish_request(flush_rq, error); spin_lock_irqsave(&q->mq_flush_lock, flags); + } running = &q->flush_queue[q->flush_running_idx]; BUG_ON(q->flush_pending_idx == q->flush_running_idx); @@ -280,8 +284,9 @@ static void mq_flush_work(struct work_struct *work) q = container_of(work, struct request_queue, mq_flush_work); - /* We don't set REQ_FLUSH_SEQ, so rq will be freed after IO finish */ - rq = blk_mq_alloc_request(q, WRITE_FLUSH, __GFP_WAIT|GFP_ATOMIC); + /* We don't need set REQ_FLUSH_SEQ, it's for consistency */ + rq = blk_mq_alloc_request(q, WRITE_FLUSH|REQ_FLUSH_SEQ, + __GFP_WAIT|GFP_ATOMIC); rq->cmd_type = REQ_TYPE_FS; rq->end_io = flush_end_io; diff --git a/block/blk-mq.c b/block/blk-mq.c index 46dbd25..2ffd067 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -159,7 +159,7 @@ static void blk_mq_rq_init(struct blk_mq_hw_ctx *hctx, struct request *rq) { blk_rq_init(hctx->queue, rq); - if (hctx->rq_pdu) + if (hctx->cmd_size) rq->special = blk_mq_rq_to_pdu(rq); } @@ -185,7 +185,7 @@ void blk_mq_free_request(struct request *rq) } EXPORT_SYMBOL(blk_mq_free_request); -static void blk_mq_finish_request(struct request *rq, int error) +void blk_mq_finish_request(struct request *rq, int error) { struct bio *bio = rq->bio; unsigned int bytes = 0; @@ -1021,7 +1021,7 @@ static int blk_mq_init_rq_map(struct blk_mq_hw_ctx *hctx, * rq_size is the size of the request plus driver payload, rounded * to the cacheline size */ - rq_size = round_up(sizeof(struct request) + hctx->rq_pdu, + rq_size = round_up(sizeof(struct request) + hctx->cmd_size, cache_line_size()); left = rq_size * hctx->queue_depth; @@ -1103,7 +1103,7 @@ static int blk_mq_init_hw_queues(struct request_queue *q, hctx->queue_num = i; hctx->flags = reg->flags; hctx->queue_depth = reg->queue_depth; - hctx->rq_pdu = reg->rq_pdu; + hctx->cmd_size = reg->cmd_size; blk_mq_init_cpu_notifier(&hctx->cpu_notifier, blk_mq_hctx_notify, hctx); diff --git a/block/blk-mq.h b/block/blk-mq.h index 935e765..4d11154 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -26,6 +26,7 @@ void __blk_mq_end_io(struct request *rq, int error); void blk_mq_run_request(struct request *rq, bool run_queue, bool async); void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); void blk_mq_init_flush(struct request_queue *q); +void blk_mq_finish_request(struct request *rq, int error); /* * CPU hotplug helpers diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 69a7c8a..68d9021 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3583,7 +3583,7 @@ static struct blk_mq_reg mtip_mq_reg = { .nr_hw_queues = 1, .queue_depth = 256, .reserved_tags = 1, - .rq_pdu = sizeof(struct mtip_cmd), + .cmd_size = sizeof(struct mtip_cmd), .numa_node = NUMA_NO_NODE, .flags = BLK_MQ_F_SHOULD_MERGE, }; diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c index c64acd6..1ef54d8 100644 --- a/drivers/block/null_blk.c +++ b/drivers/block/null_blk.c @@ -379,7 +379,7 @@ static struct blk_mq_ops null_mq_ops = { static struct blk_mq_reg null_mq_reg = { .ops = &null_mq_ops, .queue_depth = 64, - .rq_pdu = sizeof(struct nullb_cmd), + .cmd_size = sizeof(struct nullb_cmd), .flags = BLK_MQ_F_SHOULD_MERGE, }; diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index f677939..53c2e6f 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -757,7 +757,7 @@ static int virtblk_probe(struct virtio_device *vdev) goto out_mempool; } - virtio_mq_reg.rq_pdu = pool_size; + virtio_mq_reg.cmd_size = pool_size; q = vblk->disk->queue = blk_mq_init_queue(&virtio_mq_reg, vblk); if (q) blk_mq_init_commands(q, virtblk_init_vbr, vblk); @@ -907,7 +907,8 @@ static void virtblk_remove(struct virtio_device *vdev) refc = atomic_read(&disk_to_dev(vblk->disk)->kobj.kref.refcount); put_disk(vblk->disk); - mempool_destroy(vblk->pool); + if (use_bio) + mempool_destroy(vblk->pool); vdev->config->del_vqs(vdev); kfree(vblk); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 49f7ad1..2fea261 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -43,7 +43,7 @@ struct blk_mq_hw_ctx { unsigned int queue_depth; unsigned int numa_node; - unsigned int rq_pdu; /* per-request extra data */ + unsigned int cmd_size; /* per-request extra data */ struct blk_mq_cpu_notifier cpu_notifier; struct kobject kobj; @@ -54,7 +54,7 @@ struct blk_mq_reg { unsigned int nr_hw_queues; unsigned int queue_depth; unsigned int reserved_tags; - unsigned int rq_pdu; /* per-request extra data */ + unsigned int cmd_size; /* per-request extra data */ int numa_node; unsigned int timeout; unsigned int flags; /* BLK_MQ_F_* */ -- 1.7.4.4