dm: count sync and async requests separatedly

Change the pending field, so that it has separate counters for read/write
sync/async bios. We need to differentiate sync and async bios when we
impose limits on the number of pending bios, so that the async bios do not
stall sync bios.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>

---
 drivers/md/dm-core.h |    2 +-
 drivers/md/dm-rq.c   |    4 ++--
 drivers/md/dm.c      |   30 ++++++++++++++++++++----------
 3 files changed, 23 insertions(+), 13 deletions(-)

Index: linux-4.13-rc5/drivers/md/dm.c
===================================================================
--- linux-4.13-rc5.orig/drivers/md/dm.c
+++ linux-4.13-rc5/drivers/md/dm.c
@@ -506,8 +506,10 @@ static void free_tio(struct dm_target_io
 
 int md_in_flight(struct mapped_device *md)
 {
-	return atomic_read(&md->pending[READ]) +
-	       atomic_read(&md->pending[WRITE]);
+	return atomic_read(&md->pending[READ][0]) +
+	       atomic_read(&md->pending[READ][1]) +
+	       atomic_read(&md->pending[WRITE][0]) +
+	       atomic_read(&md->pending[WRITE][1]);
 }
 
 /*
@@ -517,15 +519,15 @@ int md_in_flight(struct mapped_device *m
 static void update_in_flight(struct mapped_device *md, int rw)
 {
 	atomic_set(&dm_disk(md)->part0.in_flight[rw],
-		   atomic_read(&md->pending[rw]));
+		   atomic_read(&md->pending[rw][0]) +
+		   atomic_read(&md->pending[rw][1]));
 }
 
 static void start_io_acct(struct dm_io *io)
 {
 	struct mapped_device *md = io->md;
 	struct bio *bio = io->bio;
-	int cpu;
-	int rw = bio_data_dir(bio);
+	int cpu, rw, sync;
 
 	io->start_time = jiffies;
 
@@ -533,7 +535,10 @@ static void start_io_acct(struct dm_io *
 	part_round_stats(cpu, &dm_disk(md)->part0);
 	part_stat_unlock();
 
-	atomic_inc(&md->pending[rw]);
+	rw = bio_data_dir(bio);
+	sync = op_is_sync(bio->bi_opf);
+
+	atomic_inc(&md->pending[rw][sync]);
 	update_in_flight(md, rw);
 
 	if (unlikely(dm_stats_used(&md->stats)))
@@ -549,6 +554,7 @@ static void end_io_acct(struct dm_io *io
 	unsigned long duration = jiffies - io->start_time;
 	int pending;
 	int rw = bio_data_dir(bio);
+	int sync = op_is_sync(bio->bi_opf);
 
 	generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);
 
@@ -561,11 +567,13 @@ static void end_io_acct(struct dm_io *io
 	 * After this is decremented the bio must not be touched if it is
 	 * a flush.
 	 */
-	pending = atomic_dec_return(&md->pending[rw]);
+	pending = atomic_dec_return(&md->pending[rw][sync]);
 
 	update_in_flight(md, rw);
 
-	pending += atomic_read(&md->pending[rw^0x1]);
+	pending += atomic_read(&md->pending[rw][sync^1]);
+	pending += atomic_read(&md->pending[rw^1][sync]);
+	pending += atomic_read(&md->pending[rw^1][sync^1]);
 
 	/* nudge anyone waiting on suspend queue */
 	if (!pending)
@@ -1805,8 +1813,10 @@ static struct mapped_device *alloc_dev(i
 	if (!md->disk)
 		goto bad;
 
-	atomic_set(&md->pending[0], 0);
-	atomic_set(&md->pending[1], 0);
+	atomic_set(&md->pending[0][0], 0);
+	atomic_set(&md->pending[0][1], 0);
+	atomic_set(&md->pending[1][0], 0);
+	atomic_set(&md->pending[1][1], 0);
 	init_waitqueue_head(&md->wait);
 	INIT_WORK(&md->work, dm_wq_work);
 	init_waitqueue_head(&md->eventq);
Index: linux-4.13-rc5/drivers/md/dm-core.h
===================================================================
--- linux-4.13-rc5.orig/drivers/md/dm-core.h
+++ linux-4.13-rc5/drivers/md/dm-core.h
@@ -66,7 +66,7 @@ struct mapped_device {
 	/*
 	 * A list of ios that arrived while we were suspended.
 	 */
-	atomic_t pending[2];
+	atomic_t pending[2][2];
 	wait_queue_head_t wait;
 	struct work_struct work;
 	spinlock_t deferred_lock;
Index: linux-4.13-rc5/drivers/md/dm-rq.c
===================================================================
--- linux-4.13-rc5.orig/drivers/md/dm-rq.c
+++ linux-4.13-rc5/drivers/md/dm-rq.c
@@ -187,7 +187,7 @@ static void rq_completed(struct mapped_d
 	struct request_queue *q = md->queue;
 	unsigned long flags;
 
-	atomic_dec(&md->pending[rw]);
+	atomic_dec(&md->pending[rw][0]);
 
 	/* nudge anyone waiting on suspend queue */
 	if (!md_in_flight(md))
@@ -522,7 +522,7 @@ static void dm_start_request(struct mapp
 		blk_start_request(orig);
 	else
 		blk_mq_start_request(orig);
-	atomic_inc(&md->pending[rq_data_dir(orig)]);
+	atomic_inc(&md->pending[rq_data_dir(orig)][0]);
 
 	if (md->seq_rq_merge_deadline_usecs) {
 		md->last_rq_pos = rq_end_sector(orig);