dm: count sync and async requests separatedly

Change the pending field, so that it has separate counters for read/write
sync/async bios. We need to differentiate sync and async bios when we
impose limits on the number of pending bios, so that the async bios do not
stall sync bios.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>

---
 drivers/md/dm.c |   43 +++++++++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 16 deletions(-)

Index: linux-4.7-rc1/drivers/md/dm.c
===================================================================
--- linux-4.7-rc1.orig/drivers/md/dm.c	2016-05-30 18:01:33.000000000 +0200
+++ linux-4.7-rc1/drivers/md/dm.c	2016-05-30 18:01:35.000000000 +0200
@@ -160,7 +160,7 @@ struct mapped_device {
 	/*
 	 * A list of ios that arrived while we were suspended.
 	 */
-	atomic_t pending[2];
+	atomic_t pending[2][2];
 	wait_queue_head_t wait;
 	struct work_struct work;
 	spinlock_t deferred_lock;
@@ -703,8 +703,10 @@ static void free_old_clone_request(struc
 
 static int md_in_flight(struct mapped_device *md)
 {
-	return atomic_read(&md->pending[READ]) +
-	       atomic_read(&md->pending[WRITE]);
+	return atomic_read(&md->pending[READ][0]) +
+	       atomic_read(&md->pending[READ][1]) +
+	       atomic_read(&md->pending[WRITE][0]) +
+	       atomic_read(&md->pending[WRITE][1]);
 }
 
 /*
@@ -714,15 +716,15 @@ static int md_in_flight(struct mapped_de
 static void update_in_flight(struct mapped_device *md, int rw)
 {
 	atomic_set(&dm_disk(md)->part0.in_flight[rw],
-		   atomic_read(&md->pending[rw]));
+		   atomic_read(&md->pending[rw][0]) +
+		   atomic_read(&md->pending[rw][1]));
 }
 
 static void start_io_acct(struct dm_io *io)
 {
 	struct mapped_device *md = io->md;
 	struct bio *bio = io->bio;
-	int cpu;
-	int rw = bio_data_dir(bio);
+	int cpu, rw, sync;
 
 	io->start_time = jiffies;
 
@@ -730,7 +732,10 @@ static void start_io_acct(struct dm_io *
 	part_round_stats(cpu, &dm_disk(md)->part0);
 	part_stat_unlock();
 
-	atomic_inc(&md->pending[rw]);
+	rw = bio_data_dir(bio);
+	sync = rw_is_sync(bio->bi_rw);
+
+	atomic_inc(&md->pending[rw][sync]);
 	update_in_flight(md, rw);
 
 	if (unlikely(dm_stats_used(&md->stats)))
@@ -743,24 +748,28 @@ static void end_io_acct(struct dm_io *io
 	struct mapped_device *md = io->md;
 	struct bio *bio = io->bio;
 	unsigned long duration = jiffies - io->start_time;
-	int pending;
-	int rw = bio_data_dir(bio);
+	int pending, rw, sync;
 
-	generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);
+	generic_end_io_acct(bio_data_dir(bio), &dm_disk(md)->part0, io->start_time);
 
 	if (unlikely(dm_stats_used(&md->stats)))
 		dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector,
 				    bio_sectors(bio), true, duration, &io->stats_aux);
 
+	rw = bio_data_dir(bio);
+	sync = rw_is_sync(bio->bi_rw);
+
 	/*
 	 * After this is decremented the bio must not be touched if it is
 	 * a flush.
 	 */
-	pending = atomic_dec_return(&md->pending[rw]);
+	pending = atomic_dec_return(&md->pending[rw][sync]);
 
 	update_in_flight(md, rw);
 
-	pending += atomic_read(&md->pending[rw^0x1]);
+	pending += atomic_read(&md->pending[rw][sync^1]);
+	pending += atomic_read(&md->pending[rw^1][sync]);
+	pending += atomic_read(&md->pending[rw^1][sync^1]);
 
 	/* nudge anyone waiting on suspend queue */
 	if (!pending)
@@ -1147,7 +1156,7 @@ static void rq_end_stats(struct mapped_d
  */
 static void rq_completed(struct mapped_device *md, int rw, bool run_queue)
 {
-	atomic_dec(&md->pending[rw]);
+	atomic_dec(&md->pending[rw][0]);
 
 	/* nudge anyone waiting on suspend queue */
 	if (!md_in_flight(md))
@@ -2118,7 +2127,7 @@ static void dm_start_request(struct mapp
 		blk_start_request(orig);
 	else
 		blk_mq_start_request(orig);
-	atomic_inc(&md->pending[rq_data_dir(orig)]);
+	atomic_inc(&md->pending[rq_data_dir(orig)][0]);
 
 	if (md->seq_rq_merge_deadline_usecs) {
 		md->last_rq_pos = rq_end_sector(orig);
@@ -2433,8 +2442,10 @@ static struct mapped_device *alloc_dev(i
 	if (!md->disk)
 		goto bad;
 
-	atomic_set(&md->pending[0], 0);
-	atomic_set(&md->pending[1], 0);
+	atomic_set(&md->pending[0][0], 0);
+	atomic_set(&md->pending[0][1], 0);
+	atomic_set(&md->pending[1][0], 0);
+	atomic_set(&md->pending[1][1], 0);
 	init_waitqueue_head(&md->wait);
 	INIT_WORK(&md->work, dm_wq_work);
 	init_waitqueue_head(&md->eventq);