dm optimize: introduce dm_get_live_table_fast and dm_put_live_table_fast

If the code needs to take a current dm_table it uses dm_get_live_table
and dm_table_put. dm_get_live_table  increases a table reference count.
dm_table_put decreases the table reference count.

The table reference count is changed by every process doing I/O, so it
subject to cache line bouncing between multiple CPUs.

This patch introduces functions dm_get_live_table_fast and
dm_put_live_table_fast. dm_get_live_table_fast takes the rcu read lock
and returns a dm table (without changing table reference count).
dm_put_live_table_fast drops the rcu read lock. Consequently, there is
no cache line bouncing.

These fast functions can be used if the caller does not sleep between
dm_get_live_table_fast and dm_put_live_table_fast. If the caller could
sleep, old functions dm_get_live_table and dm_table_put must be used.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>

---
 drivers/md/dm.c |   36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

Index: linux-3.3-fast/drivers/md/dm.c
===================================================================
--- linux-3.3-fast.orig/drivers/md/dm.c	2012-04-21 03:21:56.000000000 +0200
+++ linux-3.3-fast/drivers/md/dm.c	2012-04-21 03:25:36.000000000 +0200
@@ -679,6 +679,26 @@ struct dm_table *dm_get_live_table(struc
 }
 
 /*
+ * A fast alternative of dm_get_live_table.
+ *
+ * Use dm_put_live_table_fast to release the table. dm_put_live_table_fast must
+ * be called in all cases, regardless if this function returns NULL or not.
+ *
+ * This function doesn't increase table reference count, but holds rcu instead.
+ * The caller must not sleep untill dm_put_live_table_fast is called.
+ */
+struct dm_table *dm_get_live_table_fast(struct mapped_device *md)
+{
+	rcu_read_lock();
+	return rcu_dereference(md->map);
+}
+
+void dm_put_live_table_fast(struct mapped_device *md)
+{
+	rcu_read_unlock();
+}
+
+/*
  * Get the geometry associated with a dm device
  */
 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
@@ -1465,7 +1485,7 @@ static int dm_merge_bvec(struct request_
 			 struct bio_vec *biovec)
 {
 	struct mapped_device *md = q->queuedata;
-	struct dm_table *map = dm_get_live_table(md);
+	struct dm_table *map = dm_get_live_table_fast(md);
 	struct dm_target *ti;
 	sector_t max_sectors;
 	int max_size = 0;
@@ -1475,7 +1495,7 @@ static int dm_merge_bvec(struct request_
 
 	ti = dm_table_find_target(map, bvm->bi_sector);
 	if (!dm_target_is_valid(ti))
-		goto out_table;
+		goto out;
 
 	/*
 	 * Find maximum amount of I/O that won't need splitting
@@ -1504,10 +1524,10 @@ static int dm_merge_bvec(struct request_
 
 		max_size = 0;
 
-out_table:
-	dm_table_put(map);
 
 out:
+	dm_put_live_table_fast(md);
+
 	/*
 	 * Always allow an entire first page
 	 */
@@ -1798,14 +1818,14 @@ static int dm_lld_busy(struct request_qu
 {
 	int r;
 	struct mapped_device *md = q->queuedata;
-	struct dm_table *map = dm_get_live_table(md);
+	struct dm_table *map = dm_get_live_table_fast(md);
 
 	if (!map || test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))
 		r = 1;
 	else
 		r = dm_table_any_busy_target(map);
 
-	dm_table_put(map);
+	dm_put_live_table_fast(md);
 
 	return r;
 }
@@ -1817,7 +1837,7 @@ static int dm_any_congested(void *conges
 	struct dm_table *map;
 
 	if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
-		map = dm_get_live_table(md);
+		map = dm_get_live_table_fast(md);
 		if (map) {
 			/*
 			 * Request-based dm cares about only own queue for
@@ -1829,8 +1849,8 @@ static int dm_any_congested(void *conges
 			else
 				r = dm_table_any_congested(map, bdi_bits);
 
-			dm_table_put(map);
 		}
+		dm_put_live_table_fast(md);
 	}
 
 	return r;