--- drivers/md/dm-crypt.c | 84 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 4 deletions(-) Index: linux-3.2-fast/drivers/md/dm-crypt.c =================================================================== --- linux-3.2-fast.orig/drivers/md/dm-crypt.c 2012-01-31 02:30:57.000000000 +0100 +++ linux-3.2-fast/drivers/md/dm-crypt.c 2012-01-31 02:30:59.000000000 +0100 @@ -47,6 +47,8 @@ struct dm_crypt_io { int error; sector_t sector; + + struct list_head list; }; struct dm_crypt_request { @@ -116,6 +118,10 @@ struct crypt_config { wait_queue_head_t crypt_thread_wait; struct list_head crypt_thread_list; + struct task_struct *write_thread; + wait_queue_head_t write_thread_wait; + struct list_head write_thread_list; + char *cipher; char *cipher_string; @@ -1042,9 +1048,8 @@ static int kcryptd_io_read(struct dm_cry return 0; } -static void kcryptd_io_write(struct work_struct *work) +static void kcryptd_io_write(struct dm_crypt_io *io) { - struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); struct crypt_config *cc = io->cc; struct bio *clone = io->bio_out; @@ -1060,12 +1065,68 @@ static void kcryptd_io_write(struct work generic_make_request(clone); } +static int dmcrypt_write(void *data) +{ + struct crypt_config *cc = data; + while (1) { + struct dm_crypt_io *dmreqs[DMREQ_PULL_BATCH]; + unsigned n_dmreqs; + unsigned i; + + DECLARE_WAITQUEUE(wait, current); + + spin_lock_irq(&cc->write_thread_wait.lock); + + if (!list_empty(&cc->write_thread_list)) + goto pop_from_list; + + __set_current_state(TASK_INTERRUPTIBLE); + __add_wait_queue_exclusive(&cc->write_thread_wait, &wait); + spin_unlock_irq(&cc->write_thread_wait.lock); + + if (unlikely(kthread_should_stop())) { + set_task_state(current, TASK_RUNNING); + remove_wait_queue(&cc->write_thread_wait, &wait); + break; + } + + schedule(); + + set_task_state(current, TASK_RUNNING); + remove_wait_queue(&cc->write_thread_wait, &wait); + continue; + +pop_from_list: + n_dmreqs = 0; + do { + struct dm_crypt_io *io = container_of( + cc->write_thread_list.next, + struct dm_crypt_io, list); + list_del(&io->list); + dmreqs[n_dmreqs++] = io; + } while (n_dmreqs < DMREQ_PULL_BATCH && + !list_empty(&cc->write_thread_list)); + + spin_unlock_irq(&cc->write_thread_wait.lock); + + i = 0; + do { + struct dm_crypt_io *io = dmreqs[i]; + kcryptd_io_write(io); + } while (++i < n_dmreqs); + } + return 0; +} + static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io) { struct crypt_config *cc = io->cc; + unsigned long flags; - INIT_WORK(&io->work, kcryptd_io_write); - queue_work(cc->io_queue, &io->work); + spin_lock_irqsave(&cc->write_thread_wait.lock, flags); + list_add_tail(&io->list, &cc->write_thread_list); + wake_up_locked(&cc->write_thread_wait); + spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags); } static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) @@ -1275,6 +1336,9 @@ static void crypt_dtr(struct dm_target * kfree(cc->crypt_threads); } + if (cc->write_thread) + kthread_stop(cc->write_thread); + if (cc->io_queue) destroy_workqueue(cc->io_queue); @@ -1625,6 +1689,18 @@ static int crypt_ctr(struct dm_target *t } } + init_waitqueue_head(&cc->write_thread_wait); + INIT_LIST_HEAD(&cc->write_thread_list); + + cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write"); + if (IS_ERR(cc->write_thread)) { + ret = PTR_ERR(cc->write_thread); + cc->write_thread = NULL; + ti->error = "Couldn't spawn write thread"; + goto bad; + } + wake_up_process(cc->write_thread); + ti->num_flush_requests = 1; ti->discard_zeroes_data_unsupported = 1;