From: Ben Slusky The attached patch changes the loop device transfer functions (including cryptoloop transfers) to accept page/offset pairs instead of virtual addresses, and removes the redundant kmaps in do_lo_send, do_lo_receive, and loop_transfer_bio. Per Andrew Morton's request a while back. drivers/block/cryptoloop.c | 80 ++++++++++++++++++++++--------------- drivers/block/loop.c | 95 ++++++++++++++++++++++++--------------------- include/linux/loop.h | 11 +++-- 3 files changed, 107 insertions(+), 79 deletions(-) diff -puN drivers/block/cryptoloop.c~loop-highmem drivers/block/cryptoloop.c --- 25/drivers/block/cryptoloop.c~loop-highmem 2003-12-29 22:31:22.000000000 -0800 +++ 25-akpm/drivers/block/cryptoloop.c 2003-12-29 22:31:22.000000000 -0800 @@ -87,43 +87,49 @@ typedef int (*encdec_ecb_t)(struct crypt static int -cryptoloop_transfer_ecb(struct loop_device *lo, int cmd, char *raw_buf, - char *loop_buf, int size, sector_t IV) +cryptoloop_transfer_ecb(struct loop_device *lo, int cmd, + struct page *raw_page, unsigned raw_off, + struct page *loop_page, unsigned loop_off, + int size, sector_t IV) { struct crypto_tfm *tfm = (struct crypto_tfm *) lo->key_data; struct scatterlist sg_out = { 0, }; struct scatterlist sg_in = { 0, }; encdec_ecb_t encdecfunc; - char const *in; - char *out; + struct page *in_page, *out_page; + unsigned in_offs, out_offs; if (cmd == READ) { - in = raw_buf; - out = loop_buf; + in_page = raw_page; + in_offs = raw_off; + out_page = loop_page; + out_offs = loop_off; encdecfunc = tfm->crt_u.cipher.cit_decrypt; } else { - in = loop_buf; - out = raw_buf; + in_page = loop_page; + in_offs = loop_off; + out_page = raw_page; + out_offs = raw_off; encdecfunc = tfm->crt_u.cipher.cit_encrypt; } while (size > 0) { const int sz = min(size, LOOP_IV_SECTOR_SIZE); - sg_in.page = virt_to_page(in); - sg_in.offset = (unsigned long)in & ~PAGE_MASK; + sg_in.page = in_page; + sg_in.offset = in_offs; sg_in.length = sz; - sg_out.page = virt_to_page(out); - sg_out.offset = (unsigned long)out & ~PAGE_MASK; + sg_out.page = out_page; + sg_out.offset = out_offs; sg_out.length = sz; encdecfunc(tfm, &sg_out, &sg_in, sz); size -= sz; - in += sz; - out += sz; + in_offs += sz; + out_offs += sz; } return 0; @@ -135,24 +141,30 @@ typedef int (*encdec_cbc_t)(struct crypt unsigned int nsg, u8 *iv); static int -cryptoloop_transfer_cbc(struct loop_device *lo, int cmd, char *raw_buf, - char *loop_buf, int size, sector_t IV) +cryptoloop_transfer_cbc(struct loop_device *lo, int cmd, + struct page *raw_page, unsigned raw_off, + struct page *loop_page, unsigned loop_off, + int size, sector_t IV) { struct crypto_tfm *tfm = (struct crypto_tfm *) lo->key_data; struct scatterlist sg_out = { 0, }; struct scatterlist sg_in = { 0, }; encdec_cbc_t encdecfunc; - char const *in; - char *out; + struct page *in_page, *out_page; + unsigned in_offs, out_offs; if (cmd == READ) { - in = raw_buf; - out = loop_buf; + in_page = raw_page; + in_offs = raw_off; + out_page = loop_page; + out_offs = loop_off; encdecfunc = tfm->crt_u.cipher.cit_decrypt_iv; } else { - in = loop_buf; - out = raw_buf; + in_page = loop_page; + in_offs = loop_off; + out_page = raw_page; + out_offs = raw_off; encdecfunc = tfm->crt_u.cipher.cit_encrypt_iv; } @@ -161,39 +173,43 @@ cryptoloop_transfer_cbc(struct loop_devi u32 iv[4] = { 0, }; iv[0] = cpu_to_le32(IV & 0xffffffff); - sg_in.page = virt_to_page(in); - sg_in.offset = offset_in_page(in); + sg_in.page = in_page; + sg_in.offset = in_offs; sg_in.length = sz; - sg_out.page = virt_to_page(out); - sg_out.offset = offset_in_page(out); + sg_out.page = out_page; + sg_out.offset = out_offs; sg_out.length = sz; encdecfunc(tfm, &sg_out, &sg_in, sz, (u8 *)iv); IV++; size -= sz; - in += sz; - out += sz; + in_offs += sz; + out_offs += sz; } return 0; } static int -cryptoloop_transfer(struct loop_device *lo, int cmd, char *raw_buf, - char *loop_buf, int size, sector_t IV) +cryptoloop_transfer(struct loop_device *lo, int cmd, + struct page *raw_page, unsigned raw_off, + struct page *loop_page, unsigned loop_off, + int size, sector_t IV) { struct crypto_tfm *tfm = (struct crypto_tfm *) lo->key_data; if(tfm->crt_cipher.cit_mode == CRYPTO_TFM_MODE_ECB) { lo->transfer = cryptoloop_transfer_ecb; - return cryptoloop_transfer_ecb(lo, cmd, raw_buf, loop_buf, size, IV); + return cryptoloop_transfer_ecb(lo, cmd, raw_page, raw_off, + loop_page, loop_off, size, IV); } if(tfm->crt_cipher.cit_mode == CRYPTO_TFM_MODE_CBC) { lo->transfer = cryptoloop_transfer_cbc; - return cryptoloop_transfer_cbc(lo, cmd, raw_buf, loop_buf, size, IV); + return cryptoloop_transfer_cbc(lo, cmd, raw_page, raw_off, + loop_page, loop_off, size, IV); } /* This is not supposed to happen */ diff -puN drivers/block/loop.c~loop-highmem drivers/block/loop.c --- 25/drivers/block/loop.c~loop-highmem 2003-12-29 22:31:22.000000000 -0800 +++ 25-akpm/drivers/block/loop.c 2003-12-29 22:31:22.000000000 -0800 @@ -76,24 +76,34 @@ static struct gendisk **disks; /* * Transfer functions */ -static int transfer_none(struct loop_device *lo, int cmd, char *raw_buf, - char *loop_buf, int size, sector_t real_block) +static int transfer_none(struct loop_device *lo, int cmd, + struct page *raw_page, unsigned raw_off, + struct page *loop_page, unsigned loop_off, + int size, sector_t real_block) { - if (raw_buf != loop_buf) { - if (cmd == READ) - memcpy(loop_buf, raw_buf, size); - else - memcpy(raw_buf, loop_buf, size); - } + char *raw_buf = kmap_atomic(raw_page, KM_USER0) + raw_off; + char *loop_buf = kmap_atomic(loop_page, KM_USER1) + loop_off; + if (cmd == READ) + memcpy(loop_buf, raw_buf, size); + else + memcpy(raw_buf, loop_buf, size); + + kunmap_atomic(raw_buf, KM_USER0); + kunmap_atomic(loop_buf, KM_USER1); + cond_resched(); return 0; } -static int transfer_xor(struct loop_device *lo, int cmd, char *raw_buf, - char *loop_buf, int size, sector_t real_block) -{ - char *in, *out, *key; - int i, keysize; +static int transfer_xor(struct loop_device *lo, int cmd, + struct page *raw_page, unsigned raw_off, + struct page *loop_page, unsigned loop_off, + int size, sector_t real_block) +{ + char *raw_buf = kmap_atomic(raw_page, KM_USER0) + raw_off; + char *loop_buf = kmap_atomic(loop_page, KM_USER1) + loop_off; + char *in, *out, *key; + int i, keysize; if (cmd == READ) { in = raw_buf; @@ -107,6 +117,10 @@ static int transfer_xor(struct loop_devi keysize = lo->lo_encrypt_key_size; for (i = 0; i < size; i++) *out++ = *in++ ^ key[(i & 511) % keysize]; + + kunmap_atomic(raw_buf, KM_USER0); + kunmap_atomic(loop_buf, KM_USER1); + cond_resched(); return 0; } @@ -162,13 +176,15 @@ figure_loop_size(struct loop_device *lo) } static inline int -lo_do_transfer(struct loop_device *lo, int cmd, char *rbuf, - char *lbuf, int size, sector_t rblock) +lo_do_transfer(struct loop_device *lo, int cmd, + struct page *rpage, unsigned roffs, + struct page *lpage, unsigned loffs, + int size, sector_t rblock) { if (!lo->transfer) return 0; - return lo->transfer(lo, cmd, rbuf, lbuf, size, rblock); + return lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock); } static int @@ -178,16 +194,15 @@ do_lo_send(struct loop_device *lo, struc struct address_space *mapping = file->f_mapping; struct address_space_operations *aops = mapping->a_ops; struct page *page; - char *kaddr, *data; pgoff_t index; - unsigned size, offset; + unsigned size, offset, bv_offs; int len; int ret = 0; down(&mapping->host->i_sem); index = pos >> PAGE_CACHE_SHIFT; offset = pos & ((pgoff_t)PAGE_CACHE_SIZE - 1); - data = kmap(bvec->bv_page) + bvec->bv_offset; + bv_offs = bvec->bv_offset; len = bvec->bv_len; while (len > 0) { sector_t IV; @@ -204,25 +219,28 @@ do_lo_send(struct loop_device *lo, struc goto fail; if (aops->prepare_write(file, page, offset, offset+size)) goto unlock; - kaddr = kmap(page); - transfer_result = lo_do_transfer(lo, WRITE, kaddr + offset, - data, size, IV); + transfer_result = lo_do_transfer(lo, WRITE, page, offset, + bvec->bv_page, bv_offs, + size, IV); if (transfer_result) { + char *kaddr; + /* * The transfer failed, but we still write the data to * keep prepare/commit calls balanced. */ printk(KERN_ERR "loop: transfer error block %llu\n", (unsigned long long)index); + kaddr = kmap_atomic(page, KM_USER0); memset(kaddr + offset, 0, size); + kunmap_atomic(kaddr, KM_USER0); } flush_dcache_page(page); - kunmap(page); if (aops->commit_write(file, page, offset, offset+size)) goto unlock; if (transfer_result) goto unlock; - data += size; + bv_offs += size; len -= size; offset = 0; index++; @@ -232,7 +250,6 @@ do_lo_send(struct loop_device *lo, struc } up(&mapping->host->i_sem); out: - kunmap(bvec->bv_page); return ret; unlock: @@ -261,7 +278,8 @@ lo_send(struct loop_device *lo, struct b struct lo_read_data { struct loop_device *lo; - char *data; + struct page *page; + unsigned offset; int bsize; }; @@ -269,7 +287,6 @@ static int lo_read_actor(read_descriptor_t *desc, struct page *page, unsigned long offset, unsigned long size) { - char *kaddr; unsigned long count = desc->count; struct lo_read_data *p = (struct lo_read_data*)desc->buf; struct loop_device *lo = p->lo; @@ -280,18 +297,16 @@ lo_read_actor(read_descriptor_t *desc, s if (size > count) size = count; - kaddr = kmap(page); - if (lo_do_transfer(lo, READ, kaddr + offset, p->data, size, IV)) { + if (lo_do_transfer(lo, READ, page, offset, p->page, p->offset, size, IV)) { size = 0; printk(KERN_ERR "loop: transfer error block %ld\n", page->index); desc->error = -EINVAL; } - kunmap(page); desc->count = count - size; desc->written += size; - p->data += size; + p->offset += size; return size; } @@ -304,12 +319,12 @@ do_lo_receive(struct loop_device *lo, int retval; cookie.lo = lo; - cookie.data = kmap(bvec->bv_page) + bvec->bv_offset; + cookie.page = bvec->bv_page; + cookie.offset = bvec->bv_offset; cookie.bsize = bsize; file = lo->lo_backing_file; retval = file->f_op->sendfile(file, &pos, bvec->bv_len, lo_read_actor, &cookie); - kunmap(bvec->bv_page); return (retval < 0)? retval: 0; } @@ -495,22 +510,16 @@ static int loop_transfer_bio(struct loop { sector_t IV; struct bio_vec *from_bvec, *to_bvec; - char *vto, *vfrom; int ret = 0, i; IV = from_bio->bi_sector + (lo->lo_offset >> 9); bio_for_each_segment(from_bvec, from_bio, i) { to_bvec = &to_bio->bi_io_vec[i]; - - kmap(from_bvec->bv_page); - kmap(to_bvec->bv_page); - vfrom = page_address(from_bvec->bv_page) + from_bvec->bv_offset; - vto = page_address(to_bvec->bv_page) + to_bvec->bv_offset; - ret |= lo_do_transfer(lo, bio_data_dir(to_bio), vto, vfrom, - from_bvec->bv_len, IV); - kunmap(from_bvec->bv_page); - kunmap(to_bvec->bv_page); + ret |= lo_do_transfer(lo, bio_data_dir(to_bio), + to_bvec->bv_page, to_bvec->bv_offset, + from_bvec->bv_page, from_bvec->bv_offset, + from_bvec->bv_len, IV); IV += from_bvec->bv_len >> 9; } diff -puN include/linux/loop.h~loop-highmem include/linux/loop.h --- 25/include/linux/loop.h~loop-highmem 2003-12-29 22:31:22.000000000 -0800 +++ 25-akpm/include/linux/loop.h 2003-12-29 22:31:22.000000000 -0800 @@ -34,8 +34,9 @@ struct loop_device { loff_t lo_sizelimit; int lo_flags; int (*transfer)(struct loop_device *, int cmd, - char *raw_buf, char *loop_buf, int size, - sector_t real_block); + struct page *raw_page, unsigned raw_off, + struct page *loop_page, unsigned loop_off, + int size, sector_t real_block); char lo_file_name[LO_NAME_SIZE]; char lo_crypt_name[LO_NAME_SIZE]; char lo_encrypt_key[LO_KEY_SIZE]; @@ -128,8 +129,10 @@ struct loop_info64 { /* Support for loadable transfer modules */ struct loop_func_table { int number; /* filter type */ - int (*transfer)(struct loop_device *lo, int cmd, char *raw_buf, - char *loop_buf, int size, sector_t real_block); + int (*transfer)(struct loop_device *lo, int cmd, + struct page *raw_page, unsigned raw_off, + struct page *loop_page, unsigned loop_off, + int size, sector_t real_block); int (*init)(struct loop_device *, const struct loop_info64 *); /* release is called from loop_unregister_transfer or clr_fd */ int (*release)(struct loop_device *); _