iomap: support RWF_UNCACHED for buffered writesno-cache-io

This adds support for RWF_UNCACHED for file systems using iomap to
perform buffered writes. We use the generic infrastructure for this,
by tracking pages we created and calling write_drop_cached_pages()
to issue writeback and prune those pages.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

6 files changed, 86 insertions, 11 deletions

diff --git a/fs/iomap/apply.c b/fs/iomap/apply.c
index e76148db03b8f7..f2f87fb412d56b 100644
--- a/fs/iomap/apply.c
+++ b/fs/iomap/apply.c
@@ -24,7 +24,10 @@ loff_t
 iomap_apply(struct iomap_data *data, const struct iomap_ops *ops,
 	    iomap_actor_t actor)
 {
-	struct iomap iomap = { .type = IOMAP_HOLE };
+	struct iomap iomap = {
+		.type		= IOMAP_HOLE,
+		.page_list	= LIST_HEAD_INIT(iomap.page_list)
+	};
 	struct iomap srcmap = { .type = IOMAP_HOLE };
 	loff_t written = 0, ret;
 	u64 end;
@@ -92,5 +95,8 @@ iomap_apply(struct iomap_data *data, const struct iomap_ops *ops,
 				     data->flags, &iomap);
 	}
 
+	if (!list_empty(&iomap.page_list))
+		uncached_write_pages(data->inode->i_mapping, &iomap.page_list);
+
 	return written ? written : ret;
 }
diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
index 0a1a195ed1cc25..681607807d8021 100644
--- a/fs/iomap/buffered-io.c
+++ b/fs/iomap/buffered-io.c
@@ -582,6 +582,7 @@ EXPORT_SYMBOL_GPL(iomap_migrate_page);
 
 enum {
 	IOMAP_WRITE_F_UNSHARE		= (1 << 0),
+	IOMAP_WRITE_F_UNCACHED		= (1 << 1),
 };
 
 static void
@@ -659,6 +660,7 @@ iomap_write_begin(struct inode *inode, loff_t pos, unsigned len, unsigned flags,
 		struct page **pagep, struct iomap *iomap, struct iomap *srcmap)
 {
 	const struct iomap_page_ops *page_ops = iomap->page_ops;
+	unsigned aop_flags;
 	struct page *page;
 	int status = 0;
 
@@ -675,8 +677,11 @@ iomap_write_begin(struct inode *inode, loff_t pos, unsigned len, unsigned flags,
 			return status;
 	}
 
+	aop_flags = AOP_FLAG_NOFS;
+	if (flags & IOMAP_WRITE_F_UNCACHED)
+		aop_flags |= AOP_FLAG_UNCACHED;
 	page = grab_cache_page_write_begin(inode->i_mapping, pos >> PAGE_SHIFT,
-			AOP_FLAG_NOFS);
+						aop_flags);
 	if (!page) {
 		status = -ENOMEM;
 		goto out_no_page;
@@ -820,9 +825,13 @@ iomap_write_actor(const struct iomap_data *data, struct iomap *iomap,
 	struct iov_iter *i = data->priv;
 	loff_t length = data->len;
 	loff_t pos = data->pos;
+	unsigned flags = 0;
 	long status = 0;
 	ssize_t written = 0;
 
+	if (data->flags & IOMAP_UNCACHED)
+		flags |= IOMAP_WRITE_F_UNCACHED;
+
 	do {
 		struct page *page;
 		unsigned long offset;	/* Offset into pagecache page */
@@ -851,8 +860,8 @@ again:
 			break;
 		}
 
-		status = iomap_write_begin(inode, pos, bytes, 0, &page, iomap,
-				srcmap);
+		status = iomap_write_begin(inode, pos, bytes, flags,
+						&page, iomap, srcmap);
 		if (unlikely(status))
 			break;
 
@@ -889,7 +898,10 @@ again:
 		written += copied;
 		length -= copied;
 
-		balance_dirty_pages_ratelimited(inode->i_mapping);
+		if (!PagePrivio(page))
+			balance_dirty_pages_ratelimited(inode->i_mapping);
+		else
+			list_add_tail(&page->lru, &iomap->page_list);
 	} while (iov_iter_count(i) && length);
 
 	return written ? written : status;
@@ -907,6 +919,9 @@ iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *iter,
 	};
 	loff_t ret = 0, written = 0;
 
+	if (iocb->ki_flags & IOCB_UNCACHED)
+		data.flags |= IOMAP_UNCACHED;
+
 	while (iov_iter_count(iter)) {
 		data.len = iov_iter_count(iter);
 		ret = iomap_apply(&data, ops, iomap_write_actor);
diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h
index 6dc227b8c47ecc..63c771e3eef547 100644
--- a/fs/iomap/trace.h
+++ b/fs/iomap/trace.h
@@ -93,7 +93,8 @@ DEFINE_PAGE_EVENT(iomap_invalidatepage);
 	{ IOMAP_REPORT,		"REPORT" }, \
 	{ IOMAP_FAULT,		"FAULT" }, \
 	{ IOMAP_DIRECT,		"DIRECT" }, \
-	{ IOMAP_NOWAIT,		"NOWAIT" }
+	{ IOMAP_NOWAIT,		"NOWAIT" }, \
+	{ IOMAP_UNCACHED,	"UNCACHED" }
 
 #define IOMAP_F_FLAGS_STRINGS \
 	{ IOMAP_F_NEW,		"NEW" }, \
@@ -101,6 +102,7 @@ DEFINE_PAGE_EVENT(iomap_invalidatepage);
 	{ IOMAP_F_SHARED,	"SHARED" }, \
 	{ IOMAP_F_MERGED,	"MERGED" }, \
 	{ IOMAP_F_BUFFER_HEAD,	"BH" }, \
+	{ IOMAP_F_PAGE_CREATE,	"PAGE_CREATE" }, \
 	{ IOMAP_F_SIZE_CHANGED,	"SIZE_CHANGED" }
 
 DECLARE_EVENT_CLASS(iomap_class,
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 5ea5fc167524b2..36b96c98ddd488 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -3107,6 +3107,8 @@ extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *);
 extern ssize_t generic_perform_write(struct file *, struct iov_iter *,
 				     struct kiocb *);
 
+extern void uncached_write_pages(struct address_space *, struct list_head *);
+
 ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
 		rwf_t flags);
 ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
diff --git a/include/linux/iomap.h b/include/linux/iomap.h
index 30f40145a9e9b5..259848d2dfd94d 100644
--- a/include/linux/iomap.h
+++ b/include/linux/iomap.h
@@ -48,12 +48,16 @@ struct vm_fault;
  *
  * IOMAP_F_BUFFER_HEAD indicates that the file system requires the use of
  * buffer heads for this mapping.
+ *
+ * IOMAP_F_PAGE_CREATE indicates that pages had to be allocated to satisfy
+ * this operation.
  */
 #define IOMAP_F_NEW		0x01
 #define IOMAP_F_DIRTY		0x02
 #define IOMAP_F_SHARED		0x04
 #define IOMAP_F_MERGED		0x08
 #define IOMAP_F_BUFFER_HEAD	0x10
+#define IOMAP_F_PAGE_CREATE	0x20
 
 /*
  * Flags set by the core iomap code during operations:
@@ -87,6 +91,7 @@ struct iomap {
 	void			*inline_data;
 	void			*private; /* filesystem private */
 	const struct iomap_page_ops *page_ops;
+	struct list_head	page_list;
 };
 
 static inline sector_t
@@ -121,6 +126,7 @@ struct iomap_page_ops {
 #define IOMAP_FAULT		(1 << 3) /* mapping for page fault */
 #define IOMAP_DIRECT		(1 << 4) /* direct I/O */
 #define IOMAP_NOWAIT		(1 << 5) /* do not block */
+#define IOMAP_UNCACHED		(1 << 6) /* uncached IO */
 
 struct iomap_ops {
 	/*
diff --git a/mm/filemap.c b/mm/filemap.c
index 42d9457afdf45f..5d04ee3260023b 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -3268,23 +3268,67 @@ EXPORT_SYMBOL(generic_file_direct_write);
 struct page *grab_cache_page_write_begin(struct address_space *mapping,
 					pgoff_t index, unsigned flags)
 {
-	struct page *page;
+	gfp_t gfp = mapping_gfp_mask(mapping);
 	int fgp_flags = FGP_LOCK|FGP_WRITE;
+	struct page *page;
 
 	if (flags & AOP_FLAG_NOFS)
 		fgp_flags |= FGP_NOFS;
 	if (!(flags & AOP_FLAG_UNCACHED))
 		fgp_flags |= FGP_CREAT;
-
-	page = pagecache_get_page(mapping, index, fgp_flags,
-			mapping_gfp_mask(mapping));
-	if (page)
+	page = pagecache_get_page(mapping, index, fgp_flags, gfp);
+	if (!page && (flags & AOP_FLAG_UNCACHED)) {
+		if (flags & AOP_FLAG_NOFS)
+			gfp &= ~__GFP_FS;
+		page = __page_cache_alloc(gfp);
+		if (page) {
+			page->mapping = mapping;
+			page->index = index;
+			__SetPageLocked(page);
+			__SetPagePrivio(page);
+			get_page(page);
+		}
+	} else if (page)
 		wait_for_stable_page(page);
 
 	return page;
 }
 EXPORT_SYMBOL(grab_cache_page_write_begin);
 
+void uncached_write_pages(struct address_space *mapping,
+			  struct list_head *wb_list)
+{
+	struct writeback_control wbc = {
+		.sync_mode	= WB_SYNC_ALL,
+		.for_sync	= 1,
+	};
+	struct blk_plug plug;
+	struct page *page;
+
+	wbc_attach_fdatawrite_inode(&wbc, mapping->host);
+	blk_start_plug(&plug);
+
+	list_for_each_entry(page, wb_list, lru) {
+		lock_page(page);
+		wbc.nr_to_write = 1;
+		wbc.pages_skipped = 0;
+		test_clear_page_writeback(page);
+		mapping->a_ops->writepage(page, &wbc);
+		if (wbc.pages_skipped)
+			printk("wp done: skipped %ld\n", wbc.pages_skipped);
+	}
+	while (!list_empty(wb_list)) {
+		page = list_first_entry(wb_list, struct page, lru);
+		list_del(&page->lru);
+		wait_on_page_writeback(page);
+		page->mapping = NULL;
+		put_page(page);
+	}
+
+	blk_finish_plug(&plug);
+	wbc_detach_inode(&wbc);
+}
+
 ssize_t generic_perform_write(struct file *file,
 				struct iov_iter *i, struct kiocb *iocb)
 {