Enable scatter ABD in userspace

This patch enables ztest to use scatter ABD. Part of this patch comes from: DeHackEd@845369d Suggested-by: DHE <[email protected]> Signed-off-by: Chunwei Chen <[email protected]>
tuxoko · Feb 28, 2014 · 85f5ea5 · 85f5ea5
1 parent 1cef6a8
commit 85f5ea5
Show file tree

Hide file tree

Showing 3 changed files with 142 additions and 47 deletions.
diff --git a/include/sys/abd.h b/include/sys/abd.h
@@ -61,7 +61,6 @@ extern "C" {
 	abd;				\
 })
 
-#ifdef _KERNEL
 
 /* purely virtual structure to prevent dereferencing */
 struct abd;
@@ -134,16 +133,16 @@ void do_abd_copy_to_buf_off(void *, abd_t *, size_t, size_t);
 int do_abd_cmp(abd_t *, abd_t *, size_t);
 int do_abd_cmp_buf_off(abd_t *, const void *, size_t, size_t);
 void do_abd_zero_off(abd_t *, size_t, size_t);
+#ifdef _KERNEL
 int do_abd_copy_to_user_off(void __user *, abd_t *, size_t, size_t);
 int do_abd_copy_from_user_off(abd_t *, const void __user *, size_t, size_t);
 int do_abd_uiomove_off(abd_t *, size_t, enum uio_rw, uio_t *, size_t);
-int do_abd_uiomove(abd_t *, size_t, enum uio_rw, uio_t *);
 int do_abd_uiocopy_off(abd_t *, size_t, enum uio_rw, uio_t *, size_t *, size_t);
-int do_abd_uiocopy(abd_t *, size_t, enum uio_rw, uio_t *, size_t *);
-abd_t *do_abd_get_offset(abd_t *, size_t);
-void do_abd_put_offset(abd_t *);
 unsigned int do_abd_bio_map_off(struct bio *, abd_t *, unsigned int, size_t);
 unsigned long do_abd_bio_nr_pages_off(abd_t *, unsigned int, size_t);
+#endif
+abd_t *do_abd_get_offset(abd_t *, size_t);
+void do_abd_put_offset(abd_t *);
 #define do_abd_borrow_linear(a, n) zio_buf_alloc(n)
 #define do_abd_borrow_linear_copy(a, n)		\
 ({						\
@@ -159,28 +158,6 @@ do {						\
 	zio_buf_free(b, n);			\
 } while (0)
 
-#else	/* _KERNEL */
-
-typedef void abd_t;
-
-#define ABD_IS_SCATTER(abd)	(0)
-#define ABD_IS_LINEAR(abd)	(1)
-#define ASSERT_ABD_SCATTER(abd)	((void)0)
-#define ASSERT_ABD_LINEAR(abd)	((void)0)
-
-#define abd_alloc	zio_data_buf_alloc
-#define abd_free	zio_data_buf_free
-
-/*
- * Userspace has only linear buffers, so no multiplexing
- */
-#define ABD_FUNC_WRAPPER(abd, func, ...)		u_##func(__VA_ARGS__)
-#define ABD_RET_FUNC_WRAPPER(type, abd, func, ...)	u_##func(__VA_ARGS__)
-#define ABD_FUNC2_WRAPPER(abd1, abd2, func, ...)	u_##func(__VA_ARGS__)
-#define ABD_RET_FUNC2_WRAPPER(type, abd1, abd2, func, ...) u_##func(__VA_ARGS__)
-
-#endif	/* _KERNEL */
-
 /*
  * ABD functions for userspace and linear buffer
  * Should not be used directly
@@ -194,13 +171,13 @@ typedef void abd_t;
 #define u_abd_cmp(a, b, n)			memcmp(a, b, n)
 #define u_abd_cmp_buf_off(a, b, n, off)		memcmp((void *)(a)+(off), b, n)
 #define u_abd_zero_off(a, n, off)		(void)memset((void *)(a)+(off), 0, n)
+#define u_abd_get_offset(a, off)		(abd_t *)((void *)(a)+(off))
+#define u_abd_put_offset(a)			do { } while (0)
+#ifdef _KERNEL
 #define u_abd_copy_to_user_off(a, b, n, off)	copy_to_user(a, (void *)(b)+(off), n)
 #define u_abd_copy_from_user_off(a, b, n, off)	copy_from_user((void *)(a)+(off), b, n)
 #define u_abd_uiomove_off(p, n, rw, uio, off)	uiomove((void *)(p)+(off), n, rw, uio)
 #define u_abd_uiocopy_off(p, n, rw, uio, c, off) uiocopy((void *)(p)+(off), n, rw, uio, c)
-#define u_abd_get_offset(a, off)		(abd_t *)((void *)(a)+(off))
-#define u_abd_put_offset(a)			do { } while (0)
-#ifdef _KERNEL
 #define u_abd_bio_map_off(bio, a, n, off)	bio_map(bio, (void *)(a)+(off), n)
 #define u_abd_bio_nr_pages_off(a, n, off)	bio_nr_pages((void *)(a)+(off), n)
 #endif
@@ -289,6 +266,7 @@ typedef void abd_t;
 #define abd_zero(abd, size) \
 	abd_zero_off(abd, size, 0)
 
+#ifdef _KERNEL
 /*
  * Copy from ABD to user buffer.
  */
@@ -325,6 +303,19 @@ typedef void abd_t;
 #define abd_uiocopy(abd, n, rw, uio, c) \
 	abd_uiocopy_off(abd, n, rw, uio, c, 0)
 
+/*
+ * bio_map for ABD.
+ */
+#define abd_bio_map_off(bio, abd, bio_size, off) \
+	ABD_RET_FUNC_WRAPPER(unsigned int, abd, abd_bio_map_off, bio, abd, bio_size, off)
+
+/*
+ * bio_nr_pages for ABD.
+ */
+#define abd_bio_nr_pages_off(abd, bio_size, off) \
+	ABD_RET_FUNC_WRAPPER(unsigned long, abd, abd_bio_nr_pages_off, abd, bio_size, off)
+#endif	/* _KERNEL */
+
 /*
  * Allocate a new ABD to point to offset @off of the original ABD.
  * It shares the underlying buffer with the original ABD.
@@ -341,20 +332,6 @@ typedef void abd_t;
 #define abd_put_offset(abd) \
 	ABD_FUNC_WRAPPER(abd, abd_put_offset, abd)
 
-#ifdef _KERNEL
-/*
- * bio_map for ABD.
- */
-#define abd_bio_map_off(bio, abd, bio_size, off) \
-	ABD_RET_FUNC_WRAPPER(unsigned int, abd, abd_bio_map_off, bio, abd, bio_size, off)
-
-/*
- * bio_nr_pages for ABD.
- */
-#define abd_bio_nr_pages_off(abd, bio_size, off) \
-	ABD_RET_FUNC_WRAPPER(unsigned long, abd, abd_bio_nr_pages_off, abd, bio_size, off)
-#endif	/* _KERNEL */
-
 /*
  * Borrow a linear buffer for an ABD
  * Will allocate if ABD is scatter

diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am
@@ -20,6 +20,7 @@ libzpool_la_SOURCES = \
 	$(top_srcdir)/module/zcommon/zfs_uio.c \
 	$(top_srcdir)/module/zcommon/zpool_prop.c \
 	$(top_srcdir)/module/zcommon/zprop_common.c \
+	$(top_srcdir)/module/zfs/abd.c \
 	$(top_srcdir)/module/zfs/arc.c \
 	$(top_srcdir)/module/zfs/bplist.c \
 	$(top_srcdir)/module/zfs/bpobj.c \
@@ -105,7 +106,6 @@ libzpool_la_LIBADD += $(ZLIB)
 libzpool_la_LDFLAGS = -version-info 2:0:0
 
 EXTRA_DIST = \
-	$(top_srcdir)/module/zfs/abd.c \
 	$(top_srcdir)/module/zfs/vdev_disk.c \
 	$(top_srcdir)/module/zfs/zfs_acl.c \
 	$(top_srcdir)/module/zfs/zfs_ctldir.c \

diff --git a/module/zfs/abd.c b/module/zfs/abd.c
@@ -24,6 +24,7 @@
  */
 
 #include <sys/abd.h>
+#ifdef _KERNEL
 #include <linux/kernel.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
@@ -32,6 +33,121 @@
 #include <linux/pagemap.h>
 #include <linux/kmap_compat.h>
 
+#else	/* _KERNEL */
+
+/*
+ * Userspace compatibility layer
+ */
+
+/*
+ * page
+ */
+#ifndef PAGE_SIZE
+#define PAGE_SIZE 4096
+#endif
+
+struct page;
+
+#define alloc_page(gfp)		((struct page *)umem_alloc(PAGE_SIZE, UMEM_DEFAULT))
+#define __free_page(page)	umem_free(page, PAGE_SIZE)
+
+/*
+ * scatterlist
+ */
+struct scatterlist {
+	struct page *page;
+	int length;
+	int end;
+};
+
+static void
+sg_init_table(struct scatterlist *sg, int nr) {
+	memset(sg, 0, nr * sizeof(struct scatterlist));
+	sg[nr - 1].end = 1;
+}
+
+static inline void
+sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len,
+    unsigned int offset) {
+	/* currently we don't use offset */
+	ASSERT(offset == 0);
+	sg->page = page;
+	sg->length = len;
+}
+
+static inline struct page *
+sg_page(struct scatterlist *sg) {
+	return sg->page;
+}
+
+/*
+ * sg_mapping_iter
+ */
+struct sg_mapping_iter {
+	struct scatterlist *sg;
+	int started;
+	int nents;
+	int length;
+	void *addr;
+};
+
+void
+__sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sg,
+    unsigned int nents) {
+	memset(miter, 0, sizeof(struct sg_mapping_iter));
+	miter->sg = sg;
+	miter->nents = nents;
+}
+#define sg_miter_start(miter, sg, nents, flags) __sg_miter_start(miter, sg, nents)
+#define sg_miter_stop(miter) do { } while (0)
+
+int
+sg_miter_next(struct sg_mapping_iter *miter) {
+	if (!miter->nents)
+		return 0;
+
+	if (!miter->started)
+		miter->started = 1;
+	else if (miter->sg->end)
+		return 0;
+	else
+		miter->sg++;
+
+	miter->nents--;
+	miter->length = miter->sg->length;
+	miter->addr = (void *)miter->sg->page;
+	return 1;
+}
+
+/*
+ * misc
+ */
+#ifndef DIV_ROUND_UP
+#define DIV_ROUND_UP(n,d)		(((n) + (d) - 1) / (d))
+#endif
+
+#ifndef unlikely
+#define unlikely(x)			(x)
+#endif
+
+#define kmap(page)			((void *)page)
+#define kunmap(page)			do { } while (0)
+#define zfs_kmap_atomic(page, type)	((void *)page)
+#define zfs_kunmap_atomic(addr, type)	do { } while (0)
+#define local_irq_save(flags)		do { flags = 0; } while (0)
+#define local_irq_restore(flags)	do { } while (0)
+#define flush_kernel_dcache_page(page)	do { } while (0)
+#define flush_dcache_page(page)		do { } while (0)
+#define set_current_state(state)	do { } while (0)
+static inline long
+schedule_timeout(long timeout)
+{
+	sleep(timeout);
+	return 0;
+}
+
+#endif	/* _KERNEL */
+
 #define ABD_MITER_WFLAGS (SG_MITER_ATOMIC|SG_MITER_TO_SG)
 #define ABD_MITER_RFLAGS (SG_MITER_ATOMIC|SG_MITER_FROM_SG)
 
@@ -426,6 +542,7 @@ do_abd_zero_off(abd_t *__abd, size_t size, size_t off)
 	local_irq_restore(flags);
 }
 
+#ifdef _KERNEL
 int
 do_abd_copy_to_user_off(void __user *buf, abd_t *__abd, size_t size, size_t off)
 {
@@ -637,16 +754,17 @@ do_abd_bio_nr_pages_off(abd_t *__abd, unsigned int bio_size, size_t off)
 	return ((abd->abd_offset + off + bio_size + PAGE_SIZE-1)>>PAGE_SHIFT) -
 	    ((abd->abd_offset + off)>>PAGE_SHIFT);
 }
+#endif	/* _KERNEL */
 
 abd_t *
 do_abd_get_offset(abd_t *__sabd, size_t off)
 {
 	arc_buf_data_t *abd;
 	arc_buf_data_t *sabd = ABD_CHECK(__sabd);
-
+#ifdef container_of
 	ASSERTV(arc_buf_data_t *oabd = container_of(sabd->abd_sgl, arc_buf_data_t, __abd_sgl[0]));
 	ASSERT(oabd->abd_magic == ARC_BUF_DATA_MAGIC);
-
+#endif
 	abd = kmem_alloc(sizeof(arc_buf_data_t), KM_PUSHPAGE);
 
 	ASSERT(abd);