Move ARC data buffers out of vmalloc

cmd/zdb/zdb.c

@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
  */
 
 #include <stdio.h>
@@ -30,6 +31,7 @@
 #include <stdlib.h>
 #include <ctype.h>
 #include <sys/zfs_context.h>
+#include <sys/abd.h>
 #include <sys/spa.h>
 #include <sys/spa_impl.h>
 #include <sys/dmu.h>
@@ -1262,13 +1264,13 @@ visit_indirect(spa_t *spa, const dnode_phys_t *dnp,
 		ASSERT(buf->b_data);
 
 		/* recursively visit blocks below this */
-		cbp = buf->b_data;
-		for (i = 0; i < epb; i++, cbp++) {
+		for (i = 0; i < epb; i++) {
 			zbookmark_phys_t czb;
 
 			SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
 			    zb->zb_level - 1,
 			    zb->zb_blkid * epb + i);
+			cbp = abd_array(buf->b_data, i, blkptr_t);
 			err = visit_indirect(spa, dnp, cbp, &czb);
 			if (err)
 				break;
@@ -1434,7 +1436,7 @@ dump_bptree(objset_t *os, uint64_t obj, char *name)
 		return;
 
 	VERIFY3U(0, ==, dmu_bonus_hold(os, obj, FTAG, &db));
-	bt = db->db_data;
+	bt = ABD_TO_BUF(db->db_data);
 	zdb_nicenum(bt->bt_bytes, bytes);
 	(void) printf("\n    %s: %llu datasets, %s\n",
 	    name, (unsigned long long)(bt->bt_end - bt->bt_begin), bytes);
@@ -1885,7 +1887,7 @@ dump_object(objset_t *os, uint64_t object, int verbosity, int *print_header)
 		if (error)
 			fatal("dmu_bonus_hold(%llu) failed, errno %u",
 			    object, error);
-		bonus = db->db_data;
+		bonus = ABD_TO_BUF(db->db_data);
 		bsize = db->db_size;
 		dn = DB_DNODE((dmu_buf_impl_t *)db);
 	}
@@ -2108,7 +2110,7 @@ dump_config(spa_t *spa)
 	    spa->spa_config_object, FTAG, &db);
 
 	if (error == 0) {
-		nvsize = *(uint64_t *)db->db_data;
+		nvsize = *(uint64_t *)ABD_TO_BUF(db->db_data);
 		dmu_buf_rele(db, FTAG);
 
 		(void) printf("\nMOS Configuration:\n");
@@ -2431,7 +2433,7 @@ zdb_blkptr_done(zio_t *zio)
 	zdb_cb_t *zcb = zio->io_private;
 	zbookmark_phys_t *zb = &zio->io_bookmark;
 
-	zio_data_buf_free(zio->io_data, zio->io_size);
+	abd_free(zio->io_data, zio->io_size);
 
 	mutex_enter(&spa->spa_scrub_lock);
 	spa->spa_scrub_inflight--;
@@ -2494,7 +2496,7 @@ zdb_blkptr_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	if (!BP_IS_EMBEDDED(bp) &&
 	    (dump_opt['c'] > 1 || (dump_opt['c'] && is_metadata))) {
 		size_t size = BP_GET_PSIZE(bp);
-		void *data = zio_data_buf_alloc(size);
+		abd_t *data = abd_alloc_linear(size);
 		int flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCRUB | ZIO_FLAG_RAW;
 
 		/* If it's an intent log block, failure is expected. */
@@ -3285,6 +3287,7 @@ zdb_read_block(char *thing, spa_t *spa)
 	zio_t *zio;
 	vdev_t *vd;
 	void *pbuf, *lbuf, *buf;
+	abd_t *pbuf_abd;
 	char *s, *p, *dup, *vdev, *flagstr;
 	int i, error;
 
@@ -3356,6 +3359,7 @@ zdb_read_block(char *thing, spa_t *spa)
 	lsize = size;
 
 	pbuf = umem_alloc_aligned(SPA_MAXBLOCKSIZE, 512, UMEM_NOFAIL);
+	pbuf_abd = abd_get_from_buf(pbuf, SPA_MAXBLOCKSIZE);
 	lbuf = umem_alloc(SPA_MAXBLOCKSIZE, UMEM_NOFAIL);
 
 	BP_ZERO(bp);
@@ -3383,15 +3387,15 @@ zdb_read_block(char *thing, spa_t *spa)
 		/*
 		 * Treat this as a normal block read.
 		 */
-		zio_nowait(zio_read(zio, spa, bp, pbuf, psize, NULL, NULL,
+		zio_nowait(zio_read(zio, spa, bp, pbuf_abd, psize, NULL, NULL,
 		    ZIO_PRIORITY_SYNC_READ,
 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_RAW, NULL));
 	} else {
 		/*
 		 * Treat this as a vdev child I/O.
 		 */
-		zio_nowait(zio_vdev_child_io(zio, bp, vd, offset, pbuf, psize,
-		    ZIO_TYPE_READ, ZIO_PRIORITY_SYNC_READ,
+		zio_nowait(zio_vdev_child_io(zio, bp, vd, offset, pbuf_abd,
+		    psize, ZIO_TYPE_READ, ZIO_PRIORITY_SYNC_READ,
 		    ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_QUEUE |
 		    ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY |
 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_RAW, NULL, NULL));
@@ -3465,6 +3469,7 @@ zdb_read_block(char *thing, spa_t *spa)
 		zdb_dump_block(thing, buf, size, flags);
 
 out:
+	abd_put(pbuf_abd);
 	umem_free(pbuf, SPA_MAXBLOCKSIZE);
 	umem_free(lbuf, SPA_MAXBLOCKSIZE);
 	free(dup);

cmd/zdb/zdb_il.c

@@ -26,6 +26,7 @@
 
 /*
  * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
  */
 
 /*
@@ -36,6 +37,7 @@
 #include <stdlib.h>
 #include <ctype.h>
 #include <sys/zfs_context.h>
+#include <sys/abd.h>
 #include <sys/spa.h>
 #include <sys/dmu.h>
 #include <sys/stat.h>
@@ -125,6 +127,7 @@ zil_prt_rec_write(zilog_t *zilog, int txtype, lr_write_t *lr)
 	blkptr_t *bp = &lr->lr_blkptr;
 	zbookmark_phys_t zb;
 	char buf[SPA_MAXBLOCKSIZE];
+	abd_t *abd;
 	int verbose = MAX(dump_opt['d'], dump_opt['i']);
 	int error;
 
@@ -158,9 +161,11 @@ zil_prt_rec_write(zilog_t *zilog, int txtype, lr_write_t *lr)
 		    lr->lr_foid, ZB_ZIL_LEVEL,
 		    lr->lr_offset / BP_GET_LSIZE(bp));
 
+		abd = abd_get_from_buf(buf, BP_GET_LSIZE(bp));
 		error = zio_wait(zio_read(NULL, zilog->zl_spa,
-		    bp, buf, BP_GET_LSIZE(bp), NULL, NULL,
+		    bp, abd, BP_GET_LSIZE(bp), NULL, NULL,
 		    ZIO_PRIORITY_SYNC_READ, ZIO_FLAG_CANFAIL, &zb));
+		abd_put(abd);
 		if (error)
 			return;
 		data = buf;

cmd/ztest/ztest.c

@@ -23,6 +23,7 @@
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
+ * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
  */
 
 /*
@@ -86,6 +87,7 @@
  */
 
 #include <sys/zfs_context.h>
+#include <sys/abd.h>
 #include <sys/spa.h>
 #include <sys/dmu.h>
 #include <sys/txg.h>
@@ -1310,26 +1312,31 @@ ztest_tx_assign(dmu_tx_t *tx, uint64_t txg_how, const char *tag)
 }
 
 static void
-ztest_pattern_set(void *buf, uint64_t size, uint64_t value)
+ztest_pattern_set(abd_t *abd, uint64_t size, uint64_t value)
 {
+	void *buf = abd_borrow_buf(abd, size);
 	uint64_t *ip = buf;
 	uint64_t *ip_end = (uint64_t *)((uintptr_t)buf + (uintptr_t)size);
 
 	while (ip < ip_end)
 		*ip++ = value;
+
+	abd_return_buf_copy(abd, buf, size);
 }
 
 #ifndef NDEBUG
 static boolean_t
-ztest_pattern_match(void *buf, uint64_t size, uint64_t value)
+ztest_pattern_match(abd_t *abd, uint64_t size, uint64_t value)
 {
+	void *buf = abd_borrow_buf_copy(abd, size);
 	uint64_t *ip = buf;
 	uint64_t *ip_end = (uint64_t *)((uintptr_t)buf + (uintptr_t)size);
 	uint64_t diff = 0;
 
 	while (ip < ip_end)
 		diff |= (value - *ip++);
 
+	abd_return_buf(abd, buf, size);
 	return (diff == 0);
 }
 #endif
@@ -1369,7 +1376,8 @@ ztest_bt_bonus(dmu_buf_t *db)
 	dmu_object_info_from_db(db, &doi);
 	ASSERT3U(doi.doi_bonus_size, <=, db->db_size);
 	ASSERT3U(doi.doi_bonus_size, >=, sizeof (*bt));
-	bt = (void *)((char *)db->db_data + doi.doi_bonus_size - sizeof (*bt));
+	bt = (void *)((char *)ABD_TO_BUF(db->db_data) + doi.doi_bonus_size -
+	    sizeof (*bt));
 
 	return (bt);
 }
@@ -1731,7 +1739,7 @@ ztest_replay_write(ztest_ds_t *zd, lr_write_t *lr, boolean_t byteswap)
 	if (abuf == NULL) {
 		dmu_write(os, lr->lr_foid, offset, length, data, tx);
 	} else {
-		bcopy(data, abuf->b_data, length);
+		abd_copy_from_buf(abuf->b_data, data, length);
 		dmu_assign_arcbuf(db, offset, abuf, tx);
 	}
 
@@ -4126,16 +4134,19 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 		for (off = bigoff, j = 0; j < s; j++, off += chunksize) {
 			dmu_buf_t *dbt;
 			if (i != 5) {
-				bcopy((caddr_t)bigbuf + (off - bigoff),
-				    bigbuf_arcbufs[j]->b_data, chunksize);
+				abd_copy_from_buf(bigbuf_arcbufs[j]->b_data,
+				    (caddr_t)bigbuf + (off - bigoff),
+				    chunksize);
 			} else {
-				bcopy((caddr_t)bigbuf + (off - bigoff),
+				abd_copy_from_buf(
 				    bigbuf_arcbufs[2 * j]->b_data,
+				    (caddr_t)bigbuf + (off - bigoff),
 				    chunksize / 2);
-				bcopy((caddr_t)bigbuf + (off - bigoff) +
-				    chunksize / 2,
+
+				abd_copy_from_buf(
 				    bigbuf_arcbufs[2 * j + 1]->b_data,
-				    chunksize / 2);
+				    (caddr_t)bigbuf + (off - bigoff) +
+				    chunksize / 2, chunksize / 2);
 			}
 
 			if (i == 1) {
@@ -5189,7 +5200,7 @@ ztest_ddt_repair(ztest_ds_t *zd, uint64_t id)
 	enum zio_checksum checksum = spa_dedup_checksum(spa);
 	dmu_buf_t *db;
 	dmu_tx_t *tx;
-	void *buf;
+	abd_t *buf;
 	blkptr_t blk;
 	int copies = 2 * ZIO_DEDUPDITTO_MIN;
 	int i;
@@ -5270,14 +5281,14 @@ ztest_ddt_repair(ztest_ds_t *zd, uint64_t id)
 	 * Damage the block.  Dedup-ditto will save us when we read it later.
 	 */
 	psize = BP_GET_PSIZE(&blk);
-	buf = zio_buf_alloc(psize);
+	buf = abd_alloc_linear(psize);
 	ztest_pattern_set(buf, psize, ~pattern);
 
 	(void) zio_wait(zio_rewrite(NULL, spa, 0, &blk,
 	    buf, psize, NULL, NULL, ZIO_PRIORITY_SYNC_WRITE,
 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_INDUCE_DAMAGE, NULL));
 
-	zio_buf_free(buf, psize);
+	abd_free(buf, psize);
 
 	(void) rw_unlock(&ztest_name_lock);
 	umem_free(od, sizeof (ztest_od_t));

config/kernel-kmap-atomic-args.m4

@@ -0,0 +1,20 @@
+dnl #
+dnl # 2.6.37 API change
+dnl # kmap_atomic changed from assigning hard-coded named slot to using
+dnl # push/pop based dynamical allocation.
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_KMAP_ATOMIC_ARGS], [
+	AC_MSG_CHECKING([whether kmap_atomic wants 1 args])
+	ZFS_LINUX_TRY_COMPILE([
+		#include <linux/pagemap.h>
+	],[
+		struct page page;
+		kmap_atomic(&page);
+	],[
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_1ARG_KMAP_ATOMIC, 1,
+		          [kmap_atomic wants 1 args])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])

config/kernel.m4

@@ -95,6 +95,7 @@ AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [
 	ZFS_AC_KERNEL_5ARG_SGET
 	ZFS_AC_KERNEL_LSEEK_EXECUTE
 	ZFS_AC_KERNEL_VFS_ITERATE
+	ZFS_AC_KERNEL_KMAP_ATOMIC_ARGS
 
 	AS_IF([test "$LINUX_OBJ" != "$LINUX"], [
 		KERNELMAKE_PARAMS="$KERNELMAKE_PARAMS O=$LINUX_OBJ"

include/linux/Makefile.am

@@ -5,7 +5,8 @@ KERNEL_H = \
 	$(top_srcdir)/include/linux/xattr_compat.h \
 	$(top_srcdir)/include/linux/vfs_compat.h \
 	$(top_srcdir)/include/linux/blkdev_compat.h \
-	$(top_srcdir)/include/linux/utsname_compat.h
+	$(top_srcdir)/include/linux/utsname_compat.h \
+	$(top_srcdir)/include/linux/kmap_compat.h
 
 USER_H =
 

include/linux/kmap_compat.h

@@ -0,0 +1,40 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
+ */
+
+#ifndef _ZFS_KMAP_H
+#define	_ZFS_KMAP_H
+
+#include <linux/highmem.h>
+
+#ifdef HAVE_1ARG_KMAP_ATOMIC
+/* 2.6.37 API change */
+#define	zfs_kmap_atomic(page, km_type)		kmap_atomic(page)
+#define	zfs_kunmap_atomic(addr, km_type)	kunmap_atomic(addr)
+#else
+#define	zfs_kmap_atomic(page, km_type)		kmap_atomic(page, km_type)
+#define	zfs_kunmap_atomic(addr, km_type)	kunmap_atomic(addr, km_type)
+#endif
+
+#endif	/* _ZFS_KMAP_H */

include/sys/Makefile.am

@@ -3,6 +3,7 @@ SUBDIRS = fm fs
 COMMON_H = \
 	$(top_srcdir)/include/sys/arc.h \
 	$(top_srcdir)/include/sys/arc_impl.h \
+	$(top_srcdir)/include/sys/abd.h \
 	$(top_srcdir)/include/sys/avl.h \
 	$(top_srcdir)/include/sys/avl_impl.h \
 	$(top_srcdir)/include/sys/blkptr.h \