Merge pull request openzfs#670 from ahrens/merge

merge with upstream

.github/workflows/build-dependencies.txt

@@ -6,6 +6,9 @@ bc
 build-essential
 curl
 dbench
+debhelper-compat
+dh-python
+dkms
 fakeroot
 fio
 gdb
@@ -33,12 +36,15 @@ mdadm
 nfs-kernel-server
 pamtester
 parted
+po-debconf
 python3
+python3-all-dev
 python3-cffi
 python3-dev
 python3-packaging
 python3-pip
 python3-setuptools
+python3-sphinx
 rng-tools-debian
 rsync
 samba

.github/workflows/zfs-tests-dlpx-functional.yml

@@ -32,15 +32,18 @@ jobs:
         ./configure --enable-debug --enable-debuginfo
     - name: Make
       run: |
-        make -j$(nproc) --no-print-directory --silent pkg-utils pkg-kmod
+        make --no-print-directory --silent native-deb-utils native-deb-kmod
+        mv ../*.deb .
+        rm ./openzfs-zfs-dkms*.deb ./openzfs-zfs-dracut*.deb
     - name: Install
       run: |
-        sudo dpkg -i *.deb
         # Update order of directories to search for modules, otherwise
         #   Ubuntu will load kernel-shipped ones.
         sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
-        sudo depmod
-        sudo modprobe zfs
+        sudo dpkg -i *.deb
+        # Native Debian packages enable and start the services
+        # Stop zfs-zed daemon, as it may interfere with some ZTS test cases
+        sudo systemctl stop zfs-zed
         # Workaround for cloud-init bug
         # see https://github.com/openzfs/zfs/issues/12644
         FILE=/lib/udev/rules.d/10-cloud-init-hook-hotplug.rules

cmd/zed/zed.d/statechange-notify.sh

@@ -38,7 +38,7 @@ if [ "${ZEVENT_VDEV_STATE_STR}" != "FAULTED" ] \
 fi
 
 umask 077
-note_subject="ZFS device fault for pool ${ZEVENT_POOL_GUID} on $(hostname)"
+note_subject="ZFS device fault for pool ${ZEVENT_POOL} on $(hostname)"
 note_pathname="$(mktemp)"
 {
     if [ "${ZEVENT_VDEV_STATE_STR}" = "FAULTED" ] ; then
@@ -66,7 +66,7 @@ note_pathname="$(mktemp)"
     [ -n "${ZEVENT_VDEV_GUID}" ] && echo "  vguid: ${ZEVENT_VDEV_GUID}"
     [ -n "${ZEVENT_VDEV_DEVID}" ] && echo "  devid: ${ZEVENT_VDEV_DEVID}"
 
-    echo "   pool: ${ZEVENT_POOL_GUID}"
+    echo "   pool: ${ZEVENT_POOL} (${ZEVENT_POOL_GUID})"
 
 } > "${note_pathname}"
 

cmd/zfs/zfs_iter.c

@@ -143,19 +143,20 @@ zfs_callback(zfs_handle_t *zhp, void *data)
 		    (cb->cb_types &
 		    (ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME))) &&
 		    zfs_get_type(zhp) == ZFS_TYPE_FILESYSTEM) {
-			(void) zfs_iter_filesystems(zhp, zfs_callback, data);
+			(void) zfs_iter_filesystems(zhp, cb->cb_flags,
+			    zfs_callback, data);
 		}
 
 		if (((zfs_get_type(zhp) & (ZFS_TYPE_SNAPSHOT |
 		    ZFS_TYPE_BOOKMARK)) == 0) && include_snaps) {
-			(void) zfs_iter_snapshots(zhp,
-			    (cb->cb_flags & ZFS_ITER_SIMPLE) != 0,
+			(void) zfs_iter_snapshots(zhp, cb->cb_flags,
 			    zfs_callback, data, 0, 0);
 		}
 
 		if (((zfs_get_type(zhp) & (ZFS_TYPE_SNAPSHOT |
 		    ZFS_TYPE_BOOKMARK)) == 0) && include_bmarks) {
-			(void) zfs_iter_bookmarks(zhp, zfs_callback, data);
+			(void) zfs_iter_bookmarks(zhp, cb->cb_flags,
+			    zfs_callback, data);
 		}
 
 		cb->cb_depth--;
@@ -211,18 +212,58 @@ zfs_free_sort_columns(zfs_sort_column_t *sc)
 	}
 }
 
-int
-zfs_sort_only_by_name(const zfs_sort_column_t *sc)
+/*
+ * Return true if all of the properties to be sorted are populated by
+ * dsl_dataset_fast_stat(). Note that sc == NULL (no sort) means we
+ * don't need any extra properties, so returns true.
+ */
+boolean_t
+zfs_sort_only_by_fast(const zfs_sort_column_t *sc)
 {
-	return (sc != NULL && sc->sc_next == NULL &&
-	    sc->sc_prop == ZFS_PROP_NAME);
+	while (sc != NULL) {
+		switch (sc->sc_prop) {
+		case ZFS_PROP_NAME:
+		case ZFS_PROP_GUID:
+		case ZFS_PROP_CREATETXG:
+		case ZFS_PROP_NUMCLONES:
+		case ZFS_PROP_INCONSISTENT:
+		case ZFS_PROP_REDACTED:
+		case ZFS_PROP_ORIGIN:
+			break;
+		default:
+			return (B_FALSE);
+		}
+		sc = sc->sc_next;
+	}
+
+	return (B_TRUE);
 }
 
-int
-zfs_sort_only_by_createtxg(const zfs_sort_column_t *sc)
+boolean_t
+zfs_list_only_by_fast(const zprop_list_t *p)
 {
-	return (sc != NULL && sc->sc_next == NULL &&
-	    sc->sc_prop == ZFS_PROP_CREATETXG);
+	if (p == NULL) {
+		/* NULL means 'all' so we can't use simple mode */
+		return (B_FALSE);
+	}
+
+	while (p != NULL) {
+		switch (p->pl_prop) {
+		case ZFS_PROP_NAME:
+		case ZFS_PROP_GUID:
+		case ZFS_PROP_CREATETXG:
+		case ZFS_PROP_NUMCLONES:
+		case ZFS_PROP_INCONSISTENT:
+		case ZFS_PROP_REDACTED:
+		case ZFS_PROP_ORIGIN:
+			break;
+		default:
+			return (B_FALSE);
+		}
+		p = p->pl_next;
+	}
+
+	return (B_TRUE);
 }
 
 static int

cmd/zfs/zfs_iter.h

@@ -40,20 +40,12 @@ typedef struct zfs_sort_column {
 	boolean_t		sc_reverse;
 } zfs_sort_column_t;
 
-#define	ZFS_ITER_RECURSE	   (1 << 0)
-#define	ZFS_ITER_ARGS_CAN_BE_PATHS (1 << 1)
-#define	ZFS_ITER_PROP_LISTSNAPS    (1 << 2)
-#define	ZFS_ITER_DEPTH_LIMIT	   (1 << 3)
-#define	ZFS_ITER_RECVD_PROPS	   (1 << 4)
-#define	ZFS_ITER_LITERAL_PROPS	   (1 << 5)
-#define	ZFS_ITER_SIMPLE		   (1 << 6)
-
 int zfs_for_each(int, char **, int options, zfs_type_t,
     zfs_sort_column_t *, zprop_list_t **, int, zfs_iter_f, void *);
 int zfs_add_sort_column(zfs_sort_column_t **, const char *, boolean_t);
 void zfs_free_sort_columns(zfs_sort_column_t *);
-int zfs_sort_only_by_name(const zfs_sort_column_t *);
-int zfs_sort_only_by_createtxg(const zfs_sort_column_t *);
+boolean_t zfs_sort_only_by_fast(const zfs_sort_column_t *);
+boolean_t zfs_list_only_by_fast(const zprop_list_t *);
 
 #ifdef	__cplusplus
 }

cmd/zfs/zfs_main.c

@@ -1532,7 +1532,7 @@ destroy_print_snapshots(zfs_handle_t *fs_zhp, destroy_cbdata_t *cb)
 	int err;
 	assert(cb->cb_firstsnap == NULL);
 	assert(cb->cb_prevsnap == NULL);
-	err = zfs_iter_snapshots_sorted(fs_zhp, destroy_print_cb, cb, 0, 0);
+	err = zfs_iter_snapshots_sorted(fs_zhp, 0, destroy_print_cb, cb, 0, 0);
 	if (cb->cb_firstsnap != NULL) {
 		uint64_t used = 0;
 		if (err == 0) {
@@ -1558,7 +1558,7 @@ snapshot_to_nvl_cb(zfs_handle_t *zhp, void *arg)
 	if (!cb->cb_doclones && !cb->cb_defer_destroy) {
 		cb->cb_target = zhp;
 		cb->cb_first = B_TRUE;
-		err = zfs_iter_dependents(zhp, B_TRUE,
+		err = zfs_iter_dependents(zhp, 0, B_TRUE,
 		    destroy_check_dependent, cb);
 	}
 
@@ -1576,7 +1576,8 @@ gather_snapshots(zfs_handle_t *zhp, void *arg)
 	destroy_cbdata_t *cb = arg;
 	int err = 0;
 
-	err = zfs_iter_snapspec(zhp, cb->cb_snapspec, snapshot_to_nvl_cb, cb);
+	err = zfs_iter_snapspec(zhp, 0, cb->cb_snapspec,
+	    snapshot_to_nvl_cb, cb);
 	if (err == ENOENT)
 		err = 0;
 	if (err != 0)
@@ -1589,7 +1590,7 @@ gather_snapshots(zfs_handle_t *zhp, void *arg)
 	}
 
 	if (cb->cb_recurse)
-		err = zfs_iter_filesystems(zhp, gather_snapshots, cb);
+		err = zfs_iter_filesystems(zhp, 0, gather_snapshots, cb);
 
 out:
 	zfs_close(zhp);
@@ -1614,7 +1615,7 @@ destroy_clones(destroy_cbdata_t *cb)
 			 * false while destroying the clones.
 			 */
 			cb->cb_defer_destroy = B_FALSE;
-			err = zfs_iter_dependents(zhp, B_FALSE,
+			err = zfs_iter_dependents(zhp, 0, B_FALSE,
 			    destroy_callback, cb);
 			cb->cb_defer_destroy = defer;
 			zfs_close(zhp);
@@ -1825,7 +1826,7 @@ zfs_do_destroy(int argc, char **argv)
 		 */
 		cb.cb_first = B_TRUE;
 		if (!cb.cb_doclones &&
-		    zfs_iter_dependents(zhp, B_TRUE, destroy_check_dependent,
+		    zfs_iter_dependents(zhp, 0, B_TRUE, destroy_check_dependent,
 		    &cb) != 0) {
 			rv = 1;
 			goto out;
@@ -1836,7 +1837,7 @@ zfs_do_destroy(int argc, char **argv)
 			goto out;
 		}
 		cb.cb_batchedsnaps = fnvlist_alloc();
-		if (zfs_iter_dependents(zhp, B_FALSE, destroy_callback,
+		if (zfs_iter_dependents(zhp, 0, B_FALSE, destroy_callback,
 		    &cb) != 0) {
 			rv = 1;
 			goto out;
@@ -3660,16 +3661,6 @@ found3:;
 	argc -= optind;
 	argv += optind;
 
-	/*
-	 * If we are only going to list snapshot names and sort by name or
-	 * by createtxg, then we can use faster version.
-	 */
-	if (strcmp(fields, "name") == 0 &&
-	    (zfs_sort_only_by_name(sortcol) ||
-	    zfs_sort_only_by_createtxg(sortcol))) {
-		flags |= ZFS_ITER_SIMPLE;
-	}
-
 	/*
 	 * If "-o space" and no types were specified, don't display snapshots.
 	 */
@@ -3697,6 +3688,15 @@ found3:;
 
 	cb.cb_first = B_TRUE;
 
+	/*
+	 * If we are only going to list and sort by properties that are "fast"
+	 * then we can use "simple" mode and avoid populating the properties
+	 * nvlist.
+	 */
+	if (zfs_list_only_by_fast(cb.cb_proplist) &&
+	    zfs_sort_only_by_fast(sortcol))
+		flags |= ZFS_ITER_SIMPLE;
+
 	ret = zfs_for_each(argc, argv, flags, types, sortcol, &cb.cb_proplist,
 	    limit, list_callback, &cb);
 
@@ -4007,7 +4007,7 @@ rollback_check(zfs_handle_t *zhp, void *data)
 		}
 
 		if (cbp->cb_recurse) {
-			if (zfs_iter_dependents(zhp, B_TRUE,
+			if (zfs_iter_dependents(zhp, 0, B_TRUE,
 			    rollback_check_dependent, cbp) != 0) {
 				zfs_close(zhp);
 				return (-1);
@@ -4106,10 +4106,10 @@ zfs_do_rollback(int argc, char **argv)
 	if (cb.cb_create > 0)
 		min_txg = cb.cb_create;
 
-	if ((ret = zfs_iter_snapshots(zhp, B_FALSE, rollback_check, &cb,
+	if ((ret = zfs_iter_snapshots(zhp, 0, rollback_check, &cb,
 	    min_txg, 0)) != 0)
 		goto out;
-	if ((ret = zfs_iter_bookmarks(zhp, rollback_check, &cb)) != 0)
+	if ((ret = zfs_iter_bookmarks(zhp, 0, rollback_check, &cb)) != 0)
 		goto out;
 
 	if ((ret = cb.cb_error) != 0)
@@ -4251,7 +4251,7 @@ zfs_snapshot_cb(zfs_handle_t *zhp, void *arg)
 	free(name);
 
 	if (sd->sd_recursive)
-		rv = zfs_iter_filesystems(zhp, zfs_snapshot_cb, sd);
+		rv = zfs_iter_filesystems(zhp, 0, zfs_snapshot_cb, sd);
 	zfs_close(zhp);
 	return (rv);
 }
@@ -6311,7 +6311,7 @@ zfs_do_allow_unallow_impl(int argc, char **argv, boolean_t un)
 
 		if (un && opts.recursive) {
 			struct deleg_perms data = { un, update_perm_nvl };
-			if (zfs_iter_filesystems(zhp, set_deleg_perms,
+			if (zfs_iter_filesystems(zhp, 0, set_deleg_perms,
 			    &data) != 0)
 				goto cleanup0;
 		}
@@ -6689,7 +6689,7 @@ get_one_dataset(zfs_handle_t *zhp, void *data)
 	/*
 	 * Iterate over any nested datasets.
 	 */
-	if (zfs_iter_filesystems(zhp, get_one_dataset, data) != 0) {
+	if (zfs_iter_filesystems(zhp, 0, get_one_dataset, data) != 0) {
 		zfs_close(zhp);
 		return (1);
 	}

cmd/zpool/zpool_main.c

@@ -9339,7 +9339,7 @@ check_unsupp_fs(zfs_handle_t *zhp, void *unsupp_fs)
 		(*count)++;
 	}
 
-	zfs_iter_filesystems(zhp, check_unsupp_fs, unsupp_fs);
+	zfs_iter_filesystems(zhp, 0, check_unsupp_fs, unsupp_fs);
 
 	zfs_close(zhp);
 

config/deb.am

@@ -1,14 +1,17 @@
-PHONY += deb-kmod deb-dkms deb-utils deb deb-local
+PHONY += deb-kmod deb-dkms deb-utils deb deb-local native-deb-local \
+	native-deb-utils native-deb-kmod native-deb
 
-deb-local:
+native-deb-local:
 	@(if test "${HAVE_DPKGBUILD}" = "no"; then \
 		echo -e "\n" \
 	"*** Required util ${DPKGBUILD} missing.  Please install the\n" \
 	"*** package for your distribution which provides ${DPKGBUILD},\n" \
 	"*** re-run configure, and try again.\n"; \
 		exit 1; \
-	fi; \
-	if test "${HAVE_ALIEN}" = "no"; then \
+	fi)
+
+deb-local: native-deb-local
+	@(if test "${HAVE_ALIEN}" = "no"; then \
 		echo -e "\n" \
 	"*** Required util ${ALIEN} missing.  Please install the\n" \
 	"*** package for your distribution which provides ${ALIEN},\n" \
@@ -85,3 +88,16 @@ deb-utils: deb-local rpm-utils-initramfs
 	    $$pkg8 $$pkg9 $$pkg10 $$pkg11;
 
 deb: deb-kmod deb-dkms deb-utils
+
+debian:
+	cp -r contrib/debian debian; chmod +x debian/rules;
+
+native-deb-utils: native-deb-local debian
+	cp contrib/debian/control debian/control; \
+	$(DPKGBUILD) -b -rfakeroot -us -uc;
+
+native-deb-kmod: native-deb-local debian
+	sh scripts/make_gitrev.sh; \
+	fakeroot debian/rules override_dh_binary-modules;
+
+native-deb: native-deb-utils native-deb-kmod

config/zfs-build.m4

@@ -471,6 +471,7 @@ AC_DEFUN([ZFS_AC_DPKG], [
 	AC_SUBST(HAVE_DPKGBUILD)
 	AC_SUBST(DPKGBUILD)
 	AC_SUBST(DPKGBUILD_VERSION)
+	AC_SUBST([CFGOPTS], ["$CFGOPTS"])
 ])
 
 dnl #

configure.ac

@@ -32,6 +32,7 @@
 
 AC_INIT(m4_esyscmd(awk '/^Name:/ {printf $2}' META),
 	m4_esyscmd(awk '/^Version:/ {printf $2}' META))
+CFGOPTS="$*"
 AC_LANG(C)
 ZFS_AC_META
 AC_CONFIG_AUX_DIR([config])
@@ -65,6 +66,7 @@ ZFS_AC_DEBUG_KMEM_TRACKING
 ZFS_AC_DEBUG_INVARIANTS
 
 AC_CONFIG_FILES([
+	contrib/debian/rules
 	Makefile
 	cmd/zfs_object_agent/Makefile
 	cmd/zfs_object_agent/scripts/Makefile