Update ZTS to work on FreeBSD

Update the common ZTS scripts and individual test cases as needed 
in order to allow them to be run on FreeBSD.  The high level goal
is to provide compatibility wrappers whenever possible to minimize
changes to individual test cases.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Signed-off-by: Matt Macy <mmacy@FreeBSD.org>
Signed-off-by: Ryan Moeller <ryan@ixsystems.com>
Closes openzfs#9692

scripts/zfs-tests.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # CDDL HEADER START
 #
@@ -47,10 +47,19 @@ TAGS=""
 ITERATIONS=1
 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
 ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
-ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
-TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
-LOSETUP=${LOSETUP:-/sbin/losetup}
-DMSETUP=${DMSETUP:-/sbin/dmsetup}
+UNAME=$(uname -s)
+
+# Override some defaults if on FreeBSD
+if [ "$UNAME" = "FreeBSD" ] ; then
+	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
+	LOSETUP=/sbin/mdconfig
+	DMSETUP=/sbin/gpart
+else
+	ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
+	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
+	LOSETUP=${LOSETUP:-/sbin/losetup}
+	DMSETUP=${DMSETUP:-/sbin/dmsetup}
+fi
 
 #
 # Log an informational message when additional verbosity is enabled.
@@ -70,6 +79,33 @@ fail() {
 	exit 1
 }
 
+cleanup_freebsd_loopback() {
+	for TEST_LOOPBACK in ${LOOPBACKS}; do
+		if [ -c "/dev/${TEST_LOOPBACK}" ]; then
+			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
+			    echo "Failed to destroy: ${TEST_LOOPBACK}"
+		fi
+	done
+}
+
+cleanup_linux_loopback() {
+	for TEST_LOOPBACK in ${LOOPBACKS}; do
+		LOOP_DEV=$(basename "$TEST_LOOPBACK")
+		DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
+		    grep "${LOOP_DEV}" | cut -f1)
+
+		if [ -n "$DM_DEV" ]; then
+			sudo "${DMSETUP}" remove "${DM_DEV}" ||
+			    echo "Failed to remove: ${DM_DEV}"
+		fi
+
+		if [ -n "${TEST_LOOPBACK}" ]; then
+			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
+			    echo "Failed to remove: ${TEST_LOOPBACK}"
+		fi
+	done
+}
+
 #
 # Attempt to remove loopback devices and files which where created earlier
 # by this script to run the test framework.  The '-k' option may be passed
@@ -80,22 +116,13 @@ cleanup() {
 		return 0
 	fi
 
-	if [ "$LOOPBACK" = "yes" ]; then
-		for TEST_LOOPBACK in ${LOOPBACKS}; do
-			LOOP_DEV=$(basename "$TEST_LOOPBACK")
-			DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
-			    grep "${LOOP_DEV}" | cut -f1)
-
-			if [ -n "$DM_DEV" ]; then
-				sudo "${DMSETUP}" remove "${DM_DEV}" ||
-				    echo "Failed to remove: ${DM_DEV}"
-			fi
 
-			if [ -n "${TEST_LOOPBACK}" ]; then
-				sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
-				    echo "Failed to remove: ${TEST_LOOPBACK}"
-			fi
-		done
+	if [ "$LOOPBACK" = "yes" ]; then
+		if [ "$UNAME" = "FreeBSD" ] ; then
+			cleanup_freebsd_loopback
+		else
+			cleanup_linux_loopback
+		fi
 	fi
 
 	for TEST_FILE in ${FILES}; do
@@ -118,7 +145,11 @@ cleanup_all() {
 	local TEST_POOLS
 	TEST_POOLS=$(sudo "$ZPOOL" list -H -o name | grep testpool)
 	local TEST_LOOPBACKS
-	TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
+	if [ "$UNAME" = "FreeBSD" ] ; then
+		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
+	else
+		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
+	fi
 	local TEST_FILES
 	TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
 
@@ -129,13 +160,19 @@ cleanup_all() {
 		sudo "$ZPOOL" destroy "${TEST_POOL}"
 	done
 
-	msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
-	    grep loop | tr '\n' ' ')"
-	sudo "${DMSETUP}" remove_all
+	if [ "$UNAME" != "FreeBSD" ] ; then
+		msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
+		    grep loop | tr '\n' ' ')"
+		sudo "${DMSETUP}" remove_all
+	fi
 
 	msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
 	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
-		sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
+		if [ "$UNAME" = "FreeBSD" ] ; then
+			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
+		else
+			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
+		fi
 	done
 
 	msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
@@ -202,6 +239,9 @@ create_links() {
 constrain_path() {
 	. "$STF_SUITE/include/commands.cfg"
 
+	SYSTEM_DIRS="/bin /sbin /usr/bin /usr/sbin"
+	SYSTEM_DIRS+=" /usr/local/bin /usr/local/sbin"
+
 	if [ "$INTREE" = "yes" ]; then
 		# Constrained path set to ./zfs/bin/
 		STF_PATH="$BIN_DIR"
@@ -224,30 +264,40 @@ constrain_path() {
 	else
 		# Constrained path set to /var/tmp/constrained_path.*
 		SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXX}
-		STF_PATH=$(/bin/mktemp -d "$SYSTEMDIR")
+		STF_PATH=$(mktemp -d "$SYSTEMDIR")
 		STF_PATH_REMOVE="yes"
 		STF_MISSING_BIN=""
 
 		chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 
 		# Special case links for standard zfs utilities
-		create_links "/bin /usr/bin /sbin /usr/sbin" "$ZFS_FILES"
+		create_links "$SYSTEM_DIRS" "$ZFS_FILES"
 
 		# Special case links for zfs test suite utilities
 		create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
 	fi
 
 	# Standard system utilities
-	create_links "/bin /usr/bin /sbin /usr/sbin" "$SYSTEM_FILES"
+	SYSTEM_FILES="$SYSTEM_FILES_COMMON"
+	if [ "$UNAME" = "FreeBSD" ] ; then
+		SYSTEM_FILES+=" $SYSTEM_FILES_FREEBSD"
+	else
+		SYSTEM_FILES+=" $SYSTEM_FILES_LINUX"
+	fi
+	create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
 
 	# Exceptions
 	ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
-	ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
-	ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
-	ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
-	ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
-	ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
-	ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
+	if [ "$UNAME" = "Linux" ] ; then
+		ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
+		ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
+		ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
+		ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
+		ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
+		ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
+	elif [ "$UNAME" = "FreeBSD" ] ; then
+		ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
+	fi
 
 	if [ -L "$STF_PATH/arc_summary3" ]; then
 		ln -fs "$STF_PATH/arc_summary3" "$STF_PATH/arc_summary"
@@ -466,6 +516,9 @@ constrain_path
 #
 # Check if ksh exists
 #
+if [ "$UNAME" = "FreeBSD" ]; then
+	sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
+fi
 [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
 [ -e "$STF_SUITE/include/default.cfg" ] || fail \
     "Missing $STF_SUITE/include/default.cfg file."
@@ -509,7 +562,11 @@ fi
 #
 # See libzfs/libzfs_config.c for more information.
 #
-__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
+if [ "$UNAME" = "FreeBSD" ] ; then
+	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | tr -s '\n' ' ')"
+else
+	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
+fi
 
 . "$STF_SUITE/include/default.cfg"
 
@@ -549,15 +606,28 @@ if [ -z "${DISKS}" ]; then
 		test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
 
 		for TEST_FILE in ${FILES}; do
-			TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
-			sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
-			    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
-			LOOPBACKS="${LOOPBACKS}${TEST_LOOPBACK} "
-			BASELOOPBACKS=$(basename "$TEST_LOOPBACK")
-			if [[ "$DISKS" ]]; then
-				DISKS="$DISKS $BASELOOPBACKS"
+			if [ "$UNAME" = "FreeBSD" ] ; then
+				MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
+				if [ -z "$MDDEVICE" ] ; then
+					fail "Failed: ${TEST_FILE} -> loopback"
+				fi
+				LOOPBACKS="${LOOPBACKS}${MDDEVICE} "
+				if [[ "$DISKS" ]]; then
+					DISKS="$DISKS $MDDEVICE"
+				else
+					DISKS="$MDDEVICE"
+				fi
 			else
-				DISKS="$BASELOOPBACKS"
+				TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
+				sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
+				    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
+				LOOPBACKS="${LOOPBACKS}${TEST_LOOPBACK} "
+				BASELOOPBACKS=$(basename "$TEST_LOOPBACK")
+				if [[ "$DISKS" ]]; then
+					DISKS="$DISKS $BASELOOPBACKS"
+				else
+					DISKS="$BASELOOPBACKS"
+				fi
 			fi
 		done
 	fi
@@ -604,8 +674,14 @@ export __ZFS_POOL_EXCLUDE
 export TESTFAIL_CALLBACKS
 export PATH=$STF_PATH
 
-RESULTS_FILE=$(mktemp -u -t zts-results.XXXX -p "$FILEDIR")
-REPORT_FILE=$(mktemp -u -t zts-report.XXXX -p "$FILEDIR")
+if [ "$UNAME" = "FreeBSD" ] ; then
+	mkdir -p "$FILEDIR" || true
+	RESULTS_FILE=$(mktemp -u "${FILEDIR}/zts-results.XXXX")
+	REPORT_FILE=$(mktemp -u "${FILEDIR}/zts-report.XXXX")
+else
+	RESULTS_FILE=$(mktemp -u -t zts-results.XXXX -p "$FILEDIR")
+	REPORT_FILE=$(mktemp -u -t zts-report.XXXX -p "$FILEDIR")
+fi
 
 #
 # Run all the tests as specified.

tests/zfs-tests/cmd/Makefile.am

@@ -2,7 +2,6 @@ EXTRA_DIST = file_common.h
 
 SUBDIRS = \
 	chg_usr_exec \
-	user_ns_exec \
 	devname2devid \
 	dir_rd_update \
 	file_check \
@@ -19,11 +18,16 @@ SUBDIRS = \
 	mmap_libaio \
 	mmapwrite \
 	nvlist_to_lua \
-	randfree_file \
 	randwritecomp \
 	readmmap \
 	rename_dir \
 	rm_lnkcnt_zero_file \
-	threadsappend \
-	xattrtest \
-	stride_dd
+	stride_dd \
+	threadsappend
+
+if BUILD_LINUX
+SUBDIRS += \
+	randfree_file \
+	user_ns_exec \
+	xattrtest
+endif

tests/zfs-tests/cmd/mkfile/Makefile.am

@@ -4,3 +4,6 @@ pkgexecdir = $(datadir)/@PACKAGE@/zfs-tests/bin
 
 pkgexec_PROGRAMS = mkfile
 mkfile_SOURCES = mkfile.c
+if BUILD_FREEBSD
+mkfile_LDADD = -L/usr/local/lib -lintl
+endif

tests/zfs-tests/cmd/mkfile/mkfile.c

@@ -137,8 +137,17 @@ main(int argc, char **argv)
 			argv++;
 			argc--;
 			continue;
-		}
-		if (lseek(fd, (off_t)size-1, SEEK_SET) < 0) {
+		} else if (fchown(fd, getuid(), getgid()) < 0) {
+			saverr = errno;
+			(void) fprintf(stderr, gettext(
+			    "Could not set owner/group of %s: %s\n"),
+			    argv[1], strerror(saverr));
+			(void) close(fd);
+			errors++;
+			argv++;
+			argc--;
+			continue;
+		} else if (lseek(fd, (off_t)size-1, SEEK_SET) < 0) {
 			saverr = errno;
 			(void) fprintf(stderr, gettext(
 			    "Could not seek to offset %ld in %s: %s\n"),

tests/zfs-tests/cmd/mkfiles/mkfiles.c

@@ -55,6 +55,10 @@ main(int argc, char **argv)
 			(void) fprintf(stderr, "Failed to create %s %s\n", buf,
 			    strerror(errno));
 			return (-4);
+		} else if (fchown(fd, getuid(), getgid()) < 0) {
+			(void) fprintf(stderr, "Failed to chown %s %s\n", buf,
+			    strerror(errno));
+			return (-5);
 		}
 		(void) close(fd);
 	}

tests/zfs-tests/cmd/mktree/mktree.c

@@ -30,7 +30,9 @@
 #include <unistd.h>
 #include <string.h>
 #include <fcntl.h>
+#ifdef __linux__
 #include <sys/xattr.h>
+#endif
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/param.h>
@@ -176,11 +178,13 @@ crtfile(char *pname)
 		exit(errno);
 	}
 
+#ifdef __linux__
 	if (fsetxattr(fd, "user.xattr", pbuf, 1024, 0) < 0) {
 		(void) fprintf(stderr, "fsetxattr(fd, \"xattr\", pbuf, "
 		    "1024, 0) failed.\n[%d]: %s.\n", errno, strerror(errno));
 		exit(errno);
 	}
+#endif
 
 	(void) close(fd);
 	free(pbuf);