zfs-2.4.0-rc2 patchset

.github/workflows/scripts/generate-ci-type.py

@@ -65,7 +65,7 @@ def output_type(type, reason):
 
     # check last (HEAD) commit message
     last_commit_message_raw = subprocess.run([
-        'git', 'show', '-s', '--format=%B', 'HEAD'
+        'git', 'show', '-s', '--format=%B', head
     ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     for line in last_commit_message_raw.stdout.decode().splitlines():

.github/workflows/scripts/qemu-2-start.sh

@@ -121,7 +121,7 @@ case "$OS" in
     KSRC="$FREEBSD_SNAP/../amd64/$FreeBSD/src.txz"
     ;;
   freebsd15-0c)
-    FreeBSD="15.0-PRERELEASE"
+    FreeBSD="15.0-ALPHA2"
     OSNAME="FreeBSD $FreeBSD"
     OSv="freebsd14.0"
     URLxz="$FREEBSD_SNAP/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI-ufs.raw.xz"

.github/workflows/zfs-qemu.yml

@@ -44,7 +44,7 @@ jobs:
             os_selection="$FULL_OS"
           fi
 
-          if [ ${{ github.event.inputs.fedora_kernel_ver }} != "" ] ; then
+          if ${{ github.event.inputs.fedora_kernel_ver != '' }}; then
               # They specified a custom kernel version for Fedora.  Use only
               # Fedora runners.
               os_json=$(echo ${os_selection} | jq -c '[.[] | select(startswith("fedora"))]')
@@ -53,9 +53,8 @@ jobs:
               os_json=$(echo ${os_selection} | jq -c)
           fi
 
-          echo $os_json
-          echo "os=$os_json" >> $GITHUB_OUTPUT
-          echo "ci_type=$ci_type" >> $GITHUB_OUTPUT
+          echo "os=$os_json" | tee -a $GITHUB_OUTPUT
+          echo "ci_type=$ci_type" | tee -a $GITHUB_OUTPUT
 
   qemu-vm:
     name: qemu-x86

Makefile.am

@@ -1,6 +1,7 @@
 CLEANFILES =
 dist_noinst_DATA =
 INSTALL_DATA_HOOKS =
+INSTALL_EXEC_HOOKS =
 ALL_LOCAL =
 CLEAN_LOCAL =
 CHECKS = shellcheck checkbashisms
@@ -71,6 +72,9 @@ all: gitrev
 PHONY += install-data-hook $(INSTALL_DATA_HOOKS)
 install-data-hook: $(INSTALL_DATA_HOOKS)
 
+PHONY += install-exec-hook $(INSTALL_EXEC_HOOKS)
+install-exec-hook: $(INSTALL_EXEC_HOOKS)
+
 PHONY += maintainer-clean-local
 maintainer-clean-local:
 	-$(RM) $(GITREV)

cmd/Makefile.am

@@ -98,17 +98,16 @@ endif
 
 
 if USING_PYTHON
-bin_SCRIPTS      += arc_summary     arcstat        dbufstat        zilstat
-CLEANFILES       += arc_summary     arcstat        dbufstat        zilstat
-dist_noinst_DATA += %D%/arc_summary %D%/arcstat.in %D%/dbufstat.in %D%/zilstat.in
+bin_SCRIPTS      += zarcsummary     zarcstat        dbufstat        zilstat
+CLEANFILES       += zarcsummary     zarcstat        dbufstat        zilstat
+dist_noinst_DATA += %D%/zarcsummary %D%/zarcstat.in %D%/dbufstat.in %D%/zilstat.in
 
-$(call SUBST,arcstat,%D%/)
+$(call SUBST,zarcstat,%D%/)
 $(call SUBST,dbufstat,%D%/)
 $(call SUBST,zilstat,%D%/)
-arc_summary: %D%/arc_summary
+zarcsummary: %D%/zarcsummary
 	$(AM_V_at)cp $< $@
 endif
 
-
 PHONY += cmd
 cmd: $(bin_SCRIPTS) $(bin_PROGRAMS) $(sbin_SCRIPTS) $(sbin_PROGRAMS) $(dist_bin_SCRIPTS) $(zfsexec_PROGRAMS) $(mounthelper_PROGRAMS)

cmd/arcstat.in → cmd/zarcstat.in

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: CDDL-1.0
 #
 # Print out ZFS ARC Statistics exported via kstat(1)
-# For a definition of fields, or usage, use arcstat -v
+# For a definition of fields, or usage, use zarcstat -v
 #
 # This script was originally a fork of the original arcstat.pl (0.1)
 # by Neelakanth Nadgir, originally published on his Sun blog on
@@ -56,6 +56,7 @@ import time
 import getopt
 import re
 import copy
+import os
 
 from signal import signal, SIGINT, SIGWINCH, SIG_DFL
 
@@ -171,7 +172,7 @@ cols = {
     "zactive":    [7, 1000, "zfetch prefetches active per second"],
 }
 
-# ARC structural breakdown from arc_summary
+# ARC structural breakdown from zarcsummary
 structfields = {
     "cmp":      ["compressed", "Compressed"],
     "ovh":      ["overhead", "Overhead"],
@@ -187,7 +188,7 @@ structstats = {                             # size stats
     "sz":       ["_size", "size"],
 }
 
-# ARC types breakdown from arc_summary
+# ARC types breakdown from zarcsummary
 typefields = {
     "data":     ["data", "ARC data"],
     "meta":     ["metadata", "ARC metadata"],
@@ -198,7 +199,7 @@ typestats = {                               # size stats
     "sz":       ["_size", "size"],
 }
 
-# ARC states breakdown from arc_summary
+# ARC states breakdown from zarcsummary
 statefields = {
     "ano":      ["anon", "Anonymous"],
     "mfu":      ["mfu", "MFU"],
@@ -261,7 +262,7 @@ hdr_intr = 20          # Print header every 20 lines of output
 opfile = None
 sep = "  "              # Default separator is 2 spaces
 l2exist = False
-cmd = ("Usage: arcstat [-havxp] [-f fields] [-o file] [-s string] [interval "
+cmd = ("Usage: zarcstat [-havxp] [-f fields] [-o file] [-s string] [interval "
        "[count]]\n")
 cur = {}
 d = {}
@@ -348,10 +349,10 @@ def usage():
                      "character or string\n")
     sys.stderr.write("\t -p : Disable auto-scaling of numerical fields\n")
     sys.stderr.write("\nExamples:\n")
-    sys.stderr.write("\tarcstat -o /tmp/a.log 2 10\n")
-    sys.stderr.write("\tarcstat -s \",\" -o /tmp/a.log 2 10\n")
-    sys.stderr.write("\tarcstat -v\n")
-    sys.stderr.write("\tarcstat -f time,hit%,dh%,ph%,mh% 1\n")
+    sys.stderr.write("\tzarcstat -o /tmp/a.log 2 10\n")
+    sys.stderr.write("\tzarcstat -s \",\" -o /tmp/a.log 2 10\n")
+    sys.stderr.write("\tzarcstat -v\n")
+    sys.stderr.write("\tzarcstat -f time,hit%,dh%,ph%,mh% 1\n")
     sys.stderr.write("\n")
 
     sys.exit(1)
@@ -366,7 +367,7 @@ def snap_stats():
 
     cur = kstat
 
-    # fill in additional values from arc_summary
+    # fill in additional values from zarcsummary
     cur["caches_size"] = caches_size = cur["anon_data"]+cur["anon_metadata"]+\
         cur["mfu_data"]+cur["mfu_metadata"]+cur["mru_data"]+cur["mru_metadata"]+\
         cur["uncached_data"]+cur["uncached_metadata"]
@@ -766,6 +767,7 @@ def calculate():
 
 
 def main():
+
     global sint
     global count
     global hdr_intr

cmd/arc_summary → cmd/zarcsummary

@@ -34,7 +34,7 @@ Provides basic information on the ARC, its efficiency, the L2ARC (if present),
 the Data Management Unit (DMU), Virtual Devices (VDEVs), and tunables. See
 the in-source documentation and code at
 https://github.com/openzfs/zfs/blob/master/module/zfs/arc.c for details.
-The original introduction to arc_summary can be found at
+The original introduction to zarcsummary can be found at
 http://cuddletech.com/?p=454
 """
 
@@ -161,7 +161,7 @@ elif sys.platform.startswith('linux'):
         return get_params(TUNABLES_PATH)
 
     def get_version_impl(request):
-        # The original arc_summary called /sbin/modinfo/{spl,zfs} to get
+        # The original zarcsummary called /sbin/modinfo/{spl,zfs} to get
         # the version information. We switch to /sys/module/{spl,zfs}/version
         # to make sure we get what is really loaded in the kernel
         try:
@@ -439,7 +439,7 @@ def print_header():
     """
 
     # datetime is now recommended over time but we keep the exact formatting
-    # from the older version of arc_summary in case there are scripts
+    # from the older version of zarcsummary in case there are scripts
     # that expect it in this way
     daydate = time.strftime(DATE_FORMAT)
     spc_date = LINE_LENGTH-len(daydate)

cmd/zdb/zdb.c

@@ -107,7 +107,9 @@ extern uint_t zfs_reconstruct_indirect_combinations_max;
 extern uint_t zfs_btree_verify_intensity;
 
 static const char cmdname[] = "zdb";
-uint8_t dump_opt[256];
+uint8_t dump_opt[512];
+
+#define	ALLOCATED_OPT	256
 
 typedef void object_viewer_t(objset_t *, uint64_t, void *data, size_t size);
 
@@ -1666,6 +1668,16 @@ dump_metaslab_stats(metaslab_t *msp)
 	dump_histogram(rt->rt_histogram, ZFS_RANGE_TREE_HISTOGRAM_SIZE, 0);
 }
 
+static void
+dump_allocated(void *arg, uint64_t start, uint64_t size)
+{
+	uint64_t *off = arg;
+	if (*off != start)
+		(void) printf("ALLOC: %"PRIu64" %"PRIu64"\n", *off,
+		    start - *off);
+	*off = start + size;
+}
+
 static void
 dump_metaslab(metaslab_t *msp)
 {
@@ -1682,13 +1694,24 @@ dump_metaslab(metaslab_t *msp)
 	    (u_longlong_t)msp->ms_id, (u_longlong_t)msp->ms_start,
 	    (u_longlong_t)space_map_object(sm), freebuf);
 
-	if (dump_opt['m'] > 2 && !dump_opt['L']) {
+	if (dump_opt[ALLOCATED_OPT] ||
+	    (dump_opt['m'] > 2 && !dump_opt['L'])) {
 		mutex_enter(&msp->ms_lock);
 		VERIFY0(metaslab_load(msp));
+	}
+
+	if (dump_opt['m'] > 2 && !dump_opt['L']) {
 		zfs_range_tree_stat_verify(msp->ms_allocatable);
 		dump_metaslab_stats(msp);
-		metaslab_unload(msp);
-		mutex_exit(&msp->ms_lock);
+	}
+
+	if (dump_opt[ALLOCATED_OPT]) {
+		uint64_t off = msp->ms_start;
+		zfs_range_tree_walk(msp->ms_allocatable, dump_allocated,
+		    &off);
+		if (off != msp->ms_start + msp->ms_size)
+			(void) printf("ALLOC: %"PRIu64" %"PRIu64"\n", off,
+			    msp->ms_size - off);
 	}
 
 	if (dump_opt['m'] > 1 && sm != NULL &&
@@ -1703,6 +1726,12 @@ dump_metaslab(metaslab_t *msp)
 		    SPACE_MAP_HISTOGRAM_SIZE, sm->sm_shift);
 	}
 
+	if (dump_opt[ALLOCATED_OPT] ||
+	    (dump_opt['m'] > 2 && !dump_opt['L'])) {
+		metaslab_unload(msp);
+		mutex_exit(&msp->ms_lock);
+	}
+
 	if (vd->vdev_ops == &vdev_draid_ops)
 		ASSERT3U(msp->ms_size, <=, 1ULL << vd->vdev_ms_shift);
 	else
@@ -1739,8 +1768,9 @@ print_vdev_metaslab_header(vdev_t *vd)
 		}
 	}
 
-	(void) printf("\tvdev %10llu   %s",
-	    (u_longlong_t)vd->vdev_id, bias_str);
+	(void) printf("\tvdev %10llu\t%s  metaslab shift %4llu",
+	    (u_longlong_t)vd->vdev_id, bias_str,
+	    (u_longlong_t)vd->vdev_ms_shift);
 
 	if (ms_flush_data_obj != 0) {
 		(void) printf("   ms_unflushed_phys object %llu",
@@ -2634,8 +2664,8 @@ print_indirect(spa_t *spa, blkptr_t *bp, const zbookmark_phys_t *zb,
 		}
 		if (BP_GET_LEVEL(bp) != zb->zb_level) {
 			(void) printf(" (ERROR: Block pointer level "
-			    "(%llu) does not match bookmark level (%ld))",
-			    BP_GET_LEVEL(bp), zb->zb_level);
+			    "(%llu) does not match bookmark level (%lld))",
+			    BP_GET_LEVEL(bp), (longlong_t)zb->zb_level);
 			corruption_found = B_TRUE;
 		}
 	}
@@ -2688,8 +2718,8 @@ visit_indirect(spa_t *spa, const dnode_phys_t *dnp,
 			if (fill != BP_GET_FILL(bp)) {
 				(void) printf("%16llx: Block pointer "
 				    "fill (%llu) does not match calculated "
-				    "value (%lu)\n", offset, BP_GET_FILL(bp),
-				    fill);
+				    "value (%llu)\n", offset, BP_GET_FILL(bp),
+				    (u_longlong_t)fill);
 				corruption_found = B_TRUE;
 			}
 		}
@@ -9375,6 +9405,8 @@ main(int argc, char **argv)
 		{"all-reconstruction",	no_argument,		NULL, 'Y'},
 		{"livelist",		no_argument,		NULL, 'y'},
 		{"zstd-headers",	no_argument,		NULL, 'Z'},
+		{"allocated-map",	no_argument,		NULL,
+		    ALLOCATED_OPT},
 		{0, 0, 0, 0}
 	};
 
@@ -9405,6 +9437,7 @@ main(int argc, char **argv)
 		case 'u':
 		case 'y':
 		case 'Z':
+		case ALLOCATED_OPT:
 			dump_opt[c]++;
 			dump_all = 0;
 			break;

cmd/zdb/zdb.h

@@ -29,6 +29,6 @@
 #define	_ZDB_H
 
 void dump_intent_log(zilog_t *);
-extern uint8_t dump_opt[256];
+extern uint8_t dump_opt[512];
 
 #endif	/* _ZDB_H */

cmd/zdb/zdb_il.c

@@ -48,8 +48,6 @@
 
 #include "zdb.h"
 
-extern uint8_t dump_opt[256];
-
 static char tab_prefix[4] = "\t\t\t";
 
 static void

cmd/zed/zed.d/Makefile.am

@@ -9,18 +9,18 @@ dist_zedexec_SCRIPTS = \
 	%D%/all-debug.sh \
 	%D%/all-syslog.sh \
 	%D%/data-notify.sh \
-	%D%/deadman-slot_off.sh \
+	%D%/deadman-sync-slot_off.sh \
 	%D%/generic-notify.sh \
-	%D%/pool_import-led.sh \
+	%D%/pool_import-sync-led.sh \
 	%D%/resilver_finish-notify.sh \
 	%D%/resilver_finish-start-scrub.sh \
 	%D%/scrub_finish-notify.sh \
-	%D%/statechange-led.sh \
+	%D%/statechange-sync-led.sh \
 	%D%/statechange-notify.sh \
-	%D%/statechange-slot_off.sh \
+	%D%/statechange-sync-slot_off.sh \
 	%D%/trim_finish-notify.sh \
-	%D%/vdev_attach-led.sh \
-	%D%/vdev_clear-led.sh
+	%D%/vdev_attach-sync-led.sh \
+	%D%/vdev_clear-sync-led.sh
 
 nodist_zedexec_SCRIPTS = \
 	%D%/history_event-zfs-list-cacher.sh
@@ -30,17 +30,17 @@ SUBSTFILES += $(nodist_zedexec_SCRIPTS)
 zedconfdefaults = \
 	all-syslog.sh \
 	data-notify.sh \
-	deadman-slot_off.sh \
+	deadman-sync-slot_off.sh \
 	history_event-zfs-list-cacher.sh \
-	pool_import-led.sh \
+	pool_import-sync-led.sh \
 	resilver_finish-notify.sh \
 	resilver_finish-start-scrub.sh \
 	scrub_finish-notify.sh \
-	statechange-led.sh \
+	statechange-sync-led.sh \
 	statechange-notify.sh \
-	statechange-slot_off.sh \
-	vdev_attach-led.sh \
-	vdev_clear-led.sh
+	statechange-sync-slot_off.sh \
+	vdev_attach-sync-led.sh \
+	vdev_clear-sync-led.sh
 
 dist_noinst_DATA += %D%/README
 

cmd/zed/zed.d/pool_import-sync-led.sh

@@ -0,0 +1 @@
+statechange-sync-led.sh

cmd/zed/zed.d/vdev_attach-sync-led.sh

@@ -0,0 +1 @@
+statechange-sync-led.sh

cmd/zed/zed.d/vdev_clear-sync-led.sh

@@ -0,0 +1 @@
+statechange-sync-led.sh