ZTS: Add test case for slow vdev sit out

Add ZTS test to verify that the slow vdev sit out code is working
correctly.

Signed-off-by: Tony Hutter <[email protected]>

tests/runfiles/common.run

@@ -702,6 +702,10 @@ tests = ['dio_aligned_block', 'dio_async_always', 'dio_async_fio_ioengines',
     'dio_unaligned_block', 'dio_unaligned_filesize']
 tags = ['functional', 'direct']
 
+[tests/functional/events]
+tests = ['slow_vdev_sit_out']
+tags = ['functional', 'events']
+
 [tests/functional/exec]
 tests = ['exec_001_pos', 'exec_002_neg']
 tags = ['functional', 'exec']

tests/zfs-tests/include/libtest.shlib

@@ -1109,6 +1109,16 @@ function get_pool_prop # property pool
 	zpool get -Hpo value "$prop" "$pool" || log_fail "zpool get $prop $pool"
 }
 
+# Get the specified vdev property in parsable format or fail
+function get_vdev_prop
+{
+	typeset prop="$1"
+	typeset pool="$2"
+	typeset vdev="$3"
+
+	zpool get -Hpo value "$prop" "$pool" "$vdev" || log_fail "zpool get $prop $pool $vdev"
+}
+
 # Return 0 if a pool exists; $? otherwise
 #
 # $1 - pool name

tests/zfs-tests/include/tunables.cfg

@@ -69,6 +69,7 @@ MULTIHOST_INTERVAL		multihost.interval		zfs_multihost_interval
 OVERRIDE_ESTIMATE_RECORDSIZE	send.override_estimate_recordsize	zfs_override_estimate_recordsize
 PREFETCH_DISABLE		prefetch.disable		zfs_prefetch_disable
 RAIDZ_EXPAND_MAX_REFLOW_BYTES	vdev.expand_max_reflow_bytes	raidz_expand_max_reflow_bytes
+RAIDZ_READ_SIT_OUT_SECS	vdev.raidz_read_sit_out_secs	raidz_read_sit_out_secs
 REBUILD_SCRUB_ENABLED		rebuild_scrub_enabled		zfs_rebuild_scrub_enabled
 REMOVAL_SUSPEND_PROGRESS	removal_suspend_progress	zfs_removal_suspend_progress
 REMOVE_MAX_SEGMENT		remove_max_segment		zfs_remove_max_segment

tests/zfs-tests/tests/Makefile.am

@@ -1497,6 +1497,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/events/events_001_pos.ksh \
 	functional/events/events_002_pos.ksh \
 	functional/events/setup.ksh \
+	functional/events/slow_vdev_sit_out.ksh \
 	functional/events/zed_cksum_config.ksh \
 	functional/events/zed_cksum_reported.ksh \
 	functional/events/zed_diagnose_multiple.ksh \

tests/zfs-tests/tests/functional/events/slow_vdev_sit_out.ksh

@@ -0,0 +1,95 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+# Copyright (c) 2024 by Lawrence Livermore National Security, LLC.
+
+# DESCRIPTION:
+#	Verify that vdev 'sit out' when they are slow
+#
+# STRATEGY:
+#	1. Create various raidz/draid pools
+#	2. Inject delays into one of the disks
+#	3. Verify it gets set to 'sit out'
+#	4. Wait for RAIDZ_READ_SIT_OUT_SECS and verify 'sit out' state is lifted
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+# Avoid name conflicts with the default pool
+TESTPOOL2=testpool2
+
+function cleanup
+{
+	restore_tunable RAIDZ_READ_SIT_OUT_SECS
+	log_must zinject -c all
+	destroy_pool $TESTPOOL2
+	log_must rm -f $TEST_BASE_DIR/vdev.$$.*
+}
+
+log_assert "Verify sit_out_reads works"
+
+log_onexit cleanup
+
+# shorten sit out period for testing
+save_tunable RAIDZ_READ_SIT_OUT_SECS
+set_tunable32 RAIDZ_READ_SIT_OUT_SECS 5
+
+log_must truncate -s 150M $TEST_BASE_DIR/vdev.$$.{0..9}
+
+for raidtype in raidz raidz2 raidz3 draid1 draid2 draid3 ; do
+	log_must zpool create $TESTPOOL2 $raidtype $TEST_BASE_DIR/vdev.$$.{0..9}
+	log_must dd if=/dev/urandom of=/$TESTPOOL2/bigfile bs=1M count=100
+	log_must zpool export $TESTPOOL2
+	log_must zpool import -d $TEST_BASE_DIR $TESTPOOL2
+
+	BAD_VDEV=$TEST_BASE_DIR/vdev.$$.9
+
+	log_note "sit_out: $(get_vdev_prop sit_out_reads $TESTPOOL2 $BAD_VDEV)"
+
+	# Initial state should not be sitting out
+	log_must eval [[ "$(get_vdev_prop sit_out_reads $TESTPOOL2 $BAD_VDEV)" == "off" ]]
+
+	# Delay our reads 200ms to trigger sit out
+	log_must zinject -d $BAD_VDEV -D200:1 -T read $TESTPOOL2
+
+	# Do some reads and wait for us to sit out
+	for i in {1..100} ; do
+		dd if=/$TESTPOOL2/bigfile skip=$i bs=1M count=1 of=/dev/null &> /dev/null
+
+		sit_out_reads=$(get_vdev_prop sit_out_reads $TESTPOOL2 $BAD_VDEV)
+		if [[ "$sit_out_reads" == "on" ]] ; then
+			break
+		fi
+	done
+
+	log_must test "$(get_vdev_prop sit_out_reads $TESTPOOL2 $BAD_VDEV)" == "on"
+
+	# Clear fault injection
+	log_must zinject -c all
+
+	# Wait for us to exit our sit out period
+	sleep 6
+	log_must test "$(get_vdev_prop sit_out_reads $TESTPOOL2 $BAD_VDEV)" == "off"
+	destroy_pool $TESTPOOL2
+done
+
+log_pass "sit_out_reads works correctly"