btrfs-progs: ci: wait for loop devices before mount

Since a few days the CI started to fail randomly when there were loop
devices used in the tests. The mount fails because some device is
reported to be missing:

  $ losetup --show --find
  /dev/loop3
  ...

  $ mkfs ...
  ERROR: device scan failed on '/dev/loop3': No such file or directory
  ...

  $ mount
  mount: /home/runner/work/btrfs-progs/btrfs-progs/tests/mnt: wrong fs
  type, bad option, bad superblock on /dev/loop3, missing codepage or
  helper program, or other error.

  $ dmesg
  ...
  BTRFS error (device loop0): devid 3 uuid 11d9c345-9527-433e-a024-7102659fa0ee is missing
  BTRFS error (device loop0): failed to read the system array: -2
  BTRFS error (device loop0): open_ctree failed

This was reproducible in the "cli" tests, but also happened on a local
machine.

To fix that wait for all loop devices before mount, the command
'btrfs device ready' should block until that. The convenience helper
does that, for any standalone 'mount' used with loop devices this must
be done manually.

Signed-off-by: David Sterba <dsterba@suse.com>

10 files changed, 32 insertions, 0 deletions

diff --git a/tests/cli-tests/017-fi-show-missing/test.sh b/tests/cli-tests/017-fi-show-missing/test.sh
index a6520c86..653bde32 100755
--- a/tests/cli-tests/017-fi-show-missing/test.sh
+++ b/tests/cli-tests/017-fi-show-missing/test.sh
@@ -20,6 +20,7 @@ run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d raid1 "${loopdevs[@]}"
 # Move the device, changing its path, simulating the device being missing
 run_check $SUDO_HELPER mv "$dev2" /dev/loop-non-existent
 
+cond_wait_for_loopdevs
 run_check $SUDO_HELPER mount -o degraded $dev1 $TEST_MNT
 
 if ! run_check_stdout $SUDO_HELPER "$TOP/btrfs" filesystem show "$TEST_MNT" | \
diff --git a/tests/common b/tests/common
index 602a4122..9621600c 100644
--- a/tests/common
+++ b/tests/common
@@ -648,6 +648,9 @@ run_check_mount_test_dev()
 		_fail "Invalid \$TEST_MNT: $TEST_MNT"
 	}
 
+	# Wait for loopdevs before mount as this could fail in the CI for some reason
+	cond_wait_for_loopdevs
+
 	run_check $SUDO_HELPER mount -t btrfs $loop_opt "$@" "$TEST_DEV" "$TEST_MNT"
 }
 
@@ -834,6 +837,7 @@ prepare_loopdevs()
 		chmod a+rw "$loopdev_prefix$i"
 		truncate -s0 "$loopdev_prefix$i"
 		truncate -s2g "$loopdev_prefix$i"
+		run_check sync "$loopdev_prefix$i"
 		loopdevs[$i]=`run_check_stdout $SUDO_HELPER losetup --find --show "$loopdev_prefix$i"`
 	done
 }
@@ -851,6 +855,22 @@ cleanup_loopdevs()
 	run_check $SUDO_HELPER losetup --all
 }
 
+# Loop devices may not be ready for mount after mkfs (observed in the CI), wait
+# for them explicitly. Errors are reported but tolerated, the mount could succeed
+# eventually or the whole test will fail anyway.
+wait_for_loopdevs()
+{
+	for dev in ${loopdevs[@]}; do
+		run_mayfail $SUDO_HELPER "$TOP/btrfs" device ready "$dev"
+	done
+}
+
+cond_wait_for_loopdevs() {
+	if [ -n "${loopdevs[1]}" ]; then
+		wait_for_loopdevs
+	fi
+}
+
 init_env()
 {
 	TEST_MNT="${TEST_MNT:-$TEST_TOP/mnt}"
diff --git a/tests/fsck-tests/052-init-csum-tree/test.sh b/tests/fsck-tests/052-init-csum-tree/test.sh
index cb7d8845..edf096b6 100755
--- a/tests/fsck-tests/052-init-csum-tree/test.sh
+++ b/tests/fsck-tests/052-init-csum-tree/test.sh
@@ -21,6 +21,7 @@ run_check $SUDO_HELPER dd if=/dev/urandom of="$TEST_MNT/nodatasum_file" \
 	bs=16k count=1 status=noxfer > /dev/null 2>&1
 
 # Revert to default datasum
+cond_wait_for_loopdevs
 run_check $SUDO_HELPER mount -o remount,datasum "$TEST_MNT"
 
 # Then create an inode with datasum, but all preallocated extents
diff --git a/tests/misc-tests/006-image-on-missing-device/test.sh b/tests/misc-tests/006-image-on-missing-device/test.sh
index f4efe8bc..992c0cda 100755
--- a/tests/misc-tests/006-image-on-missing-device/test.sh
+++ b/tests/misc-tests/006-image-on-missing-device/test.sh
@@ -24,6 +24,7 @@ test_run()
 	run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d raid1 -m raid1 "$dev1" "$dev2"
 
 	# we need extents to trigger reading from all devices
+	cond_wait_for_loopdevs
 	run_check $SUDO_HELPER mount "$dev1" "$TEST_MNT"
 	run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/a" bs=1M count=10
 	run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/b" bs=4k count=1000 conv=sync
diff --git a/tests/misc-tests/021-image-multi-devices/test.sh b/tests/misc-tests/021-image-multi-devices/test.sh
index ae157cda..25f3b775 100755
--- a/tests/misc-tests/021-image-multi-devices/test.sh
+++ b/tests/misc-tests/021-image-multi-devices/test.sh
@@ -19,6 +19,7 @@ loop3=${loopdevs[3]}
 # Create the test file system.
 
 run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$loop1" "$loop2"
+cond_wait_for_loopdevs
 run_check $SUDO_HELPER mount "$loop1" "$TEST_MNT"
 run_check $SUDO_HELPER dd bs=1M count=1 if=/dev/zero of="$TEST_MNT/foobar"
 orig_md5=$(run_check_stdout stat "$TEST_MNT/foobar" | md5sum | cut -d ' ' -f 1)
@@ -38,6 +39,7 @@ run_check $SUDO_HELPER "$TOP/btrfs-image" -r "$IMAGE" "$loop3"
 # Run check to make sure there is nothing wrong for the recovered image
 run_check $SUDO_HELPER "$TOP/btrfs" check "$loop3"
 
+cond_wait_for_loopdevs
 run_check $SUDO_HELPER mount "$loop3" "$TEST_MNT"
 new_md5=$(run_check_stdout stat "$TEST_MNT/foobar" | md5sum | cut -d ' ' -f 1)
 run_check $SUDO_HELPER umount "$TEST_MNT"
diff --git a/tests/misc-tests/030-missing-device-image/test.sh b/tests/misc-tests/030-missing-device-image/test.sh
index 762d7ea8..588d3e88 100755
--- a/tests/misc-tests/030-missing-device-image/test.sh
+++ b/tests/misc-tests/030-missing-device-image/test.sh
@@ -32,6 +32,7 @@ test_missing()
 	run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f -d raid1 -m raid1 "$dev1" "$dev2"
 
 	# fill the fs with some data, we could create space cache
+	cond_wait_for_loopdevs
 	run_check $SUDO_HELPER mount "$dev1" "$TEST_MNT"
 	run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/a" bs=1M count=10
 	run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/b" bs=4k count=1000 conv=sync
diff --git a/tests/misc-tests/035-receive-common-mount-point-prefix/test.sh b/tests/misc-tests/035-receive-common-mount-point-prefix/test.sh
index e89402e0..6b985734 100755
--- a/tests/misc-tests/035-receive-common-mount-point-prefix/test.sh
+++ b/tests/misc-tests/035-receive-common-mount-point-prefix/test.sh
@@ -23,9 +23,11 @@ loop2=$(run_check_stdout $SUDO_HELPER losetup --find --show dev2)
 run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$loop1"
 run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$loop2"
 
+cond_wait_for_loopdevs
 run_check $SUDO_HELPER mount "$loop1" "$TEST_MNT"
 run_check $SUDO_HELPER mkdir "$TEST_MNT/ddis"
 run_check $SUDO_HELPER mkdir "$TEST_MNT/ddis-not-a-mount"
+cond_wait_for_loopdevs
 run_check $SUDO_HELPER mount "$loop2" "$TEST_MNT/ddis"
 
 echo "some data" | $SUDO_HELPER tee "$TEST_MNT/ddis/file" > /dev/null
diff --git a/tests/misc-tests/046-seed-multi-mount/test.sh b/tests/misc-tests/046-seed-multi-mount/test.sh
index 95654fd9..38be4259 100755
--- a/tests/misc-tests/046-seed-multi-mount/test.sh
+++ b/tests/misc-tests/046-seed-multi-mount/test.sh
@@ -47,6 +47,7 @@ nextdevice() {
 	run_check $SUDO_HELPER "$TOP/btrfs" device add ${loopdevs[$nextdev]} "$TEST_MNT"
 	# Although seed sprout would make the fs RW, explicitly remount it RW
 	# just in case of future behavior change.
+	cond_wait_for_loopdevs
 	run_check $SUDO_HELPER mount -o remount,rw "$TEST_MNT"
 	# Rewrite the file
 	md5sum=$(run_check_stdout md5sum "$TEST_MNT/file$nextdev" | awk '{print $1}')
diff --git a/tests/mkfs-tests/001-basic-profiles/test.sh b/tests/mkfs-tests/001-basic-profiles/test.sh
index c37db87f..07df6faa 100755
--- a/tests/mkfs-tests/001-basic-profiles/test.sh
+++ b/tests/mkfs-tests/001-basic-profiles/test.sh
@@ -24,6 +24,7 @@ test_get_info()
 	# device RAID0 as the same.
 	# Thus kernel may create new SINGLE chunks, causing extra warning
 	# when testing single device RAID0.
+	cond_wait_for_loopdevs
 	run_check $SUDO_HELPER mount -o ro "$dev1" "$TEST_MNT"
 	run_check_stdout "$TOP/btrfs" filesystem df "$TEST_MNT" > "$tmp_out"
 	if grep -q "Multiple block group profiles detected" "$tmp_out"; then
diff --git a/tests/mkfs-tests/027-rootdir-inode/test.sh b/tests/mkfs-tests/027-rootdir-inode/test.sh
index df80f0c0..3f55aa6b 100755
--- a/tests/mkfs-tests/027-rootdir-inode/test.sh
+++ b/tests/mkfs-tests/027-rootdir-inode/test.sh
@@ -25,6 +25,7 @@ check_global_prereq getfattr
 # doesn't support xattr.
 # Instead we go $TEST_TOP/btrfs-progs-mkfs-tests-027.XXXXXX/ instead.
 run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$tmp_dev"
+cond_wait_for_loopdevs
 run_check $SUDO_HELPER mount -t btrfs "$tmp_dev" "$TEST_MNT"
 
 run_check $SUDO_HELPER mkdir "$TEST_MNT/source_dir/"
@@ -39,6 +40,7 @@ run_check $SUDO_HELPER setfattr -n user.foobar "$TEST_MNT/source_dir/foobar"
 run_check $SUDO_HELPER "$TOP/mkfs.btrfs" --rootdir "$TEST_MNT/source_dir" -f "$real_dev"
 run_check $SUDO_HELPER umount "$TEST_MNT"
 
+cond_wait_for_loopdevs
 run_check $SUDO_HELPER mount -t btrfs "$real_dev" "$TEST_MNT"
 
 new_mode=$(run_check_stdout $SUDO_HELPER stat "$TEST_MNT/" | grep "Uid:")