WIP: Add support for high-priority dm-crypt flag.high-priority-flags

10 files changed, 69 insertions, 8 deletions

diff --git a/lib/libcryptsetup.h b/lib/libcryptsetup.h
index 88d03bab..72bc12fc 100644
--- a/lib/libcryptsetup.h
+++ b/lib/libcryptsetup.h
@@ -1517,6 +1517,8 @@ int crypt_keyslot_destroy(struct crypt_device *cd, int keyslot);
 #define CRYPT_ACTIVATE_RECALCULATE_RESET (UINT32_C(1) << 26)
 /** dm-verity: try to use tasklets */
 #define CRYPT_ACTIVATE_TASKLETS (UINT32_C(1) << 27)
+/** dm-crypt: use high-priority workqueues */
+#define CRYPT_ACTIVATE_HIGH_PRIORITY (UINT32_C(1) << 28)
 
 /**
  * Active device runtime attributes
diff --git a/lib/libdevmapper.c b/lib/libdevmapper.c
index ebee542a..2e77849c 100644
--- a/lib/libdevmapper.c
+++ b/lib/libdevmapper.c
@@ -170,6 +170,9 @@ static void _dm_set_crypt_compat(struct crypt_device *cd,
 	if (_dm_satisfies_version(1, 22, 0, crypt_maj, crypt_min, crypt_patch))
 		_dm_flags |= DM_CRYPT_NO_WORKQUEUE_SUPPORTED;
 
+	if (_dm_satisfies_version(1, 26, 0, crypt_maj, crypt_min, crypt_patch))
+		_dm_flags |= DM_CRYPT_HIGH_PRIORITY_SUPPORTED;
+
 	_dm_crypt_checked = true;
 }
 
@@ -571,19 +574,22 @@ static char *get_dm_crypt_params(const struct dm_target *tgt, uint32_t flags)
 		num_options++;
 	if (flags & CRYPT_ACTIVATE_IV_LARGE_SECTORS)
 		num_options++;
+	if (flags & CRYPT_ACTIVATE_HIGH_PRIORITY)
+		num_options++;
 	if (tgt->u.crypt.integrity)
 		num_options++;
 	if (tgt->u.crypt.sector_size != SECTOR_SIZE)
 		num_options++;
 
-	if (num_options) { /* MAX length  int32 + 15 + 15 + 23 + 18 + 19 + 17 + 13 + int32 + integrity_str */
-		r = snprintf(features, sizeof(features), " %d%s%s%s%s%s%s%s%s", num_options,
+	if (num_options) { /* MAX length  int32 + 15 + 15 + 23 + 18 + 19 + 17 + 14 + 13 + int32 + integrity_str */
+		r = snprintf(features, sizeof(features), " %d%s%s%s%s%s%s%s%s%s", num_options,
 		(flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? " allow_discards" : "",
 		(flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? " same_cpu_crypt" : "",
 		(flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? " submit_from_crypt_cpus" : "",
 		(flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) ? " no_read_workqueue" : "",
 		(flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? " no_write_workqueue" : "",
 		(flags & CRYPT_ACTIVATE_IV_LARGE_SECTORS) ? " iv_large_sectors" : "",
+		(flags & CRYPT_ACTIVATE_HIGH_PRIORITY) ? " high_priority" : "",
 		(tgt->u.crypt.sector_size != SECTOR_SIZE) ?
 			_uf(sector_feature, sizeof(sector_feature), "sector_size", tgt->u.crypt.sector_size) : "",
 		integrity_dm);
@@ -1593,6 +1599,14 @@ static int check_retry(struct crypt_device *cd, uint32_t *dmd_flags, uint32_t dm
 		ret = 1;
 	}
 
+	/* Drop high-priority workqueue options if not supported */
+	if ((*dmd_flags & CRYPT_ACTIVATE_HIGH_PRIORITY) &&
+	    !(dmt_flags & DM_CRYPT_HIGH_PRIORITY_SUPPORTED)) {
+		log_dbg(cd, "dm-crypt does not support high-priority option");
+		*dmd_flags = *dmd_flags & ~CRYPT_ACTIVATE_HIGH_PRIORITY;
+		ret = 1;
+	}
+
 	return ret;
 }
 
@@ -1986,6 +2000,8 @@ static int _dm_target_query_crypt(struct crypt_device *cd, uint32_t get_flags,
 				*act_flags |= CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE;
 			else if (!strcasecmp(arg, "iv_large_sectors"))
 				*act_flags |= CRYPT_ACTIVATE_IV_LARGE_SECTORS;
+			else if (!strcasecmp(arg, "high_priority"))
+				*act_flags |= CRYPT_ACTIVATE_HIGH_PRIORITY;
 			else if (sscanf(arg, "integrity:%u:", &val) == 1) {
 				tgt->u.crypt.tag_size = val;
 				rintegrity = strchr(arg + strlen("integrity:"), ':');
diff --git a/lib/luks2/luks2_json_metadata.c b/lib/luks2/luks2_json_metadata.c
index 22f3e3df..00f111f4 100644
--- a/lib/luks2/luks2_json_metadata.c
+++ b/lib/luks2/luks2_json_metadata.c
@@ -1566,6 +1566,7 @@ static const struct  {
 	{ CRYPT_ACTIVATE_NO_JOURNAL,             "no-journal" },
 	{ CRYPT_ACTIVATE_NO_READ_WORKQUEUE,      "no-read-workqueue" },
 	{ CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE,     "no-write-workqueue" },
+	{ CRYPT_ACTIVATE_HIGH_PRIORITY,          "high_priority" },
 	{ 0, NULL }
 };
 
diff --git a/lib/utils_dm.h b/lib/utils_dm.h
index dbbd4701..a9ca9add 100644
--- a/lib/utils_dm.h
+++ b/lib/utils_dm.h
@@ -76,6 +76,7 @@ static inline uint32_t act2dmflags(uint32_t act_flags)
 #define DM_INTEGRITY_FIX_HMAC_SUPPORTED (1 << 26) /* hmac covers also superblock */
 #define DM_INTEGRITY_RESET_RECALC_SUPPORTED (1 << 27) /* dm-integrity automatic recalculation supported */
 #define DM_VERITY_TASKLETS_SUPPORTED (1 << 28) /* dm-verity tasklets supported */
+#define DM_CRYPT_HIGH_PRIORITY_SUPPORTED (1 << 29) /* dm-crypt high priority workqueue flag supported  */
 
 typedef enum { DM_CRYPT = 0, DM_VERITY, DM_INTEGRITY, DM_LINEAR, DM_ERROR, DM_ZERO, DM_UNKNOWN } dm_target_type;
 enum tdirection { TARGET_EMPTY = 0, TARGET_SET, TARGET_QUERY };
diff --git a/man/common_options.adoc b/man/common_options.adoc
index 841929bd..4cba277d 100644
--- a/man/common_options.adoc
+++ b/man/common_options.adoc
@@ -774,6 +774,17 @@ count is lower. This option is not available for PBKDF2.
 endif::[]
 
 ifdef::ACTION_REFRESH,ACTION_OPEN[]
+*--perf-high_priority*::
+Set dm-crypt workqueues and the writer thread to high priority.
+This i mproves throughput and latency of dm-crypt while degrading general
+responsiveness of the system.
++
+*NOTE:* This option is available only for low-level dm-crypt
+performance tuning, use only if you need a change to default dm-crypt
+behaviour. Needs kernel 6.10 or later.
+endif::[]
+
+ifdef::ACTION_REFRESH,ACTION_OPEN[]
 *--perf-no_read_workqueue, --perf-no_write_workqueue*::
 Bypass dm-crypt internal workqueue and process read or write requests
 synchronously.
diff --git a/src/cryptsetup.c b/src/cryptsetup.c
index fd580d73..57344e51 100644
--- a/src/cryptsetup.c
+++ b/src/cryptsetup.c
@@ -1036,13 +1036,15 @@ static int action_status(void)
 				 CRYPT_ACTIVATE_SAME_CPU_CRYPT|
 				 CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS|
 				 CRYPT_ACTIVATE_NO_READ_WORKQUEUE|
-				 CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE))
-			log_std("  flags:   %s%s%s%s%s\n",
+				 CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE|
+				 CRYPT_ACTIVATE_HIGH_PRIORITY))
+			log_std("  flags:   %s%s%s%s%s%s\n",
 				(cad.flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? "discards " : "",
 				(cad.flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? "same_cpu_crypt " : "",
 				(cad.flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? "submit_from_crypt_cpus " : "",
 				(cad.flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) ? "no_read_workqueue " : "",
-				(cad.flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? "no_write_workqueue" : "");
+				(cad.flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? "no_write_workqueue" : "",
+				(cad.flags & CRYPT_ACTIVATE_HIGH_PRIORITY) ? "high_priority" : "");
 	}
 out:
 	crypt_free(cd);
diff --git a/src/cryptsetup_arg_list.h b/src/cryptsetup_arg_list.h
index 74967487..510b52d4 100644
--- a/src/cryptsetup_arg_list.h
+++ b/src/cryptsetup_arg_list.h
@@ -141,6 +141,8 @@ ARG(OPT_PBKDF_MEMORY, '\0', POPT_ARG_STRING, N_("PBKDF memory cost limit"), N_("
 
 ARG(OPT_PBKDF_PARALLEL, '\0', POPT_ARG_STRING, N_("PBKDF parallel cost"), N_("threads"), CRYPT_ARG_UINT32, { .u32_value = DEFAULT_LUKS2_PARALLEL_THREADS }, {})
 
+ARG(OPT_PERF_HIGH_PRIORITY, '\0', POPT_ARG_NONE, N_("Set dm-crypt workqueues and the writer thread to high priority"), NULL, CRYPT_ARG_BOOL, {}, {})
+
 ARG(OPT_PERF_NO_READ_WORKQUEUE, '\0', POPT_ARG_NONE, N_("Bypass dm-crypt workqueue and process read requests synchronously"), NULL, CRYPT_ARG_BOOL, {}, {})
 
 ARG(OPT_PERF_NO_WRITE_WORKQUEUE, '\0', POPT_ARG_NONE, N_("Bypass dm-crypt workqueue and process write requests synchronously"), NULL, CRYPT_ARG_BOOL, {}, {})
diff --git a/src/utils_arg_names.h b/src/utils_arg_names.h
index 4ec5510b..604e1710 100644
--- a/src/utils_arg_names.h
+++ b/src/utils_arg_names.h
@@ -125,6 +125,7 @@
 #define OPT_PBKDF_FORCE_ITERATIONS	"pbkdf-force-iterations"
 #define OPT_PBKDF_MEMORY		"pbkdf-memory"
 #define OPT_PBKDF_PARALLEL		"pbkdf-parallel"
+#define OPT_PERF_HIGH_PRIORITY		"perf-high_priority"
 #define OPT_PERF_NO_READ_WORKQUEUE	"perf-no_read_workqueue"
 #define OPT_PERF_NO_WRITE_WORKQUEUE	"perf-no_write_workqueue"
 #define OPT_PERF_SAME_CPU_CRYPT		"perf-same_cpu_crypt"
diff --git a/src/utils_luks.c b/src/utils_luks.c
index 5c51a5a2..95f768b8 100644
--- a/src/utils_luks.c
+++ b/src/utils_luks.c
@@ -92,6 +92,9 @@ void set_activation_flags(uint32_t *flags)
 	if (ARG_SET(OPT_PERF_NO_WRITE_WORKQUEUE_ID))
 		*flags |= CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE;
 
+	if (ARG_SET(OPT_PERF_HIGH_PRIORITY_ID))
+		*flags |= CRYPT_ACTIVATE_HIGH_PRIORITY;
+
 	if (ARG_SET(OPT_INTEGRITY_NO_JOURNAL_ID))
 		*flags |= CRYPT_ACTIVATE_NO_JOURNAL;
 
diff --git a/tests/device-test b/tests/device-test
index 9aaf03c8..212cc865 100755
--- a/tests/device-test
+++ b/tests/device-test
@@ -106,6 +106,9 @@ function dm_crypt_features()
 
 	[ $VER_MIN -lt 22 ] && return
 	DM_PERF_NO_WORKQUEUE=1
+
+	[ $VER_MIN -lt 26 ] && return
+	DM_PERF_HIGH_PRIORITY=1
 }
 
 format() # format
@@ -183,12 +186,18 @@ else
 	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt && fail
 	echo -e "$PWD1" | $CRYPTSETUP refresh $PLAIN_OPT $DEV $DEV_NAME2 2>/dev/null && fail
 	if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then
-		echo -n "no_read_workqueue no_write_workqueue"
+		echo -n "no_read_workqueue no_write_workqueue "
 		echo -e "$PWD1" | $CRYPTSETUP refresh $PLAIN_OPT -q $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue || fail
 		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
 		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
 		check_io
 	fi
+	if [ -n "$DM_PERF_HIGH_PRIORITY" ]; then
+		echo -n "high_priority "
+		echo -e "$PWD1" | $CRYPTSETUP refresh $PLAIN_OPT -q $DEV_NAME --perf-high_priority || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q high_priority || fail
+		check_io
+	fi
 	$CRYPTSETUP close $DEV_NAME || fail
 	echo
 
@@ -215,11 +224,16 @@ else
 	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt && fail
 	echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME2 2>/dev/null && fail
 	if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then
-		echo -n "no_read_workqueue no_write_workqueue"
+		echo -n "no_read_workqueue no_write_workqueue "
 		echo -e "$PWD1" | $CRYPTSETUP refresh $DEV_NAME  --perf-no_read_workqueue --perf-no_write_workqueue || fail
 		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
 		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
 	fi
+	if [ -n "$DM_PERF_HIGH_PRIORITY" ]; then
+		echo -n "high_priority "
+		echo -e "$PWD1" | $CRYPTSETUP refresh $DEV_NAME  --perf-high_priority || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q high_priority || fail
+	fi
 	$CRYPTSETUP close $DEV_NAME || fail
 	echo
 
@@ -280,7 +294,7 @@ else
 		fi
 	fi
 	if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then
-		echo -n "no_read_workqueue no_write_workqueue"
+		echo -n "no_read_workqueue no_write_workqueue "
 		echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue --persistent || fail
 		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
 		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
@@ -289,6 +303,14 @@ else
 		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
 		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
 	fi
+	if [ -n "$DM_PERF_HIGH_PRIORITY" ]; then
+		echo -n "high_priority "
+		echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME --perf-high_priority --persistent || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q high_priority || fail
+		$CRYPTSETUP close $DEV_NAME || fail
+		echo -e "$PWD1" | $CRYPTSETUP open $DEV $DEV_NAME || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q high_priority || fail
+	fi
 	echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME2 2>/dev/null && fail
 	$CRYPTSETUP close $DEV_NAME || fail
 	echo