cyclictest: add align thread wakeup times option

This patch provides and additional -A/--align flag to cyclictest to align
thread wakeup times of all threads as closly defined as possible.

When running multiple threads in cyclictest (-S or -t # option) the threads
are launched in an unsynchronized manner. Basically the creation order and
time for thread creation determines the start time. For provoking a maximum
congestion situation (e.g. cache evictions) and to improve reproducibility
or run conditions the start time should be defined distances appart. The
well defined distance is implemented as a offset parameter to -A/--align
and will offset each threads start time by the parameter * the sequentially
assigned thread number (par->tnum), together with the -d0 (distance in the
intervals of the individual threads) this alignment option allows to get
the thread wakeup times as closely synchronized as possible.

The method to sync is simply that the thread with par->tnum == 0 is chosen
to set a globally shared timestamp, and all other threads use this timestamp
as their starting time rather than each calling clock_gettime() at startup.
To ensure synchronization of the thread startup the setting of the global
time is guarded by pthread_barriers.

Signed-off-by: Nicholas Mc Guire <der.herr@hofr.at>
Reviewed-by: Andreas Platschek <andreas.platschek@opentech.at>
Signed-off-by: Clark Williams <clark.williams@gmail.com>

2 files changed, 44 insertions, 6 deletions

diff --git a/src/cyclictest/cyclictest.8 b/src/cyclictest/cyclictest.8
index fbb6174..60102df 100644
--- a/src/cyclictest/cyclictest.8
+++ b/src/cyclictest/cyclictest.8
@@ -16,7 +16,7 @@
 cyclictest \- High resolution test program
 .SH SYNOPSIS
 .B cyclictest
-.RI "[ \-hfmnqrsvMS ] [\-a " proc " ] [\-b " usec " ] [\-c " clock " ] [\-d " dist " ] \
+.RI "[ \-hfmnqrsvMS ] [\-a " proc " ] [\-A " align " ] [\-b " usec " ] [\-c " clock " ] [\-d " dist " ] \
 [\-h " histogram " ] [\-i " intv " ] [\-l " loop " ] [\-o " red " ] [\-p " prio " ] \
 [\-t " num " ] [\-D " time "] [\-w] [\-W] [\-y " policy " ] [ \-S | \-U ]"
 
@@ -36,6 +36,9 @@ A summary of options is included below.
 .B \-a, \-\-affinity[=PROC]
 Run all threads on procesor number PROC. If PROC is not specified, run thread #N on processor #N.
 .TP
+.B \-A, \-\-align=USEC
+Align measurement thread wakeups to a specific offset in microseconds
+.TP
 .B \-b, \-\-breaktrace=USEC
 Send break trace command when latency > USEC. This is a debugging option to control the latency tracer in the realtime preemption patch.
 It is useful to track down unexpected large latencies on a system. This option does only work with following kernel config options.
@@ -149,14 +152,14 @@ where n=task number c=count v=latency value in us. Use this option in combinatio
 .B \\-D, \-\-duration=TIME
 Run the test for the specified time, which defaults to seconds. Append 'm', 'h', or 'd' to specify minutes, hours or days
 .TP
-.B \\-w, \-\-wakeup 
+.B \\-w, \-\-wakeup
 task wakeup tracing (used with \-b)
 .TP
 .B \\-W, \-\-wakeuprt
 rt-task wakeup tracing (used with \-b)
 .TP
 .B \\-y, \-\-policy=NAME
-set the scheduler policy of the measurement threads 
+set the scheduler policy of the measurement threads
 where NAME is one of: other, normal, batch, idle, fifo, rr
 .TP
 .B \\-M, \-\-refresh_on_max
@@ -166,7 +169,7 @@ running cyclictest on low-bandwidth connections)
 .B \\-S, \-\-smp
 Set options for standard testing on SMP systems. Equivalent to using
 the options: "\-t \-a \-n" as well keeping any specified priority
-equal across all threads 
+equal across all threads
 .TP
 .B \\-U, \-\-numa
 Similar to the above \-\-smp option, this implies the "\-t \-a \-n"
diff --git a/src/cyclictest/cyclictest.c b/src/cyclictest/cyclictest.c
index d0b9dfe..bbdcf93 100644
--- a/src/cyclictest/cyclictest.c
+++ b/src/cyclictest/cyclictest.c
@@ -140,6 +140,7 @@ struct thread_param {
 	unsigned long interval;
 	int cpu;
 	int node;
+	int tnum;
 };
 
 /* Struct for statistics */
@@ -183,6 +184,8 @@ static int check_clock_resolution;
 static int ct_debug;
 static int use_fifo = 0;
 static pthread_t fifo_threadid;
+static int aligned = 0;
+static int disaligned = 0;
 
 static pthread_cond_t refresh_on_max_cond = PTHREAD_COND_INITIALIZER;
 static pthread_mutex_t refresh_on_max_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -191,6 +194,10 @@ static pthread_mutex_t break_thread_id_lock = PTHREAD_MUTEX_INITIALIZER;
 static pid_t break_thread_id = 0;
 static uint64_t break_thread_value = 0;
 
+static pthread_barrier_t align_barr;
+static pthread_barrier_t globalt_barr;
+static struct timespec globalt;
+
 /* Backup of kernel variables that we modify */
 static struct kvars {
 	char name[KVARNAMELEN];
@@ -768,7 +775,20 @@ void *timerthread(void *param)
 		fatal("timerthread%d: failed to set priority to %d\n", par->cpu, par->prio);
 
 	/* Get current time */
-	clock_gettime(par->clock, &now);
+	if(aligned){
+		pthread_barrier_wait(&globalt_barr);
+		if(par->tnum==0){
+			clock_gettime(par->clock, &globalt);
+		}
+		pthread_barrier_wait(&align_barr);
+		now = globalt;
+		if(disaligned){
+			now.tv_nsec += disaligned * par->tnum;
+			tsnorm(&now);
+		}
+	}
+	else
+		clock_gettime(par->clock, &now);
 
 	next = now;
 	next.tv_sec += interval.tv_sec;
@@ -958,6 +978,7 @@ static void display_help(int error)
 	       "cyclictest <options>\n\n"
 	       "-a [NUM] --affinity        run thread #N on processor #N, if possible\n"
 	       "                           with NUM pin all threads to the processor NUM\n"
+	       "-A USEC  --aligned=USEC    align thread wakeups to a specific offset\n"
 	       "-b USEC  --breaktrace=USEC send break trace command when latency > USEC\n"
 	       "-B       --preemptirqs     both preempt and irqsoff tracing (used with -b)\n"
 	       "-c CLOCK --clock=CLOCK     select clock\n"
@@ -1095,6 +1116,7 @@ enum option_values {
 	OPT_QUIET, OPT_PRIOSPREAD, OPT_RELATIVE, OPT_RESOLUTION, OPT_SYSTEM,
 	OPT_SMP, OPT_THREADS, OPT_TRACER, OPT_UNBUFFERED, OPT_NUMA, OPT_VERBOSE,
 	OPT_WAKEUP, OPT_WAKEUPRT, OPT_DBGCYCLIC, OPT_POLICY, OPT_HELP, OPT_NUMOPTS,
+	OPT_ALIGNED,
 };
 
 /* Process commandline options */
@@ -1113,6 +1135,7 @@ static void process_options (int argc, char *argv[])
 		static struct option long_options[] = {
 			{"affinity",         optional_argument, NULL, OPT_AFFINITY},
 			{"notrace",          no_argument,       NULL, OPT_NOTRACE },
+			{"aligned",          optional_argument, NULL, OPT_ALIGNED },
 			{"breaktrace",       required_argument, NULL, OPT_BREAKTRACE },
 			{"preemptirqs",      no_argument,       NULL, OPT_PREEMPTIRQ },
 			{"clock",            required_argument, NULL, OPT_CLOCK },
@@ -1154,7 +1177,7 @@ static void process_options (int argc, char *argv[])
 			{"help",             no_argument,       NULL, OPT_HELP },
 			{NULL, 0, NULL, 0}
 		};
-		int c = getopt_long(argc, argv, "a::b:Bc:Cd:D:Efh:H:i:Il:MnNo:O:p:PmqrRsSt::uUvD:wWT:",
+		int c = getopt_long(argc, argv, "a::A:b:Bc:Cd:D:Efh:H:i:Il:MnNo:O:p:PmqrRsSt::uUvD:wWT:",
 				    long_options, &option_index);
 		if (c == -1)
 			break;
@@ -1174,6 +1197,12 @@ static void process_options (int argc, char *argv[])
 				setaffinity = AFFINITY_USEALL;
 			}
 			break;
+		case 'A':
+		case OPT_ALIGNED:
+			aligned=1;
+			if (optarg != NULL)
+				disaligned = atoi(optarg);
+			break;
 		case 'b':
 		case OPT_BREAKTRACE:
 			tracelimit = atoi(optarg); break;
@@ -1413,6 +1442,11 @@ static void process_options (int argc, char *argv[])
 	if (num_threads < 1)
 		error = 1;
 
+	if (aligned) {
+		pthread_barrier_init (&globalt_barr, NULL, num_threads);
+		pthread_barrier_init (&align_barr, NULL, num_threads);
+	}
+
 	if (error)
 		display_help(1);
 }
@@ -1889,6 +1923,7 @@ int main(int argc, char **argv)
 		par->max_cycles = max_cycles;
 		par->stats = stat;
 		par->node = node;
+		par->tnum = i;
 		switch (setaffinity) {
 		case AFFINITY_UNSPECIFIED: par->cpu = -1; break;
 		case AFFINITY_SPECIFIED: par->cpu = affinity; break;