Merge branch 'work' into temp

Conflicts:
	.gitignore
	Makefile

7 files changed, 610 insertions, 10 deletions

diff --git a/.gitignore b/.gitignore
index 0d77a66..568021c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,4 @@ sendme
 sigwaittest
 svsematest
 pip_stress
+hackbench
diff --git a/Makefile b/Makefile
index 52f8d64..f02e4c3 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,8 @@
 VERSION_STRING = 0.66
 
 sources = cyclictest.c signaltest.c pi_stress.c rt-migrate-test.c	\
-	  ptsematest.c sigwaittest.c svsematest.c sendme.c pip_stress.c
+	  ptsematest.c sigwaittest.c svsematest.c sendme.c pip_stress.c \
+	  hackbench.c
 
 TARGETS = $(sources:.c=)
 
@@ -37,6 +38,7 @@ VPATH	+= src/sigwaittest:
 VPATH	+= src/svsematest:
 VPATH	+= src/backfire:
 VPATH	+= src/lib
+VPATH	+= src/hackbench
 
 %.o: %.c
 	$(CC) -D VERSION_STRING=$(VERSION_STRING) -c $< $(CFLAGS)
@@ -82,6 +84,9 @@ sendme: sendme.o rt-utils.o rt-get_cpu.o
 pip_stress: pip_stress.o error.o rt-utils.o
 	$(CC) $(CFLAGS) -o $@ $^ $(LIBS)
 
+hackbench: hackbench.o
+	$(CC) $(CFLAGS) -o $@ $^ $(LIBS)
+
 CLEANUP  = $(TARGETS) *.o .depend *.*~ *.orig *.rej rt-tests.spec *.d
 CLEANUP += $(if $(wildcard .git), ChangeLog)
 
@@ -116,6 +121,7 @@ install: all
 	gzip src/sigwaittest/sigwaittest.8 -c >"$(DESTDIR)$(mandir)/man8/sigwaittest.8.gz"
 	gzip src/svsematest/svsematest.8 -c >"$(DESTDIR)$(mandir)/man8/svsematest.8.gz"
 	gzip src/backfire/sendme.8 -c >"$(DESTDIR)$(mandir)/man8/sendme.8.gz"
+	gzip src/hackbench/hackbench.8 -c >"$(DESTDIR)$(mandir)/man8/hackbench.8.gz"
 
 .PHONY: release
 release: clean changelog
diff --git a/rt-tests.spec-in b/rt-tests.spec-in
index 0338f34..9c0c215 100644
--- a/rt-tests.spec-in
+++ b/rt-tests.spec-in
@@ -13,9 +13,9 @@ Obsoletes: cyclictest signaltest pi_tests
 BuildRequires: numactl-devel
 
 %description
-A set of programs that test and measure various components of "realtime"
-kernel behavior, such as timer latency, signal latency and the functioning
-of priority-inheritance mutexes.
+rt-tests is a set of programs that test and measure various components of
+real-time kernel behavior. This package measures timer, signal, and hardware
+latency. It also tests the functioning of priority-inheritance mutexes. 
 
 %prep
 %setup -qn rt-tests
@@ -45,6 +45,7 @@ rm -rf $RPM_BUILD_ROOT
 /usr/bin/sendme
 /usr/bin/sigwaittest
 /usr/bin/svsematest
+/usr/bin/hackbench
 %doc
 /usr/share/man/man8/cyclictest.8.gz
 /usr/share/man/man8/pi_stress.8.gz
@@ -54,8 +55,12 @@ rm -rf $RPM_BUILD_ROOT
 /usr/share/man/man8/sendme.8.gz
 /usr/share/man/man8/sigwaittest.8.gz
 /usr/share/man/man8/svsematest.8.gz
+/usr/share/man/man8/hackbench.8.gz
 
 %changelog
+* Fri Feb 19 2010 Clark Williams <williams@redhat.com> - 0.67-1
+- modified specfile to add hackbench
+
 * Mon Feb 15 2010 Clark Williams <williams@redhat.com> - 0.66-1
 - fix incorrect usage of sched_setscheduler in check_privs()
 
diff --git a/src/cyclictest/cyclictest.c b/src/cyclictest/cyclictest.c
index 3a6a0d0..1c00e2b 100644
--- a/src/cyclictest/cyclictest.c
+++ b/src/cyclictest/cyclictest.c
@@ -1,7 +1,7 @@
 /*
  * High resolution timer test software
  *
- * (C) 2008-2009 Clark Williams <williams@redhat.com>
+ * (C) 2008-2010 Clark Williams <williams@redhat.com>
  * (C) 2005-2007 Thomas Gleixner <tglx@linutronix.de>
  *
  * This program is free software; you can redistribute it and/or
@@ -107,6 +107,7 @@ enum {
 	IRQPREEMPTOFF,
 	WAKEUP,
 	WAKEUPRT,
+	CUSTOM,
 };
 
 /* Struct to transfer parameters to the thread */
@@ -146,7 +147,7 @@ struct thread_stat {
 
 static int shutdown;
 static int tracelimit = 0;
-static int ftrace = 0;
+static int ftrace = 1;
 static int kernelversion;
 static int verbose = 0;
 static int oscope_reduction = 1;
@@ -311,14 +312,14 @@ void tracing(int on)
 		switch (kernelversion) {
 		case KV_26_LT18: gettimeofday(0,(struct timezone *)1); break;
 		case KV_26_LT24: prctl(0, 1); break;
-		case KV_26_CURR: setkernvar("tracing_enabled", "1"); break;
+		case KV_26_CURR: setkernvar("tracing_on", "1"); break;
 		default:	 break;
 		}
 	} else {
 		switch (kernelversion) {
 		case KV_26_LT18: gettimeofday(0,0); break;
 		case KV_26_LT24: prctl(0, 0); break;
-		case KV_26_CURR: setkernvar("tracing_enabled", "0"); break;
+		case KV_26_CURR: setkernvar("tracing_on", "0"); break;
 		default:	 break;
 		}
 	}
@@ -388,6 +389,8 @@ static void setup_tracer(void)
 		char buffer[32];
 		int ret;
 
+		setkernvar("tracing_enabled", "1");
+
 		sprintf(buffer, "%d", tracelimit);
 		setkernvar("tracing_thresh", buffer);
 
@@ -446,6 +449,7 @@ static void setup_tracer(void)
 			fprintf(stderr, "Requested tracer '%s' not available\n", tracer);
 
 		setkernvar(traceroptions, "print-parent");
+		setkernvar(traceroptions, "latency-format");
 		if (verbose) {
 			setkernvar(traceroptions, "sym-offset");
 			setkernvar(traceroptions, "sym-addr");
@@ -902,7 +906,7 @@ static void process_options (int argc, char *argv[])
 			{"numa", no_argument, NULL, 'U'},
 			{NULL, 0, NULL, 0}
 		};
-		int c = getopt_long(argc, argv, "a::b:Bc:Cd:Efh:i:Il:MnNo:O:p:PmqrsSt::uUvD:wWTy:",
+		int c = getopt_long(argc, argv, "a::b:Bc:Cd:Efh:i:Il:MnNo:O:p:PmqrsSt::uUvD:wWT:y:",
 				    long_options, &option_index);
 		if (c == -1)
 			break;
@@ -958,7 +962,10 @@ static void process_options (int argc, char *argv[])
 			else
 				num_threads = max_cpus;
 			break;
-		case 'T': strncpy(tracer, optarg, sizeof(tracer)); break;
+		case 'T': 
+			tracetype = CUSTOM;
+			strncpy(tracer, optarg, sizeof(tracer)); 
+			break;
 		case 'u': setvbuf(stdout, NULL, _IONBF, 0); break;
 		case 'v': verbose = 1; break;
 		case 'm': lockall = 1; break;
diff --git a/src/hackbench/Makefile b/src/hackbench/Makefile
new file mode 100644
index 0000000..469cf25
--- /dev/null
+++ b/src/hackbench/Makefile
@@ -0,0 +1,5 @@
+hackbench: hackbench.c
+	$(CC) $(CFLAGS) -o hackbench hackbench.c -g -Wall -O2 -lpthread
+
+clean :
+	rm -f hackbench
diff --git a/src/hackbench/hackbench.8 b/src/hackbench/hackbench.8
new file mode 100644
index 0000000..d86c52f
--- /dev/null
+++ b/src/hackbench/hackbench.8
@@ -0,0 +1,105 @@
+.TH "hackbench" "8" "February  23, 2010" "" ""
+.SH "NAME"
+hackbench \- scheduler benchmark/stress test
+.SH "SYNOPSIS"
+.B hackbench
+.RI "[\-p|\-\-pipe] [\-s|\-\-datasize " <bytes> "] " 
+.RI "[\-l|\-\-loops " <num\-loops> "] "
+.RI "[\-g|\-\-groups "<num\-groups> "] "
+.RI "[\-f|\-\-fds <num\-fds>] "
+.RI "[\-T|\-\-threads] [\-P|\-\-process] [\-\-help]"
+
+.SH "DESCRIPTION"
+Hackbench is both a benchmark and a stress test for the Linux kernel
+scheduler. It's main job is to create a specified number of pairs of
+schedulable entities (either threads or traditional processes) which
+communicate via either sockets or pipes and time how long it takes for
+each pair to send data back and forth.
+
+.SH "OPTIONS"
+These programs follow the usual GNU command line syntax, with long
+options starting with two dashes ("\-\-").
+.br 
+A summary of options is included below.
+.TP 
+.B \-p, \-\-pipe
+Sends the data via a pipe instead of the socket (default)
+.TP 
+.B \-s, \-\-datasize=<size in bytes>
+Sets the amount of data to send in each message
+.TP 
+.B \-l, \-\-loops=<number of loops>
+How many messages each sender/receiver pair should send
+.TP 
+.B \-g, \-\-groups=<number of groups>
+Defines how many groups of senders and receivers should be started
+.TP 
+.B \-f, \-\-fds=<number of file descriptors>
+Defines how many file descriptors each child should use. 
+Note that the effective number will be twice the amount you set here,
+as the sender and receiver children will each open the given amount of 
+file descriptors.
+.TP 
+.B \-T, \-\-threads
+Each sender/receiver child will be a POSIX thread of the parent.
+.TP 
+.B \-P, \-\-process
+Hackbench will use fork() on all children (default behaviour)
+.TP 
+.B \-\-help
+.br 
+Shows a simple help screen
+.SH "EXAMPLES"
+.LP 
+Running hackbench without any options will give default behaviour,
+using fork() and sending data between senders and receivers via sockets.
+.LP 
+user@host: ~ $ hackbench
+.br 
+Running in process mode with 10 groups using 40 file descriptors each (== 400 tasks)
+.br 
+Each sender will pass 100 messages of 100 bytes
+.br 
+Time: 0.890
+.LP 
+To use pipes between senders and receivers and using threads instead of fork(), run
+.LP 
+user@host: ~ $ hackbench \-\-pipe \-\-threads   (or hackbench \-p \-T)
+.br 
+Running in threaded mode with 10 groups using 40 file descriptors each (== 400 tasks)
+.br 
+Each sender will pass 100 messages of 100 bytes
+.br 
+Time: 0.497
+.LP 
+Set the datasize to 512 bytes, do 200 messages per sender/receiver pairs and use 15 groups
+using 25 file descriptors per child, in process mode.
+.LP 
+user@host: ~ $ hackbench \-s 512 \-l 200 \-g 15 \-f 25 \-P
+.br 
+Running in process mode with 15 groups using 50 file descriptors each (== 750 tasks)
+.br 
+Each sender will pass 200 messages of 512 bytes
+.br 
+Time: 4.497
+.SH "AUTHORS"
+.LP 
+hackbench was written by Rusty Russell <rusty@rustcorp.com.au>
+with contributions from Yanmin Zhang <yanmin_zhang@linux.intel.com>,
+Ingo Molnar <mingo@elte.hu> and David Sommerseth <davids@redhat.com>
+
+This manual page was written by Clark Williams <williams@redhat.com> 
+and David Sommerseth <davids@redhat.com>
+.SH "HISTORY"
+This version of hackbench is based on the code downloaded from http://people.redhat.com/mingo/cfs\-scheduler/tools/hackbench.c. 
+Yanmin Zhang merged the original hackbench code from
+.br 
+http://devresources.linuxfoundation.org/craiger/hackbench/src/hackbench.c
+which uses fork() and a modified version from
+.br 
+http://www.bullopensource.org/posix/pi\-futex/hackbench_pth.c
+which uses pthread only and gave the possibility to change 
+behaviour at run time.  Hackbench have since then gone through some
+more rewriting to improve error handling and proper tracking of fork()ed
+children, to avoid leaving zombies on the system if hackbench stops
+unexpectedly.
diff --git a/src/hackbench/hackbench.c b/src/hackbench/hackbench.c
new file mode 100644
index 0000000..c8d12bd
--- /dev/null
+++ b/src/hackbench/hackbench.c
@@ -0,0 +1,471 @@
+/*
+ * This is the latest version of hackbench.c, that tests scheduler and
+ * unix-socket (or pipe) performance.
+ *
+ * Usage: hackbench [-pipe] <num groups> [process|thread] [loops]
+ *
+ * Build it with:
+ *   gcc -g -Wall -O2 -o hackbench hackbench.c -lpthread
+ *
+ * Downloaded from http://people.redhat.com/mingo/cfs-scheduler/tools/hackbench.c
+ * February 19 2010.
+ *
+ */
+
+/* Test groups of 20 processes spraying to 20 receivers */
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <sys/time.h>
+#include <sys/poll.h>
+#include <limits.h>
+#include <getopt.h>
+#include <signal.h>
+
+static unsigned int datasize = 100;
+static unsigned int loops = 100;
+static unsigned int num_groups = 10;
+static unsigned int num_fds = 20;
+
+/*
+ * 0 means thread mode and others mean process (default)
+ */
+static unsigned int process_mode = 1;
+
+static int use_pipes = 0;
+
+struct sender_context {
+	unsigned int num_fds;
+	int ready_out;
+	int wakefd;
+	int out_fds[0];
+};
+
+struct receiver_context {
+	unsigned int num_packets;
+	int in_fds[2];
+	int ready_out;
+	int wakefd;
+};
+
+
+typedef union {
+	pthread_t threadid;
+	pid_t	  pid;
+	long long error;
+} childinfo_t;
+
+childinfo_t *child_tab = NULL;
+unsigned int total_children = 0;
+unsigned int signal_caught = 0;
+
+inline static void sneeze(const char *msg) {
+	/* Avoid calling these functions when called from a code path
+	 * which involves sigcatcher(), as they are not reentrant safe.
+	 */
+	if( !signal_caught ) {
+		fprintf(stderr, "%s (error: %s)\n", msg, strerror(errno));
+	}
+}
+
+static void barf(const char *msg)
+{
+	sneeze(msg);
+	exit(1);
+}
+
+static void print_usage_exit()
+{
+	printf("Usage: hackbench [-p|--pipe] [-s|--datasize <bytes>] [-l|--loops <num loops>]\n"
+	       "\t\t [-g|--groups <num groups] [-f|--fds <num fds>]\n"
+	       "\t\t [-T|--threads] [-P|--process] [--help]\n");
+	exit(1);
+}
+
+static void fdpair(int fds[2])
+{
+	if (use_pipes) {
+		if (pipe(fds) == 0)
+			return;
+	} else {
+		if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == 0)
+			return;
+	}
+	barf("Creating fdpair");
+}
+
+/* Block until we're ready to go */
+static void ready(int ready_out, int wakefd)
+{
+	char dummy = '*';
+	struct pollfd pollfd = { .fd = wakefd, .events = POLLIN };
+
+	/* Tell them we're ready. */
+	if (write(ready_out, &dummy, 1) != 1)
+		barf("CLIENT: ready write");
+
+	/* Wait for "GO" signal */
+	if (poll(&pollfd, 1, -1) != 1)
+		barf("poll");
+}
+
+/* Sender sprays loops messages down each file descriptor */
+static void *sender(struct sender_context *ctx)
+{
+	char data[datasize];
+	unsigned int i, j;
+
+	ready(ctx->ready_out, ctx->wakefd);
+	memset(&data, '-', datasize);
+
+	/* Now pump to every receiver. */
+	for (i = 0; i < loops; i++) {
+		for (j = 0; j < ctx->num_fds; j++) {
+			int ret, done = 0;
+
+again:
+			ret = write(ctx->out_fds[j], data + done, sizeof(data)-done);
+			if (ret < 0)
+				barf("SENDER: write");
+			done += ret;
+			if (done < sizeof(data))
+				goto again;
+		}
+	}
+
+	return NULL;
+}
+
+
+/* One receiver per fd */
+static void *receiver(struct receiver_context* ctx)
+{
+	unsigned int i;
+
+	if (process_mode)
+		close(ctx->in_fds[1]);
+
+	/* Wait for start... */
+	ready(ctx->ready_out, ctx->wakefd);
+
+	/* Receive them all */
+	for (i = 0; i < ctx->num_packets; i++) {
+		char data[datasize];
+		int ret, done = 0;
+
+again:
+		ret = read(ctx->in_fds[0], data + done, datasize - done);
+		if (ret < 0)
+			barf("SERVER: read");
+		done += ret;
+		if (done < datasize)
+			goto again;
+	}
+	if (ctx) {
+		free(ctx);
+	}
+	return NULL;
+}
+
+childinfo_t create_worker(void *ctx, void *(*func)(void *))
+{
+	pthread_attr_t attr;
+	int err;
+	childinfo_t child;
+	pid_t childpid;
+
+	switch (process_mode) {
+	case 1: /* process mode */
+		/* Fork the sender/receiver child. */
+		switch ((childpid = fork())) {
+			case -1:
+				sneeze("fork()");
+				child.error = -1;
+				return child;
+			case 0:
+				(*func) (ctx);
+				exit(0);
+		}
+		child.pid = childpid;
+		break;
+
+	case 0: /* threaded mode */
+		if (pthread_attr_init(&attr) != 0) {
+			sneeze("pthread_attr_init()");
+			child.error = -1;
+			return child;
+		}
+
+#ifndef __ia64__
+		if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN) != 0) {
+			sneeze("pthread_attr_setstacksize()");
+			child.error = -1;
+			return child;
+		}
+#endif
+
+		if ((err=pthread_create(&child.threadid, &attr, func, ctx)) != 0) {
+			sneeze("pthread_create failed()");
+			child.error = -1;
+			return child;
+		}
+		break;
+	}
+	return child;
+}
+
+unsigned int reap_workers(childinfo_t *child, unsigned int totchld, unsigned int dokill)
+{
+	unsigned int i, rc = 0;
+	int status, err;
+	void *thr_status;
+
+	for( i = 0; i < totchld; i++ ) {
+		switch( process_mode ) {
+		case 1: /* process mode */
+			if( dokill ) {
+				kill(child[i].pid, SIGTERM);
+			}
+			fflush(stdout);
+			waitpid(child[i].pid, &status, 0);
+			if (!WIFEXITED(status))
+				rc++;
+			break;
+		case 0: /* threaded mode */
+			if( dokill ) {
+				pthread_kill(child[i].threadid, SIGTERM);
+			}
+			err = pthread_join(child[i].threadid, &thr_status);
+			if( err != 0 ) {
+				sneeze("pthread_join()");
+				rc++;
+			}
+			break;
+		}
+	}
+	return rc;
+}
+
+/* One group of senders and receivers */
+static unsigned int group(childinfo_t *child,
+			  unsigned int tab_offset,
+			  unsigned int num_fds,
+			  int ready_out,
+			  int wakefd)
+{
+	unsigned int i;
+	struct sender_context* snd_ctx = malloc (sizeof(struct sender_context)
+			+num_fds*sizeof(int));
+
+	if (!snd_ctx) {
+		sneeze("malloc() [sender ctx]");
+		return 0;
+	}
+
+
+	for (i = 0; i < num_fds; i++) {
+		int fds[2];
+		struct receiver_context* ctx = malloc (sizeof(*ctx));
+
+		if (!ctx) {
+			sneeze("malloc() [receiver ctx]");
+			return (i > 0 ? i-1 : 0);
+		}
+
+
+		/* Create the pipe between client and server */
+		fdpair(fds);
+
+		ctx->num_packets = num_fds*loops;
+		ctx->in_fds[0] = fds[0];
+		ctx->in_fds[1] = fds[1];
+		ctx->ready_out = ready_out;
+		ctx->wakefd = wakefd;
+
+		child[tab_offset+i] = create_worker(ctx, (void *)(void *)receiver);
+		if( child[tab_offset+i].error < 0 ) {
+			return (i > 0 ? i-1 : 0);
+		}
+		snd_ctx->out_fds[i] = fds[1];
+		if (process_mode)
+			close(fds[0]);
+	}
+
+	/* Now we have all the fds, fork the senders */
+	for (i = 0; i < num_fds; i++) {
+		snd_ctx->ready_out = ready_out;
+		snd_ctx->wakefd = wakefd;
+		snd_ctx->num_fds = num_fds;
+
+		child[tab_offset+num_fds+i] = create_worker(snd_ctx, (void *)(void *)sender);
+		if( child[tab_offset+num_fds+i].error < 0 ) {
+			return (num_fds+i)-1;
+		}
+	}
+
+	/* Close the fds we have left */
+	if (process_mode)
+		for (i = 0; i < num_fds; i++)
+			close(snd_ctx->out_fds[i]);
+
+	/* Return number of children to reap */
+	return num_fds * 2;
+}
+
+static void process_options (int argc, char *argv[])
+{
+	int error = 0;
+
+	while( 1 ) {
+		int optind = 0;
+
+		static struct option longopts[] = {
+			{"pipe",      no_argument,	 NULL, 'p'},
+			{"datasize",  required_argument, NULL, 's'},
+			{"loops",     required_argument, NULL, 'l'},
+			{"groups",    required_argument, NULL, 'g'},
+			{"fds",	      required_argument, NULL, 'f'},
+			{"threads",   no_argument,	 NULL, 'T'},
+			{"processes", no_argument,	 NULL, 'P'},
+			{"help",      no_argument,	 NULL, 'h'},
+			{NULL, 0, NULL, 0}
+		};
+
+		int c = getopt_long(argc, argv, "ps:l:g:f:TPh",
+				    longopts, &optind);
+		if (c == -1) {
+			break;
+		}
+		switch (c) {
+		case 'p':
+			use_pipes = 1;
+			break;
+
+		case 's':
+			if (!(argv[optind] && (datasize = atoi(optarg)) > 0)) {
+				fprintf(stderr, "%s: --datasize|-s requires an integer > 0\n", argv[0]);
+				error = 1;
+			}
+			break;
+
+		case 'l':
+			if (!(argv[optind] && (loops = atoi(optarg)) > 0)) {
+				fprintf(stderr, "%s: --loops|-l requires an integer > 0\n", argv[0]);
+				error = 1;
+			}
+			break;
+
+		case 'g':
+			if (!(argv[optind] && (num_groups = atoi(optarg)) > 0)) {
+				fprintf(stderr, "%s: --groups|-g requires an integer > 0\n", argv[0]);
+				error = 1;
+			}
+			break;
+
+		case 'f':
+			if (!(argv[optind] && (num_fds = atoi(optarg)) > 0)) {
+				fprintf(stderr, "%s: --fds|-f requires an integer > 0\n", argv[0]);
+				error = 1;
+			}
+			break;
+
+		case 'T':
+			process_mode = 0;
+			break;
+		case 'P':
+			process_mode = 1;
+			break;
+
+		case 'h':
+			print_usage_exit();
+
+		default:
+			error = 1;
+		}
+	}
+
+	if( error ) {
+		exit(1);
+	}
+}
+
+void sigcatcher(int sig) {
+	/* All caught signals will cause the program to exit */
+	signal_caught = 1;
+	if( child_tab && (total_children > 0) ) {
+		reap_workers(child_tab, total_children, 1);
+	}
+	fprintf(stderr, "** Operation aborted **\n");
+	exit(0);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct timeval start, stop, diff;
+	int readyfds[2], wakefds[2];
+	char dummy;
+
+	process_options (argc, argv);
+
+	printf("Running in %s mode with %d groups using %d file descriptors each (== %d tasks)\n",
+	       (process_mode == 0 ? "threaded" : "process"),
+	       num_groups, 2*num_fds, num_groups*(num_fds*2));
+	printf("Each sender will pass %d messages of %d bytes\n", loops, datasize);
+	fflush(NULL);
+
+	child_tab = calloc(num_fds * 2 * num_groups, sizeof(childinfo_t));
+	if (!child_tab)
+		barf("main:malloc()");
+
+	fdpair(readyfds);
+	fdpair(wakefds);
+
+	/* Catch some signals */
+	signal(SIGINT, sigcatcher);
+	signal(SIGTERM, sigcatcher);
+	signal(SIGHUP, SIG_IGN);
+
+	total_children = 0;
+	for (i = 0; i < num_groups; i++) {
+		int c = group(child_tab, total_children, num_fds, readyfds[1], wakefds[0]);
+		if( c != (num_fds*2) ) {
+			fprintf(stderr, "%i children started.  Expected %i\n", c, num_fds*2);
+			reap_workers(child_tab, total_children + c, 1);
+			barf("Creating workers");
+		}
+		total_children += c;
+	}
+
+	/* Wait for everyone to be ready */
+	for (i = 0; i < total_children; i++)
+		if (read(readyfds[0], &dummy, 1) != 1) {
+			reap_workers(child_tab, total_children, 1);
+			barf("Reading for readyfds");
+		}
+
+	gettimeofday(&start, NULL);
+
+	/* Kick them off */
+	if (write(wakefds[1], &dummy, 1) != 1) {
+		reap_workers(child_tab, total_children, 1);
+		barf("Writing to start senders");
+	}
+
+	/* Reap them all */
+	reap_workers(child_tab, total_children, 0);
+
+	gettimeofday(&stop, NULL);
+
+	/* Print time... */
+	timersub(&stop, &start, &diff);
+	printf("Time: %lu.%03lu\n", diff.tv_sec, diff.tv_usec/1000);
+	free(child_tab);
+	exit(0);
+}