added files to git repo

Signed-off-by: Clark Williams <williams@redhat.com>

4 files changed, 814 insertions, 0 deletions

diff --git a/src/backfire/Makefile.module b/src/backfire/Makefile.module
new file mode 100644
index 0000000..30011b8
--- /dev/null
+++ b/src/backfire/Makefile.module
@@ -0,0 +1,6 @@
+obj-m	:=	backfire.o
+KDIR	:=	/lib/modules/$(shell uname -r)/build
+PWD	:=	$(shell pwd)
+
+default:
+	$(MAKE) -C $(KDIR) SUBDIRS=$(PWD) modules
diff --git a/src/include/rt-tests.h b/src/include/rt-tests.h
new file mode 100644
index 0000000..d7eefd9
--- /dev/null
+++ b/src/include/rt-tests.h
@@ -0,0 +1,124 @@
+#ifndef __RT_TESTS_H__
+#define __RT_TESTS_H__
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#define __USE_GNU
+#include <fcntl.h>
+#include <getopt.h>
+#include <pthread.h>
+#include <signal.h>
+#include <sched.h>
+#include <string.h>
+#include <time.h>
+#include <errno.h>
+
+#include <linux/unistd.h>
+
+#include <sys/prctl.h>
+#include <sys/stat.h>
+#include <sys/sysinfo.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/utsname.h>
+#include <sys/mman.h>
+
+#ifndef SCHED_IDLE
+#define SCHED_IDLE 5
+#endif
+#ifndef SCHED_NORMAL
+#define SCHED_NORMAL SCHED_OTHER
+#endif
+
+/* Ugly, but .... */
+#define gettid() syscall(__NR_gettid)
+#define sigev_notify_thread_id _sigev_un._tid
+
+#ifdef __UCLIBC__
+#define MAKE_PROCESS_CPUCLOCK(pid, clock) \
+	((~(clockid_t) (pid) << 3) | (clockid_t) (clock))
+#define CPUCLOCK_SCHED          2
+
+static int clock_nanosleep(clockid_t clock_id, int flags, const struct timespec *req,
+		struct timespec *rem)
+{
+	if (clock_id == CLOCK_THREAD_CPUTIME_ID)
+		return -EINVAL;
+	if (clock_id == CLOCK_PROCESS_CPUTIME_ID)
+		clock_id = MAKE_PROCESS_CPUCLOCK (0, CPUCLOCK_SCHED);
+
+	return syscall(__NR_clock_nanosleep, clock_id, flags, req, rem);
+}
+
+static int sched_setaffinity(pid_t pid, unsigned int cpusetsize,
+		cpu_set_t *mask)
+{
+	return -EINVAL;
+}
+
+static void CPU_SET(int cpu, cpu_set_t *set) { }
+static void CPU_ZERO(cpu_set_t *set) { }
+#else
+extern int clock_nanosleep(clockid_t __clock_id, int __flags,
+			   __const struct timespec *__req,
+			   struct timespec *__rem);
+#endif
+
+#define USEC_PER_SEC		1000000
+#define NSEC_PER_SEC		1000000000
+
+#define HIST_MAX		1000000
+
+/* Struct to transfer parameters to the thread */
+struct thread_param {
+	int prio;
+	int policy;
+	int mode;
+	int timermode;
+	int signal;
+	int clock;
+	unsigned long max_cycles;
+	struct thread_stat *stats;
+	int bufmsk;
+	unsigned long interval;
+	int cpu;
+};
+
+/* Struct for statistics */
+struct thread_stat {
+	unsigned long cycles;
+	unsigned long cyclesread;
+	long min;
+	long max;
+	long act;
+	double avg;
+	long *values;
+	long *hist_array;
+	pthread_t thread;
+	int threadstarted;
+	int tid;
+	long reduce;
+	long redmax;
+	long cycleofmax;
+	long hist_overflow;
+};
+
+enum kernelversion {
+	KV_NOT_26,	/* not a 2.6 kernel */
+	KV_26_LT18,	/* less than 2.6.18 */
+	KV_26_LT24,	/* less than 2.6.24 */
+	KV_26_LT28,	/* less than 2.6.28 */
+	KV_26_CURR,	/* 2.6.28+          */
+};
+
+enum kernelversion check_kernel(void);
+
+enum {
+	ERROR_GENERAL	= -1,
+	ERROR_NOTFOUND	= -2,
+};
+
+#define TIMER_RELTIME		0
+
+#endif
diff --git a/src/pi_tests/Makefile b/src/pi_tests/Makefile
new file mode 100644
index 0000000..6b9b595
--- /dev/null
+++ b/src/pi_tests/Makefile
@@ -0,0 +1,71 @@
+PACKAGE	:=	pi_tests
+SPEC	:=	$(PACKAGE).spec
+VERSION	:=	$(shell awk '/^Version:/ { print $$2 }' $(SPEC))
+RELEASE	:=	$(shell awk '/Release:/ { print $$2 }' ${PACKAGE}.spec)
+SRPM	:=	$(PACKAGE)-$(VERSION)-$(RELEASE).src.rpm
+HERE	:= 	$(shell pwd)
+TARGET	:=	$(shell uname -p)
+RPMARGS	:=	--define "_sourcedir $(HERE)" 	\
+		--define "_builddir $(HERE)/BUILD"	\
+		--define "_rpmdir $(HERE)/RPMS" 	\
+		--define "_srcrpmdir $(HERE)/SRPMS"
+CC	:= gcc
+OPT	:= -g -O
+CFLAGS 	:= $(OPT) -Wall -D_GNU_SOURCE -DVERSION=\"$(VERSION)\"
+LIBS	:= -lpthread -lrt
+
+SRC	:= classic_pi.c allprios.c tst-mutexpi10.c pi_stress.c sigtest.c
+DOC	:= pi_stress.8 README COPYING
+TARGETS	:= $(subst .c,,$(SRC))
+
+all: classic_pi pi_stress
+
+classic_pi: classic_pi.c
+	$(CC) $(CFLAGS) -o $@ $< $(LIBS)
+
+pi_stress: pi_stress.c
+	$(CC) $(CFLAGS) -o $@ $< $(LIBS)
+
+tst-mutexpi10:  tst-mutexpi10.c
+	$(CC) $(CFLAGS) -o $@ $< $(LIBS)
+
+#tst-mutexpi10.c:	classic_pi.c
+#	indent --gnu-style classic_pi.c -o tst-mutexpi10.c
+
+allprios:	allprios.c
+	$(CC) $(CFLAGS) -o $@ $< $(LIBS)
+
+tarball:  pi_tests-$(VERSION).tar.gz
+
+libctest: tst-mutexpi10
+
+sigtest: sigtest.c
+	$(CC) $(CFLAGS) -o $@ $< $(LIBS)
+
+pi_tests-$(VERSION).tar.gz:	clean $(SRC) $(DOC) Makefile pi_tests.spec
+	rm -rf $@ pi_tests-$(VERSION)
+	mkdir pi_tests-$(VERSION)
+	cp $(SRC) $(DOC) Makefile pi_tests.spec pi_tests-$(VERSION)
+	tar -czvf pi_tests-$(VERSION).tar.gz pi_tests-$(VERSION)
+
+clean:
+	rm -rf *~ *.[oi] $(TARGETS) *.tar.gz pi_tests-*
+
+rpm:  rpmdirs $(SPEC) tarball
+	rm -rf RPMS/$(TARGET)
+	rpmbuild -ba --target $(TARGET) $(RPMARGS) $(SPEC)
+
+rpmdirs:
+	@[ -d BUILD ]  || mkdir BUILD
+	@[ -d RPMS ]   || mkdir RPMS
+	@[ -d SRPMS ]  || mkdir SRPMS
+
+rpmlint lint: rpm
+	rpmlint -vi RPMS/*/*
+	rpmlint -vi SRPMS/*
+
+DESTINATION	:= 	people.redhat.com:~williams/public_html/tests
+
+push: rpm
+	scp pi_tests-$(VERSION).tar.gz $(DESTINATION)
+	scp SRPMS/* $(DESTINATION)
diff --git a/src/pi_tests/classic_pi.c b/src/pi_tests/classic_pi.c
new file mode 100644
index 0000000..59e907b
--- /dev/null
+++ b/src/pi_tests/classic_pi.c
@@ -0,0 +1,613 @@
+/*
+  classic_pi - Classic Priority Inversion deadlock test case
+  
+   Copyright (C) 2006, 2007 Clark Williams <williams@redhat.com>
+  
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+   USA */
+
+/* This program tests Priority Inheritence mutexes and their ability
+   to avoid Priority Inversion deadlocks
+  
+   The basic premise here is to set up a deadlock scenario and confirm that PI
+   mutexes resolve the situation. Three worker threads will be created from the
+   main thread: low, medium and high priority threads that use SCHED_FIFO as 
+   their scheduling policy. The low priority thread claims a mutex and then
+   starts "working". The medium priority thread starts and preempts the low 
+   priority thread. Then the high priority thread runs and attempts to claim
+   the mutex owned by the low priority thread. Without priority inheritance, 
+   this will deadlock the program. With priority inheritance, the low priority 
+   thread receives a priority boost, finishes it's "work" and releases the mutex,
+   which allows the high priority thread to run and finish and then the medium
+   priority thread finishes. 
+   
+   That's the theory, anyway... 
+   
+   CW - 2006  */
+
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <pthread.h>
+#include <sched.h>
+#include <unistd.h>
+#include <string.h>
+#include <signal.h>
+#include <getopt.h>
+
+/* test timeout */
+#define TIMEOUT 2
+
+/* determine if the C library supports Priority Inheritance mutexes */
+#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT != -1
+#define HAVE_PI_MUTEX 1
+#else
+#define HAVE_PI_MUTEX 0
+#endif
+
+int use_pi_mutex = HAVE_PI_MUTEX;
+
+#define SUCCESS 0
+#define FAILURE 1
+
+/* the number of times we cause a priority inversion situation */
+int inversions = 1;
+
+int verbose = 0;
+
+struct option options [] = {
+	{ "verbose", no_argument, NULL, 'v' },
+	{ "quiet", no_argument, NULL, 'q' },
+	{ "no-pi", no_argument, NULL, 'n'}, 
+	{ "inversions" , required_argument, NULL, 'i'},
+};
+
+/* define priorities for the threads */
+#define SKEL_PRIO(x) (x)
+#define MAIN_PRIO(x) (x - 1)
+#define HIGH_PRIO(x) (x - 2)
+#define MED_PRIO(x)  (x - 3)
+#define LOW_PRIO(x)  (x - 4)
+
+enum thread_names {LOW=0, MEDIUM, HIGH, NUM_WORKER_THREADS};
+
+pthread_mutex_t mutex;
+pthread_mutexattr_t mutex_attr;
+
+pthread_barrier_t all_threads_ready;
+pthread_barrier_t all_threads_done;
+
+// state barriers
+pthread_barrier_t start_barrier;
+pthread_barrier_t locked_barrier;
+pthread_barrier_t elevate_barrier;
+pthread_barrier_t finish_barrier;
+
+volatile int deadlocked = 0;
+volatile int high_has_run = 0;
+volatile int low_unlocked = 0;
+
+cpu_set_t cpu_mask;
+
+struct thread_parameters {
+	pthread_t tid;
+	int inversions;
+} thread_parameters[NUM_WORKER_THREADS];
+
+/* forward prototypes */
+void *low_priority(void *arg);
+void *med_priority(void *arg);
+void *high_priority(void *arg);
+int setup_thread_attr(pthread_attr_t *attr, int prio, cpu_set_t *mask);
+int set_cpu_affinity(cpu_set_t *mask);
+void error(char *, ...);
+void info(char *, ...);
+
+int
+initialize_barriers(void)
+{
+	int status;
+
+	status = pthread_barrier_init(&all_threads_ready, NULL, NUM_WORKER_THREADS+1);
+	if (status) {
+		error("initialize_barriers: failed to initialize all_threads_ready\n");
+		return FAILURE;
+	}
+	status = pthread_barrier_init(&all_threads_done, NULL, NUM_WORKER_THREADS+1);
+	if (status) {
+		error("initialize_barriers: failed to initialize all_threads_done\n");
+		return FAILURE;
+	}
+	status = pthread_barrier_init(&start_barrier, NULL, NUM_WORKER_THREADS);
+	if (status) {
+		error("initialize_barriers: failed to initialize start_barrier\n");
+		return FAILURE;
+	}
+	status = pthread_barrier_init(&locked_barrier, NULL, 2);
+	if (status) {
+		error("initializing_barriers: failed to intialize locked_barrier\n");
+		return FAILURE;
+	}
+	status = pthread_barrier_init(&elevate_barrier, NULL, 2);
+	if (status) {
+		error("initializing_barriers: failed to initialize elevate_barrier\n");
+		return FAILURE;
+	}
+	status = pthread_barrier_init(&finish_barrier, NULL, NUM_WORKER_THREADS);
+	if (status) {
+		error("initializing_barriers: failed to initialize finish_barrier\n");
+		return FAILURE;
+	}
+	return SUCCESS;
+}	
+
+void cleanup(void)
+{
+	int i;
+	int status;
+	for (i = 0; i < NUM_WORKER_THREADS; i++) {
+		status = pthread_kill(thread_parameters[i].tid, SIGQUIT);
+		if (status)
+			error("cleanup: error sending SIGQUIT to thread %d\n", 
+			      thread_parameters[i].tid);
+	}
+}
+
+void handler(int signal)
+{
+	info("handler: %s fired\n", sys_siglist[signal]);
+	cleanup();
+	if (signal == SIGALRM) {
+		error("handler: DEADLOCK detected!\n");
+		deadlocked = 1;
+	}
+}
+
+void usage(void)
+{
+	printf ("classic_pi [options]\n");
+	printf ("   options:\n");
+	printf ("        -v|--verbose\n");
+	printf ("        -q|--quiet\n");
+	printf ("        -n|--no-pi\n");
+	printf ("        -i <n> |--inversions=<n>\n");
+}
+
+int main(int argc, char **argv)
+{
+	int status;
+	int prio_max;
+	pthread_attr_t thread_attr;
+	struct sched_param thread_param;
+	int opt;
+
+	/* Make sure we see all message, even those on stdout.  */
+	setvbuf (stdout, NULL, _IONBF, 0);
+
+	/* process command line arguments */
+	while ((opt = getopt_long(argc, argv, "+", options, NULL)) != -1) {
+		switch (opt) {
+		case '?':
+			usage();
+			exit(1);
+		case 'v':
+			verbose = 1;
+			break;
+		case 'q':
+			verbose = 0;
+			break;
+		case 'n':
+			use_pi_mutex = 0;
+			break;
+		case 'i':
+			inversions = strtol(optarg, NULL, 10);
+			info("main: doing %d inversion loops\n", inversions);
+			break;
+		}
+	}
+
+	/* initialize default attributes for the mutex */
+	status = pthread_mutexattr_init(&mutex_attr);
+	if (status) {
+		error("main: initializing mutex attribute: 0x%x\n", status);
+		return FAILURE;
+	}
+
+	if (use_pi_mutex) {
+		/* set priority inheritance attribute for mutex */
+		status = pthread_mutexattr_setprotocol(&mutex_attr, 
+						       PTHREAD_PRIO_INHERIT);
+		if (status) {
+			error("main: setting mutex attribute policy: 0x%x\n", status);
+			return FAILURE;
+		}
+	}
+	info("main: Priority Inheritance turned %s\n", use_pi_mutex ? "on" : "off");
+
+	/* initialize our mutex */
+	status = pthread_mutex_init(&mutex, &mutex_attr);
+	if (status) {
+		error("main: initializing mutex: 0x%x\n", status);
+		return FAILURE;
+	}
+
+	/* set up our barriers */
+	status = initialize_barriers();
+	if (status)
+		return FAILURE;
+
+	/* set up CPU affinity so we only use one processor */
+	if (set_cpu_affinity(&cpu_mask))
+		return FAILURE;
+		
+	/* boost us to max priority (so we keep running) :) */
+	prio_max = sched_get_priority_max(SCHED_FIFO);
+	thread_param.sched_priority = MAIN_PRIO(prio_max);
+	status = pthread_setschedparam(pthread_self(), SCHED_FIFO, &thread_param);
+	if (status) {
+		error("main: boosting to max priority: 0x%x\n", status);
+		/* Don't fail if we don't have the right privledges */
+		return SUCCESS;
+	}
+
+	/* start the low priority thread */
+	info("main: creating low priority thread\n");
+	setup_thread_attr(&thread_attr, LOW_PRIO(prio_max), &cpu_mask);
+	thread_parameters[LOW].inversions = inversions;
+	status = pthread_create(&thread_parameters[LOW].tid, 
+				&thread_attr, 
+				low_priority, 
+				&thread_parameters[LOW]);
+	if (status != 0) {
+		error("main: creating low_priority thread: 0x%x\n", status);
+		return FAILURE;
+	}
+
+	/* create the medium priority thread */
+	info("main: creating medium priority thread\n");
+	setup_thread_attr(&thread_attr, MED_PRIO(prio_max), &cpu_mask);
+	thread_parameters[MEDIUM].inversions = inversions;
+	status = pthread_create(&thread_parameters[MEDIUM].tid, 
+				&thread_attr, 
+				med_priority, 
+				&thread_parameters[MEDIUM]);
+	if (status != 0) {
+		error("main: creating med_priority thread: 0x%x\n", status);
+		return FAILURE;
+	}
+
+	/* create the high priority thread */
+	info("main: creating high priority thread\n");
+	if (setup_thread_attr(&thread_attr, HIGH_PRIO(prio_max), &cpu_mask))
+		return FAILURE;
+	thread_parameters[HIGH].inversions = inversions;
+	status = pthread_create(&thread_parameters[HIGH].tid, 
+				&thread_attr, 
+				high_priority, 
+				&thread_parameters[HIGH]);
+	if (status != 0) {
+		error("main: creating high_priority thread: 0x%x\n", status);
+		cleanup();
+		return FAILURE;
+	}
+
+	signal(SIGINT, handler);
+	signal(SIGALRM, handler);
+
+	info("main: releasing all threads\n");
+	status = pthread_barrier_wait(&all_threads_ready);
+	if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+		error("main: pthread_barrier_wait(all_threads_ready): 0x%x\n", status);
+		cleanup();
+		return FAILURE;
+	}
+	info("main: all threads initialized, waiting for mutex to be claimed\n");
+
+	alarm(TIMEOUT * inversions);
+	info("main: waiting for threads to finish\n");
+	
+	status = pthread_barrier_wait(&all_threads_done);
+	if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+		error("main: pthread_barrier_wait(all_threads_done): 0x%x\n", status);
+		cleanup();
+		return FAILURE;
+	}
+	alarm(0);
+	info("main: all threads terminated!\n");
+	if (deadlocked) {
+		info("main: test failed\n");
+		return FAILURE;
+	}
+	info("main: test passed\n");
+	return SUCCESS;
+}
+
+
+int setup_thread_attr(pthread_attr_t *attr, int prio, cpu_set_t *mask)
+{
+	int status;
+	struct sched_param thread_param;
+
+	status = pthread_attr_init(attr);
+	if (status) {
+		error("setup_thread_attr: initializing thread attribute: 0x%x\n", status);
+		return FAILURE;
+	}
+	status = pthread_attr_setschedpolicy(attr, SCHED_FIFO);
+	if (status) {
+		error("setup_thread_attr: setting attribute policy to SCHED_FIFO: 0x%x\n", status);
+		return FAILURE;
+	}
+	status = pthread_attr_setinheritsched(attr, PTHREAD_EXPLICIT_SCHED);
+	if (status) {
+		error("setup_thread_attr: setting explicit scheduling inheritance: 0x%x\n", status);
+		return FAILURE;
+	}
+	thread_param.sched_priority = prio;
+	status = pthread_attr_setschedparam(attr, &thread_param);
+	if (status) {
+		error("setup_thread_attr: setting scheduler param: 0x%x\n", status);
+		return FAILURE;
+	}
+	status = pthread_attr_setaffinity_np(attr, sizeof(cpu_set_t), mask);
+	if (status) {
+		error("setup_thread_attr: setting affinity attribute: 0x%x\n", status);
+		return FAILURE;
+	}
+	return SUCCESS;
+}
+
+int set_cpu_affinity(cpu_set_t *cpu_set)
+{
+	int status, i;
+	cpu_set_t current_mask, new_mask;
+
+	/* Now set our CPU affinity to only run one one processor */
+	status = sched_getaffinity(0, sizeof(cpu_set_t), &current_mask);
+	if (status) {
+		error("set_cpu_affinity: getting CPU affinity mask: 0x%x\n", status);
+		return FAILURE;
+	}
+	for (i = 0; i < sizeof(cpu_set_t) * 8; i++) {
+		if (CPU_ISSET(i, &current_mask))
+			break;
+	}
+	if (i >= sizeof(cpu_set_t) * 8) {
+		error("set_cpu_affinity: No schedulable CPU found!\n");
+		return FAILURE;
+	}
+	CPU_ZERO(&new_mask);
+	CPU_SET(i, &new_mask);
+	status = sched_setaffinity(0, sizeof(cpu_set_t), &new_mask);
+	if (status) {
+		error("set_cpu_affinity: setting CPU affinity mask: 0x%x\n", status);
+		return FAILURE;
+	}
+	info("set_cpu_affinity: using processr %d\n", i);
+	*cpu_set = new_mask;
+	return SUCCESS;
+}
+
+void report_threadinfo(char *name)
+{
+	int status;
+	struct sched_param thread_param;
+	int thread_policy;
+
+	status = pthread_getschedparam(pthread_self(), &thread_policy, &thread_param);
+	if (status) {
+		error("report_threadinfo: failed to get scheduler param: 0x%x\n", status);
+		exit(FAILURE);
+	}
+	info("%s: running as %s thread at priority %d\n",
+	     name, thread_policy == SCHED_FIFO ? "FIFO" :
+	     thread_policy == SCHED_RR ? "RR" : "OTHER",
+	     thread_param.sched_priority);
+}
+
+void *low_priority(void *arg)
+{
+	int status;
+	struct thread_parameters *p = (struct thread_parameters *)arg;
+
+	report_threadinfo("low_priority");
+
+	info("low_priority: entering ready state\n");
+
+	/* wait for all threads to be ready */
+	status = pthread_barrier_wait(&all_threads_ready);
+	if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+		error("low_priority: pthread_barrier_wait(all_threads_ready): %x", status);
+		return NULL;
+	}
+
+	info("low_priority: starting inversion loop (%d)\n", p->inversions);
+	while (p->inversions-- > 0) {
+		/* initial state */
+		info("low_priority: entering start wait (%d)\n", p->inversions+1);
+		status = pthread_barrier_wait(&start_barrier);
+		if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+			error("low_priority: pthread_barrier_wait(start): %x\n", status);
+			return NULL;
+		}
+		info("low_priority: claiming mutex\n");
+		pthread_mutex_lock(&mutex);
+		info("low_priority: mutex locked\n");
+
+		info("low_priority: entering locked wait\n");
+		status = pthread_barrier_wait(&locked_barrier);
+		if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+			error("low_priority: pthread_barrier_wait(locked): %x\n", status);
+			return NULL;
+		}
+
+		/* wait for priority boost */
+		info("low_priority: entering elevated wait\n");
+		low_unlocked = 0; /* prevent race with med_priority */
+		status = pthread_barrier_wait(&elevate_barrier);
+		if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+			error("low_priority: pthread_barrier_wait(elevate): %x\n", status);
+			return NULL;
+		}
+		low_unlocked = 1;
+
+		/* release the mutex */
+		info("low_priority: unlocking mutex\n");
+		pthread_mutex_unlock(&mutex);
+
+		/* finish state */
+		info("low_priority: entering finish wait\n");
+		status = pthread_barrier_wait(&finish_barrier);
+		if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+			error("low_priority: pthread_barrier_wait(elevate): %x\n", status);
+			return NULL;
+		}
+		
+	}
+	/* let main know we're done */
+	info("low_priority: entering exit state\n");
+	status = pthread_barrier_wait(&all_threads_done);
+	if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+		error("low_priority: pthread_barrier_wait(all_threads_done): %x", status);
+		return NULL;
+	}
+	info("low_priority: exiting\n");
+	return NULL;
+}
+
+void *med_priority(void *arg)
+{
+	int status;
+	struct thread_parameters *p = (struct thread_parameters *)arg;
+
+	report_threadinfo("med_priority");
+
+	info("med_priority: entering ready state\n");
+	/* wait for all threads to be ready */
+	status = pthread_barrier_wait(&all_threads_ready);
+	if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+		error("med_priority: pthread_barrier_wait(all_threads_ready): %x", status);
+		return NULL;
+	}
+
+	info("med_priority: starting inversion loop (%d)\n", p->inversions);
+	while (p->inversions-- > 0) {
+		/* start state */
+		info("med_priority: entering start state (%d)\n", p->inversions+1);
+		status = pthread_barrier_wait(&start_barrier);
+		if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+			error("med_priority: pthread_barrier_wait(start): %x", status);
+			return NULL;
+		}
+		info("med_priority: entering elevate state\n");
+		do {
+			status = pthread_barrier_wait(&elevate_barrier);
+			if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+				error("med_priority: pthread_barrier_wait(elevate): %x", status);
+				return NULL;
+			}
+		} while (!high_has_run && !low_unlocked);
+		info("med_priority: entering finish state\n");
+		status = pthread_barrier_wait(&finish_barrier);
+		if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+			error("med_priority: pthread_barrier_wait(finished): %x", status);
+			return NULL;
+		}
+	}
+
+	info("med_priority: entering exit state\n");
+	status = pthread_barrier_wait(&all_threads_done);
+	if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+		error("med_priority: pthread_barrier_wait(all_threads_done): %x", status);
+		return NULL;
+	}
+	info("med_priority: exiting\n");
+	return NULL;
+}
+
+void *high_priority(void *arg)
+{
+	int status;
+	struct thread_parameters *p = (struct thread_parameters *)arg;
+
+	report_threadinfo("high_priority");
+
+	info("high_priority: entering ready state\n");
+
+	/* wait for all threads to be ready */
+	status = pthread_barrier_wait(&all_threads_ready);
+	if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+		error("high_priority: pthread_barrier_wait(all_threads_ready): %x", status);
+		return NULL;
+	}
+
+	info("high_priority: starting inversion loop (%d)\n", p->inversions);
+	while (p->inversions-- > 0) {
+		high_has_run = 0;
+		info("high_priority: entering start state (%d)\n", p->inversions+1);
+		status = pthread_barrier_wait(&start_barrier);
+		if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+			error("high_priority: pthread_barrier_wait(start): %x", status);
+			return NULL;
+		}
+		info("high_priority: entering running state\n");
+		status = pthread_barrier_wait(&locked_barrier);
+		if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+			error("high_priority: pthread_barrier_wait(running): %x", status);
+			return NULL;
+		}
+		info("high_priority: locking mutex\n");
+		pthread_mutex_lock(&mutex);
+		info("high_priority: got mutex\n");
+		high_has_run = 1;
+		info("high_priority: unlocking mutex\n");
+		pthread_mutex_unlock(&mutex);
+		info("high_priority: entering finish state\n");
+		status = pthread_barrier_wait(&finish_barrier);
+		if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+			error("high_priority: pthread_barrier_wait(finish): %x", status);
+			return NULL;
+		}
+	}
+       
+	info("high_priority: entering exit state\n");
+	status = pthread_barrier_wait(&all_threads_done);
+	if (status && status != PTHREAD_BARRIER_SERIAL_THREAD) {
+		error("high_priority: pthread_barrier_wait(all_threads_done): %x", status);
+		return NULL;
+	}
+	info("high_priority: exiting\n");	
+	return NULL;
+}
+
+void error(char *fmt, ...)
+{
+	va_list ap;
+	fputs("ERROR: ", stderr);
+	va_start(ap, fmt);
+	vfprintf(stderr, fmt, ap);
+	va_end(ap);
+}
+
+void info(char *fmt, ...)
+{
+	if (verbose) {
+		va_list ap;
+		va_start(ap, fmt);
+		vprintf(fmt, ap);
+		va_end(ap);
+	}
+}