path: root/src/pi_tests/pip_stress.c

/*
	Pip stress - Priority Inheritance with processes

    Copyright (C) 2009, John Kacur <jkacur@redhat.com>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/*
 * This program demonstrates the technique of using priority inheritance (PI)
 * mutexes with processes instead of threads.
 * The way to do this is to obtain some shared memory - in this case with
 * mmap that backs a pthread_mutex_t since this will support PI.
 * Pay particular attention to how this is intialized to support processes.
 * Function init_shared_pthread_mutex() does this by setting the
 * pthread_mutexattr to PTHREAD_PROCESS_SHARED and the mutex protocol to
 * PTHREAD_PRIO_INHERIT.
 * In this program we purposely try to invoke a classic priority inversion.
 * A low priority process grabs the mutex and does some work.
 * A high priority process comes a long and is blocked since the mutex is taken.
 * A medium priority process that doesn't require the mutex then takes the
 * processor. Because the processes are restricted to one cpu, the low priority
 * processes never makes any progress because the medium priority process
 * runs in an infinite loop. This is a priority inversion because the
 * medium priority process is running at the expensive of the high priority
 * process. However, since we have used PRIO_INHERIT and are running on a
 * machine that supports preemption, the high priority process will lend it's
 * priority to the low priority process which will preempt the medium priority
 * process. The low priority process will then release the mutex which the
 * high priority process can obtain. When the high priority process gets to run
 * it kills the medium priority process.
 * The state structure keeps track of the progress. Although this program
 * is set up to likely trigger an inversion, there is no guarantee that
 * scheduling will make that happen. After the program completes it reports
 * whether a priority inversion occurred or not. In either case this program
 * demonstrates how to use priority inheritance mutexes with processes.
 * In fact, you would be better off to avoid scenarios in which a priority
 * inversion occurs if possible - this program tries to trigger them just
 * to show that it works. If you are having difficulty triggering an inversion,
 * merely increase the time that the low priority process sleeps while
 * holding the lock. (usleep);
 * Also note that you have to run as a user with permission to change
 * scheduling priorities.
 */

#include "pip_stress.h"

#include <unistd.h>
#include <getopt.h>

/* default time for low priority thread usleep */
useconds_t usleep_val = 500;

pthread_mutex_t *resource;

/* This records the state to determine whether a priority inversion occured */
struct State {
	int low_owns_resource;
	int high_started;
	int high_owns_resource;
	int medium_started;
	int inversion;
	pthread_mutex_t *mutex;
};

struct State *statep;

const int policy = SCHED_FIFO;
const int prio_min;	/* Initialized for the minimum priority of policy */

struct option long_options[] = {
    { "usleep", required_argument, 0, 0 },
    { 0,        0,                 0, 0 },
};

int main(void)
{
	void *mptr;	/* memory pointer */
	pid_t pid1, pid2;
	cpu_set_t set, *setp = &set;
	int res;
	int *minimum_priority = (int*)&prio_min;
	int c;

	*minimum_priority = sched_get_priority_min(policy);

	if (check_privs())
		exit(-1);

	mptr = mmap_page();	/* Get a page of shared memory */
	resource = (pthread_mutex_t*)mptr;	/* point our lock to it */
	mptr += sizeof(pthread_mutex_t);	/* advance the memory pointer */

	/* Initialize our mutex via the resource pointer */
	init_shared_pthread_mutex(resource, PTHREAD_PRIO_INHERIT, policy);

	statep = (struct State*)mptr;
	mptr += sizeof(struct State);

	init_state();	/* Initialize the state structure */

	statep->mutex = (pthread_mutex_t*)mptr;	/* point the next lock to it */
	mptr += sizeof(pthread_mutex_t);	/* advance the memory pointer */

	/* Initialize our State mutex */
	init_shared_pthread_mutex(statep->mutex, PTHREAD_PRIO_NONE, policy);

	set_rt_prio(0, prio_min, policy);

	/* We restrict this program to the first cpu, inorder to increase
	 * the likelihood of a priority inversion */
	CPU_ZERO(setp);
	CPU_SET(0, setp);
	res = sched_setaffinity(0, sizeof(set), setp);
	if (res == -1) {
		int err = errno;
		err_msg("sched_setaffinity: ");
		err_exit(err, NULL);
	}

	pid1 = fork();
	if (pid1 == -1) {
		perror("fork");
		exit(1);
	} else if (pid1 != 0) {		/* parent code */
		low(pid1);
	} else {			/* child code */
		pid2 = fork();		/* parent code */
		if (pid2 == -1) {
			perror("fork: ");
			exit(-1);
		} else if (pid2 != 0) {		/* parent code */
			high(pid2);
		} else {			/* child code */
			medium();
		}
	}

	exit(0);
}

/* Initialize the structure that tracks when a priority inversion occurs */
void init_state(void)
{
	/* Init the State structure */
	statep->low_owns_resource = 0;
	statep->high_started = 0;
	statep->high_owns_resource = 0;
	statep->medium_started = 0;
	/* Assume an inversion will occur until proven false */
	statep->inversion = 1;
}

/* @pid = high priority process pid */
void low(pid_t pid)
{
	int status;
	Pthread_mutex_lock(resource);
		Pthread_mutex_lock(statep->mutex);
			statep->low_owns_resource = 1;
			if (statep->high_owns_resource ||
					statep->medium_started) {
				statep->inversion = 0;
			}
		Pthread_mutex_unlock(statep->mutex);
		usleep(usleep_val);
	Pthread_mutex_unlock(resource);
	waitpid(pid, &status, 0);
}

void medium(void)
{
	set_rt_prio(0, prio_min+1, policy);
	Pthread_mutex_lock(statep->mutex);
		statep->medium_started = 1;
		if (!statep->high_started)
			statep->inversion = 0;
	Pthread_mutex_unlock(statep->mutex);

	for(;;);	/* infinite loop */
}

/* @pid = medium priority process pid */
void high(pid_t pid)
{
	int status;
	set_rt_prio(0, prio_min+2, policy);

	/* Must come after raising the priority */
	Pthread_mutex_lock(statep->mutex);
		statep->high_started = 1;
	Pthread_mutex_unlock(statep->mutex);

	Pthread_mutex_lock(resource);
	Pthread_mutex_lock(statep->mutex);
		statep->high_owns_resource = 1;
		if (!statep->low_owns_resource || !statep->medium_started) {
			statep->inversion = 0;
		}
	Pthread_mutex_unlock(statep->mutex);
	Pthread_mutex_unlock(resource);
	kill(pid, SIGKILL);	/* kill the medium thread */
	waitpid(pid, &status, 0);

	Pthread_mutex_lock(statep->mutex);

	if (statep->inversion)
		printf("Successfully used priority inheritance to handle an inversion\n");
	else {
		printf("No inversion incurred\n");
	}
	Pthread_mutex_unlock(statep->mutex);
}

/* mmap a page of anonymous shared memory */
void *mmap_page(void)
{
	void *mptr;
	long pgsize = sysconf(_SC_PAGE_SIZE);

	mptr = mmap(NULL, pgsize, PROTRW, MMAP_FLAGS, 0, 0);
	if (mptr == MAP_FAILED) {
		perror("In function mmap_page - mmap");
		exit(-1);
	}

	return mptr;
}

long process_shared_mutex_available(void)
{
	long res = -1;	/* undefined */
#ifdef _POSIX_THREAD_PROCESS_SHARED
	res = sysconf(_SC_THREAD_PROCESS_SHARED);
	if (res == -1) {
		int err = errno;	/* save the error number */
		err_msg("%s: sysconf(_SC_THREAD_PROCESS_SHARED): ");
		err_exit(err, NULL);
	}
#else
#error _POSIX_THREAD_PROCESS_SHARED is not defined
#endif
	return res;
}

void Pthread_mutexattr_init(pthread_mutexattr_t *attr)
{
	int err;
	err = pthread_mutexattr_init(attr);
	if (err) {
		err_msg("%s: pthread_mutexattr_init(): ", __func__);
		err_exit(err, NULL);
	}
}

void Pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
{
	int err;
	err = pthread_mutexattr_setpshared(attr, pshared);
	if (err) {
		err_msg("%s: pthread_mutexattr_setpshared(): ", __func__);
		err_exit(err, NULL);
	}
}

void Pthread_mutexattr_setprotocol(pthread_mutexattr_t *attr, int protocol)
{
	int err;
	err = pthread_mutexattr_setprotocol(attr, protocol);
	if (err) {
		err_msg("%s: pthread_mutexattr_setprotocol(): ", __func__);
		err_exit(err, NULL);
	}
}

void Pthread_mutex_init(pthread_mutex_t *mutex, pthread_mutexattr_t *attr)
{
	int err;
	err = pthread_mutex_init(mutex, attr);
	if (err) {
		err_msg("%s: pthread_mutex_init(): ", __func__);
		err_exit(err, NULL);
	}
}

void Pthread_mutex_lock(pthread_mutex_t *mutex)
{
	int err;
	err = pthread_mutex_lock(mutex);
	if (err) {
		err_msg("%s: pthread_mutex_lock(): ", __func__);
		err_exit(err, NULL);
	}
}

void Pthread_mutex_unlock(pthread_mutex_t *mutex)
{
	int err;
	err = pthread_mutex_unlock(mutex);
	if (err) {
		err_msg("%s: pthread_mutex_unlock(): ", __func__);
		err_exit(err, NULL);
	}
}

void init_shared_pthread_mutex(pthread_mutex_t *mutex, int protocol, int policy)
{
	pthread_mutexattr_t attr;

	process_shared_mutex_available();

	Pthread_mutexattr_init(&attr);
	Pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
	Pthread_mutexattr_setprotocol(&attr, protocol);

	Pthread_mutex_init(mutex, &attr);
}

/* Set the priority and policy of a process */
int set_rt_prio(pid_t pid, int prio, int policy)
{
	int err;
	struct sched_param param;
	struct sched_param *pparam = &param;
	pparam->sched_priority = prio;
	err = sched_setscheduler(pid, policy, pparam);
	if (err) {
		err = errno;	/* save the errno */
		err_msg_n(err, "%s: sched_setscheduler(): ", __func__);
		err_msg("%s: prio = %d\n", __func__, prio);
		err_msg("%s: pparam->sched_priority = %d\n", __func__, pparam->sched_priority);
		err_msg("%s: policy = %d\n", __func__, policy);
	}
	return err;	/* 0 on success */
}

int get_rt_prio(pid_t pid)
{
	int err;
	struct sched_param param;
	err = sched_getparam(pid, &param);
	if (err) {
		err = errno;	/* save the errno */
		err_msg_n(err, "%s: get_rt_prio(): ", __func__);
		return -1;
	}
	return param.sched_priority;
}