implement RT throttling management and refactor source fileshandle_rt_throttling

This got a little bigger than I intended. Originally I wanted
to just add a function to disable RT throttling while running
and restore it on exit, but it grew to some refactoring when
I added the signal handler, then moved functions into separate
files.

The function turn_off_rt_throttling() just reads the file
/proc/sys/kernel/sched_rt_runtime_us and saves it's value. If
the value is -1, no action is performed, but if it's > 0 then
we save it and write a -1 to disable throttling, then register
an on_exit handler to restore it on exit.

That's when I realized that we don't shutdown gracefully on signals
so added a 'running' variable for loop termination and set that to
zero in the signal handler.

The signal handling logic and the parser utility functions moved
into src/utils.c. The throttling logic is in src/throttling.c.
I commented out the daemonize() signal calls to prevent conflict
with the other signal handling and then added a couple
of log_msg calls to indicate boost operations happening. Finally,
changed conservative_main() and aggressive_main() to be void returns
since we never check their return value.

There will be more refactoring to come since I want to do a little
namespace cleanup/protection using static (poor-man's C++).

Signed-off-by: Clark Williams <williams@redhat.com>

5 files changed, 365 insertions, 142 deletions

diff --git a/Makefile b/Makefile
index 940e85f..c8f5b19 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,9 @@ CFLAGS	:=	-Wall -O2 -g
 LDFLAGS	:=	-ggdb
 LIBS	:=	 -lpthread
 
+SRC	:=	$(wildcard src/*.c)
+HDR	:=	$(wildcard src/*.h)
+OBJ	:=	$(SRC:.c=.o)
 DIRS	:=	src redhat man
 FILES	:=	Makefile README.md gpl-2.0.txt
 TARBALL	:=	$(NAME)-$(VERSION).tar.xz
@@ -17,15 +20,15 @@ DOCDIR	:=	$(DATADIR)/doc
 MANDIR	:=	$(DATADIR)/man
 LICDIR	:=	$(DATADIR)/licenses
 
-.PHONY:	tests
+.PHONY:	all tests
 
 all:	stalld tests
-	$(CC) -o stalld	 $(LDFLAGS) src/stalld.o $(LIBS)
 
-stalld: src/stalld.o
+stalld: $(OBJ)
+	$(CC) -o stalld	 $(LDFLAGS) $(OBJ) $(LIBS)
 
-static: src/stalld.o
-	$(CC) -o stalld-static $(LDFLAGS) --static src/stalld.o $(LIBS)
+static: $(OBJ)
+	$(CC) -o stalld-static $(LDFLAGS) --static $(OBJ) $(LIBS)
 
 tests:
 	make -C tests
@@ -49,7 +52,7 @@ clean:
 	@test ! -f $(TARBALL) || rm -f $(TARBALL)
 	@make -C redhat clean
 	@make -C tests clean
-	@rm -rf *~
+	@rm -rf *~ $(OBJ)
 
 tarball:  clean
 	rm -rf $(NAME)-$(VERSION) && mkdir $(NAME)-$(VERSION)
diff --git a/src/stalld.c b/src/stalld.c
index 258bf67..c0e84db 100644
--- a/src/stalld.c
+++ b/src/stalld.c
@@ -36,73 +36,7 @@
 #include <unistd.h>
 #include <linux/sched.h>
 
-#define BUFFER_SIZE		(1024 * 1000)
-#define MAX_WAITING_PIDS	30
-
-/*
- * See kernel/sched/debug.c:print_task().
- */
-struct task_info {
-	int pid;
-	int prio;
-	int ctxsw;
-	time_t since;
-	char comm[15];
-};
-
-struct cpu_info {
-	int id;
-	int nr_running;
-	int nr_rt_running;
-	int ctxsw;
-	int nr_waiting_tasks;
-	int thread_running;
-	struct task_info *starving;
-	pthread_t thread;
-};
-
-#ifdef __x86_64__
-# define __NR_sched_setattr 314
-# define __NR_sched_getattr 315
-#elif __i386__
-# define __NR_sched_setattr 351
-# define __NR_sched_getattr 352
-#elif __arm__
-# define __NR_sched_setattr 380
-# define __NR_sched_getattr 381
-#elif __aarch64__
-# define __NR_sched_setattr 274
-# define __NR_sched_getattr 275
-#elif __powerpc__
-# define __NR_sched_setattr 355
-# define __NR_sched_getattr 356
-#elif __s390x__
-# define __NR_sched_setattr 345
-# define __NR_sched_getattr 346
-#endif
-
-struct sched_attr {
-	uint32_t size;
-	uint32_t sched_policy;
-	uint64_t sched_flags;
-	int32_t sched_nice;
-	uint32_t sched_priority;
-	uint64_t sched_runtime;
-	uint64_t sched_deadline;
-	uint64_t sched_period;
-};
-
-int sched_setattr(pid_t pid, const struct sched_attr *attr,
-		  unsigned int flags) {
-	return syscall(__NR_sched_setattr, pid, attr, flags);
-}
-
-int sched_getattr(pid_t pid, struct sched_attr *attr,
-		  unsigned int size, unsigned int flags)
-{
-	return syscall (__NR_sched_getattr, pid , attr, size, flags);
-}
-
+#include "stalld.h"
 /*
  * logging.
  */
@@ -131,8 +65,6 @@ long config_starving_threshold = 60;
 long config_boost_duration = 3;
 long config_aggressive = 0;
 
-#define NS_PER_SEC 1000000000
-
 /*
  * XXX: Make it a cpu mask, lazy Daniel!
  */
@@ -146,6 +78,13 @@ char *config_monitored_cpus;
 int boost_policy;
 
 /*
+ * variable to indicate if stalld is running or
+ * shutting down
+ */
+
+int running = 1;
+
+/*
  * print any error messages and exit
  */
 void die(const char *fmt, ...)
@@ -285,8 +224,8 @@ void deamonize(void)
 	 * Catch, ignore and handle signals.
 	 * XXX: Implement a working signal handler.
 	 */
-	signal(SIGCHLD, SIG_IGN);
-	signal(SIGHUP, SIG_IGN);
+	//signal(SIGCHLD, SIG_IGN);
+	//signal(SIGHUP, SIG_IGN);
 
 	/*
 	 * Fork off for the second time.
@@ -516,57 +455,6 @@ char *alloc_and_fill_cpu_buffer(int cpu, char *sched_dbg, int sched_dbg_size)
 
 	return cpu_buffer;
 }
-
-long get_long_from_str(char *start)
-{
-	long value;
-	char *end;
-
-	errno = 0;
-	value = strtol(start, &end, 10);
-	if (errno || start == end) {
-		warn("Invalid ID '%s'", value);
-		return -1;
-	}
-
-	return value;
-}
-
-long get_long_after_colon(char *start)
-{
-	/*
-	 * Find the ":"
-	 */
-	start = strstr(start, ":");
-	if (!start)
-		return -1;
-
-	/*
-	 * skip ":"
-	 */
-	start++;
-
-	return get_long_from_str(start);
-}
-
-long get_variable_long_value(char *buffer, const char *variable)
-{
-	char *start;
-	/*
-	 * Line:
-	 * '  .nr_running                    : 0'
-	 */
-
-	/*
-	 * Find the ".nr_running"
-	 */
-	start = strstr(buffer, variable);
-	if (!start)
-		return -1;
-
-	return get_long_after_colon(start);
-}
-
 /*
  * Example:
  * ' S           task   PID         tree-key  switches  prio     wait-time             sum-exec        sum-sleep'
@@ -774,6 +662,7 @@ int boost_with_deadline(int pid)
 	    log_msg("boost_with_deadline failed to boost pid %d: %s\n", pid, strerror(errno));
 	    return ret;
 	}
+	log_msg("boosted pid %d using SCHED_DEADLINE\n", pid);
 	return ret;
 }
 
@@ -793,6 +682,7 @@ int boost_with_fifo(int pid)
 	    log_msg("boost_with_fifo failed to boost pid %d: %s\n", pid, strerror(errno));
 	    return ret;
 	}
+	log_msg("boosted pid %d using SCHED_FIFO\n", pid);
 	return ret;
 }
 
@@ -808,14 +698,6 @@ int restore_policy(int pid, struct sched_attr *attr)
 	return ret;
 }
 
-void normalize_timespec(struct timespec *ts)
-{
-        while (ts->tv_nsec >= NS_PER_SEC) {
-                ts->tv_nsec -= NS_PER_SEC;
-                ts->tv_sec++;
-        }
-}
-
 /*
  * this function emulates the behavior of SCHED_DEADLINE but
  * using SCHED_FIFO by boosting the thread, sleeping for runtime,
@@ -1199,7 +1081,7 @@ void *cpu_main(void *data)
 	int nothing_to_do = 0;
 	int retval;
 
-	while (cpu->thread_running) {
+	while (cpu->thread_running && running) {
 
 		retval = read_sched_debug(buffer, BUFFER_SIZE);
 		if(!retval) {
@@ -1249,7 +1131,7 @@ static const char *join_thread(pthread_t *thread)
 	return result;
 }
 
-int aggressive_main(struct cpu_info *cpus, int nr_cpus)
+void aggressive_main(struct cpu_info *cpus, int nr_cpus)
 {
 	int i;
 
@@ -1268,11 +1150,9 @@ int aggressive_main(struct cpu_info *cpus, int nr_cpus)
 
 		join_thread(&cpus[i].thread);
 	}
-
-	return 0;
 }
 
-int conservative_main(struct cpu_info *cpus, int nr_cpus)
+void conservative_main(struct cpu_info *cpus, int nr_cpus)
 {
 	char buffer[BUFFER_SIZE];
 	pthread_attr_t dettached;
@@ -1287,7 +1167,7 @@ int conservative_main(struct cpu_info *cpus, int nr_cpus)
 		cpus[i].thread_running = 0;
 	}
 
-	while (1) {
+	while (running) {
 		retval = read_sched_debug(buffer, BUFFER_SIZE);
 		if(!retval) {
 			warn("Dazed and confused, but trying to continue");
@@ -1405,6 +1285,9 @@ int main(int argc, char **argv)
 	if (config_log_syslog)
 		openlog("stalld", 0, LOG_DAEMON);
 
+	setup_signal_handling();
+	turn_off_rt_throttling();
+
 	if (!config_foreground)
 		deamonize();
 
diff --git a/src/stalld.h b/src/stalld.h
new file mode 100644
index 0000000..16b05b6
--- /dev/null
+++ b/src/stalld.h
@@ -0,0 +1,107 @@
+/*
+ * Data structures, constants and function prototypes
+ * used by stalld
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2020 Red Hat Inc, Daniel Bristot de Oliveira <bristot@redhat.com>
+ *
+ */
+#ifndef __STALLD_H__
+#define __STALLD_H__
+
+#define BUFFER_SIZE		(1024 * 1000)
+#define MAX_WAITING_PIDS	30
+
+/* informnation about running tasks on a cpu */
+struct task_info {
+       int pid;
+       int prio;
+       int ctxsw;
+       time_t since;
+       char comm[15];
+};
+
+/* information about cpus */
+struct cpu_info {
+       int id;
+       int nr_running;
+       int nr_rt_running;
+       int ctxsw;
+       int nr_waiting_tasks;
+       int thread_running;
+       struct task_info *starving;
+       pthread_t thread;
+};
+
+#ifdef __x86_64__
+# define __NR_sched_setattr 314
+# define __NR_sched_getattr 315
+#elif __i386__
+# define __NR_sched_setattr 351
+# define __NR_sched_getattr 352
+#elif __arm__
+# define __NR_sched_setattr 380
+# define __NR_sched_getattr 381
+#elif __aarch64__
+# define __NR_sched_setattr 274
+# define __NR_sched_getattr 275
+#elif __powerpc__
+# define __NR_sched_setattr 355
+# define __NR_sched_getattr 356
+#elif __s390x__
+# define __NR_sched_setattr 345
+# define __NR_sched_getattr 346
+#endif
+
+struct sched_attr {
+       uint32_t size;
+       uint32_t sched_policy;
+       uint64_t sched_flags;
+       int32_t sched_nice;
+       uint32_t sched_priority;
+       uint64_t sched_runtime;
+       uint64_t sched_deadline;
+       uint64_t sched_period;
+};
+
+static inline int sched_setattr(pid_t pid, const struct sched_attr *attr,
+                 unsigned int flags) {
+       return syscall(__NR_sched_setattr, pid, attr, flags);
+}
+
+static inline int sched_getattr(pid_t pid, struct sched_attr *attr,
+                 unsigned int size, unsigned int flags)
+{
+       return syscall (__NR_sched_getattr, pid , attr, size, flags);
+}
+
+#define NS_PER_SEC 1000000000
+static inline void normalize_timespec(struct timespec *ts)
+{
+        while (ts->tv_nsec >= NS_PER_SEC) {
+                ts->tv_nsec -= NS_PER_SEC;
+                ts->tv_sec++;
+        }
+}
+
+/*
+ * forward function definitions
+ */
+
+void die(const char *fmt, ...);
+void warn(const char *fmt, ...);
+void log_msg(const char *fmt, ...);
+
+long get_long_from_str(char *start);
+long get_long_after_colon(char *start);
+long get_variable_long_value(char *buffer, const char *variable);
+
+int turn_off_rt_throttling(void);
+int setup_signal_handling(void);
+
+/*
+ * shared variables 
+ */
+extern int running;
+
+#endif /* __STALLD_H__ */
diff --git a/src/throttling.c b/src/throttling.c
new file mode 100644
index 0000000..3cf716f
--- /dev/null
+++ b/src/throttling.c
@@ -0,0 +1,85 @@
+/*
+ * stalld code to handle automatically turning off RT throttling while running
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2020 Red Hat Inc, Daniel Bristot de Oliveira <bristot@redhat.com>
+ *
+ */
+
+#define _GNU_SOURCE
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <syslog.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+#include <linux/sched.h>
+
+#include "stalld.h"
+
+#define RT_RUNTIME_PATH "/proc/sys/kernel/sched_rt_runtime_us"
+
+static long rt_runtime_us = 0;
+
+static void restore_rt_throttling(int status, void *arg)
+{
+	if (rt_runtime_us != -1) {
+		int fd = open(RT_RUNTIME_PATH, O_WRONLY);
+		char buffer[80];
+
+		if (fd < 0)
+			die("restore_rt_throttling: failed to open %s\n", RT_RUNTIME_PATH);
+		sprintf(buffer, "%ld", rt_runtime_us);
+		write(fd, buffer, strlen(buffer));
+		close(fd);
+		log_msg("RT Throttling runtime restored to %d\n", rt_runtime_us);
+	}
+}
+
+int turn_off_rt_throttling(void)
+{
+	int fd;
+	char buffer[80];
+	int status;
+	
+
+	/* get the current value of the throttling runtime */
+	fd = open(RT_RUNTIME_PATH, O_RDWR);
+	status = read(fd, buffer, sizeof(buffer));
+	if (status < 0)
+		die("turn_off_rt_throttling: failed to read %s\n",
+		    RT_RUNTIME_PATH);
+	
+	rt_runtime_us = strtol(buffer, NULL, 10);
+
+	if (rt_runtime_us == -1) {
+		log_msg("RT throttling already disabled, doing nothing\n");
+		close(fd);
+		return 0;
+	}
+
+	/* turn off throttling and register an exit handler to restore it */
+	status = lseek(fd, 0, SEEK_SET);
+	if (status < 0)
+		die("turn_off_rt_throttling: unable to seek on %s", RT_RUNTIME_PATH);
+	status = write(fd, "-1", 2);
+	if (status < 0)
+		die("turn_off_rt_throttling: unable to write -1 to  %s", RT_RUNTIME_PATH);
+	close(fd);
+	on_exit(restore_rt_throttling, NULL);
+	log_msg("RT Throttling disabled\n");
+	return 0;
+}
diff --git a/src/utils.c b/src/utils.c
new file mode 100644
index 0000000..1352bd8
--- /dev/null
+++ b/src/utils.c
@@ -0,0 +1,145 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2020 Red Hat Inc, Clark Williams <williams@redhat.com>
+ *
+ */
+
+#define _GNU_SOURCE
+#include <ctype.h>
+#include <sys/types.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <syslog.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <unistd.h>
+#include <linux/sched.h>
+#include <sys/sysinfo.h>
+
+#include "stalld.h"
+
+long get_long_from_str(char *start)
+{
+	long value;
+	char *end;
+
+	errno = 0;
+	value = strtol(start, &end, 10);
+	if (errno || start == end) {
+		warn("Invalid ID '%s'", value);
+		return -1;
+	}
+
+	return value;
+}
+
+long get_long_after_colon(char *start)
+{
+	/*
+	 * Find the ":"
+	 */
+	start = strstr(start, ":");
+	if (!start)
+		return -1;
+
+	/*
+	 * skip ":"
+	 */
+	start++;
+
+	return get_long_from_str(start);
+}
+
+long get_variable_long_value(char *buffer, const char *variable)
+{
+	char *start;
+	/*
+	 * Line:
+	 * '  .nr_running                    : 0'
+	 */
+
+	/*
+	 * Find the ".nr_running"
+	 */
+	start = strstr(buffer, variable);
+	if (!start)
+		return -1;
+
+	return get_long_after_colon(start);
+}
+
+/*
+ * SIGINT handler for main
+ */
+static void inthandler (int signo, siginfo_t *info, void *extra)
+{
+	log_msg("received signal %d, starting shutdown\n", signo);
+	running = 0;
+}
+
+static void set_sig_handler()
+{
+	struct sigaction action;
+	action.sa_flags = SA_SIGINFO;
+	action.sa_sigaction = inthandler;
+	if (sigaction(SIGINT, &action, NULL) == -1) {
+		warn("error setting SIGINT handler: %s\n",
+		      strerror(errno));
+		exit(errno);
+	}
+}
+
+int setup_signal_handling(void)
+{
+	int status;
+	sigset_t sigset;
+
+	/* mask off all signals */
+	status = sigfillset(&sigset);
+	if (status) {
+		warn("setting up full signal set %s\n", strerror(status));
+		return status;
+	}
+	status = pthread_sigmask(SIG_BLOCK, &sigset, NULL);
+	if (status) {
+		warn("setting signal mask: %s\n", strerror(status));
+		return status;
+	}
+
+	/* now allow SIGINT and SIGTERM to be delivered */
+	status = sigemptyset(&sigset);
+	if (status) {
+		warn("creating empty signal set: %s\n", strerror(status));
+		return status;
+	}
+	status = sigaddset(&sigset, SIGINT);
+	if (status) {
+		warn("adding SIGINT to signal set: %s\n", strerror(status));
+		return status;
+	}
+	status = sigaddset(&sigset, SIGTERM);
+	if (status) {
+		warn("adding SIGTERM to signal set: %s\n", strerror(status));
+		return status;
+	}
+	status = pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
+	if (status) {
+		warn("unblocking signals: %s\n", strerror(status));
+		return status;
+	}
+
+	/* now register our signal handler */
+	set_sig_handler();
+	return 0;
+}