iowatcher: Per process IO graphs

Add support for displaying different processes with different color in
the IO graph and movie.

Signed-off-by: Jan Kara <jack@suse.cz>
Signed-off-by: Chris Mason <chris.mason@fusionio.com>

5 files changed, 508 insertions, 176 deletions

diff --git a/iowatcher/blkparse.c b/iowatcher/blkparse.c
index a0fb8c5..f432f4d 100644
--- a/iowatcher/blkparse.c
+++ b/iowatcher/blkparse.c
@@ -41,7 +41,12 @@
 static struct list_head io_hash_table[IO_HASH_TABLE_SIZE];
 static u64 ios_in_flight = 0;
 
+#define PROCESS_HASH_TABLE_BITS 7
+#define PROCESS_HASH_TABLE_SIZE (1 << PROCESS_HASH_TABLE_BITS)
+static struct list_head process_hash_table[PROCESS_HASH_TABLE_SIZE];
+
 extern int plot_io_action;
+extern int io_per_process;
 
 /*
  * Trace categories
@@ -158,6 +163,15 @@ struct pending_io {
 	/* time this IO was finished */
 	u64 completion_time;
 	struct list_head hash_list;
+	/* process which queued this IO */
+	u32 pid;
+};
+
+struct pid_map {
+	struct list_head hash_list;
+	u32 pid;
+	int index;
+	char name[0];
 };
 
 #define MINORBITS 20
@@ -202,7 +216,7 @@ static inline u64 hash_sector(u64 val)
 	return hash >> (64 - IO_HASH_TABLE_BITS);
 }
 
-static int hash_table_insert(struct pending_io *ins_pio)
+static int io_hash_table_insert(struct pending_io *ins_pio)
 {
 	u64 sector = ins_pio->sector;
 	int slot = hash_sector(sector);
@@ -218,7 +232,7 @@ static int hash_table_insert(struct pending_io *ins_pio)
 	return 0;
 }
 
-static struct pending_io *hash_table_search(u64 sector)
+static struct pending_io *io_hash_table_search(u64 sector)
 {
 	int slot = hash_sector(sector);
 	struct list_head *head;
@@ -232,40 +246,124 @@ static struct pending_io *hash_table_search(u64 sector)
 	return NULL;
 }
 
-static int hash_dispatched_io(struct blk_io_trace *io)
+static int hash_queued_io(struct blk_io_trace *io)
 {
 	struct pending_io *pio;
 	int ret;
 
 	pio = calloc(1, sizeof(*pio));
 	pio->sector = io->sector;
-	pio->dispatch_time = io->time;
+	pio->pid = io->pid;
 
-	ret = hash_table_insert(pio);
-	if (ret == -EEXIST) {
-		/* crud, the IO isn't here */
+	ret = io_hash_table_insert(pio);
+	if (ret < 0) {
+		/* crud, the IO is there already */
 		free(pio);
+		return ret;
 	}
-	return ret;
+	return 0;
+}
+
+static int hash_dispatched_io(struct blk_io_trace *io)
+{
+	struct pending_io *pio;
+
+	pio = io_hash_table_search(io->sector);
+	if (!pio) {
+		/* crud, the IO isn't here */
+		return -EEXIST;
+	}
+	pio->dispatch_time = io->time;
+	return 0;
 }
 
 static struct pending_io *hash_completed_io(struct blk_io_trace *io)
 {
 	struct pending_io *pio;
 
-	pio = hash_table_search(io->sector);
+	pio = io_hash_table_search(io->sector);
 
 	if (!pio)
 		return NULL;
 	return pio;
 }
 
+void init_process_hash_table(void)
+{
+	int i;
+	struct list_head *head;
+
+	for (i = 0; i < PROCESS_HASH_TABLE_SIZE; i++) {
+		head = process_hash_table + i;
+		INIT_LIST_HEAD(head);
+	}
+}
+
+static u32 hash_pid(u32 pid)
+{
+	u32 hash = pid;
+
+	hash ^= pid >> 3;
+	hash ^= pid >> 3;
+	hash ^= pid >> 4;
+	hash ^= pid >> 6;
+	return (hash & (PROCESS_HASH_TABLE_SIZE - 1));
+}
+
+static struct pid_map *process_hash_search(u32 pid)
+{
+	int slot = hash_pid(pid);
+	struct list_head *head;
+	struct pid_map *pm;
+
+	head = process_hash_table + slot;
+	list_for_each_entry(pm, head, hash_list) {
+		if (pm->pid == pid)
+			return pm;
+	}
+	return NULL;
+}
+
+static struct pid_map *process_hash_insert(u32 pid, char *name)
+{
+	int slot = hash_pid(pid);
+	struct pid_map *pm;
+	int old_index = 0;
+	char buf[16];
+
+	pm = process_hash_search(pid);
+	if (pm) {
+		/* Entry exists and name shouldn't be changed? */
+		if (!name || !strcmp(name, pm->name))
+			return pm;
+		list_del(&pm->hash_list);
+		old_index = pm->index;
+		free(pm);
+	}
+	if (!name) {
+		sprintf(buf, "[%u]", pid);
+		name = buf;
+	}
+	pm = malloc(sizeof(struct pid_map) + strlen(name) + 1);
+	pm->pid = pid;
+	pm->index = old_index;
+	strcpy(pm->name, name);
+	list_add_tail(&pm->hash_list, process_hash_table + slot);
+
+	return pm;
+}
+
 static void handle_notify(struct trace *trace)
 {
 	struct blk_io_trace *io = trace->io;
 	void *payload = (char *)io + sizeof(*io);
 	u32 two32[2];
 
+	if (io->action == BLK_TN_PROCESS) {
+		if (io_per_process)
+			process_hash_insert(io->pid, payload);
+		return;
+	}
 
 	if (io->action != BLK_TN_TIMESTAMP)
 		return;
@@ -663,12 +761,49 @@ void add_tput(struct trace *trace, struct graph_line_data *gld)
 		gld->max = gld->data[seconds].sum;
 }
 
-void add_io(struct trace *trace, struct graph_dot_data *gdd_writes,
-	    struct graph_dot_data *gdd_reads)
+#define GDD_PTR_ALLOC_STEP 16
+
+static struct pid_map *get_pid_map(struct trace_file *tf, u32 pid)
+{
+	struct pid_map *pm;
+
+	if (!io_per_process) {
+		if (!tf->io_plots)
+			tf->io_plots = 1;
+		return NULL;
+	}
+
+	pm = process_hash_insert(pid, NULL);
+	/* New entry? */
+	if (!pm->index) {
+		if (tf->io_plots == tf->io_plots_allocated) {
+			tf->io_plots_allocated += GDD_PTR_ALLOC_STEP;
+			tf->gdd_reads = realloc(tf->gdd_reads, tf->io_plots_allocated * sizeof(struct graph_dot_data *));
+			if (!tf->gdd_reads)
+				abort();
+			tf->gdd_writes = realloc(tf->gdd_writes, tf->io_plots_allocated * sizeof(struct graph_dot_data *));
+			if (!tf->gdd_writes)
+				abort();
+			memset(tf->gdd_reads + tf->io_plots_allocated - GDD_PTR_ALLOC_STEP,
+			       0, GDD_PTR_ALLOC_STEP * sizeof(struct graph_dot_data *));
+			memset(tf->gdd_writes + tf->io_plots_allocated - GDD_PTR_ALLOC_STEP,
+			       0, GDD_PTR_ALLOC_STEP * sizeof(struct graph_dot_data *));
+		}
+		pm->index = tf->io_plots++;
+
+		return pm;
+	}
+	return pm;
+}
+
+void add_io(struct trace *trace, struct trace_file *tf)
 {
 	struct blk_io_trace *io = trace->io;
 	int action = io->action & BLK_TA_MASK;
 	u64 offset;
+	int index;
+	char *label;
+	struct pid_map *pm;
 
 	if (io->action & BLK_TC_ACT(BLK_TC_NOTIFY))
 		return;
@@ -678,10 +813,23 @@ void add_io(struct trace *trace, struct graph_dot_data *gdd_writes,
 
 	offset = io->sector << 9;
 
-	if (BLK_DATADIR(io->action) & BLK_TC_READ)
-		set_gdd_bit(gdd_reads, offset, io->bytes, io->time);
-	else if (BLK_DATADIR(io->action) & BLK_TC_WRITE)
-		set_gdd_bit(gdd_writes, offset, io->bytes, io->time);
+	pm = get_pid_map(tf, io->pid);
+	if (!pm) {
+		index = 0;
+		label = "";
+	} else {
+		index = pm->index;
+		label = pm->name;
+	}
+	if (BLK_DATADIR(io->action) & BLK_TC_READ) {
+		if (!tf->gdd_reads[index])
+			tf->gdd_reads[index] = alloc_dot_data(tf->min_seconds, tf->max_seconds, tf->min_offset, tf->max_offset, tf->stop_seconds, pick_color(), strdup(label));
+		set_gdd_bit(tf->gdd_reads[index], offset, io->bytes, io->time);
+	} else if (BLK_DATADIR(io->action) & BLK_TC_WRITE) {
+		if (!tf->gdd_writes[index])
+			tf->gdd_writes[index] = alloc_dot_data(tf->min_seconds, tf->max_seconds, tf->min_offset, tf->max_offset, tf->stop_seconds, pick_color(), strdup(label));
+		set_gdd_bit(tf->gdd_writes[index], offset, io->bytes, io->time);
+	}
 }
 
 void add_pending_io(struct trace *trace, struct graph_line_data *gld)
@@ -695,6 +843,10 @@ void add_pending_io(struct trace *trace, struct graph_line_data *gld)
 	if (io->action & BLK_TC_ACT(BLK_TC_NOTIFY))
 		return;
 
+	if (action == __BLK_TA_QUEUE) {
+		hash_queued_io(trace->io);
+		return;
+	}
 	if (action != __BLK_TA_ISSUE)
 		return;
 
diff --git a/iowatcher/blkparse.h b/iowatcher/blkparse.h
index 6300ced..0222e5e 100644
--- a/iowatcher/blkparse.h
+++ b/iowatcher/blkparse.h
@@ -50,6 +50,36 @@ struct trace {
 	int mpstat_num_cpus;
 };
 
+struct trace_file {
+	struct list_head list;
+	char *filename;
+	char *label;
+	struct trace *trace;
+	int stop_seconds;	/* Time when trace stops */
+	int min_seconds;	/* Beginning of the interval we should plot */
+	int max_seconds;	/* End of the interval we should plot */
+	u64 min_offset;
+	u64 max_offset;
+
+	char *line_color;
+
+	struct graph_line_data *tput_gld;
+	struct graph_line_data *iop_gld;
+	struct graph_line_data *latency_gld;
+	struct graph_line_data *queue_depth_gld;
+	/* Number of entries in gdd_writes / gdd_reads */
+	int io_plots;
+	/* Allocated array size for gdd_writes / gdd_reads */
+	int io_plots_allocated;
+	struct graph_dot_data **gdd_writes;
+	struct graph_dot_data **gdd_reads;
+
+	int mpstat_min_seconds;
+	int mpstat_max_seconds;
+	int mpstat_stop_seconds;
+	struct graph_line_data **mpstat_gld;
+};
+
 static inline unsigned int MAJOR(unsigned int dev)
 {
 	return dev >> MINORBITS;
@@ -61,6 +91,7 @@ static inline unsigned int MINOR(unsigned int dev)
 }
 
 void init_io_hash_table(void);
+void init_process_hash_table(void);
 struct trace *open_trace(char *filename);
 u64 find_last_time(struct trace *trace);
 void find_extreme_offsets(struct trace *trace, u64 *min_ret, u64 *max_ret,
@@ -72,8 +103,7 @@ void add_iop(struct trace *trace, struct graph_line_data *gld);
 void check_record(struct trace *trace);
 void add_completed_io(struct trace *trace,
 		      struct graph_line_data *latency_gld);
-void add_io(struct trace *trace, struct graph_dot_data *gdd_writes,
-	    struct graph_dot_data *gdd_reads);
+void add_io(struct trace *trace, struct trace_file *tf);
 void add_tput(struct trace *trace, struct graph_line_data *gld);
 void add_pending_io(struct trace *trace, struct graph_line_data *gld);
 int next_record(struct trace *trace);
diff --git a/iowatcher/main.c b/iowatcher/main.c
index 6798273..c400580 100644
--- a/iowatcher/main.c
+++ b/iowatcher/main.c
@@ -45,8 +45,6 @@ LIST_HEAD(all_traces);
 static char line[1024];
 static int line_len = 1024;
 static int found_mpstat = 0;
-static int cpu_color_index = 0;
-static int color_index = 0;
 static int make_movie = 0;
 static int opt_graph_width = 0;
 static int opt_graph_height = 0;
@@ -61,6 +59,8 @@ static unsigned long long min_mb = 0;
 static unsigned long long max_mb = ULLONG_MAX >> 20;
 
 int plot_io_action = 0;
+int io_per_process = 0;
+unsigned int longest_proc_name = 0;
 
 /*
  * this doesn't include the IO graph,
@@ -68,36 +68,6 @@ int plot_io_action = 0;
  */
 static int total_graphs_written = 1;
 
-char *colors[] = {
-	"blue", "darkgreen",
-	"red", "aqua",
-	"orange", "darkviolet",
-	"brown", "#00FF00",
-	"yellow", "coral",
-	"black", "darkred",
-	"fuchsia", "crimson",
-	NULL };
-
-char *pick_color(void) {
-	char *ret = colors[color_index];
-	if (!ret) {
-		color_index = 0;
-		ret = colors[color_index];
-	}
-	color_index++;
-	return ret;
-}
-
-char *pick_cpu_color(void) {
-	char *ret = colors[cpu_color_index];
-	if (!ret) {
-		color_index = 0;
-		ret = colors[cpu_color_index];
-	}
-	cpu_color_index++;
-	return ret;
-}
-
 enum {
 	IO_GRAPH_INDEX = 0,
 	TPUT_GRAPH_INDEX,
@@ -166,33 +136,6 @@ static int label_index = 0;
 static int num_traces = 0;
 static int longest_label = 0;
 
-struct trace_file {
-	struct list_head list;
-	char *filename;
-	char *label;
-	struct trace *trace;
-	int stop_seconds;	/* Time when trace stops */
-	int min_seconds;	/* Beginning of the interval we should plot */
-	int max_seconds;	/* End of the interval we should plot */
-	u64 min_offset;
-	u64 max_offset;
-
-	char *read_color;
-	char *write_color;
-
-	struct graph_line_data *tput_gld;
-	struct graph_line_data *iop_gld;
-	struct graph_line_data *latency_gld;
-	struct graph_line_data *queue_depth_gld;
-	struct graph_dot_data *gdd_writes;
-	struct graph_dot_data *gdd_reads;
-
-	int mpstat_min_seconds;
-	int mpstat_max_seconds;
-	int mpstat_stop_seconds;
-	struct graph_line_data **mpstat_gld;
-};
-
 static void alloc_mpstat_gld(struct trace_file *tf)
 {
 	struct graph_line_data **ptr;
@@ -281,8 +224,7 @@ static void add_trace_file(char *filename)
 	tf->label = "";
 	tf->filename = strdup(filename);
 	list_add_tail(&tf->list, &all_traces);
-	tf->read_color = pick_color();
-	tf->write_color = pick_color();
+	tf->line_color = "black";
 	num_traces++;
 }
 
@@ -290,14 +232,20 @@ static void setup_trace_file_graphs(void)
 {
 	struct trace_file *tf;
 	int i;
+	int alloc_ptrs;
 
+	if (io_per_process)
+		alloc_ptrs = 16;
+	else
+		alloc_ptrs = 1;
 	list_for_each_entry(tf, &all_traces, list) {
 		tf->tput_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
 		tf->latency_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
 		tf->queue_depth_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
 		tf->iop_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
-		tf->gdd_writes = alloc_dot_data(tf->min_seconds, tf->max_seconds, tf->min_offset, tf->max_offset, tf->stop_seconds);
-		tf->gdd_reads = alloc_dot_data(tf->min_seconds, tf->max_seconds, tf->min_offset, tf->max_offset, tf->stop_seconds);
+		tf->gdd_writes = calloc(alloc_ptrs, sizeof(struct graph_dot_data));
+		tf->gdd_reads = calloc(alloc_ptrs, sizeof(struct graph_dot_data));
+		tf->io_plots_allocated = alloc_ptrs;
 
 		if (tf->trace->mpstat_num_cpus == 0)
 			continue;
@@ -346,6 +294,25 @@ static void read_traces(void)
 	}
 }
 
+static void pick_line_graph_color(void)
+{
+	struct trace_file *tf;
+	int i;
+
+	list_for_each_entry(tf, &all_traces, list) {
+		for (i = 0; i < tf->io_plots; i++) {
+			if (tf->gdd_reads[i]) {
+				tf->line_color = tf->gdd_reads[i]->color;
+				break;
+			}
+			if (tf->gdd_writes[i]) {
+				tf->line_color = tf->gdd_writes[i]->color;
+				break;
+			}
+		}
+	}
+}
+
 static void read_trace_events(void)
 {
 
@@ -365,7 +332,7 @@ static void read_trace_events(void)
 			check_record(trace);
 			add_tput(trace, tf->tput_gld);
 			add_iop(trace, tf->iop_gld);
-			add_io(trace, tf->gdd_writes, tf->gdd_reads);
+			add_io(trace, tf);
 			add_pending_io(trace, tf->queue_depth_gld);
 			add_completed_io(trace, tf->latency_gld);
 			ret = next_record(trace);
@@ -508,28 +475,73 @@ static char *create_movie_temp_dir(void)
 	return ret;
 }
 
-static struct plot_history *alloc_plot_history(char *color)
+static struct pid_plot_history *alloc_pid_plot_history(char *color)
+{
+	struct pid_plot_history *pph;
+
+	pph = calloc(1, sizeof(struct pid_plot_history));
+	if (!pph) {
+		perror("memory allocation failed");
+		exit(1);
+	}
+	pph->history = malloc(sizeof(double) * 4096);
+	if (!pph->history) {
+		perror("memory allocation failed");
+		exit(1);
+	}
+	pph->history_len = 4096;
+	pph->color = color;
+
+	return pph;
+}
+
+static struct plot_history *alloc_plot_history(struct trace_file *tf)
 {
 	struct plot_history *ph = calloc(1, sizeof(struct plot_history));
+	int i;
 
 	if (!ph) {
 		perror("memory allocation failed");
 		exit(1);
 	}
-	ph->history = calloc(4096, sizeof(double));
-	if (!ph->history) {
+	ph->read_pid_history = calloc(tf->io_plots, sizeof(struct pid_plot_history *));
+	if (!ph->read_pid_history) {
 		perror("memory allocation failed");
 		exit(1);
 	}
-	ph->history_len = 4096;
-	ph->color = color;
+	ph->write_pid_history = calloc(tf->io_plots, sizeof(struct pid_plot_history *));
+	if (!ph->write_pid_history) {
+		perror("memory allocation failed");
+		exit(1);
+	}
+	ph->pid_history_count = tf->io_plots;
+	for (i = 0; i < tf->io_plots; i++) {
+		if (tf->gdd_reads[i])
+			ph->read_pid_history[i] = alloc_pid_plot_history(tf->gdd_reads[i]->color);
+		if (tf->gdd_writes[i])
+			ph->write_pid_history[i] = alloc_pid_plot_history(tf->gdd_writes[i]->color);
+	}
 	return ph;
 }
 
-LIST_HEAD(movie_history_writes);
-LIST_HEAD(movie_history_reads);
+LIST_HEAD(movie_history);
 int num_histories = 0;
 
+static void free_plot_history(struct plot_history *ph)
+{
+	int pid;
+
+	for (pid = 0; pid < ph->pid_history_count; pid++) {
+		if (ph->read_pid_history[pid])
+			free(ph->read_pid_history[pid]);
+		if (ph->write_pid_history[pid])
+			free(ph->write_pid_history[pid]);
+	}
+	free(ph->read_pid_history);
+	free(ph->write_pid_history);
+	free(ph);
+}
+
 static void add_history(struct plot_history *ph, struct list_head *list)
 {
 	struct plot_history *entry;
@@ -541,47 +553,81 @@ static void add_history(struct plot_history *ph, struct list_head *list)
 		num_histories--;
 		entry = list_entry(list->next, struct plot_history, list);
 		list_del(&entry->list);
-		free(entry->history);
-		free(entry);
+		free_plot_history(entry);
 	}
 }
 
 static void plot_movie_history(struct plot *plot, struct list_head *list)
 {
 	struct plot_history *ph;
+	int pid;
 
 	if (num_histories > 2)
 		rewind_spindle_steps(num_histories - 1);
 
 	list_for_each_entry(ph, list, list) {
-		if (movie_style == MOVIE_SPINDLE)
-			svg_io_graph_movie_array_spindle(plot, ph);
-		else
-			svg_io_graph_movie_array(plot, ph);
+		for (pid = 0; pid < ph->pid_history_count; pid++) {
+			if (ph->read_pid_history[pid]) {
+				if (movie_style == MOVIE_SPINDLE) {
+					svg_io_graph_movie_array_spindle(plot,
+						ph->read_pid_history[pid]);
+				} else {
+					svg_io_graph_movie_array(plot,
+						ph->read_pid_history[pid]);
+				}
+			}
+			if (ph->write_pid_history[pid]) {
+				if (movie_style == MOVIE_SPINDLE) {
+					svg_io_graph_movie_array_spindle(plot,
+						ph->write_pid_history[pid]);
+				} else {
+					svg_io_graph_movie_array(plot,
+						ph->write_pid_history[pid]);
+				}
+			}
+		}
 	 }
 }
 
 static void free_all_plot_history(struct list_head *head)
 {
 	struct plot_history *entry;
+
 	while (!list_empty(head)) {
 		entry = list_entry(head->next, struct plot_history, list);
 		list_del(&entry->list);
-		free(entry->history);
-		free(entry);
+		free_plot_history(entry);
 	}
 }
 
+static int count_io_plot_types(void)
+{
+	struct trace_file *tf;
+	int i;
+	int total_io_types = 0;
+
+	list_for_each_entry(tf, &all_traces, list) {
+		for (i = 0; i < tf->io_plots; i++) {
+			if (tf->gdd_reads[i])
+				total_io_types++;
+			if (tf->gdd_writes[i])
+				total_io_types++;
+		}
+	}
+	return total_io_types;
+}
+
 static void plot_io(struct plot *plot, int min_seconds, int max_seconds, u64 min_offset, u64 max_offset)
 {
 	struct trace_file *tf;
+	int i;
 
 	if (active_graphs[IO_GRAPH_INDEX] == 0)
 		return;
 
 	setup_axis(plot);
 
-	svg_alloc_legend(plot, num_traces * 2);
+	svg_alloc_legend(plot, count_io_plot_types() * 2);
 
 	set_plot_label(plot, "Device IO");
 	set_ylabel(plot, "Offset (MB)");
@@ -590,17 +636,32 @@ static void plot_io(struct plot *plot, int min_seconds, int max_seconds, u64 min
 	set_xticks(plot, num_xticks, min_seconds, max_seconds);
 
 	list_for_each_entry(tf, &all_traces, list) {
-		char *label = tf->label;
-
-		if (!label)
-			label = "";
-		svg_io_graph(plot, tf->gdd_reads, tf->read_color);
-		if (tf->gdd_reads->total_ios)
-			svg_add_legend(plot, label, " Reads", tf->read_color);
+		char label[256];
+		char *pos;
+
+		if (!tf->label)
+			label[0] = 0;
+		else {
+			strcpy(label, tf->label);
+			if (io_per_process)
+				strcat(label, " ");
+		}
+		pos = label + strlen(label);
+
+		for (i = 0; i < tf->io_plots; i++) {
+			if (tf->gdd_reads[i]) {
+				svg_io_graph(plot, tf->gdd_reads[i]);
+				if (io_per_process)
+					strcpy(pos, tf->gdd_reads[i]->label);
+				svg_add_legend(plot, label, " Reads", tf->gdd_reads[i]->color);
+			}
 
-		svg_io_graph(plot, tf->gdd_writes, tf->write_color);
-		if (tf->gdd_writes->total_ios) {
-			svg_add_legend(plot, label, " Writes", tf->write_color);
+			if (tf->gdd_writes[i]) {
+				svg_io_graph(plot, tf->gdd_writes[i]);
+				if (io_per_process)
+					strcpy(pos, tf->gdd_writes[i]->label);
+				svg_add_legend(plot, label, " Writes", tf->gdd_writes[i]->color);
+			}
 		}
 	}
 	if (plot->add_xlabel)
@@ -640,9 +701,9 @@ static void plot_tput(struct plot *plot, int min_seconds, int max_seconds)
 	set_xticks(plot, num_xticks, min_seconds, max_seconds);
 
 	list_for_each_entry(tf, &all_traces, list) {
-		svg_line_graph(plot, tf->tput_gld, tf->read_color, 0, 0);
+		svg_line_graph(plot, tf->tput_gld, tf->line_color, 0, 0);
 		if (num_traces > 1)
-			svg_add_legend(plot, tf->label, "", tf->read_color);
+			svg_add_legend(plot, tf->label, "", tf->line_color);
 	}
 
 	if (plot->add_xlabel)
@@ -692,7 +753,7 @@ static void plot_cpu(struct plot *plot, int max_seconds, char *label,
 	set_ylabel(plot, "Percent");
 	set_xticks(plot, num_xticks, tf->mpstat_min_seconds, max_seconds);
 
-	cpu_color_index = 0;
+	reset_cpu_color();
 	list_for_each_entry(tf, &all_traces, list) {
 		if (tf->mpstat_gld == 0)
 			break;
@@ -774,9 +835,9 @@ static void plot_queue_depth(struct plot *plot, int min_seconds, int max_seconds
 	set_xticks(plot, num_xticks, min_seconds, max_seconds);
 
 	list_for_each_entry(tf, &all_traces, list) {
-		svg_line_graph(plot, tf->queue_depth_gld, tf->read_color, 0, 0);
+		svg_line_graph(plot, tf->queue_depth_gld, tf->line_color, 0, 0);
 		if (num_traces > 1)
-			svg_add_legend(plot, tf->label, "", tf->read_color);
+			svg_add_legend(plot, tf->label, "", tf->line_color);
 	}
 
 	if (plot->add_xlabel)
@@ -817,9 +878,8 @@ static void plot_io_movie(struct plot *plot)
 {
 	struct trace_file *tf;
 	char *movie_dir = create_movie_temp_dir();
-	int i;
-	struct plot_history *read_history;
-	struct plot_history *write_history;
+	int i, pid;
+	struct plot_history *history;
 	int batch_i;
 	int movie_len = 30;
 	int movie_frames_per_sec = 20;
@@ -836,9 +896,17 @@ static void plot_io_movie(struct plot *plot)
 		batch_count = 1;
 
 	list_for_each_entry(tf, &all_traces, list) {
-		char *label = tf->label;
-		if (!label)
-			label = "";
+		char label[256];
+		char *pos;
+
+		if (!tf->label)
+			label[0] = 0;
+		else {
+			strcpy(label, tf->label);
+			if (io_per_process)
+				strcat(label, " ");
+		}
+		pos = label + strlen(label);
 
 		i = 0;
 		while (i < cols) {
@@ -852,15 +920,14 @@ static void plot_io_movie(struct plot *plot)
 			set_graph_size(cols / graph_width_factor, rows / 8);
 			plot->timeline = i / graph_width_factor;
 
-			plot_tput(plot, tf->gdd_reads->min_seconds,
-				  tf->gdd_reads->max_seconds);
+			plot_tput(plot, tf->min_seconds,
+				  tf->max_seconds);
 
-			plot_cpu(plot, tf->gdd_reads->max_seconds,
+			plot_cpu(plot, tf->max_seconds,
 				   "CPU System Time", CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
 
 			plot->direction = PLOT_ACROSS;
-			plot_queue_depth(plot, tf->gdd_reads->min_seconds,
-					 tf->gdd_reads->max_seconds);
+			plot_queue_depth(plot, tf->min_seconds, tf->max_seconds);
 
 			/* movie graph starts here */
 			plot->start_y_offset = orig_y_offset;
@@ -874,31 +941,45 @@ static void plot_io_movie(struct plot *plot)
 			else
 				setup_axis(plot);
 
-			svg_alloc_legend(plot, num_traces * 2);
+			svg_alloc_legend(plot, count_io_plot_types() * 2);
 
-			read_history = alloc_plot_history(tf->read_color);
-			write_history = alloc_plot_history(tf->write_color);
-			read_history->col = i;
-			write_history->col = i;
+			history = alloc_plot_history(tf);
+			history->col = i;
 
-			if (tf->gdd_reads->total_ios)
-				svg_add_legend(plot, label, " Reads", tf->read_color);
-			if (tf->gdd_writes->total_ios)
-				svg_add_legend(plot, label, " Writes", tf->write_color);
+			for (pid = 0; pid < tf->io_plots; pid++) {
+				if (tf->gdd_reads[pid]) {
+					if (io_per_process)
+						strcpy(pos, tf->gdd_reads[pid]->label);
+					svg_add_legend(plot, label, " Reads", tf->gdd_reads[pid]->color);
+				}
+				if (tf->gdd_writes[pid]) {
+					if (io_per_process)
+						strcpy(pos, tf->gdd_writes[pid]->label);
+					svg_add_legend(plot, label, " Writes", tf->gdd_writes[pid]->color);
+				}
+			}
 
 			batch_i = 0;
 			while (i < cols && batch_i < batch_count) {
-				svg_io_graph_movie(tf->gdd_reads, read_history, i);
-				svg_io_graph_movie(tf->gdd_writes, write_history, i);
+				for (pid = 0; pid < tf->io_plots; pid++) {
+					if (tf->gdd_reads[pid]) {
+						svg_io_graph_movie(tf->gdd_reads[pid],
+								   history->read_pid_history[pid],
+								   i);
+					}
+					if (tf->gdd_writes[pid]) {
+						svg_io_graph_movie(tf->gdd_writes[pid],
+								   history->write_pid_history[pid],
+								   i);
+					}
+				}
 				i++;
 				batch_i++;
 			}
 
-			add_history(read_history, &movie_history_reads);
-			add_history(write_history, &movie_history_writes);
+			add_history(history, &movie_history);
 
-			plot_movie_history(plot, &movie_history_reads);
-			plot_movie_history(plot, &movie_history_writes);
+			plot_movie_history(plot, &movie_history);
 
 			svg_write_legend(plot);
 			close_plot(plot);
@@ -906,8 +987,7 @@ static void plot_io_movie(struct plot *plot)
 
 			close_plot_file(plot);
 		}
-		free_all_plot_history(&movie_history_reads);
-		free_all_plot_history(&movie_history_writes);
+		free_all_plot_history(&movie_history);
 	}
 	convert_movie_files(movie_dir);
 	mencode_movie(movie_dir);
@@ -948,9 +1028,9 @@ static void plot_latency(struct plot *plot, int min_seconds, int max_seconds)
 	set_xticks(plot, num_xticks, min_seconds, max_seconds);
 
 	list_for_each_entry(tf, &all_traces, list) {
-		svg_line_graph(plot, tf->latency_gld, tf->read_color, 0, 0);
+		svg_line_graph(plot, tf->latency_gld, tf->line_color, 0, 0);
 		if (num_traces > 1)
-			svg_add_legend(plot, tf->label, "", tf->read_color);
+			svg_add_legend(plot, tf->label, "", tf->line_color);
 	}
 
 	if (plot->add_xlabel)
@@ -992,9 +1072,9 @@ static void plot_iops(struct plot *plot, int min_seconds, int max_seconds)
 	set_xticks(plot, num_xticks, min_seconds, max_seconds);
 
 	list_for_each_entry(tf, &all_traces, list) {
-		svg_line_graph(plot, tf->iop_gld, tf->read_color, 0, 0);
+		svg_line_graph(plot, tf->iop_gld, tf->line_color, 0, 0);
 		if (num_traces > 1)
-			svg_add_legend(plot, tf->label, "", tf->read_color);
+			svg_add_legend(plot, tf->label, "", tf->line_color);
 	}
 
 	if (plot->add_xlabel)
@@ -1032,7 +1112,7 @@ enum {
 	HELP_LONG_OPT = 1,
 };
 
-char *option_string = "T:t:o:l:r:O:N:d:p:m::h:w:c:x:y:a:";
+char *option_string = "T:t:o:l:r:O:N:d:p:m::h:w:c:x:y:a:P";
 static struct option long_options[] = {
 	{"columns", required_argument, 0, 'c'},
 	{"title", required_argument, 0, 'T'},
@@ -1050,6 +1130,7 @@ static struct option long_options[] = {
 	{"xzoom", required_argument, 0, 'x'},
 	{"yzoom", required_argument, 0, 'y'},
 	{"io-plot-action", required_argument, 0, 'a'},
+	{"per-process-io", no_argument, 0, 'P'},
 	{"help", no_argument, 0, HELP_LONG_OPT},
 	{0, 0, 0, 0}
 };
@@ -1074,6 +1155,7 @@ static void print_usage(void)
 		"\t-x (--xzoom): limit processed time to min:max\n"
 		"\t-y (--yzoom): limit processed sectors to min:max\n"
 		"\t-a (--io-plot-action): plot given action (one of Q,D,C) in IO graph\n"
+		"\t-P (--per-process-io): distinguish between processes in IO graph\n"
 	       );
 	exit(1);
 }
@@ -1233,6 +1315,9 @@ action_err:
 			if (plot_io_action < 0)
 				goto action_err;
 			break;
+		case 'P':
+			io_per_process = 1;
+			break;
 		case '?':
 		case HELP_LONG_OPT:
 			print_usage();
@@ -1257,6 +1342,7 @@ int main(int ac, char **av)
 	int rows, cols;
 
 	init_io_hash_table();
+	init_process_hash_table();
 
 	enable_all_graphs();
 
@@ -1335,9 +1421,12 @@ int main(int ac, char **av)
 	/* run through all the traces and read their events */
 	read_trace_events();
 
+	pick_line_graph_color();
+
 	plot = alloc_plot();
 
 	if (make_movie) {
+		set_legend_width(longest_label + longest_proc_name + 1 + strlen("writes"));
 		plot_io_movie(plot);
 		exit(0);
 	}
@@ -1345,7 +1434,7 @@ int main(int ac, char **av)
 	set_plot_output(plot, output_filename);
 
 	if (active_graphs[IO_GRAPH_INDEX] || found_mpstat)
-		set_legend_width(longest_label + strlen("writes"));
+		set_legend_width(longest_label + longest_proc_name + 1 + strlen("writes"));
 	else if (num_traces > 1)
 		set_legend_width(longest_label);
 	else
diff --git a/iowatcher/plot.c b/iowatcher/plot.c
index d3505e8..79e5d3c 100644
--- a/iowatcher/plot.c
+++ b/iowatcher/plot.c
@@ -69,6 +69,49 @@ static char line[1024];
 static int final_height = 0;
 static int final_width = 0;
 
+static char *colors[] = {
+	"blue", "darkgreen",
+	"red", "aqua",
+	"orange", "darkviolet",
+	"brown", "#00FF00",
+	"yellow", "coral",
+	"black", "darkred",
+	"fuchsia", "crimson",
+	NULL };
+
+extern unsigned int longest_proc_name;
+
+char *pick_color(void)
+{
+	static int color_index;
+	char *ret = colors[color_index];
+
+	if (!ret) {
+		color_index = 0;
+		ret = colors[color_index];
+	}
+	color_index++;
+	return ret;
+}
+
+static int cpu_color_index;
+
+char *pick_cpu_color(void)
+{
+	char *ret = colors[cpu_color_index];
+	if (!ret) {
+		cpu_color_index = 0;
+		ret = colors[cpu_color_index];
+	}
+	cpu_color_index++;
+	return ret;
+}
+
+void reset_cpu_color(void)
+{
+	cpu_color_index = 0;
+}
+
 struct graph_line_data *alloc_line_data(int min_seconds, int max_seconds, int stop_seconds)
 {
 	int size = sizeof(struct graph_line_data) + (stop_seconds + 1) * sizeof(struct graph_line_pair);
@@ -91,7 +134,7 @@ void free_line_data(struct graph_line_data *gld)
 	free(gld);
 }
 
-struct graph_dot_data *alloc_dot_data(int min_seconds, int max_seconds, u64 min_offset, u64 max_offset, int stop_seconds)
+struct graph_dot_data *alloc_dot_data(int min_seconds, int max_seconds, u64 min_offset, u64 max_offset, int stop_seconds, char *color, char *label)
 {
 	int size;
 	int arr_size;
@@ -119,6 +162,12 @@ struct graph_dot_data *alloc_dot_data(int min_seconds, int max_seconds, u64 min_
 	gdd->cols = cols;
 	gdd->min_offset = min_offset;
 	gdd->max_offset = max_offset;
+	gdd->color = color;
+	gdd->label = label;
+
+	if (strlen(label) > longest_proc_name)
+		longest_proc_name = strlen(label);
+
 	return gdd;
 }
 
@@ -838,18 +887,18 @@ static int svg_add_io(int fd, double row, double col, double width, double heigh
 	return write(fd, line, strlen(line));
 }
 
-int svg_io_graph_movie_array(struct plot *plot, struct plot_history *ph)
+int svg_io_graph_movie_array(struct plot *plot, struct pid_plot_history *pph)
 {
 	double cell_index;
 	double movie_row;
 	double movie_col;
 	int i;
 
-	for (i = 0; i < ph->num_used; i++) {
-		cell_index = ph->history[i];
+	for (i = 0; i < pph->num_used; i++) {
+		cell_index = pph->history[i];
 		movie_row = floor(cell_index / graph_width);
 		movie_col = cell_index - movie_row * graph_width;
-		svg_add_io(plot->fd, movie_row, movie_col, 4, 4, ph->color);
+		svg_add_io(plot->fd, movie_row, movie_col, 4, 4, pph->color);
 	}
 	return 0;
 }
@@ -861,7 +910,7 @@ void rewind_spindle_steps(int num)
 	spindle_steps -= num * 0.01;
 }
 
-int svg_io_graph_movie_array_spindle(struct plot *plot, struct plot_history *ph)
+int svg_io_graph_movie_array_spindle(struct plot *plot, struct pid_plot_history *pph)
 {
 	double cell_index;
 	int i;
@@ -901,11 +950,11 @@ int svg_io_graph_movie_array_spindle(struct plot *plot, struct plot_history *ph)
 
 	radius = floor(radius / 2);
 	num_circles = radius / 4 - 3;
-	cells_per_circle = ph->history_max / num_circles;
+	cells_per_circle = pph->history_max / num_circles;
 	degrees_per_cell = 360 / cells_per_circle;
 
-	for (i = 0; i < ph->num_used; i++) {
-		cell_index = ph->history[i];
+	for (i = 0; i < pph->num_used; i++) {
+		cell_index = pph->history[i];
 		circle_num = floor(cell_index / cells_per_circle);
 		rot = cell_index - circle_num * cells_per_circle;
 		circle_num = num_circles - circle_num;
@@ -918,29 +967,29 @@ int svg_io_graph_movie_array_spindle(struct plot *plot, struct plot_history *ph)
 			 "stroke=\"%s\" stroke-width=\"4\"/>\n",
 			 rot, center_x, center_y,
 			 axis_x_off_double(graph_width_extra / 2 + radius) + 8, center_y,
-			 radius, radius, ph->color);
+			 radius, radius, pph->color);
 
 		write(plot->fd, line, strlen(line));
 	}
 	return 0;
 }
 
-static int add_plot_history(struct plot_history *ph, double val)
+static int add_plot_history(struct pid_plot_history *pph, double val)
 {
-	if (ph->num_used == ph->history_len) {
-		ph->history = realloc(ph->history,
-				      (ph->history_len + 4096) * sizeof(double));
-		if (!ph->history) {
+	if (pph->num_used == pph->history_len) {
+		pph->history_len += 4096;
+		pph->history = realloc(pph->history,
+				       pph->history_len * sizeof(double));
+		if (!pph->history) {
 			perror("Unable to allocate memory");
 			exit(1);
 		}
-		ph->history_len += 4096;
 	}
-	ph->history[ph->num_used++] = val;
+	pph->history[pph->num_used++] = val;
 	return 0;
 }
 
-int svg_io_graph_movie(struct graph_dot_data *gdd, struct plot_history *ph, int col)
+int svg_io_graph_movie(struct graph_dot_data *gdd, struct pid_plot_history *pph, int col)
 {
 	int row = 0;
 	int arr_index;
@@ -953,7 +1002,7 @@ int svg_io_graph_movie(struct graph_dot_data *gdd, struct plot_history *ph, int
 	int margin_orig = graph_inner_y_margin;
 
 	graph_inner_y_margin += 5;
-	ph->history_max = (gdd->max_offset - gdd->min_offset + 1) / movie_blocks_per_cell;
+	pph->history_max = (gdd->max_offset - gdd->min_offset + 1) / movie_blocks_per_cell;
 
 	for (row = gdd->rows - 1; row >= 0; row--) {
 		bit_index = row * gdd->cols + col;
@@ -970,14 +1019,14 @@ int svg_io_graph_movie(struct graph_dot_data *gdd, struct plot_history *ph, int
 			/* a cell number in the graph */
 			cell_index /= movie_blocks_per_cell;
 
-			add_plot_history(ph, cell_index);
+			add_plot_history(pph, cell_index);
 		}
 	}
 	graph_inner_y_margin = margin_orig;
 	return 0;
 }
 
-int svg_io_graph(struct plot *plot, struct graph_dot_data *gdd, char *color)
+int svg_io_graph(struct plot *plot, struct graph_dot_data *gdd)
 {
 	int fd = plot->fd;;
 	int col = 0;
@@ -997,7 +1046,7 @@ int svg_io_graph(struct plot *plot, struct graph_dot_data *gdd, char *color)
 				continue;
 			val = gdd->data[arr_index];
 			if (val & (1 << bit_mod))
-				svg_add_io(fd, floor(row / io_graph_scale), col, 1.5, 1.5, color);
+				svg_add_io(fd, floor(row / io_graph_scale), col, 1.5, 1.5, gdd->color);
 		}
 	}
 	return 0;
diff --git a/iowatcher/plot.h b/iowatcher/plot.h
index fb9c63b..bc85f45 100644
--- a/iowatcher/plot.h
+++ b/iowatcher/plot.h
@@ -109,25 +109,37 @@ struct graph_dot_data {
 	/* label for the legend */
 	char *label;
 
+	/* color for plotting data */
+	char *color;
+
 	/* bitmap, one bit for each cell to light up */
 	unsigned char data[];
 };
 
-struct plot_history {
-	struct list_head list;
+struct pid_plot_history {
 	double history_max;
 	int history_len;
 	int num_used;
-	int col;
 	char *color;
 	double *history;
 };
 
-int svg_io_graph(struct plot *plot, struct graph_dot_data *gdd, char *color);
+struct plot_history {
+	struct list_head list;
+	int pid_history_count;
+	int col;
+	struct pid_plot_history **read_pid_history;
+	struct pid_plot_history **write_pid_history;
+};
+
+char *pick_color(void);
+char *pick_cpu_color(void);
+void reset_cpu_color(void);
+int svg_io_graph(struct plot *plot, struct graph_dot_data *gdd);
 int svg_line_graph(struct plot *plot, struct graph_line_data *gld, char *color, int thresh1, int thresh2);
 struct graph_line_data *alloc_line_data(int min_seconds, int max_seconds, int stop_seconds);
 void free_line_data(struct graph_line_data *gld);
-struct graph_dot_data *alloc_dot_data(int min_seconds, int max_seconds, u64 min_offset, u64 max_offset, int stop_seconds);
+struct graph_dot_data *alloc_dot_data(int min_seconds, int max_seconds, u64 min_offset, u64 max_offset, int stop_seconds, char *color, char *label);
 void free_dot_data(struct graph_dot_data *gdd);
 void set_gdd_bit(struct graph_dot_data *gdd, u64 offset, double bytes, double time);
 void print_gdd(struct graph_dot_data *gdd);
@@ -156,13 +168,13 @@ void set_io_graph_scale(int scale);
 void set_plot_output(struct plot *plot, char *filename);
 void set_graph_size(int width, int height);
 void get_graph_size(int *width, int *height);
-int svg_io_graph_movie(struct graph_dot_data *gdd, struct plot_history *ph, int col);
-int svg_io_graph_movie_array(struct plot *plot, struct plot_history *ph);
+int svg_io_graph_movie(struct graph_dot_data *gdd, struct pid_plot_history *ph, int col);
+int svg_io_graph_movie_array(struct plot *plot, struct pid_plot_history *ph);
 void svg_write_time_line(struct plot *plot, int col);
 void set_graph_height(int h);
 void set_graph_width(int w);
 int close_plot_file(struct plot *plot);
-int svg_io_graph_movie_array_spindle(struct plot *plot, struct plot_history *ph);
+int svg_io_graph_movie_array_spindle(struct plot *plot, struct pid_plot_history *ph);
 void rewind_spindle_steps(int num);
 void setup_axis_spindle(struct plot *plot);
 int close_plot_col(struct plot *plot);