Merge the iowatcher repository

16 files changed, 6259 insertions, 0 deletions

diff --git a/iowatcher/COPYING b/iowatcher/COPYING
new file mode 100644
index 0000000..d159169
--- /dev/null
+++ b/iowatcher/COPYING
@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/iowatcher/Makefile b/iowatcher/Makefile
new file mode 100644
index 0000000..7b5101c
--- /dev/null
+++ b/iowatcher/Makefile
@@ -0,0 +1,37 @@
+C      = gcc
+CFLAGS  = -Wall -O0 -g -W
+ALL_CFLAGS = $(CFLAGS) -D_GNU_SOURCE -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
+
+PROGS = iowatcher
+INSTALL = install
+prefix = /usr/local
+bindir = $(prefix)/bin
+
+export prefix INSTALL
+
+ALL = $(PROGS)
+
+$(PROGS): | depend
+
+all: $(ALL)
+
+%.o: %.c
+	$(CC) -o $*.o -c $(ALL_CFLAGS) $<
+
+iowatcher: blkparse.o plot.o main.o tracers.o mpstat.o fio.o
+	$(CC) $(ALL_CFLAGS) -o $@ $(filter %.o,$^) -lm
+
+depend:
+	@$(CC) -MM $(ALL_CFLAGS) *.c 1> .depend
+
+clean:
+	-rm -f *.o $(PROGS) .depend
+
+install: all
+	$(INSTALL) -m 755 -d $(DESTDIR)$(bindir)
+	$(INSTALL) -m 755 $(ALL) $(DESTDIR)$(bindir)
+
+ifneq ($(wildcard .depend),)
+include .depend
+endif
+
diff --git a/iowatcher/README b/iowatcher/README
new file mode 100644
index 0000000..83db494
--- /dev/null
+++ b/iowatcher/README
@@ -0,0 +1,97 @@
+iowatcher graphs the results of a blktrace run.  It has a few different modes:
+
+	* Graph the result of an existing blktrace
+
+	* Start a new blktrace
+
+	* Start a new blktrace and a benchmark run
+
+	* Make a movie of the IO from a given trace (only mp4 for now)
+
+Output:
+
+	iowatcher can produce either svg files or mp4 movies.  Most browsers
+	can view the svg files, or you can use rsvg-view-3 from librsvg.
+	rsvg-convert can turn the svgs into many other formats.
+
+Building:
+
+	Type make and make install.  We need ffmpeg or png2theora, and
+	librsvg to make movies, otherwise there are no dependencies.
+
+The basic options:
+
+	-d controls which device you are tracing.  You can only trace one device
+	   at a time for now.  It is sent directly to blktrace, and only
+	   needed when you are making a new trace.
+
+	-t controls the name of the blktrace file.  iowatcher uses a dump from
+	   blkparse, so -t tries to guess the name of the corresponding
+	   per CPU blktrace data files if the dump file doesn't already exist.
+
+	   If you want more than one trace in a given graph, you can specify
+	   -t more than once.
+
+	-F Add a fio bandwidth log graph.  You need to run fio --bandwidth-log
+	   to get one of these, and then pass either the read log or the write
+	   log into iowatcher.
+
+	-l Sets a label in the graph for a trace file.  The labels are added in
+	   the same order the trace files are added.
+
+	-m Create a movie.  The file format depends on the extension used in the
+	   -o filename.* option.  If you specify an .ogv or .ogg extension, the
+	   result will be Ogg Theora video, if png2theora is available.
+	   If you use an .mp4 extension, the result will be an mp4 video if
+	   ffmpeg is avilable.  You can use any other extension, but the end
+	   result will be an mp4.
+
+	   You can use --movie=spindle or --movie=rect, which changes the
+	   style of the IO mapping.
+
+	-T Set a title for the graph.  This goes at the top of the image.
+
+	-o output filename.  The default is trace.svg.  iowatcher is
+	   only able to create svg for now.
+
+	-r control the duration in seconds for the rolling average.
+	   iowatcher tries to smooth out bumpy graphs by averaging the
+	   current second with seconds from the past.  Longer numbers here
+	   give you flatter graphs.
+
+	-P add per-process tags to the IO.  Each process responsible for
+	   submitting the IO gets a different color.
+
+	-O add a single graph to the output.  By default all the graphs
+	   are included, but with -O you get only the graphs you ask for.
+	   -O may be used more than once.
+
+	-N remove a single graph from the output.  This may also be used more
+	   than once.
+
+	Choices for -O and -N are:
+	   io, fio, tput, latency, queue_depth, iops, cpu-sys, cpu-io,
+	   cpu-irq, cpu-user, cpu-soft
+
+Examples:
+	
+	# generate graph from the existing trace.dump
+	iowatcher -t trace.dump -o trace.svg
+	
+	# skip the IO graph
+	iowatcher -t trace.dump -o trace.svg -N io
+
+	# only graph tput and latency
+	iowatcher -t trace.dump -o trace.svg -O tput -O latency
+
+	# generate a graph from two runs, and label them
+	iowatcher -t ext4.dump -t xfs.dump -l Ext4 -l XFS -o trace.svg
+
+	# Run a fio benchmark and store the trace in trace.dump
+	# add a title to the top.  Use /dev/sda for blktrace
+	iowatcher -d /dev/sda -t trace.dump -T 'Fio Benchmark' -p 'fio some_job_file'
+
+	# Make a movie from an existing trace
+	iowatcher -t trace --movie -o trace.mp4
+
+Please email chris.mason@fusionio.com with any questions
diff --git a/iowatcher/blkparse.c b/iowatcher/blkparse.c
new file mode 100644
index 0000000..ef33d5b
--- /dev/null
+++ b/iowatcher/blkparse.c
@@ -0,0 +1,1196 @@
+/*
+ * Copyright (C) 2012 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ *  Parts of this file were imported from Jens Axboe's blktrace sources (also GPL)
+ */
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+#include <inttypes.h>
+#include <string.h>
+#include <asm/types.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <math.h>
+#include <dirent.h>
+
+#include "plot.h"
+#include "blkparse.h"
+#include "list.h"
+#include "tracers.h"
+
+#define IO_HASH_TABLE_BITS  11
+#define IO_HASH_TABLE_SIZE (1 << IO_HASH_TABLE_BITS)
+static struct list_head io_hash_table[IO_HASH_TABLE_SIZE];
+static u64 ios_in_flight = 0;
+
+#define PROCESS_HASH_TABLE_BITS 7
+#define PROCESS_HASH_TABLE_SIZE (1 << PROCESS_HASH_TABLE_BITS)
+static struct list_head process_hash_table[PROCESS_HASH_TABLE_SIZE];
+
+extern int plot_io_action;
+extern int io_per_process;
+
+/*
+ * Trace categories
+ */
+enum {
+	BLK_TC_READ	= 1 << 0,	/* reads */
+	BLK_TC_WRITE	= 1 << 1,	/* writes */
+	BLK_TC_FLUSH	= 1 << 2,	/* flush */
+	BLK_TC_SYNC	= 1 << 3,	/* sync */
+	BLK_TC_QUEUE	= 1 << 4,	/* queueing/merging */
+	BLK_TC_REQUEUE	= 1 << 5,	/* requeueing */
+	BLK_TC_ISSUE	= 1 << 6,	/* issue */
+	BLK_TC_COMPLETE	= 1 << 7,	/* completions */
+	BLK_TC_FS	= 1 << 8,	/* fs requests */
+	BLK_TC_PC	= 1 << 9,	/* pc requests */
+	BLK_TC_NOTIFY	= 1 << 10,	/* special message */
+	BLK_TC_AHEAD	= 1 << 11,	/* readahead */
+	BLK_TC_META	= 1 << 12,	/* metadata */
+	BLK_TC_DISCARD	= 1 << 13,	/* discard requests */
+	BLK_TC_DRV_DATA	= 1 << 14,	/* binary driver data */
+	BLK_TC_FUA	= 1 << 15,	/* fua requests */
+
+	BLK_TC_END	= 1 << 15,	/* we've run out of bits! */
+};
+
+#define BLK_TC_SHIFT		(16)
+#define BLK_TC_ACT(act)		((act) << BLK_TC_SHIFT)
+#define BLK_DATADIR(a) (((a) >> BLK_TC_SHIFT) & (BLK_TC_READ | BLK_TC_WRITE))
+
+/*
+ * Basic trace actions
+ */
+enum {
+	__BLK_TA_QUEUE = 1,		/* queued */
+	__BLK_TA_BACKMERGE,		/* back merged to existing rq */
+	__BLK_TA_FRONTMERGE,		/* front merge to existing rq */
+	__BLK_TA_GETRQ,			/* allocated new request */
+	__BLK_TA_SLEEPRQ,		/* sleeping on rq allocation */
+	__BLK_TA_REQUEUE,		/* request requeued */
+	__BLK_TA_ISSUE,			/* sent to driver */
+	__BLK_TA_COMPLETE,		/* completed by driver */
+	__BLK_TA_PLUG,			/* queue was plugged */
+	__BLK_TA_UNPLUG_IO,		/* queue was unplugged by io */
+	__BLK_TA_UNPLUG_TIMER,		/* queue was unplugged by timer */
+	__BLK_TA_INSERT,		/* insert request */
+	__BLK_TA_SPLIT,			/* bio was split */
+	__BLK_TA_BOUNCE,		/* bio was bounced */
+	__BLK_TA_REMAP,			/* bio was remapped */
+	__BLK_TA_ABORT,			/* request aborted */
+	__BLK_TA_DRV_DATA,		/* binary driver data */
+};
+
+#define BLK_TA_MASK ((1 << BLK_TC_SHIFT) - 1)
+
+/*
+ * Notify events.
+ */
+enum blktrace_notify {
+	__BLK_TN_PROCESS = 0,		/* establish pid/name mapping */
+	__BLK_TN_TIMESTAMP,		/* include system clock */
+	__BLK_TN_MESSAGE,               /* Character string message */
+};
+
+/*
+ * Trace actions in full. Additionally, read or write is masked
+ */
+#define BLK_TA_QUEUE		(__BLK_TA_QUEUE | BLK_TC_ACT(BLK_TC_QUEUE))
+#define BLK_TA_BACKMERGE	(__BLK_TA_BACKMERGE | BLK_TC_ACT(BLK_TC_QUEUE))
+#define BLK_TA_FRONTMERGE	(__BLK_TA_FRONTMERGE | BLK_TC_ACT(BLK_TC_QUEUE))
+#define	BLK_TA_GETRQ		(__BLK_TA_GETRQ | BLK_TC_ACT(BLK_TC_QUEUE))
+#define	BLK_TA_SLEEPRQ		(__BLK_TA_SLEEPRQ | BLK_TC_ACT(BLK_TC_QUEUE))
+#define	BLK_TA_REQUEUE		(__BLK_TA_REQUEUE | BLK_TC_ACT(BLK_TC_REQUEUE))
+#define BLK_TA_ISSUE		(__BLK_TA_ISSUE | BLK_TC_ACT(BLK_TC_ISSUE))
+#define BLK_TA_COMPLETE		(__BLK_TA_COMPLETE| BLK_TC_ACT(BLK_TC_COMPLETE))
+#define BLK_TA_PLUG		(__BLK_TA_PLUG | BLK_TC_ACT(BLK_TC_QUEUE))
+#define BLK_TA_UNPLUG_IO	(__BLK_TA_UNPLUG_IO | BLK_TC_ACT(BLK_TC_QUEUE))
+#define BLK_TA_UNPLUG_TIMER	(__BLK_TA_UNPLUG_TIMER | BLK_TC_ACT(BLK_TC_QUEUE))
+#define BLK_TA_INSERT		(__BLK_TA_INSERT | BLK_TC_ACT(BLK_TC_QUEUE))
+#define BLK_TA_SPLIT		(__BLK_TA_SPLIT)
+#define BLK_TA_BOUNCE		(__BLK_TA_BOUNCE)
+#define BLK_TA_REMAP		(__BLK_TA_REMAP | BLK_TC_ACT(BLK_TC_QUEUE))
+#define BLK_TA_ABORT		(__BLK_TA_ABORT | BLK_TC_ACT(BLK_TC_QUEUE))
+#define BLK_TA_DRV_DATA		(__BLK_TA_DRV_DATA | BLK_TC_ACT(BLK_TC_DRV_DATA))
+
+#define BLK_TN_PROCESS		(__BLK_TN_PROCESS | BLK_TC_ACT(BLK_TC_NOTIFY))
+#define BLK_TN_TIMESTAMP	(__BLK_TN_TIMESTAMP | BLK_TC_ACT(BLK_TC_NOTIFY))
+#define BLK_TN_MESSAGE		(__BLK_TN_MESSAGE | BLK_TC_ACT(BLK_TC_NOTIFY))
+
+#define BLK_IO_TRACE_MAGIC	0x65617400
+#define BLK_IO_TRACE_VERSION	0x07
+/*
+ * The trace itself
+ */
+struct blk_io_trace {
+	__u32 magic;		/* MAGIC << 8 | version */
+	__u32 sequence;		/* event number */
+	__u64 time;		/* in nanoseconds */
+	__u64 sector;		/* disk offset */
+	__u32 bytes;		/* transfer length */
+	__u32 action;		/* what happened */
+	__u32 pid;		/* who did it */
+	__u32 device;		/* device identifier (dev_t) */
+	__u32 cpu;		/* on what cpu did it happen */
+	__u16 error;		/* completion error */
+	__u16 pdu_len;		/* length of data after this trace */
+};
+
+struct pending_io {
+	/* sector offset of this IO */
+	u64 sector;
+
+	/* dev_t for this IO */
+	u32 device;
+
+	/* time this IO was dispatched */
+	u64 dispatch_time;
+	/* time this IO was finished */
+	u64 completion_time;
+	struct list_head hash_list;
+	/* process which queued this IO */
+	u32 pid;
+};
+
+struct pid_map {
+	struct list_head hash_list;
+	u32 pid;
+	int index;
+	char name[0];
+};
+
+u64 get_record_time(struct trace *trace)
+{
+	return trace->io->time;
+}
+
+void init_io_hash_table(void)
+{
+	int i;
+	struct list_head *head;
+
+	for (i = 0; i < IO_HASH_TABLE_SIZE; i++) {
+		head = io_hash_table + i;
+		INIT_LIST_HEAD(head);
+	}
+}
+
+/* taken from the kernel hash.h */
+static inline u64 hash_sector(u64 val)
+{
+	u64 hash = val;
+
+	/*  Sigh, gcc can't optimise this alone like it does for 32 bits. */
+	u64 n = hash;
+	n <<= 18;
+	hash -= n;
+	n <<= 33;
+	hash -= n;
+	n <<= 3;
+	hash += n;
+	n <<= 3;
+	hash -= n;
+	n <<= 4;
+	hash += n;
+	n <<= 2;
+	hash += n;
+
+	/* High bits are more random, so use them. */
+	return hash >> (64 - IO_HASH_TABLE_BITS);
+}
+
+static int io_hash_table_insert(struct pending_io *ins_pio)
+{
+	u64 sector = ins_pio->sector;
+	u32 dev = ins_pio->device;
+	int slot = hash_sector(sector);
+	struct list_head *head;
+	struct pending_io *pio;
+
+	head = io_hash_table + slot;
+	list_for_each_entry(pio, head, hash_list) {
+		if (pio->sector == sector && pio->device == dev)
+			return -EEXIST;
+	}
+	list_add_tail(&ins_pio->hash_list, head);
+	return 0;
+}
+
+static struct pending_io *io_hash_table_search(u64 sector, u32 dev)
+{
+	int slot = hash_sector(sector);
+	struct list_head *head;
+	struct pending_io *pio;
+
+	head = io_hash_table + slot;
+	list_for_each_entry(pio, head, hash_list) {
+		if (pio->sector == sector && pio->device == dev)
+			return pio;
+	}
+	return NULL;
+}
+
+static struct pending_io *hash_queued_io(struct blk_io_trace *io)
+{
+	struct pending_io *pio;
+	int ret;
+
+	pio = calloc(1, sizeof(*pio));
+	pio->sector = io->sector;
+	pio->device = io->device;
+	pio->pid = io->pid;
+
+	ret = io_hash_table_insert(pio);
+	if (ret < 0) {
+		/* crud, the IO is there already */
+		free(pio);
+		return NULL;
+	}
+	return pio;
+}
+
+static struct pending_io *hash_dispatched_io(struct blk_io_trace *io)
+{
+	struct pending_io *pio;
+
+	pio = io_hash_table_search(io->sector, io->device);
+	if (!pio) {
+		pio = hash_queued_io(io);
+		if (!pio)
+			return NULL;
+	}
+	pio->dispatch_time = io->time;
+	return pio;
+}
+
+static struct pending_io *hash_completed_io(struct blk_io_trace *io)
+{
+	struct pending_io *pio;
+
+	pio = io_hash_table_search(io->sector, io->device);
+
+	if (!pio)
+		return NULL;
+	return pio;
+}
+
+void init_process_hash_table(void)
+{
+	int i;
+	struct list_head *head;
+
+	for (i = 0; i < PROCESS_HASH_TABLE_SIZE; i++) {
+		head = process_hash_table + i;
+		INIT_LIST_HEAD(head);
+	}
+}
+
+static u32 hash_pid(u32 pid)
+{
+	u32 hash = pid;
+
+	hash ^= pid >> 3;
+	hash ^= pid >> 3;
+	hash ^= pid >> 4;
+	hash ^= pid >> 6;
+	return (hash & (PROCESS_HASH_TABLE_SIZE - 1));
+}
+
+static struct pid_map *process_hash_search(u32 pid)
+{
+	int slot = hash_pid(pid);
+	struct list_head *head;
+	struct pid_map *pm;
+
+	head = process_hash_table + slot;
+	list_for_each_entry(pm, head, hash_list) {
+		if (pm->pid == pid)
+			return pm;
+	}
+	return NULL;
+}
+
+static struct pid_map *process_hash_insert(u32 pid, char *name)
+{
+	int slot = hash_pid(pid);
+	struct pid_map *pm;
+	int old_index = 0;
+	char buf[16];
+
+	pm = process_hash_search(pid);
+	if (pm) {
+		/* Entry exists and name shouldn't be changed? */
+		if (!name || !strcmp(name, pm->name))
+			return pm;
+		list_del(&pm->hash_list);
+		old_index = pm->index;
+		free(pm);
+	}
+	if (!name) {
+		sprintf(buf, "[%u]", pid);
+		name = buf;
+	}
+	pm = malloc(sizeof(struct pid_map) + strlen(name) + 1);
+	pm->pid = pid;
+	pm->index = old_index;
+	strcpy(pm->name, name);
+	list_add_tail(&pm->hash_list, process_hash_table + slot);
+
+	return pm;
+}
+
+static void handle_notify(struct trace *trace)
+{
+	struct blk_io_trace *io = trace->io;
+	void *payload = (char *)io + sizeof(*io);
+	u32 two32[2];
+
+	if (io->action == BLK_TN_PROCESS) {
+		if (io_per_process)
+			process_hash_insert(io->pid, payload);
+		return;
+	}
+
+	if (io->action != BLK_TN_TIMESTAMP)
+		return;
+
+	if (io->pdu_len != sizeof(two32))
+		return;
+
+	memcpy(two32, payload, sizeof(two32));
+	trace->start_timestamp = io->time;
+	trace->abs_start_time.tv_sec = two32[0];
+	trace->abs_start_time.tv_nsec = two32[1];
+	if (trace->abs_start_time.tv_nsec < 0) {
+		trace->abs_start_time.tv_sec--;
+		trace->abs_start_time.tv_nsec += 1000000000;
+	}
+}
+
+int next_record(struct trace *trace)
+{
+	int skip = trace->io->pdu_len;
+	u64 offset;
+
+	trace->cur += sizeof(*trace->io) + skip;
+	offset = trace->cur - trace->start;
+	if (offset >= trace->len)
+		return 1;
+
+	trace->io = (struct blk_io_trace *)trace->cur;
+	return 0;
+}
+
+void first_record(struct trace *trace)
+{
+	trace->cur = trace->start;
+	trace->io = (struct blk_io_trace *)trace->cur;
+}
+
+static int is_io_event(struct blk_io_trace *test)
+{
+	char *message;
+	if (!(test->action & BLK_TC_ACT(BLK_TC_NOTIFY)))
+		return 1;
+	if (test->action == BLK_TN_MESSAGE) {
+		int len = test->pdu_len;
+		if (len < 3)
+			return 0;
+		message = (char *)(test + 1);
+		if (strncmp(message, "fio ", 4) == 0) {
+			return 1;
+		}
+	}
+	return 0;
+}
+
+u64 find_last_time(struct trace *trace)
+{
+	char *p = trace->start + trace->len;
+	struct blk_io_trace *test;
+	int search_len = 0;
+	u64 found = 0;
+
+	if (trace->len < sizeof(*trace->io))
+		return 0;
+	p -= sizeof(*trace->io);
+	while (p >= trace->start) {
+		test = (struct blk_io_trace *)p;
+		if (CHECK_MAGIC(test) && is_io_event(test)) {
+			u64 offset = p - trace->start;
+			if (offset + sizeof(*test) + test->pdu_len == trace->len) {
+				return test->time;
+			}
+		}
+		p--;
+		search_len++;
+		if (search_len > 8192) {
+			break;
+		}
+	}
+
+	/* searching backwards didn't work out, we'll have to scan the file */
+	first_record(trace);
+	while (1) {
+		if (is_io_event(trace->io))
+			found = trace->io->time;
+		if (next_record(trace))
+			break;
+	}
+	first_record(trace);
+	return found;
+}
+
+static int parse_fio_bank_message(struct trace *trace, u64 *bank_ret, u64 *offset_ret,
+			   u64 *num_banks_ret)
+{
+	char *s;
+	char *next;
+	char *message;
+	struct blk_io_trace *test = trace->io;
+	int len = test->pdu_len;
+	u64 bank;
+	u64 offset;
+	u64 num_banks;
+
+	if (!(test->action & BLK_TC_ACT(BLK_TC_NOTIFY)))
+		return -1;
+	if (test->action != BLK_TN_MESSAGE)
+		return -1;
+
+	/* the message is fio rw bank offset num_banks */
+	if (len < 3)
+		return -1;
+	message = (char *)(test + 1);
+	if (strncmp(message, "fio r ", 6) != 0)
+		return -1;
+
+	message = strndup(message, len);
+	s = strchr(message, ' ');
+	if (!s)
+		goto out;
+	s++;
+	s = strchr(s, ' ');
+	if (!s)
+		goto out;
+
+	bank = strtoll(s, &next, 10);
+	if (s == next)
+		goto out;
+	s = next;
+
+	offset = strtoll(s, &next, 10);
+	if (s == next)
+		goto out;
+	s = next;
+
+	num_banks = strtoll(s, &next, 10);
+	if (s == next)
+		goto out;
+
+	*bank_ret = bank;
+	*offset_ret = offset;
+	*num_banks_ret = num_banks;
+
+	return 0;
+out:
+	free(message);
+	return -1;
+}
+
+static struct dev_info *lookup_dev(struct trace *trace, struct blk_io_trace *io)
+{
+	u32 dev = io->device;
+	int i;
+	struct dev_info *di = NULL;
+
+	for (i = 0; i < trace->num_devices; i++) {
+		if (trace->devices[i].device == dev) {
+			di = trace->devices + i;
+			goto found;
+		}
+	}
+	i = trace->num_devices++;
+	if (i >= MAX_DEVICES_PER_TRACE) {
+		fprintf(stderr, "Trace contains too many devices (%d)\n", i);
+		exit(1);
+	}
+	di = trace->devices + i;
+	di->device = dev;
+found:
+	return di;
+}
+
+static void map_devices(struct trace *trace)
+{
+	struct dev_info *di;
+	u64 found;
+	u64 map_start = 0;
+	int i;
+
+	first_record(trace);
+	while (1) {
+		if (!(trace->io->action & BLK_TC_ACT(BLK_TC_NOTIFY))) {
+			di = lookup_dev(trace, trace->io);
+			found = trace->io->sector << 9;
+			if (found < di->min)
+				di->min = found;
+
+			found += trace->io->bytes;
+			if (di->max < found)
+				di->max = found;
+		}
+		if (next_record(trace))
+			break;
+	}
+	first_record(trace);
+	for (i = 0; i < trace->num_devices; i++) {
+		di = trace->devices + i;
+		di->map = map_start;
+		map_start += di->max - di->min;
+	}
+}
+
+static u64 map_io(struct trace *trace, struct blk_io_trace *io)
+{
+	struct dev_info *di = lookup_dev(trace, io);
+	u64 val = trace->io->sector << 9;
+	return di->map + val - di->min;
+}
+
+void find_extreme_offsets(struct trace *trace, u64 *min_ret, u64 *max_ret, u64 *max_bank_ret,
+			  u64 *max_offset_ret)
+{
+	u64 found = 0;
+	u64 max = 0, min = ~(u64)0;
+	u64 max_bank = 0;
+	u64 max_bank_offset = 0;
+	u64 num_banks = 0;
+
+	map_devices(trace);
+
+	first_record(trace);
+	while (1) {
+		if (!(trace->io->action & BLK_TC_ACT(BLK_TC_NOTIFY))) {
+			found = map_io(trace, trace->io);
+			if (found < min)
+				min = found;
+
+			found += trace->io->bytes;
+			if (max < found)
+				max = found;
+		} else {
+			u64 bank;
+			u64 offset;
+			if (!parse_fio_bank_message(trace, &bank,
+						    &offset, &num_banks)) {
+				if (bank > max_bank)
+					max_bank = bank;
+				if (offset > max_bank_offset)
+					max_bank_offset = offset;
+			}
+		}
+		if (next_record(trace))
+			break;
+	}
+	first_record(trace);
+	*min_ret = min;
+	*max_ret = max;
+	*max_bank_ret = max_bank;
+	*max_offset_ret = max_bank_offset;
+}
+
+static void check_io_types(struct trace *trace)
+{
+	struct blk_io_trace *io = trace->io;
+	int action = io->action & BLK_TA_MASK;
+
+	if (!(io->action & BLK_TC_ACT(BLK_TC_NOTIFY))) {
+		switch (action) {
+		case __BLK_TA_COMPLETE:
+			trace->found_completion = 1;
+			break;
+		case __BLK_TA_ISSUE:
+			trace->found_issue = 1;
+			break;
+		case __BLK_TA_QUEUE:
+			trace->found_queue = 1;
+			break;
+		};
+	}
+}
+
+
+int filter_outliers(struct trace *trace, u64 min_offset, u64 max_offset,
+		    u64 *yzoom_min, u64 *yzoom_max)
+{
+	int hits[11];
+	u64 max_per_bucket[11];
+	u64 min_per_bucket[11];
+	u64 bytes_per_bucket = (max_offset - min_offset + 1) / 10;
+	int slot;
+	int fat_count = 0;
+
+	memset(hits, 0, sizeof(int) * 11);
+	memset(max_per_bucket, 0, sizeof(u64) * 11);
+	memset(min_per_bucket, 0xff, sizeof(u64) * 11);
+	first_record(trace);
+	while (1) {
+		check_io_types(trace);
+		if (!(trace->io->action & BLK_TC_ACT(BLK_TC_NOTIFY)) &&
+		    (trace->io->action & BLK_TA_MASK) == __BLK_TA_QUEUE) {
+			u64 off = map_io(trace, trace->io) - min_offset;
+
+			slot = (int)(off / bytes_per_bucket);
+			hits[slot]++;
+			if (off < min_per_bucket[slot])
+				min_per_bucket[slot] = off;
+
+			off += trace->io->bytes;
+			slot = (int)(off / bytes_per_bucket);
+			hits[slot]++;
+			if (off > max_per_bucket[slot])
+				max_per_bucket[slot] = off;
+		}
+		if (next_record(trace))
+			break;
+	}
+	first_record(trace);
+	for (slot = 0; slot < 11; slot++) {
+		if (hits[slot] > fat_count) {
+			fat_count = hits[slot];
+		}
+	}
+
+	*yzoom_max = max_offset;
+	for (slot = 10; slot >= 0; slot--) {
+		double d = hits[slot];
+
+		if (d >= (double)fat_count * .05) {
+			*yzoom_max = max_per_bucket[slot] + min_offset;
+			break;
+		}
+	}
+
+	*yzoom_min = min_offset;
+	for (slot = 0; slot < 10; slot++) {
+		double d = hits[slot];
+
+		if (d >= (double)fat_count * .05) {
+			*yzoom_min = min_per_bucket[slot] + min_offset;
+			break;
+		}
+	}
+	return 0;
+}
+
+static char footer[] = ".blktrace.0";
+static int footer_len = sizeof(footer) - 1;
+
+static int match_trace(char *name, int *len)
+{
+	int match_len;
+	int footer_start;
+
+	match_len = strlen(name);
+	if (match_len <= footer_len)
+		return 0;
+
+	footer_start = match_len - footer_len;
+	if (strcmp(name + footer_start, footer) != 0)
+		return 0;
+
+	if (len)
+		*len = match_len;
+	return 1;
+}
+
+struct tracelist {
+	struct tracelist *next;
+	char *name;
+};
+
+static struct tracelist *traces_list(char *dir_name, int *len)
+{
+	int count = 0;
+	struct tracelist *traces = NULL;
+	int dlen = strlen(dir_name);
+	DIR *dir = opendir(dir_name);
+	if (!dir)
+		return NULL;
+
+	while (1) {
+		int n = 0;
+		struct tracelist *tl;
+		struct dirent *d = readdir(dir);
+		if (!d)
+			break;
+
+		if (!match_trace(d->d_name, &n))
+			continue;
+
+		n += dlen + 1; /* dir + '/' + file */
+		/* Allocate space for tracelist + filename */
+		tl = calloc(1, sizeof(struct tracelist) + (sizeof(char) * (n + 1)));
+		if (!tl) {
+			closedir(dir);
+			return NULL;
+		}
+		tl->next = traces;
+		tl->name = (char *)(tl + 1);
+		snprintf(tl->name, n, "%s/%s", dir_name, d->d_name);
+		traces = tl;
+		count++;
+	}
+
+	closedir(dir);
+
+	if (len)
+		*len = count;
+
+	return traces;
+}
+
+static void traces_free(struct tracelist *traces)
+{
+	while (traces) {
+		struct tracelist *tl = traces;
+		traces = traces->next;
+		free(tl);
+	}
+}
+
+static int dump_traces(struct tracelist *traces, int count, char *dumpfile)
+{
+	struct tracelist *tl;
+	char **argv = NULL;
+	int argc = 0;
+	int i;
+	int err = 0;
+
+	argc = count * 2; /* {"-i", trace } */
+	argc += 4; /* See below */
+	argv = calloc(argc + 1, sizeof(char *));
+	if (!argv)
+		return -errno;
+
+	i = 0;
+	argv[i++] = "blkparse";
+	argv[i++] = "-O";
+	argv[i++] = "-d";
+	argv[i++] = dumpfile;
+	for (tl = traces; tl != NULL; tl = tl->next) {
+		argv[i++] = "-i";
+		argv[i++] = tl->name;
+	}
+
+	err = run_program(argc, argv, 1, NULL, NULL);
+	if (err)
+		fprintf(stderr, "%s exited with %d, expected 0\n", argv[0], err);
+	free(argv);
+	return err;
+}
+
+static char *find_trace_file(char *filename)
+{
+	int ret;
+	struct stat st;
+	char *dot;
+	int found_dir = 0;
+	char *dumpfile;
+	int len = strlen(filename);
+
+	/* look for an exact match of whatever they pass in.
+	 * If it is a file, assume it is the dump file.
+	 * If a directory, remember that it existed so we
+	 * can combine traces in that directory later
+	 */
+	ret = stat(filename, &st);
+	if (ret == 0) {
+		if (S_ISREG(st.st_mode))
+			return strdup(filename);
+
+		if (S_ISDIR(st.st_mode))
+			found_dir = 1;
+	}
+
+	if (found_dir) {
+		int i;
+		/* Eat up trailing '/'s */
+		for (i = len - 1; filename[i] == '/'; i--)
+			filename[i] = '\0';
+	}
+
+	/*
+	 * try tacking .dump onto the end and see if that already
+	 * has been generated
+	 */
+	ret = asprintf(&dumpfile, "%s.dump", filename);
+	if (ret == -1) {
+		perror("Error building dump file name");
+		return NULL;
+	}
+	ret = stat(dumpfile, &st);
+	if (ret == 0)
+		return dumpfile;
+
+	/*
+	 * try to generate the .dump from all the traces in
+	 * a single dir.
+	 */
+	if (found_dir) {
+		int count;
+		struct tracelist *traces = traces_list(filename, &count);
+		if (traces) {
+			ret = dump_traces(traces, count, dumpfile);
+			traces_free(traces);
+			if (ret == 0)
+				return dumpfile;
+		}
+	}
+	free(dumpfile);
+
+	/*
+	 * try to generate the .dump from all the blktrace
+	 * files for a named trace
+	 */
+	dot = strrchr(filename, '.');
+	if (!dot || strcmp(".dump", dot) != 0) {
+		struct tracelist trace = {0 ,NULL};
+		if (dot && dot != filename)
+			len = dot - filename;
+
+		ret = asprintf(&trace.name, "%*s.blktrace.0", len, filename);
+		if (ret == -1)
+			return NULL;
+		ret = asprintf(&dumpfile, "%*s.dump", len, filename);
+		if (ret == -1) {
+			free(trace.name);
+			return NULL;
+		}
+
+		ret = dump_traces(&trace, 1, dumpfile);
+		if (ret == 0) {
+			free(trace.name);
+			return dumpfile;
+		}
+		free(trace.name);
+		free(dumpfile);
+	}
+	return NULL;
+}
+struct trace *open_trace(char *filename)
+{
+	int fd;
+	char *p;
+	struct stat st;
+	int ret;
+	struct trace *trace;
+	char *found_filename;
+
+	trace = calloc(1, sizeof(*trace));
+	if (!trace) {
+		fprintf(stderr, "unable to allocate memory for trace\n");
+		return NULL;
+	}
+
+	found_filename = find_trace_file(filename);
+	if (!found_filename) {
+		fprintf(stderr, "Unable to find trace file %s\n", filename);
+		goto fail;
+	}
+	filename = found_filename;
+
+	fd = open(filename, O_RDONLY);
+	if (fd < 0) {
+		fprintf(stderr, "Unable to open trace file %s err %s\n", filename, strerror(errno));
+		goto fail;
+	}
+	ret = fstat(fd, &st);
+	if (ret < 0) {
+		fprintf(stderr, "stat failed on %s err %s\n", filename, strerror(errno));
+		goto fail_fd;
+	}
+	p = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	if (p == MAP_FAILED) {
+		fprintf(stderr, "Unable to mmap trace file %s, err %s\n", filename, strerror(errno));
+		goto fail_fd;
+	}
+	trace->fd = fd;
+	trace->len = st.st_size;
+	trace->start = p;
+	trace->cur = p;
+	trace->io = (struct blk_io_trace *)p;
+	return trace;
+
+fail_fd:
+	close(fd);
+fail:
+	free(trace);
+	return NULL;
+}
+static inline int tput_event(struct trace *trace)
+{
+	if (trace->found_completion)
+		return __BLK_TA_COMPLETE;
+	if (trace->found_issue)
+		return __BLK_TA_ISSUE;
+	if (trace->found_queue)
+		return __BLK_TA_QUEUE;
+
+	return __BLK_TA_COMPLETE;
+}
+
+int action_char_to_num(char action)
+{
+	switch (action) {
+	case 'Q':
+		return __BLK_TA_QUEUE;
+	case 'D':
+		return __BLK_TA_ISSUE;
+	case 'C':
+		return __BLK_TA_COMPLETE;
+	}
+	return -1;
+}
+
+static inline int io_event(struct trace *trace)
+{
+	if (plot_io_action)
+		return plot_io_action;
+	if (trace->found_queue)
+		return __BLK_TA_QUEUE;
+	if (trace->found_issue)
+		return __BLK_TA_ISSUE;
+	if (trace->found_completion)
+		return __BLK_TA_COMPLETE;
+
+	return __BLK_TA_COMPLETE;
+}
+
+void add_tput(struct trace *trace, struct graph_line_data *writes_gld,
+	      struct graph_line_data *reads_gld)
+{
+	struct blk_io_trace *io = trace->io;
+	struct graph_line_data *gld;
+	int action = io->action & BLK_TA_MASK;
+	int seconds;
+
+	if (io->action & BLK_TC_ACT(BLK_TC_NOTIFY))
+		return;
+
+	if (action != tput_event(trace))
+		return;
+
+	if (BLK_DATADIR(io->action) & BLK_TC_READ)
+		gld = reads_gld;
+	else
+		gld = writes_gld;
+
+	seconds = SECONDS(io->time);
+	gld->data[seconds].sum += io->bytes;
+
+	gld->data[seconds].count = 1;
+	if (gld->data[seconds].sum > gld->max)
+		gld->max = gld->data[seconds].sum;
+}
+
+#define GDD_PTR_ALLOC_STEP 16
+
+static struct pid_map *get_pid_map(struct trace_file *tf, u32 pid)
+{
+	struct pid_map *pm;
+
+	if (!io_per_process) {
+		if (!tf->io_plots)
+			tf->io_plots = 1;
+		return NULL;
+	}
+
+	pm = process_hash_insert(pid, NULL);
+	/* New entry? */
+	if (!pm->index) {
+		if (tf->io_plots == tf->io_plots_allocated) {
+			tf->io_plots_allocated += GDD_PTR_ALLOC_STEP;
+			tf->gdd_reads = realloc(tf->gdd_reads, tf->io_plots_allocated * sizeof(struct graph_dot_data *));
+			if (!tf->gdd_reads)
+				abort();
+			tf->gdd_writes = realloc(tf->gdd_writes, tf->io_plots_allocated * sizeof(struct graph_dot_data *));
+			if (!tf->gdd_writes)
+				abort();
+			memset(tf->gdd_reads + tf->io_plots_allocated - GDD_PTR_ALLOC_STEP,
+			       0, GDD_PTR_ALLOC_STEP * sizeof(struct graph_dot_data *));
+			memset(tf->gdd_writes + tf->io_plots_allocated - GDD_PTR_ALLOC_STEP,
+			       0, GDD_PTR_ALLOC_STEP * sizeof(struct graph_dot_data *));
+		}
+		pm->index = tf->io_plots++;
+
+		return pm;
+	}
+	return pm;
+}
+
+void add_io(struct trace *trace, struct trace_file *tf)
+{
+	struct blk_io_trace *io = trace->io;
+	int action = io->action & BLK_TA_MASK;
+	u64 offset;
+	int index;
+	char *label;
+	struct pid_map *pm;
+
+	if (io->action & BLK_TC_ACT(BLK_TC_NOTIFY))
+		return;
+
+	if (action != io_event(trace))
+		return;
+
+	offset = map_io(trace, io);
+
+	pm = get_pid_map(tf, io->pid);
+	if (!pm) {
+		index = 0;
+		label = "";
+	} else {
+		index = pm->index;
+		label = pm->name;
+	}
+	if (BLK_DATADIR(io->action) & BLK_TC_READ) {
+		if (!tf->gdd_reads[index])
+			tf->gdd_reads[index] = alloc_dot_data(tf->min_seconds, tf->max_seconds, tf->min_offset, tf->max_offset, tf->stop_seconds, pick_color(), strdup(label));
+		set_gdd_bit(tf->gdd_reads[index], offset, io->bytes, io->time);
+	} else if (BLK_DATADIR(io->action) & BLK_TC_WRITE) {
+		if (!tf->gdd_writes[index])
+			tf->gdd_writes[index] = alloc_dot_data(tf->min_seconds, tf->max_seconds, tf->min_offset, tf->max_offset, tf->stop_seconds, pick_color(), strdup(label));
+		set_gdd_bit(tf->gdd_writes[index], offset, io->bytes, io->time);
+	}
+}
+
+void add_pending_io(struct trace *trace, struct graph_line_data *gld)
+{
+	unsigned int seconds;
+	struct blk_io_trace *io = trace->io;
+	int action = io->action & BLK_TA_MASK;
+	double avg;
+	struct pending_io *pio;
+
+	if (io->action & BLK_TC_ACT(BLK_TC_NOTIFY))
+		return;
+
+	if (action == __BLK_TA_QUEUE) {
+		if (trace->found_issue || trace->found_completion)
+			hash_queued_io(trace->io);
+		return;
+	}
+	if (action == __BLK_TA_REQUEUE) {
+		if (ios_in_flight > 0)
+			ios_in_flight--;
+		return;
+	}
+	if (action != __BLK_TA_ISSUE)
+		return;
+
+	pio = hash_dispatched_io(trace->io);
+	if (!pio)
+		return;
+
+	if (!trace->found_completion) {
+		list_del(&pio->hash_list);
+		free(pio);
+	}
+
+	ios_in_flight++;
+
+	seconds = SECONDS(io->time);
+	gld->data[seconds].sum += ios_in_flight;
+	gld->data[seconds].count++;
+
+	avg = (double)gld->data[seconds].sum / gld->data[seconds].count;
+	if (gld->max < (u64)avg) {
+		gld->max = avg;
+	}
+}
+
+void add_completed_io(struct trace *trace,
+		      struct graph_line_data *latency_gld)
+{
+	struct blk_io_trace *io = trace->io;
+	int seconds;
+	int action = io->action & BLK_TA_MASK;
+	struct pending_io *pio;
+	double avg;
+	u64 latency;
+
+	if (io->action & BLK_TC_ACT(BLK_TC_NOTIFY))
+		return;
+
+	if (action != __BLK_TA_COMPLETE)
+		return;
+
+	seconds = SECONDS(io->time);
+
+	pio = hash_completed_io(trace->io);
+	if (!pio)
+		return;
+
+	if (ios_in_flight > 0)
+		ios_in_flight--;
+	if (io->time >= pio->dispatch_time) {
+		latency = io->time - pio->dispatch_time;
+		latency_gld->data[seconds].sum += latency;
+		latency_gld->data[seconds].count++;
+	}
+
+	list_del(&pio->hash_list);
+	free(pio);
+
+	avg = (double)latency_gld->data[seconds].sum /
+		latency_gld->data[seconds].count;
+	if (latency_gld->max < (u64)avg) {
+		latency_gld->max = avg;
+	}
+}
+
+void add_iop(struct trace *trace, struct graph_line_data *gld)
+{
+	struct blk_io_trace *io = trace->io;
+	int action = io->action & BLK_TA_MASK;
+	int seconds;
+
+	if (io->action & BLK_TC_ACT(BLK_TC_NOTIFY))
+		return;
+
+	/* iops and tput use the same events */
+	if (action != tput_event(trace))
+		return;
+
+	seconds = SECONDS(io->time);
+	gld->data[seconds].sum += 1;
+	gld->data[seconds].count = 1;
+	if (gld->data[seconds].sum > gld->max)
+		gld->max = gld->data[seconds].sum;
+}
+
+void check_record(struct trace *trace)
+{
+	handle_notify(trace);
+}
diff --git a/iowatcher/blkparse.h b/iowatcher/blkparse.h
new file mode 100644
index 0000000..fce9d01
--- /dev/null
+++ b/iowatcher/blkparse.h
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2012 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ *  Parts of this file were imported from Jens Axboe's blktrace sources (also GPL)
+ */
+#ifndef __IOWATCH_BLKPARSE__
+#define __IOWATCH_BLKPARSE__
+#define MINORBITS 20
+#define MINORMASK ((1 << MINORBITS) - 1)
+#define SECONDS(x)              ((unsigned long long)(x) / 1000000000)
+#define NANO_SECONDS(x)         ((unsigned long long)(x) % 1000000000)
+#define DOUBLE_TO_NANO_ULL(d)   ((unsigned long long)((d) * 1000000000))
+#define CHECK_MAGIC(t)          (((t)->magic & 0xffffff00) == BLK_IO_TRACE_MAGIC)
+
+struct dev_info {
+	u32 device;
+	u64 min;
+	u64 max;
+	u64 map;
+};
+
+#define MAX_DEVICES_PER_TRACE 64
+
+struct trace {
+	int fd;
+	u64 len;
+	char *start;
+	char *cur;
+	struct blk_io_trace *io;
+	u64 start_timestamp;
+	struct timespec abs_start_time;
+
+	/*
+	 * flags for the things we find in the stream
+	 * we prefer different events for different things
+	 */
+	int found_issue;
+	int found_completion;
+	int found_queue;
+
+	char *mpstat_start;
+	char *mpstat_cur;
+	u64 mpstat_len;
+	int mpstat_fd;
+	int mpstat_seconds;
+	int mpstat_num_cpus;
+
+	char *fio_start;
+	char *fio_cur;
+	u64 fio_len;
+	int fio_fd;
+	int fio_seconds;
+	int num_devices;
+	struct dev_info devices[MAX_DEVICES_PER_TRACE];
+};
+
+struct trace_file {
+	struct list_head list;
+	char *filename;
+	char *label;
+	struct trace *trace;
+	unsigned int stop_seconds;	/* Time when trace stops */
+	unsigned int min_seconds;	/* Beginning of the interval we should plot */
+	unsigned int max_seconds;	/* End of the interval we should plot */
+	u64 min_offset;
+	u64 max_offset;
+
+	char *reads_color;
+	char *writes_color;
+	char *line_color;
+
+	struct graph_line_data *tput_writes_gld;
+	struct graph_line_data *tput_reads_gld;
+	struct graph_line_data *iop_gld;
+	struct graph_line_data *latency_gld;
+	struct graph_line_data *queue_depth_gld;
+
+	int fio_trace;
+	struct graph_line_data *fio_gld;
+
+	/* Number of entries in gdd_writes / gdd_reads */
+	int io_plots;
+
+	/* Allocated array size for gdd_writes / gdd_reads */
+	int io_plots_allocated;
+	struct graph_dot_data **gdd_writes;
+	struct graph_dot_data **gdd_reads;
+
+	unsigned int mpstat_min_seconds;
+	unsigned int mpstat_max_seconds;
+	unsigned int mpstat_stop_seconds;
+	struct graph_line_data **mpstat_gld;
+};
+
+static inline unsigned int MAJOR(unsigned int dev)
+{
+	return dev >> MINORBITS;
+}
+
+static inline unsigned int MINOR(unsigned int dev)
+{
+	return dev & MINORMASK;
+}
+
+void init_io_hash_table(void);
+void init_process_hash_table(void);
+struct trace *open_trace(char *filename);
+u64 find_last_time(struct trace *trace);
+void find_extreme_offsets(struct trace *trace, u64 *min_ret, u64 *max_ret,
+			  u64 *max_bank_ret, u64 *max_offset_ret);
+int filter_outliers(struct trace *trace, u64 min_offset, u64 max_offset,
+		    u64 *yzoom_min, u64 *yzoom_max);
+int action_char_to_num(char action);
+void add_iop(struct trace *trace, struct graph_line_data *gld);
+void check_record(struct trace *trace);
+void add_completed_io(struct trace *trace,
+		      struct graph_line_data *latency_gld);
+void add_io(struct trace *trace, struct trace_file *tf);
+void add_tput(struct trace *trace, struct graph_line_data *writes_gld,
+	      struct graph_line_data *reads_gld);
+void add_pending_io(struct trace *trace, struct graph_line_data *gld);
+int next_record(struct trace *trace);
+u64 get_record_time(struct trace *trace);
+void first_record(struct trace *trace);
+#endif
diff --git a/iowatcher/fio.c b/iowatcher/fio.c
new file mode 100644
index 0000000..b680f66
--- /dev/null
+++ b/iowatcher/fio.c
@@ -0,0 +1,217 @@
+/*
+ * Copyright (C) 2013 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+#include <inttypes.h>
+#include <string.h>
+#include <asm/types.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <math.h>
+
+#include "plot.h"
+#include "blkparse.h"
+#include "list.h"
+#include "tracers.h"
+#include "fio.h"
+
+static int past_eof(struct trace *trace, char *cur)
+{
+	if (cur >= trace->fio_start + trace->fio_len)
+		return 1;
+	return 0;
+}
+
+static int parse_fio_line(struct trace *trace, int *time, int *rate, int *dir, int *bs)
+{
+	char *cur = trace->fio_cur;
+	char *p;
+	int *res[] = { time, rate, dir, bs, NULL };
+	int val;
+	int i = 0;
+	int *t;
+	char *end = index(cur, '\n');
+	char *tmp;
+
+	if (!end)
+		return 1;
+
+	tmp = strndup(cur, end - cur);
+	if (!tmp)
+		return 1;
+	p = strtok(tmp, ",");
+	while (p && *res) {
+		val = atoi(p);
+		t = res[i++];
+		*t = val;
+		p = strtok(NULL, ",");
+	}
+
+	free(tmp);
+
+	if (i < 3)
+		return 1;
+	return 0;
+}
+
+int next_fio_line(struct trace *trace)
+{
+	char *next;
+	char *cur = trace->fio_cur;
+
+	next = strchr(cur, '\n');
+	if (!next)
+		return 1;
+	next++;
+	if (past_eof(trace, next))
+		return 1;
+	trace->fio_cur = next;
+	return 0;
+}
+
+char *first_fio(struct trace *trace)
+{
+	trace->fio_cur = trace->fio_start;
+	return trace->fio_cur;
+}
+
+static void find_last_fio_time(struct trace *trace)
+{
+	double d;
+	int time, rate, dir, bs;
+	int ret;
+	int last_time = 0;
+
+	if (trace->fio_len == 0)
+		return;
+
+	first_fio(trace);
+	while (1) {
+		ret = parse_fio_line(trace, &time, &rate, &dir, &bs);
+		if (ret)
+			break;
+		if (dir <= 1 && time > last_time)
+			last_time = time;
+		ret = next_fio_line(trace);
+		if (ret)
+			break;
+	}
+	d = (double)time / 1000;
+	trace->fio_seconds = ceil(d);
+	return;
+}
+
+static int read_fio(struct trace *trace, char *trace_name)
+{
+	int fd;
+	struct stat st;
+	int ret;
+	char *p;
+
+	fd = open(trace_name, O_RDONLY);
+	if (fd < 0)
+		return 1;
+
+	ret = fstat(fd, &st);
+	if (ret < 0) {
+		fprintf(stderr, "stat failed on %s err %s\n",
+			trace_name, strerror(errno));
+		goto fail_fd;
+	}
+	p = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	if (p == MAP_FAILED) {
+		fprintf(stderr, "Unable to mmap trace file %s, err %s\n",
+			trace_name, strerror(errno));
+		goto fail_fd;
+	}
+	trace->fio_start = p;
+	trace->fio_len = st.st_size;
+	trace->fio_cur = p;
+	trace->fio_fd = fd;
+	find_last_fio_time(trace);
+	first_fio(trace);
+	return 0;
+
+fail_fd:
+	close(fd);
+	return 1;
+}
+
+struct trace *open_fio_trace(char *path)
+{
+	int ret;
+	struct trace *trace;
+
+	trace = calloc(1, sizeof(*trace));
+	if (!trace) {
+		fprintf(stderr, "unable to allocate memory for trace\n");
+		exit(1);
+	}
+
+	ret = read_fio(trace, path);
+	if (ret) {
+		free(trace);
+		return NULL;
+	}
+
+	return trace;
+}
+
+int read_fio_event(struct trace *trace, int *time_ret, u64 *bw_ret, int *dir_ret)
+{
+	char *cur = trace->fio_cur;
+	int time, rate, dir, bs;
+	int ret;
+
+	if (past_eof(trace, cur))
+		return 1;
+
+	ret = parse_fio_line(trace, &time, &rate, &dir, &bs);
+	if (ret)
+		return 1;
+
+	time = floor((double)time / 1000);
+	*time_ret = time;
+	*bw_ret = (u64)rate * 1024;
+
+	*dir_ret = dir;
+	return 0;
+}
+
+int add_fio_gld(unsigned int time, u64 bw, struct graph_line_data *gld)
+{
+	double val;
+
+	if (time > gld->max_seconds)
+		return 0;
+
+	gld->data[time].sum += bw;
+	gld->data[time].count++;
+
+	val = ((double)gld->data[time].sum) / gld->data[time].count;
+
+	if (val > gld->max)
+		gld->max = ceil(val);
+	return 0;
+
+}
diff --git a/iowatcher/fio.h b/iowatcher/fio.h
new file mode 100644
index 0000000..31ee473
--- /dev/null
+++ b/iowatcher/fio.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2013 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef __FIO__
+#define __FIO__
+
+int read_fio_event(struct trace *trace, int *time, u64 *bw, int *dir);
+int add_fio_gld(unsigned int time, u64 bw, struct graph_line_data *gld);
+int next_fio_line(struct trace *trace);
+struct trace *open_fio_trace(char *path);
+char *first_fio(struct trace *trace);
+
+#endif
diff --git a/iowatcher/iowatcher.1 b/iowatcher/iowatcher.1
new file mode 100644
index 0000000..3044abf
--- /dev/null
+++ b/iowatcher/iowatcher.1
@@ -0,0 +1,162 @@
+.TH iowatcher "1" "April 2014" "iowatcher" "User Commands"
+
+.SH NAME
+iowatcher - Create visualizations from blktrace results
+
+.SH SYNOPSIS
+.B iowatcher
+\fI[options]\fR [--] \fI[program arguments ...]\fR
+
+.SH DESCRIPTION
+iowatcher graphs the results of a blktrace run.  It can graph the result of an existing blktrace, start a new blktrace, or start a new blktrace and a benchmark run.  It can then create an image or movie of the IO from a given trace.  iowatcher can produce either SVG files or movies in mp4 format (with ffmpeg) or ogg format (with png2theora).
+
+.SH OPTIONS
+.TP
+\fB--help\fP
+Print a brief usage summary.
+.TP
+\fB-d, --device\fP \fIdevice\fP
+Controls which device you are tracing.  You can only trace one device at a time for now.  It is sent directly to blktrace, and only needed when you are making a new trace.
+.TP
+\fB-D, --blktrace-destination\fP \fIdestination\fP
+Destination for blktrace.
+.TP
+\fB-p, --prog\fP
+Run a program while blktrace is run. The program and its arguments must be
+specified after all other options.  Note that this option previously required
+the program to be given as a single argument but it now tells \fBiowatcher\fP
+to expect extra arguments which it should be run during the trace.
+.TP
+\fB--\fP
+End option parsing. If \fB--prog\fP is specified, everything after \fB--\fP is
+the program to be run. This can be useful if the program name could otherwise
+be mistaken for an option.
+.TP
+\fB-K, --keep-movie-svgs\fP
+Keep the SVG files generated for movie mode.
+.TP
+\fB-t, --trace\fP \fIpath\fP
+Specify the name of the file or directory in which blktrace output is located.
+\fBiowatcher\fP uses a dump from blkparse, so this option tries to guess the
+name of the corresponding per-CPU blktrace data files if the dump file doesn't
+already exist.  To add multiple traces to a given graph, you can specify
+\fB--trace\fP more than once.  If \fIpath\fP is a directory, \fBiowatcher\fP
+will use the name of the directory as the base name of the dump file and all
+trace files found inside the directory will be processed.
+.TP
+\fB-l, --label\fP \fIlabel\fP
+Sets a label in the graph for a trace file.  The labels are added in the same order as the trace files.
+.TP
+\fB-m, --movie\fP \fI[style]\fP
+Create a movie.  The file format depends on the extension used in the \fB-o\fP
+\fIfile\fP option.  If you specify an .ogv or .ogg extension, the result will
+be Ogg Theora video, if png2theora is available.  If you use an .mp4 extension,
+the result will be an mp4 video if ffmpeg is available.  You can use any other
+extension, but the end result will be an mp4.  The accepted \fIstyle\fP values
+are \fBspindle\fP for a circular disc-like effect (default) or \fBrect\fP for a
+rectangular graph style.
+.TP
+\fB-T, --title\fP \fItitle\fP
+Set a \fItitle\fP to be placed at the top of the graph.
+.TP
+\fB-o, --output\fP \fIfile\fP
+Output filename for the SVG image or video. The video format used will depend
+on the file name extension. See \fB--movie\fP for details.
+.TP
+\fB-r, --rolling\fP \fIseconds\fP
+Control the duration for the rolling average.  \fBiowatcher\fP tries to smooth out bumpy graphs by averaging the current second with seconds from the past.  Larger numbers here give you flatter graphs.
+.TP
+\fB-h, --height\fP \fIheight\fP
+Set the height of each graph
+.TP
+\fB-w, --width\fP \fIwidth\fP
+Set the width of each graph
+.TP
+\fB-c, --columns\fP \fIcolumns\fP
+Number of columns in graph output
+.TP
+\fB-x, --xzoom\fP \fImin:max\fP
+Limit processed time range to \fImin:max\fP.
+.TP
+\fB-y, --yzoom\fP \fImin:max\fP
+Limit processed sectors to \fImin:max\fP.
+.TP
+\fB-a, --io-plot-action\fP \fIaction\fP
+Plot \fIaction\fP (one of Q, D, or C) in the IO graph.
+.TP
+\fB-P, --per-process-io\fP
+Distinguish between processes in the IO graph.
+.TP
+\fB-O, --only-graph\fP \fIgraph\fP
+Add a single graph to the output (see section \fBGRAPHS\fP for options).  By
+default all graphs are included. Use \fB-O\fP to generate only the required
+graphs.  \fB-O\fP may be used more than once.
+.TP
+\fB-N, --no-graph\fP \fItype\fP
+Remove a single graph from the output (see section \fBGRAPHS\fP for options).
+This option may be used more than once.
+.SH GRAPHS
+Values accepted by the \fB-O\fP and \fB-N\fP options are:
+
+   io, tput, latency, queue_depth, iops, cpu-sys, cpu-io, cpu-irq, cpu-user, cpu-soft
+
+.SH EXAMPLES
+Generate graph from the existing trace.dump:
+.PP
+.RS
+# iowatcher -t trace
+.RE
+.PP
+Skip the IO graph:
+.PP
+.RS
+# iowatcher -t trace.dump -o trace.svg -N io
+.RE
+.PP
+Only graph tput and latency:
+.PP
+.RS
+# iowatcher -t trace.dump -o trace.svg -O tput -O latency
+.RE
+.PP
+Generate a graph from two runs, and label them:
+.PP
+.RS
+# iowatcher -t ext4.dump -t xfs.dump -l Ext4 -l XFS -o trace.svg
+.RE
+.PP
+Run a fio benchmark and store the trace in trace.dump, add a title to the top, use /dev/sda for blktrace:
+.PP
+.RS
+# iowatcher -d /dev/sda -t trace.dump -T 'Fio Benchmark' -p fio some_job_file
+.RE
+.PP
+Make a movie from an existing trace:
+.PP
+.RS
+# iowatcher -t trace --movie -o trace.mp4
+.RE
+
+.SH AUTHORS
+iowatcher was created and is maintained by Chris Mason.
+
+This man page was largely written by Andrew Price based on Chris's original README.
+
+.SH COPYRIGHT
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License v2 as published by the Free
+Software Foundation.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
+Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+.SH "SEE ALSO"
+.BR blktrace (8),
+.BR blkparse (1),
+.BR fio (1),
+.BR mpstat (1)
diff --git a/iowatcher/list.h b/iowatcher/list.h
new file mode 100644
index 0000000..52f980f
--- /dev/null
+++ b/iowatcher/list.h
@@ -0,0 +1,449 @@
+/*
+ * Copyright (C) 2012 Fusion-io.  All rights reserved.
+ *
+ * This header was taken from the Linux kernel
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _LINUX_LIST_H
+#define _LINUX_LIST_H
+
+#define LIST_POISON1  ((void *) 0x00100100)
+#define LIST_POISON2  ((void *) 0x00200200)
+
+#undef offsetof
+#ifdef __compiler_offsetof
+#define offsetof(TYPE,MEMBER) __compiler_offsetof(TYPE,MEMBER)
+#else
+#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#endif
+
+#define container_of(ptr, type, member) ({                      \
+	 const typeof( ((type *)0)->member ) *__mptr = (ptr);    \
+	(type *)( (char *)__mptr - offsetof(type,member) );})
+
+/*
+ * Simple doubly linked list implementation.
+ *
+ * Some of the internal functions ("__xxx") are useful when
+ * manipulating whole lists rather than single entries, as
+ * sometimes we already know the next/prev entries and we can
+ * generate better code by using them directly rather than
+ * using the generic single-entry routines.
+ */
+
+struct list_head {
+	struct list_head *next, *prev;
+};
+
+#define LIST_HEAD_INIT(name) { &(name), &(name) }
+
+#define LIST_HEAD(name) \
+	struct list_head name = LIST_HEAD_INIT(name)
+
+static inline void INIT_LIST_HEAD(struct list_head *list)
+{
+	list->next = list;
+	list->prev = list;
+}
+
+/*
+ * Insert a new entry between two known consecutive entries.
+ *
+ * This is only for internal list manipulation where we know
+ * the prev/next entries already!
+ */
+#ifndef CONFIG_DEBUG_LIST
+static inline void __list_add(struct list_head *new,
+			      struct list_head *prev,
+			      struct list_head *next)
+{
+	next->prev = new;
+	new->next = next;
+	new->prev = prev;
+	prev->next = new;
+}
+#else
+extern void __list_add(struct list_head *new,
+			      struct list_head *prev,
+			      struct list_head *next);
+#endif
+
+/**
+ * list_add - add a new entry
+ * @new: new entry to be added
+ * @head: list head to add it after
+ *
+ * Insert a new entry after the specified head.
+ * This is good for implementing stacks.
+ */
+#ifndef CONFIG_DEBUG_LIST
+static inline void list_add(struct list_head *new, struct list_head *head)
+{
+	__list_add(new, head, head->next);
+}
+#else
+extern void list_add(struct list_head *new, struct list_head *head);
+#endif
+
+
+/**
+ * list_add_tail - add a new entry
+ * @new: new entry to be added
+ * @head: list head to add it before
+ *
+ * Insert a new entry before the specified head.
+ * This is useful for implementing queues.
+ */
+static inline void list_add_tail(struct list_head *new, struct list_head *head)
+{
+	__list_add(new, head->prev, head);
+}
+
+/*
+ * Delete a list entry by making the prev/next entries
+ * point to each other.
+ *
+ * This is only for internal list manipulation where we know
+ * the prev/next entries already!
+ */
+static inline void __list_del(struct list_head * prev, struct list_head * next)
+{
+	next->prev = prev;
+	prev->next = next;
+}
+
+/**
+ * list_del - deletes entry from list.
+ * @entry: the element to delete from the list.
+ * Note: list_empty on entry does not return true after this, the entry is
+ * in an undefined state.
+ */
+#ifndef CONFIG_DEBUG_LIST
+static inline void list_del(struct list_head *entry)
+{
+	__list_del(entry->prev, entry->next);
+	entry->next = LIST_POISON1;
+	entry->prev = LIST_POISON2;
+}
+#else
+extern void list_del(struct list_head *entry);
+#endif
+
+/**
+ * list_replace - replace old entry by new one
+ * @old : the element to be replaced
+ * @new : the new element to insert
+ * Note: if 'old' was empty, it will be overwritten.
+ */
+static inline void list_replace(struct list_head *old,
+				struct list_head *new)
+{
+	new->next = old->next;
+	new->next->prev = new;
+	new->prev = old->prev;
+	new->prev->next = new;
+}
+
+static inline void list_replace_init(struct list_head *old,
+					struct list_head *new)
+{
+	list_replace(old, new);
+	INIT_LIST_HEAD(old);
+}
+/**
+ * list_del_init - deletes entry from list and reinitialize it.
+ * @entry: the element to delete from the list.
+ */
+static inline void list_del_init(struct list_head *entry)
+{
+	__list_del(entry->prev, entry->next);
+	INIT_LIST_HEAD(entry);
+}
+
+/**
+ * list_move - delete from one list and add as another's head
+ * @list: the entry to move
+ * @head: the head that will precede our entry
+ */
+static inline void list_move(struct list_head *list, struct list_head *head)
+{
+        __list_del(list->prev, list->next);
+        list_add(list, head);
+}
+
+/**
+ * list_move_tail - delete from one list and add as another's tail
+ * @list: the entry to move
+ * @head: the head that will follow our entry
+ */
+static inline void list_move_tail(struct list_head *list,
+				  struct list_head *head)
+{
+        __list_del(list->prev, list->next);
+        list_add_tail(list, head);
+}
+
+/**
+ * list_is_last - tests whether @list is the last entry in list @head
+ * @list: the entry to test
+ * @head: the head of the list
+ */
+static inline int list_is_last(const struct list_head *list,
+				const struct list_head *head)
+{
+	return list->next == head;
+}
+
+/**
+ * list_empty - tests whether a list is empty
+ * @head: the list to test.
+ */
+static inline int list_empty(const struct list_head *head)
+{
+	return head->next == head;
+}
+
+/**
+ * list_empty_careful - tests whether a list is empty and not being modified
+ * @head: the list to test
+ *
+ * Description:
+ * tests whether a list is empty _and_ checks that no other CPU might be
+ * in the process of modifying either member (next or prev)
+ *
+ * NOTE: using list_empty_careful() without synchronization
+ * can only be safe if the only activity that can happen
+ * to the list entry is list_del_init(). Eg. it cannot be used
+ * if another CPU could re-list_add() it.
+ */
+static inline int list_empty_careful(const struct list_head *head)
+{
+	struct list_head *next = head->next;
+	return (next == head) && (next == head->prev);
+}
+
+static inline void __list_splice(struct list_head *list,
+				 struct list_head *head)
+{
+	struct list_head *first = list->next;
+	struct list_head *last = list->prev;
+	struct list_head *at = head->next;
+
+	first->prev = head;
+	head->next = first;
+
+	last->next = at;
+	at->prev = last;
+}
+
+/**
+ * list_splice - join two lists
+ * @list: the new list to add.
+ * @head: the place to add it in the first list.
+ */
+static inline void list_splice(struct list_head *list, struct list_head *head)
+{
+	if (!list_empty(list))
+		__list_splice(list, head);
+}
+
+/**
+ * list_splice_init - join two lists and reinitialise the emptied list.
+ * @list: the new list to add.
+ * @head: the place to add it in the first list.
+ *
+ * The list at @list is reinitialised
+ */
+static inline void list_splice_init(struct list_head *list,
+				    struct list_head *head)
+{
+	if (!list_empty(list)) {
+		__list_splice(list, head);
+		INIT_LIST_HEAD(list);
+	}
+}
+
+/**
+ * list_entry - get the struct for this entry
+ * @ptr:	the &struct list_head pointer.
+ * @type:	the type of the struct this is embedded in.
+ * @member:	the name of the list_struct within the struct.
+ */
+#define list_entry(ptr, type, member) \
+	container_of(ptr, type, member)
+
+/**
+ * list_for_each	-	iterate over a list
+ * @pos:	the &struct list_head to use as a loop cursor.
+ * @head:	the head for your list.
+ */
+#define list_for_each(pos, head) \
+	for (pos = (head)->next; pos != (head); \
+        	pos = pos->next)
+
+/**
+ * __list_for_each	-	iterate over a list
+ * @pos:	the &struct list_head to use as a loop cursor.
+ * @head:	the head for your list.
+ *
+ * This variant differs from list_for_each() in that it's the
+ * simplest possible list iteration code, no prefetching is done.
+ * Use this for code that knows the list to be very short (empty
+ * or 1 entry) most of the time.
+ */
+#define __list_for_each(pos, head) \
+	for (pos = (head)->next; pos != (head); pos = pos->next)
+
+/**
+ * list_for_each_prev	-	iterate over a list backwards
+ * @pos:	the &struct list_head to use as a loop cursor.
+ * @head:	the head for your list.
+ */
+#define list_for_each_prev(pos, head) \
+	for (pos = (head)->prev; pos != (head); \
+        	pos = pos->prev)
+
+/**
+ * list_for_each_safe - iterate over a list safe against removal of list entry
+ * @pos:	the &struct list_head to use as a loop cursor.
+ * @n:		another &struct list_head to use as temporary storage
+ * @head:	the head for your list.
+ */
+#define list_for_each_safe(pos, n, head) \
+	for (pos = (head)->next, n = pos->next; pos != (head); \
+		pos = n, n = pos->next)
+
+/**
+ * list_for_each_entry	-	iterate over list of given type
+ * @pos:	the type * to use as a loop cursor.
+ * @head:	the head for your list.
+ * @member:	the name of the list_struct within the struct.
+ */
+#define list_for_each_entry(pos, head, member)				\
+	for (pos = list_entry((head)->next, typeof(*pos), member);	\
+	     &pos->member != (head); 	\
+	     pos = list_entry(pos->member.next, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_reverse - iterate backwards over list of given type.
+ * @pos:	the type * to use as a loop cursor.
+ * @head:	the head for your list.
+ * @member:	the name of the list_struct within the struct.
+ */
+#define list_for_each_entry_reverse(pos, head, member)			\
+	for (pos = list_entry((head)->prev, typeof(*pos), member);	\
+	     &pos->member != (head); 	\
+	     pos = list_entry(pos->member.prev, typeof(*pos), member))
+
+/**
+ * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue
+ * @pos:	the type * to use as a start point
+ * @head:	the head of the list
+ * @member:	the name of the list_struct within the struct.
+ *
+ * Prepares a pos entry for use as a start point in list_for_each_entry_continue.
+ */
+#define list_prepare_entry(pos, head, member) \
+	((pos) ? : list_entry(head, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_continue - continue iteration over list of given type
+ * @pos:	the type * to use as a loop cursor.
+ * @head:	the head for your list.
+ * @member:	the name of the list_struct within the struct.
+ *
+ * Continue to iterate over list of given type, continuing after
+ * the current position.
+ */
+#define list_for_each_entry_continue(pos, head, member) 		\
+	for (pos = list_entry(pos->member.next, typeof(*pos), member);	\
+	     &pos->member != (head);	\
+	     pos = list_entry(pos->member.next, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_from - iterate over list of given type from the current point
+ * @pos:	the type * to use as a loop cursor.
+ * @head:	the head for your list.
+ * @member:	the name of the list_struct within the struct.
+ *
+ * Iterate over list of given type, continuing from current position.
+ */
+#define list_for_each_entry_from(pos, head, member) 			\
+	for (; &pos->member != (head);	\
+	     pos = list_entry(pos->member.next, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
+ * @pos:	the type * to use as a loop cursor.
+ * @n:		another type * to use as temporary storage
+ * @head:	the head for your list.
+ * @member:	the name of the list_struct within the struct.
+ */
+#define list_for_each_entry_safe(pos, n, head, member)			\
+	for (pos = list_entry((head)->next, typeof(*pos), member),	\
+		n = list_entry(pos->member.next, typeof(*pos), member);	\
+	     &pos->member != (head); 					\
+	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
+
+/**
+ * list_for_each_entry_safe_continue
+ * @pos:	the type * to use as a loop cursor.
+ * @n:		another type * to use as temporary storage
+ * @head:	the head for your list.
+ * @member:	the name of the list_struct within the struct.
+ *
+ * Iterate over list of given type, continuing after current point,
+ * safe against removal of list entry.
+ */
+#define list_for_each_entry_safe_continue(pos, n, head, member) 		\
+	for (pos = list_entry(pos->member.next, typeof(*pos), member), 		\
+		n = list_entry(pos->member.next, typeof(*pos), member);		\
+	     &pos->member != (head);						\
+	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
+
+/**
+ * list_for_each_entry_safe_from
+ * @pos:	the type * to use as a loop cursor.
+ * @n:		another type * to use as temporary storage
+ * @head:	the head for your list.
+ * @member:	the name of the list_struct within the struct.
+ *
+ * Iterate over list of given type from current point, safe against
+ * removal of list entry.
+ */
+#define list_for_each_entry_safe_from(pos, n, head, member) 			\
+	for (n = list_entry(pos->member.next, typeof(*pos), member);		\
+	     &pos->member != (head);						\
+	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
+
+/**
+ * list_for_each_entry_safe_reverse
+ * @pos:	the type * to use as a loop cursor.
+ * @n:		another type * to use as temporary storage
+ * @head:	the head for your list.
+ * @member:	the name of the list_struct within the struct.
+ *
+ * Iterate backwards over list of given type, safe against removal
+ * of list entry.
+ */
+#define list_for_each_entry_safe_reverse(pos, n, head, member)		\
+	for (pos = list_entry((head)->prev, typeof(*pos), member),	\
+		n = list_entry(pos->member.prev, typeof(*pos), member);	\
+	     &pos->member != (head); 					\
+	     pos = n, n = list_entry(n->member.prev, typeof(*n), member))
+
+#endif
diff --git a/iowatcher/main.c b/iowatcher/main.c
new file mode 100644
index 0000000..cbf6c9a
--- /dev/null
+++ b/iowatcher/main.c
@@ -0,0 +1,1740 @@
+/*
+ * Copyright (C) 2012 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ *  Parts of this file were imported from Jens Axboe's blktrace sources (also GPL)
+ */
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+#include <inttypes.h>
+#include <string.h>
+#include <asm/types.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <math.h>
+#include <getopt.h>
+#include <limits.h>
+#include <float.h>
+#include <signal.h>
+
+#include "plot.h"
+#include "blkparse.h"
+#include "list.h"
+#include "tracers.h"
+#include "mpstat.h"
+#include "fio.h"
+
+LIST_HEAD(all_traces);
+LIST_HEAD(fio_traces);
+
+static char line[1024];
+static int line_len = 1024;
+static int found_mpstat = 0;
+static int make_movie = 0;
+static int keep_movie_svgs = 0;
+static int opt_graph_width = 0;
+static int opt_graph_height = 0;
+
+static int columns = 1;
+static int num_xticks = 9;
+static int num_yticks = 4;
+
+static double min_time = 0;
+static double max_time = DBL_MAX;
+static unsigned long long min_mb = 0;
+static unsigned long long max_mb = ULLONG_MAX >> 20;
+
+int plot_io_action = 0;
+int io_per_process = 0;
+unsigned int longest_proc_name = 0;
+
+/*
+ * this doesn't include the IO graph,
+ * but it counts the other graphs as they go out
+ */
+static int total_graphs_written = 1;
+
+enum {
+	IO_GRAPH_INDEX = 0,
+	TPUT_GRAPH_INDEX,
+	FIO_GRAPH_INDEX,
+	CPU_SYS_GRAPH_INDEX,
+	CPU_IO_GRAPH_INDEX,
+	CPU_IRQ_GRAPH_INDEX,
+	CPU_SOFT_GRAPH_INDEX,
+	CPU_USER_GRAPH_INDEX,
+	LATENCY_GRAPH_INDEX,
+	QUEUE_DEPTH_GRAPH_INDEX,
+	IOPS_GRAPH_INDEX,
+	TOTAL_GRAPHS
+};
+
+enum {
+	MPSTAT_SYS = 0,
+	MPSTAT_IRQ,
+	MPSTAT_IO,
+	MPSTAT_SOFT,
+	MPSTAT_USER,
+	MPSTAT_GRAPHS
+};
+
+static char *graphs_by_name[] = {
+	"io",
+	"tput",
+	"fio",
+	"cpu-sys",
+	"cpu-io",
+	"cpu-irq",
+	"cpu-soft",
+	"cpu-user",
+	"latency",
+	"queue-depth",
+	"iops",
+};
+
+enum {
+	MOVIE_SPINDLE,
+	MOVIE_RECT,
+	NUM_MOVIE_STYLES,
+};
+
+char *movie_styles[] = {
+	"spindle",
+	"rect",
+	NULL
+};
+
+static int movie_style = 0;
+
+static int lookup_movie_style(char *str)
+{
+	int i;
+
+	for (i = 0; i < NUM_MOVIE_STYLES; i++) {
+		if (strcmp(str, movie_styles[i]) == 0)
+			return i;
+	}
+	return -1;
+}
+
+static int active_graphs[TOTAL_GRAPHS];
+static int last_active_graph = IOPS_GRAPH_INDEX;
+
+static int label_index = 0;
+static int num_traces = 0;
+static int num_fio_traces = 0;
+static int longest_label = 0;
+
+static char *graph_title = "";
+static char *output_filename = "trace.svg";
+static char *blktrace_devices[MAX_DEVICES_PER_TRACE];
+static int num_blktrace_devices = 0;
+static char *blktrace_outfile = "trace";
+static char *blktrace_dest_dir = ".";
+static char **prog_argv = NULL;
+static int prog_argc = 0;
+static char *ffmpeg_codec = "libx264";
+
+static void alloc_mpstat_gld(struct trace_file *tf)
+{
+	struct graph_line_data **ptr;
+
+	if (tf->trace->mpstat_num_cpus == 0)
+		return;
+
+	ptr = calloc((tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS,
+		     sizeof(struct graph_line_data *));
+	if (!ptr) {
+		perror("Unable to allocate mpstat arrays\n");
+		exit(1);
+	}
+	tf->mpstat_gld = ptr;
+}
+
+static void enable_all_graphs(void)
+{
+	int i;
+	for (i = 0; i < TOTAL_GRAPHS; i++)
+		active_graphs[i] = 1;
+}
+
+static void disable_all_graphs(void)
+{
+	int i;
+	for (i = 0; i < TOTAL_GRAPHS; i++)
+		active_graphs[i] = 0;
+}
+
+static int enable_one_graph(char *name)
+{
+	int i;
+	for (i = 0; i < TOTAL_GRAPHS; i++) {
+		if (strcmp(name, graphs_by_name[i]) == 0) {
+			active_graphs[i] = 1;
+			return 0;
+		}
+	}
+	return -ENOENT;
+}
+
+static int disable_one_graph(char *name)
+{
+	int i;
+	for (i = 0; i < TOTAL_GRAPHS; i++) {
+		if (strcmp(name, graphs_by_name[i]) == 0) {
+			active_graphs[i] = 0;
+			return 0;
+		}
+	}
+	return -ENOENT;
+}
+
+static int last_graph(void)
+{
+	int i;
+	for (i = TOTAL_GRAPHS - 1; i >= 0; i--) {
+		if (active_graphs[i]) {
+			return i;
+		}
+	}
+	return -ENOENT;
+}
+
+static int graphs_left(int cur)
+{
+	int i;
+	int left = 0;
+	for (i = cur; i < TOTAL_GRAPHS; i++) {
+		if (active_graphs[i])
+			left++;
+	}
+	return left;
+}
+
+static char * join_path(char *dest_dir, char *filename)
+{
+	/* alloc 2 extra bytes for '/' and '\0' */
+	char *path = malloc(strlen(dest_dir) + strlen(filename) + 2);
+	sprintf(path, "%s%s%s", dest_dir, "/", filename);
+
+	return path;
+}
+
+static void add_trace_file(char *filename)
+{
+	struct trace_file *tf;
+
+	tf = calloc(1, sizeof(*tf));
+	if (!tf) {
+		fprintf(stderr, "Unable to allocate memory\n");
+		exit(1);
+	}
+	tf->label = "";
+	tf->filename = strdup(filename);
+	list_add_tail(&tf->list, &all_traces);
+	tf->line_color = "black";
+	num_traces++;
+}
+
+static void add_fio_trace_file(char *filename)
+{
+	struct trace_file *tf;
+
+	tf = calloc(1, sizeof(*tf));
+	if (!tf) {
+		fprintf(stderr, "Unable to allocate memory\n");
+		exit(1);
+	}
+	tf->label = "";
+	tf->filename = strdup(filename);
+	list_add_tail(&tf->list, &fio_traces);
+	tf->line_color = pick_fio_color();
+	tf->fio_trace = 1;
+	num_fio_traces++;
+}
+
+static void setup_trace_file_graphs(void)
+{
+	struct trace_file *tf;
+	int i;
+	int alloc_ptrs;
+
+	if (io_per_process)
+		alloc_ptrs = 16;
+	else
+		alloc_ptrs = 1;
+
+	list_for_each_entry(tf, &all_traces, list) {
+		tf->tput_reads_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
+		tf->tput_writes_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
+		tf->latency_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
+		tf->queue_depth_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
+
+		tf->iop_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
+		tf->gdd_writes = calloc(alloc_ptrs, sizeof(struct graph_dot_data *));
+		tf->gdd_reads = calloc(alloc_ptrs, sizeof(struct graph_dot_data *));
+		tf->io_plots_allocated = alloc_ptrs;
+
+		if (tf->trace->mpstat_num_cpus == 0)
+			continue;
+
+		alloc_mpstat_gld(tf);
+		for (i = 0; i < (tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i++) {
+			tf->mpstat_gld[i] =
+				alloc_line_data(tf->mpstat_min_seconds,
+						tf->mpstat_max_seconds,
+						tf->mpstat_max_seconds);
+			tf->mpstat_gld[i]->max = 100;
+		}
+	}
+
+	list_for_each_entry(tf, &fio_traces, list) {
+		if (tf->trace->fio_seconds > 0) {
+			tf->fio_gld = alloc_line_data(tf->min_seconds,
+						      tf->max_seconds,
+						      tf->stop_seconds);
+		}
+	}
+
+}
+
+static void read_traces(void)
+{
+	struct trace_file *tf;
+	struct trace *trace;
+	u64 last_time;
+	u64 ymin;
+	u64 ymax;
+	u64 max_bank;
+	u64 max_bank_offset;
+	char *path = NULL;
+
+	list_for_each_entry(tf, &all_traces, list) {
+		path = join_path(blktrace_dest_dir, tf->filename);
+
+		trace = open_trace(path);
+		if (!trace)
+			exit(1);
+
+		last_time = find_last_time(trace);
+		tf->trace = trace;
+		tf->max_seconds = SECONDS(last_time) + 1;
+		tf->stop_seconds = SECONDS(last_time) + 1;
+
+		find_extreme_offsets(trace, &tf->min_offset, &tf->max_offset,
+				    &max_bank, &max_bank_offset);
+		filter_outliers(trace, tf->min_offset, tf->max_offset, &ymin, &ymax);
+		tf->min_offset = ymin;
+		tf->max_offset = ymax;
+
+		read_mpstat(trace, path);
+		tf->mpstat_stop_seconds = trace->mpstat_seconds;
+		tf->mpstat_max_seconds = trace->mpstat_seconds;
+		if (tf->mpstat_max_seconds)
+			found_mpstat = 1;
+
+		free(path);
+	}
+
+	list_for_each_entry(tf, &fio_traces, list) {
+		trace = open_fio_trace(tf->filename);
+		if (!trace)
+			exit(1);
+		tf->trace = trace;
+		tf->max_seconds = tf->trace->fio_seconds;
+		tf->stop_seconds = tf->trace->fio_seconds;
+	}
+}
+
+static void pick_line_graph_color(void)
+{
+	struct trace_file *tf;
+	int i;
+
+	list_for_each_entry(tf, &all_traces, list) {
+		for (i = 0; i < tf->io_plots; i++) {
+			if (tf->gdd_reads[i]) {
+				tf->line_color = tf->gdd_reads[i]->color;
+				tf->reads_color = tf->gdd_reads[i]->color;
+			}
+			if (tf->gdd_writes[i]) {
+				tf->line_color = tf->gdd_writes[i]->color;
+				tf->writes_color = tf->gdd_writes[i]->color;
+			}
+			if (tf->writes_color && tf->reads_color)
+				break;
+		}
+		if (!tf->reads_color)
+			tf->reads_color = tf->line_color;
+		if (!tf->writes_color)
+			tf->writes_color = tf->line_color;
+	}
+}
+
+static void read_fio_events(struct trace_file *tf)
+{
+	u64 bw = 0;
+	int time = 0;
+	int dir = 0;
+	int ret;
+
+	first_fio(tf->trace);
+	while (1) {
+		ret = read_fio_event(tf->trace, &time, &bw, &dir);
+		if (ret)
+			break;
+
+		if (dir <= 1)
+			add_fio_gld(time, bw, tf->fio_gld);
+		if (next_fio_line(tf->trace))
+			break;
+	}
+}
+
+static void read_trace_events(void)
+{
+
+	struct trace_file *tf;
+	struct trace *trace;
+	int ret;
+	int i;
+	unsigned int time;
+	double user, sys, iowait, irq, soft;
+	double max_user = 0, max_sys = 0, max_iowait = 0,
+	       max_irq = 0, max_soft = 0;
+
+	list_for_each_entry(tf, &fio_traces, list)
+		read_fio_events(tf);
+
+	list_for_each_entry(tf, &all_traces, list) {
+		trace = tf->trace;
+
+		first_record(trace);
+		do {
+			if (SECONDS(get_record_time(trace)) > tf->max_seconds)
+				continue;
+			check_record(trace);
+			add_tput(trace, tf->tput_writes_gld, tf->tput_reads_gld);
+			add_iop(trace, tf->iop_gld);
+			add_io(trace, tf);
+			add_pending_io(trace, tf->queue_depth_gld);
+			add_completed_io(trace, tf->latency_gld);
+		} while (!(ret = next_record(trace)));
+	}
+	list_for_each_entry(tf, &all_traces, list) {
+		trace = tf->trace;
+
+		if (trace->mpstat_num_cpus == 0)
+			continue;
+
+		first_mpstat(trace);
+
+		for (time = 0; time < tf->mpstat_stop_seconds; time++) {
+			for (i = 0; i < (trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i += MPSTAT_GRAPHS) {
+				ret = read_mpstat_event(trace, &user, &sys,
+							&iowait, &irq, &soft);
+				if (ret)
+					goto mpstat_done;
+				if (next_mpstat_line(trace))
+					goto mpstat_done;
+
+				if (sys > max_sys)
+					max_sys = sys;
+				if (user > max_user)
+					max_user = user;
+				if (irq > max_irq)
+					max_irq = irq;
+				if (iowait > max_iowait)
+					max_iowait = iowait;
+
+				add_mpstat_gld(time, sys, tf->mpstat_gld[i + MPSTAT_SYS]);
+				add_mpstat_gld(time, irq, tf->mpstat_gld[i + MPSTAT_IRQ]);
+				add_mpstat_gld(time, soft, tf->mpstat_gld[i + MPSTAT_SOFT]);
+				add_mpstat_gld(time, user, tf->mpstat_gld[i + MPSTAT_USER]);
+				add_mpstat_gld(time, iowait, tf->mpstat_gld[i + MPSTAT_IO]);
+			}
+			if (next_mpstat(trace) == NULL)
+				break;
+		}
+	}
+
+mpstat_done:
+	list_for_each_entry(tf, &all_traces, list) {
+		trace = tf->trace;
+
+		if (trace->mpstat_num_cpus == 0)
+			continue;
+
+		tf->mpstat_gld[MPSTAT_SYS]->max = max_sys;
+		tf->mpstat_gld[MPSTAT_IRQ]->max = max_irq;
+		tf->mpstat_gld[MPSTAT_SOFT]->max = max_soft;
+		tf->mpstat_gld[MPSTAT_USER]->max = max_user;
+		tf->mpstat_gld[MPSTAT_IO]->max = max_iowait;;
+	}
+	return;
+}
+
+static void set_trace_label(char *label)
+{
+	int cur = 0;
+	struct trace_file *tf;
+	int len = strlen(label);
+
+	if (len > longest_label)
+		longest_label = len;
+
+	list_for_each_entry(tf, &all_traces, list) {
+		if (cur == label_index) {
+			tf->label = strdup(label);
+			label_index++;
+			return;
+			break;
+		}
+		cur++;
+	}
+	list_for_each_entry(tf, &fio_traces, list) {
+		if (cur == label_index) {
+			tf->label = strdup(label);
+			label_index++;
+			break;
+		}
+		cur++;
+	}
+}
+
+static void set_blktrace_outfile(char *arg)
+{
+	char *s = strdup(arg);
+	char *last_dot = strrchr(s, '.');
+
+	if (last_dot) {
+		if (strcmp(last_dot, ".dump") == 0)
+			*last_dot = '\0';
+	}
+	blktrace_outfile = s;
+}
+
+
+static void compare_minmax_tf(struct trace_file *tf, unsigned int *max_seconds,
+			      u64 *min_offset, u64 *max_offset)
+{
+	if (tf->max_seconds > *max_seconds)
+		*max_seconds = tf->max_seconds;
+	if (tf->max_offset > *max_offset)
+		*max_offset = tf->max_offset;
+	if (tf->min_offset < *min_offset)
+		*min_offset = tf->min_offset;
+}
+
+static void set_all_minmax_tf(unsigned int min_seconds,
+			      unsigned int max_seconds,
+			      u64 min_offset, u64 max_offset)
+{
+	struct trace_file *tf;
+	struct list_head *traces = &all_traces;
+again:
+	list_for_each_entry(tf, traces, list) {
+		tf->min_seconds = min_seconds;
+		tf->max_seconds = max_seconds;
+		if (tf->stop_seconds > max_seconds)
+			tf->stop_seconds = max_seconds;
+		if (tf->mpstat_max_seconds) {
+			tf->mpstat_min_seconds = min_seconds;
+			tf->mpstat_max_seconds = max_seconds;
+			if (tf->mpstat_stop_seconds > max_seconds)
+				tf->mpstat_stop_seconds = max_seconds;
+		}
+		tf->min_offset = min_offset;
+		tf->max_offset = max_offset;
+	}
+	if (traces == &all_traces) {
+		traces = &fio_traces;
+		goto again;
+	}
+}
+
+static struct pid_plot_history *alloc_pid_plot_history(char *color)
+{
+	struct pid_plot_history *pph;
+
+	pph = calloc(1, sizeof(struct pid_plot_history));
+	if (!pph) {
+		perror("memory allocation failed");
+		exit(1);
+	}
+	pph->history = malloc(sizeof(double) * 4096);
+	if (!pph->history) {
+		perror("memory allocation failed");
+		exit(1);
+	}
+	pph->history_len = 4096;
+	pph->color = color;
+
+	return pph;
+}
+
+static struct plot_history *alloc_plot_history(struct trace_file *tf)
+{
+	struct plot_history *ph = calloc(1, sizeof(struct plot_history));
+	int i;
+
+	if (!ph) {
+		perror("memory allocation failed");
+		exit(1);
+	}
+	ph->read_pid_history = calloc(tf->io_plots, sizeof(struct pid_plot_history *));
+	if (!ph->read_pid_history) {
+		perror("memory allocation failed");
+		exit(1);
+	}
+	ph->write_pid_history = calloc(tf->io_plots, sizeof(struct pid_plot_history *));
+	if (!ph->write_pid_history) {
+		perror("memory allocation failed");
+		exit(1);
+	}
+	ph->pid_history_count = tf->io_plots;
+	for (i = 0; i < tf->io_plots; i++) {
+		if (tf->gdd_reads[i])
+			ph->read_pid_history[i] = alloc_pid_plot_history(tf->gdd_reads[i]->color);
+		if (tf->gdd_writes[i])
+			ph->write_pid_history[i] = alloc_pid_plot_history(tf->gdd_writes[i]->color);
+	}
+	return ph;
+}
+
+LIST_HEAD(movie_history);
+int num_histories = 0;
+
+static void free_plot_history(struct plot_history *ph)
+{
+	int pid;
+
+	for (pid = 0; pid < ph->pid_history_count; pid++) {
+		if (ph->read_pid_history[pid])
+			free(ph->read_pid_history[pid]);
+		if (ph->write_pid_history[pid])
+			free(ph->write_pid_history[pid]);
+	}
+	free(ph->read_pid_history);
+	free(ph->write_pid_history);
+	free(ph);
+}
+
+static void add_history(struct plot_history *ph, struct list_head *list)
+{
+	struct plot_history *entry;
+
+	list_add_tail(&ph->list, list);
+	num_histories++;
+
+	if (num_histories > 12) {
+		num_histories--;
+		entry = list_entry(list->next, struct plot_history, list);
+		list_del(&entry->list);
+		free_plot_history(entry);
+	}
+}
+
+static void plot_movie_history(struct plot *plot, struct list_head *list)
+{
+	struct plot_history *ph;
+	int pid;
+
+	if (num_histories > 2)
+		rewind_spindle_steps(num_histories - 1);
+
+	list_for_each_entry(ph, list, list) {
+		for (pid = 0; pid < ph->pid_history_count; pid++) {
+			if (ph->read_pid_history[pid]) {
+				if (movie_style == MOVIE_SPINDLE) {
+					svg_io_graph_movie_array_spindle(plot,
+						ph->read_pid_history[pid]);
+				} else {
+					svg_io_graph_movie_array(plot,
+						ph->read_pid_history[pid]);
+				}
+			}
+			if (ph->write_pid_history[pid]) {
+				if (movie_style == MOVIE_SPINDLE) {
+					svg_io_graph_movie_array_spindle(plot,
+						ph->write_pid_history[pid]);
+				} else {
+					svg_io_graph_movie_array(plot,
+						ph->write_pid_history[pid]);
+				}
+			}
+		}
+	 }
+}
+
+static void free_all_plot_history(struct list_head *head)
+{
+	struct plot_history *entry;
+
+	while (!list_empty(head)) {
+		entry = list_entry(head->next, struct plot_history, list);
+		list_del(&entry->list);
+		free_plot_history(entry);
+	}
+}
+
+static int count_io_plot_types(void)
+{
+	struct trace_file *tf;
+	int i;
+	int total_io_types = 0;
+
+	list_for_each_entry(tf, &all_traces, list) {
+		for (i = 0; i < tf->io_plots; i++) {
+			if (tf->gdd_reads[i])
+				total_io_types++;
+			if (tf->gdd_writes[i])
+				total_io_types++;
+		}
+	}
+	return total_io_types;
+}
+
+static void plot_io_legend(struct plot *plot, struct graph_dot_data *gdd, char *prefix, char *rw)
+{
+	int ret = 0;
+	char *label = NULL;
+	if (io_per_process)
+		ret = asprintf(&label, "%s %s", prefix, gdd->label);
+	else
+		ret = asprintf(&label, "%s", prefix);
+	if (ret < 0) {
+		perror("Failed to process labels");
+		exit(1);
+	}
+	svg_add_legend(plot, label, rw, gdd->color);
+	free(label);
+}
+
+static void plot_io(struct plot *plot, unsigned int min_seconds,
+		    unsigned int max_seconds, u64 min_offset, u64 max_offset)
+{
+	struct trace_file *tf;
+	int i;
+
+	if (active_graphs[IO_GRAPH_INDEX] == 0)
+		return;
+
+	setup_axis(plot);
+
+	svg_alloc_legend(plot, count_io_plot_types() * 2);
+
+	set_plot_label(plot, "Device IO");
+	set_ylabel(plot, "Offset (MB)");
+	set_yticks(plot, num_yticks, min_offset / (1024 * 1024),
+		   max_offset / (1024 * 1024), "");
+	set_xticks(plot, num_xticks, min_seconds, max_seconds);
+
+	list_for_each_entry(tf, &all_traces, list) {
+		char *prefix = tf->label ? tf->label : "";
+
+		for (i = 0; i < tf->io_plots; i++) {
+			if (tf->gdd_writes[i]) {
+				svg_io_graph(plot, tf->gdd_writes[i]);
+				plot_io_legend(plot, tf->gdd_writes[i], prefix, " Writes");
+			}
+			if (tf->gdd_reads[i]) {
+				svg_io_graph(plot, tf->gdd_reads[i]);
+				plot_io_legend(plot, tf->gdd_reads[i], prefix, " Reads");
+			}
+		}
+	}
+	if (plot->add_xlabel)
+		set_xlabel(plot, "Time (seconds)");
+	svg_write_legend(plot);
+	close_plot(plot);
+}
+
+static void plot_tput(struct plot *plot, unsigned int min_seconds,
+		      unsigned int max_seconds, int with_legend)
+{
+	struct trace_file *tf;
+	char *units;
+	char line[128];
+	u64 max = 0;
+
+	if (active_graphs[TPUT_GRAPH_INDEX] == 0)
+		return;
+
+	if (with_legend)
+		svg_alloc_legend(plot, num_traces * 2);
+
+	list_for_each_entry(tf, &all_traces, list) {
+		if (tf->tput_writes_gld->max > max)
+			max = tf->tput_writes_gld->max;
+		if (tf->tput_reads_gld->max > max)
+			max = tf->tput_reads_gld->max;
+	}
+	list_for_each_entry(tf, &all_traces, list) {
+		if (tf->tput_writes_gld->max > 0)
+			tf->tput_writes_gld->max = max;
+		if (tf->tput_reads_gld->max > 0)
+			tf->tput_reads_gld->max = max;
+	}
+
+	setup_axis(plot);
+	set_plot_label(plot, "Throughput");
+
+	tf = list_entry(all_traces.next, struct trace_file, list);
+
+	scale_line_graph_bytes(&max, &units, 1024);
+	sprintf(line, "%sB/s", units);
+	set_ylabel(plot, line);
+	set_yticks(plot, num_yticks, 0, max, "");
+	set_xticks(plot, num_xticks, min_seconds, max_seconds);
+
+	list_for_each_entry(tf, &all_traces, list) {
+		if (tf->tput_writes_gld->max > 0) {
+			svg_line_graph(plot, tf->tput_writes_gld, tf->writes_color, 0, 0);
+			if (with_legend)
+				svg_add_legend(plot, tf->label, "Writes", tf->writes_color);
+		}
+		if (tf->tput_reads_gld->max > 0) {
+			svg_line_graph(plot, tf->tput_reads_gld, tf->reads_color, 0, 0);
+			if (with_legend)
+				svg_add_legend(plot, tf->label, "Reads", tf->reads_color);
+		}
+	}
+
+	if (plot->add_xlabel)
+		set_xlabel(plot, "Time (seconds)");
+
+	if (with_legend)
+		svg_write_legend(plot);
+
+	close_plot(plot);
+	total_graphs_written++;
+}
+
+static void plot_fio_tput(struct plot *plot,
+			  unsigned int min_seconds, unsigned int max_seconds)
+{
+	struct trace_file *tf;
+	char *units;
+	char line[128];
+	u64 max = 0;
+
+	if (num_fio_traces == 0 || active_graphs[FIO_GRAPH_INDEX] == 0)
+		return;
+
+	if (num_fio_traces > 1)
+		svg_alloc_legend(plot, num_fio_traces);
+
+	list_for_each_entry(tf, &fio_traces, list) {
+		if (tf->fio_gld->max > max)
+			max = tf->fio_gld->max;
+	}
+
+	list_for_each_entry(tf, &fio_traces, list) {
+		if (tf->fio_gld->max > 0)
+			tf->fio_gld->max = max;
+	}
+
+	setup_axis(plot);
+	set_plot_label(plot, "Fio Throughput");
+
+	tf = list_entry(all_traces.next, struct trace_file, list);
+
+	scale_line_graph_bytes(&max, &units, 1024);
+	sprintf(line, "%sB/s", units);
+	set_ylabel(plot, line);
+	set_yticks(plot, num_yticks, 0, max, "");
+
+	set_xticks(plot, num_xticks, min_seconds, max_seconds);
+	list_for_each_entry(tf, &fio_traces, list) {
+		if (tf->fio_gld->max > 0) {
+			svg_line_graph(plot, tf->fio_gld, tf->line_color, 0, 0);
+			if (num_fio_traces > 1)
+				svg_add_legend(plot, tf->label, "", tf->line_color);
+		}
+	}
+
+	if (plot->add_xlabel)
+		set_xlabel(plot, "Time (seconds)");
+
+	if (num_fio_traces > 1)
+		svg_write_legend(plot);
+	close_plot(plot);
+	total_graphs_written++;
+}
+
+static void plot_cpu(struct plot *plot, unsigned int max_seconds, char *label,
+		     int active_index, int gld_index)
+{
+	struct trace_file *tf;
+	int max = 0;
+	int i;
+	unsigned int gld_i;
+	char *color;
+	double avg = 0;
+	int ymax;
+	int plotted = 0;
+
+	if (active_graphs[active_index] == 0)
+		return;
+
+	list_for_each_entry(tf, &all_traces, list) {
+		if (tf->trace->mpstat_num_cpus > max)
+			max = tf->trace->mpstat_num_cpus;
+	}
+	if (max == 0)
+		return;
+
+	tf = list_entry(all_traces.next, struct trace_file, list);
+
+	ymax = tf->mpstat_gld[gld_index]->max;
+	if (ymax == 0)
+		return;
+
+	svg_alloc_legend(plot, num_traces * max);
+
+	setup_axis(plot);
+	set_plot_label(plot, label);
+
+	max_seconds = tf->mpstat_max_seconds;
+
+	set_yticks(plot, num_yticks, 0, tf->mpstat_gld[gld_index]->max, "");
+	set_ylabel(plot, "Percent");
+	set_xticks(plot, num_xticks, tf->mpstat_min_seconds, max_seconds);
+
+	reset_cpu_color();
+	list_for_each_entry(tf, &all_traces, list) {
+		if (tf->mpstat_gld == 0)
+			break;
+		for (gld_i = tf->mpstat_gld[0]->min_seconds;
+		     gld_i < tf->mpstat_gld[0]->stop_seconds; gld_i++) {
+			if (tf->mpstat_gld[gld_index]->data[gld_i].count) {
+				avg += (tf->mpstat_gld[gld_index]->data[gld_i].sum /
+					tf->mpstat_gld[gld_index]->data[gld_i].count);
+			}
+		}
+		avg /= tf->mpstat_gld[gld_index]->stop_seconds -
+		       tf->mpstat_gld[gld_index]->min_seconds;
+		color = pick_cpu_color();
+		svg_line_graph(plot, tf->mpstat_gld[0], color, 0, 0);
+		svg_add_legend(plot, tf->label, " avg", color);
+
+		for (i = 1; i < tf->trace->mpstat_num_cpus + 1; i++) {
+			struct graph_line_data *gld = tf->mpstat_gld[i * MPSTAT_GRAPHS + gld_index];
+			double this_avg = 0;
+
+			for (gld_i = gld->min_seconds;
+			     gld_i < gld->stop_seconds; gld_i++) {
+				if (gld->data[i].count) {
+					this_avg += gld->data[i].sum /
+						gld->data[i].count;
+				}
+			}
+
+			this_avg /= gld->stop_seconds - gld->min_seconds;
+
+			for (gld_i = gld->min_seconds;
+			     gld_i < gld->stop_seconds; gld_i++) {
+				double val;
+
+				if (gld->data[gld_i].count == 0)
+					continue;
+				val = (double)gld->data[gld_i].sum /
+					gld->data[gld_i].count;
+
+				if (this_avg > avg + 30 || val > 95) {
+					color = pick_cpu_color();
+					svg_line_graph(plot, gld, color, avg + 30, 95);
+					snprintf(line, line_len, " CPU %d\n", i - 1);
+					svg_add_legend(plot, tf->label, line, color);
+					plotted++;
+					break;
+				}
+
+			}
+		}
+	}
+
+	if (plot->add_xlabel)
+		set_xlabel(plot, "Time (seconds)");
+
+	if (!plot->no_legend) {
+		svg_write_legend(plot);
+		svg_free_legend(plot);
+	}
+	close_plot(plot);
+	total_graphs_written++;
+}
+
+static void plot_queue_depth(struct plot *plot, unsigned int min_seconds,
+			     unsigned int max_seconds)
+{
+	struct trace_file *tf;
+
+	if (active_graphs[QUEUE_DEPTH_GRAPH_INDEX] == 0)
+		return;
+
+	setup_axis(plot);
+	set_plot_label(plot, "Queue Depth");
+	if (num_traces > 1)
+		svg_alloc_legend(plot, num_traces);
+
+	tf = list_entry(all_traces.next, struct trace_file, list);
+	set_ylabel(plot, "Pending IO");
+	set_yticks(plot, num_yticks, 0, tf->queue_depth_gld->max, "");
+	set_xticks(plot, num_xticks, min_seconds, max_seconds);
+
+	list_for_each_entry(tf, &all_traces, list) {
+		svg_line_graph(plot, tf->queue_depth_gld, tf->line_color, 0, 0);
+		if (num_traces > 1)
+			svg_add_legend(plot, tf->label, "", tf->line_color);
+	}
+
+	if (plot->add_xlabel)
+		set_xlabel(plot, "Time (seconds)");
+	if (num_traces > 1)
+		svg_write_legend(plot);
+	close_plot(plot);
+	total_graphs_written++;
+}
+
+static void convert_movie_files(char *movie_dir)
+{
+	fprintf(stderr, "Converting svg files in %s\n", movie_dir);
+	snprintf(line, line_len, "find %s -name \\*.svg | xargs -I{} -n 1 -P 8 rsvg-convert -o {}.png {}",
+		 movie_dir);
+	system(line);
+}
+
+static void mencode_movie(char *movie_dir)
+{
+	fprintf(stderr, "Creating movie %s with ffmpeg\n", movie_dir);
+	snprintf(line, line_len, "ffmpeg -r 20 -y -i %s/%%10d-%s.svg.png -b:v 250k "
+		 "-vcodec %s %s", movie_dir, output_filename, ffmpeg_codec,
+		 output_filename);
+	system(line);
+}
+
+static void tencode_movie(char *movie_dir)
+{
+	fprintf(stderr, "Creating movie %s with png2theora\n", movie_dir);
+	snprintf(line, line_len, "png2theora -o %s %s/%%010d-%s.svg.png",
+			output_filename, movie_dir, output_filename);
+	system(line);
+}
+
+static void encode_movie(char *movie_dir)
+{
+	char *last_dot = strrchr(output_filename, '.');
+
+	if (last_dot &&
+	    (!strncmp(last_dot, ".ogg", 4) || !strncmp(last_dot, ".ogv", 4))) {
+		tencode_movie(movie_dir);
+	} else
+		mencode_movie(movie_dir);
+}
+
+static void cleanup_movie(char *movie_dir)
+{
+	if (keep_movie_svgs) {
+		fprintf(stderr, "Keeping movie dir %s\n", movie_dir);
+		return;
+	}
+	fprintf(stderr, "Removing movie dir %s\n", movie_dir);
+	snprintf(line, line_len, "rm %s/*", movie_dir);
+	system(line);
+
+	snprintf(line, line_len, "rmdir %s", movie_dir);
+	system(line);
+}
+
+static void plot_io_movie(struct plot *plot)
+{
+	struct trace_file *tf;
+	int i, pid;
+	struct plot_history *history;
+	int batch_i;
+	int movie_len = 30;
+	int movie_frames_per_sec = 20;
+	int total_frames = movie_len * movie_frames_per_sec;
+	int rows, cols;
+	int batch_count;
+	int graph_width_factor = 5;
+	int orig_y_offset;
+	char movie_dir[] = "io-movie-XXXXXX";
+
+	if (mkdtemp(movie_dir) == NULL) {
+		perror("Unable to create temp directory for movie files");
+		exit(1);
+	}
+
+	get_graph_size(&cols, &rows);
+	batch_count = cols / total_frames;
+
+	if (batch_count == 0)
+		batch_count = 1;
+
+	list_for_each_entry(tf, &all_traces, list) {
+		char *prefix = tf->label ? tf->label : "";
+
+		i = 0;
+		while (i < cols) {
+			snprintf(line, line_len, "%s/%010d-%s.svg", movie_dir, i, output_filename);
+			set_plot_output(plot, line);
+			set_plot_title(plot, graph_title);
+			orig_y_offset = plot->start_y_offset;
+
+			plot->no_legend = 1;
+
+			set_graph_size(cols / graph_width_factor, rows / 8);
+			plot->timeline = i / graph_width_factor;
+
+			plot_tput(plot, tf->min_seconds, tf->max_seconds, 0);
+
+			plot_cpu(plot, tf->max_seconds,
+				   "CPU System Time", CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
+
+			plot->direction = PLOT_ACROSS;
+			plot_queue_depth(plot, tf->min_seconds, tf->max_seconds);
+
+			/* movie graph starts here */
+			plot->start_y_offset = orig_y_offset;
+			set_graph_size(cols - cols / graph_width_factor, rows);
+			plot->no_legend = 0;
+			plot->timeline = 0;
+			plot->direction = PLOT_DOWN;;
+
+			if (movie_style == MOVIE_SPINDLE)
+				setup_axis_spindle(plot);
+			else
+				setup_axis(plot);
+
+			svg_alloc_legend(plot, count_io_plot_types() * 2);
+
+			history = alloc_plot_history(tf);
+			history->col = i;
+
+			for (pid = 0; pid < tf->io_plots; pid++) {
+				if (tf->gdd_reads[pid])
+					plot_io_legend(plot, tf->gdd_reads[pid], prefix, " Reads");
+				if (tf->gdd_writes[pid])
+					plot_io_legend(plot, tf->gdd_writes[pid], prefix, " Writes");
+			}
+
+			batch_i = 0;
+			while (i < cols && batch_i < batch_count) {
+				for (pid = 0; pid < tf->io_plots; pid++) {
+					if (tf->gdd_reads[pid]) {
+						svg_io_graph_movie(tf->gdd_reads[pid],
+								   history->read_pid_history[pid],
+								   i);
+					}
+					if (tf->gdd_writes[pid]) {
+						svg_io_graph_movie(tf->gdd_writes[pid],
+								   history->write_pid_history[pid],
+								   i);
+					}
+				}
+				i++;
+				batch_i++;
+			}
+
+			add_history(history, &movie_history);
+
+			plot_movie_history(plot, &movie_history);
+
+			svg_write_legend(plot);
+			close_plot(plot);
+			close_plot(plot);
+
+			close_plot_file(plot);
+		}
+		free_all_plot_history(&movie_history);
+	}
+	convert_movie_files(movie_dir);
+	encode_movie(movie_dir);
+	cleanup_movie(movie_dir);
+}
+
+static void plot_latency(struct plot *plot, unsigned int min_seconds,
+			 unsigned int max_seconds)
+{
+	struct trace_file *tf;
+	char *units;
+	char line[128];
+	u64 max = 0;
+
+	if (active_graphs[LATENCY_GRAPH_INDEX] == 0)
+		return;
+
+	if (num_traces > 1)
+		svg_alloc_legend(plot, num_traces);
+
+	list_for_each_entry(tf, &all_traces, list) {
+		if (tf->latency_gld->max > max)
+			max = tf->latency_gld->max;
+	}
+
+	list_for_each_entry(tf, &all_traces, list)
+		tf->latency_gld->max = max;
+
+	setup_axis(plot);
+	set_plot_label(plot, "IO Latency");
+
+	tf = list_entry(all_traces.next, struct trace_file, list);
+
+	scale_line_graph_time(&max, &units);
+	sprintf(line, "latency (%ss)", units);
+	set_ylabel(plot, line);
+	set_yticks(plot, num_yticks, 0, max, "");
+	set_xticks(plot, num_xticks, min_seconds, max_seconds);
+
+	list_for_each_entry(tf, &all_traces, list) {
+		svg_line_graph(plot, tf->latency_gld, tf->line_color, 0, 0);
+		if (num_traces > 1)
+			svg_add_legend(plot, tf->label, "", tf->line_color);
+	}
+
+	if (plot->add_xlabel)
+		set_xlabel(plot, "Time (seconds)");
+	if (num_traces > 1)
+		svg_write_legend(plot);
+	close_plot(plot);
+	total_graphs_written++;
+}
+
+static void plot_iops(struct plot *plot, unsigned int min_seconds,
+		      unsigned int max_seconds)
+{
+	struct trace_file *tf;
+	char *units;
+	u64 max = 0;
+
+	if (active_graphs[IOPS_GRAPH_INDEX] == 0)
+		return;
+
+	list_for_each_entry(tf, &all_traces, list) {
+		if (tf->iop_gld->max > max)
+			max = tf->iop_gld->max;
+	}
+
+	list_for_each_entry(tf, &all_traces, list)
+		tf->iop_gld->max = max;
+
+	setup_axis(plot);
+	set_plot_label(plot, "IOPs");
+	if (num_traces > 1)
+		svg_alloc_legend(plot, num_traces);
+
+	tf = list_entry(all_traces.next, struct trace_file, list);
+
+	scale_line_graph_bytes(&max, &units, 1000);
+	set_ylabel(plot, "IO/s");
+
+	set_yticks(plot, num_yticks, 0, max, units);
+	set_xticks(plot, num_xticks, min_seconds, max_seconds);
+
+	list_for_each_entry(tf, &all_traces, list) {
+		svg_line_graph(plot, tf->iop_gld, tf->line_color, 0, 0);
+		if (num_traces > 1)
+			svg_add_legend(plot, tf->label, "", tf->line_color);
+	}
+
+	if (plot->add_xlabel)
+		set_xlabel(plot, "Time (seconds)");
+	if (num_traces > 1)
+		svg_write_legend(plot);
+
+	close_plot(plot);
+	total_graphs_written++;
+}
+
+static void check_plot_columns(struct plot *plot, int index)
+{
+	int count;
+
+	if (columns > 1 && (total_graphs_written == 0 ||
+	    total_graphs_written % columns != 0)) {
+		count = graphs_left(index);
+		if (plot->direction == PLOT_DOWN) {
+			plot->start_x_offset = 0;
+			if (count <= columns)
+				plot->add_xlabel = 1;
+		}
+		plot->direction = PLOT_ACROSS;
+
+	} else {
+		plot->direction = PLOT_DOWN;
+		if (index == last_active_graph)
+			plot->add_xlabel = 1;
+	}
+
+}
+
+enum {
+	HELP_LONG_OPT = 1,
+};
+
+char *option_string = "+F:T:t:o:l:r:O:N:d:D:pm::h:w:c:x:y:a:C:PK";
+static struct option long_options[] = {
+	{"columns", required_argument, 0, 'c'},
+	{"fio-trace", required_argument, 0, 'F'},
+	{"title", required_argument, 0, 'T'},
+	{"trace", required_argument, 0, 't'},
+	{"output", required_argument, 0, 'o'},
+	{"label", required_argument, 0, 'l'},
+	{"rolling", required_argument, 0, 'r'},
+	{"no-graph", required_argument, 0, 'N'},
+	{"only-graph", required_argument, 0, 'O'},
+	{"device", required_argument, 0, 'd'},
+	{"blktrace-destination", required_argument, 0, 'D'},
+	{"prog", no_argument, 0, 'p'},
+	{"movie", optional_argument, 0, 'm'},
+	{"codec", optional_argument, 0, 'C'},
+	{"keep-movie-svgs", no_argument, 0, 'K'},
+	{"width", required_argument, 0, 'w'},
+	{"height", required_argument, 0, 'h'},
+	{"xzoom", required_argument, 0, 'x'},
+	{"yzoom", required_argument, 0, 'y'},
+	{"io-plot-action", required_argument, 0, 'a'},
+	{"per-process-io", no_argument, 0, 'P'},
+	{"help", no_argument, 0, HELP_LONG_OPT},
+	{0, 0, 0, 0}
+};
+
+static void print_usage(void)
+{
+	fprintf(stderr, "iowatcher usage:\n"
+		"\t-d (--device): device for blktrace to trace\n"
+		"\t-D (--blktrace-destination): destination for blktrace\n"
+		"\t-t (--trace): trace file name (more than one allowed)\n"
+		"\t-F (--fio-trace): fio bandwidth trace (more than one allowed)\n"
+		"\t-l (--label): trace label in the graph\n"
+		"\t-o (--output): output file name for the SVG image or video\n"
+		"\t-p (--prog): run a program while blktrace is run\n"
+		"\t-K (--keep-movie-svgs keep svgs generated for movie mode\n"
+		"\t-m (--movie [=spindle|rect]): create IO animations\n"
+		"\t-C (--codec): ffmpeg codec. Use ffmpeg -codecs to list\n"
+		"\t-r (--rolling): number of seconds in the rolling averge\n"
+		"\t-T (--title): graph title\n"
+		"\t-N (--no-graph): skip a single graph (io, tput, latency, queue_depth, \n"
+		"\t\t\tiops, cpu-sys, cpu-io, cpu-irq cpu-soft cpu-user)\n"
+		"\t-O (--only-graph): add a single graph to the output\n"
+		"\t-h (--height): set the height of each graph\n"
+		"\t-w (--width): set the width of each graph\n"
+		"\t-c (--columns): numbers of columns in graph output\n"
+		"\t-x (--xzoom): limit processed time to min:max\n"
+		"\t-y (--yzoom): limit processed sectors to min:max\n"
+		"\t-a (--io-plot-action): plot given action (one of Q,D,C) in IO graph\n"
+		"\t-P (--per-process-io): distinguish between processes in IO graph\n"
+	       );
+	exit(1);
+}
+
+static int parse_double_range(char *str, double *min, double *max)
+{
+	char *end;
+
+	/* Empty lower bound - leave original value */
+	if (str[0] != ':') {
+		*min = strtod(str, &end);
+		if (*min == HUGE_VAL || *min == -HUGE_VAL)
+			return -ERANGE;
+		if (*end != ':')
+			return -EINVAL;
+	} else
+		end = str;
+	/* Empty upper bound - leave original value */
+	if (end[1]) {
+		*max = strtod(end+1, &end);
+		if (*max == HUGE_VAL || *max == -HUGE_VAL)
+			return -ERANGE;
+		if (*end != 0)
+			return -EINVAL;
+	}
+	if (*min > *max)
+		return -EINVAL;
+	return 0;
+}
+
+static int parse_ull_range(char *str, unsigned long long *min,
+			   unsigned long long *max)
+{
+	char *end;
+
+	/* Empty lower bound - leave original value */
+	if (str[0] != ':') {
+		*min = strtoull(str, &end, 10);
+		if (*min == ULLONG_MAX && errno == ERANGE)
+			return -ERANGE;
+		if (*end != ':')
+			return -EINVAL;
+	} else
+		end = str;
+	/* Empty upper bound - leave original value */
+	if (end[1]) {
+		*max = strtoull(end+1, &end, 10);
+		if (*max == ULLONG_MAX && errno == ERANGE)
+			return -ERANGE;
+		if (*end != 0)
+			return -EINVAL;
+	}
+	if (*min > *max)
+		return -EINVAL;
+	return 0;
+}
+
+static int parse_options(int ac, char **av)
+{
+	int c;
+	int disabled = 0;
+	int p_flagged = 0;
+
+	while (1) {
+		int option_index = 0;
+
+		c = getopt_long(ac, av, option_string,
+				long_options, &option_index);
+
+		if (c == -1)
+			break;
+
+		switch(c) {
+		case 'T':
+			graph_title = strdup(optarg);
+			break;
+		case 't':
+			add_trace_file(optarg);
+			set_blktrace_outfile(optarg);
+			break;
+		case 'F':
+			add_fio_trace_file(optarg);
+			break;
+		case 'o':
+			output_filename = strdup(optarg);
+			break;
+		case 'l':
+			set_trace_label(optarg);
+			break;
+		case 'r':
+			set_rolling_avg(atoi(optarg));
+			break;
+		case 'O':
+			if (!disabled) {
+				disable_all_graphs();
+				disabled = 1;
+			}
+			enable_one_graph(optarg);
+			break;
+		case 'N':
+			disable_one_graph(optarg);
+			break;
+		case 'd':
+			if (num_blktrace_devices == MAX_DEVICES_PER_TRACE - 1) {
+				fprintf(stderr, "Too many blktrace devices provided\n");
+				exit(1);
+			}
+			blktrace_devices[num_blktrace_devices++] = strdup(optarg);
+			break;
+		case 'D':
+			blktrace_dest_dir = strdup(optarg);
+			if (!strcmp(blktrace_dest_dir, "")) {
+				fprintf(stderr, "Need a directory\n");
+				print_usage();
+			}
+			break;
+		case 'p':
+			p_flagged = 1;
+			break;
+		case 'K':
+			keep_movie_svgs = 1;
+			break;
+		case 'm':
+			make_movie = 1;
+			if (optarg) {
+				movie_style = lookup_movie_style(optarg);
+				if (movie_style < 0) {
+					fprintf(stderr, "Unknown movie style %s\n", optarg);
+					print_usage();
+				}
+			}
+			fprintf(stderr, "Using movie style: %s\n",
+				movie_styles[movie_style]);
+			break;
+		case 'C':
+			ffmpeg_codec = strdup(optarg);
+			break;
+		case 'h':
+			opt_graph_height = atoi(optarg);
+			break;
+		case 'w':
+			opt_graph_width = atoi(optarg);
+			break;
+		case 'c':
+			columns = atoi(optarg);
+			break;
+		case 'x':
+			if (parse_double_range(optarg, &min_time, &max_time)
+			    < 0) {
+				fprintf(stderr, "Cannot parse time range %s\n",
+					optarg);
+				exit(1);
+			}
+			break;
+		case 'y':
+			if (parse_ull_range(optarg, &min_mb, &max_mb)
+			    < 0) {
+				fprintf(stderr,
+					"Cannot parse offset range %s\n",
+					optarg);
+				exit(1);
+			}
+			if (max_mb > ULLONG_MAX >> 20) {
+				fprintf(stderr,
+					"Upper range limit too big."
+					" Maximum is %llu.\n", ULLONG_MAX >> 20);
+				exit(1);
+			}
+			break;
+		case 'a':
+			if (strlen(optarg) != 1) {
+action_err:
+				fprintf(stderr, "Action must be one of Q, D, C.");
+				exit(1);
+			}
+			plot_io_action = action_char_to_num(optarg[0]);
+			if (plot_io_action < 0)
+				goto action_err;
+			break;
+		case 'P':
+			io_per_process = 1;
+			break;
+		case '?':
+		case HELP_LONG_OPT:
+			print_usage();
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (optind < ac && p_flagged) {
+		prog_argv = &av[optind];
+		prog_argc = ac - optind;
+	} else if (p_flagged) {
+		fprintf(stderr, "--prog or -p given but no program specified\n");
+		exit(1);
+	} else if (optind < ac) {
+		fprintf(stderr, "Extra arguments '%s'... (and --prog not specified)\n", av[optind]);
+		exit(1);
+	}
+
+	return 0;
+}
+
+static void dest_mkdir(char *dir)
+{
+	int ret;
+
+	ret = mkdir(dir, 0777);
+	if (ret && errno != EEXIST) {
+		fprintf(stderr, "failed to mkdir error %s\n", strerror(errno));
+		exit(errno);
+	}
+}
+
+int main(int ac, char **av)
+{
+	struct plot *plot;
+	unsigned int min_seconds = 0;
+	unsigned int max_seconds = 0;
+	u64 max_offset = 0;
+	u64 min_offset = ~(u64)0;
+	struct trace_file *tf;
+	int ret;
+	int rows, cols;
+
+	init_io_hash_table();
+	init_process_hash_table();
+
+	enable_all_graphs();
+
+	parse_options(ac, av);
+
+	last_active_graph = last_graph();
+	if (make_movie) {
+		set_io_graph_scale(256);
+		if (movie_style == MOVIE_SPINDLE)
+			set_graph_size(750, 550);
+		else
+			set_graph_size(700, 400);
+
+		/*
+		 * the plots in the movie don't have a seconds
+		 * line yet, this makes us skip it
+		 */
+		last_active_graph = TOTAL_GRAPHS + 1;
+	}
+	if (opt_graph_height)
+		set_graph_height(opt_graph_height);
+
+	if (opt_graph_width)
+		set_graph_width(opt_graph_width);
+
+	if (list_empty(&all_traces) && list_empty(&fio_traces)) {
+		fprintf(stderr, "No traces found, exiting\n");
+		exit(1);
+	}
+
+	if (num_blktrace_devices) {
+		char *path = join_path(blktrace_dest_dir, blktrace_outfile);
+		dest_mkdir(blktrace_dest_dir);
+		dest_mkdir(path);
+
+		snprintf(line, line_len, "%s.dump", path);
+		unlink(line);
+
+		ret = start_blktrace(blktrace_devices, num_blktrace_devices,
+				     blktrace_outfile, blktrace_dest_dir);
+		if (ret) {
+			perror("Exiting due to blktrace failure");
+			exit(ret);
+		}
+
+		snprintf(line, line_len, "%s.mpstat", path);
+		ret = start_mpstat(line);
+		if (ret) {
+			perror("Exiting due to mpstat failure");
+			exit(ret);
+		}
+
+		if (prog_argv && prog_argc) {
+			run_program(prog_argc, prog_argv, 1, NULL, NULL);
+			wait_for_tracers(SIGINT);
+		} else {
+			printf("Tracing until interrupted...\n");
+			wait_for_tracers(0);
+		}
+		free(path);
+	}
+
+	/* step one, read all the traces */
+	read_traces();
+
+	/* step two, find the maxes for time and offset */
+	list_for_each_entry(tf, &all_traces, list)
+		compare_minmax_tf(tf, &max_seconds, &min_offset, &max_offset);
+	list_for_each_entry(tf, &fio_traces, list)
+		compare_minmax_tf(tf, &max_seconds, &min_offset, &max_offset);
+	min_seconds = min_time;
+	if (max_seconds > max_time)
+		max_seconds = ceil(max_time);
+	if (min_offset < min_mb << 20)
+		min_offset = min_mb << 20;
+	if (max_offset > max_mb << 20)
+		max_offset = max_mb << 20;
+
+	/* push the max we found into all the tfs */
+	set_all_minmax_tf(min_seconds, max_seconds, min_offset, max_offset);
+
+	/* alloc graphing structs for all the traces */
+	setup_trace_file_graphs();
+
+	/* run through all the traces and read their events */
+	read_trace_events();
+
+	pick_line_graph_color();
+
+	plot = alloc_plot();
+
+	if (make_movie) {
+		set_legend_width(longest_label + longest_proc_name + 1 + strlen("writes"));
+		plot_io_movie(plot);
+		exit(0);
+	}
+
+	set_plot_output(plot, output_filename);
+
+	if (active_graphs[IO_GRAPH_INDEX] || found_mpstat)
+		set_legend_width(longest_label + longest_proc_name + 1 + strlen("writes"));
+	else if (num_traces >= 1 || num_fio_traces >= 1)
+		set_legend_width(longest_label);
+	else
+		set_legend_width(0);
+
+	get_graph_size(&cols, &rows);
+	if (columns > 1)
+		plot->add_xlabel = 1;
+	set_plot_title(plot, graph_title);
+
+	check_plot_columns(plot, IO_GRAPH_INDEX);
+	plot_io(plot, min_seconds, max_seconds, min_offset, max_offset);
+	plot->add_xlabel = 0;
+
+	if (columns > 1) {
+		set_graph_size(cols / columns, rows);
+		num_xticks /= columns;
+		if (num_xticks < 2)
+			num_xticks = 2;
+	}
+	if (rows <= 50)
+		num_yticks--;
+
+	check_plot_columns(plot, TPUT_GRAPH_INDEX);
+	plot_tput(plot, min_seconds, max_seconds, 1);
+
+	check_plot_columns(plot, FIO_GRAPH_INDEX);
+	plot_fio_tput(plot, min_seconds, max_seconds);
+
+	check_plot_columns(plot, CPU_IO_GRAPH_INDEX);
+	plot_cpu(plot, max_seconds, "CPU IO Wait Time",
+		 CPU_IO_GRAPH_INDEX, MPSTAT_IO);
+
+	check_plot_columns(plot, CPU_SYS_GRAPH_INDEX);
+	plot_cpu(plot, max_seconds, "CPU System Time",
+		 CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
+
+	check_plot_columns(plot, CPU_IRQ_GRAPH_INDEX);
+	plot_cpu(plot, max_seconds, "CPU IRQ Time",
+		 CPU_IRQ_GRAPH_INDEX, MPSTAT_IRQ);
+
+	check_plot_columns(plot, CPU_SOFT_GRAPH_INDEX);
+	plot_cpu(plot, max_seconds, "CPU SoftIRQ Time",
+		 CPU_SOFT_GRAPH_INDEX, MPSTAT_SOFT);
+
+	check_plot_columns(plot, CPU_USER_GRAPH_INDEX);
+	plot_cpu(plot, max_seconds, "CPU User Time",
+		 CPU_USER_GRAPH_INDEX, MPSTAT_USER);
+
+	check_plot_columns(plot, LATENCY_GRAPH_INDEX);
+	plot_latency(plot, min_seconds, max_seconds);
+
+	check_plot_columns(plot, QUEUE_DEPTH_GRAPH_INDEX);
+	plot_queue_depth(plot, min_seconds, max_seconds);
+
+	check_plot_columns(plot, IOPS_GRAPH_INDEX);
+	plot_iops(plot, min_seconds, max_seconds);
+
+	/* once for all */
+	close_plot(plot);
+	close_plot_file(plot);
+	return 0;
+}
diff --git a/iowatcher/mpstat.c b/iowatcher/mpstat.c
new file mode 100644
index 0000000..db7b8e0
--- /dev/null
+++ b/iowatcher/mpstat.c
@@ -0,0 +1,315 @@
+/*
+ * Copyright (C) 2012 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+#include <inttypes.h>
+#include <string.h>
+#include <asm/types.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <math.h>
+
+#include "plot.h"
+#include "blkparse.h"
+#include "list.h"
+#include "tracers.h"
+#include "mpstat.h"
+
+char line[1024];
+int line_len = 1024;
+
+static char record_header[] = "CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest   %idle\n";
+static char record_header_v2[] = "CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle\n";
+
+int record_header_len = sizeof(record_header);
+int record_header_v2_len = sizeof(record_header_v2);
+
+static int past_eof(struct trace *trace, char *cur)
+{
+	if (cur >= trace->mpstat_start + trace->mpstat_len)
+		return 1;
+	return 0;
+}
+
+int next_mpstat_line(struct trace *trace)
+{
+	char *next;
+	char *cur = trace->mpstat_cur;
+
+	next = strchr(cur, '\n');
+	if (!next)
+		return 1;
+	next++;
+	if (past_eof(trace, next))
+		return 1;
+	trace->mpstat_cur = next;
+	return 0;
+}
+
+char *next_mpstat(struct trace *trace)
+{
+	char *cur;
+
+	cur = strstr(trace->mpstat_cur, record_header);
+	if (cur) {
+		cur += record_header_len;
+	} else {
+		cur = strstr(trace->mpstat_cur, record_header_v2);
+		if (cur)
+			cur += record_header_v2_len;
+	}
+	if (!cur)
+		return NULL;
+
+	if (past_eof(trace, cur))
+		return NULL;
+	trace->mpstat_cur = cur;
+	return cur;
+}
+
+char *first_mpstat(struct trace *trace)
+{
+	char *cur = trace->mpstat_cur;
+
+	trace->mpstat_cur = trace->mpstat_start;
+
+	cur = next_mpstat(trace);
+	if (!cur)
+		return NULL;
+	return cur;
+}
+
+static void find_last_mpstat_time(struct trace *trace)
+{
+	int num_mpstats = 0;
+	char *cur;
+
+	first_mpstat(trace);
+
+	cur = first_mpstat(trace);
+	while (cur) {
+		num_mpstats++;
+		cur = next_mpstat(trace);
+	}
+	first_mpstat(trace);
+	trace->mpstat_seconds = num_mpstats;
+}
+
+static int guess_mpstat_cpus(struct trace *trace)
+{
+	char *cur;
+	int ret;
+	int count = 0;
+
+	cur = first_mpstat(trace);
+	if (!cur)
+		return 0;
+
+	while (1) {
+		ret = next_mpstat_line(trace);
+		if (ret)
+			break;
+
+		cur = trace->mpstat_cur;
+		count++;
+
+		if (!cur)
+			break;
+
+		if (cur[0] == '\n')
+			break;
+	}
+	trace->mpstat_num_cpus = count - 1;
+	return 0;
+}
+
+static int count_mpstat_cpus(struct trace *trace)
+{
+	char *cur = trace->mpstat_start;
+	char *cpu;
+	char *record;
+	int len; char *line;
+
+	first_mpstat(trace);
+	cpu = strstr(cur, " CPU)");
+	if (!cpu)
+		return guess_mpstat_cpus(trace);
+
+	line = strndup(cur, cpu - cur);
+
+	record = strrchr(line, '(');
+	if (!record) {
+		free(line);
+		return 0;
+	}
+	record++;
+
+	len = line + strlen(line) - record;
+
+	cur = strndup(record, len);
+	trace->mpstat_num_cpus = atoi(cur);
+	first_mpstat(trace);
+	free(line);
+
+	return trace->mpstat_num_cpus;
+}
+
+static char *guess_filename(char *trace_name)
+{
+	struct stat st;
+	int ret;
+	char *cur;
+	char *tmp;
+
+	snprintf(line, line_len, "%s.mpstat", trace_name);
+	ret = stat(line, &st);
+	if (ret == 0)
+		return trace_name;
+
+	cur = strrchr(trace_name, '.');
+	if (!cur) {
+		return trace_name;
+	}
+
+	tmp = strndup(trace_name, cur - trace_name);
+	snprintf(line, line_len, "%s.mpstat", tmp);
+	ret = stat(line, &st);
+	if (ret == 0)
+		return tmp;
+
+	free(tmp);
+	return trace_name;
+}
+
+int read_mpstat(struct trace *trace, char *trace_name)
+{
+	int fd;
+	struct stat st;
+	int ret;
+	char *p;
+
+	if (record_header_len == 0) {
+		record_header_len = strlen(record_header);
+	}
+
+	snprintf(line, line_len, "%s.mpstat", guess_filename(trace_name));
+	fd = open(line, O_RDONLY);
+	if (fd < 0)
+		return 0;
+
+	ret = fstat(fd, &st);
+	if (ret < 0) {
+		fprintf(stderr, "stat failed on %s err %s\n", line, strerror(errno));
+		goto fail_fd;
+	}
+	p = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	if (p == MAP_FAILED) {
+		fprintf(stderr, "Unable to mmap trace file %s, err %s\n", line, strerror(errno));
+		goto fail_fd;
+	}
+	trace->mpstat_start = p;
+	trace->mpstat_len = st.st_size;
+	trace->mpstat_cur = p;
+	trace->mpstat_fd = fd;
+	find_last_mpstat_time(trace);
+	count_mpstat_cpus(trace);
+
+	first_mpstat(trace);
+
+	return 0;
+
+fail_fd:
+	close(fd);
+	return 0;
+}
+
+/*
+ * 09:56:26 AM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest   %idle
+ *
+ * or
+ *
+ * 10:18:51 AM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
+ *
+ *
+ * this reads just one line in the mpstat
+ */
+int read_mpstat_event(struct trace *trace, double *user,
+		      double *sys, double *iowait, double *irq,
+		      double *soft)
+{
+	char *cur = trace->mpstat_cur;
+	char *nptr;
+	double val;
+
+	/* jump past the date and CPU number */
+	cur += 16;
+	if (past_eof(trace, cur))
+		return 1;
+
+	/* usr time */
+	val = strtod(cur, &nptr);
+	if (val == 0 && cur == nptr)
+		return 1;
+	*user = val;
+
+	/* nice time, pitch this one */
+	cur = nptr;
+	val = strtod(cur, &nptr);
+	if (val == 0 && cur == nptr)
+		return 1;
+
+	/* system time */
+	cur = nptr;
+	val = strtod(cur, &nptr);
+	if (val == 0 && cur == nptr)
+		return 1;
+	*sys = val;
+
+	cur = nptr;
+	val = strtod(cur, &nptr);
+	if (val == 0 && cur == nptr)
+		return 1;
+	*iowait = val;
+
+	cur = nptr;
+	val = strtod(cur, &nptr);
+	if (val == 0 && cur == nptr)
+		return 1;
+	*irq = val;
+
+	cur = nptr;
+	val = strtod(cur, &nptr);
+	if (val == 0 && cur == nptr)
+		return 1;
+	*soft = val;
+
+	return 0;
+}
+
+int add_mpstat_gld(int time, double sys, struct graph_line_data *gld)
+{
+	gld->data[time].sum = sys;
+	gld->data[time].count = 1;
+	return 0;
+
+}
diff --git a/iowatcher/mpstat.h b/iowatcher/mpstat.h
new file mode 100644
index 0000000..f3aeb65
--- /dev/null
+++ b/iowatcher/mpstat.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2012 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef __MPSTAT__
+#define __MPSTAT__
+
+int read_mpstat(struct trace *trace, char *trace_name);
+char *next_mpstat(struct trace *trace);
+char *first_mpstat(struct trace *trace);
+int read_mpstat_event(struct trace *trace, double *user,
+		      double *sys, double *iowait, double *irq,
+		      double *soft);
+int next_mpstat_line(struct trace *trace);
+int add_mpstat_gld(int time, double sys, struct graph_line_data *gld);
+#endif
diff --git a/iowatcher/plot.c b/iowatcher/plot.c
new file mode 100644
index 0000000..8dd112d
--- /dev/null
+++ b/iowatcher/plot.c
@@ -0,0 +1,1119 @@
+/*
+ * Copyright (C) 2012 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ *  Parts of this file were imported from Jens Axboe's blktrace sources (also GPL)
+ */
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+#include <inttypes.h>
+#include <string.h>
+#include <asm/types.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <math.h>
+
+#include "plot.h"
+
+static int io_graph_scale = 8;
+static int graph_width = 700;
+static int graph_height = 250;
+static int graph_circle_extra = 30;
+static int graph_inner_x_margin = 2;
+static int graph_inner_y_margin = 2;
+static int graph_tick_len = 5;
+static int graph_left_pad = 120;
+static int tick_label_pad = 16;
+static int tick_font_size = 15;
+static char *font_family = "sans-serif";
+
+/* this is the title for the whole page */
+static int plot_title_height = 50;
+static int plot_title_font_size = 25;
+
+/* this is the label at the top of each plot */
+static int plot_label_height = 60;
+static int plot_label_font_size = 20;
+
+/* label for each axis is slightly smaller */
+static int axis_label_font_size = 16;
+
+int legend_x_off = 45;
+int legend_y_off = -10;
+int legend_font_size = 15;
+int legend_width = 80;
+
+static int rolling_avg_secs = 0;
+
+static int line_len = 1024;
+static char line[1024];
+
+static int final_height = 0;
+static int final_width = 0;
+
+static char *colors[] = {
+	"blue", "darkgreen",
+	"red",
+	"darkviolet",
+	"orange",
+	"aqua",
+	"brown", "#00FF00",
+	"yellow", "coral",
+	"black", "darkred",
+	"fuchsia", "crimson",
+	NULL };
+
+extern unsigned int longest_proc_name;
+
+char *pick_color(void)
+{
+	static int color_index;
+	char *ret = colors[color_index];
+
+	if (!ret) {
+		color_index = 0;
+		ret = colors[color_index];
+	}
+	color_index++;
+	return ret;
+}
+
+char *pick_fio_color(void)
+{
+	static int fio_color_index;
+	char *ret = colors[fio_color_index];
+
+	if (!ret) {
+		fio_color_index = 0;
+		ret = colors[fio_color_index];
+	}
+	fio_color_index += 2;
+	return ret;
+}
+
+static int cpu_color_index;
+
+char *pick_cpu_color(void)
+{
+	char *ret = colors[cpu_color_index];
+	if (!ret) {
+		cpu_color_index = 0;
+		ret = colors[cpu_color_index];
+	}
+	cpu_color_index++;
+	return ret;
+}
+
+void reset_cpu_color(void)
+{
+	cpu_color_index = 0;
+}
+
+struct graph_line_data *alloc_line_data(unsigned int min_seconds,
+					unsigned int max_seconds,
+					unsigned int stop_seconds)
+{
+	int size = sizeof(struct graph_line_data) + (stop_seconds + 1) * sizeof(struct graph_line_pair);
+	struct graph_line_data *gld;
+
+	gld = calloc(1, size);
+	if (!gld) {
+		fprintf(stderr, "Unable to allocate memory for graph data\n");
+		exit(1);
+	}
+	gld->min_seconds = min_seconds;
+	gld->max_seconds = max_seconds;
+	gld->stop_seconds = stop_seconds;
+	return gld;
+}
+
+struct graph_dot_data *alloc_dot_data(unsigned int min_seconds,
+				      unsigned int max_seconds,
+				      u64 min_offset, u64 max_offset,
+				      unsigned int stop_seconds,
+				      char *color, char *label)
+{
+	int size;
+	int arr_size;
+	int rows = graph_height * io_graph_scale;
+	int cols = graph_width;
+	struct graph_dot_data *gdd;
+
+	size = sizeof(struct graph_dot_data);
+
+	/* the number of bits */
+	arr_size = (rows + 1) * cols;
+
+	/* the number of bytes */
+	arr_size = (arr_size + 7) / 8;
+
+	gdd = calloc(1, size + arr_size);
+	if (!gdd) {
+		fprintf(stderr, "Unable to allocate memory for graph data\n");
+		exit(1);
+	}
+	gdd->min_seconds = min_seconds;
+	gdd->max_seconds = max_seconds;
+	gdd->stop_seconds = stop_seconds;
+	gdd->rows = rows;
+	gdd->cols = cols;
+	gdd->min_offset = min_offset;
+	gdd->max_offset = max_offset;
+	gdd->color = color;
+	gdd->label = label;
+
+	if (strlen(label) > longest_proc_name)
+		longest_proc_name = strlen(label);
+
+	return gdd;
+}
+
+void set_gdd_bit(struct graph_dot_data *gdd, u64 offset, double bytes, double time)
+{
+	double bytes_per_row = (double)(gdd->max_offset - gdd->min_offset + 1) / gdd->rows;
+	double secs_per_col = (double)(gdd->max_seconds - gdd->min_seconds) / gdd->cols;
+	double col;
+	double row;
+	int col_int;
+	int row_int;
+	int bit_index;
+	int arr_index;
+	int bit_mod;
+	double mod = bytes_per_row;
+
+	if (offset > gdd->max_offset || offset < gdd->min_offset)
+		return;
+	time = time / 1000000000.0;
+	if (time < gdd->min_seconds || time > gdd->max_seconds)
+		return;
+	gdd->total_ios++;
+	while (bytes > 0 && offset <= gdd->max_offset) {
+		row = (double)(offset - gdd->min_offset) / bytes_per_row;
+		col = (time - gdd->min_seconds) / secs_per_col;
+
+		col_int = floor(col);
+		row_int = floor(row);
+		bit_index = row_int * gdd->cols + col_int;
+		arr_index = bit_index / 8;
+		bit_mod = bit_index % 8;
+
+		gdd->data[arr_index] |= 1 << bit_mod;
+		offset += mod;
+		bytes -= mod;
+	}
+}
+
+static double rolling_avg(struct graph_line_pair *data, int index, int distance)
+{
+	double sum = 0;
+	int start;
+
+	if (distance < 0)
+		distance = 1;
+	if (distance > index) {
+		start = 0;
+	} else {
+		start = index - distance;
+	}
+	distance = 0;
+	while (start <= index) {
+		double avg;
+
+		if (data[start].count)
+			avg = ((double)data[start].sum) / data[start].count;
+		else
+			avg= 0;
+
+		sum += avg;
+		distance++;
+		start++;
+	}
+	return sum / distance;
+}
+
+void write_svg_header(int fd)
+{
+	char *spaces = "                                                    \n";
+	char *header = "<svg  xmlns=\"http://www.w3.org/2000/svg\">\n";
+	char *filter1 ="<filter id=\"shadow\">\n "
+		"<feOffset result=\"offOut\" in=\"SourceAlpha\" dx=\"4\" dy=\"4\" />\n "
+		"<feGaussianBlur result=\"blurOut\" in=\"offOut\" stdDeviation=\"2\" />\n "
+		"<feBlend in=\"SourceGraphic\" in2=\"blurOut\" mode=\"normal\" />\n "
+		"</filter>\n";
+	char *filter2 ="<filter id=\"textshadow\" x=\"0\" y=\"0\" width=\"200%\" height=\"200%\">\n "
+		"<feOffset result=\"offOut\" in=\"SourceAlpha\" dx=\"1\" dy=\"1\" />\n "
+		"<feGaussianBlur result=\"blurOut\" in=\"offOut\" stdDeviation=\"1.5\" />\n "
+		"<feBlend in=\"SourceGraphic\" in2=\"blurOut\" mode=\"normal\" />\n "
+		"</filter>\n";
+	char *filter3 ="<filter id=\"labelshadow\" x=\"0\" y=\"0\" width=\"200%\" height=\"200%\">\n "
+		"<feOffset result=\"offOut\" in=\"SourceGraphic\" dx=\"3\" dy=\"3\" />\n "
+		"<feColorMatrix result=\"matrixOut\" in=\"offOut\" type=\"matrix\" "
+		"values=\"0.2 0 0 0 0 0 0.2 0 0 0 0 0 0.2 0 0 0 0 0 1 0\" /> "
+		"<feGaussianBlur result=\"blurOut\" in=\"offOut\" stdDeviation=\"2\" />\n "
+		"<feBlend in=\"SourceGraphic\" in2=\"blurOut\" mode=\"normal\" />\n "
+		"</filter>\n";
+	char *defs_start = "<defs>\n";
+	char *defs_close = "</defs>\n";
+	final_width = 0;
+	final_height = 0;
+
+	write(fd, header, strlen(header));
+	/* write a bunch of spaces so we can stuff in the width and height later */
+	write(fd, spaces, strlen(spaces));
+	write(fd, spaces, strlen(spaces));
+	write(fd, spaces, strlen(spaces));
+
+	write(fd, defs_start, strlen(defs_start));
+	write(fd, filter1, strlen(filter1));
+	write(fd, filter2, strlen(filter2));
+	write(fd, filter3, strlen(filter3));
+	write(fd, defs_close, strlen(defs_close));
+}
+
+/* svg y offset for the traditional 0,0 (bottom left corner) of the plot */
+static int axis_y(void)
+{
+	return plot_label_height + graph_height + graph_inner_y_margin;
+}
+
+/* this gives you the correct pixel for a given offset from the bottom left y axis */
+static double axis_y_off_double(double y)
+{
+	return plot_label_height + graph_height - y;
+}
+
+static int axis_y_off(int y)
+{
+	return axis_y_off_double(y);
+}
+
+/* svg x axis offset from 0 */
+static int axis_x(void)
+{
+	return graph_left_pad;
+}
+
+/* the correct pixel for a given X offset */
+static double axis_x_off_double(double x)
+{
+	return graph_left_pad + graph_inner_x_margin + x;
+}
+
+static int axis_x_off(int x)
+{
+	return (int)axis_x_off_double(x);
+}
+
+/*
+ * this draws a backing rectangle for the plot and it
+ * also creates a new svg element so our offsets can
+ * be relative to this one plot.
+ */
+void setup_axis(struct plot *plot)
+{
+	int ret;
+	int len;
+	int fd = plot->fd;
+	int bump_height = tick_font_size * 3 + axis_label_font_size;
+	int local_legend_width = legend_width;
+
+	if (plot->no_legend)
+		local_legend_width = 0;
+
+	plot->total_width = axis_x_off(graph_width) + graph_left_pad / 2 + local_legend_width;
+	plot->total_height = axis_y() + tick_label_pad + tick_font_size;
+
+	if (plot->add_xlabel)
+		plot->total_height += bump_height;
+
+	/* backing rect */
+	snprintf(line, line_len, "<rect x=\"%d\" y=\"%d\" width=\"%d\" "
+		 "height=\"%d\" fill=\"white\" stroke=\"none\"/>",
+		 plot->start_x_offset,
+		plot->start_y_offset, plot->total_width + 40,
+		plot->total_height + 20);
+	len = strlen(line);
+	write(fd, line, len);
+
+	snprintf(line, line_len, "<rect x=\"%d\" y=\"%d\" width=\"%d\" "
+		 "filter=\"url(#shadow)\" "
+		 "height=\"%d\" fill=\"white\" stroke=\"none\"/>",
+		 plot->start_x_offset + 15,
+		plot->start_y_offset, plot->total_width, plot->total_height);
+	len = strlen(line);
+	write(fd, line, len);
+	plot->total_height += 20;
+	plot->total_width += 20;
+
+	if (plot->total_height + plot->start_y_offset > final_height)
+		final_height = plot->total_height + plot->start_y_offset;
+	if (plot->start_x_offset + plot->total_width + 40 > final_width)
+		final_width = plot->start_x_offset + plot->total_width + 40;
+
+	/* create an svg object for all our coords to be relative against */
+	snprintf(line, line_len, "<svg x=\"%d\" y=\"%d\">\n", plot->start_x_offset, plot->start_y_offset);
+	write(fd, line, strlen(line));
+
+	snprintf(line, 1024, "<path d=\"M%d %d h %d V %d H %d Z\" stroke=\"black\" stroke-width=\"2\" fill=\"none\"/>\n",
+		 axis_x(), axis_y(),
+		 graph_width + graph_inner_x_margin * 2, axis_y_off(graph_height) - graph_inner_y_margin,
+		 axis_x());
+	len = strlen(line);
+	ret = write(fd, line, len);
+	if (ret != len) {
+		fprintf(stderr, "failed to write svg axis\n");
+		exit(1);
+	}
+}
+
+/*
+ * this draws a backing rectangle for the plot and it
+ * also creates a new svg element so our offsets can
+ * be relative to this one plot.
+ */
+void setup_axis_spindle(struct plot *plot)
+{
+	int len;
+	int fd = plot->fd;
+	int bump_height = tick_font_size * 3 + axis_label_font_size;
+
+	legend_x_off = -60;
+
+	plot->total_width = axis_x_off(graph_width) + legend_width;
+	plot->total_height = axis_y() + tick_label_pad + tick_font_size;
+
+	if (plot->add_xlabel)
+		plot->total_height += bump_height;
+
+	/* backing rect */
+	snprintf(line, line_len, "<rect x=\"%d\" y=\"%d\" width=\"%d\" "
+		 "height=\"%d\" fill=\"white\" stroke=\"none\"/>",
+		 plot->start_x_offset,
+		plot->start_y_offset, plot->total_width + 10,
+		plot->total_height + 20);
+	len = strlen(line);
+	write(fd, line, len);
+
+	snprintf(line, line_len, "<rect x=\"%d\" y=\"%d\" width=\"%d\" "
+		 "filter=\"url(#shadow)\" "
+		 "height=\"%d\" fill=\"white\" stroke=\"none\"/>",
+		 plot->start_x_offset + 15,
+		plot->start_y_offset, plot->total_width - 30,
+		plot->total_height);
+	len = strlen(line);
+	write(fd, line, len);
+	plot->total_height += 20;
+
+	if (plot->total_height + plot->start_y_offset > final_height)
+		final_height = plot->total_height + plot->start_y_offset;
+	if (plot->start_x_offset + plot->total_width + 40 > final_width)
+		final_width = plot->start_x_offset + plot->total_width + 40;
+
+	/* create an svg object for all our coords to be relative against */
+	snprintf(line, line_len, "<svg x=\"%d\" y=\"%d\">\n", plot->start_x_offset, plot->start_y_offset);
+	write(fd, line, strlen(line));
+
+}
+
+/* draw a plot title.  This should be done only once,
+ * and it bumps the plot width/height numbers by
+ * what it draws.
+ *
+ * Call this before setting up the first axis
+ */
+void set_plot_title(struct plot *plot, char *title)
+{
+	int len;
+	int fd = plot->fd;
+
+	plot->total_height = plot_title_height;
+	plot->total_width = axis_x_off(graph_width) + graph_left_pad / 2 + legend_width;
+
+	/* backing rect */
+	snprintf(line, line_len, "<rect x=\"0\" y=\"%d\" width=\"%d\" height=\"%d\" fill=\"white\" stroke=\"none\"/>",
+		plot->start_y_offset, plot->total_width + 40, plot_title_height + 20);
+	len = strlen(line);
+	write(fd, line, len);
+
+	snprintf(line, line_len, "<text x=\"%d\" y=\"%d\" font-family=\"%s\" font-size=\"%d\" "
+		 "font-weight=\"bold\" fill=\"black\" style=\"text-anchor: %s\">%s</text>\n",
+		 axis_x_off(graph_width / 2),
+		plot->start_y_offset + plot_title_height / 2,
+		font_family, plot_title_font_size, "middle", title);
+	plot->start_y_offset += plot_title_height;
+	len = strlen(line);
+	write(fd, line, len);
+}
+
+#define TICK_MINI_STEPS 3
+
+static double find_step(double first, double last, int num_ticks)
+{
+	int mini_step[TICK_MINI_STEPS] = { 1, 2, 5 };
+	int cur_mini_step = 0;
+	double step = (last - first) / num_ticks;
+	double log10 = log(10);
+
+	/* Round to power of 10 */
+	step = exp(floor(log(step) / log10) * log10);
+	/* Scale down step to provide enough ticks */
+	while (cur_mini_step < TICK_MINI_STEPS
+	       && (last - first) / (step * mini_step[cur_mini_step]) > num_ticks)
+		cur_mini_step++;
+
+	if (cur_mini_step > 0)
+		step *= mini_step[cur_mini_step - 1];
+
+	return step;
+}
+
+/*
+ * create evenly spread out ticks along the xaxis.  if tick only is set
+ * this just makes the ticks, otherwise it labels each tick as it goes
+ */
+void set_xticks(struct plot *plot, int num_ticks, int first, int last)
+{
+	int pixels_per_tick;
+	double step;
+	int i;
+	int tick_y = axis_y_off(graph_tick_len) + graph_inner_y_margin;
+	int tick_x = axis_x();
+	int tick_only = plot->add_xlabel == 0;
+
+	int text_y = axis_y() + tick_label_pad;
+
+	char *middle = "middle";
+	char *start = "start";
+
+	step = find_step(first, last, num_ticks);
+	/*
+	 * We don't want last two ticks to be too close together so subtract
+	 * 20% of the step from the interval
+	 */
+	num_ticks = (double)(last - first - step) / step + 1;
+	pixels_per_tick = graph_width * step / (double)(last - first);
+
+	for (i = 0; i < num_ticks; i++) {
+		char *anchor;
+		if (i != 0) {
+			snprintf(line, line_len, "<rect x=\"%d\" y=\"%d\" width=\"2\" height=\"%d\" style=\"stroke:none;fill:black;\"/>\n",
+				tick_x, tick_y, graph_tick_len);
+			write(plot->fd, line, strlen(line));
+			anchor = middle;
+		} else {
+			anchor = start;
+		}
+
+		if (!tick_only) {
+			if (step >= 1)
+				snprintf(line, line_len, "<text x=\"%d\" y=\"%d\" font-family=\"%s\" font-size=\"%d\" "
+					"fill=\"black\" style=\"text-anchor: %s\">%d</text>\n",
+					tick_x, text_y, font_family, tick_font_size, anchor,
+					(int)(first + step * i));
+			else
+				snprintf(line, line_len, "<text x=\"%d\" y=\"%d\" font-family=\"%s\" font-size=\"%d\" "
+					"fill=\"black\" style=\"text-anchor: %s\">%.2f</text>\n",
+					tick_x, text_y, font_family, tick_font_size, anchor,
+					first + step * i);
+			write(plot->fd, line, strlen(line));
+		}
+		tick_x += pixels_per_tick;
+	}
+
+	if (!tick_only) {
+		if (step >= 1)
+			snprintf(line, line_len, "<text x=\"%d\" y=\"%d\" font-family=\"%s\" font-size=\"%d\" "
+				"fill=\"black\" style=\"text-anchor: middle\">%d</text>\n",
+				axis_x_off(graph_width - 2),
+				text_y, font_family, tick_font_size, last);
+		else
+			snprintf(line, line_len, "<text x=\"%d\" y=\"%d\" font-family=\"%s\" font-size=\"%d\" "
+				"fill=\"black\" style=\"text-anchor: middle\">%.2f</text>\n",
+				axis_x_off(graph_width - 2),
+				text_y, font_family, tick_font_size, (double)last);
+		write(plot->fd, line, strlen(line));
+	}
+}
+
+void set_ylabel(struct plot *plot, char *label)
+{
+	int len;
+	int fd = plot->fd;
+
+	snprintf(line, line_len, "<text x=\"%d\" y=\"%d\" font-family=\"%s\" "
+		 "transform=\"rotate(-90 %d %d)\" font-weight=\"bold\" "
+		 "font-size=\"%d\" fill=\"black\" style=\"text-anchor: %s\">%s</text>\n",
+		 graph_left_pad / 2 - axis_label_font_size,
+		 axis_y_off(graph_height / 2),
+		 font_family,
+		 graph_left_pad / 2 - axis_label_font_size,
+		 (int)axis_y_off(graph_height / 2),
+		 axis_label_font_size, "middle", label);
+	len = strlen(line);
+	write(fd, line, len);
+}
+
+void set_xlabel(struct plot *plot, char *label)
+{
+	int len;
+	int fd = plot->fd;
+	snprintf(line, line_len, "<text x=\"%d\" y=\"%d\" font-family=\"%s\" "
+		 "font-weight=\"bold\" "
+		 "font-size=\"%d\" fill=\"black\" style=\"text-anchor: %s\">%s</text>\n",
+		 axis_x_off(graph_width / 2),
+		 axis_y() + tick_font_size * 3 + axis_label_font_size / 2,
+		 font_family,
+		 axis_label_font_size, "middle", label);
+	len = strlen(line);
+	write(fd, line, len);
+
+}
+
+/*
+ * create evenly spread out ticks along the y axis.
+ * The ticks are labeled as it goes
+ */
+void set_yticks(struct plot *plot, int num_ticks, int first, int last, char *units)
+{
+	int pixels_per_tick = graph_height / num_ticks;
+	int step = (last - first) / num_ticks;
+	int i;
+	int tick_y = 0;
+	int text_x = axis_x() - 6;
+	int tick_x = axis_x();
+	char *anchor = "end";
+
+	for (i = 0; i < num_ticks; i++) {
+		if (i != 0) {
+			snprintf(line, line_len, "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" "
+				 "style=\"stroke:lightgray;stroke-width:2;stroke-dasharray:9,12;\"/>\n",
+				tick_x, axis_y_off(tick_y),
+				axis_x_off(graph_width), axis_y_off(tick_y));
+			write(plot->fd, line, strlen(line));
+		}
+
+		snprintf(line, line_len, "<text x=\"%d\" y=\"%d\" font-family=\"%s\" font-size=\"%d\" "
+			 "fill=\"black\" style=\"text-anchor: %s\">%d%s</text>\n",
+			text_x,
+			axis_y_off(tick_y - tick_font_size / 2),
+			font_family, tick_font_size, anchor, first + step * i, units);
+		write(plot->fd, line, strlen(line));
+		tick_y += pixels_per_tick;
+	}
+	snprintf(line, line_len, "<text x=\"%d\" y=\"%d\" font-family=\"%s\" font-size=\"%d\" "
+		 "fill=\"black\" style=\"text-anchor: %s\">%d%s</text>\n",
+		 text_x, axis_y_off(graph_height), font_family, tick_font_size, anchor, last, units);
+	write(plot->fd, line, strlen(line));
+}
+
+void set_plot_label(struct plot *plot, char *label)
+{
+	int len;
+	int fd = plot->fd;
+
+	snprintf(line, line_len, "<text x=\"%d\" y=\"%d\" font-family=\"%s\" "
+		 "font-size=\"%d\" fill=\"black\" style=\"text-anchor: %s\">%s</text>\n",
+		 axis_x() + graph_width / 2,
+		 plot_label_height / 2,
+		font_family, plot_label_font_size, "middle", label);
+	len = strlen(line);
+	write(fd, line, len);
+}
+
+static void close_svg(int fd)
+{
+	char *close_line = "</svg>\n";
+
+	write(fd, close_line, strlen(close_line));
+}
+
+int close_plot(struct plot *plot)
+{
+	close_svg(plot->fd);
+	if (plot->direction == PLOT_DOWN)
+		plot->start_y_offset += plot->total_height;
+	else if (plot->direction == PLOT_ACROSS)
+		plot->start_x_offset += plot->total_width;
+	return 0;
+}
+
+struct plot *alloc_plot(void)
+{
+	struct plot *plot;
+	plot = calloc(1, sizeof(*plot));
+	if (!plot) {
+		fprintf(stderr, "Unable to allocate memory %s\n", strerror(errno));
+		exit(1);
+	}
+	plot->fd = 0;
+	return plot;
+}
+
+int close_plot_file(struct plot *plot)
+{
+	int ret;
+	ret = lseek(plot->fd, 0, SEEK_SET);
+	if (ret == (off_t)-1) {
+		perror("seek");
+		exit(1);
+	}
+	final_width = ((final_width  + 1) / 2) * 2;
+	final_height = ((final_height  + 1) / 2) * 2;
+	snprintf(line, line_len, "<svg  xmlns=\"http://www.w3.org/2000/svg\" "
+		 "width=\"%d\" height=\"%d\">\n",
+		 final_width, final_height);
+	write(plot->fd, line, strlen(line));
+	snprintf(line, line_len, "<rect x=\"0\" y=\"0\" width=\"%d\" "
+		 "height=\"%d\" fill=\"white\"/>\n", final_width, final_height);
+	write(plot->fd, line, strlen(line));
+	close(plot->fd);
+	plot->fd = 0;
+	return 0;
+}
+
+void set_plot_output(struct plot *plot, char *filename)
+{
+	int fd;
+
+	if (plot->fd)
+		close_plot_file(plot);
+	fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, 0600);
+	if (fd < 0) {
+		fprintf(stderr, "Unable to open output file %s err %s\n", filename, strerror(errno));
+		exit(1);
+	}
+	plot->fd = fd;
+	plot->start_y_offset = plot->start_x_offset = 0;
+	write_svg_header(fd);
+}
+
+char *byte_unit_names[] = { "", "K", "M", "G", "T", "P", "E", "Z", "Y", "unobtainium" };
+int MAX_BYTE_UNIT_SCALE = 9;
+
+char *time_unit_names[] = { "n", "u", "m", "s" };
+int MAX_TIME_UNIT_SCALE = 3;
+
+void scale_line_graph_bytes(u64 *max, char **units, u64 factor)
+{
+	int scale = 0;
+	u64 val = *max;
+	u64 div = 1;
+	while (val > factor * 64) {
+		val /= factor;
+		scale++;
+		div *= factor;
+	}
+	*units = byte_unit_names[scale];
+	if (scale == 0)
+		return;
+
+	if (scale > MAX_BYTE_UNIT_SCALE)
+		scale = MAX_BYTE_UNIT_SCALE;
+
+	*max /= div;
+}
+
+void scale_line_graph_time(u64 *max, char **units)
+{
+	int scale = 0;
+	u64 val = *max;
+	u64 div = 1;
+	while (val > 1000 * 10) {
+		val /= 1000;
+		scale++;
+		div *= 1000;
+		if (scale == MAX_TIME_UNIT_SCALE)
+			break;
+	}
+	*units = time_unit_names[scale];
+	if (scale == 0)
+		return;
+
+	*max /= div;
+}
+
+int svg_line_graph(struct plot *plot, struct graph_line_data *gld, char *color, int thresh1, int thresh2)
+{
+	unsigned int i;
+	double val;
+	double avg;
+	int rolling;
+	int fd = plot->fd;
+	char *start = "<path d=\"";
+	double yscale = ((double)gld->max) / graph_height;
+	double xscale = (double)(gld->max_seconds - gld->min_seconds - 1) / graph_width;
+	char c = 'M';
+	double x;
+	int printed_header = 0;
+	int printed_lines = 0;
+
+	if (thresh1 && thresh2)
+		rolling = 0;
+	else if (rolling_avg_secs)
+		rolling = rolling_avg_secs;
+	else
+		rolling = (gld->stop_seconds - gld->min_seconds) / 25;
+
+	for (i = gld->min_seconds; i < gld->stop_seconds; i++) {
+		avg = rolling_avg(gld->data, i, rolling);
+		if (yscale == 0)
+			val = 0;
+		else
+			val = avg / yscale;
+
+		if (val > graph_height)
+			val = graph_height;
+		if (val < 0)
+			val = 0;
+
+		x = (double)(i - gld->min_seconds) / xscale;
+		if (!thresh1 && !thresh2) {
+
+			if (!printed_header) {
+				write(fd, start, strlen(start));
+				printed_header = 1;
+			}
+
+			/* in full line mode, everything in the graph is connected */
+			snprintf(line, line_len, "%c %d %d ", c, axis_x_off(x), axis_y_off(val));
+			c = 'L';
+			write(fd, line, strlen(line));
+			printed_lines = 1;
+		} else if (avg > thresh1 || avg > thresh2) {
+			int len = 10;
+			if (!printed_header) {
+				write(fd, start, strlen(start));
+				printed_header = 1;
+			}
+
+			/* otherwise, we just print a bar up there to show this one data point */
+			if (i >= gld->stop_seconds - 2)
+				len = -10;
+
+			/*
+			 * we don't use the rolling averages here to show high
+			 * points in the data
+			 */
+			snprintf(line, line_len, "M %d %d h %d ", axis_x_off(x),
+				 axis_y_off(val), len);
+			write(fd, line, strlen(line));
+			printed_lines = 1;
+		}
+
+	}
+	if (printed_lines) {
+		snprintf(line, line_len, "\" fill=\"none\" stroke=\"%s\" stroke-width=\"2\"/>\n", color);
+		write(fd, line, strlen(line));
+	}
+	if (plot->timeline)
+		svg_write_time_line(plot, plot->timeline);
+
+	return 0;
+}
+
+void svg_write_time_line(struct plot *plot, int col)
+{
+	snprintf(line, line_len, "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" "
+				 "style=\"stroke:black;stroke-width:2;\"/>\n",
+				 axis_x_off(col), axis_y_off(0),
+				 axis_x_off(col), axis_y_off(graph_height));
+	write(plot->fd, line, strlen(line));
+}
+
+static int svg_add_io(int fd, double row, double col, double width, double height, char *color)
+{
+	float rx = 0;
+
+	snprintf(line, line_len, "<rect x=\"%.2f\" y=\"%.2f\" width=\"%.1f\" height=\"%.1f\" "
+		 "rx=\"%.2f\" style=\"stroke:none;fill:%s;stroke-width:0\"/>\n",
+		 axis_x_off_double(col), axis_y_off_double(row), width, height, rx, color);
+	return write(fd, line, strlen(line));
+}
+
+int svg_io_graph_movie_array(struct plot *plot, struct pid_plot_history *pph)
+{
+	double cell_index;
+	double movie_row;
+	double movie_col;
+	int i;
+
+	for (i = 0; i < pph->num_used; i++) {
+		cell_index = pph->history[i];
+		movie_row = floor(cell_index / graph_width);
+		movie_col = cell_index - movie_row * graph_width;
+		svg_add_io(plot->fd, movie_row, movie_col, 4, 4, pph->color);
+	}
+	return 0;
+}
+
+static float spindle_steps = 0;
+
+void rewind_spindle_steps(int num)
+{
+	spindle_steps -= num * 0.01;
+}
+
+int svg_io_graph_movie_array_spindle(struct plot *plot, struct pid_plot_history *pph)
+{
+	double cell_index;
+	int i;
+	int num_circles = 0;
+	double cells_per_circle;
+	double circle_num;
+	double degrees_per_cell;
+	double rot;
+	double center_x;
+	double center_y;
+	double graph_width_extra = graph_width + graph_circle_extra;
+	double graph_height_extra = graph_height + graph_circle_extra;
+	double radius;;
+
+	if (graph_width_extra > graph_height_extra)
+		graph_width_extra = graph_height_extra;
+
+	if (graph_width_extra < graph_height_extra)
+		graph_height_extra = graph_width_extra;
+
+	radius = graph_width_extra;
+
+	center_x = axis_x_off_double(graph_width_extra / 2);
+	center_y = axis_y_off_double(graph_height_extra / 2);
+
+	snprintf(line, line_len, "<g transform=\"rotate(%.4f, %.2f, %.2f)\"> "
+		 "<circle cx=\"%.2f\" cy=\"%.2f\" "
+		 "stroke=\"black\" stroke-width=\"6\" "
+		 "r=\"%.2f\" fill=\"none\"/>\n",
+		 spindle_steps * 1.2, center_x, center_y, center_x, center_y, graph_width_extra / 2);
+	write(plot->fd, line, strlen(line));
+	snprintf(line, line_len, "<circle cx=\"%.2f\" cy=\"%.2f\" "
+		"stroke=\"none\" fill=\"red\" r=\"%.2f\"/>\n</g>\n",
+		axis_x_off_double(graph_width_extra), center_y, 4.5);
+	write(plot->fd, line, strlen(line));
+	spindle_steps += 0.01;
+
+	radius = floor(radius / 2);
+	num_circles = radius / 4 - 3;
+	cells_per_circle = pph->history_max / num_circles;
+	degrees_per_cell = 360 / cells_per_circle;
+
+	for (i = 0; i < pph->num_used; i++) {
+		cell_index = pph->history[i];
+		circle_num = floor(cell_index / cells_per_circle);
+		rot = cell_index - circle_num * cells_per_circle;
+		circle_num = num_circles - circle_num;
+		radius = circle_num * 4;
+
+		rot = rot * degrees_per_cell;
+		rot -= spindle_steps;
+		snprintf(line, line_len, "<path transform=\"rotate(%.4f, %.2f, %.2f)\" "
+			 "d=\"M %.2f %.2f a %.2f %.2f 0 0 1 0 5\" "
+			 "stroke=\"%s\" stroke-width=\"4\"/>\n",
+			 -rot, center_x, center_y,
+			 axis_x_off_double(graph_width_extra / 2 + radius) + 8, center_y,
+			 radius, radius, pph->color);
+
+		write(plot->fd, line, strlen(line));
+	}
+	return 0;
+}
+
+static int add_plot_history(struct pid_plot_history *pph, double val)
+{
+	if (pph->num_used == pph->history_len) {
+		pph->history_len += 4096;
+		pph->history = realloc(pph->history,
+				       pph->history_len * sizeof(double));
+		if (!pph->history) {
+			perror("Unable to allocate memory");
+			exit(1);
+		}
+	}
+	pph->history[pph->num_used++] = val;
+	return 0;
+}
+
+int svg_io_graph_movie(struct graph_dot_data *gdd, struct pid_plot_history *pph, int col)
+{
+	int row = 0;
+	int arr_index;
+	unsigned char val;
+	int bit_index;
+	int bit_mod;
+	double blocks_per_row = (gdd->max_offset - gdd->min_offset + 1) / gdd->rows;
+	double movie_blocks_per_cell = (gdd->max_offset - gdd->min_offset + 1) / (graph_width * graph_height);
+	double cell_index;
+	int margin_orig = graph_inner_y_margin;
+
+	graph_inner_y_margin += 5;
+	pph->history_max = (gdd->max_offset - gdd->min_offset + 1) / movie_blocks_per_cell;
+
+	for (row = gdd->rows - 1; row >= 0; row--) {
+		bit_index = row * gdd->cols + col;
+		arr_index = bit_index / 8;
+		bit_mod = bit_index % 8;
+
+		if (arr_index < 0)
+			continue;
+		val = gdd->data[arr_index];
+		if (val & (1 << bit_mod)) {
+			/* in bytes, linear offset from the start of the drive */
+			cell_index = (double)row * blocks_per_row;
+
+			/* a cell number in the graph */
+			cell_index /= movie_blocks_per_cell;
+
+			add_plot_history(pph, cell_index);
+		}
+	}
+	graph_inner_y_margin = margin_orig;
+	return 0;
+}
+
+int svg_io_graph(struct plot *plot, struct graph_dot_data *gdd)
+{
+	int fd = plot->fd;;
+	int col = 0;
+	int row = 0;
+	int arr_index;
+	unsigned char val;
+	int bit_index;
+	int bit_mod;
+
+	for (row = gdd->rows - 1; row >= 0; row--) {
+		for (col = 0; col < gdd->cols; col++) {
+			bit_index = row * gdd->cols + col;
+			arr_index = bit_index / 8;
+			bit_mod = bit_index % 8;
+
+			if (arr_index < 0)
+				continue;
+			val = gdd->data[arr_index];
+			if (val & (1 << bit_mod))
+				svg_add_io(fd, floor(row / io_graph_scale), col, 1.5, 1.5, gdd->color);
+		}
+	}
+	return 0;
+}
+
+void svg_alloc_legend(struct plot *plot, int num_lines)
+{
+	char **lines = calloc(num_lines, sizeof(char *));
+	plot->legend_index = 0;
+	plot->legend_lines = lines;
+	plot->num_legend_lines = num_lines;
+}
+
+void svg_free_legend(struct plot *plot)
+{
+	int i;
+	for (i = 0; i < plot->legend_index; i++)
+		free(plot->legend_lines[i]);
+	free(plot->legend_lines);
+	plot->legend_lines = NULL;
+	plot->legend_index = 0;
+}
+
+void svg_write_legend(struct plot *plot)
+{
+	int legend_line_x = axis_x_off(graph_width) + legend_x_off;
+	int legend_line_y = axis_y_off(graph_height) + legend_y_off;
+	int i;
+
+	if (plot->legend_index == 0)
+		return;
+
+	snprintf(line, line_len, "<rect x=\"%d\" y=\"%d\" width=\"%d\" height=\"%d\" "
+		 "fill=\"white\" filter=\"url(#shadow)\"/>\n",
+		 legend_line_x - 15,
+		 legend_line_y - 12,
+		 legend_width,
+		 plot->legend_index * legend_font_size + legend_font_size / 2 + 12);
+
+	write(plot->fd, line, strlen(line));
+	for (i = 0; i < plot->legend_index; i++) {
+		write(plot->fd, plot->legend_lines[i],
+		      strlen(plot->legend_lines[i]));
+		free(plot->legend_lines[i]);
+	}
+	free(plot->legend_lines);
+	plot->legend_lines = NULL;
+	plot->legend_index = 0;
+}
+
+void svg_add_legend(struct plot *plot, char *text, char *extra, char *color)
+{
+	int legend_line_x = axis_x_off(graph_width) + legend_x_off;
+	int legend_line_y = axis_y_off(graph_height) + legend_y_off;
+
+	if (!text && (!extra || strlen(extra) == 0))
+		return;
+
+	legend_line_y += plot->legend_index * legend_font_size + legend_font_size / 2;
+	snprintf(line, line_len, "<path d=\"M %d %d h 8\" stroke=\"%s\" stroke-width=\"8\" "
+		 "filter=\"url(#labelshadow)\"/> "
+		 "<text x=\"%d\" y=\"%d\" font-family=\"%s\" font-size=\"%d\" "
+		 "fill=\"black\" style=\"text-anchor: left\">%s%s</text>\n",
+		 legend_line_x, legend_line_y,
+		 color, legend_line_x + 13,
+		 legend_line_y + 4, font_family, legend_font_size,
+		 text, extra);
+
+	plot->legend_lines[plot->legend_index++] = strdup(line);
+}
+
+void set_legend_width(int longest_str)
+{
+	if (longest_str)
+		legend_width = longest_str * (legend_font_size * 3 / 4) + 25;
+	else
+		legend_width = 0;
+}
+
+void set_rolling_avg(int rolling)
+{
+	rolling_avg_secs = rolling;
+}
+
+void set_io_graph_scale(int scale)
+{
+	io_graph_scale = scale;
+}
+
+void set_graph_size(int width, int height)
+{
+	graph_width = width;
+	graph_height = height;
+}
+
+void get_graph_size(int *width, int *height)
+{
+	*width = graph_width;
+	*height = graph_height;
+}
+
+void set_graph_height(int h)
+{
+	graph_height = h;
+}
+void set_graph_width(int w)
+{
+	graph_width = w;
+}
diff --git a/iowatcher/plot.h b/iowatcher/plot.h
new file mode 100644
index 0000000..7e87b1d
--- /dev/null
+++ b/iowatcher/plot.h
@@ -0,0 +1,179 @@
+/*
+ * Copyright (C) 2012 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ */
+#ifndef __IOWATCH_PLOT__
+#define __IOWATCH_PLOT__
+#define MAX_TICKS 10
+
+#include "list.h"
+
+typedef __u64 u64;
+typedef __u32 u32;
+typedef __u16 u16;
+
+
+/* values for the plot direction field */
+#define PLOT_DOWN 0
+#define PLOT_ACROSS 1
+
+struct plot {
+	int fd;
+
+	/* svg style y = 0 is the top of the graph */
+	int start_y_offset;
+
+	/* abs coords of the start of X start of the plot */
+	int start_x_offset;
+
+	int add_xlabel;
+	int no_legend;
+
+	/*
+	 * these two are for anyone that wants
+	 * to add a plot after this one, it tells
+	 * them how much space we took up
+	 */
+	int total_height;
+	int total_width;
+	char **legend_lines;
+	int legend_index;
+	int num_legend_lines;
+	int direction;
+
+	/*
+	 * timeline is a vertical line through line graphs that
+	 * is used by the movie mode to show where in the graph
+	 * our current frame lives
+	 */
+	int timeline;
+};
+
+struct graph_line_pair {
+	u64 count;
+	u64 sum;
+};
+
+struct graph_line_data {
+	/* beginning of an interval displayed by this graph */
+	unsigned int min_seconds;
+
+	/* end of an interval displayed by this graph */
+	unsigned int max_seconds;
+
+	unsigned int stop_seconds;
+
+	/* Y max */
+	u64 max;
+
+	/* label for this graph */
+	char *label;
+	struct graph_line_pair data[];
+};
+
+struct graph_dot_data {
+	u64 min_offset;
+	u64 max_offset;
+	u64 max_bank;
+	u64 max_bank_offset;
+	u64 total_ios;
+	u64 total_bank_ios;
+
+	int add_bank_ios;
+
+	/* in pixels, number of rows in our bitmap */
+	int rows;
+	/* in pixels, number of cols in our bitmap */
+	int cols;
+
+	/* beginning of an interval displayed by this graph */
+	int min_seconds;
+
+	/* end of an interval displayed by this graph */
+	unsigned int max_seconds;
+	unsigned int stop_seconds;
+
+	/* label for the legend */
+	char *label;
+
+	/* color for plotting data */
+	char *color;
+
+	/* bitmap, one bit for each cell to light up */
+	unsigned char data[];
+};
+
+struct pid_plot_history {
+	double history_max;
+	int history_len;
+	int num_used;
+	char *color;
+	double *history;
+};
+
+struct plot_history {
+	struct list_head list;
+	int pid_history_count;
+	int col;
+	struct pid_plot_history **read_pid_history;
+	struct pid_plot_history **write_pid_history;
+};
+
+char *pick_color(void);
+char *pick_fio_color(void);
+char *pick_cpu_color(void);
+void reset_cpu_color(void);
+int svg_io_graph(struct plot *plot, struct graph_dot_data *gdd);
+int svg_line_graph(struct plot *plot, struct graph_line_data *gld, char *color, int thresh1, int thresh2);
+struct graph_line_data *alloc_line_data(unsigned int min_seconds, unsigned int max_seconds, unsigned int stop_seconds);
+struct graph_dot_data *alloc_dot_data(unsigned int min_seconds, unsigned int max_seconds, u64 min_offset, u64 max_offset, unsigned int stop_seconds, char *color, char *label);
+void set_gdd_bit(struct graph_dot_data *gdd, u64 offset, double bytes, double time);
+void write_svg_header(int fd);
+struct plot *alloc_plot(void);
+int close_plot(struct plot *plot);
+int close_plot_no_height(struct plot *plot);
+void setup_axis(struct plot *plot);
+void set_xticks(struct plot *plot, int num_ticks, int first, int last);
+void set_yticks(struct plot *plot, int num_ticks, int first, int last, char *units);
+void set_plot_title(struct plot *plot, char *title);
+void set_plot_label(struct plot *plot, char *label);
+void set_xlabel(struct plot *plot, char *label);
+void set_ylabel(struct plot *plot, char *label);
+void scale_line_graph_bytes(u64 *max, char **units, u64 factor);
+void scale_line_graph_time(u64 *max, char **units);
+void write_drop_shadow_line(struct plot *plot);
+void svg_write_legend(struct plot *plot);
+void svg_add_legend(struct plot *plot, char *text, char *extra, char *color);
+void svg_alloc_legend(struct plot *plot, int num_lines);
+void set_legend_width(int longest_str);
+void set_rolling_avg(int rolling);
+void svg_free_legend(struct plot *plot);
+void set_io_graph_scale(int scale);
+void set_plot_output(struct plot *plot, char *filename);
+void set_graph_size(int width, int height);
+void get_graph_size(int *width, int *height);
+int svg_io_graph_movie(struct graph_dot_data *gdd, struct pid_plot_history *ph, int col);
+int svg_io_graph_movie_array(struct plot *plot, struct pid_plot_history *ph);
+void svg_write_time_line(struct plot *plot, int col);
+void set_graph_height(int h);
+void set_graph_width(int w);
+int close_plot_file(struct plot *plot);
+int svg_io_graph_movie_array_spindle(struct plot *plot, struct pid_plot_history *ph);
+void rewind_spindle_steps(int num);
+void setup_axis_spindle(struct plot *plot);
+int close_plot_col(struct plot *plot);
+
+#endif
diff --git a/iowatcher/tracers.c b/iowatcher/tracers.c
new file mode 100644
index 0000000..4c3d10d
--- /dev/null
+++ b/iowatcher/tracers.c
@@ -0,0 +1,189 @@
+/*
+ * Copyright (C) 2012 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ *  Parts of this file were imported from Jens Axboe's blktrace sources (also GPL)
+ */
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+#include <inttypes.h>
+#include <string.h>
+#include <asm/types.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <signal.h>
+#include <sys/wait.h>
+#include <spawn.h>
+
+#include "plot.h"
+#include "blkparse.h"
+#include "list.h"
+#include "tracers.h"
+
+extern char **environ;
+
+static pid_t blktrace_pid = 0;
+static pid_t mpstat_pid = 0;
+
+static void sig_handler_for_quit(int val)
+{
+	fprintf(stderr, "Received signal %d. Terminating tracers.\n", val);
+	wait_for_tracers(SIGTERM);
+}
+
+int start_blktrace(char **devices, int num_devices, char *trace_name, char *dest)
+{
+	int ret;
+	int i;
+	char *argv[15 + MAX_DEVICES_PER_TRACE * 2];
+	int argc = 0;
+
+	if (!trace_name)
+		trace_name = "trace";
+	if (!dest)
+		dest = ".";
+
+	argv[argc++] = "blktrace";
+	argv[argc++] = "-b";
+	argv[argc++] = "8192";
+	argv[argc++] = "-a";
+	argv[argc++] = "queue";
+	argv[argc++] = "-a";
+	argv[argc++] = "complete";
+	argv[argc++] = "-a";
+	argv[argc++] = "issue";
+	argv[argc++] = "-a";
+	argv[argc++] = "notify";
+
+	if (num_devices == 1) {
+		argv[argc++] = "-o";
+		argv[argc++] = trace_name;
+	} else {
+		/* Multiple devices output to a directory named trace_name */
+		dest = trace_name;
+	}
+	argv[argc++] = "-D";
+	argv[argc++] = dest;
+
+	for (i = 0; i < num_devices; i++) {
+		argv[argc++] = "-d";
+		argv[argc++] = devices[i];
+	}
+	argv[argc] = NULL;
+	signal(SIGTERM, sig_handler_for_quit);
+	signal(SIGINT, sig_handler_for_quit);
+	ret = run_program(argc, argv, 0, &blktrace_pid, NULL);
+	return ret;
+}
+
+int wait_program(pid_t pid, const char *pname, int sig)
+{
+	int status;
+	int ret = 0;
+
+	if (sig) {
+		ret = kill(pid, sig);
+		if (ret) {
+			fprintf(stderr, "Failed to send signal %d to %s (%lu): %s\n",
+			        sig, pname, (long)pid, strerror(errno));
+			return ret;
+		}
+		fprintf(stderr, "Kill (%d): %s (%ld)\n", sig, pname, (long)pid);
+	}
+
+	waitpid(pid, &status, 0);
+	if (WIFEXITED(status)) {
+		ret = WEXITSTATUS(status);
+		if (ret == 127) /* spawnp failed after forking */
+			fprintf(stderr, "Failed to run '%s'\n", pname);
+		else
+			fprintf(stderr, "Exit (%d): %s\n", ret, pname);
+	} else if (WIFSIGNALED(status) && sig && WTERMSIG(status) != sig) {
+		fprintf(stderr, "'%s' killed by signal %d\n", pname, WTERMSIG(status));
+		ret = -1;
+	}
+	return ret;
+}
+
+int run_program(int argc, char **argv, int wait, pid_t *pid, char *outpath)
+{
+	int i;
+	int err;
+	pid_t _pid;
+	posix_spawn_file_actions_t facts;
+	posix_spawn_file_actions_t *factp = NULL;
+
+	if (outpath != NULL) {
+		posix_spawn_file_actions_init(&facts);
+		posix_spawn_file_actions_addopen(&facts, 1, outpath, O_WRONLY|O_CREAT|O_TRUNC, 0600);
+		factp = &facts;
+	}
+
+	fprintf(stderr, "Start");
+	for (i = 0; i < argc; i++)
+		fprintf(stderr, " %s", argv[i]);
+	fprintf(stderr, "\n");
+
+	err = posix_spawnp(&_pid, argv[0], factp, NULL, argv, environ);
+	if (err != 0) {
+		fprintf(stderr, "Could not run '%s': %s\n", argv[0], strerror(err));
+	} else if (wait) {
+		err = wait_program(_pid, argv[0], 0);
+	} else if (!pid) {
+		fprintf(stderr, "Warning: %s (%ld): Not saving pid and not waiting for it.\n",
+		        argv[0], (long)_pid);
+	} else {
+		*pid = _pid;
+	}
+	return err;
+}
+
+int wait_for_tracers(int sig)
+{
+	int err;
+	if (blktrace_pid != 0) {
+		err = wait_program(blktrace_pid, "blktrace", sig);
+		if (err)
+			exit(1);
+		blktrace_pid = 0;
+	}
+	if (mpstat_pid != 0) {
+		err = wait_program(mpstat_pid, "mpstat", sig);
+		if (err)
+			exit(1);
+		mpstat_pid = 0;
+	}
+	return 0;
+}
+
+int start_mpstat(char *path)
+{
+	int ret;
+	int argc = 4;
+	char *argv[] = {
+		"mpstat",
+		"-P", "ALL", "1",
+		NULL,
+	};
+
+	ret = run_program(argc, argv, 0, &mpstat_pid, path);
+	return ret;
+}
diff --git a/iowatcher/tracers.h b/iowatcher/tracers.h
new file mode 100644
index 0000000..9b4f6a5
--- /dev/null
+++ b/iowatcher/tracers.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2012 Fusion-io
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public
+ *  License v2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ */
+#ifndef __IOWATCH_TRACERS
+#define __IOWATCH_TRACERS
+int run_program(int argc, char **argv, int wait, pid_t *pid, char *stdoutpath);
+int wait_program(pid_t pid, const char *pname, int signal);
+int stop_blktrace(void);
+int start_blktrace(char **devices, int num_devices, char *trace_name, char *dest);
+int start_mpstat(char *trace_name);
+int wait_for_tracers(int sig);
+
+
+#endif