€•˜	     Œsphinx.addnodes”Œdocument”“”)”}”(Œ	rawsource”Œ ”Œchildren”]”(Œtranslations”ŒLanguagesNode”“”)”}”(hhh]”(h Œpending_xref”“”)”}”(hhh]”Œdocutils.nodes”ŒText”“”ŒChinese (Simplified)”…””}”Œparent”hsbaŒ
attributes”}”(Œids”]”Œclasses”]”Œnames”]”Œdupnames”]”Œbackrefs”]”Œ	refdomain”Œstd”Œreftype”Œdoc”Œ	reftarget”Œ /translations/zh_CN/trace/ftrace”Œmodname”NŒ	classname”NŒrefexplicit”ˆuŒtagname”hhhubh)”}”(hhh]”hŒChinese (Traditional)”…””}”hh2sbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ /translations/zh_TW/trace/ftrace”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubh)”}”(hhh]”hŒItalian”…””}”hhFsbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ /translations/it_IT/trace/ftrace”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubh)”}”(hhh]”hŒJapanese”…””}”hhZsbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ /translations/ja_JP/trace/ftrace”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubh)”}”(hhh]”hŒKorean”…””}”hhnsbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ /translations/ko_KR/trace/ftrace”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubh)”}”(hhh]”hŒSpanish”…””}”hh‚sbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ /translations/sp_SP/trace/ftrace”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubeh}”(h]”h ]”h"]”h$]”h&]”Œcurrent_language”ŒEnglish”uh1h
hhŒ	_document”hŒsource”NŒline”NubhŒsection”“”)”}”(hhh]”(hŒtitle”“”)”}”(hŒftrace - Function Tracer”h]”hŒftrace - Function Tracer”…””}”(hh¨hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hh£hžhhŸŒ:/var/lib/git/docbuild/linux/Documentation/trace/ftrace.rst”h KubhŒ	paragraph”“”)”}”(hŒCopyright 2008 Red Hat Inc.”h]”hŒCopyright 2008 Red Hat Inc.”…””}”(hh¹hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Khh£hžhubhŒ
field_list”“”)”}”(hhh]”(hŒfield”“”)”}”(hhh]”(hŒ
field_name”“”)”}”(hŒAuthor”h]”hŒAuthor”…””}”(hhÓhžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÑhhÎhŸh¶h K ubhŒ
field_body”“”)”}”(hŒ$Steven Rostedt <srostedt@redhat.com>”h]”h¸)”}”(hhåh]”(hŒSteven Rostedt <”…””}”(hhçhžhhŸNh NubhŒ	reference”“”)”}”(hŒsrostedt@redhat.com”h]”hŒsrostedt@redhat.com”…””}”(hhðhžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”Œrefuri”Œmailto:srostedt@redhat.com”uh1hîhhçubhŒ>”…””}”(hhçhžhhŸNh Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Khhãubah}”(h]”h ]”h"]”h$]”h&]”uh1háhhÎubeh}”(h]”h ]”h"]”h$]”h&]”uh1hÌhŸh¶h KhhÉhžhubhÍ)”}”(hhh]”(hÒ)”}”(hŒLicense”h]”hŒLicense”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÑhj  hŸh¶h K ubhâ)”}”(hŒPThe GNU Free Documentation License, Version 1.2
(dual licensed under the GPL v2)”h]”h¸)”}”(hŒPThe GNU Free Documentation License, Version 1.2
(dual licensed under the GPL v2)”h]”hŒPThe GNU Free Documentation License, Version 1.2
(dual licensed under the GPL v2)”…””}”(hj+  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Khj'  ubah}”(h]”h ]”h"]”h$]”h&]”uh1háhj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1hÌhŸh¶h KhhÉhžhubhÍ)”}”(hhh]”(hÒ)”}”(hŒOriginal Reviewers”h]”hŒOriginal Reviewers”…””}”(hjH  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÑhjE  hŸh¶h K ubhâ)”}”(hŒMElias Oltmanns, Randy Dunlap, Andrew Morton,
John Kacur, and David Teigland.
”h]”h¸)”}”(hŒLElias Oltmanns, Randy Dunlap, Andrew Morton,
John Kacur, and David Teigland.”h]”hŒLElias Oltmanns, Randy Dunlap, Andrew Morton,
John Kacur, and David Teigland.”…””}”(hjZ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K
hjV  ubah}”(h]”h ]”h"]”h$]”h&]”uh1háhjE  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1hÌhŸh¶h K
hhÉhžhubeh}”(h]”h ]”h"]”h$]”h&]”uh1hÇhh£hžhhŸh¶h KubhŒbullet_list”“”)”}”(hhh]”(hŒ	list_item”“”)”}”(hŒWritten for: 2.6.28-rc2”h]”h¸)”}”(hjƒ  h]”hŒWritten for: 2.6.28-rc2”…””}”(hj…  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Khj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj|  hžhhŸh¶h Nubj€  )”}”(hŒUpdated for: 3.10”h]”h¸)”}”(hjš  h]”hŒUpdated for: 3.10”…””}”(hjœ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Khj˜  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj|  hžhhŸh¶h Nubj€  )”}”(hŒ=Updated for: 4.13 - Copyright 2017 VMware Inc. Steven Rostedt”h]”h¸)”}”(hj±  h]”hŒ=Updated for: 4.13 - Copyright 2017 VMware Inc. Steven Rostedt”…””}”(hj³  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Khj¯  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj|  hžhhŸh¶h Nubj€  )”}”(hŒ>Converted to rst format - Changbin Du <changbin.du@intel.com>
”h]”h¸)”}”(hŒ=Converted to rst format - Changbin Du <changbin.du@intel.com>”h]”(hŒ'Converted to rst format - Changbin Du <”…””}”(hjÊ  hžhhŸNh Nubhï)”}”(hŒchangbin.du@intel.com”h]”hŒchangbin.du@intel.com”…””}”(hjÒ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”Œrefuri”Œmailto:changbin.du@intel.com”uh1hîhjÊ  ubhŒ>”…””}”(hjÊ  hžhhŸNh Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KhjÆ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj|  hžhhŸh¶h Nubeh}”(h]”h ]”h"]”h$]”h&]”Œbullet”Œ-”uh1jz  hŸh¶h Khh£hžhubh¢)”}”(hhh]”(h§)”}”(hŒIntroduction”h]”hŒIntroduction”…””}”(hjý  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjú  hžhhŸh¶h Kubh¸)”}”(hŒóFtrace is an internal tracer designed to help out developers and
designers of systems to find what is going on inside the kernel.
It can be used for debugging or analyzing latencies and
performance issues that take place outside of user-space.”h]”hŒóFtrace is an internal tracer designed to help out developers and
designers of systems to find what is going on inside the kernel.
It can be used for debugging or analyzing latencies and
performance issues that take place outside of user-space.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Khjú  hžhubh¸)”}”(hX4  Although ftrace is typically considered the function tracer, it
is really a framework of several assorted tracing utilities.
There's latency tracing to examine what occurs between interrupts
disabled and enabled, as well as for preemption and from a time
a task is woken to the task is actually scheduled in.”h]”hX6  Although ftrace is typically considered the function tracer, it
is really a framework of several assorted tracing utilities.
Thereâ€™s latency tracing to examine what occurs between interrupts
disabled and enabled, as well as for preemption and from a time
a task is woken to the task is actually scheduled in.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Khjú  hžhubh¸)”}”(hŒÜOne of the most common uses of ftrace is the event tracing.
Throughout the kernel is hundreds of static event points that
can be enabled via the tracefs file system to see what is
going on in certain parts of the kernel.”h]”hŒÜOne of the most common uses of ftrace is the event tracing.
Throughout the kernel is hundreds of static event points that
can be enabled via the tracefs file system to see what is
going on in certain parts of the kernel.”…””}”(hj'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K hjú  hžhubh¸)”}”(hŒ$See events.rst for more information.”h]”hŒ$See events.rst for more information.”…””}”(hj5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K%hjú  hžhubeh}”(h]”Œintroduction”ah ]”h"]”Œintroduction”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Kubh¢)”}”(hhh]”(h§)”}”(hŒImplementation Details”h]”hŒImplementation Details”…””}”(hjN  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjK  hžhhŸh¶h K)ubh¸)”}”(hŒPSee Documentation/trace/ftrace-design.rst for details for arch porters and such.”h]”hŒPSee Documentation/trace/ftrace-design.rst for details for arch porters and such.”…””}”(hj\  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K+hjK  hžhubeh}”(h]”Œimplementation-details”ah ]”h"]”Œimplementation details”ah$]”h&]”uh1h¡hh£hžhhŸh¶h K)ubh¢)”}”(hhh]”(h§)”}”(hŒThe File System”h]”hŒThe File System”…””}”(hju  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjr  hžhhŸh¶h K/ubh¸)”}”(hŒeFtrace uses the tracefs file system to hold the control files as
well as the files to display output.”h]”hŒeFtrace uses the tracefs file system to hold the control files as
well as the files to display output.”…””}”(hjƒ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K1hjr  hžhubh¸)”}”(hŒÈWhen tracefs is configured into the kernel (which selecting any ftrace
option will do) the directory /sys/kernel/tracing will be created. To mount
this directory, you can add to your /etc/fstab file::”h]”hŒÇWhen tracefs is configured into the kernel (which selecting any ftrace
option will do) the directory /sys/kernel/tracing will be created. To mount
this directory, you can add to your /etc/fstab file:”…””}”(hj‘  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K4hjr  hžhubhŒliteral_block”“”)”}”(hŒItracefs       /sys/kernel/tracing       tracefs defaults        0       0”h]”hŒItracefs       /sys/kernel/tracing       tracefs defaults        0       0”…””}”hj¡  sbah}”(h]”h ]”h"]”h$]”h&]”Œ	xml:space”Œpreserve”uh1jŸ  hŸh¶h K8hjr  hžhubh¸)”}”(hŒ&Or you can mount it at run time with::”h]”hŒ%Or you can mount it at run time with:”…””}”(hj±  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K:hjr  hžhubj   )”}”(hŒ*mount -t tracefs nodev /sys/kernel/tracing”h]”hŒ*mount -t tracefs nodev /sys/kernel/tracing”…””}”hj¿  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h K<hjr  hžhubh¸)”}”(hŒMFor quicker access to that directory you may want to make a soft link to
it::”h]”hŒLFor quicker access to that directory you may want to make a soft link to
it:”…””}”(hjÍ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K>hjr  hžhubj   )”}”(hŒ"ln -s /sys/kernel/tracing /tracing”h]”hŒ"ln -s /sys/kernel/tracing /tracing”…””}”hjÛ  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h KAhjr  hžhubhŒ	attention”“”)”}”(hX  Before 4.1, all ftrace tracing control files were within the debugfs
file system, which is typically located at /sys/kernel/debug/tracing.
For backward compatibility, when mounting the debugfs file system,
the tracefs file system will be automatically mounted at:

/sys/kernel/debug/tracing

All files located in the tracefs file system will be located in that
debugfs file system directory as well.”h]”(h¸)”}”(hX  Before 4.1, all ftrace tracing control files were within the debugfs
file system, which is typically located at /sys/kernel/debug/tracing.
For backward compatibility, when mounting the debugfs file system,
the tracefs file system will be automatically mounted at:”h]”hX  Before 4.1, all ftrace tracing control files were within the debugfs
file system, which is typically located at /sys/kernel/debug/tracing.
For backward compatibility, when mounting the debugfs file system,
the tracefs file system will be automatically mounted at:”…””}”(hjï  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KEhjë  ubh¸)”}”(hŒ/sys/kernel/debug/tracing”h]”hŒ/sys/kernel/debug/tracing”…””}”(hjý  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KJhjë  ubh¸)”}”(hŒkAll files located in the tracefs file system will be located in that
debugfs file system directory as well.”h]”hŒkAll files located in the tracefs file system will be located in that
debugfs file system directory as well.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KLhjë  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jé  hjr  hžhhŸh¶h Nubjê  )”}”(hX9  Any selected ftrace option will also create the tracefs file system.
The rest of the document will assume that you are in the ftrace directory
(cd /sys/kernel/tracing) and will only concentrate on the files within that
directory and not distract from the content with the extended
"/sys/kernel/tracing" path name.”h]”h¸)”}”(hX9  Any selected ftrace option will also create the tracefs file system.
The rest of the document will assume that you are in the ftrace directory
(cd /sys/kernel/tracing) and will only concentrate on the files within that
directory and not distract from the content with the extended
"/sys/kernel/tracing" path name.”h]”hX=  Any selected ftrace option will also create the tracefs file system.
The rest of the document will assume that you are in the ftrace directory
(cd /sys/kernel/tracing) and will only concentrate on the files within that
directory and not distract from the content with the extended
â€œ/sys/kernel/tracingâ€ path name.”…””}”(hj#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KQhj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jé  hjr  hžhhŸh¶h Nubh¸)”}”(hŒFThat's it! (assuming that you have ftrace configured into your kernel)”h]”hŒHThatâ€™s it! (assuming that you have ftrace configured into your kernel)”…””}”(hj7  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KWhjr  hžhubh¸)”}”(hŒAfter mounting tracefs you will have access to the control and output files
of ftrace. Here is a list of some of the key files:”h]”hŒAfter mounting tracefs you will have access to the control and output files
of ftrace. Here is a list of some of the key files:”…””}”(hjE  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KYhjr  hžhubhŒblock_quote”“”)”}”(hX¹k  Note: all time values are in microseconds.

 current_tracer:

       This is used to set or display the current tracer
       that is configured. Changing the current tracer clears
       the ring buffer content as well as the "snapshot" buffer.

 available_tracers:

       This holds the different types of tracers that
       have been compiled into the kernel. The
       tracers listed here can be configured by
       echoing their name into current_tracer.

 tracing_on:

       This sets or displays whether writing to the trace
       ring buffer is enabled. Echo 0 into this file to disable
       the tracer or 1 to enable it. Note, this only disables
       writing to the ring buffer, the tracing overhead may
       still be occurring.

       The kernel function tracing_off() can be used within the
       kernel to disable writing to the ring buffer, which will
       set this file to "0". User space can re-enable tracing by
       echoing "1" into the file.

       Note, the function and event trigger "traceoff" will also
       set this file to zero and stop tracing. Which can also
       be re-enabled by user space using this file.

 trace:

       This file holds the output of the trace in a human
       readable format (described below). Opening this file for
       writing with the O_TRUNC flag clears the ring buffer content.
       Note, this file is not a consumer. If tracing is off
       (no tracer running, or tracing_on is zero), it will produce
       the same output each time it is read. When tracing is on,
       it may produce inconsistent results as it tries to read
       the entire buffer without consuming it.

 trace_pipe:

       The output is the same as the "trace" file but this
       file is meant to be streamed with live tracing.
       Reads from this file will block until new data is
       retrieved.  Unlike the "trace" file, this file is a
       consumer. This means reading from this file causes
       sequential reads to display more current data. Once
       data is read from this file, it is consumed, and
       will not be read again with a sequential read. The
       "trace" file is static, and if the tracer is not
       adding more data, it will display the same
       information every time it is read.

 trace_options:

       This file lets the user control the amount of data
       that is displayed in one of the above output
       files. Options also exist to modify how a tracer
       or events work (stack traces, timestamps, etc).

 options:

       This is a directory that has a file for every available
       trace option (also in trace_options). Options may also be set
       or cleared by writing a "1" or "0" respectively into the
       corresponding file with the option name.

 tracing_max_latency:

       Some of the tracers record the max latency.
       For example, the maximum time that interrupts are disabled.
       The maximum time is saved in this file. The max trace will also be
       stored, and displayed by "trace". A new max trace will only be
       recorded if the latency is greater than the value in this file
       (in microseconds).

       By echoing in a time into this file, no latency will be recorded
       unless it is greater than the time in this file.

 tracing_thresh:

       Some latency tracers will record a trace whenever the
       latency is greater than the number in this file.
       Only active when the file contains a number greater than 0.
       (in microseconds)

 buffer_percent:

       This is the watermark for how much the ring buffer needs to be filled
       before a waiter is woken up. That is, if an application calls a
       blocking read syscall on one of the per_cpu trace_pipe_raw files, it
       will block until the given amount of data specified by buffer_percent
       is in the ring buffer before it wakes the reader up. This also
       controls how the splice system calls are blocked on this file::

         0   - means to wake up as soon as there is any data in the ring buffer.
         50  - means to wake up when roughly half of the ring buffer sub-buffers
               are full.
         100 - means to block until the ring buffer is totally full and is
               about to start overwriting the older data.

 buffer_size_kb:

       This sets or displays the number of kilobytes each CPU
       buffer holds. By default, the trace buffers are the same size
       for each CPU. The displayed number is the size of the
       CPU buffer and not total size of all buffers. The
       trace buffers are allocated in pages (blocks of memory
       that the kernel uses for allocation, usually 4 KB in size).
       A few extra pages may be allocated to accommodate buffer management
       meta-data. If the last page allocated has room for more bytes
       than requested, the rest of the page will be used,
       making the actual allocation bigger than requested or shown.
       ( Note, the size may not be a multiple of the page size
       due to buffer management meta-data. )

       Buffer sizes for individual CPUs may vary
       (see "per_cpu/cpu0/buffer_size_kb" below), and if they do
       this file will show "X".

 buffer_total_size_kb:

       This displays the total combined size of all the trace buffers.

 buffer_subbuf_size_kb:

       This sets or displays the sub buffer size. The ring buffer is broken up
       into several same size "sub buffers". An event can not be bigger than
       the size of the sub buffer. Normally, the sub buffer is the size of the
       architecture's page (4K on x86). The sub buffer also contains meta data
       at the start which also limits the size of an event.  That means when
       the sub buffer is a page size, no event can be larger than the page
       size minus the sub buffer meta data.

       Note, the buffer_subbuf_size_kb is a way for the user to specify the
       minimum size of the subbuffer. The kernel may make it bigger due to the
       implementation details, or simply fail the operation if the kernel can
       not handle the request.

       Changing the sub buffer size allows for events to be larger than the
       page size.

       Note: When changing the sub-buffer size, tracing is stopped and any
       data in the ring buffer and the snapshot buffer will be discarded.

 free_buffer:

       If a process is performing tracing, and the ring buffer should be
       shrunk "freed" when the process is finished, even if it were to be
       killed by a signal, this file can be used for that purpose. On close
       of this file, the ring buffer will be resized to its minimum size.
       Having a process that is tracing also open this file, when the process
       exits its file descriptor for this file will be closed, and in doing so,
       the ring buffer will be "freed".

       It may also stop tracing if disable_on_free option is set.

 tracing_cpumask:

       This is a mask that lets the user only trace on specified CPUs.
       The format is a hex string representing the CPUs.

 set_ftrace_filter:

       When dynamic ftrace is configured in (see the
       section below "dynamic ftrace"), the code is dynamically
       modified (code text rewrite) to disable calling of the
       function profiler (mcount). This lets tracing be configured
       in with practically no overhead in performance.  This also
       has a side effect of enabling or disabling specific functions
       to be traced. Echoing names of functions into this file
       will limit the trace to only those functions.
       This influences the tracers "function" and "function_graph"
       and thus also function profiling (see "function_profile_enabled").

       The functions listed in "available_filter_functions" are what
       can be written into this file.

       This interface also allows for commands to be used. See the
       "Filter commands" section for more details.

       As a speed up, since processing strings can be quite expensive
       and requires a check of all functions registered to tracing, instead
       an index can be written into this file. A number (starting with "1")
       written will instead select the same corresponding at the line position
       of the "available_filter_functions" file.

 set_ftrace_notrace:

       This has an effect opposite to that of
       set_ftrace_filter. Any function that is added here will not
       be traced. If a function exists in both set_ftrace_filter
       and set_ftrace_notrace, the function will _not_ be traced.

 set_ftrace_pid:

       Have the function tracer only trace the threads whose PID are
       listed in this file.

       If the "function-fork" option is set, then when a task whose
       PID is listed in this file forks, the child's PID will
       automatically be added to this file, and the child will be
       traced by the function tracer as well. This option will also
       cause PIDs of tasks that exit to be removed from the file.

 set_ftrace_notrace_pid:

       Have the function tracer ignore threads whose PID are listed in
       this file.

       If the "function-fork" option is set, then when a task whose
       PID is listed in this file forks, the child's PID will
       automatically be added to this file, and the child will not be
       traced by the function tracer as well. This option will also
       cause PIDs of tasks that exit to be removed from the file.

       If a PID is in both this file and "set_ftrace_pid", then this
       file takes precedence, and the thread will not be traced.

 set_event_pid:

       Have the events only trace a task with a PID listed in this file.
       Note, sched_switch and sched_wake_up will also trace events
       listed in this file.

       To have the PIDs of children of tasks with their PID in this file
       added on fork, enable the "event-fork" option. That option will also
       cause the PIDs of tasks to be removed from this file when the task
       exits.

 set_event_notrace_pid:

       Have the events not trace a task with a PID listed in this file.
       Note, sched_switch and sched_wakeup will trace threads not listed
       in this file, even if a thread's PID is in the file if the
       sched_switch or sched_wakeup events also trace a thread that should
       be traced.

       To have the PIDs of children of tasks with their PID in this file
       added on fork, enable the "event-fork" option. That option will also
       cause the PIDs of tasks to be removed from this file when the task
       exits.

 set_graph_function:

       Functions listed in this file will cause the function graph
       tracer to only trace these functions and the functions that
       they call. (See the section "dynamic ftrace" for more details).
       Note, set_ftrace_filter and set_ftrace_notrace still affects
       what functions are being traced.

 set_graph_notrace:

       Similar to set_graph_function, but will disable function graph
       tracing when the function is hit until it exits the function.
       This makes it possible to ignore tracing functions that are called
       by a specific function.

 available_filter_functions:

       This lists the functions that ftrace has processed and can trace.
       These are the function names that you can pass to
       "set_ftrace_filter", "set_ftrace_notrace",
       "set_graph_function", or "set_graph_notrace".
       (See the section "dynamic ftrace" below for more details.)

 available_filter_functions_addrs:

       Similar to available_filter_functions, but with address displayed
       for each function. The displayed address is the patch-site address
       and can differ from /proc/kallsyms address.

 dyn_ftrace_total_info:

       This file is for debugging purposes. The number of functions that
       have been converted to nops and are available to be traced.

 enabled_functions:

       This file is more for debugging ftrace, but can also be useful
       in seeing if any function has a callback attached to it.
       Not only does the trace infrastructure use ftrace function
       trace utility, but other subsystems might too. This file
       displays all functions that have a callback attached to them
       as well as the number of callbacks that have been attached.
       Note, a callback may also call multiple functions which will
       not be listed in this count.

       If the callback registered to be traced by a function with
       the "save regs" attribute (thus even more overhead), a 'R'
       will be displayed on the same line as the function that
       is returning registers.

       If the callback registered to be traced by a function with
       the "ip modify" attribute (thus the regs->ip can be changed),
       an 'I' will be displayed on the same line as the function that
       can be overridden.

       If a non ftrace trampoline is attached (BPF) a 'D' will be displayed.
       Note, normal ftrace trampolines can also be attached, but only one
       "direct" trampoline can be attached to a given function at a time.

       Some architectures can not call direct trampolines, but instead have
       the ftrace ops function located above the function entry point. In
       such cases an 'O' will be displayed.

       If a function had either the "ip modify" or a "direct" call attached to
       it in the past, a 'M' will be shown. This flag is never cleared. It is
       used to know if a function was every modified by the ftrace infrastructure,
       and can be used for debugging.

       If the architecture supports it, it will also show what callback
       is being directly called by the function. If the count is greater
       than 1 it most likely will be ftrace_ops_list_func().

       If the callback of a function jumps to a trampoline that is
       specific to the callback and which is not the standard trampoline,
       its address will be printed as well as the function that the
       trampoline calls.

 touched_functions:

       This file contains all the functions that ever had a function callback
       to it via the ftrace infrastructure. It has the same format as
       enabled_functions but shows all functions that have every been
       traced.

       To see any function that has every been modified by "ip modify" or a
       direct trampoline, one can perform the following command:

       grep ' M ' /sys/kernel/tracing/touched_functions

 function_profile_enabled:

       When set it will enable all functions with either the function
       tracer, or if configured, the function graph tracer. It will
       keep a histogram of the number of functions that were called
       and if the function graph tracer was configured, it will also keep
       track of the time spent in those functions. The histogram
       content can be displayed in the files:

       trace_stat/function<cpu> ( function0, function1, etc).

 trace_stat:

       A directory that holds different tracing stats.

 kprobe_events:

       Enable dynamic trace points. See kprobetrace.rst.

 kprobe_profile:

       Dynamic trace points stats. See kprobetrace.rst.

 max_graph_depth:

       Used with the function graph tracer. This is the max depth
       it will trace into a function. Setting this to a value of
       one will show only the first kernel function that is called
       from user space.

 printk_formats:

       This is for tools that read the raw format files. If an event in
       the ring buffer references a string, only a pointer to the string
       is recorded into the buffer and not the string itself. This prevents
       tools from knowing what that string was. This file displays the string
       and address for the string allowing tools to map the pointers to what
       the strings were.

 saved_cmdlines:

       Only the pid of the task is recorded in a trace event unless
       the event specifically saves the task comm as well. Ftrace
       makes a cache of pid mappings to comms to try to display
       comms for events. If a pid for a comm is not listed, then
       "<...>" is displayed in the output.

       If the option "record-cmd" is set to "0", then comms of tasks
       will not be saved during recording. By default, it is enabled.

 saved_cmdlines_size:

       By default, 128 comms are saved (see "saved_cmdlines" above). To
       increase or decrease the amount of comms that are cached, echo
       the number of comms to cache into this file.

 saved_tgids:

       If the option "record-tgid" is set, on each scheduling context switch
       the Task Group ID of a task is saved in a table mapping the PID of
       the thread to its TGID. By default, the "record-tgid" option is
       disabled.

 snapshot:

       This displays the "snapshot" buffer and also lets the user
       take a snapshot of the current running trace.
       See the "Snapshot" section below for more details.

 stack_max_size:

       When the stack tracer is activated, this will display the
       maximum stack size it has encountered.
       See the "Stack Trace" section below.

 stack_trace:

       This displays the stack back trace of the largest stack
       that was encountered when the stack tracer is activated.
       See the "Stack Trace" section below.

 stack_trace_filter:

       This is similar to "set_ftrace_filter" but it limits what
       functions the stack tracer will check.

 trace_clock:

       Whenever an event is recorded into the ring buffer, a
       "timestamp" is added. This stamp comes from a specified
       clock. By default, ftrace uses the "local" clock. This
       clock is very fast and strictly per cpu, but on some
       systems it may not be monotonic with respect to other
       CPUs. In other words, the local clocks may not be in sync
       with local clocks on other CPUs.

       Usual clocks for tracing::

         # cat trace_clock
         [local] global counter x86-tsc

       The clock with the square brackets around it is the one in effect.

       local:
               Default clock, but may not be in sync across CPUs

       global:
               This clock is in sync with all CPUs but may
               be a bit slower than the local clock.

       counter:
               This is not a clock at all, but literally an atomic
               counter. It counts up one by one, but is in sync
               with all CPUs. This is useful when you need to
               know exactly the order events occurred with respect to
               each other on different CPUs.

       uptime:
               This uses the jiffies counter and the time stamp
               is relative to the time since boot up.

       perf:
               This makes ftrace use the same clock that perf uses.
               Eventually perf will be able to read ftrace buffers
               and this will help out in interleaving the data.

       x86-tsc:
               Architectures may define their own clocks. For
               example, x86 uses its own TSC cycle clock here.

       ppc-tb:
               This uses the powerpc timebase register value.
               This is in sync across CPUs and can also be used
               to correlate events across hypervisor/guest if
               tb_offset is known.

       mono:
               This uses the fast monotonic clock (CLOCK_MONOTONIC)
               which is monotonic and is subject to NTP rate adjustments.

       mono_raw:
               This is the raw monotonic clock (CLOCK_MONOTONIC_RAW)
               which is monotonic but is not subject to any rate adjustments
               and ticks at the same rate as the hardware clocksource.

       boot:
               This is the boot clock (CLOCK_BOOTTIME) and is based on the
               fast monotonic clock, but also accounts for time spent in
               suspend. Since the clock access is designed for use in
               tracing in the suspend path, some side effects are possible
               if clock is accessed after the suspend time is accounted before
               the fast mono clock is updated. In this case, the clock update
               appears to happen slightly sooner than it normally would have.
               Also on 32-bit systems, it's possible that the 64-bit boot offset
               sees a partial update. These effects are rare and post
               processing should be able to handle them. See comments in the
               ktime_get_boot_fast_ns() function for more information.

       tai:
               This is the tai clock (CLOCK_TAI) and is derived from the wall-
               clock time. However, this clock does not experience
               discontinuities and backwards jumps caused by NTP inserting leap
               seconds. Since the clock access is designed for use in tracing,
               side effects are possible. The clock access may yield wrong
               readouts in case the internal TAI offset is updated e.g., caused
               by setting the system time or using adjtimex() with an offset.
               These effects are rare and post processing should be able to
               handle them. See comments in the ktime_get_tai_fast_ns()
               function for more information.

       To set a clock, simply echo the clock name into this file::

         # echo global > trace_clock

       Setting a clock clears the ring buffer content as well as the
       "snapshot" buffer.

 trace_marker:

       This is a very useful file for synchronizing user space
       with events happening in the kernel. Writing strings into
       this file will be written into the ftrace buffer.

       It is useful in applications to open this file at the start
       of the application and just reference the file descriptor
       for the file::

               void trace_write(const char *fmt, ...)
               {
                       va_list ap;
                       char buf[256];
                       int n;

                       if (trace_fd < 0)
                               return;

                       va_start(ap, fmt);
                       n = vsnprintf(buf, 256, fmt, ap);
                       va_end(ap);

                       write(trace_fd, buf, n);
               }

       start::

               trace_fd = open("trace_marker", O_WRONLY);

       Note: Writing into the trace_marker file can also initiate triggers
             that are written into /sys/kernel/tracing/events/ftrace/print/trigger
             See "Event triggers" in Documentation/trace/events.rst and an
             example in Documentation/trace/histogram.rst (Section 3.)

 trace_marker_raw:

       This is similar to trace_marker above, but is meant for binary data
       to be written to it, where a tool can be used to parse the data
       from trace_pipe_raw.

 uprobe_events:

       Add dynamic tracepoints in programs.
       See uprobetracer.rst

 uprobe_profile:

       Uprobe statistics. See uprobetrace.txt

 instances:

       This is a way to make multiple trace buffers where different
       events can be recorded in different buffers.
       See "Instances" section below.

 events:

       This is the trace event directory. It holds event tracepoints
       (also known as static tracepoints) that have been compiled
       into the kernel. It shows what event tracepoints exist
       and how they are grouped by system. There are "enable"
       files at various levels that can enable the tracepoints
       when a "1" is written to them.

       See events.rst for more information.

 set_event:

       By echoing in the event into this file, will enable that event.

       See events.rst for more information.

 available_events:

       A list of events that can be enabled in tracing.

       See events.rst for more information.

 timestamp_mode:

       Certain tracers may change the timestamp mode used when
       logging trace events into the event buffer.  Events with
       different modes can coexist within a buffer but the mode in
       effect when an event is logged determines which timestamp mode
       is used for that event.  The default timestamp mode is
       'delta'.

       Usual timestamp modes for tracing:

         # cat timestamp_mode
         [delta] absolute

         The timestamp mode with the square brackets around it is the
         one in effect.

         delta: Default timestamp mode - timestamp is a delta against
                a per-buffer timestamp.

         absolute: The timestamp is a full timestamp, not a delta
                against some other value.  As such it takes up more
                space and is less efficient.

 hwlat_detector:

       Directory for the Hardware Latency Detector.
       See "Hardware Latency Detector" section below.

 per_cpu:

       This is a directory that contains the trace per_cpu information.

 per_cpu/cpu0/buffer_size_kb:

       The ftrace buffer is defined per_cpu. That is, there's a separate
       buffer for each CPU to allow writes to be done atomically,
       and free from cache bouncing. These buffers may have different
       size buffers. This file is similar to the buffer_size_kb
       file, but it only displays or sets the buffer size for the
       specific CPU. (here cpu0).

 per_cpu/cpu0/trace:

       This is similar to the "trace" file, but it will only display
       the data specific for the CPU. If written to, it only clears
       the specific CPU buffer.

 per_cpu/cpu0/trace_pipe

       This is similar to the "trace_pipe" file, and is a consuming
       read, but it will only display (and consume) the data specific
       for the CPU.

 per_cpu/cpu0/trace_pipe_raw

       For tools that can parse the ftrace ring buffer binary format,
       the trace_pipe_raw file can be used to extract the data
       from the ring buffer directly. With the use of the splice()
       system call, the buffer data can be quickly transferred to
       a file or to the network where a server is collecting the
       data.

       Like trace_pipe, this is a consuming reader, where multiple
       reads will always produce different data.

 per_cpu/cpu0/snapshot:

       This is similar to the main "snapshot" file, but will only
       snapshot the current CPU (if supported). It only displays
       the content of the snapshot for a given CPU, and if
       written to, only clears this CPU buffer.

 per_cpu/cpu0/snapshot_raw:

       Similar to the trace_pipe_raw, but will read the binary format
       from the snapshot buffer for the given CPU.

 per_cpu/cpu0/stats:

       This displays certain stats about the ring buffer:

       entries:
               The number of events that are still in the buffer.

       overrun:
               The number of lost events due to overwriting when
               the buffer was full.

       commit overrun:
               Should always be zero.
               This gets set if so many events happened within a nested
               event (ring buffer is re-entrant), that it fills the
               buffer and starts dropping events.

       bytes:
               Bytes actually read (not overwritten).

       oldest event ts:
               The oldest timestamp in the buffer

       now ts:
               The current timestamp

       dropped events:
               Events lost due to overwrite option being off.

       read events:
               The number of events read.
”h]”(h¸)”}”(hŒ*Note: all time values are in microseconds.”h]”hŒ*Note: all time values are in microseconds.”…””}”(hjY  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K]hjU  ubjT  )”}”(hXi  current_tracer:

      This is used to set or display the current tracer
      that is configured. Changing the current tracer clears
      the ring buffer content as well as the "snapshot" buffer.

available_tracers:

      This holds the different types of tracers that
      have been compiled into the kernel. The
      tracers listed here can be configured by
      echoing their name into current_tracer.

tracing_on:

      This sets or displays whether writing to the trace
      ring buffer is enabled. Echo 0 into this file to disable
      the tracer or 1 to enable it. Note, this only disables
      writing to the ring buffer, the tracing overhead may
      still be occurring.

      The kernel function tracing_off() can be used within the
      kernel to disable writing to the ring buffer, which will
      set this file to "0". User space can re-enable tracing by
      echoing "1" into the file.

      Note, the function and event trigger "traceoff" will also
      set this file to zero and stop tracing. Which can also
      be re-enabled by user space using this file.

trace:

      This file holds the output of the trace in a human
      readable format (described below). Opening this file for
      writing with the O_TRUNC flag clears the ring buffer content.
      Note, this file is not a consumer. If tracing is off
      (no tracer running, or tracing_on is zero), it will produce
      the same output each time it is read. When tracing is on,
      it may produce inconsistent results as it tries to read
      the entire buffer without consuming it.

trace_pipe:

      The output is the same as the "trace" file but this
      file is meant to be streamed with live tracing.
      Reads from this file will block until new data is
      retrieved.  Unlike the "trace" file, this file is a
      consumer. This means reading from this file causes
      sequential reads to display more current data. Once
      data is read from this file, it is consumed, and
      will not be read again with a sequential read. The
      "trace" file is static, and if the tracer is not
      adding more data, it will display the same
      information every time it is read.

trace_options:

      This file lets the user control the amount of data
      that is displayed in one of the above output
      files. Options also exist to modify how a tracer
      or events work (stack traces, timestamps, etc).

options:

      This is a directory that has a file for every available
      trace option (also in trace_options). Options may also be set
      or cleared by writing a "1" or "0" respectively into the
      corresponding file with the option name.

tracing_max_latency:

      Some of the tracers record the max latency.
      For example, the maximum time that interrupts are disabled.
      The maximum time is saved in this file. The max trace will also be
      stored, and displayed by "trace". A new max trace will only be
      recorded if the latency is greater than the value in this file
      (in microseconds).

      By echoing in a time into this file, no latency will be recorded
      unless it is greater than the time in this file.

tracing_thresh:

      Some latency tracers will record a trace whenever the
      latency is greater than the number in this file.
      Only active when the file contains a number greater than 0.
      (in microseconds)

buffer_percent:

      This is the watermark for how much the ring buffer needs to be filled
      before a waiter is woken up. That is, if an application calls a
      blocking read syscall on one of the per_cpu trace_pipe_raw files, it
      will block until the given amount of data specified by buffer_percent
      is in the ring buffer before it wakes the reader up. This also
      controls how the splice system calls are blocked on this file::

        0   - means to wake up as soon as there is any data in the ring buffer.
        50  - means to wake up when roughly half of the ring buffer sub-buffers
              are full.
        100 - means to block until the ring buffer is totally full and is
              about to start overwriting the older data.

buffer_size_kb:

      This sets or displays the number of kilobytes each CPU
      buffer holds. By default, the trace buffers are the same size
      for each CPU. The displayed number is the size of the
      CPU buffer and not total size of all buffers. The
      trace buffers are allocated in pages (blocks of memory
      that the kernel uses for allocation, usually 4 KB in size).
      A few extra pages may be allocated to accommodate buffer management
      meta-data. If the last page allocated has room for more bytes
      than requested, the rest of the page will be used,
      making the actual allocation bigger than requested or shown.
      ( Note, the size may not be a multiple of the page size
      due to buffer management meta-data. )

      Buffer sizes for individual CPUs may vary
      (see "per_cpu/cpu0/buffer_size_kb" below), and if they do
      this file will show "X".

buffer_total_size_kb:

      This displays the total combined size of all the trace buffers.

buffer_subbuf_size_kb:

      This sets or displays the sub buffer size. The ring buffer is broken up
      into several same size "sub buffers". An event can not be bigger than
      the size of the sub buffer. Normally, the sub buffer is the size of the
      architecture's page (4K on x86). The sub buffer also contains meta data
      at the start which also limits the size of an event.  That means when
      the sub buffer is a page size, no event can be larger than the page
      size minus the sub buffer meta data.

      Note, the buffer_subbuf_size_kb is a way for the user to specify the
      minimum size of the subbuffer. The kernel may make it bigger due to the
      implementation details, or simply fail the operation if the kernel can
      not handle the request.

      Changing the sub buffer size allows for events to be larger than the
      page size.

      Note: When changing the sub-buffer size, tracing is stopped and any
      data in the ring buffer and the snapshot buffer will be discarded.

free_buffer:

      If a process is performing tracing, and the ring buffer should be
      shrunk "freed" when the process is finished, even if it were to be
      killed by a signal, this file can be used for that purpose. On close
      of this file, the ring buffer will be resized to its minimum size.
      Having a process that is tracing also open this file, when the process
      exits its file descriptor for this file will be closed, and in doing so,
      the ring buffer will be "freed".

      It may also stop tracing if disable_on_free option is set.

tracing_cpumask:

      This is a mask that lets the user only trace on specified CPUs.
      The format is a hex string representing the CPUs.

set_ftrace_filter:

      When dynamic ftrace is configured in (see the
      section below "dynamic ftrace"), the code is dynamically
      modified (code text rewrite) to disable calling of the
      function profiler (mcount). This lets tracing be configured
      in with practically no overhead in performance.  This also
      has a side effect of enabling or disabling specific functions
      to be traced. Echoing names of functions into this file
      will limit the trace to only those functions.
      This influences the tracers "function" and "function_graph"
      and thus also function profiling (see "function_profile_enabled").

      The functions listed in "available_filter_functions" are what
      can be written into this file.

      This interface also allows for commands to be used. See the
      "Filter commands" section for more details.

      As a speed up, since processing strings can be quite expensive
      and requires a check of all functions registered to tracing, instead
      an index can be written into this file. A number (starting with "1")
      written will instead select the same corresponding at the line position
      of the "available_filter_functions" file.

set_ftrace_notrace:

      This has an effect opposite to that of
      set_ftrace_filter. Any function that is added here will not
      be traced. If a function exists in both set_ftrace_filter
      and set_ftrace_notrace, the function will _not_ be traced.

set_ftrace_pid:

      Have the function tracer only trace the threads whose PID are
      listed in this file.

      If the "function-fork" option is set, then when a task whose
      PID is listed in this file forks, the child's PID will
      automatically be added to this file, and the child will be
      traced by the function tracer as well. This option will also
      cause PIDs of tasks that exit to be removed from the file.

set_ftrace_notrace_pid:

      Have the function tracer ignore threads whose PID are listed in
      this file.

      If the "function-fork" option is set, then when a task whose
      PID is listed in this file forks, the child's PID will
      automatically be added to this file, and the child will not be
      traced by the function tracer as well. This option will also
      cause PIDs of tasks that exit to be removed from the file.

      If a PID is in both this file and "set_ftrace_pid", then this
      file takes precedence, and the thread will not be traced.

set_event_pid:

      Have the events only trace a task with a PID listed in this file.
      Note, sched_switch and sched_wake_up will also trace events
      listed in this file.

      To have the PIDs of children of tasks with their PID in this file
      added on fork, enable the "event-fork" option. That option will also
      cause the PIDs of tasks to be removed from this file when the task
      exits.

set_event_notrace_pid:

      Have the events not trace a task with a PID listed in this file.
      Note, sched_switch and sched_wakeup will trace threads not listed
      in this file, even if a thread's PID is in the file if the
      sched_switch or sched_wakeup events also trace a thread that should
      be traced.

      To have the PIDs of children of tasks with their PID in this file
      added on fork, enable the "event-fork" option. That option will also
      cause the PIDs of tasks to be removed from this file when the task
      exits.

set_graph_function:

      Functions listed in this file will cause the function graph
      tracer to only trace these functions and the functions that
      they call. (See the section "dynamic ftrace" for more details).
      Note, set_ftrace_filter and set_ftrace_notrace still affects
      what functions are being traced.

set_graph_notrace:

      Similar to set_graph_function, but will disable function graph
      tracing when the function is hit until it exits the function.
      This makes it possible to ignore tracing functions that are called
      by a specific function.

available_filter_functions:

      This lists the functions that ftrace has processed and can trace.
      These are the function names that you can pass to
      "set_ftrace_filter", "set_ftrace_notrace",
      "set_graph_function", or "set_graph_notrace".
      (See the section "dynamic ftrace" below for more details.)

available_filter_functions_addrs:

      Similar to available_filter_functions, but with address displayed
      for each function. The displayed address is the patch-site address
      and can differ from /proc/kallsyms address.

dyn_ftrace_total_info:

      This file is for debugging purposes. The number of functions that
      have been converted to nops and are available to be traced.

enabled_functions:

      This file is more for debugging ftrace, but can also be useful
      in seeing if any function has a callback attached to it.
      Not only does the trace infrastructure use ftrace function
      trace utility, but other subsystems might too. This file
      displays all functions that have a callback attached to them
      as well as the number of callbacks that have been attached.
      Note, a callback may also call multiple functions which will
      not be listed in this count.

      If the callback registered to be traced by a function with
      the "save regs" attribute (thus even more overhead), a 'R'
      will be displayed on the same line as the function that
      is returning registers.

      If the callback registered to be traced by a function with
      the "ip modify" attribute (thus the regs->ip can be changed),
      an 'I' will be displayed on the same line as the function that
      can be overridden.

      If a non ftrace trampoline is attached (BPF) a 'D' will be displayed.
      Note, normal ftrace trampolines can also be attached, but only one
      "direct" trampoline can be attached to a given function at a time.

      Some architectures can not call direct trampolines, but instead have
      the ftrace ops function located above the function entry point. In
      such cases an 'O' will be displayed.

      If a function had either the "ip modify" or a "direct" call attached to
      it in the past, a 'M' will be shown. This flag is never cleared. It is
      used to know if a function was every modified by the ftrace infrastructure,
      and can be used for debugging.

      If the architecture supports it, it will also show what callback
      is being directly called by the function. If the count is greater
      than 1 it most likely will be ftrace_ops_list_func().

      If the callback of a function jumps to a trampoline that is
      specific to the callback and which is not the standard trampoline,
      its address will be printed as well as the function that the
      trampoline calls.

touched_functions:

      This file contains all the functions that ever had a function callback
      to it via the ftrace infrastructure. It has the same format as
      enabled_functions but shows all functions that have every been
      traced.

      To see any function that has every been modified by "ip modify" or a
      direct trampoline, one can perform the following command:

      grep ' M ' /sys/kernel/tracing/touched_functions

function_profile_enabled:

      When set it will enable all functions with either the function
      tracer, or if configured, the function graph tracer. It will
      keep a histogram of the number of functions that were called
      and if the function graph tracer was configured, it will also keep
      track of the time spent in those functions. The histogram
      content can be displayed in the files:

      trace_stat/function<cpu> ( function0, function1, etc).

trace_stat:

      A directory that holds different tracing stats.

kprobe_events:

      Enable dynamic trace points. See kprobetrace.rst.

kprobe_profile:

      Dynamic trace points stats. See kprobetrace.rst.

max_graph_depth:

      Used with the function graph tracer. This is the max depth
      it will trace into a function. Setting this to a value of
      one will show only the first kernel function that is called
      from user space.

printk_formats:

      This is for tools that read the raw format files. If an event in
      the ring buffer references a string, only a pointer to the string
      is recorded into the buffer and not the string itself. This prevents
      tools from knowing what that string was. This file displays the string
      and address for the string allowing tools to map the pointers to what
      the strings were.

saved_cmdlines:

      Only the pid of the task is recorded in a trace event unless
      the event specifically saves the task comm as well. Ftrace
      makes a cache of pid mappings to comms to try to display
      comms for events. If a pid for a comm is not listed, then
      "<...>" is displayed in the output.

      If the option "record-cmd" is set to "0", then comms of tasks
      will not be saved during recording. By default, it is enabled.

saved_cmdlines_size:

      By default, 128 comms are saved (see "saved_cmdlines" above). To
      increase or decrease the amount of comms that are cached, echo
      the number of comms to cache into this file.

saved_tgids:

      If the option "record-tgid" is set, on each scheduling context switch
      the Task Group ID of a task is saved in a table mapping the PID of
      the thread to its TGID. By default, the "record-tgid" option is
      disabled.

snapshot:

      This displays the "snapshot" buffer and also lets the user
      take a snapshot of the current running trace.
      See the "Snapshot" section below for more details.

stack_max_size:

      When the stack tracer is activated, this will display the
      maximum stack size it has encountered.
      See the "Stack Trace" section below.

stack_trace:

      This displays the stack back trace of the largest stack
      that was encountered when the stack tracer is activated.
      See the "Stack Trace" section below.

stack_trace_filter:

      This is similar to "set_ftrace_filter" but it limits what
      functions the stack tracer will check.

trace_clock:

      Whenever an event is recorded into the ring buffer, a
      "timestamp" is added. This stamp comes from a specified
      clock. By default, ftrace uses the "local" clock. This
      clock is very fast and strictly per cpu, but on some
      systems it may not be monotonic with respect to other
      CPUs. In other words, the local clocks may not be in sync
      with local clocks on other CPUs.

      Usual clocks for tracing::

        # cat trace_clock
        [local] global counter x86-tsc

      The clock with the square brackets around it is the one in effect.

      local:
              Default clock, but may not be in sync across CPUs

      global:
              This clock is in sync with all CPUs but may
              be a bit slower than the local clock.

      counter:
              This is not a clock at all, but literally an atomic
              counter. It counts up one by one, but is in sync
              with all CPUs. This is useful when you need to
              know exactly the order events occurred with respect to
              each other on different CPUs.

      uptime:
              This uses the jiffies counter and the time stamp
              is relative to the time since boot up.

      perf:
              This makes ftrace use the same clock that perf uses.
              Eventually perf will be able to read ftrace buffers
              and this will help out in interleaving the data.

      x86-tsc:
              Architectures may define their own clocks. For
              example, x86 uses its own TSC cycle clock here.

      ppc-tb:
              This uses the powerpc timebase register value.
              This is in sync across CPUs and can also be used
              to correlate events across hypervisor/guest if
              tb_offset is known.

      mono:
              This uses the fast monotonic clock (CLOCK_MONOTONIC)
              which is monotonic and is subject to NTP rate adjustments.

      mono_raw:
              This is the raw monotonic clock (CLOCK_MONOTONIC_RAW)
              which is monotonic but is not subject to any rate adjustments
              and ticks at the same rate as the hardware clocksource.

      boot:
              This is the boot clock (CLOCK_BOOTTIME) and is based on the
              fast monotonic clock, but also accounts for time spent in
              suspend. Since the clock access is designed for use in
              tracing in the suspend path, some side effects are possible
              if clock is accessed after the suspend time is accounted before
              the fast mono clock is updated. In this case, the clock update
              appears to happen slightly sooner than it normally would have.
              Also on 32-bit systems, it's possible that the 64-bit boot offset
              sees a partial update. These effects are rare and post
              processing should be able to handle them. See comments in the
              ktime_get_boot_fast_ns() function for more information.

      tai:
              This is the tai clock (CLOCK_TAI) and is derived from the wall-
              clock time. However, this clock does not experience
              discontinuities and backwards jumps caused by NTP inserting leap
              seconds. Since the clock access is designed for use in tracing,
              side effects are possible. The clock access may yield wrong
              readouts in case the internal TAI offset is updated e.g., caused
              by setting the system time or using adjtimex() with an offset.
              These effects are rare and post processing should be able to
              handle them. See comments in the ktime_get_tai_fast_ns()
              function for more information.

      To set a clock, simply echo the clock name into this file::

        # echo global > trace_clock

      Setting a clock clears the ring buffer content as well as the
      "snapshot" buffer.

trace_marker:

      This is a very useful file for synchronizing user space
      with events happening in the kernel. Writing strings into
      this file will be written into the ftrace buffer.

      It is useful in applications to open this file at the start
      of the application and just reference the file descriptor
      for the file::

              void trace_write(const char *fmt, ...)
              {
                      va_list ap;
                      char buf[256];
                      int n;

                      if (trace_fd < 0)
                              return;

                      va_start(ap, fmt);
                      n = vsnprintf(buf, 256, fmt, ap);
                      va_end(ap);

                      write(trace_fd, buf, n);
              }

      start::

              trace_fd = open("trace_marker", O_WRONLY);

      Note: Writing into the trace_marker file can also initiate triggers
            that are written into /sys/kernel/tracing/events/ftrace/print/trigger
            See "Event triggers" in Documentation/trace/events.rst and an
            example in Documentation/trace/histogram.rst (Section 3.)

trace_marker_raw:

      This is similar to trace_marker above, but is meant for binary data
      to be written to it, where a tool can be used to parse the data
      from trace_pipe_raw.

uprobe_events:

      Add dynamic tracepoints in programs.
      See uprobetracer.rst

uprobe_profile:

      Uprobe statistics. See uprobetrace.txt

instances:

      This is a way to make multiple trace buffers where different
      events can be recorded in different buffers.
      See "Instances" section below.

events:

      This is the trace event directory. It holds event tracepoints
      (also known as static tracepoints) that have been compiled
      into the kernel. It shows what event tracepoints exist
      and how they are grouped by system. There are "enable"
      files at various levels that can enable the tracepoints
      when a "1" is written to them.

      See events.rst for more information.

set_event:

      By echoing in the event into this file, will enable that event.

      See events.rst for more information.

available_events:

      A list of events that can be enabled in tracing.

      See events.rst for more information.

timestamp_mode:

      Certain tracers may change the timestamp mode used when
      logging trace events into the event buffer.  Events with
      different modes can coexist within a buffer but the mode in
      effect when an event is logged determines which timestamp mode
      is used for that event.  The default timestamp mode is
      'delta'.

      Usual timestamp modes for tracing:

        # cat timestamp_mode
        [delta] absolute

        The timestamp mode with the square brackets around it is the
        one in effect.

        delta: Default timestamp mode - timestamp is a delta against
               a per-buffer timestamp.

        absolute: The timestamp is a full timestamp, not a delta
               against some other value.  As such it takes up more
               space and is less efficient.

hwlat_detector:

      Directory for the Hardware Latency Detector.
      See "Hardware Latency Detector" section below.

per_cpu:

      This is a directory that contains the trace per_cpu information.

per_cpu/cpu0/buffer_size_kb:

      The ftrace buffer is defined per_cpu. That is, there's a separate
      buffer for each CPU to allow writes to be done atomically,
      and free from cache bouncing. These buffers may have different
      size buffers. This file is similar to the buffer_size_kb
      file, but it only displays or sets the buffer size for the
      specific CPU. (here cpu0).

per_cpu/cpu0/trace:

      This is similar to the "trace" file, but it will only display
      the data specific for the CPU. If written to, it only clears
      the specific CPU buffer.

per_cpu/cpu0/trace_pipe

      This is similar to the "trace_pipe" file, and is a consuming
      read, but it will only display (and consume) the data specific
      for the CPU.

per_cpu/cpu0/trace_pipe_raw

      For tools that can parse the ftrace ring buffer binary format,
      the trace_pipe_raw file can be used to extract the data
      from the ring buffer directly. With the use of the splice()
      system call, the buffer data can be quickly transferred to
      a file or to the network where a server is collecting the
      data.

      Like trace_pipe, this is a consuming reader, where multiple
      reads will always produce different data.

per_cpu/cpu0/snapshot:

      This is similar to the main "snapshot" file, but will only
      snapshot the current CPU (if supported). It only displays
      the content of the snapshot for a given CPU, and if
      written to, only clears this CPU buffer.

per_cpu/cpu0/snapshot_raw:

      Similar to the trace_pipe_raw, but will read the binary format
      from the snapshot buffer for the given CPU.

per_cpu/cpu0/stats:

      This displays certain stats about the ring buffer:

      entries:
              The number of events that are still in the buffer.

      overrun:
              The number of lost events due to overwriting when
              the buffer was full.

      commit overrun:
              Should always be zero.
              This gets set if so many events happened within a nested
              event (ring buffer is re-entrant), that it fills the
              buffer and starts dropping events.

      bytes:
              Bytes actually read (not overwritten).

      oldest event ts:
              The oldest timestamp in the buffer

      now ts:
              The current timestamp

      dropped events:
              Events lost due to overwrite option being off.

      read events:
              The number of events read.
”•       h]”(h¸)”}”(hŒcurrent_tracer:”h]”hŒcurrent_tracer:”…””}”(hjk  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K_hjg  ubjT  )”}”(hŒ£This is used to set or display the current tracer
that is configured. Changing the current tracer clears
the ring buffer content as well as the "snapshot" buffer.
”h]”h¸)”}”(hŒ¢This is used to set or display the current tracer
that is configured. Changing the current tracer clears
the ring buffer content as well as the "snapshot" buffer.”h]”hŒ¦This is used to set or display the current tracer
that is configured. Changing the current tracer clears
the ring buffer content as well as the â€œsnapshotâ€ buffer.”…””}”(hj}  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kahjy  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Kahjg  ubh¸)”}”(hŒavailable_tracers:”h]”hŒavailable_tracers:”…””}”(hj‘  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kehjg  ubjT  )”}”(hŒ¨This holds the different types of tracers that
have been compiled into the kernel. The
tracers listed here can be configured by
echoing their name into current_tracer.
”h]”h¸)”}”(hŒ§This holds the different types of tracers that
have been compiled into the kernel. The
tracers listed here can be configured by
echoing their name into current_tracer.”h]”hŒ§This holds the different types of tracers that
have been compiled into the kernel. The
tracers listed here can be configured by
echoing their name into current_tracer.”…””}”(hj£  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KghjŸ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Kghjg  ubh¸)”}”(hŒtracing_on:”h]”hŒtracing_on:”…””}”(hj·  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Klhjg  ubjT  )”}”(hXS  This sets or displays whether writing to the trace
ring buffer is enabled. Echo 0 into this file to disable
the tracer or 1 to enable it. Note, this only disables
writing to the ring buffer, the tracing overhead may
still be occurring.

The kernel function tracing_off() can be used within the
kernel to disable writing to the ring buffer, which will
set this file to "0". User space can re-enable tracing by
echoing "1" into the file.

Note, the function and event trigger "traceoff" will also
set this file to zero and stop tracing. Which can also
be re-enabled by user space using this file.
”h]”(h¸)”}”(hŒëThis sets or displays whether writing to the trace
ring buffer is enabled. Echo 0 into this file to disable
the tracer or 1 to enable it. Note, this only disables
writing to the ring buffer, the tracing overhead may
still be occurring.”h]”hŒëThis sets or displays whether writing to the trace
ring buffer is enabled. Echo 0 into this file to disable
the tracer or 1 to enable it. Note, this only disables
writing to the ring buffer, the tracing overhead may
still be occurring.”…””}”(hjÉ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KnhjÅ  ubh¸)”}”(hŒÆThe kernel function tracing_off() can be used within the
kernel to disable writing to the ring buffer, which will
set this file to "0". User space can re-enable tracing by
echoing "1" into the file.”h]”hŒÎThe kernel function tracing_off() can be used within the
kernel to disable writing to the ring buffer, which will
set this file to â€œ0â€. User space can re-enable tracing by
echoing â€œ1â€ into the file.”…””}”(hj×  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KthjÅ  ubh¸)”}”(hŒNote, the function and event trigger "traceoff" will also
set this file to zero and stop tracing. Which can also
be re-enabled by user space using this file.”h]”hŒ¡Note, the function and event trigger â€œtraceoffâ€ will also
set this file to zero and stop tracing. Which can also
be re-enabled by user space using this file.”…””}”(hjå  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KyhjÅ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Knhjg  ubh¸)”}”(hŒtrace:”h]”hŒtrace:”…””}”(hjù  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K}hjg  ubjT  )”}”(hXµ  This file holds the output of the trace in a human
readable format (described below). Opening this file for
writing with the O_TRUNC flag clears the ring buffer content.
Note, this file is not a consumer. If tracing is off
(no tracer running, or tracing_on is zero), it will produce
the same output each time it is read. When tracing is on,
it may produce inconsistent results as it tries to read
the entire buffer without consuming it.
”h]”h¸)”}”(hX´  This file holds the output of the trace in a human
readable format (described below). Opening this file for
writing with the O_TRUNC flag clears the ring buffer content.
Note, this file is not a consumer. If tracing is off
(no tracer running, or tracing_on is zero), it will produce
the same output each time it is read. When tracing is on,
it may produce inconsistent results as it tries to read
the entire buffer without consuming it.”h]”hX´  This file holds the output of the trace in a human
readable format (described below). Opening this file for
writing with the O_TRUNC flag clears the ring buffer content.
Note, this file is not a consumer. If tracing is off
(no tracer running, or tracing_on is zero), it will produce
the same output each time it is read. When tracing is on,
it may produce inconsistent results as it tries to read
the entire buffer without consuming it.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Khj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Khjg  ubh¸)”}”(hŒtrace_pipe:”h]”hŒtrace_pipe:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kˆhjg  ubjT  )”}”(hX  The output is the same as the "trace" file but this
file is meant to be streamed with live tracing.
Reads from this file will block until new data is
retrieved.  Unlike the "trace" file, this file is a
consumer. This means reading from this file causes
sequential reads to display more current data. Once
data is read from this file, it is consumed, and
will not be read again with a sequential read. The
"trace" file is static, and if the tracer is not
adding more data, it will display the same
information every time it is read.
”h]”h¸)”}”(hX  The output is the same as the "trace" file but this
file is meant to be streamed with live tracing.
Reads from this file will block until new data is
retrieved.  Unlike the "trace" file, this file is a
consumer. This means reading from this file causes
sequential reads to display more current data. Once
data is read from this file, it is consumed, and
will not be read again with a sequential read. The
"trace" file is static, and if the tracer is not
adding more data, it will display the same
information every time it is read.”h]”hX  The output is the same as the â€œtraceâ€ file but this
file is meant to be streamed with live tracing.
Reads from this file will block until new data is
retrieved.  Unlike the â€œtraceâ€ file, this file is a
consumer. This means reading from this file causes
sequential reads to display more current data. Once
data is read from this file, it is consumed, and
will not be read again with a sequential read. The
â€œtraceâ€ file is static, and if the tracer is not
adding more data, it will display the same
information every time it is read.”…””}”(hj1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KŠhj-  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h KŠhjg  ubh¸)”}”(hŒtrace_options:”h]”hŒtrace_options:”…””}”(hjE  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K–hjg  ubjT  )”}”(hŒÁThis file lets the user control the amount of data
that is displayed in one of the above output
files. Options also exist to modify how a tracer
or events work (stack traces, timestamps, etc).
”h]”h¸)”}”(hŒÀThis file lets the user control the amount of data
that is displayed in one of the above output
files. Options also exist to modify how a tracer
or events work (stack traces, timestamps, etc).”h]”hŒÀThis file lets the user control the amount of data
that is displayed in one of the above output
files. Options also exist to modify how a tracer
or events work (stack traces, timestamps, etc).”…””}”(hjW  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K˜hjS  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h K˜hjg  ubh¸)”}”(hŒoptions:”h]”hŒoptions:”…””}”(hjk  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Khjg  ubjT  )”}”(hŒØThis is a directory that has a file for every available
trace option (also in trace_options). Options may also be set
or cleared by writing a "1" or "0" respectively into the
corresponding file with the option name.
”h]”h¸)”}”(hŒ×This is a directory that has a file for every available
trace option (also in trace_options). Options may also be set
or cleared by writing a "1" or "0" respectively into the
corresponding file with the option name.”h]”hŒßThis is a directory that has a file for every available
trace option (also in trace_options). Options may also be set
or cleared by writing a â€œ1â€ or â€œ0â€ respectively into the
corresponding file with the option name.”…””}”(hj}  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KŸhjy  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h KŸhjg  ubh¸)”}”(hŒtracing_max_latency:”h]”hŒtracing_max_latency:”…””}”(hj‘  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K¤hjg  ubjT  )”}”(hX¯  Some of the tracers record the max latency.
For example, the maximum time that interrupts are disabled.
The maximum time is saved in this file. The max trace will also be
stored, and displayed by "trace". A new max trace will only be
recorded if the latency is greater than the value in this file
(in microseconds).

By echoing in a time into this file, no latency will be recorded
unless it is greater than the time in this file.
”h]”(h¸)”}”(hX;  Some of the tracers record the max latency.
For example, the maximum time that interrupts are disabled.
The maximum time is saved in this file. The max trace will also be
stored, and displayed by "trace". A new max trace will only be
recorded if the latency is greater than the value in this file
(in microseconds).”h]”hX?  Some of the tracers record the max latency.
For example, the maximum time that interrupts are disabled.
The maximum time is saved in this file. The max trace will also be
stored, and displayed by â€œtraceâ€. A new max trace will only be
recorded if the latency is greater than the value in this file
(in microseconds).”…””}”(hj£  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K¦hjŸ  ubh¸)”}”(hŒqBy echoing in a time into this file, no latency will be recorded
unless it is greater than the time in this file.”h]”hŒqBy echoing in a time into this file, no latency will be recorded
unless it is greater than the time in this file.”…””}”(hj±  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K­hjŸ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h K¦hjg  ubh¸)”}”(hŒtracing_thresh:”h]”hŒtracing_thresh:”…””}”(hjÅ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K°hjg  ubjT  )”}”(hŒµSome latency tracers will record a trace whenever the
latency is greater than the number in this file.
Only active when the file contains a number greater than 0.
(in microseconds)
”h]”h¸)”}”(hŒ´Some latency tracers will record a trace whenever the
latency is greater than the number in this file.
Only active when the file contains a number greater than 0.
(in microseconds)”h]”hŒ´Some latency tracers will record a trace whenever the
latency is greater than the number in this file.
Only active when the file contains a number greater than 0.
(in microseconds)”…””}”(hj×  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K²hjÓ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h K²hjg  ubh¸)”}”(hŒbuffer_percent:”h]”hŒbuffer_percent:”…””}”(hjë  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K·hjg  ubjT  )”}”(hX®  This is the watermark for how much the ring buffer needs to be filled
before a waiter is woken up. That is, if an application calls a
blocking read syscall on one of the per_cpu trace_pipe_raw files, it
will block until the given amount of data specified by buffer_percent
is in the ring buffer before it wakes the reader up. This also
controls how the splice system calls are blocked on this file::

  0   - means to wake up as soon as there is any data in the ring buffer.
  50  - means to wake up when roughly half of the ring buffer sub-buffers
        are full.
  100 - means to block until the ring buffer is totally full and is
        about to start overwriting the older data.
”h]”(h¸)”}”(hX  This is the watermark for how much the ring buffer needs to be filled
before a waiter is woken up. That is, if an application calls a
blocking read syscall on one of the per_cpu trace_pipe_raw files, it
will block until the given amount of data specified by buffer_percent
is in the ring buffer before it wakes the reader up. This also
controls how the splice system calls are blocked on this file::”h]”hXŽ  This is the watermark for how much the ring buffer needs to be filled
before a waiter is woken up. That is, if an application calls a
blocking read syscall on one of the per_cpu trace_pipe_raw files, it
will block until the given amount of data specified by buffer_percent
is in the ring buffer before it wakes the reader up. This also
controls how the splice system calls are blocked on this file:”…””}”(hjý  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h K¹hjù  ubj   )”}”(hX  0   - means to wake up as soon as there is any data in the ring buffer.
50  - means to wake up when roughly half of the ring buffer sub-buffers
      are full.
100 - means to block until the ring buffer is totally full and is
      about to start overwriting the older data.”h]”hX  0   - means to wake up as soon as there is any data in the ring buffer.
50  - means to wake up when roughly half of the ring buffer sub-buffers
      are full.
100 - means to block until the ring buffer is totally full and is
      about to start overwriting the older data.”…””}”hj  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h KÀhjù  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h K¹hjg  ubh¸)”}”(hŒbuffer_size_kb:”h]”hŒbuffer_size_kb:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KÆhjg  ubjT  )”}”(hX  This sets or displays the number of kilobytes each CPU
buffer holds. By default, the trace buffers are the same size
for each CPU. The displayed number is the size of the
CPU buffer and not total size of all buffers. The
trace buffers are allocated in pages (blocks of memory
that the kernel uses for allocation, usually 4 KB in size).
A few extra pages may be allocated to accommodate buffer management
meta-data. If the last page allocated has room for more bytes
than requested, the rest of the page will be used,
making the actual allocation bigger than requested or shown.
( Note, the size may not be a multiple of the page size
due to buffer management meta-data. )

Buffer sizes for individual CPUs may vary
(see "per_cpu/cpu0/buffer_size_kb" below), and if they do
this file will show "X".
”h]”(h¸)”}”(hXŸ  This sets or displays the number of kilobytes each CPU
buffer holds. By default, the trace buffers are the same size
for each CPU. The displayed number is the size of the
CPU buffer and not total size of all buffers. The
trace buffers are allocated in pages (blocks of memory
that the kernel uses for allocation, usually 4 KB in size).
A few extra pages may be allocated to accommodate buffer management
meta-data. If the last page allocated has room for more bytes
than requested, the rest of the page will be used,
making the actual allocation bigger than requested or shown.
( Note, the size may not be a multiple of the page size
due to buffer management meta-data. )”h]”hXŸ  This sets or displays the number of kilobytes each CPU
buffer holds. By default, the trace buffers are the same size
for each CPU. The displayed number is the size of the
CPU buffer and not total size of all buffers. The
trace buffers are allocated in pages (blocks of memory
that the kernel uses for allocation, usually 4 KB in size).
A few extra pages may be allocated to accommodate buffer management
meta-data. If the last page allocated has room for more bytes
than requested, the rest of the page will be used,
making the actual allocation bigger than requested or shown.
( Note, the size may not be a multiple of the page size
due to buffer management meta-data. )”…””}”(hj1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KÈhj-  ubh¸)”}”(hŒ|Buffer sizes for individual CPUs may vary
(see "per_cpu/cpu0/buffer_size_kb" below), and if they do
this file will show "X".”h]”hŒ„Buffer sizes for individual CPUs may vary
(see â€œper_cpu/cpu0/buffer_size_kbâ€ below), and if they do
this file will show â€œXâ€.”…””}”(hj?  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KÕhj-  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h KÈhjg  ubh¸)”}”(hŒbuffer_total_size_kb:”h]”hŒbuffer_total_size_kb:”…””}”(hjS  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KÙhjg  ubjT  )”}”(hŒ@This displays the total combined size of all the trace buffers.
”h]”h¸)”}”(hŒ?This displays the total combined size of all the trace buffers.”h]”hŒ?This displays the total combined size of all the trace buffers.”…””}”(hje  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KÛhja  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h KÛhjg  ubh¸)”}”(hŒbuffer_subbuf_size_kb:”h]”hŒbuffer_subbuf_size_kb:”…””}”(hjy  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h KÝhjg  ubjT  )”}”(hX“  This sets or displays the sub buffer size. The ring buffer is broken up
into several same size "sub buffers". An event can not be bigger than
the size of the sub buffer. Normally, the sub buffer is the size of the
architecture's page (4K on x86). The sub buffer also contains meta data
at the start which also limits the size of an event.  That means when
the sub buffer is a page size, no event can be larger than the page
size minus the sub buffer meta data.

Note, the buffer_subbuf_size_kb is a way for the user to specify the
minimum size of the subbuffer. The kernel may make it bigger due to the
implementation details, or simply fail the operation if the kernel can
not handle the request.

Changing the sub buffer size allows for events to be larger than the
page size.

Note: When changing the sub-buffer size, tracing is stopped and any
data in the ring buffer and the snapshot buffer will be discarded.
”h]”(h¸)”}”(hXÌ  This sets or displays the sub buffer size. The ring buffer is broken up
into several same size "sub buffers". An event can not be bigger than
the size of the sub buffer. Normally, the sub buffer is the size of the
architecture's page (4K on x86). The sub buffer also contains meta data
at the start which also limits the size of an event.  That means when
the sub buffer is a page size, no event can be larger than the page
size minus the sub buffer meta data.”h]”hXÒ  This sets or displays the sub buffer size. The ring buffer is broken up
into several same size â€œsub buffersâ€. An event can not be bigger than
the size of the sub buffer. Normally, the sub buffer is the size of the
architectureâ€™s page (4K on x86). The sub buffer also contains meta data
at the start which also limits the size of an event.  That means when
the sub buffer is a page size, no event can be larger than the page
size minus the sub buffer meta data.”…””}”(hj‹  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kßhj‡  ubh¸)”}”(hŒëNote, the buffer_subbuf_size_kb is a way for the user to specify the
minimum size of the subbuffer. The kernel may make it bigger due to the
implementation details, or simply fail the operation if the kernel can
not handle the request.”h]”hŒëNote, the buffer_subbuf_size_kb is a way for the user to specify the
minimum size of the subbuffer. The kernel may make it bigger due to the
implementation details, or simply fail the operation if the kernel can
not handle the request.”…””}”(hj™  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kçhj‡  ubh¸)”}”(hŒOChanging the sub buffer size allows for events to be larger than the
page size.”h]”hŒOChanging the sub buffer size allows for events to be larger than the
page size.”…””}”(hj§  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kìhj‡  ubh¸)”}”(hŒ†Note: When changing the sub-buffer size, tracing is stopped and any
data in the ring buffer and the snapshot buffer will be discarded.”h]”hŒ†Note: When changing the sub-buffer size, tracing is stopped and any
data in the ring buffer and the snapshot buffer will be discarded.”…””}”(hjµ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kïhj‡  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Kßhjg  ubh¸)”}”(hŒfree_buffer:”h]”hŒfree_buffer:”…””}”(hjÉ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kòhjg  ubjT  )”}”(hXú  If a process is performing tracing, and the ring buffer should be
shrunk "freed" when the process is finished, even if it were to be
killed by a signal, this file can be used for that purpose. On close
of this file, the ring buffer will be resized to its minimum size.
Having a process that is tracing also open this file, when the process
exits its file descriptor for this file will be closed, and in doing so,
the ring buffer will be "freed".

It may also stop tracing if disable_on_free option is set.
”h]”(h¸)”}”(hX½  If a process is performing tracing, and the ring buffer should be
shrunk "freed" when the process is finished, even if it were to be
killed by a signal, this file can be used for that purpose. On close
of this file, the ring buffer will be resized to its minimum size.
Having a process that is tracing also open this file, when the process
exits its file descriptor for this file will be closed, and in doing so,
the ring buffer will be "freed".”h]”hXÅ  If a process is performing tracing, and the ring buffer should be
shrunk â€œfreedâ€ when the process is finished, even if it were to be
killed by a signal, this file can be used for that purpose. On close
of this file, the ring buffer will be resized to its minimum size.
Having a process that is tracing also open this file, when the process
exits its file descriptor for this file will be closed, and in doing so,
the ring buffer will be â€œfreedâ€.”…””}”(hjÛ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kôhj×  ubh¸)”}”(hŒ:It may also stop tracing if disable_on_free option is set.”h]”hŒ:It may also stop tracing if disable_on_free option is set.”…””}”(hjé  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kühj×  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Kôhjg  ubh¸)”}”(hŒtracing_cpumask:”h]”hŒtracing_cpumask:”…””}”(hjý  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Kþhjg  ubjT  )”}”(hŒrThis is a mask that lets the user only trace on specified CPUs.
The format is a hex string representing the CPUs.
”h]”h¸)”}”(hŒqThis is a mask that lets the user only trace on specified CPUs.
The format is a hex string representing the CPUs.”h]”hŒqThis is a mask that lets the user only trace on specified CPUs.
The format is a hex string representing the CPUs.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M hj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M hjg  ubh¸)”}”(hŒset_ftrace_filter:”h]”hŒset_ftrace_filter:”…””}”(hj#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjg  ubjT  )”}”(hX;  When dynamic ftrace is configured in (see the
section below "dynamic ftrace"), the code is dynamically
modified (code text rewrite) to disable calling of the
function profiler (mcount). This lets tracing be configured
in with practically no overhead in performance.  This also
has a side effect of enabling or disabling specific functions
to be traced. Echoing names of functions into this file
will limit the trace to only those functions.
This influences the tracers "function" and "function_graph"
and thus also function profiling (see "function_profile_enabled").

The functions listed in "available_filter_functions" are what
can be written into this file.

This interface also allows for commands to be used. See the
"Filter commands" section for more details.

As a speed up, since processing strings can be quite expensive
and requires a check of all functions registered to tracing, instead
an index can be written into this file. A number (starting with "1")
written will instead select the same corresponding at the line position
of the "available_filter_functions" file.
”h]”(h¸)”}”(hX7  When dynamic ftrace is configured in (see the
section below "dynamic ftrace"), the code is dynamically
modified (code text rewrite) to disable calling of the
function profiler (mcount). This lets tracing be configured
in with practically no overhead in performance.  This also
has a side effect of enabling or disabling specific functions
to be traced. Echoing names of functions into this file
will limit the trace to only those functions.
This influences the tracers "function" and "function_graph"
and thus also function profiling (see "function_profile_enabled").”h]”hXG  When dynamic ftrace is configured in (see the
section below â€œdynamic ftraceâ€), the code is dynamically
modified (code text rewrite) to disable calling of the
function profiler (mcount). This lets tracing be configured
in with practically no overhead in performance.  This also
has a side effect of enabling or disabling specific functions
to be traced. Echoing names of functions into this file
will limit the trace to only those functions.
This influences the tracers â€œfunctionâ€ and â€œfunction_graphâ€
and thus also function profiling (see â€œfunction_profile_enabledâ€).”…””}”(hj5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj1  ubh¸)”}”(hŒ\The functions listed in "available_filter_functions" are what
can be written into this file.”h]”hŒ`The functions listed in â€œavailable_filter_functionsâ€ are what
can be written into this file.”…””}”(hjC  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj1  ubh¸)”}”(hŒgThis interface also allows for commands to be used. See the
"Filter commands" section for more details.”h]”hŒkThis interface also allows for commands to be used. See the
â€œFilter commandsâ€ section for more details.”…””}”(hjQ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj1  ubh¸)”}”(hX:  As a speed up, since processing strings can be quite expensive
and requires a check of all functions registered to tracing, instead
an index can be written into this file. A number (starting with "1")
written will instead select the same corresponding at the line position
of the "available_filter_functions" file.”h]”hXB  As a speed up, since processing strings can be quite expensive
and requires a check of all functions registered to tracing, instead
an index can be written into this file. A number (starting with â€œ1â€)
written will instead select the same corresponding at the line position
of the â€œavailable_filter_functionsâ€ file.”…””}”(hj_  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj1  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mhjg  ubh¸)”}”(hŒset_ftrace_notrace:”h]”hŒset_ftrace_notrace:”…””}”(hjs  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjg  ubjT  )”}”(hŒØThis has an effect opposite to that of
set_ftrace_filter. Any function that is added here will not
be traced. If a function exists in both set_ftrace_filter
and set_ftrace_notrace, the function will _not_ be traced.
”h]”h¸)”}”(hŒ×This has an effect opposite to that of
set_ftrace_filter. Any function that is added here will not
be traced. If a function exists in both set_ftrace_filter
and set_ftrace_notrace, the function will _not_ be traced.”h]”hŒ×This has an effect opposite to that of
set_ftrace_filter. Any function that is added here will not
be traced. If a function exists in both set_ftrace_filter
and set_ftrace_notrace, the function will _not_ be traced.”…””}”(hj…  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mhjg  ubh¸)”}”(hŒset_ftrace_pid:”h]”hŒset_ftrace_pid:”…””}”(hj™  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M#hjg  ubjT  )”}”(hX{  Have the function tracer only trace the threads whose PID are
listed in this file.

If the "function-fork" option is set, then when a task whose
PID is listed in this file forks, the child's PID will
automatically be added to this file, and the child will be
traced by the function tracer as well. This option will also
cause PIDs of tasks that exit to be removed from the file.
”h]”(h¸)”}”(hŒRHave the function tracer only trace the threads whose PID are
listed in this file.”h]”hŒRHave the function tracer only trace the threads whose PID are
listed in this file.”…””}”(hj«  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M%hj§  ubh¸)”}”(hX&  If the "function-fork" option is set, then when a task whose
PID is listed in this file forks, the child's PID will
automatically be added to this file, and the child will be
traced by the function tracer as well. This option will also
cause PIDs of tasks that exit to be removed from the file.”h]”hX,  If the â€œfunction-forkâ€ option is set, then when a task whose
PID is listed in this file forks, the childâ€™s PID will
automatically be added to this file, and the child will be
traced by the function tracer as well. This option will also
cause PIDs of tasks that exit to be removed from the file.”…””}”(hj¹  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M(hj§  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M%hjg  ubh¸)”}”(hŒset_ftrace_notrace_pid:”h]”hŒset_ftrace_notrace_pid:”…””}”(hjÍ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M.hjg  ubjT  )”}”(hXð  Have the function tracer ignore threads whose PID are listed in
this file.

If the "function-fork" option is set, then when a task whose
PID is listed in this file forks, the child's PID will
automatically be added to this file, and the child will not be
traced by the function tracer as well. This option will also
cause PIDs of tasks that exit to be removed from the file.

If a PID is in both this file and "set_ftrace_pid", then this
file takes precedence, and the thread will not be traced.
”h]”(h¸)”}”(hŒJHave the function tracer ignore threads whose PID are listed in
this file.”h]”hŒJHave the function tracer ignore threads whose PID are listed in
this file.”…””}”(hjß  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M0hjÛ  ubh¸)”}”(hX*  If the "function-fork" option is set, then when a task whose
PID is listed in this file forks, the child's PID will
automatically be added to this file, and the child will not be
traced by the function tracer as well. This option will also
cause PIDs of tasks that exit to be removed from the file.”h]”hX0  If the â€œfunction-forkâ€ option is set, then when a task whose
PID is listed in this file forks, the childâ€™s PID will
automatically be added to this file, and the child will not be
traced by the function tracer as well. This option will also
cause PIDs of tasks that exit to be removed from the file.”…””}”(hjí  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M3hjÛ  ubh¸)”}”(hŒwIf a PID is in both this file and "set_ftrace_pid", then this
file takes precedence, and the thread will not be traced.”h]”hŒ{If a PID is in both this file and â€œset_ftrace_pidâ€, then this
file takes precedence, and the thread will not be traced.”…””}”(hjû  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M9hjÛ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M0hjg  ubh¸)”}”(hŒset_event_pid:”h]”hŒset_event_pid:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M<hjg  ubjT  )”}”(hXe  Have the events only trace a task with a PID listed in this file.
Note, sched_switch and sched_wake_up will also trace events
listed in this file.

To have the PIDs of children of tasks with their PID in this file
added on fork, enable the "event-fork" option. That option will also
cause the PIDs of tasks to be removed from this file when the task
exits.
”h]”(h¸)”}”(hŒ’Have the events only trace a task with a PID listed in this file.
Note, sched_switch and sched_wake_up will also trace events
listed in this file.”h]”hŒ’Have the events only trace a task with a PID listed in this file.
Note, sched_switch and sched_wake_up will also trace events
listed in this file.”…””}”(hj!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M>hj  ubh¸)”}”(hŒÐTo have the PIDs of children of tasks with their PID in this file
added on fork, enable the "event-fork" option. That option will also
cause the PIDs of tasks to be removed from this file when the task
exits.”h]”hŒÔTo have the PIDs of children of tasks with their PID in this file
added on fork, enable the â€œevent-forkâ€ option. That option will also
cause the PIDs of tasks to be removed from this file when the task
exits.”…””}”(hj/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MBhj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M>hjg  ubh¸)”}”(hŒset_event_notrace_pid:”h]”hŒset_event_notrace_pid:”…””}”(hjC  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MGhjg  ubjT  )”}”(hXß  Have the events not trace a task with a PID listed in this file.
Note, sched_switch and sched_wakeup will trace threads not listed
in this file, even if a thread's PID is in the file if the
sched_switch or sched_wakeup events also trace a thread that should
be traced.

To have the PIDs of children of tasks with their PID in this file
added on fork, enable the "event-fork" option. That option will also
cause the PIDs of tasks to be removed from this file when the task
exits.
”h]”(h¸)”}”(hX  Have the events not trace a task with a PID listed in this file.
Note, sched_switch and sched_wakeup will trace threads not listed
in this file, even if a thread's PID is in the file if the
sched_switch or sched_wakeup events also trace a thread that should
be traced.”h]”hX  Have the events not trace a task with a PID listed in this file.
Note, sched_switch and sched_wakeup will trace threads not listed
in this file, even if a threadâ€™s PID is in the file if the
sched_switch or sched_wakeup events also trace a thread that should
be traced.”…””}”(hjU  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MIhjQ  ubh¸)”}”(hŒÐTo have the PIDs of children of tasks with their PID in this file
added on fork, enable the "event-fork" option. That option will also
cause the PIDs of tasks to be removed from this file when the task
exits.”h]”hŒÔTo have the PIDs of children of tasks with their PID in this file
added on fork, enable the â€œevent-forkâ€ option. That option will also
cause the PIDs of tasks to be removed from this file when the task
exits.”…””}”(hjc  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MOhjQ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MIhjg  ubh¸)”}”(hŒset_graph_function:”h]”hŒset_graph_function:”…””}”(hjw  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MThjg  ubjT  )”}”(hX  Functions listed in this file will cause the function graph
tracer to only trace these functions and the functions that
they call. (See the section "dynamic ftrace" for more details).
Note, set_ftrace_filter and set_ftrace_notrace still affects
what functions are being traced.
”h]”h¸)”}”(hX  Functions listed in this file will cause the function graph
tracer to only trace these functions and the functions that
they call. (See the section "dynamic ftrace" for more details).
Note, set_ftrace_filter and set_ftrace_notrace still affects
what functions are being traced.”h]”hX  Functions listed in this file will cause the function graph
tracer to only trace these functions and the functions that
they call. (See the section â€œdynamic ftraceâ€ for more details).
Note, set_ftrace_filter and set_ftrace_notrace still affects
what functions are being traced.”…””}”(hj‰  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MVhj…  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MVhjg  ubh¸)”}”(hŒset_graph_notrace:”h]”hŒset_graph_notrace:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M\hjg  ubjT  )”}”(hŒØSimilar to set_graph_function, but will disable function graph
tracing when the function is hit until it exits the function.
This makes it possible to ignore tracing functions that are called
by a specific function.
”h]”h¸)”}”(hŒ×Similar to set_graph_function, but will disable function graph
tracing when the function is hit until it exits the function.
This makes it possible to ignore tracing functions that are called
by a specific function.”h]”hŒ×Similar to set_graph_function, but will disable function graph
tracing when the function is hit until it exits the function.
This makes it possible to ignore tracing functions that are called
by a specific function.”…””}”(hj¯  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M^hj«  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M^hjg  ubh¸)”}”(hŒavailable_filter_functions:”h]”hŒavailable_filter_functions:”…””}”(hjÃ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mchjg  ubjT  )”}”(hX  This lists the functions that ftrace has processed and can trace.
These are the function names that you can pass to
"set_ftrace_filter", "set_ftrace_notrace",
"set_graph_function", or "set_graph_notrace".
(See the section "dynamic ftrace" below for more details.)
”h]”h¸)”}”(hX  This lists the functions that ftrace has processed and can trace.
These are the function names that you can pass to
"set_ftrace_filter", "set_ftrace_notrace",
"set_graph_function", or "set_graph_notrace".
(See the section "dynamic ftrace" below for more details.)”h]”hX  This lists the functions that ftrace has processed and can trace.
These are the function names that you can pass to
â€œset_ftrace_filterâ€, â€œset_ftrace_notraceâ€,
â€œset_graph_functionâ€, or â€œset_graph_notraceâ€.
(See the section â€œdynamic ftraceâ€ below for more details.)”…””}”(hjÕ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MehjÑ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mehjg  ubh¸)”}”(hŒ!available_filter_functions_addrs:”h]”hŒ!available_filter_functions_addrs:”…””}”(hjé  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mkhjg  ubjT  )”}”(hŒ±Similar to available_filter_functions, but with address displayed
for each function. The displayed address is the patch-site address
and can differ from /proc/kallsyms address.
”h]”h¸)”}”(hŒ°Similar to available_filter_functions, but with address displayed
for each function. The displayed address is the patch-site address
and can differ from /proc/kallsyms address.”h]”hŒ°Similar to available_filter_functions, but with address displayed
for each function. The displayed address is the patch-site address
and can differ from /proc/kallsyms address.”…””}”(hjû  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mmhj÷  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mmhjg  ubh¸)”}”(hŒdyn_ftrace_total_info:”h]”hŒdyn_ftrace_total_info:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mqhjg  ubjT  )”}”(hŒ~This file is for debugging purposes. The number of functions that
have been converted to nops and are available to be traced.
”h]”h¸)”}”(hŒ}This file is for debugging purposes. The number of functions that
have been converted to nops and are available to be traced.”h]”hŒ}This file is for debugging purposes. The number of functions that
have been converted to nops and are available to be traced.”…””}”(hj!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mshj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mshjg  ubh¸)”}”(hŒenabled_functions:”h]”hŒenabled_functions:”…””}”(hj5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mvhjg  ubjT  )”}”(hXQ  This file is more for debugging ftrace, but can also be useful
in seeing if any function has a callback attached to it.
Not only does the trace infrastructure use ftrace function
trace utility, but other subsystems might too. This file
displays all functions that have a callback attached to them
as well as the number of callbacks that have been attached.
Note, a callback may also call multiple functions which will
not be listed in this count.

If the callback registered to be traced by a function with
the "save regs" attribute (thus even more overhead), a 'R'
will be displayed on the same line as the function that
is returning registers.

If the callback registered to be traced by a function with
the "ip modify" attribute (thus the regs->ip can be changed),
an 'I' will be displayed on the same line as the function that
can be overridden.

If a non ftrace trampoline is attached (BPF) a 'D' will be displayed.
Note, normal ftrace trampolines can also be attached, but only one
"direct" trampoline can be attached to a given function at a time.

Some architectures can not call direct trampolines, but instead have
the ftrace ops function located above the function entry point. In
such cases an 'O' will be displayed.

If a function had either the "ip modify" or a "direct" call attached to
it in the past, a 'M' will be shown. This flag is never cleared. It is
used to know if a function was every modified by the ftrace infrastructure,
and can be used for debugging.

If the architecture supports it, it will also show what callback
is being directly called by the function. If the count is greater
than 1 it most likely will be ftrace_ops_list_func().

If the callback of a function jumps to a trampoline that is
specific to the callback and which is not the standard trampoline,
its address will be printed as well as the function that the
trampoline calls.
”h]”(h¸)”}”(hX¾  This file is more for debugging ftrace, but can also be useful
in seeing if any function has a callback attached to it.
Not only does the trace infrastructure use ftrace function
trace utility, but other subsystems might too. This file
displays all functions that have a callback attached to them
as well as the number of callbacks that have been attached.
Note, a callback may also call multiple functions which will
not be listed in this count.”h]”hX¾  This file is more for debugging ftrace, but can also be useful
in seeing if any function has a callback attached to it.
Not only does the trace infrastructure use ftrace function
trace utility, but other subsystems might too. This file
displays all functions that have a callback attached to them
as well as the number of callbacks that have been attached.
Note, a callback may also call multiple functions which will
not be listed in this count.”…””}”(hjG  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MxhjC  ubh¸)”}”(hŒÅIf the callback registered to be traced by a function with
the "save regs" attribute (thus even more overhead), a 'R'
will be displayed on the same line as the function that
is returning registers.”h]”hŒÍIf the callback registered to be traced by a function with
the â€œsave regsâ€ attribute (thus even more overhead), a â€˜Râ€™
will be displayed on the same line as the function that
is returning registers.”…””}”(hjU  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjC  ubh¸)”}”(hŒÊIf the callback registered to be traced by a function with
the "ip modify" attribute (thus the regs->ip can be changed),
an 'I' will be displayed on the same line as the function that
can be overridden.”h]”hŒÒIf the callback registered to be traced by a function with
the â€œip modifyâ€ attribute (thus the regs->ip can be changed),
an â€˜Iâ€™ will be displayed on the same line as the function that
can be overridden.”…””}”(hjc  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M†hjC  ubh¸)”}”(hŒËIf a non ftrace trampoline is attached (BPF) a 'D' will be displayed.
Note, normal ftrace trampolines can also be attached, but only one
"direct" trampoline can be attached to a given function at a time.”h]”hŒÓIf a non ftrace trampoline is attached (BPF) a â€˜Dâ€™ will be displayed.
Note, normal ftrace trampolines can also be attached, but only one
â€œdirectâ€ trampoline can be attached to a given function at a time.”…””}”(hjq  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M‹hjC  ubh¸)”}”(hŒ¬Some architectures can not call direct trampolines, but instead have
the ftrace ops function located above the function entry point. In
such cases an 'O' will be displayed.”h]”hŒ°Some architectures can not call direct trampolines, but instead have
the ftrace ops function located above the function entry point. In
such cases an â€˜Oâ€™ will be displayed.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjC  ubh¸)”}”(hŒùIf a function had either the "ip modify" or a "direct" call attached to
it in the past, a 'M' will be shown. This flag is never cleared. It is
used to know if a function was every modified by the ftrace infrastructure,
and can be used for debugging.”h]”hX  If a function had either the â€œip modifyâ€ or a â€œdirectâ€ call attached to
it in the past, a â€˜Mâ€™ will be shown. This flag is never cleared. It is
used to know if a function was every modified by the ftrace infrastructure,
and can be used for debugging.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M“hjC  ubh¸)”}”(hŒ¸If the architecture supports it, it will also show what callback
is being directly called by the function. If the count is greater
than 1 it most likely will be ftrace_ops_list_func().”h]”hŒ¸If the architecture supports it, it will also show what callback
is being directly called by the function. If the count is greater
than 1 it most likely will be ftrace_ops_list_func().”…””}”(hj›  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M˜hjC  ubh¸)”}”(hŒÍIf the callback of a function jumps to a trampoline that is
specific to the callback and which is not the standard trampoline,
its address will be printed as well as the function that the
trampoline calls.”h]”hŒÍIf the callback of a function jumps to a trampoline that is
specific to the callback and which is not the standard trampoline,
its address will be printed as well as the function that the
trampoline calls.”…””}”(hj©  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MœhjC  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mxhjg  ubh¸)”}”(hŒtouched_functions:”h]”hŒtouched_functions:”…””}”(hj½  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¡hjg  ubjT  )”}”(hX  This file contains all the functions that ever had a function callback
to it via the ftrace infrastructure. It has the same format as
enabled_functions but shows all functions that have every been
traced.

To see any function that has every been modified by "ip modify" or a
direct trampoline, one can perform the following command:

grep ' M ' /sys/kernel/tracing/touched_functions
”h]”(h¸)”}”(hŒÌThis file contains all the functions that ever had a function callback
to it via the ftrace infrastructure. It has the same format as
enabled_functions but shows all functions that have every been
traced.”h]”hŒÌThis file contains all the functions that ever had a function callback
to it via the ftrace infrastructure. It has the same format as
enabled_functions but shows all functions that have every been
traced.”…””}”(hjÏ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M£hjË  ubh¸)”}”(hŒ~To see any function that has every been modified by "ip modify" or a
direct trampoline, one can perform the following command:”h]”hŒ‚To see any function that has every been modified by â€œip modifyâ€ or a
direct trampoline, one can perform the following command:”…””}”(hjÝ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¨hjË  ubh¸)”}”(hŒ0grep ' M ' /sys/kernel/tracing/touched_functions”h]”hŒ4grep â€˜ M â€˜ /sys/kernel/tracing/touched_functions”…””}”(hjë  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M«hjË  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M£hjg  ubh¸)”}”(hŒfunction_profile_enabled:”h]”hŒfunction_profile_enabled:”…””}”(hjÿ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M­hjg  ubjT  )”}”(hX•  When set it will enable all functions with either the function
tracer, or if configured, the function graph tracer. It will
keep a histogram of the number of functions that were called
and if the function graph tracer was configured, it will also keep
track of the time spent in those functions. The histogram
content can be displayed in the files:

trace_stat/function<cpu> ( function0, function1, etc).
”h]”(h¸)”}”(hX\  When set it will enable all functions with either the function
tracer, or if configured, the function graph tracer. It will
keep a histogram of the number of functions that were called
and if the function graph tracer was configured, it will also keep
track of the time spent in those functions. The histogram
content can be displayed in the files:”h]”hX\  When set it will enable all functions with either the function
tracer, or if configured, the function graph tracer. It will
keep a histogram of the number of functions that were called
and if the function graph tracer was configured, it will also keep
track of the time spent in those functions. The histogram
content can be displayed in the files:”…””}”(hj	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¯hj	  ubh¸)”}”(hŒ6trace_stat/function<cpu> ( function0, function1, etc).”h]”hŒ6trace_stat/function<cpu> ( function0, function1, etc).”…””}”(hj	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¶hj	  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M¯hjg  ubh¸)”}”(hŒtrace_stat:”h]”hŒtrace_stat:”…””}”(hj3	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¸hjg  ubjT  )”}”(hŒ0A directory that holds different tracing stats.
”h]”h¸)”}”(hŒ/A directory that holds different tracing stats.”h]”hŒ/A directory that holds different tracing stats.”…””}”(hjE	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MºhjA	  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mºhjg  ubh¸)”}”(hŒkprobe_events:”h]”hŒkprobe_events:”…””}”(hjY	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¼hjg  ubjT  )”}”(hŒ2Enable dynamic trace points. See kprobetrace.rst.
”h]”h¸)”}”(hŒ1Enable dynamic trace points. See kprobetrace.rst.”h]”hŒ1Enable dynamic trace points. See kprobetrace.rst.”…””}”(hjk	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¾hjg	  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M¾hjg  ubh¸)”}”(hŒkprobe_profile:”h]”hŒkprobe_profile:”…””}”(hj	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÀhjg  ubjT  )”}”(hŒ1Dynamic trace points stats. See kprobetrace.rst.
”h]”h¸)”}”(hŒ0Dynamic trace points stats. See kprobetrace.rst.”h]”hŒ0Dynamic trace points stats. See kprobetrace.rst.”…””}”(hj‘	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÂhj	  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÂhjg  ubh¸)”}”(hŒmax_graph_depth:”h]”hŒmax_graph_depth:”…””}”(hj¥	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÄhjg  ubjT  )”}”(hŒÂUsed with the function graph tracer. This is the max depth
it will trace into a function. Setting this to a value of
one will show only the first kernel function that is called
from user space.
”h]”h¸)”}”(hŒÁUsed with the function graph tracer. This is the max depth
it will trace into a function. Setting this to a value of
one will show only the first kernel function that is called
from user space.”h]”hŒÁUsed with the function graph tracer. This is the max depth
it will trace into a function. Setting this to a value of
one will show only the first kernel function that is called
from user space.”…””}”(hj·	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÆhj³	  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÆhjg  ubh¸)”}”(hŒprintk_formats:”h]”hŒprintk_formats:”…””}”(hjË	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MËhjg  ubjT  )”}”(hXg  This is for tools that read the raw format files. If an event in
the ring buffer references a string, only a pointer to the string
is recorded into the buffer and not the string itself. This prevents
tools from knowing what that string was. This file displays the string
and address for the string allowing tools to map the pointers to what
the strings were.
”h]”h¸)”}”(hXf  This is for tools that read the raw format files. If an event in
the ring buffer references a string, only a pointer to the string
is recorded into the buffer and not the string itself. This prevents
tools from knowing what that string was. This file displays the string
and address for the string allowing tools to map the pointers to what
the strings were.”h]”hXf  This is for tools that read the raw format files. If an event in
the ring buffer references a string, only a pointer to the string
is recorded into the buffer and not the string itself. This prevents
tools from knowing what that string was. This file displays the string
and address for the string allowing tools to map the pointers to what
the strings were.”…””}”(hjÝ	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÍhjÙ	  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÍhjg  ubh¸)”}”(hŒsaved_cmdlines:”h]”hŒsaved_cmdlines:”…””}”(hjñ	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÔhjg  ubjT  )”}”(hX  Only the pid of the task is recorded in a trace event unless
the event specifically saves the task comm as well. Ftrace
makes a cache of pid mappings to comms to try to display
comms for events. If a pid for a comm is not listed, then
"<...>" is displayed in the output.

If the option "record-cmd" is set to "0", then comms of tasks
will not be saved during recording. By default, it is enabled.
”h]”(h¸)”}”(hX  Only the pid of the task is recorded in a trace event unless
the event specifically saves the task comm as well. Ftrace
makes a cache of pid mappings to comms to try to display
comms for events. If a pid for a comm is not listed, then
"<...>" is displayed in the output.”h]”hX  Only the pid of the task is recorded in a trace event unless
the event specifically saves the task comm as well. Ftrace
makes a cache of pid mappings to comms to try to display
comms for events. If a pid for a comm is not listed, then
â€œ<...>â€ is displayed in the output.”…””}”(hj
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÖhjÿ	  ubh¸)”}”(hŒ|If the option "record-cmd" is set to "0", then comms of tasks
will not be saved during recording. By default, it is enabled.”h]”hŒ„If the option â€œrecord-cmdâ€ is set to â€œ0â€, then comms of tasks
will not be saved during recording. By default, it is enabled.”…””}”(hj
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÜhjÿ	  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÖhjg  ubh¸)”}”(hŒsaved_cmdlines_size:”h]”hŒsaved_cmdlines_size:”…””}”(hj%
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mßhjg  ubjT  )”}”(hŒ­By default, 128 comms are saved (see "saved_cmdlines" above). To
increase or decrease the amount of comms that are cached, echo
the number of comms to cache into this file.
”h]”h¸)”}”(hŒ¬By default, 128 comms are saved (see "saved_cmdlines" above). To
increase or decrease the amount of comms that are cached, echo
the number of comms to cache into this file.”h]”hŒ°By default, 128 comms are saved (see â€œsaved_cmdlinesâ€ above). To
increase or decrease the amount of comms that are cached, echo
the number of comms to cache into this file.”…””}”(hj7
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Máhj3
  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Máhjg  ubh¸)”}”(hŒsaved_tgids:”h]”hŒsaved_tgids:”…””}”(hjK
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Måhjg  ubjT  )”}”(hŒÓIf the option "record-tgid" is set, on each scheduling context switch
the Task Group ID of a task is saved in a table mapping the PID of
the thread to its TGID. By default, the "record-tgid" option is
disabled.
”h]”h¸)”}”(hŒÒIf the option "record-tgid" is set, on each scheduling context switch
the Task Group ID of a task is saved in a table mapping the PID of
the thread to its TGID. By default, the "record-tgid" option is
disabled.”h]”hŒÚIf the option â€œrecord-tgidâ€ is set, on each scheduling context switch
the Task Group ID of a task is saved in a table mapping the PID of
the thread to its TGID. By default, the â€œrecord-tgidâ€ option is
disabled.”…””}”(hj]
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MçhjY
  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mçhjg  ubh¸)”}”(hŒ	snapshot:”h]”hŒ	snapshot:”…””}”(hjq
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mìhjg  ubjT  )”}”(hŒœThis displays the "snapshot" buffer and also lets the user
take a snapshot of the current running trace.
See the "Snapshot" section below for more details.
”h]”h¸)”}”(hŒ›This displays the "snapshot" buffer and also lets the user
take a snapshot of the current running trace.
See the "Snapshot" section below for more details.”h]”hŒ£This displays the â€œsnapshotâ€ buffer and also lets the user
take a snapshot of the current running trace.
See the â€œSnapshotâ€ section below for more details.”…””}”(hjƒ
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mîhj
  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mîhjg  ubh¸)”}”(hŒstack_max_size:”h]”hŒstack_max_size:”…””}”(hj—
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mòhjg  ubjT  )”}”(hŒ†When the stack tracer is activated, this will display the
maximum stack size it has encountered.
See the "Stack Trace" section below.
”h]”h¸)”}”(hŒ…When the stack tracer is activated, this will display the
maximum stack size it has encountered.
See the "Stack Trace" section below.”h]”hŒ‰When the stack tracer is activated, this will display the
maximum stack size it has encountered.
See the â€œStack Traceâ€ section below.”…””}”(hj©
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Môhj¥
  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Môhjg  ubh¸)”}”(hŒstack_trace:”h]”hŒstack_trace:”…””}”(hj½
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Møhjg  ubjT  )”}”(hŒ–This displays the stack back trace of the largest stack
that was encountered when the stack tracer is activated.
See the "Stack Trace" section below.
”h]”h¸)”}”(hŒ•This displays the stack back trace of the largest stack
that was encountered when the stack tracer is activated.
See the "Stack Trace" section below.”h]”hŒ™This displays the stack back trace of the largest stack
that was encountered when the stack tracer is activated.
See the â€œStack Traceâ€ section below.”…””}”(hjÏ
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MúhjË
  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Múhjg  ubh¸)”}”(hŒstack_trace_filter:”h]”hŒstack_trace_filter:”…””}”(hjã
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mþhjg  ubjT  )”}”(hŒaThis is similar to "set_ftrace_filter" but it limits what
functions the stack tracer will check.
”h]”h¸)”}”(hŒ`This is similar to "set_ftrace_filter" but it limits what
functions the stack tracer will check.”h]”hŒdThis is similar to â€œset_ftrace_filterâ€ but it limits what
functions the stack tracer will check.”…””}”(hjõ
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M hjñ
  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M hjg  ubh¸)”}”(hŒtrace_clock:”h]”hŒtrace_clock:”…””}”(hj	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjg  ubjT  )”}”(hXÍ  Whenever an event is recorded into the ring buffer, a
"timestamp" is added. This stamp comes from a specified
clock. By default, ftrace uses the "local" clock. This
clock is very fast and strictly per cpu, but on some
systems it may not be monotonic with respect to other
CPUs. In other words, the local clocks may not be in sync
with local clocks on other CPUs.

Usual clocks for tracing::

  # cat trace_clock
  [local] global counter x86-tsc

The clock with the square brackets around it is the one in effect.

local:
        Default clock, but may not be in sync across CPUs

global:
        This clock is in sync with all CPUs but may
        be a bit slower than the local clock.

counter:
        This is not a clock at all, but literally an atomic
        counter. It counts up one by one, but is in sync
        with all CPUs. This is useful when you need to
        know exactly the order events occurred with respect to
        each other on different CPUs.

uptime:
        This uses the jiffies counter and the time stamp
        is relative to the time since boot up.

perf:
        This makes ftrace use the same clock that perf uses.
        Eventually perf will be able to read ftrace buffers
        and this will help out in interleaving the data.

x86-tsc:
        Architectures may define their own clocks. For
        example, x86 uses its own TSC cycle clock here.

ppc-tb:
        This uses the powerpc timebase register value.
        This is in sync across CPUs and can also be used
        to correlate events across hypervisor/guest if
        tb_offset is known.

mono:
        This uses the fast monotonic clock (CLOCK_MONOTONIC)
        which is monotonic and is subject to NTP rate adjustments.

mono_raw:
        This is the raw monotonic clock (CLOCK_MONOTONIC_RAW)
        which is monotonic but is not subject to any rate adjustments
        and ticks at the same rate as the hardware clocksource.

boot:
        This is the boot clock (CLOCK_BOOTTIME) and is based on the
        fast monotonic clock, but also accounts for time spent in
        suspend. Since the clock access is designed for use in
        tracing in the suspend path, some side effects are possible
        if clock is accessed after the suspend time is accounted before
        the fast mono clock is updated. In this case, the clock update
        appears to happen slightly sooner than it normally would have.
        Also on 32-bit systems, it's possible that the 64-bit boot offset
        sees a partial update. These effects are rare and post
        processing should be able to handle them. See comments in the
        ktime_get_boot_fast_ns() function for more information.

tai:
        This is the tai clock (CLOCK_TAI) and is derived from the wall-
        clock time. However, this clock does not experience
        discontinuities and backwards jumps caused by NTP inserting leap
        seconds. Since the clock access is designed for use in tracing,
        side effects are possible. The clock access may yield wrong
        readouts in case the internal TAI offset is updated e.g., caused
        by setting the system time or using adjtimex() with an offset.
        These effects are rare and post processing should be able to
        handle them. See comments in the ktime_get_tai_fast_ns()
        function for more information.

To set a clock, simply echo the clock name into this file::

  # echo global > trace_clock

Setting a clock clears the ring buffer content as well as the
"snapshot" buffer.
”h]”(h¸)”}”(hXj  Whenever an event is recorded into the ring buffer, a
"timestamp" is added. This stamp comes from a specified
clock. By default, ftrace uses the "local" clock. This
clock is very fast and strictly per cpu, but on some
systems it may not be monotonic with respect to other
CPUs. In other words, the local clocks may not be in sync
with local clocks on other CPUs.”h]”hXr  Whenever an event is recorded into the ring buffer, a
â€œtimestampâ€ is added. This stamp comes from a specified
clock. By default, ftrace uses the â€œlocalâ€ clock. This
clock is very fast and strictly per cpu, but on some
systems it may not be monotonic with respect to other
CPUs. In other words, the local clocks may not be in sync
with local clocks on other CPUs.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj  ubh¸)”}”(hŒUsual clocks for tracing::”h]”hŒUsual clocks for tracing:”…””}”(hj)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj  ubj   )”}”(hŒ0# cat trace_clock
[local] global counter x86-tsc”h]”hŒ0# cat trace_clock
[local] global counter x86-tsc”…””}”hj7  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhj  ubh¸)”}”(hŒBThe clock with the square brackets around it is the one in effect.”h]”hŒBThe clock with the square brackets around it is the one in effect.”…””}”(hjE  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj  ubhŒdefinition_list”“”)”}”(hhh]”(hŒdefinition_list_item”“”)”}”(hŒ9local:
Default clock, but may not be in sync across CPUs
”h]”(hŒterm”“”)”}”(hŒlocal:”h]”hŒlocal:”…””}”(hj`  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MhjZ  ubhŒ
definition”“”)”}”(hhh]”h¸)”}”(hŒ1Default clock, but may not be in sync across CPUs”h]”hŒ1Default clock, but may not be in sync across CPUs”…””}”(hjs  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjp  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjZ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MhjU  ubjY  )”}”(hŒZglobal:
This clock is in sync with all CPUs but may
be a bit slower than the local clock.
”h]”(j_  )”}”(hŒglobal:”h]”hŒglobal:”…””}”(hj‘  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mhj  ubjo  )”}”(hhh]”h¸)”}”(hŒQThis clock is in sync with all CPUs but may
be a bit slower than the local clock.”h]”hŒQThis clock is in sync with all CPUs but may
be a bit slower than the local clock.”…””}”(hj¢  hžhhŸNh Nubah}”(•      h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjŸ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MhjU  ubjY  )”}”(hŒòcounter:
This is not a clock at all, but literally an atomic
counter. It counts up one by one, but is in sync
with all CPUs. This is useful when you need to
know exactly the order events occurred with respect to
each other on different CPUs.
”h]”(j_  )”}”(hŒcounter:”h]”hŒcounter:”…””}”(hjÀ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M hj¼  ubjo  )”}”(hhh]”h¸)”}”(hŒèThis is not a clock at all, but literally an atomic
counter. It counts up one by one, but is in sync
with all CPUs. This is useful when you need to
know exactly the order events occurred with respect to
each other on different CPUs.”h]”hŒèThis is not a clock at all, but literally an atomic
counter. It counts up one by one, but is in sync
with all CPUs. This is useful when you need to
know exactly the order events occurred with respect to
each other on different CPUs.”…””}”(hjÑ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjÎ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj¼  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M hjU  ubjY  )”}”(hŒ`uptime:
This uses the jiffies counter and the time stamp
is relative to the time since boot up.
”h]”(j_  )”}”(hŒuptime:”h]”hŒuptime:”…””}”(hjï  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M$hjë  ubjo  )”}”(hhh]”h¸)”}”(hŒWThis uses the jiffies counter and the time stamp
is relative to the time since boot up.”h]”hŒWThis uses the jiffies counter and the time stamp
is relative to the time since boot up.”…””}”(hj   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M#hjý  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjë  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M$hjU  ubjY  )”}”(hŒ perf:
This makes ftrace use the same clock that perf uses.
Eventually perf will be able to read ftrace buffers
and this will help out in interleaving the data.
”h]”(j_  )”}”(hŒperf:”h]”hŒperf:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M)hj  ubjo  )”}”(hhh]”h¸)”}”(hŒ™This makes ftrace use the same clock that perf uses.
Eventually perf will be able to read ftrace buffers
and this will help out in interleaving the data.”h]”hŒ™This makes ftrace use the same clock that perf uses.
Eventually perf will be able to read ftrace buffers
and this will help out in interleaving the data.”…””}”(hj/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M'hj,  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M)hjU  ubjY  )”}”(hŒhx86-tsc:
Architectures may define their own clocks. For
example, x86 uses its own TSC cycle clock here.
”h]”(j_  )”}”(hŒx86-tsc:”h]”hŒx86-tsc:”…””}”(hjM  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M-hjI  ubjo  )”}”(hhh]”h¸)”}”(hŒ^Architectures may define their own clocks. For
example, x86 uses its own TSC cycle clock here.”h]”hŒ^Architectures may define their own clocks. For
example, x86 uses its own TSC cycle clock here.”…””}”(hj^  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M,hj[  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjI  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M-hjU  ubjY  )”}”(hŒ«ppc-tb:
This uses the powerpc timebase register value.
This is in sync across CPUs and can also be used
to correlate events across hypervisor/guest if
tb_offset is known.
”h]”(j_  )”}”(hŒppc-tb:”h]”hŒppc-tb:”…””}”(hj|  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M3hjx  ubjo  )”}”(hhh]”h¸)”}”(hŒ¢This uses the powerpc timebase register value.
This is in sync across CPUs and can also be used
to correlate events across hypervisor/guest if
tb_offset is known.”h]”hŒ¢This uses the powerpc timebase register value.
This is in sync across CPUs and can also be used
to correlate events across hypervisor/guest if
tb_offset is known.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M0hjŠ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjx  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M3hjU  ubjY  )”}”(hŒvmono:
This uses the fast monotonic clock (CLOCK_MONOTONIC)
which is monotonic and is subject to NTP rate adjustments.
”h]”(j_  )”}”(hŒmono:”h]”hŒmono:”…””}”(hj«  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M7hj§  ubjo  )”}”(hhh]”h¸)”}”(hŒoThis uses the fast monotonic clock (CLOCK_MONOTONIC)
which is monotonic and is subject to NTP rate adjustments.”h]”hŒoThis uses the fast monotonic clock (CLOCK_MONOTONIC)
which is monotonic and is subject to NTP rate adjustments.”…””}”(hj¼  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M6hj¹  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj§  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M7hjU  ubjY  )”}”(hŒ¶mono_raw:
This is the raw monotonic clock (CLOCK_MONOTONIC_RAW)
which is monotonic but is not subject to any rate adjustments
and ticks at the same rate as the hardware clocksource.
”h]”(j_  )”}”(hŒ	mono_raw:”h]”hŒ	mono_raw:”…””}”(hjÚ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M<hjÖ  ubjo  )”}”(hhh]”h¸)”}”(hŒ«This is the raw monotonic clock (CLOCK_MONOTONIC_RAW)
which is monotonic but is not subject to any rate adjustments
and ticks at the same rate as the hardware clocksource.”h]”hŒ«This is the raw monotonic clock (CLOCK_MONOTONIC_RAW)
which is monotonic but is not subject to any rate adjustments
and ticks at the same rate as the hardware clocksource.”…””}”(hjë  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M:hjè  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjÖ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M<hjU  ubjY  )”}”(hXœ  boot:
This is the boot clock (CLOCK_BOOTTIME) and is based on the
fast monotonic clock, but also accounts for time spent in
suspend. Since the clock access is designed for use in
tracing in the suspend path, some side effects are possible
if clock is accessed after the suspend time is accounted before
the fast mono clock is updated. In this case, the clock update
appears to happen slightly sooner than it normally would have.
Also on 32-bit systems, it's possible that the 64-bit boot offset
sees a partial update. These effects are rare and post
processing should be able to handle them. See comments in the
ktime_get_boot_fast_ns() function for more information.
”h]”(j_  )”}”(hŒboot:”h]”hŒboot:”…””}”(hj	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MIhj  ubjo  )”}”(hhh]”h¸)”}”(hX•  This is the boot clock (CLOCK_BOOTTIME) and is based on the
fast monotonic clock, but also accounts for time spent in
suspend. Since the clock access is designed for use in
tracing in the suspend path, some side effects are possible
if clock is accessed after the suspend time is accounted before
the fast mono clock is updated. In this case, the clock update
appears to happen slightly sooner than it normally would have.
Also on 32-bit systems, it's possible that the 64-bit boot offset
sees a partial update. These effects are rare and post
processing should be able to handle them. See comments in the
ktime_get_boot_fast_ns() function for more information.”h]”hX—  This is the boot clock (CLOCK_BOOTTIME) and is based on the
fast monotonic clock, but also accounts for time spent in
suspend. Since the clock access is designed for use in
tracing in the suspend path, some side effects are possible
if clock is accessed after the suspend time is accounted before
the fast mono clock is updated. In this case, the clock update
appears to happen slightly sooner than it normally would have.
Also on 32-bit systems, itâ€™s possible that the 64-bit boot offset
sees a partial update. These effects are rare and post
processing should be able to handle them. See comments in the
ktime_get_boot_fast_ns() function for more information.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M?hj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MIhjU  ubjY  )”}”(hXK  tai:
This is the tai clock (CLOCK_TAI) and is derived from the wall-
clock time. However, this clock does not experience
discontinuities and backwards jumps caused by NTP inserting leap
seconds. Since the clock access is designed for use in tracing,
side effects are possible. The clock access may yield wrong
readouts in case the internal TAI offset is updated e.g., caused
by setting the system time or using adjtimex() with an offset.
These effects are rare and post processing should be able to
handle them. See comments in the ktime_get_tai_fast_ns()
function for more information.
”h]”(j_  )”}”(hŒtai:”h]”hŒtai:”…””}”(hj8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MUhj4  ubjo  )”}”(hhh]”h¸)”}”(hXE  This is the tai clock (CLOCK_TAI) and is derived from the wall-
clock time. However, this clock does not experience
discontinuities and backwards jumps caused by NTP inserting leap
seconds. Since the clock access is designed for use in tracing,
side effects are possible. The clock access may yield wrong
readouts in case the internal TAI offset is updated e.g., caused
by setting the system time or using adjtimex() with an offset.
These effects are rare and post processing should be able to
handle them. See comments in the ktime_get_tai_fast_ns()
function for more information.”h]”hXE  This is the tai clock (CLOCK_TAI) and is derived from the wall-
clock time. However, this clock does not experience
discontinuities and backwards jumps caused by NTP inserting leap
seconds. Since the clock access is designed for use in tracing,
side effects are possible. The clock access may yield wrong
readouts in case the internal TAI offset is updated e.g., caused
by setting the system time or using adjtimex() with an offset.
These effects are rare and post processing should be able to
handle them. See comments in the ktime_get_tai_fast_ns()
function for more information.”…””}”(hjI  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MLhjF  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj4  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MUhjU  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hj  ubh¸)”}”(hŒ;To set a clock, simply echo the clock name into this file::”h]”hŒ:To set a clock, simply echo the clock name into this file:”…””}”(hji  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MWhj  ubj   )”}”(hŒ# echo global > trace_clock”h]”hŒ# echo global > trace_clock”…””}”hjw  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MYhj  ubh¸)”}”(hŒPSetting a clock clears the ring buffer content as well as the
"snapshot" buffer.”h]”hŒTSetting a clock clears the ring buffer content as well as the
â€œsnapshotâ€ buffer.”…””}”(hj…  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M[hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mhjg  ubh¸)”}”(hŒtrace_marker:”h]”hŒtrace_marker:”…””}”(hj™  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M^hjg  ubjT  )”}”(hXñ  This is a very useful file for synchronizing user space
with events happening in the kernel. Writing strings into
this file will be written into the ftrace buffer.

It is useful in applications to open this file at the start
of the application and just reference the file descriptor
for the file::

        void trace_write(const char *fmt, ...)
        {
                va_list ap;
                char buf[256];
                int n;

                if (trace_fd < 0)
                        return;

                va_start(ap, fmt);
                n = vsnprintf(buf, 256, fmt, ap);
                va_end(ap);

                write(trace_fd, buf, n);
        }

start::

        trace_fd = open("trace_marker", O_WRONLY);

Note: Writing into the trace_marker file can also initiate triggers
      that are written into /sys/kernel/tracing/events/ftrace/print/trigger
      See "Event triggers" in Documentation/trace/events.rst and an
      example in Documentation/trace/histogram.rst (Section 3.)
”h]”(h¸)”}”(hŒ£This is a very useful file for synchronizing user space
with events happening in the kernel. Writing strings into
this file will be written into the ftrace buffer.”h]”hŒ£This is a very useful file for synchronizing user space
with events happening in the kernel. Writing strings into
this file will be written into the ftrace buffer.”…””}”(hj«  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M`hj§  ubh¸)”}”(hŒ„It is useful in applications to open this file at the start
of the application and just reference the file descriptor
for the file::”h]”hŒƒIt is useful in applications to open this file at the start
of the application and just reference the file descriptor
for the file:”…””}”(hj¹  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mdhj§  ubj   )”}”(hX  void trace_write(const char *fmt, ...)
{
        va_list ap;
        char buf[256];
        int n;

        if (trace_fd < 0)
                return;

        va_start(ap, fmt);
        n = vsnprintf(buf, 256, fmt, ap);
        va_end(ap);

        write(trace_fd, buf, n);
}”h]”hX  void trace_write(const char *fmt, ...)
{
        va_list ap;
        char buf[256];
        int n;

        if (trace_fd < 0)
                return;

        va_start(ap, fmt);
        n = vsnprintf(buf, 256, fmt, ap);
        va_end(ap);

        write(trace_fd, buf, n);
}”…””}”hjÇ  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhhj§  ubh¸)”}”(hŒstart::”h]”hŒstart:”…””}”(hjÕ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mxhj§  ubj   )”}”(hŒ*trace_fd = open("trace_marker", O_WRONLY);”h]”hŒ*trace_fd = open("trace_marker", O_WRONLY);”…””}”hjã  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mzhj§  ubjT  )”}”(hhh]”jY  )”}”(hX  Note: Writing into the trace_marker file can also initiate triggers
that are written into /sys/kernel/tracing/events/ftrace/print/trigger
See "Event triggers" in Documentation/trace/events.rst and an
example in Documentation/trace/histogram.rst (Section 3.)
”h]”(j_  )”}”(hŒCNote: Writing into the trace_marker file can also initiate triggers”h]”hŒCNote: Writing into the trace_marker file can also initiate triggers”…””}”(hjø  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mhjô  ubjo  )”}”(hhh]”h¸)”}”(hŒ½that are written into /sys/kernel/tracing/events/ftrace/print/trigger
See "Event triggers" in Documentation/trace/events.rst and an
example in Documentation/trace/histogram.rst (Section 3.)”h]”hŒÁthat are written into /sys/kernel/tracing/events/ftrace/print/trigger
See â€œEvent triggersâ€ in Documentation/trace/events.rst and an
example in Documentation/trace/histogram.rst (Section 3.)”…””}”(hj	  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M}hj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjô  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhjñ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hj§  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M`hjg  ubh¸)”}”(hŒtrace_marker_raw:”h]”hŒtrace_marker_raw:”…””}”(hj/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjg  ubjT  )”}”(hŒ™This is similar to trace_marker above, but is meant for binary data
to be written to it, where a tool can be used to parse the data
from trace_pipe_raw.
”h]”h¸)”}”(hŒ˜This is similar to trace_marker above, but is meant for binary data
to be written to it, where a tool can be used to parse the data
from trace_pipe_raw.”h]”hŒ˜This is similar to trace_marker above, but is meant for binary data
to be written to it, where a tool can be used to parse the data
from trace_pipe_raw.”…””}”(hjA  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mƒhj=  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mƒhjg  ubh¸)”}”(hŒuprobe_events:”h]”hŒuprobe_events:”…””}”(hjU  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M‡hjg  ubjT  )”}”(hŒ:Add dynamic tracepoints in programs.
See uprobetracer.rst
”h]”h¸)”}”(hŒ9Add dynamic tracepoints in programs.
See uprobetracer.rst”h]”hŒ9Add dynamic tracepoints in programs.
See uprobetracer.rst”…””}”(hjg  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M‰hjc  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M‰hjg  ubh¸)”}”(hŒuprobe_profile:”h]”hŒuprobe_profile:”…””}”(hj{  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MŒhjg  ubjT  )”}”(hŒ'Uprobe statistics. See uprobetrace.txt
”h]”h¸)”}”(hŒ&Uprobe statistics. See uprobetrace.txt”h]”hŒ&Uprobe statistics. See uprobetrace.txt”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MŽhj‰  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MŽhjg  ubh¸)”}”(hŒ
instances:”h]”hŒ
instances:”…””}”(hj¡  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjg  ubjT  )”}”(hŒ‰This is a way to make multiple trace buffers where different
events can be recorded in different buffers.
See "Instances" section below.
”h]”h¸)”}”(hŒˆThis is a way to make multiple trace buffers where different
events can be recorded in different buffers.
See "Instances" section below.”h]”hŒŒThis is a way to make multiple trace buffers where different
events can be recorded in different buffers.
See â€œInstancesâ€ section below.”…””}”(hj³  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M’hj¯  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M’hjg  ubh¸)”}”(hŒevents:”h]”hŒevents:”…””}”(hjÇ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M–hjg  ubjT  )”}”(hXd  This is the trace event directory. It holds event tracepoints
(also known as static tracepoints) that have been compiled
into the kernel. It shows what event tracepoints exist
and how they are grouped by system. There are "enable"
files at various levels that can enable the tracepoints
when a "1" is written to them.

See events.rst for more information.
”h]”(h¸)”}”(hX=  This is the trace event directory. It holds event tracepoints
(also known as static tracepoints) that have been compiled
into the kernel. It shows what event tracepoints exist
and how they are grouped by system. There are "enable"
files at various levels that can enable the tracepoints
when a "1" is written to them.”h]”hXE  This is the trace event directory. It holds event tracepoints
(also known as static tracepoints) that have been compiled
into the kernel. It shows what event tracepoints exist
and how they are grouped by system. There are â€œenableâ€
files at various levels that can enable the tracepoints
when a â€œ1â€ is written to them.”…””}”(hjÙ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M˜hjÕ  ubh¸)”}”(hŒ$See events.rst for more information.”h]”hŒ$See events.rst for more information.”…””}”(hjç  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MŸhjÕ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M˜hjg  ubh¸)”}”(hŒ
set_event:”h]”hŒ
set_event:”…””}”(hjû  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¡hjg  ubjT  )”}”(hŒfBy echoing in the event into this file, will enable that event.

See events.rst for more information.
”h]”(h¸)”}”(hŒ?By echoing in the event into this file, will enable that event.”h]”hŒ?By echoing in the event into this file, will enable that event.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M£hj	  ubh¸)”}”(hŒ$See events.rst for more information.”h]”hŒ$See events.rst for more information.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¥hj	  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M£hjg  ubh¸)”}”(hŒavailable_events:”h]”hŒavailable_events:”…””}”(hj/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M§hjg  ubjT  )”}”(hŒWA list of events that can be enabled in tracing.

See events.rst for more information.
”h]”(h¸)”}”(hŒ0A list of events that can be enabled in tracing.”h]”hŒ0A list of events that can be enabled in tracing.”…””}”(hjA  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M©hj=  ubh¸)”}”(hŒ$See events.rst for more information.”h]”hŒ$See events.rst for more information.”…””}”(hjO  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M«hj=  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M©hjg  ubh¸)”}”(hŒtimestamp_mode:”h]”hŒtimestamp_mode:”…””}”(hjc  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M­hjg  ubjT  )”}”(hXÌ  Certain tracers may change the timestamp mode used when
logging trace events into the event buffer.  Events with
different modes can coexist within a buffer but the mode in
effect when an event is logged determines which timestamp mode
is used for that event.  The default timestamp mode is
'delta'.

Usual timestamp modes for tracing:

  # cat timestamp_mode
  [delta] absolute

  The timestamp mode with the square brackets around it is the
  one in effect.

  delta: Default timestamp mode - timestamp is a delta against
         a per-buffer timestamp.

  absolute: The timestamp is a full timestamp, not a delta
         against some other value.  As such it takes up more
         space and is less efficient.
”h]”(h¸)”}”(hX+  Certain tracers may change the timestamp mode used when
logging trace events into the event buffer.  Events with
different modes can coexist within a buffer but the mode in
effect when an event is logged determines which timestamp mode
is used for that event.  The default timestamp mode is
'delta'.”h]”hX/  Certain tracers may change the timestamp mode used when
logging trace events into the event buffer.  Events with
different modes can coexist within a buffer but the mode in
effect when an event is logged determines which timestamp mode
is used for that event.  The default timestamp mode is
â€˜deltaâ€™.”…””}”(hju  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¯hjq  ubh¸)”}”(hŒ"Usual timestamp modes for tracing:”h]”hŒ"Usual timestamp modes for tracing:”…””}”(hjƒ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¶hjq  ubjT  )”}”(hXi  # cat timestamp_mode
[delta] absolute

The timestamp mode with the square brackets around it is the
one in effect.

delta: Default timestamp mode - timestamp is a delta against
       a per-buffer timestamp.

absolute: The timestamp is a full timestamp, not a delta
       against some other value.  As such it takes up more
       space and is less efficient.
”h]”(h¸)”}”(hŒ%# cat timestamp_mode
[delta] absolute”h]”hŒ%# cat timestamp_mode
[delta] absolute”…””}”(hj•  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¸hj‘  ubh¸)”}”(hŒKThe timestamp mode with the square brackets around it is the
one in effect.”h]”hŒKThe timestamp mode with the square brackets around it is the
one in effect.”…””}”(hj£  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M»hj‘  ubjT  )”}”(hhh]”(jY  )”}”(hŒUdelta: Default timestamp mode - timestamp is a delta against
a per-buffer timestamp.
”h]”(j_  )”}”(hŒ<delta: Default timestamp mode - timestamp is a delta against”h]”hŒ<delta: Default timestamp mode - timestamp is a delta against”…””}”(hj¸  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M¿hj´  ubjo  )”}”(hhh]”h¸)”}”(hŒa per-buffer timestamp.”h]”hŒa per-buffer timestamp.”…””}”(hjÉ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¿hjÆ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj´  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M¿hj±  ubjY  )”}”(hŒŠabsolute: The timestamp is a full timestamp, not a delta
against some other value.  As such it takes up more
space and is less efficient.
”h]”(j_  )”}”(hŒ8absolute: The timestamp is a full timestamp, not a delta”h]”hŒ8absolute: The timestamp is a full timestamp, not a delta”…””}”(hjç  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MÃhjã  ubjo  )”}”(hhh]”h¸)”}”(hŒPagainst some other value.  As such it takes up more
space and is less efficient.”h]”hŒPagainst some other value.  As such it takes up more
space and is less efficient.”…””}”(hjø  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÂhjõ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjã  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MÃhj±  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hj‘  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M¸hjq  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M¯hjg  ubh¸)”}”(hŒhwlat_detector:”h]”hŒhwlat_detector:”…””}”(hj$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÅhjg  ubjT  )”}”(hŒ\Directory for the Hardware Latency Detector.
See "Hardware Latency Detector" section below.
”h]”h¸)”}”(hŒ[Directory for the Hardware Latency Detector.
See "Hardware Latency Detector" section below.”h]”hŒ_Directory for the Hardware Latency Detector.
See â€œHardware Latency Detectorâ€ section below.”…””}”(hj6  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÇhj2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÇhjg  ubh¸)”}”(hŒper_cpu:”h]”hŒper_cpu:”…””}”(hjJ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÊhjg  ubjT  )”}”(hŒAThis is a directory that contains the trace per_cpu information.
”h]”h¸)”}”(hŒ@This is a directory that contains the trace per_cpu information.”h]”hŒ@This is a directory that contains the trace per_cpu information.”…””}”(hj\  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÌhjX  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÌhjg  ubh¸)”}”(hŒper_cpu/cpu0/buffer_size_kb:”h]”hŒper_cpu/cpu0/buffer_size_kb:”…””}”(hjp  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÎhjg  ubjT  )”}”(hXK  The ftrace buffer is defined per_cpu. That is, there's a separate
buffer for each CPU to allow writes to be done atomically,
and free from cache bouncing. These buffers may have different
size buffers. This file is similar to the buffer_size_kb
file, but it only displays or sets the buffer size for the
specific CPU. (here cpu0).
”h]”h¸)”}”(hXJ  The ftrace buffer is defined per_cpu. That is, there's a separate
buffer for each CPU to allow writes to be done atomically,
and free from cache bouncing. These buffers may have different
size buffers. This file is similar to the buffer_size_kb
file, but it only displays or sets the buffer size for the
specific CPU. (here cpu0).”h]”hXL  The ftrace buffer is defined per_cpu. That is, thereâ€™s a separate
buffer for each CPU to allow writes to be done atomically,
and free from cache bouncing. These buffers may have different
size buffers. This file is similar to the buffer_size_kb
file, but it only displays or sets the buffer size for the
specific CPU. (here cpu0).”…””}”(hj‚  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÐhj~  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÐhjg  ubh¸)”}”(hŒper_cpu/cpu0/trace:”h]”hŒper_cpu/cpu0/trace:”…””}”(hj–  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M×hjg  ubjT  )”}”(hŒ”This is similar to the "trace" file, but it will only display
the data specific for the CPU. If written to, it only clears
the specific CPU buffer.
”h]”h¸)”}”(hŒ“This is similar to the "trace" file, but it will only display
the data specific for the CPU. If written to, it only clears
the specific CPU buffer.”h]”hŒ—This is similar to the â€œtraceâ€ file, but it will only display
the data specific for the CPU. If written to, it only clears
the specific CPU buffer.”…””}”(hj¨  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÙhj¤  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÙhjg  ubh¸)”}”(hŒper_cpu/cpu0/trace_pipe”h]”hŒper_cpu/cpu0/trace_pipe”…””}”(hj¼  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÝhjg  ubjT  )”}”(hŒ‰This is similar to the "trace_pipe" file, and is a consuming
read, but it will only display (and consume) the data specific
for the CPU.
”h]”h¸)”}”(hŒˆThis is similar to the "trace_pipe" file, and is a consuming
read, but it will only display (and consume) the data specific
for the CPU.”h]”hŒŒThis is similar to the â€œtrace_pipeâ€ file, and is a consuming
read, but it will only display (and consume) the data specific
for the CPU.”…””}”(hjÎ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MßhjÊ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mßhjg  ubh¸)”}”(hŒper_cpu/cpu0/trace_pipe_raw”h]”hŒper_cpu/cpu0/trace_pipe_raw”…””}”(hjâ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mãhjg  ubjT  )”}”(hX•  For tools that can parse the ftrace ring buffer binary format,
the trace_pipe_raw file can be used to extract the data
from the ring buffer directly. With the use of the splice()
system call, the buffer data can be quickly transferred to
a file or to the network where a server is collecting the
data.

Like trace_pipe, this is a consuming reader, where multiple
reads will always produce different data.
”h]”(h¸)”}”(hX-  For tools that can parse the ftrace ring buffer binary format,
the trace_pipe_raw file can be used to extract the data
from the ring buffer directly. With the use of the splice()
system call, the buffer data can be quickly transferred to
a file or to the network where a server is collecting the
data.”h]”hX-  For tools that can parse the ftrace ring buffer binary format,
the trace_pipe_raw file can be used to extract the data
from the ring buffer directly. With the use of the splice()
system call, the buffer data can be quickly transferred to
a file or to the network where a server is collecting the
data.”…””}”(hjô  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Måhjð  ubh¸)”}”(hŒeLike trace_pipe, this is a consuming reader, where multiple
reads will always produce different data.”h]”hŒeLike trace_pipe, this is a consuming reader, where multiple
reads will always produce different data.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mìhjð  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Måhjg  ubh¸)”}”(hŒper_cpu/cpu0/snapshot:”h]”hŒper_cpu/cpu0/snapshot:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mïhjg  ubjT  )”}”(hŒÒThis is similar to the main "snapshot" file, but will only
snapshot the current CPU (if supported). It only displays
the content of the snapshot for a given CPU, and if
written to, only clears this CPU buffer.
”h]”h¸)”}”(hŒÑThis is similar to the main "snapshot" file, but will only
snapshot the current CPU (if supported). It only displays
the content of the snapshot for a given CPU, and if
written to, only clears this CPU buffer.”h]”hŒÕThis is similar to the main â€œsnapshotâ€ file, but will only
snapshot the current CPU (if supported). It only displays
the content of the snapshot for a given CPU, and if
written to, only clears this CPU buffer.”…””}”(hj(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mñhj$  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mñhjg  ubh¸)”}”(hŒper_cpu/cpu0/snapshot_raw:”h]”hŒper_cpu/cpu0/snapshot_raw:”…””}”(hj<  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Möhjg  ubjT  )”}”(hŒkSimilar to the trace_pipe_raw, but will read the binary format
from the snapshot buffer for the given CPU.
”h]”h¸)”}”(hŒjSimilar to the trace_pipe_raw, but will read the binary format
from the snapshot buffer for the given CPU.”h]”hŒjSimilar to the trace_pipe_raw, but will read the binary format
from the snapshot buffer for the given CPU.”…””}”(hjN  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MøhjJ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Møhjg  ubh¸)”}”(hŒper_cpu/cpu0/stats:”h]”hŒper_cpu/cpu0/stats:”…””}”(hjb  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mûhjg  ubjT  )”}”(hXÆ  This displays certain stats about the ring buffer:

entries:
        The number of events that are still in the buffer.

overrun:
        The number of lost events due to overwriting when
        the buffer was full.

commit overrun:
        Should always be zero.
        This gets set if so many events happened within a nested
        event (ring buffer is re-entrant), that it fills the
        buffer and starts dropping events.

bytes:
        Bytes actually read (not overwritten).

oldest event ts:
        The oldest timestamp in the buffer

now ts:
        The current timestamp

dropped events:
        Events lost due to overwrite option being off.

read events:
        The number of events read.
”h]”(h¸)”}”(hŒ2This displays certain stats about the ring buffer:”h]”hŒ2This displays certain stats about the ring buffer:”…””}”(hjt  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mýhjp  ubjT  )”}”(hhh]”(jY  )”}”(hŒ<entries:
The number of events that are still in the buffer.
”h]”(j_  )”}”(hŒentries:”h]”hŒentries:”…””}”(hj‰  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M hj…  ubjo  )”}”(hhh]”h¸)”}”(hŒ2The number of events that are still in the buffer.”h]”hŒ2The number of events that are still in the buffer.”…””}”(hjš  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M hj—  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj…  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M hj‚  ubjY  )”}”(hŒPoverrun:
The number of lost events due to overwriting when
the buffer was full.
”h]”(j_  )”}”(hŒoverrun:”h]”hŒoverrun:”…””}”(hj¸  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mhj´  ubjo  )”}”(hhh]”h¸)”}”(hŒFThe number of lost events due to overwriting when
the buffer was full.”h]”hŒFThe number of lost events due to overwriting when
the buffer was full.”…””}”(hjÉ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjÆ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj´  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj‚  ubjY  )”}”(hŒ¸commit overrun:
Should always be zero.
This gets set if so many events happened within a nested
event (ring buffer is re-entrant), that it fills the
buffer and starts dropping events.
”h]”(j_  )”}”(hŒcommit overrun:”h]”hŒcommit overrun:”…””}”(hjç  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M
hjã  ubjo  )”}”(hhh]”h¸)”}”(hŒ§Should always be zero.
This gets set if so many events happened within a nested
event (ring buffer is re-entrant), that it fills the
buffer and starts dropping events.”h]”hŒ§Should always be zero.
This gets set if so many events happened within a nested
event (ring buffer is re-entrant), that it fills the
buffer and starts dropping events.”…””}”(hjø  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjõ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjã  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M
hj‚  ubjY  )”}”(hŒ.bytes:
Bytes actually read (not overwritten).
”h]”(j_  )”}”(hŒbytes:”h]”hŒbytes:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mhj  ubjo  )”}”(hhh]”h¸)”}”(hŒ&Bytes actually read (not overwritten).”h]”hŒ&Bytes actually read (not overwritten).”…””}”(hj'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj$  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj‚  ubjY  )”}”(hŒ4oldest event ts:
The oldest timestamp in the buffer
”h]”(j_  )”}”(hŒoldest event ts:”h]”hŒoldest event ts:”…””}”(hjE  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MhjA  ubjo  )”}”(hhh]”h¸)”}”(hŒ"The oldest timestamp in the buffer”h]”hŒ"The oldest timestamp in the buffer”…””}”(hjV  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjS  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjA  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj‚  ubjY  )”}”(hŒnow ts:
The current timestamp
”h]”(j_  )”}”(hŒnow ts:”h]”hŒnow ts:”…””}”(hjt  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mhjp  ubjo  )”}”(hhh]”h¸)”}”(hŒThe current timestamp”h]”hŒThe current timestamp”…””}”(hj…  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj‚  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjp  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj‚  ubjY  )”}”(hŒ?dropped events:
Events lost due to overwrite option being off.
”h]”(j_  )”}”(hŒdropped events:”h]”hŒdropped events:”…””}”(hj£  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MhjŸ  ubjo  )”}”(hhh]”h¸)”}”(hŒ.Events lost due to overwrite option being off.”h]”hŒ.Events lost due to overwrite option being off.”…””}”(hj´  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj±  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjŸ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj‚  ubjY  )”}”(hŒ(read events:
The number of events read.
”h]”(j_  )”}”(hŒread events:”h]”hŒread events:”…””}”(hjÒ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MhjÎ  ubjo  )”}”(hhh]”h¸)”}”(hŒThe number of events read.”h]”hŒThe number of events read.”…””}”(hjã  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjà  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjÎ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj‚  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hjp  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mýhjg  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h K_hjU  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h K]hjr  hžhubeh}”(h]”Œthe-file-system”ah ]”h"]”Œthe file system”ah$]”h&]”uh1h¡hh£hžhhŸh¶h K/ubh¢)”}”(hhh]”(h§)”}”(hŒThe Tracers”h]”hŒThe Tracers”…””}”(hj   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj  hžhhŸh¶h Mubh¸)”}”(hŒ;Here is the list of current tracers that may be configured.”h]”hŒ;Here is the list of current tracers that may be configured.”…””}”(hj.  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj  hžhubjT  )”}”(hXÖ
  "function"

      Function call tracer to trace all kernel functions.

"function_graph"

      Similar to the function tracer except that the
      function tracer probes the functions on their entry
      whereas the function graph tracer traces on both entry
      and exit of the functions. It then provides the ability
      to draw a graph of function calls similar to C code
      source.

      Note that the function graph calculates the timings of when the
      function starts and returns internally and for each instance. If
      there are two instances that run function graph tracer and traces
      the same functions, the length of the timings may be slightly off as
      each read the timestamp separately and not at the same time.

"blk"

      The block tracer. The tracer used by the blktrace user
      application.

"hwlat"

      The Hardware Latency tracer is used to detect if the hardware
      produces any latency. See "Hardware Latency Detector" section
      below.

"irqsoff"

      Traces the areas that disable interrupts and saves
      the trace with the longest max latency.
      See tracing_max_latency. When a new max is recorded,
      it replaces the old trace. It is best to view this
      trace with the latency-format option enabled, which
      happens automatically when the tracer is selected.

"preemptoff"

      Similar to irqsoff but traces and records the amount of
      time for which preemption is disabled.

"preemptirqsoff"

      Similar to irqsoff and preemptoff, but traces and
      records the largest time for which irqs and/or preemption
      is disabled.

"wakeup"

      Traces and records the max latency that it takes for
      the highest priority task to get scheduled after
      it has been woken up.
      Traces all tasks as an average developer would expect.

"wakeup_rt"

      Traces and records the max latency that it takes for just
      RT tasks (as the current "wakeup" does). This is useful
      for those interested in wake up timings of RT tasks.

"wakeup_dl"

      Traces and records the max latency that it takes for
      a SCHED_DEADLINE task to be woken (as the "wakeup" and
      "wakeup_rt" does).

"mmiotrace"

      A special tracer that is used to trace binary module.
      It will trace all the calls that a module makes to the
      hardware. Everything it writes and reads from the I/O
      as well.

"branch"

      This tracer can be configured when tracing likely/unlikely
      calls within the kernel. It will trace when a likely and
      unlikely branch is hit and if it was correct in its prediction
      of being correct.

"nop"

      This is the "trace nothing" tracer. To remove all
      tracers from tracing simply echo "nop" into
      current_tracer.
”h]”(h¸)”}”(hŒ
"function"”h]”hŒâ€œfunctionâ€”…””}”(hj@  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M hj<  ubjT  )”}”(hŒ4Function call tracer to trace all kernel functions.
”h]”h¸)”}”(hŒ3Function call tracer to trace all kernel functions.”h]”hŒ3Function call tracer to trace all kernel functions.”…””}”(hjR  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M"hjN  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M"hj<  ubh¸)”}”(hŒ"function_graph"”h]”hŒâ€œfunction_graphâ€”…””}”(hjf  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M$hj<  ubjT  )”}”(hXT  Similar to the function tracer except that the
function tracer probes the functions on their entry
whereas the function graph tracer traces on both entry
and exit of the functions. It then provides the ability
to draw a graph of function calls similar to C code
source.

Note that the function graph calculates the timings of when the
function starts and returns internally and for each instance. If
there are two instances that run function graph tracer and traces
the same functions, the length of the timings may be slightly off as
each read the timestamp separately and not at the same time.
”h]”(h¸)”}”(hX  Similar to the function tracer except that the
function tracer probes the functions on their entry
whereas the function graph tracer traces on both entry
and exit of the functions. It then provides the ability
to draw a graph of function calls similar to C code
source.”h]”hX  Similar to the function tracer except that the
function tracer probes the functions on their entry
whereas the function graph tracer traces on both entry
and exit of the functions. It then provides the ability
to draw a graph of function calls similar to C code
source.”…””}”(hjx  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M&hjt  ubh¸)”}”(hXD  Note that the function graph calculates the timings of when the
function starts and returns internally and for each instance. If
there are two instances that run function graph tracer and traces
the same functions, the length of the timings may be slightly off as
each read the timestamp separately and not at the same time.”h]”hXD  Note that the function graph calculates the timings of when the
function starts and returns internally and for each instance. If
there are two instances that run function graph tracer and traces
the same functions, the length of the timings may be slightly off as
each read the timestamp separately and not at the same time.”…””}”(hj†  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M-hjt  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M&hj<  ubh¸)”}”(hŒ"blk"”h]”hŒ	â€œblkâ€”…””}”(hjš  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M3hj<  ubjT  )”}”(hŒDThe block tracer. The tracer used by the blktrace user
application.
”h]”h¸)”}”(hŒCThe block tracer. The tracer used by the blktrace user
application.”h]”hŒCThe block tracer. The tracer used by the blktrace user
application.”…””}”(hj¬  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M5hj¨  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M5hj<  ubh¸)”}”(hŒ"hwlat"”h]”hŒâ€œhwlatâ€”…””}”(hjÀ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M8hj<  ubjT  )”}”(hŒƒThe Hardware Latency tracer is used to detect if the hardware
produces any latency. See "Hardware Latency Detector" section
below.
”h]”h¸)”}”(hŒ‚The Hardware Latency tracer is used to detect if the hardware
produces any latency. See "Hardware Latency Detector" section
below.”h]”hŒ†The Hardware Latency tracer is used to detect if the hardware
produces any latency. See â€œHardware Latency Detectorâ€ section
below.”…””}”(hjÒ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M:hjÎ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M:hj<  ubh¸)”}”(hŒ	"irqsoff"”h]”hŒâ€œirqsoffâ€”…””}”(hjæ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M>hj<  ubjT  )”}”(hX*  Traces the areas that disable interrupts and saves
the trace with the longest max latency.
See tracing_max_latency. When a new max is recorded,
it replaces the old trace. It is best to view this
trace with the latency-format option enabled, which
happens automatically when the tracer is selected.
”h]”h¸)”}”(hX)  Traces the areas that disable interrupts and saves
the trace with the longest max latency.
See tracing_max_latency. When a new max is recorded,
it replaces the old trace. It is best to view this
trace with the latency-format option enabled, which
happens automatically when the tracer is selected.”h]”hX)  Traces the areas that disable interrupts and saves
the trace with the longest max latency.
See tracing_max_latency. When a new max is recorded,
it replaces the old trace. It is best to view this
trace with the latency-format option enabled, which
happens automatically when the tracer is selected.”…””}”(hjø  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M@hjô  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M@hj<  ubh¸)”}”(hŒ"preemptoff"”h]”hŒâ€œpreemptoffâ€”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MGhj<  ubjT  )”}”(hŒ_Similar to irqsoff but traces and records the amount of
time for which preemption is disabled.
”h]”h¸)”}”(hŒ^Similar to irqsoff but traces and records the amount of
time for which preemption is disabled.”h]”hŒ^Similar to irqsoff but traces and records the amount of
time for which preemption is disabled.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MIhj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MIhj<  ubh¸)”}”(hŒ"preemptirqsoff"”h]”hŒâ€œpreemptirqsoffâ€”…””}”(hj2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MLhj<  ubjT  )”}”(hŒySimilar to irqsoff and preemptoff, but traces and
records the largest time for which irqs and/or preemption
is disabled.
”h]”h¸)”}”(hŒxSimilar to irqsoff and preemptoff, but traces and
records the largest time for which irqs and/or preemption
is disabled.”h]”hŒxSimilar to irqsoff and preemptoff, but traces and
records the largest time for which irqs and/or preemption
is disabled.”…””}”(hjD  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MNhj@  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MNhj<  ubh¸)”}”(hŒ"wakeup"”h]”hŒâ€œwakeupâ€”…””}”(hjX  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MRhj<  ubjT  )”}”(hŒ³Traces and records the max latency that it takes for
the highest priority task to get scheduled after
it has been woken up.
Traces all tasks as an average developer would expect.
”h]”h¸)”}”(hŒ²Traces and records the max latency that it takes for
the highest priority task to get scheduled after
it has been woken up.
Traces all tasks as an average developer would expect.”h]”hŒ²Traces and records the max latency that it takes for
the highest priority task to get scheduled after
it has been woken up.
Traces all tasks as an average developer would expect.”…””}”(hjj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MThjf  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MThj<  ubh¸)”}”(hŒ"wakeup_rt"”h]”hŒâ€œwakeup_rtâ€”…””}”(hj~  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MYhj<  ubjT  )”}”(hŒ§Traces and records the max latency that it takes for just
RT tasks (as the current "wakeup" does). This is useful
for those interested in wake up timings of RT tasks.
”h]”h¸)”}”(hŒ¦Traces and records the max latency that it takes for just
RT tasks (as the current "wakeup" does). This is useful
for those interested in wake up timings of RT tasks.”h]”hŒªTraces and records the max latency that it takes for just
RT tasks (as the current â€œwakeupâ€ does). This is useful
for those interested in wake up timings of RT tasks.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M[hjŒ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M[hj<  ubh¸)”}”(hŒ"wakeup_dl"”h]”hŒâ€œwakeup_dlâ€”…””}”(hj¤  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M_hj<  ubjT  )”}”(hŒTraces and records the max latency that it takes for
a SCHED_DEADLINE task to be woken (as the "wakeup" and
"wakeup_rt" does).
”h]”h¸)”}”(hŒ~Traces and records the max latency that it takes for
a SCHED_DEADLINE task to be woken (as the "wakeup" and
"wakeup_rt" does).”h]”hŒ†Traces and records the max latency that it takes for
a SCHED_DEADLINE task to be woken (as the â€œwakeupâ€ and
â€œwakeup_rtâ€ does).”…””}”(hj¶  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mahj²  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mahj<  ubh¸)”}”(hŒ"mmiotrace"”h]”hŒâ€œmmiotraceâ€”…””}”(hjÊ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mehj<  ubjT  )”}”(hŒ¬A special tracer that is used to trace binary module.
It will trace all the calls that a module makes to the
hardware. Everything it writes and reads from the I/O
as well.
”h]”h¸)”}”(hŒ«A special tracer that is used to trace binary module.
It will trace all the calls that a module makes to the
hardware. Everything it writes and reads from the I/O
as well.”h]”hŒ«A special tracer that is used to trace binary module.
It will trace all the calls that a module makes to the
hardware. Everything it writes and reads from the I/O
as well.”…””}”(hjÜ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MghjØ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mghj<  ubh¸)”}”(hŒ"branch"”h]”hŒâ€œbranchâ€”…””}”(hjð  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mlhj<  ubjT  )”}”(hŒÅThis tracer can be configured when tracing likely/unlikely
calls within the kernel. It will trace when a likely and
unlikely branch is hit and if it was correct in its prediction
of being correct.
”h]”h¸)”}”(hŒÄThis tracer can be configured when tracing likely/unlikely
calls within the kernel. It will trace when a likely and
unlikely branch is hit and if it was correct in its prediction
of being correct.”h]”hŒÄThis tracer can be configured when tracing likely/unlikely
calls within the kernel. It will trace when a likely and
unlikely branch is hit and if it was correct in its prediction
of being correct.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mnhjþ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mnhj<  ubh¸)”}”(hŒ"nop"”h]”hŒ	â€œnopâ€”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mshj<  ubjT  )”}”(hŒnThis is the "trace nothing" tracer. To remove all
tracers from tracing simply echo "nop" into
current_tracer.
”h]”h¸)”}”(hŒmThis is the "trace nothing" tracer. To remove all
tracers from tracing simply echo "nop" into
current_tracer.”h]”hŒuThis is the â€œtrace nothingâ€ tracer. To remove all
tracers from tracing simply echo â€œnopâ€ into
current_tracer.”…””}”(hj(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Muhj$  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Muhj<  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M hj  hžhubeh}”(h]”Œthe-tracers”ah ]”h"]”Œthe tracers”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Mubh¢)”}”(hhh]”(h§)”}”(hŒError conditions”h]”hŒError conditions”…””}”(hjM  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjJ  hžhhŸh¶h MzubjT  )”}”(hX  For most ftrace commands, failure modes are obvious and communicated
using standard return codes.

For other more involved commands, extended error information may be
available via the tracing/error_log file.  For the commands that
support it, reading the tracing/error_log file after an error will
display more detailed information about what went wrong, if
information is available.  The tracing/error_log file is a circular
error log displaying a small number (currently, 8) of ftrace errors
for the last (8) failed commands.

The extended error information and usage takes the form shown in
this example::

  # echo xxx > /sys/kernel/tracing/events/sched/sched_wakeup/trigger
  echo: write error: Invalid argument

  # cat /sys/kernel/tracing/error_log
  [ 5348.887237] location: error: Couldn't yyy: zzz
    Command: xxx
             ^
  [ 7517.023364] location: error: Bad rrr: sss
    Command: ppp qqq
                 ^

To clear the error log, echo the empty string into it::

  # echo > /sys/kernel/tracing/error_log
”h]”(h¸)”}”(hŒaFor most ftrace commands, failure modes are obvious and communicated
using standard return codes.”h]”hŒaFor most ftrace commands, failure modes are obvious and communicated
using standard return codes.”…””}”(hj_  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M|hj[  ubh¸)”}”(hX­  For other more involved commands, extended error information may be
available via the tracing/error_log file.  For the commands that
support it, reading the tracing/error_log file after an error will
display more detailed information about what went wrong, if
information is available.  The tracing/error_log file is a circular
error log displaying a small number (currently, 8) of ftrace errors
for the last (8) failed commands.”h]”hX­  For other more involved commands, extended error information may be
available via the tracing/error_log file.  For the commands that
support it, reading the tracing/error_log file after an error will
display more detailed information about what went wrong, if
information is available.  The tracing/error_log file is a circular
error log displaying a small number (currently, 8) of ftrace errors
for the last (8) failed commands.”…””}”(hjm  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj[  ubh¸)”}”(hŒOThe extended error information and usage takes the form shown in
this example::”h]”hŒNThe extended error information and usage takes the form shown in
this example:”…””}”(hj{  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M‡hj[  ubj   )”}”(hX*  # echo xxx > /sys/kernel/tracing/events/sched/sched_wakeup/trigger
echo: write error: Invalid argument

# cat /sys/kernel/tracing/error_log
[ 5348.887237] location: error: Couldn't yyy: zzz
  Command: xxx
           ^
[ 7517.023364] location: error: Bad rrr: sss
  Command: ppp qqq
               ^”h]”hX*  # echo xxx > /sys/kernel/tracing/events/sched/sched_wakeup/trigger
echo: write error: Invalid argument

# cat /sys/kernel/tracing/error_log
[ 5348.887237] location: error: Couldn't yyy: zzz
  Command: xxx
           ^
[ 7517.023364] location: error: Bad rrr: sss
  Command: ppp qqq
               ^”…””}”hj‰  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MŠhj[  ubh¸)”}”(hŒ7To clear the error log, echo the empty string into it::”h]”hŒ6To clear the error log, echo the empty string into it:”…””}”(hj—  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M•hj[  ubj   )”}”(hŒ&# echo > /sys/kernel/tracing/error_log”h]”hŒ&# echo > /sys/kernel/tracing/error_log”…””}”hj¥  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M—hj[  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M|hjJ  hžhubeh}”(h]”Œerror-conditions”ah ]”h"]”Œerror conditions”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Mzubh¢)”}”(hhh]”(h§)”}”(hŒExamples of using the tracer”h]”hŒExamples of using the tracer”…””}”(hjÄ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjÁ  hžhhŸh¶h Mšubh¸)”}”(hŒHere are typical examples of using the tracers when controlling
them only with the tracefs interface (without using any
user-land utilities).”h]”hŒHere are typical examples of using the tracers when controlling
them only with the tracefs interface (without using any
user-land utilities).”…””}”(hjÒ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MœhjÁ  hžhubeh}”(h]”Œexamples-of-using-the-tracer”ah ]”h"]”Œexamples of using the tracer”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Mšubh¢)”}”(hhh]”(h§)”}”(hŒOutput format:”h]”hŒOutput format:”…””}”(hjë  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjè  hžhhŸh¶h M¡ubh¸)”}”(hŒ=Here is an example of the output format of the file "trace"::”h]”hŒ@Here is an example of the output format of the file â€œtraceâ€:”…””}”(hjù  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M£hjè  hžhubj   )”}”(hXÜ  # tracer: function
#
# entries-in-buffer/entries-written: 140080/250280   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
            bash-1977  [000] .... 17284.993652: sys_close <-system_call_fastpath
            bash-1977  [000] .... 17284.993653: __close_fd <-sys_close
            bash-1977  [000] .... 17284.993653: _raw_spin_lock <-__close_fd
            sshd-1974  [003] .... 17284.993653: __srcu_read_unlock <-fsnotify
            bash-1977  [000] .... 17284.993654: add_preempt_count <-_raw_spin_lock
            bash-1977  [000] ...1 17284.993655: _raw_spin_unlock <-__close_fd
            bash-1977  [000] ...1 17284.993656: sub_preempt_count <-_raw_spin_unlock
            bash-1977  [000] .... 17284.993657: filp_close <-__close_fd
            bash-1977  [000] .... 17284.993657: dnotify_flush <-filp_close
            sshd-1974  [003] .... 17284.993658: sys_select <-system_call_fastpath
            ....”h]”hXÜ  # tracer: function
#
# entries-in-buffer/entries-written: 140080/250280   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
            bash-1977  [000] .... 17284.993652: sys_close <-system_call_fastpath
            bash-1977  [000] .... 17284.993653: __close_fd <-sys_close
            bash-1977  [000] .... 17284.993653: _raw_spin_lock <-__close_fd
            sshd-1974  [003] .... 17284.993653: __srcu_read_unlock <-fsnotify
            bash-1977  [000] .... 17284.993654: add_preempt_count <-_raw_spin_lock
            bash-1977  [000] ...1 17284.993655: _raw_spin_unlock <-__close_fd
            bash-1977  [000] ...1 17284.993656: sub_preempt_count <-_raw_spin_unlock
            bash-1977  [000] .... 17284.993657: filp_close <-__close_fd
            bash-1977  [000] .... 17284.993657: dnotify_flush <-filp_close
            sshd-1974  [003] .... 17284.993658: sys_select <-system_call_fastpath
            ....”…””}”hj  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M¥hjè  hžhubh¸)”}”(hXX  A header is printed with the tracer name that is represented by
the trace. In this case the tracer is "function". Then it shows the
number of events in the buffer as well as the total number of entries
that were written. The difference is the number of entries that were
lost due to the buffer filling up (250280 - 140080 = 110200 events
lost).”h]”hX\  A header is printed with the tracer name that is represented by
the trace. In this case the tracer is â€œfunctionâ€. Then it shows the
number of events in the buffer as well as the total number of entries
that were written. The difference is the number of entries that were
lost due to the buffer filling up (250280 - 140080 = 110200 events
lost).”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¼hjè  hžhubh¸)”}”(hX~  The header explains the content of the events. Task name "bash", the task
PID "1977", the CPU that it was running on "000", the latency format
(explained below), the timestamp in <secs>.<usecs> format, the
function name that was traced "sys_close" and the parent function that
called this function "system_call_fastpath". The timestamp is the time
at which the function was entered.”h]”hX’  The header explains the content of the events. Task name â€œbashâ€, the task
PID â€œ1977â€, the CPU that it was running on â€œ000â€, the latency format
(explained below), the timestamp in <secs>.<usecs> format, the
function name that was traced â€œsys_closeâ€ and the parent function that
called this function â€œsystem_call_fastpathâ€. The timestamp is the time
at which the function was entered.”…””}”(hj#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÃhjè  hžhubeh}”(h]”Œoutput-format”ah ]”h"]”Œoutput format:”ah$]”h&]”uh1h¡hh£hžhhŸh¶h M¡ubh¢)”}”(hhh]”(h§)”}”(hŒLatency trace format”h]”hŒLatency trace format”…””}”(hj<  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj9  hžhhŸh¶h MËubh¸)”}”(hŒ¼When the latency-format option is enabled or when one of the latency
tracers is set, the trace file gives somewhat more information to see
why a latency happened. Here is a typical trace::”h]”hŒ»When the latency-format option is enabled or when one of the latency
tracers is set, the trace file gives somewhat more information to see
why a latency happened. Here is a typical trace:”…””}”(hjJ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÍhj9  hžhubj   )”}”(hX¦  # tracer: irqsoff
#
# irqsoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 259 us, #4/4, CPU#2 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: ps-6143 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: __lock_task_sighand
#  => ended at:   _raw_spin_unlock_irqrestore
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
      ps-6143    2d...    0us!: trace_hardirqs_off <-__lock_task_sighand
      ps-6143    2d..1  259us+: trace_hardirqs_on <-_raw_spin_unlock_irqrestore
      ps-6143    2d..1  263us+: time_hardirqs_on <-_raw_spin_unlock_irqrestore
      ps-6143    2d..1  306us : <stack trace>
 => trace_hardirqs_on_caller
 => trace_hardirqs_on
 => _raw_spin_unlock_irqrestore
 => do_task_stat
 => proc_tgid_stat
 => proc_single_show
 => seq_read
 => vfs_read
 => sys_read
 => system_call_fastpath”h]”hX¦  # tracer: irqsoff
#
# irqsoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 259 us, #4/4, CPU#2 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: ps-6143 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: __lock_task_sighand
#  => ended at:   _raw_spin_unlock_irqrestore
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
      ps-6143    2d...    0us!: trace_hardirqs_off <-__lock_task_sighand
      ps-6143    2d..1  259us+: trace_hardirqs_on <-_raw_spin_unlock_irqrestore
      ps-6143    2d..1  263us+: time_hardirqs_on <-_raw_spin_unlock_irqrestore
      ps-6143    2d..1  306us : <stack trace>
 => trace_hardirqs_on_caller
 => trace_hardirqs_on
 => _raw_spin_unlock_irqrestore
 => do_task_stat
 => proc_tgid_stat
 => proc_single_show
 => seq_read
 => vfs_read
 => sys_read
 => system_call_fastpath”…””}”hjX  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MÑhj9  hžhubh¸)”}”(hXÒ  This shows that the current tracer is "irqsoff" tracing the time
for which interrupts were disabled. It gives the trace version (which
never changes) and the version of the kernel upon which this was executed on
(3.8). Then it displays the max latency in microseconds (259 us). The number
of trace entries displayed and the total number (both are four: #4/4).
VP, KP, SP, and HP are always zero and are reserved for later use.
#P is the number of online CPUs (#P:4).”h]”hXÖ  This shows that the current tracer is â€œirqsoffâ€ tracing the time
for which interrupts were disabled. It gives the trace version (which
never changes) and the version of the kernel upon which this was executed on
(3.8). Then it displays the max latency in microseconds (259 us). The number
of trace entries displayed and the total number (both are four: #4/4).
VP, KP, SP, and HP are always zero and are reserved for later use.
#P is the number of online CPUs (#P:4).”…””}”(hjf  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mõhj9  hžhubh¸)”}”(hŒSThe task is the process that was running when the latency
occurred. (ps pid: 6143).”h]”hŒSThe task is the process that was running when the latency
occurred. (ps pid: 6143).”…””}”(hjt  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mýhj9  hžhubh¸)”}”(hŒ|The start and stop (the functions in which the interrupts were
disabled and enabled respectively) that caused the latencies:”h]”hŒ|The start and stop (the functions in which the interrupts were
disabled and enabled respectively) that caused the latencies:”…””}”(hj‚  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M hj9  hžhubjT  )”}”(hŒ}- __lock_task_sighand is where the interrupts were disabled.
- _raw_spin_unlock_irqrestore is where they were enabled again.
”h]”j{  )”}”(hhh]”(j€  )”}”(hŒ:__lock_task_sighand is where the interrupts were disabled.”h]”h¸)”}”(hj™  h]”hŒ:__lock_task_sighand is where the interrupts were disabled.”…””}”(hj›  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj—  uba•     h}”(h]”h ]”h"]”h$]”h&]”uh1j  hj”  ubj€  )”}”(hŒ>_raw_spin_unlock_irqrestore is where they were enabled again.
”h]”h¸)”}”(hŒ=_raw_spin_unlock_irqrestore is where they were enabled again.”h]”hŒ=_raw_spin_unlock_irqrestore is where they were enabled again.”…””}”(hj²  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj®  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj”  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h Mhj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mhj9  hžhubh¸)”}”(hŒYThe next lines after the header are the trace itself. The header
explains which is which.”h]”hŒYThe next lines after the header are the trace itself. The header
explains which is which.”…””}”(hjÒ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj9  hžhubjT  )”}”(hX  cmd: The name of the process in the trace.

pid: The PID of that process.

CPU#: The CPU which the process was running on.

irqs-off: 'd' interrupts are disabled. '.' otherwise.

need-resched:
      - 'B' all, TIF_NEED_RESCHED, PREEMPT_NEED_RESCHED and TIF_RESCHED_LAZY is set,
      - 'N' both TIF_NEED_RESCHED and PREEMPT_NEED_RESCHED is set,
      - 'n' only TIF_NEED_RESCHED is set,
      - 'p' only PREEMPT_NEED_RESCHED is set,
      - 'L' both PREEMPT_NEED_RESCHED and TIF_RESCHED_LAZY is set,
      - 'b' both TIF_NEED_RESCHED and TIF_RESCHED_LAZY is set,
      - 'l' only TIF_RESCHED_LAZY is set
      - '.' otherwise.

hardirq/softirq:
      - 'Z' - NMI occurred inside a hardirq
      - 'z' - NMI is running
      - 'H' - hard irq occurred inside a softirq.
      - 'h' - hard irq is running
      - 's' - soft irq is running
      - '.' - normal context.

preempt-depth: The level of preempt_disabled
”h]”(h¸)”}”(hŒ*cmd: The name of the process in the trace.”h]”hŒ*cmd: The name of the process in the trace.”…””}”(hjä  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M	hjà  ubh¸)”}”(hŒpid: The PID of that process.”h]”hŒpid: The PID of that process.”…””}”(hjò  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjà  ubh¸)”}”(hŒ/CPU#: The CPU which the process was running on.”h]”hŒ/CPU#: The CPU which the process was running on.”…””}”(hj   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjà  ubh¸)”}”(hŒ5irqs-off: 'd' interrupts are disabled. '.' otherwise.”h]”hŒ=irqs-off: â€˜dâ€™ interrupts are disabled. â€˜.â€™ otherwise.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjà  ubjT  )”}”(hhh]”(jY  )”}”(hX  need-resched:
- 'B' all, TIF_NEED_RESCHED, PREEMPT_NEED_RESCHED and TIF_RESCHED_LAZY is set,
- 'N' both TIF_NEED_RESCHED and PREEMPT_NEED_RESCHED is set,
- 'n' only TIF_NEED_RESCHED is set,
- 'p' only PREEMPT_NEED_RESCHED is set,
- 'L' both PREEMPT_NEED_RESCHED and TIF_RESCHED_LAZY is set,
- 'b' both TIF_NEED_RESCHED and TIF_RESCHED_LAZY is set,
- 'l' only TIF_RESCHED_LAZY is set
- '.' otherwise.
”h]”(j_  )”}”(hŒneed-resched:”h]”hŒneed-resched:”…””}”(hj#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mhj  ubjo  )”}”(hhh]”j{  )”}”(hhh]”(j€  )”}”(hŒL'B' all, TIF_NEED_RESCHED, PREEMPT_NEED_RESCHED and TIF_RESCHED_LAZY is set,”h]”h¸)”}”(hj9  h]”hŒPâ€˜Bâ€™ all, TIF_NEED_RESCHED, PREEMPT_NEED_RESCHED and TIF_RESCHED_LAZY is set,”…””}”(hj;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj7  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj4  ubj€  )”}”(hŒ:'N' both TIF_NEED_RESCHED and PREEMPT_NEED_RESCHED is set,”h]”h¸)”}”(hjP  h]”hŒ>â€˜Nâ€™ both TIF_NEED_RESCHED and PREEMPT_NEED_RESCHED is set,”…””}”(hjR  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjN  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj4  ubj€  )”}”(hŒ!'n' only TIF_NEED_RESCHED is set,”h]”h¸)”}”(hjg  h]”hŒ%â€˜nâ€™ only TIF_NEED_RESCHED is set,”…””}”(hji  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhje  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj4  ubj€  )”}”(hŒ%'p' only PREEMPT_NEED_RESCHED is set,”h]”h¸)”}”(hj~  h]”hŒ)â€˜pâ€™ only PREEMPT_NEED_RESCHED is set,”…””}”(hj€  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj|  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj4  ubj€  )”}”(hŒ:'L' both PREEMPT_NEED_RESCHED and TIF_RESCHED_LAZY is set,”h]”h¸)”}”(hj•  h]”hŒ>â€˜Lâ€™ both PREEMPT_NEED_RESCHED and TIF_RESCHED_LAZY is set,”…””}”(hj—  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj“  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj4  ubj€  )”}”(hŒ6'b' both TIF_NEED_RESCHED and TIF_RESCHED_LAZY is set,”h]”h¸)”}”(hj¬  h]”hŒ:â€˜bâ€™ both TIF_NEED_RESCHED and TIF_RESCHED_LAZY is set,”…””}”(hj®  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjª  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj4  ubj€  )”}”(hŒ 'l' only TIF_RESCHED_LAZY is set”h]”h¸)”}”(hjÃ  h]”hŒ$â€˜lâ€™ only TIF_RESCHED_LAZY is set”…””}”(hjÅ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjÁ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj4  ubj€  )”}”(hŒ'.' otherwise.
”h]”h¸)”}”(hŒ'.' otherwise.”h]”hŒâ€˜.â€™ otherwise.”…””}”(hjÜ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjØ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj4  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h Mhj1  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj  ubjY  )”}”(hŒÊhardirq/softirq:
- 'Z' - NMI occurred inside a hardirq
- 'z' - NMI is running
- 'H' - hard irq occurred inside a softirq.
- 'h' - hard irq is running
- 's' - soft irq is running
- '.' - normal context.
”h]”(j_  )”}”(hŒhardirq/softirq:”h]”hŒhardirq/softirq:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M!hj  ubjo  )”}”(hhh]”j{  )”}”(hhh]”(j€  )”}”(hŒ#'Z' - NMI occurred inside a hardirq”h]”h¸)”}”(hj  h]”hŒ'â€˜Zâ€™ - NMI occurred inside a hardirq”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj  ubj€  )”}”(hŒ'z' - NMI is running”h]”h¸)”}”(hj3  h]”hŒâ€˜zâ€™ - NMI is running”…””}”(hj5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj1  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj  ubj€  )”}”(hŒ)'H' - hard irq occurred inside a softirq.”h]”h¸)”}”(hjJ  h]”hŒ-â€˜Hâ€™ - hard irq occurred inside a softirq.”…””}”(hjL  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjH  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj  ubj€  )”}”(hŒ'h' - hard irq is running”h]”h¸)”}”(hja  h]”hŒâ€˜hâ€™ - hard irq is running”…””}”(hjc  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj_  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj  ubj€  )”}”(hŒ's' - soft irq is running”h]”h¸)”}”(hjx  h]”hŒâ€˜sâ€™ - soft irq is running”…””}”(hjz  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M hjv  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj  ubj€  )”}”(hŒ'.' - normal context.
”h]”h¸)”}”(hŒ'.' - normal context.”h]”hŒâ€˜.â€™ - normal context.”…””}”(hj‘  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M!hj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h Mhj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M!hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hjà  ubh¸)”}”(hŒ,preempt-depth: The level of preempt_disabled”h]”hŒ,preempt-depth: The level of preempt_disabled”…””}”(hj½  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M#hjà  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M	hj9  hžhubh¸)”}”(hŒ5The above is mostly meaningful for kernel developers.”h]”hŒ5The above is mostly meaningful for kernel developers.”…””}”(hjÑ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M%hj9  hžhubjT  )”}”(hX´  time:
      When the latency-format option is enabled, the trace file
      output includes a timestamp relative to the start of the
      trace. This differs from the output when latency-format
      is disabled, which includes an absolute timestamp.

delay:
      This is just to help catch your eye a bit better. And
      needs to be fixed to be only relative to the same CPU.
      The marks are determined by the difference between this
      current trace and the next trace.

        - '$' - greater than 1 second
        - '@' - greater than 100 millisecond
        - '*' - greater than 10 millisecond
        - '#' - greater than 1000 microsecond
        - '!' - greater than 100 microsecond
        - '+' - greater than 10 microsecond
        - ' ' - less than or equal to 10 microsecond.

The rest is the same as the 'trace' file.

Note, the latency tracers will usually end with a back trace
to easily find where the latency occurred.
”h]”(jT  )”}”(hhh]”(jY  )”}”(hŒätime:
When the latency-format option is enabled, the trace file
output includes a timestamp relative to the start of the
trace. This differs from the output when latency-format
is disabled, which includes an absolute timestamp.
”h]”(j_  )”}”(hŒtime:”h]”hŒtime:”…””}”(hjê  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M+hjæ  ubjo  )”}”(hhh]”h¸)”}”(hŒÝWhen the latency-format option is enabled, the trace file
output includes a timestamp relative to the start of the
trace. This differs from the output when latency-format
is disabled, which includes an absolute timestamp.”h]”hŒÝWhen the latency-format option is enabled, the trace file
output includes a timestamp relative to the start of the
trace. This differs from the output when latency-format
is disabled, which includes an absolute timestamp.”…””}”(hjû  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M(hjø  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjæ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M+hjã  ubjY  )”}”(hXá  delay:
This is just to help catch your eye a bit better. And
needs to be fixed to be only relative to the same CPU.
The marks are determined by the difference between this
current trace and the next trace.

  - '$' - greater than 1 second
  - '@' - greater than 100 millisecond
  - '*' - greater than 10 millisecond
  - '#' - greater than 1000 microsecond
  - '!' - greater than 100 microsecond
  - '+' - greater than 10 microsecond
  - ' ' - less than or equal to 10 microsecond.
”h]”(j_  )”}”(hŒdelay:”h]”hŒdelay:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M9hj  ubjo  )”}”(hhh]”(h¸)”}”(hŒÆThis is just to help catch your eye a bit better. And
needs to be fixed to be only relative to the same CPU.
The marks are determined by the difference between this
current trace and the next trace.”h]”hŒÆThis is just to help catch your eye a bit better. And
needs to be fixed to be only relative to the same CPU.
The marks are determined by the difference between this
current trace and the next trace.”…””}”(hj*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M.hj'  ubjT  )”}”(hX  - '$' - greater than 1 second
- '@' - greater than 100 millisecond
- '*' - greater than 10 millisecond
- '#' - greater than 1000 microsecond
- '!' - greater than 100 microsecond
- '+' - greater than 10 microsecond
- ' ' - less than or equal to 10 microsecond.
”h]”j{  )”}”(hhh]”(j€  )”}”(hŒ'$' - greater than 1 second”h]”h¸)”}”(hjA  h]”hŒâ€˜$â€™ - greater than 1 second”…””}”(hjC  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M3hj?  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj<  ubj€  )”}”(hŒ"'@' - greater than 100 millisecond”h]”h¸)”}”(hjX  h]”hŒ&â€˜@â€™ - greater than 100 millisecond”…””}”(hjZ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M4hjV  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj<  ubj€  )”}”(hŒ!'*' - greater than 10 millisecond”h]”h¸)”}”(hjo  h]”hŒ%â€˜*â€™ - greater than 10 millisecond”…””}”(hjq  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M5hjm  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj<  ubj€  )”}”(hŒ#'#' - greater than 1000 microsecond”h]”h¸)”}”(hj†  h]”hŒ'â€˜#â€™ - greater than 1000 microsecond”…””}”(hjˆ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M6hj„  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj<  ubj€  )”}”(hŒ"'!' - greater than 100 microsecond”h]”h¸)”}”(hj  h]”hŒ&â€˜!â€™ - greater than 100 microsecond”…””}”(hjŸ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M7hj›  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj<  ubj€  )”}”(hŒ!'+' - greater than 10 microsecond”h]”h¸)”}”(hj´  h]”hŒ%â€˜+â€™ - greater than 10 microsecond”…””}”(hj¶  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M8hj²  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj<  ubj€  )”}”(hŒ,' ' - less than or equal to 10 microsecond.
”h]”h¸)”}”(hŒ+' ' - less than or equal to 10 microsecond.”h]”hŒ/â€˜ â€˜ - less than or equal to 10 microsecond.”…””}”(hjÍ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M9hjÉ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj<  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h M3hj8  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M3hj'  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M9hjã  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hjß  ubh¸)”}”(hŒ)The rest is the same as the 'trace' file.”h]”hŒ-The rest is the same as the â€˜traceâ€™ file.”…””}”(hjÿ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M;hjß  ubh¸)”}”(hŒgNote, the latency tracers will usually end with a back trace
to easily find where the latency occurred.”h]”hŒgNote, the latency tracers will usually end with a back trace
to easily find where the latency occurred.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M=hjß  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M'hj9  hžhubeh}”(h]”Œlatency-trace-format”ah ]”h"]”Œlatency trace format”ah$]”h&]”uh1h¡hh£hžhhŸh¶h MËubh¢)”}”(hhh]”(h§)”}”(hŒtrace_options”h]”hŒtrace_options”…””}”(hj,  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj)  hžhhŸh¶h MAubh¸)”}”(hŒ¶The trace_options file (or the options directory) is used to control
what gets printed in the trace output, or manipulate the tracers.
To see what is available, simply cat the file::”h]”hŒµThe trace_options file (or the options directory) is used to control
what gets printed in the trace output, or manipulate the tracers.
To see what is available, simply cat the file:”…””}”(hj:  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MChj)  hžhubj   )”}”(hX  cat trace_options
      print-parent
      nosym-offset
      nosym-addr
      noverbose
      noraw
      nohex
      nobin
      noblock
      nofields
      trace_printk
      annotate
      nouserstacktrace
      nosym-userobj
      noprintk-msg-only
      context-info
      nolatency-format
      record-cmd
      norecord-tgid
      overwrite
      nodisable_on_free
      irq-info
      markers
      noevent-fork
      function-trace
      nofunction-fork
      nodisplay-graph
      nostacktrace
      nobranch”h]”hX  cat trace_options
      print-parent
      nosym-offset
      nosym-addr
      noverbose
      noraw
      nohex
      nobin
      noblock
      nofields
      trace_printk
      annotate
      nouserstacktrace
      nosym-userobj
      noprintk-msg-only
      context-info
      nolatency-format
      record-cmd
      norecord-tgid
      overwrite
      nodisable_on_free
      irq-info
      markers
      noevent-fork
      function-trace
      nofunction-fork
      nodisplay-graph
      nostacktrace
      nobranch”…””}”hjH  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MGhj)  hžhubh¸)”}”(hŒGTo disable one of the options, echo in the option prepended with
"no"::”h]”hŒJTo disable one of the options, echo in the option prepended with
â€œnoâ€:”…””}”(hjV  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mehj)  hžhubj   )”}”(hŒ#echo noprint-parent > trace_options”h]”hŒ#echo noprint-parent > trace_options”…””}”hjd  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhhj)  hžhubh¸)”}”(hŒ)To enable an option, leave off the "no"::”h]”hŒ,To enable an option, leave off the â€œnoâ€:”…””}”(hjr  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mjhj)  hžhubj   )”}”(hŒecho sym-offset > trace_options”h]”hŒecho sym-offset > trace_options”…””}”hj€  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mlhj)  hžhubh¸)”}”(hŒHere are the available options:”h]”hŒHere are the available options:”…””}”(hjŽ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mnhj)  hžhubjT  )”}”(hX:   print-parent
      On function traces, display the calling (parent)
      function as well as the function being traced.
      ::

        print-parent:
         bash-4000  [01]  1477.606694: simple_strtoul <-kstrtoul

        noprint-parent:
         bash-4000  [01]  1477.606694: simple_strtoul


sym-offset
      Display not only the function name, but also the
      offset in the function. For example, instead of
      seeing just "ktime_get", you will see
      "ktime_get+0xb/0x20".
      ::

        sym-offset:
         bash-4000  [01]  1477.606694: simple_strtoul+0x6/0xa0

sym-addr
      This will also display the function address as well
      as the function name.
      ::

        sym-addr:
         bash-4000  [01]  1477.606694: simple_strtoul <c0339346>

verbose
      This deals with the trace file when the
      latency-format option is enabled.
      ::

          bash  4000 1 0 00000000 00010a95 [58127d26] 1720.415ms \
          (+0.000ms): simple_strtoul (kstrtoul)

raw
      This will display raw numbers. This option is best for
      use with user applications that can translate the raw
      numbers better than having it done in the kernel.

hex
      Similar to raw, but the numbers will be in a hexadecimal format.

bin
      This will print out the formats in raw binary.

block
      When set, reading trace_pipe will not block when polled.

fields
      Print the fields as described by their types. This is a better
      option than using hex, bin or raw, as it gives a better parsing
      of the content of the event.

trace_printk
      Can disable trace_printk() from writing into the buffer.

trace_printk_dest
      Set to have trace_printk() and similar internal tracing functions
      write into this instance. Note, only one trace instance can have
      this set. By setting this flag, it clears the trace_printk_dest flag
      of the instance that had it set previously. By default, the top
      level trace has this set, and will get it set again if another
      instance has it set then clears it.

      This flag cannot be cleared by the top level instance, as it is the
      default instance. The only way the top level instance has this flag
      cleared, is by it being set in another instance.

annotate
      It is sometimes confusing when the CPU buffers are full
      and one CPU buffer had a lot of events recently, thus
      a shorter time frame, were another CPU may have only had
      a few events, which lets it have older events. When
      the trace is reported, it shows the oldest events first,
      and it may look like only one CPU ran (the one with the
      oldest events). When the annotate option is set, it will
      display when a new CPU buffer started::

                        <idle>-0     [001] dNs4 21169.031481: wake_up_idle_cpu <-add_timer_on
                        <idle>-0     [001] dNs4 21169.031482: _raw_spin_unlock_irqrestore <-add_timer_on
                        <idle>-0     [001] .Ns4 21169.031484: sub_preempt_count <-_raw_spin_unlock_irqrestore
              ##### CPU 2 buffer started ####
                        <idle>-0     [002] .N.1 21169.031484: rcu_idle_exit <-cpu_idle
                        <idle>-0     [001] .Ns3 21169.031484: _raw_spin_unlock <-clocksource_watchdog
                        <idle>-0     [001] .Ns3 21169.031485: sub_preempt_count <-_raw_spin_unlock

userstacktrace
      This option changes the trace. It records a
      stacktrace of the current user space thread after
      each trace event.

sym-userobj
      when user stacktrace are enabled, look up which
      object the address belongs to, and print a
      relative address. This is especially useful when
      ASLR is on, otherwise you don't get a chance to
      resolve the address to object/file/line after
      the app is no longer running

      The lookup is performed when you read
      trace,trace_pipe. Example::

                a.out-1623  [000] 40874.465068: /root/a.out[+0x480] <-/root/a.out[+0
                x494] <- /root/a.out[+0x4a8] <- /lib/libc-2.7.so[+0x1e1a6]


printk-msg-only
      When set, trace_printk()s will only show the format
      and not their parameters (if trace_bprintk() or
      trace_bputs() was used to save the trace_printk()).

context-info
      Show only the event data. Hides the comm, PID,
      timestamp, CPU, and other useful data.

latency-format
      This option changes the trace output. When it is enabled,
      the trace displays additional information about the
      latency, as described in "Latency trace format".

pause-on-trace
      When set, opening the trace file for read, will pause
      writing to the ring buffer (as if tracing_on was set to zero).
      This simulates the original behavior of the trace file.
      When the file is closed, tracing will be enabled again.

hash-ptr
      When set, "%p" in the event printk format displays the
      hashed pointer value instead of real address.
      This will be useful if you want to find out which hashed
      value is corresponding to the real value in trace log.

record-cmd
      When any event or tracer is enabled, a hook is enabled
      in the sched_switch trace point to fill comm cache
      with mapped pids and comms. But this may cause some
      overhead, and if you only care about pids, and not the
      name of the task, disabling this option can lower the
      impact of tracing. See "saved_cmdlines".

record-tgid
      When any event or tracer is enabled, a hook is enabled
      in the sched_switch trace point to fill the cache of
      mapped Thread Group IDs (TGID) mapping to pids. See
      "saved_tgids".

overwrite
      This controls what happens when the trace buffer is
      full. If "1" (default), the oldest events are
      discarded and overwritten. If "0", then the newest
      events are discarded.
      (see per_cpu/cpu0/stats for overrun and dropped)

disable_on_free
      When the free_buffer is closed, tracing will
      stop (tracing_on set to 0).

irq-info
      Shows the interrupt, preempt count, need resched data.
      When disabled, the trace looks like::

              # tracer: function
              #
              # entries-in-buffer/entries-written: 144405/9452052   #P:4
              #
              #           TASK-PID   CPU#      TIMESTAMP  FUNCTION
              #              | |       |          |         |
                        <idle>-0     [002]  23636.756054: ttwu_do_activate.constprop.89 <-try_to_wake_up
                        <idle>-0     [002]  23636.756054: activate_task <-ttwu_do_activate.constprop.89
                        <idle>-0     [002]  23636.756055: enqueue_task <-activate_task


markers
      When set, the trace_marker is writable (only by root).
      When disabled, the trace_marker will error with EINVAL
      on write.

event-fork
      When set, tasks with PIDs listed in set_event_pid will have
      the PIDs of their children added to set_event_pid when those
      tasks fork. Also, when tasks with PIDs in set_event_pid exit,
      their PIDs will be removed from the file.

      This affects PIDs listed in set_event_notrace_pid as well.

function-trace
      The latency tracers will enable function tracing
      if this option is enabled (default it is). When
      it is disabled, the latency tracers do not trace
      functions. This keeps the overhead of the tracer down
      when performing latency tests.

function-fork
      When set, tasks with PIDs listed in set_ftrace_pid will
      have the PIDs of their children added to set_ftrace_pid
      when those tasks fork. Also, when tasks with PIDs in
      set_ftrace_pid exit, their PIDs will be removed from the
      file.

      This affects PIDs in set_ftrace_notrace_pid as well.

display-graph
      When set, the latency tracers (irqsoff, wakeup, etc) will
      use function graph tracing instead of function tracing.

stacktrace
      When set, a stack trace is recorded after any trace event
      is recorded.

branch
      Enable branch tracing with the tracer. This enables branch
      tracer along with the currently set tracer. Enabling this
      with the "nop" tracer is the same as just enabling the
      "branch" tracer.
”h]”jT  )”}”(hhh]”(jY  )”}”(hX   print-parent
On function traces, display the calling (parent)
function as well as the function being traced.
::

  print-parent:
   bash-4000  [01]  1477.606694: simple_strtoul <-kstrtoul

  noprint-parent:
   bash-4000  [01]  1477.606694: simple_strtoul

”h]”(j_  )”}”(hŒprint-parent”h]”hŒprint-parent”…””}”(hj§  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mzhj£  ubjo  )”}”(hhh]”(h¸)”}”(hŒbOn function traces, display the calling (parent)
function as well as the function being traced.
::”h]”hŒ_On function traces, display the calling (parent)
function as well as the function being traced.”…””}”(hj¸  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mqhjµ  ubj   )”}”(hŒ…print-parent:
 bash-4000  [01]  1477.606694: simple_strtoul <-kstrtoul

noprint-parent:
 bash-4000  [01]  1477.606694: simple_strtoul”h]”hŒ…print-parent:
 bash-4000  [01]  1477.606694: simple_strtoul <-kstrtoul

noprint-parent:
 bash-4000  [01]  1477.606694: simple_strtoul”…””}”hjÆ  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Muhjµ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj£  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mzhj   ubjY  )”}”(hŒósym-offset
Display not only the function name, but also the
offset in the function. For example, instead of
seeing just "ktime_get", you will see
"ktime_get+0xb/0x20".
::

  sym-offset:
   bash-4000  [01]  1477.606694: simple_strtoul+0x6/0xa0
”h]”(j_  )”}”(hŒ
sym-offset”h]”hŒ
sym-offset”…””}”(hjä  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M„hjà  ubjo  )”}”(hhh]”(h¸)”}”(hŒŸDisplay not only the function name, but also the
offset in the function. For example, instead of
seeing just "ktime_get", you will see
"ktime_get+0xb/0x20".
::”h]”hŒ¤Display not only the function name, but also the
offset in the function. For example, instead of
seeing just â€œktime_getâ€, you will see
â€œktime_get+0xb/0x20â€.”…””}”(hjõ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M}hjò  ubj   )”}”(hŒBsym-offset:
 bash-4000  [01]  1477.606694: simple_strtoul+0x6/0xa0”h]”hŒBsym-offset:
 bash-4000  [01]  1477.606694: simple_strtoul+0x6/0xa0”…””}”hj  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mƒhjò  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjà  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M„hj   ubjY  )”}”(hŒžsym-addr
This will also display the function address as well
as the function name.
::

  sym-addr:
   bash-4000  [01]  1477.606694: simple_strtoul <c0339346>
”h]”(j_  )”}”(hŒsym-addr”h]”hŒsym-addr”…””}”(hj!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MŒhj  ubjo  )”}”(hhh]”(h¸)”}”(hŒLThis will also display the function address as well
as the function name.
::”h]”hŒIThis will also display the function address as well
as the function name.”…””}”(hj2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M‡hj/  ubj   )”}”(hŒBsym-addr:
 bash-4000  [01]  1477.606694: simple_strtoul <c0339346>”h]”hŒBsym-addr:
 bash-4000  [01]  1477.606694: simple_strtoul <c0339346>”…””}”hj@  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M‹hj/  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MŒhj   ubjY  )”}”(hŒ½verbose
This deals with the trace file when the
latency-format option is enabled.
::

    bash  4000 1 0 00000000 00010a95 [58127d26] 1720.415ms \
    (+0.000ms): simple_strtoul (kstrtoul)
”h]”(j_  )”}”(hŒverbose”h]”hŒverbose”…””}”(hj^  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M”hjZ  ubjo  )”}”(hhh]”(h¸)”}”(hŒLThis deals with the trace file when the
latency-format option is enabled.
::”h]”hŒIThis deals with the trace file when the
latency-format option is enabled.”…””}”(hjo  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjl  ubj   )”}”(hŒ^bash  4000 1 0 00000000 00010a95 [58127d26] 1720.415ms \
(+0.000ms): simple_strtoul (kstrtoul)”h]”hŒ^bash  4000 1 0 00000000 00010a95 [58127d26] 1720.415ms \
(+0.000ms): simple_strtoul (kstrtoul)”…””}”hj}  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M“hjl  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjZ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M”hj   ubjY  )”}”(hŒ£raw
This will display raw numbers. This option is best for
use with user applications that can translate the raw
numbers better than having it done in the kernel.
”h]”(j_  )”}”(hŒraw”h]”hŒraw”…””}”(hj›  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M™hj—  ubjo  )”}”(hhh]”h¸)”}”(hŒžThis will display raw numbers. This option is best for
use with user applications that can translate the raw
numbers better than having it done in the kernel.”h]”hŒžThis will display raw numbers. This option is best for
use with user applications that can translate the raw
numbers better than having it done in the kernel.”…””}”(hj¬  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M—hj©  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj—  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M™hj   ubjY  )”}”(hŒEhex
Similar to raw, but the numbers will be in a hexadecimal format.
”h]”(j_  )”}”(hŒhex”h]”hŒhex”…””}”(hjÊ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MœhjÆ  ubjo  )”}”(hhh]”h¸)”}”(hŒ@Similar to raw, but the numbers will be in a hexadecimal format.”h]”hŒ@Similar to raw, but the numbers will be in a hexadecimal format.”…””}”(hjÛ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MœhjØ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjÆ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mœhj   ubjY  )”}”(hŒ3bin
This will print out the formats in raw binary.
”h]”(j_  )”}”(hŒbin”h]”hŒbin”…””}”(hjù  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MŸhjõ  ubjo  )”}”(hhh]”h¸)”}”(hŒ.This will print out the formats in raw binary.”h]”hŒ.This will print out the formats in raw binary.”…””}”(hj
  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MŸhj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjõ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MŸhj   ubjY  )”}”(hŒ?block
When set, reading trace_pipe will not block when polled.
”h]”(j_  )”}”(hŒblock”h]”hŒblock”…””}”(hj(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M¢hj$  ubjo  )”}”(hhh]”h¸)”}”(hŒ8When set, reading trace_pipe will not block when polled.”h]”hŒ8When set, reading trace_pipe will not block when polled.”…””}”(hj9  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¢hj6  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj$  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M¢hj   ubjY  )”}”(hŒ£fields
Print the fields as described by their types. This is a better
option than using hex, bin or raw, as it gives a better parsing
of the content of the event.
”h]”(j_  )”}”(hŒfields”h]”hŒfields”…””}”(hjW  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M§hjS  ubjo  )”}”(hhh]”h¸)”}”(hŒ›Print the fields as described by their types. This is a better
option than using hex, bin or raw, as it gives a better parsing
of the content of the event.”h]”hŒ›Print the fields as described by their types. This is a better
option than using hex, bin or raw, as it gives a better parsing
of the content of the event.”…””}”(hjh  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¥hje  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjS  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M§hj   ubjY  )”}”(hŒFtrace_printk
Can disable trace_printk() from writing into the buffer.
”h]”(j_  )”}”(hŒtrace_printk”h]”hŒtrace_printk”…””}”(hj†  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mªhj‚  ubjo  )”}”(hhh]”h¸)”}”(hŒ8Can disable trace_printk() from writing into the buffer.”h]”hŒ8Can disable trace_printk() from writing into the buffer.”…””}”(hj—  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mªhj”  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj‚  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mªhj   ubjY  )”}”(hX7  trace_printk_dest
Set to have trace_printk() and similar internal tracing functions
write into this instance. Note, only one trace instance can have
this set. By setting this flag, it clears the trace_printk_dest flag
of the instance that had it set previously. By default, the top
level trace has this set, and will get it set again if another
instance has it set then clears it.

This flag cannot be cleared by the top level instance, as it is the
default instance. The only way the top level instance has this flag
cleared, is by it being set in another instance.
”h]”(j_  )”}”(hŒtrace_printk_dest”h]”hŒtrace_printk_dest”…””}”(hjµ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M¶hj±  ubjo  )”}”(hhh]”(h¸)”}”(hXj  Set to have trace_printk() and similar internal tracing functions
write into this instance. Note, only one trace instance can have
this set. By setting this flag, it clears the trace_printk_dest flag
of the instance that had it set previously. By default, the top
level trace has this set, and will get it set again if another
instance has it set then clears it.”h]”hXj  Set to have trace_printk() and similar internal tracing functions
write into this instance. Note, only one trace instance can have
this set. By setting this flag, it clears the trace_printk_dest flag
of the instance that had it set previously. By default, the top
level trace has this set, and will get it set again if another
instance has it set then clears it.”…””}”(hjÆ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M­hjÃ  ubh¸)”}”(hŒ¸This flag cannot be cleared by the top level instance, as it is the
default instance. The only way the top level instance has this flag
cleared, is by it being set in another instance.”h]”hŒ¸This flag cannot be cleared by the top level instance, as it is the
default instance. The only way the top level instance has this flag
cleared, is by it being set in another instance.”…””}”(hjÔ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M´hjÃ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj±  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M¶hj   ubjY  )”}”(hX  annotate
It is sometimes confusing when the CPU buffers are full
and one CPU buffer had a lot of events recently, thus
a shorter time frame, were another CPU may have only had
a few events, which lets it have older events. When
the trace is reported, it shows the oldest events first,
and it may look like only one CPU ran (the one with the
oldest events). When the annotate option is set, it will
display when a new CPU buffer started::

                  <idle>-0     [001] dNs4 21169.031481: wake_up_idle_cpu <-add_timer_on
                  <idle>-0     [001] dNs4 21169.031482: _raw_spin_unlock_irqrestore <-add_timer_on
                  <idle>-0     [001] .Ns4 21169.031484: sub_preempt_count <-_raw_spin_unlock_irqrestore
        ##### CPU 2 buffer started ####
                  <idle>-0     [002] .N.1 21169.031484: rcu_idle_exit <-cpu_idle
                  <idle>-0     [001] .Ns3 21169.031484: _raw_spin_unlock <-clocksource_watchdog
                  <idle>-0     [001] .Ns3 21169.031485: sub_preempt_count <-_raw_spin_unlock
”h]”(j_  )”}”(hŒannotate”h]”hŒannotate”…””}”(hjò  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MÈhjî  ubjo  )”}”(hhh]”(h¸)”}”(hX¬  It is sometimes confusing when the CPU buffers are full
and one CPU buffer had a lot of events recently, thus
a shorter time frame, were another CPU may have only had
a few events, which lets it have older events. When
the trace is reported, it shows the oldest events first,
and it may look like only one CPU ran (the one with the
oldest events). When the annotate option is set, it will
display when a new CPU buffer started::”h]”hX«  It is sometimes confusing when the CPU buffers are full
and one CPU buffer had a lot of events recently, thus
a shorter time frame, were another CPU may have only had
a few events, which lets it have older events. When
the trace is reported, it shows the oldest events first,
and it may look like only one CPU ran (the one with the
oldest events). When the annotate option is set, it will
display when a new CPU buffer started:”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¹hj   ubj   )”}”(hX             <idle>-0     [001] dNs4 21169.031481: wake_up_idle_cpu <-add_timer_on
          <idle>-0     [001] dNs4 21169.031482: _raw_spin_unlock_irqrestore <-add_timer_on
          <idle>-0     [001] .Ns4 21169.031484: sub_preempt_count <-_raw_spin_unlock_irqrestore
##### CPU 2 buffer started ####
          <idle>-0     [002] .N.1 21169.031484: rcu_idle_exit <-cpu_idle
          <idle>-0     [001] .Ns3 21169.031484: _raw_spin_unlock <-clocksource_watchdog
          <idle>-0     [001] .Ns3 21169.031485: sub_preempt_count <-_raw_spin_unlock”h]”hX             <idle>-0     [001] dNs4 21169.031481: wake_up_idle_cpu <-add_timer_on
          <idle>-0     [001] dNs4 21169.031482: _raw_spin_unlock_irqrestore <-add_timer_on
          <idle>-0     [001] .Ns4 21169.031484: sub_preempt_count <-_raw_spin_unlock_irqrestore
##### CPU 2 buffer started ####
          <idle>-0     [002] .N.1 21169.031484: rcu_idle_exit <-cpu_idle
          <idle>-0     [001] .Ns3 21169.031484: _raw_spin_unlock <-clocksource_watchdog
          <idle>-0     [001] .Ns3 21169.031485: sub_preempt_count <-_raw_spin_unlock”…””}”hj  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MÂhj   ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjî  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MÈhj   ubjY  )”}”(hŒuserstacktrace
This option changes the trace. It records a
stacktrace of the current user space thread after
each trace event.
”h]”(j_  )”}”(hŒuserstacktrace”h]”hŒuserstacktrace”…””}”(hj/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MÍhj+  ubjo  )”}”(hhh]”h¸)”}”(hŒoThis option changes the trace. It records a
stacktrace of the current user space thread after
each trace event.”h]”hŒoThis option changes the trace. It records a
stacktrace of the current user space thread after
each trace event.”…””}”(hj@  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MËhj=  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj+  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MÍhj   ubjY  )”}”(hXì  sym-userobj
when user stacktrace are enabled, look up which
object the address belongs to, and print a
relative address. This is especially useful when
ASLR is on, otherwise you don't get a chance to
resolve the address to object/file/line after
the app is no longer running

The lookup is performed when you read
trace,trace_pipe. Example::

          a.out-1623  [000] 40874.465068: /root/a.out[+0x480] <-/root/a.out[+0
          x494] <- /root/a.out[+0x4a8] <- /lib/libc-2.7.so[+0x1e1a6]

”h]”(j_  )”}”(hŒsym-userobj”h]”hŒsym-userobj”…””}”(hj^  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MÜhjZ  ubjo  )”}”(hhh]”(h¸)”}”(hX  when user stacktrace are enabled, look up which
object the address belongs to, and print a
relative address. This is especially useful when
ASLR is on, otherwise you don't get a chance to
resolve the address to object/file/line after
the app is no longer running”h]”hX  when user stacktrace are enabled, look up which
object the address belongs to, and print a
relative address. This is especially useful when
ASLR is on, otherwise you donâ€™t get a chance to
resolve the address to object/file/line after
the app is no longer running”…””}”(hjo  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÐhjl  ubh¸)”}”(hŒAThe lookup is performed when you read
trace,trace_pipe. Example::”h]”hŒ@The lookup is performed when you read
trace,trace_pipe. Example:”…””}”(hj}  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M×hjl  ubj   )”}”(hŒa.out-1623  [000] 40874.465068: /root/a.out[+0x480] <-/root/a.out[+0
x494] <- /root/a.out[+0x4a8] <- /lib/libc-2.7.so[+0x1e1a6]”h]”hŒa.out-1623  [000] 40874.465068: /root/a.out[+0x480] <-/root/a.out[+0
x494] <- /root/a.out[+0x4a8] <- /lib/libc-2.7.so[+0x1e1a6]”…””}”hj‹  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MÚhjl  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjZ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MÜhj   ubjY  )”}”(hŒ¨printk-msg-only
When set, trace_printk()s will only show the format
and not their parameters (if trace_bprintk() or
trace_bputs() was used to save the trace_printk()).
”h]”(j_  )”}”(hŒprintk-msg-only”h]”hŒprintk-msg-only”…””}”(hj©  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Máhj¥  ubjo  )”}”(hhh]”h¸)”}”(hŒ—When set, trace_printk()s will only show the format
and not their parameters (if trace_bprintk() or
trace_bputs() was used to save the trace_printk()).”h]”hŒ—When set, trace_printk()s will only show the format
and not their parameters (if trace_bprintk() or
trace_bputs() was used to save the trace_printk()).”…””}”(hjº  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mßhj·  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj¥  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Máhj   ubjY  )”}”(hŒccontext-info
Show only the event data. Hides the comm, PID,
timestamp, CPU, and other useful data.
”h]”(j_  )”}”(hŒcontext-info”h]”hŒcontext-info”…””}”(hjØ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MåhjÔ  ubjo  )”}”(hhh]”h¸)”}”(hŒUShow only the event data. Hides the comm, PID,
timestamp, CPU, and other useful data.”h]”hŒUShow only the event data. Hides the comm, PID,
timestamp, CPU, and other useful data.”…””}”(hjé  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mähjæ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjÔ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Måhj   ubjY  )”}”(hŒ®latency-format
This option changes the trace output. When it is enabled,
the trace displays additional information about the
latency, as described in "Latency trace format".
”h]”(j_  )”}”(hŒlatency-format”h]”hŒlatency-format”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mêhj  ubjo  )”}”(hhh]”h¸)”}”(hŒžThis option changes the trace output. When it is enabled,
the trace displays additional information about the
latency, as described in "Latency trace format".”h]”hŒ¢This option changes the trace output. When it is enabled,
the trace displays additional information about the
latency, as described in â€œLatency trace formatâ€.”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mèhj  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mêhj   ubjY  )”}”(hŒôpause-on-trace
When set, opening the trace file for read, will pause
writing to the ring buffer (as if tracing_on was set to zero).
This simulates the original behavior of the trace file.
When the file is closed, tracing will be enabled again.
”h]”(j_  )”}”(hŒpause-on-trace”h]”hŒpause-on-trace”…””}”(hj6  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mðhj2  ubjo  )”}”(hhh]”h¸)”}”(hŒäWhen set, opening the trace file for read, will pause
writing to the ring buffer (as if tracing_on was set to zero).
This simulates the original behavior of the trace file.
When the file is closed, tracing will be enabled again.”h]”hŒäWhen set, opening the trace file for read, will pause
writing to the ring buffer (as if tracing_on was set to zero).
This simulates the original behavior of the trace file.
When the file is closed, tracing will be enabled again.”…””}”(hjG  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MíhjD  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj2  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mðhj   ubjY  )”}”(hŒÞhash-ptr
When set, "%p" in the event printk format displays the
hashed pointer value instead of real address.
This will be useful if you want to find out which hashed
value is corresponding to the real value in trace log.
”h]”(j_  )”}”(hŒhash-ptr”h]”hŒhash-ptr”…””}”(hje  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Möhja  ubjo  )”}”(hhh]”h¸)”}”(hŒÔWhen set, "%p" in the event printk format displays the
hashed pointer value instead of real address.
This will be useful if you want to find out which hashed
value is corresponding to the real value in trace log.”h]”hŒØWhen set, â€œ%pâ€ in the event printk format displays the
hashed pointer value instead of real address.
This will be useful if you want to find out which hashed
value is corresponding to the real value in trace log.”…””}”(hjv  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Móhjs  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hja  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Möhj   ubjY  )”}”(hX?  record-cmd
When any event or tracer is enabled, a hook is enabled
in the sched_switch trace point to fill comm cache
with mapped pids and comms. But this may cause some
overhead, and if you only care about pids, and not the
name of the task, disabling this option can lower the
impact of tracing. See "saved_cmdlines".
”h]”(j_  )”}”(hŒ
record-cmd”h]”hŒ
record-cmd”…””}”(hj”  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mþhj  ubjo  )”}”(hhh]”h¸)”}”(hX3  When any event or tracer is enabled, a hook is enabled
in the sched_switch trace point to fill comm cache
with mapped pids and comms. But this may cause some
overhead, and if you only care about pids, and not the
name of the task, disabling this option can lower the
impact of tracing. See "saved_cmdlines".”h]”hX7  When any event or tracer is enabled, a hook is enabled
in the sched_switch trace point to fill comm cache
with mapped pids and comms. But this may cause some
overhead, and if you only care about pids, and not the
name of the task, disabling this option can lower the
impact of tracing. See â€œsaved_cmdlinesâ€.”…””}”(hj¥  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mùhj¢  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mþhj   ubjY  )”}”(hŒ»record-tgid
When any event or tracer is enabled, a hook is enabled
in the sched_switch trace point to fill the cache of
mapped Thread Group IDs (TGID) mapping to pids. See
"saved_tgids".
”h]”(j_  )”}”(hŒrecord-tgid”h]”hŒrecord-tgid”…””}”(hjÃ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mhj¿  ubjo  )”}”(hhh]”h¸)”}”(hŒ®When any event or tracer is enabled, a hook is enabled
in the sched_switch trace point to fill the cache of
mapped Thread Group IDs (TGID) mapping to pids. See
"saved_tgids".”h]”hŒ²When any event or tracer is enabled, a hook is enabled
in the sched_switch trace point to fill the cache of
mapped Thread Group IDs (TGID) mapping to pids. See
â€œsaved_tgidsâ€.”…””}”(hjÔ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjÑ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj¿  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj   ubjY  )”}”(hŒæoverwrite
This controls what happens when the trace buffer is
full. If "1" (default), the oldest events are
discarded and overwritten. If "0", then the newest
events are discarded.
(see per_cpu/cpu0/stats for overrun and dropped)
”h]”(j_  )”}”(hŒ	overwrite”h]”hŒ	overwrite”…””}”(hjò  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mhjî  ubjo  )”}”(hhh]”h¸)”}”(hŒÛThis controls what happens when the trace buffer is
full. If "1" (default), the oldest events are
discarded and overwritten. If "0", then the newest
events are discarded.
(see per_cpu/cpu0/stats for overrun and dropped)”h]”hŒãThis controls what happens when the trace buffer is
full. If â€œ1â€ (default), the oldest events are
discarded and overwritten. If â€œ0â€, then the newest
events are discarded.
(see per_cpu/cpu0/stats for overrun and dropped)”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj   ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjî  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj   ubjY  )”}”(hŒYdisable_on_free
When the free_buffer is closed, tracing will
stop (tracing_on set to 0).
”h]”(j_  )”}”(hŒdisable_on_free”h]”hŒdisable_on_free”…””}”(hj!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mhj  ubjo  )”}”(hhh]”h¸)”}”(hŒHWhen the free_buffer is closed, tracing will
stop (tracing_on set to 0).”h]”hŒHWhen the free_buffer is closed, tracing will
stop (tracing_on set to 0).”…””}”(hj2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj/  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj   ubjY  )”}”(hXe  irq-info
Shows the interrupt, preempt count, need resched data.
When disabled, the trace looks like::

        # tracer: function
        #
        # entries-in-buffer/entries-written: 144405/9452052   #P:4
        #
        #           TASK-PID   CPU#      TIMESTAMP  FUNCTION
        #              | |       |          |         |
                  <idle>-0     [002]  23636.756054: ttwu_do_activate.constprop.89 <-try_to_wake_up
                  <idle>-0     [002]  23636.756054: activate_task <-ttwu_do_activate.constprop.89
                  <idle>-0     [002]  23636.756055: enqueue_task <-activate_task

”h]”(j_  )”}”(hŒirq-info”h]”hŒirq-info”…””}”(hjP  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MhjL  ubjo  )”}”(hhh]”(h¸)”}”(hŒ\Shows the interrupt, preempt count, need resched data.
When disabled, the trace looks like::”h]”hŒ[Shows the interrupt, preempt count, need resched data.
When disabled, the trace looks like:”…””}”(hja  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj^  ubj   )”}”(hX´  # tracer: function
#
# entries-in-buffer/entries-written: 144405/9452052   #P:4
#
#           TASK-PID   CPU#      TIMESTAMP  FUNCTION
#              | |       |          |         |
          <idle>-0     [002]  23636.756054: ttwu_do_activate.constprop.89 <-try_to_wake_up
          <idle>-0     [002]  23636.756054: activate_task <-ttwu_do_activate.constprop.89
          <idle>-0     [002]  23636.756055: enqueue_task <-activate_task”h]”hX´  # tracer: function
#
# entries-in-buffer/entries-written: 144405/9452052   #P:4
#
#           TASK-PID   CPU#      TIMESTAMP  FUNCTION
#              | |       |          |         |
          <idle>-0     [002]  23636.756054: ttwu_do_activate.constprop.89 <-try_to_wake_up
          <idle>-0     [002]  23636.756054: activate_task <-ttwu_do_activate.constprop.89
          <idle>-0     [002]  23636.756055: enqueue_task <-activate_task”…””}”hjo  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhj^  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjL  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj   ubjY  )”}”(hŒ€markers
When set, the trace_marker is writable (only by root).
When disabled, the trace_marker will error with EINVAL
on write.
”h]”(j_  )”}”(hŒmarkers”h]”hŒmarkers”…””}”(hj  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M#hj‰  ubjo  )”}”(hhh]”h¸)”}”(hŒwWhen set, the trace_marker is writable (only by root).
When disabled, the trace_marker will error with EINVAL
on write.”h]”hŒwWhen set, the trace_marker is writable (only by root).
When disabled, the trace_marker will error with EINVAL
on write.”…””}”(hjž  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M!hj›  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj‰  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M#hj   ubjY  )”}”(hX(  event-fork
When set, tasks with PIDs listed in set_event_pid will have
the PIDs of their children added to set_event_pid when those
tasks fork. Also, when tasks with PIDs in set_event_pid exit,
their PIDs will be removed from the file.

This affects PIDs listed in set_event_notrace_pid as well.
”h]”(j_  )”}”(hŒ
event-fork”h]”hŒ
event-fork”…””}”(hj¼  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M+hj¸  ubjo  )”}”(hhh]”(h¸)”}”(hŒàWhen set, tasks with PIDs listed in set_event_pid will have
the PIDs of their children added to set_event_pid when those
tasks fork. Also, when tasks with PIDs in set_event_pid exit,
their PIDs will be removed from the file.”h]”hŒàWhen set, tasks with PIDs listed in set_event_pid will have
the PIDs of their children added to set_event_pid when those
tasks fork. Also, when tasks with PIDs in set_event_pid exit,
their PIDs will be removed from the file.”…””}”(hjÍ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M&hjÊ  ubh¸)”}”(hŒ:This affects PIDs listed in set_event_notrace_pid as well.”h]”hŒ:This affects PIDs listed in set_event_notrace_pid as well.”…””}”(hjÛ  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M+hjÊ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj¸  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M+hj   ubjY  )”}”(hŒöfunction-trace
The latency tracers will enable function tracing
if this option is enabled (default it is). When
it is disabled, the latency tracers do not trace
functions. This keeps the overhead of the tracer down
when performing latency tests.
”h]”(j_  )”}”(hŒfunction-trace”h]”hŒfunction-trace”…””}”(hjù  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M2hjõ  ubjo  )”}”(hhh]”h¸)”}”(hŒæThe latency tracers will enable function tracing
if this option is enabled (default it is). When
it is disabled, the latency tracers do not trace
functions. This keeps the overhead of the tracer down
when performing latency tests.”h]”hŒæThe latency tracers will enable function tracing
if this option is enabled (default it is). When
it is disabled, the latency tracers do not trace
functions. This keeps the overhead of the tracer down
when performing latency tests.”…””}”(hj
   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M.hj   ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjõ  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M2hj   ubjY  )”}”(hX(  function-fork
When set, tasks with PIDs listed in set_ftrace_pid will
have the PIDs of their children added to set_ftrace_pid
when those tasks fork. Also, when tasks with PIDs in
set_ftrace_pid exit, their PIDs will be removed from the
file.

This affects PIDs in set_ftrace_notrace_pid as well.
”h]”(j_  )”}”(hŒfunction-fork”h]”hŒfunction-fork”…””}”(hj(   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M;hj$   ubjo  )”}”(hhh]”(h¸)”}”(hŒãWhen set, tasks with PIDs listed in set_ftrace_pid will
have the PIDs of their children added to set_ftrace_pid
when those tasks fork. Also, when tasks with PIDs in
set_ftrace_pid exit, their PIDs will be removed from the
file.”h]”hŒãWhen set, tasks with PIDs listed in set_ftrace_pid will
have the PIDs of their children added to set_ftrace_pid
when those tasks fork. Also, when tasks with PIDs in
set_ftrace_pid exit, their PIDs will be removed from the
file.”…””}”(hj9   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M5hj6   ubh¸)”}”(hŒ4This affects PIDs in set_ftrace_notrace_pid as well.”h]”hŒ4This affects PIDs in set_ftrace_notrace_pid as well.”…””}”(hjG   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M;hj6   ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj$   ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M;hj   ubjY  )”}”(hŒ€display-graph
When set, the latency tracers (irqsoff, wakeup, etc) will
use function graph tracing instead of function tracing.
”h]”(j_  )”}”(hŒdisplay-graph”h]”hŒdisplay-graph”…””}”(hje   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M?hja   ubjo  )”}”(hhh]”h¸)”}”(hŒqWhen set, the latency tracers (irqsoff, wakeup, etc) will
use function graph tracing instead of function tracing.”h]”hŒqWhen set, the latency tracers (irqsoff, wakeup, etc) will
use function graph tracing instead of function tracing.”…””}”(hjv   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M>hjs   ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hja   ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M?hj   ubjY  )”}”(hŒRstacktrace
When set, a stack trace is recorded after any trace event
is recorded.
”h]”(j_  )”}”(hŒ
stacktrace”h]”hŒ
stacktrace”…””}”(hj”   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MChj   ubjo  )”}”(hhh]”h¸)”}”(hŒFWhen set, a stack trace is recorded after any trace event
is recorded.”h]”hŒFWhen set, a stack trace is recorded after any trace event
is recorded.”…””}”(hj¥   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MBhj¢   ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj   ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MChj   ubjY  )”}”(hŒÄbranch
Enable branch tracing with the tracer. This enables branch
tracer along with the currently set tracer. Enabling this
with the "nop" tracer is the same as just enabling the
"branch" tracer.
”h]”(j_  )”}”(hŒbranch”h]”hŒbranch”…””}”(hjÃ   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MIhj¿   ubjo  )”}”(hhh]”h¸)”}”(hŒ¼Enable branch tracing with the tracer. This enables branch
tracer along with the currently set tracer. Enabling this
with the "nop" tracer is the same as just enabling the
"branch" tracer.”h]”hŒÄEnable branch tracing with the tracer. This enables branch
tracer along with the currently set tracer. Enabling this
with the â€œnopâ€ tracer is the same as just enabling the
â€œbranchâ€ tracer.”…””}”(hjÔ   hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MFhjÑ   ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj¿   ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MIhj   ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hjœ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mphj)  hžhubhŒtip”“”)”}”(hŒŠSome tracers have their own options. They only appear in this
file when the tracer is active. They always appear in the
options directory.”h]”h¸)”}”(hŒŠSome tracers have their own options. They only appear in this
file when the tracer is active. They always appear in the
options directory.”h]”hŒŠSome tracers have their own options. They only appear in this
file when the tracer is active. They always appear in the
options directory.”…””}”(hj !  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MKhjü   ubah}”(h]”h ]”h"]”h$]”h&]”uh1jú   hj)  hžhhŸh¶h Nubh¸)”}”(hŒ Here are the per tracer options:”h]”hŒ Here are the per tracer options:”…””}”(hj!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MPhj)  hžhubh¸)”}”(hŒOptions for function tracer:”h]”hŒOptions for function tracer:”…””}”(hj"!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MRhj)  hžhubjT  )”}”(hX^  func_stack_trace
      When set, a stack trace is recorded after every
      function that is recorded. NOTE! Limit the functions
      that are recorded before enabling this, with
      "set_ftrace_filter" otherwise the system performance
      will be critically degraded. Remember to disable
      this option before clearing the function filter.
”h]”jT  )”}”(hhh]”jY  )”}”(hX:  func_stack_trace
When set, a stack trace is recorded after every
function that is recorded. NOTE! Limit the functions
that are recorded before enabling this, with
"set_ftrace_filter" otherwise the system performance
will be critically degraded. Remember to disable
this option before clearing the function filter.
”h]”(j_  )”}”(hŒfunc_stack_trace”h]”hŒfunc_stack_trace”…””}”(hj;!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MZhj7!  ubjo  )”}”(hhh]”h¸)”}”(hX(  When set, a stack trace is recorded after every
function that is recorded. NOTE! Limit the functions
that are recorded before enabling this, with
"set_ftrace_filter" otherwise the system performance
will be critically degraded. Remember to disable
this option before clearing the function filter.”h]”hX,  When set, a stack trace is recorded after every
function that is recorded. NOTE! Limit the functions
that are recorded before enabling this, with
â€œset_ftrace_filterâ€ otherwise the system performance
will be critically degraded. Remember to disable
this option before clearing the function filter.”…””}”(hjL!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MUhjI!  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj7!  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MZhj4!  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hj0!  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MThj)  hžhubh¸)”}”(hŒ"Options for function_graph tracer:”h]”hŒ"Options for function_graph tracer:”…””}”(hjr!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M\hj)  hžhubjT  )”}”(hXâ
  Since the function_graph tracer has a slightly different output
it has its own options to control what is displayed.

 funcgraph-overrun
       When set, the "overrun" of the graph stack is
       displayed after each function traced. The
       overrun, is when the stack depth of the calls
       is greater than what is reserved for each task.
       Each task has a fixed array of functions to
       trace in the call graph. If the depth of the
       calls exceeds that, the function is not traced.
       The overrun is the number of functions missed
       due to exceeding this array.

 funcgraph-cpu
       When set, the CPU number of the CPU where the trace
       occurred is displayed.

 funcgraph-overhead
       When set, if the function takes longer than
       A certain amount, then a delay marker is
       displayed. See "delay" above, under the
       header description.

 funcgraph-proc
       Unlike other tracers, the process' command line
       is not displayed by default, but instead only
       when a task is traced in and out during a context
       switch. Enabling this options has the command
       of each process displayed at every line.

 funcgraph-duration
       At the end of each function (the return)
       the duration of the amount of time in the
       function is displayed in microseconds.

 funcgraph-abstime
       When set, the timestamp is displayed at each line.

 funcgraph-irqs
       When disabled, functions that happen inside an
       interrupt will not be traced.

 funcgraph-tail
       When set, the return event will include the function
       that it represents. By default this is off, and
       only a closing curly bracket "}" is displayed for
       the return of a function.

 funcgraph-retval
       When set, the return value of each traced function
       will be printed after an equal sign "=". By default
       this is off.

 funcgraph-retval-hex
       When set, the return value will always be printed
       in hexadecimal format. If the option is not set and
       the return value is an error code, it will be printed
       in signed decimal format; otherwise it will also be
       printed in hexadecimal format. By default, this option
       is off.

 sleep-time
       When running function graph tracer, to include
       the time a task schedules out in its function.
       When enabled, it will account time the task has been
       scheduled out as part of the function call.

 graph-time
       When running function profiler with function graph tracer,
       to include the time to call nested functions. When this is
       not set, the time reported for the function will only
       include the time the function itself executed for, not the
       time for functions that it called.
”h]”(h¸)”}”(hŒtSince the function_graph tracer has a slightly different output
it has its own options to control what is displayed.”h]”hŒtSince the function_graph tracer has a slightly different output
it has its own options to control what is displayed.”…””}”(hj„!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M^hj€!  ubjT  )”}”(hX0
  funcgraph-overrun
      When set, the "overrun" of the graph stack is
      displayed after each function traced. The
      overrun, is when the stack depth of the calls
      is greater than what is reserved for each task.
      Each task has a fixed array of functions to
      trace in the call graph. If the depth of the
      calls exceeds that, the function is not traced.
      The overrun is the number of functions missed
      due to exceeding this array.

funcgraph-cpu
      When set, the CPU number of the CPU where the trace
      occurred is displayed.

funcgraph-overhead
      When set, if the function takes longer than
      A certain amount, then a delay marker is
      displayed. See "delay" above, under the
      header description.

funcgraph-proc
      Unlike other tracers, the process' command line
      is not displayed by default, but instead only
      when a task is traced in and out during a context
      switch. Enabling this options has the command
      of each process displayed at every line.

funcgraph-duration
      At the end of each function (the return)
      the duration of the amount of time in the
      function is displayed in microseconds.

funcgraph-abstime
      When set, the timestamp is displayed at each line.

funcgraph-irqs
      When disabled, functions that happen inside an
      interrupt will not be traced.

funcgraph-tail
      When set, the return event will include the function
      that it represents. By default this is off, and
      only a closing curly bracket "}" is displayed for
      the return of a function.

funcgraph-retval
      When set, the return value of each traced function
      will be printed after an equal sign "=". By default
      this is off.

funcgraph-retval-hex
      When set, the return value will always be printed
      in hexadecimal format. If the option is not set and
      the return value is an error code, it will be printed
      in signed decimal format; otherwise it will also be
      printed in hexadecimal format. By default, this option
      is off.

sleep-time
      When running function graph tracer, to include
      the time a task schedules out in its function.
      When enabled, it will account time the task has been
      scheduled out as part of the function call.

graph-time
      When running function profiler with function graph tracer,
      to include the time to call nested functions. When this is
      not set, the time reported for the function will only
      include the time the function itself executed for, not the
      time for functions that it called.
”•      h]”jT  )”}”(hhh]”(jY  )”}”(hXœ  funcgraph-overrun
When set, the "overrun" of the graph stack is
displayed after each function traced. The
overrun, is when the stack depth of the calls
is greater than what is reserved for each task.
Each task has a fixed array of functions to
trace in the call graph. If the depth of the
calls exceeds that, the function is not traced.
The overrun is the number of functions missed
due to exceeding this array.
”h]”(j_  )”}”(hŒfuncgraph-overrun”h]”hŒfuncgraph-overrun”…””}”(hj!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mjhj™!  ubjo  )”}”(hhh]”h¸)”}”(hX‰  When set, the "overrun" of the graph stack is
displayed after each function traced. The
overrun, is when the stack depth of the calls
is greater than what is reserved for each task.
Each task has a fixed array of functions to
trace in the call graph. If the depth of the
calls exceeds that, the function is not traced.
The overrun is the number of functions missed
due to exceeding this array.”h]”hX  When set, the â€œoverrunâ€ of the graph stack is
displayed after each function traced. The
overrun, is when the stack depth of the calls
is greater than what is reserved for each task.
Each task has a fixed array of functions to
trace in the call graph. If the depth of the
calls exceeds that, the function is not traced.
The overrun is the number of functions missed
due to exceeding this array.”…””}”(hj®!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mbhj«!  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj™!  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mjhj–!  ubjY  )”}”(hŒYfuncgraph-cpu
When set, the CPU number of the CPU where the trace
occurred is displayed.
”h]”(j_  )”}”(hŒfuncgraph-cpu”h]”hŒfuncgraph-cpu”…””}”(hjÌ!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MnhjÈ!  ubjo  )”}”(hhh]”h¸)”}”(hŒJWhen set, the CPU number of the CPU where the trace
occurred is displayed.”h]”hŒJWhen set, the CPU number of the CPU where the trace
occurred is displayed.”…””}”(hjÝ!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MmhjÚ!  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjÈ!  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mnhj–!  ubjY  )”}”(hŒ¤funcgraph-overhead
When set, if the function takes longer than
A certain amount, then a delay marker is
displayed. See "delay" above, under the
header description.
”h]”(j_  )”}”(hŒfuncgraph-overhead”h]”hŒfuncgraph-overhead”…””}”(hjû!  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mthj÷!  ubjo  )”}”(hhh]”h¸)”}”(hŒWhen set, if the function takes longer than
A certain amount, then a delay marker is
displayed. See "delay" above, under the
header description.”h]”hŒ”When set, if the function takes longer than
A certain amount, then a delay marker is
displayed. See â€œdelayâ€ above, under the
header description.”…””}”(hj"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mqhj	"  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj÷!  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mthj–!  ubjY  )”}”(hŒöfuncgraph-proc
Unlike other tracers, the process' command line
is not displayed by default, but instead only
when a task is traced in and out during a context
switch. Enabling this options has the command
of each process displayed at every line.
”h]”(j_  )”}”(hŒfuncgraph-proc”h]”hŒfuncgraph-proc”…””}”(hj*"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M{hj&"  ubjo  )”}”(hhh]”h¸)”}”(hŒæUnlike other tracers, the process' command line
is not displayed by default, but instead only
when a task is traced in and out during a context
switch. Enabling this options has the command
of each process displayed at every line.”h]”hŒèUnlike other tracers, the processâ€™ command line
is not displayed by default, but instead only
when a task is traced in and out during a context
switch. Enabling this options has the command
of each process displayed at every line.”…””}”(hj;"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mwhj8"  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj&"  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M{hj–!  ubjY  )”}”(hŒfuncgraph-duration
At the end of each function (the return)
the duration of the amount of time in the
function is displayed in microseconds.
”h]”(j_  )”}”(hŒfuncgraph-duration”h]”hŒfuncgraph-duration”…””}”(hjY"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M€hjU"  ubjo  )”}”(hhh]”h¸)”}”(hŒyAt the end of each function (the return)
the duration of the amount of time in the
function is displayed in microseconds.”h]”hŒyAt the end of each function (the return)
the duration of the amount of time in the
function is displayed in microseconds.”…””}”(hjj"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M~hjg"  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjU"  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M€hj–!  ubjY  )”}”(hŒEfuncgraph-abstime
When set, the timestamp is displayed at each line.
”h]”(j_  )”}”(hŒfuncgraph-abstime”h]”hŒfuncgraph-abstime”…””}”(hjˆ"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mƒhj„"  ubjo  )”}”(hhh]”h¸)”}”(hŒ2When set, the timestamp is displayed at each line.”h]”hŒ2When set, the timestamp is displayed at each line.”…””}”(hj™"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mƒhj–"  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj„"  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mƒhj–!  ubjY  )”}”(hŒ\funcgraph-irqs
When disabled, functions that happen inside an
interrupt will not be traced.
”h]”(j_  )”}”(hŒfuncgraph-irqs”h]”hŒfuncgraph-irqs”…””}”(hj·"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M‡hj³"  ubjo  )”}”(hhh]”h¸)”}”(hŒLWhen disabled, functions that happen inside an
interrupt will not be traced.”h]”hŒLWhen disabled, functions that happen inside an
interrupt will not be traced.”…””}”(hjÈ"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M†hjÅ"  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj³"  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M‡hj–!  ubjY  )”}”(hŒÀfuncgraph-tail
When set, the return event will include the function
that it represents. By default this is off, and
only a closing curly bracket "}" is displayed for
the return of a function.
”h]”(j_  )”}”(hŒfuncgraph-tail”h]”hŒfuncgraph-tail”…””}”(hjæ"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mhjâ"  ubjo  )”}”(hhh]”h¸)”}”(hŒ°When set, the return event will include the function
that it represents. By default this is off, and
only a closing curly bracket "}" is displayed for
the return of a function.”h]”hŒ´When set, the return event will include the function
that it represents. By default this is off, and
only a closing curly bracket â€œ}â€ is displayed for
the return of a function.”…””}”(hj÷"  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MŠhjô"  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjâ"  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mhj–!  ubjY  )”}”(hŒ…funcgraph-retval
When set, the return value of each traced function
will be printed after an equal sign "=". By default
this is off.
”h]”(j_  )”}”(hŒfuncgraph-retval”h]”hŒfuncgraph-retval”…””}”(hj#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M’hj#  ubjo  )”}”(hhh]”h¸)”}”(hŒsWhen set, the return value of each traced function
will be printed after an equal sign "=". By default
this is off.”h]”hŒwWhen set, the return value of each traced function
will be printed after an equal sign â€œ=â€. By default
this is off.”…””}”(hj&#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj##  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj#  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M’hj–!  ubjY  )”}”(hX$  funcgraph-retval-hex
When set, the return value will always be printed
in hexadecimal format. If the option is not set and
the return value is an error code, it will be printed
in signed decimal format; otherwise it will also be
printed in hexadecimal format. By default, this option
is off.
”h]”(j_  )”}”(hŒfuncgraph-retval-hex”h]”hŒfuncgraph-retval-hex”…””}”(hjD#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Mšhj@#  ubjo  )”}”(hhh]”h¸)”}”(hX  When set, the return value will always be printed
in hexadecimal format. If the option is not set and
the return value is an error code, it will be printed
in signed decimal format; otherwise it will also be
printed in hexadecimal format. By default, this option
is off.”h]”hX  When set, the return value will always be printed
in hexadecimal format. If the option is not set and
the return value is an error code, it will be printed
in signed decimal format; otherwise it will also be
printed in hexadecimal format. By default, this option
is off.”…””}”(hjU#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M•hjR#  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj@#  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Mšhj–!  ubjY  )”}”(hŒÊsleep-time
When running function graph tracer, to include
the time a task schedules out in its function.
When enabled, it will account time the task has been
scheduled out as part of the function call.
”h]”(j_  )”}”(hŒ
sleep-time”h]”hŒ
sleep-time”…””}”(hjs#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M hjo#  ubjo  )”}”(hhh]”h¸)”}”(hŒ¾When running function graph tracer, to include
the time a task schedules out in its function.
When enabled, it will account time the task has been
scheduled out as part of the function call.”h]”hŒ¾When running function graph tracer, to include
the time a task schedules out in its function.
When enabled, it will account time the task has been
scheduled out as part of the function call.”…””}”(hj„#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj#  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjo#  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M hj–!  ubjY  )”}”(hX  graph-time
When running function profiler with function graph tracer,
to include the time to call nested functions. When this is
not set, the time reported for the function will only
include the time the function itself executed for, not the
time for functions that it called.
”h]”(j_  )”}”(hŒ
graph-time”h]”hŒ
graph-time”…””}”(hj¢#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M§hjž#  ubjo  )”}”(hhh]”h¸)”}”(hX	  When running function profiler with function graph tracer,
to include the time to call nested functions. When this is
not set, the time reported for the function will only
include the time the function itself executed for, not the
time for functions that it called.”h]”hX	  When running function profiler with function graph tracer,
to include the time to call nested functions. When this is
not set, the time reported for the function will only
include the time the function itself executed for, not the
time for functions that it called.”…””}”(hj³#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M£hj°#  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjž#  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M§hj–!  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hj’!  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mahj€!  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M^hj)  hžhubh¸)”}”(hŒOptions for blk tracer:”h]”hŒOptions for blk tracer:”…””}”(hjß#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M©hj)  hžhubjT  )”}”(hŒ5blk_classic
      Shows a more minimalistic output.

”h]”jT  )”}”(hhh]”jY  )”}”(hŒ/blk_classic
Shows a more minimalistic output.

”h]”(j_  )”}”(hŒblk_classic”h]”hŒblk_classic”…””}”(hjø#  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M­hjô#  ubjo  )”}”(hhh]”h¸)”}”(hŒ!Shows a more minimalistic output.”h]”hŒ!Shows a more minimalistic output.”…””}”(hj	$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¬hj$  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjô#  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M­hjñ#  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hjí#  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M«hj)  hžhubeh}”(h]”Œtrace-options”ah ]”h"]”Œtrace_options”ah$]”h&]”uh1h¡hh£hžhhŸh¶h MAubh¢)”}”(hhh]”(h§)”}”(hŒirqsoff”h]”hŒirqsoff”…””}”(hj:$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj7$  hžhhŸh¶h M°ubh¸)”}”(hX  When interrupts are disabled, the CPU can not react to any other
external event (besides NMIs and SMIs). This prevents the timer
interrupt from triggering or the mouse interrupt from letting
the kernel know of a new mouse event. The result is a latency
with the reaction time.”h]”hX  When interrupts are disabled, the CPU can not react to any other
external event (besides NMIs and SMIs). This prevents the timer
interrupt from triggering or the mouse interrupt from letting
the kernel know of a new mouse event. The result is a latency
with the reaction time.”…””}”(hjH$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M²hj7$  hžhubh¸)”}”(hX  The irqsoff tracer tracks the time for which interrupts are
disabled. When a new maximum latency is hit, the tracer saves
the trace leading up to that latency point so that every time a
new maximum is reached, the old saved trace is discarded and the
new trace is saved.”h]”hX  The irqsoff tracer tracks the time for which interrupts are
disabled. When a new maximum latency is hit, the tracer saves
the trace leading up to that latency point so that every time a
new maximum is reached, the old saved trace is discarded and the
new trace is saved.”…””}”(hjV$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¸hj7$  hžhubh¸)”}”(hŒKTo reset the maximum, echo 0 into tracing_max_latency. Here is
an example::”h]”hŒJTo reset the maximum, echo 0 into tracing_max_latency. Here is
an example:”…””}”(hjd$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¾hj7$  hžhubj   )”}”(hX  # echo 0 > options/function-trace
# echo irqsoff > current_tracer
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# ls -ltr
[...]
# echo 0 > tracing_on
# cat trace
# tracer: irqsoff
#
# irqsoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 16 us, #4/4, CPU#0 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: swapper/0-0 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: run_timer_softirq
#  => ended at:   run_timer_softirq
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
  <idle>-0       0d.s2    0us+: _raw_spin_lock_irq <-run_timer_softirq
  <idle>-0       0dNs3   17us : _raw_spin_unlock_irq <-run_timer_softirq
  <idle>-0       0dNs3   17us+: trace_hardirqs_on <-run_timer_softirq
  <idle>-0       0dNs3   25us : <stack trace>
 => _raw_spin_unlock_irq
 => run_timer_softirq
 => __do_softirq
 => call_softirq
 => do_softirq
 => irq_exit
 => smp_apic_timer_interrupt
 => apic_timer_interrupt
 => rcu_idle_exit
 => cpu_idle
 => rest_init
 => start_kernel
 => x86_64_start_reservations
 => x86_64_start_kernel”h]”hX  # echo 0 > options/function-trace
# echo irqsoff > current_tracer
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# ls -ltr
[...]
# echo 0 > tracing_on
# cat trace
# tracer: irqsoff
#
# irqsoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 16 us, #4/4, CPU#0 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: swapper/0-0 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: run_timer_softirq
#  => ended at:   run_timer_softirq
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
  <idle>-0       0d.s2    0us+: _raw_spin_lock_irq <-run_timer_softirq
  <idle>-0       0dNs3   17us : _raw_spin_unlock_irq <-run_timer_softirq
  <idle>-0       0dNs3   17us+: trace_hardirqs_on <-run_timer_softirq
  <idle>-0       0dNs3   25us : <stack trace>
 => _raw_spin_unlock_irq
 => run_timer_softirq
 => __do_softirq
 => call_softirq
 => do_softirq
 => irq_exit
 => smp_apic_timer_interrupt
 => apic_timer_interrupt
 => rcu_idle_exit
 => cpu_idle
 => rest_init
 => start_kernel
 => x86_64_start_reservations
 => x86_64_start_kernel”…””}”hjr$  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MÁhj7$  hžhubh¸)”}”(hXa  Here we see that we had a latency of 16 microseconds (which is
very good). The _raw_spin_lock_irq in run_timer_softirq disabled
interrupts. The difference between the 16 and the displayed
timestamp 25us occurred because the clock was incremented
between the time of recording the max latency and the time of
recording the function that had that latency.”h]”hXa  Here we see that we had a latency of 16 microseconds (which is
very good). The _raw_spin_lock_irq in run_timer_softirq disabled
interrupts. The difference between the 16 and the displayed
timestamp 25us occurred because the clock was incremented
between the time of recording the max latency and the time of
recording the function that had that latency.”…””}”(hj€$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mðhj7$  hžhubh¸)”}”(hŒjNote the above example had function-trace not set. If we set
function-trace, we get a much larger output::”h]”hŒiNote the above example had function-trace not set. If we set
function-trace, we get a much larger output:”…””}”(hjŽ$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M÷hj7$  hžhubj   )”}”(hX
  with echo 1 > options/function-trace

 # tracer: irqsoff
 #
 # irqsoff latency trace v1.1.5 on 3.8.0-test+
 # --------------------------------------------------------------------
 # latency: 71 us, #168/168, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
 #    -----------------
 #    | task: bash-2042 (uid:0 nice:0 policy:0 rt_prio:0)
 #    -----------------
 #  => started at: ata_scsi_queuecmd
 #  => ended at:   ata_scsi_queuecmd
 #
 #
 #                  _------=> CPU#
 #                 / _-----=> irqs-off
 #                | / _----=> need-resched
 #                || / _---=> hardirq/softirq
 #                ||| / _--=> preempt-depth
 #                |||| /     delay
 #  cmd     pid   ||||| time  |   caller
 #     \   /      |||||  \    |   /
     bash-2042    3d...    0us : _raw_spin_lock_irqsave <-ata_scsi_queuecmd
     bash-2042    3d...    0us : add_preempt_count <-_raw_spin_lock_irqsave
     bash-2042    3d..1    1us : ata_scsi_find_dev <-ata_scsi_queuecmd
     bash-2042    3d..1    1us : __ata_scsi_find_dev <-ata_scsi_find_dev
     bash-2042    3d..1    2us : ata_find_dev.part.14 <-__ata_scsi_find_dev
     bash-2042    3d..1    2us : ata_qc_new_init <-__ata_scsi_queuecmd
     bash-2042    3d..1    3us : ata_sg_init <-__ata_scsi_queuecmd
     bash-2042    3d..1    4us : ata_scsi_rw_xlat <-__ata_scsi_queuecmd
     bash-2042    3d..1    4us : ata_build_rw_tf <-ata_scsi_rw_xlat
 [...]
     bash-2042    3d..1   67us : delay_tsc <-__delay
     bash-2042    3d..1   67us : add_preempt_count <-delay_tsc
     bash-2042    3d..2   67us : sub_preempt_count <-delay_tsc
     bash-2042    3d..1   67us : add_preempt_count <-delay_tsc
     bash-2042    3d..2   68us : sub_preempt_count <-delay_tsc
     bash-2042    3d..1   68us+: ata_bmdma_start <-ata_bmdma_qc_issue
     bash-2042    3d..1   71us : _raw_spin_unlock_irqrestore <-ata_scsi_queuecmd
     bash-2042    3d..1   71us : _raw_spin_unlock_irqrestore <-ata_scsi_queuecmd
     bash-2042    3d..1   72us+: trace_hardirqs_on <-ata_scsi_queuecmd
     bash-2042    3d..1  120us : <stack trace>
  => _raw_spin_unlock_irqrestore
  => ata_scsi_queuecmd
  => scsi_dispatch_cmd
  => scsi_request_fn
  => __blk_run_queue_uncond
  => __blk_run_queue
  => blk_queue_bio
  => submit_bio_noacct
  => submit_bio
  => submit_bh
  => __ext3_get_inode_loc
  => ext3_iget
  => ext3_lookup
  => lookup_real
  => __lookup_hash
  => walk_component
  => lookup_last
  => path_lookupat
  => filename_lookup
  => user_path_at_empty
  => user_path_at
  => vfs_fstatat
  => vfs_stat
  => sys_newstat
  => system_call_fastpath”h]”hX
  with echo 1 > options/function-trace

 # tracer: irqsoff
 #
 # irqsoff latency trace v1.1.5 on 3.8.0-test+
 # --------------------------------------------------------------------
 # latency: 71 us, #168/168, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
 #    -----------------
 #    | task: bash-2042 (uid:0 nice:0 policy:0 rt_prio:0)
 #    -----------------
 #  => started at: ata_scsi_queuecmd
 #  => ended at:   ata_scsi_queuecmd
 #
 #
 #                  _------=> CPU#
 #                 / _-----=> irqs-off
 #                | / _----=> need-resched
 #                || / _---=> hardirq/softirq
 #                ||| / _--=> preempt-depth
 #                |||| /     delay
 #  cmd     pid   ||||| time  |   caller
 #     \   /      |||||  \    |   /
     bash-2042    3d...    0us : _raw_spin_lock_irqsave <-ata_scsi_queuecmd
     bash-2042    3d...    0us : add_preempt_count <-_raw_spin_lock_irqsave
     bash-2042    3d..1    1us : ata_scsi_find_dev <-ata_scsi_queuecmd
     bash-2042    3d..1    1us : __ata_scsi_find_dev <-ata_scsi_find_dev
     bash-2042    3d..1    2us : ata_find_dev.part.14 <-__ata_scsi_find_dev
     bash-2042    3d..1    2us : ata_qc_new_init <-__ata_scsi_queuecmd
     bash-2042    3d..1    3us : ata_sg_init <-__ata_scsi_queuecmd
     bash-2042    3d..1    4us : ata_scsi_rw_xlat <-__ata_scsi_queuecmd
     bash-2042    3d..1    4us : ata_build_rw_tf <-ata_scsi_rw_xlat
 [...]
     bash-2042    3d..1   67us : delay_tsc <-__delay
     bash-2042    3d..1   67us : add_preempt_count <-delay_tsc
     bash-2042    3d..2   67us : sub_preempt_count <-delay_tsc
     bash-2042    3d..1   67us : add_preempt_count <-delay_tsc
     bash-2042    3d..2   68us : sub_preempt_count <-delay_tsc
     bash-2042    3d..1   68us+: ata_bmdma_start <-ata_bmdma_qc_issue
     bash-2042    3d..1   71us : _raw_spin_unlock_irqrestore <-ata_scsi_queuecmd
     bash-2042    3d..1   71us : _raw_spin_unlock_irqrestore <-ata_scsi_queuecmd
     bash-2042    3d..1   72us+: trace_hardirqs_on <-ata_scsi_queuecmd
     bash-2042    3d..1  120us : <stack trace>
  => _raw_spin_unlock_irqrestore
  => ata_scsi_queuecmd
  => scsi_dispatch_cmd
  => scsi_request_fn
  => __blk_run_queue_uncond
  => __blk_run_queue
  => blk_queue_bio
  => submit_bio_noacct
  => submit_bio
  => submit_bh
  => __ext3_get_inode_loc
  => ext3_iget
  => ext3_lookup
  => lookup_real
  => __lookup_hash
  => walk_component
  => lookup_last
  => path_lookupat
  => filename_lookup
  => user_path_at_empty
  => user_path_at
  => vfs_fstatat
  => vfs_stat
  => sys_newstat
  => system_call_fastpath”…””}”hjœ$  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Múhj7$  hžhubh¸)”}”(hX0  Here we traced a 71 microsecond latency. But we also see all the
functions that were called during that time. Note that by
enabling function tracing, we incur an added overhead. This
overhead may extend the latency times. But nevertheless, this
trace has provided some very helpful debugging information.”h]”hX0  Here we traced a 71 microsecond latency. But we also see all the
functions that were called during that time. Note that by
enabling function tracing, we incur an added overhead. This
overhead may extend the latency times. But nevertheless, this
trace has provided some very helpful debugging information.”…””}”(hjª$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M?hj7$  hžhubh¸)”}”(hŒYIf we prefer function graph output instead of function, we can set
display-graph option::”h]”hŒXIf we prefer function graph output instead of function, we can set
display-graph option:”…””}”(hj¸$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MEhj7$  hžhubj   )”}”(hX	  with echo 1 > options/display-graph

 # tracer: irqsoff
 #
 # irqsoff latency trace v1.1.5 on 4.20.0-rc6+
 # --------------------------------------------------------------------
 # latency: 3751 us, #274/274, CPU#0 | (M:desktop VP:0, KP:0, SP:0 HP:0 #P:4)
 #    -----------------
 #    | task: bash-1507 (uid:0 nice:0 policy:0 rt_prio:0)
 #    -----------------
 #  => started at: free_debug_processing
 #  => ended at:   return_to_handler
 #
 #
 #                                       _-----=> irqs-off
 #                                      / _----=> need-resched
 #                                     | / _---=> hardirq/softirq
 #                                     || / _--=> preempt-depth
 #                                     ||| /
 #   REL TIME      CPU  TASK/PID       ||||     DURATION                  FUNCTION CALLS
 #      |          |     |    |        ||||      |   |                     |   |   |   |
         0 us |   0)   bash-1507    |  d... |   0.000 us    |  _raw_spin_lock_irqsave();
         0 us |   0)   bash-1507    |  d..1 |   0.378 us    |    do_raw_spin_trylock();
         1 us |   0)   bash-1507    |  d..2 |               |    set_track() {
         2 us |   0)   bash-1507    |  d..2 |               |      save_stack_trace() {
         2 us |   0)   bash-1507    |  d..2 |               |        __save_stack_trace() {
         3 us |   0)   bash-1507    |  d..2 |               |          __unwind_start() {
         3 us |   0)   bash-1507    |  d..2 |               |            get_stack_info() {
         3 us |   0)   bash-1507    |  d..2 |   0.351 us    |              in_task_stack();
         4 us |   0)   bash-1507    |  d..2 |   1.107 us    |            }
 [...]
      3750 us |   0)   bash-1507    |  d..1 |   0.516 us    |      do_raw_spin_unlock();
      3750 us |   0)   bash-1507    |  d..1 |   0.000 us    |  _raw_spin_unlock_irqrestore();
      3764 us |   0)   bash-1507    |  d..1 |   0.000 us    |  tracer_hardirqs_on();
     bash-1507    0d..1 3792us : <stack trace>
  => free_debug_processing
  => __slab_free
  => kmem_cache_free
  => vm_area_free
  => remove_vma
  => exit_mmap
  => mmput
  => begin_new_exec
  => load_elf_binary
  => search_binary_handler
  => __do_execve_file.isra.32
  => __x64_sys_execve
  => do_syscall_64
  => entry_SYSCALL_64_after_hwframe”h]”hX	  with echo 1 > options/display-graph

 # tracer: irqsoff
 #
 # irqsoff latency trace v1.1.5 on 4.20.0-rc6+
 # --------------------------------------------------------------------
 # latency: 3751 us, #274/274, CPU#0 | (M:desktop VP:0, KP:0, SP:0 HP:0 #P:4)
 #    -----------------
 #    | task: bash-1507 (uid:0 nice:0 policy:0 rt_prio:0)
 #    -----------------
 #  => started at: free_debug_processing
 #  => ended at:   return_to_handler
 #
 #
 #                                       _-----=> irqs-off
 #                                      / _----=> need-resched
 #                                     | / _---=> hardirq/softirq
 #                                     || / _--=> preempt-depth
 #                                     ||| /
 #   REL TIME      CPU  TASK/PID       ||||     DURATION                  FUNCTION CALLS
 #      |          |     |    |        ||||      |   |                     |   |   |   |
         0 us |   0)   bash-1507    |  d... |   0.000 us    |  _raw_spin_lock_irqsave();
         0 us |   0)   bash-1507    |  d..1 |   0.378 us    |    do_raw_spin_trylock();
         1 us |   0)   bash-1507    |  d..2 |               |    set_track() {
         2 us |   0)   bash-1507    |  d..2 |               |      save_stack_trace() {
         2 us |   0)   bash-1507    |  d..2 |               |        __save_stack_trace() {
         3 us |   0)   bash-1507    |  d..2 |               |          __unwind_start() {
         3 us |   0)   bash-1507    |  d..2 |               |            get_stack_info() {
         3 us |   0)   bash-1507    |  d..2 |   0.351 us    |              in_task_stack();
         4 us |   0)   bash-1507    |  d..2 |   1.107 us    |            }
 [...]
      3750 us |   0)   bash-1507    |  d..1 |   0.516 us    |      do_raw_spin_unlock();
      3750 us |   0)   bash-1507    |  d..1 |   0.000 us    |  _raw_spin_unlock_irqrestore();
      3764 us |   0)   bash-1507    |  d..1 |   0.000 us    |  tracer_hardirqs_on();
     bash-1507    0d..1 3792us : <stack trace>
  => free_debug_processing
  => __slab_free
  => kmem_cache_free
  => vm_area_free
  => remove_vma
  => exit_mmap
  => mmput
  => begin_new_exec
  => load_elf_binary
  => search_binary_handler
  => __do_execve_file.isra.32
  => __x64_sys_execve
  => do_syscall_64
  => entry_SYSCALL_64_after_hwframe”…””}”hjÆ$  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MHhj7$  hžhubeh}”(h]”Œirqsoff”ah ]”h"]”Œirqsoff”ah$]”h&]”uh1h¡hh£hžhhŸh¶h M°ubh¢)”}”(hhh]”(h§)”}”(hŒ
preemptoff”h]”hŒ
preemptoff”…””}”(hjß$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjÜ$  hžhhŸh¶h M{ubh¸)”}”(hŒ×When preemption is disabled, we may be able to receive
interrupts but the task cannot be preempted and a higher
priority task must wait for preemption to be enabled again
before it can preempt a lower priority task.”h]”hŒ×When preemption is disabled, we may be able to receive
interrupts but the task cannot be preempted and a higher
priority task must wait for preemption to be enabled again
before it can preempt a lower priority task.”…””}”(hjí$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M}hjÜ$  hžhubh¸)”}”(hŒàThe preemptoff tracer traces the places that disable preemption.
Like the irqsoff tracer, it records the maximum latency for
which preemption was disabled. The control of preemptoff tracer
is much like the irqsoff tracer.
::”h]”hŒÝThe preemptoff tracer traces the places that disable preemption.
Like the irqsoff tracer, it records the maximum latency for
which preemption was disabled. The control of preemptoff tracer
is much like the irqsoff tracer.”…””}”(hjû$  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M‚hjÜ$  hžhubj   )”}”(hXb  # echo 0 > options/function-trace
# echo preemptoff > current_tracer
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# ls -ltr
[...]
# echo 0 > tracing_on
# cat trace
# tracer: preemptoff
#
# preemptoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 46 us, #4/4, CPU#1 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: sshd-1991 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: do_IRQ
#  => ended at:   do_IRQ
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
    sshd-1991    1d.h.    0us+: irq_enter <-do_IRQ
    sshd-1991    1d..1   46us : irq_exit <-do_IRQ
    sshd-1991    1d..1   47us+: trace_preempt_on <-do_IRQ
    sshd-1991    1d..1   52us : <stack trace>
 => sub_preempt_count
 => irq_exit
 => do_IRQ
 => ret_from_intr”h]”hXb  # echo 0 > options/function-trace
# echo preemptoff > current_tracer
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# ls -ltr
[...]
# echo 0 > tracing_on
# cat trace
# tracer: preemptoff
#
# preemptoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 46 us, #4/4, CPU#1 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: sshd-1991 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: do_IRQ
#  => ended at:   do_IRQ
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
    sshd-1991    1d.h.    0us+: irq_enter <-do_IRQ
    sshd-1991    1d..1   46us : irq_exit <-do_IRQ
    sshd-1991    1d..1   47us+: trace_preempt_on <-do_IRQ
    sshd-1991    1d..1   52us : <stack trace>
 => sub_preempt_count
 => irq_exit
 => do_IRQ
 => ret_from_intr”…””}”hj	%  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MˆhjÜ$  hžhubh¸)”}”(hXI  This has some more changes. Preemption was disabled when an
interrupt came in (notice the 'h'), and was enabled on exit.
But we also see that interrupts have been disabled when entering
the preempt off section and leaving it (the 'd'). We do not know if
interrupts were enabled in the mean time or shortly after this
was over.
::”h]”hXN  This has some more changes. Preemption was disabled when an
interrupt came in (notice the â€˜hâ€™), and was enabled on exit.
But we also see that interrupts have been disabled when entering
the preempt off section and leaving it (the â€˜dâ€™). We do not know if
interrupts were enabled in the mean time or shortly after this
was over.”…””}”(hj%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M®hjÜ$  hžhubj   )”}”(hX¢  # tracer: preemptoff
#
# preemptoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 83 us, #241/241, CPU#1 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: bash-1994 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: wake_up_new_task
#  => ended at:   task_rq_unlock
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
    bash-1994    1d..1    0us : _raw_spin_lock_irqsave <-wake_up_new_task
    bash-1994    1d..1    0us : select_task_rq_fair <-select_task_rq
    bash-1994    1d..1    1us : __rcu_read_lock <-select_task_rq_fair
    bash-1994    1d..1    1us : source_load <-select_task_rq_fair
    bash-1994    1d..1    1us : source_load <-select_task_rq_fair
[...]
    bash-1994    1d..1   12us : irq_enter <-smp_apic_timer_interrupt
    bash-1994    1d..1   12us : rcu_irq_enter <-irq_enter
    bash-1994    1d..1   13us : add_preempt_count <-irq_enter
    bash-1994    1d.h1   13us : exit_idle <-smp_apic_timer_interrupt
    bash-1994    1d.h1   13us : hrtimer_interrupt <-smp_apic_timer_interrupt
    bash-1994    1d.h1   13us : _raw_spin_lock <-hrtimer_interrupt
    bash-1994    1d.h1   14us : add_preempt_count <-_raw_spin_lock
    bash-1994    1d.h2   14us : ktime_get_update_offsets <-hrtimer_interrupt
[...]
    bash-1994    1d.h1   35us : lapic_next_event <-clockevents_program_event
    bash-1994    1d.h1   35us : irq_exit <-smp_apic_timer_interrupt
    bash-1994    1d.h1   36us : sub_preempt_count <-irq_exit
    bash-1994    1d..2   36us : do_softirq <-irq_exit
    bash-1994    1d..2   36us : __do_softirq <-call_softirq
    bash-1994    1d..2   36us : __local_bh_disable <-__do_softirq
    bash-1994    1d.s2   37us : add_preempt_count <-_raw_spin_lock_irq
    bash-1994    1d.s3   38us : _raw_spin_unlock <-run_timer_softirq
    bash-1994    1d.s3   39us : sub_preempt_count <-_raw_spin_unlock
    bash-1994    1d.s2   39us : call_timer_fn <-run_timer_softirq
[...]
    bash-1994    1dNs2   81us : cpu_needs_another_gp <-rcu_process_callbacks
    bash-1994    1dNs2   82us : __local_bh_enable <-__do_softirq
    bash-1994    1dNs2   82us : sub_preempt_count <-__local_bh_enable
    bash-1994    1dN.2   82us : idle_cpu <-irq_exit
    bash-1994    1dN.2   83us : rcu_irq_exit <-irq_exit
    bash-1994    1dN.2   83us : sub_preempt_count <-irq_exit
    bash-1994    1.N.1   84us : _raw_spin_unlock_irqrestore <-task_rq_unlock
    bash-1994    1.N.1   84us+: trace_preempt_on <-task_rq_unlock
    bash-1994    1.N.1  104us : <stack trace>
 => sub_preempt_count
 => _raw_spin_unlock_irqrestore
 => task_rq_unlock
 => wake_up_new_task
 => do_fork
 => sys_clone
 => stub_clone”h]”hX¢  # tracer: preemptoff
#
# preemptoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 83 us, #241/241, CPU#1 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: bash-1994 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: wake_up_new_task
#  => ended at:   task_rq_unlock
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
    bash-1994    1d..1    0us : _raw_spin_lock_irqsave <-wake_up_new_task
    bash-1994    1d..1    0us : select_task_rq_fair <-select_task_rq
    bash-1994    1d..1    1us : __rcu_read_lock <-select_task_rq_fair
    bash-1994    1d..1    1us : source_load <-select_task_rq_fair
    bash-1994    1d..1    1us : source_load <-select_task_rq_fair
[...]
    bash-1994    1d..1   12us : irq_enter <-smp_apic_timer_interrupt
    bash-1994    1d..1   12us : rcu_irq_enter <-irq_enter
    bash-1994    1d..1   13us : add_preempt_count <-irq_enter
    bash-1994    1d.h1   13us : exit_idle <-smp_apic_timer_interrupt
    bash-1994    1d.h1   13us : hrtimer_interrupt <-smp_apic_timer_interrupt
    bash-1994    1d.h1   13us : _raw_spin_lock <-hrtimer_interrupt
    bash-1994    1d.h1   14us : add_preempt_count <-_raw_spin_lock
    bash-1994    1d.h2   14us : ktime_get_update_offsets <-hrtimer_interrupt
[...]
    bash-1994    1d.h1   35us : lapic_next_event <-clockevents_program_event
    bash-1994    1d.h1   35us : irq_exit <-smp_apic_timer_interrupt
    bash-1994    1d.h1   36us : sub_preempt_count <-irq_exit
    bash-1994    1d..2   36us : do_softirq <-irq_exit
    bash-1994    1d..2   36us : __do_softirq <-call_softirq
    bash-1994    1d..2   36us : __local_bh_disable <-__do_softirq
    bash-1994    1d.s2   37us : add_preempt_count <-_raw_spin_lock_irq
    bash-1994    1d.s3   38us : _raw_spin_unlock <-run_timer_softirq
    bash-1994    1d.s3   39us : sub_preempt_count <-_raw_spin_unlock
    bash-1994    1d.s2   39us : call_timer_fn <-run_timer_softirq
[...]
    bash-1994    1dNs2   81us : cpu_needs_another_gp <-rcu_process_callbacks
    bash-1994    1dNs2   82us : __local_bh_enable <-__do_softirq
    bash-1994    1dNs2   82us : sub_preempt_count <-__local_bh_enable
    bash-1994    1dN.2   82us : idle_cpu <-irq_exit
    bash-1994    1dN.2   83us : rcu_irq_exit <-irq_exit
    bash-1994    1dN.2   83us : sub_preempt_count <-irq_exit
    bash-1994    1.N.1   84us : _raw_spin_unlock_irqrestore <-task_rq_unlock
    bash-1994    1.N.1   84us+: trace_preempt_on <-task_rq_unlock
    bash-1994    1.N.1  104us : <stack trace>
 => sub_preempt_count
 => _raw_spin_unlock_irqrestore
 => task_rq_unlock
 => wake_up_new_task
 => do_fork
 => sys_clone
 => stub_clone”…””}”hj%%  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M¶hjÜ$  hžhubh¸)”}”(hXd  The above is an example of the preemptoff trace with
function-trace set. Here we see that interrupts were not disabled
the entire time. The irq_enter code lets us know that we entered
an interrupt 'h'. Before that, the functions being traced still
show that it is not in an interrupt, but we can see from the
functions themselves that this is not the case.”h]”hXh  The above is an example of the preemptoff trace with
function-trace set. Here we see that interrupts were not disabled
the entire time. The irq_enter code lets us know that we entered
an interrupt â€˜hâ€™. Before that, the functions being traced still
show that it is not in an interrupt, but we can see from the
functions themselves that this is not the case.”…””}”(hj3%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MöhjÜ$  hžhubeh}”(h]”Œ
preemptoff”ah ]”h"]”Œ
preemptoff”ah$]”h&]”uh1h¡hh£hžhhŸh¶h M{ubh¢)”}”(hhh]”(h§)”}”(hŒpreemptirqsoff”h]”hŒpreemptirqsoff”…””}”(hjL%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjI%  hžhhŸh¶h Mþubh¸)”}”(hŒÇKnowing the locations that have interrupts disabled or
preemption disabled for the longest times is helpful. But
sometimes we would like to know when either preemption and/or
interrupts are disabled.”h]”hŒÇKnowing the locations that have interrupts disabled or
preemption disabled for the longest times is helpful. But
sometimes we would like to know when either preemption and/or
interrupts are disabled.”…””}”(hjZ%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M hjI%  hžhubh¸)”}”(hŒConsider the following code::”h]”hŒConsider the following code:”…””}”(hjh%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjI%  hžhubj   )”}”(hŒ¿local_irq_disable();
call_function_with_irqs_off();
preempt_disable();
call_function_with_irqs_and_preemption_off();
local_irq_enable();
call_function_with_preemption_off();
preempt_enable();”h]”hŒ¿local_irq_disable();
call_function_with_irqs_off();
preempt_disable();
call_function_with_irqs_and_preemption_off();
local_irq_enable();
call_function_with_preemption_off();
preempt_enable();”…””}”hjv%  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MhjI%  hžhubh¸)”}”(hŒ‚The irqsoff tracer will record the total length of
call_function_with_irqs_off() and
call_function_with_irqs_and_preemption_off().”h]”hŒ‚The irqsoff tracer will record the total length of
call_function_with_irqs_off() and
call_function_with_irqs_and_preemption_off().”…””}”(hj„%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjI%  hžhubh¸)”}”(hŒ‹The preemptoff tracer will record the total length of
call_function_with_irqs_and_preemption_off() and
call_function_with_preemption_off().”h]”hŒ‹The preemptoff tracer will record the total length of
call_function_with_irqs_and_preemption_off() and
call_function_with_preemption_off().”…””}”(hj’%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjI%  hžhubh¸)”}”(hŒ¸But neither will trace the time that interrupts and/or
preemption is disabled. This total time is the time that we can
not schedule. To record this time, use the preemptirqsoff
tracer.”h]”hŒ¸But neither will trace the time that interrupts and/or
preemption is disabled. This total time is the time that we can
not schedule. To record this time, use the preemptirqsoff
tracer.”…””}”(hj %  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjI%  hžhubh¸)”}”(hŒKAgain, using this trace is much like the irqsoff and preemptoff
tracers.
::”h]”hŒHAgain, using this trace is much like the irqsoff and preemptoff
tracers.”…””}”(hj®%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjI%  hžhubj   )”}”(hX  # echo 0 > options/function-trace
# echo preemptirqsoff > current_tracer
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# ls -ltr
[...]
# echo 0 > tracing_on
# cat trace
# tracer: preemptirqsoff
#
# preemptirqsoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 100 us, #4/4, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: ls-2230 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: ata_scsi_queuecmd
#  => ended at:   ata_scsi_queuecmd
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
      ls-2230    3d...    0us+: _raw_spin_lock_irqsave <-ata_scsi_queuecmd
      ls-2230    3...1  100us : _raw_spin_unlock_irqrestore <-ata_scsi_queuecmd
      ls-2230    3...1  101us+: trace_preempt_on <-ata_scsi_queuecmd
      ls-2230    3...1  111us : <stack trace>
 => sub_preempt_count
 => _raw_spin_unlock_irqrestore
 => ata_scsi_queuecmd
 => scsi_dispatch_cmd
 => scsi_request_fn
 => __blk_run_queue_uncond
 => __blk_run_queue
 => blk_queue_bio
 => submit_bio_noacct
 => submit_bio
 => submit_bh
 => ext3_bread
 => ext3_dir_bread
 => htree_dirblock_to_tree
 => ext3_htree_fill_tree
 => ext3_readdir
 => vfs_readdir
 => sys_getdents
 => system_call_fastpath”h]”hX  # echo 0 > options/function-trace
# echo preemptirqsoff > current_tracer
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# ls -ltr
[...]
# echo 0 > tracing_on
# cat trace
# tracer: preemptirqsoff
#
# preemptirqsoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 100 us, #4/4, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: ls-2230 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: ata_scsi_queuecmd
#  => ended at:   ata_scsi_queuecmd
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
      ls-2230    3d...    0us+: _raw_spin_lock_irqsave <-ata_scsi_queuecmd
      ls-2230    3...1  100us : _raw_spin_unlock_irqrestore <-ata_scsi_queuecmd
      ls-2230    3...1  101us+: trace_preempt_on <-ata_scsi_queuecmd
      ls-2230    3...1  111us : <stack trace>
 => sub_preempt_count
 => _raw_spin_unlock_irqrestore
 => ata_scsi_queuecmd
 => scsi_dispatch_cmd
 => scsi_request_fn
 => __blk_run_queue_uncond
 => __blk_run_queue
 => blk_queue_bio
 => submit_bio_noacct
 => submit_bio
 => submit_bh
 => ext3_bread
 => ext3_dir_bread
 => htree_dirblock_to_tree
 => ext3_htree_fill_tree
 => ext3_readdir
 => vfs_readdir
 => sys_getdents
 => system_call_fastpath”…””}”hj¼%  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M hjI%  hžhubh¸)”}”(hX  The trace_hardirqs_off_thunk is called from assembly on x86 when
interrupts are disabled in the assembly code. Without the
function tracing, we do not know if interrupts were enabled
within the preemption points. We do see that it started with
preemption enabled.”h]”hX  The trace_hardirqs_off_thunk is called from assembly on x86 when
interrupts are disabled in the assembly code. Without the
function tracing, we do not know if interrupts were enabled
within the preemption points. We do see that it started with
preemption enabled.”…””}”(hjÊ%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MUhjI%  hžhubh¸)”}”(hŒ)Here is a trace with function-trace set::”h]”hŒ(Here is a trace with function-trace set:”…””}”(hjØ%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M[hjI%  hžhubj   )”}”(hX  # tracer: preemptirqsoff
#
# preemptirqsoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 161 us, #339/339, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: ls-2269 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: schedule
#  => ended at:   mutex_unlock
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
kworker/-59      3...1    0us : __schedule <-schedule
kworker/-59      3d..1    0us : rcu_preempt_qs <-rcu_note_context_switch
kworker/-59      3d..1    1us : add_preempt_count <-_raw_spin_lock_irq
kworker/-59      3d..2    1us : deactivate_task <-__schedule
kworker/-59      3d..2    1us : dequeue_task <-deactivate_task
kworker/-59      3d..2    2us : update_rq_clock <-dequeue_task
kworker/-59      3d..2    2us : dequeue_task_fair <-dequeue_task
kworker/-59      3d..2    2us : update_curr <-dequeue_task_fair
kworker/-59      3d..2    2us : update_min_vruntime <-update_curr
kworker/-59      3d..2    3us : cpuacct_charge <-update_curr
kworker/-59      3d..2    3us : __rcu_read_lock <-cpuacct_charge
kworker/-59      3d..2    3us : __rcu_read_unlock <-cpuacct_charge
kworker/-59      3d..2    3us : update_cfs_rq_blocked_load <-dequeue_task_fair
kworker/-59      3d..2    4us : clear_buddies <-dequeue_task_fair
kworker/-59      3d..2    4us : account_entity_dequeue <-dequeue_task_fair
kworker/-59      3d..2    4us : update_min_vruntime <-dequeue_task_fair
kworker/-59      3d..2    4us : update_cfs_shares <-dequeue_task_fair
kworker/-59      3d..2    5us : hrtick_update <-dequeue_task_fair
kworker/-59      3d..2    5us : wq_worker_sleeping <-__schedule
kworker/-59      3d..2    5us : kthread_data <-wq_worker_sleeping
kworker/-59      3d..2    5us : put_prev_task_fair <-__schedule
kworker/-59      3d..2    6us : pick_next_task_fair <-pick_next_task
kworker/-59      3d..2    6us : clear_buddies <-pick_next_task_fair
kworker/-59      3d..2    6us : set_next_entity <-pick_next_task_fair
kworker/-59      3d..2    6us : update_stats_wait_end <-set_next_entity
      ls-2269    3d..2    7us : finish_task_switch <-__schedule
      ls-2269    3d..2    7us : _raw_spin_unlock_irq <-finish_task_switch
      ls-2269    3d..2    8us : do_IRQ <-ret_from_intr
      ls-2269    3d..2    8us : irq_enter <-do_IRQ
      ls-2269    3d..2    8us : rcu_irq_enter <-irq_enter
      ls-2269    3d..2    9us : add_preempt_count <-irq_enter
      ls-2269    3d.h2    9us : exit_idle <-do_IRQ
[...]
      ls-2269    3d.h3   20us : sub_preempt_count <-_raw_spin_unlock
      ls-2269    3d.h2   20us : irq_exit <-do_IRQ
      ls-2269    3d.h2   21us : sub_preempt_count <-irq_exit
      ls-2269    3d..3   21us : do_softirq <-irq_exit
      ls-2269    3d..3   21us : __do_softirq <-call_softirq
      ls-2269    3d..3   21us+: __local_bh_disable <-__do_softirq
      ls-2269    3d.s4   29us : sub_preempt_count <-_local_bh_enable_ip
      ls-2269    3d.s5   29us : sub_preempt_count <-_local_bh_enable_ip
      ls-2269    3d.s5   31us : do_IRQ <-ret_from_intr
      ls-2269    3d.s5   31us : irq_enter <-do_IRQ
      ls-2269    3d.s5   31us : rcu_irq_enter <-irq_enter
[...]
      ls-2269    3d.s5   31us : rcu_irq_enter <-irq_enter
      ls-2269    3d.s5   32us : add_preempt_count <-irq_enter
      ls-2269    3d.H5   32us : exit_idle <-do_IRQ
      ls-2269    3d.H5   32us : handle_irq <-do_IRQ
      ls-2269    3d.H5   32us : irq_to_desc <-handle_irq
      ls-2269    3d.H5   33us : handle_fasteoi_irq <-handle_irq
[...]
      ls-2269    3d.s5  158us : _raw_spin_unlock_irqrestore <-rtl8139_poll
      ls-2269    3d.s3  158us : net_rps_action_and_irq_enable.isra.65 <-net_rx_action
      ls-2269    3d.s3  159us : __local_bh_enable <-__do_softirq
      ls-2269    3d.s3  159us : sub_preempt_count <-__local_bh_enable
      ls-2269    3d..3  159us : idle_cpu <-irq_exit
      ls-2269    3d..3  159us : rcu_irq_exit <-irq_exit
      ls-2269    3d..3  160us : sub_preempt_count <-irq_exit
      ls-2269    3d...  161us : __mutex_unlock_slowpath <-mutex_unlock
      ls-2269    3d...  162us+: trace_hardirqs_on <-mutex_unlock
      ls-2269    3d...  186us : <stack trace>
 => __mutex_unlock_slowpath
 => mutex_unlock
 => process_output
 => n_tty_write
 => tty_write
 => vfs_write
 => sys_write
 => system_call_fastpath”h]”hX  # tracer: preemptirqsoff
#
# preemptirqsoff latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 161 us, #339/339, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: ls-2269 (uid:0 nice:0 policy:0 rt_prio:0)
#    -----------------
#  => started at: schedule
#  => ended at:   mutex_unlock
#
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
kworker/-59      3...1    0us : __schedule <-schedule
kworker/-59      3d..1    0us : rcu_preempt_qs <-rcu_note_context_switch
kworker/-59      3d..1    1us : add_preempt_count <-_raw_spin_lock_irq
kworker/-59      3d..2    1us : deactivate_task <-__schedule
kworker/-59      3d..2    1us : dequeue_task <-deactivate_task
kworker/-59      3d..2    2us : update_rq_clock <-dequeue_task
kworker/-59      3d..2    2us : dequeue_task_fair <-dequeue_task
kworker/-59      3d..2    2us : update_curr <-dequeue_task_fair
kworker/-59      3d..2    2us : update_min_vruntime <-update_curr
kworker/-59      3d..2    3us : cpuacct_charge <-update_curr
kworker/-59      3d..2    3us : __rcu_read_lock <-cpuacct_charge
kworker/-59      3d..2    3us : __rcu_read_unlock <-cpuacct_charge
kworker/-59      3d..2    3us : update_cfs_rq_blocked_load <-dequeue_task_fair
kworker/-59      3d..2    4us : clear_buddies <-dequeue_task_fair
kworker/-59      3d..2    4us : account_entity_dequeue <-dequeue_task_fair
kworker/-59      3d..2    4us : update_min_vruntime <-dequeue_task_fair
kworker/-59      3d..2    4us : update_cfs_shares <-dequeue_task_fair
kworker/-59      3d..2    5us : hrtick_update <-dequeue_task_fair
kworker/-59      3d..2    5us : wq_worker_sleeping <-__schedule
kworker/-59      3d..2    5us : kthread_data <-wq_worker_sleeping
kworker/-59      3d..2    5us : put_prev_task_fair <-__schedule
kworker/-59      3d..2    6us : pick_next_task_fair <-pick_next_task
kworker/-59      3d..2    6us : clear_buddies <-pick_next_task_fair
kworker/-59      3d..2    6us : set_next_entity <-pick_next_task_fair
kworker/-59      3d..2    6us : update_stats_wait_end <-set_next_entity
      ls-2269    3d..2    7us : finish_task_switch <-__schedule
      ls-2269    3d..2    7us : _raw_spin_unlock_irq <-finish_task_switch
      ls-2269    3d..2    8us : do_IRQ <-ret_from_intr
      ls-2269    3d..2    8us : irq_enter <-do_IRQ
      ls-2269    3d..2    8us : rcu_irq_enter <-irq_enter
      ls-2269    3d..2    9us : add_preempt_count <-irq_enter
      ls-2269    3d.h2    9us : exit_idle <-do_IRQ
[...]
      ls-2269    3d.h3   20us : sub_preempt_count <-_raw_spin_unlock
      ls-2269    3d.h2   20us : irq_exit <-do_IRQ
      ls-2269    3d.h2   21us : sub_preempt_count <-irq_exit
      ls-2269    3d..3   21us : do_softirq <-irq_exit
      ls-2269    3d..3   21us : __do_softirq <-call_softirq
      ls-2269    3d..3   21us+: __local_bh_disable <-__do_softirq
      ls-2269    3d.s4   29us : sub_preempt_count <-_local_bh_enable_ip
      ls-2269    3d.s5   29us : sub_preempt_count <-_local_bh_enable_ip
      ls-2269    3d.s5   31us : do_IRQ <-ret_from_intr
      ls-2269    3d.s5   31us : irq_enter <-do_IRQ
      ls-2269    3d.s5   31us : rcu_irq_enter <-irq_enter
[...]
      ls-2269    3d.s5   31us : rcu_irq_enter <-irq_enter
      ls-2269    3d.s5   32us : add_preempt_count <-irq_enter
      ls-2269    3d.H5   32us : exit_idle <-do_IRQ
      ls-2269    3d.H5   32us : handle_irq <-do_IRQ
      ls-2269    3d.H5   32us : irq_to_desc <-handle_irq
      ls-2269    3d.H5   33us : handle_fasteoi_irq <-handle_irq
[...]
      ls-2269    3d.s5  158us : _raw_spin_unlock_irqrestore <-rtl8139_poll
      ls-2269    3d.s3  158us : net_rps_action_and_irq_enable.isra.65 <-net_rx_action
      ls-2269    3d.s3  159us : __local_bh_enable <-__do_softirq
      ls-2269    3d.s3  159us : sub_preempt_count <-__local_bh_enable
      ls-2269    3d..3  159us : idle_cpu <-irq_exit
      ls-2269    3d..3  159us : rcu_irq_exit <-irq_exit
      ls-2269    3d..3  160us : sub_preempt_count <-irq_exit
      ls-2269    3d...  161us : __mutex_unlock_slowpath <-mutex_unlock
      ls-2269    3d...  162us+: trace_hardirqs_on <-mutex_unlock
      ls-2269    3d...  186us : <stack trace>
 => __mutex_unlock_slowpath
 => mutex_unlock
 => process_output
 => n_tty_write
 => tty_write
 => vfs_write
 => sys_write
 => system_call_fastpath”…””}”hjæ%  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M]hjI%  hžhubh¸)”}”(hX  This is an interesting trace. It started with kworker running and
scheduling out and ls taking over. But as soon as ls released the
rq lock and enabled interrupts (but not preemption) an interrupt
triggered. When the interrupt finished, it started running softirqs.
But while the softirq was running, another interrupt triggered.
When an interrupt is running inside a softirq, the annotation is 'H'.”h]”hX“  This is an interesting trace. It started with kworker running and
scheduling out and ls taking over. But as soon as ls released the
rq lock and enabled interrupts (but not preemption) an interrupt
triggered. When the interrupt finished, it started running softirqs.
But while the softirq was running, another interrupt triggered.
When an interrupt is running inside a softirq, the annotation is â€˜Hâ€™.”…””}”(hjô%  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¸hjI%  hžhubeh}”(h]”Œpreemptirqsoff”ah ]”h"]”Œpreemptirqsoff”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Mþubh¢)”}”(hhh]”(h§)”}”(hŒwakeup”h]”hŒwakeup”…””}”(hj&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj
&  hžhhŸh¶h MÁubh¸)”}”(hŒÛOne common case that people are interested in tracing is the
time it takes for a task that is woken to actually wake up.
Now for non Real-Time tasks, this can be arbitrary. But tracing
it nonetheless can be interesting.”h]”hŒÛOne common case that people are interested in tracing is the
time it takes for a task that is woken to actually wake up.
Now for non Real-Time tasks, this can be arbitrary. But tracing
it nonetheless can be interesting.”…””}”(hj&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÃhj
&  hžhubh¸)”}”(hŒWithout function tracing::”h]”hŒWithout function tracing:”…””}”(hj)&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÈhj
&  hžhubj   )”}”(hXG  # echo 0 > options/function-trace
# echo wakeup > current_tracer
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# chrt -f 5 sleep 1
# echo 0 > tracing_on
# cat trace
# tracer: wakeup
#
# wakeup latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 15 us, #4/4, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: kworker/3:1H-312 (uid:0 nice:-20 policy:0 rt_prio:0)
#    -----------------
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
  <idle>-0       3dNs7    0us :      0:120:R   + [003]   312:100:R kworker/3:1H
  <idle>-0       3dNs7    1us+: ttwu_do_activate.constprop.87 <-try_to_wake_up
  <idle>-0       3d..3   15us : __schedule <-schedule
  <idle>-0       3d..3   15us :      0:120:R ==> [003]   312:100:R kworker/3:1H”h]”hXG  # echo 0 > options/function-trace
# echo wakeup > current_tracer
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# chrt -f 5 sleep 1
# echo 0 > tracing_on
# cat trace
# tracer: wakeup
#
# wakeup latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 15 us, #4/4, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: kworker/3:1H-312 (uid:0 nice:-20 policy:0 rt_prio:0)
#    -----------------
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
  <idle>-0       3dNs7    0us :      0:120:R   + [003]   312:100:R kworker/3:1H
  <idle>-0       3dNs7    1us+: ttwu_do_activate.constprop.87 <-try_to_wake_up
  <idle>-0       3d..3   15us : __schedule <-schedule
  <idle>-0       3d..3   15us :      0:120:R ==> [003]   312:100:R kworker/3:1H”…””}”hj7&  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MÊhj
&  hžhubh¸)”}”(hŒûThe tracer only traces the highest priority task in the system
to avoid tracing the normal circumstances. Here we see that
the kworker with a nice priority of -20 (not very nice), took
just 15 microseconds from the time it woke up, to the time it
ran.”h]”hŒûThe tracer only traces the highest priority task in the system
to avoid tracing the normal circumstances. Here we see that
the kworker with a nice priority of -20 (not very nice), took
just 15 microseconds from the time it woke up, to the time it
ran.”…””}”(hjE&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mçhj
&  hžhubh¸)”}”(hŒqNon Real-Time tasks are not that interesting. A more interesting
trace is to concentrate only on Real-Time tasks.”h]”hŒqNon Real-Time tasks are not that interesting. A more interesting
trace is to concentrate only on Real-Time tasks.”…””}”(hjS&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Míhj
&  hžhubeh}”(h]”Œwakeup”ah ]”h"]”Œwakeup”ah$]”h&]”uh1h¡hh£hžhhŸh¶h MÁubh¢)”}”(hhh]”(h§)”}”(hŒ	wakeup_rt”h]”hŒ	wakeup_rt”…””}”(hjl&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hji&  hžhhŸh¶h Mñubh¸)”}”(hX»  In a Real-Time environment it is very important to know the
wakeup time it takes for the highest priority task that is woken
up to the time that it executes. This is also known as "schedule
latency". I stress the point that this is about RT tasks. It is
also important to know the scheduling latency of non-RT tasks,
but the average schedule latency is better for non-RT tasks.
Tools like LatencyTop are more appropriate for such
measurements.”h]”hX¿  In a Real-Time environment it is very important to know the
wakeup time it takes for the highest priority task that is woken
up to the time that it executes. This is also known as â€œschedule
latencyâ€. I stress the point that this is about RT tasks. It is
also important to know the scheduling latency of non-RT tasks,
but the average schedule latency is better for non-RT tasks.
Tools like LatencyTop are more appropriate for such
measurements.”…””}”(hjz&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Móhji&  hžhubh¸)”}”(hX`  Real-Time environments are interested in the worst case latency.
That is the longest latency it takes for something to happen,
and not the average. We can have a very fast scheduler that may
only have a large latency once in a while, but that would not
work well with Real-Time tasks.  The wakeup_rt tracer was designed
to record the worst case wakeups of RT tasks. Non-RT tasks are
not recorded because the tracer only records one worst case and
tracing non-RT tasks that are unpredictable will overwrite the
worst case latency of RT tasks (just run the normal wakeup
tracer for a while to see that effect).”h]”hX`  Real-Time environments are interested in the worst case latency.
That is the longest latency it takes for something to happen,
and not the average. We can have a very fast scheduler that may
only have a large latency once in a while, but that would not
work well with Real-Time tasks.  The wakeup_rt tracer was designed
to record the worst case wakeups of RT tasks. Non-RT tasks are
not recorded because the tracer only records one worst case and
tracing non-RT tasks that are unpredictable will overwrite the
worst case latency of RT tasks (just run the normal wakeup
tracer for a while to see that effect).”…””}”(hjˆ&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mühji&  hžhubh¸)”}”(hŒåSince this tracer only deals with RT tasks, we will run this
slightly differently than we did with the previous tracers.
Instead of performing an 'ls', we will run 'sleep 1' under
'chrt' which changes the priority of the task.
::”h]”hŒîSince this tracer only deals with RT tasks, we will run this
slightly differently than we did with the previous tracers.
Instead of performing an â€˜lsâ€™, we will run â€˜sleep 1â€™ under
â€˜chrtâ€™ which changes the priority of the task.”…””}”(hj–&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhji&  hžhubj   )”}”(hXL  # echo 0 > options/function-trace
# echo wakeup_rt > current_tracer
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# chrt -f 5 sleep 1
# echo 0 > tracing_on
# cat trace
# tracer: wakeup
#
# tracer: wakeup_rt
#
# wakeup_rt latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 5 us, #4/4, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: sleep-2389 (uid:0 nice:0 policy:1 rt_prio:5)
#    -----------------
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
  <idle>-0       3d.h4    0us :      0:120:R   + [003]  2389: 94:R sleep
  <idle>-0       3d.h4    1us+: ttwu_do_activate.constprop.87 <-try_to_wake_up
  <idle>-0       3d..3    5us : __schedule <-schedule
  <idle>-0       3d..3    5us :      0:120:R ==> [003]  2389: 94:R sleep”•‡     h]”hXL  # echo 0 > options/function-trace
# echo wakeup_rt > current_tracer
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# chrt -f 5 sleep 1
# echo 0 > tracing_on
# cat trace
# tracer: wakeup
#
# tracer: wakeup_rt
#
# wakeup_rt latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 5 us, #4/4, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: sleep-2389 (uid:0 nice:0 policy:1 rt_prio:5)
#    -----------------
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
  <idle>-0       3d.h4    0us :      0:120:R   + [003]  2389: 94:R sleep
  <idle>-0       3d.h4    1us+: ttwu_do_activate.constprop.87 <-try_to_wake_up
  <idle>-0       3d..3    5us : __schedule <-schedule
  <idle>-0       3d..3    5us :      0:120:R ==> [003]  2389: 94:R sleep”…””}”hj¤&  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhji&  hžhubh¸)”}”(hX7  Running this on an idle system, we see that it only took 5 microseconds
to perform the task switch.  Note, since the trace point in the schedule
is before the actual "switch", we stop the tracing when the recorded task
is about to schedule in. This may change if we add a new marker at the
end of the scheduler.”h]”hX;  Running this on an idle system, we see that it only took 5 microseconds
to perform the task switch.  Note, since the trace point in the schedule
is before the actual â€œswitchâ€, we stop the tracing when the recorded task
is about to schedule in. This may change if we add a new marker at the
end of the scheduler.”…””}”(hj²&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M-hji&  hžhubh¸)”}”(hŒ×Notice that the recorded task is 'sleep' with the PID of 2389
and it has an rt_prio of 5. This priority is user-space priority
and not the internal kernel priority. The policy is 1 for
SCHED_FIFO and 2 for SCHED_RR.”h]”hŒÛNotice that the recorded task is â€˜sleepâ€™ with the PID of 2389
and it has an rt_prio of 5. This priority is user-space priority
and not the internal kernel priority. The policy is 1 for
SCHED_FIFO and 2 for SCHED_RR.”…””}”(hjÀ&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M3hji&  hžhubh¸)”}”(hŒGNote, that the trace data shows the internal priority (99 - rtprio).
::”h]”hŒDNote, that the trace data shows the internal priority (99 - rtprio).”…””}”(hjÎ&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M8hji&  hžhubj   )”}”(hŒF<idle>-0       3d..3    5us :      0:120:R ==> [003]  2389: 94:R sleep”h]”hŒF<idle>-0       3d..3    5us :      0:120:R ==> [003]  2389: 94:R sleep”…””}”hjÜ&  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M;hji&  hžhubh¸)”}”(hŒóThe 0:120:R means idle was running with a nice priority of 0 (120 - 120)
and in the running state 'R'. The sleep task was scheduled in with
2389: 94:R. That is the priority is the kernel rtprio (99 - 5 = 94)
and it too is in the running state.”h]”hŒ÷The 0:120:R means idle was running with a nice priority of 0 (120 - 120)
and in the running state â€˜Râ€™. The sleep task was scheduled in with
2389: 94:R. That is the priority is the kernel rtprio (99 - 5 = 94)
and it too is in the running state.”…””}”(hjê&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M=hji&  hžhubh¸)”}”(hŒ8Doing the same with chrt -r 5 and function-trace set.
::”h]”hŒ5Doing the same with chrt -r 5 and function-trace set.”…””}”(hjø&  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MBhji&  hžhubj   )”}”(hXP  echo 1 > options/function-trace

# tracer: wakeup_rt
#
# wakeup_rt latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 29 us, #85/85, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: sleep-2448 (uid:0 nice:0 policy:1 rt_prio:5)
#    -----------------
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
  <idle>-0       3d.h4    1us+:      0:120:R   + [003]  2448: 94:R sleep
  <idle>-0       3d.h4    2us : ttwu_do_activate.constprop.87 <-try_to_wake_up
  <idle>-0       3d.h3    3us : check_preempt_curr <-ttwu_do_wakeup
  <idle>-0       3d.h3    3us : resched_curr <-check_preempt_curr
  <idle>-0       3dNh3    4us : task_woken_rt <-ttwu_do_wakeup
  <idle>-0       3dNh3    4us : _raw_spin_unlock <-try_to_wake_up
  <idle>-0       3dNh3    4us : sub_preempt_count <-_raw_spin_unlock
  <idle>-0       3dNh2    5us : ttwu_stat <-try_to_wake_up
  <idle>-0       3dNh2    5us : _raw_spin_unlock_irqrestore <-try_to_wake_up
  <idle>-0       3dNh2    6us : sub_preempt_count <-_raw_spin_unlock_irqrestore
  <idle>-0       3dNh1    6us : _raw_spin_lock <-__run_hrtimer
  <idle>-0       3dNh1    6us : add_preempt_count <-_raw_spin_lock
  <idle>-0       3dNh2    7us : _raw_spin_unlock <-hrtimer_interrupt
  <idle>-0       3dNh2    7us : sub_preempt_count <-_raw_spin_unlock
  <idle>-0       3dNh1    7us : tick_program_event <-hrtimer_interrupt
  <idle>-0       3dNh1    7us : clockevents_program_event <-tick_program_event
  <idle>-0       3dNh1    8us : ktime_get <-clockevents_program_event
  <idle>-0       3dNh1    8us : lapic_next_event <-clockevents_program_event
  <idle>-0       3dNh1    8us : irq_exit <-smp_apic_timer_interrupt
  <idle>-0       3dNh1    9us : sub_preempt_count <-irq_exit
  <idle>-0       3dN.2    9us : idle_cpu <-irq_exit
  <idle>-0       3dN.2    9us : rcu_irq_exit <-irq_exit
  <idle>-0       3dN.2   10us : rcu_eqs_enter_common.isra.45 <-rcu_irq_exit
  <idle>-0       3dN.2   10us : sub_preempt_count <-irq_exit
  <idle>-0       3.N.1   11us : rcu_idle_exit <-cpu_idle
  <idle>-0       3dN.1   11us : rcu_eqs_exit_common.isra.43 <-rcu_idle_exit
  <idle>-0       3.N.1   11us : tick_nohz_idle_exit <-cpu_idle
  <idle>-0       3dN.1   12us : menu_hrtimer_cancel <-tick_nohz_idle_exit
  <idle>-0       3dN.1   12us : ktime_get <-tick_nohz_idle_exit
  <idle>-0       3dN.1   12us : tick_do_update_jiffies64 <-tick_nohz_idle_exit
  <idle>-0       3dN.1   13us : cpu_load_update_nohz <-tick_nohz_idle_exit
  <idle>-0       3dN.1   13us : _raw_spin_lock <-cpu_load_update_nohz
  <idle>-0       3dN.1   13us : add_preempt_count <-_raw_spin_lock
  <idle>-0       3dN.2   13us : __cpu_load_update <-cpu_load_update_nohz
  <idle>-0       3dN.2   14us : sched_avg_update <-__cpu_load_update
  <idle>-0       3dN.2   14us : _raw_spin_unlock <-cpu_load_update_nohz
  <idle>-0       3dN.2   14us : sub_preempt_count <-_raw_spin_unlock
  <idle>-0       3dN.1   15us : calc_load_nohz_stop <-tick_nohz_idle_exit
  <idle>-0       3dN.1   15us : touch_softlockup_watchdog <-tick_nohz_idle_exit
  <idle>-0       3dN.1   15us : hrtimer_cancel <-tick_nohz_idle_exit
  <idle>-0       3dN.1   15us : hrtimer_try_to_cancel <-hrtimer_cancel
  <idle>-0       3dN.1   16us : lock_hrtimer_base.isra.18 <-hrtimer_try_to_cancel
  <idle>-0       3dN.1   16us : _raw_spin_lock_irqsave <-lock_hrtimer_base.isra.18
  <idle>-0       3dN.1   16us : add_preempt_count <-_raw_spin_lock_irqsave
  <idle>-0       3dN.2   17us : __remove_hrtimer <-remove_hrtimer.part.16
  <idle>-0       3dN.2   17us : hrtimer_force_reprogram <-__remove_hrtimer
  <idle>-0       3dN.2   17us : tick_program_event <-hrtimer_force_reprogram
  <idle>-0       3dN.2   18us : clockevents_program_event <-tick_program_event
  <idle>-0       3dN.2   18us : ktime_get <-clockevents_program_event
  <idle>-0       3dN.2   18us : lapic_next_event <-clockevents_program_event
  <idle>-0       3dN.2   19us : _raw_spin_unlock_irqrestore <-hrtimer_try_to_cancel
  <idle>-0       3dN.2   19us : sub_preempt_count <-_raw_spin_unlock_irqrestore
  <idle>-0       3dN.1   19us : hrtimer_forward <-tick_nohz_idle_exit
  <idle>-0       3dN.1   20us : ktime_add_safe <-hrtimer_forward
  <idle>-0       3dN.1   20us : ktime_add_safe <-hrtimer_forward
  <idle>-0       3dN.1   20us : hrtimer_start_range_ns <-hrtimer_start_expires.constprop.11
  <idle>-0       3dN.1   20us : __hrtimer_start_range_ns <-hrtimer_start_range_ns
  <idle>-0       3dN.1   21us : lock_hrtimer_base.isra.18 <-__hrtimer_start_range_ns
  <idle>-0       3dN.1   21us : _raw_spin_lock_irqsave <-lock_hrtimer_base.isra.18
  <idle>-0       3dN.1   21us : add_preempt_count <-_raw_spin_lock_irqsave
  <idle>-0       3dN.2   22us : ktime_add_safe <-__hrtimer_start_range_ns
  <idle>-0       3dN.2   22us : enqueue_hrtimer <-__hrtimer_start_range_ns
  <idle>-0       3dN.2   22us : tick_program_event <-__hrtimer_start_range_ns
  <idle>-0       3dN.2   23us : clockevents_program_event <-tick_program_event
  <idle>-0       3dN.2   23us : ktime_get <-clockevents_program_event
  <idle>-0       3dN.2   23us : lapic_next_event <-clockevents_program_event
  <idle>-0       3dN.2   24us : _raw_spin_unlock_irqrestore <-__hrtimer_start_range_ns
  <idle>-0       3dN.2   24us : sub_preempt_count <-_raw_spin_unlock_irqrestore
  <idle>-0       3dN.1   24us : account_idle_ticks <-tick_nohz_idle_exit
  <idle>-0       3dN.1   24us : account_idle_time <-account_idle_ticks
  <idle>-0       3.N.1   25us : sub_preempt_count <-cpu_idle
  <idle>-0       3.N..   25us : schedule <-cpu_idle
  <idle>-0       3.N..   25us : __schedule <-preempt_schedule
  <idle>-0       3.N..   26us : add_preempt_count <-__schedule
  <idle>-0       3.N.1   26us : rcu_note_context_switch <-__schedule
  <idle>-0       3.N.1   26us : rcu_sched_qs <-rcu_note_context_switch
  <idle>-0       3dN.1   27us : rcu_preempt_qs <-rcu_note_context_switch
  <idle>-0       3.N.1   27us : _raw_spin_lock_irq <-__schedule
  <idle>-0       3dN.1   27us : add_preempt_count <-_raw_spin_lock_irq
  <idle>-0       3dN.2   28us : put_prev_task_idle <-__schedule
  <idle>-0       3dN.2   28us : pick_next_task_stop <-pick_next_task
  <idle>-0       3dN.2   28us : pick_next_task_rt <-pick_next_task
  <idle>-0       3dN.2   29us : dequeue_pushable_task <-pick_next_task_rt
  <idle>-0       3d..3   29us : __schedule <-preempt_schedule
  <idle>-0       3d..3   30us :      0:120:R ==> [003]  2448: 94:R sleep”h]”hXP  echo 1 > options/function-trace

# tracer: wakeup_rt
#
# wakeup_rt latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 29 us, #85/85, CPU#3 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: sleep-2448 (uid:0 nice:0 policy:1 rt_prio:5)
#    -----------------
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
  <idle>-0       3d.h4    1us+:      0:120:R   + [003]  2448: 94:R sleep
  <idle>-0       3d.h4    2us : ttwu_do_activate.constprop.87 <-try_to_wake_up
  <idle>-0       3d.h3    3us : check_preempt_curr <-ttwu_do_wakeup
  <idle>-0       3d.h3    3us : resched_curr <-check_preempt_curr
  <idle>-0       3dNh3    4us : task_woken_rt <-ttwu_do_wakeup
  <idle>-0       3dNh3    4us : _raw_spin_unlock <-try_to_wake_up
  <idle>-0       3dNh3    4us : sub_preempt_count <-_raw_spin_unlock
  <idle>-0       3dNh2    5us : ttwu_stat <-try_to_wake_up
  <idle>-0       3dNh2    5us : _raw_spin_unlock_irqrestore <-try_to_wake_up
  <idle>-0       3dNh2    6us : sub_preempt_count <-_raw_spin_unlock_irqrestore
  <idle>-0       3dNh1    6us : _raw_spin_lock <-__run_hrtimer
  <idle>-0       3dNh1    6us : add_preempt_count <-_raw_spin_lock
  <idle>-0       3dNh2    7us : _raw_spin_unlock <-hrtimer_interrupt
  <idle>-0       3dNh2    7us : sub_preempt_count <-_raw_spin_unlock
  <idle>-0       3dNh1    7us : tick_program_event <-hrtimer_interrupt
  <idle>-0       3dNh1    7us : clockevents_program_event <-tick_program_event
  <idle>-0       3dNh1    8us : ktime_get <-clockevents_program_event
  <idle>-0       3dNh1    8us : lapic_next_event <-clockevents_program_event
  <idle>-0       3dNh1    8us : irq_exit <-smp_apic_timer_interrupt
  <idle>-0       3dNh1    9us : sub_preempt_count <-irq_exit
  <idle>-0       3dN.2    9us : idle_cpu <-irq_exit
  <idle>-0       3dN.2    9us : rcu_irq_exit <-irq_exit
  <idle>-0       3dN.2   10us : rcu_eqs_enter_common.isra.45 <-rcu_irq_exit
  <idle>-0       3dN.2   10us : sub_preempt_count <-irq_exit
  <idle>-0       3.N.1   11us : rcu_idle_exit <-cpu_idle
  <idle>-0       3dN.1   11us : rcu_eqs_exit_common.isra.43 <-rcu_idle_exit
  <idle>-0       3.N.1   11us : tick_nohz_idle_exit <-cpu_idle
  <idle>-0       3dN.1   12us : menu_hrtimer_cancel <-tick_nohz_idle_exit
  <idle>-0       3dN.1   12us : ktime_get <-tick_nohz_idle_exit
  <idle>-0       3dN.1   12us : tick_do_update_jiffies64 <-tick_nohz_idle_exit
  <idle>-0       3dN.1   13us : cpu_load_update_nohz <-tick_nohz_idle_exit
  <idle>-0       3dN.1   13us : _raw_spin_lock <-cpu_load_update_nohz
  <idle>-0       3dN.1   13us : add_preempt_count <-_raw_spin_lock
  <idle>-0       3dN.2   13us : __cpu_load_update <-cpu_load_update_nohz
  <idle>-0       3dN.2   14us : sched_avg_update <-__cpu_load_update
  <idle>-0       3dN.2   14us : _raw_spin_unlock <-cpu_load_update_nohz
  <idle>-0       3dN.2   14us : sub_preempt_count <-_raw_spin_unlock
  <idle>-0       3dN.1   15us : calc_load_nohz_stop <-tick_nohz_idle_exit
  <idle>-0       3dN.1   15us : touch_softlockup_watchdog <-tick_nohz_idle_exit
  <idle>-0       3dN.1   15us : hrtimer_cancel <-tick_nohz_idle_exit
  <idle>-0       3dN.1   15us : hrtimer_try_to_cancel <-hrtimer_cancel
  <idle>-0       3dN.1   16us : lock_hrtimer_base.isra.18 <-hrtimer_try_to_cancel
  <idle>-0       3dN.1   16us : _raw_spin_lock_irqsave <-lock_hrtimer_base.isra.18
  <idle>-0       3dN.1   16us : add_preempt_count <-_raw_spin_lock_irqsave
  <idle>-0       3dN.2   17us : __remove_hrtimer <-remove_hrtimer.part.16
  <idle>-0       3dN.2   17us : hrtimer_force_reprogram <-__remove_hrtimer
  <idle>-0       3dN.2   17us : tick_program_event <-hrtimer_force_reprogram
  <idle>-0       3dN.2   18us : clockevents_program_event <-tick_program_event
  <idle>-0       3dN.2   18us : ktime_get <-clockevents_program_event
  <idle>-0       3dN.2   18us : lapic_next_event <-clockevents_program_event
  <idle>-0       3dN.2   19us : _raw_spin_unlock_irqrestore <-hrtimer_try_to_cancel
  <idle>-0       3dN.2   19us : sub_preempt_count <-_raw_spin_unlock_irqrestore
  <idle>-0       3dN.1   19us : hrtimer_forward <-tick_nohz_idle_exit
  <idle>-0       3dN.1   20us : ktime_add_safe <-hrtimer_forward
  <idle>-0       3dN.1   20us : ktime_add_safe <-hrtimer_forward
  <idle>-0       3dN.1   20us : hrtimer_start_range_ns <-hrtimer_start_expires.constprop.11
  <idle>-0       3dN.1   20us : __hrtimer_start_range_ns <-hrtimer_start_range_ns
  <idle>-0       3dN.1   21us : lock_hrtimer_base.isra.18 <-__hrtimer_start_range_ns
  <idle>-0       3dN.1   21us : _raw_spin_lock_irqsave <-lock_hrtimer_base.isra.18
  <idle>-0       3dN.1   21us : add_preempt_count <-_raw_spin_lock_irqsave
  <idle>-0       3dN.2   22us : ktime_add_safe <-__hrtimer_start_range_ns
  <idle>-0       3dN.2   22us : enqueue_hrtimer <-__hrtimer_start_range_ns
  <idle>-0       3dN.2   22us : tick_program_event <-__hrtimer_start_range_ns
  <idle>-0       3dN.2   23us : clockevents_program_event <-tick_program_event
  <idle>-0       3dN.2   23us : ktime_get <-clockevents_program_event
  <idle>-0       3dN.2   23us : lapic_next_event <-clockevents_program_event
  <idle>-0       3dN.2   24us : _raw_spin_unlock_irqrestore <-__hrtimer_start_range_ns
  <idle>-0       3dN.2   24us : sub_preempt_count <-_raw_spin_unlock_irqrestore
  <idle>-0       3dN.1   24us : account_idle_ticks <-tick_nohz_idle_exit
  <idle>-0       3dN.1   24us : account_idle_time <-account_idle_ticks
  <idle>-0       3.N.1   25us : sub_preempt_count <-cpu_idle
  <idle>-0       3.N..   25us : schedule <-cpu_idle
  <idle>-0       3.N..   25us : __schedule <-preempt_schedule
  <idle>-0       3.N..   26us : add_preempt_count <-__schedule
  <idle>-0       3.N.1   26us : rcu_note_context_switch <-__schedule
  <idle>-0       3.N.1   26us : rcu_sched_qs <-rcu_note_context_switch
  <idle>-0       3dN.1   27us : rcu_preempt_qs <-rcu_note_context_switch
  <idle>-0       3.N.1   27us : _raw_spin_lock_irq <-__schedule
  <idle>-0       3dN.1   27us : add_preempt_count <-_raw_spin_lock_irq
  <idle>-0       3dN.2   28us : put_prev_task_idle <-__schedule
  <idle>-0       3dN.2   28us : pick_next_task_stop <-pick_next_task
  <idle>-0       3dN.2   28us : pick_next_task_rt <-pick_next_task
  <idle>-0       3dN.2   29us : dequeue_pushable_task <-pick_next_task_rt
  <idle>-0       3d..3   29us : __schedule <-preempt_schedule
  <idle>-0       3d..3   30us :      0:120:R ==> [003]  2448: 94:R sleep”…””}”hj'  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MEhji&  hžhubh¸)”}”(hŒcThis isn't that big of a trace, even with function tracing enabled,
so I included the entire trace.”h]”hŒeThis isnâ€™t that big of a trace, even with function tracing enabled,
so I included the entire trace.”…””}”(hj'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M®hji&  hžhubh¸)”}”(hŒ½The interrupt went off while when the system was idle. Somewhere
before task_woken_rt() was called, the NEED_RESCHED flag was set,
this is indicated by the first occurrence of the 'N' flag.”h]”hŒÁThe interrupt went off while when the system was idle. Somewhere
before task_woken_rt() was called, the NEED_RESCHED flag was set,
this is indicated by the first occurrence of the â€˜Nâ€™ flag.”…””}”(hj"'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M±hji&  hžhubeh}”(h]”Œ	wakeup-rt”ah ]”h"]”Œ	wakeup_rt”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Mñubh¢)”}”(hhh]”(h§)”}”(hŒLatency tracing and events”h]”hŒLatency tracing and events”…””}”(hj;'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj8'  hžhhŸh¶h M¶ubh¸)”}”(hŒÊAs function tracing can induce a much larger latency, but without
seeing what happens within the latency it is hard to know what
caused it. There is a middle ground, and that is with enabling
events.
::”h]”hŒÇAs function tracing can induce a much larger latency, but without
seeing what happens within the latency it is hard to know what
caused it. There is a middle ground, and that is with enabling
events.”…””}”(hjI'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M·hj8'  hžhubj   )”}”(hXä  # echo 0 > options/function-trace
# echo wakeup_rt > current_tracer
# echo 1 > events/enable
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# chrt -f 5 sleep 1
# echo 0 > tracing_on
# cat trace
# tracer: wakeup_rt
#
# wakeup_rt latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 6 us, #12/12, CPU#2 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: sleep-5882 (uid:0 nice:0 policy:1 rt_prio:5)
#    -----------------
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
  <idle>-0       2d.h4    0us :      0:120:R   + [002]  5882: 94:R sleep
  <idle>-0       2d.h4    0us : ttwu_do_activate.constprop.87 <-try_to_wake_up
  <idle>-0       2d.h4    1us : sched_wakeup: comm=sleep pid=5882 prio=94 success=1 target_cpu=002
  <idle>-0       2dNh2    1us : hrtimer_expire_exit: hrtimer=ffff88007796feb8
  <idle>-0       2.N.2    2us : power_end: cpu_id=2
  <idle>-0       2.N.2    3us : cpu_idle: state=4294967295 cpu_id=2
  <idle>-0       2dN.3    4us : hrtimer_cancel: hrtimer=ffff88007d50d5e0
  <idle>-0       2dN.3    4us : hrtimer_start: hrtimer=ffff88007d50d5e0 function=tick_sched_timer expires=34311211000000 softexpires=34311211000000
  <idle>-0       2.N.2    5us : rcu_utilization: Start context switch
  <idle>-0       2.N.2    5us : rcu_utilization: End context switch
  <idle>-0       2d..3    6us : __schedule <-schedule
  <idle>-0       2d..3    6us :      0:120:R ==> [002]  5882: 94:R sleep”h]”hXä  # echo 0 > options/function-trace
# echo wakeup_rt > current_tracer
# echo 1 > events/enable
# echo 1 > tracing_on
# echo 0 > tracing_max_latency
# chrt -f 5 sleep 1
# echo 0 > tracing_on
# cat trace
# tracer: wakeup_rt
#
# wakeup_rt latency trace v1.1.5 on 3.8.0-test+
# --------------------------------------------------------------------
# latency: 6 us, #12/12, CPU#2 | (M:preempt VP:0, KP:0, SP:0 HP:0 #P:4)
#    -----------------
#    | task: sleep-5882 (uid:0 nice:0 policy:1 rt_prio:5)
#    -----------------
#
#                  _------=> CPU#
#                 / _-----=> irqs-off
#                | / _----=> need-resched
#                || / _---=> hardirq/softirq
#                ||| / _--=> preempt-depth
#                |||| /     delay
#  cmd     pid   ||||| time  |   caller
#     \   /      |||||  \    |   /
  <idle>-0       2d.h4    0us :      0:120:R   + [002]  5882: 94:R sleep
  <idle>-0       2d.h4    0us : ttwu_do_activate.constprop.87 <-try_to_wake_up
  <idle>-0       2d.h4    1us : sched_wakeup: comm=sleep pid=5882 prio=94 success=1 target_cpu=002
  <idle>-0       2dNh2    1us : hrtimer_expire_exit: hrtimer=ffff88007796feb8
  <idle>-0       2.N.2    2us : power_end: cpu_id=2
  <idle>-0       2.N.2    3us : cpu_idle: state=4294967295 cpu_id=2
  <idle>-0       2dN.3    4us : hrtimer_cancel: hrtimer=ffff88007d50d5e0
  <idle>-0       2dN.3    4us : hrtimer_start: hrtimer=ffff88007d50d5e0 function=tick_sched_timer expires=34311211000000 softexpires=34311211000000
  <idle>-0       2.N.2    5us : rcu_utilization: Start context switch
  <idle>-0       2.N.2    5us : rcu_utilization: End context switch
  <idle>-0       2d..3    6us : __schedule <-schedule
  <idle>-0       2d..3    6us :      0:120:R ==> [002]  5882: 94:R sleep”…””}”hjW'  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M½hj8'  hžhubeh}”(h]”Œlatency-tracing-and-events”ah ]”h"]”Œlatency tracing and events”ah$]”h&]”uh1h¡hh£hžhhŸh¶h M¶ubh¢)”}”(hhh]”(h§)”}”(hŒHardware Latency Detector”h]”hŒHardware Latency Detector”…””}”(hjp'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjm'  hžhhŸh¶h Måubh¸)”}”(hŒIThe hardware latency detector is executed by enabling the "hwlat" tracer.”h]”hŒMThe hardware latency detector is executed by enabling the â€œhwlatâ€ tracer.”…””}”(hj~'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mçhjm'  hžhubh¸)”}”(hŒ‹NOTE, this tracer will affect the performance of the system as it will
periodically make a CPU constantly busy with interrupts disabled.
::”h]”hŒˆNOTE, this tracer will affect the performance of the system as it will
periodically make a CPU constantly busy with interrupts disabled.”…””}”(hjŒ'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Méhjm'  hžhubj   )”}”(hX}  # echo hwlat > current_tracer
# sleep 100
# cat trace
# tracer: hwlat
#
# entries-in-buffer/entries-written: 13/13   #P:8
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
           <...>-1729  [001] d...   678.473449: #1     inner/outer(us):   11/12    ts:1581527483.343962693 count:6
           <...>-1729  [004] d...   689.556542: #2     inner/outer(us):   16/9     ts:1581527494.889008092 count:1
           <...>-1729  [005] d...   714.756290: #3     inner/outer(us):   16/16    ts:1581527519.678961629 count:5
           <...>-1729  [001] d...   718.788247: #4     inner/outer(us):    9/17    ts:1581527523.889012713 count:1
           <...>-1729  [002] d...   719.796341: #5     inner/outer(us):   13/9     ts:1581527524.912872606 count:1
           <...>-1729  [006] d...   844.787091: #6     inner/outer(us):    9/12    ts:1581527649.889048502 count:2
           <...>-1729  [003] d...   849.827033: #7     inner/outer(us):   18/9     ts:1581527654.889013793 count:1
           <...>-1729  [007] d...   853.859002: #8     inner/outer(us):    9/12    ts:1581527658.889065736 count:1
           <...>-1729  [001] d...   855.874978: #9     inner/outer(us):    9/11    ts:1581527660.861991877 count:1
           <...>-1729  [001] d...   863.938932: #10    inner/outer(us):    9/11    ts:1581527668.970010500 count:1 nmi-total:7 nmi-count:1
           <...>-1729  [007] d...   878.050780: #11    inner/outer(us):    9/12    ts:1581527683.385002600 count:1 nmi-total:5 nmi-count:1
           <...>-1729  [007] d...   886.114702: #12    inner/outer(us):    9/12    ts:1581527691.385001600 count:1”h]”hX}  # echo hwlat > current_tracer
# sleep 100
# cat trace
# tracer: hwlat
#
# entries-in-buffer/entries-written: 13/13   #P:8
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
           <...>-1729  [001] d...   678.473449: #1     inner/outer(us):   11/12    ts:1581527483.343962693 count:6
           <...>-1729  [004] d...   689.556542: #2     inner/outer(us):   16/9     ts:1581527494.889008092 count:1
           <...>-1729  [005] d...   714.756290: #3     inner/outer(us):   16/16    ts:1581527519.678961629 count:5
           <...>-1729  [001] d...   718.788247: #4     inner/outer(us):    9/17    ts:1581527523.889012713 count:1
           <...>-1729  [002] d...   719.796341: #5     inner/outer(us):   13/9     ts:1581527524.912872606 count:1
           <...>-1729  [006] d...   844.787091: #6     inner/outer(us):    9/12    ts:1581527649.889048502 count:2
           <...>-1729  [003] d...   849.827033: #7     inner/outer(us):   18/9     ts:1581527654.889013793 count:1
           <...>-1729  [007] d...   853.859002: #8     inner/outer(us):    9/12    ts:1581527658.889065736 count:1
           <...>-1729  [001] d...   855.874978: #9     inner/outer(us):    9/11    ts:1581527660.861991877 count:1
           <...>-1729  [001] d...   863.938932: #10    inner/outer(us):    9/11    ts:1581527668.970010500 count:1 nmi-total:7 nmi-count:1
           <...>-1729  [007] d...   878.050780: #11    inner/outer(us):    9/12    ts:1581527683.385002600 count:1 nmi-total:5 nmi-count:1
           <...>-1729  [007] d...   886.114702: #12    inner/outer(us):    9/12    ts:1581527691.385001600 count:1”…””}”hjš'  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Míhjm'  hžhubh¸)”}”(hŒ…The above output is somewhat the same in the header. All events will have
interrupts disabled 'd'. Under the FUNCTION title there is:”h]”hŒ‰The above output is somewhat the same in the header. All events will have
interrupts disabled â€˜dâ€™. Under the FUNCTION title there is:”…””}”(hj¨'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M		hjm'  hžhubjT  )”}”(hX¡  #1
       This is the count of events recorded that were greater than the
       tracing_threshold (See below).

inner/outer(us):   11/11

     This shows two numbers as "inner latency" and "outer latency". The test
     runs in a loop checking a timestamp twice. The latency detected within
     the two timestamps is the "inner latency" and the latency detected
     after the previous timestamp and the next timestamp in the loop is
     the "outer latency".

ts:1581527483.343962693

     The absolute timestamp that the first latency was recorded in the window.

count:6

     The number of times a latency was detected during the window.

nmi-total:7 nmi-count:1

     On architectures that support it, if an NMI comes in during the
     test, the time spent in NMI is reported in "nmi-total" (in
     microseconds).

     All architectures that have NMIs will show the "nmi-count" if an
     NMI comes in during the test.
”h]”(jT  )”}”(hhh]”jY  )”}”(hŒb#1
This is the count of events recorded that were greater than the
tracing_threshold (See below).
”h]”(j_  )”}”(hŒ#1”h]”hŒ#1”…””}”(hjÁ'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M	hj½'  ubjo  )”}”(hhh]”h¸)”}”(hŒ^This is the count of events recorded that were greater than the
tracing_threshold (See below).”h]”hŒ^This is the count of events recorded that were greater than the
tracing_threshold (See below).”…””}”(hjÒ'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M	hjÏ'  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj½'  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M	hjº'  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hj¶'  ubh¸)”}”(hŒinner/outer(us):   11/11”h]”hŒinner/outer(us):   11/11”…””}”(hjò'  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M	hj¶'  ubjT  )”}”(hX*  This shows two numbers as "inner latency" and "outer latency". The test
runs in a loop checking a timestamp twice. The latency detected within
the two timestamps is the "inner latency" and the latency detected
after the previous timestamp and the next timestamp in the loop is
the "outer latency".
”h]”h¸)”}”(hX)  This shows two numbers as "inner latency" and "outer latency". The test
runs in a loop checking a timestamp twice. The latency detected within
the two timestamps is the "inner latency" and the latency detected
after the previous timestamp and the next timestamp in the loop is
the "outer latency".”h]”hX9  This shows two numbers as â€œinner latencyâ€ and â€œouter latencyâ€. The test
runs in a loop checking a timestamp twice. The latency detected within
the two timestamps is the â€œinner latencyâ€ and the latency detected
after the previous timestamp and the next timestamp in the loop is
the â€œouter latencyâ€.”…””}”(hj(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M	hj (  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M	hj¶'  ubh¸)”}”(hŒts:1581527483.343962693”h]”hŒts:1581527483.343962693”…””}”(hj(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M	hj¶'  ubjT  )”}”(hŒJThe absolute timestamp that the first latency was recorded in the window.
”h]”h¸)”}”(hŒIThe absolute timestamp that the first latency was recorded in the window.”h]”hŒIThe absolute timestamp that the first latency was recorded in the window.”…””}”(hj*(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M	hj&(  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M	hj¶'  ubh¸)”}”(hŒcount:6”h]”hŒcount:6”…””}”(hj>(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M	hj¶'  ubjT  )”}”(hŒ>The number of times a latency was detected during the window.
”h]”h¸)”}”(hŒ=The number of times a latency was detected during the window.”h]”hŒ=The number of times a latency was detected during the window.”…””}”(hjP(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M	hjL(  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M	hj¶'  ubh¸)”}”(hŒnmi-total:7 nmi-count:1”h]”hŒnmi-total:7 nmi-count:1”…””}”(hjd(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M 	hj¶'  ubjT  )”}”(hŒêOn architectures that support it, if an NMI comes in during the
test, the time spent in NMI is reported in "nmi-total" (in
microseconds).

All architectures that have NMIs will show the "nmi-count" if an
NMI comes in during the test.
”h]”(h¸)”}”(hŒ‰On architectures that support it, if an NMI comes in during the
test, the time spent in NMI is reported in "nmi-total" (in
microseconds).”h]”hŒOn architectures that support it, if an NMI comes in during the
test, the time spent in NMI is reported in â€œnmi-totalâ€ (in
microseconds).”…””}”(hjv(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M"	hjr(  ubh¸)”}”(hŒ^All architectures that have NMIs will show the "nmi-count" if an
NMI comes in during the test.”h]”hŒbAll architectures that have NMIs will show the â€œnmi-countâ€ if an
NMI comes in during the test.”…””}”(hj„(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M&	hjr(  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M"	hj¶'  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M	hjm'  hžhubh¸)”}”(hŒhwlat files:”h]”hŒhwlat files:”…””}”(hjž(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M)	hjm'  hžhubjT  )”}”(hXÍ  tracing_threshold
      This gets automatically set to "10" to represent 10
      microseconds. This is the threshold of latency that
      needs to be detected before the trace will be recorded.

      Note, when hwlat tracer is finished (another tracer is
      written into "current_tracer"), the original value for
      tracing_threshold is placed back into this file.

hwlat_detector/width
      The length of time the test runs with interrupts disabled.

hwlat_detector/window
      The length of time of the window which the test
      runs. That is, the test will run for "width"
      microseconds per "window" microseconds

tracing_cpumask
      When the test is started. A kernel thread is created that
      runs the test. This thread will alternate between CPUs
      listed in the tracing_cpumask between each period
      (one "window"). To limit the test to specific CPUs
      set the mask in this file to only the CPUs that the test
      should run on.
”h]”jT  )”}”(hhh]”(jY  )”}”(hXR  tracing_threshold
This gets automatically set to "10" to represent 10
microseconds. This is the threshold of latency that
needs to be detected before the trace will be recorded.

Note, when hwlat tracer is finished (another tracer is
written into "current_tracer"), the original value for
tracing_threshold is placed back into this file.
”h]”(j_  )”}”(hŒtracing_threshold”h]”hŒtracing_threshold”…””}”(hj·(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M2	hj³(  ubjo  )”}”(hhh]”(h¸)”}”(hŒŸThis gets automatically set to "10" to represent 10
microseconds. This is the threshold of latency that
needs to be detected before the trace will be recorded.”h]”hŒ£This gets automatically set to â€œ10â€ to represent 10
microseconds. This is the threshold of latency that
needs to be detected before the trace will be recorded.”…””}”(hjÈ(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M,	hjÅ(  ubh¸)”}”(hŒžNote, when hwlat tracer is finished (another tracer is
written into "current_tracer"), the original value for
tracing_threshold is placed back into this file.”h]”hŒ¢Note, when hwlat tracer is finished (another tracer is
written into â€œcurrent_tracerâ€), the original value for
tracing_threshold is placed back into this file.”…””}”(hjÖ(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M0	hjÅ(  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj³(  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M2	hj°(  ubjY  )”}”(hŒPhwlat_detector/width
The length of time the test runs with interrupts disabled.
”h]”(j_  )”}”(hŒhwlat_detector/width”h]”hŒhwlat_detector/width”…””}”(hjô(  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M5	hjð(  ubjo  )”}”(hhh]”h¸)”}”(hŒ:The length of time the test runs with interrupts disabled.”h]”hŒ:The length of time the test runs with interrupts disabled.”…””}”(hj)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M5	hj)  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjð(  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M5	hj°(  ubjY  )”}”(hŒšhwlat_detector/window
The length of time of the window which the test
runs. That is, the test will run for "width"
microseconds per "window" microseconds
”h]”(j_  )”}”(hŒhwlat_detector/window”h]”hŒhwlat_detector/window”…””}”(hj#)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h M:	hj)  ubjo  )”}”(hhh]”h¸)”}”(hŒƒThe length of time of the window which the test
runs. That is, the test will run for "width"
microseconds per "window" microseconds”h]”hŒ‹The length of time of the window which the test
runs. That is, the test will run for â€œwidthâ€
microseconds per â€œwindowâ€ microseconds”…””}”(hj4)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M8	hj1)  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj)  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h M:	hj°(  ubjY  )”}”(hX.  tracing_cpumask
When the test is started. A kernel thread is created that
runs the test. This thread will alternate between CPUs
listed in the tracing_cpumask between each period
(one "window"). To limit the test to specific CPUs
set the mask in this file to only the CPUs that the test
should run on.
”h]”(j_  )”}”(hŒtracing_cpumask”h]”hŒtracing_cpumask”…””}”(hjR)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MB	hjN)  ubjo  )”}”(hhh]”h¸)”}”(hX  When the test is started. A kernel thread is created that
runs the test. This thread will alternate between CPUs
listed in the tracing_cpumask between each period
(one "window"). To limit the test to specific CPUs
set the mask in this file to only the CPUs that the test
should run on.”h]”hX!  When the test is started. A kernel thread is created that
runs the test. This thread will alternate between CPUs
listed in the tracing_cpumask between each period
(one â€œwindowâ€). To limit the test to specific CPUs
set the mask in this file to only the CPUs that the test
should run on.”…””}”(hjc)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M=	hj`)  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjN)  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MB	hj°(  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hj¬(  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M+	hjm'  hžhubeh}”(h]”Œhardware-latency-detector”ah ]”h"]”Œhardware latency detector”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Måubh¢)”}”(hhh]”(h§)”}”(hŒfunction”h]”hŒfunction”…””}”(hj”)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj‘)  hžhhŸh¶h ME	ubh¸)”}”(hŒØThis tracer is the function tracer. Enabling the function tracer
can be done from the debug file system. Make sure the
ftrace_enabled is set; otherwise this tracer is a nop.
See the "ftrace_enabled" section below.
::”h]”hŒÙThis tracer is the function tracer. Enabling the function tracer
can be done from the debug file system. Make sure the
ftrace_enabled is set; otherwise this tracer is a nop.
See the â€œftrace_enabledâ€ section below.”…””}”(hj¢)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MG	hj‘)  hžhubj   )”}”(hXÐ  # sysctl kernel.ftrace_enabled=1
# echo function > current_tracer
# echo 1 > tracing_on
# usleep 1
# echo 0 > tracing_on
# cat trace
# tracer: function
#
# entries-in-buffer/entries-written: 24799/24799   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
            bash-1994  [002] ....  3082.063030: mutex_unlock <-rb_simple_write
            bash-1994  [002] ....  3082.063031: __mutex_unlock_slowpath <-mutex_unlock
            bash-1994  [002] ....  3082.063031: __fsnotify_parent <-fsnotify_modify
            bash-1994  [002] ....  3082.063032: fsnotify <-fsnotify_modify
            bash-1994  [002] ....  3082.063032: __srcu_read_lock <-fsnotify
            bash-1994  [002] ....  3082.063032: add_preempt_count <-__srcu_read_lock
            bash-1994  [002] ...1  3082.063032: sub_preempt_count <-__srcu_read_lock
            bash-1994  [002] ....  3082.063033: __srcu_read_unlock <-fsnotify
[...]”h]”hXÐ  # sysctl kernel.ftrace_enabled=1
# echo function > current_tracer
# echo 1 > tracing_on
# usleep 1
# echo 0 > tracing_on
# cat trace
# tracer: function
#
# entries-in-buffer/entries-written: 24799/24799   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
            bash-1994  [002] ....  3082.063030: mutex_unlock <-rb_simple_write
            bash-1994  [002] ....  3082.063031: __mutex_unlock_slowpath <-mutex_unlock
            bash-1994  [002] ....  3082.063031: __fsnotify_parent <-fsnotify_modify
            bash-1994  [002] ....  3082.063032: fsnotify <-fsnotify_modify
            bash-1994  [002] ....  3082.063032: __srcu_read_lock <-fsnotify
            bash-1994  [002] ....  3082.063032: add_preempt_count <-__srcu_read_lock
            bash-1994  [002] ...1  3082.063032: sub_preempt_count <-__srcu_read_lock
            bash-1994  [002] ....  3082.063033: __srcu_read_unlock <-fsnotify
[...]”…””}”hj°)  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MM	hj‘)  hžhubh¸)”}”(hX  Note: function tracer uses ring buffers to store the above
entries. The newest data may overwrite the oldest data.
Sometimes using echo to stop the trace is not sufficient because
the tracing could have overwritten the data that you wanted to
record. For this reason, it is sometimes better to disable
tracing directly from a program. This allows you to stop the
tracing at the point that you hit the part that you are
interested in. To disable the tracing directly from a C program,
something like following code snippet can be used::”h]”hX  Note: function tracer uses ring buffers to store the above
entries. The newest data may overwrite the oldest data.
Sometimes using echo to stop the trace is not sufficient because
the tracing could have overwritten the data that you wanted to
record. For this reason, it is sometimes better to disable
tracing directly from a program. This allows you to stop the
tracing at the point that you hit the part that you are
interested in. To disable the tracing directly from a C program,
something like following code snippet can be used:”…””}”(hj¾)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mi	hj‘)  hžhubj   )”}”(hŒóint trace_fd;
[...]
int main(int argc, char *argv[]) {
        [...]
        trace_fd = open(tracing_file("tracing_on"), O_WRONLY);
        [...]
        if (condition_hit()) {
                write(trace_fd, "0", 1);
        }
        [...]
}”h]”hŒóint trace_fd;
[...]
int main(int argc, char *argv[]) {
        [...]
        trace_fd = open(tracing_file("tracing_on"), O_WRONLY);
        [...]
        if (condition_hit()) {
                write(trace_fd, "0", 1);
        }
        [...]
}”…””}”hjÌ)  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Ms	hj‘)  hžhubeh}”(h]”Œfunction”ah ]”h"]”Œfunction”ah$]”h&]”uh1h¡hh£hžhhŸh¶h ME	ubh¢)”}”(hhh]”(h§)”}”(hŒSingle thread tracing”h]”hŒSingle thread tracing”…””}”(hjå)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjâ)  hžhhŸh¶h M	ubh¸)”}”(hŒKBy writing into set_ftrace_pid you can trace a
single thread. For example::”h]”hŒJBy writing into set_ftrace_pid you can trace a
single thread. For example:”…””}”(hjó)  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mƒ	hjâ)  hžhubj   )”}”(hX  # cat set_ftrace_pid
no pid
# echo 3111 > set_ftrace_pid
# cat set_ftrace_pid
3111
# echo function > current_tracer
# cat trace | head
# tracer: function
#
#           TASK-PID    CPU#    TIMESTAMP  FUNCTION
#              | |       |          |         |
    yum-updatesd-3111  [003]  1637.254676: finish_task_switch <-thread_return
    yum-updatesd-3111  [003]  1637.254681: hrtimer_cancel <-schedule_hrtimeout_range
    yum-updatesd-3111  [003]  1637.254682: hrtimer_try_to_cancel <-hrtimer_cancel
    yum-updatesd-3111  [003]  1637.254683: lock_hrtimer_base <-hrtimer_try_to_cancel
    yum-updatesd-3111  [003]  1637.254685: fget_light <-do_sys_poll
    yum-updatesd-3111  [003]  1637.254686: pipe_poll <-do_sys_poll
# echo > set_ftrace_pid
# cat trace |head
# tracer: function
#
#           TASK-PID    CPU#    TIMESTAMP  FUNCTION
#              | |       |          |         |
##### CPU 3 buffer started ####
    yum-updatesd-3111  [003]  1701.957688: free_poll_entry <-poll_freewait
    yum-updatesd-3111  [003]  1701.957689: remove_wait_queue <-free_poll_entry
    yum-updatesd-3111  [003]  1701.957691: fput <-free_poll_entry
    yum-updatesd-3111  [003]  1701.957692: audit_syscall_exit <-sysret_audit
    yum-updatesd-3111  [003]  1701.957693: path_put <-audit_syscall_exit”h]”hX  # cat set_ftrace_pid
no pid
# echo 3111 > set_ftrace_pid
# cat set_ftrace_pid
3111
# echo function > current_tracer
# cat trace | head
# tracer: function
#
#           TASK-PID    CPU#    TIMESTAMP  FUNCTION
#              | |       |          |         |
    yum-updatesd-3111  [003]  1637.254676: finish_task_switch <-thread_return
    yum-updatesd-3111  [003]  1637.254681: hrtimer_cancel <-schedule_hrtimeout_range
    yum-updatesd-3111  [003]  1637.254682: hrtimer_try_to_cancel <-hrtimer_cancel
    yum-updatesd-3111  [003]  1637.254683: lock_hrtimer_base <-hrtimer_try_to_cancel
    yum-updatesd-3111  [003]  1637.254685: fget_light <-do_sys_poll
    yum-updatesd-3111  [003]  1637.254686: pipe_poll <-do_sys_poll
# echo > set_ftrace_pid
# cat trace |head
# tracer: function
#
#           TASK-PID    CPU#    TIMESTAMP  FUNCTION
#              | |       |          |         |
##### CPU 3 buffer started ####
    yum-updatesd-3111  [003]  1701.957688: free_poll_entry <-poll_freewait
    yum-updatesd-3111  [003]  1701.957689: remove_wait_queue <-free_poll_entry
    yum-updatesd-3111  [003]  1701.957691: fput <-free_poll_entry
    yum-updatesd-3111  [003]  1701.957692: audit_syscall_exit <-sysret_audit
    yum-updatesd-3111  [003]  1701.957693: path_put <-audit_syscall_exit”…””}”hj*  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M†	hjâ)  hžhubh¸)”}”(hŒdIf you want to trace a function when executing, you could use
something like this simple program.
::”h]”hŒaIf you want to trace a function when executing, you could use
something like this simple program.”…””}”(hj*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¤	hjâ)  hžhubj   )”}”(hX‰  #include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>

#define _STR(x) #x
#define STR(x) _STR(x)
#define MAX_PATH 256

const char *find_tracefs(void)
{
       static char tracefs[MAX_PATH+1];
       static int tracefs_found;
       char type[100];
       FILE *fp;

       if (tracefs_found)
               return tracefs;

       if ((fp = fopen("/proc/mounts","r")) == NULL) {
               perror("/proc/mounts");
               return NULL;
       }

       while (fscanf(fp, "%*s %"
                     STR(MAX_PATH)
                     "s %99s %*s %*d %*d\n",
                     tracefs, type) == 2) {
               if (strcmp(type, "tracefs") == 0)
                       break;
       }
       fclose(fp);

       if (strcmp(type, "tracefs") != 0) {
               fprintf(stderr, "tracefs not mounted");
               return NULL;
       }

       strcat(tracefs, "/tracing/");
       tracefs_found = 1;

       return tracefs;
}

const char *tracing_file(const char *file_name)
{
       static char trace_file[MAX_PATH+1];
       snprintf(trace_file, MAX_PATH, "%s/%s", find_tracefs(), file_name);
       return trace_file;
}

int main (int argc, char **argv)
{
        if (argc < 1)
                exit(-1);

        if (fork() > 0) {
                int fd, ffd;
                char line[64];
                int s;

                ffd = open(tracing_file("current_tracer"), O_WRONLY);
                if (ffd < 0)
                        exit(-1);
                write(ffd, "nop", 3);

                fd = open(tracing_file("set_ftrace_pid"), O_WRONLY);
                s = sprintf(line, "%d\n", getpid());
                write(fd, line, s);

                write(ffd, "function", 8);

                close(fd);
                close(ffd);

                execvp(argv[1], argv+1);
        }

        return 0;
}”h]”hX‰  #include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>

#define _STR(x) #x
#define STR(x) _STR(x)
#define MAX_PATH 256

const char *find_tracefs(void)
{
       static char tracefs[MAX_PATH+1];
       static int tracefs_found;
       char type[100];
       FILE *fp;

       if (tracefs_found)
               return tracefs;

       if ((fp = fopen("/proc/mounts","r")) == NULL) {
               perror("/proc/mounts");
               return NULL;
       }

       while (fscanf(fp, "%*s %"
                     STR(MAX_PATH)
                     "s %99s %*s %*d %*d\n",
                     tracefs, type) == 2) {
               if (strcmp(type, "tracefs") == 0)
                       break;
       }
       fclose(fp);

       if (strcmp(type, "tracefs") != 0) {
               fprintf(stderr, "tracefs not mounted");
               return NULL;
       }

       strcat(tracefs, "/tracing/");
       tracefs_found = 1;

       return tracefs;
}

const char *tracing_file(const char *file_name)
{
       static char trace_file[MAX_PATH+1];
       snprintf(trace_file, MAX_PATH, "%s/%s", find_tracefs(), file_name);
       return trace_file;
}

int main (int argc, char **argv)
{
        if (argc < 1)
                exit(-1);

        if (fork() > 0) {
                int fd, ffd;
                char line[64];
                int s;

                ffd = open(tracing_file("current_tracer"), O_WRONLY);
                if (ffd < 0)
                        exit(-1);
                write(ffd, "nop", 3);

                fd = open(tracing_file("set_ftrace_pid"), O_WRONLY);
                s = sprintf(line, "%d\n", getpid());
                write(fd, line, s);

                write(ffd, "function", 8);

                close(fd);
                close(ffd);

                execvp(argv[1], argv+1);
        }

        return 0;
}”…””}”hj*  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M¨	hjâ)  hžhubh¸)”}”(hŒOr this simple script!
::”h]”hŒOr this simple script!”…””}”(hj+*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mü	hjâ)  hžhubj   )”}”(hŒÜ#!/bin/bash

tracefs=`sed -ne 's/^tracefs \(.*\) tracefs.*/\1/p' /proc/mounts`
echo 0 > $tracefs/tracing_on
echo $$ > $tracefs/set_ftrace_pid
echo function > $tracefs/current_tracer
echo 1 > $tracefs/tracing_on
exec "$@"”h]”hŒÜ#!/bin/bash

tracefs=`sed -ne 's/^tracefs \(.*\) tracefs.*/\1/p' /proc/mounts`
echo 0 > $tracefs/tracing_on
echo $$ > $tracefs/set_ftrace_pid
echo function > $tracefs/current_tracer
echo 1 > $tracefs/tracing_on
exec "$@"”…””}”hj9*  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mÿ	hjâ)  hžhubeh}”(h]”Œsingle-thread-tracing”ah ]”h"]”Œsingle thread tracing”ah$]”h&]”uh1h¡hh£hžhhŸh¶h M	ubh¢)”}”(hhh]”(h§)”}”(hŒfunction graph tracer”h]”hŒfunction graph tracer”…””}”(hjR*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjO*  hžhhŸh¶h M

ubh¸)”}”(hXŽ  This tracer is similar to the function tracer except that it
probes a function on its entry and its exit. This is done by
using a dynamically allocated stack of return addresses in each
task_struct. On function entry the tracer overwrites the return
address of each function traced to set a custom probe. Thus the
original return address is stored on the stack of return address
in the task_struct.”h]”hXŽ  This tracer is similar to the function tracer except that it
probes a function on its entry and its exit. This is done by
using a dynamically allocated stack of return addresses in each
task_struct. On function entry the tracer overwrites the return
address of each function traced to set a custom probe. Thus the
original return address is stored on the stack of return address
in the task_struct.”…””}”(hj`*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M
hjO*  hžhubh¸)”}”(hŒEProbing on both ends of a function leads to special features
such as:”h]”hŒEProbing on both ends of a function leads to special features
such as:”…””}”(hjn*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M
hjO*  hžhubj{  )”}”(hhh]”(j€  )”}”(hŒ&measure of a function's time execution”h]”h¸)”}”(hj*  h]”hŒ(measure of a functionâ€™s time execution”…””}”(hjƒ*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M
hj*  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj|*  hžhhŸh¶h Nubj€  )”}”(hŒ:having a reliable call stack to draw function calls graph
”h]”h¸)”}”(hŒ9having a reliable call stack to draw function calls graph”h]”hŒ9having a reliable call stack to draw function calls graph”…””}”(hjš*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M
hj–*  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj|*  hžhhŸh¶h Nubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h M
hjO*  hžhubh¸)”}”(hŒ,This tracer is useful in several situations:”h]”hŒ,This tracer is useful in several situations:”…””}”(hj´*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M
hjO*  hžhubj{  )”}”(hhh]”(j€  )”}”(hŒyou want to find the reason of a strange kernel behavior and
need to see what happens in detail on any areas (or specific
ones).
”h]”h¸)”}”(hŒ€you want to find the reason of a strange kernel behavior and
need to see what happens in detail on any areas (or specific
ones).”h]”hŒ€you want to find the reason of a strange kernel behavior and
need to see what happens in detail on any areas (or specific
ones).”…””}”(hjÉ*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M
hjÅ*  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjÂ*  hžhhŸh¶h Nubj€  )”}”(hŒLyou are experiencing weird latencies but it's difficult to
find its origin.
”h]”h¸)”}”(hŒKyou are experiencing weird latencies but it's difficult to
find its origin.”h]”hŒMyou are experiencing weird latencies but itâ€™s difficult to
find its origin.”…””}”(hjá*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M 
hjÝ*  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjÂ*  hžhhŸh¶h Nubj€  )”}”(hŒDyou want to find quickly which path is taken by a specific
function
”h]”h¸)”}”(hŒCyou want to find quickly which path is taken by a specific
function”h]”hŒCyou want to find quickly which path is taken by a specific
function”…””}”(hjù*  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M#
hjõ*  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjÂ*  hžhhŸh¶h Nubj€  )”}”(hŒRyou just want to peek inside a working kernel and want to see
what happens there.
”h]”h¸)”}”(hŒQyou just want to peek inside a working kernel and want to see
what happens there.”h]”hŒQyou just want to peek inside a working kernel and want to see
what happens there.”…””}”(hj+  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M&
hj+  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjÂ*  hžhhŸh¶h Nubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h M
hjO*  hžhubj   )”}”(hXî  # tracer: function_graph
#
# CPU  DURATION                  FUNCTION CALLS
# |     |   |                     |   |   |   |

 0)               |  sys_open() {
 0)               |    do_sys_open() {
 0)               |      getname() {
 0)               |        kmem_cache_alloc() {
 0)   1.382 us    |          __might_sleep();
 0)   2.478 us    |        }
 0)               |        strncpy_from_user() {
 0)               |          might_fault() {
 0)   1.389 us    |            __might_sleep();
 0)   2.553 us    |          }
 0)   3.807 us    |        }
 0)   7.876 us    |      }
 0)               |      alloc_fd() {
 0)   0.668 us    |        _spin_lock();
 0)   0.570 us    |        expand_files();
 0)   0.586 us    |        _spin_unlock();”h]”hXî  # tracer: function_graph
#
# CPU  DURATION                  FUNCTION CALLS
# |     |   |                     |   |   |   |

 0)               |  sys_open() {
 0)               |    do_sys_open() {
 0)               |      getname() {
 0)               |        kmem_cache_alloc() {
 0)   1.382 us    |          __might_sleep();
 0)   2.478 us    |        }
 0)               |        strncpy_from_user() {
 0)               |          might_fault() {
 0)   1.389 us    |            __might_sleep();
 0)   2.553 us    |          }
 0)   3.807 us    |        }
 0)   7.876 us    |      }
 0)               |      alloc_fd() {
 0)   0.668 us    |        _spin_lock();
 0)   0.570 us    |        expand_files();
 0)   0.586 us    |        _spin_unlock();”…””}”hj++  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M+
hjO*  hžhubh¸)”}”(hŒThere are several columns that can be dynamically
enabled/disabled. You can use every combination of options you
want, depending on your needs.”h]”hŒThere are several columns that can be dynamically
enabled/disabled. You can use every combination of options you
want, depending on your needs.”…””}”(hj9+  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MB
hjO*  hžhubj{  )”}”(hhh]”(j€  )”}”(hX>  The cpu number on which the function executed is default
enabled.  It is sometimes better to only trace one cpu (see
tracing_cpumask file) or you might sometimes see unordered
function calls while cpu tracing switch.

      - hide: echo nofuncgraph-cpu > trace_options
      - show: echo funcgraph-cpu > trace_options
”h]”(h¸)”}”(hŒØThe cpu number on which the function executed is default
enabled.  It is sometimes better to only trace one cpu (see
tracing_cpumask file) or you might sometimes see unordered
function calls while cpu tracing switch.”h]”hŒØThe cpu number on which the function executed is default
enabled.  It is sometimes better to only trace one cpu (see
tracing_cpumask file) or you might sometimes see unordered
function calls while cpu tracing switch.”…””}”(hjN+  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MF
hjJ+  ubjT  )”}”(hŒX- hide: echo nofuncgraph-cpu > trace_options
- show: echo funcgraph-cpu > trace_options
”h]”j{  )”}”(hhh]”(j€  )”}”(hŒ*hide: echo nofuncgraph-cpu > trace_options”h]”h¸)”}”(hje+  h]”hŒ*hide: echo nofuncgraph-cpu > trace_options”…””}”(hjg+  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MK
hjc+  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj`+  ubj€  )”}”(hŒ)show: echo funcgraph-cpu > trace_options
”h]”h¸)”}”(hŒ(show: echo funcgraph-cpu > trace_options”h]”hŒ(show: echo funcgraph-cpu > trace_options”…””}”(hj~+  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h ML
hjz+  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj`+  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h MK
hj\+  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MK
hjJ+  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1j  hjG+  hžhhŸh¶h Nubj€  )”}”(hX.  The duration (function's time of execution) is displayed on
the closing bracket line of a function or on the same line
than the current function in case of a leaf one. It is default
enabled.

      - hide: echo nofuncgraph-duration > trace_options
      - show: echo funcgraph-duration > trace_options
”h]”(h¸)”}”(hŒ¾The duration (function's time of execution) is displayed on
the closing bracket line of a function or on the same line
than the current function in case of a leaf one. It is default
enabled.”h]”hŒÀThe duration (functionâ€™s time of execution) is displayed on
the closing bracket line of a function or on the same line
than the current function in case of a leaf one. It is default
enabled.”…””}”(hj¨+  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MN
hj¤+  ubjT  )”}”(hŒb- hide: echo nofuncgraph-duration > trace_options
- show: echo funcgraph-duration > trace_options
”h]”j{  )”}”(hhh]”(j€  )”}”(hŒ/hide: echo nofuncgraph-duration > trace_options”h]”h¸)”}”(hj¿+  h]”hŒ/hide: echo nofuncgraph-duration > trace_options”…””}”(hjÁ+  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MS
hj½+  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjº+  ubj€  )”}”(hŒ.show: echo funcgraph-duration > trace_options
”h]”h¸)”}”(hŒ-show: echo funcgraph-duration > trace_options”h]”hŒ-show: echo funcgraph-duration > trace_options”…””}”(hjØ+  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MT
hjÔ+  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjº+  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h MS
hj¶+  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MS
hj¤+  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1j  hjG+  hžhhŸh¶h Nubj€  )”}”(hXæ  The overhead field precedes the duration field in case of
reached duration thresholds.

      - hide: echo nofuncgraph-overhead > trace_options
      - show: echo funcgraph-overhead > trace_options
      - depends on: funcgraph-duration

ie::

  3) # 1837.709 us |          } /* __switch_to */
  3)               |          finish_task_switch() {
  3)   0.313 us    |            _raw_spin_unlock_irq();
  3)   3.177 us    |          }
  3) # 1889.063 us |        } /* __schedule */
  3) ! 140.417 us  |      } /* __schedule */
  3) # 2034.948 us |    } /* schedule */
  3) * 33998.59 us |  } /* schedule_preempt_disabled */

  [...]

  1)   0.260 us    |              msecs_to_jiffies();
  1)   0.313 us    |              __rcu_read_unlock();
  1) + 61.770 us   |            }
  1) + 64.479 us   |          }
  1)   0.313 us    |          rcu_bh_qs();
  1)   0.313 us    |          __local_bh_enable();
  1) ! 217.240 us  |        }
  1)   0.365 us    |        idle_cpu();
  1)               |        rcu_irq_exit() {
  1)   0.417 us    |          rcu_eqs_enter_common.isra.47();
  1)   3.125 us    |        }
  1) ! 227.812 us  |      }
  1) ! 457.395 us  |    }
  1) @ 119760.2 us |  }

  [...]

  2)               |    handle_IPI() {
  1)   6.979 us    |                  }
  2)   0.417 us    |      scheduler_ipi();
  1)   9.791 us    |                }
  1) + 12.917 us   |              }
  2)   3.490 us    |    }
  1) + 15.729 us   |            }
  1) + 18.542 us   |          }
  2) $ 3594274 us  |  }
”h]”(h¸)”}”(hŒVThe overhead field precedes the duration field in case of
reached duration thresholds.”h]”hŒVThe overhead field precedes the duration field in case of
reached duration thresholds.”…””}”(hj,  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MV
hjþ+  ubjT  )”}”(hŒƒ- hide: echo nofuncgraph-overhead > trace_options
- show: echo funcgraph-overhead > trace_options
- depends on: funcgraph-duration
”h]”j{  )”}”(hhh]”(j€  )”}”(hŒ/hide: echo nofuncgraph-overhead > trace_options”h]”h¸)”}”(hj,  h]”hŒ/hide: echo nofuncgraph-overhead > trace_options”…””}”(hj,  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MY
hj,  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj,  ubj€  )”}”(hŒ-show: echo funcgraph-overhead > trace_options”h]”h¸)”}”(hj0,  h]”hŒ-show: echo funcgraph-overhead > trace_options”…””}”(hj2,  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MZ
hj.,  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj,  ubj€  )”}”(hŒdepends on: funcgraph-duration
”h]”h¸)”}”(hŒdepends on: funcgraph-duration”h]”hŒdepends on: funcgraph-duration”…””}”(hjI,  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M[
hjE,  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj,  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h MY
hj,  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MY
hjþ+  ubh¸)”}”(hŒie::”h]”hŒie:”…””}”(hji,  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M]
hjþ+  ubj   )”}”(hX¯  3) # 1837.709 us |          } /* __switch_to */
3)               |          finish_task_switch() {
3)   0.313 us    |            _raw_spin_unlock_irq();
3)   3.177 us    |          }
3) # 1889.063 us |        } /* __schedule */
3) ! 140.417 us  |      } /* __schedule */
3) # 2034.948 us |    } /* schedule */
3) * 33998.59 us |  } /* schedule_preempt_disabled */

[...]

1)   0.260 us    |              msecs_to_jiffies();
1)   0.313 us    |              __rcu_read_unlock();
1) + 61.770 us   |            }
1) + 64.479 us   |          }
1)   0.313 us    |          rcu_bh_qs();
1)   0.313 us    |          __local_bh_enable();
1) ! 217.240 us  |        }
1)   0.365 us    |        idle_cpu();
1)               |        rcu_irq_exit() {
1)   0.417 us    |          rcu_eqs_enter_common.isra.47();
1)   3.125 us    |        }
1) ! 227.812 us  |      }
1) ! 457.395 us  |    }
1) @ 119760.2 us |  }

[...]

2)               |    handle_IPI() {
1)   6.979 us    |                  }
2)   0.417 us    |      scheduler_ipi();
1)   9.791 us    |                }
1) + 12.917 us   |              }
2)   3.490 us    |    }
1) + 15.729 us   |            }
1) + 18.542 us   |          }
2) $ 3594274 us  |  }”h]”hX¯  3) # 1837.709 us |          } /* __switch_to */
3)               |          finish_task_switch() {
3)   0.313 us    |            _raw_spin_unlock_irq();
3)   3.177 us    |          }
3) # 1889.063 us |        } /* __schedule */
3) ! 140.417 us  |      } /* __schedule */
3) # 2034.948 us |    } /* schedule */
3) * 33998.59 us |  } /* schedule_preempt_disabled */

[...]

1)   0.260 us    |              msecs_to_jiffies();
1)   0.313 us    |              __rcu_read_unlock();
1) + 61.770 us   |            }
1) + 64.479 us   |          }
1)   0.313 us    |          rcu_bh_qs();
1)   0.313 us    |          __local_bh_enable();
1) ! 217.240 us  |        }
1)   0.365 us    |        idle_cpu();
1)               |        rcu_irq_exit() {
1)   0.417 us    |          rcu_eqs_enter_common.isra.47();
1)   3.125 us    |        }
1) ! 227.812 us  |      }
1) ! 457.395 us  |    }
1) @ 119760.2 us |  }

[...]

2)               |    handle_IPI() {
1)   6.979 us    |                  }
2)   0.417 us    |      scheduler_ipi();
1)   9.791 us    |                }
1) + 12.917 us   |              }
2)   3.490 us    |    }
1) + 15.729 us   |            }
1) + 18.542 us   |          }
2) $ 3594274 us  |  }”…””•      }”hjw,  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M_
hjþ+  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1j  hjG+  hžhhŸh¶h Nubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h MF
hjO*  hžhubh¸)”}”(hŒFlags::”h]”hŒFlags:”…””}”(hj‘,  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M…
hjO*  hžhubj   )”}”(hX  + means that the function exceeded 10 usecs.
! means that the function exceeded 100 usecs.
# means that the function exceeded 1000 usecs.
* means that the function exceeded 10 msecs.
@ means that the function exceeded 100 msecs.
$ means that the function exceeded 1 sec.”h]”hX  + means that the function exceeded 10 usecs.
! means that the function exceeded 100 usecs.
# means that the function exceeded 1000 usecs.
* means that the function exceeded 10 msecs.
@ means that the function exceeded 100 msecs.
$ means that the function exceeded 1 sec.”…””}”hjŸ,  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M‡
hjO*  hžhubj{  )”}”(hhh]”(j€  )”}”(hXæ  The task/pid field displays the thread cmdline and pid which
executed the function. It is default disabled.

      - hide: echo nofuncgraph-proc > trace_options
      - show: echo funcgraph-proc > trace_options

ie::

  # tracer: function_graph
  #
  # CPU  TASK/PID        DURATION                  FUNCTION CALLS
  # |    |    |           |   |                     |   |   |   |
  0)    sh-4802     |               |                  d_free() {
  0)    sh-4802     |               |                    call_rcu() {
  0)    sh-4802     |               |                      __call_rcu() {
  0)    sh-4802     |   0.616 us    |                        rcu_process_gp_end();
  0)    sh-4802     |   0.586 us    |                        check_for_new_grace_period();
  0)    sh-4802     |   2.899 us    |                      }
  0)    sh-4802     |   4.040 us    |                    }
  0)    sh-4802     |   5.151 us    |                  }
  0)    sh-4802     | + 49.370 us   |                }

”h]”(h¸)”}”(hŒkThe task/pid field displays the thread cmdline and pid which
executed the function. It is default disabled.”h]”hŒkThe task/pid field displays the thread cmdline and pid which
executed the function. It is default disabled.”…””}”(hj´,  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M
hj°,  ubjT  )”}”(hŒZ- hide: echo nofuncgraph-proc > trace_options
- show: echo funcgraph-proc > trace_options
”h]”j{  )”}”(hhh]”(j€  )”}”(hŒ+hide: echo nofuncgraph-proc > trace_options”h]”h¸)”}”(hjË,  h]”hŒ+hide: echo nofuncgraph-proc > trace_options”…””}”(hjÍ,  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M’
hjÉ,  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjÆ,  ubj€  )”}”(hŒ*show: echo funcgraph-proc > trace_options
”h]”h¸)”}”(hŒ)show: echo funcgraph-proc > trace_options”h]”hŒ)show: echo funcgraph-proc > trace_options”…””}”(hjä,  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M“
hjà,  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjÆ,  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h M’
hjÂ,  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M’
hj°,  ubh¸)”}”(hŒie::”h]”hŒie:”…””}”(hj-  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M•
hj°,  ubj   )”}”(hXð  # tracer: function_graph
#
# CPU  TASK/PID        DURATION                  FUNCTION CALLS
# |    |    |           |   |                     |   |   |   |
0)    sh-4802     |               |                  d_free() {
0)    sh-4802     |               |                    call_rcu() {
0)    sh-4802     |               |                      __call_rcu() {
0)    sh-4802     |   0.616 us    |                        rcu_process_gp_end();
0)    sh-4802     |   0.586 us    |                        check_for_new_grace_period();
0)    sh-4802     |   2.899 us    |                      }
0)    sh-4802     |   4.040 us    |                    }
0)    sh-4802     |   5.151 us    |                  }
0)    sh-4802     | + 49.370 us   |                }”h]”hXð  # tracer: function_graph
#
# CPU  TASK/PID        DURATION                  FUNCTION CALLS
# |    |    |           |   |                     |   |   |   |
0)    sh-4802     |               |                  d_free() {
0)    sh-4802     |               |                    call_rcu() {
0)    sh-4802     |               |                      __call_rcu() {
0)    sh-4802     |   0.616 us    |                        rcu_process_gp_end();
0)    sh-4802     |   0.586 us    |                        check_for_new_grace_period();
0)    sh-4802     |   2.899 us    |                      }
0)    sh-4802     |   4.040 us    |                    }
0)    sh-4802     |   5.151 us    |                  }
0)    sh-4802     | + 49.370 us   |                }”…””}”hj-  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M—
hj°,  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1j  hj­,  hžhhŸh¶h Nubj€  )”}”(hXë  The absolute time field is an absolute timestamp given by the
system clock since it started. A snapshot of this time is
given on each entry/exit of functions

      - hide: echo nofuncgraph-abstime > trace_options
      - show: echo funcgraph-abstime > trace_options

ie::

  #
  #      TIME       CPU  DURATION                  FUNCTION CALLS
  #       |         |     |   |                     |   |   |   |
  360.774522 |   1)   0.541 us    |                                          }
  360.774522 |   1)   4.663 us    |                                        }
  360.774523 |   1)   0.541 us    |                                        __wake_up_bit();
  360.774524 |   1)   6.796 us    |                                      }
  360.774524 |   1)   7.952 us    |                                    }
  360.774525 |   1)   9.063 us    |                                  }
  360.774525 |   1)   0.615 us    |                                  journal_mark_dirty();
  360.774527 |   1)   0.578 us    |                                  __brelse();
  360.774528 |   1)               |                                  reiserfs_prepare_for_journal() {
  360.774528 |   1)               |                                    unlock_buffer() {
  360.774529 |   1)               |                                      wake_up_bit() {
  360.774529 |   1)               |                                        bit_waitqueue() {
  360.774530 |   1)   0.594 us    |                                          __phys_addr();

”h]”(h¸)”}”(hŒThe absolute time field is an absolute timestamp given by the
system clock since it started. A snapshot of this time is
given on each entry/exit of functions”h]”hŒThe absolute time field is an absolute timestamp given by the
system clock since it started. A snapshot of this time is
given on each entry/exit of functions”…””}”(hj*-  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¦
hj&-  ubjT  )”}”(hŒ`- hide: echo nofuncgraph-abstime > trace_options
- show: echo funcgraph-abstime > trace_options
”h]”j{  )”}”(hhh]”(j€  )”}”(hŒ.hide: echo nofuncgraph-abstime > trace_options”h]”h¸)”}”(hjA-  h]”hŒ.hide: echo nofuncgraph-abstime > trace_options”…””}”(hjC-  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mª
hj?-  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj<-  ubj€  )”}”(hŒ-show: echo funcgraph-abstime > trace_options
”h]”h¸)”}”(hŒ,show: echo funcgraph-abstime > trace_options”h]”hŒ,show: echo funcgraph-abstime > trace_options”…””}”(hjZ-  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M«
hjV-  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hj<-  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h Mª
hj8-  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mª
hj&-  ubh¸)”}”(hŒie::”h]”hŒie:”…””}”(hjz-  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M­
hj&-  ubj   )”}”(hX·  #
#      TIME       CPU  DURATION                  FUNCTION CALLS
#       |         |     |   |                     |   |   |   |
360.774522 |   1)   0.541 us    |                                          }
360.774522 |   1)   4.663 us    |                                        }
360.774523 |   1)   0.541 us    |                                        __wake_up_bit();
360.774524 |   1)   6.796 us    |                                      }
360.774524 |   1)   7.952 us    |                                    }
360.774525 |   1)   9.063 us    |                                  }
360.774525 |   1)   0.615 us    |                                  journal_mark_dirty();
360.774527 |   1)   0.578 us    |                                  __brelse();
360.774528 |   1)               |                                  reiserfs_prepare_for_journal() {
360.774528 |   1)               |                                    unlock_buffer() {
360.774529 |   1)               |                                      wake_up_bit() {
360.774529 |   1)               |                                        bit_waitqueue() {
360.774530 |   1)   0.594 us    |                                          __phys_addr();”h]”hX·  #
#      TIME       CPU  DURATION                  FUNCTION CALLS
#       |         |     |   |                     |   |   |   |
360.774522 |   1)   0.541 us    |                                          }
360.774522 |   1)   4.663 us    |                                        }
360.774523 |   1)   0.541 us    |                                        __wake_up_bit();
360.774524 |   1)   6.796 us    |                                      }
360.774524 |   1)   7.952 us    |                                    }
360.774525 |   1)   9.063 us    |                                  }
360.774525 |   1)   0.615 us    |                                  journal_mark_dirty();
360.774527 |   1)   0.578 us    |                                  __brelse();
360.774528 |   1)               |                                  reiserfs_prepare_for_journal() {
360.774528 |   1)               |                                    unlock_buffer() {
360.774529 |   1)               |                                      wake_up_bit() {
360.774529 |   1)               |                                        bit_waitqueue() {
360.774530 |   1)   0.594 us    |                                          __phys_addr();”…””}”hjˆ-  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M¯
hj&-  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1j  hj­,  hžhhŸh¶h Nubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h M
hjO*  hžhubh¸)”}”(hŒˆThe function name is always displayed after the closing bracket
for a function if the start of that function is not in the
trace buffer.”h]”hŒˆThe function name is always displayed after the closing bracket
for a function if the start of that function is not in the
trace buffer.”…””}”(hj¢-  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÁ
hjO*  hžhubh¸)”}”(hŒËDisplay of the function name after the closing bracket may be
enabled for functions whose start is in the trace buffer,
allowing easier searching with grep for function durations.
It is default disabled.”h]”hŒËDisplay of the function name after the closing bracket may be
enabled for functions whose start is in the trace buffer,
allowing easier searching with grep for function durations.
It is default disabled.”…””}”(hj°-  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÅ
hjO*  hžhubjT  )”}”(hXT        - hide: echo nofuncgraph-tail > trace_options
      - show: echo funcgraph-tail > trace_options

Example with nofuncgraph-tail (default)::

  0)               |      putname() {
  0)               |        kmem_cache_free() {
  0)   0.518 us    |          __phys_addr();
  0)   1.757 us    |        }
  0)   2.861 us    |      }

Example with funcgraph-tail::

  0)               |      putname() {
  0)               |        kmem_cache_free() {
  0)   0.518 us    |          __phys_addr();
  0)   1.757 us    |        } /* kmem_cache_free() */
  0)   2.861 us    |      } /* putname() */
”h]”(jT  )”}”(hŒZ- hide: echo nofuncgraph-tail > trace_options
- show: echo funcgraph-tail > trace_options
”h]”j{  )”}”(hhh]”(j€  )”}”(hŒ+hide: echo nofuncgraph-tail > trace_options”h]”h¸)”}”(hjË-  h]”hŒ+hide: echo nofuncgraph-tail > trace_options”…””}”(hjÍ-  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÊ
hjÉ-  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjÆ-  ubj€  )”}”(hŒ*show: echo funcgraph-tail > trace_options
”h]”h¸)”}”(hŒ)show: echo funcgraph-tail > trace_options”h]”hŒ)show: echo funcgraph-tail > trace_options”…””}”(hjä-  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MË
hjà-  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjÆ-  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h MÊ
hjÂ-  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÊ
hj¾-  ubh¸)”}”(hŒ)Example with nofuncgraph-tail (default)::”h]”hŒ(Example with nofuncgraph-tail (default):”…””}”(hj.  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÍ
hj¾-  ubj   )”}”(hŒ²0)               |      putname() {
0)               |        kmem_cache_free() {
0)   0.518 us    |          __phys_addr();
0)   1.757 us    |        }
0)   2.861 us    |      }”h]”hŒ²0)               |      putname() {
0)               |        kmem_cache_free() {
0)   0.518 us    |          __phys_addr();
0)   1.757 us    |        }
0)   2.861 us    |      }”…””}”hj.  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MÏ
hj¾-  ubh¸)”}”(hŒExample with funcgraph-tail::”h]”hŒExample with funcgraph-tail:”…””}”(hj .  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÕ
hj¾-  ubj   )”}”(hŒÚ0)               |      putname() {
0)               |        kmem_cache_free() {
0)   0.518 us    |          __phys_addr();
0)   1.757 us    |        } /* kmem_cache_free() */
0)   2.861 us    |      } /* putname() */”h]”hŒÚ0)               |      putname() {
0)               |        kmem_cache_free() {
0)   0.518 us    |          __phys_addr();
0)   1.757 us    |        } /* kmem_cache_free() */
0)   2.861 us    |      } /* putname() */”…””}”hj..  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M×
hj¾-  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÊ
hjO*  hžhubh¸)”}”(hŒÒThe return value of each traced function can be displayed after
an equal sign "=". When encountering system call failures, it
can be very helpful to quickly locate the function that first
returns an error code.”h]”hŒÖThe return value of each traced function can be displayed after
an equal sign â€œ=â€. When encountering system call failures, it
can be very helpful to quickly locate the function that first
returns an error code.”…””}”(hjB.  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÝ
hjO*  hžhubjT  )”}”(hX@        - hide: echo nofuncgraph-retval > trace_options
      - show: echo funcgraph-retval > trace_options

Example with funcgraph-retval::

  1)               |    cgroup_migrate() {
  1)   0.651 us    |      cgroup_migrate_add_task(); /* = 0xffff93fcfd346c00 */
  1)               |      cgroup_migrate_execute() {
  1)               |        cpu_cgroup_can_attach() {
  1)               |          cgroup_taskset_first() {
  1)   0.732 us    |            cgroup_taskset_next(); /* = 0xffff93fc8fb20000 */
  1)   1.232 us    |          } /* cgroup_taskset_first = 0xffff93fc8fb20000 */
  1)   0.380 us    |          sched_rt_can_attach(); /* = 0x0 */
  1)   2.335 us    |        } /* cpu_cgroup_can_attach = -22 */
  1)   4.369 us    |      } /* cgroup_migrate_execute = -22 */
  1)   7.143 us    |    } /* cgroup_migrate = -22 */
”h]”(jT  )”}”(hŒ^- hide: echo nofuncgraph-retval > trace_options
- show: echo funcgraph-retval > trace_options
”h]”j{  )”}”(hhh]”(j€  )”}”(hŒ-hide: echo nofuncgraph-retval > trace_options”h]”h¸)”}”(hj].  h]”hŒ-hide: echo nofuncgraph-retval > trace_options”…””}”(hj_.  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mâ
hj[.  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjX.  ubj€  )”}”(hŒ,show: echo funcgraph-retval > trace_options
”h]”h¸)”}”(hŒ+show: echo funcgraph-retval > trace_options”h]”hŒ+show: echo funcgraph-retval > trace_options”…””}”(hjv.  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mã
hjr.  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjX.  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h Mâ
hjT.  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mâ
hjP.  ubh¸)”}”(hŒExample with funcgraph-retval::”h]”hŒExample with funcgraph-retval:”…””}”(hj–.  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Må
hjP.  ubj   )”}”(hX  1)               |    cgroup_migrate() {
1)   0.651 us    |      cgroup_migrate_add_task(); /* = 0xffff93fcfd346c00 */
1)               |      cgroup_migrate_execute() {
1)               |        cpu_cgroup_can_attach() {
1)               |          cgroup_taskset_first() {
1)   0.732 us    |            cgroup_taskset_next(); /* = 0xffff93fc8fb20000 */
1)   1.232 us    |          } /* cgroup_taskset_first = 0xffff93fc8fb20000 */
1)   0.380 us    |          sched_rt_can_attach(); /* = 0x0 */
1)   2.335 us    |        } /* cpu_cgroup_can_attach = -22 */
1)   4.369 us    |      } /* cgroup_migrate_execute = -22 */
1)   7.143 us    |    } /* cgroup_migrate = -22 */”h]”hX  1)               |    cgroup_migrate() {
1)   0.651 us    |      cgroup_migrate_add_task(); /* = 0xffff93fcfd346c00 */
1)               |      cgroup_migrate_execute() {
1)               |        cpu_cgroup_can_attach() {
1)               |          cgroup_taskset_first() {
1)   0.732 us    |            cgroup_taskset_next(); /* = 0xffff93fc8fb20000 */
1)   1.232 us    |          } /* cgroup_taskset_first = 0xffff93fc8fb20000 */
1)   0.380 us    |          sched_rt_can_attach(); /* = 0x0 */
1)   2.335 us    |        } /* cpu_cgroup_can_attach = -22 */
1)   4.369 us    |      } /* cgroup_migrate_execute = -22 */
1)   7.143 us    |    } /* cgroup_migrate = -22 */”…””}”hj¤.  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mç
hjP.  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mâ
hjO*  hžhubh¸)”}”(hŒ¦The above example shows that the function cpu_cgroup_can_attach
returned the error code -22 firstly, then we can read the code
of this function to get the root cause.”h]”hŒ¦The above example shows that the function cpu_cgroup_can_attach
returned the error code -22 firstly, then we can read the code
of this function to get the root cause.”…””}”(hj¸.  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mó
hjO*  hžhubh¸)”}”(hŒæWhen the option funcgraph-retval-hex is not set, the return value can
be displayed in a smart way. Specifically, if it is an error code,
it will be printed in signed decimal format, otherwise it will
printed in hexadecimal format.”h]”hŒæWhen the option funcgraph-retval-hex is not set, the return value can
be displayed in a smart way. Specifically, if it is an error code,
it will be printed in signed decimal format, otherwise it will
printed in hexadecimal format.”…””}”(hjÆ.  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M÷
hjO*  hžhubjT  )”}”(hX        - smart: echo nofuncgraph-retval-hex > trace_options
      - hexadecimal: echo funcgraph-retval-hex > trace_options

Example with funcgraph-retval-hex::

  1)               |      cgroup_migrate() {
  1)   0.651 us    |        cgroup_migrate_add_task(); /* = 0xffff93fcfd346c00 */
  1)               |        cgroup_migrate_execute() {
  1)               |          cpu_cgroup_can_attach() {
  1)               |            cgroup_taskset_first() {
  1)   0.732 us    |              cgroup_taskset_next(); /* = 0xffff93fc8fb20000 */
  1)   1.232 us    |            } /* cgroup_taskset_first = 0xffff93fc8fb20000 */
  1)   0.380 us    |            sched_rt_can_attach(); /* = 0x0 */
  1)   2.335 us    |          } /* cpu_cgroup_can_attach = 0xffffffea */
  1)   4.369 us    |        } /* cgroup_migrate_execute = 0xffffffea */
  1)   7.143 us    |      } /* cgroup_migrate = 0xffffffea */
”h]”(jT  )”}”(hŒn- smart: echo nofuncgraph-retval-hex > trace_options
- hexadecimal: echo funcgraph-retval-hex > trace_options
”h]”j{  )”}”(hhh]”(j€  )”}”(hŒ2smart: echo nofuncgraph-retval-hex > trace_options”h]”h¸)”}”(hjá.  h]”hŒ2smart: echo nofuncgraph-retval-hex > trace_options”…””}”(hjã.  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mü
hjß.  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjÜ.  ubj€  )”}”(hŒ7hexadecimal: echo funcgraph-retval-hex > trace_options
”h]”h¸)”}”(hŒ6hexadecimal: echo funcgraph-retval-hex > trace_options”h]”hŒ6hexadecimal: echo funcgraph-retval-hex > trace_options”…””}”(hjú.  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mý
hjö.  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjÜ.  ubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h Mü
hjØ.  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mü
hjÔ.  ubh¸)”}”(hŒ#Example with funcgraph-retval-hex::”h]”hŒ"Example with funcgraph-retval-hex:”…””}”(hj/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mÿ
hjÔ.  ubj   )”}”(hXÈ  1)               |      cgroup_migrate() {
1)   0.651 us    |        cgroup_migrate_add_task(); /* = 0xffff93fcfd346c00 */
1)               |        cgroup_migrate_execute() {
1)               |          cpu_cgroup_can_attach() {
1)               |            cgroup_taskset_first() {
1)   0.732 us    |              cgroup_taskset_next(); /* = 0xffff93fc8fb20000 */
1)   1.232 us    |            } /* cgroup_taskset_first = 0xffff93fc8fb20000 */
1)   0.380 us    |            sched_rt_can_attach(); /* = 0x0 */
1)   2.335 us    |          } /* cpu_cgroup_can_attach = 0xffffffea */
1)   4.369 us    |        } /* cgroup_migrate_execute = 0xffffffea */
1)   7.143 us    |      } /* cgroup_migrate = 0xffffffea */”h]”hXÈ  1)               |      cgroup_migrate() {
1)   0.651 us    |        cgroup_migrate_add_task(); /* = 0xffff93fcfd346c00 */
1)               |        cgroup_migrate_execute() {
1)               |          cpu_cgroup_can_attach() {
1)               |            cgroup_taskset_first() {
1)   0.732 us    |              cgroup_taskset_next(); /* = 0xffff93fc8fb20000 */
1)   1.232 us    |            } /* cgroup_taskset_first = 0xffff93fc8fb20000 */
1)   0.380 us    |            sched_rt_can_attach(); /* = 0x0 */
1)   2.335 us    |          } /* cpu_cgroup_can_attach = 0xffffffea */
1)   4.369 us    |        } /* cgroup_migrate_execute = 0xffffffea */
1)   7.143 us    |      } /* cgroup_migrate = 0xffffffea */”…””}”hj(/  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MhjÔ.  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mü
hjO*  hžhubh¸)”}”(hŒ†At present, there are some limitations when using the funcgraph-retval
option, and these limitations will be eliminated in the future:”h]”hŒ†At present, there are some limitations when using the funcgraph-retval
option, and these limitations will be eliminated in the future:”…””}”(hj</  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjO*  hžhubj{  )”}”(hhh]”(j€  )”}”(hŒlEven if the function return type is void, a return value will still
be printed, and you can just ignore it.
”h]”h¸)”}”(hŒkEven if the function return type is void, a return value will still
be printed, and you can just ignore it.”h]”hŒkEven if the function return type is void, a return value will still
be printed, and you can just ignore it.”…””}”(hjQ/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjM/  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjJ/  hžhhŸh¶h Nubj€  )”}”(hXn  Even if return values are stored in multiple registers, only the
value contained in the first register will be recorded and printed.
To illustrate, in the x86 architecture, eax and edx are used to store
a 64-bit return value, with the lower 32 bits saved in eax and the
upper 32 bits saved in edx. However, only the value stored in eax
will be recorded and printed.
”h]”h¸)”}”(hXm  Even if return values are stored in multiple registers, only the
value contained in the first register will be recorded and printed.
To illustrate, in the x86 architecture, eax and edx are used to store
a 64-bit return value, with the lower 32 bits saved in eax and the
upper 32 bits saved in edx. However, only the value stored in eax
will be recorded and printed.”h]”hXm  Even if return values are stored in multiple registers, only the
value contained in the first register will be recorded and printed.
To illustrate, in the x86 architecture, eax and edx are used to store
a 64-bit return value, with the lower 32 bits saved in eax and the
upper 32 bits saved in edx. However, only the value stored in eax
will be recorded and printed.”…””}”(hji/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhje/  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hjJ/  hžhhŸh¶h Nubj€  )”}”(hX  In certain procedure call standards, such as arm64's AAPCS64, when a
type is smaller than a GPR, it is the responsibility of the consumer
to perform the narrowing, and the upper bits may contain UNKNOWN values.
Therefore, it is advisable to check the code for such cases. For instance,
when using a u8 in a 64-bit GPR, bits [63:8] may contain arbitrary values,
especially when larger types are truncated, whether explicitly or implicitly.
Here are some specific cases to illustrate this point:

**Case One**:

The function narrow_to_u8 is defined as follows::

      u8 narrow_to_u8(u64 val)
      {
              // implicitly truncated
              return val;
      }

It may be compiled to::

      narrow_to_u8:
              < ... ftrace instrumentation ... >
              RET

If you pass 0x123456789abcdef to this function and want to narrow it,
it may be recorded as 0x123456789abcdef instead of 0xef.

**Case Two**:

The function error_if_not_4g_aligned is defined as follows::

      int error_if_not_4g_aligned(u64 val)
      {
              if (val & GENMASK(31, 0))
                      return -EINVAL;

              return 0;
      }

It could be compiled to::

      error_if_not_4g_aligned:
              CBNZ    w0, .Lnot_aligned
              RET                     // bits [31:0] are zero, bits
                                      // [63:32] are UNKNOWN
      .Lnot_aligned:
              MOV    x0, #-EINVAL
              RET

When passing 0x2_0000_0000 to it, the return value may be recorded as
0x2_0000_0000 instead of 0.
”h]”(h¸)”}”(hXí  In certain procedure call standards, such as arm64's AAPCS64, when a
type is smaller than a GPR, it is the responsibility of the consumer
to perform the narrowing, and the upper bits may contain UNKNOWN values.
Therefore, it is advisable to check the code for such cases. For instance,
when using a u8 in a 64-bit GPR, bits [63:8] may contain arbitrary values,
especially when larger types are truncated, whether explicitly or implicitly.
Here are some specific cases to illustrate this point:”h]”hXï  In certain procedure call standards, such as arm64â€™s AAPCS64, when a
type is smaller than a GPR, it is the responsibility of the consumer
to perform the narrowing, and the upper bits may contain UNKNOWN values.
Therefore, it is advisable to check the code for such cases. For instance,
when using a u8 in a 64-bit GPR, bits [63:8] may contain arbitrary values,
especially when larger types are truncated, whether explicitly or implicitly.
Here are some specific cases to illustrate this point:”…””}”(hj/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj}/  ubh¸)”}”(hŒ**Case One**:”h]”(hŒstrong”“”)”}”(hŒ**Case One**”h]”hŒCase One”…””}”(hj•/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j“/  hj/  ubhŒ:”…””}”(hj/  hžhhŸNh Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M"hj}/  ubh¸)”}”(hŒ1The function narrow_to_u8 is defined as follows::”h]”hŒ0The function narrow_to_u8 is defined as follows:”…””}”(hj­/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M$hj}/  ubj   )”}”(hŒPu8 narrow_to_u8(u64 val)
{
        // implicitly truncated
        return val;
}”h]”hŒPu8 narrow_to_u8(u64 val)
{
        // implicitly truncated
        return val;
}”…””}”hj»/  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M&hj}/  ubh¸)”}”(hŒIt may be compiled to::”h]”hŒIt may be compiled to:”…””}”(hjÉ/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M,hj}/  ubj   )”}”(hŒDnarrow_to_u8:
        < ... ftrace instrumentation ... >
        RET”h]”hŒDnarrow_to_u8:
        < ... ftrace instrumentation ... >
        RET”…””}”hj×/  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M.hj}/  ubh¸)”}”(hŒ~If you pass 0x123456789abcdef to this function and want to narrow it,
it may be recorded as 0x123456789abcdef instead of 0xef.”h]”hŒ~If you pass 0x123456789abcdef to this function and want to narrow it,
it may be recorded as 0x123456789abcdef instead of 0xef.”…””}”(hjå/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M2hj}/  ubh¸)”}”(hŒ**Case Two**:”h]”(j”/  )”}”(hŒ**Case Two**”h]”hŒCase Two”…””}”(hj÷/  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j“/  hjó/  ubhŒ:”…””}”(hjó/  hžhhŸNh Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M5hj}/  ubh¸)”}”(hŒ<The function error_if_not_4g_aligned is defined as follows::”h]”hŒ;The function error_if_not_4g_aligned is defined as follows:”…””}”(hj0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M7hj}/  ubj   )”}”(hŒ}int error_if_not_4g_aligned(u64 val)
{
        if (val & GENMASK(31, 0))
                return -EINVAL;

        return 0;
}”h]”hŒ}int error_if_not_4g_aligned(u64 val)
{
        if (val & GENMASK(31, 0))
                return -EINVAL;

        return 0;
}”…””}”hj0  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M9hj}/  ubh¸)”}”(hŒIt could be compiled to::”h]”hŒIt could be compiled to:”…””}”(hj+0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MAhj}/  ubj   )”}”(hŒæerror_if_not_4g_aligned:
        CBNZ    w0, .Lnot_aligned
        RET                     // bits [31:0] are zero, bits
                                // [63:32] are UNKNOWN
.Lnot_aligned:
        MOV    x0, #-EINVAL
        RET”h]”hŒæerror_if_not_4g_aligned:
        CBNZ    w0, .Lnot_aligned
        RET                     // bits [31:0] are zero, bits
                                // [63:32] are UNKNOWN
.Lnot_aligned:
        MOV    x0, #-EINVAL
        RET”…””}”hj90  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MChj}/  ubh¸)”}”(hŒaWhen passing 0x2_0000_0000 to it, the return value may be recorded as
0x2_0000_0000 instead of 0.”h]”hŒaWhen passing 0x2_0000_0000 to it, the return value may be recorded as
0x2_0000_0000 instead of 0.”…””}”(hjG0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MKhj}/  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1j  hjJ/  hžhhŸh¶h Nubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h MhjO*  hžhubh¸)”}”(hŒñYou can put some comments on specific functions by using
trace_printk() For example, if you want to put a comment inside
the __might_sleep() function, you just have to include
<linux/ftrace.h> and call trace_printk() inside __might_sleep()::”h]”hŒðYou can put some comments on specific functions by using
trace_printk() For example, if you want to put a comment inside
the __might_sleep() function, you just have to include
<linux/ftrace.h> and call trace_printk() inside __might_sleep():”…””}”(hja0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MNhjO*  hžhubj   )”}”(hŒ trace_printk("I'm a comment!\n")”h]”hŒ trace_printk("I'm a comment!\n")”…””}”hjo0  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MShjO*  hžhubh¸)”}”(hŒwill produce::”h]”hŒwill produce:”…””}”(hj}0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MUhjO*  hžhubj   )”}”(hŒˆ1)               |             __might_sleep() {
1)               |                /* I'm a comment! */
1)   1.449 us    |             }”h]”hŒˆ1)               |             __might_sleep() {
1)               |                /* I'm a comment! */
1)   1.449 us    |             }”…””}”hj‹0  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MWhjO*  hžhubh¸)”}”(hŒYou might find other useful features for this tracer in the
following "dynamic ftrace" section such as tracing only specific
functions or tasks.”h]”hŒ”You might find other useful features for this tracer in the
following â€œdynamic ftraceâ€ section such as tracing only specific
functions or tasks.”…””}”(hj™0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M\hjO*  hžhubeh}”(h]”Œfunction-graph-tracer”ah ]”h"]”Œfunction graph tracer”ah$]”h&]”uh1h¡hh£hžhhŸh¶h M

ubh¢)”}”(hhh]”(h§)”}”(hŒdynamic ftrace”h]”hŒdynamic ftrace”…””}”(hj²0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj¯0  hžhhŸh¶h Maubh¸)”}”(hXi  If CONFIG_DYNAMIC_FTRACE is set, the system will run with
virtually no overhead when function tracing is disabled. The way
this works is the mcount function call (placed at the start of
every kernel function, produced by the -pg switch in gcc),
starts of pointing to a simple return. (Enabling FTRACE will
include the -pg switch in the compiling of the kernel.)”h]”hXi  If CONFIG_DYNAMIC_FTRACE is set, the system will run with
virtually no overhead when function tracing is disabled. The way
this works is the mcount function call (placed at the start of
every kernel function, produced by the -pg switch in gcc),
starts of pointing to a simple return. (Enabling FTRACE will
include the -pg switch in the compiling of the kernel.)”…””}”(hjÀ0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mchj¯0  hžhubh¸)”}”(hX’  At compile time every C file object is run through the
recordmcount program (located in the scripts directory). This
program will parse the ELF headers in the C object to find all
the locations in the .text section that call mcount. Starting
with gcc version 4.6, the -mfentry has been added for x86, which
calls "__fentry__" instead of "mcount". Which is called before
the creation of the stack frame.”h]”hXš  At compile time every C file object is run through the
recordmcount program (located in the scripts directory). This
program will parse the ELF headers in the C object to find all
the locations in the .text section that call mcount. Starting
with gcc version 4.6, the -mfentry has been added for x86, which
calls â€œ__fentry__â€ instead of â€œmcountâ€. Which is called before
the creation of the stack frame.”…””}”(hjÎ0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mjhj¯0  hžhubh¸)”}”(hŒÞNote, not all sections are traced. They may be prevented by either
a notrace, or blocked another way and all inline functions are not
traced. Check the "available_filter_functions" file to see what functions
can be traced.”h]”hŒâNote, not all sections are traced. They may be prevented by either
a notrace, or blocked another way and all inline functions are not
traced. Check the â€œavailable_filter_functionsâ€ file to see what functions
can be traced.”…””}”(hjÜ0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mrhj¯0  hžhubh¸)”}”(hX!  A section called "__mcount_loc" is created that holds
references to all the mcount/fentry call sites in the .text section.
The recordmcount program re-links this section back into the
original object. The final linking stage of the kernel will add all these
references into a single table.”h]”hX%  A section called â€œ__mcount_locâ€ is created that holds
references to all the mcount/fentry call sites in the .text section.
The recordmcount program re-links this section back into the
original object. The final linking stage of the kernel will add all these
references into a single table.”…””}”(hjê0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mwhj¯0  hžhubh¸)”}”(hXÔ  On boot up, before SMP is initialized, the dynamic ftrace code
scans this table and updates all the locations into nops. It
also records the locations, which are added to the
available_filter_functions list.  Modules are processed as they
are loaded and before they are executed.  When a module is
unloaded, it also removes its functions from the ftrace function
list. This is automatic in the module unload code, and the
module author does not need to worry about it.”h]”hXÔ  On boot up, before SMP is initialized, the dynamic ftrace code
scans this table and updates all the locations into nops. It
also records the locations, which are added to the
available_filter_functions list.  Modules are processed as they
are loaded and before they are executed.  When a module is
unloaded, it also removes its functions from the ftrace function
list. This is automatic in the module unload code, and the
module author does not need to worry about it.”…””}”(hjø0  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M}hj¯0  hžhubh¸)”}”(hXâ  When tracing is enabled, the process of modifying the function
tracepoints is dependent on architecture. The old method is to use
kstop_machine to prevent races with the CPUs executing code being
modified (which can cause the CPU to do undesirable things, especially
if the modified code crosses cache (or page) boundaries), and the nops are
patched back to calls. But this time, they do not call mcount
(which is just a function stub). They now call into the ftrace
infrastructure.”h]”hXâ  When tracing is enabled, the process of modifying the function
tracepoints is dependent on architecture. The old method is to use
kstop_machine to prevent races with the CPUs executing code being
modified (which can cause the CPU to do undesirable things, especially
if the modified code crosses cache (or page) boundaries), and the nops are
patched back to calls. But this time, they do not call mcount
(which is just a function stub). They now call into the ftrace
infrastructure.”…””}”(hj1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M†hj¯0  hžhubh¸)”}”(hX%  The new method of modifying the function tracepoints is to place
a breakpoint at the location to be modified, sync all CPUs, modify
the rest of the instruction not covered by the breakpoint. Sync
all CPUs again, and then remove the breakpoint with the finished
version to the ftrace call site.”h]”hX%  The new method of modifying the function tracepoints is to place
a breakpoint at the location to be modified, sync all CPUs, modify
the rest of the instruction not covered by the breakpoint. Sync
all CPUs again, and then remove the breakpoint with the finished
version to the ftrace call site.”…””}”(hj1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj¯0  hžhubh¸)”}”(hŒ»Some archs do not even need to monkey around with the synchronization,
and can just slap the new code on top of the old without any
problems with other CPUs executing it at the same time.”h]”hŒ»Some archs do not even need to monkey around with the synchronization,
and can just slap the new code on top of the old without any
problems with other CPUs executing it at the same time.”…””}”(hj"1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M•hj¯0  hžhubh¸)”}”(hŒÈOne special side-effect to the recording of the functions being
traced is that we can now selectively choose which functions we
wish to trace and which ones we want the mcount calls to remain
as nops.”h]”hŒÈOne special side-effect to the recording of the functions being
traced is that we can now selectively choose which functions we
wish to trace and which ones we want the mcount calls to remain
as nops.”…””}”(hj01  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M™hj¯0  hžhubh¸)”}”(hŒhTwo files are used, one for enabling and one for disabling the
tracing of specified functions. They are:”h]”hŒhTwo files are used, one for enabling and one for disabling the
tracing of specified functions. They are:”…””}”(hj>1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mžhj¯0  hžhubjT  )”}”(hŒset_ftrace_filter
”h]”h¸)”}”(hŒset_ftrace_filter”h]”hŒset_ftrace_filter”…””}”(hjP1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¡hjL1  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M¡hj¯0  hžhubh¸)”}”(hŒand”h]”hŒand”…””}”(hjd1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M£hj¯0  hžhubjT  )”}”(hŒset_ftrace_notrace
”h]”h¸)”}”(hŒset_ftrace_notrace”h]”hŒset_ftrace_notrace”…””}”(hjv1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¥hjr1  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h M¥hj¯0  hžhubh¸)”}”(hŒKA list of available functions that you can add to these files is
listed in:”h]”hŒKA list of available functions that you can add to these files is
listed in:”…””}”(hjŠ1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M§hj¯0  hžhubjT  )”}”(hŒavailable_filter_functions
”h]”h¸)”}”(hŒavailable_filter_functions”h]”hŒavailable_filter_functions”…””}”(hjœ1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mªhj˜1  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mªhj¯0  hžhubj   )”}”(hŒ‹# cat available_filter_functions
put_prev_task_idle
kmem_cache_create
pick_next_task_rt
cpus_read_lock
pick_next_task_fair
mutex_lock
[...]”h]”hŒ‹# cat available_filter_functions
put_prev_task_idle
kmem_cache_create
pick_next_task_rt
cpus_read_lock
pick_next_task_fair
mutex_lock
[...]”…””}”hj°1  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M®hj¯0  hžhubh¸)”}”(hŒ@If I am only interested in sys_nanosleep and hrtimer_interrupt::”h]”hŒ?If I am only interested in sys_nanosleep and hrtimer_interrupt:”…””}”(hj¾1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M·hj¯0  hžhubj   )”}”(hX  # echo sys_nanosleep hrtimer_interrupt > set_ftrace_filter
# echo function > current_tracer
# echo 1 > tracing_on
# usleep 1
# echo 0 > tracing_on
# cat trace
# tracer: function
#
# entries-in-buffer/entries-written: 5/5   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
          usleep-2665  [001] ....  4186.475355: sys_nanosleep <-system_call_fastpath
          <idle>-0     [001] d.h1  4186.475409: hrtimer_interrupt <-smp_apic_timer_interrupt
          usleep-2665  [001] d.h1  4186.475426: hrtimer_interrupt <-smp_apic_timer_interrupt
          <idle>-0     [003] d.h1  4186.475426: hrtimer_interrupt <-smp_apic_timer_interrupt
          <idle>-0     [002] d.h1  4186.475427: hrtimer_interrupt <-smp_apic_timer_interrupt”h]”hX  # echo sys_nanosleep hrtimer_interrupt > set_ftrace_filter
# echo function > current_tracer
# echo 1 > tracing_on
# usleep 1
# echo 0 > tracing_on
# cat trace
# tracer: function
#
# entries-in-buffer/entries-written: 5/5   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
          usleep-2665  [001] ....  4186.475355: sys_nanosleep <-system_call_fastpath
          <idle>-0     [001] d.h1  4186.475409: hrtimer_interrupt <-smp_apic_timer_interrupt
          usleep-2665  [001] d.h1  4186.475426: hrtimer_interrupt <-smp_apic_timer_interrupt
          <idle>-0     [003] d.h1  4186.475426: hrtimer_interrupt <-smp_apic_timer_interrupt
          <idle>-0     [002] d.h1  4186.475427: hrtimer_interrupt <-smp_apic_timer_interrupt”…””}”hjÌ1  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M¹hj¯0  hžhubh¸)”}”(hŒATo see which functions are being traced, you can cat the file:
::”h]”hŒ>To see which functions are being traced, you can cat the file:”…””}”(hjÚ1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÐhj¯0  hžhubj   )”}”(hŒ7# cat set_ftrace_filter
hrtimer_interrupt
sys_nanosleep”h]”hŒ7# cat set_ftrace_filter
hrtimer_interrupt
sys_nanosleep”…””}”hjè1  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MÓhj¯0  hžhubh¸)”}”(hŒDPerhaps this is not enough. The filters also allow glob(7) matching.”h]”hŒDPerhaps this is not enough. The filters also allow glob(7) matching.”…””}”(hjö1  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MØhj¯0  hžhubjT  )”}”(hX  ``<match>*``
      will match functions that begin with <match>
``*<match>``
      will match functions that end with <match>
``*<match>*``
      will match functions that have <match> in it
``<match1>*<match2>``
      will match functions that begin with <match1> and end with <match2>
”h]”jT  )”}”(hhh]”(jY  )”}”(hŒ9``<match>*``
will match functions that begin with <match>”h]”(j_  )”}”(hŒ``<match>*``”h]”hŒliteral”“”)”}”(hj2  h]”hŒ<match>*”…””}”(hj2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j2  hj2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MÚhj2  ubjo  )”}”(hhh]”h¸)”}”(hŒ,will match functions that begin with <match>”h]”hŒ,will match functions that begin with <match>”…””}”(hj+2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÛhj(2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj2  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MÚhj2  ubjY  )”}”(hŒ7``*<match>``
will match functions that end with <match>”h]”(j_  )”}”(hŒ``*<match>``”h]”j2  )”}”(hjK2  h]”hŒ*<match>”…””}”(hjM2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j2  hjI2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MÜhjE2  ubjo  )”}”(hhh]”h¸)”}”(hŒ*will match functions that end with <match>”h]”hŒ*will match functions that end with <match>”…””}”(hjc2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÝhj`2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjE2  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MÜhj2  ubjY  )”}”(hŒ:``*<match>*``
will match functions that have <match> in it”h]”(j_  )”}”(hŒ``*<match>*``”h]”j2  )”}”(hjƒ2  h]”hŒ	*<match>*”…””}”(hj…2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j2  hj2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h MÞhj}2  ubjo  )”}”(hhh]”h¸)”}”(hŒ,will match functions that have <match> in it”h]”hŒ,will match functions that have <match> in it”…””}”(hj›2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mßhj˜2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hj}2  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h MÞhj2  ubjY  )”}”(hŒZ``<match1>*<match2>``
will match functions that begin with <match1> and end with <match2>
”h]”(j_  )”}”(hŒ``<match1>*<match2>``”h]”j2  )”}”(hj»2  h]”hŒ<match1>*<match2>”…””}”(hj½2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1j2  hj¹2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j^  hŸh¶h Máhjµ2  ubjo  )”}”(hhh]”h¸)”}”(hŒCwill match functions that begin with <match1> and end with <match2>”h]”hŒCwill match functions that begin with <match1> and end with <match2>”…””}”(hjÓ2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MáhjÐ2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jn  hjµ2  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jX  hŸh¶h Máhj2  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hj2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÚhj¯0  hžhubhŒnote”“”)”}”(hŒIt is better to use quotes to enclose the wild cards,
otherwise the shell may expand the parameters into names
of files in the local directory.”h]”h¸)”}”(hŒIt is better to use quotes to enclose the wild cards,
otherwise the shell may expand the parameters into names
of files in the local directory.”h]”hŒIt is better to use quotes to enclose the wild cards,
otherwise the shell may expand the parameters into names
of files in the local directory.”…””}”(hjÿ2  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mähjû2  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jù2  hj¯0  hžhhŸh¶h Nubj   )”}”(hŒ&# echo 'hrtimer_*' > set_ftrace_filter”h]”hŒ&# echo 'hrtimer_*' > set_ftrace_filter”…””}”hj3  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mêhj¯0  hžhubh¸)”}”(hŒ
Produces::”h]”hŒ	Produces:”…””}”(hj!3  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mìhj¯0  hžhubj   )”}”(hXŠ  # tracer: function
#
# entries-in-buffer/entries-written: 897/897   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
          <idle>-0     [003] dN.1  4228.547803: hrtimer_cancel <-tick_nohz_idle_exit
          <idle>-0     [003] dN.1  4228.547804: hrtimer_try_to_cancel <-hrtimer_cancel
          <idle>-0     [003] dN.2  4228.547805: hrtimer_force_reprogram <-__remove_hrtimer
          <idle>-0     [003] dN.1  4228.547805: hrtimer_forward <-tick_nohz_idle_exit
          <idle>-0     [003] dN.1  4228.547805: hrtimer_start_range_ns <-hrtimer_start_expires.constprop.11
          <idle>-0     [003] d..1  4228.547858: hrtimer_get_next_event <-get_next_timer_interrupt
          <idle>-0     [003] d..1  4228.547859: hrtimer_start <-__tick_nohz_idle_enter
          <idle>-0     [003] d..2  4228.547860: hrtimer_force_reprogram <-__rem”h]”hXŠ  # tracer: function
#
# entries-in-buffer/entries-written: 897/897   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
          <idle>-0     [003] dN.1  4228.547803: hrtimer_cancel <-tick_nohz_idle_exit
          <idle>-0     [003] dN.1  4228.547804: hrtimer_try_to_cancel <-hrtimer_cancel
          <idle>-0     [003] dN.2  4228.547805: hrtimer_force_reprogram <-__remove_hrtimer
          <idle>-0     [003] dN.1  4228.547805: hrtimer_forward <-tick_nohz_idle_exit
          <idle>-0     [003] dN.1  4228.547805: hrtimer_start_range_ns <-hrtimer_start_expires.constprop.11
          <idle>-0     [003] d..1  4228.547858: hrtimer_get_next_event <-get_next_timer_interrupt
          <idle>-0     [003] d..1  4228.547859: hrtimer_start <-__tick_nohz_idle_enter
          <idle>-0     [003] d..2  4228.547860: hrtimer_force_reprogram <-__rem”…””}”hj/3  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mîhj¯0  hžhubh¸)”}”(hŒ)Notice that we lost the sys_nanosleep.
::”h]”hŒ&Notice that we lost the sys_nanosleep.”…””}”(hj=3  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj¯0  hžhubj   )”}”(hX>  # cat set_ftrace_filter
hrtimer_run_queues
hrtimer_run_pending
hrtimer_setup
hrtimer_cancel
hrtimer_try_to_cancel
hrtimer_forward
hrtimer_start
hrtimer_reprogram
hrtimer_force_reprogram
hrtimer_get_next_event
hrtimer_interrupt
hrtimer_nanosleep
hrtimer_wakeup
hrtimer_get_remaining
hrtimer_get_res
hrtimer_init_sleeper”h]”hX>  # cat set_ftrace_filter
hrtimer_run_queues
hrtimer_run_pending
hrtimer_setup
hrtimer_cancel
hrtimer_try_to_cancel
hrtimer_forward
hrtimer_start
hrtimer_reprogram
hrtimer_force_reprogram
hrtimer_get_next_event
hrtimer_interrupt
hrtimer_nanosleep
hrtimer_wakeup
hrtimer_get_remaining
hrtimer_get_res
hrtimer_init_sleeper”…””}”hjK3  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhj¯0  hžhubh¸)”}”(hŒ‚This is because the '>' and '>>' act just like they do in bash.
To rewrite the filters, use '>'
To append to the filters, use '>>'”h]”hŒ’This is because the â€˜>â€™ and â€˜>>â€™ act just like they do in bash.
To rewrite the filters, use â€˜>â€™
To append to the filters, use â€˜>>â€™”…””}”(hjY3  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj¯0  hžhubh¸)”}”(hŒDTo clear out a filter so that all functions will be recorded
again::”h]”hŒCTo clear out a filter so that all functions will be recorded
again:”…””}”(hjg3  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj¯0  hžhubj   )”}”(hŒ4# echo > set_ftrace_filter
# cat set_ftrace_filter
#”h]”hŒ4# echo > set_ftrace_filter
# cat set_ftrace_filter
#”…””}”hju3  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhj¯0  hžhubh¸)”}”(hŒAgain, now we want to append.”h]”hŒAgain, now we want to append.”…””}”(hjƒ3  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M#hj¯0  hžhubj   )”}”(hXÃ  # echo sys_nanosleep > set_ftrace_filter
# cat set_ftrace_filter
sys_nanosleep
# echo 'hrtimer_*' >> set_ftrace_filter
# cat set_ftrace_filter
hrtimer_run_queues
hrtimer_run_pending
hrtimer_setup
hrtimer_cancel
hrtimer_try_to_cancel
hrtimer_forward
hrtimer_start
hrtimer_reprogram
hrtimer_force_reprogram
hrtimer_get_next_event
hrtimer_interrupt
sys_nanosleep
hrtimer_nanosleep
hrtimer_wakeup
hrtimer_get_remaining
hrtimer_get_res
hrtimer_init_sleeper”h]”hXÃ  # echo sys_nanosleep > set_ftrace_filter
# cat set_ftrace_filter
sys_nanosleep
# echo 'hrtimer_*' >> set_ftrace_filter
# cat set_ftrace_filter
hrtimer_run_queues
hrtimer_run_pending
hrtimer_setup
hrtimer_cancel
hrtimer_try_to_cancel
hrtimer_forward
hrtimer_start
hrtimer_reprogram
hrtimer_force_reprogram
hrtimer_get_next_event
hrtimer_interrupt
sys_nanosleep
hrtimer_nanosleep
hrtimer_wakeup
hrtimer_get_remaining
hrtimer_get_res
hrtimer_init_sleeper”…””}”hj‘3  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M'hj¯0  hžhubh¸)”}”(hŒEThe set_ftrace_notrace prevents those functions from being
traced.
::”h]”hŒBThe set_ftrace_notrace prevents those functions from being
traced.”…””}”(hjŸ3  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M?hj¯0  hžhubj   )”}”(hŒ0# echo '*preempt*' '*lock*' > set_ftrace_notrace”h]”hŒ0# echo '*preempt*' '*lock*' > set_ftrace_notrace”…””}”hj­3  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MChj¯0  hžhubh¸)”}”(hŒ
Produces::”h]”hŒ	Produces:”…””}”(hj»3  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MEhj¯0  hžhubj   )”}”(hX†  # tracer: function
#
# entries-in-buffer/entries-written: 39608/39608   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
            bash-1994  [000] ....  4342.324896: file_ra_state_init <-do_dentry_open
            bash-1994  [000] ....  4342.324897: open_check_o_direct <-do_last
            bash-1994  [000] ....  4342.324897: ima_file_check <-do_last
            bash-1994  [000] ....  4342.324898: process_measurement <-ima_file_check
            bash-1994  [000] ....  4342.324898: ima_get_action <-process_measurement
            bash-1994  [000] ....  4342.324898: ima_match_policy <-ima_get_action
            bash-1994  [000] ....  4342.324899: do_truncate <-do_last
            bash-1994  [000] ....  4342.324899: setattr_should_drop_suidgid <-do_truncate
            bash-1994  [000] ....  4342.324899: notify_change <-do_truncate
            bash-1994  [000] ....  4342.324900: current_fs_time <-notify_change
            bash-1994  [000] ....  4342.324900: current_kernel_time <-current_fs_time
            bash-1994  [000] ....  4342.324900: timespec_trunc <-current_fs_time”h]”hX†  # tracer: function
#
# entries-in-buffer/entries-written: 39608/39608   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
            bash-1994  [000] ....  4342.324896: file_ra_state_init <-do_dentry_open
            bash-1994  [000] ....  4342.324897: open_check_o_direct <-do_last
            bash-1994  [000] ....  4342.324897: ima_file_check <-do_last
            bash-1994  [000] ....  4342.324898: process_measurement <-ima_file_check
            bash-1994  [000] ....  4342.324898: ima_get_action <-process_measurement
            bash-1994  [000] ....  4342.324898: ima_match_policy <-ima_get_action
            bash-1994  [000] ....  4342.324899: do_truncate <-do_last
            bash-1994  [000] ....  4342.324899: setattr_should_drop_suidgid <-do_truncate
            bash-1994  [000] ....  4342.324899: notify_change <-do_truncate
            bash-1994  [000] ....  4342.324900: current_fs_time <-notify_change
            bash-1994  [000] ....  4342.324900: current_kernel_time <-current_fs_time
            bash-1994  [000] ....  4342.324900: timespec_trunc <-current_fs_time”…””}”hjÉ3  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MGhj¯0  hžhubh¸)”}”(hŒ8We can see that there's no more lock or preempt tracing.”h]”hŒ:We can see that thereâ€™s no more lock or preempt tracing.”…””}”(hj×3  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M_hj¯0  hžhubeh}”(h]”Œdynamic-ftrace”ah ]”h"]”Œdynamic ftrace”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Maubh¢)”}”(hhh]”(h§)”}”(hŒ$Selecting function filters via index”h]”hŒ$Selecting function filters via index”…””}”(hjð3  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjí3  hžhhŸh¶h Mbubh¸)”}”(hXô  Because processing of strings is expensive (the address of the function
needs to be looked up before comparing to the string being passed in),
an index can be used as well to enable functions. This is useful in the
case of setting thousands of specific functions at a time. By passing
in a list of numbers, no string processing will occur. Instead, the function
at the specific location in the internal array (which corresponds to the
functions in the "available_filter_functions" file), is selected.”h]”hXø  Because processing of strings is expensive (the address of the function
needs to be looked up before comparing to the string being passed in),
an index can be used as well to enable functions. This is useful in the
case of setting thousands of specific functions at a time. By passing
in a list of numbers, no string processing will occur. Instead, the function
at the specific location in the internal array (which corresponds to the
functions in the â€œavailable_filter_functionsâ€ file), is selected.”…””}”(hjþ3  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mdhjí3  hžhubj   )”}”(hŒ# echo 1 > set_ftrace_filter”h]”hŒ# echo 1 > set_ftrace_filter”…””}”hj4  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mnhjí3  hžhubh¸)”}”(hŒEWill select the first function listed in "available_filter_functions"”h]”hŒIWill select the first function listed in â€œavailable_filter_functionsâ€”…””}”(hj4  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mphjí3  hžhubj   )”}”(hX  # head -1 available_filter_functions
trace_initcall_finish_cb

# cat set_ftrace_filter
trace_initcall_finish_cb

# head -50 available_filter_functions | tail -1
x86_pmu_commit_txn

# echo 1 50 > set_ftrace_filter
# cat set_ftrace_filter
trace_initcall_finish_cb
x86_pmu_commit_txn”h]”hX  # head -1 available_filter_functions
trace_initcall_finish_cb

# cat set_ftrace_filter
trace_initcall_finish_cb

# head -50 available_filter_functions | tail -1
x86_pmu_commit_txn

# echo 1 50 > set_ftrace_filter
# cat set_ftrace_filter
trace_initcall_finish_cb
x86_pmu_commit_txn”…””}”hj(4  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mthjí3  hžhubeh}”(h]”Œ$selecting-function-filters-via-index”ah ]”h"]”Œ$selecting function filters via index”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Mbubh¢)”}”(hhh]”(h§)”}”(hŒ-Dynamic ftrace with the function graph tracer”h]”hŒ-Dynamic ftrace with the function graph tracer”…””}”(hjA4  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj>4  hžhhŸh¶h Mƒubh¸)”}”(hŒ´Although what has been explained above concerns both the
function tracer and the function-graph-tracer, there are some
special features only available in the function-graph tracer.”h]”hŒ´Although what has been explained above concerns both the
function tracer and the function-graph-tracer, there are some
special features only available in the function-graph tracer.”…””}”(hjO4  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M…hj>4  hžhubh¸)”}”(hŒxIf you want to trace only one function and all of its children,
you just have to echo its name into set_graph_function::”h]”hŒwIf you want to trace only one function and all of its children,
you just have to echo its name into set_graph_function:”…””}”(hj]4  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M‰hj>4  hžhubj   )”}”(hŒ$echo __do_fault > set_graph_function”h]”hŒ$echo __do_fault > set_graph_function”…””}”hjk4  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MŒhj>4  hžhubh¸)”}”(hŒJwill produce the following "expanded" trace of the __do_fault()
function::”h]”hŒMwill produce the following â€œexpandedâ€ trace of the __do_fault()
function:”…””}”(hjy4  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MŽhj>4  hžhubj   )”}”(hXÅ  0)               |  __do_fault() {
0)               |    filemap_fault() {
0)               |      find_lock_page() {
0)   0.804 us    |        find_get_page();
0)               |        __might_sleep() {
0)   1.329 us    |        }
0)   3.904 us    |      }
0)   4.979 us    |    }
0)   0.653 us    |    _spin_lock();
0)   0.578 us    |    page_add_file_rmap();
0)   0.525 us    |    native_set_pte_at();
0)   0.585 us    |    _spin_unlock();
0)               |    unlock_page() {
0)   0.541 us    |      page_waitqueue();
0)   0.639 us    |      __wake_up_bit();
0)   2.786 us    |    }
0) + 14.237 us   |  }
0)               |  __do_fault() {
0)               |    filemap_fault() {
0)               |      find_lock_page() {
0)   0.698 us    |        find_get_page();
0)               |        __might_sleep() {
0)   1.412 us    |        }
0)   3.950 us    |      }
0)   5.098 us    |    }
0)   0.631 us    |    _spin_lock();
0)   0.571 us    |    page_add_file_rmap();
0)   0.526 us    |    native_set_pte_at();
0)   0.586 us    |    _spin_unlock();
0)               |    unlock_page() {
0)   0.533 us    |      page_waitqueue();
0)   0.638 us    |      __wake_up_bit();
0)   2.793 us    |    }
0) + 14.012 us   |  }”h]”hXÅ  0)               |  __do_fault() {
0)               |    filemap_fault() {
0)               |      find_lock_page() {
0)   0.804 us    |        find_get_page();
0)               |        __might_sleep() {
0)   1.329 us    |        }
0)   3.904 us    |      }
0)   4.979 us    |    }
0)   0.653 us    |    _spin_lock();
0)   0.578 us    |    page_add_file_rmap();
0)   0.525 us    |    native_set_pte_at();
0)   0.585 us    |    _spin_unlock();
0)               |    unlock_page() {
0)   0.541 us    |      page_waitqueue();
0)   0.639 us    |      __wake_up_bit();
0)   2.786 us    |    }
0) + 14.237 us   |  }
0)               |  __do_fault() {
0)               |    filemap_fault() {
0)               |      find_lock_page() {
0)   0.698 us    |        find_get_page();
0)               |        __might_sleep() {
0)   1.412 us    |        }
0)   3.950 us    |      }
0)   5.098 us    |    }
0)   0.631 us    |    _spin_lock();
0)   0.571 us    |    page_add_file_rmap();
0)   0.526 us    |    native_set_pte_at();
0)   0.586 us    |    _spin_unlock();
0)               |    unlock_page() {
0)   0.533 us    |      page_waitqueue();
0)   0.638 us    |      __wake_up_bit();
0)   2.793 us    |    }
0) + 14.012 us   |  }”…””}”hj‡4  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M‘hj>4  hžhubh¸)”}”(hŒ/You can also expand several functions at once::”h]”hŒ.You can also expand several functions at once:”…””}”(hj•4  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M´hj>4  hžhubj   )”}”(hŒGecho sys_open > set_graph_function
echo sys_close >> set_graph_function”h]”hŒGecho sys_open > set_graph_function
echo sys_close >> set_graph_function”…””}”hj£4  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M¶hj>4  hžhubh¸)”}”(hŒYNow if you want to go back to trace all functions you can clear
this special filter via::”h]”hŒXNow if you want to go back to trace all functions you can clear
this special filter via:”…””}”(hj±4  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¹hj>4  hžhubj   )”}”(hŒecho > set_graph_function”h]”hŒecho > set_graph_function”…””}”hj¿4  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M¼hj>4  hžhubeh}”(h]”Œ-dynamic-ftrace-with-the-function-graph-tracer”ah ]”h"]”Œ-dynamic ftrace with the function graph tracer”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Mƒubh¢)”}”(hhh]”(h§)”}”(hŒftrace_enabled”h]”hŒftrace_enabled”…””}”(hjØ4  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjÕ4  hžhhŸh¶h MÀubh¸)”}”(hX±  Note, the proc sysctl ftrace_enable is a big on/off switch for the
function tracer. By default it is enabled (when function tracing is
enabled in the kernel). If it is disabled, all function tracing is
disabled. This includes not only the function tracers for ftrace, but
also for any other uses (perf, kprobes, stack tracing, profiling, etc). It
cannot be disabled if there is a callback with FTRACE_OPS_FL_PERMANENT set
registered.”h]”hX±  Note, the proc sysctl ftrace_enable is a big on/off switch for the
function tracer. By default it is enabled (when function tracing is
enabled in the kernel). If it is disabled, all function tracing is
disabled. This includes not only the function tracers for ftrace, but
also for any other uses (perf, kprobes, stack tracing, profiling, etc). It
cannot be disabled if there is a callback with FTRACE_OPS_FL_PERMANENT set
registered.”…””}”(hjæ4  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&•      ]”uh1h·hŸh¶h MÂhjÕ4  hžhubh¸)”}”(hŒPlease disable this with care.”h]”hŒPlease disable this with care.”…””}”(hjô4  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÊhjÕ4  hžhubh¸)”}”(hŒ(This can be disable (and enabled) with::”h]”hŒ'This can be disable (and enabled) with:”…””}”(hj5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÌhjÕ4  hžhubj   )”}”(hŒ˜ sysctl kernel.ftrace_enabled=0
 sysctl kernel.ftrace_enabled=1

or

 echo 0 > /proc/sys/kernel/ftrace_enabled
 echo 1 > /proc/sys/kernel/ftrace_enabled”h]”hŒ˜ sysctl kernel.ftrace_enabled=0
 sysctl kernel.ftrace_enabled=1

or

 echo 0 > /proc/sys/kernel/ftrace_enabled
 echo 1 > /proc/sys/kernel/ftrace_enabled”…””}”hj5  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MÎhjÕ4  hžhubeh}”(h]”Œftrace-enabled”ah ]”h"]”Œftrace_enabled”ah$]”h&]”uh1h¡hh£hžhhŸh¶h MÀubh¢)”}”(hhh]”(h§)”}”(hŒFilter commands”h]”hŒFilter commands”…””}”(hj)5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj&5  hžhhŸh¶h MØubh¸)”}”(hŒkA few commands are supported by the set_ftrace_filter interface.
Trace commands have the following format::”h]”hŒjA few commands are supported by the set_ftrace_filter interface.
Trace commands have the following format:”…””}”(hj75  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÚhj&5  hžhubj   )”}”(hŒ <function>:<command>:<parameter>”h]”hŒ <function>:<command>:<parameter>”…””}”hjE5  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MÝhj&5  hžhubh¸)”}”(hŒ%The following commands are supported:”h]”hŒ%The following commands are supported:”…””}”(hjS5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mßhj&5  hžhubj{  )”}”(hhh]”(j€  )”}”(hX™  mod:
This command enables function filtering per module. The
parameter defines the module. For example, if only the write*
functions in the ext3 module are desired, run:

 echo 'write*:mod:ext3' > set_ftrace_filter

This command interacts with the filter in the same way as
filtering based on function names. Thus, adding more functions
in a different module is accomplished by appending (>>) to the
filter file. Remove specific module functions by prepending
'!'::

 echo '!writeback*:mod:ext3' >> set_ftrace_filter

Mod command supports module globbing. Disable tracing for all
functions except a specific module::

 echo '!*:mod:!ext3' >> set_ftrace_filter

Disable tracing for all modules, but still trace kernel::

 echo '!*:mod:*' >> set_ftrace_filter

Enable filter only for kernel::

 echo '*write*:mod:!*' >> set_ftrace_filter

Enable filter for module globbing::

 echo '*write*:mod:*snd*' >> set_ftrace_filter
”h]”(h¸)”}”(hŒ©mod:
This command enables function filtering per module. The
parameter defines the module. For example, if only the write*
functions in the ext3 module are desired, run:”h]”hŒ©mod:
This command enables function filtering per module. The
parameter defines the module. For example, if only the write*
functions in the ext3 module are desired, run:”…””}”(hjh5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Máhjd5  ubjT  )”}”(hŒ+echo 'write*:mod:ext3' > set_ftrace_filter
”h]”h¸)”}”(hŒ*echo 'write*:mod:ext3' > set_ftrace_filter”h]”hŒ.echo â€˜write*:mod:ext3â€™ > set_ftrace_filter”…””}”(hjz5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mæhjv5  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h Mæhjd5  ubh¸)”}”(hŒùThis command interacts with the filter in the same way as
filtering based on function names. Thus, adding more functions
in a different module is accomplished by appending (>>) to the
filter file. Remove specific module functions by prepending
'!'::”h]”hŒüThis command interacts with the filter in the same way as
filtering based on function names. Thus, adding more functions
in a different module is accomplished by appending (>>) to the
filter file. Remove specific module functions by prepending
â€˜!â€™:”…””}”(hjŽ5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mèhjd5  ubj   )”}”(hŒ0echo '!writeback*:mod:ext3' >> set_ftrace_filter”h]”hŒ0echo '!writeback*:mod:ext3' >> set_ftrace_filter”…””}”hjœ5  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mîhjd5  ubh¸)”}”(hŒbMod command supports module globbing. Disable tracing for all
functions except a specific module::”h]”hŒaMod command supports module globbing. Disable tracing for all
functions except a specific module:”…””}”(hjª5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mðhjd5  ubj   )”}”(hŒ(echo '!*:mod:!ext3' >> set_ftrace_filter”h]”hŒ(echo '!*:mod:!ext3' >> set_ftrace_filter”…””}”hj¸5  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Móhjd5  ubh¸)”}”(hŒ9Disable tracing for all modules, but still trace kernel::”h]”hŒ8Disable tracing for all modules, but still trace kernel:”…””}”(hjÆ5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mõhjd5  ubj   )”}”(hŒ$echo '!*:mod:*' >> set_ftrace_filter”h]”hŒ$echo '!*:mod:*' >> set_ftrace_filter”…””}”hjÔ5  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M÷hjd5  ubh¸)”}”(hŒEnable filter only for kernel::”h]”hŒEnable filter only for kernel:”…””}”(hjâ5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mùhjd5  ubj   )”}”(hŒ*echo '*write*:mod:!*' >> set_ftrace_filter”h]”hŒ*echo '*write*:mod:!*' >> set_ftrace_filter”…””}”hjð5  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mûhjd5  ubh¸)”}”(hŒ#Enable filter for module globbing::”h]”hŒ"Enable filter for module globbing:”…””}”(hjþ5  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mýhjd5  ubj   )”}”(hŒ-echo '*write*:mod:*snd*' >> set_ftrace_filter”h]”hŒ-echo '*write*:mod:*snd*' >> set_ftrace_filter”…””}”hj6  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mÿhjd5  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1j  hja5  hžhhŸh¶h Nubj€  )”}”(hXE  traceon/traceoff:
These commands turn tracing on and off when the specified
functions are hit. The parameter determines how many times the
tracing system is turned on and off. If unspecified, there is
no limit. For example, to disable tracing when a schedule bug
is hit the first 5 times, run::

 echo '__schedule_bug:traceoff:5' > set_ftrace_filter

To always disable tracing when __schedule_bug is hit::

 echo '__schedule_bug:traceoff' > set_ftrace_filter

These commands are cumulative whether or not they are appended
to set_ftrace_filter. To remove a command, prepend it by '!'
and drop the parameter::

 echo '!__schedule_bug:traceoff:0' > set_ftrace_filter

The above removes the traceoff command for __schedule_bug
that have a counter. To remove commands without counters::

 echo '!__schedule_bug:traceoff' > set_ftrace_filter
”h]”(h¸)”}”(hX&  traceon/traceoff:
These commands turn tracing on and off when the specified
functions are hit. The parameter determines how many times the
tracing system is turned on and off. If unspecified, there is
no limit. For example, to disable tracing when a schedule bug
is hit the first 5 times, run::”h]”hX%  traceon/traceoff:
These commands turn tracing on and off when the specified
functions are hit. The parameter determines how many times the
tracing system is turned on and off. If unspecified, there is
no limit. For example, to disable tracing when a schedule bug
is hit the first 5 times, run:”…””}”(hj$6  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj 6  ubj   )”}”(hŒ4echo '__schedule_bug:traceoff:5' > set_ftrace_filter”h]”hŒ4echo '__schedule_bug:traceoff:5' > set_ftrace_filter”…””}”hj26  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhj 6  ubh¸)”}”(hŒ6To always disable tracing when __schedule_bug is hit::”h]”hŒ5To always disable tracing when __schedule_bug is hit:”…””}”(hj@6  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M
hj 6  ubj   )”}”(hŒ2echo '__schedule_bug:traceoff' > set_ftrace_filter”h]”hŒ2echo '__schedule_bug:traceoff' > set_ftrace_filter”…””}”hjN6  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhj 6  ubh¸)”}”(hŒ”These commands are cumulative whether or not they are appended
to set_ftrace_filter. To remove a command, prepend it by '!'
and drop the parameter::”h]”hŒ—These commands are cumulative whether or not they are appended
to set_ftrace_filter. To remove a command, prepend it by â€˜!â€™
and drop the parameter:”…””}”(hj\6  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj 6  ubj   )”}”(hŒ5echo '!__schedule_bug:traceoff:0' > set_ftrace_filter”h]”hŒ5echo '!__schedule_bug:traceoff:0' > set_ftrace_filter”…””}”hjj6  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhj 6  ubh¸)”}”(hŒtThe above removes the traceoff command for __schedule_bug
that have a counter. To remove commands without counters::”h]”hŒsThe above removes the traceoff command for __schedule_bug
that have a counter. To remove commands without counters:”…””}”(hjx6  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj 6  ubj   )”}”(hŒ3echo '!__schedule_bug:traceoff' > set_ftrace_filter”h]”hŒ3echo '!__schedule_bug:traceoff' > set_ftrace_filter”…””}”hj†6  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhj 6  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1j  hja5  hžhhŸh¶h Nubj€  )”}”(hX…  snapshot:
Will cause a snapshot to be triggered when the function is hit.
::

 echo 'native_flush_tlb_others:snapshot' > set_ftrace_filter

To only snapshot once:
::

 echo 'native_flush_tlb_others:snapshot:1' > set_ftrace_filter

To remove the above commands::

 echo '!native_flush_tlb_others:snapshot' > set_ftrace_filter
 echo '!native_flush_tlb_others:snapshot:0' > set_ftrace_filter
”h]”(h¸)”}”(hŒLsnapshot:
Will cause a snapshot to be triggered when the function is hit.
::”h]”hŒIsnapshot:
Will cause a snapshot to be triggered when the function is hit.”…””}”(hjž6  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjš6  ubj   )”}”(hŒ;echo 'native_flush_tlb_others:snapshot' > set_ftrace_filter”h]”hŒ;echo 'native_flush_tlb_others:snapshot' > set_ftrace_filter”…””}”hj¬6  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhjš6  ubh¸)”}”(hŒTo only snapshot once:
::”h]”hŒTo only snapshot once:”…””}”(hjº6  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhjš6  ubj   )”}”(hŒ=echo 'native_flush_tlb_others:snapshot:1' > set_ftrace_filter”h]”hŒ=echo 'native_flush_tlb_others:snapshot:1' > set_ftrace_filter”…””}”hjÈ6  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M"hjš6  ubh¸)”}”(hŒTo remove the above commands::”h]”hŒTo remove the above commands:”…””}”(hjÖ6  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M$hjš6  ubj   )”}”(hŒ{echo '!native_flush_tlb_others:snapshot' > set_ftrace_filter
echo '!native_flush_tlb_others:snapshot:0' > set_ftrace_filter”h]”hŒ{echo '!native_flush_tlb_others:snapshot' > set_ftrace_filter
echo '!native_flush_tlb_others:snapshot:0' > set_ftrace_filter”…””}”hjä6  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M&hjš6  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1j  hja5  hžhhŸh¶h Nubj€  )”}”(hX&  enable_event/disable_event:
These commands can enable or disable a trace event. Note, because
function tracing callbacks are very sensitive, when these commands
are registered, the trace point is activated, but disabled in
a "soft" mode. That is, the tracepoint will be called, but
just will not be traced. The event tracepoint stays in this mode
as long as there's a command that triggers it.
::

 echo 'try_to_wake_up:enable_event:sched:sched_switch:2' > \
       set_ftrace_filter

The format is::

  <function>:enable_event:<system>:<event>[:count]
  <function>:disable_event:<system>:<event>[:count]

To remove the events commands::

 echo '!try_to_wake_up:enable_event:sched:sched_switch:0' > \
       set_ftrace_filter
 echo '!schedule:disable_event:sched:sched_switch' > \
       set_ftrace_filter
”h]”(h¸)”}”(hXŒ  enable_event/disable_event:
These commands can enable or disable a trace event. Note, because
function tracing callbacks are very sensitive, when these commands
are registered, the trace point is activated, but disabled in
a "soft" mode. That is, the tracepoint will be called, but
just will not be traced. The event tracepoint stays in this mode
as long as there's a command that triggers it.
::”h]”hX  enable_event/disable_event:
These commands can enable or disable a trace event. Note, because
function tracing callbacks are very sensitive, when these commands
are registered, the trace point is activated, but disabled in
a â€œsoftâ€ mode. That is, the tracepoint will be called, but
just will not be traced. The event tracepoint stays in this mode
as long as thereâ€™s a command that triggers it.”…””}”(hjü6  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M)hjø6  ubj   )”}”(hŒSecho 'try_to_wake_up:enable_event:sched:sched_switch:2' > \
      set_ftrace_filter”h]”hŒSecho 'try_to_wake_up:enable_event:sched:sched_switch:2' > \
      set_ftrace_filter”…””}”hj
7  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M2hjø6  ubh¸)”}”(hŒThe format is::”h]”hŒThe format is:”…””}”(hj7  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M5hjø6  ubj   )”}”(hŒb<function>:enable_event:<system>:<event>[:count]
<function>:disable_event:<system>:<event>[:count]”h]”hŒb<function>:enable_event:<system>:<event>[:count]
<function>:disable_event:<system>:<event>[:count]”…””}”hj&7  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M7hjø6  ubh¸)”}”(hŒTo remove the events commands::”h]”hŒTo remove the events commands:”…””}”(hj47  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M:hjø6  ubj   )”}”(hŒ¢echo '!try_to_wake_up:enable_event:sched:sched_switch:0' > \
      set_ftrace_filter
echo '!schedule:disable_event:sched:sched_switch' > \
      set_ftrace_filter”h]”hŒ¢echo '!try_to_wake_up:enable_event:sched:sched_switch:0' > \
      set_ftrace_filter
echo '!schedule:disable_event:sched:sched_switch' > \
      set_ftrace_filter”…””}”hjB7  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M<hjø6  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1j  hja5  hžhhŸh¶h Nubj€  )”}”(hXA  dump:
When the function is hit, it will dump the contents of the ftrace
ring buffer to the console. This is useful if you need to debug
something, and want to dump the trace when a certain function
is hit. Perhaps it's a function that is called before a triple
fault happens and does not allow you to get a regular dump.
”h]”h¸)”}”(hX@  dump:
When the function is hit, it will dump the contents of the ftrace
ring buffer to the console. This is useful if you need to debug
something, and want to dump the trace when a certain function
is hit. Perhaps it's a function that is called before a triple
fault happens and does not allow you to get a regular dump.”h]”hXB  dump:
When the function is hit, it will dump the contents of the ftrace
ring buffer to the console. This is useful if you need to debug
something, and want to dump the trace when a certain function
is hit. Perhaps itâ€™s a function that is called before a triple
fault happens and does not allow you to get a regular dump.”…””}”(hjZ7  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MAhjV7  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hja5  hžhhŸh¶h Nubj€  )”}”(hX	  cpudump:
When the function is hit, it will dump the contents of the ftrace
ring buffer for the current CPU to the console. Unlike the "dump"
command, it only prints out the contents of the ring buffer for the
CPU that executed the function that triggered the dump.
”h]”h¸)”}”(hX  cpudump:
When the function is hit, it will dump the contents of the ftrace
ring buffer for the current CPU to the console. Unlike the "dump"
command, it only prints out the contents of the ring buffer for the
CPU that executed the function that triggered the dump.”h]”hX  cpudump:
When the function is hit, it will dump the contents of the ftrace
ring buffer for the current CPU to the console. Unlike the â€œdumpâ€
command, it only prints out the contents of the ring buffer for the
CPU that executed the function that triggered the dump.”…””}”(hjr7  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MHhjn7  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hja5  hžhhŸh¶h Nubj€  )”}”(hŒAstacktrace:
When the function is hit, a stack trace is recorded.
”h]”h¸)”}”(hŒ@stacktrace:
When the function is hit, a stack trace is recorded.”h]”hŒ@stacktrace:
When the function is hit, a stack trace is recorded.”…””}”(hjŠ7  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MNhj†7  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j  hja5  hžhhŸh¶h Nubeh}”(h]”h ]”h"]”h$]”h&]”jø  jù  uh1jz  hŸh¶h Máhj&5  hžhubeh}”(h]”Œfilter-commands”ah ]”h"]”Œfilter commands”ah$]”h&]”uh1h¡hh£hžhhŸh¶h MØubh¢)”}”(hhh]”(h§)”}”(hŒ
trace_pipe”h]”hŒ
trace_pipe”…””}”(hj¯7  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj¬7  hžhhŸh¶h MRubh¸)”}”(hŒØThe trace_pipe outputs the same content as the trace file, but
the effect on the tracing is different. Every read from
trace_pipe is consumed. This means that subsequent reads will be
different. The trace is live.
::”h]”hŒÕThe trace_pipe outputs the same content as the trace file, but
the effect on the tracing is different. Every read from
trace_pipe is consumed. This means that subsequent reads will be
different. The trace is live.”…””}”(hj½7  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MThj¬7  hžhubj   )”}”(hX-  # echo function > current_tracer
# cat trace_pipe > /tmp/trace.out &
[1] 4153
# echo 1 > tracing_on
# usleep 1
# echo 0 > tracing_on
# cat trace
# tracer: function
#
# entries-in-buffer/entries-written: 0/0   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |

#
# cat /tmp/trace.out
           bash-1994  [000] ....  5281.568961: mutex_unlock <-rb_simple_write
           bash-1994  [000] ....  5281.568963: __mutex_unlock_slowpath <-mutex_unlock
           bash-1994  [000] ....  5281.568963: __fsnotify_parent <-fsnotify_modify
           bash-1994  [000] ....  5281.568964: fsnotify <-fsnotify_modify
           bash-1994  [000] ....  5281.568964: __srcu_read_lock <-fsnotify
           bash-1994  [000] ....  5281.568964: add_preempt_count <-__srcu_read_lock
           bash-1994  [000] ...1  5281.568965: sub_preempt_count <-__srcu_read_lock
           bash-1994  [000] ....  5281.568965: __srcu_read_unlock <-fsnotify
           bash-1994  [000] ....  5281.568967: sys_dup2 <-system_call_fastpath”h]”hX-  # echo function > current_tracer
# cat trace_pipe > /tmp/trace.out &
[1] 4153
# echo 1 > tracing_on
# usleep 1
# echo 0 > tracing_on
# cat trace
# tracer: function
#
# entries-in-buffer/entries-written: 0/0   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |

#
# cat /tmp/trace.out
           bash-1994  [000] ....  5281.568961: mutex_unlock <-rb_simple_write
           bash-1994  [000] ....  5281.568963: __mutex_unlock_slowpath <-mutex_unlock
           bash-1994  [000] ....  5281.568963: __fsnotify_parent <-fsnotify_modify
           bash-1994  [000] ....  5281.568964: fsnotify <-fsnotify_modify
           bash-1994  [000] ....  5281.568964: __srcu_read_lock <-fsnotify
           bash-1994  [000] ....  5281.568964: add_preempt_count <-__srcu_read_lock
           bash-1994  [000] ...1  5281.568965: sub_preempt_count <-__srcu_read_lock
           bash-1994  [000] ....  5281.568965: __srcu_read_unlock <-fsnotify
           bash-1994  [000] ....  5281.568967: sys_dup2 <-system_call_fastpath”…””}”hjË7  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MZhj¬7  hžhubh¸)”}”(hX  Note, reading the trace_pipe file will block until more input is
added. This is contrary to the trace file. If any process opened
the trace file for reading, it will actually disable tracing and
prevent new entries from being added. The trace_pipe file does
not have this limitation.”h]”hX  Note, reading the trace_pipe file will block until more input is
added. This is contrary to the trace file. If any process opened
the trace file for reading, it will actually disable tracing and
prevent new entries from being added. The trace_pipe file does
not have this limitation.”…””}”(hjÙ7  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mzhj¬7  hžhubeh}”(h]”Œ
trace-pipe”ah ]”h"]”Œ
trace_pipe”ah$]”h&]”uh1h¡hh£hžhhŸh¶h MRubh¢)”}”(hhh]”(h§)”}”(hŒtrace entries”h]”hŒtrace entries”…””}”(hjò7  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjï7  hžhhŸh¶h Mubh¸)”}”(hXQ  Having too much or not enough data can be troublesome in
diagnosing an issue in the kernel. The file buffer_size_kb is
used to modify the size of the internal trace buffers. The
number listed is the number of entries that can be recorded per
CPU. To know the full size, multiply the number of possible CPUs
with the number of entries.
::”h]”hXN  Having too much or not enough data can be troublesome in
diagnosing an issue in the kernel. The file buffer_size_kb is
used to modify the size of the internal trace buffers. The
number listed is the number of entries that can be recorded per
CPU. To know the full size, multiply the number of possible CPUs
with the number of entries.”…””}”(hj 8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mƒhjï7  hžhubj   )”}”(hŒ+# cat buffer_size_kb
1408 (units kilobytes)”h]”hŒ+# cat buffer_size_kb
1408 (units kilobytes)”…””}”hj8  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M‹hjï7  hžhubh¸)”}”(hŒ&Or simply read buffer_total_size_kb
::”h]”hŒ#Or simply read buffer_total_size_kb”…””}”(hj8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MŽhjï7  hžhubj   )”}”(hŒ# cat buffer_total_size_kb
5632”h]”hŒ# cat buffer_total_size_kb
5632”…””}”hj*8  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M‘hjï7  hžhubh¸)”}”(hŒITo modify the buffer, simple echo in a number (in 1024 byte segments).
::”h]”hŒFTo modify the buffer, simple echo in a number (in 1024 byte segments).”…””}”(hj88  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M”hjï7  hžhubj   )”}”(hŒJ# echo 10000 > buffer_size_kb
# cat buffer_size_kb
10000 (units kilobytes)”h]”hŒJ# echo 10000 > buffer_size_kb
# cat buffer_size_kb
10000 (units kilobytes)”…””}”hjF8  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M—hjï7  hžhubh¸)”}”(hŒpIt will try to allocate as much as possible. If you allocate too
much, it can cause Out-Of-Memory to trigger.
::”h]”hŒmIt will try to allocate as much as possible. If you allocate too
much, it can cause Out-Of-Memory to trigger.”…””}”(hjT8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M›hjï7  hžhubj   )”}”(hŒn# echo 1000000000000 > buffer_size_kb
-bash: echo: write error: Cannot allocate memory
# cat buffer_size_kb
85”h]”hŒn# echo 1000000000000 > buffer_size_kb
-bash: echo: write error: Cannot allocate memory
# cat buffer_size_kb
85”…””}”hjb8  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MŸhjï7  hžhubh¸)”}”(hŒ;The per_cpu buffers can be changed individually as well:
::”h]”hŒ8The per_cpu buffers can be changed individually as well:”…””}”(hjp8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¤hjï7  hžhubj   )”}”(hŒS# echo 10000 > per_cpu/cpu0/buffer_size_kb
# echo 100 > per_cpu/cpu1/buffer_size_kb”h]”hŒS# echo 10000 > per_cpu/cpu0/buffer_size_kb
# echo 100 > per_cpu/cpu1/buffer_size_kb”…””}”hj~8  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M§hjï7  hžhubh¸)”}”(hŒeWhen the per_cpu buffers are not the same, the buffer_size_kb
at the top level will just show an X
::”h]”hŒbWhen the per_cpu buffers are not the same, the buffer_size_kb
at the top level will just show an X”…””}”(hjŒ8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mªhjï7  hžhubj   )”}”(hŒ# cat buffer_size_kb
X”h]”hŒ# cat buffer_size_kb
X”…””}”hjš8  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M®hjï7  hžhubh¸)”}”(hŒ4This is where the buffer_total_size_kb is useful:
::”h]”hŒ1This is where the buffer_total_size_kb is useful:”…””}”(hj¨8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M±hjï7  hžhubj   )”}”(hŒ # cat buffer_total_size_kb
12916”h]”hŒ # cat buffer_total_size_kb
12916”…””}”hj¶8  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M´hjï7  hžhubh¸)”}”(hŒXWriting to the top level buffer_size_kb will reset all the buffers
to be the same again.”h]”hŒXWriting to the top level buffer_size_kb will reset all the buffers
to be the same again.”…””}”(hjÄ8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M·hjï7  hžhubeh}”(h]”Œtrace-entries”ah ]”h"]”Œtrace entries”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Mubh¢)”}”(hhh]”(h§)”}”(hŒSnapshot”h]”hŒSnapshot”…””}”(hjÝ8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjÚ8  hžhhŸh¶h M»ubh¸)”}”(hX  CONFIG_TRACER_SNAPSHOT makes a generic snapshot feature
available to all non latency tracers. (Latency tracers which
record max latency, such as "irqsoff" or "wakeup", can't use
this feature, since those are already using the snapshot
mechanism internally.)”h]”hX  CONFIG_TRACER_SNAPSHOT makes a generic snapshot feature
available to all non latency tracers. (Latency tracers which
record max latency, such as â€œirqsoffâ€ or â€œwakeupâ€, canâ€™t use
this feature, since those are already using the snapshot
mechanism internally.)”…””}”(hjë8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¼hjÚ8  hžhubh¸)”}”(hŒÙSnapshot preserves a current trace buffer at a particular point
in time without stopping tracing. Ftrace swaps the current
buffer with a spare buffer, and tracing continues in the new
current (=previous spare) buffer.”h]”hŒÙSnapshot preserves a current trace buffer at a particular point
in time without stopping tracing. Ftrace swaps the current
buffer with a spare buffer, and tracing continues in the new
current (=previous spare) buffer.”…””}”(hjù8  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÂhjÚ8  hžhubh¸)”}”(hŒEThe following tracefs files in "tracing" are related to this
feature:”h]”hŒIThe following tracefs files in â€œtracingâ€ are related to this
feature:”…””}”(hj9  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÇhjÚ8  hžhubjT  )”}”(hXé  snapshot:

      This is used to take a snapshot and to read the output
      of the snapshot. Echo 1 into this file to allocate a
      spare buffer and to take a snapshot (swap), then read
      the snapshot from this file in the same format as
      "trace" (described above in the section "The File
      System"). Both reads snapshot and tracing are executable
      in parallel. When the spare buffer is allocated, echoing
      0 frees it, and echoing else (positive) values clear the
      snapshot contents.
      More details are shown in the table below.

      +--------------+------------+------------+------------+
      |status\\input |     0      |     1      |    else    |
      +==============+============+============+============+
      |not allocated |(do nothing)| alloc+swap |(do nothing)|
      +--------------+------------+------------+------------+
      |allocated     |    free    |    swap    |   clear    |
      +--------------+------------+------------+------------+
”h]”(h¸)”}”(hŒ	snapshot:”h]”hŒ	snapshot:”…””}”(hj9  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÊhj9  ubjT  )”}”(hXx  This is used to take a snapshot and to read the output
of the snapshot. Echo 1 into this file to allocate a
spare buffer and to take a snapshot (swap), then read
the snapshot from this file in the same format as
"trace" (described above in the section "The File
System"). Both reads snapshot and tracing are executable
in parallel. When the spare buffer is allocated, echoing
0 frees it, and echoing else (positive) values clear the
snapshot contents.
More details are shown in the table below.

+--------------+------------+------------+------------+
|status\\input |     0      |     1      |    else    |
+==============+============+============+============+
|not allocated |(do nothing)| alloc+swap |(do nothing)|
+--------------+------------+------------+------------+
|allocated     |    free    |    swap    |   clear    |
+--------------+------------+------------+------------+
”h]”(h¸)”}”(hXî  This is used to take a snapshot and to read the output
of the snapshot. Echo 1 into this file to allocate a
spare buffer and to take a snapshot (swap), then read
the snapshot from this file in the same format as
"trace" (described above in the section "The File
System"). Both reads snapshot and tracing are executable
in parallel. When the spare buffer is allocated, echoing
0 frees it, and echoing else (positive) values clear the
snapshot contents.
More details are shown in the table below.”h]”hXö  This is used to take a snapshot and to read the output
of the snapshot. Echo 1 into this file to allocate a
spare buffer and to take a snapshot (swap), then read
the snapshot from this file in the same format as
â€œtraceâ€ (described above in the section â€œThe File
Systemâ€). Both reads snapshot and tracing are executable
in parallel. When the spare buffer is allocated, echoing
0 frees it, and echoing else (positive) values clear the
snapshot contents.
More details are shown in the table below.”…””}”(hj+9  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÌhj'9  ubhŒtable”“”)”}”(hhh]”hŒtgroup”“”)”}”(hhh]”(hŒcolspec”“”)”}”(hhh]”h}”(h]”h ]”h"]”h$]”h&]”Œcolwidth”Kuh1jC9  hj@9  ubjD9  )”}”(hhh]”h}”(h]”h ]”h"]”h$]”h&]”Œcolwidth”Kuh1jC9  hj@9  ubjD9  )”}”(hhh]”h}”(h]”h ]”h"]”h$]”h&]”Œcolwidth”Kuh1jC9  hj@9  ubjD9  )”}”(hhh]”h}”(h]”h ]”h"]”h$]”h&]”Œcolwidth”Kuh1jC9  hj@9  ubhŒthead”“”)”}”(hhh]”hŒrow”“”)”}”(hhh]”(hŒentry”“”)”}”(hhh]”h¸)”}”(hŒstatus\\input”h]”hŒstatus \input”…””}”(hj|9  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MØhjy9  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjt9  ubjx9  )”}”(hhh]”h¸)”}”(hŒ0”h]”hŒ0”…””}”(hj“9  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MØhj9  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjt9  ubjx9  )”}”(hhh]”h¸)”}”(hŒ1”h]”hŒ1”…””}”(hjª9  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MØhj§9  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjt9  ubjx9  )”}”(hhh]”h¸)”}”(hŒelse”h]”hŒelse”…””}”(hjÁ9  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MØhj¾9  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjt9  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jr9  hjo9  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jm9  hj@9  ubhŒtbody”“”)”}”(hhh]”(js9  )”}”(hhh]”(jx9  )”}”(hhh]”h¸)”}”(hŒnot allocated”h]”hŒnot allocated”…””}”(hjì9  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÚhjé9  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjæ9  ubjx9  )”}”(hhh]”h¸)”}”(hŒ(do nothing)”h]”hŒ(do nothing)”…””}”(hj:  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÚhj :  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjæ9  ubjx9  )”}”(hhh]”h¸)”}”(hŒ
alloc+swap”h]”hŒ
alloc+swap”…””}”(hj:  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÚhj:  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjæ9  ubjx9  )”}”(hhh]”h¸)”}”(hŒ(do nothing)”h]”hŒ(do nothing)”…””}”(hj1:  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÚhj.:  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjæ9  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jr9  hjã9  ubjs9  )”}”(hhh]”(jx9  )”}”(hhh]”h¸)”}”(hŒ	allocated”h]”hŒ	allocated”…””}”(hjQ:  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÜhjN:  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjK:  ubjx9  )”}”(hhh]”h¸)”}”(hŒfree”h]”hŒfree”…””}”(hjh:  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÜhje:  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjK:  ubjx9  )”}”(hhh]”h¸)”}”(hŒswap”h]”hŒswap”…””}”(hj:  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÜhj|:  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjK:  ubjx9  )”}”(hhh]”h¸)”}”(hŒclear”h]”hŒclear”…””}”(hj–:  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÜhj“:  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jw9  hjK:  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jr9  hjã9  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1já9  hj@9  ubeh}”(h]”h ]”h"]”h$]”h&]”Œcols”Kuh1j>9  hj;9  ubah}”(h]”h ]”h"]”h$]”h&]”uh1j99  hj'9  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÌhj9  ubeh}”(h]”h ]”h"]”h$]”h&]”uh1jS  hŸh¶h MÊhjÚ8  hžhubh¸)”}”(hŒ4Here is an example of using the snapshot feature.
::”h]”hŒ1Here is an example of using the snapshot feature.”…””}”(hjÏ:  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MßhjÚ8  hžhubj   )”}”(hX=  # echo 1 > events/sched/enable
# echo 1 > snapshot
# cat snapshot
# tracer: nop
#
# entries-in-buffer/entries-written: 71/71   #P:8
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
          <idle>-0     [005] d...  2440.603828: sched_switch: prev_comm=swapper/5 prev_pid=0 prev_prio=120   prev_state=R ==> next_comm=snapshot-test-2 next_pid=2242 next_prio=120
           sleep-2242  [005] d...  2440.603846: sched_switch: prev_comm=snapshot-test-2 prev_pid=2242 prev_prio=120   prev_state=R ==> next_comm=kworker/5:1 next_pid=60 next_prio=120
[...]
        <idle>-0     [002] d...  2440.707230: sched_switch: prev_comm=swapper/2 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=snapshot-test-2 next_pid=2229 next_prio=120

# cat trace
# tracer: nop
#
# entries-in-buffer/entries-written: 77/77   #P:8
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
          <idle>-0     [007] d...  2440.707395: sched_switch: prev_comm=swapper/7 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=snapshot-test-2 next_pid=2243 next_prio=120
 snapshot-test-2-2229  [002] d...  2440.707438: sched_switch: prev_comm=snapshot-test-2 prev_pid=2229 prev_prio=120 prev_state=S ==> next_comm=swapper/2 next_pid=0 next_prio=120
[...]”h]”hX=  # echo 1 > events/sched/enable
# echo 1 > snapshot
# cat snapshot
# tracer: nop
#
# entries-in-buffer/entries-written: 71/71   #P:8
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
          <idle>-0     [005] d...  2440.603828: sched_switch: prev_comm=swapper/5 prev_pid=0 prev_prio=120   prev_state=R ==> next_comm=snapshot-test-2 next_pid=2242 next_prio=120
           sleep-2242  [005] d...  2440.603846: sched_switch: prev_comm=snapshot-test-2 prev_pid=2242 prev_prio=120   prev_state=R ==> next_comm=kworker/5:1 next_pid=60 next_prio=120
[...]
        <idle>-0     [002] d...  2440.707230: sched_switch: prev_comm=swapper/2 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=snapshot-test-2 next_pid=2229 next_prio=120

# cat trace
# tracer: nop
#
# entries-in-buffer/entries-written: 77/77   #P:8
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
          <idle>-0     [007] d...  2440.707395: sched_switch: prev_comm=swapper/7 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=snapshot-test-2 next_pid=2243 next_prio=120
 snapshot-test-2-2229  [002] d...  2440.707438: sched_switch: prev_comm=snapshot-test-2 prev_pid=2229 prev_prio=120 prev_state=S ==> next_comm=swapper/2 next_pid=0 next_prio=120
[...]”…””}”hjÝ:  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h MâhjÚ8  hžhubh¸)”}”(hŒIf you try to use this snapshot feature when current tracer is
one of the latency tracers, you will get the following results.
::”h]”hŒ~If you try to use this snapshot feature when current tracer is
one of the latency tracers, you will get the following results.”…””}”(hjë:  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MhjÚ8  hžhubj   )”}”(hŒ™# echo wakeup > current_tracer
# echo 1 > snapshot
bash: echo: write error: Device or resource busy
# cat snapshot
cat: snapshot: Device or resource busy”h]”hŒ™# echo wakeup > current_tracer
# echo 1 > snapshot
bash: echo: write error: Device or resource busy
# cat snapshot
cat: snapshot: Device or resource busy”…””}”hjù:  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M
hjÚ8  hžhubeh}”(h]”Œsnapshot”ah ]”h"]”Œsnapshot”ah$]”h&]”uh1h¡hh£hžhhŸh¶h M»ubh¢)”}”(hhh]”(h§)”}”(hŒ	Instances”h]”hŒ	Instances”…””}”(hj;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj;  hžhhŸh¶h Mubh¸)”}”(hX5  In the tracefs tracing directory, there is a directory called "instances".
This directory can have new directories created inside of it using
mkdir, and removing directories with rmdir. The directory created
with mkdir in this directory will already contain files and other
directories after it is created.
::”h]”hX6  In the tracefs tracing directory, there is a directory called â€œinstancesâ€.
This directory can have new directories created inside of it using
mkdir, and removing directories with rmdir. The directory created
with mkdir in this directory will already contain files and other
directories after it is created.”…””}”(hj ;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj;  hžhubj   )”}”(hŒÇ# mkdir instances/foo
# ls instances/foo
buffer_size_kb  buffer_total_size_kb  events  free_buffer  per_cpu
set_event  snapshot  trace  trace_clock  trace_marker  trace_options
trace_pipe  tracing_on”h]”hŒÇ# mkdir instances/foo
# ls instances/foo
buffer_size_kb  buffer_total_size_kb  events  free_buffer  per_cpu
set_event  snapshot  trace  trace_clock  trace_marker  trace_options
trace_pipe  tracing_on”…””}”hj.;  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mhj;  hžhubh¸)”}”(hŒãAs you can see, the new directory looks similar to the tracing directory
itself. In fact, it is very similar, except that the buffer and
events are agnostic from the main directory, or from any other
instances that are created.”h]”hŒãAs you can see, the new directory looks similar to the tracing directory
itself. In fact, it is very similar, except that the buffer and
events are agnostic from the main directory, or from any other
instances that are created.”…””}”(hj<;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M hj;  hžhubh¸)”}”(hXÆ  The files in the new directory work just like the files with the
same name in the tracing directory except the buffer that is used
is a separate and new buffer. The files affect that buffer but do not
affect the main buffer with the exception of trace_options. Currently,
the trace_options affect all instances and the top level buffer
the same, but this may change in future releases. That is, options
may become specific to the instance they reside in.”h]”hXÆ  The files in the new directory work just like the files with the
same name in the tracing directory except the buffer that is used
is a separate and new buffer. The files affect that buffer but do not
affect the main buffer with the exception of trace_options. Currently,
the trace_options affect all instances and the top level buffer
the same, but this may change in future releases. That is, options
may become specific to the instance they reside in.”…””}”(hjJ;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M%hj;  hžhubh¸)”}”(hŒµNotice that none of the function tracer files are there, nor is
current_tracer and available_tracers. This is because the buffers
can currently only have events enabled for them.
::”h]”hŒ²Notice that none of the function tracer files are there, nor is
current_tracer and available_tracers. This is because the buffers
can currently only have events enabled for them.”…””}”(hjX;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M-hj;  hžhubj   )”}”(hXÞ  # mkdir instances/foo
# mkdir instances/bar
# mkdir instances/zoot
# echo 100000 > buffer_size_kb
# echo 1000 > instances/foo/buffer_size_kb
# echo 5000 > instances/bar/per_cpu/cpu1/buffer_size_kb
# echo function > current_trace
# echo 1 > instances/foo/events/sched/sched_wakeup/enable
# echo 1 > instances/foo/events/sched/sched_wakeup_new/enable
# echo 1 > instances/foo/events/sched/sched_switch/enable
# echo 1 > instances/bar/events/irq/enable
# echo 1 > instances/zoot/events/syscalls/enable
# cat trace_pipe
CPU:2 [LOST 11745 EVENTS]
            bash-2044  [002] .... 10594.481032: _raw_spin_lock_irqsave <-get_page_from_freelist
            bash-2044  [002] d... 10594.481032: add_preempt_count <-_raw_spin_lock_irqsave
            bash-2044  [002] d..1 10594.481032: __rmqueue <-get_page_from_freelist
            bash-2044  [002] d..1 10594.481033: _raw_spin_unlock <-get_page_from_freelist
            bash-2044  [002] d..1 10594.481033: sub_preempt_count <-_raw_spin_unlock
            bash-2044  [002] d... 10594.481033: get_pageblock_flags_group <-get_pageblock_migratetype
            bash-2044  [002] d... 10594.481034: __mod_zone_page_state <-get_page_from_freelist
            bash-2044  [002] d... 10594.481034: zone_statistics <-get_page_from_freelist
            bash-2044  [002] d... 10594.481034: __inc_zone_state <-zone_statistics
            bash-2044  [002] d... 10594.481034: __inc_zone_state <-zone_statistics
            bash-2044  [002] .... 10594.481035: arch_dup_task_struct <-copy_process
[...]

# cat instances/foo/trace_pipe
            bash-1998  [000] d..4   136.676759: sched_wakeup: comm=kworker/0:1 pid=59 prio=120 success=1 target_cpu=000
            bash-1998  [000] dN.4   136.676760: sched_wakeup: comm=bash pid=1998 prio=120 success=1 target_cpu=000
          <idle>-0     [003] d.h3   136.676906: sched_wakeup: comm=rcu_preempt pid=9 prio=120 success=1 target_cpu=003
          <idle>-0     [003] d..3   136.676909: sched_switch: prev_comm=swapper/3 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=rcu_preempt next_pid=9 next_prio=120
     rcu_preempt-9     [003] d..3   136.676916: sched_switch: prev_comm=rcu_preempt prev_pid=9 prev_prio=120 prev_state=S ==> next_comm=swapper/3 next_pid=0 next_prio=120
            bash-1998  [000] d..4   136.677014: sched_wakeup: comm=kworker/0:1 pid=59 prio=120 success=1 target_cpu=000
            bash-1998  [000] dN.4   136.677016: sched_wakeup: comm=bash pid=1998 prio=120 success=1 target_cpu=000
            bash-1998  [000] d..3   136.677018: sched_switch: prev_comm=bash prev_pid=1998 prev_prio=120 prev_state=R+ ==> next_comm=kworker/0:1 next_pid=59 next_prio=120
     kworker/0:1-59    [000] d..4   136.677022: sched_wakeup: comm=sshd pid=1995 prio=120 success=1 target_cpu=001
     kworker/0:1-59    [000] d..3   136.677025: sched_switch: prev_comm=kworker/0:1 prev_pid=59 prev_prio=120 prev_state=S ==> next_comm=bash next_pid=1998 next_prio=120
[...]

# cat instances/bar/trace_pipe
     migration/1-14    [001] d.h3   138.732674: softirq_raise: vec=3 [action=NET_RX]
          <idle>-0     [001] dNh3   138.732725: softirq_raise: vec=3 [action=NET_RX]
            bash-1998  [000] d.h1   138.733101: softirq_raise: vec=1 [action=TIMER]
            bash-1998  [000] d.h1   138.733102: softirq_raise: vec=9 [action=RCU]
            bash-1998  [000] ..s2   138.733105: softirq_entry: vec=1 [action=TIMER]
            bash-1998  [000] ..s2   138.733106: softirq_exit: vec=1 [action=TIMER]
            bash-1998  [000] ..s2   138.733106: softirq_entry: vec=9 [action=RCU]
            bash-1998  [000] ..s2   138.733109: softirq_exit: vec=9 [action=RCU]
            sshd-1995  [001] d.h1   138.733278: irq_handler_entry: irq=21 name=uhci_hcd:usb4
            sshd-1995  [001] d.h1   138.733280: irq_handler_exit: irq=21 ret=unhandled
            sshd-1995  [001] d.h1   138.733281: irq_handler_entry: irq=21 name=eth0
            sshd-1995  [001] d.h1   138.733283: irq_handler_exit: irq=21 ret=handled
[...]

# cat instances/zoot/trace
# tracer: nop
#
# entries-in-buffer/entries-written: 18996/18996   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
            bash-1998  [000] d...   140.733501: sys_write -> 0x2
            bash-1998  [000] d...   140.733504: sys_dup2(oldfd: a, newfd: 1)
            bash-1998  [000] d...   140.733506: sys_dup2 -> 0x1
            bash-1998  [000] d...   140.733508: sys_fcntl(fd: a, cmd: 1, arg: 0)
            bash-1998  [000] d...   140.733509: sys_fcntl -> 0x1
            bash-1998  [000] d...   140.733510: sys_close(fd: a)
            bash-1998  [000] d...   140.733510: sys_close -> 0x0
            bash-1998  [000] d...   140.733514: sys_rt_sigprocmask(how: 0, nset: 0, oset: 6e2768, sigsetsize: 8)
            bash-1998  [000] d...   140.733515: sys_rt_sigprocmask -> 0x0
            bash-1998  [000] d...   140.733516: sys_rt_sigaction(sig: 2, act: 7fff718846f0, oact: 7fff71884650, sigsetsize: 8)
            bash-1998  [000] d...   140.733516: sys_rt_sigaction -> 0x0”h]”hXÞ  # mkdir instances/foo
# mkdir instances/bar
# mkdir instances/zoot
# echo 100000 > buffer_size_kb
# echo 1000 > instances/foo/buffer_size_kb
# echo 5000 > instances/bar/per_cpu/cpu1/buffer_size_kb
# echo function > current_trace
# echo 1 > instances/foo/events/sched/sched_wakeup/enable
# echo 1 > instances/foo/events/sched/sched_wakeup_new/enable
# echo 1 > instances/foo/events/sched/sched_switch/enable
# echo 1 > instances/bar/events/irq/enable
# echo 1 > instances/zoot/events/syscalls/enable
# cat trace_pipe
CPU:2 [LOST 11745 EVENTS]
            bash-2044  [002] .... 10594.481032: _raw_spin_lock_irqsave <-get_page_from_freelist
            bash-2044  [002] d... 10594.481032: add_preempt_count <-_raw_spin_lock_irqsave
            bash-2044  [002] d..1 10594.481032: __rmqueue <-get_page_from_freelist
            bash-2044  [002] d..1 10594.481033: _raw_spin_unlock <-get_page_from_freelist
            bash-2044  [002] d..1 10594.481033: sub_preempt_count <-_raw_spin_unlock
            bash-2044  [002] d... 10594.481033: get_pageblock_flags_group <-get_pageblock_migratetype
            bash-2044  [002] d... 10594.481034: __mod_zone_page_state <-get_page_from_freelist
            bash-2044  [002] d... 10594.481034: zone_statistics <-get_page_from_freelist
            bash-2044  [002] d... 10594.481034: __inc_zone_state <-zone_statistics
            bash-2044  [002] d... 10594.481034: __inc_zone_state <-zone_statistics
            bash-2044  [002] .... 10594.481035: arch_dup_task_struct <-copy_process
[...]

# cat instances/foo/trace_pipe
            bash-1998  [000] d..4   136.676759: sched_wakeup: comm=kworker/0:1 pid=59 prio=120 success=1 target_cpu=000
            bash-1998  [000] dN.4   136.676760: sched_wakeup: comm=bash pid=1998 prio=120 success=1 target_cpu=000
          <idle>-0     [003] d.h3   136.676906: sched_wakeup: comm=rcu_preempt pid=9 prio=120 success=1 target_cpu=003
          <idle>-0     [003] d..3   136.676909: sched_switch: prev_comm=swapper/3 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=rcu_preempt next_pid=9 next_prio=120
     rcu_preempt-9     [003] d..3   136.676916: sched_switch: prev_comm=rcu_preempt prev_pid=9 prev_prio=120 prev_state=S ==> next_comm=swapper/3 next_pid=0 next_prio=120
            bash-1998  [000] d..4   136.677014: sched_wakeup: comm=kworker/0:1 pid=59 prio=120 success=1 target_cpu=000
            bash-1998  [000] dN.4   136.677016: sched_wakeup: comm=bash pid=1998 prio=120 success=1 target_cpu=000
            bash-1998  [000] d..3   136.677018: sched_switch: prev_comm=bash prev_pid=1998 prev_prio=120 prev_state=R+ ==> next_comm=kworker/0:1 next_pid=59 next_prio=120
     kworker/0:1-59    [000] d..4   136.677022: sched_wakeup: comm=sshd pid=1995 prio=120 success=1 target_cpu=001
     kworker/0:1-59    [000] d..3   136.677025: sched_switch: prev_comm=kworker/0:1 prev_pid=59 prev_prio=120 prev_state=S ==> next_comm=bash next_pid=1998 next_prio=120
[...]

# cat instances/bar/trace_pipe
     migration/1-14    [001] d.h3   138.732674: softirq_raise: vec=3 [action=NET_RX]
          <idle>-0     [001] dNh3   138.732725: softirq_raise: vec=3 [action=NET_RX]
            bash-1998  [000] d.h1   138.733101: softirq_raise: vec=1 [action=TIMER]
            bash-1998  [000] d.h1   138.733102: softirq_raise: vec=9 [action=RCU]
            bash-1998  [000] ..s2   138.733105: softirq_entry: vec=1 [action=TIMER]
            bash-1998  [000] ..s2   138.733106: softirq_exit: vec=1 [action=TIMER]
            bash-1998  [000] ..s2   138.733106: softirq_entry: vec=9 [action=RCU]
            bash-1998  [000] ..s2   138.733109: softirq_exit: vec=9 [action=RCU]
            sshd-1995  [001] d.h1   138.733278: irq_handler_entry: irq=21 name=uhci_hcd:usb4
            sshd-1995  [001] d.h1   138.733280: irq_handler_exit: irq=21 ret=unhandled
            sshd-1995  [001] d.h1   138.733281: irq_handler_entry: irq=21 name=eth0
            sshd-1995  [001] d.h1   138.733283: irq_handler_exit: irq=21 ret=handled
[...]

# cat instances/zoot/trace
# tracer: nop
#
# entries-in-buffer/entries-written: 18996/18996   #P:4
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
            bash-1998  [000] d...   140.733501: sys_write -> 0x2
            bash-1998  [000] d...   140.733504: sys_dup2(oldfd: a, newfd: 1)
            bash-1998  [000] d...   140.733506: sys_dup2 -> 0x1
            bash-1998  [000] d...   140.733508: sys_fcntl(fd: a, cmd: 1, arg: 0)
            bash-1998  [000] d...   140.733509: sys_fcntl -> 0x1
            bash-1998  [000] d...   140.733510: sys_close(fd: a)
            bash-1998  [000] d...   140.733510: sys_close -> 0x0
            bash-1998  [000] d...   140.733514: sys_rt_sigprocmask(how: 0, nset: 0, oset: 6e2768, sigsetsize: 8)
            bash-1998  [000] d...   140.733515: sys_rt_sigprocmask -> 0x0
            bash-1998  [000] d...   140.733516: sys_rt_sigaction(sig: 2, act: 7fff718846f0, oact: 7fff71884650, sigsetsize: 8)
            bash-1998  [000] d...   140.733516: sys_rt_sigaction -> 0x0”…””}”hjf;  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M2hj;  hžhubh¸)”}”(hŒYou can see that the trace of the top most trace buffer shows only
the function tracing. The foo instance displays wakeups and task
switches.”h]”hŒYou can see that the trace of the top most trace buffer shows only
the function tracing. The foo instance displays wakeups and task
switches.”…””}”(hjt;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mhj;  hžhubh¸)”}”(hŒ<To remove the instances, simply delete their directories:
::”h]”hŒ9To remove the instances, simply delete their directories:”…””}”(hj‚;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M…hj;  hžhubj   )”}”(hŒB# rmdir instances/foo
# rmdir instances/bar
# rmdir instances/zoot”h]”hŒB# rmdir instances/foo
# rmdir instances/bar
# rmdir instances/zoot”…””}”hj;  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h Mˆhj;  hžhubh¸)”}”(hŒlNote, if a process has a trace file open in one of the instance
directories, the rmdir will fail with EBUSY.”h]”hŒlNote, if a process has a trace file open in one of the instance
directories, the rmdir will fail with EBUSY.”…””}”(hjž;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MŒhj;  hžhubeh}”(h]”Œ	instances”ah ]”h"]”Œ	instances”ah$]”h&]”uh1h¡hh£hžhhŸh¶h Mubh¢)”}”(hhh]”(h§)”}”(hŒStack trace”h]”hŒStack trace”…””}”(hj·;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hj´;  hžhhŸh¶h M‘ubh¸)”}”(hX%  Since the kernel has a fixed sized stack, it is important not to
waste it in functions. A kernel developer must be conscious of
what they allocate on the stack. If they add too much, the system
can be in danger of a stack overflow, and corruption will occur,
usually leading to a system panic.”h]”hX%  Since the kernel has a fixed sized stack, it is important not to
waste it in functions. A kernel developer must be conscious of
what they allocate on the stack. If they add too much, the system
can be in danger of a stack overflow, and corruption will occur,
usually leading to a system panic.”…””}”(hjÅ;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M’hj´;  hžhubh¸)”}”(hXA  There are some tools that check this, usually with interrupts
periodically checking usage. But if you can perform a check
at every function call that will become very useful. As ftrace provides
a function tracer, it makes it convenient to check the stack size
at every function call. This is enabled via the stack tracer.”h]”hXA  There are some tools that check this, usually with interrupts
periodically checking usage. But if you can perform a check
at every function call that will become very useful. As ftrace provides
a function tracer, it makes it convenient to check the stack size
at every function call. This is enabled via the stack tracer.”…””}”(hjÓ;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M˜hj´;  hžhubh¸)”}”(hŒŒCONFIG_STACK_TRACER enables the ftrace stack tracing functionality.
To enable it, write a '1' into /proc/sys/kernel/stack_tracer_enabled.
::”h]”hŒCONFIG_STACK_TRACER enables the ftrace stack tracing functionality.
To enable it, write a â€˜1â€™ into /proc/sys/kernel/stack_tracer_enabled.”…””}”(hjá;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h Mžhj´;  hžhubj   )”}”(hŒ0# echo 1 > /proc/sys/kernel/stack_tracer_enabled”h]”hŒ0# echo 1 > /proc/sys/kernel/stack_tracer_enabled”…””}”hjï;  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M¢hj´;  hžhubh¸)”}”(hŒ¦You can also enable it from the kernel command line to trace
the stack size of the kernel during boot up, by adding "stacktrace"
to the kernel command line parameter.”h]”hŒªYou can also enable it from the kernel command line to trace
the stack size of the kernel during boot up, by adding â€œstacktraceâ€
to the kernel command line parameter.”…””}”(hjý;  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¤hj´;  hžhubh¸)”}”(hŒ=After running it for a few minutes, the output looks like:
::”h]”hŒ:After running it for a few minutes, the output looks like:”…””}”(hj<  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h M¨hj´;  hžhubj   )”}”(hXõ  # cat stack_max_size
2928

# cat stack_trace
        Depth    Size   Location    (18 entries)
        -----    ----   --------
  0)     2928     224   update_sd_lb_stats+0xbc/0x4ac
  1)     2704     160   find_busiest_group+0x31/0x1f1
  2)     2544     256   load_balance+0xd9/0x662
  3)     2288      80   idle_balance+0xbb/0x130
  4)     2208     128   __schedule+0x26e/0x5b9
  5)     2080      16   schedule+0x64/0x66
  6)     2064     128   schedule_timeout+0x34/0xe0
  7)     1936     112   wait_for_common+0x97/0xf1
  8)     1824      16   wait_for_completion+0x1d/0x1f
  9)     1808     128   flush_work+0xfe/0x119
 10)     1680      16   tty_flush_to_ldisc+0x1e/0x20
 11)     1664      48   input_available_p+0x1d/0x5c
 12)     1616      48   n_tty_poll+0x6d/0x134
 13)     1568      64   tty_poll+0x64/0x7f
 14)     1504     880   do_select+0x31e/0x511
 15)      624     400   core_sys_select+0x177/0x216
 16)      224      96   sys_select+0x91/0xb9
 17)      128     128   system_call_fastpath+0x16/0x1b”h]”hXõ  # cat stack_max_size
2928

# cat stack_trace
        Depth    Size   Location    (18 entries)
        -----    ----   --------
  0)     2928     224   update_sd_lb_stats+0xbc/0x4ac
  1)     2704     160   find_busiest_group+0x31/0x1f1
  2)     2544     256   load_balance+0xd9/0x662
  3)     2288      80   idle_balance+0xbb/0x130
  4)     2208     128   __schedule+0x26e/0x5b9
  5)     2080      16   schedule+0x64/0x66
  6)     2064     128   schedule_timeout+0x34/0xe0
  7)     1936     112   wait_for_common+0x97/0xf1
  8)     1824      16   wait_for_completion+0x1d/0x1f
  9)     1808     128   flush_work+0xfe/0x119
 10)     1680      16   tty_flush_to_ldisc+0x1e/0x20
 11)     1664      48   input_available_p+0x1d/0x5c
 12)     1616      48   n_tty_poll+0x6d/0x134
 13)     1568      64   tty_poll+0x64/0x7f
 14)     1504     880   do_select+0x31e/0x511
 15)      624     400   core_sys_select+0x177/0x216
 16)      224      96   sys_select+0x91/0xb9
 17)      128     128   system_call_fastpath+0x16/0x1b”…””}”hj<  sbah}”(h]”h ]”h"]”h$]”h&]”j¯  j°  uh1jŸ  hŸh¶h M«hj´;  hžhubh¸)”}”(hŒ¿Note, if -mfentry is being used by gcc, functions get traced before
they set up the stack frame. This means that leaf level functions
are not tested by the stack tracer when -mfentry is used.”h]”hŒ¿Note, if -mfentry is being used by gcc, functions get traced before
they set up the stack frame. This means that leaf level functions
are not tested by the stack tracer when -mfentry is used.”…””}”(hj'<  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÄhj´;  hžhubh¸)”}”(hŒ?Currently, -mfentry is used by gcc 4.6.0 and above on x86 only.”h]”hŒ?Currently, -mfentry is used by gcc 4.6.0 and above on x86 only.”…””}”(hj5<  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÈhj´;  hžhubeh}”(h]”Œstack-trace”ah ]”h"]”Œstack trace”ah$]”h&]”uh1h¡hh£hžhhŸh¶h M‘ubh¢)”}”(hhh]”(h§)”}”(hŒMore”h]”hŒMore”…””}”(hjN<  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h¦hjK<  hžhhŸh¶h MËubh¸)”}”(hŒNMore details can be found in the source code, in the `kernel/trace/*.c` files.”h]”(hŒ5More details can be found in the source code, in the ”…””}”(hj\<  hžhhŸNh NubhŒtitle_reference”“”)”}”(hŒ`kernel/trace/*.c`”h]”hŒkernel/trace/*.c”…””}”(hjf<  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1jd<  hj\<  ubhŒ files.”…””}”(hj\<  hžhhŸNh Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1h·hŸh¶h MÌhjK<  hžhubeh}”(h]”Œmore”ah ]”h"]”Œmore”ah$]”h&]”uh1h¡hh£hžhhŸh¶h MËubeh}”(h]”Œftrace-function-tracer”ah ]”h"]”Œftrace - function tracer”ah$]”h&]”uh1h¡hhhžhhŸh¶h Kubeh}”(h]”h ]”h"]”h$]”h&]”Œsource”h¶uh1hŒcurrent_source”NŒcurrent_line”NŒsettings”Œdocutils.frontend”ŒValues”“”)”}”(h¦NŒ	generator”NŒ	datestamp”NŒsource_link”NŒ
source_url”NŒtoc_backlinks”jw9  Œfootnote_backlinks”KŒsectnum_xform”KŒstrip_comments”NŒstrip_elements_with_classes”NŒstrip_classes”NŒreport_level”KŒ
halt_level”KŒexit_status_level”KŒdebug”NŒwarning_stream”NŒ	traceback”ˆŒinput_encoding”Œ	utf-8-sig”Œinput_encoding_error_handler”Œstrict”Œoutput_encoding”Œutf-8”Œoutput_encoding_error_handler”j°<  Œerror_encoding”Œutf-8”Œerror_encoding_error_handler”Œbackslashreplace”Œlanguage_code”Œen”Œrecord_dependencies”NŒconfig”NŒ	id_prefix”hŒauto_id_prefix”Œid”Œdump_settings”NŒdump_internals”NŒdump_transforms”NŒdump_pseudo_xml”NŒexpose_internals”NŒstrict_visitor”NŒ_disable_config”NŒ_source”h¶Œ_destination”NŒ_config_files”]”Œ7/var/lib/git/docbuild/linux/Documentation/docutils.conf”aŒfile_insertion_enabled”ˆŒraw_enabled”KŒline_length_limit”M'Œpep_references”NŒpep_base_url”Œhttps://peps.python.org/”Œpep_file_url_template”Œpep-%04d”Œrfc_references”NŒrfc_base_url”Œ&https://datatracker.ietf.org/doc/html/”Œ	tab_width”KŒtrim_footnote_reference_space”‰Œsyntax_highlight”Œlong”Œsmart_quotes”ˆŒsmartquotes_locales”]”Œcharacter_level_inline_markup”‰Œdoctitle_xform”‰Œdocinfo_xform”KŒsectsubtitle_xform”‰Œimage_loading”Œlink”Œembed_stylesheet”‰Œcloak_email_addresses”ˆŒsection_self_link”‰Œenv”NubŒreporter”NŒindirect_targets”]”Œsubstitution_defs”}”Œsubstitution_names”}”Œrefnames”}”Œrefids”}”Œnameids”}”(j‹<  jˆ<  jH  jE  jo  jl  j  j  jG  jD  j¾  j»  jå  jâ  j6  j3  j&  j#  j4$  j1$  jÙ$  jÖ$  jF%  jC%  j&  j&  jf&  jc&  j5'  j2'  jj'  jg'  jŽ)  j‹)  jß)  jÜ)  jL*  jI*  j¬0  j©0  jê3  jç3  j;4  j84  jÒ4  jÏ4  j#5  j 5  j©7  j¦7  jì7  jé7  j×8  jÔ8  j;  j	;  j±;  j®;  jH<  jE<  jƒ<  j€<  uŒ	nametypes”}”(j‹<  ‰jH  ‰jo  ‰j  ‰jG  ‰j¾  ‰jå  ‰j6  ‰j&  ‰j4$  ‰jÙ$  ‰jF%  ‰j&  ‰jf&  ‰j5'  ‰jj'  ‰jŽ)  ‰jß)  ‰jL*  ‰j¬0  ‰jê3  ‰j;4  ‰jÒ4  ‰j#5  ‰j©7  ‰jì7  ‰j×8  ‰j;  ‰j±;  ‰jH<  ‰jƒ<  ‰uh}”(jˆ<  h£jE  jú  jl  jK  j  jr  jD  j  j»  jJ  jâ  jÁ  j3  jè  j#  j9  j1$  j)  jÖ$  j7$  jC%  jÜ$  j&  jI%  jc&  j
&  j2'  ji&  jg'  j8'  j‹)  jm'  jÜ)  j‘)  jI*  jâ)  j©0  jO*  jç3  j¯0  j84  jí3  jÏ4  j>4  j 5  jÕ4  j¦7  j&5  jé7  j¬7  jÔ8  jï7  j	;  jÚ8  j®;  j;  jE<  j´;  j€<  jK<  uŒfootnote_refs”}”Œcitation_refs”}”Œautofootnotes”]”Œautofootnote_refs”]”Œsymbol_footnotes”]”Œsymbol_footnote_refs”]”Œ	footnotes”]”Œ	citations”]”Œautofootnote_start”KŒsymbol_footnote_start”K Œ
id_counter”Œcollections”ŒCounter”“”}”…”R”Œparse_messages”]”(hŒsystem_message”“”)”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hj=  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hj=  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”ŒINFO”Œline”M9Œsource”h¶uh1j=  hji&  hžhhŸh¶h M9ubj=  )”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hj3=  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hj0=  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”MCŒsource”h¶uh1j=  hji&  hžhhŸh¶h MCubj=  )”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hjN=  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hjK=  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”Mý	Œsource”h¶uh1j=  hjâ)  hžhhŸh¶h Mý	ubj=  )”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hji=  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hjf=  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”MÑŒsource”h¶uh1j=  hj¯0  hžhhŸh¶h MÑubj=  )”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hj„=  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hj=  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”MŒsource”h¶uh1j=  hj¯0  hžhhŸh¶h Mubj=  )”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hjŸ=  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hjœ=  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”MŒsource”h¶uh1j=  hjï7  hžhhŸh¶h Mubj=  )”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hjº=  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hj·=  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”M•Œsource”h¶uh1j=  hjï7  hžhhŸh¶h M•ubj=  )”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hjÕ=  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hjÒ=  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”M¥Œsource”h¶uh1j=  hjï7  hžhhŸh¶h M¥ubj=  )”}”(hhh]”h¸)”}”(•¡      hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hjð=  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hjí=  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”M²Œsource”h¶uh1j=  hjï7  hžhhŸh¶h M²ubj=  )”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hj>  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hj>  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”MàŒsource”h¶uh1j=  hjÚ8  hžhhŸh¶h Màubj=  )”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hj&>  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hj#>  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”M†Œsource”h¶uh1j=  hj;  hžhhŸh¶h M†ubj=  )”}”(hhh]”h¸)”}”(hŒfPossible title underline, too short for the title.
Treating it as ordinary text because it's so short.”h]”hŒhPossible title underline, too short for the title.
Treating it as ordinary text because itâ€™s so short.”…””}”(hjA>  hžhhŸNh Nubah}”(h]”h ]”h"]”h$]”h&]”uh1h·hj>>  ubah}”(h]”h ]”h"]”h$]”h&]”Œlevel”KŒtype”j-=  Œline”M©Œsource”h¶uh1j=  hj´;  hžhhŸh¶h M©ubeŒtransform_messages”]”Œtransformer”NŒinclude_log”]”Œ
decoration”Nhžhub.