€•Ä      Œsphinx.addnodes”Œdocument”“”)”}”(Œ	rawsource”Œ ”Œchildren”]”(Œtranslations”ŒLanguagesNode”“”)”}”(hhh]”(h Œpending_xref”“”)”}”(hhh]”Œdocutils.nodes”ŒText”“”ŒChinese (Simplified)”…””}”Œparent”hsbaŒ
attributes”}”(Œids”]”Œclasses”]”Œnames”]”Œdupnames”]”Œbackrefs”]”Œ	refdomain”Œstd”Œreftype”Œdoc”Œ	reftarget”Œ)/translations/zh_CN/trace/timerlat-tracer”Œmodname”NŒ	classname”NŒrefexplicit”ˆuŒtagname”hhhubh)”}”(hhh]”hŒChinese (Traditional)”…””}”hh2sbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ)/translations/zh_TW/trace/timerlat-tracer”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubh)”}”(hhh]”hŒItalian”…””}”hhFsbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ)/translations/it_IT/trace/timerlat-tracer”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubh)”}”(hhh]”hŒJapanese”…””}”hhZsbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ)/translations/ja_JP/trace/timerlat-tracer”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubh)”}”(hhh]”hŒKorean”…””}”hhnsbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ)/translations/ko_KR/trace/timerlat-tracer”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubh)”}”(hhh]”hŒPortuguese (Brazilian)”…””}”hh‚sbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ)/translations/pt_BR/trace/timerlat-tracer”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubh)”}”(hhh]”hŒSpanish”…””}”hh–sbah}”(h]”h ]”h"]”h$]”h&]”Œ	refdomain”h)Œreftype”h+Œ	reftarget”Œ)/translations/sp_SP/trace/timerlat-tracer”Œmodname”NŒ	classname”NŒrefexplicit”ˆuh1hhhubeh}”(h]”h ]”h"]”h$]”h&]”Œcurrent_language”ŒEnglish”uh1h
hhŒ	_document”hŒsource”NŒline”NubhŒsection”“”)”}”(hhh]”(hŒtitle”“”)”}”(hŒTimerlat tracer”h]”hŒTimerlat tracer”…””}”(hh¼h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hºhh·h²hh³ŒC/var/lib/git/docbuild/linux/Documentation/trace/timerlat-tracer.rst”h´KubhŒ	paragraph”“”)”}”(hX¬  The timerlat tracer aims to help the preemptive kernel developers to
find sources of wakeup latencies of real-time threads. Like cyclictest,
the tracer sets a periodic timer that wakes up a thread. The thread then
computes a *wakeup latency* value as the difference between the *current
time* and the *absolute time* that the timer was set to expire. The main
goal of timerlat is tracing in such a way to help kernel developers.”h]”(hŒáThe timerlat tracer aims to help the preemptive kernel developers to
find sources of wakeup latencies of real-time threads. Like cyclictest,
the tracer sets a periodic timer that wakes up a thread. The thread then
computes a ”…””}”(hhÍh²hh³Nh´NubhŒemphasis”“”)”}”(hŒ*wakeup latency*”h]”hŒwakeup latency”…””}”(hh×h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhhÍubhŒ% value as the difference between the ”…””}”(hhÍh²hh³Nh´NubhÖ)”}”(hŒ*current
time*”h]”hŒcurrent
time”…””}”(hhéh²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhhÍubhŒ	 and the ”…””}”(hhÍh²hh³Nh´NubhÖ)”}”(hŒ*absolute time*”h]”hŒabsolute time”…””}”(hhûh²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhhÍubhŒp that the timer was set to expire. The main
goal of timerlat is tracing in such a way to help kernel developers.”…””}”(hhÍh²hh³Nh´Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´Khh·h²hubh¶)”}”(hhh]”(h»)”}”(hŒUsage”h]”hŒUsage”…””}”(hj  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hºhj  h²hh³hÊh´KubhÌ)”}”(hŒ~Write the ASCII text "timerlat" into the current_tracer file of the
tracing system (generally mounted at /sys/kernel/tracing).”h]”hŒ‚Write the ASCII text â€œtimerlatâ€ into the current_tracer file of the
tracing system (generally mounted at /sys/kernel/tracing).”…””}”(hj$  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´Khj  h²hubhÌ)”}”(hŒFor example::”h]”hŒFor example:”…””}”(hj2  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´Khj  h²hubhŒliteral_block”“”)”}”(hŒX[root@f32 ~]# cd /sys/kernel/tracing/
[root@f32 tracing]# echo timerlat > current_tracer”h]”hŒX[root@f32 ~]# cd /sys/kernel/tracing/
[root@f32 tracing]# echo timerlat > current_tracer”…””}”hjB  sbah}”(h]”h ]”h"]”h$]”h&]”Œ	xml:space”Œpreserve”uh1j@  h³hÊh´Khj  h²hubhÌ)”}”(hŒ>It is possible to follow the trace by reading the trace file::”h]”hŒ=It is possible to follow the trace by reading the trace file:”…””}”(hjR  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´Khj  h²hubjA  )”}”(hX%  [root@f32 tracing]# cat trace
# tracer: timerlat
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            || /
#                            ||||             ACTIVATION
#         TASK-PID      CPU# ||||   TIMESTAMP    ID            CONTEXT                LATENCY
#            | |         |   ||||      |         |                  |                       |
        <idle>-0       [000] d.h1    54.029328: #1     context    irq timer_latency       932 ns
         <...>-867     [000] ....    54.029339: #1     context thread timer_latency     11700 ns
        <idle>-0       [001] dNh1    54.029346: #1     context    irq timer_latency      2833 ns
         <...>-868     [001] ....    54.029353: #1     context thread timer_latency      9820 ns
        <idle>-0       [000] d.h1    54.030328: #2     context    irq timer_latency       769 ns
         <...>-867     [000] ....    54.030330: #2     context thread timer_latency      3070 ns
        <idle>-0       [001] d.h1    54.030344: #2     context    irq timer_latency       935 ns
         <...>-868     [001] ....    54.030347: #2     context thread timer_latency      4351 ns”h]”hX%  [root@f32 tracing]# cat trace
# tracer: timerlat
#
#                              _-----=> irqs-off
#                             / _----=> need-resched
#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            || /
#                            ||||             ACTIVATION
#         TASK-PID      CPU# ||||   TIMESTAMP    ID            CONTEXT                LATENCY
#            | |         |   ||||      |         |                  |                       |
        <idle>-0       [000] d.h1    54.029328: #1     context    irq timer_latency       932 ns
         <...>-867     [000] ....    54.029339: #1     context thread timer_latency     11700 ns
        <idle>-0       [001] dNh1    54.029346: #1     context    irq timer_latency      2833 ns
         <...>-868     [001] ....    54.029353: #1     context thread timer_latency      9820 ns
        <idle>-0       [000] d.h1    54.030328: #2     context    irq timer_latency       769 ns
         <...>-867     [000] ....    54.030330: #2     context thread timer_latency      3070 ns
        <idle>-0       [001] d.h1    54.030344: #2     context    irq timer_latency       935 ns
         <...>-868     [001] ....    54.030347: #2     context thread timer_latency      4351 ns”…””}”hj`  sbah}”(h]”h ]”h"]”h$]”h&]”jP  jQ  uh1j@  h³hÊh´Khj  h²hubhÌ)”}”(hX€  The tracer creates a per-cpu kernel thread with real-time priority
SCHED_FIFO:95 that prints two lines at every activation. The first is
the *timer latency* observed at the *hardirq* context before the activation
of the thread. The second is the *timer latency* observed by the thread.
The ACTIVATION ID field serves to relate the *irq* execution to its
respective *thread* execution.”h]”(hŒThe tracer creates a per-cpu kernel thread with real-time priority
SCHED_FIFO:95 that prints two lines at every activation. The first is
the ”…””}”(hjn  h²hh³Nh´NubhÖ)”}”(hŒ*timer latency*”h]”hŒtimer latency”…””}”(hjv  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjn  ubhŒ observed at the ”…””}”(hjn  h²hh³Nh´NubhÖ)”}”(hŒ	*hardirq*”h]”hŒhardirq”…””}”(hjˆ  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjn  ubhŒ@ context before the activation
of the thread. The second is the ”…””}”(hjn  h²hh³Nh´NubhÖ)”}”(hŒ*timer latency*”h]”hŒtimer latency”…””}”(hjš  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjn  ubhŒF observed by the thread.
The ACTIVATION ID field serves to relate the ”…””}”(hjn  h²hh³Nh´NubhÖ)”}”(hŒ*irq*”h]”hŒirq”…””}”(hj¬  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjn  ubhŒ execution to its
respective ”…””}”(hjn  h²hh³Nh´NubhÖ)”}”(hŒ*thread*”h]”hŒthread”…””}”(hj¾  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjn  ubhŒ execution.”…””}”(hjn  h²hh³Nh´Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´K.hj  h²hubhÌ)”}”(hX  The *irq*/*thread* splitting is important to clarify in which context
the unexpected high value is coming from. The *irq* context can be
delayed by hardware-related actions, such as SMIs, NMIs, IRQs,
or by thread masking interrupts. Once the timer happens, the delay
can also be influenced by blocking caused by threads. For example, by
postponing the scheduler execution via preempt_disable(), scheduler
execution, or masking interrupts. Threads can also be delayed by the
interference from other threads and IRQs.”h]”(hŒThe ”…””}”(hjÖ  h²hh³Nh´NubhÖ)”}”(hŒ*irq*”h]”hŒirq”…””}”(hjÞ  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjÖ  ubhŒ/”…””}”(hjÖ  h²hh³Nh´NubhÖ)”}”(hŒ*thread*”h]”hŒthread”…””}”(hjð  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjÖ  ubhŒb splitting is important to clarify in which context
the unexpected high value is coming from. The ”…””}”(hjÖ  h²hh³Nh´NubhÖ)”}”(hŒ*irq*”h]”hŒirq”…””}”(hj  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjÖ  ubhXŠ   context can be
delayed by hardware-related actions, such as SMIs, NMIs, IRQs,
or by thread masking interrupts. Once the timer happens, the delay
can also be influenced by blocking caused by threads. For example, by
postponing the scheduler execution via preempt_disable(), scheduler
execution, or masking interrupts. Threads can also be delayed by the
interference from other threads and IRQs.”…””}”(hjÖ  h²hh³Nh´Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´K5hj  h²hubeh}”(h]”Œusage”ah ]”h"]”Œusage”ah$]”h&]”uh1hµhh·h²hh³hÊh´Kubh¶)”}”(hhh]”(h»)”}”(hŒTracer options”h]”hŒTracer options”…””}”(hj%  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hºhj"  h²hh³hÊh´K?ubhÌ)”}”(hŒ”The timerlat tracer is built on top of osnoise tracer.
So its configuration is also done in the osnoise/ config
directory. The timerlat configs are:”h]”hŒ”The timerlat tracer is built on top of osnoise tracer.
So its configuration is also done in the osnoise/ config
directory. The timerlat configs are:”…””}”(hj3  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KAhj"  h²hubhŒblock_quote”“”)”}”(hX`  - cpus: CPUs at which a timerlat thread will execute.
- timerlat_period_us: the period of the timerlat thread.
- stop_tracing_us: stop the system tracing if a
  timer latency at the *irq* context higher than the configured
  value happens. Writing 0 disables this option.
- stop_tracing_total_us: stop the system tracing if a
  timer latency at the *thread* context is higher than the configured
  value happens. Writing 0 disables this option.
- print_stack: save the stack of the IRQ occurrence. The stack is printed
  after the *thread context* event, or at the IRQ handler if *stop_tracing_us*
  is hit.
”h]”hŒbullet_list”“”)”}”(hhh]”(hŒ	list_item”“”)”}”(hŒ3cpus: CPUs at which a timerlat thread will execute.”h]”hÌ)”}”(hjP  h]”hŒ3cpus: CPUs at which a timerlat thread will execute.”…””}”(hjR  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KEhjN  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jL  hjI  ubjM  )”}”(hŒ6timerlat_period_us: the period of the timerlat thread.”h]”hÌ)”}”(hjg  h]”hŒ6timerlat_period_us: the period of the timerlat thread.”…””}”(hji  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KFhje  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jL  hjI  ubjM  )”}”(hŒšstop_tracing_us: stop the system tracing if a
timer latency at the *irq* context higher than the configured
value happens. Writing 0 disables this option.”h]”hÌ)”}”(hŒšstop_tracing_us: stop the system tracing if a
timer latency at the *irq* context higher than the configured
value happens. Writing 0 disables this option.”h]”(hŒCstop_tracing_us: stop the system tracing if a
timer latency at the ”…””}”(hj€  h²hh³Nh´NubhÖ)”}”(hŒ*irq*”h]”hŒirq”…””}”(hjˆ  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhj€  ubhŒR context higher than the configured
value happens. Writing 0 disables this option.”…””}”(hj€  h²hh³Nh´Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KGhj|  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jL  hjI  ubjM  )”}”(hŒ¦stop_tracing_total_us: stop the system tracing if a
timer latency at the *thread* context is higher than the configured
value happens. Writing 0 disables this option.”h]”hÌ)”}”(hŒ¦stop_tracing_total_us: stop the system tracing if a
timer latency at the *thread* context is higher than the configured
value happens. Writing 0 disables this option.”h]”(hŒIstop_tracing_total_us: stop the system tracing if a
timer latency at the ”…””}”(hjª  h²hh³Nh´NubhÖ)”}”(hŒ*thread*”h]”hŒthread”…””}”(hj²  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjª  ubhŒU context is higher than the configured
value happens. Writing 0 disables this option.”…””}”(hjª  h²hh³Nh´Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KJhj¦  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jL  hjI  ubjM  )”}”(hŒprint_stack: save the stack of the IRQ occurrence. The stack is printed
after the *thread context* event, or at the IRQ handler if *stop_tracing_us*
is hit.
”h]”hÌ)”}”(hŒœprint_stack: save the stack of the IRQ occurrence. The stack is printed
after the *thread context* event, or at the IRQ handler if *stop_tracing_us*
is hit.”h]”(hŒRprint_stack: save the stack of the IRQ occurrence. The stack is printed
after the ”…””}”(hjÔ  h²hh³Nh´NubhÖ)”}”(hŒ*thread context*”h]”hŒthread context”…””}”(hjÜ  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjÔ  ubhŒ! event, or at the IRQ handler if ”…””}”(hjÔ  h²hh³Nh´NubhÖ)”}”(hŒ*stop_tracing_us*”h]”hŒstop_tracing_us”…””}”(hjî  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjÔ  ubhŒ
is hit.”…””}”(hjÔ  h²hh³Nh´Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KMhjÐ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jL  hjI  ubeh}”(h]”h ]”h"]”h$]”h&]”Œbullet”Œ-”uh1jG  h³hÊh´KEhjC  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jA  h³hÊh´KEhj"  h²hubeh}”(h]”Œtracer-options”ah ]”h"]”Œtracer options”ah$]”h&]”uh1hµhh·h²hh³hÊh´K?ubh¶)”}”(hhh]”(h»)”}”(hŒtimerlat and osnoise”h]”hŒtimerlat and osnoise”…””}”(hj%  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hºhj"  h²hh³hÊh´KRubhÌ)”}”(hŒOThe timerlat can also take advantage of the osnoise: traceevents.
For example::”h]”hŒNThe timerlat can also take advantage of the osnoise: traceevents.
For example:”…””}”(hj3  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KThj"  h²hubjA  )”}”(hXü    [root@f32 ~]# cd /sys/kernel/tracing/
  [root@f32 tracing]# echo timerlat > current_tracer
  [root@f32 tracing]# echo 1 > events/osnoise/enable
  [root@f32 tracing]# echo 25 > osnoise/stop_tracing_total_us
  [root@f32 tracing]# tail -10 trace
       cc1-87882   [005] d..h...   548.771078: #402268 context    irq timer_latency     13585 ns
       cc1-87882   [005] dNLh1..   548.771082: irq_noise: local_timer:236 start 548.771077442 duration 7597 ns
       cc1-87882   [005] dNLh2..   548.771099: irq_noise: qxl:21 start 548.771085017 duration 7139 ns
       cc1-87882   [005] d...3..   548.771102: thread_noise:      cc1:87882 start 548.771078243 duration 9909 ns
timerlat/5-1035    [005] .......   548.771104: #402268 context thread timer_latency     39960 ns”h]”hXü    [root@f32 ~]# cd /sys/kernel/tracing/
  [root@f32 tracing]# echo timerlat > current_tracer
  [root@f32 tracing]# echo 1 > events/osnoise/enable
  [root@f32 tracing]# echo 25 > osnoise/stop_tracing_total_us
  [root@f32 tracing]# tail -10 trace
       cc1-87882   [005] d..h...   548.771078: #402268 context    irq timer_latency     13585 ns
       cc1-87882   [005] dNLh1..   548.771082: irq_noise: local_timer:236 start 548.771077442 duration 7597 ns
       cc1-87882   [005] dNLh2..   548.771099: irq_noise: qxl:21 start 548.771085017 duration 7139 ns
       cc1-87882   [005] d...3..   548.771102: thread_noise:      cc1:87882 start 548.771078243 duration 9909 ns
timerlat/5-1035    [005] .......   548.771104: #402268 context thread timer_latency     39960 ns”…””}”hjA  sbah}”(h]”h ]”h"]”h$]”h&]”jP  jQ  uh1j@  h³hÊh´KWhj"  h²hubhÌ)”}”(hX(  In this case, the root cause of the timer latency does not point to a
single cause but to multiple ones. Firstly, the timer IRQ was delayed
for 13 us, which may point to a long IRQ disabled section (see IRQ
stacktrace section). Then the timer interrupt that wakes up the timerlat
thread took 7597 ns, and the qxl:21 device IRQ took 7139 ns. Finally,
the cc1 thread noise took 9909 ns of time before the context switch.
Such pieces of evidence are useful for the developer to use other
tracing methods to figure out how to debug and optimize the system.”h]”hX(  In this case, the root cause of the timer latency does not point to a
single cause but to multiple ones. Firstly, the timer IRQ was delayed
for 13 us, which may point to a long IRQ disabled section (see IRQ
stacktrace section). Then the timer interrupt that wakes up the timerlat
thread took 7597 ns, and the qxl:21 device IRQ took 7139 ns. Finally,
the cc1 thread noise took 9909 ns of time before the context switch.
Such pieces of evidence are useful for the developer to use other
tracing methods to figure out how to debug and optimize the system.”…””}”(hjO  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´Kbhj"  h²hubhÌ)”}”(hXH  It is worth mentioning that the *duration* values reported
by the osnoise: events are *net* values. For example, the
thread_noise does not include the duration of the overhead caused
by the IRQ execution (which indeed accounted for 12736 ns). But
the values reported by the timerlat tracer (timerlat_latency)
are *gross* values.”h]”(hŒ It is worth mentioning that the ”…””}”(hj]  h²hh³Nh´NubhÖ)”}”(hŒ
*duration*”h]”hŒduration”…””}”(hje  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhj]  ubhŒ, values reported
by the osnoise: events are ”…””}”(hj]  h²hh³Nh´NubhÖ)”}”(hŒ*net*”h]”hŒnet”…””}”(hjw  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhj]  ubhŒÞ values. For example, the
thread_noise does not include the duration of the overhead caused
by the IRQ execution (which indeed accounted for 12736 ns). But
the values reported by the timerlat tracer (timerlat_latency)
are ”…””}”(hj]  h²hh³Nh´NubhÖ)”}”(hŒ*gross*”h]”hŒgross”…””}”(hj‰  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhj]  ubhŒ values.”…””}”(hj]  h²hh³Nh´Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´Kkhj"  h²hubhÌ)”}”(hŒÇThe art below illustrates a CPU timeline and how the timerlat tracer
observes it at the top and the osnoise: events at the bottom. Each "-"
in the timelines means circa 1 us, and the time moves ==>::”h]”hŒÊThe art below illustrates a CPU timeline and how the timerlat tracer
observes it at the top and the osnoise: events at the bottom. Each â€œ-â€
in the timelines means circa 1 us, and the time moves ==>:”…””}”(hj¡  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´Krhj"  h²hubjA  )”}”(hX×      External     timer irq                   thread
     clock        latency                    latency
     event        13585 ns                   39960 ns
       |             ^                         ^
       v             |                         |
       |-------------|                         |
       |-------------+-------------------------|
                     ^                         ^
========================================================================
                  [tmr irq]  [dev irq]
[another thread...^       v..^       v.......][timerlat/ thread]  <-- CPU timeline
=========================================================================
                  |-------|  |-------|
                          |--^       v-------|
                          |          |       |
                          |          |       + thread_noise: 9909 ns
                          |          +-> irq_noise: 6139 ns
                          +-> irq_noise: 7597 ns”h]”hX×      External     timer irq                   thread
     clock        latency                    latency
     event        13585 ns                   39960 ns
       |             ^                         ^
       v             |                         |
       |-------------|                         |
       |-------------+-------------------------|
                     ^                         ^
========================================================================
                  [tmr irq]  [dev irq]
[another thread...^       v..^       v.......][timerlat/ thread]  <-- CPU timeline
=========================================================================
                  |-------|  |-------|
                          |--^       v-------|
                          |          |       |
                          |          |       + thread_noise: 9909 ns
                          |          +-> irq_noise: 6139 ns
                          +-> irq_noise: 7597 ns”…””}”hj¯  sbah}”(h]”h ]”h"]”h$]”h&]”jP  jQ  uh1j@  h³hÊh´Kvhj"  h²hubeh}”(h]”Œtimerlat-and-osnoise”ah ]”h"]”Œtimerlat and osnoise”ah$]”h&]”uh1hµhh·h²hh³hÊh´KRubh¶)”}”(hhh]”(h»)”}”(hŒIRQ stacktrace”h]”hŒIRQ stacktrace”…””}”(hjÈ  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hºhjÅ  h²hh³hÊh´KŠubhÌ)”}”(hŒ°The osnoise/print_stack option is helpful for the cases in which a thread
noise causes the major factor for the timer latency, because of preempt or
irq disabled. For example::”h]”hŒ¯The osnoise/print_stack option is helpful for the cases in which a thread
noise causes the major factor for the timer latency, because of preempt or
irq disabled. For example:”…””}”(hjÖ  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KŒhjÅ  h²hubjA  )”}”(hX        [root@f32 tracing]# echo 500 > osnoise/stop_tracing_total_us
      [root@f32 tracing]# echo 500 > osnoise/print_stack
      [root@f32 tracing]# echo timerlat > current_tracer
      [root@f32 tracing]# tail -21 per_cpu/cpu7/trace
        insmod-1026    [007] dN.h1..   200.201948: irq_noise: local_timer:236 start 200.201939376 duration 7872 ns
        insmod-1026    [007] d..h1..   200.202587: #29800 context    irq timer_latency      1616 ns
        insmod-1026    [007] dN.h2..   200.202598: irq_noise: local_timer:236 start 200.202586162 duration 11855 ns
        insmod-1026    [007] dN.h3..   200.202947: irq_noise: local_timer:236 start 200.202939174 duration 7318 ns
        insmod-1026    [007] d...3..   200.203444: thread_noise:   insmod:1026 start 200.202586933 duration 838681 ns
    timerlat/7-1001    [007] .......   200.203445: #29800 context thread timer_latency    859978 ns
    timerlat/7-1001    [007] ....1..   200.203446: <stack trace>
=> timerlat_irq
=> __hrtimer_run_queues
=> hrtimer_interrupt
=> __sysvec_apic_timer_interrupt
=> asm_call_irq_on_stack
=> sysvec_apic_timer_interrupt
=> asm_sysvec_apic_timer_interrupt
=> delay_tsc
=> dummy_load_1ms_pd_init
=> do_one_initcall
=> do_init_module
=> __do_sys_finit_module
=> do_syscall_64
=> entry_SYSCALL_64_after_hwframe”h]”hX        [root@f32 tracing]# echo 500 > osnoise/stop_tracing_total_us
      [root@f32 tracing]# echo 500 > osnoise/print_stack
      [root@f32 tracing]# echo timerlat > current_tracer
      [root@f32 tracing]# tail -21 per_cpu/cpu7/trace
        insmod-1026    [007] dN.h1..   200.201948: irq_noise: local_timer:236 start 200.201939376 duration 7872 ns
        insmod-1026    [007] d..h1..   200.202587: #29800 context    irq timer_latency      1616 ns
        insmod-1026    [007] dN.h2..   200.202598: irq_noise: local_timer:236 start 200.202586162 duration 11855 ns
        insmod-1026    [007] dN.h3..   200.202947: irq_noise: local_timer:236 start 200.202939174 duration 7318 ns
        insmod-1026    [007] d...3..   200.203444: thread_noise:   insmod:1026 start 200.202586933 duration 838681 ns
    timerlat/7-1001    [007] .......   200.203445: #29800 context thread timer_latency    859978 ns
    timerlat/7-1001    [007] ....1..   200.203446: <stack trace>
=> timerlat_irq
=> __hrtimer_run_queues
=> hrtimer_interrupt
=> __sysvec_apic_timer_interrupt
=> asm_call_irq_on_stack
=> sysvec_apic_timer_interrupt
=> asm_sysvec_apic_timer_interrupt
=> delay_tsc
=> dummy_load_1ms_pd_init
=> do_one_initcall
=> do_init_module
=> __do_sys_finit_module
=> do_syscall_64
=> entry_SYSCALL_64_after_hwframe”…””}”hjä  sbah}”(h]”h ]”h"]”h$]”h&]”jP  jQ  uh1j@  h³hÊh´KhjÅ  h²hubhÌ)”}”(hX  In this case, it is possible to see that the thread added the highest
contribution to the *timer latency* and the stack trace, saved during
the timerlat IRQ handler, points to a function named
dummy_load_1ms_pd_init, which had the following code (on purpose)::”h]”(hŒZIn this case, it is possible to see that the thread added the highest
contribution to the ”…””}”(hjò  h²hh³Nh´NubhÖ)”}”(hŒ*timer latency*”h]”hŒtimer latency”…””}”(hjú  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hÕhjò  ubhŒš and the stack trace, saved during
the timerlat IRQ handler, points to a function named
dummy_load_1ms_pd_init, which had the following code (on purpose):”…””}”(hjò  h²hh³Nh´Nubeh}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KªhjÅ  h²hubjA  )”}”(hŒstatic int __init dummy_load_1ms_pd_init(void)
{
        preempt_disable();
        mdelay(1);
        preempt_enable();
        return 0;

}”h]”hŒstatic int __init dummy_load_1ms_pd_init(void)
{
        preempt_disable();
        mdelay(1);
        preempt_enable();
        return 0;

}”…””}”hj  sbah}”(h]”h ]”h"]”h$]”h&]”jP  jQ  uh1j@  h³hÊh´K¯hjÅ  h²hubeh}”(h]”Œirq-stacktrace”ah ]”h"]”Œirq stacktrace”ah$]”h&]”uh1hµhh·h²hh³hÊh´KŠubh¶)”}”(hhh]”(h»)”}”(hŒUser-space interface”h]”hŒUser-space interface”…””}”(hj+  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hºhj(  h²hh³hÊh´K¹ubhÌ)”}”(hŒÔTimerlat allows user-space threads to use timerlat infra-structure to
measure scheduling latency. This interface is accessible via a per-CPU
file descriptor inside $tracing_dir/osnoise/per_cpu/cpu$ID/timerlat_fd.”h]”hŒÔTimerlat allows user-space threads to use timerlat infra-structure to
measure scheduling latency. This interface is accessible via a per-CPU
file descriptor inside $tracing_dir/osnoise/per_cpu/cpu$ID/timerlat_fd.”…””}”(hj9  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´K»hj(  h²hubhÌ)”}”(hŒ<This interface is accessible under the following conditions:”h]”hŒ<This interface is accessible under the following conditions:”…””}”(hjG  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´K¿hj(  h²hubjB  )”}”(hŒþ- timerlat tracer is enable
- osnoise workload option is set to NO_OSNOISE_WORKLOAD
- The user-space thread is affined to a single processor
- The thread opens the file associated with its single processor
- Only one thread can access the file at a time
”h]”jH  )”}”(hhh]”(jM  )”}”(hŒtimerlat tracer is enable”h]”hÌ)”}”(hj^  h]”hŒtimerlat tracer is enable”…””}”(hj`  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÁhj\  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jL  hjY  ubjM  )”}”(hŒ5osnoise workload option is set to NO_OSNOISE_WORKLOAD”h]”hÌ)”}”(hju  h]”hŒ5osnoise workload option is set to NO_OSNOISE_WORKLOAD”…””}”(hjw  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÂhjs  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jL  hjY  ubjM  )”}”(hŒ6The user-space thread is affined to a single processor”h]”hÌ)”}”(hjŒ  h]”hŒ6The user-space thread is affined to a single processor”…””}”(hjŽ  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÃhjŠ  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jL  hjY  ubjM  )”}”(hŒ>The thread opens the file associated with its single processor”h]”hÌ)”}”(hj£  h]”hŒ>The thread opens the file associated with its single processor”…””}”(hj¥  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÄhj¡  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jL  hjY  ubjM  )”}”(hŒ.Only one thread can access the file at a time
”h]”hÌ)”}”(hŒ-Only one thread can access the file at a time”h]”hŒ-Only one thread can access the file at a time”…””}”(hj¼  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÅhj¸  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jL  hjY  ubeh}”(h]”h ]”h"]”h$]”h&]”j  j  uh1jG  h³hÊh´KÁhjU  ubah}”(h]”h ]”h"]”h$]”h&]”uh1jA  h³hÊh´KÁhj(  h²hubhÌ)”}”(hŒˆThe open() syscall will fail if any of these conditions are not met.
After opening the file descriptor, the user space can read from it.”h]”hŒˆThe open() syscall will fail if any of these conditions are not met.
After opening the file descriptor, the user space can read from it.”…””}”(hjÜ  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÇhj(  h²hubhÌ)”}”(hŒˆThe read() system call will run a timerlat code that will arm the
timer in the future and wait for it as the regular kernel thread does.”h]”hŒˆThe read() system call will run a timerlat code that will arm the
timer in the future and wait for it as the regular kernel thread does.”…””}”(hjê  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÊhj(  h²hubhÌ)”}”(hŒÝWhen the timer IRQ fires, the timerlat IRQ will execute, report the
IRQ latency and wake up the thread waiting in the read. The thread will be
scheduled and report the thread latency via tracer - as for the kernel
thread.”h]”hŒÝWhen the timer IRQ fires, the timerlat IRQ will execute, report the
IRQ latency and wake up the thread waiting in the read. The thread will be
scheduled and report the thread latency via tracer - as for the kernel
thread.”…””}”(hjø  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÍhj(  h²hubhÌ)”}”(hŒ­The difference from the in-kernel timerlat is that, instead of re-arming
the timer, timerlat will return to the read() system call. At this point,
the user can run any code.”h]”hŒ­The difference from the in-kernel timerlat is that, instead of re-arming
the timer, timerlat will return to the read() system call. At this point,
the user can run any code.”…””}”(hj  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÒhj(  h²hubhÌ)”}”(hŒðIf the application rereads the file timerlat file descriptor, the tracer
will report the return from user-space latency, which is the total
latency. If this is the end of the work, it can be interpreted as the
response time for the request.”h]”hŒðIf the application rereads the file timerlat file descriptor, the tracer
will report the return from user-space latency, which is the total
latency. If this is the end of the work, it can be interpreted as the
response time for the request.”…””}”(hj  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÖhj(  h²hubhÌ)”}”(hŒ~After reporting the total latency, timerlat will restart the cycle, arm
a timer, and go to sleep for the following activation.”h]”hŒ~After reporting the total latency, timerlat will restart the cycle, arm
a timer, and go to sleep for the following activation.”…””}”(hj"  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÛhj(  h²hubhÌ)”}”(hŒ½If at any time one of the conditions is broken, e.g., the thread migrates
while in user space, or the timerlat tracer is disabled, the SIG_KILL
signal will be sent to the user-space thread.”h]”hŒ½If at any time one of the conditions is broken, e.g., the thread migrates
while in user space, or the timerlat tracer is disabled, the SIG_KILL
signal will be sent to the user-space thread.”…””}”(hj0  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´KÞhj(  h²hubhÌ)”}”(hŒ:Here is an basic example of user-space code for timerlat::”h]”hŒ9Here is an basic example of user-space code for timerlat:”…””}”(hj>  h²hh³Nh´Nubah}”(h]”h ]”h"]”h$]”h&]”uh1hËh³hÊh´Kâhj(  h²hubjA  )”}”(hX#  int main(void)
{
       char buffer[1024];
       int timerlat_fd;
       int retval;
       long cpu = 0;   /* place in CPU 0 */
       cpu_set_t set;

       CPU_ZERO(&set);
       CPU_SET(cpu, &set);

       if (sched_setaffinity(gettid(), sizeof(set), &set) == -1)
               return 1;

       snprintf(buffer, sizeof(buffer),
               "/sys/kernel/tracing/osnoise/per_cpu/cpu%ld/timerlat_fd",
               cpu);

       timerlat_fd = open(buffer, O_RDONLY);
       if (timerlat_fd < 0) {
               printf("error opening %s: %s\n", buffer, strerror(errno));
               exit(1);
       }

       for (;;) {
               retval = read(timerlat_fd, buffer, 1024);
               if (retval < 0)
                       break;
       }

       close(timerlat_fd);
       exit(0);
}”h]”hX#  int main(void)
{
       char buffer[1024];
       int timerlat_fd;
       int retval;
       long cpu = 0;   /* place in CPU 0 */
       cpu_set_t set;

       CPU_ZERO(&set);
       CPU_SET(cpu, &set);

       if (sched_setaffinity(gettid(), sizeof(set), &set) == -1)
               return 1;

       snprintf(buffer, sizeof(buffer),
               "/sys/kernel/tracing/osnoise/per_cpu/cpu%ld/timerlat_fd",
               cpu);

       timerlat_fd = open(buffer, O_RDONLY);
       if (timerlat_fd < 0) {
               printf("error opening %s: %s\n", buffer, strerror(errno));
               exit(1);
       }

       for (;;) {
               retval = read(timerlat_fd, buffer, 1024);
               if (retval < 0)
                       break;
       }

       close(timerlat_fd);
       exit(0);
}”…””}”hjL  sbah}”(h]”h ]”h"]”h$]”h&]”jP  jQ  uh1j@  h³hÊh´Kähj(  h²hubeh}”(h]”Œuser-space-interface”ah ]”h"]”Œuser-space interface”ah$]”h&]”uh1hµhh·h²hh³hÊh´K¹ubeh}”(h]”Œtimerlat-tracer”ah ]”h"]”Œtimerlat tracer”ah$]”h&]”uh1hµhhh²hh³hÊh´Kubeh}”(h]”h ]”h"]”h$]”h&]”Œsource”hÊuh1hŒcurrent_source”NŒcurrent_line”NŒsettings”Œdocutils.frontend”ŒValues”“”)”}”(hºNŒ	generator”NŒ	datestamp”NŒsource_link”NŒ
source_url”NŒtoc_backlinks”Œentry”Œfootnote_backlinks”KŒsectnum_xform”KŒstrip_comments”NŒstrip_elements_with_classes”NŒstrip_classes”NŒreport_level”KŒ
halt_level”KŒexit_status_level”KŒdebug”NŒwarning_stream”NŒ	traceback”ˆŒinput_encoding”Œ	utf-8-sig”Œinput_encoding_error_handler”Œstrict”Œoutput_encoding”Œutf-8”Œoutput_encoding_error_handler”j  Œerror_encoding”Œutf-8”Œerror_encoding_error_handler”Œbackslashreplace”Œlanguage_code”Œen”Œrecord_dependencies”NŒconfig”NŒ	id_prefix”hŒauto_id_prefix”Œid”Œdump_settings”NŒdump_internals”NŒdump_transforms”NŒdump_pseudo_xml”NŒexpose_internals”NŒstrict_visitor”NŒ_disable_config”NŒ_source”hÊŒ_destination”NŒ_config_files”]”Œ7/var/lib/git/docbuild/linux/Documentation/docutils.conf”aŒfile_insertion_enabled”ˆŒraw_enabled”KŒline_length_limit”M'Œpep_references”NŒpep_base_url”Œhttps://peps.python.org/”Œpep_file_url_template”Œpep-%04d”Œrfc_references”NŒrfc_base_url”Œ&https://datatracker.ietf.org/doc/html/”Œ	tab_width”KŒtrim_footnote_reference_space”‰Œsyntax_highlight”Œlong”Œsmart_quotes”ˆŒsmartquotes_locales”]”Œcharacter_level_inline_markup”‰Œdoctitle_xform”‰Œdocinfo_xform”KŒsectsubtitle_xform”‰Œimage_loading”Œlink”Œembed_stylesheet”‰Œcloak_email_addresses”ˆŒsection_self_link”‰Œenv”NubŒreporter”NŒindirect_targets”]”Œsubstitution_defs”}”Œsubstitution_names”}”Œrefnames”}”Œrefids”}”Œnameids”}”(jg  jd  j  j  j  j  jÂ  j¿  j%  j"  j_  j\  uŒ	nametypes”}”(jg  ‰j  ‰j  ‰jÂ  ‰j%  ‰j_  ‰uh}”(jd  h·j  j  j  j"  j¿  j"  j"  jÅ  j\  j(  uŒfootnote_refs”}”Œcitation_refs”}”Œautofootnotes”]”Œautofootnote_refs”]”Œsymbol_footnotes”]”Œsymbol_footnote_refs”]”Œ	footnotes”]”Œ	citations”]”Œautofootnote_start”KŒsymbol_footnote_start”K Œ
id_counter”Œcollections”ŒCounter”“”}”…”R”Œparse_messages”]”Œtransform_messages”]”Œtransformer”NŒinclude_log”]”Œ
decoration”Nh²hub.