diff options
author | Alexei Starovoitov <ast@kernel.org> | 2017-12-20 13:29:19 -0800 |
---|---|---|
committer | Alexei Starovoitov <ast@kernel.org> | 2018-02-12 21:42:46 -0800 |
commit | ce3bcc614aa456b75bbcf11a8dfa6bd75e074ec4 (patch) | |
tree | 7e616a4eec3294593686d7fa00ef4f64dae59271 | |
parent | cf19e5e2054f5172c07a152f9e04eb3bae3d86dd (diff) | |
download | bpf-tp_args_v6.tar.gz |
bpf: raw tracepoint args v6tp_args_v6
in assembler it looks like:
(gdb) disassemble __bpf_trace_xdp_exception
Dump of assembler code for function __bpf_trace_xdp_exception:
0xffffffff81132080 <+0>: mov %ecx,%ecx
0xffffffff81132082 <+2>: jmpq 0xffffffff811231f0 <bpf_trace_run3>
where
TRACE_EVENT(xdp_exception,
TP_PROTO(const struct net_device *dev,
const struct bpf_prog *xdp, u32 act),
The above assembler snippet is casting 32-bit 'act' field into 'u64'
to pass into bpf_trace_run3() and all other registers are passed as-is.
So all of ~500 of __bpf_trace_*() functions are only 5-10 _byte_ long
and in total this approach adds 7k bytes to .text and 8k bytes
to .rodata since I want them to appear in kallsyms, so I don't
have to add another 8-byte fields to 'struct trace_event_class'
and 'struct trace_event_call'
Such approach, I think, is the lowest overhead we can possibly have
while calling trace_xdp_exception() from kernel C code and
transitioning into bpf land.
Since many folks are starting to use tracepoint+bpf at speeds of
1M+ events per second this is very valuable optimization.
Since I'm not touching anything on ftrace or perf side,
I'm thinking to extend BPF_PROG_ATTACH command to specify
which tracepoint to attach to.
The user space will look like:
prog_fd = bpf_prog_load(...);
bpf_prog_attach(prog_fd, "xdp_exception");
On the kernel side I will walk kallsyms to
find __tracepoint_xdp_exception record, check that
tp->name == "xdp_exception",
then find __bpf_trace_xdp_exception() address (also in kallsyms),
and finally use tracepoint_probe_register() to connect the two.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
-rw-r--r-- | arch/x86/xen/mmu_pv.c | 16 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/i915_trace.h | 13 | ||||
-rw-r--r-- | drivers/infiniband/hw/hfi1/file_ops.c | 2 | ||||
-rw-r--r-- | drivers/infiniband/hw/hfi1/trace_ctxts.h | 12 | ||||
-rw-r--r-- | drivers/s390/cio/ioasm.c | 18 | ||||
-rw-r--r-- | drivers/s390/cio/trace.h | 50 | ||||
-rw-r--r-- | fs/dax.c | 2 | ||||
-rw-r--r-- | include/linux/trace_events.h | 47 | ||||
-rw-r--r-- | include/linux/tracepoint-defs.h | 1 | ||||
-rw-r--r-- | include/linux/tracepoint.h | 22 | ||||
-rw-r--r-- | include/trace/bpf_probe.h | 87 | ||||
-rw-r--r-- | include/trace/define_trace.h | 15 | ||||
-rw-r--r-- | include/trace/events/f2fs.h | 2 | ||||
-rw-r--r-- | include/trace/events/fs_dax.h | 6 | ||||
-rw-r--r-- | include/trace/events/rcu.h | 4 | ||||
-rw-r--r-- | include/trace/events/xen.h | 32 | ||||
-rw-r--r-- | include/uapi/linux/bpf.h | 6 | ||||
-rw-r--r-- | kernel/rcu/tree.c | 10 | ||||
-rw-r--r-- | kernel/trace/bpf_trace.c | 184 | ||||
-rw-r--r-- | kernel/tracepoint.c | 27 | ||||
-rw-r--r-- | net/wireless/trace.h | 2 | ||||
-rw-r--r-- | sound/firewire/amdtp-stream-trace.h | 2 |
22 files changed, 451 insertions, 109 deletions
diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index d85076223a696d..9c98e5eeef9dce 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -218,7 +218,7 @@ static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val) static void xen_set_pmd(pmd_t *ptr, pmd_t val) { - trace_xen_mmu_set_pmd(ptr, val); + trace_xen_mmu_set_pmd(ptr, &val); /* If page is not pinned, we can just update the entry directly */ @@ -277,14 +277,14 @@ static inline void __xen_set_pte(pte_t *ptep, pte_t pteval) static void xen_set_pte(pte_t *ptep, pte_t pteval) { - trace_xen_mmu_set_pte(ptep, pteval); + trace_xen_mmu_set_pte(ptep, &pteval); __xen_set_pte(ptep, pteval); } static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval) { - trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval); + trace_xen_mmu_set_pte_at(mm, addr, ptep, &pteval); __xen_set_pte(ptep, pteval); } @@ -292,7 +292,7 @@ pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { /* Just return the pte as-is. We preserve the bits on commit */ - trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep); + trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, ptep); return *ptep; } @@ -301,7 +301,7 @@ void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, { struct mmu_update u; - trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte); + trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, &pte); xen_mc_batch(); u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD; @@ -409,7 +409,7 @@ static void xen_set_pud_hyper(pud_t *ptr, pud_t val) static void xen_set_pud(pud_t *ptr, pud_t val) { - trace_xen_mmu_set_pud(ptr, val); + trace_xen_mmu_set_pud(ptr, &val); /* If page is not pinned, we can just update the entry directly */ @@ -424,7 +424,7 @@ static void xen_set_pud(pud_t *ptr, pud_t val) #ifdef CONFIG_X86_PAE static void xen_set_pte_atomic(pte_t *ptep, pte_t pte) { - trace_xen_mmu_set_pte_atomic(ptep, pte); + trace_xen_mmu_set_pte_atomic(ptep, &pte); set_64bit((u64 *)ptep, native_pte_val(pte)); } @@ -514,7 +514,7 @@ static void xen_set_p4d(p4d_t *ptr, p4d_t val) pgd_t *user_ptr = xen_get_user_pgd((pgd_t *)ptr); pgd_t pgd_val; - trace_xen_mmu_set_p4d(ptr, (p4d_t *)user_ptr, val); + trace_xen_mmu_set_p4d(ptr, (p4d_t *)user_ptr, &val); /* If page is not pinned, we can just update the entry directly */ diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index e1169c02eb2bc7..681da1f5191117 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -849,8 +849,8 @@ TRACE_EVENT(i915_flip_complete, TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj) ); -TRACE_EVENT_CONDITION(i915_reg_rw, - TP_PROTO(bool write, i915_reg_t reg, u64 val, int len, bool trace), +TRACE_EVENT_CONDITION(i915_reg_rw__, + TP_PROTO(bool write, u32 reg, u64 val, int len, bool trace), TP_ARGS(write, reg, val, len, trace), @@ -865,7 +865,7 @@ TRACE_EVENT_CONDITION(i915_reg_rw, TP_fast_assign( __entry->val = (u64)val; - __entry->reg = i915_mmio_reg_offset(reg); + __entry->reg = reg; __entry->write = write; __entry->len = len; ), @@ -876,6 +876,13 @@ TRACE_EVENT_CONDITION(i915_reg_rw, (u32)(__entry->val & 0xffffffff), (u32)(__entry->val >> 32)) ); +#if !defined(CREATE_TRACE_POINTS) && !defined(TRACE_HEADER_MULTI_READ) +static inline void trace_i915_reg_rw(bool write, i915_reg_t reg, u64 val, + int len, bool trace) +{ + trace_i915_reg_rw__(write, i915_mmio_reg_offset(reg), val, len, trace); +} +#endif TRACE_EVENT(intel_gpu_freq_change, TP_PROTO(u32 freq), diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index 41fafebe3b0d97..da4aa1a95b110b 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c @@ -1153,7 +1153,7 @@ static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len) cinfo.sdma_ring_size = fd->cq->nentries; cinfo.rcvegr_size = uctxt->egrbufs.rcvtid_size; - trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, cinfo); + trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, &cinfo); if (copy_to_user((void __user *)arg, &cinfo, len)) return -EFAULT; diff --git a/drivers/infiniband/hw/hfi1/trace_ctxts.h b/drivers/infiniband/hw/hfi1/trace_ctxts.h index 4eb4cc798035ff..e00c8a7d559cd8 100644 --- a/drivers/infiniband/hw/hfi1/trace_ctxts.h +++ b/drivers/infiniband/hw/hfi1/trace_ctxts.h @@ -106,7 +106,7 @@ TRACE_EVENT(hfi1_uctxtdata, TRACE_EVENT(hfi1_ctxt_info, TP_PROTO(struct hfi1_devdata *dd, unsigned int ctxt, unsigned int subctxt, - struct hfi1_ctxt_info cinfo), + struct hfi1_ctxt_info *cinfo), TP_ARGS(dd, ctxt, subctxt, cinfo), TP_STRUCT__entry(DD_DEV_ENTRY(dd) __field(unsigned int, ctxt) @@ -120,11 +120,11 @@ TRACE_EVENT(hfi1_ctxt_info, TP_fast_assign(DD_DEV_ASSIGN(dd); __entry->ctxt = ctxt; __entry->subctxt = subctxt; - __entry->egrtids = cinfo.egrtids; - __entry->rcvhdrq_cnt = cinfo.rcvhdrq_cnt; - __entry->rcvhdrq_size = cinfo.rcvhdrq_entsize; - __entry->sdma_ring_size = cinfo.sdma_ring_size; - __entry->rcvegr_size = cinfo.rcvegr_size; + __entry->egrtids = cinfo->egrtids; + __entry->rcvhdrq_cnt = cinfo->rcvhdrq_cnt; + __entry->rcvhdrq_size = cinfo->rcvhdrq_entsize; + __entry->sdma_ring_size = cinfo->sdma_ring_size; + __entry->rcvegr_size = cinfo->rcvegr_size; ), TP_printk("[%s] ctxt %u:%u " CINFO_FMT, __get_str(dev), diff --git a/drivers/s390/cio/ioasm.c b/drivers/s390/cio/ioasm.c index 4fa9ee1d09fa93..0aecb6314e6f38 100644 --- a/drivers/s390/cio/ioasm.c +++ b/drivers/s390/cio/ioasm.c @@ -35,7 +35,7 @@ int stsch(struct subchannel_id schid, struct schib *addr) int ccode; ccode = __stsch(schid, addr); - trace_s390_cio_stsch(schid, addr, ccode); + trace_s390_cio_stsch(&schid, addr, ccode); return ccode; } @@ -63,7 +63,7 @@ int msch(struct subchannel_id schid, struct schib *addr) int ccode; ccode = __msch(schid, addr); - trace_s390_cio_msch(schid, addr, ccode); + trace_s390_cio_msch(&schid, addr, ccode); return ccode; } @@ -88,7 +88,7 @@ int tsch(struct subchannel_id schid, struct irb *addr) int ccode; ccode = __tsch(schid, addr); - trace_s390_cio_tsch(schid, addr, ccode); + trace_s390_cio_tsch(&schid, addr, ccode); return ccode; } @@ -115,7 +115,7 @@ int ssch(struct subchannel_id schid, union orb *addr) int ccode; ccode = __ssch(schid, addr); - trace_s390_cio_ssch(schid, addr, ccode); + trace_s390_cio_ssch(&schid, addr, ccode); return ccode; } @@ -141,7 +141,7 @@ int csch(struct subchannel_id schid) int ccode; ccode = __csch(schid); - trace_s390_cio_csch(schid, ccode); + trace_s390_cio_csch(&schid, ccode); return ccode; } @@ -202,7 +202,7 @@ int rchp(struct chp_id chpid) int ccode; ccode = __rchp(chpid); - trace_s390_cio_rchp(chpid, ccode); + trace_s390_cio_rchp(&chpid, ccode); return ccode; } @@ -228,7 +228,7 @@ int rsch(struct subchannel_id schid) int ccode; ccode = __rsch(schid); - trace_s390_cio_rsch(schid, ccode); + trace_s390_cio_rsch(&schid, ccode); return ccode; } @@ -253,7 +253,7 @@ int hsch(struct subchannel_id schid) int ccode; ccode = __hsch(schid); - trace_s390_cio_hsch(schid, ccode); + trace_s390_cio_hsch(&schid, ccode); return ccode; } @@ -278,7 +278,7 @@ int xsch(struct subchannel_id schid) int ccode; ccode = __xsch(schid); - trace_s390_cio_xsch(schid, ccode); + trace_s390_cio_xsch(&schid, ccode); return ccode; } diff --git a/drivers/s390/cio/trace.h b/drivers/s390/cio/trace.h index 1f8d1c1e566de7..4aa6d1426106fa 100644 --- a/drivers/s390/cio/trace.h +++ b/drivers/s390/cio/trace.h @@ -22,7 +22,7 @@ #include <linux/tracepoint.h> DECLARE_EVENT_CLASS(s390_class_schib, - TP_PROTO(struct subchannel_id schid, struct schib *schib, int cc), + TP_PROTO(struct subchannel_id *schid, struct schib *schib, int cc), TP_ARGS(schid, schib, cc), TP_STRUCT__entry( __field(u8, cssid) @@ -33,9 +33,9 @@ DECLARE_EVENT_CLASS(s390_class_schib, __field(int, cc) ), TP_fast_assign( - __entry->cssid = schid.cssid; - __entry->ssid = schid.ssid; - __entry->schno = schid.sch_no; + __entry->cssid = schid->cssid; + __entry->ssid = schid->ssid; + __entry->schno = schid->sch_no; __entry->devno = schib->pmcw.dev; __entry->schib = *schib; __entry->cc = cc; @@ -60,7 +60,7 @@ DECLARE_EVENT_CLASS(s390_class_schib, * @cc: Condition code */ DEFINE_EVENT(s390_class_schib, s390_cio_stsch, - TP_PROTO(struct subchannel_id schid, struct schib *schib, int cc), + TP_PROTO(struct subchannel_id *schid, struct schib *schib, int cc), TP_ARGS(schid, schib, cc) ); @@ -71,7 +71,7 @@ DEFINE_EVENT(s390_class_schib, s390_cio_stsch, * @cc: Condition code */ DEFINE_EVENT(s390_class_schib, s390_cio_msch, - TP_PROTO(struct subchannel_id schid, struct schib *schib, int cc), + TP_PROTO(struct subchannel_id *schid, struct schib *schib, int cc), TP_ARGS(schid, schib, cc) ); @@ -82,7 +82,7 @@ DEFINE_EVENT(s390_class_schib, s390_cio_msch, * @cc: Condition code */ TRACE_EVENT(s390_cio_tsch, - TP_PROTO(struct subchannel_id schid, struct irb *irb, int cc), + TP_PROTO(struct subchannel_id *schid, struct irb *irb, int cc), TP_ARGS(schid, irb, cc), TP_STRUCT__entry( __field(u8, cssid) @@ -92,9 +92,9 @@ TRACE_EVENT(s390_cio_tsch, __field(int, cc) ), TP_fast_assign( - __entry->cssid = schid.cssid; - __entry->ssid = schid.ssid; - __entry->schno = schid.sch_no; + __entry->cssid = schid->cssid; + __entry->ssid = schid->ssid; + __entry->schno = schid->sch_no; __entry->irb = *irb; __entry->cc = cc; ), @@ -151,7 +151,7 @@ TRACE_EVENT(s390_cio_tpi, * @cc: Condition code */ TRACE_EVENT(s390_cio_ssch, - TP_PROTO(struct subchannel_id schid, union orb *orb, int cc), + TP_PROTO(struct subchannel_id *schid, union orb *orb, int cc), TP_ARGS(schid, orb, cc), TP_STRUCT__entry( __field(u8, cssid) @@ -161,9 +161,9 @@ TRACE_EVENT(s390_cio_ssch, __field(int, cc) ), TP_fast_assign( - __entry->cssid = schid.cssid; - __entry->ssid = schid.ssid; - __entry->schno = schid.sch_no; + __entry->cssid = schid->cssid; + __entry->ssid = schid->ssid; + __entry->schno = schid->sch_no; __entry->orb = *orb; __entry->cc = cc; ), @@ -173,7 +173,7 @@ TRACE_EVENT(s390_cio_ssch, ); DECLARE_EVENT_CLASS(s390_class_schid, - TP_PROTO(struct subchannel_id schid, int cc), + TP_PROTO(struct subchannel_id *schid, int cc), TP_ARGS(schid, cc), TP_STRUCT__entry( __field(u8, cssid) @@ -182,9 +182,9 @@ DECLARE_EVENT_CLASS(s390_class_schid, __field(int, cc) ), TP_fast_assign( - __entry->cssid = schid.cssid; - __entry->ssid = schid.ssid; - __entry->schno = schid.sch_no; + __entry->cssid = schid->cssid; + __entry->ssid = schid->ssid; + __entry->schno = schid->sch_no; __entry->cc = cc; ), TP_printk("schid=%x.%x.%04x cc=%d", __entry->cssid, __entry->ssid, @@ -198,7 +198,7 @@ DECLARE_EVENT_CLASS(s390_class_schid, * @cc: Condition code */ DEFINE_EVENT(s390_class_schid, s390_cio_csch, - TP_PROTO(struct subchannel_id schid, int cc), + TP_PROTO(struct subchannel_id *schid, int cc), TP_ARGS(schid, cc) ); @@ -208,7 +208,7 @@ DEFINE_EVENT(s390_class_schid, s390_cio_csch, * @cc: Condition code */ DEFINE_EVENT(s390_class_schid, s390_cio_hsch, - TP_PROTO(struct subchannel_id schid, int cc), + TP_PROTO(struct subchannel_id *schid, int cc), TP_ARGS(schid, cc) ); @@ -218,7 +218,7 @@ DEFINE_EVENT(s390_class_schid, s390_cio_hsch, * @cc: Condition code */ DEFINE_EVENT(s390_class_schid, s390_cio_xsch, - TP_PROTO(struct subchannel_id schid, int cc), + TP_PROTO(struct subchannel_id *schid, int cc), TP_ARGS(schid, cc) ); @@ -228,7 +228,7 @@ DEFINE_EVENT(s390_class_schid, s390_cio_xsch, * @cc: Condition code */ DEFINE_EVENT(s390_class_schid, s390_cio_rsch, - TP_PROTO(struct subchannel_id schid, int cc), + TP_PROTO(struct subchannel_id *schid, int cc), TP_ARGS(schid, cc) ); @@ -238,7 +238,7 @@ DEFINE_EVENT(s390_class_schid, s390_cio_rsch, * @cc: Condition code */ TRACE_EVENT(s390_cio_rchp, - TP_PROTO(struct chp_id chpid, int cc), + TP_PROTO(struct chp_id *chpid, int cc), TP_ARGS(chpid, cc), TP_STRUCT__entry( __field(u8, cssid) @@ -246,8 +246,8 @@ TRACE_EVENT(s390_cio_rchp, __field(int, cc) ), TP_fast_assign( - __entry->cssid = chpid.cssid; - __entry->id = chpid.id; + __entry->cssid = chpid->cssid; + __entry->id = chpid->id; __entry->cc = cc; ), TP_printk("chpid=%x.%02x cc=%d", __entry->cssid, __entry->id, diff --git a/fs/dax.c b/fs/dax.c index 0276df90e86c58..6d03ead8e78880 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1429,7 +1429,7 @@ static int dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, goto finish_iomap; } - trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry); + trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, &pfn, entry); result = vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn, write); break; diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 8a1442c4e5131f..6b5ab510fdeabe 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -546,6 +546,53 @@ extern void ftrace_profile_free_filter(struct perf_event *event); void perf_trace_buf_update(void *record, u16 type); void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp); +void bpf_trace_run1(struct bpf_prog *prog, u64 arg1); +void bpf_trace_run2(struct bpf_prog *prog, u64 arg1, u64 arg2); +void bpf_trace_run3(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3); +void bpf_trace_run4(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4); +void bpf_trace_run5(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5); +void bpf_trace_run6(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6); +void bpf_trace_run7(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7); +void bpf_trace_run8(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8); +void bpf_trace_run9(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9); +void bpf_trace_run10(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10); +void bpf_trace_run11(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10, u64 arg11); +void bpf_trace_run12(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12); +void bpf_trace_run13(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12, + u64 arg13); +void bpf_trace_run14(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12, + u64 arg13, u64 arg14); +void bpf_trace_run15(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12, + u64 arg13, u64 arg14, u64 arg15); +void bpf_trace_run16(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12, + u64 arg13, u64 arg14, u64 arg15, u64 arg16); +void bpf_trace_run17(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12, + u64 arg13, u64 arg14, u64 arg15, u64 arg16, u64 arg17); void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx, struct trace_event_call *call, u64 count, struct pt_regs *regs, struct hlist_head *head, diff --git a/include/linux/tracepoint-defs.h b/include/linux/tracepoint-defs.h index 64ed7064f1fa65..39a283c61c51ca 100644 --- a/include/linux/tracepoint-defs.h +++ b/include/linux/tracepoint-defs.h @@ -33,6 +33,7 @@ struct tracepoint { int (*regfunc)(void); void (*unregfunc)(void); struct tracepoint_func __rcu *funcs; + u32 num_args; }; #endif diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index c94f466d57ef1e..2fbcd8e4057523 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -40,9 +40,9 @@ tracepoint_probe_register_prio(struct tracepoint *tp, void *probe, void *data, int prio); extern int tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data); -extern void -for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv), - void *priv); +extern void * +for_each_kernel_tracepoint(void *(*fct)(struct tracepoint *tp, void *priv), + void *priv); #ifdef CONFIG_MODULES struct tp_module { @@ -225,23 +225,27 @@ extern void syscall_unregfunc(void); return static_key_false(&__tracepoint_##name.key); \ } +#define ___FN_COUNT(fn,n0,n1,n2,n3,n4,n5,n6,n7,n8,n9,n10,n11,n12,n13,n14,n15,n16,n,...) fn##n +#define __FN_COUNT(fn,...) ___FN_COUNT(fn,##__VA_ARGS__,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0) +#define __COUNT(...) __FN_COUNT(/**/,##__VA_ARGS__) + /* * We have no guarantee that gcc and the linker won't up-align the tracepoint * structures, so we create an array of pointers that will be used for iteration * on the tracepoints. */ -#define DEFINE_TRACE_FN(name, reg, unreg) \ +#define DEFINE_TRACE_FN(name, reg, unreg, num_args) \ static const char __tpstrtab_##name[] \ __attribute__((section("__tracepoints_strings"))) = #name; \ struct tracepoint __tracepoint_##name \ __attribute__((section("__tracepoints"))) = \ - { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\ + { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL, num_args };\ static struct tracepoint * const __tracepoint_ptr_##name __used \ __attribute__((section("__tracepoints_ptrs"))) = \ &__tracepoint_##name; -#define DEFINE_TRACE(name) \ - DEFINE_TRACE_FN(name, NULL, NULL); +#define DEFINE_TRACE(name, num_args) \ + DEFINE_TRACE_FN(name, NULL, NULL, num_args); #define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \ EXPORT_SYMBOL_GPL(__tracepoint_##name) @@ -275,8 +279,8 @@ extern void syscall_unregfunc(void); return false; \ } -#define DEFINE_TRACE_FN(name, reg, unreg) -#define DEFINE_TRACE(name) +#define DEFINE_TRACE_FN(name, reg, unreg, num_args) +#define DEFINE_TRACE(name, num_args) #define EXPORT_TRACEPOINT_SYMBOL_GPL(name) #define EXPORT_TRACEPOINT_SYMBOL(name) diff --git a/include/trace/bpf_probe.h b/include/trace/bpf_probe.h new file mode 100644 index 00000000000000..cfbdf6082a959e --- /dev/null +++ b/include/trace/bpf_probe.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#undef TRACE_SYSTEM_VAR + +#ifdef CONFIG_BPF_EVENTS + +#undef __entry +#define __entry entry + +#undef __get_dynamic_array +#define __get_dynamic_array(field) \ + ((void *)__entry + (__entry->__data_loc_##field & 0xffff)) + +#undef __get_dynamic_array_len +#define __get_dynamic_array_len(field) \ + ((__entry->__data_loc_##field >> 16) & 0xffff) + +#undef __get_str +#define __get_str(field) ((char *)__get_dynamic_array(field)) + +#undef __get_bitmask +#define __get_bitmask(field) (char *)__get_dynamic_array(field) + +#undef __perf_count +#define __perf_count(c) (c) + +#undef __perf_task +#define __perf_task(t) (t) + +/* + * cast any interger or pointer type to u64 without warnings + * on 32 and 64 bit archs + */ +#define __CAST_TO_U64(expr) \ + (u64) __builtin_choose_expr(sizeof(long) < sizeof(expr), \ + (expr), \ + (long) expr) +#define __CAST1(a,...) __CAST_TO_U64(a) +#define __CAST2(a,...) __CAST_TO_U64(a), __CAST1(__VA_ARGS__) +#define __CAST3(a,...) __CAST_TO_U64(a), __CAST2(__VA_ARGS__) +#define __CAST4(a,...) __CAST_TO_U64(a), __CAST3(__VA_ARGS__) +#define __CAST5(a,...) __CAST_TO_U64(a), __CAST4(__VA_ARGS__) +#define __CAST6(a,...) __CAST_TO_U64(a), __CAST5(__VA_ARGS__) +#define __CAST7(a,...) __CAST_TO_U64(a), __CAST6(__VA_ARGS__) +#define __CAST8(a,...) __CAST_TO_U64(a), __CAST7(__VA_ARGS__) +#define __CAST9(a,...) __CAST_TO_U64(a), __CAST8(__VA_ARGS__) +#define __CAST10(a,...) __CAST_TO_U64(a), __CAST9(__VA_ARGS__) +#define __CAST11(a,...) __CAST_TO_U64(a), __CAST10(__VA_ARGS__) +#define __CAST12(a,...) __CAST_TO_U64(a), __CAST11(__VA_ARGS__) +#define __CAST13(a,...) __CAST_TO_U64(a), __CAST12(__VA_ARGS__) +#define __CAST14(a,...) __CAST_TO_U64(a), __CAST13(__VA_ARGS__) +#define __CAST15(a,...) __CAST_TO_U64(a), __CAST14(__VA_ARGS__) +#define __CAST16(a,...) __CAST_TO_U64(a), __CAST15(__VA_ARGS__) +#define __CAST17(a,...) __CAST_TO_U64(a), __CAST16(__VA_ARGS__) + +#define CAST_TO_U64(...) __FN_COUNT(__CAST,##__VA_ARGS__)(__VA_ARGS__) + +#undef DECLARE_EVENT_CLASS +#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ +/* no 'static' here. The bpf probe functions are global */ \ +notrace void \ +__bpf_trace_##call(void *__data, proto) \ +{ \ + struct bpf_prog *prog = __data; \ + \ + __FN_COUNT(bpf_trace_run, args)(prog, CAST_TO_U64(args)); \ +} + +/* + * This part is compiled out, it is only here as a build time check + * to make sure that if the tracepoint handling changes, the + * bpf probe will fail to compile unless it too is updated. + */ +#undef DEFINE_EVENT +#define DEFINE_EVENT(template, call, proto, args) \ +static inline void bpf_test_probe_##call(void) \ +{ \ + check_trace_callback_type_##call(__bpf_trace_##template); \ +} + + +#undef DEFINE_EVENT_PRINT +#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ + DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) + +#include TRACE_INCLUDE(TRACE_INCLUDE_FILE) +#endif /* CONFIG_BPF_EVENTS */ diff --git a/include/trace/define_trace.h b/include/trace/define_trace.h index d9e3d4aa3f6e3b..3bbd3b88177f1e 100644 --- a/include/trace/define_trace.h +++ b/include/trace/define_trace.h @@ -25,7 +25,7 @@ #undef TRACE_EVENT #define TRACE_EVENT(name, proto, args, tstruct, assign, print) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, __COUNT(args)) #undef TRACE_EVENT_CONDITION #define TRACE_EVENT_CONDITION(name, proto, args, cond, tstruct, assign, print) \ @@ -39,24 +39,24 @@ #undef TRACE_EVENT_FN #define TRACE_EVENT_FN(name, proto, args, tstruct, \ assign, print, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, __COUNT(args)) #undef TRACE_EVENT_FN_COND #define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \ assign, print, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, __COUNT(args)) #undef DEFINE_EVENT #define DEFINE_EVENT(template, name, proto, args) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, __COUNT(args)) #undef DEFINE_EVENT_FN #define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, __COUNT(args)) #undef DEFINE_EVENT_PRINT #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, __COUNT(args)) #undef DEFINE_EVENT_CONDITION #define DEFINE_EVENT_CONDITION(template, name, proto, args, cond) \ @@ -64,7 +64,7 @@ #undef DECLARE_TRACE #define DECLARE_TRACE(name, proto, args) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, __COUNT(args)) #undef TRACE_INCLUDE #undef __TRACE_INCLUDE @@ -95,6 +95,7 @@ #ifdef TRACEPOINTS_ENABLED #include <trace/trace_events.h> #include <trace/perf.h> +#include <trace/bpf_probe.h> #endif #undef TRACE_EVENT diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index 06c87f9f720c2e..795698925d206e 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -491,7 +491,7 @@ DEFINE_EVENT(f2fs__truncate_node, f2fs_truncate_node, TRACE_EVENT(f2fs_truncate_partial_nodes, - TP_PROTO(struct inode *inode, nid_t nid[], int depth, int err), + TP_PROTO(struct inode *inode, nid_t *nid, int depth, int err), TP_ARGS(inode, nid, depth, err), diff --git a/include/trace/events/fs_dax.h b/include/trace/events/fs_dax.h index 97b09fcf7e5287..5a6a8285750f5f 100644 --- a/include/trace/events/fs_dax.h +++ b/include/trace/events/fs_dax.h @@ -104,7 +104,7 @@ DEFINE_PMD_LOAD_HOLE_EVENT(dax_pmd_load_hole_fallback); DECLARE_EVENT_CLASS(dax_pmd_insert_mapping_class, TP_PROTO(struct inode *inode, struct vm_fault *vmf, - long length, pfn_t pfn, void *radix_entry), + long length, pfn_t *pfn, void *radix_entry), TP_ARGS(inode, vmf, length, pfn, radix_entry), TP_STRUCT__entry( __field(unsigned long, ino) @@ -123,7 +123,7 @@ DECLARE_EVENT_CLASS(dax_pmd_insert_mapping_class, __entry->address = vmf->address; __entry->write = vmf->flags & FAULT_FLAG_WRITE; __entry->length = length; - __entry->pfn_val = pfn.val; + __entry->pfn_val = pfn->val; __entry->radix_entry = radix_entry; ), TP_printk("dev %d:%d ino %#lx %s %s address %#lx length %#lx " @@ -145,7 +145,7 @@ DECLARE_EVENT_CLASS(dax_pmd_insert_mapping_class, #define DEFINE_PMD_INSERT_MAPPING_EVENT(name) \ DEFINE_EVENT(dax_pmd_insert_mapping_class, name, \ TP_PROTO(struct inode *inode, struct vm_fault *vmf, \ - long length, pfn_t pfn, void *radix_entry), \ + long length, pfn_t *pfn, void *radix_entry), \ TP_ARGS(inode, vmf, length, pfn, radix_entry)) DEFINE_PMD_INSERT_MAPPING_EVENT(dax_pmd_insert_mapping); diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index 0b50fda80db0e0..4b463294306f95 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h @@ -436,7 +436,7 @@ TRACE_EVENT(rcu_fqs, */ TRACE_EVENT(rcu_dyntick, - TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t dynticks), + TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t *dynticks), TP_ARGS(polarity, oldnesting, newnesting, dynticks), @@ -451,7 +451,7 @@ TRACE_EVENT(rcu_dyntick, __entry->polarity = polarity; __entry->oldnesting = oldnesting; __entry->newnesting = newnesting; - __entry->dynticks = atomic_read(&dynticks); + __entry->dynticks = atomic_read(dynticks); ), TP_printk("%s %lx %lx %#3x", __entry->polarity, diff --git a/include/trace/events/xen.h b/include/trace/events/xen.h index b8adf05c534e72..d1965815d9cfdd 100644 --- a/include/trace/events/xen.h +++ b/include/trace/events/xen.h @@ -128,14 +128,14 @@ TRACE_EVENT(xen_mc_extend_args, TRACE_DEFINE_SIZEOF(pteval_t); /* mmu */ DECLARE_EVENT_CLASS(xen_mmu__set_pte, - TP_PROTO(pte_t *ptep, pte_t pteval), + TP_PROTO(pte_t *ptep, pte_t *pteval), TP_ARGS(ptep, pteval), TP_STRUCT__entry( __field(pte_t *, ptep) __field(pteval_t, pteval) ), TP_fast_assign(__entry->ptep = ptep; - __entry->pteval = pteval.pte), + __entry->pteval = pteval->pte), TP_printk("ptep %p pteval %0*llx (raw %0*llx)", __entry->ptep, (int)sizeof(pteval_t) * 2, (unsigned long long)pte_val(native_make_pte(__entry->pteval)), @@ -144,14 +144,14 @@ DECLARE_EVENT_CLASS(xen_mmu__set_pte, #define DEFINE_XEN_MMU_SET_PTE(name) \ DEFINE_EVENT(xen_mmu__set_pte, name, \ - TP_PROTO(pte_t *ptep, pte_t pteval), \ + TP_PROTO(pte_t *ptep, pte_t *pteval), \ TP_ARGS(ptep, pteval)) DEFINE_XEN_MMU_SET_PTE(xen_mmu_set_pte); TRACE_EVENT(xen_mmu_set_pte_at, TP_PROTO(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pteval), + pte_t *ptep, pte_t *pteval), TP_ARGS(mm, addr, ptep, pteval), TP_STRUCT__entry( __field(struct mm_struct *, mm) @@ -162,7 +162,7 @@ TRACE_EVENT(xen_mmu_set_pte_at, TP_fast_assign(__entry->mm = mm; __entry->addr = addr; __entry->ptep = ptep; - __entry->pteval = pteval.pte), + __entry->pteval = pteval->pte), TP_printk("mm %p addr %lx ptep %p pteval %0*llx (raw %0*llx)", __entry->mm, __entry->addr, __entry->ptep, (int)sizeof(pteval_t) * 2, (unsigned long long)pte_val(native_make_pte(__entry->pteval)), @@ -172,14 +172,14 @@ TRACE_EVENT(xen_mmu_set_pte_at, TRACE_DEFINE_SIZEOF(pmdval_t); TRACE_EVENT(xen_mmu_set_pmd, - TP_PROTO(pmd_t *pmdp, pmd_t pmdval), + TP_PROTO(pmd_t *pmdp, pmd_t *pmdval), TP_ARGS(pmdp, pmdval), TP_STRUCT__entry( __field(pmd_t *, pmdp) __field(pmdval_t, pmdval) ), TP_fast_assign(__entry->pmdp = pmdp; - __entry->pmdval = pmdval.pmd), + __entry->pmdval = pmdval->pmd), TP_printk("pmdp %p pmdval %0*llx (raw %0*llx)", __entry->pmdp, (int)sizeof(pmdval_t) * 2, (unsigned long long)pmd_val(native_make_pmd(__entry->pmdval)), @@ -220,14 +220,14 @@ TRACE_EVENT(xen_mmu_pmd_clear, TRACE_DEFINE_SIZEOF(pudval_t); TRACE_EVENT(xen_mmu_set_pud, - TP_PROTO(pud_t *pudp, pud_t pudval), + TP_PROTO(pud_t *pudp, pud_t *pudval), TP_ARGS(pudp, pudval), TP_STRUCT__entry( __field(pud_t *, pudp) __field(pudval_t, pudval) ), TP_fast_assign(__entry->pudp = pudp; - __entry->pudval = native_pud_val(pudval)), + __entry->pudval = native_pud_val(*pudval)), TP_printk("pudp %p pudval %0*llx (raw %0*llx)", __entry->pudp, (int)sizeof(pudval_t) * 2, (unsigned long long)pud_val(native_make_pud(__entry->pudval)), @@ -237,7 +237,7 @@ TRACE_EVENT(xen_mmu_set_pud, TRACE_DEFINE_SIZEOF(p4dval_t); TRACE_EVENT(xen_mmu_set_p4d, - TP_PROTO(p4d_t *p4dp, p4d_t *user_p4dp, p4d_t p4dval), + TP_PROTO(p4d_t *p4dp, p4d_t *user_p4dp, p4d_t *p4dval), TP_ARGS(p4dp, user_p4dp, p4dval), TP_STRUCT__entry( __field(p4d_t *, p4dp) @@ -246,7 +246,7 @@ TRACE_EVENT(xen_mmu_set_p4d, ), TP_fast_assign(__entry->p4dp = p4dp; __entry->user_p4dp = user_p4dp; - __entry->p4dval = p4d_val(p4dval)), + __entry->p4dval = p4d_val(*p4dval)), TP_printk("p4dp %p user_p4dp %p p4dval %0*llx (raw %0*llx)", __entry->p4dp, __entry->user_p4dp, (int)sizeof(p4dval_t) * 2, (unsigned long long)pgd_val(native_make_pgd(__entry->p4dval)), @@ -255,14 +255,14 @@ TRACE_EVENT(xen_mmu_set_p4d, #else TRACE_EVENT(xen_mmu_set_pud, - TP_PROTO(pud_t *pudp, pud_t pudval), + TP_PROTO(pud_t *pudp, pud_t *pudval), TP_ARGS(pudp, pudval), TP_STRUCT__entry( __field(pud_t *, pudp) __field(pudval_t, pudval) ), TP_fast_assign(__entry->pudp = pudp; - __entry->pudval = native_pud_val(pudval)), + __entry->pudval = native_pud_val(*pudval)), TP_printk("pudp %p pudval %0*llx (raw %0*llx)", __entry->pudp, (int)sizeof(pudval_t) * 2, (unsigned long long)pgd_val(native_make_pgd(__entry->pudval)), @@ -273,7 +273,7 @@ TRACE_EVENT(xen_mmu_set_pud, DECLARE_EVENT_CLASS(xen_mmu_ptep_modify_prot, TP_PROTO(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pteval), + pte_t *ptep, pte_t *pteval), TP_ARGS(mm, addr, ptep, pteval), TP_STRUCT__entry( __field(struct mm_struct *, mm) @@ -284,7 +284,7 @@ DECLARE_EVENT_CLASS(xen_mmu_ptep_modify_prot, TP_fast_assign(__entry->mm = mm; __entry->addr = addr; __entry->ptep = ptep; - __entry->pteval = pteval.pte), + __entry->pteval = pteval->pte), TP_printk("mm %p addr %lx ptep %p pteval %0*llx (raw %0*llx)", __entry->mm, __entry->addr, __entry->ptep, (int)sizeof(pteval_t) * 2, (unsigned long long)pte_val(native_make_pte(__entry->pteval)), @@ -293,7 +293,7 @@ DECLARE_EVENT_CLASS(xen_mmu_ptep_modify_prot, #define DEFINE_XEN_MMU_PTEP_MODIFY_PROT(name) \ DEFINE_EVENT(xen_mmu_ptep_modify_prot, name, \ TP_PROTO(struct mm_struct *mm, unsigned long addr, \ - pte_t *ptep, pte_t pteval), \ + pte_t *ptep, pte_t *pteval), \ TP_ARGS(mm, addr, ptep, pteval)) DEFINE_XEN_MMU_PTEP_MODIFY_PROT(xen_mmu_ptep_modify_prot_start); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index db6bdc37512683..11ac17f78d9442 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -133,6 +133,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_SOCK_OPS, BPF_PROG_TYPE_SK_SKB, BPF_PROG_TYPE_CGROUP_DEVICE, + BPF_PROG_TYPE_RAW_TRACEPOINT, }; enum bpf_attach_type { @@ -143,6 +144,7 @@ enum bpf_attach_type { BPF_SK_SKB_STREAM_PARSER, BPF_SK_SKB_STREAM_VERDICT, BPF_CGROUP_DEVICE, + BPF_RAW_TRACEPOINT, __MAX_BPF_ATTACH_TYPE }; @@ -1106,4 +1108,8 @@ struct bpf_cgroup_dev_ctx { __u32 minor; }; +struct bpf_raw_tp_ctx { + __u64 args[16]; +}; + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 491bdf39f276cd..43c0f899f78c8a 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -772,7 +772,7 @@ static void rcu_eqs_enter(bool user) } lockdep_assert_irqs_disabled(); - trace_rcu_dyntick(TPS("Start"), rdtp->dynticks_nesting, 0, rdtp->dynticks); + trace_rcu_dyntick(TPS("Start"), rdtp->dynticks_nesting, 0, &rdtp->dynticks); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); for_each_rcu_flavor(rsp) { rdp = this_cpu_ptr(rsp->rda); @@ -848,14 +848,14 @@ void rcu_nmi_exit(void) * leave it in non-RCU-idle state. */ if (rdtp->dynticks_nmi_nesting != 1) { - trace_rcu_dyntick(TPS("--="), rdtp->dynticks_nmi_nesting, rdtp->dynticks_nmi_nesting - 2, rdtp->dynticks); + trace_rcu_dyntick(TPS("--="), rdtp->dynticks_nmi_nesting, rdtp->dynticks_nmi_nesting - 2, &rdtp->dynticks); WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */ rdtp->dynticks_nmi_nesting - 2); return; } /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ - trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks); + trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, &rdtp->dynticks); WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ rcu_dynticks_eqs_enter(); } @@ -930,7 +930,7 @@ static void rcu_eqs_exit(bool user) rcu_dynticks_task_exit(); rcu_dynticks_eqs_exit(); rcu_cleanup_after_idle(); - trace_rcu_dyntick(TPS("End"), rdtp->dynticks_nesting, 1, rdtp->dynticks); + trace_rcu_dyntick(TPS("End"), rdtp->dynticks_nesting, 1, &rdtp->dynticks); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); WRITE_ONCE(rdtp->dynticks_nesting, 1); WRITE_ONCE(rdtp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE); @@ -1004,7 +1004,7 @@ void rcu_nmi_enter(void) } trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), rdtp->dynticks_nmi_nesting, - rdtp->dynticks_nmi_nesting + incby, rdtp->dynticks); + rdtp->dynticks_nmi_nesting + incby, &rdtp->dynticks); WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */ rdtp->dynticks_nmi_nesting + incby); barrier(); diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index fc2838ac8b7877..41d79d8504f1c1 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -882,3 +882,187 @@ int perf_event_query_prog_array(struct perf_event *event, void __user *info) return ret; } + +static __always_inline +void __bpf_trace_run(struct bpf_prog *prog, u64 *args) +{ +} + +#define EVAL1(FN, X) FN(X) +#define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y) +#define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y) +#define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y) +#define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y) +#define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y) + +#define COPY(X) args[X - 1] = arg##X; + +void bpf_trace_run1(struct bpf_prog *prog, u64 arg1) +{ + u64 args[1]; + + EVAL1(COPY, 1); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run1); +void bpf_trace_run2(struct bpf_prog *prog, u64 arg1, u64 arg2) +{ + u64 args[2]; + + EVAL2(COPY, 1, 2); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run2); +void bpf_trace_run3(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3) +{ + u64 args[3]; + + EVAL3(COPY, 1, 2, 3); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run3); +void bpf_trace_run4(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4) +{ + u64 args[4]; + + EVAL4(COPY, 1, 2, 3, 4); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run4); +void bpf_trace_run5(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5) +{ + u64 args[5]; + + EVAL5(COPY, 1, 2, 3, 4, 5); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run5); +void bpf_trace_run6(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6) +{ + u64 args[6]; + + EVAL6(COPY, 1, 2, 3, 4, 5, 6); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run6); +void bpf_trace_run7(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7) +{ + u64 args[7]; + + EVAL6(COPY, 1, 2, 3, 4, 5, 6); + EVAL1(COPY, 7); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run7); +void bpf_trace_run8(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8) +{ + u64 args[8]; + + EVAL6(COPY, 1, 2, 3, 4, 5, 6); + EVAL2(COPY, 7, 8); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run8); +void bpf_trace_run9(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9) +{ + u64 args[9]; + + EVAL6(COPY, 1, 2, 3, 4, 5, 6); + EVAL3(COPY, 7, 8, 9); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run9); +void bpf_trace_run10(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10) +{ + u64 args[10]; + + EVAL6(COPY, 1, 2, 3, 4, 5, 6); + EVAL4(COPY, 7, 8, 9, 10); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run10); +void bpf_trace_run11(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10, u64 arg11) +{ + u64 args[11]; + + EVAL6(COPY, 1, 2, 3, 4, 5, 6); + EVAL5(COPY, 7, 8, 9, 10, 11); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run11); +void bpf_trace_run12(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12) +{ + u64 args[12]; + + EVAL6(COPY, 1, 2, 3, 4, 5, 6); + EVAL6(COPY, 7, 8, 9, 10, 11, 12); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run12); +void bpf_trace_run17(struct bpf_prog *prog, u64 arg1, u64 arg2, + u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, + u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12, + u64 arg13, u64 arg14, u64 arg15, u64 arg16, u64 arg17) +{ + u64 args[17]; + + EVAL6(COPY, 1, 2, 3, 4, 5, 6); + EVAL6(COPY, 7, 8, 9, 10, 11, 12); + EVAL5(COPY, 13, 14, 15, 16, 17); + __bpf_trace_run(prog, args); +} +EXPORT_SYMBOL_GPL(bpf_trace_run17); + +static void *__find_tp(struct tracepoint *tp, void *priv) +{ + char *name = priv; + + if (!strcmp(tp->name, name)) + return tp; + return NULL; +} + +int bpf_probe_register(char *name, struct bpf_prog *prog) +{ + struct tracepoint *tp; + unsigned long addr; + char buf[128]; + int err; + + tp = for_each_kernel_tracepoint(__find_tp, name); + if (!tp) + return -ENOENT; + + /* + * check that program doesn't access arguments beyond what's + * available in this tracepoint + */ + if (prog->aux->max_ctx_offset > tp->num_args * sizeof(u64)) + return -EINVAL; + + snprintf(buf, sizeof(buf), "__bpf_trace_%s", name); + addr = kallsyms_lookup_name(buf); + if (!addr) + return -ENOENT; + + err = tracepoint_probe_register(tp, (void *)addr, prog); + if (err) + return err; + return 0; +} + diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c index 671b134573876d..3f2dc5738c2bd1 100644 --- a/kernel/tracepoint.c +++ b/kernel/tracepoint.c @@ -502,17 +502,22 @@ static __init int init_tracepoints(void) __initcall(init_tracepoints); #endif /* CONFIG_MODULES */ -static void for_each_tracepoint_range(struct tracepoint * const *begin, - struct tracepoint * const *end, - void (*fct)(struct tracepoint *tp, void *priv), - void *priv) +static void *for_each_tracepoint_range(struct tracepoint * const *begin, + struct tracepoint * const *end, + void *(*fct)(struct tracepoint *tp, void *priv), + void *priv) { struct tracepoint * const *iter; + void *ret; if (!begin) - return; - for (iter = begin; iter < end; iter++) - fct(*iter, priv); + return NULL; + for (iter = begin; iter < end; iter++) { + ret = fct(*iter, priv); + if (ret) + return ret; + } + return NULL; } /** @@ -520,11 +525,11 @@ static void for_each_tracepoint_range(struct tracepoint * const *begin, * @fct: callback * @priv: private data */ -void for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv), - void *priv) +void *for_each_kernel_tracepoint(void *(*fct)(struct tracepoint *tp, void *priv), + void *priv) { - for_each_tracepoint_range(__start___tracepoints_ptrs, - __stop___tracepoints_ptrs, fct, priv); + return for_each_tracepoint_range(__start___tracepoints_ptrs, + __stop___tracepoints_ptrs, fct, priv); } EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint); diff --git a/net/wireless/trace.h b/net/wireless/trace.h index bcfedd39e7a365..392189bb0a0db3 100644 --- a/net/wireless/trace.h +++ b/net/wireless/trace.h @@ -3114,7 +3114,7 @@ TRACE_EVENT(rdev_start_radar_detection, TRACE_EVENT(rdev_set_mcast_rate, TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, - int mcast_rate[NUM_NL80211_BANDS]), + int *mcast_rate), TP_ARGS(wiphy, netdev, mcast_rate), TP_STRUCT__entry( WIPHY_ENTRY diff --git a/sound/firewire/amdtp-stream-trace.h b/sound/firewire/amdtp-stream-trace.h index ea0d486652c838..54cdd4ffa9ceb1 100644 --- a/sound/firewire/amdtp-stream-trace.h +++ b/sound/firewire/amdtp-stream-trace.h @@ -14,7 +14,7 @@ #include <linux/tracepoint.h> TRACE_EVENT(in_packet, - TP_PROTO(const struct amdtp_stream *s, u32 cycles, u32 cip_header[2], unsigned int payload_length, unsigned int index), + TP_PROTO(const struct amdtp_stream *s, u32 cycles, u32 *cip_header, unsigned int payload_length, unsigned int index), TP_ARGS(s, cycles, cip_header, payload_length, index), TP_STRUCT__entry( __field(unsigned int, second) |