aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexei Starovoitov <ast@kernel.org>2017-12-20 13:29:19 -0800
committerAlexei Starovoitov <ast@kernel.org>2018-02-12 21:42:46 -0800
commitce3bcc614aa456b75bbcf11a8dfa6bd75e074ec4 (patch)
tree7e616a4eec3294593686d7fa00ef4f64dae59271
parentcf19e5e2054f5172c07a152f9e04eb3bae3d86dd (diff)
downloadbpf-tp_args_v6.tar.gz
bpf: raw tracepoint args v6tp_args_v6
in assembler it looks like: (gdb) disassemble __bpf_trace_xdp_exception Dump of assembler code for function __bpf_trace_xdp_exception: 0xffffffff81132080 <+0>: mov %ecx,%ecx 0xffffffff81132082 <+2>: jmpq 0xffffffff811231f0 <bpf_trace_run3> where TRACE_EVENT(xdp_exception, TP_PROTO(const struct net_device *dev, const struct bpf_prog *xdp, u32 act), The above assembler snippet is casting 32-bit 'act' field into 'u64' to pass into bpf_trace_run3() and all other registers are passed as-is. So all of ~500 of __bpf_trace_*() functions are only 5-10 _byte_ long and in total this approach adds 7k bytes to .text and 8k bytes to .rodata since I want them to appear in kallsyms, so I don't have to add another 8-byte fields to 'struct trace_event_class' and 'struct trace_event_call' Such approach, I think, is the lowest overhead we can possibly have while calling trace_xdp_exception() from kernel C code and transitioning into bpf land. Since many folks are starting to use tracepoint+bpf at speeds of 1M+ events per second this is very valuable optimization. Since I'm not touching anything on ftrace or perf side, I'm thinking to extend BPF_PROG_ATTACH command to specify which tracepoint to attach to. The user space will look like: prog_fd = bpf_prog_load(...); bpf_prog_attach(prog_fd, "xdp_exception"); On the kernel side I will walk kallsyms to find __tracepoint_xdp_exception record, check that tp->name == "xdp_exception", then find __bpf_trace_xdp_exception() address (also in kallsyms), and finally use tracepoint_probe_register() to connect the two. Signed-off-by: Alexei Starovoitov <ast@kernel.org>
-rw-r--r--arch/x86/xen/mmu_pv.c16
-rw-r--r--drivers/gpu/drm/i915/i915_trace.h13
-rw-r--r--drivers/infiniband/hw/hfi1/file_ops.c2
-rw-r--r--drivers/infiniband/hw/hfi1/trace_ctxts.h12
-rw-r--r--drivers/s390/cio/ioasm.c18
-rw-r--r--drivers/s390/cio/trace.h50
-rw-r--r--fs/dax.c2
-rw-r--r--include/linux/trace_events.h47
-rw-r--r--include/linux/tracepoint-defs.h1
-rw-r--r--include/linux/tracepoint.h22
-rw-r--r--include/trace/bpf_probe.h87
-rw-r--r--include/trace/define_trace.h15
-rw-r--r--include/trace/events/f2fs.h2
-rw-r--r--include/trace/events/fs_dax.h6
-rw-r--r--include/trace/events/rcu.h4
-rw-r--r--include/trace/events/xen.h32
-rw-r--r--include/uapi/linux/bpf.h6
-rw-r--r--kernel/rcu/tree.c10
-rw-r--r--kernel/trace/bpf_trace.c184
-rw-r--r--kernel/tracepoint.c27
-rw-r--r--net/wireless/trace.h2
-rw-r--r--sound/firewire/amdtp-stream-trace.h2
22 files changed, 451 insertions, 109 deletions
diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index d85076223a696d..9c98e5eeef9dce 100644
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -218,7 +218,7 @@ static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
static void xen_set_pmd(pmd_t *ptr, pmd_t val)
{
- trace_xen_mmu_set_pmd(ptr, val);
+ trace_xen_mmu_set_pmd(ptr, &val);
/* If page is not pinned, we can just update the entry
directly */
@@ -277,14 +277,14 @@ static inline void __xen_set_pte(pte_t *ptep, pte_t pteval)
static void xen_set_pte(pte_t *ptep, pte_t pteval)
{
- trace_xen_mmu_set_pte(ptep, pteval);
+ trace_xen_mmu_set_pte(ptep, &pteval);
__xen_set_pte(ptep, pteval);
}
static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pteval)
{
- trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval);
+ trace_xen_mmu_set_pte_at(mm, addr, ptep, &pteval);
__xen_set_pte(ptep, pteval);
}
@@ -292,7 +292,7 @@ pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
/* Just return the pte as-is. We preserve the bits on commit */
- trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep);
+ trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, ptep);
return *ptep;
}
@@ -301,7 +301,7 @@ void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
{
struct mmu_update u;
- trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte);
+ trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, &pte);
xen_mc_batch();
u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
@@ -409,7 +409,7 @@ static void xen_set_pud_hyper(pud_t *ptr, pud_t val)
static void xen_set_pud(pud_t *ptr, pud_t val)
{
- trace_xen_mmu_set_pud(ptr, val);
+ trace_xen_mmu_set_pud(ptr, &val);
/* If page is not pinned, we can just update the entry
directly */
@@ -424,7 +424,7 @@ static void xen_set_pud(pud_t *ptr, pud_t val)
#ifdef CONFIG_X86_PAE
static void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
{
- trace_xen_mmu_set_pte_atomic(ptep, pte);
+ trace_xen_mmu_set_pte_atomic(ptep, &pte);
set_64bit((u64 *)ptep, native_pte_val(pte));
}
@@ -514,7 +514,7 @@ static void xen_set_p4d(p4d_t *ptr, p4d_t val)
pgd_t *user_ptr = xen_get_user_pgd((pgd_t *)ptr);
pgd_t pgd_val;
- trace_xen_mmu_set_p4d(ptr, (p4d_t *)user_ptr, val);
+ trace_xen_mmu_set_p4d(ptr, (p4d_t *)user_ptr, &val);
/* If page is not pinned, we can just update the entry
directly */
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index e1169c02eb2bc7..681da1f5191117 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -849,8 +849,8 @@ TRACE_EVENT(i915_flip_complete,
TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj)
);
-TRACE_EVENT_CONDITION(i915_reg_rw,
- TP_PROTO(bool write, i915_reg_t reg, u64 val, int len, bool trace),
+TRACE_EVENT_CONDITION(i915_reg_rw__,
+ TP_PROTO(bool write, u32 reg, u64 val, int len, bool trace),
TP_ARGS(write, reg, val, len, trace),
@@ -865,7 +865,7 @@ TRACE_EVENT_CONDITION(i915_reg_rw,
TP_fast_assign(
__entry->val = (u64)val;
- __entry->reg = i915_mmio_reg_offset(reg);
+ __entry->reg = reg;
__entry->write = write;
__entry->len = len;
),
@@ -876,6 +876,13 @@ TRACE_EVENT_CONDITION(i915_reg_rw,
(u32)(__entry->val & 0xffffffff),
(u32)(__entry->val >> 32))
);
+#if !defined(CREATE_TRACE_POINTS) && !defined(TRACE_HEADER_MULTI_READ)
+static inline void trace_i915_reg_rw(bool write, i915_reg_t reg, u64 val,
+ int len, bool trace)
+{
+ trace_i915_reg_rw__(write, i915_mmio_reg_offset(reg), val, len, trace);
+}
+#endif
TRACE_EVENT(intel_gpu_freq_change,
TP_PROTO(u32 freq),
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index 41fafebe3b0d97..da4aa1a95b110b 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -1153,7 +1153,7 @@ static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len)
cinfo.sdma_ring_size = fd->cq->nentries;
cinfo.rcvegr_size = uctxt->egrbufs.rcvtid_size;
- trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, cinfo);
+ trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, &cinfo);
if (copy_to_user((void __user *)arg, &cinfo, len))
return -EFAULT;
diff --git a/drivers/infiniband/hw/hfi1/trace_ctxts.h b/drivers/infiniband/hw/hfi1/trace_ctxts.h
index 4eb4cc798035ff..e00c8a7d559cd8 100644
--- a/drivers/infiniband/hw/hfi1/trace_ctxts.h
+++ b/drivers/infiniband/hw/hfi1/trace_ctxts.h
@@ -106,7 +106,7 @@ TRACE_EVENT(hfi1_uctxtdata,
TRACE_EVENT(hfi1_ctxt_info,
TP_PROTO(struct hfi1_devdata *dd, unsigned int ctxt,
unsigned int subctxt,
- struct hfi1_ctxt_info cinfo),
+ struct hfi1_ctxt_info *cinfo),
TP_ARGS(dd, ctxt, subctxt, cinfo),
TP_STRUCT__entry(DD_DEV_ENTRY(dd)
__field(unsigned int, ctxt)
@@ -120,11 +120,11 @@ TRACE_EVENT(hfi1_ctxt_info,
TP_fast_assign(DD_DEV_ASSIGN(dd);
__entry->ctxt = ctxt;
__entry->subctxt = subctxt;
- __entry->egrtids = cinfo.egrtids;
- __entry->rcvhdrq_cnt = cinfo.rcvhdrq_cnt;
- __entry->rcvhdrq_size = cinfo.rcvhdrq_entsize;
- __entry->sdma_ring_size = cinfo.sdma_ring_size;
- __entry->rcvegr_size = cinfo.rcvegr_size;
+ __entry->egrtids = cinfo->egrtids;
+ __entry->rcvhdrq_cnt = cinfo->rcvhdrq_cnt;
+ __entry->rcvhdrq_size = cinfo->rcvhdrq_entsize;
+ __entry->sdma_ring_size = cinfo->sdma_ring_size;
+ __entry->rcvegr_size = cinfo->rcvegr_size;
),
TP_printk("[%s] ctxt %u:%u " CINFO_FMT,
__get_str(dev),
diff --git a/drivers/s390/cio/ioasm.c b/drivers/s390/cio/ioasm.c
index 4fa9ee1d09fa93..0aecb6314e6f38 100644
--- a/drivers/s390/cio/ioasm.c
+++ b/drivers/s390/cio/ioasm.c
@@ -35,7 +35,7 @@ int stsch(struct subchannel_id schid, struct schib *addr)
int ccode;
ccode = __stsch(schid, addr);
- trace_s390_cio_stsch(schid, addr, ccode);
+ trace_s390_cio_stsch(&schid, addr, ccode);
return ccode;
}
@@ -63,7 +63,7 @@ int msch(struct subchannel_id schid, struct schib *addr)
int ccode;
ccode = __msch(schid, addr);
- trace_s390_cio_msch(schid, addr, ccode);
+ trace_s390_cio_msch(&schid, addr, ccode);
return ccode;
}
@@ -88,7 +88,7 @@ int tsch(struct subchannel_id schid, struct irb *addr)
int ccode;
ccode = __tsch(schid, addr);
- trace_s390_cio_tsch(schid, addr, ccode);
+ trace_s390_cio_tsch(&schid, addr, ccode);
return ccode;
}
@@ -115,7 +115,7 @@ int ssch(struct subchannel_id schid, union orb *addr)
int ccode;
ccode = __ssch(schid, addr);
- trace_s390_cio_ssch(schid, addr, ccode);
+ trace_s390_cio_ssch(&schid, addr, ccode);
return ccode;
}
@@ -141,7 +141,7 @@ int csch(struct subchannel_id schid)
int ccode;
ccode = __csch(schid);
- trace_s390_cio_csch(schid, ccode);
+ trace_s390_cio_csch(&schid, ccode);
return ccode;
}
@@ -202,7 +202,7 @@ int rchp(struct chp_id chpid)
int ccode;
ccode = __rchp(chpid);
- trace_s390_cio_rchp(chpid, ccode);
+ trace_s390_cio_rchp(&chpid, ccode);
return ccode;
}
@@ -228,7 +228,7 @@ int rsch(struct subchannel_id schid)
int ccode;
ccode = __rsch(schid);
- trace_s390_cio_rsch(schid, ccode);
+ trace_s390_cio_rsch(&schid, ccode);
return ccode;
}
@@ -253,7 +253,7 @@ int hsch(struct subchannel_id schid)
int ccode;
ccode = __hsch(schid);
- trace_s390_cio_hsch(schid, ccode);
+ trace_s390_cio_hsch(&schid, ccode);
return ccode;
}
@@ -278,7 +278,7 @@ int xsch(struct subchannel_id schid)
int ccode;
ccode = __xsch(schid);
- trace_s390_cio_xsch(schid, ccode);
+ trace_s390_cio_xsch(&schid, ccode);
return ccode;
}
diff --git a/drivers/s390/cio/trace.h b/drivers/s390/cio/trace.h
index 1f8d1c1e566de7..4aa6d1426106fa 100644
--- a/drivers/s390/cio/trace.h
+++ b/drivers/s390/cio/trace.h
@@ -22,7 +22,7 @@
#include <linux/tracepoint.h>
DECLARE_EVENT_CLASS(s390_class_schib,
- TP_PROTO(struct subchannel_id schid, struct schib *schib, int cc),
+ TP_PROTO(struct subchannel_id *schid, struct schib *schib, int cc),
TP_ARGS(schid, schib, cc),
TP_STRUCT__entry(
__field(u8, cssid)
@@ -33,9 +33,9 @@ DECLARE_EVENT_CLASS(s390_class_schib,
__field(int, cc)
),
TP_fast_assign(
- __entry->cssid = schid.cssid;
- __entry->ssid = schid.ssid;
- __entry->schno = schid.sch_no;
+ __entry->cssid = schid->cssid;
+ __entry->ssid = schid->ssid;
+ __entry->schno = schid->sch_no;
__entry->devno = schib->pmcw.dev;
__entry->schib = *schib;
__entry->cc = cc;
@@ -60,7 +60,7 @@ DECLARE_EVENT_CLASS(s390_class_schib,
* @cc: Condition code
*/
DEFINE_EVENT(s390_class_schib, s390_cio_stsch,
- TP_PROTO(struct subchannel_id schid, struct schib *schib, int cc),
+ TP_PROTO(struct subchannel_id *schid, struct schib *schib, int cc),
TP_ARGS(schid, schib, cc)
);
@@ -71,7 +71,7 @@ DEFINE_EVENT(s390_class_schib, s390_cio_stsch,
* @cc: Condition code
*/
DEFINE_EVENT(s390_class_schib, s390_cio_msch,
- TP_PROTO(struct subchannel_id schid, struct schib *schib, int cc),
+ TP_PROTO(struct subchannel_id *schid, struct schib *schib, int cc),
TP_ARGS(schid, schib, cc)
);
@@ -82,7 +82,7 @@ DEFINE_EVENT(s390_class_schib, s390_cio_msch,
* @cc: Condition code
*/
TRACE_EVENT(s390_cio_tsch,
- TP_PROTO(struct subchannel_id schid, struct irb *irb, int cc),
+ TP_PROTO(struct subchannel_id *schid, struct irb *irb, int cc),
TP_ARGS(schid, irb, cc),
TP_STRUCT__entry(
__field(u8, cssid)
@@ -92,9 +92,9 @@ TRACE_EVENT(s390_cio_tsch,
__field(int, cc)
),
TP_fast_assign(
- __entry->cssid = schid.cssid;
- __entry->ssid = schid.ssid;
- __entry->schno = schid.sch_no;
+ __entry->cssid = schid->cssid;
+ __entry->ssid = schid->ssid;
+ __entry->schno = schid->sch_no;
__entry->irb = *irb;
__entry->cc = cc;
),
@@ -151,7 +151,7 @@ TRACE_EVENT(s390_cio_tpi,
* @cc: Condition code
*/
TRACE_EVENT(s390_cio_ssch,
- TP_PROTO(struct subchannel_id schid, union orb *orb, int cc),
+ TP_PROTO(struct subchannel_id *schid, union orb *orb, int cc),
TP_ARGS(schid, orb, cc),
TP_STRUCT__entry(
__field(u8, cssid)
@@ -161,9 +161,9 @@ TRACE_EVENT(s390_cio_ssch,
__field(int, cc)
),
TP_fast_assign(
- __entry->cssid = schid.cssid;
- __entry->ssid = schid.ssid;
- __entry->schno = schid.sch_no;
+ __entry->cssid = schid->cssid;
+ __entry->ssid = schid->ssid;
+ __entry->schno = schid->sch_no;
__entry->orb = *orb;
__entry->cc = cc;
),
@@ -173,7 +173,7 @@ TRACE_EVENT(s390_cio_ssch,
);
DECLARE_EVENT_CLASS(s390_class_schid,
- TP_PROTO(struct subchannel_id schid, int cc),
+ TP_PROTO(struct subchannel_id *schid, int cc),
TP_ARGS(schid, cc),
TP_STRUCT__entry(
__field(u8, cssid)
@@ -182,9 +182,9 @@ DECLARE_EVENT_CLASS(s390_class_schid,
__field(int, cc)
),
TP_fast_assign(
- __entry->cssid = schid.cssid;
- __entry->ssid = schid.ssid;
- __entry->schno = schid.sch_no;
+ __entry->cssid = schid->cssid;
+ __entry->ssid = schid->ssid;
+ __entry->schno = schid->sch_no;
__entry->cc = cc;
),
TP_printk("schid=%x.%x.%04x cc=%d", __entry->cssid, __entry->ssid,
@@ -198,7 +198,7 @@ DECLARE_EVENT_CLASS(s390_class_schid,
* @cc: Condition code
*/
DEFINE_EVENT(s390_class_schid, s390_cio_csch,
- TP_PROTO(struct subchannel_id schid, int cc),
+ TP_PROTO(struct subchannel_id *schid, int cc),
TP_ARGS(schid, cc)
);
@@ -208,7 +208,7 @@ DEFINE_EVENT(s390_class_schid, s390_cio_csch,
* @cc: Condition code
*/
DEFINE_EVENT(s390_class_schid, s390_cio_hsch,
- TP_PROTO(struct subchannel_id schid, int cc),
+ TP_PROTO(struct subchannel_id *schid, int cc),
TP_ARGS(schid, cc)
);
@@ -218,7 +218,7 @@ DEFINE_EVENT(s390_class_schid, s390_cio_hsch,
* @cc: Condition code
*/
DEFINE_EVENT(s390_class_schid, s390_cio_xsch,
- TP_PROTO(struct subchannel_id schid, int cc),
+ TP_PROTO(struct subchannel_id *schid, int cc),
TP_ARGS(schid, cc)
);
@@ -228,7 +228,7 @@ DEFINE_EVENT(s390_class_schid, s390_cio_xsch,
* @cc: Condition code
*/
DEFINE_EVENT(s390_class_schid, s390_cio_rsch,
- TP_PROTO(struct subchannel_id schid, int cc),
+ TP_PROTO(struct subchannel_id *schid, int cc),
TP_ARGS(schid, cc)
);
@@ -238,7 +238,7 @@ DEFINE_EVENT(s390_class_schid, s390_cio_rsch,
* @cc: Condition code
*/
TRACE_EVENT(s390_cio_rchp,
- TP_PROTO(struct chp_id chpid, int cc),
+ TP_PROTO(struct chp_id *chpid, int cc),
TP_ARGS(chpid, cc),
TP_STRUCT__entry(
__field(u8, cssid)
@@ -246,8 +246,8 @@ TRACE_EVENT(s390_cio_rchp,
__field(int, cc)
),
TP_fast_assign(
- __entry->cssid = chpid.cssid;
- __entry->id = chpid.id;
+ __entry->cssid = chpid->cssid;
+ __entry->id = chpid->id;
__entry->cc = cc;
),
TP_printk("chpid=%x.%02x cc=%d", __entry->cssid, __entry->id,
diff --git a/fs/dax.c b/fs/dax.c
index 0276df90e86c58..6d03ead8e78880 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -1429,7 +1429,7 @@ static int dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
goto finish_iomap;
}
- trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);
+ trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, &pfn, entry);
result = vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn,
write);
break;
diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h
index 8a1442c4e5131f..6b5ab510fdeabe 100644
--- a/include/linux/trace_events.h
+++ b/include/linux/trace_events.h
@@ -546,6 +546,53 @@ extern void ftrace_profile_free_filter(struct perf_event *event);
void perf_trace_buf_update(void *record, u16 type);
void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp);
+void bpf_trace_run1(struct bpf_prog *prog, u64 arg1);
+void bpf_trace_run2(struct bpf_prog *prog, u64 arg1, u64 arg2);
+void bpf_trace_run3(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3);
+void bpf_trace_run4(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4);
+void bpf_trace_run5(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5);
+void bpf_trace_run6(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6);
+void bpf_trace_run7(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7);
+void bpf_trace_run8(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8);
+void bpf_trace_run9(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9);
+void bpf_trace_run10(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10);
+void bpf_trace_run11(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10, u64 arg11);
+void bpf_trace_run12(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12);
+void bpf_trace_run13(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12,
+ u64 arg13);
+void bpf_trace_run14(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12,
+ u64 arg13, u64 arg14);
+void bpf_trace_run15(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12,
+ u64 arg13, u64 arg14, u64 arg15);
+void bpf_trace_run16(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12,
+ u64 arg13, u64 arg14, u64 arg15, u64 arg16);
+void bpf_trace_run17(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12,
+ u64 arg13, u64 arg14, u64 arg15, u64 arg16, u64 arg17);
void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
struct trace_event_call *call, u64 count,
struct pt_regs *regs, struct hlist_head *head,
diff --git a/include/linux/tracepoint-defs.h b/include/linux/tracepoint-defs.h
index 64ed7064f1fa65..39a283c61c51ca 100644
--- a/include/linux/tracepoint-defs.h
+++ b/include/linux/tracepoint-defs.h
@@ -33,6 +33,7 @@ struct tracepoint {
int (*regfunc)(void);
void (*unregfunc)(void);
struct tracepoint_func __rcu *funcs;
+ u32 num_args;
};
#endif
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index c94f466d57ef1e..2fbcd8e4057523 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -40,9 +40,9 @@ tracepoint_probe_register_prio(struct tracepoint *tp, void *probe, void *data,
int prio);
extern int
tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data);
-extern void
-for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv),
- void *priv);
+extern void *
+for_each_kernel_tracepoint(void *(*fct)(struct tracepoint *tp, void *priv),
+ void *priv);
#ifdef CONFIG_MODULES
struct tp_module {
@@ -225,23 +225,27 @@ extern void syscall_unregfunc(void);
return static_key_false(&__tracepoint_##name.key); \
}
+#define ___FN_COUNT(fn,n0,n1,n2,n3,n4,n5,n6,n7,n8,n9,n10,n11,n12,n13,n14,n15,n16,n,...) fn##n
+#define __FN_COUNT(fn,...) ___FN_COUNT(fn,##__VA_ARGS__,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)
+#define __COUNT(...) __FN_COUNT(/**/,##__VA_ARGS__)
+
/*
* We have no guarantee that gcc and the linker won't up-align the tracepoint
* structures, so we create an array of pointers that will be used for iteration
* on the tracepoints.
*/
-#define DEFINE_TRACE_FN(name, reg, unreg) \
+#define DEFINE_TRACE_FN(name, reg, unreg, num_args) \
static const char __tpstrtab_##name[] \
__attribute__((section("__tracepoints_strings"))) = #name; \
struct tracepoint __tracepoint_##name \
__attribute__((section("__tracepoints"))) = \
- { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
+ { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL, num_args };\
static struct tracepoint * const __tracepoint_ptr_##name __used \
__attribute__((section("__tracepoints_ptrs"))) = \
&__tracepoint_##name;
-#define DEFINE_TRACE(name) \
- DEFINE_TRACE_FN(name, NULL, NULL);
+#define DEFINE_TRACE(name, num_args) \
+ DEFINE_TRACE_FN(name, NULL, NULL, num_args);
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \
EXPORT_SYMBOL_GPL(__tracepoint_##name)
@@ -275,8 +279,8 @@ extern void syscall_unregfunc(void);
return false; \
}
-#define DEFINE_TRACE_FN(name, reg, unreg)
-#define DEFINE_TRACE(name)
+#define DEFINE_TRACE_FN(name, reg, unreg, num_args)
+#define DEFINE_TRACE(name, num_args)
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name)
#define EXPORT_TRACEPOINT_SYMBOL(name)
diff --git a/include/trace/bpf_probe.h b/include/trace/bpf_probe.h
new file mode 100644
index 00000000000000..cfbdf6082a959e
--- /dev/null
+++ b/include/trace/bpf_probe.h
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#undef TRACE_SYSTEM_VAR
+
+#ifdef CONFIG_BPF_EVENTS
+
+#undef __entry
+#define __entry entry
+
+#undef __get_dynamic_array
+#define __get_dynamic_array(field) \
+ ((void *)__entry + (__entry->__data_loc_##field & 0xffff))
+
+#undef __get_dynamic_array_len
+#define __get_dynamic_array_len(field) \
+ ((__entry->__data_loc_##field >> 16) & 0xffff)
+
+#undef __get_str
+#define __get_str(field) ((char *)__get_dynamic_array(field))
+
+#undef __get_bitmask
+#define __get_bitmask(field) (char *)__get_dynamic_array(field)
+
+#undef __perf_count
+#define __perf_count(c) (c)
+
+#undef __perf_task
+#define __perf_task(t) (t)
+
+/*
+ * cast any interger or pointer type to u64 without warnings
+ * on 32 and 64 bit archs
+ */
+#define __CAST_TO_U64(expr) \
+ (u64) __builtin_choose_expr(sizeof(long) < sizeof(expr), \
+ (expr), \
+ (long) expr)
+#define __CAST1(a,...) __CAST_TO_U64(a)
+#define __CAST2(a,...) __CAST_TO_U64(a), __CAST1(__VA_ARGS__)
+#define __CAST3(a,...) __CAST_TO_U64(a), __CAST2(__VA_ARGS__)
+#define __CAST4(a,...) __CAST_TO_U64(a), __CAST3(__VA_ARGS__)
+#define __CAST5(a,...) __CAST_TO_U64(a), __CAST4(__VA_ARGS__)
+#define __CAST6(a,...) __CAST_TO_U64(a), __CAST5(__VA_ARGS__)
+#define __CAST7(a,...) __CAST_TO_U64(a), __CAST6(__VA_ARGS__)
+#define __CAST8(a,...) __CAST_TO_U64(a), __CAST7(__VA_ARGS__)
+#define __CAST9(a,...) __CAST_TO_U64(a), __CAST8(__VA_ARGS__)
+#define __CAST10(a,...) __CAST_TO_U64(a), __CAST9(__VA_ARGS__)
+#define __CAST11(a,...) __CAST_TO_U64(a), __CAST10(__VA_ARGS__)
+#define __CAST12(a,...) __CAST_TO_U64(a), __CAST11(__VA_ARGS__)
+#define __CAST13(a,...) __CAST_TO_U64(a), __CAST12(__VA_ARGS__)
+#define __CAST14(a,...) __CAST_TO_U64(a), __CAST13(__VA_ARGS__)
+#define __CAST15(a,...) __CAST_TO_U64(a), __CAST14(__VA_ARGS__)
+#define __CAST16(a,...) __CAST_TO_U64(a), __CAST15(__VA_ARGS__)
+#define __CAST17(a,...) __CAST_TO_U64(a), __CAST16(__VA_ARGS__)
+
+#define CAST_TO_U64(...) __FN_COUNT(__CAST,##__VA_ARGS__)(__VA_ARGS__)
+
+#undef DECLARE_EVENT_CLASS
+#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
+/* no 'static' here. The bpf probe functions are global */ \
+notrace void \
+__bpf_trace_##call(void *__data, proto) \
+{ \
+ struct bpf_prog *prog = __data; \
+ \
+ __FN_COUNT(bpf_trace_run, args)(prog, CAST_TO_U64(args)); \
+}
+
+/*
+ * This part is compiled out, it is only here as a build time check
+ * to make sure that if the tracepoint handling changes, the
+ * bpf probe will fail to compile unless it too is updated.
+ */
+#undef DEFINE_EVENT
+#define DEFINE_EVENT(template, call, proto, args) \
+static inline void bpf_test_probe_##call(void) \
+{ \
+ check_trace_callback_type_##call(__bpf_trace_##template); \
+}
+
+
+#undef DEFINE_EVENT_PRINT
+#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
+ DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
+
+#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
+#endif /* CONFIG_BPF_EVENTS */
diff --git a/include/trace/define_trace.h b/include/trace/define_trace.h
index d9e3d4aa3f6e3b..3bbd3b88177f1e 100644
--- a/include/trace/define_trace.h
+++ b/include/trace/define_trace.h
@@ -25,7 +25,7 @@
#undef TRACE_EVENT
#define TRACE_EVENT(name, proto, args, tstruct, assign, print) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, __COUNT(args))
#undef TRACE_EVENT_CONDITION
#define TRACE_EVENT_CONDITION(name, proto, args, cond, tstruct, assign, print) \
@@ -39,24 +39,24 @@
#undef TRACE_EVENT_FN
#define TRACE_EVENT_FN(name, proto, args, tstruct, \
assign, print, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, __COUNT(args))
#undef TRACE_EVENT_FN_COND
#define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \
assign, print, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, __COUNT(args))
#undef DEFINE_EVENT
#define DEFINE_EVENT(template, name, proto, args) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, __COUNT(args))
#undef DEFINE_EVENT_FN
#define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, __COUNT(args))
#undef DEFINE_EVENT_PRINT
#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, __COUNT(args))
#undef DEFINE_EVENT_CONDITION
#define DEFINE_EVENT_CONDITION(template, name, proto, args, cond) \
@@ -64,7 +64,7 @@
#undef DECLARE_TRACE
#define DECLARE_TRACE(name, proto, args) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, __COUNT(args))
#undef TRACE_INCLUDE
#undef __TRACE_INCLUDE
@@ -95,6 +95,7 @@
#ifdef TRACEPOINTS_ENABLED
#include <trace/trace_events.h>
#include <trace/perf.h>
+#include <trace/bpf_probe.h>
#endif
#undef TRACE_EVENT
diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h
index 06c87f9f720c2e..795698925d206e 100644
--- a/include/trace/events/f2fs.h
+++ b/include/trace/events/f2fs.h
@@ -491,7 +491,7 @@ DEFINE_EVENT(f2fs__truncate_node, f2fs_truncate_node,
TRACE_EVENT(f2fs_truncate_partial_nodes,
- TP_PROTO(struct inode *inode, nid_t nid[], int depth, int err),
+ TP_PROTO(struct inode *inode, nid_t *nid, int depth, int err),
TP_ARGS(inode, nid, depth, err),
diff --git a/include/trace/events/fs_dax.h b/include/trace/events/fs_dax.h
index 97b09fcf7e5287..5a6a8285750f5f 100644
--- a/include/trace/events/fs_dax.h
+++ b/include/trace/events/fs_dax.h
@@ -104,7 +104,7 @@ DEFINE_PMD_LOAD_HOLE_EVENT(dax_pmd_load_hole_fallback);
DECLARE_EVENT_CLASS(dax_pmd_insert_mapping_class,
TP_PROTO(struct inode *inode, struct vm_fault *vmf,
- long length, pfn_t pfn, void *radix_entry),
+ long length, pfn_t *pfn, void *radix_entry),
TP_ARGS(inode, vmf, length, pfn, radix_entry),
TP_STRUCT__entry(
__field(unsigned long, ino)
@@ -123,7 +123,7 @@ DECLARE_EVENT_CLASS(dax_pmd_insert_mapping_class,
__entry->address = vmf->address;
__entry->write = vmf->flags & FAULT_FLAG_WRITE;
__entry->length = length;
- __entry->pfn_val = pfn.val;
+ __entry->pfn_val = pfn->val;
__entry->radix_entry = radix_entry;
),
TP_printk("dev %d:%d ino %#lx %s %s address %#lx length %#lx "
@@ -145,7 +145,7 @@ DECLARE_EVENT_CLASS(dax_pmd_insert_mapping_class,
#define DEFINE_PMD_INSERT_MAPPING_EVENT(name) \
DEFINE_EVENT(dax_pmd_insert_mapping_class, name, \
TP_PROTO(struct inode *inode, struct vm_fault *vmf, \
- long length, pfn_t pfn, void *radix_entry), \
+ long length, pfn_t *pfn, void *radix_entry), \
TP_ARGS(inode, vmf, length, pfn, radix_entry))
DEFINE_PMD_INSERT_MAPPING_EVENT(dax_pmd_insert_mapping);
diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h
index 0b50fda80db0e0..4b463294306f95 100644
--- a/include/trace/events/rcu.h
+++ b/include/trace/events/rcu.h
@@ -436,7 +436,7 @@ TRACE_EVENT(rcu_fqs,
*/
TRACE_EVENT(rcu_dyntick,
- TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t dynticks),
+ TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t *dynticks),
TP_ARGS(polarity, oldnesting, newnesting, dynticks),
@@ -451,7 +451,7 @@ TRACE_EVENT(rcu_dyntick,
__entry->polarity = polarity;
__entry->oldnesting = oldnesting;
__entry->newnesting = newnesting;
- __entry->dynticks = atomic_read(&dynticks);
+ __entry->dynticks = atomic_read(dynticks);
),
TP_printk("%s %lx %lx %#3x", __entry->polarity,
diff --git a/include/trace/events/xen.h b/include/trace/events/xen.h
index b8adf05c534e72..d1965815d9cfdd 100644
--- a/include/trace/events/xen.h
+++ b/include/trace/events/xen.h
@@ -128,14 +128,14 @@ TRACE_EVENT(xen_mc_extend_args,
TRACE_DEFINE_SIZEOF(pteval_t);
/* mmu */
DECLARE_EVENT_CLASS(xen_mmu__set_pte,
- TP_PROTO(pte_t *ptep, pte_t pteval),
+ TP_PROTO(pte_t *ptep, pte_t *pteval),
TP_ARGS(ptep, pteval),
TP_STRUCT__entry(
__field(pte_t *, ptep)
__field(pteval_t, pteval)
),
TP_fast_assign(__entry->ptep = ptep;
- __entry->pteval = pteval.pte),
+ __entry->pteval = pteval->pte),
TP_printk("ptep %p pteval %0*llx (raw %0*llx)",
__entry->ptep,
(int)sizeof(pteval_t) * 2, (unsigned long long)pte_val(native_make_pte(__entry->pteval)),
@@ -144,14 +144,14 @@ DECLARE_EVENT_CLASS(xen_mmu__set_pte,
#define DEFINE_XEN_MMU_SET_PTE(name) \
DEFINE_EVENT(xen_mmu__set_pte, name, \
- TP_PROTO(pte_t *ptep, pte_t pteval), \
+ TP_PROTO(pte_t *ptep, pte_t *pteval), \
TP_ARGS(ptep, pteval))
DEFINE_XEN_MMU_SET_PTE(xen_mmu_set_pte);
TRACE_EVENT(xen_mmu_set_pte_at,
TP_PROTO(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep, pte_t pteval),
+ pte_t *ptep, pte_t *pteval),
TP_ARGS(mm, addr, ptep, pteval),
TP_STRUCT__entry(
__field(struct mm_struct *, mm)
@@ -162,7 +162,7 @@ TRACE_EVENT(xen_mmu_set_pte_at,
TP_fast_assign(__entry->mm = mm;
__entry->addr = addr;
__entry->ptep = ptep;
- __entry->pteval = pteval.pte),
+ __entry->pteval = pteval->pte),
TP_printk("mm %p addr %lx ptep %p pteval %0*llx (raw %0*llx)",
__entry->mm, __entry->addr, __entry->ptep,
(int)sizeof(pteval_t) * 2, (unsigned long long)pte_val(native_make_pte(__entry->pteval)),
@@ -172,14 +172,14 @@ TRACE_EVENT(xen_mmu_set_pte_at,
TRACE_DEFINE_SIZEOF(pmdval_t);
TRACE_EVENT(xen_mmu_set_pmd,
- TP_PROTO(pmd_t *pmdp, pmd_t pmdval),
+ TP_PROTO(pmd_t *pmdp, pmd_t *pmdval),
TP_ARGS(pmdp, pmdval),
TP_STRUCT__entry(
__field(pmd_t *, pmdp)
__field(pmdval_t, pmdval)
),
TP_fast_assign(__entry->pmdp = pmdp;
- __entry->pmdval = pmdval.pmd),
+ __entry->pmdval = pmdval->pmd),
TP_printk("pmdp %p pmdval %0*llx (raw %0*llx)",
__entry->pmdp,
(int)sizeof(pmdval_t) * 2, (unsigned long long)pmd_val(native_make_pmd(__entry->pmdval)),
@@ -220,14 +220,14 @@ TRACE_EVENT(xen_mmu_pmd_clear,
TRACE_DEFINE_SIZEOF(pudval_t);
TRACE_EVENT(xen_mmu_set_pud,
- TP_PROTO(pud_t *pudp, pud_t pudval),
+ TP_PROTO(pud_t *pudp, pud_t *pudval),
TP_ARGS(pudp, pudval),
TP_STRUCT__entry(
__field(pud_t *, pudp)
__field(pudval_t, pudval)
),
TP_fast_assign(__entry->pudp = pudp;
- __entry->pudval = native_pud_val(pudval)),
+ __entry->pudval = native_pud_val(*pudval)),
TP_printk("pudp %p pudval %0*llx (raw %0*llx)",
__entry->pudp,
(int)sizeof(pudval_t) * 2, (unsigned long long)pud_val(native_make_pud(__entry->pudval)),
@@ -237,7 +237,7 @@ TRACE_EVENT(xen_mmu_set_pud,
TRACE_DEFINE_SIZEOF(p4dval_t);
TRACE_EVENT(xen_mmu_set_p4d,
- TP_PROTO(p4d_t *p4dp, p4d_t *user_p4dp, p4d_t p4dval),
+ TP_PROTO(p4d_t *p4dp, p4d_t *user_p4dp, p4d_t *p4dval),
TP_ARGS(p4dp, user_p4dp, p4dval),
TP_STRUCT__entry(
__field(p4d_t *, p4dp)
@@ -246,7 +246,7 @@ TRACE_EVENT(xen_mmu_set_p4d,
),
TP_fast_assign(__entry->p4dp = p4dp;
__entry->user_p4dp = user_p4dp;
- __entry->p4dval = p4d_val(p4dval)),
+ __entry->p4dval = p4d_val(*p4dval)),
TP_printk("p4dp %p user_p4dp %p p4dval %0*llx (raw %0*llx)",
__entry->p4dp, __entry->user_p4dp,
(int)sizeof(p4dval_t) * 2, (unsigned long long)pgd_val(native_make_pgd(__entry->p4dval)),
@@ -255,14 +255,14 @@ TRACE_EVENT(xen_mmu_set_p4d,
#else
TRACE_EVENT(xen_mmu_set_pud,
- TP_PROTO(pud_t *pudp, pud_t pudval),
+ TP_PROTO(pud_t *pudp, pud_t *pudval),
TP_ARGS(pudp, pudval),
TP_STRUCT__entry(
__field(pud_t *, pudp)
__field(pudval_t, pudval)
),
TP_fast_assign(__entry->pudp = pudp;
- __entry->pudval = native_pud_val(pudval)),
+ __entry->pudval = native_pud_val(*pudval)),
TP_printk("pudp %p pudval %0*llx (raw %0*llx)",
__entry->pudp,
(int)sizeof(pudval_t) * 2, (unsigned long long)pgd_val(native_make_pgd(__entry->pudval)),
@@ -273,7 +273,7 @@ TRACE_EVENT(xen_mmu_set_pud,
DECLARE_EVENT_CLASS(xen_mmu_ptep_modify_prot,
TP_PROTO(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep, pte_t pteval),
+ pte_t *ptep, pte_t *pteval),
TP_ARGS(mm, addr, ptep, pteval),
TP_STRUCT__entry(
__field(struct mm_struct *, mm)
@@ -284,7 +284,7 @@ DECLARE_EVENT_CLASS(xen_mmu_ptep_modify_prot,
TP_fast_assign(__entry->mm = mm;
__entry->addr = addr;
__entry->ptep = ptep;
- __entry->pteval = pteval.pte),
+ __entry->pteval = pteval->pte),
TP_printk("mm %p addr %lx ptep %p pteval %0*llx (raw %0*llx)",
__entry->mm, __entry->addr, __entry->ptep,
(int)sizeof(pteval_t) * 2, (unsigned long long)pte_val(native_make_pte(__entry->pteval)),
@@ -293,7 +293,7 @@ DECLARE_EVENT_CLASS(xen_mmu_ptep_modify_prot,
#define DEFINE_XEN_MMU_PTEP_MODIFY_PROT(name) \
DEFINE_EVENT(xen_mmu_ptep_modify_prot, name, \
TP_PROTO(struct mm_struct *mm, unsigned long addr, \
- pte_t *ptep, pte_t pteval), \
+ pte_t *ptep, pte_t *pteval), \
TP_ARGS(mm, addr, ptep, pteval))
DEFINE_XEN_MMU_PTEP_MODIFY_PROT(xen_mmu_ptep_modify_prot_start);
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index db6bdc37512683..11ac17f78d9442 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -133,6 +133,7 @@ enum bpf_prog_type {
BPF_PROG_TYPE_SOCK_OPS,
BPF_PROG_TYPE_SK_SKB,
BPF_PROG_TYPE_CGROUP_DEVICE,
+ BPF_PROG_TYPE_RAW_TRACEPOINT,
};
enum bpf_attach_type {
@@ -143,6 +144,7 @@ enum bpf_attach_type {
BPF_SK_SKB_STREAM_PARSER,
BPF_SK_SKB_STREAM_VERDICT,
BPF_CGROUP_DEVICE,
+ BPF_RAW_TRACEPOINT,
__MAX_BPF_ATTACH_TYPE
};
@@ -1106,4 +1108,8 @@ struct bpf_cgroup_dev_ctx {
__u32 minor;
};
+struct bpf_raw_tp_ctx {
+ __u64 args[16];
+};
+
#endif /* _UAPI__LINUX_BPF_H__ */
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 491bdf39f276cd..43c0f899f78c8a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -772,7 +772,7 @@ static void rcu_eqs_enter(bool user)
}
lockdep_assert_irqs_disabled();
- trace_rcu_dyntick(TPS("Start"), rdtp->dynticks_nesting, 0, rdtp->dynticks);
+ trace_rcu_dyntick(TPS("Start"), rdtp->dynticks_nesting, 0, &rdtp->dynticks);
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
for_each_rcu_flavor(rsp) {
rdp = this_cpu_ptr(rsp->rda);
@@ -848,14 +848,14 @@ void rcu_nmi_exit(void)
* leave it in non-RCU-idle state.
*/
if (rdtp->dynticks_nmi_nesting != 1) {
- trace_rcu_dyntick(TPS("--="), rdtp->dynticks_nmi_nesting, rdtp->dynticks_nmi_nesting - 2, rdtp->dynticks);
+ trace_rcu_dyntick(TPS("--="), rdtp->dynticks_nmi_nesting, rdtp->dynticks_nmi_nesting - 2, &rdtp->dynticks);
WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */
rdtp->dynticks_nmi_nesting - 2);
return;
}
/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
- trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
+ trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, &rdtp->dynticks);
WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
rcu_dynticks_eqs_enter();
}
@@ -930,7 +930,7 @@ static void rcu_eqs_exit(bool user)
rcu_dynticks_task_exit();
rcu_dynticks_eqs_exit();
rcu_cleanup_after_idle();
- trace_rcu_dyntick(TPS("End"), rdtp->dynticks_nesting, 1, rdtp->dynticks);
+ trace_rcu_dyntick(TPS("End"), rdtp->dynticks_nesting, 1, &rdtp->dynticks);
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
WRITE_ONCE(rdtp->dynticks_nesting, 1);
WRITE_ONCE(rdtp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
@@ -1004,7 +1004,7 @@ void rcu_nmi_enter(void)
}
trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
rdtp->dynticks_nmi_nesting,
- rdtp->dynticks_nmi_nesting + incby, rdtp->dynticks);
+ rdtp->dynticks_nmi_nesting + incby, &rdtp->dynticks);
WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */
rdtp->dynticks_nmi_nesting + incby);
barrier();
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index fc2838ac8b7877..41d79d8504f1c1 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -882,3 +882,187 @@ int perf_event_query_prog_array(struct perf_event *event, void __user *info)
return ret;
}
+
+static __always_inline
+void __bpf_trace_run(struct bpf_prog *prog, u64 *args)
+{
+}
+
+#define EVAL1(FN, X) FN(X)
+#define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y)
+#define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y)
+#define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y)
+#define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y)
+#define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y)
+
+#define COPY(X) args[X - 1] = arg##X;
+
+void bpf_trace_run1(struct bpf_prog *prog, u64 arg1)
+{
+ u64 args[1];
+
+ EVAL1(COPY, 1);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run1);
+void bpf_trace_run2(struct bpf_prog *prog, u64 arg1, u64 arg2)
+{
+ u64 args[2];
+
+ EVAL2(COPY, 1, 2);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run2);
+void bpf_trace_run3(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3)
+{
+ u64 args[3];
+
+ EVAL3(COPY, 1, 2, 3);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run3);
+void bpf_trace_run4(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4)
+{
+ u64 args[4];
+
+ EVAL4(COPY, 1, 2, 3, 4);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run4);
+void bpf_trace_run5(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5)
+{
+ u64 args[5];
+
+ EVAL5(COPY, 1, 2, 3, 4, 5);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run5);
+void bpf_trace_run6(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6)
+{
+ u64 args[6];
+
+ EVAL6(COPY, 1, 2, 3, 4, 5, 6);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run6);
+void bpf_trace_run7(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7)
+{
+ u64 args[7];
+
+ EVAL6(COPY, 1, 2, 3, 4, 5, 6);
+ EVAL1(COPY, 7);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run7);
+void bpf_trace_run8(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8)
+{
+ u64 args[8];
+
+ EVAL6(COPY, 1, 2, 3, 4, 5, 6);
+ EVAL2(COPY, 7, 8);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run8);
+void bpf_trace_run9(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9)
+{
+ u64 args[9];
+
+ EVAL6(COPY, 1, 2, 3, 4, 5, 6);
+ EVAL3(COPY, 7, 8, 9);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run9);
+void bpf_trace_run10(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10)
+{
+ u64 args[10];
+
+ EVAL6(COPY, 1, 2, 3, 4, 5, 6);
+ EVAL4(COPY, 7, 8, 9, 10);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run10);
+void bpf_trace_run11(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10, u64 arg11)
+{
+ u64 args[11];
+
+ EVAL6(COPY, 1, 2, 3, 4, 5, 6);
+ EVAL5(COPY, 7, 8, 9, 10, 11);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run11);
+void bpf_trace_run12(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12)
+{
+ u64 args[12];
+
+ EVAL6(COPY, 1, 2, 3, 4, 5, 6);
+ EVAL6(COPY, 7, 8, 9, 10, 11, 12);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run12);
+void bpf_trace_run17(struct bpf_prog *prog, u64 arg1, u64 arg2,
+ u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
+ u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12,
+ u64 arg13, u64 arg14, u64 arg15, u64 arg16, u64 arg17)
+{
+ u64 args[17];
+
+ EVAL6(COPY, 1, 2, 3, 4, 5, 6);
+ EVAL6(COPY, 7, 8, 9, 10, 11, 12);
+ EVAL5(COPY, 13, 14, 15, 16, 17);
+ __bpf_trace_run(prog, args);
+}
+EXPORT_SYMBOL_GPL(bpf_trace_run17);
+
+static void *__find_tp(struct tracepoint *tp, void *priv)
+{
+ char *name = priv;
+
+ if (!strcmp(tp->name, name))
+ return tp;
+ return NULL;
+}
+
+int bpf_probe_register(char *name, struct bpf_prog *prog)
+{
+ struct tracepoint *tp;
+ unsigned long addr;
+ char buf[128];
+ int err;
+
+ tp = for_each_kernel_tracepoint(__find_tp, name);
+ if (!tp)
+ return -ENOENT;
+
+ /*
+ * check that program doesn't access arguments beyond what's
+ * available in this tracepoint
+ */
+ if (prog->aux->max_ctx_offset > tp->num_args * sizeof(u64))
+ return -EINVAL;
+
+ snprintf(buf, sizeof(buf), "__bpf_trace_%s", name);
+ addr = kallsyms_lookup_name(buf);
+ if (!addr)
+ return -ENOENT;
+
+ err = tracepoint_probe_register(tp, (void *)addr, prog);
+ if (err)
+ return err;
+ return 0;
+}
+
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index 671b134573876d..3f2dc5738c2bd1 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -502,17 +502,22 @@ static __init int init_tracepoints(void)
__initcall(init_tracepoints);
#endif /* CONFIG_MODULES */
-static void for_each_tracepoint_range(struct tracepoint * const *begin,
- struct tracepoint * const *end,
- void (*fct)(struct tracepoint *tp, void *priv),
- void *priv)
+static void *for_each_tracepoint_range(struct tracepoint * const *begin,
+ struct tracepoint * const *end,
+ void *(*fct)(struct tracepoint *tp, void *priv),
+ void *priv)
{
struct tracepoint * const *iter;
+ void *ret;
if (!begin)
- return;
- for (iter = begin; iter < end; iter++)
- fct(*iter, priv);
+ return NULL;
+ for (iter = begin; iter < end; iter++) {
+ ret = fct(*iter, priv);
+ if (ret)
+ return ret;
+ }
+ return NULL;
}
/**
@@ -520,11 +525,11 @@ static void for_each_tracepoint_range(struct tracepoint * const *begin,
* @fct: callback
* @priv: private data
*/
-void for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv),
- void *priv)
+void *for_each_kernel_tracepoint(void *(*fct)(struct tracepoint *tp, void *priv),
+ void *priv)
{
- for_each_tracepoint_range(__start___tracepoints_ptrs,
- __stop___tracepoints_ptrs, fct, priv);
+ return for_each_tracepoint_range(__start___tracepoints_ptrs,
+ __stop___tracepoints_ptrs, fct, priv);
}
EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint);
diff --git a/net/wireless/trace.h b/net/wireless/trace.h
index bcfedd39e7a365..392189bb0a0db3 100644
--- a/net/wireless/trace.h
+++ b/net/wireless/trace.h
@@ -3114,7 +3114,7 @@ TRACE_EVENT(rdev_start_radar_detection,
TRACE_EVENT(rdev_set_mcast_rate,
TP_PROTO(struct wiphy *wiphy, struct net_device *netdev,
- int mcast_rate[NUM_NL80211_BANDS]),
+ int *mcast_rate),
TP_ARGS(wiphy, netdev, mcast_rate),
TP_STRUCT__entry(
WIPHY_ENTRY
diff --git a/sound/firewire/amdtp-stream-trace.h b/sound/firewire/amdtp-stream-trace.h
index ea0d486652c838..54cdd4ffa9ceb1 100644
--- a/sound/firewire/amdtp-stream-trace.h
+++ b/sound/firewire/amdtp-stream-trace.h
@@ -14,7 +14,7 @@
#include <linux/tracepoint.h>
TRACE_EVENT(in_packet,
- TP_PROTO(const struct amdtp_stream *s, u32 cycles, u32 cip_header[2], unsigned int payload_length, unsigned int index),
+ TP_PROTO(const struct amdtp_stream *s, u32 cycles, u32 *cip_header, unsigned int payload_length, unsigned int index),
TP_ARGS(s, cycles, cip_header, payload_length, index),
TP_STRUCT__entry(
__field(unsigned int, second)