Commit 14bef1ab authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf: introduce bpf_get_branch_snapshot' 

Song Liu says:

====================

Changes v6 => v7:
1. Improve/fix intel_pmu_snapshot_branch_stack() logic. (Peter).

Changes v5 => v6:
1. Add local_irq_save/restore to intel_pmu_snapshot_branch_stack. (Peter)
2. Remove buf and size check in bpf_get_branch_snapshot, move flags check
   to later fo the function. (Peter, Andrii)
3. Revise comments for bpf_get_branch_snapshot in bpf.h (Andrii)

Changes v4 => v5:
1. Modify perf_snapshot_branch_stack_t to save some memcpy. (Andrii)
2. Minor fixes in selftests. (Andrii)

Changes v3 => v4:
1. Do not reshuffle intel_pmu_disable_all(). Use some inline to save LBR
   entries. (Peter)
2. Move static_call(perf_snapshot_branch_stack) to the helper. (Alexei)
3. Add argument flags to bpf_get_branch_snapshot. (Andrii)
4. Make MAX_BRANCH_SNAPSHOT an enum (Andrii). And rename it as
   PERF_MAX_BRANCH_SNAPSHOT
5. Make bpf_get_branch_snapshot similar to bpf_read_branch_records.
   (Andrii)
6. Move the test target function to bpf_testmod. Updated kallsyms_find_next
   to work properly with modules. (Andrii)

Changes v2 => v3:
1. Fix the use of static_call. (Peter)
2. Limit the use to perfmon version >= 2. (Peter)
3. Modify intel_pmu_snapshot_branch_stack() to use intel_pmu_disable_all
   and intel_pmu_enable_all().

Changes v1 => v2:
1. Rename the helper as bpf_get_branch_snapshot;
2. Fix/simplify the use of static_call;
3. Instead of percpu variables, let intel_pmu_snapshot_branch_stack output
   branch records to an output argument of type perf_branch_snapshot.

Branch stack can be very useful in understanding software events. For
example, when a long function, e.g. sys_perf_event_open, returns an errno,
it is not obvious why the function failed. Branch stack could provide very
helpful information in this type of scenarios.

This set adds support to read branch stack with a new BPF helper
bpf_get_branch_trace(). Currently, this is only supported in Intel systems.
It is also possible to support the same feaure for PowerPC.

The hardware that records the branch stace is not stopped automatically on
software events. Therefore, it is necessary to stop it in software soon.
Otherwise, the hardware buffers/registers will be flushed. One of the key
design consideration in this set is to minimize the number of branch record
entries between the event triggers and the hardware recorder is stopped.
Based on this goal, current design is different from the discussions in
original RFC [1]:
 1) Static call is used when supported, to save function pointer
    dereference;
 2) intel_pmu_lbr_disable_all is used instead of perf_pmu_disable(),
    because the latter uses about 10 entries before stopping LBR.

With current code, on Intel CPU, LBR is stopped after 7 branch entries
after fexit triggers:

ID: 0 from bpf_get_branch_snapshot+18 to intel_pmu_snapshot_branch_stack+0
ID: 1 from __brk_limit+477143934 to bpf_get_branch_snapshot+0
ID: 2 from __brk_limit+477192263 to __brk_limit+477143880  # trampoline
ID: 3 from __bpf_prog_enter+34 to __brk_limit+477192251
ID: 4 from migrate_disable+60 to __bpf_prog_enter+9
ID: 5 from __bpf_prog_enter+4 to migrate_disable+0
ID: 6 from bpf_testmod_loop_test+20 to __bpf_prog_enter+0
ID: 7 from bpf_testmod_loop_test+20 to bpf_testmod_loop_test+13
ID: 8 from bpf_testmod_loop_test+20 to bpf_testmod_loop_test+13
...

[1] https://lore.kernel.org/bpf/20210818012937.2522409-1-songliubraving@fb.com/


====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 3384c7c7 025bd7c7

arch/x86/events/intel/core.c

+61 −6
Original line number Diff line number Diff line
@@ -2143,19 +2143,19 @@ static __initconst const u64 knl_hw_cache_extra_regs 
 * However, there are some cases which may change PEBS status, e.g. PMI

 * throttle. The PEBS_ENABLE should be updated where the status changes.

 */

static void __intel_pmu_disable_all(void)

static __always_inline void __intel_pmu_disable_all(bool bts)

{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);



	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);



	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))

	if (bts && test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))

		intel_pmu_disable_bts();

}



static void intel_pmu_disable_all(void)

static __always_inline void intel_pmu_disable_all(void)

{

	__intel_pmu_disable_all();

	__intel_pmu_disable_all(true);

	intel_pmu_pebs_disable_all();

	intel_pmu_lbr_disable_all();

}
@@ -2186,6 +2186,49 @@ static void intel_pmu_enable_all(int added) 
	__intel_pmu_enable_all(added, false);

}



static noinline int

__intel_pmu_snapshot_branch_stack(struct perf_branch_entry *entries,

				  unsigned int cnt, unsigned long flags)

{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);



	intel_pmu_lbr_read();

	cnt = min_t(unsigned int, cnt, x86_pmu.lbr_nr);



	memcpy(entries, cpuc->lbr_entries, sizeof(struct perf_branch_entry) * cnt);

	intel_pmu_enable_all(0);

	local_irq_restore(flags);

	return cnt;

}



static int

intel_pmu_snapshot_branch_stack(struct perf_branch_entry *entries, unsigned int cnt)

{

	unsigned long flags;



	/* must not have branches... */

	local_irq_save(flags);

	__intel_pmu_disable_all(false); /* we don't care about BTS */

	__intel_pmu_pebs_disable_all();

	__intel_pmu_lbr_disable();

	/*            ... until here */

	return __intel_pmu_snapshot_branch_stack(entries, cnt, flags);

}



static int

intel_pmu_snapshot_arch_branch_stack(struct perf_branch_entry *entries, unsigned int cnt)

{

	unsigned long flags;



	/* must not have branches... */

	local_irq_save(flags);

	__intel_pmu_disable_all(false); /* we don't care about BTS */

	__intel_pmu_pebs_disable_all();

	__intel_pmu_arch_lbr_disable();

	/*            ... until here */

	return __intel_pmu_snapshot_branch_stack(entries, cnt, flags);

}



/*

 * Workaround for:

 *   Intel Errata AAK100 (model 26)
@@ -2929,7 +2972,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs) 
		apic_write(APIC_LVTPC, APIC_DM_NMI);

	intel_bts_disable_local();

	cpuc->enabled = 0;

	__intel_pmu_disable_all();

	__intel_pmu_disable_all(true);

	handled = intel_pmu_drain_bts_buffer();

	handled += intel_bts_interrupt();

	status = intel_pmu_get_status();
@@ -6283,9 +6326,21 @@ __init int intel_pmu_init(void) 
			x86_pmu.lbr_nr = 0;

	}



	if (x86_pmu.lbr_nr)

	if (x86_pmu.lbr_nr) {

		pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr);



		/* only support branch_stack snapshot for perfmon >= v2 */

		if (x86_pmu.disable_all == intel_pmu_disable_all) {

			if (boot_cpu_has(X86_FEATURE_ARCH_LBR)) {

				static_call_update(perf_snapshot_branch_stack,

						   intel_pmu_snapshot_arch_branch_stack);

			} else {

				static_call_update(perf_snapshot_branch_stack,

						   intel_pmu_snapshot_branch_stack);

			}

		}

	}



	intel_pmu_check_extra_regs(x86_pmu.extra_regs);



	/* Support full width counters using alternative MSR range */

arch/x86/events/intel/ds.c

+1 −1
Original line number Diff line number Diff line
@@ -1301,7 +1301,7 @@ void intel_pmu_pebs_disable_all(void) 
	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);



	if (cpuc->pebs_enabled)

		wrmsrl(MSR_IA32_PEBS_ENABLE, 0);

		__intel_pmu_pebs_disable_all();

}



static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)

arch/x86/events/intel/lbr.c

+5 −15
Original line number Diff line number Diff line
@@ -228,20 +228,6 @@ static void __intel_pmu_lbr_enable(bool pmi) 
		wrmsrl(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN);

}



static void __intel_pmu_lbr_disable(void)

{

	u64 debugctl;



	if (static_cpu_has(X86_FEATURE_ARCH_LBR)) {

		wrmsrl(MSR_ARCH_LBR_CTL, 0);

		return;

	}



	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);

	debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);

	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);

}



void intel_pmu_lbr_reset_32(void)

{

	int i;
@@ -779,9 +765,13 @@ void intel_pmu_lbr_disable_all(void) 
{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);



	if (cpuc->lbr_users && !vlbr_exclude_host())

	if (cpuc->lbr_users && !vlbr_exclude_host()) {

		if (static_cpu_has(X86_FEATURE_ARCH_LBR))

			return __intel_pmu_arch_lbr_disable();



		__intel_pmu_lbr_disable();

	}

}



void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)

{

arch/x86/events/perf_event.h

+19 −0
Original line number Diff line number Diff line
@@ -1240,6 +1240,25 @@ static inline bool intel_pmu_has_bts(struct perf_event *event) 
	return intel_pmu_has_bts_period(event, hwc->sample_period);

}



static __always_inline void __intel_pmu_pebs_disable_all(void)

{

	wrmsrl(MSR_IA32_PEBS_ENABLE, 0);

}



static __always_inline void __intel_pmu_arch_lbr_disable(void)

{

	wrmsrl(MSR_ARCH_LBR_CTL, 0);

}



static __always_inline void __intel_pmu_lbr_disable(void)

{

	u64 debugctl;



	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);

	debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);

	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);

}



int intel_pmu_save_and_restart(struct perf_event *event);



struct event_constraint *

include/linux/perf_event.h

+23 −0
Original line number Diff line number Diff line
@@ -57,6 +57,7 @@ struct perf_guest_info_callbacks { 
#include <linux/cgroup.h>

#include <linux/refcount.h>

#include <linux/security.h>

#include <linux/static_call.h>

#include <asm/local.h>



struct perf_callchain_entry {
@@ -1612,4 +1613,26 @@ extern void __weak arch_perf_update_userpage(struct perf_event *event, 
extern __weak u64 arch_perf_get_page_size(struct mm_struct *mm, unsigned long addr);

#endif



/*

 * Snapshot branch stack on software events.

 *

 * Branch stack can be very useful in understanding software events. For

 * example, when a long function, e.g. sys_perf_event_open, returns an

 * errno, it is not obvious why the function failed. Branch stack could

 * provide very helpful information in this type of scenarios.

 *

 * On software event, it is necessary to stop the hardware branch recorder

 * fast. Otherwise, the hardware register/buffer will be flushed with

 * entries of the triggering event. Therefore, static call is used to

 * stop the hardware recorder.

 */



/*

 * cnt is the number of entries allocated for entries.

 * Return number of entries copied to .

 */

typedef int (perf_snapshot_branch_stack_t)(struct perf_branch_entry *entries,

					   unsigned int cnt);

DECLARE_STATIC_CALL(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);



#endif /* _LINUX_PERF_EVENT_H */