Commit 8739c681 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

sched/clock/x86: Mark sched_clock() noinstr 



In order to use sched_clock() from noinstr code, mark it and all it's
implenentations noinstr.

The whole pvclock thing (used by KVM/Xen) is a bit of a pain,
since it calls out to watchdogs, create a
pvclock_clocksource_read_nowd() variant doesn't do that and can be
noinstr.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20230126151323.702003578@infradead.org
parent 5c9da9fe

arch/x86/include/asm/kvmclock.h

+1 −1
Original line number Diff line number Diff line
@@ -8,7 +8,7 @@ extern struct clocksource kvm_clock; 


DECLARE_PER_CPU(struct pvclock_vsyscall_time_info *, hv_clock_per_cpu);



static inline struct pvclock_vcpu_time_info *this_cpu_pvti(void)

static __always_inline struct pvclock_vcpu_time_info *this_cpu_pvti(void)

{

	return &this_cpu_read(hv_clock_per_cpu)->pvti;

}

arch/x86/include/asm/paravirt.h

+1 −1
Original line number Diff line number Diff line
@@ -26,7 +26,7 @@ DECLARE_STATIC_CALL(pv_sched_clock, dummy_sched_clock); 


void paravirt_set_sched_clock(u64 (*func)(void));



static inline u64 paravirt_sched_clock(void)

static __always_inline u64 paravirt_sched_clock(void)

{

	return static_call(pv_sched_clock)();

}

arch/x86/include/asm/pvclock.h

+2 −1
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ 


/* some helper functions for xen and kvm pv clock sources */

u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src);

u64 pvclock_clocksource_read_nowd(struct pvclock_vcpu_time_info *src);

u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src);

void pvclock_set_flags(u8 flags);

unsigned long pvclock_tsc_khz(struct pvclock_vcpu_time_info *src);
@@ -39,7 +40,7 @@ bool pvclock_read_retry(const struct pvclock_vcpu_time_info *src, 
 * Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction,

 * yielding a 64-bit result.

 */

static inline u64 pvclock_scale_delta(u64 delta, u32 mul_frac, int shift)

static __always_inline u64 pvclock_scale_delta(u64 delta, u32 mul_frac, int shift)

{

	u64 product;

#ifdef __i386__

arch/x86/kernel/cpu/vmware.c

+1 −1
Original line number Diff line number Diff line
@@ -143,7 +143,7 @@ static __init int parse_no_stealacc(char *arg) 
}

early_param("no-steal-acc", parse_no_stealacc);



static unsigned long long notrace vmware_sched_clock(void)

static noinstr u64 vmware_sched_clock(void)

{

	unsigned long long ns;

arch/x86/kernel/kvmclock.c

+3 −3
Original line number Diff line number Diff line
@@ -71,12 +71,12 @@ static int kvm_set_wallclock(const struct timespec64 *now) 
	return -ENODEV;

}



static u64 kvm_clock_read(void)

static noinstr u64 kvm_clock_read(void)

{

	u64 ret;



	preempt_disable_notrace();

	ret = pvclock_clocksource_read(this_cpu_pvti());

	ret = pvclock_clocksource_read_nowd(this_cpu_pvti());

	preempt_enable_notrace();

	return ret;

}
@@ -86,7 +86,7 @@ static u64 kvm_clock_get_cycles(struct clocksource *cs) 
	return kvm_clock_read();

}



static u64 kvm_sched_clock_read(void)

static noinstr u64 kvm_sched_clock_read(void)

{

	return kvm_clock_read() - kvm_sched_clock_offset;

}