Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
SUSE:SLE-12-SP2:GA
xen.7653
57a1e64c-x86-time-introduce-and-use-rdtsc_order...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 57a1e64c-x86-time-introduce-and-use-rdtsc_ordered.patch of Package xen.7653
References: bsc#970135 # Commit fa74e70500fd73dd2fc441c7dc00b190fb37cee5 # Date 2016-08-03 14:40:44 +0200 # Author Jan Beulich <jbeulich@suse.com> # Committer Jan Beulich <jbeulich@suse.com> x86/time: introduce and use rdtsc_ordered() Matching Linux commit 03b9730b76 ("x86/asm/tsc: Add rdtsc_ordered() and use it in trivial call sites") and earlier ones it builds upon, let's make sure timing loops don't have their rdtsc()-s re-ordered, as that would harm precision of the result (values were observed to be several hundred clocks off without this adjustment). Signed-off-by: Jan Beulich <jbeulich@suse.com> Tested-by: Dario Faggioli <dario.faggioli@citrix.com> Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com> Tested-by: Joao Martins <joao.m.martins@oracle.com> --- a/xen/arch/x86/apic.c +++ b/xen/arch/x86/apic.c @@ -1160,8 +1160,7 @@ static int __init calibrate_APIC_clock(v /* * We wrapped around just now. Let's start: */ - if (cpu_has_tsc) - rdtscll(t1); + t1 = rdtsc_ordered(); tt1 = apic_read(APIC_TMCCT); /* @@ -1171,8 +1170,7 @@ static int __init calibrate_APIC_clock(v wait_8254_wraparound(); tt2 = apic_read(APIC_TMCCT); - if (cpu_has_tsc) - rdtscll(t2); + t2 = rdtsc_ordered(); /* * The APIC bus clock counter is 32 bits only, it --- a/xen/arch/x86/cpu/amd.c +++ b/xen/arch/x86/cpu/amd.c @@ -464,6 +464,9 @@ static void __devinit init_amd(struct cp wrmsr_amd_safe(0xc001100d, l, h & ~1); } + /* MFENCE stops RDTSC speculation */ + __set_bit(X86_FEATURE_MFENCE_RDTSC, c->x86_capability); + switch(c->x86) { case 0xf ... 0x17: --- a/xen/arch/x86/delay.c +++ b/xen/arch/x86/delay.c @@ -21,10 +21,10 @@ void __udelay(unsigned long usecs) unsigned long ticks = usecs * (cpu_khz / 1000); unsigned long s, e; - rdtscl(s); + s = rdtsc_ordered(); do { rep_nop(); - rdtscl(e); + e = rdtsc_ordered(); } while ((e-s) < ticks); } --- a/xen/arch/x86/smpboot.c +++ b/xen/arch/x86/smpboot.c @@ -142,7 +142,7 @@ static void synchronize_tsc_master(unsig for ( i = 1; i <= 5; i++ ) { - rdtscll(tsc_value); + tsc_value = rdtsc_ordered(); wmb(); atomic_inc(&tsc_count); while ( atomic_read(&tsc_count) != (i<<1) ) --- a/xen/arch/x86/time.c +++ b/xen/arch/x86/time.c @@ -260,10 +260,10 @@ static u64 init_pit_and_calibrate_tsc(vo outb(CALIBRATE_LATCH & 0xff, PIT_CH2); /* LSB of count */ outb(CALIBRATE_LATCH >> 8, PIT_CH2); /* MSB of count */ - rdtscll(start); + start = rdtsc_ordered(); for ( count = 0; (inb(0x61) & 0x20) == 0; count++ ) continue; - rdtscll(end); + end = rdtsc_ordered(); /* Error if the CTC doesn't behave itself. */ if ( count == 0 ) @@ -763,7 +763,7 @@ s_time_t get_s_time_fixed(u64 at_tsc) if ( at_tsc ) tsc = at_tsc; else - rdtscll(tsc); + tsc = rdtsc_ordered(); delta = tsc - t->local_tsc_stamp; now = t->stime_local_stamp + scale_delta(delta, &t->tsc_scale); @@ -958,7 +958,7 @@ int cpu_frequency_change(u64 freq) /* TSC-extrapolated time may be bogus after frequency change. */ /*t->stime_local_stamp = get_s_time();*/ t->stime_local_stamp = t->stime_master_stamp; - rdtscll(curr_tsc); + curr_tsc = rdtsc_ordered(); t->local_tsc_stamp = curr_tsc; set_time_scale(&t->tsc_scale, freq); local_irq_enable(); @@ -1170,16 +1170,13 @@ static void local_time_calibration(void) */ static void check_tsc_warp(unsigned long tsc_khz, unsigned long *max_warp) { -#define rdtsc_barrier() mb() static DEFINE_SPINLOCK(sync_lock); static cycles_t last_tsc; cycles_t start, now, prev, end; int i; - rdtsc_barrier(); - start = get_cycles(); - rdtsc_barrier(); + start = rdtsc_ordered(); /* The measurement runs for 20 msecs: */ end = start + tsc_khz * 20ULL; @@ -1194,9 +1191,7 @@ static void check_tsc_warp(unsigned long */ spin_lock(&sync_lock); prev = last_tsc; - rdtsc_barrier(); - now = get_cycles(); - rdtsc_barrier(); + now = rdtsc_ordered(); last_tsc = now; spin_unlock(&sync_lock); @@ -1294,7 +1289,7 @@ static void time_calibration_tsc_rendezv if ( r->master_stime == 0 ) { r->master_stime = read_platform_stime(); - rdtscll(r->master_tsc_stamp); + r->master_tsc_stamp = rdtsc_ordered(); } atomic_inc(&r->semaphore); @@ -1320,7 +1315,7 @@ static void time_calibration_tsc_rendezv } } - rdtscll(c->local_tsc_stamp); + c->local_tsc_stamp = rdtsc_ordered(); c->stime_local_stamp = get_s_time_fixed(c->local_tsc_stamp); c->stime_master_stamp = r->master_stime; @@ -1350,7 +1345,7 @@ static void time_calibration_std_rendezv mb(); /* receive signal /then/ read r->master_stime */ } - rdtscll(c->local_tsc_stamp); + c->local_tsc_stamp = rdtsc_ordered(); c->stime_local_stamp = get_s_time_fixed(c->local_tsc_stamp); c->stime_master_stamp = r->master_stime; @@ -1385,7 +1380,7 @@ void time_latch_stamps(void) local_irq_save(flags); ap_bringup_ref.master_stime = read_platform_stime(); - rdtscll(tsc); + tsc = rdtsc_ordered(); local_irq_restore(flags); ap_bringup_ref.local_stime = get_s_time_fixed(tsc); @@ -1403,7 +1398,7 @@ void init_percpu_time(void) local_irq_save(flags); now = read_platform_stime(); - rdtscll(tsc); + tsc = rdtsc_ordered(); local_irq_restore(flags); t->stime_master_stamp = now; --- a/xen/include/asm-x86/cpufeature.h +++ b/xen/include/asm-x86/cpufeature.h @@ -63,6 +63,7 @@ /* Other features, Linux-defined mapping, word 3 */ /* This range is used for feature bits which conflict or are synthesized */ +#define X86_FEATURE_MFENCE_RDTSC (3*32+ 7) /* MFENCE synchronizes RDTSC */ #define X86_FEATURE_CONSTANT_TSC (3*32+ 8) /* TSC ticks at a constant rate */ #define X86_FEATURE_NONSTOP_TSC (3*32+ 9) /* TSC does not stop in C states */ #define X86_FEATURE_ARAT (3*32+ 10) /* Always running APIC timer */ --- a/xen/include/asm-x86/msr.h +++ b/xen/include/asm-x86/msr.h @@ -83,6 +83,26 @@ static inline int wrmsr_safe(unsigned in (val) = ((unsigned long)_eax) | (((unsigned long)_edx)<<32); \ } while(0) +static inline uint64_t rdtsc_ordered(void) +{ + uint64_t tsc; + + /* + * The RDTSC instruction is not ordered relative to memory access. + * The Intel SDM and the AMD APM are both vague on this point, but + * empirically an RDTSC instruction can be speculatively executed + * before prior loads. An RDTSC immediately after an appropriate + * barrier appears to be ordered as a normal load, that is, it + * provides the same ordering guarantees as reading from a global + * memory location that some other imaginary CPU is updating + * continuously with a time stamp. + */ + alternative("lfence", "mfence", X86_FEATURE_MFENCE_RDTSC); + rdtscll(tsc); + + return tsc; +} + #define __write_tsc(val) wrmsrl(MSR_IA32_TSC, val) #define write_tsc(val) ({ \ /* Reliable TSCs are in lockstep across all CPUs. We should \
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
