Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
SUSE:SLE-12-SP2:Update
xen.12882
57a1e64c-x86-time-introduce-and-use-rdtsc_order...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 57a1e64c-x86-time-introduce-and-use-rdtsc_ordered.patch of Package xen.12882
References: bsc#970135 # Commit fa74e70500fd73dd2fc441c7dc00b190fb37cee5 # Date 2016-08-03 14:40:44 +0200 # Author Jan Beulich <jbeulich@suse.com> # Committer Jan Beulich <jbeulich@suse.com> x86/time: introduce and use rdtsc_ordered() Matching Linux commit 03b9730b76 ("x86/asm/tsc: Add rdtsc_ordered() and use it in trivial call sites") and earlier ones it builds upon, let's make sure timing loops don't have their rdtsc()-s re-ordered, as that would harm precision of the result (values were observed to be several hundred clocks off without this adjustment). Signed-off-by: Jan Beulich <jbeulich@suse.com> Tested-by: Dario Faggioli <dario.faggioli@citrix.com> Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com> Tested-by: Joao Martins <joao.m.martins@oracle.com> # Commit 7fb0a87d97201f9c3639f85615eacd93110dc1c5 # Date 2016-08-05 18:00:45 +0200 # Author Jan Beulich <jbeulich@suse.com> # Committer Jan Beulich <jbeulich@suse.com> x86/time: also use rdtsc_ordered() in check_tsc_warp() This really was meant to be added in a v2 of what became commit fa74e70500 ("x86/time: introduce and use rdtsc_ordered()"). Signed-off-by: Jan Beulich <jbeulich@suse.com> Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com> --- a/xen/arch/x86/apic.c +++ b/xen/arch/x86/apic.c @@ -1137,7 +1137,7 @@ static int __init calibrate_APIC_clock(v /* * We wrapped around just now. Let's start: */ - t1 = rdtsc(); + t1 = rdtsc_ordered(); tt1 = apic_read(APIC_TMCCT); /* @@ -1147,7 +1147,7 @@ static int __init calibrate_APIC_clock(v wait_8254_wraparound(); tt2 = apic_read(APIC_TMCCT); - t2 = rdtsc(); + t2 = rdtsc_ordered(); /* * The APIC bus clock counter is 32 bits only, it --- a/xen/arch/x86/cpu/amd.c +++ b/xen/arch/x86/cpu/amd.c @@ -611,6 +611,10 @@ static void init_amd(struct cpuinfo_x86 } } + /* MFENCE stops RDTSC speculation */ + if (!cpu_has_lfence_dispatch) + __set_bit(X86_FEATURE_MFENCE_RDTSC, c->x86_capability); + switch(c->x86) { case 0xf ... 0x17: --- a/xen/arch/x86/delay.c +++ b/xen/arch/x86/delay.c @@ -21,10 +21,10 @@ void __udelay(unsigned long usecs) unsigned long ticks = usecs * (cpu_khz / 1000); unsigned long s, e; - s = rdtsc(); + s = rdtsc_ordered(); do { rep_nop(); - e = rdtsc(); + e = rdtsc_ordered(); } while ((e-s) < ticks); } --- a/xen/arch/x86/smpboot.c +++ b/xen/arch/x86/smpboot.c @@ -124,7 +124,7 @@ static void synchronize_tsc_master(unsig for ( i = 1; i <= 5; i++ ) { - tsc_value = rdtsc(); + tsc_value = rdtsc_ordered(); wmb(); atomic_inc(&tsc_count); while ( atomic_read(&tsc_count) != (i<<1) ) --- a/xen/arch/x86/time.c +++ b/xen/arch/x86/time.c @@ -258,10 +258,10 @@ static u64 init_pit_and_calibrate_tsc(vo outb(CALIBRATE_LATCH & 0xff, PIT_CH2); /* LSB of count */ outb(CALIBRATE_LATCH >> 8, PIT_CH2); /* MSB of count */ - start = rdtsc(); + start = rdtsc_ordered(); for ( count = 0; (inb(0x61) & 0x20) == 0; count++ ) continue; - end = rdtsc(); + end = rdtsc_ordered(); /* Error if the CTC doesn't behave itself. */ if ( count == 0 ) @@ -761,7 +761,7 @@ s_time_t get_s_time_fixed(u64 at_tsc) if ( at_tsc ) tsc = at_tsc; else - tsc = rdtsc(); + tsc = rdtsc_ordered(); delta = tsc - t->local_tsc_stamp; now = t->stime_local_stamp + scale_delta(delta, &t->tsc_scale); @@ -934,7 +934,7 @@ int cpu_frequency_change(u64 freq) /* TSC-extrapolated time may be bogus after frequency change. */ /*t->stime_local_stamp = get_s_time();*/ t->stime_local_stamp = t->stime_master_stamp; - curr_tsc = rdtsc(); + curr_tsc = rdtsc_ordered(); t->local_tsc_stamp = curr_tsc; set_time_scale(&t->tsc_scale, freq); local_irq_enable(); @@ -1125,16 +1125,13 @@ static void local_time_calibration(void) */ static void check_tsc_warp(unsigned long tsc_khz, unsigned long *max_warp) { -#define rdtsc_barrier() mb() static DEFINE_SPINLOCK(sync_lock); static cycles_t last_tsc; cycles_t start, now, prev, end; int i; - rdtsc_barrier(); - start = get_cycles(); - rdtsc_barrier(); + start = rdtsc_ordered(); /* The measurement runs for 20 msecs: */ end = start + tsc_khz * 20ULL; @@ -1149,9 +1146,7 @@ static void check_tsc_warp(unsigned long */ spin_lock(&sync_lock); prev = last_tsc; - rdtsc_barrier(); - now = get_cycles(); - rdtsc_barrier(); + now = rdtsc_ordered(); last_tsc = now; spin_unlock(&sync_lock); @@ -1249,7 +1244,7 @@ static void time_calibration_tsc_rendezv if ( r->master_stime == 0 ) { r->master_stime = read_platform_stime(); - r->master_tsc_stamp = rdtsc(); + r->master_tsc_stamp = rdtsc_ordered(); } atomic_inc(&r->semaphore); @@ -1275,7 +1270,7 @@ static void time_calibration_tsc_rendezv } } - c->local_tsc_stamp = rdtsc(); + c->local_tsc_stamp = rdtsc_ordered(); c->stime_local_stamp = get_s_time_fixed(c->local_tsc_stamp); c->stime_master_stamp = r->master_stime; @@ -1305,7 +1300,7 @@ static void time_calibration_std_rendezv mb(); /* receive signal /then/ read r->master_stime */ } - c->local_tsc_stamp = rdtsc(); + c->local_tsc_stamp = rdtsc_ordered(); c->stime_local_stamp = get_s_time_fixed(c->local_tsc_stamp); c->stime_master_stamp = r->master_stime; @@ -1340,7 +1335,7 @@ void time_latch_stamps(void) local_irq_save(flags); ap_bringup_ref.master_stime = read_platform_stime(); - tsc = rdtsc(); + tsc = rdtsc_ordered(); local_irq_restore(flags); ap_bringup_ref.local_stime = get_s_time_fixed(tsc); @@ -1358,7 +1353,7 @@ void init_percpu_time(void) local_irq_save(flags); now = read_platform_stime(); - tsc = rdtsc(); + tsc = rdtsc_ordered(); local_irq_restore(flags); t->stime_master_stamp = now; --- a/xen/include/asm-x86/cpufeature.h +++ b/xen/include/asm-x86/cpufeature.h @@ -27,6 +27,7 @@ XEN_CPUFEATURE(SC_RSB_PV, (FSCAPIN XEN_CPUFEATURE(SC_RSB_HVM, (FSCAPINTS+0)*32+ 17) /* RSB overwrite needed for HVM */ XEN_CPUFEATURE(NO_XPTI, (FSCAPINTS+0)*32+ 18) /* XPTI mitigation not in use */ XEN_CPUFEATURE(SC_MSR_IDLE, (FSCAPINTS+0)*32+ 19) /* (SC_MSR_PV || SC_MSR_HVM) && default_xen_spec_ctrl */ +XEN_CPUFEATURE(MFENCE_RDTSC, (FSCAPINTS+0)*32+ 20) /* MFENCE synchronizes RDTSC */ #define NCAPINTS (FSCAPINTS + 1) /* N 32-bit words worth of info */ --- a/xen/include/asm-x86/msr.h +++ b/xen/include/asm-x86/msr.h @@ -80,6 +80,22 @@ static inline uint64_t rdtsc(void) return ((uint64_t)high << 32) | low; } +static inline uint64_t rdtsc_ordered(void) +{ + /* + * The RDTSC instruction is not ordered relative to memory access. + * The Intel SDM and the AMD APM are both vague on this point, but + * empirically an RDTSC instruction can be speculatively executed + * before prior loads. An RDTSC immediately after an appropriate + * barrier appears to be ordered as a normal load, that is, it + * provides the same ordering guarantees as reading from a global + * memory location that some other imaginary CPU is updating + * continuously with a time stamp. + */ + alternative("lfence", "mfence", X86_FEATURE_MFENCE_RDTSC); + return rdtsc(); +} + #define __write_tsc(val) wrmsrl(MSR_IA32_TSC, val) #define write_tsc(val) ({ \ /* Reliable TSCs are in lockstep across all CPUs. We should \
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
