| .. | .. |
|---|
| 28 | 28 | |
|---|
| 29 | 29 | #include "xen-ops.h" |
|---|
| 30 | 30 | |
|---|
| 31 | | -/* Xen may fire a timer up to this many ns early */ |
|---|
| 31 | +/* Minimum amount of time until next clock event fires */ |
|---|
| 32 | 32 | #define TIMER_SLOP 100000 |
|---|
| 33 | 33 | |
|---|
| 34 | 34 | static u64 xen_sched_clock_offset __read_mostly; |
|---|
| .. | .. |
|---|
| 39 | 39 | struct pvclock_vcpu_time_info *info = |
|---|
| 40 | 40 | &HYPERVISOR_shared_info->vcpu_info[0].time; |
|---|
| 41 | 41 | |
|---|
| 42 | + setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); |
|---|
| 42 | 43 | return pvclock_tsc_khz(info); |
|---|
| 43 | 44 | } |
|---|
| 44 | 45 | |
|---|
| .. | .. |
|---|
| 145 | 146 | .notifier_call = xen_pvclock_gtod_notify, |
|---|
| 146 | 147 | }; |
|---|
| 147 | 148 | |
|---|
| 149 | +static int xen_cs_enable(struct clocksource *cs) |
|---|
| 150 | +{ |
|---|
| 151 | + vclocks_set_used(VDSO_CLOCKMODE_PVCLOCK); |
|---|
| 152 | + return 0; |
|---|
| 153 | +} |
|---|
| 154 | + |
|---|
| 148 | 155 | static struct clocksource xen_clocksource __read_mostly = { |
|---|
| 149 | | - .name = "xen", |
|---|
| 150 | | - .rating = 400, |
|---|
| 151 | | - .read = xen_clocksource_get_cycles, |
|---|
| 152 | | - .mask = ~0, |
|---|
| 153 | | - .flags = CLOCK_SOURCE_IS_CONTINUOUS, |
|---|
| 156 | + .name = "xen", |
|---|
| 157 | + .rating = 400, |
|---|
| 158 | + .read = xen_clocksource_get_cycles, |
|---|
| 159 | + .mask = CLOCKSOURCE_MASK(64), |
|---|
| 160 | + .flags = CLOCK_SOURCE_IS_CONTINUOUS, |
|---|
| 161 | + .enable = xen_cs_enable, |
|---|
| 154 | 162 | }; |
|---|
| 155 | 163 | |
|---|
| 156 | 164 | /* |
|---|
| .. | .. |
|---|
| 212 | 220 | return 0; |
|---|
| 213 | 221 | } |
|---|
| 214 | 222 | |
|---|
| 215 | | -static const struct clock_event_device xen_timerop_clockevent = { |
|---|
| 223 | +static struct clock_event_device xen_timerop_clockevent __ro_after_init = { |
|---|
| 216 | 224 | .name = "xen", |
|---|
| 217 | 225 | .features = CLOCK_EVT_FEAT_ONESHOT, |
|---|
| 218 | 226 | |
|---|
| .. | .. |
|---|
| 273 | 281 | return ret; |
|---|
| 274 | 282 | } |
|---|
| 275 | 283 | |
|---|
| 276 | | -static const struct clock_event_device xen_vcpuop_clockevent = { |
|---|
| 284 | +static struct clock_event_device xen_vcpuop_clockevent __ro_after_init = { |
|---|
| 277 | 285 | .name = "xen", |
|---|
| 278 | 286 | .features = CLOCK_EVT_FEAT_ONESHOT, |
|---|
| 279 | 287 | |
|---|
| .. | .. |
|---|
| 412 | 420 | ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t); |
|---|
| 413 | 421 | |
|---|
| 414 | 422 | /* |
|---|
| 415 | | - * We don't disable VCLOCK_PVCLOCK entirely if it fails to register the |
|---|
| 416 | | - * secondary time info with Xen or if we migrated to a host without the |
|---|
| 417 | | - * necessary flags. On both of these cases what happens is either |
|---|
| 418 | | - * process seeing a zeroed out pvti or seeing no PVCLOCK_TSC_STABLE_BIT |
|---|
| 419 | | - * bit set. Userspace checks the latter and if 0, it discards the data |
|---|
| 420 | | - * in pvti and fallbacks to a system call for a reliable timestamp. |
|---|
| 423 | + * We don't disable VDSO_CLOCKMODE_PVCLOCK entirely if it fails to |
|---|
| 424 | + * register the secondary time info with Xen or if we migrated to a |
|---|
| 425 | + * host without the necessary flags. On both of these cases what |
|---|
| 426 | + * happens is either process seeing a zeroed out pvti or seeing no |
|---|
| 427 | + * PVCLOCK_TSC_STABLE_BIT bit set. Userspace checks the latter and |
|---|
| 428 | + * if 0, it discards the data in pvti and fallbacks to a system |
|---|
| 429 | + * call for a reliable timestamp. |
|---|
| 421 | 430 | */ |
|---|
| 422 | 431 | if (ret != 0) |
|---|
| 423 | 432 | pr_notice("Cannot restore secondary vcpu_time_info (err %d)", |
|---|
| .. | .. |
|---|
| 443 | 452 | |
|---|
| 444 | 453 | ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t); |
|---|
| 445 | 454 | if (ret) { |
|---|
| 446 | | - pr_notice("xen: VCLOCK_PVCLOCK not supported (err %d)\n", ret); |
|---|
| 455 | + pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (err %d)\n", ret); |
|---|
| 447 | 456 | free_page((unsigned long)ti); |
|---|
| 448 | 457 | return; |
|---|
| 449 | 458 | } |
|---|
| .. | .. |
|---|
| 460 | 469 | if (!ret) |
|---|
| 461 | 470 | free_page((unsigned long)ti); |
|---|
| 462 | 471 | |
|---|
| 463 | | - pr_notice("xen: VCLOCK_PVCLOCK not supported (tsc unstable)\n"); |
|---|
| 472 | + pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (tsc unstable)\n"); |
|---|
| 464 | 473 | return; |
|---|
| 465 | 474 | } |
|---|
| 466 | 475 | |
|---|
| 467 | 476 | xen_clock = ti; |
|---|
| 468 | 477 | pvclock_set_pvti_cpu0_va(xen_clock); |
|---|
| 469 | 478 | |
|---|
| 470 | | - xen_clocksource.archdata.vclock_mode = VCLOCK_PVCLOCK; |
|---|
| 479 | + xen_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_PVCLOCK; |
|---|
| 471 | 480 | } |
|---|
| 472 | 481 | |
|---|
| 473 | 482 | static void __init xen_time_init(void) |
|---|
| .. | .. |
|---|
| 519 | 528 | void __init xen_init_time_ops(void) |
|---|
| 520 | 529 | { |
|---|
| 521 | 530 | xen_sched_clock_offset = xen_clocksource_read(); |
|---|
| 522 | | - pv_time_ops = xen_time_ops; |
|---|
| 531 | + pv_ops.time = xen_time_ops; |
|---|
| 523 | 532 | |
|---|
| 524 | 533 | x86_init.timers.timer_init = xen_time_init; |
|---|
| 525 | 534 | x86_init.timers.setup_percpu_clockev = x86_init_noop; |
|---|
| .. | .. |
|---|
| 547 | 556 | |
|---|
| 548 | 557 | void __init xen_hvm_init_time_ops(void) |
|---|
| 549 | 558 | { |
|---|
| 559 | + static bool hvm_time_initialized; |
|---|
| 560 | + |
|---|
| 561 | + if (hvm_time_initialized) |
|---|
| 562 | + return; |
|---|
| 563 | + |
|---|
| 550 | 564 | /* |
|---|
| 551 | 565 | * vector callback is needed otherwise we cannot receive interrupts |
|---|
| 552 | 566 | * on cpu > 0 and at this point we don't know how many cpus are |
|---|
| .. | .. |
|---|
| 556 | 570 | return; |
|---|
| 557 | 571 | |
|---|
| 558 | 572 | if (!xen_feature(XENFEAT_hvm_safe_pvclock)) { |
|---|
| 559 | | - pr_info("Xen doesn't support pvclock on HVM, disable pv timer"); |
|---|
| 573 | + pr_info_once("Xen doesn't support pvclock on HVM, disable pv timer"); |
|---|
| 574 | + return; |
|---|
| 575 | + } |
|---|
| 576 | + |
|---|
| 577 | + /* |
|---|
| 578 | + * Only MAX_VIRT_CPUS 'vcpu_info' are embedded inside 'shared_info'. |
|---|
| 579 | + * The __this_cpu_read(xen_vcpu) is still NULL when Xen HVM guest |
|---|
| 580 | + * boots on vcpu >= MAX_VIRT_CPUS (e.g., kexec), To access |
|---|
| 581 | + * __this_cpu_read(xen_vcpu) via xen_clocksource_read() will panic. |
|---|
| 582 | + * |
|---|
| 583 | + * The xen_hvm_init_time_ops() should be called again later after |
|---|
| 584 | + * __this_cpu_read(xen_vcpu) is available. |
|---|
| 585 | + */ |
|---|
| 586 | + if (!__this_cpu_read(xen_vcpu)) { |
|---|
| 587 | + pr_info("Delay xen_init_time_common() as kernel is running on vcpu=%d\n", |
|---|
| 588 | + xen_vcpu_nr(0)); |
|---|
| 560 | 589 | return; |
|---|
| 561 | 590 | } |
|---|
| 562 | 591 | |
|---|
| 563 | 592 | xen_sched_clock_offset = xen_clocksource_read(); |
|---|
| 564 | | - pv_time_ops = xen_time_ops; |
|---|
| 593 | + pv_ops.time = xen_time_ops; |
|---|
| 565 | 594 | x86_init.timers.setup_percpu_clockev = xen_time_init; |
|---|
| 566 | 595 | x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents; |
|---|
| 567 | 596 | |
|---|
| 568 | 597 | x86_platform.calibrate_tsc = xen_tsc_khz; |
|---|
| 569 | 598 | x86_platform.get_wallclock = xen_get_wallclock; |
|---|
| 570 | 599 | x86_platform.set_wallclock = xen_set_wallclock; |
|---|
| 600 | + |
|---|
| 601 | + hvm_time_initialized = true; |
|---|
| 571 | 602 | } |
|---|
| 572 | 603 | #endif |
|---|
| 604 | + |
|---|
| 605 | +/* Kernel parameter to specify Xen timer slop */ |
|---|
| 606 | +static int __init parse_xen_timer_slop(char *ptr) |
|---|
| 607 | +{ |
|---|
| 608 | + unsigned long slop = memparse(ptr, NULL); |
|---|
| 609 | + |
|---|
| 610 | + xen_timerop_clockevent.min_delta_ns = slop; |
|---|
| 611 | + xen_timerop_clockevent.min_delta_ticks = slop; |
|---|
| 612 | + xen_vcpuop_clockevent.min_delta_ns = slop; |
|---|
| 613 | + xen_vcpuop_clockevent.min_delta_ticks = slop; |
|---|
| 614 | + |
|---|
| 615 | + return 0; |
|---|
| 616 | +} |
|---|
| 617 | +early_param("xen_timer_slop", parse_xen_timer_slop); |
|---|