| .. | .. |
|---|
| 7 | 7 | /* Forward declaration, a strange C thing */ |
|---|
| 8 | 8 | struct task_struct; |
|---|
| 9 | 9 | struct mm_struct; |
|---|
| 10 | +struct io_bitmap; |
|---|
| 10 | 11 | struct vm86; |
|---|
| 11 | 12 | |
|---|
| 12 | 13 | #include <asm/math_emu.h> |
|---|
| .. | .. |
|---|
| 24 | 25 | #include <asm/special_insns.h> |
|---|
| 25 | 26 | #include <asm/fpu/types.h> |
|---|
| 26 | 27 | #include <asm/unwind_hints.h> |
|---|
| 28 | +#include <asm/vmxfeatures.h> |
|---|
| 27 | 29 | #include <asm/vdso/processor.h> |
|---|
| 28 | 30 | |
|---|
| 29 | 31 | #include <linux/personality.h> |
|---|
| .. | .. |
|---|
| 43 | 45 | #define NET_IP_ALIGN 0 |
|---|
| 44 | 46 | |
|---|
| 45 | 47 | #define HBP_NUM 4 |
|---|
| 46 | | -/* |
|---|
| 47 | | - * Default implementation of macro that returns current |
|---|
| 48 | | - * instruction pointer ("program counter"). |
|---|
| 49 | | - */ |
|---|
| 50 | | -static inline void *current_text_addr(void) |
|---|
| 51 | | -{ |
|---|
| 52 | | - void *pc; |
|---|
| 53 | | - |
|---|
| 54 | | - asm volatile("mov $1f, %0; 1:":"=r" (pc)); |
|---|
| 55 | | - |
|---|
| 56 | | - return pc; |
|---|
| 57 | | -} |
|---|
| 58 | 48 | |
|---|
| 59 | 49 | /* |
|---|
| 60 | 50 | * These alignment constraints are for performance in the vSMP case, |
|---|
| .. | .. |
|---|
| 97 | 87 | /* Number of 4K pages in DTLB/ITLB combined(in pages): */ |
|---|
| 98 | 88 | int x86_tlbsize; |
|---|
| 99 | 89 | #endif |
|---|
| 90 | +#ifdef CONFIG_X86_VMX_FEATURE_NAMES |
|---|
| 91 | + __u32 vmx_capability[NVMXINTS]; |
|---|
| 92 | +#endif |
|---|
| 100 | 93 | __u8 x86_virt_bits; |
|---|
| 101 | 94 | __u8 x86_phys_bits; |
|---|
| 102 | 95 | /* CPUID returned core id bits: */ |
|---|
| .. | .. |
|---|
| 106 | 99 | __u32 extended_cpuid_level; |
|---|
| 107 | 100 | /* Maximum supported CPUID level, -1=no CPUID: */ |
|---|
| 108 | 101 | int cpuid_level; |
|---|
| 109 | | - __u32 x86_capability[NCAPINTS + NBUGINTS]; |
|---|
| 102 | + /* |
|---|
| 103 | + * Align to size of unsigned long because the x86_capability array |
|---|
| 104 | + * is passed to bitops which require the alignment. Use unnamed |
|---|
| 105 | + * union to enforce the array is aligned to size of unsigned long. |
|---|
| 106 | + */ |
|---|
| 107 | + union { |
|---|
| 108 | + __u32 x86_capability[NCAPINTS + NBUGINTS]; |
|---|
| 109 | + unsigned long x86_capability_alignment; |
|---|
| 110 | + }; |
|---|
| 110 | 111 | char x86_vendor_id[16]; |
|---|
| 111 | 112 | char x86_model_id[64]; |
|---|
| 112 | 113 | /* in KB - valid for CPUS which support this call: */ |
|---|
| 113 | 114 | unsigned int x86_cache_size; |
|---|
| 114 | 115 | int x86_cache_alignment; /* In bytes */ |
|---|
| 115 | | - /* Cache QoS architectural values: */ |
|---|
| 116 | + /* Cache QoS architectural values, valid only on the BSP: */ |
|---|
| 116 | 117 | int x86_cache_max_rmid; /* max index */ |
|---|
| 117 | 118 | int x86_cache_occ_scale; /* scale to bytes */ |
|---|
| 119 | + int x86_cache_mbm_width_offset; |
|---|
| 118 | 120 | int x86_power; |
|---|
| 119 | 121 | unsigned long loops_per_jiffy; |
|---|
| 120 | 122 | /* cpuid returned max cores value: */ |
|---|
| 121 | | - u16 x86_max_cores; |
|---|
| 123 | + u16 x86_max_cores; |
|---|
| 122 | 124 | u16 apicid; |
|---|
| 123 | 125 | u16 initial_apicid; |
|---|
| 124 | 126 | u16 x86_clflush_size; |
|---|
| .. | .. |
|---|
| 130 | 132 | u16 logical_proc_id; |
|---|
| 131 | 133 | /* Core id: */ |
|---|
| 132 | 134 | u16 cpu_core_id; |
|---|
| 135 | + u16 cpu_die_id; |
|---|
| 136 | + u16 logical_die_id; |
|---|
| 133 | 137 | /* Index into per_cpu list: */ |
|---|
| 134 | 138 | u16 cpu_index; |
|---|
| 135 | 139 | u32 microcode; |
|---|
| .. | .. |
|---|
| 156 | 160 | #define X86_VENDOR_CENTAUR 5 |
|---|
| 157 | 161 | #define X86_VENDOR_TRANSMETA 7 |
|---|
| 158 | 162 | #define X86_VENDOR_NSC 8 |
|---|
| 159 | | -#define X86_VENDOR_NUM 9 |
|---|
| 163 | +#define X86_VENDOR_HYGON 9 |
|---|
| 164 | +#define X86_VENDOR_ZHAOXIN 10 |
|---|
| 165 | +#define X86_VENDOR_NUM 11 |
|---|
| 160 | 166 | |
|---|
| 161 | 167 | #define X86_VENDOR_UNKNOWN 0xff |
|---|
| 162 | 168 | |
|---|
| .. | .. |
|---|
| 166 | 172 | extern struct cpuinfo_x86 boot_cpu_data; |
|---|
| 167 | 173 | extern struct cpuinfo_x86 new_cpu_data; |
|---|
| 168 | 174 | |
|---|
| 169 | | -extern struct x86_hw_tss doublefault_tss; |
|---|
| 170 | 175 | extern __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS]; |
|---|
| 171 | 176 | extern __u32 cpu_caps_set[NCAPINTS + NBUGINTS]; |
|---|
| 172 | 177 | |
|---|
| .. | .. |
|---|
| 316 | 321 | */ |
|---|
| 317 | 322 | u64 sp1; |
|---|
| 318 | 323 | |
|---|
| 324 | + /* |
|---|
| 325 | + * Since Linux does not use ring 2, the 'sp2' slot is unused by |
|---|
| 326 | + * hardware. entry_SYSCALL_64 uses it as scratch space to stash |
|---|
| 327 | + * the user RSP value. |
|---|
| 328 | + */ |
|---|
| 319 | 329 | u64 sp2; |
|---|
| 330 | + |
|---|
| 320 | 331 | u64 reserved2; |
|---|
| 321 | 332 | u64 ist[7]; |
|---|
| 322 | 333 | u32 reserved3; |
|---|
| .. | .. |
|---|
| 331 | 342 | * IO-bitmap sizes: |
|---|
| 332 | 343 | */ |
|---|
| 333 | 344 | #define IO_BITMAP_BITS 65536 |
|---|
| 334 | | -#define IO_BITMAP_BYTES (IO_BITMAP_BITS/8) |
|---|
| 335 | | -#define IO_BITMAP_LONGS (IO_BITMAP_BYTES/sizeof(long)) |
|---|
| 336 | | -#define IO_BITMAP_OFFSET (offsetof(struct tss_struct, io_bitmap) - offsetof(struct tss_struct, x86_tss)) |
|---|
| 337 | | -#define INVALID_IO_BITMAP_OFFSET 0x8000 |
|---|
| 345 | +#define IO_BITMAP_BYTES (IO_BITMAP_BITS / BITS_PER_BYTE) |
|---|
| 346 | +#define IO_BITMAP_LONGS (IO_BITMAP_BYTES / sizeof(long)) |
|---|
| 347 | + |
|---|
| 348 | +#define IO_BITMAP_OFFSET_VALID_MAP \ |
|---|
| 349 | + (offsetof(struct tss_struct, io_bitmap.bitmap) - \ |
|---|
| 350 | + offsetof(struct tss_struct, x86_tss)) |
|---|
| 351 | + |
|---|
| 352 | +#define IO_BITMAP_OFFSET_VALID_ALL \ |
|---|
| 353 | + (offsetof(struct tss_struct, io_bitmap.mapall) - \ |
|---|
| 354 | + offsetof(struct tss_struct, x86_tss)) |
|---|
| 355 | + |
|---|
| 356 | +#ifdef CONFIG_X86_IOPL_IOPERM |
|---|
| 357 | +/* |
|---|
| 358 | + * sizeof(unsigned long) coming from an extra "long" at the end of the |
|---|
| 359 | + * iobitmap. The limit is inclusive, i.e. the last valid byte. |
|---|
| 360 | + */ |
|---|
| 361 | +# define __KERNEL_TSS_LIMIT \ |
|---|
| 362 | + (IO_BITMAP_OFFSET_VALID_ALL + IO_BITMAP_BYTES + \ |
|---|
| 363 | + sizeof(unsigned long) - 1) |
|---|
| 364 | +#else |
|---|
| 365 | +# define __KERNEL_TSS_LIMIT \ |
|---|
| 366 | + (offsetof(struct tss_struct, x86_tss) + sizeof(struct x86_hw_tss) - 1) |
|---|
| 367 | +#endif |
|---|
| 368 | + |
|---|
| 369 | +/* Base offset outside of TSS_LIMIT so unpriviledged IO causes #GP */ |
|---|
| 370 | +#define IO_BITMAP_OFFSET_INVALID (__KERNEL_TSS_LIMIT + 1) |
|---|
| 338 | 371 | |
|---|
| 339 | 372 | struct entry_stack { |
|---|
| 340 | 373 | char stack[PAGE_SIZE]; |
|---|
| .. | .. |
|---|
| 344 | 377 | struct entry_stack stack; |
|---|
| 345 | 378 | } __aligned(PAGE_SIZE); |
|---|
| 346 | 379 | |
|---|
| 380 | +/* |
|---|
| 381 | + * All IO bitmap related data stored in the TSS: |
|---|
| 382 | + */ |
|---|
| 383 | +struct x86_io_bitmap { |
|---|
| 384 | + /* The sequence number of the last active bitmap. */ |
|---|
| 385 | + u64 prev_sequence; |
|---|
| 386 | + |
|---|
| 387 | + /* |
|---|
| 388 | + * Store the dirty size of the last io bitmap offender. The next |
|---|
| 389 | + * one will have to do the cleanup as the switch out to a non io |
|---|
| 390 | + * bitmap user will just set x86_tss.io_bitmap_base to a value |
|---|
| 391 | + * outside of the TSS limit. So for sane tasks there is no need to |
|---|
| 392 | + * actually touch the io_bitmap at all. |
|---|
| 393 | + */ |
|---|
| 394 | + unsigned int prev_max; |
|---|
| 395 | + |
|---|
| 396 | + /* |
|---|
| 397 | + * The extra 1 is there because the CPU will access an |
|---|
| 398 | + * additional byte beyond the end of the IO permission |
|---|
| 399 | + * bitmap. The extra byte must be all 1 bits, and must |
|---|
| 400 | + * be within the limit. |
|---|
| 401 | + */ |
|---|
| 402 | + unsigned long bitmap[IO_BITMAP_LONGS + 1]; |
|---|
| 403 | + |
|---|
| 404 | + /* |
|---|
| 405 | + * Special I/O bitmap to emulate IOPL(3). All bytes zero, |
|---|
| 406 | + * except the additional byte at the end. |
|---|
| 407 | + */ |
|---|
| 408 | + unsigned long mapall[IO_BITMAP_LONGS + 1]; |
|---|
| 409 | +}; |
|---|
| 410 | + |
|---|
| 347 | 411 | struct tss_struct { |
|---|
| 348 | 412 | /* |
|---|
| 349 | 413 | * The fixed hardware portion. This must not cross a page boundary |
|---|
| .. | .. |
|---|
| 352 | 416 | */ |
|---|
| 353 | 417 | struct x86_hw_tss x86_tss; |
|---|
| 354 | 418 | |
|---|
| 355 | | - /* |
|---|
| 356 | | - * The extra 1 is there because the CPU will access an |
|---|
| 357 | | - * additional byte beyond the end of the IO permission |
|---|
| 358 | | - * bitmap. The extra byte must be all 1 bits, and must |
|---|
| 359 | | - * be within the limit. |
|---|
| 360 | | - */ |
|---|
| 361 | | - unsigned long io_bitmap[IO_BITMAP_LONGS + 1]; |
|---|
| 419 | + struct x86_io_bitmap io_bitmap; |
|---|
| 362 | 420 | } __aligned(PAGE_SIZE); |
|---|
| 363 | 421 | |
|---|
| 364 | 422 | DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw); |
|---|
| 365 | 423 | |
|---|
| 366 | | -/* |
|---|
| 367 | | - * sizeof(unsigned long) coming from an extra "long" at the end |
|---|
| 368 | | - * of the iobitmap. |
|---|
| 369 | | - * |
|---|
| 370 | | - * -1? seg base+limit should be pointing to the address of the |
|---|
| 371 | | - * last valid byte |
|---|
| 372 | | - */ |
|---|
| 373 | | -#define __KERNEL_TSS_LIMIT \ |
|---|
| 374 | | - (IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1) |
|---|
| 424 | +/* Per CPU interrupt stacks */ |
|---|
| 425 | +struct irq_stack { |
|---|
| 426 | + char stack[IRQ_STACK_SIZE]; |
|---|
| 427 | +} __aligned(IRQ_STACK_SIZE); |
|---|
| 428 | + |
|---|
| 429 | +DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); |
|---|
| 375 | 430 | |
|---|
| 376 | 431 | #ifdef CONFIG_X86_32 |
|---|
| 377 | 432 | DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack); |
|---|
| .. | .. |
|---|
| 380 | 435 | #define cpu_current_top_of_stack cpu_tss_rw.x86_tss.sp1 |
|---|
| 381 | 436 | #endif |
|---|
| 382 | 437 | |
|---|
| 383 | | -/* |
|---|
| 384 | | - * Save the original ist values for checking stack pointers during debugging |
|---|
| 385 | | - */ |
|---|
| 386 | | -struct orig_ist { |
|---|
| 387 | | - unsigned long ist[7]; |
|---|
| 388 | | -}; |
|---|
| 389 | | - |
|---|
| 390 | 438 | #ifdef CONFIG_X86_64 |
|---|
| 391 | | -DECLARE_PER_CPU(struct orig_ist, orig_ist); |
|---|
| 392 | | - |
|---|
| 393 | | -union irq_stack_union { |
|---|
| 394 | | - char irq_stack[IRQ_STACK_SIZE]; |
|---|
| 439 | +struct fixed_percpu_data { |
|---|
| 395 | 440 | /* |
|---|
| 396 | 441 | * GCC hardcodes the stack canary as %gs:40. Since the |
|---|
| 397 | 442 | * irq_stack is the object at %gs:0, we reserve the bottom |
|---|
| 398 | 443 | * 48 bytes of the irq stack for the canary. |
|---|
| 399 | 444 | */ |
|---|
| 400 | | - struct { |
|---|
| 401 | | - char gs_base[40]; |
|---|
| 402 | | - unsigned long stack_canary; |
|---|
| 403 | | - }; |
|---|
| 445 | + char gs_base[40]; |
|---|
| 446 | + unsigned long stack_canary; |
|---|
| 404 | 447 | }; |
|---|
| 405 | 448 | |
|---|
| 406 | | -DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible; |
|---|
| 407 | | -DECLARE_INIT_PER_CPU(irq_stack_union); |
|---|
| 449 | +DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible; |
|---|
| 450 | +DECLARE_INIT_PER_CPU(fixed_percpu_data); |
|---|
| 408 | 451 | |
|---|
| 409 | 452 | static inline unsigned long cpu_kernelmode_gs_base(int cpu) |
|---|
| 410 | 453 | { |
|---|
| 411 | | - return (unsigned long)per_cpu(irq_stack_union.gs_base, cpu); |
|---|
| 454 | + return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu); |
|---|
| 412 | 455 | } |
|---|
| 413 | 456 | |
|---|
| 414 | | -DECLARE_PER_CPU(char *, irq_stack_ptr); |
|---|
| 415 | 457 | DECLARE_PER_CPU(unsigned int, irq_count); |
|---|
| 416 | 458 | extern asmlinkage void ignore_sysret(void); |
|---|
| 417 | 459 | |
|---|
| 418 | | -#if IS_ENABLED(CONFIG_KVM) |
|---|
| 419 | 460 | /* Save actual FS/GS selectors and bases to current->thread */ |
|---|
| 420 | | -void save_fsgs_for_kvm(void); |
|---|
| 421 | | -#endif |
|---|
| 461 | +void current_save_fsgs(void); |
|---|
| 422 | 462 | #else /* X86_64 */ |
|---|
| 423 | 463 | #ifdef CONFIG_STACKPROTECTOR |
|---|
| 424 | 464 | /* |
|---|
| .. | .. |
|---|
| 433 | 473 | }; |
|---|
| 434 | 474 | DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary); |
|---|
| 435 | 475 | #endif |
|---|
| 436 | | -/* |
|---|
| 437 | | - * per-CPU IRQ handling stacks |
|---|
| 438 | | - */ |
|---|
| 439 | | -struct irq_stack { |
|---|
| 440 | | - u32 stack[THREAD_SIZE/sizeof(u32)]; |
|---|
| 441 | | -} __aligned(THREAD_SIZE); |
|---|
| 442 | | - |
|---|
| 443 | | -DECLARE_PER_CPU(struct irq_stack *, hardirq_stack); |
|---|
| 444 | | -DECLARE_PER_CPU(struct irq_stack *, softirq_stack); |
|---|
| 476 | +/* Per CPU softirq stack pointer */ |
|---|
| 477 | +DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr); |
|---|
| 445 | 478 | #endif /* X86_64 */ |
|---|
| 446 | 479 | |
|---|
| 447 | 480 | extern unsigned int fpu_kernel_xstate_size; |
|---|
| 448 | 481 | extern unsigned int fpu_user_xstate_size; |
|---|
| 449 | 482 | |
|---|
| 450 | 483 | struct perf_event; |
|---|
| 451 | | - |
|---|
| 452 | | -typedef struct { |
|---|
| 453 | | - unsigned long seg; |
|---|
| 454 | | -} mm_segment_t; |
|---|
| 455 | 484 | |
|---|
| 456 | 485 | struct thread_struct { |
|---|
| 457 | 486 | /* Cached TLS descriptors: */ |
|---|
| .. | .. |
|---|
| 484 | 513 | /* Save middle states of ptrace breakpoints */ |
|---|
| 485 | 514 | struct perf_event *ptrace_bps[HBP_NUM]; |
|---|
| 486 | 515 | /* Debug status used for traps, single steps, etc... */ |
|---|
| 487 | | - unsigned long debugreg6; |
|---|
| 516 | + unsigned long virtual_dr6; |
|---|
| 488 | 517 | /* Keep track of the exact dr7 value set by the user */ |
|---|
| 489 | 518 | unsigned long ptrace_dr7; |
|---|
| 490 | 519 | /* Fault info: */ |
|---|
| .. | .. |
|---|
| 496 | 525 | struct vm86 *vm86; |
|---|
| 497 | 526 | #endif |
|---|
| 498 | 527 | /* IO permissions: */ |
|---|
| 499 | | - unsigned long *io_bitmap_ptr; |
|---|
| 500 | | - unsigned long iopl; |
|---|
| 501 | | - /* Max allowed port in the bitmap, in bytes: */ |
|---|
| 502 | | - unsigned io_bitmap_max; |
|---|
| 528 | + struct io_bitmap *io_bitmap; |
|---|
| 503 | 529 | |
|---|
| 504 | | - mm_segment_t addr_limit; |
|---|
| 530 | + /* |
|---|
| 531 | + * IOPL. Priviledge level dependent I/O permission which is |
|---|
| 532 | + * emulated via the I/O bitmap to prevent user space from disabling |
|---|
| 533 | + * interrupts. |
|---|
| 534 | + */ |
|---|
| 535 | + unsigned long iopl_emul; |
|---|
| 505 | 536 | |
|---|
| 537 | + unsigned int iopl_warn:1; |
|---|
| 506 | 538 | unsigned int sig_on_uaccess_err:1; |
|---|
| 507 | | - unsigned int uaccess_err:1; /* uaccess failed */ |
|---|
| 508 | 539 | |
|---|
| 509 | 540 | /* Floating point and extended processor state */ |
|---|
| 510 | 541 | struct fpu fpu; |
|---|
| .. | .. |
|---|
| 522 | 553 | *size = fpu_kernel_xstate_size; |
|---|
| 523 | 554 | } |
|---|
| 524 | 555 | |
|---|
| 525 | | -/* |
|---|
| 526 | | - * Set IOPL bits in EFLAGS from given mask |
|---|
| 527 | | - */ |
|---|
| 528 | | -static inline void native_set_iopl_mask(unsigned mask) |
|---|
| 529 | | -{ |
|---|
| 530 | | -#ifdef CONFIG_X86_32 |
|---|
| 531 | | - unsigned int reg; |
|---|
| 532 | | - |
|---|
| 533 | | - asm volatile ("pushfl;" |
|---|
| 534 | | - "popl %0;" |
|---|
| 535 | | - "andl %1, %0;" |
|---|
| 536 | | - "orl %2, %0;" |
|---|
| 537 | | - "pushl %0;" |
|---|
| 538 | | - "popfl" |
|---|
| 539 | | - : "=&r" (reg) |
|---|
| 540 | | - : "i" (~X86_EFLAGS_IOPL), "r" (mask)); |
|---|
| 541 | | -#endif |
|---|
| 542 | | -} |
|---|
| 543 | | - |
|---|
| 544 | 556 | static inline void |
|---|
| 545 | 557 | native_load_sp0(unsigned long sp0) |
|---|
| 546 | 558 | { |
|---|
| 547 | 559 | this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0); |
|---|
| 548 | 560 | } |
|---|
| 549 | 561 | |
|---|
| 550 | | -static inline void native_swapgs(void) |
|---|
| 562 | +static __always_inline void native_swapgs(void) |
|---|
| 551 | 563 | { |
|---|
| 552 | 564 | #ifdef CONFIG_X86_64 |
|---|
| 553 | 565 | asm volatile("swapgs" ::: "memory"); |
|---|
| .. | .. |
|---|
| 570 | 582 | current_stack_pointer) < THREAD_SIZE; |
|---|
| 571 | 583 | } |
|---|
| 572 | 584 | |
|---|
| 573 | | -#ifdef CONFIG_PARAVIRT |
|---|
| 585 | +#ifdef CONFIG_PARAVIRT_XXL |
|---|
| 574 | 586 | #include <asm/paravirt.h> |
|---|
| 575 | 587 | #else |
|---|
| 576 | 588 | #define __cpuid native_cpuid |
|---|
| .. | .. |
|---|
| 580 | 592 | native_load_sp0(sp0); |
|---|
| 581 | 593 | } |
|---|
| 582 | 594 | |
|---|
| 583 | | -#define set_iopl_mask native_set_iopl_mask |
|---|
| 584 | | -#endif /* CONFIG_PARAVIRT */ |
|---|
| 595 | +#endif /* CONFIG_PARAVIRT_XXL */ |
|---|
| 585 | 596 | |
|---|
| 586 | 597 | /* Free all resources held by a thread. */ |
|---|
| 587 | 598 | extern void release_thread(struct task_struct *); |
|---|
| .. | .. |
|---|
| 651 | 662 | return edx; |
|---|
| 652 | 663 | } |
|---|
| 653 | 664 | |
|---|
| 654 | | -/* |
|---|
| 655 | | - * This function forces the icache and prefetched instruction stream to |
|---|
| 656 | | - * catch up with reality in two very specific cases: |
|---|
| 657 | | - * |
|---|
| 658 | | - * a) Text was modified using one virtual address and is about to be executed |
|---|
| 659 | | - * from the same physical page at a different virtual address. |
|---|
| 660 | | - * |
|---|
| 661 | | - * b) Text was modified on a different CPU, may subsequently be |
|---|
| 662 | | - * executed on this CPU, and you want to make sure the new version |
|---|
| 663 | | - * gets executed. This generally means you're calling this in a IPI. |
|---|
| 664 | | - * |
|---|
| 665 | | - * If you're calling this for a different reason, you're probably doing |
|---|
| 666 | | - * it wrong. |
|---|
| 667 | | - */ |
|---|
| 668 | | -static inline void sync_core(void) |
|---|
| 669 | | -{ |
|---|
| 670 | | - /* |
|---|
| 671 | | - * There are quite a few ways to do this. IRET-to-self is nice |
|---|
| 672 | | - * because it works on every CPU, at any CPL (so it's compatible |
|---|
| 673 | | - * with paravirtualization), and it never exits to a hypervisor. |
|---|
| 674 | | - * The only down sides are that it's a bit slow (it seems to be |
|---|
| 675 | | - * a bit more than 2x slower than the fastest options) and that |
|---|
| 676 | | - * it unmasks NMIs. The "push %cs" is needed because, in |
|---|
| 677 | | - * paravirtual environments, __KERNEL_CS may not be a valid CS |
|---|
| 678 | | - * value when we do IRET directly. |
|---|
| 679 | | - * |
|---|
| 680 | | - * In case NMI unmasking or performance ever becomes a problem, |
|---|
| 681 | | - * the next best option appears to be MOV-to-CR2 and an |
|---|
| 682 | | - * unconditional jump. That sequence also works on all CPUs, |
|---|
| 683 | | - * but it will fault at CPL3 (i.e. Xen PV). |
|---|
| 684 | | - * |
|---|
| 685 | | - * CPUID is the conventional way, but it's nasty: it doesn't |
|---|
| 686 | | - * exist on some 486-like CPUs, and it usually exits to a |
|---|
| 687 | | - * hypervisor. |
|---|
| 688 | | - * |
|---|
| 689 | | - * Like all of Linux's memory ordering operations, this is a |
|---|
| 690 | | - * compiler barrier as well. |
|---|
| 691 | | - */ |
|---|
| 692 | | -#ifdef CONFIG_X86_32 |
|---|
| 693 | | - asm volatile ( |
|---|
| 694 | | - "pushfl\n\t" |
|---|
| 695 | | - "pushl %%cs\n\t" |
|---|
| 696 | | - "pushl $1f\n\t" |
|---|
| 697 | | - "iret\n\t" |
|---|
| 698 | | - "1:" |
|---|
| 699 | | - : ASM_CALL_CONSTRAINT : : "memory"); |
|---|
| 700 | | -#else |
|---|
| 701 | | - unsigned int tmp; |
|---|
| 702 | | - |
|---|
| 703 | | - asm volatile ( |
|---|
| 704 | | - UNWIND_HINT_SAVE |
|---|
| 705 | | - "mov %%ss, %0\n\t" |
|---|
| 706 | | - "pushq %q0\n\t" |
|---|
| 707 | | - "pushq %%rsp\n\t" |
|---|
| 708 | | - "addq $8, (%%rsp)\n\t" |
|---|
| 709 | | - "pushfq\n\t" |
|---|
| 710 | | - "mov %%cs, %0\n\t" |
|---|
| 711 | | - "pushq %q0\n\t" |
|---|
| 712 | | - "pushq $1f\n\t" |
|---|
| 713 | | - "iretq\n\t" |
|---|
| 714 | | - UNWIND_HINT_RESTORE |
|---|
| 715 | | - "1:" |
|---|
| 716 | | - : "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory"); |
|---|
| 717 | | -#endif |
|---|
| 718 | | -} |
|---|
| 719 | | - |
|---|
| 720 | 665 | extern void select_idle_routine(const struct cpuinfo_x86 *c); |
|---|
| 721 | 666 | extern void amd_e400_c1e_apic_setup(void); |
|---|
| 722 | 667 | |
|---|
| .. | .. |
|---|
| 728 | 673 | extern void enable_sep_cpu(void); |
|---|
| 729 | 674 | extern int sysenter_setup(void); |
|---|
| 730 | 675 | |
|---|
| 731 | | -void early_trap_pf_init(void); |
|---|
| 732 | 676 | |
|---|
| 733 | 677 | /* Defined in head.S */ |
|---|
| 734 | 678 | extern struct desc_ptr early_gdt_descr; |
|---|
| .. | .. |
|---|
| 738 | 682 | extern void load_fixmap_gdt(int); |
|---|
| 739 | 683 | extern void load_percpu_segment(int); |
|---|
| 740 | 684 | extern void cpu_init(void); |
|---|
| 685 | +extern void cpu_init_secondary(void); |
|---|
| 686 | +extern void cpu_init_exception_handling(void); |
|---|
| 687 | +extern void cr4_init(void); |
|---|
| 741 | 688 | |
|---|
| 742 | 689 | static inline unsigned long get_debugctlmsr(void) |
|---|
| 743 | 690 | { |
|---|
| .. | .. |
|---|
| 798 | 745 | * Useful for spinlocks to avoid one state transition in the |
|---|
| 799 | 746 | * cache coherency protocol: |
|---|
| 800 | 747 | */ |
|---|
| 801 | | -static inline void prefetchw(const void *x) |
|---|
| 748 | +static __always_inline void prefetchw(const void *x) |
|---|
| 802 | 749 | { |
|---|
| 803 | 750 | alternative_input(BASE_PREFETCH, "prefetchw %P1", |
|---|
| 804 | 751 | X86_FEATURE_3DNOWPREFETCH, |
|---|
| .. | .. |
|---|
| 823 | 770 | }) |
|---|
| 824 | 771 | |
|---|
| 825 | 772 | #ifdef CONFIG_X86_32 |
|---|
| 826 | | -/* |
|---|
| 827 | | - * User space process size: 3GB (default). |
|---|
| 828 | | - */ |
|---|
| 829 | | -#define IA32_PAGE_OFFSET PAGE_OFFSET |
|---|
| 830 | | -#define TASK_SIZE PAGE_OFFSET |
|---|
| 831 | | -#define TASK_SIZE_LOW TASK_SIZE |
|---|
| 832 | | -#define TASK_SIZE_MAX TASK_SIZE |
|---|
| 833 | | -#define DEFAULT_MAP_WINDOW TASK_SIZE |
|---|
| 834 | | -#define STACK_TOP TASK_SIZE |
|---|
| 835 | | -#define STACK_TOP_MAX STACK_TOP |
|---|
| 836 | | - |
|---|
| 837 | 773 | #define INIT_THREAD { \ |
|---|
| 838 | 774 | .sp0 = TOP_OF_INIT_STACK, \ |
|---|
| 839 | 775 | .sysenter_cs = __KERNEL_CS, \ |
|---|
| 840 | | - .io_bitmap_ptr = NULL, \ |
|---|
| 841 | | - .addr_limit = KERNEL_DS, \ |
|---|
| 842 | 776 | } |
|---|
| 843 | 777 | |
|---|
| 844 | 778 | #define KSTK_ESP(task) (task_pt_regs(task)->sp) |
|---|
| 845 | 779 | |
|---|
| 846 | 780 | #else |
|---|
| 847 | | -/* |
|---|
| 848 | | - * User space process size. This is the first address outside the user range. |
|---|
| 849 | | - * There are a few constraints that determine this: |
|---|
| 850 | | - * |
|---|
| 851 | | - * On Intel CPUs, if a SYSCALL instruction is at the highest canonical |
|---|
| 852 | | - * address, then that syscall will enter the kernel with a |
|---|
| 853 | | - * non-canonical return address, and SYSRET will explode dangerously. |
|---|
| 854 | | - * We avoid this particular problem by preventing anything executable |
|---|
| 855 | | - * from being mapped at the maximum canonical address. |
|---|
| 856 | | - * |
|---|
| 857 | | - * On AMD CPUs in the Ryzen family, there's a nasty bug in which the |
|---|
| 858 | | - * CPUs malfunction if they execute code from the highest canonical page. |
|---|
| 859 | | - * They'll speculate right off the end of the canonical space, and |
|---|
| 860 | | - * bad things happen. This is worked around in the same way as the |
|---|
| 861 | | - * Intel problem. |
|---|
| 862 | | - * |
|---|
| 863 | | - * With page table isolation enabled, we map the LDT in ... [stay tuned] |
|---|
| 864 | | - */ |
|---|
| 865 | | -#define TASK_SIZE_MAX ((1UL << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE) |
|---|
| 866 | | - |
|---|
| 867 | | -#define DEFAULT_MAP_WINDOW ((1UL << 47) - PAGE_SIZE) |
|---|
| 868 | | - |
|---|
| 869 | | -/* This decides where the kernel will search for a free chunk of vm |
|---|
| 870 | | - * space during mmap's. |
|---|
| 871 | | - */ |
|---|
| 872 | | -#define IA32_PAGE_OFFSET ((current->personality & ADDR_LIMIT_3GB) ? \ |
|---|
| 873 | | - 0xc0000000 : 0xFFFFe000) |
|---|
| 874 | | - |
|---|
| 875 | | -#define TASK_SIZE_LOW (test_thread_flag(TIF_ADDR32) ? \ |
|---|
| 876 | | - IA32_PAGE_OFFSET : DEFAULT_MAP_WINDOW) |
|---|
| 877 | | -#define TASK_SIZE (test_thread_flag(TIF_ADDR32) ? \ |
|---|
| 878 | | - IA32_PAGE_OFFSET : TASK_SIZE_MAX) |
|---|
| 879 | | -#define TASK_SIZE_OF(child) ((test_tsk_thread_flag(child, TIF_ADDR32)) ? \ |
|---|
| 880 | | - IA32_PAGE_OFFSET : TASK_SIZE_MAX) |
|---|
| 881 | | - |
|---|
| 882 | | -#define STACK_TOP TASK_SIZE_LOW |
|---|
| 883 | | -#define STACK_TOP_MAX TASK_SIZE_MAX |
|---|
| 884 | | - |
|---|
| 885 | | -#define INIT_THREAD { \ |
|---|
| 886 | | - .addr_limit = KERNEL_DS, \ |
|---|
| 887 | | -} |
|---|
| 781 | +#define INIT_THREAD { } |
|---|
| 888 | 782 | |
|---|
| 889 | 783 | extern unsigned long KSTK_ESP(struct task_struct *task); |
|---|
| 890 | 784 | |
|---|
| .. | .. |
|---|
| 911 | 805 | |
|---|
| 912 | 806 | DECLARE_PER_CPU(u64, msr_misc_features_shadow); |
|---|
| 913 | 807 | |
|---|
| 914 | | -/* Register/unregister a process' MPX related resource */ |
|---|
| 915 | | -#define MPX_ENABLE_MANAGEMENT() mpx_enable_management() |
|---|
| 916 | | -#define MPX_DISABLE_MANAGEMENT() mpx_disable_management() |
|---|
| 917 | | - |
|---|
| 918 | | -#ifdef CONFIG_X86_INTEL_MPX |
|---|
| 919 | | -extern int mpx_enable_management(void); |
|---|
| 920 | | -extern int mpx_disable_management(void); |
|---|
| 921 | | -#else |
|---|
| 922 | | -static inline int mpx_enable_management(void) |
|---|
| 923 | | -{ |
|---|
| 924 | | - return -EINVAL; |
|---|
| 925 | | -} |
|---|
| 926 | | -static inline int mpx_disable_management(void) |
|---|
| 927 | | -{ |
|---|
| 928 | | - return -EINVAL; |
|---|
| 929 | | -} |
|---|
| 930 | | -#endif /* CONFIG_X86_INTEL_MPX */ |
|---|
| 931 | | - |
|---|
| 932 | 808 | #ifdef CONFIG_CPU_SUP_AMD |
|---|
| 933 | 809 | extern u16 amd_get_nb_id(int cpu); |
|---|
| 934 | 810 | extern u32 amd_get_nodes_per_socket(void); |
|---|
| 811 | +extern bool cpu_has_ibpb_brtype_microcode(void); |
|---|
| 812 | +extern void amd_clear_divider(void); |
|---|
| 935 | 813 | #else |
|---|
| 936 | 814 | static inline u16 amd_get_nb_id(int cpu) { return 0; } |
|---|
| 937 | 815 | static inline u32 amd_get_nodes_per_socket(void) { return 0; } |
|---|
| 816 | +static inline bool cpu_has_ibpb_brtype_microcode(void) { return false; } |
|---|
| 817 | +static inline void amd_clear_divider(void) { } |
|---|
| 938 | 818 | #endif |
|---|
| 939 | 819 | |
|---|
| 940 | 820 | static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves) |
|---|
| .. | .. |
|---|
| 953 | 833 | } |
|---|
| 954 | 834 | |
|---|
| 955 | 835 | extern unsigned long arch_align_stack(unsigned long sp); |
|---|
| 956 | | -extern void free_init_pages(char *what, unsigned long begin, unsigned long end); |
|---|
| 957 | | -extern void free_kernel_image_pages(void *begin, void *end); |
|---|
| 836 | +void free_init_pages(const char *what, unsigned long begin, unsigned long end); |
|---|
| 837 | +extern void free_kernel_image_pages(const char *what, void *begin, void *end); |
|---|
| 958 | 838 | |
|---|
| 959 | 839 | void default_idle(void); |
|---|
| 960 | 840 | #ifdef CONFIG_XEN |
|---|
| .. | .. |
|---|
| 963 | 843 | #define xen_set_default_idle 0 |
|---|
| 964 | 844 | #endif |
|---|
| 965 | 845 | |
|---|
| 966 | | -void stop_this_cpu(void *dummy); |
|---|
| 967 | | -void df_debug(struct pt_regs *regs, long error_code); |
|---|
| 968 | | -void microcode_check(void); |
|---|
| 846 | +void __noreturn stop_this_cpu(void *dummy); |
|---|
| 847 | +void microcode_check(struct cpuinfo_x86 *prev_info); |
|---|
| 848 | +void store_cpu_caps(struct cpuinfo_x86 *info); |
|---|
| 969 | 849 | |
|---|
| 970 | 850 | enum l1tf_mitigations { |
|---|
| 971 | 851 | L1TF_MITIGATION_OFF, |
|---|
| .. | .. |
|---|
| 984 | 864 | MDS_MITIGATION_VMWERV, |
|---|
| 985 | 865 | }; |
|---|
| 986 | 866 | |
|---|
| 987 | | -enum taa_mitigations { |
|---|
| 988 | | - TAA_MITIGATION_OFF, |
|---|
| 989 | | - TAA_MITIGATION_UCODE_NEEDED, |
|---|
| 990 | | - TAA_MITIGATION_VERW, |
|---|
| 991 | | - TAA_MITIGATION_TSX_DISABLED, |
|---|
| 992 | | -}; |
|---|
| 867 | +extern bool gds_ucode_mitigated(void); |
|---|
| 993 | 868 | |
|---|
| 994 | 869 | #endif /* _ASM_X86_PROCESSOR_H */ |
|---|