| .. | .. | 
|---|
| 7 | 7 |  /* Forward declaration, a strange C thing */ | 
|---|
| 8 | 8 |  struct task_struct; | 
|---|
| 9 | 9 |  struct mm_struct; | 
|---|
 | 10 | +struct io_bitmap;  | 
|---|
| 10 | 11 |  struct vm86; | 
|---|
| 11 | 12 |   | 
|---|
| 12 | 13 |  #include <asm/math_emu.h> | 
|---|
| .. | .. | 
|---|
| 24 | 25 |  #include <asm/special_insns.h> | 
|---|
| 25 | 26 |  #include <asm/fpu/types.h> | 
|---|
| 26 | 27 |  #include <asm/unwind_hints.h> | 
|---|
 | 28 | +#include <asm/vmxfeatures.h>  | 
|---|
| 27 | 29 |  #include <asm/vdso/processor.h> | 
|---|
| 28 | 30 |   | 
|---|
| 29 | 31 |  #include <linux/personality.h> | 
|---|
| .. | .. | 
|---|
| 43 | 45 |  #define NET_IP_ALIGN	0 | 
|---|
| 44 | 46 |   | 
|---|
| 45 | 47 |  #define HBP_NUM 4 | 
|---|
| 46 |  | -/*  | 
|---|
| 47 |  | - * Default implementation of macro that returns current  | 
|---|
| 48 |  | - * instruction pointer ("program counter").  | 
|---|
| 49 |  | - */  | 
|---|
| 50 |  | -static inline void *current_text_addr(void)  | 
|---|
| 51 |  | -{  | 
|---|
| 52 |  | -	void *pc;  | 
|---|
| 53 |  | -  | 
|---|
| 54 |  | -	asm volatile("mov $1f, %0; 1:":"=r" (pc));  | 
|---|
| 55 |  | -  | 
|---|
| 56 |  | -	return pc;  | 
|---|
| 57 |  | -}  | 
|---|
| 58 | 48 |   | 
|---|
| 59 | 49 |  /* | 
|---|
| 60 | 50 |   * These alignment constraints are for performance in the vSMP case, | 
|---|
| .. | .. | 
|---|
| 97 | 87 |  	/* Number of 4K pages in DTLB/ITLB combined(in pages): */ | 
|---|
| 98 | 88 |  	int			x86_tlbsize; | 
|---|
| 99 | 89 |  #endif | 
|---|
 | 90 | +#ifdef CONFIG_X86_VMX_FEATURE_NAMES  | 
|---|
 | 91 | +	__u32			vmx_capability[NVMXINTS];  | 
|---|
 | 92 | +#endif  | 
|---|
| 100 | 93 |  	__u8			x86_virt_bits; | 
|---|
| 101 | 94 |  	__u8			x86_phys_bits; | 
|---|
| 102 | 95 |  	/* CPUID returned core id bits: */ | 
|---|
| .. | .. | 
|---|
| 106 | 99 |  	__u32			extended_cpuid_level; | 
|---|
| 107 | 100 |  	/* Maximum supported CPUID level, -1=no CPUID: */ | 
|---|
| 108 | 101 |  	int			cpuid_level; | 
|---|
| 109 |  | -	__u32			x86_capability[NCAPINTS + NBUGINTS];  | 
|---|
 | 102 | +	/*  | 
|---|
 | 103 | +	 * Align to size of unsigned long because the x86_capability array  | 
|---|
 | 104 | +	 * is passed to bitops which require the alignment. Use unnamed  | 
|---|
 | 105 | +	 * union to enforce the array is aligned to size of unsigned long.  | 
|---|
 | 106 | +	 */  | 
|---|
 | 107 | +	union {  | 
|---|
 | 108 | +		__u32		x86_capability[NCAPINTS + NBUGINTS];  | 
|---|
 | 109 | +		unsigned long	x86_capability_alignment;  | 
|---|
 | 110 | +	};  | 
|---|
| 110 | 111 |  	char			x86_vendor_id[16]; | 
|---|
| 111 | 112 |  	char			x86_model_id[64]; | 
|---|
| 112 | 113 |  	/* in KB - valid for CPUS which support this call: */ | 
|---|
| 113 | 114 |  	unsigned int		x86_cache_size; | 
|---|
| 114 | 115 |  	int			x86_cache_alignment;	/* In bytes */ | 
|---|
| 115 |  | -	/* Cache QoS architectural values: */  | 
|---|
 | 116 | +	/* Cache QoS architectural values, valid only on the BSP: */  | 
|---|
| 116 | 117 |  	int			x86_cache_max_rmid;	/* max index */ | 
|---|
| 117 | 118 |  	int			x86_cache_occ_scale;	/* scale to bytes */ | 
|---|
 | 119 | +	int			x86_cache_mbm_width_offset;  | 
|---|
| 118 | 120 |  	int			x86_power; | 
|---|
| 119 | 121 |  	unsigned long		loops_per_jiffy; | 
|---|
| 120 | 122 |  	/* cpuid returned max cores value: */ | 
|---|
| 121 |  | -	u16			 x86_max_cores;  | 
|---|
 | 123 | +	u16			x86_max_cores;  | 
|---|
| 122 | 124 |  	u16			apicid; | 
|---|
| 123 | 125 |  	u16			initial_apicid; | 
|---|
| 124 | 126 |  	u16			x86_clflush_size; | 
|---|
| .. | .. | 
|---|
| 130 | 132 |  	u16			logical_proc_id; | 
|---|
| 131 | 133 |  	/* Core id: */ | 
|---|
| 132 | 134 |  	u16			cpu_core_id; | 
|---|
 | 135 | +	u16			cpu_die_id;  | 
|---|
 | 136 | +	u16			logical_die_id;  | 
|---|
| 133 | 137 |  	/* Index into per_cpu list: */ | 
|---|
| 134 | 138 |  	u16			cpu_index; | 
|---|
| 135 | 139 |  	u32			microcode; | 
|---|
| .. | .. | 
|---|
| 156 | 160 |  #define X86_VENDOR_CENTAUR	5 | 
|---|
| 157 | 161 |  #define X86_VENDOR_TRANSMETA	7 | 
|---|
| 158 | 162 |  #define X86_VENDOR_NSC		8 | 
|---|
| 159 |  | -#define X86_VENDOR_NUM		9  | 
|---|
 | 163 | +#define X86_VENDOR_HYGON	9  | 
|---|
 | 164 | +#define X86_VENDOR_ZHAOXIN	10  | 
|---|
 | 165 | +#define X86_VENDOR_NUM		11  | 
|---|
| 160 | 166 |   | 
|---|
| 161 | 167 |  #define X86_VENDOR_UNKNOWN	0xff | 
|---|
| 162 | 168 |   | 
|---|
| .. | .. | 
|---|
| 166 | 172 |  extern struct cpuinfo_x86	boot_cpu_data; | 
|---|
| 167 | 173 |  extern struct cpuinfo_x86	new_cpu_data; | 
|---|
| 168 | 174 |   | 
|---|
| 169 |  | -extern struct x86_hw_tss	doublefault_tss;  | 
|---|
| 170 | 175 |  extern __u32			cpu_caps_cleared[NCAPINTS + NBUGINTS]; | 
|---|
| 171 | 176 |  extern __u32			cpu_caps_set[NCAPINTS + NBUGINTS]; | 
|---|
| 172 | 177 |   | 
|---|
| .. | .. | 
|---|
| 316 | 321 |  	 */ | 
|---|
| 317 | 322 |  	u64			sp1; | 
|---|
| 318 | 323 |   | 
|---|
 | 324 | +	/*  | 
|---|
 | 325 | +	 * Since Linux does not use ring 2, the 'sp2' slot is unused by  | 
|---|
 | 326 | +	 * hardware.  entry_SYSCALL_64 uses it as scratch space to stash  | 
|---|
 | 327 | +	 * the user RSP value.  | 
|---|
 | 328 | +	 */  | 
|---|
| 319 | 329 |  	u64			sp2; | 
|---|
 | 330 | +  | 
|---|
| 320 | 331 |  	u64			reserved2; | 
|---|
| 321 | 332 |  	u64			ist[7]; | 
|---|
| 322 | 333 |  	u32			reserved3; | 
|---|
| .. | .. | 
|---|
| 331 | 342 |   * IO-bitmap sizes: | 
|---|
| 332 | 343 |   */ | 
|---|
| 333 | 344 |  #define IO_BITMAP_BITS			65536 | 
|---|
| 334 |  | -#define IO_BITMAP_BYTES			(IO_BITMAP_BITS/8)  | 
|---|
| 335 |  | -#define IO_BITMAP_LONGS			(IO_BITMAP_BYTES/sizeof(long))  | 
|---|
| 336 |  | -#define IO_BITMAP_OFFSET		(offsetof(struct tss_struct, io_bitmap) - offsetof(struct tss_struct, x86_tss))  | 
|---|
| 337 |  | -#define INVALID_IO_BITMAP_OFFSET	0x8000  | 
|---|
 | 345 | +#define IO_BITMAP_BYTES			(IO_BITMAP_BITS / BITS_PER_BYTE)  | 
|---|
 | 346 | +#define IO_BITMAP_LONGS			(IO_BITMAP_BYTES / sizeof(long))  | 
|---|
 | 347 | +  | 
|---|
 | 348 | +#define IO_BITMAP_OFFSET_VALID_MAP				\  | 
|---|
 | 349 | +	(offsetof(struct tss_struct, io_bitmap.bitmap) -	\  | 
|---|
 | 350 | +	 offsetof(struct tss_struct, x86_tss))  | 
|---|
 | 351 | +  | 
|---|
 | 352 | +#define IO_BITMAP_OFFSET_VALID_ALL				\  | 
|---|
 | 353 | +	(offsetof(struct tss_struct, io_bitmap.mapall) -	\  | 
|---|
 | 354 | +	 offsetof(struct tss_struct, x86_tss))  | 
|---|
 | 355 | +  | 
|---|
 | 356 | +#ifdef CONFIG_X86_IOPL_IOPERM  | 
|---|
 | 357 | +/*  | 
|---|
 | 358 | + * sizeof(unsigned long) coming from an extra "long" at the end of the  | 
|---|
 | 359 | + * iobitmap. The limit is inclusive, i.e. the last valid byte.  | 
|---|
 | 360 | + */  | 
|---|
 | 361 | +# define __KERNEL_TSS_LIMIT	\  | 
|---|
 | 362 | +	(IO_BITMAP_OFFSET_VALID_ALL + IO_BITMAP_BYTES + \  | 
|---|
 | 363 | +	 sizeof(unsigned long) - 1)  | 
|---|
 | 364 | +#else  | 
|---|
 | 365 | +# define __KERNEL_TSS_LIMIT	\  | 
|---|
 | 366 | +	(offsetof(struct tss_struct, x86_tss) + sizeof(struct x86_hw_tss) - 1)  | 
|---|
 | 367 | +#endif  | 
|---|
 | 368 | +  | 
|---|
 | 369 | +/* Base offset outside of TSS_LIMIT so unpriviledged IO causes #GP */  | 
|---|
 | 370 | +#define IO_BITMAP_OFFSET_INVALID	(__KERNEL_TSS_LIMIT + 1)  | 
|---|
| 338 | 371 |   | 
|---|
| 339 | 372 |  struct entry_stack { | 
|---|
| 340 | 373 |  	char	stack[PAGE_SIZE]; | 
|---|
| .. | .. | 
|---|
| 344 | 377 |  	struct entry_stack stack; | 
|---|
| 345 | 378 |  } __aligned(PAGE_SIZE); | 
|---|
| 346 | 379 |   | 
|---|
 | 380 | +/*  | 
|---|
 | 381 | + * All IO bitmap related data stored in the TSS:  | 
|---|
 | 382 | + */  | 
|---|
 | 383 | +struct x86_io_bitmap {  | 
|---|
 | 384 | +	/* The sequence number of the last active bitmap. */  | 
|---|
 | 385 | +	u64			prev_sequence;  | 
|---|
 | 386 | +  | 
|---|
 | 387 | +	/*  | 
|---|
 | 388 | +	 * Store the dirty size of the last io bitmap offender. The next  | 
|---|
 | 389 | +	 * one will have to do the cleanup as the switch out to a non io  | 
|---|
 | 390 | +	 * bitmap user will just set x86_tss.io_bitmap_base to a value  | 
|---|
 | 391 | +	 * outside of the TSS limit. So for sane tasks there is no need to  | 
|---|
 | 392 | +	 * actually touch the io_bitmap at all.  | 
|---|
 | 393 | +	 */  | 
|---|
 | 394 | +	unsigned int		prev_max;  | 
|---|
 | 395 | +  | 
|---|
 | 396 | +	/*  | 
|---|
 | 397 | +	 * The extra 1 is there because the CPU will access an  | 
|---|
 | 398 | +	 * additional byte beyond the end of the IO permission  | 
|---|
 | 399 | +	 * bitmap. The extra byte must be all 1 bits, and must  | 
|---|
 | 400 | +	 * be within the limit.  | 
|---|
 | 401 | +	 */  | 
|---|
 | 402 | +	unsigned long		bitmap[IO_BITMAP_LONGS + 1];  | 
|---|
 | 403 | +  | 
|---|
 | 404 | +	/*  | 
|---|
 | 405 | +	 * Special I/O bitmap to emulate IOPL(3). All bytes zero,  | 
|---|
 | 406 | +	 * except the additional byte at the end.  | 
|---|
 | 407 | +	 */  | 
|---|
 | 408 | +	unsigned long		mapall[IO_BITMAP_LONGS + 1];  | 
|---|
 | 409 | +};  | 
|---|
 | 410 | +  | 
|---|
| 347 | 411 |  struct tss_struct { | 
|---|
| 348 | 412 |  	/* | 
|---|
| 349 | 413 |  	 * The fixed hardware portion.  This must not cross a page boundary | 
|---|
| .. | .. | 
|---|
| 352 | 416 |  	 */ | 
|---|
| 353 | 417 |  	struct x86_hw_tss	x86_tss; | 
|---|
| 354 | 418 |   | 
|---|
| 355 |  | -	/*  | 
|---|
| 356 |  | -	 * The extra 1 is there because the CPU will access an  | 
|---|
| 357 |  | -	 * additional byte beyond the end of the IO permission  | 
|---|
| 358 |  | -	 * bitmap. The extra byte must be all 1 bits, and must  | 
|---|
| 359 |  | -	 * be within the limit.  | 
|---|
| 360 |  | -	 */  | 
|---|
| 361 |  | -	unsigned long		io_bitmap[IO_BITMAP_LONGS + 1];  | 
|---|
 | 419 | +	struct x86_io_bitmap	io_bitmap;  | 
|---|
| 362 | 420 |  } __aligned(PAGE_SIZE); | 
|---|
| 363 | 421 |   | 
|---|
| 364 | 422 |  DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw); | 
|---|
| 365 | 423 |   | 
|---|
| 366 |  | -/*  | 
|---|
| 367 |  | - * sizeof(unsigned long) coming from an extra "long" at the end  | 
|---|
| 368 |  | - * of the iobitmap.  | 
|---|
| 369 |  | - *  | 
|---|
| 370 |  | - * -1? seg base+limit should be pointing to the address of the  | 
|---|
| 371 |  | - * last valid byte  | 
|---|
| 372 |  | - */  | 
|---|
| 373 |  | -#define __KERNEL_TSS_LIMIT	\  | 
|---|
| 374 |  | -	(IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)  | 
|---|
 | 424 | +/* Per CPU interrupt stacks */  | 
|---|
 | 425 | +struct irq_stack {  | 
|---|
 | 426 | +	char		stack[IRQ_STACK_SIZE];  | 
|---|
 | 427 | +} __aligned(IRQ_STACK_SIZE);  | 
|---|
 | 428 | +  | 
|---|
 | 429 | +DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);  | 
|---|
| 375 | 430 |   | 
|---|
| 376 | 431 |  #ifdef CONFIG_X86_32 | 
|---|
| 377 | 432 |  DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack); | 
|---|
| .. | .. | 
|---|
| 380 | 435 |  #define cpu_current_top_of_stack cpu_tss_rw.x86_tss.sp1 | 
|---|
| 381 | 436 |  #endif | 
|---|
| 382 | 437 |   | 
|---|
| 383 |  | -/*  | 
|---|
| 384 |  | - * Save the original ist values for checking stack pointers during debugging  | 
|---|
| 385 |  | - */  | 
|---|
| 386 |  | -struct orig_ist {  | 
|---|
| 387 |  | -	unsigned long		ist[7];  | 
|---|
| 388 |  | -};  | 
|---|
| 389 |  | -  | 
|---|
| 390 | 438 |  #ifdef CONFIG_X86_64 | 
|---|
| 391 |  | -DECLARE_PER_CPU(struct orig_ist, orig_ist);  | 
|---|
| 392 |  | -  | 
|---|
| 393 |  | -union irq_stack_union {  | 
|---|
| 394 |  | -	char irq_stack[IRQ_STACK_SIZE];  | 
|---|
 | 439 | +struct fixed_percpu_data {  | 
|---|
| 395 | 440 |  	/* | 
|---|
| 396 | 441 |  	 * GCC hardcodes the stack canary as %gs:40.  Since the | 
|---|
| 397 | 442 |  	 * irq_stack is the object at %gs:0, we reserve the bottom | 
|---|
| 398 | 443 |  	 * 48 bytes of the irq stack for the canary. | 
|---|
| 399 | 444 |  	 */ | 
|---|
| 400 |  | -	struct {  | 
|---|
| 401 |  | -		char gs_base[40];  | 
|---|
| 402 |  | -		unsigned long stack_canary;  | 
|---|
| 403 |  | -	};  | 
|---|
 | 445 | +	char		gs_base[40];  | 
|---|
 | 446 | +	unsigned long	stack_canary;  | 
|---|
| 404 | 447 |  }; | 
|---|
| 405 | 448 |   | 
|---|
| 406 |  | -DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;  | 
|---|
| 407 |  | -DECLARE_INIT_PER_CPU(irq_stack_union);  | 
|---|
 | 449 | +DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;  | 
|---|
 | 450 | +DECLARE_INIT_PER_CPU(fixed_percpu_data);  | 
|---|
| 408 | 451 |   | 
|---|
| 409 | 452 |  static inline unsigned long cpu_kernelmode_gs_base(int cpu) | 
|---|
| 410 | 453 |  { | 
|---|
| 411 |  | -	return (unsigned long)per_cpu(irq_stack_union.gs_base, cpu);  | 
|---|
 | 454 | +	return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);  | 
|---|
| 412 | 455 |  } | 
|---|
| 413 | 456 |   | 
|---|
| 414 |  | -DECLARE_PER_CPU(char *, irq_stack_ptr);  | 
|---|
| 415 | 457 |  DECLARE_PER_CPU(unsigned int, irq_count); | 
|---|
| 416 | 458 |  extern asmlinkage void ignore_sysret(void); | 
|---|
| 417 | 459 |   | 
|---|
| 418 |  | -#if IS_ENABLED(CONFIG_KVM)  | 
|---|
| 419 | 460 |  /* Save actual FS/GS selectors and bases to current->thread */ | 
|---|
| 420 |  | -void save_fsgs_for_kvm(void);  | 
|---|
| 421 |  | -#endif  | 
|---|
 | 461 | +void current_save_fsgs(void);  | 
|---|
| 422 | 462 |  #else	/* X86_64 */ | 
|---|
| 423 | 463 |  #ifdef CONFIG_STACKPROTECTOR | 
|---|
| 424 | 464 |  /* | 
|---|
| .. | .. | 
|---|
| 433 | 473 |  }; | 
|---|
| 434 | 474 |  DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary); | 
|---|
| 435 | 475 |  #endif | 
|---|
| 436 |  | -/*  | 
|---|
| 437 |  | - * per-CPU IRQ handling stacks  | 
|---|
| 438 |  | - */  | 
|---|
| 439 |  | -struct irq_stack {  | 
|---|
| 440 |  | -	u32                     stack[THREAD_SIZE/sizeof(u32)];  | 
|---|
| 441 |  | -} __aligned(THREAD_SIZE);  | 
|---|
| 442 |  | -  | 
|---|
| 443 |  | -DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);  | 
|---|
| 444 |  | -DECLARE_PER_CPU(struct irq_stack *, softirq_stack);  | 
|---|
 | 476 | +/* Per CPU softirq stack pointer */  | 
|---|
 | 477 | +DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);  | 
|---|
| 445 | 478 |  #endif	/* X86_64 */ | 
|---|
| 446 | 479 |   | 
|---|
| 447 | 480 |  extern unsigned int fpu_kernel_xstate_size; | 
|---|
| 448 | 481 |  extern unsigned int fpu_user_xstate_size; | 
|---|
| 449 | 482 |   | 
|---|
| 450 | 483 |  struct perf_event; | 
|---|
| 451 |  | -  | 
|---|
| 452 |  | -typedef struct {  | 
|---|
| 453 |  | -	unsigned long		seg;  | 
|---|
| 454 |  | -} mm_segment_t;  | 
|---|
| 455 | 484 |   | 
|---|
| 456 | 485 |  struct thread_struct { | 
|---|
| 457 | 486 |  	/* Cached TLS descriptors: */ | 
|---|
| .. | .. | 
|---|
| 484 | 513 |  	/* Save middle states of ptrace breakpoints */ | 
|---|
| 485 | 514 |  	struct perf_event	*ptrace_bps[HBP_NUM]; | 
|---|
| 486 | 515 |  	/* Debug status used for traps, single steps, etc... */ | 
|---|
| 487 |  | -	unsigned long           debugreg6;  | 
|---|
 | 516 | +	unsigned long           virtual_dr6;  | 
|---|
| 488 | 517 |  	/* Keep track of the exact dr7 value set by the user */ | 
|---|
| 489 | 518 |  	unsigned long           ptrace_dr7; | 
|---|
| 490 | 519 |  	/* Fault info: */ | 
|---|
| .. | .. | 
|---|
| 496 | 525 |  	struct vm86		*vm86; | 
|---|
| 497 | 526 |  #endif | 
|---|
| 498 | 527 |  	/* IO permissions: */ | 
|---|
| 499 |  | -	unsigned long		*io_bitmap_ptr;  | 
|---|
| 500 |  | -	unsigned long		iopl;  | 
|---|
| 501 |  | -	/* Max allowed port in the bitmap, in bytes: */  | 
|---|
| 502 |  | -	unsigned		io_bitmap_max;  | 
|---|
 | 528 | +	struct io_bitmap	*io_bitmap;  | 
|---|
| 503 | 529 |   | 
|---|
| 504 |  | -	mm_segment_t		addr_limit;  | 
|---|
 | 530 | +	/*  | 
|---|
 | 531 | +	 * IOPL. Priviledge level dependent I/O permission which is  | 
|---|
 | 532 | +	 * emulated via the I/O bitmap to prevent user space from disabling  | 
|---|
 | 533 | +	 * interrupts.  | 
|---|
 | 534 | +	 */  | 
|---|
 | 535 | +	unsigned long		iopl_emul;  | 
|---|
| 505 | 536 |   | 
|---|
 | 537 | +	unsigned int		iopl_warn:1;  | 
|---|
| 506 | 538 |  	unsigned int		sig_on_uaccess_err:1; | 
|---|
| 507 |  | -	unsigned int		uaccess_err:1;	/* uaccess failed */  | 
|---|
| 508 | 539 |   | 
|---|
| 509 | 540 |  	/* Floating point and extended processor state */ | 
|---|
| 510 | 541 |  	struct fpu		fpu; | 
|---|
| .. | .. | 
|---|
| 522 | 553 |  	*size = fpu_kernel_xstate_size; | 
|---|
| 523 | 554 |  } | 
|---|
| 524 | 555 |   | 
|---|
| 525 |  | -/*  | 
|---|
| 526 |  | - * Set IOPL bits in EFLAGS from given mask  | 
|---|
| 527 |  | - */  | 
|---|
| 528 |  | -static inline void native_set_iopl_mask(unsigned mask)  | 
|---|
| 529 |  | -{  | 
|---|
| 530 |  | -#ifdef CONFIG_X86_32  | 
|---|
| 531 |  | -	unsigned int reg;  | 
|---|
| 532 |  | -  | 
|---|
| 533 |  | -	asm volatile ("pushfl;"  | 
|---|
| 534 |  | -		      "popl %0;"  | 
|---|
| 535 |  | -		      "andl %1, %0;"  | 
|---|
| 536 |  | -		      "orl %2, %0;"  | 
|---|
| 537 |  | -		      "pushl %0;"  | 
|---|
| 538 |  | -		      "popfl"  | 
|---|
| 539 |  | -		      : "=&r" (reg)  | 
|---|
| 540 |  | -		      : "i" (~X86_EFLAGS_IOPL), "r" (mask));  | 
|---|
| 541 |  | -#endif  | 
|---|
| 542 |  | -}  | 
|---|
| 543 |  | -  | 
|---|
| 544 | 556 |  static inline void | 
|---|
| 545 | 557 |  native_load_sp0(unsigned long sp0) | 
|---|
| 546 | 558 |  { | 
|---|
| 547 | 559 |  	this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0); | 
|---|
| 548 | 560 |  } | 
|---|
| 549 | 561 |   | 
|---|
| 550 |  | -static inline void native_swapgs(void)  | 
|---|
 | 562 | +static __always_inline void native_swapgs(void)  | 
|---|
| 551 | 563 |  { | 
|---|
| 552 | 564 |  #ifdef CONFIG_X86_64 | 
|---|
| 553 | 565 |  	asm volatile("swapgs" ::: "memory"); | 
|---|
| .. | .. | 
|---|
| 570 | 582 |  			       current_stack_pointer) < THREAD_SIZE; | 
|---|
| 571 | 583 |  } | 
|---|
| 572 | 584 |   | 
|---|
| 573 |  | -#ifdef CONFIG_PARAVIRT  | 
|---|
 | 585 | +#ifdef CONFIG_PARAVIRT_XXL  | 
|---|
| 574 | 586 |  #include <asm/paravirt.h> | 
|---|
| 575 | 587 |  #else | 
|---|
| 576 | 588 |  #define __cpuid			native_cpuid | 
|---|
| .. | .. | 
|---|
| 580 | 592 |  	native_load_sp0(sp0); | 
|---|
| 581 | 593 |  } | 
|---|
| 582 | 594 |   | 
|---|
| 583 |  | -#define set_iopl_mask native_set_iopl_mask  | 
|---|
| 584 |  | -#endif /* CONFIG_PARAVIRT */  | 
|---|
 | 595 | +#endif /* CONFIG_PARAVIRT_XXL */  | 
|---|
| 585 | 596 |   | 
|---|
| 586 | 597 |  /* Free all resources held by a thread. */ | 
|---|
| 587 | 598 |  extern void release_thread(struct task_struct *); | 
|---|
| .. | .. | 
|---|
| 651 | 662 |  	return edx; | 
|---|
| 652 | 663 |  } | 
|---|
| 653 | 664 |   | 
|---|
| 654 |  | -/*  | 
|---|
| 655 |  | - * This function forces the icache and prefetched instruction stream to  | 
|---|
| 656 |  | - * catch up with reality in two very specific cases:  | 
|---|
| 657 |  | - *  | 
|---|
| 658 |  | - *  a) Text was modified using one virtual address and is about to be executed  | 
|---|
| 659 |  | - *     from the same physical page at a different virtual address.  | 
|---|
| 660 |  | - *  | 
|---|
| 661 |  | - *  b) Text was modified on a different CPU, may subsequently be  | 
|---|
| 662 |  | - *     executed on this CPU, and you want to make sure the new version  | 
|---|
| 663 |  | - *     gets executed.  This generally means you're calling this in a IPI.  | 
|---|
| 664 |  | - *  | 
|---|
| 665 |  | - * If you're calling this for a different reason, you're probably doing  | 
|---|
| 666 |  | - * it wrong.  | 
|---|
| 667 |  | - */  | 
|---|
| 668 |  | -static inline void sync_core(void)  | 
|---|
| 669 |  | -{  | 
|---|
| 670 |  | -	/*  | 
|---|
| 671 |  | -	 * There are quite a few ways to do this.  IRET-to-self is nice  | 
|---|
| 672 |  | -	 * because it works on every CPU, at any CPL (so it's compatible  | 
|---|
| 673 |  | -	 * with paravirtualization), and it never exits to a hypervisor.  | 
|---|
| 674 |  | -	 * The only down sides are that it's a bit slow (it seems to be  | 
|---|
| 675 |  | -	 * a bit more than 2x slower than the fastest options) and that  | 
|---|
| 676 |  | -	 * it unmasks NMIs.  The "push %cs" is needed because, in  | 
|---|
| 677 |  | -	 * paravirtual environments, __KERNEL_CS may not be a valid CS  | 
|---|
| 678 |  | -	 * value when we do IRET directly.  | 
|---|
| 679 |  | -	 *  | 
|---|
| 680 |  | -	 * In case NMI unmasking or performance ever becomes a problem,  | 
|---|
| 681 |  | -	 * the next best option appears to be MOV-to-CR2 and an  | 
|---|
| 682 |  | -	 * unconditional jump.  That sequence also works on all CPUs,  | 
|---|
| 683 |  | -	 * but it will fault at CPL3 (i.e. Xen PV).  | 
|---|
| 684 |  | -	 *  | 
|---|
| 685 |  | -	 * CPUID is the conventional way, but it's nasty: it doesn't  | 
|---|
| 686 |  | -	 * exist on some 486-like CPUs, and it usually exits to a  | 
|---|
| 687 |  | -	 * hypervisor.  | 
|---|
| 688 |  | -	 *  | 
|---|
| 689 |  | -	 * Like all of Linux's memory ordering operations, this is a  | 
|---|
| 690 |  | -	 * compiler barrier as well.  | 
|---|
| 691 |  | -	 */  | 
|---|
| 692 |  | -#ifdef CONFIG_X86_32  | 
|---|
| 693 |  | -	asm volatile (  | 
|---|
| 694 |  | -		"pushfl\n\t"  | 
|---|
| 695 |  | -		"pushl %%cs\n\t"  | 
|---|
| 696 |  | -		"pushl $1f\n\t"  | 
|---|
| 697 |  | -		"iret\n\t"  | 
|---|
| 698 |  | -		"1:"  | 
|---|
| 699 |  | -		: ASM_CALL_CONSTRAINT : : "memory");  | 
|---|
| 700 |  | -#else  | 
|---|
| 701 |  | -	unsigned int tmp;  | 
|---|
| 702 |  | -  | 
|---|
| 703 |  | -	asm volatile (  | 
|---|
| 704 |  | -		UNWIND_HINT_SAVE  | 
|---|
| 705 |  | -		"mov %%ss, %0\n\t"  | 
|---|
| 706 |  | -		"pushq %q0\n\t"  | 
|---|
| 707 |  | -		"pushq %%rsp\n\t"  | 
|---|
| 708 |  | -		"addq $8, (%%rsp)\n\t"  | 
|---|
| 709 |  | -		"pushfq\n\t"  | 
|---|
| 710 |  | -		"mov %%cs, %0\n\t"  | 
|---|
| 711 |  | -		"pushq %q0\n\t"  | 
|---|
| 712 |  | -		"pushq $1f\n\t"  | 
|---|
| 713 |  | -		"iretq\n\t"  | 
|---|
| 714 |  | -		UNWIND_HINT_RESTORE  | 
|---|
| 715 |  | -		"1:"  | 
|---|
| 716 |  | -		: "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");  | 
|---|
| 717 |  | -#endif  | 
|---|
| 718 |  | -}  | 
|---|
| 719 |  | -  | 
|---|
| 720 | 665 |  extern void select_idle_routine(const struct cpuinfo_x86 *c); | 
|---|
| 721 | 666 |  extern void amd_e400_c1e_apic_setup(void); | 
|---|
| 722 | 667 |   | 
|---|
| .. | .. | 
|---|
| 728 | 673 |  extern void enable_sep_cpu(void); | 
|---|
| 729 | 674 |  extern int sysenter_setup(void); | 
|---|
| 730 | 675 |   | 
|---|
| 731 |  | -void early_trap_pf_init(void);  | 
|---|
| 732 | 676 |   | 
|---|
| 733 | 677 |  /* Defined in head.S */ | 
|---|
| 734 | 678 |  extern struct desc_ptr		early_gdt_descr; | 
|---|
| .. | .. | 
|---|
| 738 | 682 |  extern void load_fixmap_gdt(int); | 
|---|
| 739 | 683 |  extern void load_percpu_segment(int); | 
|---|
| 740 | 684 |  extern void cpu_init(void); | 
|---|
 | 685 | +extern void cpu_init_secondary(void);  | 
|---|
 | 686 | +extern void cpu_init_exception_handling(void);  | 
|---|
 | 687 | +extern void cr4_init(void);  | 
|---|
| 741 | 688 |   | 
|---|
| 742 | 689 |  static inline unsigned long get_debugctlmsr(void) | 
|---|
| 743 | 690 |  { | 
|---|
| .. | .. | 
|---|
| 798 | 745 |   * Useful for spinlocks to avoid one state transition in the | 
|---|
| 799 | 746 |   * cache coherency protocol: | 
|---|
| 800 | 747 |   */ | 
|---|
| 801 |  | -static inline void prefetchw(const void *x)  | 
|---|
 | 748 | +static __always_inline void prefetchw(const void *x)  | 
|---|
| 802 | 749 |  { | 
|---|
| 803 | 750 |  	alternative_input(BASE_PREFETCH, "prefetchw %P1", | 
|---|
| 804 | 751 |  			  X86_FEATURE_3DNOWPREFETCH, | 
|---|
| .. | .. | 
|---|
| 823 | 770 |  }) | 
|---|
| 824 | 771 |   | 
|---|
| 825 | 772 |  #ifdef CONFIG_X86_32 | 
|---|
| 826 |  | -/*  | 
|---|
| 827 |  | - * User space process size: 3GB (default).  | 
|---|
| 828 |  | - */  | 
|---|
| 829 |  | -#define IA32_PAGE_OFFSET	PAGE_OFFSET  | 
|---|
| 830 |  | -#define TASK_SIZE		PAGE_OFFSET  | 
|---|
| 831 |  | -#define TASK_SIZE_LOW		TASK_SIZE  | 
|---|
| 832 |  | -#define TASK_SIZE_MAX		TASK_SIZE  | 
|---|
| 833 |  | -#define DEFAULT_MAP_WINDOW	TASK_SIZE  | 
|---|
| 834 |  | -#define STACK_TOP		TASK_SIZE  | 
|---|
| 835 |  | -#define STACK_TOP_MAX		STACK_TOP  | 
|---|
| 836 |  | -  | 
|---|
| 837 | 773 |  #define INIT_THREAD  {							  \ | 
|---|
| 838 | 774 |  	.sp0			= TOP_OF_INIT_STACK,			  \ | 
|---|
| 839 | 775 |  	.sysenter_cs		= __KERNEL_CS,				  \ | 
|---|
| 840 |  | -	.io_bitmap_ptr		= NULL,					  \  | 
|---|
| 841 |  | -	.addr_limit		= KERNEL_DS,				  \  | 
|---|
| 842 | 776 |  } | 
|---|
| 843 | 777 |   | 
|---|
| 844 | 778 |  #define KSTK_ESP(task)		(task_pt_regs(task)->sp) | 
|---|
| 845 | 779 |   | 
|---|
| 846 | 780 |  #else | 
|---|
| 847 |  | -/*  | 
|---|
| 848 |  | - * User space process size.  This is the first address outside the user range.  | 
|---|
| 849 |  | - * There are a few constraints that determine this:  | 
|---|
| 850 |  | - *  | 
|---|
| 851 |  | - * On Intel CPUs, if a SYSCALL instruction is at the highest canonical  | 
|---|
| 852 |  | - * address, then that syscall will enter the kernel with a  | 
|---|
| 853 |  | - * non-canonical return address, and SYSRET will explode dangerously.  | 
|---|
| 854 |  | - * We avoid this particular problem by preventing anything executable  | 
|---|
| 855 |  | - * from being mapped at the maximum canonical address.  | 
|---|
| 856 |  | - *  | 
|---|
| 857 |  | - * On AMD CPUs in the Ryzen family, there's a nasty bug in which the  | 
|---|
| 858 |  | - * CPUs malfunction if they execute code from the highest canonical page.  | 
|---|
| 859 |  | - * They'll speculate right off the end of the canonical space, and  | 
|---|
| 860 |  | - * bad things happen.  This is worked around in the same way as the  | 
|---|
| 861 |  | - * Intel problem.  | 
|---|
| 862 |  | - *  | 
|---|
| 863 |  | - * With page table isolation enabled, we map the LDT in ... [stay tuned]  | 
|---|
| 864 |  | - */  | 
|---|
| 865 |  | -#define TASK_SIZE_MAX	((1UL << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE)  | 
|---|
| 866 |  | -  | 
|---|
| 867 |  | -#define DEFAULT_MAP_WINDOW	((1UL << 47) - PAGE_SIZE)  | 
|---|
| 868 |  | -  | 
|---|
| 869 |  | -/* This decides where the kernel will search for a free chunk of vm  | 
|---|
| 870 |  | - * space during mmap's.  | 
|---|
| 871 |  | - */  | 
|---|
| 872 |  | -#define IA32_PAGE_OFFSET	((current->personality & ADDR_LIMIT_3GB) ? \  | 
|---|
| 873 |  | -					0xc0000000 : 0xFFFFe000)  | 
|---|
| 874 |  | -  | 
|---|
| 875 |  | -#define TASK_SIZE_LOW		(test_thread_flag(TIF_ADDR32) ? \  | 
|---|
| 876 |  | -					IA32_PAGE_OFFSET : DEFAULT_MAP_WINDOW)  | 
|---|
| 877 |  | -#define TASK_SIZE		(test_thread_flag(TIF_ADDR32) ? \  | 
|---|
| 878 |  | -					IA32_PAGE_OFFSET : TASK_SIZE_MAX)  | 
|---|
| 879 |  | -#define TASK_SIZE_OF(child)	((test_tsk_thread_flag(child, TIF_ADDR32)) ? \  | 
|---|
| 880 |  | -					IA32_PAGE_OFFSET : TASK_SIZE_MAX)  | 
|---|
| 881 |  | -  | 
|---|
| 882 |  | -#define STACK_TOP		TASK_SIZE_LOW  | 
|---|
| 883 |  | -#define STACK_TOP_MAX		TASK_SIZE_MAX  | 
|---|
| 884 |  | -  | 
|---|
| 885 |  | -#define INIT_THREAD  {						\  | 
|---|
| 886 |  | -	.addr_limit		= KERNEL_DS,			\  | 
|---|
| 887 |  | -}  | 
|---|
 | 781 | +#define INIT_THREAD { }  | 
|---|
| 888 | 782 |   | 
|---|
| 889 | 783 |  extern unsigned long KSTK_ESP(struct task_struct *task); | 
|---|
| 890 | 784 |   | 
|---|
| .. | .. | 
|---|
| 911 | 805 |   | 
|---|
| 912 | 806 |  DECLARE_PER_CPU(u64, msr_misc_features_shadow); | 
|---|
| 913 | 807 |   | 
|---|
| 914 |  | -/* Register/unregister a process' MPX related resource */  | 
|---|
| 915 |  | -#define MPX_ENABLE_MANAGEMENT()	mpx_enable_management()  | 
|---|
| 916 |  | -#define MPX_DISABLE_MANAGEMENT()	mpx_disable_management()  | 
|---|
| 917 |  | -  | 
|---|
| 918 |  | -#ifdef CONFIG_X86_INTEL_MPX  | 
|---|
| 919 |  | -extern int mpx_enable_management(void);  | 
|---|
| 920 |  | -extern int mpx_disable_management(void);  | 
|---|
| 921 |  | -#else  | 
|---|
| 922 |  | -static inline int mpx_enable_management(void)  | 
|---|
| 923 |  | -{  | 
|---|
| 924 |  | -	return -EINVAL;  | 
|---|
| 925 |  | -}  | 
|---|
| 926 |  | -static inline int mpx_disable_management(void)  | 
|---|
| 927 |  | -{  | 
|---|
| 928 |  | -	return -EINVAL;  | 
|---|
| 929 |  | -}  | 
|---|
| 930 |  | -#endif /* CONFIG_X86_INTEL_MPX */  | 
|---|
| 931 |  | -  | 
|---|
| 932 | 808 |  #ifdef CONFIG_CPU_SUP_AMD | 
|---|
| 933 | 809 |  extern u16 amd_get_nb_id(int cpu); | 
|---|
| 934 | 810 |  extern u32 amd_get_nodes_per_socket(void); | 
|---|
 | 811 | +extern bool cpu_has_ibpb_brtype_microcode(void);  | 
|---|
 | 812 | +extern void amd_clear_divider(void);  | 
|---|
| 935 | 813 |  #else | 
|---|
| 936 | 814 |  static inline u16 amd_get_nb_id(int cpu)		{ return 0; } | 
|---|
| 937 | 815 |  static inline u32 amd_get_nodes_per_socket(void)	{ return 0; } | 
|---|
 | 816 | +static inline bool cpu_has_ibpb_brtype_microcode(void)	{ return false; }  | 
|---|
 | 817 | +static inline void amd_clear_divider(void)		{ }  | 
|---|
| 938 | 818 |  #endif | 
|---|
| 939 | 819 |   | 
|---|
| 940 | 820 |  static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves) | 
|---|
| .. | .. | 
|---|
| 953 | 833 |  } | 
|---|
| 954 | 834 |   | 
|---|
| 955 | 835 |  extern unsigned long arch_align_stack(unsigned long sp); | 
|---|
| 956 |  | -extern void free_init_pages(char *what, unsigned long begin, unsigned long end);  | 
|---|
| 957 |  | -extern void free_kernel_image_pages(void *begin, void *end);  | 
|---|
 | 836 | +void free_init_pages(const char *what, unsigned long begin, unsigned long end);  | 
|---|
 | 837 | +extern void free_kernel_image_pages(const char *what, void *begin, void *end);  | 
|---|
| 958 | 838 |   | 
|---|
| 959 | 839 |  void default_idle(void); | 
|---|
| 960 | 840 |  #ifdef	CONFIG_XEN | 
|---|
| .. | .. | 
|---|
| 963 | 843 |  #define xen_set_default_idle 0 | 
|---|
| 964 | 844 |  #endif | 
|---|
| 965 | 845 |   | 
|---|
| 966 |  | -void stop_this_cpu(void *dummy);  | 
|---|
| 967 |  | -void df_debug(struct pt_regs *regs, long error_code);  | 
|---|
| 968 |  | -void microcode_check(void);  | 
|---|
 | 846 | +void __noreturn stop_this_cpu(void *dummy);  | 
|---|
 | 847 | +void microcode_check(struct cpuinfo_x86 *prev_info);  | 
|---|
 | 848 | +void store_cpu_caps(struct cpuinfo_x86 *info);  | 
|---|
| 969 | 849 |   | 
|---|
| 970 | 850 |  enum l1tf_mitigations { | 
|---|
| 971 | 851 |  	L1TF_MITIGATION_OFF, | 
|---|
| .. | .. | 
|---|
| 984 | 864 |  	MDS_MITIGATION_VMWERV, | 
|---|
| 985 | 865 |  }; | 
|---|
| 986 | 866 |   | 
|---|
| 987 |  | -enum taa_mitigations {  | 
|---|
| 988 |  | -	TAA_MITIGATION_OFF,  | 
|---|
| 989 |  | -	TAA_MITIGATION_UCODE_NEEDED,  | 
|---|
| 990 |  | -	TAA_MITIGATION_VERW,  | 
|---|
| 991 |  | -	TAA_MITIGATION_TSX_DISABLED,  | 
|---|
| 992 |  | -};  | 
|---|
 | 867 | +extern bool gds_ucode_mitigated(void);  | 
|---|
| 993 | 868 |   | 
|---|
| 994 | 869 |  #endif /* _ASM_X86_PROCESSOR_H */ | 
|---|