| .. | .. |
|---|
| 7 | 7 | #include <linux/swapops.h> |
|---|
| 8 | 8 | #include <linux/kmemleak.h> |
|---|
| 9 | 9 | #include <linux/sched/task.h> |
|---|
| 10 | +#include <linux/sched/mm.h> |
|---|
| 10 | 11 | |
|---|
| 11 | 12 | #include <asm/set_memory.h> |
|---|
| 13 | +#include <asm/cpu_device_id.h> |
|---|
| 12 | 14 | #include <asm/e820/api.h> |
|---|
| 13 | 15 | #include <asm/init.h> |
|---|
| 14 | 16 | #include <asm/page.h> |
|---|
| .. | .. |
|---|
| 26 | 28 | #include <asm/pti.h> |
|---|
| 27 | 29 | #include <asm/text-patching.h> |
|---|
| 28 | 30 | #include <asm/memtype.h> |
|---|
| 31 | +#include <asm/paravirt.h> |
|---|
| 29 | 32 | |
|---|
| 30 | 33 | /* |
|---|
| 31 | 34 | * We need to define the tracepoints somewhere, and tlb.c |
|---|
| .. | .. |
|---|
| 260 | 263 | } |
|---|
| 261 | 264 | } |
|---|
| 262 | 265 | |
|---|
| 266 | +#define INTEL_MATCH(_model) { .vendor = X86_VENDOR_INTEL, \ |
|---|
| 267 | + .family = 6, \ |
|---|
| 268 | + .model = _model, \ |
|---|
| 269 | + } |
|---|
| 270 | +/* |
|---|
| 271 | + * INVLPG may not properly flush Global entries |
|---|
| 272 | + * on these CPUs when PCIDs are enabled. |
|---|
| 273 | + */ |
|---|
| 274 | +static const struct x86_cpu_id invlpg_miss_ids[] = { |
|---|
| 275 | + INTEL_MATCH(INTEL_FAM6_ALDERLAKE ), |
|---|
| 276 | + INTEL_MATCH(INTEL_FAM6_ALDERLAKE_L ), |
|---|
| 277 | + INTEL_MATCH(INTEL_FAM6_ALDERLAKE_N ), |
|---|
| 278 | + INTEL_MATCH(INTEL_FAM6_RAPTORLAKE ), |
|---|
| 279 | + INTEL_MATCH(INTEL_FAM6_RAPTORLAKE_P), |
|---|
| 280 | + INTEL_MATCH(INTEL_FAM6_RAPTORLAKE_S), |
|---|
| 281 | + {} |
|---|
| 282 | +}; |
|---|
| 283 | + |
|---|
| 263 | 284 | static void setup_pcid(void) |
|---|
| 264 | 285 | { |
|---|
| 265 | 286 | if (!IS_ENABLED(CONFIG_X86_64)) |
|---|
| .. | .. |
|---|
| 267 | 288 | |
|---|
| 268 | 289 | if (!boot_cpu_has(X86_FEATURE_PCID)) |
|---|
| 269 | 290 | return; |
|---|
| 291 | + |
|---|
| 292 | + if (x86_match_cpu(invlpg_miss_ids)) { |
|---|
| 293 | + pr_info("Incomplete global flushes, disabling PCID"); |
|---|
| 294 | + setup_clear_cpu_cap(X86_FEATURE_PCID); |
|---|
| 295 | + return; |
|---|
| 296 | + } |
|---|
| 270 | 297 | |
|---|
| 271 | 298 | if (boot_cpu_has(X86_FEATURE_PGE)) { |
|---|
| 272 | 299 | /* |
|---|
| .. | .. |
|---|
| 785 | 812 | spinlock_t *ptl; |
|---|
| 786 | 813 | pte_t *ptep; |
|---|
| 787 | 814 | |
|---|
| 788 | | - poking_mm = copy_init_mm(); |
|---|
| 815 | + poking_mm = mm_alloc(); |
|---|
| 789 | 816 | BUG_ON(!poking_mm); |
|---|
| 790 | 817 | |
|---|
| 818 | + /* Xen PV guests need the PGD to be pinned. */ |
|---|
| 819 | + paravirt_arch_dup_mmap(NULL, poking_mm); |
|---|
| 820 | + |
|---|
| 791 | 821 | /* |
|---|
| 792 | 822 | * Randomize the poking address, but make sure that the following page |
|---|
| 793 | 823 | * will be mapped at the same PMD. We need 2 pages, so find space for 3, |
|---|