| .. | .. |
|---|
| 3 | 3 | #include <linux/slab.h> |
|---|
| 4 | 4 | #include <linux/memblock.h> |
|---|
| 5 | 5 | #include <linux/mem_encrypt.h> |
|---|
| 6 | +#include <linux/pgtable.h> |
|---|
| 6 | 7 | |
|---|
| 7 | 8 | #include <asm/set_memory.h> |
|---|
| 8 | | -#include <asm/pgtable.h> |
|---|
| 9 | 9 | #include <asm/realmode.h> |
|---|
| 10 | 10 | #include <asm/tlbflush.h> |
|---|
| 11 | +#include <asm/crash.h> |
|---|
| 12 | +#include <asm/sev-es.h> |
|---|
| 11 | 13 | |
|---|
| 12 | 14 | struct real_mode_header *real_mode_header; |
|---|
| 13 | 15 | u32 *trampoline_cr4_features; |
|---|
| .. | .. |
|---|
| 15 | 17 | /* Hold the pgd entry used on booting additional CPUs */ |
|---|
| 16 | 18 | pgd_t trampoline_pgd_entry; |
|---|
| 17 | 19 | |
|---|
| 18 | | -void __init set_real_mode_mem(phys_addr_t mem, size_t size) |
|---|
| 20 | +void load_trampoline_pgtable(void) |
|---|
| 19 | 21 | { |
|---|
| 20 | | - void *base = __va(mem); |
|---|
| 22 | +#ifdef CONFIG_X86_32 |
|---|
| 23 | + load_cr3(initial_page_table); |
|---|
| 24 | +#else |
|---|
| 25 | + /* |
|---|
| 26 | + * This function is called before exiting to real-mode and that will |
|---|
| 27 | + * fail with CR4.PCIDE still set. |
|---|
| 28 | + */ |
|---|
| 29 | + if (boot_cpu_has(X86_FEATURE_PCID)) |
|---|
| 30 | + cr4_clear_bits(X86_CR4_PCIDE); |
|---|
| 21 | 31 | |
|---|
| 22 | | - real_mode_header = (struct real_mode_header *) base; |
|---|
| 32 | + write_cr3(real_mode_header->trampoline_pgd); |
|---|
| 33 | +#endif |
|---|
| 34 | + |
|---|
| 35 | + /* |
|---|
| 36 | + * The CR3 write above will not flush global TLB entries. |
|---|
| 37 | + * Stale, global entries from previous page tables may still be |
|---|
| 38 | + * present. Flush those stale entries. |
|---|
| 39 | + * |
|---|
| 40 | + * This ensures that memory accessed while running with |
|---|
| 41 | + * trampoline_pgd is *actually* mapped into trampoline_pgd. |
|---|
| 42 | + */ |
|---|
| 43 | + __flush_tlb_all(); |
|---|
| 23 | 44 | } |
|---|
| 24 | 45 | |
|---|
| 25 | 46 | void __init reserve_real_mode(void) |
|---|
| .. | .. |
|---|
| 40 | 61 | } |
|---|
| 41 | 62 | |
|---|
| 42 | 63 | memblock_reserve(mem, size); |
|---|
| 43 | | - set_real_mode_mem(mem, size); |
|---|
| 64 | + set_real_mode_mem(mem); |
|---|
| 65 | + crash_reserve_low_1M(); |
|---|
| 66 | +} |
|---|
| 67 | + |
|---|
| 68 | +static void sme_sev_setup_real_mode(struct trampoline_header *th) |
|---|
| 69 | +{ |
|---|
| 70 | +#ifdef CONFIG_AMD_MEM_ENCRYPT |
|---|
| 71 | + if (sme_active()) |
|---|
| 72 | + th->flags |= TH_FLAGS_SME_ACTIVE; |
|---|
| 73 | + |
|---|
| 74 | + if (sev_es_active()) { |
|---|
| 75 | + /* |
|---|
| 76 | + * Skip the call to verify_cpu() in secondary_startup_64 as it |
|---|
| 77 | + * will cause #VC exceptions when the AP can't handle them yet. |
|---|
| 78 | + */ |
|---|
| 79 | + th->start = (u64) secondary_startup_64_no_verify; |
|---|
| 80 | + |
|---|
| 81 | + if (sev_es_setup_ap_jump_table(real_mode_header)) |
|---|
| 82 | + panic("Failed to get/update SEV-ES AP Jump Table"); |
|---|
| 83 | + } |
|---|
| 84 | +#endif |
|---|
| 44 | 85 | } |
|---|
| 45 | 86 | |
|---|
| 46 | 87 | static void __init setup_real_mode(void) |
|---|
| .. | .. |
|---|
| 110 | 151 | *trampoline_cr4_features = mmu_cr4_features; |
|---|
| 111 | 152 | |
|---|
| 112 | 153 | trampoline_header->flags = 0; |
|---|
| 113 | | - if (sme_active()) |
|---|
| 114 | | - trampoline_header->flags |= TH_FLAGS_SME_ACTIVE; |
|---|
| 115 | 154 | |
|---|
| 116 | 155 | trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd); |
|---|
| 117 | 156 | |
|---|
| .. | .. |
|---|
| 126 | 165 | for (i = pgd_index(__PAGE_OFFSET); i < PTRS_PER_PGD; i++) |
|---|
| 127 | 166 | trampoline_pgd[i] = init_top_pgt[i].pgd; |
|---|
| 128 | 167 | #endif |
|---|
| 168 | + |
|---|
| 169 | + sme_sev_setup_real_mode(trampoline_header); |
|---|
| 129 | 170 | } |
|---|
| 130 | 171 | |
|---|
| 131 | 172 | /* |
|---|