| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * handle transition of Linux booting another kernel |
|---|
| 3 | 4 | * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com> |
|---|
| 4 | | - * |
|---|
| 5 | | - * This source code is licensed under the GNU General Public License, |
|---|
| 6 | | - * Version 2. See the file COPYING for more details. |
|---|
| 7 | 5 | */ |
|---|
| 8 | 6 | |
|---|
| 9 | 7 | #define pr_fmt(fmt) "kexec: " fmt |
|---|
| .. | .. |
|---|
| 18 | 16 | #include <linux/io.h> |
|---|
| 19 | 17 | #include <linux/suspend.h> |
|---|
| 20 | 18 | #include <linux/vmalloc.h> |
|---|
| 19 | +#include <linux/efi.h> |
|---|
| 21 | 20 | |
|---|
| 22 | 21 | #include <asm/init.h> |
|---|
| 23 | | -#include <asm/pgtable.h> |
|---|
| 24 | 22 | #include <asm/tlbflush.h> |
|---|
| 25 | 23 | #include <asm/mmu_context.h> |
|---|
| 26 | 24 | #include <asm/io_apic.h> |
|---|
| .. | .. |
|---|
| 29 | 27 | #include <asm/setup.h> |
|---|
| 30 | 28 | #include <asm/set_memory.h> |
|---|
| 31 | 29 | |
|---|
| 30 | +#ifdef CONFIG_ACPI |
|---|
| 31 | +/* |
|---|
| 32 | + * Used while adding mapping for ACPI tables. |
|---|
| 33 | + * Can be reused when other iomem regions need be mapped |
|---|
| 34 | + */ |
|---|
| 35 | +struct init_pgtable_data { |
|---|
| 36 | + struct x86_mapping_info *info; |
|---|
| 37 | + pgd_t *level4p; |
|---|
| 38 | +}; |
|---|
| 39 | + |
|---|
| 40 | +static int mem_region_callback(struct resource *res, void *arg) |
|---|
| 41 | +{ |
|---|
| 42 | + struct init_pgtable_data *data = arg; |
|---|
| 43 | + unsigned long mstart, mend; |
|---|
| 44 | + |
|---|
| 45 | + mstart = res->start; |
|---|
| 46 | + mend = mstart + resource_size(res) - 1; |
|---|
| 47 | + |
|---|
| 48 | + return kernel_ident_mapping_init(data->info, data->level4p, mstart, mend); |
|---|
| 49 | +} |
|---|
| 50 | + |
|---|
| 51 | +static int |
|---|
| 52 | +map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p) |
|---|
| 53 | +{ |
|---|
| 54 | + struct init_pgtable_data data; |
|---|
| 55 | + unsigned long flags; |
|---|
| 56 | + int ret; |
|---|
| 57 | + |
|---|
| 58 | + data.info = info; |
|---|
| 59 | + data.level4p = level4p; |
|---|
| 60 | + flags = IORESOURCE_MEM | IORESOURCE_BUSY; |
|---|
| 61 | + |
|---|
| 62 | + ret = walk_iomem_res_desc(IORES_DESC_ACPI_TABLES, flags, 0, -1, |
|---|
| 63 | + &data, mem_region_callback); |
|---|
| 64 | + if (ret && ret != -EINVAL) |
|---|
| 65 | + return ret; |
|---|
| 66 | + |
|---|
| 67 | + /* ACPI tables could be located in ACPI Non-volatile Storage region */ |
|---|
| 68 | + ret = walk_iomem_res_desc(IORES_DESC_ACPI_NV_STORAGE, flags, 0, -1, |
|---|
| 69 | + &data, mem_region_callback); |
|---|
| 70 | + if (ret && ret != -EINVAL) |
|---|
| 71 | + return ret; |
|---|
| 72 | + |
|---|
| 73 | + return 0; |
|---|
| 74 | +} |
|---|
| 75 | +#else |
|---|
| 76 | +static int map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p) { return 0; } |
|---|
| 77 | +#endif |
|---|
| 78 | + |
|---|
| 32 | 79 | #ifdef CONFIG_KEXEC_FILE |
|---|
| 33 | 80 | const struct kexec_file_ops * const kexec_file_loaders[] = { |
|---|
| 34 | 81 | &kexec_bzImage64_ops, |
|---|
| 35 | 82 | NULL |
|---|
| 36 | 83 | }; |
|---|
| 37 | 84 | #endif |
|---|
| 85 | + |
|---|
| 86 | +static int |
|---|
| 87 | +map_efi_systab(struct x86_mapping_info *info, pgd_t *level4p) |
|---|
| 88 | +{ |
|---|
| 89 | +#ifdef CONFIG_EFI |
|---|
| 90 | + unsigned long mstart, mend; |
|---|
| 91 | + |
|---|
| 92 | + if (!efi_enabled(EFI_BOOT)) |
|---|
| 93 | + return 0; |
|---|
| 94 | + |
|---|
| 95 | + mstart = (boot_params.efi_info.efi_systab | |
|---|
| 96 | + ((u64)boot_params.efi_info.efi_systab_hi<<32)); |
|---|
| 97 | + |
|---|
| 98 | + if (efi_enabled(EFI_64BIT)) |
|---|
| 99 | + mend = mstart + sizeof(efi_system_table_64_t); |
|---|
| 100 | + else |
|---|
| 101 | + mend = mstart + sizeof(efi_system_table_32_t); |
|---|
| 102 | + |
|---|
| 103 | + if (!mstart) |
|---|
| 104 | + return 0; |
|---|
| 105 | + |
|---|
| 106 | + return kernel_ident_mapping_init(info, level4p, mstart, mend); |
|---|
| 107 | +#endif |
|---|
| 108 | + return 0; |
|---|
| 109 | +} |
|---|
| 38 | 110 | |
|---|
| 39 | 111 | static void free_transition_pgtable(struct kimage *image) |
|---|
| 40 | 112 | { |
|---|
| .. | .. |
|---|
| 50 | 122 | |
|---|
| 51 | 123 | static int init_transition_pgtable(struct kimage *image, pgd_t *pgd) |
|---|
| 52 | 124 | { |
|---|
| 125 | + pgprot_t prot = PAGE_KERNEL_EXEC_NOENC; |
|---|
| 126 | + unsigned long vaddr, paddr; |
|---|
| 127 | + int result = -ENOMEM; |
|---|
| 53 | 128 | p4d_t *p4d; |
|---|
| 54 | 129 | pud_t *pud; |
|---|
| 55 | 130 | pmd_t *pmd; |
|---|
| 56 | 131 | pte_t *pte; |
|---|
| 57 | | - unsigned long vaddr, paddr; |
|---|
| 58 | | - int result = -ENOMEM; |
|---|
| 59 | 132 | |
|---|
| 60 | 133 | vaddr = (unsigned long)relocate_kernel; |
|---|
| 61 | 134 | paddr = __pa(page_address(image->control_code_page)+PAGE_SIZE); |
|---|
| .. | .. |
|---|
| 92 | 165 | set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE)); |
|---|
| 93 | 166 | } |
|---|
| 94 | 167 | pte = pte_offset_kernel(pmd, vaddr); |
|---|
| 95 | | - set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC_NOENC)); |
|---|
| 168 | + |
|---|
| 169 | + if (sev_active()) |
|---|
| 170 | + prot = PAGE_KERNEL_EXEC; |
|---|
| 171 | + |
|---|
| 172 | + set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, prot)); |
|---|
| 96 | 173 | return 0; |
|---|
| 97 | 174 | err: |
|---|
| 98 | 175 | return result; |
|---|
| .. | .. |
|---|
| 129 | 206 | level4p = (pgd_t *)__va(start_pgtable); |
|---|
| 130 | 207 | clear_page(level4p); |
|---|
| 131 | 208 | |
|---|
| 209 | + if (sev_active()) { |
|---|
| 210 | + info.page_flag |= _PAGE_ENC; |
|---|
| 211 | + info.kernpg_flag |= _PAGE_ENC; |
|---|
| 212 | + } |
|---|
| 213 | + |
|---|
| 132 | 214 | if (direct_gbpages) |
|---|
| 133 | 215 | info.direct_gbpages = true; |
|---|
| 134 | 216 | |
|---|
| .. | .. |
|---|
| 158 | 240 | if (result) |
|---|
| 159 | 241 | return result; |
|---|
| 160 | 242 | } |
|---|
| 243 | + |
|---|
| 244 | + /* |
|---|
| 245 | + * Prepare EFI systab and ACPI tables for kexec kernel since they are |
|---|
| 246 | + * not covered by pfn_mapped. |
|---|
| 247 | + */ |
|---|
| 248 | + result = map_efi_systab(&info, level4p); |
|---|
| 249 | + if (result) |
|---|
| 250 | + return result; |
|---|
| 251 | + |
|---|
| 252 | + result = map_acpi_tables(&info, level4p); |
|---|
| 253 | + if (result) |
|---|
| 254 | + return result; |
|---|
| 161 | 255 | |
|---|
| 162 | 256 | return init_transition_pgtable(image, level4p); |
|---|
| 163 | 257 | } |
|---|
| .. | .. |
|---|
| 203 | 297 | ); |
|---|
| 204 | 298 | } |
|---|
| 205 | 299 | |
|---|
| 206 | | -#ifdef CONFIG_KEXEC_FILE |
|---|
| 207 | | -/* Update purgatory as needed after various image segments have been prepared */ |
|---|
| 208 | | -static int arch_update_purgatory(struct kimage *image) |
|---|
| 209 | | -{ |
|---|
| 210 | | - int ret = 0; |
|---|
| 211 | | - |
|---|
| 212 | | - if (!image->file_mode) |
|---|
| 213 | | - return 0; |
|---|
| 214 | | - |
|---|
| 215 | | - /* Setup copying of backup region */ |
|---|
| 216 | | - if (image->type == KEXEC_TYPE_CRASH) { |
|---|
| 217 | | - ret = kexec_purgatory_get_set_symbol(image, |
|---|
| 218 | | - "purgatory_backup_dest", |
|---|
| 219 | | - &image->arch.backup_load_addr, |
|---|
| 220 | | - sizeof(image->arch.backup_load_addr), 0); |
|---|
| 221 | | - if (ret) |
|---|
| 222 | | - return ret; |
|---|
| 223 | | - |
|---|
| 224 | | - ret = kexec_purgatory_get_set_symbol(image, |
|---|
| 225 | | - "purgatory_backup_src", |
|---|
| 226 | | - &image->arch.backup_src_start, |
|---|
| 227 | | - sizeof(image->arch.backup_src_start), 0); |
|---|
| 228 | | - if (ret) |
|---|
| 229 | | - return ret; |
|---|
| 230 | | - |
|---|
| 231 | | - ret = kexec_purgatory_get_set_symbol(image, |
|---|
| 232 | | - "purgatory_backup_sz", |
|---|
| 233 | | - &image->arch.backup_src_sz, |
|---|
| 234 | | - sizeof(image->arch.backup_src_sz), 0); |
|---|
| 235 | | - if (ret) |
|---|
| 236 | | - return ret; |
|---|
| 237 | | - } |
|---|
| 238 | | - |
|---|
| 239 | | - return ret; |
|---|
| 240 | | -} |
|---|
| 241 | | -#else /* !CONFIG_KEXEC_FILE */ |
|---|
| 242 | | -static inline int arch_update_purgatory(struct kimage *image) |
|---|
| 243 | | -{ |
|---|
| 244 | | - return 0; |
|---|
| 245 | | -} |
|---|
| 246 | | -#endif /* CONFIG_KEXEC_FILE */ |
|---|
| 247 | | - |
|---|
| 248 | 300 | int machine_kexec_prepare(struct kimage *image) |
|---|
| 249 | 301 | { |
|---|
| 250 | 302 | unsigned long start_pgtable; |
|---|
| .. | .. |
|---|
| 255 | 307 | |
|---|
| 256 | 308 | /* Setup the identity mapped 64bit page table */ |
|---|
| 257 | 309 | result = init_pgtable(image, start_pgtable); |
|---|
| 258 | | - if (result) |
|---|
| 259 | | - return result; |
|---|
| 260 | | - |
|---|
| 261 | | - /* update purgatory as needed */ |
|---|
| 262 | | - result = arch_update_purgatory(image); |
|---|
| 263 | 310 | if (result) |
|---|
| 264 | 311 | return result; |
|---|
| 265 | 312 | |
|---|
| .. | .. |
|---|
| 348 | 395 | #endif |
|---|
| 349 | 396 | |
|---|
| 350 | 397 | __ftrace_enabled_restore(save_ftrace_enabled); |
|---|
| 351 | | -} |
|---|
| 352 | | - |
|---|
| 353 | | -void arch_crash_save_vmcoreinfo(void) |
|---|
| 354 | | -{ |
|---|
| 355 | | - VMCOREINFO_NUMBER(phys_base); |
|---|
| 356 | | - VMCOREINFO_SYMBOL(init_top_pgt); |
|---|
| 357 | | - vmcoreinfo_append_str("NUMBER(pgtable_l5_enabled)=%d\n", |
|---|
| 358 | | - pgtable_l5_enabled()); |
|---|
| 359 | | - |
|---|
| 360 | | -#ifdef CONFIG_NUMA |
|---|
| 361 | | - VMCOREINFO_SYMBOL(node_data); |
|---|
| 362 | | - VMCOREINFO_LENGTH(node_data, MAX_NUMNODES); |
|---|
| 363 | | -#endif |
|---|
| 364 | | - vmcoreinfo_append_str("KERNELOFFSET=%lx\n", |
|---|
| 365 | | - kaslr_offset()); |
|---|
| 366 | | - VMCOREINFO_NUMBER(KERNEL_IMAGE_SIZE); |
|---|
| 367 | 398 | } |
|---|
| 368 | 399 | |
|---|
| 369 | 400 | /* arch-dependent functionality related to kexec file-based syscall */ |
|---|
| .. | .. |
|---|
| 556 | 587 | kexec_mark_crashkres(false); |
|---|
| 557 | 588 | } |
|---|
| 558 | 589 | |
|---|
| 590 | +/* |
|---|
| 591 | + * During a traditional boot under SME, SME will encrypt the kernel, |
|---|
| 592 | + * so the SME kexec kernel also needs to be un-encrypted in order to |
|---|
| 593 | + * replicate a normal SME boot. |
|---|
| 594 | + * |
|---|
| 595 | + * During a traditional boot under SEV, the kernel has already been |
|---|
| 596 | + * loaded encrypted, so the SEV kexec kernel needs to be encrypted in |
|---|
| 597 | + * order to replicate a normal SEV boot. |
|---|
| 598 | + */ |
|---|
| 559 | 599 | int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, gfp_t gfp) |
|---|
| 560 | 600 | { |
|---|
| 601 | + if (sev_active()) |
|---|
| 602 | + return 0; |
|---|
| 603 | + |
|---|
| 561 | 604 | /* |
|---|
| 562 | 605 | * If SME is active we need to be sure that kexec pages are |
|---|
| 563 | 606 | * not encrypted because when we boot to the new kernel the |
|---|
| .. | .. |
|---|
| 568 | 611 | |
|---|
| 569 | 612 | void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages) |
|---|
| 570 | 613 | { |
|---|
| 614 | + if (sev_active()) |
|---|
| 615 | + return; |
|---|
| 616 | + |
|---|
| 571 | 617 | /* |
|---|
| 572 | 618 | * If SME is active we need to reset the pages back to being |
|---|
| 573 | 619 | * an encrypted mapping before freeing them. |
|---|