| .. | .. |
|---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * relocate_kernel.S - put the kernel image in place to boot |
|---|
| 3 | 4 | * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com> |
|---|
| 4 | | - * |
|---|
| 5 | | - * This source code is licensed under the GNU General Public License, |
|---|
| 6 | | - * Version 2. See the file COPYING for more details. |
|---|
| 7 | 5 | */ |
|---|
| 8 | 6 | |
|---|
| 9 | 7 | #include <linux/linkage.h> |
|---|
| .. | .. |
|---|
| 11 | 9 | #include <asm/kexec.h> |
|---|
| 12 | 10 | #include <asm/processor-flags.h> |
|---|
| 13 | 11 | #include <asm/pgtable_types.h> |
|---|
| 12 | +#include <asm/nospec-branch.h> |
|---|
| 13 | +#include <asm/unwind_hints.h> |
|---|
| 14 | 14 | |
|---|
| 15 | 15 | /* |
|---|
| 16 | | - * Must be relocatable PIC code callable as a C function |
|---|
| 16 | + * Must be relocatable PIC code callable as a C function, in particular |
|---|
| 17 | + * there must be a plain RET and not jump to return thunk. |
|---|
| 17 | 18 | */ |
|---|
| 18 | 19 | |
|---|
| 19 | 20 | #define PTR(x) (x << 3) |
|---|
| .. | .. |
|---|
| 40 | 41 | .text |
|---|
| 41 | 42 | .align PAGE_SIZE |
|---|
| 42 | 43 | .code64 |
|---|
| 43 | | - .globl relocate_kernel |
|---|
| 44 | | -relocate_kernel: |
|---|
| 44 | +SYM_CODE_START_NOALIGN(relocate_kernel) |
|---|
| 45 | + UNWIND_HINT_EMPTY |
|---|
| 45 | 46 | /* |
|---|
| 46 | 47 | * %rdi indirection_page |
|---|
| 47 | 48 | * %rsi page_list |
|---|
| .. | .. |
|---|
| 104 | 105 | /* jump to identity mapped page */ |
|---|
| 105 | 106 | addq $(identity_mapped - relocate_kernel), %r8 |
|---|
| 106 | 107 | pushq %r8 |
|---|
| 108 | + ANNOTATE_UNRET_SAFE |
|---|
| 107 | 109 | ret |
|---|
| 110 | + int3 |
|---|
| 111 | +SYM_CODE_END(relocate_kernel) |
|---|
| 108 | 112 | |
|---|
| 109 | | -identity_mapped: |
|---|
| 113 | +SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) |
|---|
| 114 | + UNWIND_HINT_EMPTY |
|---|
| 110 | 115 | /* set return address to 0 if not preserving context */ |
|---|
| 111 | 116 | pushq $0 |
|---|
| 112 | 117 | /* store the start address on the stack */ |
|---|
| .. | .. |
|---|
| 189 | 194 | xorl %r14d, %r14d |
|---|
| 190 | 195 | xorl %r15d, %r15d |
|---|
| 191 | 196 | |
|---|
| 197 | + ANNOTATE_UNRET_SAFE |
|---|
| 192 | 198 | ret |
|---|
| 199 | + int3 |
|---|
| 193 | 200 | |
|---|
| 194 | 201 | 1: |
|---|
| 195 | 202 | popq %rdx |
|---|
| 196 | 203 | leaq PAGE_SIZE(%r10), %rsp |
|---|
| 204 | + ANNOTATE_RETPOLINE_SAFE |
|---|
| 197 | 205 | call *%rdx |
|---|
| 198 | 206 | |
|---|
| 199 | 207 | /* get the re-entry point of the peer system */ |
|---|
| 200 | 208 | movq 0(%rsp), %rbp |
|---|
| 201 | | - call 1f |
|---|
| 202 | | -1: |
|---|
| 203 | | - popq %r8 |
|---|
| 204 | | - subq $(1b - relocate_kernel), %r8 |
|---|
| 209 | + leaq relocate_kernel(%rip), %r8 |
|---|
| 205 | 210 | movq CP_PA_SWAP_PAGE(%r8), %r10 |
|---|
| 206 | 211 | movq CP_PA_BACKUP_PAGES_MAP(%r8), %rdi |
|---|
| 207 | 212 | movq CP_PA_TABLE_PAGE(%r8), %rax |
|---|
| .. | .. |
|---|
| 210 | 215 | call swap_pages |
|---|
| 211 | 216 | movq $virtual_mapped, %rax |
|---|
| 212 | 217 | pushq %rax |
|---|
| 218 | + ANNOTATE_UNRET_SAFE |
|---|
| 213 | 219 | ret |
|---|
| 220 | + int3 |
|---|
| 221 | +SYM_CODE_END(identity_mapped) |
|---|
| 214 | 222 | |
|---|
| 215 | | -virtual_mapped: |
|---|
| 223 | +SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) |
|---|
| 224 | + UNWIND_HINT_EMPTY |
|---|
| 216 | 225 | movq RSP(%r8), %rsp |
|---|
| 217 | 226 | movq CR4(%r8), %rax |
|---|
| 218 | 227 | movq %rax, %cr4 |
|---|
| .. | .. |
|---|
| 229 | 238 | popq %r12 |
|---|
| 230 | 239 | popq %rbp |
|---|
| 231 | 240 | popq %rbx |
|---|
| 241 | + ANNOTATE_UNRET_SAFE |
|---|
| 232 | 242 | ret |
|---|
| 243 | + int3 |
|---|
| 244 | +SYM_CODE_END(virtual_mapped) |
|---|
| 233 | 245 | |
|---|
| 234 | 246 | /* Do the copies */ |
|---|
| 235 | | -swap_pages: |
|---|
| 247 | +SYM_CODE_START_LOCAL_NOALIGN(swap_pages) |
|---|
| 248 | + UNWIND_HINT_EMPTY |
|---|
| 236 | 249 | movq %rdi, %rcx /* Put the page_list in %rcx */ |
|---|
| 237 | 250 | xorl %edi, %edi |
|---|
| 238 | 251 | xorl %esi, %esi |
|---|
| .. | .. |
|---|
| 284 | 297 | lea PAGE_SIZE(%rax), %rsi |
|---|
| 285 | 298 | jmp 0b |
|---|
| 286 | 299 | 3: |
|---|
| 300 | + ANNOTATE_UNRET_SAFE |
|---|
| 287 | 301 | ret |
|---|
| 302 | + int3 |
|---|
| 303 | +SYM_CODE_END(swap_pages) |
|---|
| 288 | 304 | |
|---|
| 289 | 305 | .globl kexec_control_code_size |
|---|
| 290 | 306 | .set kexec_control_code_size, . - relocate_kernel |
|---|