.. | .. |
---|
23 | 23 | #include <asm/cpufeatures.h> |
---|
24 | 24 | #include <asm/percpu.h> |
---|
25 | 25 | #include <asm/nops.h> |
---|
| 26 | +#include <asm/nospec-branch.h> |
---|
26 | 27 | #include <asm/bootparam.h> |
---|
27 | 28 | #include <asm/export.h> |
---|
28 | 29 | #include <asm/pgtable_32.h> |
---|
.. | .. |
---|
64 | 65 | * can. |
---|
65 | 66 | */ |
---|
66 | 67 | __HEAD |
---|
67 | | -ENTRY(startup_32) |
---|
| 68 | +SYM_CODE_START(startup_32) |
---|
68 | 69 | movl pa(initial_stack),%ecx |
---|
69 | 70 | |
---|
70 | | - /* test KEEP_SEGMENTS flag to see if the bootloader is asking |
---|
71 | | - us to not reload segments */ |
---|
72 | | - testb $KEEP_SEGMENTS, BP_loadflags(%esi) |
---|
73 | | - jnz 2f |
---|
74 | | - |
---|
75 | 71 | /* |
---|
76 | 72 | * Set segments to known values. |
---|
77 | 73 | */ |
---|
.. | .. |
---|
82 | 78 | movl %eax,%fs |
---|
83 | 79 | movl %eax,%gs |
---|
84 | 80 | movl %eax,%ss |
---|
85 | | -2: |
---|
86 | 81 | leal -__PAGE_OFFSET(%ecx),%esp |
---|
87 | 82 | |
---|
88 | 83 | /* |
---|
.. | .. |
---|
140 | 135 | movl %eax,pa(initial_page_table+0xffc) |
---|
141 | 136 | #endif |
---|
142 | 137 | |
---|
143 | | -#ifdef CONFIG_PARAVIRT |
---|
144 | | - /* This is can only trip for a broken bootloader... */ |
---|
145 | | - cmpw $0x207, pa(boot_params + BP_version) |
---|
146 | | - jb .Ldefault_entry |
---|
147 | | - |
---|
148 | | - /* Paravirt-compatible boot parameters. Look to see what architecture |
---|
149 | | - we're booting under. */ |
---|
150 | | - movl pa(boot_params + BP_hardware_subarch), %eax |
---|
151 | | - cmpl $num_subarch_entries, %eax |
---|
152 | | - jae .Lbad_subarch |
---|
153 | | - |
---|
154 | | - movl pa(subarch_entries)(,%eax,4), %eax |
---|
155 | | - subl $__PAGE_OFFSET, %eax |
---|
156 | | - jmp *%eax |
---|
157 | | - |
---|
158 | | -.Lbad_subarch: |
---|
159 | | -WEAK(xen_entry) |
---|
160 | | - /* Unknown implementation; there's really |
---|
161 | | - nothing we can do at this point. */ |
---|
162 | | - ud2a |
---|
163 | | - |
---|
164 | | - __INITDATA |
---|
165 | | - |
---|
166 | | -subarch_entries: |
---|
167 | | - .long .Ldefault_entry /* normal x86/PC */ |
---|
168 | | - .long xen_entry /* Xen hypervisor */ |
---|
169 | | - .long .Ldefault_entry /* Moorestown MID */ |
---|
170 | | -num_subarch_entries = (. - subarch_entries) / 4 |
---|
171 | | -.previous |
---|
172 | | -#else |
---|
173 | 138 | jmp .Ldefault_entry |
---|
174 | | -#endif /* CONFIG_PARAVIRT */ |
---|
| 139 | +SYM_CODE_END(startup_32) |
---|
175 | 140 | |
---|
176 | 141 | #ifdef CONFIG_HOTPLUG_CPU |
---|
177 | 142 | /* |
---|
.. | .. |
---|
179 | 144 | * up already except stack. We just set up stack here. Then call |
---|
180 | 145 | * start_secondary(). |
---|
181 | 146 | */ |
---|
182 | | -ENTRY(start_cpu0) |
---|
| 147 | +SYM_FUNC_START(start_cpu0) |
---|
183 | 148 | movl initial_stack, %ecx |
---|
184 | 149 | movl %ecx, %esp |
---|
185 | 150 | call *(initial_code) |
---|
186 | 151 | 1: jmp 1b |
---|
187 | | -ENDPROC(start_cpu0) |
---|
| 152 | +SYM_FUNC_END(start_cpu0) |
---|
188 | 153 | #endif |
---|
189 | 154 | |
---|
190 | 155 | /* |
---|
.. | .. |
---|
195 | 160 | * If cpu hotplug is not supported then this code can go in init section |
---|
196 | 161 | * which will be freed later |
---|
197 | 162 | */ |
---|
198 | | -ENTRY(startup_32_smp) |
---|
| 163 | +SYM_FUNC_START(startup_32_smp) |
---|
199 | 164 | cld |
---|
200 | 165 | movl $(__BOOT_DS),%eax |
---|
201 | 166 | movl %eax,%ds |
---|
.. | .. |
---|
362 | 327 | |
---|
363 | 328 | call *(initial_code) |
---|
364 | 329 | 1: jmp 1b |
---|
365 | | -ENDPROC(startup_32_smp) |
---|
| 330 | +SYM_FUNC_END(startup_32_smp) |
---|
366 | 331 | |
---|
367 | 332 | #include "verify_cpu.S" |
---|
368 | 333 | |
---|
.. | .. |
---|
390 | 355 | #endif |
---|
391 | 356 | |
---|
392 | 357 | andl $0,setup_once_ref /* Once is enough, thanks */ |
---|
393 | | - ret |
---|
| 358 | + RET |
---|
394 | 359 | |
---|
395 | | -ENTRY(early_idt_handler_array) |
---|
| 360 | +SYM_FUNC_START(early_idt_handler_array) |
---|
396 | 361 | # 36(%esp) %eflags |
---|
397 | 362 | # 32(%esp) %cs |
---|
398 | 363 | # 28(%esp) %eip |
---|
.. | .. |
---|
407 | 372 | i = i + 1 |
---|
408 | 373 | .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc |
---|
409 | 374 | .endr |
---|
410 | | -ENDPROC(early_idt_handler_array) |
---|
| 375 | +SYM_FUNC_END(early_idt_handler_array) |
---|
411 | 376 | |
---|
412 | | -early_idt_handler_common: |
---|
| 377 | +SYM_CODE_START_LOCAL(early_idt_handler_common) |
---|
413 | 378 | /* |
---|
414 | 379 | * The stack is the hardware frame, an error code or zero, and the |
---|
415 | 380 | * vector number. |
---|
.. | .. |
---|
460 | 425 | decl %ss:early_recursion_flag |
---|
461 | 426 | addl $4, %esp /* pop pt_regs->orig_ax */ |
---|
462 | 427 | iret |
---|
463 | | -ENDPROC(early_idt_handler_common) |
---|
| 428 | +SYM_CODE_END(early_idt_handler_common) |
---|
464 | 429 | |
---|
465 | 430 | /* This is the default interrupt "handler" :-) */ |
---|
466 | | -ENTRY(early_ignore_irq) |
---|
| 431 | +SYM_FUNC_START(early_ignore_irq) |
---|
467 | 432 | cld |
---|
468 | 433 | #ifdef CONFIG_PRINTK |
---|
469 | 434 | pushl %eax |
---|
.. | .. |
---|
498 | 463 | hlt_loop: |
---|
499 | 464 | hlt |
---|
500 | 465 | jmp hlt_loop |
---|
501 | | -ENDPROC(early_ignore_irq) |
---|
| 466 | +SYM_FUNC_END(early_ignore_irq) |
---|
502 | 467 | |
---|
503 | 468 | __INITDATA |
---|
504 | 469 | .align 4 |
---|
505 | | -GLOBAL(early_recursion_flag) |
---|
506 | | - .long 0 |
---|
| 470 | +SYM_DATA(early_recursion_flag, .long 0) |
---|
507 | 471 | |
---|
508 | 472 | __REFDATA |
---|
509 | 473 | .align 4 |
---|
510 | | -ENTRY(initial_code) |
---|
511 | | - .long i386_start_kernel |
---|
512 | | -ENTRY(setup_once_ref) |
---|
513 | | - .long setup_once |
---|
| 474 | +SYM_DATA(initial_code, .long i386_start_kernel) |
---|
| 475 | +SYM_DATA(setup_once_ref, .long setup_once) |
---|
514 | 476 | |
---|
515 | 477 | #ifdef CONFIG_PAGE_TABLE_ISOLATION |
---|
516 | 478 | #define PGD_ALIGN (2 * PAGE_SIZE) |
---|
.. | .. |
---|
553 | 515 | __PAGE_ALIGNED_DATA |
---|
554 | 516 | /* Page-aligned for the benefit of paravirt? */ |
---|
555 | 517 | .align PGD_ALIGN |
---|
556 | | -ENTRY(initial_page_table) |
---|
| 518 | +SYM_DATA_START(initial_page_table) |
---|
557 | 519 | .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 /* low identity map */ |
---|
558 | 520 | # if KPMDS == 3 |
---|
559 | 521 | .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 |
---|
.. | .. |
---|
571 | 533 | # error "Kernel PMDs should be 1, 2 or 3" |
---|
572 | 534 | # endif |
---|
573 | 535 | .align PAGE_SIZE /* needs to be page-sized too */ |
---|
| 536 | + |
---|
| 537 | +#ifdef CONFIG_PAGE_TABLE_ISOLATION |
---|
| 538 | + /* |
---|
| 539 | + * PTI needs another page so sync_initial_pagetable() works correctly |
---|
| 540 | + * and does not scribble over the data which is placed behind the |
---|
| 541 | + * actual initial_page_table. See clone_pgd_range(). |
---|
| 542 | + */ |
---|
| 543 | + .fill 1024, 4, 0 |
---|
| 544 | +#endif |
---|
| 545 | + |
---|
| 546 | +SYM_DATA_END(initial_page_table) |
---|
574 | 547 | #endif |
---|
575 | 548 | |
---|
576 | 549 | .data |
---|
577 | 550 | .balign 4 |
---|
578 | | -ENTRY(initial_stack) |
---|
579 | | - /* |
---|
580 | | - * The SIZEOF_PTREGS gap is a convention which helps the in-kernel |
---|
581 | | - * unwinder reliably detect the end of the stack. |
---|
582 | | - */ |
---|
583 | | - .long init_thread_union + THREAD_SIZE - SIZEOF_PTREGS - \ |
---|
584 | | - TOP_OF_KERNEL_STACK_PADDING; |
---|
| 551 | +/* |
---|
| 552 | + * The SIZEOF_PTREGS gap is a convention which helps the in-kernel unwinder |
---|
| 553 | + * reliably detect the end of the stack. |
---|
| 554 | + */ |
---|
| 555 | +SYM_DATA(initial_stack, |
---|
| 556 | + .long init_thread_union + THREAD_SIZE - |
---|
| 557 | + SIZEOF_PTREGS - TOP_OF_KERNEL_STACK_PADDING) |
---|
585 | 558 | |
---|
586 | 559 | __INITRODATA |
---|
587 | 560 | int_msg: |
---|
.. | .. |
---|
597 | 570 | */ |
---|
598 | 571 | |
---|
599 | 572 | .data |
---|
600 | | -.globl boot_gdt_descr |
---|
601 | | - |
---|
602 | 573 | ALIGN |
---|
603 | 574 | # early boot GDT descriptor (must use 1:1 address mapping) |
---|
604 | 575 | .word 0 # 32 bit align gdt_desc.address |
---|
605 | | -boot_gdt_descr: |
---|
| 576 | +SYM_DATA_START_LOCAL(boot_gdt_descr) |
---|
606 | 577 | .word __BOOT_DS+7 |
---|
607 | 578 | .long boot_gdt - __PAGE_OFFSET |
---|
| 579 | +SYM_DATA_END(boot_gdt_descr) |
---|
608 | 580 | |
---|
609 | 581 | # boot GDT descriptor (later on used by CPU#0): |
---|
610 | 582 | .word 0 # 32 bit align gdt_desc.address |
---|
611 | | -ENTRY(early_gdt_descr) |
---|
| 583 | +SYM_DATA_START(early_gdt_descr) |
---|
612 | 584 | .word GDT_ENTRIES*8-1 |
---|
613 | 585 | .long gdt_page /* Overwritten for secondary CPUs */ |
---|
| 586 | +SYM_DATA_END(early_gdt_descr) |
---|
614 | 587 | |
---|
615 | 588 | /* |
---|
616 | 589 | * The boot_gdt must mirror the equivalent in setup.S and is |
---|
617 | 590 | * used only for booting. |
---|
618 | 591 | */ |
---|
619 | 592 | .align L1_CACHE_BYTES |
---|
620 | | -ENTRY(boot_gdt) |
---|
| 593 | +SYM_DATA_START(boot_gdt) |
---|
621 | 594 | .fill GDT_ENTRY_BOOT_CS,8,0 |
---|
622 | 595 | .quad 0x00cf9a000000ffff /* kernel 4GB code at 0x00000000 */ |
---|
623 | 596 | .quad 0x00cf92000000ffff /* kernel 4GB data at 0x00000000 */ |
---|
| 597 | +SYM_DATA_END(boot_gdt) |
---|