~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,51 +1,47 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
3		- *
4		- * This program is free software; you can redistribute it and/or
5		- * modify it under the terms of the GNU General Public License
6		- * as published by the Free Software Foundation; either version 2
7		- * of the License, or (at your option) any later version.
8		- *
9		- * This program is distributed in the hope that it will be useful,
10		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12		- * GNU General Public License for more details.
13		- *
14		- * You should have received a copy of the GNU General Public License
15		- * along with this program; if not, see <http://www.gnu.org/licenses/>.
16	4	*/
17	5
18	6	#include <string.h>
19	7	#include <stdlib.h>
	8	+#include <inttypes.h>
	9	+#include <sys/mman.h>
20	10
21	11	#include "builtin.h"
22		-#include "check.h"
23		-#include "elf.h"
24		-#include "special.h"
	12	+#include "cfi.h"
25	13	#include "arch.h"
	14	+#include "check.h"
	15	+#include "special.h"
26	16	#include "warn.h"
	17	+#include "arch_elf.h"
27	18
	19	+#include <linux/objtool.h>
28	20	#include <linux/hashtable.h>
29	21	#include <linux/kernel.h>
30		-
31		-#define FAKE_JUMP_OFFSET -1
	22	+#include <linux/static_call_types.h>
32	23
33	24	struct alternative {
34	25	struct list_head list;
35	26	struct instruction *insn;
	27	+ bool skip_orig;
36	28	};
37	29
38		-const char *objname;
39		-struct cfi_state initial_func_cfi;
	30	+static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
	31	+
	32	+static struct cfi_init_state initial_func_cfi;
	33	+static struct cfi_state init_cfi;
	34	+static struct cfi_state func_cfi;
40	35
41	36	struct instruction find_insn(struct objtool_file file,
42	37	struct section *sec, unsigned long offset)
43	38	{
44	39	struct instruction *insn;
45	40
46		- hash_for_each_possible(file->insn_hash, insn, hash, offset)
	41	+ hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
47	42	if (insn->sec == sec && insn->offset == offset)
48	43	return insn;
	44	+ }
49	45
50	46	return NULL;
51	47	}
..	..	@@ -81,22 +77,33 @@
81	77	return find_insn(file, func->cfunc->sec, func->cfunc->offset);
82	78	}
83	79
84		-#define func_for_each_insn_all(file, func, insn) \
	80	+static struct instruction prev_insn_same_sym(struct objtool_file file,
	81	+ struct instruction *insn)
	82	+{
	83	+ struct instruction *prev = list_prev_entry(insn, list);
	84	+
	85	+ if (&prev->list != &file->insn_list && prev->func == insn->func)
	86	+ return prev;
	87	+
	88	+ return NULL;
	89	+}
	90	+
	91	+#define func_for_each_insn(file, func, insn) \
85	92	for (insn = find_insn(file, func->sec, func->offset); \
86	93	insn; \
87	94	insn = next_insn_same_func(file, insn))
88	95
89		-#define func_for_each_insn(file, func, insn) \
90		- for (insn = find_insn(file, func->sec, func->offset); \
	96	+#define sym_for_each_insn(file, sym, insn) \
	97	+ for (insn = find_insn(file, sym->sec, sym->offset); \
91	98	insn && &insn->list != &file->insn_list && \
92		- insn->sec == func->sec && \
93		- insn->offset < func->offset + func->len; \
	99	+ insn->sec == sym->sec && \
	100	+ insn->offset < sym->offset + sym->len; \
94	101	insn = list_next_entry(insn, list))
95	102
96		-#define func_for_each_insn_continue_reverse(file, func, insn) \
	103	+#define sym_for_each_insn_continue_reverse(file, sym, insn) \
97	104	for (insn = list_prev_entry(insn, list); \
98	105	&insn->list != &file->insn_list && \
99		- insn->sec == func->sec && insn->offset >= func->offset; \
	106	+ insn->sec == sym->sec && insn->offset >= sym->offset; \
100	107	insn = list_prev_entry(insn, list))
101	108
102	109	#define sec_for_each_insn_from(file, insn) \
..	..	@@ -106,27 +113,34 @@
106	113	for (insn = next_insn_same_sec(file, insn); insn; \
107	114	insn = next_insn_same_sec(file, insn))
108	115
109		-/*
110		- * Check if the function has been manually whitelisted with the
111		- * STACK_FRAME_NON_STANDARD macro, or if it should be automatically whitelisted
112		- * due to its use of a context switching instruction.
113		- */
114		-static bool ignore_func(struct objtool_file file, struct symbol func)
	116	+static bool is_jump_table_jump(struct instruction *insn)
115	117	{
116		- struct rela *rela;
	118	+ struct alt_group *alt_group = insn->alt_group;
117	119
118		- /* check for STACK_FRAME_NON_STANDARD */
119		- if (file->whitelist && file->whitelist->rela)
120		- list_for_each_entry(rela, &file->whitelist->rela->rela_list, list) {
121		- if (rela->sym->type == STT_SECTION &&
122		- rela->sym->sec == func->sec &&
123		- rela->addend == func->offset)
124		- return true;
125		- if (rela->sym->type == STT_FUNC && rela->sym == func)
126		- return true;
127		- }
	120	+ if (insn->jump_table)
	121	+ return true;
128	122
129		- return false;
	123	+ /* Retpoline alternative for a jump table? */
	124	+ return alt_group && alt_group->orig_group &&
	125	+ alt_group->orig_group->first_insn->jump_table;
	126	+}
	127	+
	128	+static bool is_sibling_call(struct instruction *insn)
	129	+{
	130	+ /*
	131	+ * Assume only ELF functions can make sibling calls. This ensures
	132	+ * sibling call detection consistency between vmlinux.o and individual
	133	+ * objects.
	134	+ */
	135	+ if (!insn->func)
	136	+ return false;
	137	+
	138	+ /* An indirect jump is either a sibling call or a jump to a table. */
	139	+ if (insn->type == INSN_JUMP_DYNAMIC)
	140	+ return !is_jump_table_jump(insn);
	141	+
	142	+ /* add_jump_destinations() sets insn->call_dest for sibling calls. */
	143	+ return (is_static_jump(insn) && insn->call_dest);
130	144	}
131	145
132	146	/*
..	..	@@ -137,14 +151,9 @@
137	151	*
138	152	* For local functions, we have to detect them manually by simply looking for
139	153	* the lack of a return instruction.
140		- *
141		- * Returns:
142		- * -1: error
143		- * 0: no dead end
144		- * 1: dead end
145	154	*/
146		-static int __dead_end_function(struct objtool_file file, struct symbol func,
147		- int recursion)
	155	+static bool __dead_end_function(struct objtool_file file, struct symbol func,
	156	+ int recursion)
148	157	{
149	158	int i;
150	159	struct instruction *insn;
..	..	@@ -161,97 +170,178 @@
161	170	"do_task_dead",
162	171	"__module_put_and_exit",
163	172	"complete_and_exit",
164		- "kvm_spurious_fault",
165	173	"__reiserfs_panic",
166	174	"lbug_with_loc",
167	175	"fortify_panic",
168	176	"usercopy_abort",
169	177	"machine_real_restart",
170	178	"rewind_stack_do_exit",
	179	+ "kunit_try_catch_throw",
	180	+ "xen_start_kernel",
	181	+ "cpu_bringup_and_idle",
171	182	};
172	183
	184	+ if (!func)
	185	+ return false;
	186	+
173	187	if (func->bind == STB_WEAK)
174		- return 0;
	188	+ return false;
175	189
176	190	if (func->bind == STB_GLOBAL)
177	191	for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
178	192	if (!strcmp(func->name, global_noreturns[i]))
179		- return 1;
	193	+ return true;
180	194
181	195	if (!func->len)
182		- return 0;
	196	+ return false;
183	197
184	198	insn = find_insn(file, func->sec, func->offset);
185	199	if (!insn->func)
186		- return 0;
	200	+ return false;
187	201
188		- func_for_each_insn_all(file, func, insn) {
	202	+ func_for_each_insn(file, func, insn) {
189	203	empty = false;
190	204
191	205	if (insn->type == INSN_RETURN)
192		- return 0;
	206	+ return false;
193	207	}
194	208
195	209	if (empty)
196		- return 0;
	210	+ return false;
197	211
198	212	/*
199	213	* A function can have a sibling call instead of a return. In that
200	214	* case, the function's dead-end status depends on whether the target
201	215	* of the sibling call returns.
202	216	*/
203		- func_for_each_insn_all(file, func, insn) {
204		- if (insn->type == INSN_JUMP_UNCONDITIONAL) {
	217	+ func_for_each_insn(file, func, insn) {
	218	+ if (is_sibling_call(insn)) {
205	219	struct instruction *dest = insn->jump_dest;
206	220
207	221	if (!dest)
208	222	/* sibling call to another file */
209		- return 0;
	223	+ return false;
210	224
211		- if (dest->func && dest->func->pfunc != insn->func->pfunc) {
212		-
213		- /* local sibling call */
214		- if (recursion == 5) {
215		- /*
216		- * Infinite recursion: two functions
217		- * have sibling calls to each other.
218		- * This is a very rare case. It means
219		- * they aren't dead ends.
220		- */
221		- return 0;
222		- }
223		-
224		- return __dead_end_function(file, dest->func,
225		- recursion + 1);
	225	+ /* local sibling call */
	226	+ if (recursion == 5) {
	227	+ /*
	228	+ * Infinite recursion: two functions have
	229	+ * sibling calls to each other. This is a very
	230	+ * rare case. It means they aren't dead ends.
	231	+ */
	232	+ return false;
226	233	}
227		- }
228	234
229		- if (insn->type == INSN_JUMP_DYNAMIC && list_empty(&insn->alts))
230		- /* sibling call */
231		- return 0;
	235	+ return __dead_end_function(file, dest->func, recursion+1);
	236	+ }
232	237	}
233	238
234		- return 1;
	239	+ return true;
235	240	}
236	241
237		-static int dead_end_function(struct objtool_file file, struct symbol func)
	242	+static bool dead_end_function(struct objtool_file file, struct symbol func)
238	243	{
239	244	return __dead_end_function(file, func, 0);
240	245	}
241	246
242		-static void clear_insn_state(struct insn_state *state)
	247	+static void init_cfi_state(struct cfi_state *cfi)
243	248	{
244	249	int i;
245	250
246		- memset(state, 0, sizeof(*state));
247		- state->cfa.base = CFI_UNDEFINED;
248	251	for (i = 0; i < CFI_NUM_REGS; i++) {
249		- state->regs[i].base = CFI_UNDEFINED;
250		- state->vals[i].base = CFI_UNDEFINED;
	252	+ cfi->regs[i].base = CFI_UNDEFINED;
	253	+ cfi->vals[i].base = CFI_UNDEFINED;
251	254	}
252		- state->drap_reg = CFI_UNDEFINED;
253		- state->drap_offset = -1;
	255	+ cfi->cfa.base = CFI_UNDEFINED;
	256	+ cfi->drap_reg = CFI_UNDEFINED;
	257	+ cfi->drap_offset = -1;
254	258	}
	259	+
	260	+static void init_insn_state(struct insn_state state, struct section sec)
	261	+{
	262	+ memset(state, 0, sizeof(*state));
	263	+ init_cfi_state(&state->cfi);
	264	+
	265	+ /*
	266	+ * We need the full vmlinux for noinstr validation, otherwise we can
	267	+ * not correctly determine insn->call_dest->sec (external symbols do
	268	+ * not have a section).
	269	+ */
	270	+ if (vmlinux && noinstr && sec)
	271	+ state->noinstr = sec->noinstr;
	272	+}
	273	+
	274	+static struct cfi_state *cfi_alloc(void)
	275	+{
	276	+ struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
	277	+ if (!cfi) {
	278	+ WARN("calloc failed");
	279	+ exit(1);
	280	+ }
	281	+ nr_cfi++;
	282	+ return cfi;
	283	+}
	284	+
	285	+static int cfi_bits;
	286	+static struct hlist_head *cfi_hash;
	287	+
	288	+static inline bool cficmp(struct cfi_state cfi1, struct cfi_state cfi2)
	289	+{
	290	+ return memcmp((void *)cfi1 + sizeof(cfi1->hash),
	291	+ (void *)cfi2 + sizeof(cfi2->hash),
	292	+ sizeof(struct cfi_state) - sizeof(struct hlist_node));
	293	+}
	294	+
	295	+static inline u32 cfi_key(struct cfi_state *cfi)
	296	+{
	297	+ return jhash((void *)cfi + sizeof(cfi->hash),
	298	+ sizeof(*cfi) - sizeof(cfi->hash), 0);
	299	+}
	300	+
	301	+static struct cfi_state cfi_hash_find_or_add(struct cfi_state cfi)
	302	+{
	303	+ struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
	304	+ struct cfi_state *obj;
	305	+
	306	+ hlist_for_each_entry(obj, head, hash) {
	307	+ if (!cficmp(cfi, obj)) {
	308	+ nr_cfi_cache++;
	309	+ return obj;
	310	+ }
	311	+ }
	312	+
	313	+ obj = cfi_alloc();
	314	+ obj = cfi;
	315	+ hlist_add_head(&obj->hash, head);
	316	+
	317	+ return obj;
	318	+}
	319	+
	320	+static void cfi_hash_add(struct cfi_state *cfi)
	321	+{
	322	+ struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
	323	+
	324	+ hlist_add_head(&cfi->hash, head);
	325	+}
	326	+
	327	+static void *cfi_hash_alloc(void)
	328	+{
	329	+ cfi_bits = vmlinux ? ELF_HASH_BITS - 3 : 13;
	330	+ cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
	331	+ PROT_READ\|PROT_WRITE,
	332	+ MAP_PRIVATE\|MAP_ANON, -1, 0);
	333	+ if (cfi_hash == (void *)-1L) {
	334	+ WARN("mmap fail cfi_hash");
	335	+ cfi_hash = NULL;
	336	+ } else if (stats) {
	337	+ printf("cfi_bits: %d\n", cfi_bits);
	338	+ }
	339	+
	340	+ return cfi_hash;
	341	+}
	342	+
	343	+static unsigned long nr_insns;
	344	+static unsigned long nr_insns_visited;
255	345
256	346	/*
257	347	* Call the arch-specific instruction decoder for all the instructions and add
..	..	@@ -275,6 +365,11 @@
275	365	strncmp(sec->name, ".discard.", 9))
276	366	sec->text = true;
277	367
	368	+ if (!strcmp(sec->name, ".noinstr.text") \|\|
	369	+ !strcmp(sec->name, ".entry.text") \|\|
	370	+ !strncmp(sec->name, ".text.__x86.", 12))
	371	+ sec->noinstr = true;
	372	+
278	373	for (offset = 0; offset < sec->len; offset += insn->len) {
279	374	insn = malloc(sizeof(*insn));
280	375	if (!insn) {
..	..	@@ -283,7 +378,7 @@
283	378	}
284	379	memset(insn, 0, sizeof(*insn));
285	380	INIT_LIST_HEAD(&insn->alts);
286		- clear_insn_state(&insn->state);
	381	+ INIT_LIST_HEAD(&insn->stack_ops);
287	382
288	383	insn->sec = sec;
289	384	insn->offset = offset;
..	..	@@ -292,23 +387,17 @@
292	387	sec->len - offset,
293	388	&insn->len, &insn->type,
294	389	&insn->immediate,
295		- &insn->stack_op);
	390	+ &insn->stack_ops);
296	391	if (ret)
297	392	goto err;
298	393
299		- if (!insn->type \|\| insn->type > INSN_LAST) {
300		- WARN_FUNC("invalid instruction type %d",
301		- insn->sec, insn->offset, insn->type);
302		- ret = -1;
303		- goto err;
304		- }
305		-
306		- hash_add(file->insn_hash, &insn->hash, insn->offset);
	394	+ hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
307	395	list_add_tail(&insn->list, &file->insn_list);
	396	+ nr_insns++;
308	397	}
309	398
310	399	list_for_each_entry(func, &sec->symbol_list, list) {
311		- if (func->type != STT_FUNC)
	400	+ if (func->type != STT_FUNC \|\| func->alias != func)
312	401	continue;
313	402
314	403	if (!find_insn(file, sec, func->offset)) {
..	..	@@ -317,11 +406,13 @@
317	406	return -1;
318	407	}
319	408
320		- func_for_each_insn(file, func, insn)
321		- if (!insn->func)
322		- insn->func = func;
	409	+ sym_for_each_insn(file, func, insn)
	410	+ insn->func = func;
323	411	}
324	412	}
	413	+
	414	+ if (stats)
	415	+ printf("nr_insns: %lu\n", nr_insns);
325	416
326	417	return 0;
327	418
..	..	@@ -330,15 +421,27 @@
330	421	return ret;
331	422	}
332	423
	424	+static struct instruction find_last_insn(struct objtool_file file,
	425	+ struct section *sec)
	426	+{
	427	+ struct instruction *insn = NULL;
	428	+ unsigned int offset;
	429	+ unsigned int end = (sec->len > 10) ? sec->len - 10 : 0;
	430	+
	431	+ for (offset = sec->len - 1; offset >= end && !insn; offset--)
	432	+ insn = find_insn(file, sec, offset);
	433	+
	434	+ return insn;
	435	+}
	436	+
333	437	/*
334	438	* Mark "ud2" instructions and manually annotated dead ends.
335	439	*/
336	440	static int add_dead_ends(struct objtool_file *file)
337	441	{
338	442	struct section *sec;
339		- struct rela *rela;
	443	+ struct reloc *reloc;
340	444	struct instruction *insn;
341		- bool found;
342	445
343	446	/*
344	447	* By default, "ud2" is a dead end unless otherwise annotated, because
..	..	@@ -355,31 +458,24 @@
355	458	if (!sec)
356	459	goto reachable;
357	460
358		- list_for_each_entry(rela, &sec->rela_list, list) {
359		- if (rela->sym->type != STT_SECTION) {
	461	+ list_for_each_entry(reloc, &sec->reloc_list, list) {
	462	+ if (reloc->sym->type != STT_SECTION) {
360	463	WARN("unexpected relocation symbol type in %s", sec->name);
361	464	return -1;
362	465	}
363		- insn = find_insn(file, rela->sym->sec, rela->addend);
	466	+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
364	467	if (insn)
365	468	insn = list_prev_entry(insn, list);
366		- else if (rela->addend == rela->sym->sec->len) {
367		- found = false;
368		- list_for_each_entry_reverse(insn, &file->insn_list, list) {
369		- if (insn->sec == rela->sym->sec) {
370		- found = true;
371		- break;
372		- }
373		- }
374		-
375		- if (!found) {
376		- WARN("can't find unreachable insn at %s+0x%x",
377		- rela->sym->sec->name, rela->addend);
	469	+ else if (reloc->addend == reloc->sym->sec->len) {
	470	+ insn = find_last_insn(file, reloc->sym->sec);
	471	+ if (!insn) {
	472	+ WARN("can't find unreachable insn at %s+0x%" PRIx64,
	473	+ reloc->sym->sec->name, reloc->addend);
378	474	return -1;
379	475	}
380	476	} else {
381		- WARN("can't find unreachable insn at %s+0x%x",
382		- rela->sym->sec->name, rela->addend);
	477	+ WARN("can't find unreachable insn at %s+0x%" PRIx64,
	478	+ reloc->sym->sec->name, reloc->addend);
383	479	return -1;
384	480	}
385	481
..	..	@@ -397,35 +493,251 @@
397	493	if (!sec)
398	494	return 0;
399	495
400		- list_for_each_entry(rela, &sec->rela_list, list) {
401		- if (rela->sym->type != STT_SECTION) {
	496	+ list_for_each_entry(reloc, &sec->reloc_list, list) {
	497	+ if (reloc->sym->type != STT_SECTION) {
402	498	WARN("unexpected relocation symbol type in %s", sec->name);
403	499	return -1;
404	500	}
405		- insn = find_insn(file, rela->sym->sec, rela->addend);
	501	+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
406	502	if (insn)
407	503	insn = list_prev_entry(insn, list);
408		- else if (rela->addend == rela->sym->sec->len) {
409		- found = false;
410		- list_for_each_entry_reverse(insn, &file->insn_list, list) {
411		- if (insn->sec == rela->sym->sec) {
412		- found = true;
413		- break;
414		- }
415		- }
416		-
417		- if (!found) {
418		- WARN("can't find reachable insn at %s+0x%x",
419		- rela->sym->sec->name, rela->addend);
	504	+ else if (reloc->addend == reloc->sym->sec->len) {
	505	+ insn = find_last_insn(file, reloc->sym->sec);
	506	+ if (!insn) {
	507	+ WARN("can't find reachable insn at %s+0x%" PRIx64,
	508	+ reloc->sym->sec->name, reloc->addend);
420	509	return -1;
421	510	}
422	511	} else {
423		- WARN("can't find reachable insn at %s+0x%x",
424		- rela->sym->sec->name, rela->addend);
	512	+ WARN("can't find reachable insn at %s+0x%" PRIx64,
	513	+ reloc->sym->sec->name, reloc->addend);
425	514	return -1;
426	515	}
427	516
428	517	insn->dead_end = false;
	518	+ }
	519	+
	520	+ return 0;
	521	+}
	522	+
	523	+static int create_static_call_sections(struct objtool_file *file)
	524	+{
	525	+ struct section *sec;
	526	+ struct static_call_site *site;
	527	+ struct instruction *insn;
	528	+ struct symbol *key_sym;
	529	+ char key_name, tmp;
	530	+ int idx;
	531	+
	532	+ sec = find_section_by_name(file->elf, ".static_call_sites");
	533	+ if (sec) {
	534	+ INIT_LIST_HEAD(&file->static_call_list);
	535	+ WARN("file already has .static_call_sites section, skipping");
	536	+ return 0;
	537	+ }
	538	+
	539	+ if (list_empty(&file->static_call_list))
	540	+ return 0;
	541	+
	542	+ idx = 0;
	543	+ list_for_each_entry(insn, &file->static_call_list, call_node)
	544	+ idx++;
	545	+
	546	+ sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
	547	+ sizeof(struct static_call_site), idx);
	548	+ if (!sec)
	549	+ return -1;
	550	+
	551	+ idx = 0;
	552	+ list_for_each_entry(insn, &file->static_call_list, call_node) {
	553	+
	554	+ site = (struct static_call_site *)sec->data->d_buf + idx;
	555	+ memset(site, 0, sizeof(struct static_call_site));
	556	+
	557	+ /* populate reloc for 'addr' */
	558	+ if (elf_add_reloc_to_insn(file->elf, sec,
	559	+ idx * sizeof(struct static_call_site),
	560	+ R_X86_64_PC32,
	561	+ insn->sec, insn->offset))
	562	+ return -1;
	563	+
	564	+ /* find key symbol */
	565	+ key_name = strdup(insn->call_dest->name);
	566	+ if (!key_name) {
	567	+ perror("strdup");
	568	+ return -1;
	569	+ }
	570	+ if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
	571	+ STATIC_CALL_TRAMP_PREFIX_LEN)) {
	572	+ WARN("static_call: trampoline name malformed: %s", key_name);
	573	+ return -1;
	574	+ }
	575	+ tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
	576	+ memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
	577	+
	578	+ key_sym = find_symbol_by_name(file->elf, tmp);
	579	+ if (!key_sym) {
	580	+ if (!module) {
	581	+ WARN("static_call: can't find static_call_key symbol: %s", tmp);
	582	+ return -1;
	583	+ }
	584	+
	585	+ /*
	586	+ * For modules(), the key might not be exported, which
	587	+ * means the module can make static calls but isn't
	588	+ * allowed to change them.
	589	+ *
	590	+ * In that case we temporarily set the key to be the
	591	+ * trampoline address. This is fixed up in
	592	+ * static_call_add_module().
	593	+ */
	594	+ key_sym = insn->call_dest;
	595	+ }
	596	+ free(key_name);
	597	+
	598	+ /* populate reloc for 'key' */
	599	+ if (elf_add_reloc(file->elf, sec,
	600	+ idx * sizeof(struct static_call_site) + 4,
	601	+ R_X86_64_PC32, key_sym,
	602	+ is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
	603	+ return -1;
	604	+
	605	+ idx++;
	606	+ }
	607	+
	608	+ return 0;
	609	+}
	610	+
	611	+static int create_retpoline_sites_sections(struct objtool_file *file)
	612	+{
	613	+ struct instruction *insn;
	614	+ struct section *sec;
	615	+ int idx;
	616	+
	617	+ sec = find_section_by_name(file->elf, ".retpoline_sites");
	618	+ if (sec) {
	619	+ WARN("file already has .retpoline_sites, skipping");
	620	+ return 0;
	621	+ }
	622	+
	623	+ idx = 0;
	624	+ list_for_each_entry(insn, &file->retpoline_call_list, call_node)
	625	+ idx++;
	626	+
	627	+ if (!idx)
	628	+ return 0;
	629	+
	630	+ sec = elf_create_section(file->elf, ".retpoline_sites", 0,
	631	+ sizeof(int), idx);
	632	+ if (!sec) {
	633	+ WARN("elf_create_section: .retpoline_sites");
	634	+ return -1;
	635	+ }
	636	+
	637	+ idx = 0;
	638	+ list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
	639	+
	640	+ int site = (int )sec->data->d_buf + idx;
	641	+ *site = 0;
	642	+
	643	+ if (elf_add_reloc_to_insn(file->elf, sec,
	644	+ idx * sizeof(int),
	645	+ R_X86_64_PC32,
	646	+ insn->sec, insn->offset)) {
	647	+ WARN("elf_add_reloc_to_insn: .retpoline_sites");
	648	+ return -1;
	649	+ }
	650	+
	651	+ idx++;
	652	+ }
	653	+
	654	+ return 0;
	655	+}
	656	+
	657	+static int create_return_sites_sections(struct objtool_file *file)
	658	+{
	659	+ struct instruction *insn;
	660	+ struct section *sec;
	661	+ int idx;
	662	+
	663	+ sec = find_section_by_name(file->elf, ".return_sites");
	664	+ if (sec) {
	665	+ WARN("file already has .return_sites, skipping");
	666	+ return 0;
	667	+ }
	668	+
	669	+ idx = 0;
	670	+ list_for_each_entry(insn, &file->return_thunk_list, call_node)
	671	+ idx++;
	672	+
	673	+ if (!idx)
	674	+ return 0;
	675	+
	676	+ sec = elf_create_section(file->elf, ".return_sites", 0,
	677	+ sizeof(int), idx);
	678	+ if (!sec) {
	679	+ WARN("elf_create_section: .return_sites");
	680	+ return -1;
	681	+ }
	682	+
	683	+ idx = 0;
	684	+ list_for_each_entry(insn, &file->return_thunk_list, call_node) {
	685	+
	686	+ int site = (int )sec->data->d_buf + idx;
	687	+ *site = 0;
	688	+
	689	+ if (elf_add_reloc_to_insn(file->elf, sec,
	690	+ idx * sizeof(int),
	691	+ R_X86_64_PC32,
	692	+ insn->sec, insn->offset)) {
	693	+ WARN("elf_add_reloc_to_insn: .return_sites");
	694	+ return -1;
	695	+ }
	696	+
	697	+ idx++;
	698	+ }
	699	+
	700	+ return 0;
	701	+}
	702	+
	703	+static int create_mcount_loc_sections(struct objtool_file *file)
	704	+{
	705	+ struct section *sec;
	706	+ unsigned long *loc;
	707	+ struct instruction *insn;
	708	+ int idx;
	709	+
	710	+ sec = find_section_by_name(file->elf, "__mcount_loc");
	711	+ if (sec) {
	712	+ INIT_LIST_HEAD(&file->mcount_loc_list);
	713	+ WARN("file already has __mcount_loc section, skipping");
	714	+ return 0;
	715	+ }
	716	+
	717	+ if (list_empty(&file->mcount_loc_list))
	718	+ return 0;
	719	+
	720	+ idx = 0;
	721	+ list_for_each_entry(insn, &file->mcount_loc_list, mcount_loc_node)
	722	+ idx++;
	723	+
	724	+ sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
	725	+ if (!sec)
	726	+ return -1;
	727	+
	728	+ idx = 0;
	729	+ list_for_each_entry(insn, &file->mcount_loc_list, mcount_loc_node) {
	730	+
	731	+ loc = (unsigned long *)sec->data->d_buf + idx;
	732	+ memset(loc, 0, sizeof(unsigned long));
	733	+
	734	+ if (elf_add_reloc_to_insn(file->elf, sec,
	735	+ idx * sizeof(unsigned long),
	736	+ R_X86_64_64,
	737	+ insn->sec, insn->offset))
	738	+ return -1;
	739	+
	740	+ idx++;
429	741	}
430	742
431	743	return 0;
..	..	@@ -439,18 +751,191 @@
439	751	struct instruction *insn;
440	752	struct section *sec;
441	753	struct symbol *func;
	754	+ struct reloc *reloc;
442	755
443		- for_each_sec(file, sec) {
444		- list_for_each_entry(func, &sec->symbol_list, list) {
445		- if (func->type != STT_FUNC)
	756	+ sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
	757	+ if (!sec)
	758	+ return;
	759	+
	760	+ list_for_each_entry(reloc, &sec->reloc_list, list) {
	761	+ switch (reloc->sym->type) {
	762	+ case STT_FUNC:
	763	+ func = reloc->sym;
	764	+ break;
	765	+
	766	+ case STT_SECTION:
	767	+ func = find_func_by_offset(reloc->sym->sec, reloc->addend);
	768	+ if (!func)
446	769	continue;
	770	+ break;
447	771
448		- if (!ignore_func(file, func))
449		- continue;
450		-
451		- func_for_each_insn_all(file, func, insn)
452		- insn->ignore = true;
	772	+ default:
	773	+ WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
	774	+ continue;
453	775	}
	776	+
	777	+ func_for_each_insn(file, func, insn)
	778	+ insn->ignore = true;
	779	+ }
	780	+}
	781	+
	782	+/*
	783	+ * This is a whitelist of functions that is allowed to be called with AC set.
	784	+ * The list is meant to be minimal and only contains compiler instrumentation
	785	+ * ABI and a few functions used to implement *_{to,from}_user() functions.
	786	+ *
	787	+ * These functions must not directly change AC, but may PUSHF/POPF.
	788	+ */
	789	+static const char *uaccess_safe_builtin[] = {
	790	+ /* KASAN */
	791	+ "kasan_report",
	792	+ "kasan_check_range",
	793	+ /* KASAN out-of-line */
	794	+ "__asan_loadN_noabort",
	795	+ "__asan_load1_noabort",
	796	+ "__asan_load2_noabort",
	797	+ "__asan_load4_noabort",
	798	+ "__asan_load8_noabort",
	799	+ "__asan_load16_noabort",
	800	+ "__asan_storeN_noabort",
	801	+ "__asan_store1_noabort",
	802	+ "__asan_store2_noabort",
	803	+ "__asan_store4_noabort",
	804	+ "__asan_store8_noabort",
	805	+ "__asan_store16_noabort",
	806	+ "__kasan_check_read",
	807	+ "__kasan_check_write",
	808	+ /* KASAN in-line */
	809	+ "__asan_report_load_n_noabort",
	810	+ "__asan_report_load1_noabort",
	811	+ "__asan_report_load2_noabort",
	812	+ "__asan_report_load4_noabort",
	813	+ "__asan_report_load8_noabort",
	814	+ "__asan_report_load16_noabort",
	815	+ "__asan_report_store_n_noabort",
	816	+ "__asan_report_store1_noabort",
	817	+ "__asan_report_store2_noabort",
	818	+ "__asan_report_store4_noabort",
	819	+ "__asan_report_store8_noabort",
	820	+ "__asan_report_store16_noabort",
	821	+ /* KCSAN */
	822	+ "__kcsan_check_access",
	823	+ "kcsan_found_watchpoint",
	824	+ "kcsan_setup_watchpoint",
	825	+ "kcsan_check_scoped_accesses",
	826	+ "kcsan_disable_current",
	827	+ "kcsan_enable_current_nowarn",
	828	+ /* KCSAN/TSAN */
	829	+ "__tsan_func_entry",
	830	+ "__tsan_func_exit",
	831	+ "__tsan_read_range",
	832	+ "__tsan_write_range",
	833	+ "__tsan_read1",
	834	+ "__tsan_read2",
	835	+ "__tsan_read4",
	836	+ "__tsan_read8",
	837	+ "__tsan_read16",
	838	+ "__tsan_write1",
	839	+ "__tsan_write2",
	840	+ "__tsan_write4",
	841	+ "__tsan_write8",
	842	+ "__tsan_write16",
	843	+ "__tsan_read_write1",
	844	+ "__tsan_read_write2",
	845	+ "__tsan_read_write4",
	846	+ "__tsan_read_write8",
	847	+ "__tsan_read_write16",
	848	+ "__tsan_atomic8_load",
	849	+ "__tsan_atomic16_load",
	850	+ "__tsan_atomic32_load",
	851	+ "__tsan_atomic64_load",
	852	+ "__tsan_atomic8_store",
	853	+ "__tsan_atomic16_store",
	854	+ "__tsan_atomic32_store",
	855	+ "__tsan_atomic64_store",
	856	+ "__tsan_atomic8_exchange",
	857	+ "__tsan_atomic16_exchange",
	858	+ "__tsan_atomic32_exchange",
	859	+ "__tsan_atomic64_exchange",
	860	+ "__tsan_atomic8_fetch_add",
	861	+ "__tsan_atomic16_fetch_add",
	862	+ "__tsan_atomic32_fetch_add",
	863	+ "__tsan_atomic64_fetch_add",
	864	+ "__tsan_atomic8_fetch_sub",
	865	+ "__tsan_atomic16_fetch_sub",
	866	+ "__tsan_atomic32_fetch_sub",
	867	+ "__tsan_atomic64_fetch_sub",
	868	+ "__tsan_atomic8_fetch_and",
	869	+ "__tsan_atomic16_fetch_and",
	870	+ "__tsan_atomic32_fetch_and",
	871	+ "__tsan_atomic64_fetch_and",
	872	+ "__tsan_atomic8_fetch_or",
	873	+ "__tsan_atomic16_fetch_or",
	874	+ "__tsan_atomic32_fetch_or",
	875	+ "__tsan_atomic64_fetch_or",
	876	+ "__tsan_atomic8_fetch_xor",
	877	+ "__tsan_atomic16_fetch_xor",
	878	+ "__tsan_atomic32_fetch_xor",
	879	+ "__tsan_atomic64_fetch_xor",
	880	+ "__tsan_atomic8_fetch_nand",
	881	+ "__tsan_atomic16_fetch_nand",
	882	+ "__tsan_atomic32_fetch_nand",
	883	+ "__tsan_atomic64_fetch_nand",
	884	+ "__tsan_atomic8_compare_exchange_strong",
	885	+ "__tsan_atomic16_compare_exchange_strong",
	886	+ "__tsan_atomic32_compare_exchange_strong",
	887	+ "__tsan_atomic64_compare_exchange_strong",
	888	+ "__tsan_atomic8_compare_exchange_weak",
	889	+ "__tsan_atomic16_compare_exchange_weak",
	890	+ "__tsan_atomic32_compare_exchange_weak",
	891	+ "__tsan_atomic64_compare_exchange_weak",
	892	+ "__tsan_atomic8_compare_exchange_val",
	893	+ "__tsan_atomic16_compare_exchange_val",
	894	+ "__tsan_atomic32_compare_exchange_val",
	895	+ "__tsan_atomic64_compare_exchange_val",
	896	+ "__tsan_atomic_thread_fence",
	897	+ "__tsan_atomic_signal_fence",
	898	+ /* KCOV */
	899	+ "write_comp_data",
	900	+ "check_kcov_mode",
	901	+ "__sanitizer_cov_trace_pc",
	902	+ "__sanitizer_cov_trace_const_cmp1",
	903	+ "__sanitizer_cov_trace_const_cmp2",
	904	+ "__sanitizer_cov_trace_const_cmp4",
	905	+ "__sanitizer_cov_trace_const_cmp8",
	906	+ "__sanitizer_cov_trace_cmp1",
	907	+ "__sanitizer_cov_trace_cmp2",
	908	+ "__sanitizer_cov_trace_cmp4",
	909	+ "__sanitizer_cov_trace_cmp8",
	910	+ "__sanitizer_cov_trace_switch",
	911	+ /* UBSAN */
	912	+ "ubsan_type_mismatch_common",
	913	+ "__ubsan_handle_type_mismatch",
	914	+ "__ubsan_handle_type_mismatch_v1",
	915	+ "__ubsan_handle_shift_out_of_bounds",
	916	+ /* misc */
	917	+ "csum_partial_copy_generic",
	918	+ "copy_mc_fragile",
	919	+ "copy_mc_fragile_handle_tail",
	920	+ "copy_mc_enhanced_fast_string",
	921	+ "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
	922	+ NULL
	923	+};
	924	+
	925	+static void add_uaccess_safe(struct objtool_file *file)
	926	+{
	927	+ struct symbol *func;
	928	+ const char **name;
	929	+
	930	+ if (!uaccess)
	931	+ return;
	932	+
	933	+ for (name = uaccess_safe_builtin; *name; name++) {
	934	+ func = find_symbol_by_name(file->elf, *name);
	935	+ if (!func)
	936	+ continue;
	937	+
	938	+ func->uaccess_safe = true;
454	939	}
455	940	}
456	941
..	..	@@ -460,25 +945,25 @@
460	945	* But it at least allows objtool to understand the control flow around the
461	946	* retpoline.
462	947	*/
463		-static int add_nospec_ignores(struct objtool_file *file)
	948	+static int add_ignore_alternatives(struct objtool_file *file)
464	949	{
465	950	struct section *sec;
466		- struct rela *rela;
	951	+ struct reloc *reloc;
467	952	struct instruction *insn;
468	953
469		- sec = find_section_by_name(file->elf, ".rela.discard.nospec");
	954	+ sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
470	955	if (!sec)
471	956	return 0;
472	957
473		- list_for_each_entry(rela, &sec->rela_list, list) {
474		- if (rela->sym->type != STT_SECTION) {
	958	+ list_for_each_entry(reloc, &sec->reloc_list, list) {
	959	+ if (reloc->sym->type != STT_SECTION) {
475	960	WARN("unexpected relocation symbol type in %s", sec->name);
476	961	return -1;
477	962	}
478	963
479		- insn = find_insn(file, rela->sym->sec, rela->addend);
	964	+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
480	965	if (!insn) {
481		- WARN("bad .discard.nospec entry");
	966	+ WARN("bad .discard.ignore_alts entry");
482	967	return -1;
483	968	}
484	969
..	..	@@ -488,51 +973,250 @@
488	973	return 0;
489	974	}
490	975
	976	+__weak bool arch_is_retpoline(struct symbol *sym)
	977	+{
	978	+ return false;
	979	+}
	980	+
	981	+__weak bool arch_is_rethunk(struct symbol *sym)
	982	+{
	983	+ return false;
	984	+}
	985	+
	986	+#define NEGATIVE_RELOC ((void *)-1L)
	987	+
	988	+static struct reloc insn_reloc(struct objtool_file file, struct instruction *insn)
	989	+{
	990	+ if (insn->reloc == NEGATIVE_RELOC)
	991	+ return NULL;
	992	+
	993	+ if (!insn->reloc) {
	994	+ insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
	995	+ insn->offset, insn->len);
	996	+ if (!insn->reloc) {
	997	+ insn->reloc = NEGATIVE_RELOC;
	998	+ return NULL;
	999	+ }
	1000	+ }
	1001	+
	1002	+ return insn->reloc;
	1003	+}
	1004	+
	1005	+static void remove_insn_ops(struct instruction *insn)
	1006	+{
	1007	+ struct stack_op op, tmp;
	1008	+
	1009	+ list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
	1010	+ list_del(&op->list);
	1011	+ free(op);
	1012	+ }
	1013	+}
	1014	+
	1015	+static void annotate_call_site(struct objtool_file *file,
	1016	+ struct instruction *insn, bool sibling)
	1017	+{
	1018	+ struct reloc *reloc = insn_reloc(file, insn);
	1019	+ struct symbol *sym = insn->call_dest;
	1020	+
	1021	+ if (!sym)
	1022	+ sym = reloc->sym;
	1023	+
	1024	+ /*
	1025	+ * Alternative replacement code is just template code which is
	1026	+ * sometimes copied to the original instruction. For now, don't
	1027	+ * annotate it. (In the future we might consider annotating the
	1028	+ * original instruction if/when it ever makes sense to do so.)
	1029	+ */
	1030	+ if (!strcmp(insn->sec->name, ".altinstr_replacement"))
	1031	+ return;
	1032	+
	1033	+ if (sym->static_call_tramp) {
	1034	+ list_add_tail(&insn->call_node, &file->static_call_list);
	1035	+ return;
	1036	+ }
	1037	+
	1038	+ if (sym->retpoline_thunk) {
	1039	+ list_add_tail(&insn->call_node, &file->retpoline_call_list);
	1040	+ return;
	1041	+ }
	1042	+
	1043	+ /*
	1044	+ * Many compilers cannot disable KCOV with a function attribute
	1045	+ * so they need a little help, NOP out any KCOV calls from noinstr
	1046	+ * text.
	1047	+ */
	1048	+ if (insn->sec->noinstr && sym->kcov) {
	1049	+ if (reloc) {
	1050	+ reloc->type = R_NONE;
	1051	+ elf_write_reloc(file->elf, reloc);
	1052	+ }
	1053	+
	1054	+ elf_write_insn(file->elf, insn->sec,
	1055	+ insn->offset, insn->len,
	1056	+ sibling ? arch_ret_insn(insn->len)
	1057	+ : arch_nop_insn(insn->len));
	1058	+
	1059	+ insn->type = sibling ? INSN_RETURN : INSN_NOP;
	1060	+
	1061	+ if (sibling) {
	1062	+ /*
	1063	+ * We've replaced the tail-call JMP insn by two new
	1064	+ * insn: RET; INT3, except we only have a single struct
	1065	+ * insn here. Mark it retpoline_safe to avoid the SLS
	1066	+ * warning, instead of adding another insn.
	1067	+ */
	1068	+ insn->retpoline_safe = true;
	1069	+ }
	1070	+
	1071	+ return;
	1072	+ }
	1073	+}
	1074	+
	1075	+static void add_call_dest(struct objtool_file file, struct instruction insn,
	1076	+ struct symbol *dest, bool sibling)
	1077	+{
	1078	+ insn->call_dest = dest;
	1079	+ if (!dest)
	1080	+ return;
	1081	+
	1082	+ /*
	1083	+ * Whatever stack impact regular CALLs have, should be undone
	1084	+ * by the RETURN of the called function.
	1085	+ *
	1086	+ * Annotated intra-function calls retain the stack_ops but
	1087	+ * are converted to JUMP, see read_intra_function_calls().
	1088	+ */
	1089	+ remove_insn_ops(insn);
	1090	+
	1091	+ annotate_call_site(file, insn, sibling);
	1092	+}
	1093	+
	1094	+static void add_retpoline_call(struct objtool_file file, struct instruction insn)
	1095	+{
	1096	+ /*
	1097	+ * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
	1098	+ * so convert them accordingly.
	1099	+ */
	1100	+ switch (insn->type) {
	1101	+ case INSN_CALL:
	1102	+ insn->type = INSN_CALL_DYNAMIC;
	1103	+ break;
	1104	+ case INSN_JUMP_UNCONDITIONAL:
	1105	+ insn->type = INSN_JUMP_DYNAMIC;
	1106	+ break;
	1107	+ case INSN_JUMP_CONDITIONAL:
	1108	+ insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
	1109	+ break;
	1110	+ default:
	1111	+ return;
	1112	+ }
	1113	+
	1114	+ insn->retpoline_safe = true;
	1115	+
	1116	+ /*
	1117	+ * Whatever stack impact regular CALLs have, should be undone
	1118	+ * by the RETURN of the called function.
	1119	+ *
	1120	+ * Annotated intra-function calls retain the stack_ops but
	1121	+ * are converted to JUMP, see read_intra_function_calls().
	1122	+ */
	1123	+ remove_insn_ops(insn);
	1124	+
	1125	+ annotate_call_site(file, insn, false);
	1126	+}
	1127	+
	1128	+static void add_return_call(struct objtool_file file, struct instruction insn, bool add)
	1129	+{
	1130	+ /*
	1131	+ * Return thunk tail calls are really just returns in disguise,
	1132	+ * so convert them accordingly.
	1133	+ */
	1134	+ insn->type = INSN_RETURN;
	1135	+ insn->retpoline_safe = true;
	1136	+
	1137	+ /* Skip the non-text sections, specially .discard ones */
	1138	+ if (add && insn->sec->text)
	1139	+ list_add_tail(&insn->call_node, &file->return_thunk_list);
	1140	+}
	1141	+
	1142	+/*
	1143	+ * CONFIG_CFI_CLANG: Check if the section is a CFI jump table or a
	1144	+ * compiler-generated CFI handler.
	1145	+ */
	1146	+static bool is_cfi_section(struct section *sec)
	1147	+{
	1148	+ return (sec->name &&
	1149	+ (!strncmp(sec->name, ".text..L.cfi.jumptable", 22) \|\|
	1150	+ !strcmp(sec->name, ".text.__cfi_check")));
	1151	+}
	1152	+
	1153	+/*
	1154	+ * CONFIG_CFI_CLANG: Ignore CFI jump tables.
	1155	+ */
	1156	+static void add_cfi_jumptables(struct objtool_file *file)
	1157	+{
	1158	+ struct section *sec;
	1159	+ struct symbol *func;
	1160	+ struct instruction *insn;
	1161	+
	1162	+ for_each_sec(file, sec) {
	1163	+ if (!is_cfi_section(sec))
	1164	+ continue;
	1165	+
	1166	+ list_for_each_entry(func, &sec->symbol_list, list) {
	1167	+ sym_for_each_insn(file, func, insn)
	1168	+ insn->ignore = true;
	1169	+ }
	1170	+ }
	1171	+}
	1172	+
491	1173	/*
492	1174	* Find the destination instructions for all jumps.
493	1175	*/
494	1176	static int add_jump_destinations(struct objtool_file *file)
495	1177	{
496	1178	struct instruction *insn;
497		- struct rela *rela;
	1179	+ struct reloc *reloc;
498	1180	struct section *dest_sec;
499	1181	unsigned long dest_off;
500	1182
501	1183	for_each_insn(file, insn) {
502		- if (insn->type != INSN_JUMP_CONDITIONAL &&
503		- insn->type != INSN_JUMP_UNCONDITIONAL)
	1184	+ if (!is_static_jump(insn))
504	1185	continue;
505	1186
506		- if (insn->offset == FAKE_JUMP_OFFSET)
507		- continue;
508		-
509		- rela = find_rela_by_dest_range(insn->sec, insn->offset,
510		- insn->len);
511		- if (!rela) {
	1187	+ reloc = insn_reloc(file, insn);
	1188	+ if (!reloc) {
512	1189	dest_sec = insn->sec;
513		- dest_off = insn->offset + insn->len + insn->immediate;
514		- } else if (rela->sym->type == STT_SECTION) {
515		- dest_sec = rela->sym->sec;
516		- dest_off = rela->addend + 4;
517		- } else if (rela->sym->sec->idx) {
518		- dest_sec = rela->sym->sec;
519		- dest_off = rela->sym->sym.st_value + rela->addend + 4;
520		- } else if (strstr(rela->sym->name, "_indirect_thunk_")) {
521		- /*
522		- * Retpoline jumps are really dynamic jumps in
523		- * disguise, so convert them accordingly.
524		- */
525		- insn->type = INSN_JUMP_DYNAMIC;
526		- insn->retpoline_safe = true;
	1190	+ dest_off = arch_jump_destination(insn);
	1191	+ } else if (reloc->sym->type == STT_SECTION) {
	1192	+ dest_sec = reloc->sym->sec;
	1193	+ dest_off = arch_dest_reloc_offset(reloc->addend);
	1194	+ } else if (reloc->sym->retpoline_thunk) {
	1195	+ add_retpoline_call(file, insn);
527	1196	continue;
	1197	+ } else if (reloc->sym->return_thunk) {
	1198	+ add_return_call(file, insn, true);
	1199	+ continue;
	1200	+ } else if (insn->func) {
	1201	+ /* internal or external sibling call (with reloc) */
	1202	+ add_call_dest(file, insn, reloc->sym, true);
	1203	+ continue;
	1204	+ } else if (reloc->sym->sec->idx) {
	1205	+ dest_sec = reloc->sym->sec;
	1206	+ dest_off = reloc->sym->sym.st_value +
	1207	+ arch_dest_reloc_offset(reloc->addend);
528	1208	} else {
529		- /* sibling call */
530		- insn->jump_dest = 0;
	1209	+ /* non-func asm code jumping to another file */
531	1210	continue;
532	1211	}
533	1212
534	1213	insn->jump_dest = find_insn(file, dest_sec, dest_off);
	1214	+
	1215	+ if (!insn->jump_dest && dest_sec->len == dest_off)
	1216	+ insn->jump_dest = find_last_insn(file, dest_sec);
	1217	+
535	1218	if (!insn->jump_dest) {
	1219	+ struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
536	1220
537	1221	/*
538	1222	* This is a special case where an alt instruction
..	..	@@ -542,6 +1226,22 @@
542	1226	if (!strcmp(insn->sec->name, ".altinstr_replacement"))
543	1227	continue;
544	1228
	1229	+ if (is_cfi_section(insn->sec))
	1230	+ continue;
	1231	+
	1232	+ /*
	1233	+ * This is a special case for zen_untrain_ret().
	1234	+ * It jumps to __x86_return_thunk(), but objtool
	1235	+ * can't find the thunk's starting RET
	1236	+ * instruction, because the RET is also in the
	1237	+ * middle of another instruction. Objtool only
	1238	+ * knows about the outer instruction.
	1239	+ */
	1240	+ if (sym && sym->return_thunk) {
	1241	+ add_return_call(file, insn, false);
	1242	+ continue;
	1243	+ }
	1244	+
545	1245	WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
546	1246	insn->sec, insn->offset, dest_sec->name,
547	1247	dest_off);
..	..	@@ -549,29 +1249,51 @@
549	1249	}
550	1250
551	1251	/*
552		- * For GCC 8+, create parent/child links for any cold
553		- * subfunctions. This is _mostly_ redundant with a similar
554		- * initialization in read_symbols().
555		- *
556		- * If a function has aliases, we want the first such function
557		- * in the symbol table to be the subfunction's parent. In that
558		- * case we overwrite the initialization done in read_symbols().
559		- *
560		- * However this code can't completely replace the
561		- * read_symbols() code because this doesn't detect the case
562		- * where the parent function's only reference to a subfunction
563		- * is through a switch table.
	1252	+ * Cross-function jump.
564	1253	*/
565	1254	if (insn->func && insn->jump_dest->func &&
566		- insn->func != insn->jump_dest->func &&
567		- !strstr(insn->func->name, ".cold.") &&
568		- strstr(insn->jump_dest->func->name, ".cold.")) {
569		- insn->func->cfunc = insn->jump_dest->func;
570		- insn->jump_dest->func->pfunc = insn->func;
	1255	+ insn->func != insn->jump_dest->func) {
	1256	+
	1257	+ /*
	1258	+ * For GCC 8+, create parent/child links for any cold
	1259	+ * subfunctions. This is _mostly_ redundant with a
	1260	+ * similar initialization in read_symbols().
	1261	+ *
	1262	+ * If a function has aliases, we want the first such
	1263	+ * function in the symbol table to be the subfunction's
	1264	+ * parent. In that case we overwrite the
	1265	+ * initialization done in read_symbols().
	1266	+ *
	1267	+ * However this code can't completely replace the
	1268	+ * read_symbols() code because this doesn't detect the
	1269	+ * case where the parent function's only reference to a
	1270	+ * subfunction is through a jump table.
	1271	+ */
	1272	+ if (!strstr(insn->func->name, ".cold") &&
	1273	+ strstr(insn->jump_dest->func->name, ".cold")) {
	1274	+ insn->func->cfunc = insn->jump_dest->func;
	1275	+ insn->jump_dest->func->pfunc = insn->func;
	1276	+
	1277	+ } else if (insn->jump_dest->func->pfunc != insn->func->pfunc &&
	1278	+ insn->jump_dest->offset == insn->jump_dest->func->offset) {
	1279	+ /* internal sibling call (without reloc) */
	1280	+ add_call_dest(file, insn, insn->jump_dest->func, true);
	1281	+ }
571	1282	}
572	1283	}
573	1284
574	1285	return 0;
	1286	+}
	1287	+
	1288	+static struct symbol find_call_destination(struct section sec, unsigned long offset)
	1289	+{
	1290	+ struct symbol *call_dest;
	1291	+
	1292	+ call_dest = find_func_by_offset(sec, offset);
	1293	+ if (!call_dest)
	1294	+ call_dest = find_symbol_by_offset(sec, offset);
	1295	+
	1296	+ return call_dest;
575	1297	}
576	1298
577	1299	/*
..	..	@@ -581,72 +1303,85 @@
581	1303	{
582	1304	struct instruction *insn;
583	1305	unsigned long dest_off;
584		- struct rela *rela;
	1306	+ struct symbol *dest;
	1307	+ struct reloc *reloc;
585	1308
586	1309	for_each_insn(file, insn) {
587	1310	if (insn->type != INSN_CALL)
588	1311	continue;
589	1312
590		- rela = find_rela_by_dest_range(insn->sec, insn->offset,
591		- insn->len);
592		- if (!rela) {
593		- dest_off = insn->offset + insn->len + insn->immediate;
594		- insn->call_dest = find_symbol_by_offset(insn->sec,
595		- dest_off);
	1313	+ reloc = insn_reloc(file, insn);
	1314	+ if (!reloc) {
	1315	+ dest_off = arch_jump_destination(insn);
	1316	+ dest = find_call_destination(insn->sec, dest_off);
596	1317
597		- if (!insn->call_dest && !insn->ignore) {
598		- WARN_FUNC("unsupported intra-function call",
	1318	+ add_call_dest(file, insn, dest, false);
	1319	+
	1320	+ if (insn->ignore)
	1321	+ continue;
	1322	+
	1323	+ if (!insn->call_dest) {
	1324	+ WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
	1325	+ return -1;
	1326	+ }
	1327	+
	1328	+ if (insn->func && insn->call_dest->type != STT_FUNC) {
	1329	+ WARN_FUNC("unsupported call to non-function",
599	1330	insn->sec, insn->offset);
600		- if (retpoline)
601		- WARN("If this is a retpoline, please patch it in with alternatives and annotate it with ANNOTATE_NOSPEC_ALTERNATIVE.");
602	1331	return -1;
603	1332	}
604	1333
605		- } else if (rela->sym->type == STT_SECTION) {
606		- insn->call_dest = find_symbol_by_offset(rela->sym->sec,
607		- rela->addend+4);
608		- if (!insn->call_dest \|\|
609		- insn->call_dest->type != STT_FUNC) {
610		- WARN_FUNC("can't find call dest symbol at %s+0x%x",
	1334	+ } else if (reloc->sym->type == STT_SECTION) {
	1335	+ dest_off = arch_dest_reloc_offset(reloc->addend);
	1336	+ dest = find_call_destination(reloc->sym->sec, dest_off);
	1337	+ if (!dest) {
	1338	+ if (is_cfi_section(reloc->sym->sec))
	1339	+ continue;
	1340	+
	1341	+ WARN_FUNC("can't find call dest symbol at %s+0x%lx",
611	1342	insn->sec, insn->offset,
612		- rela->sym->sec->name,
613		- rela->addend + 4);
	1343	+ reloc->sym->sec->name,
	1344	+ dest_off);
614	1345	return -1;
615	1346	}
	1347	+
	1348	+ add_call_dest(file, insn, dest, false);
	1349	+
	1350	+ } else if (reloc->sym->retpoline_thunk) {
	1351	+ add_retpoline_call(file, insn);
	1352	+
616	1353	} else
617		- insn->call_dest = rela->sym;
	1354	+ add_call_dest(file, insn, reloc->sym, false);
618	1355	}
619	1356
620	1357	return 0;
621	1358	}
622	1359
623	1360	/*
624		- * The .alternatives section requires some extra special care, over and above
625		- * what other special sections require:
626		- *
627		- * 1. Because alternatives are patched in-place, we need to insert a fake jump
628		- * instruction at the end so that validate_branch() skips all the original
629		- * replaced instructions when validating the new instruction path.
630		- *
631		- * 2. An added wrinkle is that the new instruction length might be zero. In
632		- * that case the old instructions are replaced with noops. We simulate that
633		- * by creating a fake jump as the only new instruction.
634		- *
635		- * 3. In some cases, the alternative section includes an instruction which
636		- * conditionally jumps to the _end_ of the entry. We have to modify these
637		- * jumps' destinations to point back to .text rather than the end of the
638		- * entry in .altinstr_replacement.
639		- *
640		- * 4. It has been requested that we don't validate the !POPCNT feature path
641		- * which is a "very very small percentage of machines".
	1361	+ * The .alternatives section requires some extra special care over and above
	1362	+ * other special sections because alternatives are patched in place.
642	1363	*/
643	1364	static int handle_group_alt(struct objtool_file *file,
644	1365	struct special_alt *special_alt,
645	1366	struct instruction *orig_insn,
646	1367	struct instruction **new_insn)
647	1368	{
648		- struct instruction last_orig_insn, last_new_insn, insn, fake_jump = NULL;
	1369	+ struct instruction last_orig_insn, last_new_insn = NULL, insn, nop = NULL;
	1370	+ struct alt_group orig_alt_group, new_alt_group;
649	1371	unsigned long dest_off;
	1372	+
	1373	+
	1374	+ orig_alt_group = malloc(sizeof(*orig_alt_group));
	1375	+ if (!orig_alt_group) {
	1376	+ WARN("malloc failed");
	1377	+ return -1;
	1378	+ }
	1379	+ orig_alt_group->cfi = calloc(special_alt->orig_len,
	1380	+ sizeof(struct cfi_state *));
	1381	+ if (!orig_alt_group->cfi) {
	1382	+ WARN("calloc failed");
	1383	+ return -1;
	1384	+ }
650	1385
651	1386	last_orig_insn = NULL;
652	1387	insn = orig_insn;
..	..	@@ -654,67 +1389,90 @@
654	1389	if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
655	1390	break;
656	1391
657		- if (special_alt->skip_orig)
658		- insn->type = INSN_NOP;
659		-
660		- insn->alt_group = true;
	1392	+ insn->alt_group = orig_alt_group;
661	1393	last_orig_insn = insn;
662	1394	}
	1395	+ orig_alt_group->orig_group = NULL;
	1396	+ orig_alt_group->first_insn = orig_insn;
	1397	+ orig_alt_group->last_insn = last_orig_insn;
663	1398
664		- if (next_insn_same_sec(file, last_orig_insn)) {
665		- fake_jump = malloc(sizeof(*fake_jump));
666		- if (!fake_jump) {
	1399	+
	1400	+ new_alt_group = malloc(sizeof(*new_alt_group));
	1401	+ if (!new_alt_group) {
	1402	+ WARN("malloc failed");
	1403	+ return -1;
	1404	+ }
	1405	+
	1406	+ if (special_alt->new_len < special_alt->orig_len) {
	1407	+ /*
	1408	+ * Insert a fake nop at the end to make the replacement
	1409	+ * alt_group the same size as the original. This is needed to
	1410	+ * allow propagate_alt_cfi() to do its magic. When the last
	1411	+ * instruction affects the stack, the instruction after it (the
	1412	+ * nop) will propagate the new state to the shared CFI array.
	1413	+ */
	1414	+ nop = malloc(sizeof(*nop));
	1415	+ if (!nop) {
667	1416	WARN("malloc failed");
668	1417	return -1;
669	1418	}
670		- memset(fake_jump, 0, sizeof(*fake_jump));
671		- INIT_LIST_HEAD(&fake_jump->alts);
672		- clear_insn_state(&fake_jump->state);
	1419	+ memset(nop, 0, sizeof(*nop));
	1420	+ INIT_LIST_HEAD(&nop->alts);
	1421	+ INIT_LIST_HEAD(&nop->stack_ops);
673	1422
674		- fake_jump->sec = special_alt->new_sec;
675		- fake_jump->offset = FAKE_JUMP_OFFSET;
676		- fake_jump->type = INSN_JUMP_UNCONDITIONAL;
677		- fake_jump->jump_dest = list_next_entry(last_orig_insn, list);
678		- fake_jump->func = orig_insn->func;
	1423	+ nop->sec = special_alt->new_sec;
	1424	+ nop->offset = special_alt->new_off + special_alt->new_len;
	1425	+ nop->len = special_alt->orig_len - special_alt->new_len;
	1426	+ nop->type = INSN_NOP;
	1427	+ nop->func = orig_insn->func;
	1428	+ nop->alt_group = new_alt_group;
	1429	+ nop->ignore = orig_insn->ignore_alts;
679	1430	}
680	1431
681	1432	if (!special_alt->new_len) {
682		- if (!fake_jump) {
683		- WARN("%s: empty alternative at end of section",
684		- special_alt->orig_sec->name);
685		- return -1;
686		- }
687		-
688		- *new_insn = fake_jump;
689		- return 0;
	1433	+ *new_insn = nop;
	1434	+ goto end;
690	1435	}
691	1436
692		- last_new_insn = NULL;
693	1437	insn = *new_insn;
694	1438	sec_for_each_insn_from(file, insn) {
	1439	+ struct reloc *alt_reloc;
	1440	+
695	1441	if (insn->offset >= special_alt->new_off + special_alt->new_len)
696	1442	break;
697	1443
698	1444	last_new_insn = insn;
699	1445
700	1446	insn->ignore = orig_insn->ignore_alts;
	1447	+ insn->func = orig_insn->func;
	1448	+ insn->alt_group = new_alt_group;
701	1449
702		- if (insn->type != INSN_JUMP_CONDITIONAL &&
703		- insn->type != INSN_JUMP_UNCONDITIONAL)
	1450	+ /*
	1451	+ * Since alternative replacement code is copy/pasted by the
	1452	+ * kernel after applying relocations, generally such code can't
	1453	+ * have relative-address relocation references to outside the
	1454	+ * .altinstr_replacement section, unless the arch's
	1455	+ * alternatives code can adjust the relative offsets
	1456	+ * accordingly.
	1457	+ */
	1458	+ alt_reloc = insn_reloc(file, insn);
	1459	+ if (alt_reloc &&
	1460	+ !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
	1461	+
	1462	+ WARN_FUNC("unsupported relocation in alternatives section",
	1463	+ insn->sec, insn->offset);
	1464	+ return -1;
	1465	+ }
	1466	+
	1467	+ if (!is_static_jump(insn))
704	1468	continue;
705	1469
706	1470	if (!insn->immediate)
707	1471	continue;
708	1472
709		- dest_off = insn->offset + insn->len + insn->immediate;
710		- if (dest_off == special_alt->new_off + special_alt->new_len) {
711		- if (!fake_jump) {
712		- WARN("%s: alternative jump to end of section",
713		- special_alt->orig_sec->name);
714		- return -1;
715		- }
716		- insn->jump_dest = fake_jump;
717		- }
	1473	+ dest_off = arch_jump_destination(insn);
	1474	+ if (dest_off == special_alt->new_off + special_alt->new_len)
	1475	+ insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
718	1476
719	1477	if (!insn->jump_dest) {
720	1478	WARN_FUNC("can't find alternative jump destination",
..	..	@@ -729,9 +1487,13 @@
729	1487	return -1;
730	1488	}
731	1489
732		- if (fake_jump)
733		- list_add(&fake_jump->list, &last_new_insn->list);
734		-
	1490	+ if (nop)
	1491	+ list_add(&nop->list, &last_new_insn->list);
	1492	+end:
	1493	+ new_alt_group->orig_group = orig_alt_group;
	1494	+ new_alt_group->first_insn = *new_insn;
	1495	+ new_alt_group->last_insn = nop ? : last_new_insn;
	1496	+ new_alt_group->cfi = orig_alt_group->cfi;
735	1497	return 0;
736	1498	}
737	1499
..	..	@@ -826,6 +1588,8 @@
826	1588	}
827	1589
828	1590	alt->insn = new_insn;
	1591	+ alt->skip_orig = special_alt->skip_orig;
	1592	+ orig_insn->ignore_alts \|= special_alt->skip_alt;
829	1593	list_add_tail(&alt->list, &orig_insn->alts);
830	1594
831	1595	list_del(&special_alt->list);
..	..	@@ -836,34 +1600,40 @@
836	1600	return ret;
837	1601	}
838	1602
839		-static int add_switch_table(struct objtool_file file, struct instruction insn,
840		- struct rela table, struct rela next_table)
	1603	+static int add_jump_table(struct objtool_file file, struct instruction insn,
	1604	+ struct reloc *table)
841	1605	{
842		- struct rela *rela = table;
843		- struct instruction *alt_insn;
	1606	+ struct reloc *reloc = table;
	1607	+ struct instruction *dest_insn;
844	1608	struct alternative *alt;
845	1609	struct symbol *pfunc = insn->func->pfunc;
846	1610	unsigned int prev_offset = 0;
847	1611
848		- list_for_each_entry_from(rela, &table->rela_sec->rela_list, list) {
849		- if (rela == next_table)
	1612	+ /*
	1613	+ * Each @reloc is a switch table relocation which points to the target
	1614	+ * instruction.
	1615	+ */
	1616	+ list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
	1617	+
	1618	+ /* Check for the end of the table: */
	1619	+ if (reloc != table && reloc->jump_table_start)
850	1620	break;
851	1621
852		- /* Make sure the switch table entries are consecutive: */
853		- if (prev_offset && rela->offset != prev_offset + 8)
	1622	+ /* Make sure the table entries are consecutive: */
	1623	+ if (prev_offset && reloc->offset != prev_offset + 8)
854	1624	break;
855	1625
856	1626	/* Detect function pointers from contiguous objects: */
857		- if (rela->sym->sec == pfunc->sec &&
858		- rela->addend == pfunc->offset)
	1627	+ if (reloc->sym->sec == pfunc->sec &&
	1628	+ reloc->addend == pfunc->offset)
859	1629	break;
860	1630
861		- alt_insn = find_insn(file, rela->sym->sec, rela->addend);
862		- if (!alt_insn)
	1631	+ dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
	1632	+ if (!dest_insn)
863	1633	break;
864	1634
865		- /* Make sure the jmp dest is in the function or subfunction: */
866		- if (alt_insn->func->pfunc != pfunc)
	1635	+ /* Make sure the destination is in the same function: */
	1636	+ if (!dest_insn->func \|\| dest_insn->func->pfunc != pfunc)
867	1637	break;
868	1638
869	1639	alt = malloc(sizeof(*alt));
..	..	@@ -872,9 +1642,9 @@
872	1642	return -1;
873	1643	}
874	1644
875		- alt->insn = alt_insn;
	1645	+ alt->insn = dest_insn;
876	1646	list_add_tail(&alt->list, &insn->alts);
877		- prev_offset = rela->offset;
	1647	+ prev_offset = reloc->offset;
878	1648	}
879	1649
880	1650	if (!prev_offset) {
..	..	@@ -887,56 +1657,15 @@
887	1657	}
888	1658
889	1659	/*
890		- * find_switch_table() - Given a dynamic jump, find the switch jump table in
891		- * .rodata associated with it.
892		- *
893		- * There are 3 basic patterns:
894		- *
895		- * 1. jmpq *[rodata addr](,%reg,8)
896		- *
897		- * This is the most common case by far. It jumps to an address in a simple
898		- * jump table which is stored in .rodata.
899		- *
900		- * 2. jmpq *[rodata addr](%rip)
901		- *
902		- * This is caused by a rare GCC quirk, currently only seen in three driver
903		- * functions in the kernel, only with certain obscure non-distro configs.
904		- *
905		- * As part of an optimization, GCC makes a copy of an existing switch jump
906		- * table, modifies it, and then hard-codes the jump (albeit with an indirect
907		- * jump) to use a single entry in the table. The rest of the jump table and
908		- * some of its jump targets remain as dead code.
909		- *
910		- * In such a case we can just crudely ignore all unreachable instruction
911		- * warnings for the entire object file. Ideally we would just ignore them
912		- * for the function, but that would require redesigning the code quite a
913		- * bit. And honestly that's just not worth doing: unreachable instruction
914		- * warnings are of questionable value anyway, and this is such a rare issue.
915		- *
916		- * 3. mov [rodata addr],%reg1
917		- * ... some instructions ...
918		- * jmpq *(%reg1,%reg2,8)
919		- *
920		- * This is a fairly uncommon pattern which is new for GCC 6. As of this
921		- * writing, there are 11 occurrences of it in the allmodconfig kernel.
922		- *
923		- * As of GCC 7 there are quite a few more of these and the 'in between' code
924		- * is significant. Esp. with KASAN enabled some of the code between the mov
925		- * and jmpq uses .rodata itself, which can confuse things.
926		- *
927		- * TODO: Once we have DWARF CFI and smarter instruction decoding logic,
928		- * ensure the same register is used in the mov and jump instructions.
929		- *
930		- * NOTE: RETPOLINE made it harder still to decode dynamic jumps.
	1660	+ * find_jump_table() - Given a dynamic jump, find the switch jump table
	1661	+ * associated with it.
931	1662	*/
932		-static struct rela find_switch_table(struct objtool_file file,
	1663	+static struct reloc find_jump_table(struct objtool_file file,
933	1664	struct symbol *func,
934	1665	struct instruction *insn)
935	1666	{
936		- struct rela text_rela, rodata_rela;
937		- struct instruction *orig_insn = insn;
938		- struct section *rodata_sec;
939		- unsigned long table_offset;
	1667	+ struct reloc *table_reloc;
	1668	+ struct instruction dest_insn, orig_insn = insn;
940	1669
941	1670	/*
942	1671	* Backward search using the @first_jump_src links, these help avoid
..	..	@@ -944,8 +1673,8 @@
944	1673	* it.
945	1674	*/
946	1675	for (;
947		- &insn->list != &file->insn_list && insn->func && insn->func->pfunc == func;
948		- insn = insn->first_jump_src ?: list_prev_entry(insn, list)) {
	1676	+ insn && insn->func && insn->func->pfunc == func;
	1677	+ insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
949	1678
950	1679	if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
951	1680	break;
..	..	@@ -957,58 +1686,36 @@
957	1686	insn->jump_dest->offset > orig_insn->offset))
958	1687	break;
959	1688
960		- /* look for a relocation which references .rodata */
961		- text_rela = find_rela_by_dest_range(insn->sec, insn->offset,
962		- insn->len);
963		- if (!text_rela \|\| text_rela->sym->type != STT_SECTION \|\|
964		- !text_rela->sym->sec->rodata)
	1689	+ table_reloc = arch_find_switch_table(file, insn);
	1690	+ if (!table_reloc)
	1691	+ continue;
	1692	+ dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
	1693	+ if (!dest_insn \|\| !dest_insn->func \|\| dest_insn->func->pfunc != func)
965	1694	continue;
966	1695
967		- table_offset = text_rela->addend;
968		- rodata_sec = text_rela->sym->sec;
969		-
970		- if (text_rela->type == R_X86_64_PC32)
971		- table_offset += 4;
972		-
973		- /*
974		- * Make sure the .rodata address isn't associated with a
975		- * symbol. gcc jump tables are anonymous data.
976		- */
977		- if (find_symbol_containing(rodata_sec, table_offset))
978		- continue;
979		-
980		- rodata_rela = find_rela_by_dest(rodata_sec, table_offset);
981		- if (rodata_rela) {
982		- /*
983		- * Use of RIP-relative switch jumps is quite rare, and
984		- * indicates a rare GCC quirk/bug which can leave dead
985		- * code behind.
986		- */
987		- if (text_rela->type == R_X86_64_PC32)
988		- file->ignore_unreachables = true;
989		-
990		- return rodata_rela;
991		- }
	1696	+ return table_reloc;
992	1697	}
993	1698
994	1699	return NULL;
995	1700	}
996	1701
997		-
998		-static int add_func_switch_tables(struct objtool_file *file,
999		- struct symbol *func)
	1702	+/*
	1703	+ * First pass: Mark the head of each jump table so that in the next pass,
	1704	+ * we know when a given jump table ends and the next one starts.
	1705	+ */
	1706	+static void mark_func_jump_tables(struct objtool_file *file,
	1707	+ struct symbol *func)
1000	1708	{
1001		- struct instruction insn, last = NULL, *prev_jump = NULL;
1002		- struct rela rela, prev_rela = NULL;
1003		- int ret;
	1709	+ struct instruction insn, last = NULL;
	1710	+ struct reloc *reloc;
1004	1711
1005		- func_for_each_insn_all(file, func, insn) {
	1712	+ func_for_each_insn(file, func, insn) {
1006	1713	if (!last)
1007	1714	last = insn;
1008	1715
1009	1716	/*
1010	1717	* Store back-pointers for unconditional forward jumps such
1011		- * that find_switch_table() can back-track using those and
	1718	+ * that find_jump_table() can back-track using those and
1012	1719	* avoid some potentially confusing code.
1013	1720	*/
1014	1721	if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
..	..	@@ -1023,27 +1730,25 @@
1023	1730	if (insn->type != INSN_JUMP_DYNAMIC)
1024	1731	continue;
1025	1732
1026		- rela = find_switch_table(file, func, insn);
1027		- if (!rela)
	1733	+ reloc = find_jump_table(file, func, insn);
	1734	+ if (reloc) {
	1735	+ reloc->jump_table_start = true;
	1736	+ insn->jump_table = reloc;
	1737	+ }
	1738	+ }
	1739	+}
	1740	+
	1741	+static int add_func_jump_tables(struct objtool_file *file,
	1742	+ struct symbol *func)
	1743	+{
	1744	+ struct instruction *insn;
	1745	+ int ret;
	1746	+
	1747	+ func_for_each_insn(file, func, insn) {
	1748	+ if (!insn->jump_table)
1028	1749	continue;
1029	1750
1030		- /*
1031		- * We found a switch table, but we don't know yet how big it
1032		- * is. Don't add it until we reach the end of the function or
1033		- * the beginning of another switch table in the same function.
1034		- */
1035		- if (prev_jump) {
1036		- ret = add_switch_table(file, prev_jump, prev_rela, rela);
1037		- if (ret)
1038		- return ret;
1039		- }
1040		-
1041		- prev_jump = insn;
1042		- prev_rela = rela;
1043		- }
1044		-
1045		- if (prev_jump) {
1046		- ret = add_switch_table(file, prev_jump, prev_rela, NULL);
	1751	+ ret = add_jump_table(file, insn, insn->jump_table);
1047	1752	if (ret)
1048	1753	return ret;
1049	1754	}
..	..	@@ -1056,7 +1761,7 @@
1056	1761	* section which contains a list of addresses within the function to jump to.
1057	1762	* This finds these jump tables and adds them to the insn->alts lists.
1058	1763	*/
1059		-static int add_switch_table_alts(struct objtool_file *file)
	1764	+static int add_jump_table_alts(struct objtool_file *file)
1060	1765	{
1061	1766	struct section *sec;
1062	1767	struct symbol *func;
..	..	@@ -1070,7 +1775,8 @@
1070	1775	if (func->type != STT_FUNC)
1071	1776	continue;
1072	1777
1073		- ret = add_func_switch_tables(file, func);
	1778	+ mark_func_jump_tables(file, func);
	1779	+ ret = add_func_jump_tables(file, func);
1074	1780	if (ret)
1075	1781	return ret;
1076	1782	}
..	..	@@ -1079,21 +1785,29 @@
1079	1785	return 0;
1080	1786	}
1081	1787
	1788	+static void set_func_state(struct cfi_state *state)
	1789	+{
	1790	+ state->cfa = initial_func_cfi.cfa;
	1791	+ memcpy(&state->regs, &initial_func_cfi.regs,
	1792	+ CFI_NUM_REGS * sizeof(struct cfi_reg));
	1793	+ state->stack_size = initial_func_cfi.cfa.offset;
	1794	+}
	1795	+
1082	1796	static int read_unwind_hints(struct objtool_file *file)
1083	1797	{
1084		- struct section sec, relasec;
1085		- struct rela *rela;
	1798	+ struct cfi_state cfi = init_cfi;
	1799	+ struct section sec, relocsec;
1086	1800	struct unwind_hint *hint;
1087	1801	struct instruction *insn;
1088		- struct cfi_reg *cfa;
	1802	+ struct reloc *reloc;
1089	1803	int i;
1090	1804
1091	1805	sec = find_section_by_name(file->elf, ".discard.unwind_hints");
1092	1806	if (!sec)
1093	1807	return 0;
1094	1808
1095		- relasec = sec->rela;
1096		- if (!relasec) {
	1809	+ relocsec = sec->reloc;
	1810	+ if (!relocsec) {
1097	1811	WARN("missing .rela.discard.unwind_hints section");
1098	1812	return -1;
1099	1813	}
..	..	@@ -1108,66 +1822,63 @@
1108	1822	for (i = 0; i < sec->len / sizeof(struct unwind_hint); i++) {
1109	1823	hint = (struct unwind_hint *)sec->data->d_buf + i;
1110	1824
1111		- rela = find_rela_by_dest(sec, i * sizeof(*hint));
1112		- if (!rela) {
1113		- WARN("can't find rela for unwind_hints[%d]", i);
	1825	+ reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
	1826	+ if (!reloc) {
	1827	+ WARN("can't find reloc for unwind_hints[%d]", i);
1114	1828	return -1;
1115	1829	}
1116	1830
1117		- insn = find_insn(file, rela->sym->sec, rela->addend);
	1831	+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
1118	1832	if (!insn) {
1119	1833	WARN("can't find insn for unwind_hints[%d]", i);
1120	1834	return -1;
1121	1835	}
1122	1836
1123		- cfa = &insn->state.cfa;
	1837	+ insn->hint = true;
1124	1838
1125	1839	if (hint->type == UNWIND_HINT_TYPE_SAVE) {
	1840	+ insn->hint = false;
1126	1841	insn->save = true;
1127		- continue;
1128		-
1129		- } else if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
1130		- insn->restore = true;
1131		- insn->hint = true;
1132	1842	continue;
1133	1843	}
1134	1844
1135		- insn->hint = true;
	1845	+ if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
	1846	+ insn->restore = true;
	1847	+ continue;
	1848	+ }
1136	1849
1137		- switch (hint->sp_reg) {
1138		- case ORC_REG_UNDEFINED:
1139		- cfa->base = CFI_UNDEFINED;
1140		- break;
1141		- case ORC_REG_SP:
1142		- cfa->base = CFI_SP;
1143		- break;
1144		- case ORC_REG_BP:
1145		- cfa->base = CFI_BP;
1146		- break;
1147		- case ORC_REG_SP_INDIRECT:
1148		- cfa->base = CFI_SP_INDIRECT;
1149		- break;
1150		- case ORC_REG_R10:
1151		- cfa->base = CFI_R10;
1152		- break;
1153		- case ORC_REG_R13:
1154		- cfa->base = CFI_R13;
1155		- break;
1156		- case ORC_REG_DI:
1157		- cfa->base = CFI_DI;
1158		- break;
1159		- case ORC_REG_DX:
1160		- cfa->base = CFI_DX;
1161		- break;
1162		- default:
	1850	+ if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
	1851	+ struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
	1852	+
	1853	+ if (sym && sym->bind == STB_GLOBAL) {
	1854	+ insn->entry = 1;
	1855	+ }
	1856	+ }
	1857	+
	1858	+ if (hint->type == UNWIND_HINT_TYPE_ENTRY) {
	1859	+ hint->type = UNWIND_HINT_TYPE_CALL;
	1860	+ insn->entry = 1;
	1861	+ }
	1862	+
	1863	+ if (hint->type == UNWIND_HINT_TYPE_FUNC) {
	1864	+ insn->cfi = &func_cfi;
	1865	+ continue;
	1866	+ }
	1867	+
	1868	+ if (insn->cfi)
	1869	+ cfi = *(insn->cfi);
	1870	+
	1871	+ if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
1163	1872	WARN_FUNC("unsupported unwind_hint sp base reg %d",
1164	1873	insn->sec, insn->offset, hint->sp_reg);
1165	1874	return -1;
1166	1875	}
1167	1876
1168		- cfa->offset = hint->sp_offset;
1169		- insn->state.type = hint->type;
1170		- insn->state.end = hint->end;
	1877	+ cfi.cfa.offset = hint->sp_offset;
	1878	+ cfi.type = hint->type;
	1879	+ cfi.end = hint->end;
	1880	+
	1881	+ insn->cfi = cfi_hash_find_or_add(&cfi);
1171	1882	}
1172	1883
1173	1884	return 0;
..	..	@@ -1177,32 +1888,163 @@
1177	1888	{
1178	1889	struct section *sec;
1179	1890	struct instruction *insn;
1180		- struct rela *rela;
	1891	+ struct reloc *reloc;
1181	1892
1182	1893	sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
1183	1894	if (!sec)
1184	1895	return 0;
1185	1896
1186		- list_for_each_entry(rela, &sec->rela_list, list) {
1187		- if (rela->sym->type != STT_SECTION) {
	1897	+ list_for_each_entry(reloc, &sec->reloc_list, list) {
	1898	+ if (reloc->sym->type != STT_SECTION) {
1188	1899	WARN("unexpected relocation symbol type in %s", sec->name);
1189	1900	return -1;
1190	1901	}
1191	1902
1192		- insn = find_insn(file, rela->sym->sec, rela->addend);
	1903	+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
1193	1904	if (!insn) {
1194	1905	WARN("bad .discard.retpoline_safe entry");
1195	1906	return -1;
1196	1907	}
1197	1908
1198	1909	if (insn->type != INSN_JUMP_DYNAMIC &&
1199		- insn->type != INSN_CALL_DYNAMIC) {
1200		- WARN_FUNC("retpoline_safe hint not an indirect jump/call",
	1910	+ insn->type != INSN_CALL_DYNAMIC &&
	1911	+ insn->type != INSN_RETURN &&
	1912	+ insn->type != INSN_NOP) {
	1913	+ WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop",
1201	1914	insn->sec, insn->offset);
1202	1915	return -1;
1203	1916	}
1204	1917
1205	1918	insn->retpoline_safe = true;
	1919	+ }
	1920	+
	1921	+ return 0;
	1922	+}
	1923	+
	1924	+static int read_instr_hints(struct objtool_file *file)
	1925	+{
	1926	+ struct section *sec;
	1927	+ struct instruction *insn;
	1928	+ struct reloc *reloc;
	1929	+
	1930	+ sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
	1931	+ if (!sec)
	1932	+ return 0;
	1933	+
	1934	+ list_for_each_entry(reloc, &sec->reloc_list, list) {
	1935	+ if (reloc->sym->type != STT_SECTION) {
	1936	+ WARN("unexpected relocation symbol type in %s", sec->name);
	1937	+ return -1;
	1938	+ }
	1939	+
	1940	+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
	1941	+ if (!insn) {
	1942	+ WARN("bad .discard.instr_end entry");
	1943	+ return -1;
	1944	+ }
	1945	+
	1946	+ insn->instr--;
	1947	+ }
	1948	+
	1949	+ sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
	1950	+ if (!sec)
	1951	+ return 0;
	1952	+
	1953	+ list_for_each_entry(reloc, &sec->reloc_list, list) {
	1954	+ if (reloc->sym->type != STT_SECTION) {
	1955	+ WARN("unexpected relocation symbol type in %s", sec->name);
	1956	+ return -1;
	1957	+ }
	1958	+
	1959	+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
	1960	+ if (!insn) {
	1961	+ WARN("bad .discard.instr_begin entry");
	1962	+ return -1;
	1963	+ }
	1964	+
	1965	+ insn->instr++;
	1966	+ }
	1967	+
	1968	+ return 0;
	1969	+}
	1970	+
	1971	+static int read_intra_function_calls(struct objtool_file *file)
	1972	+{
	1973	+ struct instruction *insn;
	1974	+ struct section *sec;
	1975	+ struct reloc *reloc;
	1976	+
	1977	+ sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
	1978	+ if (!sec)
	1979	+ return 0;
	1980	+
	1981	+ list_for_each_entry(reloc, &sec->reloc_list, list) {
	1982	+ unsigned long dest_off;
	1983	+
	1984	+ if (reloc->sym->type != STT_SECTION) {
	1985	+ WARN("unexpected relocation symbol type in %s",
	1986	+ sec->name);
	1987	+ return -1;
	1988	+ }
	1989	+
	1990	+ insn = find_insn(file, reloc->sym->sec, reloc->addend);
	1991	+ if (!insn) {
	1992	+ WARN("bad .discard.intra_function_call entry");
	1993	+ return -1;
	1994	+ }
	1995	+
	1996	+ if (insn->type != INSN_CALL) {
	1997	+ WARN_FUNC("intra_function_call not a direct call",
	1998	+ insn->sec, insn->offset);
	1999	+ return -1;
	2000	+ }
	2001	+
	2002	+ /*
	2003	+ * Treat intra-function CALLs as JMPs, but with a stack_op.
	2004	+ * See add_call_destinations(), which strips stack_ops from
	2005	+ * normal CALLs.
	2006	+ */
	2007	+ insn->type = INSN_JUMP_UNCONDITIONAL;
	2008	+
	2009	+ dest_off = insn->offset + insn->len + insn->immediate;
	2010	+ insn->jump_dest = find_insn(file, insn->sec, dest_off);
	2011	+ if (!insn->jump_dest) {
	2012	+ WARN_FUNC("can't find call dest at %s+0x%lx",
	2013	+ insn->sec, insn->offset,
	2014	+ insn->sec->name, dest_off);
	2015	+ return -1;
	2016	+ }
	2017	+ }
	2018	+
	2019	+ return 0;
	2020	+}
	2021	+
	2022	+static int classify_symbols(struct objtool_file *file)
	2023	+{
	2024	+ struct section *sec;
	2025	+ struct symbol *func;
	2026	+
	2027	+ for_each_sec(file, sec) {
	2028	+ list_for_each_entry(func, &sec->symbol_list, list) {
	2029	+ if (func->bind != STB_GLOBAL)
	2030	+ continue;
	2031	+
	2032	+ if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
	2033	+ strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
	2034	+ func->static_call_tramp = true;
	2035	+
	2036	+ if (arch_is_retpoline(func))
	2037	+ func->retpoline_thunk = true;
	2038	+
	2039	+ if (arch_is_rethunk(func))
	2040	+ func->return_thunk = true;
	2041	+
	2042	+ if (!strcmp(func->name, "__fentry__"))
	2043	+ func->fentry = true;
	2044	+
	2045	+ if (!strncmp(func->name, "__sanitizer_cov_", 16))
	2046	+ func->kcov = true;
	2047	+ }
1206	2048	}
1207	2049
1208	2050	return 0;
..	..	@@ -1214,9 +2056,14 @@
1214	2056	bool found = false;
1215	2057
1216	2058	/*
1217		- * This searches for the .rodata section or multiple .rodata.func_name
1218		- * sections if -fdata-sections is being used. The .str.1.1 and .str.1.8
1219		- * rodata sections are ignored as they don't contain jump tables.
	2059	+ * Search for the following rodata sections, each of which can
	2060	+ * potentially contain jump tables:
	2061	+ *
	2062	+ * - .rodata: can contain GCC switch tables
	2063	+ * - .rodata.<func>: same, if -fdata-sections is being used
	2064	+ * - .rodata..c_jump_table: contains C annotated jump tables
	2065	+ *
	2066	+ * .rodata.str1.* sections are ignored; they don't contain jump tables.
1220	2067	*/
1221	2068	for_each_sec(file, sec) {
1222	2069	if (!strncmp(sec->name, ".rodata", 7) &&
..	..	@@ -1244,11 +2091,24 @@
1244	2091	return ret;
1245	2092
1246	2093	add_ignores(file);
	2094	+ add_uaccess_safe(file);
	2095	+ add_cfi_jumptables(file);
1247	2096
1248		- ret = add_nospec_ignores(file);
	2097	+ ret = add_ignore_alternatives(file);
1249	2098	if (ret)
1250	2099	return ret;
1251	2100
	2101	+ /*
	2102	+ * Must be before add_{jump_call}_destination.
	2103	+ */
	2104	+ ret = classify_symbols(file);
	2105	+ if (ret)
	2106	+ return ret;
	2107	+
	2108	+ /*
	2109	+ * Must be before add_special_section_alts() as that depends on
	2110	+ * jump_dest being set.
	2111	+ */
1252	2112	ret = add_jump_destinations(file);
1253	2113	if (ret)
1254	2114	return ret;
..	..	@@ -1257,11 +2117,19 @@
1257	2117	if (ret)
1258	2118	return ret;
1259	2119
	2120	+ /*
	2121	+ * Must be before add_call_destination(); it changes INSN_CALL to
	2122	+ * INSN_JUMP.
	2123	+ */
	2124	+ ret = read_intra_function_calls(file);
	2125	+ if (ret)
	2126	+ return ret;
	2127	+
1260	2128	ret = add_call_destinations(file);
1261	2129	if (ret)
1262	2130	return ret;
1263	2131
1264		- ret = add_switch_table_alts(file);
	2132	+ ret = add_jump_table_alts(file);
1265	2133	if (ret)
1266	2134	return ret;
1267	2135
..	..	@@ -1273,63 +2141,75 @@
1273	2141	if (ret)
1274	2142	return ret;
1275	2143
	2144	+ ret = read_instr_hints(file);
	2145	+ if (ret)
	2146	+ return ret;
	2147	+
1276	2148	return 0;
1277	2149	}
1278	2150
1279	2151	static bool is_fentry_call(struct instruction *insn)
1280	2152	{
1281	2153	if (insn->type == INSN_CALL &&
1282		- insn->call_dest->type == STT_NOTYPE &&
1283		- !strcmp(insn->call_dest->name, "__fentry__"))
	2154	+ insn->call_dest &&
	2155	+ insn->call_dest->fentry)
1284	2156	return true;
1285	2157
1286	2158	return false;
1287	2159	}
1288	2160
1289		-static bool has_modified_stack_frame(struct insn_state *state)
	2161	+static bool has_modified_stack_frame(struct instruction insn, struct insn_state state)
1290	2162	{
	2163	+ struct cfi_state *cfi = &state->cfi;
1291	2164	int i;
1292	2165
1293		- if (state->cfa.base != initial_func_cfi.cfa.base \|\|
1294		- state->cfa.offset != initial_func_cfi.cfa.offset \|\|
1295		- state->stack_size != initial_func_cfi.cfa.offset \|\|
1296		- state->drap)
	2166	+ if (cfi->cfa.base != initial_func_cfi.cfa.base \|\| cfi->drap)
1297	2167	return true;
1298	2168
1299		- for (i = 0; i < CFI_NUM_REGS; i++)
1300		- if (state->regs[i].base != initial_func_cfi.regs[i].base \|\|
1301		- state->regs[i].offset != initial_func_cfi.regs[i].offset)
	2169	+ if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
	2170	+ return true;
	2171	+
	2172	+ if (cfi->stack_size != initial_func_cfi.cfa.offset)
	2173	+ return true;
	2174	+
	2175	+ for (i = 0; i < CFI_NUM_REGS; i++) {
	2176	+ if (cfi->regs[i].base != initial_func_cfi.regs[i].base \|\|
	2177	+ cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
1302	2178	return true;
	2179	+ }
1303	2180
1304	2181	return false;
1305	2182	}
1306	2183
1307	2184	static bool has_valid_stack_frame(struct insn_state *state)
1308	2185	{
1309		- if (state->cfa.base == CFI_BP && state->regs[CFI_BP].base == CFI_CFA &&
1310		- state->regs[CFI_BP].offset == -16)
	2186	+ struct cfi_state *cfi = &state->cfi;
	2187	+
	2188	+ if (cfi->cfa.base == CFI_BP && cfi->regs[CFI_BP].base == CFI_CFA &&
	2189	+ cfi->regs[CFI_BP].offset == -16)
1311	2190	return true;
1312	2191
1313		- if (state->drap && state->regs[CFI_BP].base == CFI_BP)
	2192	+ if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
1314	2193	return true;
1315	2194
1316	2195	return false;
1317	2196	}
1318	2197
1319		-static int update_insn_state_regs(struct instruction insn, struct insn_state state)
	2198	+static int update_cfi_state_regs(struct instruction *insn,
	2199	+ struct cfi_state *cfi,
	2200	+ struct stack_op *op)
1320	2201	{
1321		- struct cfi_reg *cfa = &state->cfa;
1322		- struct stack_op *op = &insn->stack_op;
	2202	+ struct cfi_reg *cfa = &cfi->cfa;
1323	2203
1324	2204	if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
1325	2205	return 0;
1326	2206
1327	2207	/* push */
1328		- if (op->dest.type == OP_DEST_PUSH)
	2208	+ if (op->dest.type == OP_DEST_PUSH \|\| op->dest.type == OP_DEST_PUSHF)
1329	2209	cfa->offset += 8;
1330	2210
1331	2211	/* pop */
1332		- if (op->src.type == OP_SRC_POP)
	2212	+ if (op->src.type == OP_SRC_POP \|\| op->src.type == OP_SRC_POPF)
1333	2213	cfa->offset -= 8;
1334	2214
1335	2215	/* add immediate to sp */
..	..	@@ -1340,20 +2220,19 @@
1340	2220	return 0;
1341	2221	}
1342	2222
1343		-static void save_reg(struct insn_state *state, unsigned char reg, int base,
1344		- int offset)
	2223	+static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
1345	2224	{
1346	2225	if (arch_callee_saved_reg(reg) &&
1347		- state->regs[reg].base == CFI_UNDEFINED) {
1348		- state->regs[reg].base = base;
1349		- state->regs[reg].offset = offset;
	2226	+ cfi->regs[reg].base == CFI_UNDEFINED) {
	2227	+ cfi->regs[reg].base = base;
	2228	+ cfi->regs[reg].offset = offset;
1350	2229	}
1351	2230	}
1352	2231
1353		-static void restore_reg(struct insn_state *state, unsigned char reg)
	2232	+static void restore_reg(struct cfi_state *cfi, unsigned char reg)
1354	2233	{
1355		- state->regs[reg].base = CFI_UNDEFINED;
1356		- state->regs[reg].offset = 0;
	2234	+ cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
	2235	+ cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
1357	2236	}
1358	2237
1359	2238	/*
..	..	@@ -1409,11 +2288,11 @@
1409	2288	* 41 5d pop %r13
1410	2289	* c3 retq
1411	2290	*/
1412		-static int update_insn_state(struct instruction insn, struct insn_state state)
	2291	+static int update_cfi_state(struct instruction insn, struct cfi_state cfi,
	2292	+ struct stack_op *op)
1413	2293	{
1414		- struct stack_op *op = &insn->stack_op;
1415		- struct cfi_reg *cfa = &state->cfa;
1416		- struct cfi_reg *regs = state->regs;
	2294	+ struct cfi_reg *cfa = &cfi->cfa;
	2295	+ struct cfi_reg *regs = cfi->regs;
1417	2296
1418	2297	/* stack operations don't make sense with an undefined CFA */
1419	2298	if (cfa->base == CFI_UNDEFINED) {
..	..	@@ -1424,8 +2303,9 @@
1424	2303	return 0;
1425	2304	}
1426	2305
1427		- if (state->type == ORC_TYPE_REGS \|\| state->type == ORC_TYPE_REGS_IRET)
1428		- return update_insn_state_regs(insn, state);
	2306	+ if (cfi->type == UNWIND_HINT_TYPE_REGS \|\|
	2307	+ cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
	2308	+ return update_cfi_state_regs(insn, cfi, op);
1429	2309
1430	2310	switch (op->dest.type) {
1431	2311
..	..	@@ -1440,16 +2320,16 @@
1440	2320
1441	2321	/* mov %rsp, %rbp */
1442	2322	cfa->base = op->dest.reg;
1443		- state->bp_scratch = false;
	2323	+ cfi->bp_scratch = false;
1444	2324	}
1445	2325
1446	2326	else if (op->src.reg == CFI_SP &&
1447		- op->dest.reg == CFI_BP && state->drap) {
	2327	+ op->dest.reg == CFI_BP && cfi->drap) {
1448	2328
1449	2329	/* drap: mov %rsp, %rbp */
1450	2330	regs[CFI_BP].base = CFI_BP;
1451		- regs[CFI_BP].offset = -state->stack_size;
1452		- state->bp_scratch = false;
	2331	+ regs[CFI_BP].offset = -cfi->stack_size;
	2332	+ cfi->bp_scratch = false;
1453	2333	}
1454	2334
1455	2335	else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
..	..	@@ -1464,8 +2344,8 @@
1464	2344	* ...
1465	2345	* mov %rax, %rsp
1466	2346	*/
1467		- state->vals[op->dest.reg].base = CFI_CFA;
1468		- state->vals[op->dest.reg].offset = -state->stack_size;
	2347	+ cfi->vals[op->dest.reg].base = CFI_CFA;
	2348	+ cfi->vals[op->dest.reg].offset = -cfi->stack_size;
1469	2349	}
1470	2350
1471	2351	else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
..	..	@@ -1476,14 +2356,14 @@
1476	2356	*
1477	2357	* Restore the original stack pointer (Clang).
1478	2358	*/
1479		- state->stack_size = -state->regs[CFI_BP].offset;
	2359	+ cfi->stack_size = -cfi->regs[CFI_BP].offset;
1480	2360	}
1481	2361
1482	2362	else if (op->dest.reg == cfa->base) {
1483	2363
1484	2364	/* mov %reg, %rsp */
1485	2365	if (cfa->base == CFI_SP &&
1486		- state->vals[op->src.reg].base == CFI_CFA) {
	2366	+ cfi->vals[op->src.reg].base == CFI_CFA) {
1487	2367
1488	2368	/*
1489	2369	* This is needed for the rare case
..	..	@@ -1493,8 +2373,8 @@
1493	2373	* ...
1494	2374	* mov %rcx, %rsp
1495	2375	*/
1496		- cfa->offset = -state->vals[op->src.reg].offset;
1497		- state->stack_size = cfa->offset;
	2376	+ cfa->offset = -cfi->vals[op->src.reg].offset;
	2377	+ cfi->stack_size = cfa->offset;
1498	2378
1499	2379	} else {
1500	2380	cfa->base = CFI_UNDEFINED;
..	..	@@ -1508,7 +2388,7 @@
1508	2388	if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
1509	2389
1510	2390	/* add imm, %rsp */
1511		- state->stack_size -= op->src.offset;
	2391	+ cfi->stack_size -= op->src.offset;
1512	2392	if (cfa->base == CFI_SP)
1513	2393	cfa->offset -= op->src.offset;
1514	2394	break;
..	..	@@ -1517,14 +2397,14 @@
1517	2397	if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
1518	2398
1519	2399	/* lea disp(%rbp), %rsp */
1520		- state->stack_size = -(op->src.offset + regs[CFI_BP].offset);
	2400	+ cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
1521	2401	break;
1522	2402	}
1523	2403
1524	2404	if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
1525	2405
1526	2406	/* drap: lea disp(%rsp), %drap */
1527		- state->drap_reg = op->dest.reg;
	2407	+ cfi->drap_reg = op->dest.reg;
1528	2408
1529	2409	/*
1530	2410	* lea disp(%rsp), %reg
..	..	@@ -1536,25 +2416,25 @@
1536	2416	* ...
1537	2417	* mov %rcx, %rsp
1538	2418	*/
1539		- state->vals[op->dest.reg].base = CFI_CFA;
1540		- state->vals[op->dest.reg].offset = \
1541		- -state->stack_size + op->src.offset;
	2419	+ cfi->vals[op->dest.reg].base = CFI_CFA;
	2420	+ cfi->vals[op->dest.reg].offset = \
	2421	+ -cfi->stack_size + op->src.offset;
1542	2422
1543	2423	break;
1544	2424	}
1545	2425
1546		- if (state->drap && op->dest.reg == CFI_SP &&
1547		- op->src.reg == state->drap_reg) {
	2426	+ if (cfi->drap && op->dest.reg == CFI_SP &&
	2427	+ op->src.reg == cfi->drap_reg) {
1548	2428
1549	2429	/* drap: lea disp(%drap), %rsp */
1550	2430	cfa->base = CFI_SP;
1551		- cfa->offset = state->stack_size = -op->src.offset;
1552		- state->drap_reg = CFI_UNDEFINED;
1553		- state->drap = false;
	2431	+ cfa->offset = cfi->stack_size = -op->src.offset;
	2432	+ cfi->drap_reg = CFI_UNDEFINED;
	2433	+ cfi->drap = false;
1554	2434	break;
1555	2435	}
1556	2436
1557		- if (op->dest.reg == state->cfa.base) {
	2437	+ if (op->dest.reg == cfi->cfa.base) {
1558	2438	WARN_FUNC("unsupported stack register modification",
1559	2439	insn->sec, insn->offset);
1560	2440	return -1;
..	..	@@ -1564,18 +2444,18 @@
1564	2444
1565	2445	case OP_SRC_AND:
1566	2446	if (op->dest.reg != CFI_SP \|\|
1567		- (state->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) \|\|
1568		- (state->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
	2447	+ (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) \|\|
	2448	+ (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
1569	2449	WARN_FUNC("unsupported stack pointer realignment",
1570	2450	insn->sec, insn->offset);
1571	2451	return -1;
1572	2452	}
1573	2453
1574		- if (state->drap_reg != CFI_UNDEFINED) {
	2454	+ if (cfi->drap_reg != CFI_UNDEFINED) {
1575	2455	/* drap: and imm, %rsp */
1576		- cfa->base = state->drap_reg;
1577		- cfa->offset = state->stack_size = 0;
1578		- state->drap = true;
	2456	+ cfa->base = cfi->drap_reg;
	2457	+ cfa->offset = cfi->stack_size = 0;
	2458	+ cfi->drap = true;
1579	2459	}
1580	2460
1581	2461	/*
..	..	@@ -1586,57 +2466,56 @@
1586	2466	break;
1587	2467
1588	2468	case OP_SRC_POP:
1589		- if (!state->drap && op->dest.type == OP_DEST_REG &&
1590		- op->dest.reg == cfa->base) {
	2469	+ case OP_SRC_POPF:
	2470	+ if (!cfi->drap && op->dest.reg == cfa->base) {
1591	2471
1592	2472	/* pop %rbp */
1593	2473	cfa->base = CFI_SP;
1594	2474	}
1595	2475
1596		- if (state->drap && cfa->base == CFI_BP_INDIRECT &&
1597		- op->dest.type == OP_DEST_REG &&
1598		- op->dest.reg == state->drap_reg &&
1599		- state->drap_offset == -state->stack_size) {
	2476	+ if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
	2477	+ op->dest.reg == cfi->drap_reg &&
	2478	+ cfi->drap_offset == -cfi->stack_size) {
1600	2479
1601	2480	/* drap: pop %drap */
1602		- cfa->base = state->drap_reg;
	2481	+ cfa->base = cfi->drap_reg;
1603	2482	cfa->offset = 0;
1604		- state->drap_offset = -1;
	2483	+ cfi->drap_offset = -1;
1605	2484
1606		- } else if (regs[op->dest.reg].offset == -state->stack_size) {
	2485	+ } else if (regs[op->dest.reg].offset == -cfi->stack_size) {
1607	2486
1608	2487	/* pop %reg */
1609		- restore_reg(state, op->dest.reg);
	2488	+ restore_reg(cfi, op->dest.reg);
1610	2489	}
1611	2490
1612		- state->stack_size -= 8;
	2491	+ cfi->stack_size -= 8;
1613	2492	if (cfa->base == CFI_SP)
1614	2493	cfa->offset -= 8;
1615	2494
1616	2495	break;
1617	2496
1618	2497	case OP_SRC_REG_INDIRECT:
1619		- if (state->drap && op->src.reg == CFI_BP &&
1620		- op->src.offset == state->drap_offset) {
	2498	+ if (cfi->drap && op->src.reg == CFI_BP &&
	2499	+ op->src.offset == cfi->drap_offset) {
1621	2500
1622	2501	/* drap: mov disp(%rbp), %drap */
1623		- cfa->base = state->drap_reg;
	2502	+ cfa->base = cfi->drap_reg;
1624	2503	cfa->offset = 0;
1625		- state->drap_offset = -1;
	2504	+ cfi->drap_offset = -1;
1626	2505	}
1627	2506
1628		- if (state->drap && op->src.reg == CFI_BP &&
	2507	+ if (cfi->drap && op->src.reg == CFI_BP &&
1629	2508	op->src.offset == regs[op->dest.reg].offset) {
1630	2509
1631	2510	/* drap: mov disp(%rbp), %reg */
1632		- restore_reg(state, op->dest.reg);
	2511	+ restore_reg(cfi, op->dest.reg);
1633	2512
1634	2513	} else if (op->src.reg == cfa->base &&
1635	2514	op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
1636	2515
1637	2516	/* mov disp(%rbp), %reg */
1638	2517	/* mov disp(%rsp), %reg */
1639		- restore_reg(state, op->dest.reg);
	2518	+ restore_reg(cfi, op->dest.reg);
1640	2519	}
1641	2520
1642	2521	break;
..	..	@@ -1650,78 +2529,77 @@
1650	2529	break;
1651	2530
1652	2531	case OP_DEST_PUSH:
1653		- state->stack_size += 8;
	2532	+ case OP_DEST_PUSHF:
	2533	+ cfi->stack_size += 8;
1654	2534	if (cfa->base == CFI_SP)
1655	2535	cfa->offset += 8;
1656	2536
1657	2537	if (op->src.type != OP_SRC_REG)
1658	2538	break;
1659	2539
1660		- if (state->drap) {
1661		- if (op->src.reg == cfa->base && op->src.reg == state->drap_reg) {
	2540	+ if (cfi->drap) {
	2541	+ if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
1662	2542
1663	2543	/* drap: push %drap */
1664	2544	cfa->base = CFI_BP_INDIRECT;
1665		- cfa->offset = -state->stack_size;
	2545	+ cfa->offset = -cfi->stack_size;
1666	2546
1667	2547	/* save drap so we know when to restore it */
1668		- state->drap_offset = -state->stack_size;
	2548	+ cfi->drap_offset = -cfi->stack_size;
1669	2549
1670		- } else if (op->src.reg == CFI_BP && cfa->base == state->drap_reg) {
	2550	+ } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
1671	2551
1672	2552	/* drap: push %rbp */
1673		- state->stack_size = 0;
	2553	+ cfi->stack_size = 0;
1674	2554
1675		- } else if (regs[op->src.reg].base == CFI_UNDEFINED) {
	2555	+ } else {
1676	2556
1677	2557	/* drap: push %reg */
1678		- save_reg(state, op->src.reg, CFI_BP, -state->stack_size);
	2558	+ save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
1679	2559	}
1680	2560
1681	2561	} else {
1682	2562
1683	2563	/* push %reg */
1684		- save_reg(state, op->src.reg, CFI_CFA, -state->stack_size);
	2564	+ save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
1685	2565	}
1686	2566
1687	2567	/* detect when asm code uses rbp as a scratch register */
1688	2568	if (!no_fp && insn->func && op->src.reg == CFI_BP &&
1689	2569	cfa->base != CFI_BP)
1690		- state->bp_scratch = true;
	2570	+ cfi->bp_scratch = true;
1691	2571	break;
1692	2572
1693	2573	case OP_DEST_REG_INDIRECT:
1694	2574
1695		- if (state->drap) {
1696		- if (op->src.reg == cfa->base && op->src.reg == state->drap_reg) {
	2575	+ if (cfi->drap) {
	2576	+ if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
1697	2577
1698	2578	/* drap: mov %drap, disp(%rbp) */
1699	2579	cfa->base = CFI_BP_INDIRECT;
1700	2580	cfa->offset = op->dest.offset;
1701	2581
1702	2582	/* save drap offset so we know when to restore it */
1703		- state->drap_offset = op->dest.offset;
1704		- }
1705		-
1706		- else if (regs[op->src.reg].base == CFI_UNDEFINED) {
	2583	+ cfi->drap_offset = op->dest.offset;
	2584	+ } else {
1707	2585
1708	2586	/* drap: mov reg, disp(%rbp) */
1709		- save_reg(state, op->src.reg, CFI_BP, op->dest.offset);
	2587	+ save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
1710	2588	}
1711	2589
1712	2590	} else if (op->dest.reg == cfa->base) {
1713	2591
1714	2592	/* mov reg, disp(%rbp) */
1715	2593	/* mov reg, disp(%rsp) */
1716		- save_reg(state, op->src.reg, CFI_CFA,
1717		- op->dest.offset - state->cfa.offset);
	2594	+ save_reg(cfi, op->src.reg, CFI_CFA,
	2595	+ op->dest.offset - cfi->cfa.offset);
1718	2596	}
1719	2597
1720	2598	break;
1721	2599
1722	2600	case OP_DEST_LEAVE:
1723		- if ((!state->drap && cfa->base != CFI_BP) \|\|
1724		- (state->drap && cfa->base != state->drap_reg)) {
	2601	+ if ((!cfi->drap && cfa->base != CFI_BP) \|\|
	2602	+ (cfi->drap && cfa->base != cfi->drap_reg)) {
1725	2603	WARN_FUNC("leave instruction with modified stack frame",
1726	2604	insn->sec, insn->offset);
1727	2605	return -1;
..	..	@@ -1729,10 +2607,10 @@
1729	2607
1730	2608	/* leave (mov %rbp, %rsp; pop %rbp) */
1731	2609
1732		- state->stack_size = -state->regs[CFI_BP].offset - 8;
1733		- restore_reg(state, CFI_BP);
	2610	+ cfi->stack_size = -cfi->regs[CFI_BP].offset - 8;
	2611	+ restore_reg(cfi, CFI_BP);
1734	2612
1735		- if (!state->drap) {
	2613	+ if (!cfi->drap) {
1736	2614	cfa->base = CFI_SP;
1737	2615	cfa->offset -= 8;
1738	2616	}
..	..	@@ -1740,14 +2618,14 @@
1740	2618	break;
1741	2619
1742	2620	case OP_DEST_MEM:
1743		- if (op->src.type != OP_SRC_POP) {
	2621	+ if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
1744	2622	WARN_FUNC("unknown stack-related memory operation",
1745	2623	insn->sec, insn->offset);
1746	2624	return -1;
1747	2625	}
1748	2626
1749	2627	/* pop mem */
1750		- state->stack_size -= 8;
	2628	+ cfi->stack_size -= 8;
1751	2629	if (cfa->base == CFI_SP)
1752	2630	cfa->offset -= 8;
1753	2631
..	..	@@ -1762,46 +2640,260 @@
1762	2640	return 0;
1763	2641	}
1764	2642
1765		-static bool insn_state_match(struct instruction insn, struct insn_state state)
	2643	+/*
	2644	+ * The stack layouts of alternatives instructions can sometimes diverge when
	2645	+ * they have stack modifications. That's fine as long as the potential stack
	2646	+ * layouts don't conflict at any given potential instruction boundary.
	2647	+ *
	2648	+ * Flatten the CFIs of the different alternative code streams (both original
	2649	+ * and replacement) into a single shared CFI array which can be used to detect
	2650	+ * conflicts and nicely feed a linear array of ORC entries to the unwinder.
	2651	+ */
	2652	+static int propagate_alt_cfi(struct objtool_file file, struct instruction insn)
1766	2653	{
1767		- struct insn_state state1 = &insn->state, state2 = state;
	2654	+ struct cfi_state **alt_cfi;
	2655	+ int group_off;
	2656	+
	2657	+ if (!insn->alt_group)
	2658	+ return 0;
	2659	+
	2660	+ if (!insn->cfi) {
	2661	+ WARN("CFI missing");
	2662	+ return -1;
	2663	+ }
	2664	+
	2665	+ alt_cfi = insn->alt_group->cfi;
	2666	+ group_off = insn->offset - insn->alt_group->first_insn->offset;
	2667	+
	2668	+ if (!alt_cfi[group_off]) {
	2669	+ alt_cfi[group_off] = insn->cfi;
	2670	+ } else {
	2671	+ if (cficmp(alt_cfi[group_off], insn->cfi)) {
	2672	+ WARN_FUNC("stack layout conflict in alternatives",
	2673	+ insn->sec, insn->offset);
	2674	+ return -1;
	2675	+ }
	2676	+ }
	2677	+
	2678	+ return 0;
	2679	+}
	2680	+
	2681	+static int handle_insn_ops(struct instruction insn, struct insn_state state)
	2682	+{
	2683	+ struct stack_op *op;
	2684	+
	2685	+ list_for_each_entry(op, &insn->stack_ops, list) {
	2686	+
	2687	+ if (update_cfi_state(insn, &state->cfi, op))
	2688	+ return 1;
	2689	+
	2690	+ if (op->dest.type == OP_DEST_PUSHF) {
	2691	+ if (!state->uaccess_stack) {
	2692	+ state->uaccess_stack = 1;
	2693	+ } else if (state->uaccess_stack >> 31) {
	2694	+ WARN_FUNC("PUSHF stack exhausted",
	2695	+ insn->sec, insn->offset);
	2696	+ return 1;
	2697	+ }
	2698	+ state->uaccess_stack <<= 1;
	2699	+ state->uaccess_stack \|= state->uaccess;
	2700	+ }
	2701	+
	2702	+ if (op->src.type == OP_SRC_POPF) {
	2703	+ if (state->uaccess_stack) {
	2704	+ state->uaccess = state->uaccess_stack & 1;
	2705	+ state->uaccess_stack >>= 1;
	2706	+ if (state->uaccess_stack == 1)
	2707	+ state->uaccess_stack = 0;
	2708	+ }
	2709	+ }
	2710	+ }
	2711	+
	2712	+ return 0;
	2713	+}
	2714	+
	2715	+static bool insn_cfi_match(struct instruction insn, struct cfi_state cfi2)
	2716	+{
	2717	+ struct cfi_state *cfi1 = insn->cfi;
1768	2718	int i;
1769	2719
1770		- if (memcmp(&state1->cfa, &state2->cfa, sizeof(state1->cfa))) {
	2720	+ if (!cfi1) {
	2721	+ WARN("CFI missing");
	2722	+ return false;
	2723	+ }
	2724	+
	2725	+ if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
	2726	+
1771	2727	WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
1772	2728	insn->sec, insn->offset,
1773		- state1->cfa.base, state1->cfa.offset,
1774		- state2->cfa.base, state2->cfa.offset);
	2729	+ cfi1->cfa.base, cfi1->cfa.offset,
	2730	+ cfi2->cfa.base, cfi2->cfa.offset);
1775	2731
1776		- } else if (memcmp(&state1->regs, &state2->regs, sizeof(state1->regs))) {
	2732	+ } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
1777	2733	for (i = 0; i < CFI_NUM_REGS; i++) {
1778		- if (!memcmp(&state1->regs[i], &state2->regs[i],
	2734	+ if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
1779	2735	sizeof(struct cfi_reg)))
1780	2736	continue;
1781	2737
1782	2738	WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
1783	2739	insn->sec, insn->offset,
1784		- i, state1->regs[i].base, state1->regs[i].offset,
1785		- i, state2->regs[i].base, state2->regs[i].offset);
	2740	+ i, cfi1->regs[i].base, cfi1->regs[i].offset,
	2741	+ i, cfi2->regs[i].base, cfi2->regs[i].offset);
1786	2742	break;
1787	2743	}
1788	2744
1789		- } else if (state1->type != state2->type) {
1790		- WARN_FUNC("stack state mismatch: type1=%d type2=%d",
1791		- insn->sec, insn->offset, state1->type, state2->type);
	2745	+ } else if (cfi1->type != cfi2->type) {
1792	2746
1793		- } else if (state1->drap != state2->drap \|\|
1794		- (state1->drap && state1->drap_reg != state2->drap_reg) \|\|
1795		- (state1->drap && state1->drap_offset != state2->drap_offset)) {
	2747	+ WARN_FUNC("stack state mismatch: type1=%d type2=%d",
	2748	+ insn->sec, insn->offset, cfi1->type, cfi2->type);
	2749	+
	2750	+ } else if (cfi1->drap != cfi2->drap \|\|
	2751	+ (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) \|\|
	2752	+ (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
	2753	+
1796	2754	WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
1797	2755	insn->sec, insn->offset,
1798		- state1->drap, state1->drap_reg, state1->drap_offset,
1799		- state2->drap, state2->drap_reg, state2->drap_offset);
	2756	+ cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
	2757	+ cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
1800	2758
1801	2759	} else
1802	2760	return true;
1803	2761
1804	2762	return false;
	2763	+}
	2764	+
	2765	+static inline bool func_uaccess_safe(struct symbol *func)
	2766	+{
	2767	+ if (func)
	2768	+ return func->uaccess_safe;
	2769	+
	2770	+ return false;
	2771	+}
	2772	+
	2773	+static inline const char call_dest_name(struct instruction insn)
	2774	+{
	2775	+ if (insn->call_dest)
	2776	+ return insn->call_dest->name;
	2777	+
	2778	+ return "{dynamic}";
	2779	+}
	2780	+
	2781	+static inline bool noinstr_call_dest(struct symbol *func)
	2782	+{
	2783	+ /*
	2784	+ * We can't deal with indirect function calls at present;
	2785	+ * assume they're instrumented.
	2786	+ */
	2787	+ if (!func)
	2788	+ return false;
	2789	+
	2790	+ /*
	2791	+ * If the symbol is from a noinstr section; we good.
	2792	+ */
	2793	+ if (func->sec->noinstr)
	2794	+ return true;
	2795	+
	2796	+ /*
	2797	+ * The __ubsan_handle_*() calls are like WARN(), they only happen when
	2798	+ * something 'BAD' happened. At the risk of taking the machine down,
	2799	+ * let them proceed to get the message out.
	2800	+ */
	2801	+ if (!strncmp(func->name, "__ubsan_handle_", 15))
	2802	+ return true;
	2803	+
	2804	+ return false;
	2805	+}
	2806	+
	2807	+static int validate_call(struct instruction insn, struct insn_state state)
	2808	+{
	2809	+ if (state->noinstr && state->instr <= 0 &&
	2810	+ !noinstr_call_dest(insn->call_dest)) {
	2811	+ WARN_FUNC("call to %s() leaves .noinstr.text section",
	2812	+ insn->sec, insn->offset, call_dest_name(insn));
	2813	+ return 1;
	2814	+ }
	2815	+
	2816	+ if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
	2817	+ WARN_FUNC("call to %s() with UACCESS enabled",
	2818	+ insn->sec, insn->offset, call_dest_name(insn));
	2819	+ return 1;
	2820	+ }
	2821	+
	2822	+ if (state->df) {
	2823	+ WARN_FUNC("call to %s() with DF set",
	2824	+ insn->sec, insn->offset, call_dest_name(insn));
	2825	+ return 1;
	2826	+ }
	2827	+
	2828	+ return 0;
	2829	+}
	2830	+
	2831	+static int validate_sibling_call(struct instruction insn, struct insn_state state)
	2832	+{
	2833	+ if (has_modified_stack_frame(insn, state)) {
	2834	+ WARN_FUNC("sibling call from callable instruction with modified stack frame",
	2835	+ insn->sec, insn->offset);
	2836	+ return 1;
	2837	+ }
	2838	+
	2839	+ return validate_call(insn, state);
	2840	+}
	2841	+
	2842	+static int validate_return(struct symbol func, struct instruction insn, struct insn_state *state)
	2843	+{
	2844	+ if (state->noinstr && state->instr > 0) {
	2845	+ WARN_FUNC("return with instrumentation enabled",
	2846	+ insn->sec, insn->offset);
	2847	+ return 1;
	2848	+ }
	2849	+
	2850	+ if (state->uaccess && !func_uaccess_safe(func)) {
	2851	+ WARN_FUNC("return with UACCESS enabled",
	2852	+ insn->sec, insn->offset);
	2853	+ return 1;
	2854	+ }
	2855	+
	2856	+ if (!state->uaccess && func_uaccess_safe(func)) {
	2857	+ WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
	2858	+ insn->sec, insn->offset);
	2859	+ return 1;
	2860	+ }
	2861	+
	2862	+ if (state->df) {
	2863	+ WARN_FUNC("return with DF set",
	2864	+ insn->sec, insn->offset);
	2865	+ return 1;
	2866	+ }
	2867	+
	2868	+ if (func && has_modified_stack_frame(insn, state)) {
	2869	+ WARN_FUNC("return with modified stack frame",
	2870	+ insn->sec, insn->offset);
	2871	+ return 1;
	2872	+ }
	2873	+
	2874	+ if (state->cfi.bp_scratch) {
	2875	+ WARN_FUNC("BP used as a scratch register",
	2876	+ insn->sec, insn->offset);
	2877	+ return 1;
	2878	+ }
	2879	+
	2880	+ return 0;
	2881	+}
	2882	+
	2883	+static struct instruction next_insn_to_validate(struct objtool_file file,
	2884	+ struct instruction *insn)
	2885	+{
	2886	+ struct alt_group *alt_group = insn->alt_group;
	2887	+
	2888	+ /*
	2889	+ * Simulate the fact that alternatives are patched in-place. When the
	2890	+ * end of a replacement alt_group is reached, redirect objtool flow to
	2891	+ * the end of the original alt_group.
	2892	+ */
	2893	+ if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
	2894	+ return next_insn_same_sec(file, alt_group->orig_group->last_insn);
	2895	+
	2896	+ return next_insn_same_sec(file, insn);
1805	2897	}
1806	2898
1807	2899	/*
..	..	@@ -1810,26 +2902,19 @@
1810	2902	* each instruction and validate all the rules described in
1811	2903	* tools/objtool/Documentation/stack-validation.txt.
1812	2904	*/
1813		-static int validate_branch(struct objtool_file file, struct instruction first,
1814		- struct insn_state state)
	2905	+static int validate_branch(struct objtool_file file, struct symbol func,
	2906	+ struct instruction *insn, struct insn_state state)
1815	2907	{
1816	2908	struct alternative *alt;
1817		- struct instruction insn, next_insn;
	2909	+ struct instruction next_insn, prev_insn = NULL;
1818	2910	struct section *sec;
1819		- struct symbol *func = NULL;
	2911	+ u8 visited;
1820	2912	int ret;
1821	2913
1822		- insn = first;
1823	2914	sec = insn->sec;
1824	2915
1825		- if (insn->alt_group && list_empty(&insn->alts)) {
1826		- WARN_FUNC("don't know how to handle branch to middle of alternative instruction group",
1827		- sec, insn->offset);
1828		- return 1;
1829		- }
1830		-
1831	2916	while (1) {
1832		- next_insn = next_insn_same_sec(file, insn);
	2917	+ next_insn = next_insn_to_validate(file, insn);
1833	2918
1834	2919	if (file->c_file && func && insn->func && func != insn->func->pfunc) {
1835	2920	WARN("%s() falls through to next function %s()",
..	..	@@ -1837,21 +2922,25 @@
1837	2922	return 1;
1838	2923	}
1839	2924
1840		- if (insn->func)
1841		- func = insn->func->pfunc;
1842		-
1843	2925	if (func && insn->ignore) {
1844	2926	WARN_FUNC("BUG: why am I validating an ignored function?",
1845	2927	sec, insn->offset);
1846	2928	return 1;
1847	2929	}
1848	2930
1849		- if (insn->visited) {
1850		- if (!insn->hint && !insn_state_match(insn, &state))
	2931	+ visited = VISITED_BRANCH << state.uaccess;
	2932	+ if (insn->visited & VISITED_BRANCH_MASK) {
	2933	+ if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
1851	2934	return 1;
1852	2935
1853		- return 0;
	2936	+ if (insn->visited & visited)
	2937	+ return 0;
	2938	+ } else {
	2939	+ nr_insns_visited++;
1854	2940	}
	2941	+
	2942	+ if (state.noinstr)
	2943	+ state.instr += insn->instr;
1855	2944
1856	2945	if (insn->hint) {
1857	2946	if (insn->restore) {
..	..	@@ -1859,7 +2948,8 @@
1859	2948
1860	2949	i = insn;
1861	2950	save_insn = NULL;
1862		- func_for_each_insn_continue_reverse(file, insn->func, i) {
	2951	+
	2952	+ sym_for_each_insn_continue_reverse(file, func, i) {
1863	2953	if (i->save) {
1864	2954	save_insn = i;
1865	2955	break;
..	..	@@ -1873,89 +2963,97 @@
1873	2963	}
1874	2964
1875	2965	if (!save_insn->visited) {
1876		- /*
1877		- * Oops, no state to copy yet.
1878		- * Hopefully we can reach this
1879		- * instruction from another branch
1880		- * after the save insn has been
1881		- * visited.
1882		- */
1883		- if (insn == first)
1884		- return 0;
1885		-
1886	2966	WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo",
1887	2967	sec, insn->offset);
1888	2968	return 1;
1889	2969	}
1890	2970
1891		- insn->state = save_insn->state;
	2971	+ insn->cfi = save_insn->cfi;
	2972	+ nr_cfi_reused++;
1892	2973	}
1893	2974
1894		- state = insn->state;
	2975	+ state.cfi = *insn->cfi;
	2976	+ } else {
	2977	+ /* XXX track if we actually changed state.cfi */
1895	2978
1896		- } else
1897		- insn->state = state;
1898		-
1899		- insn->visited = true;
1900		-
1901		- if (!insn->ignore_alts) {
1902		- list_for_each_entry(alt, &insn->alts, list) {
1903		- ret = validate_branch(file, alt->insn, state);
1904		- if (ret)
1905		- return 1;
	2979	+ if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
	2980	+ insn->cfi = prev_insn->cfi;
	2981	+ nr_cfi_reused++;
	2982	+ } else {
	2983	+ insn->cfi = cfi_hash_find_or_add(&state.cfi);
1906	2984	}
1907	2985	}
	2986	+
	2987	+ insn->visited \|= visited;
	2988	+
	2989	+ if (propagate_alt_cfi(file, insn))
	2990	+ return 1;
	2991	+
	2992	+ if (!insn->ignore_alts && !list_empty(&insn->alts)) {
	2993	+ bool skip_orig = false;
	2994	+
	2995	+ list_for_each_entry(alt, &insn->alts, list) {
	2996	+ if (alt->skip_orig)
	2997	+ skip_orig = true;
	2998	+
	2999	+ ret = validate_branch(file, func, alt->insn, state);
	3000	+ if (ret) {
	3001	+ if (backtrace)
	3002	+ BT_FUNC("(alt)", insn);
	3003	+ return ret;
	3004	+ }
	3005	+ }
	3006	+
	3007	+ if (skip_orig)
	3008	+ return 0;
	3009	+ }
	3010	+
	3011	+ if (handle_insn_ops(insn, &state))
	3012	+ return 1;
1908	3013
1909	3014	switch (insn->type) {
1910	3015
1911	3016	case INSN_RETURN:
1912		- if (func && has_modified_stack_frame(&state)) {
1913		- WARN_FUNC("return with modified stack frame",
1914		- sec, insn->offset);
1915		- return 1;
	3017	+ if (sls && !insn->retpoline_safe &&
	3018	+ next_insn && next_insn->type != INSN_TRAP) {
	3019	+ WARN_FUNC("missing int3 after ret",
	3020	+ insn->sec, insn->offset);
1916	3021	}
1917		-
1918		- if (state.bp_scratch) {
1919		- WARN("%s uses BP as a scratch register",
1920		- insn->func->name);
1921		- return 1;
1922		- }
1923		-
1924		- return 0;
	3022	+ return validate_return(func, insn, &state);
1925	3023
1926	3024	case INSN_CALL:
1927		- if (is_fentry_call(insn))
1928		- break;
1929		-
1930		- ret = dead_end_function(file, insn->call_dest);
1931		- if (ret == 1)
1932		- return 0;
1933		- if (ret == -1)
1934		- return 1;
1935		-
1936		- /* fallthrough */
1937	3025	case INSN_CALL_DYNAMIC:
1938		- if (!no_fp && func && !has_valid_stack_frame(&state)) {
	3026	+ ret = validate_call(insn, &state);
	3027	+ if (ret)
	3028	+ return ret;
	3029	+
	3030	+ if (!no_fp && func && !is_fentry_call(insn) &&
	3031	+ !has_valid_stack_frame(&state)) {
1939	3032	WARN_FUNC("call without frame pointer save/setup",
1940	3033	sec, insn->offset);
1941	3034	return 1;
1942	3035	}
	3036	+
	3037	+ if (dead_end_function(file, insn->call_dest))
	3038	+ return 0;
	3039	+
1943	3040	break;
1944	3041
1945	3042	case INSN_JUMP_CONDITIONAL:
1946	3043	case INSN_JUMP_UNCONDITIONAL:
1947		- if (insn->jump_dest &&
1948		- (!func \|\| !insn->jump_dest->func \|\|
1949		- insn->jump_dest->func->pfunc == func)) {
1950		- ret = validate_branch(file, insn->jump_dest,
1951		- state);
	3044	+ if (is_sibling_call(insn)) {
	3045	+ ret = validate_sibling_call(insn, &state);
1952	3046	if (ret)
1953		- return 1;
	3047	+ return ret;
1954	3048
1955		- } else if (func && has_modified_stack_frame(&state)) {
1956		- WARN_FUNC("sibling call from callable instruction with modified stack frame",
1957		- sec, insn->offset);
1958		- return 1;
	3049	+ } else if (insn->jump_dest) {
	3050	+ ret = validate_branch(file, func,
	3051	+ insn->jump_dest, state);
	3052	+ if (ret) {
	3053	+ if (backtrace)
	3054	+ BT_FUNC("(branch)", insn);
	3055	+ return ret;
	3056	+ }
1959	3057	}
1960	3058
1961	3059	if (insn->type == INSN_JUMP_UNCONDITIONAL)
..	..	@@ -1964,14 +3062,24 @@
1964	3062	break;
1965	3063
1966	3064	case INSN_JUMP_DYNAMIC:
1967		- if (func && list_empty(&insn->alts) &&
1968		- has_modified_stack_frame(&state)) {
1969		- WARN_FUNC("sibling call from callable instruction with modified stack frame",
1970		- sec, insn->offset);
1971		- return 1;
	3065	+ if (sls && !insn->retpoline_safe &&
	3066	+ next_insn && next_insn->type != INSN_TRAP) {
	3067	+ WARN_FUNC("missing int3 after indirect jump",
	3068	+ insn->sec, insn->offset);
1972	3069	}
1973	3070
1974		- return 0;
	3071	+ /* fallthrough */
	3072	+ case INSN_JUMP_DYNAMIC_CONDITIONAL:
	3073	+ if (is_sibling_call(insn)) {
	3074	+ ret = validate_sibling_call(insn, &state);
	3075	+ if (ret)
	3076	+ return ret;
	3077	+ }
	3078	+
	3079	+ if (insn->type == INSN_JUMP_DYNAMIC)
	3080	+ return 0;
	3081	+
	3082	+ break;
1975	3083
1976	3084	case INSN_CONTEXT_SWITCH:
1977	3085	if (func && (!next_insn \|\| !next_insn->hint)) {
..	..	@@ -1981,10 +3089,45 @@
1981	3089	}
1982	3090	return 0;
1983	3091
1984		- case INSN_STACK:
1985		- if (update_insn_state(insn, &state))
	3092	+ case INSN_STAC:
	3093	+ if (state.uaccess) {
	3094	+ WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
1986	3095	return 1;
	3096	+ }
1987	3097
	3098	+ state.uaccess = true;
	3099	+ break;
	3100	+
	3101	+ case INSN_CLAC:
	3102	+ if (!state.uaccess && func) {
	3103	+ WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
	3104	+ return 1;
	3105	+ }
	3106	+
	3107	+ if (func_uaccess_safe(func) && !state.uaccess_stack) {
	3108	+ WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
	3109	+ return 1;
	3110	+ }
	3111	+
	3112	+ state.uaccess = false;
	3113	+ break;
	3114	+
	3115	+ case INSN_STD:
	3116	+ if (state.df) {
	3117	+ WARN_FUNC("recursive STD", sec, insn->offset);
	3118	+ return 1;
	3119	+ }
	3120	+
	3121	+ state.df = true;
	3122	+ break;
	3123	+
	3124	+ case INSN_CLD:
	3125	+ if (!state.df && func) {
	3126	+ WARN_FUNC("redundant CLD", sec, insn->offset);
	3127	+ return 1;
	3128	+ }
	3129	+
	3130	+ state.df = false;
1988	3131	break;
1989	3132
1990	3133	default:
..	..	@@ -1995,34 +3138,186 @@
1995	3138	return 0;
1996	3139
1997	3140	if (!next_insn) {
1998		- if (state.cfa.base == CFI_UNDEFINED)
	3141	+ if (state.cfi.cfa.base == CFI_UNDEFINED)
1999	3142	return 0;
2000	3143	WARN("%s: unexpected end of section", sec->name);
2001	3144	return 1;
2002	3145	}
2003	3146
	3147	+ prev_insn = insn;
2004	3148	insn = next_insn;
2005	3149	}
2006	3150
2007	3151	return 0;
2008	3152	}
2009	3153
2010		-static int validate_unwind_hints(struct objtool_file *file)
	3154	+static int validate_unwind_hints(struct objtool_file file, struct section sec)
2011	3155	{
2012	3156	struct instruction *insn;
2013		- int ret, warnings = 0;
2014	3157	struct insn_state state;
	3158	+ int ret, warnings = 0;
2015	3159
2016	3160	if (!file->hints)
2017	3161	return 0;
2018	3162
2019		- clear_insn_state(&state);
	3163	+ init_insn_state(&state, sec);
2020	3164
2021		- for_each_insn(file, insn) {
	3165	+ if (sec) {
	3166	+ insn = find_insn(file, sec, 0);
	3167	+ if (!insn)
	3168	+ return 0;
	3169	+ } else {
	3170	+ insn = list_first_entry(&file->insn_list, typeof(*insn), list);
	3171	+ }
	3172	+
	3173	+ while (&insn->list != &file->insn_list && (!sec \|\| insn->sec == sec)) {
2022	3174	if (insn->hint && !insn->visited) {
2023		- ret = validate_branch(file, insn, state);
	3175	+ ret = validate_branch(file, insn->func, insn, state);
	3176	+ if (ret && backtrace)
	3177	+ BT_FUNC("<=== (hint)", insn);
2024	3178	warnings += ret;
2025	3179	}
	3180	+
	3181	+ insn = list_next_entry(insn, list);
	3182	+ }
	3183	+
	3184	+ return warnings;
	3185	+}
	3186	+
	3187	+/*
	3188	+ * Validate rethunk entry constraint: must untrain RET before the first RET.
	3189	+ *
	3190	+ * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes
	3191	+ * before an actual RET instruction.
	3192	+ */
	3193	+static int validate_entry(struct objtool_file file, struct instruction insn)
	3194	+{
	3195	+ struct instruction next, dest;
	3196	+ int ret, warnings = 0;
	3197	+
	3198	+ for (;;) {
	3199	+ next = next_insn_to_validate(file, insn);
	3200	+
	3201	+ if (insn->visited & VISITED_ENTRY)
	3202	+ return 0;
	3203	+
	3204	+ insn->visited \|= VISITED_ENTRY;
	3205	+
	3206	+ if (!insn->ignore_alts && !list_empty(&insn->alts)) {
	3207	+ struct alternative *alt;
	3208	+ bool skip_orig = false;
	3209	+
	3210	+ list_for_each_entry(alt, &insn->alts, list) {
	3211	+ if (alt->skip_orig)
	3212	+ skip_orig = true;
	3213	+
	3214	+ ret = validate_entry(file, alt->insn);
	3215	+ if (ret) {
	3216	+ if (backtrace)
	3217	+ BT_FUNC("(alt)", insn);
	3218	+ return ret;
	3219	+ }
	3220	+ }
	3221	+
	3222	+ if (skip_orig)
	3223	+ return 0;
	3224	+ }
	3225	+
	3226	+ switch (insn->type) {
	3227	+
	3228	+ case INSN_CALL_DYNAMIC:
	3229	+ case INSN_JUMP_DYNAMIC:
	3230	+ case INSN_JUMP_DYNAMIC_CONDITIONAL:
	3231	+ WARN_FUNC("early indirect call", insn->sec, insn->offset);
	3232	+ return 1;
	3233	+
	3234	+ case INSN_JUMP_UNCONDITIONAL:
	3235	+ case INSN_JUMP_CONDITIONAL:
	3236	+ if (!is_sibling_call(insn)) {
	3237	+ if (!insn->jump_dest) {
	3238	+ WARN_FUNC("unresolved jump target after linking?!?",
	3239	+ insn->sec, insn->offset);
	3240	+ return -1;
	3241	+ }
	3242	+ ret = validate_entry(file, insn->jump_dest);
	3243	+ if (ret) {
	3244	+ if (backtrace) {
	3245	+ BT_FUNC("(branch%s)", insn,
	3246	+ insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
	3247	+ }
	3248	+ return ret;
	3249	+ }
	3250	+
	3251	+ if (insn->type == INSN_JUMP_UNCONDITIONAL)
	3252	+ return 0;
	3253	+
	3254	+ break;
	3255	+ }
	3256	+
	3257	+ /* fallthrough */
	3258	+ case INSN_CALL:
	3259	+ dest = find_insn(file, insn->call_dest->sec,
	3260	+ insn->call_dest->offset);
	3261	+ if (!dest) {
	3262	+ WARN("Unresolved function after linking!?: %s",
	3263	+ insn->call_dest->name);
	3264	+ return -1;
	3265	+ }
	3266	+
	3267	+ ret = validate_entry(file, dest);
	3268	+ if (ret) {
	3269	+ if (backtrace)
	3270	+ BT_FUNC("(call)", insn);
	3271	+ return ret;
	3272	+ }
	3273	+ /*
	3274	+ * If a call returns without error, it must have seen UNTRAIN_RET.
	3275	+ * Therefore any non-error return is a success.
	3276	+ */
	3277	+ return 0;
	3278	+
	3279	+ case INSN_RETURN:
	3280	+ WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset);
	3281	+ return 1;
	3282	+
	3283	+ case INSN_NOP:
	3284	+ if (insn->retpoline_safe)
	3285	+ return 0;
	3286	+ break;
	3287	+
	3288	+ default:
	3289	+ break;
	3290	+ }
	3291	+
	3292	+ if (!next) {
	3293	+ WARN_FUNC("teh end!", insn->sec, insn->offset);
	3294	+ return -1;
	3295	+ }
	3296	+ insn = next;
	3297	+ }
	3298	+
	3299	+ return warnings;
	3300	+}
	3301	+
	3302	+/*
	3303	+ * Validate that all branches starting at 'insn->entry' encounter UNRET_END
	3304	+ * before RET.
	3305	+ */
	3306	+static int validate_unret(struct objtool_file *file)
	3307	+{
	3308	+ struct instruction *insn;
	3309	+ int ret, warnings = 0;
	3310	+
	3311	+ for_each_insn(file, insn) {
	3312	+ if (!insn->entry)
	3313	+ continue;
	3314	+
	3315	+ ret = validate_entry(file, insn);
	3316	+ if (ret < 0) {
	3317	+ WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset);
	3318	+ return ret;
	3319	+ }
	3320	+ warnings += ret;
2026	3321	}
2027	3322
2028	3323	return warnings;
..	..	@@ -2035,7 +3330,8 @@
2035	3330
2036	3331	for_each_insn(file, insn) {
2037	3332	if (insn->type != INSN_JUMP_DYNAMIC &&
2038		- insn->type != INSN_CALL_DYNAMIC)
	3333	+ insn->type != INSN_CALL_DYNAMIC &&
	3334	+ insn->type != INSN_RETURN)
2039	3335	continue;
2040	3336
2041	3337	if (insn->retpoline_safe)
..	..	@@ -2050,9 +3346,17 @@
2050	3346	if (!strcmp(insn->sec->name, ".init.text") && !module)
2051	3347	continue;
2052	3348
2053		- WARN_FUNC("indirect %s found in RETPOLINE build",
2054		- insn->sec, insn->offset,
2055		- insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
	3349	+ if (insn->type == INSN_RETURN) {
	3350	+ if (rethunk) {
	3351	+ WARN_FUNC("'naked' return found in RETHUNK build",
	3352	+ insn->sec, insn->offset);
	3353	+ } else
	3354	+ continue;
	3355	+ } else {
	3356	+ WARN_FUNC("indirect %s found in RETPOLINE build",
	3357	+ insn->sec, insn->offset,
	3358	+ insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
	3359	+ }
2056	3360
2057	3361	warnings++;
2058	3362	}
..	..	@@ -2073,11 +3377,12 @@
2073	3377	"__ubsan_handle_builtin_unreachable"));
2074	3378	}
2075	3379
2076		-static bool ignore_unreachable_insn(struct instruction *insn)
	3380	+static bool ignore_unreachable_insn(struct objtool_file file, struct instruction insn)
2077	3381	{
2078	3382	int i;
	3383	+ struct instruction *prev_insn;
2079	3384
2080		- if (insn->ignore \|\| insn->type == INSN_NOP)
	3385	+ if (insn->ignore \|\| insn->type == INSN_NOP \|\| insn->type == INSN_TRAP)
2081	3386	return true;
2082	3387
2083	3388	/*
..	..	@@ -2100,8 +3405,11 @@
2100	3405	* __builtin_unreachable(). The BUG() macro has an unreachable() after
2101	3406	* the UD2, which causes GCC's undefined trap logic to emit another UD2
2102	3407	* (or occasionally a JMP to UD2).
	3408	+ *
	3409	+ * It may also insert a UD2 after calling a __noreturn function.
2103	3410	*/
2104		- if (list_prev_entry(insn, list)->dead_end &&
	3411	+ prev_insn = list_prev_entry(insn, list);
	3412	+ if ((prev_insn->dead_end \|\| dead_end_function(file, prev_insn->call_dest)) &&
2105	3413	(insn->type == INSN_BUG \|\|
2106	3414	(insn->type == INSN_JUMP_UNCONDITIONAL &&
2107	3415	insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
..	..	@@ -2137,33 +3445,81 @@
2137	3445	return false;
2138	3446	}
2139	3447
	3448	+static int validate_symbol(struct objtool_file file, struct section sec,
	3449	+ struct symbol sym, struct insn_state state)
	3450	+{
	3451	+ struct instruction *insn;
	3452	+ int ret;
	3453	+
	3454	+ if (!sym->len) {
	3455	+ WARN("%s() is missing an ELF size annotation", sym->name);
	3456	+ return 1;
	3457	+ }
	3458	+
	3459	+ if (sym->pfunc != sym \|\| sym->alias != sym)
	3460	+ return 0;
	3461	+
	3462	+ insn = find_insn(file, sec, sym->offset);
	3463	+ if (!insn \|\| insn->ignore \|\| insn->visited)
	3464	+ return 0;
	3465	+
	3466	+ state->uaccess = sym->uaccess_safe;
	3467	+
	3468	+ ret = validate_branch(file, insn->func, insn, *state);
	3469	+ if (ret && backtrace)
	3470	+ BT_FUNC("<=== (sym)", insn);
	3471	+ return ret;
	3472	+}
	3473	+
	3474	+static int validate_section(struct objtool_file file, struct section sec)
	3475	+{
	3476	+ struct insn_state state;
	3477	+ struct symbol *func;
	3478	+ int warnings = 0;
	3479	+
	3480	+ list_for_each_entry(func, &sec->symbol_list, list) {
	3481	+ if (func->type != STT_FUNC)
	3482	+ continue;
	3483	+
	3484	+ init_insn_state(&state, sec);
	3485	+ set_func_state(&state.cfi);
	3486	+
	3487	+ warnings += validate_symbol(file, sec, func, &state);
	3488	+ }
	3489	+
	3490	+ return warnings;
	3491	+}
	3492	+
	3493	+static int validate_vmlinux_functions(struct objtool_file *file)
	3494	+{
	3495	+ struct section *sec;
	3496	+ int warnings = 0;
	3497	+
	3498	+ sec = find_section_by_name(file->elf, ".noinstr.text");
	3499	+ if (sec) {
	3500	+ warnings += validate_section(file, sec);
	3501	+ warnings += validate_unwind_hints(file, sec);
	3502	+ }
	3503	+
	3504	+ sec = find_section_by_name(file->elf, ".entry.text");
	3505	+ if (sec) {
	3506	+ warnings += validate_section(file, sec);
	3507	+ warnings += validate_unwind_hints(file, sec);
	3508	+ }
	3509	+
	3510	+ return warnings;
	3511	+}
	3512	+
2140	3513	static int validate_functions(struct objtool_file *file)
2141	3514	{
2142	3515	struct section *sec;
2143		- struct symbol *func;
2144		- struct instruction *insn;
2145		- struct insn_state state;
2146		- int ret, warnings = 0;
2147		-
2148		- clear_insn_state(&state);
2149		-
2150		- state.cfa = initial_func_cfi.cfa;
2151		- memcpy(&state.regs, &initial_func_cfi.regs,
2152		- CFI_NUM_REGS * sizeof(struct cfi_reg));
2153		- state.stack_size = initial_func_cfi.cfa.offset;
	3516	+ int warnings = 0;
2154	3517
2155	3518	for_each_sec(file, sec) {
2156		- list_for_each_entry(func, &sec->symbol_list, list) {
2157		- if (func->type != STT_FUNC \|\| func->pfunc != func)
2158		- continue;
	3519	+ if (!(sec->sh.sh_flags & SHF_EXECINSTR))
	3520	+ continue;
2159	3521
2160		- insn = find_insn(file, sec, func->offset);
2161		- if (!insn \|\| insn->ignore)
2162		- continue;
2163		-
2164		- ret = validate_branch(file, insn, state);
2165		- warnings += ret;
2166		- }
	3522	+ warnings += validate_section(file, sec);
2167	3523	}
2168	3524
2169	3525	return warnings;
..	..	@@ -2177,7 +3533,7 @@
2177	3533	return 0;
2178	3534
2179	3535	for_each_insn(file, insn) {
2180		- if (insn->visited \|\| ignore_unreachable_insn(insn))
	3536	+ if (insn->visited \|\| ignore_unreachable_insn(file, insn))
2181	3537	continue;
2182	3538
2183	3539	WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
..	..	@@ -2187,95 +3543,112 @@
2187	3543	return 0;
2188	3544	}
2189	3545
2190		-static void cleanup(struct objtool_file *file)
2191		-{
2192		- struct instruction insn, tmpinsn;
2193		- struct alternative alt, tmpalt;
2194		-
2195		- list_for_each_entry_safe(insn, tmpinsn, &file->insn_list, list) {
2196		- list_for_each_entry_safe(alt, tmpalt, &insn->alts, list) {
2197		- list_del(&alt->list);
2198		- free(alt);
2199		- }
2200		- list_del(&insn->list);
2201		- hash_del(&insn->hash);
2202		- free(insn);
2203		- }
2204		- elf_close(file->elf);
2205		-}
2206		-
2207		-static struct objtool_file file;
2208		-
2209		-int check(const char *_objname, bool orc)
	3546	+int check(struct objtool_file *file)
2210	3547	{
2211	3548	int ret, warnings = 0;
2212	3549
2213		- objname = _objname;
2214		-
2215		- file.elf = elf_open(objname, orc ? O_RDWR : O_RDONLY);
2216		- if (!file.elf)
2217		- return 1;
2218		-
2219		- INIT_LIST_HEAD(&file.insn_list);
2220		- hash_init(file.insn_hash);
2221		- file.whitelist = find_section_by_name(file.elf, ".discard.func_stack_frame_non_standard");
2222		- file.c_file = find_section_by_name(file.elf, ".comment");
2223		- file.ignore_unreachables = no_unreachable;
2224		- file.hints = false;
2225		-
2226	3550	arch_initial_func_cfi_state(&initial_func_cfi);
	3551	+ init_cfi_state(&init_cfi);
	3552	+ init_cfi_state(&func_cfi);
	3553	+ set_func_state(&func_cfi);
2227	3554
2228		- ret = decode_sections(&file);
	3555	+ if (!cfi_hash_alloc())
	3556	+ goto out;
	3557	+
	3558	+ cfi_hash_add(&init_cfi);
	3559	+ cfi_hash_add(&func_cfi);
	3560	+
	3561	+ ret = decode_sections(file);
2229	3562	if (ret < 0)
2230	3563	goto out;
	3564	+
2231	3565	warnings += ret;
2232	3566
2233		- if (list_empty(&file.insn_list))
	3567	+ if (list_empty(&file->insn_list))
2234	3568	goto out;
2235	3569
	3570	+ if (vmlinux && !validate_dup) {
	3571	+ ret = validate_vmlinux_functions(file);
	3572	+ if (ret < 0)
	3573	+ goto out;
	3574	+
	3575	+ warnings += ret;
	3576	+ goto out;
	3577	+ }
	3578	+
2236	3579	if (retpoline) {
2237		- ret = validate_retpoline(&file);
	3580	+ ret = validate_retpoline(file);
2238	3581	if (ret < 0)
2239	3582	return ret;
2240	3583	warnings += ret;
2241	3584	}
2242	3585
2243		- ret = validate_functions(&file);
	3586	+ ret = validate_functions(file);
2244	3587	if (ret < 0)
2245	3588	goto out;
2246	3589	warnings += ret;
2247	3590
2248		- ret = validate_unwind_hints(&file);
	3591	+ ret = validate_unwind_hints(file, NULL);
2249	3592	if (ret < 0)
2250	3593	goto out;
2251	3594	warnings += ret;
	3595	+
	3596	+ if (unret) {
	3597	+ /*
	3598	+ * Must be after validate_branch() and friends, it plays
	3599	+ * further games with insn->visited.
	3600	+ */
	3601	+ ret = validate_unret(file);
	3602	+ if (ret < 0)
	3603	+ return ret;
	3604	+ warnings += ret;
	3605	+ }
2252	3606
2253	3607	if (!warnings) {
2254		- ret = validate_reachable_instructions(&file);
	3608	+ ret = validate_reachable_instructions(file);
2255	3609	if (ret < 0)
2256	3610	goto out;
2257	3611	warnings += ret;
2258	3612	}
2259	3613
2260		- if (orc) {
2261		- ret = create_orc(&file);
2262		- if (ret < 0)
2263		- goto out;
	3614	+ ret = create_static_call_sections(file);
	3615	+ if (ret < 0)
	3616	+ goto out;
	3617	+ warnings += ret;
2264	3618
2265		- ret = create_orc_sections(&file);
	3619	+ if (mcount) {
	3620	+ ret = create_mcount_loc_sections(file);
2266	3621	if (ret < 0)
2267	3622	goto out;
	3623	+ warnings += ret;
	3624	+ }
2268	3625
2269		- ret = elf_write(file.elf);
	3626	+ if (retpoline) {
	3627	+ ret = create_retpoline_sites_sections(file);
2270	3628	if (ret < 0)
2271	3629	goto out;
	3630	+ warnings += ret;
	3631	+ }
	3632	+
	3633	+ if (rethunk) {
	3634	+ ret = create_return_sites_sections(file);
	3635	+ if (ret < 0)
	3636	+ goto out;
	3637	+ warnings += ret;
	3638	+ }
	3639	+
	3640	+ if (stats) {
	3641	+ printf("nr_insns_visited: %ld\n", nr_insns_visited);
	3642	+ printf("nr_cfi: %ld\n", nr_cfi);
	3643	+ printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
	3644	+ printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
2272	3645	}
2273	3646
2274	3647	out:
2275		- cleanup(&file);
2276		-
2277		- /* ignore warnings for now until we get all the code cleaned up */
2278		- if (ret \|\| warnings)
2279		- return 0;
	3648	+ /*
	3649	+ * For now, don't fail the kernel build on fatal warnings. These
	3650	+ * errors are still fairly common due to the growing matrix of
	3651	+ * supported toolchains and their recent pace of change.
	3652	+ */
2280	3653	return 0;
2281	3654	}