~hc/RK356X_SDK_RELEASE.git

..	..	@@ -3,16 +3,26 @@
3	3	#include <errno.h>
4	4	#include <inttypes.h>
5	5	#include <regex.h>
	6	+#include <stdlib.h>
6	7	#include "callchain.h"
7	8	#include "debug.h"
	9	+#include "dso.h"
	10	+#include "env.h"
8	11	#include "event.h"
9	12	#include "evsel.h"
10	13	#include "hist.h"
11	14	#include "machine.h"
12	15	#include "map.h"
	16	+#include "map_symbol.h"
	17	+#include "branch.h"
	18	+#include "mem-events.h"
	19	+#include "srcline.h"
	20	+#include "symbol.h"
13	21	#include "sort.h"
14	22	#include "strlist.h"
	23	+#include "target.h"
15	24	#include "thread.h"
	25	+#include "util.h"
16	26	#include "vdso.h"
17	27	#include <stdbool.h>
18	28	#include <sys/types.h>
..	..	@@ -21,12 +31,22 @@
21	31	#include "unwind.h"
22	32	#include "linux/hash.h"
23	33	#include "asm/bug.h"
	34	+#include "bpf-event.h"
	35	+#include <internal/lib.h> // page_size
	36	+#include "cgroup.h"
24	37
25		-#include "sane_ctype.h"
	38	+#include <linux/ctype.h>
26	39	#include <symbol/kallsyms.h>
27	40	#include <linux/mman.h>
	41	+#include <linux/string.h>
	42	+#include <linux/zalloc.h>
28	43
29	44	static void __machine__remove_thread(struct machine machine, struct thread th, bool lock);
	45	+
	46	+static struct dso machine__kernel_dso(struct machine machine)
	47	+{
	48	+ return machine->vmlinux_map->dso;
	49	+}
30	50
31	51	static void dsos__init(struct dsos *dsos)
32	52	{
..	..	@@ -41,7 +61,7 @@
41	61
42	62	for (i = 0; i < THREADS__TABLE_SIZE; i++) {
43	63	struct threads *threads = &machine->threads[i];
44		- threads->entries = RB_ROOT;
	64	+ threads->entries = RB_ROOT_CACHED;
45	65	init_rwsem(&threads->lock);
46	66	threads->nr = 0;
47	67	INIT_LIST_HEAD(&threads->dead);
..	..	@@ -67,7 +87,7 @@
67	87	int err = -ENOMEM;
68	88
69	89	memset(machine, 0, sizeof(*machine));
70		- map_groups__init(&machine->kmaps, machine);
	90	+ maps__init(&machine->kmaps, machine);
71	91	RB_CLEAR_NODE(&machine->rb_node);
72	92	dsos__init(&machine->dsos);
73	93
..	..	@@ -137,7 +157,7 @@
137	157	struct machine *machine = machine__new_host();
138	158	/*
139	159	* FIXME:
140		- * 1) We should switch to machine__load_kallsyms(), i.e. not explicitely
	160	+ * 1) We should switch to machine__load_kallsyms(), i.e. not explicitly
141	161	* ask for not using the kcore parsing code, once this one is fixed
142	162	* to create a map per module.
143	163	*/
..	..	@@ -179,7 +199,7 @@
179	199	for (i = 0; i < THREADS__TABLE_SIZE; i++) {
180	200	struct threads *threads = &machine->threads[i];
181	201	down_write(&threads->lock);
182		- nd = rb_first(&threads->entries);
	202	+ nd = rb_first_cached(&threads->entries);
183	203	while (nd) {
184	204	struct thread *t = rb_entry(nd, struct thread, rb_node);
185	205
..	..	@@ -198,7 +218,7 @@
198	218	return;
199	219
200	220	machine__destroy_kernel_maps(machine);
201		- map_groups__exit(&machine->kmaps);
	221	+ maps__exit(&machine->kmaps);
202	222	dsos__exit(&machine->dsos);
203	223	machine__exit_vdso(machine);
204	224	zfree(&machine->root_dir);
..	..	@@ -207,6 +227,18 @@
207	227
208	228	for (i = 0; i < THREADS__TABLE_SIZE; i++) {
209	229	struct threads *threads = &machine->threads[i];
	230	+ struct thread thread, n;
	231	+ /*
	232	+ * Forget about the dead, at this point whatever threads were
	233	+ * left in the dead lists better have a reference count taken
	234	+ * by who is using them, and then, when they drop those references
	235	+ * and it finally hits zero, thread__put() will check and see that
	236	+ * its not in the dead threads list and will not try to remove it
	237	+ * from there, just calling thread__delete() straight away.
	238	+ */
	239	+ list_for_each_entry_safe(thread, n, &threads->dead, node)
	240	+ list_del_init(&thread->node);
	241	+
210	242	exit_rwsem(&threads->lock);
211	243	}
212	244	}
..	..	@@ -222,7 +254,7 @@
222	254	void machines__init(struct machines *machines)
223	255	{
224	256	machine__init(&machines->host, "", HOST_KERNEL_ID);
225		- machines->guests = RB_ROOT;
	257	+ machines->guests = RB_ROOT_CACHED;
226	258	}
227	259
228	260	void machines__exit(struct machines *machines)
..	..	@@ -234,9 +266,10 @@
234	266	struct machine machines__add(struct machines machines, pid_t pid,
235	267	const char *root_dir)
236	268	{
237		- struct rb_node **p = &machines->guests.rb_node;
	269	+ struct rb_node **p = &machines->guests.rb_root.rb_node;
238	270	struct rb_node *parent = NULL;
239	271	struct machine pos, machine = malloc(sizeof(*machine));
	272	+ bool leftmost = true;
240	273
241	274	if (machine == NULL)
242	275	return NULL;
..	..	@@ -251,12 +284,14 @@
251	284	pos = rb_entry(parent, struct machine, rb_node);
252	285	if (pid < pos->pid)
253	286	p = &(*p)->rb_left;
254		- else
	287	+ else {
255	288	p = &(*p)->rb_right;
	289	+ leftmost = false;
	290	+ }
256	291	}
257	292
258	293	rb_link_node(&machine->rb_node, parent, p);
259		- rb_insert_color(&machine->rb_node, &machines->guests);
	294	+ rb_insert_color_cached(&machine->rb_node, &machines->guests, leftmost);
260	295
261	296	return machine;
262	297	}
..	..	@@ -267,7 +302,7 @@
267	302
268	303	machines->host.comm_exec = comm_exec;
269	304
270		- for (nd = rb_first(&machines->guests); nd; nd = rb_next(nd)) {
	305	+ for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
271	306	struct machine *machine = rb_entry(nd, struct machine, rb_node);
272	307
273	308	machine->comm_exec = comm_exec;
..	..	@@ -276,7 +311,7 @@
276	311
277	312	struct machine machines__find(struct machines machines, pid_t pid)
278	313	{
279		- struct rb_node **p = &machines->guests.rb_node;
	314	+ struct rb_node **p = &machines->guests.rb_root.rb_node;
280	315	struct rb_node *parent = NULL;
281	316	struct machine *machine;
282	317	struct machine *default_machine = NULL;
..	..	@@ -339,7 +374,7 @@
339	374	{
340	375	struct rb_node *nd;
341	376
342		- for (nd = rb_first(&machines->guests); nd; nd = rb_next(nd)) {
	377	+ for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
343	378	struct machine *pos = rb_entry(nd, struct machine, rb_node);
344	379	process(pos, data);
345	380	}
..	..	@@ -352,7 +387,8 @@
352	387
353	388	machines->host.id_hdr_size = id_hdr_size;
354	389
355		- for (node = rb_first(&machines->guests); node; node = rb_next(node)) {
	390	+ for (node = rb_first_cached(&machines->guests); node;
	391	+ node = rb_next(node)) {
356	392	machine = rb_entry(node, struct machine, rb_node);
357	393	machine->id_hdr_size = id_hdr_size;
358	394	}
..	..	@@ -377,28 +413,28 @@
377	413	if (!leader)
378	414	goto out_err;
379	415
380		- if (!leader->mg)
381		- leader->mg = map_groups__new(machine);
	416	+ if (!leader->maps)
	417	+ leader->maps = maps__new(machine);
382	418
383		- if (!leader->mg)
	419	+ if (!leader->maps)
384	420	goto out_err;
385	421
386		- if (th->mg == leader->mg)
	422	+ if (th->maps == leader->maps)
387	423	return;
388	424
389		- if (th->mg) {
	425	+ if (th->maps) {
390	426	/*
391	427	* Maps are created from MMAP events which provide the pid and
392	428	* tid. Consequently there never should be any maps on a thread
393	429	* with an unknown pid. Just print an error if there are.
394	430	*/
395		- if (!map_groups__empty(th->mg))
	431	+ if (!maps__empty(th->maps))
396	432	pr_err("Discarding thread maps for %d:%d\n",
397	433	th->pid_, th->tid);
398		- map_groups__put(th->mg);
	434	+ maps__put(th->maps);
399	435	}
400	436
401		- th->mg = map_groups__get(leader->mg);
	437	+ th->maps = maps__get(leader->maps);
402	438	out_put:
403	439	thread__put(leader);
404	440	return;
..	..	@@ -465,9 +501,10 @@
465	501	pid_t pid, pid_t tid,
466	502	bool create)
467	503	{
468		- struct rb_node **p = &threads->entries.rb_node;
	504	+ struct rb_node **p = &threads->entries.rb_root.rb_node;
469	505	struct rb_node *parent = NULL;
470	506	struct thread *th;
	507	+ bool leftmost = true;
471	508
472	509	th = threads__get_last_match(threads, machine, pid, tid);
473	510	if (th)
..	..	@@ -485,8 +522,10 @@
485	522
486	523	if (tid < th->tid)
487	524	p = &(*p)->rb_left;
488		- else
	525	+ else {
489	526	p = &(*p)->rb_right;
	527	+ leftmost = false;
	528	+ }
490	529	}
491	530
492	531	if (!create)
..	..	@@ -495,18 +534,17 @@
495	534	th = thread__new(pid, tid);
496	535	if (th != NULL) {
497	536	rb_link_node(&th->rb_node, parent, p);
498		- rb_insert_color(&th->rb_node, &threads->entries);
	537	+ rb_insert_color_cached(&th->rb_node, &threads->entries, leftmost);
499	538
500	539	/*
501		- * We have to initialize map_groups separately
502		- * after rb tree is updated.
	540	+ * We have to initialize maps separately after rb tree is updated.
503	541	*
504	542	* The reason is that we call machine__findnew_thread
505		- * within thread__init_map_groups to find the thread
	543	+ * within thread__init_maps to find the thread
506	544	* leader and that would screwed the rb tree.
507	545	*/
508		- if (thread__init_map_groups(th, machine)) {
509		- rb_erase_init(&th->rb_node, &threads->entries);
	546	+ if (thread__init_maps(th, machine)) {
	547	+ rb_erase_cached(&th->rb_node, &threads->entries);
510	548	RB_CLEAR_NODE(&th->rb_node);
511	549	thread__put(th);
512	550	return NULL;
..	..	@@ -617,10 +655,26 @@
617	655	return err;
618	656	}
619	657
	658	+int machine__process_cgroup_event(struct machine *machine,
	659	+ union perf_event *event,
	660	+ struct perf_sample *sample __maybe_unused)
	661	+{
	662	+ struct cgroup *cgrp;
	663	+
	664	+ if (dump_trace)
	665	+ perf_event__fprintf_cgroup(event, stdout);
	666	+
	667	+ cgrp = cgroup__findnew(machine->env, event->cgroup.id, event->cgroup.path);
	668	+ if (cgrp == NULL)
	669	+ return -ENOMEM;
	670	+
	671	+ return 0;
	672	+}
	673	+
620	674	int machine__process_lost_event(struct machine *machine __maybe_unused,
621	675	union perf_event event, struct perf_sample sample __maybe_unused)
622	676	{
623		- dump_printf(": id:%" PRIu64 ": lost:%" PRIu64 "\n",
	677	+ dump_printf(": id:%" PRI_lu64 ": lost:%" PRI_lu64 "\n",
624	678	event->lost.id, event->lost.lost);
625	679	return 0;
626	680	}
..	..	@@ -628,7 +682,7 @@
628	682	int machine__process_lost_samples_event(struct machine *machine __maybe_unused,
629	683	union perf_event event, struct perf_sample sample)
630	684	{
631		- dump_printf(": id:%" PRIu64 ": lost samples :%" PRIu64 "\n",
	685	+ dump_printf(": id:%" PRIu64 ": lost samples :%" PRI_lu64 "\n",
632	686	sample->id, event->lost_samples.lost);
633	687	return 0;
634	688	}
..	..	@@ -649,6 +703,7 @@
649	703
650	704	dso__set_module_info(dso, m, machine);
651	705	dso__set_long_name(dso, strdup(filename), true);
	706	+ dso->kernel = DSO_SPACE__KERNEL;
652	707	}
653	708
654	709	dso__get(dso);
..	..	@@ -681,19 +736,137 @@
681	736	return 0;
682	737	}
683	738
684		-struct map machine__findnew_module_map(struct machine machine, u64 start,
685		- const char *filename)
	739	+static int machine__process_ksymbol_register(struct machine *machine,
	740	+ union perf_event *event,
	741	+ struct perf_sample *sample __maybe_unused)
	742	+{
	743	+ struct symbol *sym;
	744	+ struct map *map = maps__find(&machine->kmaps, event->ksymbol.addr);
	745	+
	746	+ if (!map) {
	747	+ struct dso *dso = dso__new(event->ksymbol.name);
	748	+
	749	+ if (dso) {
	750	+ dso->kernel = DSO_SPACE__KERNEL;
	751	+ map = map__new2(0, dso);
	752	+ }
	753	+
	754	+ if (!dso \|\| !map) {
	755	+ dso__put(dso);
	756	+ return -ENOMEM;
	757	+ }
	758	+
	759	+ if (event->ksymbol.ksym_type == PERF_RECORD_KSYMBOL_TYPE_OOL) {
	760	+ map->dso->binary_type = DSO_BINARY_TYPE__OOL;
	761	+ map->dso->data.file_size = event->ksymbol.len;
	762	+ dso__set_loaded(map->dso);
	763	+ }
	764	+
	765	+ map->start = event->ksymbol.addr;
	766	+ map->end = map->start + event->ksymbol.len;
	767	+ maps__insert(&machine->kmaps, map);
	768	+ dso__set_loaded(dso);
	769	+
	770	+ if (is_bpf_image(event->ksymbol.name)) {
	771	+ dso->binary_type = DSO_BINARY_TYPE__BPF_IMAGE;
	772	+ dso__set_long_name(dso, "", false);
	773	+ }
	774	+ }
	775	+
	776	+ sym = symbol__new(map->map_ip(map, map->start),
	777	+ event->ksymbol.len,
	778	+ 0, 0, event->ksymbol.name);
	779	+ if (!sym)
	780	+ return -ENOMEM;
	781	+ dso__insert_symbol(map->dso, sym);
	782	+ return 0;
	783	+}
	784	+
	785	+static int machine__process_ksymbol_unregister(struct machine *machine,
	786	+ union perf_event *event,
	787	+ struct perf_sample *sample __maybe_unused)
	788	+{
	789	+ struct symbol *sym;
	790	+ struct map *map;
	791	+
	792	+ map = maps__find(&machine->kmaps, event->ksymbol.addr);
	793	+ if (!map)
	794	+ return 0;
	795	+
	796	+ if (map != machine->vmlinux_map)
	797	+ maps__remove(&machine->kmaps, map);
	798	+ else {
	799	+ sym = dso__find_symbol(map->dso, map->map_ip(map, map->start));
	800	+ if (sym)
	801	+ dso__delete_symbol(map->dso, sym);
	802	+ }
	803	+
	804	+ return 0;
	805	+}
	806	+
	807	+int machine__process_ksymbol(struct machine *machine __maybe_unused,
	808	+ union perf_event *event,
	809	+ struct perf_sample *sample)
	810	+{
	811	+ if (dump_trace)
	812	+ perf_event__fprintf_ksymbol(event, stdout);
	813	+
	814	+ if (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER)
	815	+ return machine__process_ksymbol_unregister(machine, event,
	816	+ sample);
	817	+ return machine__process_ksymbol_register(machine, event, sample);
	818	+}
	819	+
	820	+int machine__process_text_poke(struct machine machine, union perf_event event,
	821	+ struct perf_sample *sample __maybe_unused)
	822	+{
	823	+ struct map *map = maps__find(&machine->kmaps, event->text_poke.addr);
	824	+ u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
	825	+
	826	+ if (dump_trace)
	827	+ perf_event__fprintf_text_poke(event, machine, stdout);
	828	+
	829	+ if (!event->text_poke.new_len)
	830	+ return 0;
	831	+
	832	+ if (cpumode != PERF_RECORD_MISC_KERNEL) {
	833	+ pr_debug("%s: unsupported cpumode - ignoring\n", __func__);
	834	+ return 0;
	835	+ }
	836	+
	837	+ if (map && map->dso) {
	838	+ u8 *new_bytes = event->text_poke.bytes + event->text_poke.old_len;
	839	+ int ret;
	840	+
	841	+ /*
	842	+ * Kernel maps might be changed when loading symbols so loading
	843	+ * must be done prior to using kernel maps.
	844	+ */
	845	+ map__load(map);
	846	+ ret = dso__data_write_cache_addr(map->dso, map, machine,
	847	+ event->text_poke.addr,
	848	+ new_bytes,
	849	+ event->text_poke.new_len);
	850	+ if (ret != event->text_poke.new_len)
	851	+ pr_debug("Failed to write kernel text poke at %#" PRI_lx64 "\n",
	852	+ event->text_poke.addr);
	853	+ } else {
	854	+ pr_debug("Failed to find kernel text poke address map for %#" PRI_lx64 "\n",
	855	+ event->text_poke.addr);
	856	+ }
	857	+
	858	+ return 0;
	859	+}
	860	+
	861	+static struct map machine__addnew_module_map(struct machine machine, u64 start,
	862	+ const char *filename)
686	863	{
687	864	struct map *map = NULL;
688		- struct dso *dso = NULL;
689	865	struct kmod_path m;
	866	+ struct dso *dso;
690	867
691	868	if (kmod_path__parse_name(&m, filename))
692	869	return NULL;
693		-
694		- map = map_groups__find_by_name(&machine->kmaps, m.name);
695		- if (map)
696		- goto out;
697	870
698	871	dso = machine__findnew_module_dso(machine, &m, filename);
699	872	if (dso == NULL)
..	..	@@ -703,14 +876,14 @@
703	876	if (map == NULL)
704	877	goto out;
705	878
706		- map_groups__insert(&machine->kmaps, map);
	879	+ maps__insert(&machine->kmaps, map);
707	880
708		- /* Put the map here because map_groups__insert alread got it */
	881	+ /* Put the map here because maps__insert alread got it */
709	882	map__put(map);
710	883	out:
711	884	/* put the dso here, corresponding to machine__findnew_module_dso */
712	885	dso__put(dso);
713		- free(m.name);
	886	+ zfree(&m.name);
714	887	return map;
715	888	}
716	889
..	..	@@ -719,7 +892,7 @@
719	892	struct rb_node *nd;
720	893	size_t ret = __dsos__fprintf(&machines->host.dsos.head, fp);
721	894
722		- for (nd = rb_first(&machines->guests); nd; nd = rb_next(nd)) {
	895	+ for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
723	896	struct machine *pos = rb_entry(nd, struct machine, rb_node);
724	897	ret += __dsos__fprintf(&pos->dsos.head, fp);
725	898	}
..	..	@@ -739,7 +912,7 @@
739	912	struct rb_node *nd;
740	913	size_t ret = machine__fprintf_dsos_buildid(&machines->host, fp, skip, parm);
741	914
742		- for (nd = rb_first(&machines->guests); nd; nd = rb_next(nd)) {
	915	+ for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
743	916	struct machine *pos = rb_entry(nd, struct machine, rb_node);
744	917	ret += machine__fprintf_dsos_buildid(pos, fp, skip, parm);
745	918	}
..	..	@@ -750,7 +923,7 @@
750	923	{
751	924	int i;
752	925	size_t printed = 0;
753		- struct dso *kdso = machine__kernel_map(machine)->dso;
	926	+ struct dso *kdso = machine__kernel_dso(machine);
754	927
755	928	if (kdso->has_build_id) {
756	929	char filename[PATH_MAX];
..	..	@@ -779,7 +952,8 @@
779	952
780	953	ret = fprintf(fp, "Threads: %u\n", threads->nr);
781	954
782		- for (nd = rb_first(&threads->entries); nd; nd = rb_next(nd)) {
	955	+ for (nd = rb_first_cached(&threads->entries); nd;
	956	+ nd = rb_next(nd)) {
783	957	struct thread *pos = rb_entry(nd, struct thread, rb_node);
784	958
785	959	ret += thread__fprintf(pos, fp);
..	..	@@ -800,14 +974,14 @@
800	974	vmlinux_name = symbol_conf.vmlinux_name;
801	975
802	976	kernel = machine__findnew_kernel(machine, vmlinux_name,
803		- "[kernel]", DSO_TYPE_KERNEL);
	977	+ "[kernel]", DSO_SPACE__KERNEL);
804	978	} else {
805	979	if (symbol_conf.default_guest_vmlinux_name)
806	980	vmlinux_name = symbol_conf.default_guest_vmlinux_name;
807	981
808	982	kernel = machine__findnew_kernel(machine, vmlinux_name,
809	983	"[guest.kernel]",
810		- DSO_TYPE_GUEST_KERNEL);
	984	+ DSO_SPACE__KERNEL_GUEST);
811	985	}
812	986
813	987	if (kernel != NULL && (!kernel->has_build_id))
..	..	@@ -836,7 +1010,8 @@
836	1010	* symbol_name if it's not that important.
837	1011	*/
838	1012	static int machine__get_running_kernel_start(struct machine *machine,
839		- const char *symbol_name, u64 start)
	1013	+ const char **symbol_name,
	1014	+ u64 start, u64 end)
840	1015	{
841	1016	char filename[PATH_MAX];
842	1017	int i, err = -1;
..	..	@@ -861,6 +1036,11 @@
861	1036	*symbol_name = name;
862	1037
863	1038	*start = addr;
	1039	+
	1040	+ err = kallsyms__get_function_start(filename, "_etext", &addr);
	1041	+ if (!err)
	1042	+ *end = addr;
	1043	+
864	1044	return 0;
865	1045	}
866	1046
..	..	@@ -880,10 +1060,9 @@
880	1060
881	1061	kmap = map__kmap(map);
882	1062
883		- kmap->kmaps = &machine->kmaps;
884	1063	strlcpy(kmap->name, xm->name, KMAP_NAME_LEN);
885	1064
886		- map_groups__insert(&machine->kmaps, map);
	1065	+ maps__insert(&machine->kmaps, map);
887	1066
888	1067	pr_debug2("Added extra kernel map %s %" PRIx64 "-%" PRIx64 "\n",
889	1068	kmap->name, map->start, map->end);
..	..	@@ -928,8 +1107,7 @@
928	1107	int machine__map_x86_64_entry_trampolines(struct machine *machine,
929	1108	struct dso *kernel)
930	1109	{
931		- struct map_groups *kmaps = &machine->kmaps;
932		- struct maps *maps = &kmaps->maps;
	1110	+ struct maps *kmaps = &machine->kmaps;
933	1111	int nr_cpus_avail, cpu;
934	1112	bool found = false;
935	1113	struct map *map;
..	..	@@ -939,14 +1117,14 @@
939	1117	* In the vmlinux case, pgoff is a virtual address which must now be
940	1118	* mapped to a vmlinux offset.
941	1119	*/
942		- for (map = maps__first(maps); map; map = map__next(map)) {
	1120	+ maps__for_each_entry(kmaps, map) {
943	1121	struct kmap *kmap = __map__kmap(map);
944	1122	struct map *dest_map;
945	1123
946	1124	if (!kmap \|\| !is_entry_trampoline(kmap->name))
947	1125	continue;
948	1126
949		- dest_map = map_groups__find(kmaps, map->pgoff);
	1127	+ dest_map = maps__find(kmaps, map->pgoff);
950	1128	if (dest_map != map)
951	1129	map->pgoff = dest_map->map_ip(dest_map, map->pgoff);
952	1130	found = true;
..	..	@@ -991,9 +1169,6 @@
991	1169	static int
992	1170	__machine__create_kernel_maps(struct machine machine, struct dso kernel)
993	1171	{
994		- struct kmap *kmap;
995		- struct map *map;
996		-
997	1172	/* In case of renewal the kernel map, destroy previous one */
998	1173	machine__destroy_kernel_maps(machine);
999	1174
..	..	@@ -1002,14 +1177,7 @@
1002	1177	return -1;
1003	1178
1004	1179	machine->vmlinux_map->map_ip = machine->vmlinux_map->unmap_ip = identity__map_ip;
1005		- map = machine__kernel_map(machine);
1006		- kmap = map__kmap(map);
1007		- if (!kmap)
1008		- return -1;
1009		-
1010		- kmap->kmaps = &machine->kmaps;
1011		- map_groups__insert(&machine->kmaps, map);
1012		-
	1180	+ maps__insert(&machine->kmaps, machine->vmlinux_map);
1013	1181	return 0;
1014	1182	}
1015	1183
..	..	@@ -1022,7 +1190,7 @@
1022	1190	return;
1023	1191
1024	1192	kmap = map__kmap(map);
1025		- map_groups__remove(&machine->kmaps, map);
	1193	+ maps__remove(&machine->kmaps, map);
1026	1194	if (kmap && kmap->ref_reloc_sym) {
1027	1195	zfree((char **)&kmap->ref_reloc_sym->name);
1028	1196	zfree(&kmap->ref_reloc_sym);
..	..	@@ -1082,7 +1250,7 @@
1082	1250
1083	1251	void machines__destroy_kernel_maps(struct machines *machines)
1084	1252	{
1085		- struct rb_node *next = rb_first(&machines->guests);
	1253	+ struct rb_node *next = rb_first_cached(&machines->guests);
1086	1254
1087	1255	machine__destroy_kernel_maps(&machines->host);
1088	1256
..	..	@@ -1090,7 +1258,7 @@
1090	1258	struct machine *pos = rb_entry(next, struct machine, rb_node);
1091	1259
1092	1260	next = rb_next(&pos->rb_node);
1093		- rb_erase(&pos->rb_node, &machines->guests);
	1261	+ rb_erase_cached(&pos->rb_node, &machines->guests);
1094	1262	machine__delete(pos);
1095	1263	}
1096	1264	}
..	..	@@ -1117,7 +1285,7 @@
1117	1285	* kernel, with modules between them, fixup the end of all
1118	1286	* sections.
1119	1287	*/
1120		- map_groups__fixup_end(&machine->kmaps);
	1288	+ maps__fixup_end(&machine->kmaps);
1121	1289	}
1122	1290
1123	1291	return ret;
..	..	@@ -1146,9 +1314,10 @@
1146	1314	if (!file)
1147	1315	return NULL;
1148	1316
1149		- version[0] = '\0';
1150	1317	tmp = fgets(version, sizeof(version), file);
1151	1318	fclose(file);
	1319	+ if (!tmp)
	1320	+ return NULL;
1152	1321
1153	1322	name = strstr(version, prefix);
1154	1323	if (!name)
..	..	@@ -1167,11 +1336,10 @@
1167	1336	dso->symtab_type == DSO_BINARY_TYPE__GUEST_KMODULE;
1168	1337	}
1169	1338
1170		-static int map_groups__set_module_path(struct map_groups mg, const char path,
1171		- struct kmod_path *m)
	1339	+static int maps__set_module_path(struct maps maps, const char path, struct kmod_path *m)
1172	1340	{
1173	1341	char *long_name;
1174		- struct map *map = map_groups__find_by_name(mg, m->name);
	1342	+ struct map *map = maps__find_by_name(maps, m->name);
1175	1343
1176	1344	if (map == NULL)
1177	1345	return 0;
..	..	@@ -1195,8 +1363,7 @@
1195	1363	return 0;
1196	1364	}
1197	1365
1198		-static int map_groups__set_modules_path_dir(struct map_groups *mg,
1199		- const char *dir_name, int depth)
	1366	+static int maps__set_modules_path_dir(struct maps maps, const char dir_name, int depth)
1200	1367	{
1201	1368	struct dirent *dent;
1202	1369	DIR *dir = opendir(dir_name);
..	..	@@ -1228,8 +1395,7 @@
1228	1395	continue;
1229	1396	}
1230	1397
1231		- ret = map_groups__set_modules_path_dir(mg, path,
1232		- depth + 1);
	1398	+ ret = maps__set_modules_path_dir(maps, path, depth + 1);
1233	1399	if (ret < 0)
1234	1400	goto out;
1235	1401	} else {
..	..	@@ -1240,9 +1406,9 @@
1240	1406	goto out;
1241	1407
1242	1408	if (m.kmod)
1243		- ret = map_groups__set_module_path(mg, path, &m);
	1409	+ ret = maps__set_module_path(maps, path, &m);
1244	1410
1245		- free(m.name);
	1411	+ zfree(&m.name);
1246	1412
1247	1413	if (ret)
1248	1414	goto out;
..	..	@@ -1267,7 +1433,7 @@
1267	1433	machine->root_dir, version);
1268	1434	free(version);
1269	1435
1270		- return map_groups__set_modules_path_dir(&machine->kmaps, modules_path, 0);
	1436	+ return maps__set_modules_path_dir(&machine->kmaps, modules_path, 0);
1271	1437	}
1272	1438	int __weak arch__fix_module_text_start(u64 *start __maybe_unused,
1273	1439	u64 *size __maybe_unused,
..	..	@@ -1285,7 +1451,7 @@
1285	1451	if (arch__fix_module_text_start(&start, &size, name) < 0)
1286	1452	return -1;
1287	1453
1288		- map = machine__findnew_module_map(machine, start, name);
	1454	+ map = machine__addnew_module_map(machine, start, name);
1289	1455	if (map == NULL)
1290	1456	return -1;
1291	1457	map->end = start + size;
..	..	@@ -1340,11 +1506,11 @@
1340	1506	struct map *map = machine__kernel_map(machine);
1341	1507
1342	1508	map__get(map);
1343		- map_groups__remove(&machine->kmaps, map);
	1509	+ maps__remove(&machine->kmaps, map);
1344	1510
1345	1511	machine__set_kernel_mmap(machine, start, end);
1346	1512
1347		- map_groups__insert(&machine->kmaps, map);
	1513	+ maps__insert(&machine->kmaps, map);
1348	1514	map__put(map);
1349	1515	}
1350	1516
..	..	@@ -1353,7 +1519,7 @@
1353	1519	struct dso *kernel = machine__get_kernel(machine);
1354	1520	const char *name = NULL;
1355	1521	struct map *map;
1356		- u64 addr = 0;
	1522	+ u64 start = 0, end = ~0ULL;
1357	1523	int ret;
1358	1524
1359	1525	if (kernel == NULL)
..	..	@@ -1372,9 +1538,9 @@
1372	1538	"continuing anyway...\n", machine->pid);
1373	1539	}
1374	1540
1375		- if (!machine__get_running_kernel_start(machine, &name, &addr)) {
	1541	+ if (!machine__get_running_kernel_start(machine, &name, &start, &end)) {
1376	1542	if (name &&
1377		- map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map, name, addr)) {
	1543	+ map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map, name, start)) {
1378	1544	machine__destroy_kernel_maps(machine);
1379	1545	ret = -1;
1380	1546	goto out_put;
..	..	@@ -1384,16 +1550,19 @@
1384	1550	* we have a real start address now, so re-order the kmaps
1385	1551	* assume it's the last in the kmaps
1386	1552	*/
1387		- machine__update_kernel_mmap(machine, addr, ~0ULL);
	1553	+ machine__update_kernel_mmap(machine, start, end);
1388	1554	}
1389	1555
1390	1556	if (machine__create_extra_kernel_maps(machine, kernel))
1391	1557	pr_debug("Problems creating extra kernel maps, continuing anyway...\n");
1392	1558
1393		- /* update end address of the kernel map using adjacent module address */
1394		- map = map__next(machine__kernel_map(machine));
1395		- if (map)
1396		- machine__set_kernel_mmap(machine, addr, map->start);
	1559	+ if (end == ~0ULL) {
	1560	+ /* update end address of the kernel map using adjacent module address */
	1561	+ map = map__next(machine__kernel_map(machine));
	1562	+ if (map)
	1563	+ machine__set_kernel_mmap(machine, start, map->start);
	1564	+ }
	1565	+
1397	1566	out_put:
1398	1567	dso__put(kernel);
1399	1568	return ret;
..	..	@@ -1421,8 +1590,7 @@
1421	1590	static int machine__process_extra_kernel_map(struct machine *machine,
1422	1591	union perf_event *event)
1423	1592	{
1424		- struct map *kernel_map = machine__kernel_map(machine);
1425		- struct dso *kernel = kernel_map ? kernel_map->dso : NULL;
	1593	+ struct dso *kernel = machine__kernel_dso(machine);
1426	1594	struct extra_kernel_map xm = {
1427	1595	.start = event->mmap.start,
1428	1596	.end = event->mmap.start + event->mmap.len,
..	..	@@ -1441,7 +1609,7 @@
1441	1609	union perf_event *event)
1442	1610	{
1443	1611	struct map *map;
1444		- enum dso_kernel_type kernel_type;
	1612	+ enum dso_space_type dso_space;
1445	1613	bool is_kernel_mmap;
1446	1614
1447	1615	/* If we have maps from kcore then we do not need or want any others */
..	..	@@ -1449,17 +1617,17 @@
1449	1617	return 0;
1450	1618
1451	1619	if (machine__is_host(machine))
1452		- kernel_type = DSO_TYPE_KERNEL;
	1620	+ dso_space = DSO_SPACE__KERNEL;
1453	1621	else
1454		- kernel_type = DSO_TYPE_GUEST_KERNEL;
	1622	+ dso_space = DSO_SPACE__KERNEL_GUEST;
1455	1623
1456	1624	is_kernel_mmap = memcmp(event->mmap.filename,
1457	1625	machine->mmap_name,
1458	1626	strlen(machine->mmap_name) - 1) == 0;
1459	1627	if (event->mmap.filename[0] == '/' \|\|
1460	1628	(!is_kernel_mmap && event->mmap.filename[0] == '[')) {
1461		- map = machine__findnew_module_map(machine, event->mmap.start,
1462		- event->mmap.filename);
	1629	+ map = machine__addnew_module_map(machine, event->mmap.start,
	1630	+ event->mmap.filename);
1463	1631	if (map == NULL)
1464	1632	goto out_problem;
1465	1633
..	..	@@ -1511,7 +1679,7 @@
1511	1679	if (kernel == NULL)
1512	1680	goto out_problem;
1513	1681
1514		- kernel->kernel = kernel_type;
	1682	+ kernel->kernel = dso_space;
1515	1683	if (__machine__create_kernel_maps(machine, kernel) < 0) {
1516	1684	dso__put(kernel);
1517	1685	goto out_problem;
..	..	@@ -1554,6 +1722,12 @@
1554	1722	{
1555	1723	struct thread *thread;
1556	1724	struct map *map;
	1725	+ struct dso_id dso_id = {
	1726	+ .maj = event->mmap2.maj,
	1727	+ .min = event->mmap2.min,
	1728	+ .ino = event->mmap2.ino,
	1729	+ .ino_generation = event->mmap2.ino_generation,
	1730	+ };
1557	1731	int ret = 0;
1558	1732
1559	1733	if (dump_trace)
..	..	@@ -1574,10 +1748,7 @@
1574	1748
1575	1749	map = map__new(machine, event->mmap2.start,
1576	1750	event->mmap2.len, event->mmap2.pgoff,
1577		- event->mmap2.maj,
1578		- event->mmap2.min, event->mmap2.ino,
1579		- event->mmap2.ino_generation,
1580		- event->mmap2.prot,
	1751	+ &dso_id, event->mmap2.prot,
1581	1752	event->mmap2.flags,
1582	1753	event->mmap2.filename, thread);
1583	1754
..	..	@@ -1630,9 +1801,7 @@
1630	1801
1631	1802	map = map__new(machine, event->mmap.start,
1632	1803	event->mmap.len, event->mmap.pgoff,
1633		- 0, 0, 0, 0, prot, 0,
1634		- event->mmap.filename,
1635		- thread);
	1804	+ NULL, prot, 0, event->mmap.filename, thread);
1636	1805
1637	1806	if (map == NULL)
1638	1807	goto out_problem_map;
..	..	@@ -1661,10 +1830,12 @@
1661	1830	if (threads->last_match == th)
1662	1831	threads__set_last_match(threads, NULL);
1663	1832
1664		- BUG_ON(refcount_read(&th->refcnt) == 0);
1665	1833	if (lock)
1666	1834	down_write(&threads->lock);
1667		- rb_erase_init(&th->rb_node, &threads->entries);
	1835	+
	1836	+ BUG_ON(refcount_read(&th->refcnt) == 0);
	1837	+
	1838	+ rb_erase_cached(&th->rb_node, &threads->entries);
1668	1839	RB_CLEAR_NODE(&th->rb_node);
1669	1840	--threads->nr;
1670	1841	/*
..	..	@@ -1673,9 +1844,16 @@
1673	1844	* will be called and we will remove it from the dead_threads list.
1674	1845	*/
1675	1846	list_add_tail(&th->node, &threads->dead);
	1847	+
	1848	+ /*
	1849	+ * We need to do the put here because if this is the last refcount,
	1850	+ * then we will be touching the threads->dead head when removing the
	1851	+ * thread.
	1852	+ */
	1853	+ thread__put(th);
	1854	+
1676	1855	if (lock)
1677	1856	up_write(&threads->lock);
1678		- thread__put(th);
1679	1857	}
1680	1858
1681	1859	void machine__remove_thread(struct machine machine, struct thread th)
..	..	@@ -1692,6 +1870,7 @@
1692	1870	struct thread *parent = machine__findnew_thread(machine,
1693	1871	event->fork.ppid,
1694	1872	event->fork.ptid);
	1873	+ bool do_maps_clone = true;
1695	1874	int err = 0;
1696	1875
1697	1876	if (dump_trace)
..	..	@@ -1720,9 +1899,25 @@
1720	1899
1721	1900	thread = machine__findnew_thread(machine, event->fork.pid,
1722	1901	event->fork.tid);
	1902	+ /*
	1903	+ * When synthesizing FORK events, we are trying to create thread
	1904	+ * objects for the already running tasks on the machine.
	1905	+ *
	1906	+ * Normally, for a kernel FORK event, we want to clone the parent's
	1907	+ * maps because that is what the kernel just did.
	1908	+ *
	1909	+ * But when synthesizing, this should not be done. If we do, we end up
	1910	+ * with overlapping maps as we process the sythesized MMAP2 events that
	1911	+ * get delivered shortly thereafter.
	1912	+ *
	1913	+ * Use the FORK event misc flags in an internal way to signal this
	1914	+ * situation, so we can elide the map clone when appropriate.
	1915	+ */
	1916	+ if (event->fork.header.misc & PERF_RECORD_MISC_FORK_EXEC)
	1917	+ do_maps_clone = false;
1723	1918
1724	1919	if (thread == NULL \|\| parent == NULL \|\|
1725		- thread__fork(thread, parent, sample->time) < 0) {
	1920	+ thread__fork(thread, parent, sample->time, do_maps_clone) < 0) {
1726	1921	dump_printf("problem processing PERF_RECORD_FORK, skipping event.\n");
1727	1922	err = -1;
1728	1923	}
..	..	@@ -1762,6 +1957,8 @@
1762	1957	ret = machine__process_mmap_event(machine, event, sample); break;
1763	1958	case PERF_RECORD_NAMESPACES:
1764	1959	ret = machine__process_namespaces_event(machine, event, sample); break;
	1960	+ case PERF_RECORD_CGROUP:
	1961	+ ret = machine__process_cgroup_event(machine, event, sample); break;
1765	1962	case PERF_RECORD_MMAP2:
1766	1963	ret = machine__process_mmap2_event(machine, event, sample); break;
1767	1964	case PERF_RECORD_FORK:
..	..	@@ -1779,6 +1976,12 @@
1779	1976	case PERF_RECORD_SWITCH:
1780	1977	case PERF_RECORD_SWITCH_CPU_WIDE:
1781	1978	ret = machine__process_switch_event(machine, event); break;
	1979	+ case PERF_RECORD_KSYMBOL:
	1980	+ ret = machine__process_ksymbol(machine, event, sample); break;
	1981	+ case PERF_RECORD_BPF_EVENT:
	1982	+ ret = machine__process_bpf(machine, event, sample); break;
	1983	+ case PERF_RECORD_TEXT_POKE:
	1984	+ ret = machine__process_text_poke(machine, event, sample); break;
1782	1985	default:
1783	1986	ret = -1;
1784	1987	break;
..	..	@@ -1812,8 +2015,9 @@
1812	2015
1813	2016	ams->addr = ip;
1814	2017	ams->al_addr = al.addr;
1815		- ams->sym = al.sym;
1816		- ams->map = al.map;
	2018	+ ams->ms.maps = al.maps;
	2019	+ ams->ms.sym = al.sym;
	2020	+ ams->ms.map = al.map;
1817	2021	ams->phys_addr = 0;
1818	2022	}
1819	2023
..	..	@@ -1829,8 +2033,9 @@
1829	2033
1830	2034	ams->addr = addr;
1831	2035	ams->al_addr = al.addr;
1832		- ams->sym = al.sym;
1833		- ams->map = al.map;
	2036	+ ams->ms.maps = al.maps;
	2037	+ ams->ms.sym = al.sym;
	2038	+ ams->ms.map = al.map;
1834	2039	ams->phys_addr = phys_addr;
1835	2040	}
1836	2041
..	..	@@ -1850,8 +2055,9 @@
1850	2055	return mi;
1851	2056	}
1852	2057
1853		-static char callchain_srcline(struct map map, struct symbol *sym, u64 ip)
	2058	+static char callchain_srcline(struct map_symbol ms, u64 ip)
1854	2059	{
	2060	+ struct map *map = ms->map;
1855	2061	char *srcline = NULL;
1856	2062
1857	2063	if (!map \|\| callchain_param.key == CCKEY_FUNCTION)
..	..	@@ -1863,7 +2069,7 @@
1863	2069	bool show_addr = callchain_param.key == CCKEY_ADDRESS;
1864	2070
1865	2071	srcline = get_srcline(map->dso, map__rip_2objdump(map, ip),
1866		- sym, show_sym, show_addr, ip);
	2072	+ ms->sym, show_sym, show_addr, ip);
1867	2073	srcline__tree_insert(&map->dso->srclines, ip, srcline);
1868	2074	}
1869	2075
..	..	@@ -1886,6 +2092,7 @@
1886	2092	struct iterations *iter,
1887	2093	u64 branch_from)
1888	2094	{
	2095	+ struct map_symbol ms;
1889	2096	struct addr_location al;
1890	2097	int nr_loop_iter = 0;
1891	2098	u64 iter_cycles = 0;
..	..	@@ -1944,8 +2151,11 @@
1944	2151	iter_cycles = iter->cycles;
1945	2152	}
1946	2153
1947		- srcline = callchain_srcline(al.map, al.sym, al.addr);
1948		- return callchain_cursor_append(cursor, ip, al.map, al.sym,
	2154	+ ms.maps = al.maps;
	2155	+ ms.map = al.map;
	2156	+ ms.sym = al.sym;
	2157	+ srcline = callchain_srcline(&ms, al.addr);
	2158	+ return callchain_cursor_append(cursor, ip, &ms,
1949	2159	branch, flags, nr_loop_iter,
1950	2160	iter_cycles, branch_from, srcline);
1951	2161	}
..	..	@@ -1955,15 +2165,16 @@
1955	2165	{
1956	2166	unsigned int i;
1957	2167	const struct branch_stack *bs = sample->branch_stack;
	2168	+ struct branch_entry *entries = perf_sample__branch_entries(sample);
1958	2169	struct branch_info *bi = calloc(bs->nr, sizeof(struct branch_info));
1959	2170
1960	2171	if (!bi)
1961	2172	return NULL;
1962	2173
1963	2174	for (i = 0; i < bs->nr; i++) {
1964		- ip__resolve_ams(al->thread, &bi[i].to, bs->entries[i].to);
1965		- ip__resolve_ams(al->thread, &bi[i].from, bs->entries[i].from);
1966		- bi[i].flags = bs->entries[i].flags;
	2175	+ ip__resolve_ams(al->thread, &bi[i].to, entries[i].to);
	2176	+ ip__resolve_ams(al->thread, &bi[i].from, entries[i].from);
	2177	+ bi[i].flags = entries[i].flags;
1967	2178	}
1968	2179	return bi;
1969	2180	}
..	..	@@ -2032,6 +2243,303 @@
2032	2243	return nr;
2033	2244	}
2034	2245
	2246	+static int lbr_callchain_add_kernel_ip(struct thread *thread,
	2247	+ struct callchain_cursor *cursor,
	2248	+ struct perf_sample *sample,
	2249	+ struct symbol **parent,
	2250	+ struct addr_location *root_al,
	2251	+ u64 branch_from,
	2252	+ bool callee, int end)
	2253	+{
	2254	+ struct ip_callchain *chain = sample->callchain;
	2255	+ u8 cpumode = PERF_RECORD_MISC_USER;
	2256	+ int err, i;
	2257	+
	2258	+ if (callee) {
	2259	+ for (i = 0; i < end + 1; i++) {
	2260	+ err = add_callchain_ip(thread, cursor, parent,
	2261	+ root_al, &cpumode, chain->ips[i],
	2262	+ false, NULL, NULL, branch_from);
	2263	+ if (err)
	2264	+ return err;
	2265	+ }
	2266	+ return 0;
	2267	+ }
	2268	+
	2269	+ for (i = end; i >= 0; i--) {
	2270	+ err = add_callchain_ip(thread, cursor, parent,
	2271	+ root_al, &cpumode, chain->ips[i],
	2272	+ false, NULL, NULL, branch_from);
	2273	+ if (err)
	2274	+ return err;
	2275	+ }
	2276	+
	2277	+ return 0;
	2278	+}
	2279	+
	2280	+static void save_lbr_cursor_node(struct thread *thread,
	2281	+ struct callchain_cursor *cursor,
	2282	+ int idx)
	2283	+{
	2284	+ struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
	2285	+
	2286	+ if (!lbr_stitch)
	2287	+ return;
	2288	+
	2289	+ if (cursor->pos == cursor->nr) {
	2290	+ lbr_stitch->prev_lbr_cursor[idx].valid = false;
	2291	+ return;
	2292	+ }
	2293	+
	2294	+ if (!cursor->curr)
	2295	+ cursor->curr = cursor->first;
	2296	+ else
	2297	+ cursor->curr = cursor->curr->next;
	2298	+ memcpy(&lbr_stitch->prev_lbr_cursor[idx], cursor->curr,
	2299	+ sizeof(struct callchain_cursor_node));
	2300	+
	2301	+ lbr_stitch->prev_lbr_cursor[idx].valid = true;
	2302	+ cursor->pos++;
	2303	+}
	2304	+
	2305	+static int lbr_callchain_add_lbr_ip(struct thread *thread,
	2306	+ struct callchain_cursor *cursor,
	2307	+ struct perf_sample *sample,
	2308	+ struct symbol **parent,
	2309	+ struct addr_location *root_al,
	2310	+ u64 *branch_from,
	2311	+ bool callee)
	2312	+{
	2313	+ struct branch_stack *lbr_stack = sample->branch_stack;
	2314	+ struct branch_entry *entries = perf_sample__branch_entries(sample);
	2315	+ u8 cpumode = PERF_RECORD_MISC_USER;
	2316	+ int lbr_nr = lbr_stack->nr;
	2317	+ struct branch_flags *flags;
	2318	+ int err, i;
	2319	+ u64 ip;
	2320	+
	2321	+ /*
	2322	+ * The curr and pos are not used in writing session. They are cleared
	2323	+ * in callchain_cursor_commit() when the writing session is closed.
	2324	+ * Using curr and pos to track the current cursor node.
	2325	+ */
	2326	+ if (thread->lbr_stitch) {
	2327	+ cursor->curr = NULL;
	2328	+ cursor->pos = cursor->nr;
	2329	+ if (cursor->nr) {
	2330	+ cursor->curr = cursor->first;
	2331	+ for (i = 0; i < (int)(cursor->nr - 1); i++)
	2332	+ cursor->curr = cursor->curr->next;
	2333	+ }
	2334	+ }
	2335	+
	2336	+ if (callee) {
	2337	+ /* Add LBR ip from first entries.to */
	2338	+ ip = entries[0].to;
	2339	+ flags = &entries[0].flags;
	2340	+ *branch_from = entries[0].from;
	2341	+ err = add_callchain_ip(thread, cursor, parent,
	2342	+ root_al, &cpumode, ip,
	2343	+ true, flags, NULL,
	2344	+ *branch_from);
	2345	+ if (err)
	2346	+ return err;
	2347	+
	2348	+ /*
	2349	+ * The number of cursor node increases.
	2350	+ * Move the current cursor node.
	2351	+ * But does not need to save current cursor node for entry 0.
	2352	+ * It's impossible to stitch the whole LBRs of previous sample.
	2353	+ */
	2354	+ if (thread->lbr_stitch && (cursor->pos != cursor->nr)) {
	2355	+ if (!cursor->curr)
	2356	+ cursor->curr = cursor->first;
	2357	+ else
	2358	+ cursor->curr = cursor->curr->next;
	2359	+ cursor->pos++;
	2360	+ }
	2361	+
	2362	+ /* Add LBR ip from entries.from one by one. */
	2363	+ for (i = 0; i < lbr_nr; i++) {
	2364	+ ip = entries[i].from;
	2365	+ flags = &entries[i].flags;
	2366	+ err = add_callchain_ip(thread, cursor, parent,
	2367	+ root_al, &cpumode, ip,
	2368	+ true, flags, NULL,
	2369	+ *branch_from);
	2370	+ if (err)
	2371	+ return err;
	2372	+ save_lbr_cursor_node(thread, cursor, i);
	2373	+ }
	2374	+ return 0;
	2375	+ }
	2376	+
	2377	+ /* Add LBR ip from entries.from one by one. */
	2378	+ for (i = lbr_nr - 1; i >= 0; i--) {
	2379	+ ip = entries[i].from;
	2380	+ flags = &entries[i].flags;
	2381	+ err = add_callchain_ip(thread, cursor, parent,
	2382	+ root_al, &cpumode, ip,
	2383	+ true, flags, NULL,
	2384	+ *branch_from);
	2385	+ if (err)
	2386	+ return err;
	2387	+ save_lbr_cursor_node(thread, cursor, i);
	2388	+ }
	2389	+
	2390	+ /* Add LBR ip from first entries.to */
	2391	+ ip = entries[0].to;
	2392	+ flags = &entries[0].flags;
	2393	+ *branch_from = entries[0].from;
	2394	+ err = add_callchain_ip(thread, cursor, parent,
	2395	+ root_al, &cpumode, ip,
	2396	+ true, flags, NULL,
	2397	+ *branch_from);
	2398	+ if (err)
	2399	+ return err;
	2400	+
	2401	+ return 0;
	2402	+}
	2403	+
	2404	+static int lbr_callchain_add_stitched_lbr_ip(struct thread *thread,
	2405	+ struct callchain_cursor *cursor)
	2406	+{
	2407	+ struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
	2408	+ struct callchain_cursor_node *cnode;
	2409	+ struct stitch_list *stitch_node;
	2410	+ int err;
	2411	+
	2412	+ list_for_each_entry(stitch_node, &lbr_stitch->lists, node) {
	2413	+ cnode = &stitch_node->cursor;
	2414	+
	2415	+ err = callchain_cursor_append(cursor, cnode->ip,
	2416	+ &cnode->ms,
	2417	+ cnode->branch,
	2418	+ &cnode->branch_flags,
	2419	+ cnode->nr_loop_iter,
	2420	+ cnode->iter_cycles,
	2421	+ cnode->branch_from,
	2422	+ cnode->srcline);
	2423	+ if (err)
	2424	+ return err;
	2425	+ }
	2426	+ return 0;
	2427	+}
	2428	+
	2429	+static struct stitch_list get_stitch_node(struct thread thread)
	2430	+{
	2431	+ struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
	2432	+ struct stitch_list *stitch_node;
	2433	+
	2434	+ if (!list_empty(&lbr_stitch->free_lists)) {
	2435	+ stitch_node = list_first_entry(&lbr_stitch->free_lists,
	2436	+ struct stitch_list, node);
	2437	+ list_del(&stitch_node->node);
	2438	+
	2439	+ return stitch_node;
	2440	+ }
	2441	+
	2442	+ return malloc(sizeof(struct stitch_list));
	2443	+}
	2444	+
	2445	+static bool has_stitched_lbr(struct thread *thread,
	2446	+ struct perf_sample *cur,
	2447	+ struct perf_sample *prev,
	2448	+ unsigned int max_lbr,
	2449	+ bool callee)
	2450	+{
	2451	+ struct branch_stack *cur_stack = cur->branch_stack;
	2452	+ struct branch_entry *cur_entries = perf_sample__branch_entries(cur);
	2453	+ struct branch_stack *prev_stack = prev->branch_stack;
	2454	+ struct branch_entry *prev_entries = perf_sample__branch_entries(prev);
	2455	+ struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
	2456	+ int i, j, nr_identical_branches = 0;
	2457	+ struct stitch_list *stitch_node;
	2458	+ u64 cur_base, distance;
	2459	+
	2460	+ if (!cur_stack \|\| !prev_stack)
	2461	+ return false;
	2462	+
	2463	+ /* Find the physical index of the base-of-stack for current sample. */
	2464	+ cur_base = max_lbr - cur_stack->nr + cur_stack->hw_idx + 1;
	2465	+
	2466	+ distance = (prev_stack->hw_idx > cur_base) ? (prev_stack->hw_idx - cur_base) :
	2467	+ (max_lbr + prev_stack->hw_idx - cur_base);
	2468	+ /* Previous sample has shorter stack. Nothing can be stitched. */
	2469	+ if (distance + 1 > prev_stack->nr)
	2470	+ return false;
	2471	+
	2472	+ /*
	2473	+ * Check if there are identical LBRs between two samples.
	2474	+ * Identicall LBRs must have same from, to and flags values. Also,
	2475	+ * they have to be saved in the same LBR registers (same physical
	2476	+ * index).
	2477	+ *
	2478	+ * Starts from the base-of-stack of current sample.
	2479	+ */
	2480	+ for (i = distance, j = cur_stack->nr - 1; (i >= 0) && (j >= 0); i--, j--) {
	2481	+ if ((prev_entries[i].from != cur_entries[j].from) \|\|
	2482	+ (prev_entries[i].to != cur_entries[j].to) \|\|
	2483	+ (prev_entries[i].flags.value != cur_entries[j].flags.value))
	2484	+ break;
	2485	+ nr_identical_branches++;
	2486	+ }
	2487	+
	2488	+ if (!nr_identical_branches)
	2489	+ return false;
	2490	+
	2491	+ /*
	2492	+ * Save the LBRs between the base-of-stack of previous sample
	2493	+ * and the base-of-stack of current sample into lbr_stitch->lists.
	2494	+ * These LBRs will be stitched later.
	2495	+ */
	2496	+ for (i = prev_stack->nr - 1; i > (int)distance; i--) {
	2497	+
	2498	+ if (!lbr_stitch->prev_lbr_cursor[i].valid)
	2499	+ continue;
	2500	+
	2501	+ stitch_node = get_stitch_node(thread);
	2502	+ if (!stitch_node)
	2503	+ return false;
	2504	+
	2505	+ memcpy(&stitch_node->cursor, &lbr_stitch->prev_lbr_cursor[i],
	2506	+ sizeof(struct callchain_cursor_node));
	2507	+
	2508	+ if (callee)
	2509	+ list_add(&stitch_node->node, &lbr_stitch->lists);
	2510	+ else
	2511	+ list_add_tail(&stitch_node->node, &lbr_stitch->lists);
	2512	+ }
	2513	+
	2514	+ return true;
	2515	+}
	2516	+
	2517	+static bool alloc_lbr_stitch(struct thread *thread, unsigned int max_lbr)
	2518	+{
	2519	+ if (thread->lbr_stitch)
	2520	+ return true;
	2521	+
	2522	+ thread->lbr_stitch = zalloc(sizeof(*thread->lbr_stitch));
	2523	+ if (!thread->lbr_stitch)
	2524	+ goto err;
	2525	+
	2526	+ thread->lbr_stitch->prev_lbr_cursor = calloc(max_lbr + 1, sizeof(struct callchain_cursor_node));
	2527	+ if (!thread->lbr_stitch->prev_lbr_cursor)
	2528	+ goto free_lbr_stitch;
	2529	+
	2530	+ INIT_LIST_HEAD(&thread->lbr_stitch->lists);
	2531	+ INIT_LIST_HEAD(&thread->lbr_stitch->free_lists);
	2532	+
	2533	+ return true;
	2534	+
	2535	+free_lbr_stitch:
	2536	+ zfree(&thread->lbr_stitch);
	2537	+err:
	2538	+ pr_warning("Failed to allocate space for stitched LBRs. Disable LBR stitch\n");
	2539	+ thread->lbr_stitch_enable = false;
	2540	+ return false;
	2541	+}
	2542	+
2035	2543	/*
2036	2544	* Recolve LBR callstack chain sample
2037	2545	* Return:
..	..	@@ -2044,12 +2552,16 @@
2044	2552	struct perf_sample *sample,
2045	2553	struct symbol **parent,
2046	2554	struct addr_location *root_al,
2047		- int max_stack)
	2555	+ int max_stack,
	2556	+ unsigned int max_lbr)
2048	2557	{
	2558	+ bool callee = (callchain_param.order == ORDER_CALLEE);
2049	2559	struct ip_callchain *chain = sample->callchain;
2050	2560	int chain_nr = min(max_stack, (int)chain->nr), i;
2051		- u8 cpumode = PERF_RECORD_MISC_USER;
2052		- u64 ip, branch_from = 0;
	2561	+ struct lbr_stitch *lbr_stitch;
	2562	+ bool stitched_lbr = false;
	2563	+ u64 branch_from = 0;
	2564	+ int err;
2053	2565
2054	2566	for (i = 0; i < chain_nr; i++) {
2055	2567	if (chain->ips[i] == PERF_CONTEXT_USER)
..	..	@@ -2057,72 +2569,65 @@
2057	2569	}
2058	2570
2059	2571	/* LBR only affects the user callchain */
2060		- if (i != chain_nr) {
2061		- struct branch_stack *lbr_stack = sample->branch_stack;
2062		- int lbr_nr = lbr_stack->nr, j, k;
2063		- bool branch;
2064		- struct branch_flags *flags;
2065		- /*
2066		- * LBR callstack can only get user call chain.
2067		- * The mix_chain_nr is kernel call chain
2068		- * number plus LBR user call chain number.
2069		- * i is kernel call chain number,
2070		- * 1 is PERF_CONTEXT_USER,
2071		- * lbr_nr + 1 is the user call chain number.
2072		- * For details, please refer to the comments
2073		- * in callchain__printf
2074		- */
2075		- int mix_chain_nr = i + 1 + lbr_nr + 1;
	2572	+ if (i == chain_nr)
	2573	+ return 0;
2076	2574
2077		- for (j = 0; j < mix_chain_nr; j++) {
2078		- int err;
2079		- branch = false;
2080		- flags = NULL;
	2575	+ if (thread->lbr_stitch_enable && !sample->no_hw_idx &&
	2576	+ (max_lbr > 0) && alloc_lbr_stitch(thread, max_lbr)) {
	2577	+ lbr_stitch = thread->lbr_stitch;
2081	2578
2082		- if (callchain_param.order == ORDER_CALLEE) {
2083		- if (j < i + 1)
2084		- ip = chain->ips[j];
2085		- else if (j > i + 1) {
2086		- k = j - i - 2;
2087		- ip = lbr_stack->entries[k].from;
2088		- branch = true;
2089		- flags = &lbr_stack->entries[k].flags;
2090		- } else {
2091		- ip = lbr_stack->entries[0].to;
2092		- branch = true;
2093		- flags = &lbr_stack->entries[0].flags;
2094		- branch_from =
2095		- lbr_stack->entries[0].from;
2096		- }
2097		- } else {
2098		- if (j < lbr_nr) {
2099		- k = lbr_nr - j - 1;
2100		- ip = lbr_stack->entries[k].from;
2101		- branch = true;
2102		- flags = &lbr_stack->entries[k].flags;
2103		- }
2104		- else if (j > lbr_nr)
2105		- ip = chain->ips[i + 1 - (j - lbr_nr)];
2106		- else {
2107		- ip = lbr_stack->entries[0].to;
2108		- branch = true;
2109		- flags = &lbr_stack->entries[0].flags;
2110		- branch_from =
2111		- lbr_stack->entries[0].from;
2112		- }
2113		- }
	2579	+ stitched_lbr = has_stitched_lbr(thread, sample,
	2580	+ &lbr_stitch->prev_sample,
	2581	+ max_lbr, callee);
2114	2582
2115		- err = add_callchain_ip(thread, cursor, parent,
2116		- root_al, &cpumode, ip,
2117		- branch, flags, NULL,
2118		- branch_from);
2119		- if (err)
2120		- return (err < 0) ? err : 0;
	2583	+ if (!stitched_lbr && !list_empty(&lbr_stitch->lists)) {
	2584	+ list_replace_init(&lbr_stitch->lists,
	2585	+ &lbr_stitch->free_lists);
2121	2586	}
2122		- return 1;
	2587	+ memcpy(&lbr_stitch->prev_sample, sample, sizeof(*sample));
2123	2588	}
2124	2589
2125		- return 0;
	2590	+ if (callee) {
	2591	+ /* Add kernel ip */
	2592	+ err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
	2593	+ parent, root_al, branch_from,
	2594	+ true, i);
	2595	+ if (err)
	2596	+ goto error;
	2597	+
	2598	+ err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
	2599	+ root_al, &branch_from, true);
	2600	+ if (err)
	2601	+ goto error;
	2602	+
	2603	+ if (stitched_lbr) {
	2604	+ err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
	2605	+ if (err)
	2606	+ goto error;
	2607	+ }
	2608	+
	2609	+ } else {
	2610	+ if (stitched_lbr) {
	2611	+ err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
	2612	+ if (err)
	2613	+ goto error;
	2614	+ }
	2615	+ err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
	2616	+ root_al, &branch_from, false);
	2617	+ if (err)
	2618	+ goto error;
	2619	+
	2620	+ /* Add kernel ip */
	2621	+ err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
	2622	+ parent, root_al, branch_from,
	2623	+ false, i);
	2624	+ if (err)
	2625	+ goto error;
	2626	+ }
	2627	+ return 1;
	2628	+
	2629	+error:
	2630	+ return (err < 0) ? err : 0;
2126	2631	}
2127	2632
2128	2633	static int find_prev_cpumode(struct ip_callchain chain, struct thread thread,
..	..	@@ -2148,13 +2653,14 @@
2148	2653
2149	2654	static int thread__resolve_callchain_sample(struct thread *thread,
2150	2655	struct callchain_cursor *cursor,
2151		- struct perf_evsel *evsel,
	2656	+ struct evsel *evsel,
2152	2657	struct perf_sample *sample,
2153	2658	struct symbol **parent,
2154	2659	struct addr_location *root_al,
2155	2660	int max_stack)
2156	2661	{
2157	2662	struct branch_stack *branch = sample->branch_stack;
	2663	+ struct branch_entry *entries = perf_sample__branch_entries(sample);
2158	2664	struct ip_callchain *chain = sample->callchain;
2159	2665	int chain_nr = 0;
2160	2666	u8 cpumode = PERF_RECORD_MISC_USER;
..	..	@@ -2165,9 +2671,12 @@
2165	2671	if (chain)
2166	2672	chain_nr = chain->nr;
2167	2673
2168		- if (perf_evsel__has_branch_callstack(evsel)) {
	2674	+ if (evsel__has_branch_callstack(evsel)) {
	2675	+ struct perf_env *env = evsel__env(evsel);
	2676	+
2169	2677	err = resolve_lbr_callchain_sample(thread, cursor, sample, parent,
2170		- root_al, max_stack);
	2678	+ root_al, max_stack,
	2679	+ !env ? 0 : env->max_branches);
2171	2680	if (err)
2172	2681	return (err < 0) ? err : 0;
2173	2682	}
..	..	@@ -2202,7 +2711,7 @@
2202	2711
2203	2712	for (i = 0; i < nr; i++) {
2204	2713	if (callchain_param.order == ORDER_CALLEE) {
2205		- be[i] = branch->entries[i];
	2714	+ be[i] = entries[i];
2206	2715
2207	2716	if (chain == NULL)
2208	2717	continue;
..	..	@@ -2221,7 +2730,7 @@
2221	2730	be[i].from >= chain->ips[first_call] - 8)
2222	2731	first_call++;
2223	2732	} else
2224		- be[i] = branch->entries[branch->nr - i - 1];
	2733	+ be[i] = entries[branch->nr - i - 1];
2225	2734	}
2226	2735
2227	2736	memset(iter, 0, sizeof(struct iterations) * nr);
..	..	@@ -2293,9 +2802,10 @@
2293	2802	return 0;
2294	2803	}
2295	2804
2296		-static int append_inlines(struct callchain_cursor *cursor,
2297		- struct map map, struct symbol sym, u64 ip)
	2805	+static int append_inlines(struct callchain_cursor cursor, struct map_symbol ms, u64 ip)
2298	2806	{
	2807	+ struct symbol *sym = ms->sym;
	2808	+ struct map *map = ms->map;
2299	2809	struct inline_node *inline_node;
2300	2810	struct inline_list *ilist;
2301	2811	u64 addr;
..	..	@@ -2316,8 +2826,12 @@
2316	2826	}
2317	2827
2318	2828	list_for_each_entry(ilist, &inline_node->val, list) {
2319		- ret = callchain_cursor_append(cursor, ip, map,
2320		- ilist->symbol, false,
	2829	+ struct map_symbol ilist_ms = {
	2830	+ .maps = ms->maps,
	2831	+ .map = map,
	2832	+ .sym = ilist->symbol,
	2833	+ };
	2834	+ ret = callchain_cursor_append(cursor, ip, &ilist_ms, false,
2321	2835	NULL, 0, 0, 0, ilist->srcline);
2322	2836
2323	2837	if (ret != 0)
..	..	@@ -2333,34 +2847,33 @@
2333	2847	const char *srcline = NULL;
2334	2848	u64 addr = entry->ip;
2335	2849
2336		- if (symbol_conf.hide_unresolved && entry->sym == NULL)
	2850	+ if (symbol_conf.hide_unresolved && entry->ms.sym == NULL)
2337	2851	return 0;
2338	2852
2339		- if (append_inlines(cursor, entry->map, entry->sym, entry->ip) == 0)
	2853	+ if (append_inlines(cursor, &entry->ms, entry->ip) == 0)
2340	2854	return 0;
2341	2855
2342	2856	/*
2343	2857	* Convert entry->ip from a virtual address to an offset in
2344	2858	* its corresponding binary.
2345	2859	*/
2346		- if (entry->map)
2347		- addr = map__map_ip(entry->map, entry->ip);
	2860	+ if (entry->ms.map)
	2861	+ addr = map__map_ip(entry->ms.map, entry->ip);
2348	2862
2349		- srcline = callchain_srcline(entry->map, entry->sym, addr);
2350		- return callchain_cursor_append(cursor, entry->ip,
2351		- entry->map, entry->sym,
	2863	+ srcline = callchain_srcline(&entry->ms, addr);
	2864	+ return callchain_cursor_append(cursor, entry->ip, &entry->ms,
2352	2865	false, NULL, 0, 0, 0, srcline);
2353	2866	}
2354	2867
2355	2868	static int thread__resolve_callchain_unwind(struct thread *thread,
2356	2869	struct callchain_cursor *cursor,
2357		- struct perf_evsel *evsel,
	2870	+ struct evsel *evsel,
2358	2871	struct perf_sample *sample,
2359	2872	int max_stack)
2360	2873	{
2361	2874	/* Can we do dwarf post unwind? */
2362		- if (!((evsel->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
2363		- (evsel->attr.sample_type & PERF_SAMPLE_STACK_USER)))
	2875	+ if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) &&
	2876	+ (evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER)))
2364	2877	return 0;
2365	2878
2366	2879	/* Bail out if nothing was captured. */
..	..	@@ -2374,7 +2887,7 @@
2374	2887
2375	2888	int thread__resolve_callchain(struct thread *thread,
2376	2889	struct callchain_cursor *cursor,
2377		- struct perf_evsel *evsel,
	2890	+ struct evsel *evsel,
2378	2891	struct perf_sample *sample,
2379	2892	struct symbol **parent,
2380	2893	struct addr_location *root_al,
..	..	@@ -2421,7 +2934,8 @@
2421	2934
2422	2935	for (i = 0; i < THREADS__TABLE_SIZE; i++) {
2423	2936	threads = &machine->threads[i];
2424		- for (nd = rb_first(&threads->entries); nd; nd = rb_next(nd)) {
	2937	+ for (nd = rb_first_cached(&threads->entries); nd;
	2938	+ nd = rb_next(nd)) {
2425	2939	thread = rb_entry(nd, struct thread, rb_node);
2426	2940	rc = fn(thread, priv);
2427	2941	if (rc != 0)
..	..	@@ -2448,7 +2962,7 @@
2448	2962	if (rc != 0)
2449	2963	return rc;
2450	2964
2451		- for (nd = rb_first(&machines->guests); nd; nd = rb_next(nd)) {
	2965	+ for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
2452	2966	struct machine *machine = rb_entry(nd, struct machine, rb_node);
2453	2967
2454	2968	rc = machine__for_each_thread(machine, fn, priv);
..	..	@@ -2458,26 +2972,11 @@
2458	2972	return rc;
2459	2973	}
2460	2974
2461		-int __machine__synthesize_threads(struct machine machine, struct perf_tool tool,
2462		- struct target target, struct thread_map threads,
2463		- perf_event__handler_t process, bool data_mmap,
2464		- unsigned int proc_map_timeout,
2465		- unsigned int nr_threads_synthesize)
2466		-{
2467		- if (target__has_task(target))
2468		- return perf_event__synthesize_thread_map(tool, threads, process, machine, data_mmap, proc_map_timeout);
2469		- else if (target__has_cpu(target))
2470		- return perf_event__synthesize_threads(tool, process,
2471		- machine, data_mmap,
2472		- proc_map_timeout,
2473		- nr_threads_synthesize);
2474		- /* command specified */
2475		- return 0;
2476		-}
2477		-
2478	2975	pid_t machine__get_current_tid(struct machine *machine, int cpu)
2479	2976	{
2480		- if (cpu < 0 \|\| cpu >= MAX_NR_CPUS \|\| !machine->current_tid)
	2977	+ int nr_cpus = min(machine->env->nr_cpus_avail, MAX_NR_CPUS);
	2978	+
	2979	+ if (cpu < 0 \|\| cpu >= nr_cpus \|\| !machine->current_tid)
2481	2980	return -1;
2482	2981
2483	2982	return machine->current_tid[cpu];
..	..	@@ -2487,6 +2986,7 @@
2487	2986	pid_t tid)
2488	2987	{
2489	2988	struct thread *thread;
	2989	+ int nr_cpus = min(machine->env->nr_cpus_avail, MAX_NR_CPUS);
2490	2990
2491	2991	if (cpu < 0)
2492	2992	return -EINVAL;
..	..	@@ -2494,14 +2994,14 @@
2494	2994	if (!machine->current_tid) {
2495	2995	int i;
2496	2996
2497		- machine->current_tid = calloc(MAX_NR_CPUS, sizeof(pid_t));
	2997	+ machine->current_tid = calloc(nr_cpus, sizeof(pid_t));
2498	2998	if (!machine->current_tid)
2499	2999	return -ENOMEM;
2500		- for (i = 0; i < MAX_NR_CPUS; i++)
	3000	+ for (i = 0; i < nr_cpus; i++)
2501	3001	machine->current_tid[i] = -1;
2502	3002	}
2503	3003
2504		- if (cpu >= MAX_NR_CPUS) {
	3004	+ if (cpu >= nr_cpus) {
2505	3005	pr_err("Requested CPU %d too large. ", cpu);
2506	3006	pr_err("Consider raising MAX_NR_CPUS\n");
2507	3007	return -EINVAL;
..	..	@@ -2587,9 +3087,14 @@
2587	3087	return addr_cpumode;
2588	3088	}
2589	3089
	3090	+struct dso machine__findnew_dso_id(struct machine machine, const char filename, struct dso_id id)
	3091	+{
	3092	+ return dsos__findnew_id(&machine->dsos, filename, id);
	3093	+}
	3094	+
2590	3095	struct dso machine__findnew_dso(struct machine machine, const char *filename)
2591	3096	{
2592		- return dsos__findnew(&machine->dsos, filename);
	3097	+ return machine__findnew_dso_id(machine, filename, NULL);
2593	3098	}
2594	3099
2595	3100	char machine__resolve_kernel_addr(void vmachine, unsigned long long addrp, char *modp)
..	..	@@ -2605,3 +3110,15 @@
2605	3110	*addrp = map->unmap_ip(map, sym->start);
2606	3111	return sym->name;
2607	3112	}
	3113	+
	3114	+int machine__for_each_dso(struct machine machine, machine__dso_t fn, void priv)
	3115	+{
	3116	+ struct dso *pos;
	3117	+ int err = 0;
	3118	+
	3119	+ list_for_each_entry(pos, &machine->dsos.head, node) {
	3120	+ if (fn(pos, machine, priv))
	3121	+ err = -1;
	3122	+ }
	3123	+ return err;
	3124	+}