~hc/RK356X_SDK_RELEASE.git

..	..	@@ -11,7 +11,9 @@
11	11	#include <linux/kernel.h>
12	12	#include <linux/log2.h>
13	13	#include <linux/types.h>
	14	+#include <linux/zalloc.h>
14	15
	16	+#include <opencsd/ocsd_if_types.h>
15	17	#include <stdlib.h>
16	18
17	19	#include "auxtrace.h"
..	..	@@ -19,25 +21,23 @@
19	21	#include "cs-etm.h"
20	22	#include "cs-etm-decoder/cs-etm-decoder.h"
21	23	#include "debug.h"
	24	+#include "dso.h"
22	25	#include "evlist.h"
23	26	#include "intlist.h"
24	27	#include "machine.h"
25	28	#include "map.h"
26	29	#include "perf.h"
	30	+#include "session.h"
	31	+#include "map_symbol.h"
	32	+#include "branch.h"
	33	+#include "symbol.h"
	34	+#include "tool.h"
27	35	#include "thread.h"
28		-#include "thread_map.h"
29	36	#include "thread-stack.h"
30		-#include "util.h"
	37	+#include <tools/libc_compat.h>
	38	+#include "util/synthetic-events.h"
31	39
32	40	#define MAX_TIMESTAMP (~0ULL)
33		-
34		-/*
35		- * A64 instructions are always 4 bytes
36		- *
37		- * Only A64 is supported, so can use this constant for converting between
38		- * addresses and instruction counts, calculting offsets etc
39		- */
40		-#define A64_INSTR_SIZE 4
41	41
42	42	struct cs_etm_auxtrace {
43	43	struct auxtrace auxtrace;
..	..	@@ -66,33 +66,81 @@
66	66	unsigned int pmu_type;
67	67	};
68	68
69		-struct cs_etm_queue {
70		- struct cs_etm_auxtrace *etm;
71		- struct thread *thread;
72		- struct cs_etm_decoder *decoder;
73		- struct auxtrace_buffer *buffer;
74		- const struct cs_etm_state *state;
75		- union perf_event *event_buf;
76		- unsigned int queue_nr;
	69	+struct cs_etm_traceid_queue {
	70	+ u8 trace_chan_id;
77	71	pid_t pid, tid;
78		- int cpu;
79		- u64 time;
80		- u64 timestamp;
81		- u64 offset;
82	72	u64 period_instructions;
	73	+ size_t last_branch_pos;
	74	+ union perf_event *event_buf;
	75	+ struct thread *thread;
83	76	struct branch_stack *last_branch;
84	77	struct branch_stack *last_branch_rb;
85		- size_t last_branch_pos;
86	78	struct cs_etm_packet *prev_packet;
87	79	struct cs_etm_packet *packet;
	80	+ struct cs_etm_packet_queue packet_queue;
	81	+};
	82	+
	83	+struct cs_etm_queue {
	84	+ struct cs_etm_auxtrace *etm;
	85	+ struct cs_etm_decoder *decoder;
	86	+ struct auxtrace_buffer *buffer;
	87	+ unsigned int queue_nr;
	88	+ u8 pending_timestamp;
	89	+ u64 offset;
	90	+ const unsigned char *buf;
	91	+ size_t buf_len, buf_used;
	92	+ /* Conversion between traceID and index in traceid_queues array */
	93	+ struct intlist *traceid_queues_list;
	94	+ struct cs_etm_traceid_queue **traceid_queues;
88	95	};
89	96
90	97	/* RB tree for quick conversion between traceID and metadata pointers */
91	98	static struct intlist *traceid_list;
92	99
93	100	static int cs_etm__update_queues(struct cs_etm_auxtrace *etm);
	101	+static int cs_etm__process_queues(struct cs_etm_auxtrace *etm);
94	102	static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
95		- pid_t tid, u64 time_);
	103	+ pid_t tid);
	104	+static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
	105	+static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
	106	+
	107	+/* PTMs ETMIDR [11:8] set to b0011 */
	108	+#define ETMIDR_PTM_VERSION 0x00000300
	109	+
	110	+/*
	111	+ * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
	112	+ * work with. One option is to modify to auxtrace_heap_XYZ() API or simply
	113	+ * encode the etm queue number as the upper 16 bit and the channel as
	114	+ * the lower 16 bit.
	115	+ */
	116	+#define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \
	117	+ (queue_nr << 16 \| trace_chan_id)
	118	+#define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
	119	+#define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
	120	+
	121	+static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
	122	+{
	123	+ etmidr &= ETMIDR_PTM_VERSION;
	124	+
	125	+ if (etmidr == ETMIDR_PTM_VERSION)
	126	+ return CS_ETM_PROTO_PTM;
	127	+
	128	+ return CS_ETM_PROTO_ETMV3;
	129	+}
	130	+
	131	+static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
	132	+{
	133	+ struct int_node *inode;
	134	+ u64 *metadata;
	135	+
	136	+ inode = intlist__find(traceid_list, trace_chan_id);
	137	+ if (!inode)
	138	+ return -EINVAL;
	139	+
	140	+ metadata = inode->priv;
	141	+ *magic = metadata[CS_ETM_MAGIC];
	142	+ return 0;
	143	+}
96	144
97	145	int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
98	146	{
..	..	@@ -108,6 +156,233 @@
108	156	return 0;
109	157	}
110	158
	159	+void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
	160	+ u8 trace_chan_id)
	161	+{
	162	+ /*
	163	+ * Wnen a timestamp packet is encountered the backend code
	164	+ * is stopped so that the front end has time to process packets
	165	+ * that were accumulated in the traceID queue. Since there can
	166	+ * be more than one channel per cs_etm_queue, we need to specify
	167	+ * what traceID queue needs servicing.
	168	+ */
	169	+ etmq->pending_timestamp = trace_chan_id;
	170	+}
	171	+
	172	+static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
	173	+ u8 *trace_chan_id)
	174	+{
	175	+ struct cs_etm_packet_queue *packet_queue;
	176	+
	177	+ if (!etmq->pending_timestamp)
	178	+ return 0;
	179	+
	180	+ if (trace_chan_id)
	181	+ *trace_chan_id = etmq->pending_timestamp;
	182	+
	183	+ packet_queue = cs_etm__etmq_get_packet_queue(etmq,
	184	+ etmq->pending_timestamp);
	185	+ if (!packet_queue)
	186	+ return 0;
	187	+
	188	+ /* Acknowledge pending status */
	189	+ etmq->pending_timestamp = 0;
	190	+
	191	+ /* See function cs_etm_decoder__do_{hard\|soft}_timestamp() */
	192	+ return packet_queue->timestamp;
	193	+}
	194	+
	195	+static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
	196	+{
	197	+ int i;
	198	+
	199	+ queue->head = 0;
	200	+ queue->tail = 0;
	201	+ queue->packet_count = 0;
	202	+ for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
	203	+ queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
	204	+ queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
	205	+ queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
	206	+ queue->packet_buffer[i].instr_count = 0;
	207	+ queue->packet_buffer[i].last_instr_taken_branch = false;
	208	+ queue->packet_buffer[i].last_instr_size = 0;
	209	+ queue->packet_buffer[i].last_instr_type = 0;
	210	+ queue->packet_buffer[i].last_instr_subtype = 0;
	211	+ queue->packet_buffer[i].last_instr_cond = 0;
	212	+ queue->packet_buffer[i].flags = 0;
	213	+ queue->packet_buffer[i].exception_number = UINT32_MAX;
	214	+ queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
	215	+ queue->packet_buffer[i].cpu = INT_MIN;
	216	+ }
	217	+}
	218	+
	219	+static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
	220	+{
	221	+ int idx;
	222	+ struct int_node *inode;
	223	+ struct cs_etm_traceid_queue *tidq;
	224	+ struct intlist *traceid_queues_list = etmq->traceid_queues_list;
	225	+
	226	+ intlist__for_each_entry(inode, traceid_queues_list) {
	227	+ idx = (int)(intptr_t)inode->priv;
	228	+ tidq = etmq->traceid_queues[idx];
	229	+ cs_etm__clear_packet_queue(&tidq->packet_queue);
	230	+ }
	231	+}
	232	+
	233	+static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
	234	+ struct cs_etm_traceid_queue *tidq,
	235	+ u8 trace_chan_id)
	236	+{
	237	+ int rc = -ENOMEM;
	238	+ struct auxtrace_queue *queue;
	239	+ struct cs_etm_auxtrace *etm = etmq->etm;
	240	+
	241	+ cs_etm__clear_packet_queue(&tidq->packet_queue);
	242	+
	243	+ queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
	244	+ tidq->tid = queue->tid;
	245	+ tidq->pid = -1;
	246	+ tidq->trace_chan_id = trace_chan_id;
	247	+
	248	+ tidq->packet = zalloc(sizeof(struct cs_etm_packet));
	249	+ if (!tidq->packet)
	250	+ goto out;
	251	+
	252	+ tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
	253	+ if (!tidq->prev_packet)
	254	+ goto out_free;
	255	+
	256	+ if (etm->synth_opts.last_branch) {
	257	+ size_t sz = sizeof(struct branch_stack);
	258	+
	259	+ sz += etm->synth_opts.last_branch_sz *
	260	+ sizeof(struct branch_entry);
	261	+ tidq->last_branch = zalloc(sz);
	262	+ if (!tidq->last_branch)
	263	+ goto out_free;
	264	+ tidq->last_branch_rb = zalloc(sz);
	265	+ if (!tidq->last_branch_rb)
	266	+ goto out_free;
	267	+ }
	268	+
	269	+ tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
	270	+ if (!tidq->event_buf)
	271	+ goto out_free;
	272	+
	273	+ return 0;
	274	+
	275	+out_free:
	276	+ zfree(&tidq->last_branch_rb);
	277	+ zfree(&tidq->last_branch);
	278	+ zfree(&tidq->prev_packet);
	279	+ zfree(&tidq->packet);
	280	+out:
	281	+ return rc;
	282	+}
	283	+
	284	+static struct cs_etm_traceid_queue
	285	+cs_etm__etmq_get_traceid_queue(struct cs_etm_queue etmq, u8 trace_chan_id)
	286	+{
	287	+ int idx;
	288	+ struct int_node *inode;
	289	+ struct intlist *traceid_queues_list;
	290	+ struct cs_etm_traceid_queue tidq, *traceid_queues;
	291	+ struct cs_etm_auxtrace *etm = etmq->etm;
	292	+
	293	+ if (etm->timeless_decoding)
	294	+ trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
	295	+
	296	+ traceid_queues_list = etmq->traceid_queues_list;
	297	+
	298	+ /*
	299	+ * Check if the traceid_queue exist for this traceID by looking
	300	+ * in the queue list.
	301	+ */
	302	+ inode = intlist__find(traceid_queues_list, trace_chan_id);
	303	+ if (inode) {
	304	+ idx = (int)(intptr_t)inode->priv;
	305	+ return etmq->traceid_queues[idx];
	306	+ }
	307	+
	308	+ /* We couldn't find a traceid_queue for this traceID, allocate one */
	309	+ tidq = malloc(sizeof(*tidq));
	310	+ if (!tidq)
	311	+ return NULL;
	312	+
	313	+ memset(tidq, 0, sizeof(*tidq));
	314	+
	315	+ /* Get a valid index for the new traceid_queue */
	316	+ idx = intlist__nr_entries(traceid_queues_list);
	317	+ /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
	318	+ inode = intlist__findnew(traceid_queues_list, trace_chan_id);
	319	+ if (!inode)
	320	+ goto out_free;
	321	+
	322	+ /* Associate this traceID with this index */
	323	+ inode->priv = (void *)(intptr_t)idx;
	324	+
	325	+ if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
	326	+ goto out_free;
	327	+
	328	+ /* Grow the traceid_queues array by one unit */
	329	+ traceid_queues = etmq->traceid_queues;
	330	+ traceid_queues = reallocarray(traceid_queues,
	331	+ idx + 1,
	332	+ sizeof(*traceid_queues));
	333	+
	334	+ /*
	335	+ * On failure reallocarray() returns NULL and the original block of
	336	+ * memory is left untouched.
	337	+ */
	338	+ if (!traceid_queues)
	339	+ goto out_free;
	340	+
	341	+ traceid_queues[idx] = tidq;
	342	+ etmq->traceid_queues = traceid_queues;
	343	+
	344	+ return etmq->traceid_queues[idx];
	345	+
	346	+out_free:
	347	+ /*
	348	+ * Function intlist__remove() removes the inode from the list
	349	+ * and delete the memory associated to it.
	350	+ */
	351	+ intlist__remove(traceid_queues_list, inode);
	352	+ free(tidq);
	353	+
	354	+ return NULL;
	355	+}
	356	+
	357	+struct cs_etm_packet_queue
	358	+cs_etm__etmq_get_packet_queue(struct cs_etm_queue etmq, u8 trace_chan_id)
	359	+{
	360	+ struct cs_etm_traceid_queue *tidq;
	361	+
	362	+ tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
	363	+ if (tidq)
	364	+ return &tidq->packet_queue;
	365	+
	366	+ return NULL;
	367	+}
	368	+
	369	+static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm,
	370	+ struct cs_etm_traceid_queue *tidq)
	371	+{
	372	+ struct cs_etm_packet *tmp;
	373	+
	374	+ if (etm->sample_branches \|\| etm->synth_opts.last_branch \|\|
	375	+ etm->sample_instructions) {
	376	+ /*
	377	+ * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
	378	+ * the next incoming packet.
	379	+ */
	380	+ tmp = tidq->packet;
	381	+ tidq->packet = tidq->prev_packet;
	382	+ tidq->prev_packet = tmp;
	383	+ }
	384	+}
	385	+
111	386	static void cs_etm__packet_dump(const char *pkt_string)
112	387	{
113	388	const char *color = PERF_COLOR_BLUE;
..	..	@@ -121,10 +396,83 @@
121	396	fflush(stdout);
122	397	}
123	398
	399	+static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
	400	+ struct cs_etm_auxtrace *etm, int idx,
	401	+ u32 etmidr)
	402	+{
	403	+ u64 **metadata = etm->metadata;
	404	+
	405	+ t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
	406	+ t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
	407	+ t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
	408	+}
	409	+
	410	+static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
	411	+ struct cs_etm_auxtrace *etm, int idx)
	412	+{
	413	+ u64 **metadata = etm->metadata;
	414	+
	415	+ t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
	416	+ t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
	417	+ t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
	418	+ t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
	419	+ t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
	420	+ t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
	421	+ t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
	422	+}
	423	+
	424	+static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
	425	+ struct cs_etm_auxtrace *etm)
	426	+{
	427	+ int i;
	428	+ u32 etmidr;
	429	+ u64 architecture;
	430	+
	431	+ for (i = 0; i < etm->num_cpu; i++) {
	432	+ architecture = etm->metadata[i][CS_ETM_MAGIC];
	433	+
	434	+ switch (architecture) {
	435	+ case __perf_cs_etmv3_magic:
	436	+ etmidr = etm->metadata[i][CS_ETM_ETMIDR];
	437	+ cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
	438	+ break;
	439	+ case __perf_cs_etmv4_magic:
	440	+ cs_etm__set_trace_param_etmv4(t_params, etm, i);
	441	+ break;
	442	+ default:
	443	+ return -EINVAL;
	444	+ }
	445	+ }
	446	+
	447	+ return 0;
	448	+}
	449	+
	450	+static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
	451	+ struct cs_etm_queue *etmq,
	452	+ enum cs_etm_decoder_operation mode)
	453	+{
	454	+ int ret = -EINVAL;
	455	+
	456	+ if (!(mode < CS_ETM_OPERATION_MAX))
	457	+ goto out;
	458	+
	459	+ d_params->packet_printer = cs_etm__packet_dump;
	460	+ d_params->operation = mode;
	461	+ d_params->data = etmq;
	462	+ d_params->formatted = true;
	463	+ d_params->fsyncs = false;
	464	+ d_params->hsyncs = false;
	465	+ d_params->frame_aligned = true;
	466	+
	467	+ ret = 0;
	468	+out:
	469	+ return ret;
	470	+}
	471	+
124	472	static void cs_etm__dump_event(struct cs_etm_auxtrace *etm,
125	473	struct auxtrace_buffer *buffer)
126	474	{
127		- int i, ret;
	475	+ int ret;
128	476	const char *color = PERF_COLOR_BLUE;
129	477	struct cs_etm_decoder_params d_params;
130	478	struct cs_etm_trace_params *t_params;
..	..	@@ -138,32 +486,22 @@
138	486
139	487	/* Use metadata to fill in trace parameters for trace decoder */
140	488	t_params = zalloc(sizeof(t_params) etm->num_cpu);
141		- for (i = 0; i < etm->num_cpu; i++) {
142		- t_params[i].protocol = CS_ETM_PROTO_ETMV4i;
143		- t_params[i].etmv4.reg_idr0 = etm->metadata[i][CS_ETMV4_TRCIDR0];
144		- t_params[i].etmv4.reg_idr1 = etm->metadata[i][CS_ETMV4_TRCIDR1];
145		- t_params[i].etmv4.reg_idr2 = etm->metadata[i][CS_ETMV4_TRCIDR2];
146		- t_params[i].etmv4.reg_idr8 = etm->metadata[i][CS_ETMV4_TRCIDR8];
147		- t_params[i].etmv4.reg_configr =
148		- etm->metadata[i][CS_ETMV4_TRCCONFIGR];
149		- t_params[i].etmv4.reg_traceidr =
150		- etm->metadata[i][CS_ETMV4_TRCTRACEIDR];
151		- }
	489	+
	490	+ if (!t_params)
	491	+ return;
	492	+
	493	+ if (cs_etm__init_trace_params(t_params, etm))
	494	+ goto out_free;
152	495
153	496	/* Set decoder parameters to simply print the trace packets */
154		- d_params.packet_printer = cs_etm__packet_dump;
155		- d_params.operation = CS_ETM_OPERATION_PRINT;
156		- d_params.formatted = true;
157		- d_params.fsyncs = false;
158		- d_params.hsyncs = false;
159		- d_params.frame_aligned = true;
	497	+ if (cs_etm__init_decoder_params(&d_params, NULL,
	498	+ CS_ETM_OPERATION_PRINT))
	499	+ goto out_free;
160	500
161	501	decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
162	502
163		- zfree(&t_params);
164		-
165	503	if (!decoder)
166		- return;
	504	+ goto out_free;
167	505	do {
168	506	size_t consumed;
169	507
..	..	@@ -178,6 +516,9 @@
178	516	} while (buffer_used < buffer->size);
179	517
180	518	cs_etm_decoder__free(decoder);
	519	+
	520	+out_free:
	521	+ zfree(&t_params);
181	522	}
182	523
183	524	static int cs_etm__flush_events(struct perf_session *session,
..	..	@@ -193,15 +534,52 @@
193	534	if (!tool->ordered_events)
194	535	return -EINVAL;
195	536
196		- if (!etm->timeless_decoding)
197		- return -EINVAL;
198		-
199	537	ret = cs_etm__update_queues(etm);
200	538
201	539	if (ret < 0)
202	540	return ret;
203	541
204		- return cs_etm__process_timeless_queues(etm, -1, MAX_TIMESTAMP - 1);
	542	+ if (etm->timeless_decoding)
	543	+ return cs_etm__process_timeless_queues(etm, -1);
	544	+
	545	+ return cs_etm__process_queues(etm);
	546	+}
	547	+
	548	+static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
	549	+{
	550	+ int idx;
	551	+ uintptr_t priv;
	552	+ struct int_node inode, tmp;
	553	+ struct cs_etm_traceid_queue *tidq;
	554	+ struct intlist *traceid_queues_list = etmq->traceid_queues_list;
	555	+
	556	+ intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
	557	+ priv = (uintptr_t)inode->priv;
	558	+ idx = priv;
	559	+
	560	+ /* Free this traceid_queue from the array */
	561	+ tidq = etmq->traceid_queues[idx];
	562	+ thread__zput(tidq->thread);
	563	+ zfree(&tidq->event_buf);
	564	+ zfree(&tidq->last_branch);
	565	+ zfree(&tidq->last_branch_rb);
	566	+ zfree(&tidq->prev_packet);
	567	+ zfree(&tidq->packet);
	568	+ zfree(&tidq);
	569	+
	570	+ /*
	571	+ * Function intlist__remove() removes the inode from the list
	572	+ * and delete the memory associated to it.
	573	+ */
	574	+ intlist__remove(traceid_queues_list, inode);
	575	+ }
	576	+
	577	+ /* Then the RB tree itself */
	578	+ intlist__delete(traceid_queues_list);
	579	+ etmq->traceid_queues_list = NULL;
	580	+
	581	+ /* finally free the traceid_queues array */
	582	+ zfree(&etmq->traceid_queues);
205	583	}
206	584
207	585	static void cs_etm__free_queue(void *priv)
..	..	@@ -211,13 +589,8 @@
211	589	if (!etmq)
212	590	return;
213	591
214		- thread__zput(etmq->thread);
215	592	cs_etm_decoder__free(etmq->decoder);
216		- zfree(&etmq->event_buf);
217		- zfree(&etmq->last_branch);
218		- zfree(&etmq->last_branch_rb);
219		- zfree(&etmq->prev_packet);
220		- zfree(&etmq->packet);
	593	+ cs_etm__free_traceid_queues(etmq);
221	594	free(etmq);
222	595	}
223	596
..	..	@@ -261,6 +634,16 @@
261	634	zfree(&aux);
262	635	}
263	636
	637	+static bool cs_etm__evsel_is_auxtrace(struct perf_session *session,
	638	+ struct evsel *evsel)
	639	+{
	640	+ struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
	641	+ struct cs_etm_auxtrace,
	642	+ auxtrace);
	643	+
	644	+ return evsel->core.attr.type == aux->pmu_type;
	645	+}
	646	+
264	647	static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address)
265	648	{
266	649	struct machine *machine;
..	..	@@ -282,26 +665,30 @@
282	665	}
283	666	}
284	667
285		-static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u64 address,
286		- size_t size, u8 *buffer)
	668	+static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
	669	+ u64 address, size_t size, u8 *buffer)
287	670	{
288	671	u8 cpumode;
289	672	u64 offset;
290	673	int len;
291		- struct thread *thread;
292		- struct machine *machine;
293		- struct addr_location al;
	674	+ struct thread *thread;
	675	+ struct machine *machine;
	676	+ struct addr_location al;
	677	+ struct cs_etm_traceid_queue *tidq;
294	678
295	679	if (!etmq)
296		- return -1;
	680	+ return 0;
297	681
298	682	machine = etmq->etm->machine;
299	683	cpumode = cs_etm__cpu_mode(etmq, address);
	684	+ tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
	685	+ if (!tidq)
	686	+ return 0;
300	687
301		- thread = etmq->thread;
	688	+ thread = tidq->thread;
302	689	if (!thread) {
303	690	if (cpumode != PERF_RECORD_MISC_KERNEL)
304		- return -EINVAL;
	691	+ return 0;
305	692	thread = etmq->etm->unknown_thread;
306	693	}
307	694
..	..	@@ -324,51 +711,19 @@
324	711	return len;
325	712	}
326	713
327		-static struct cs_etm_queue cs_etm__alloc_queue(struct cs_etm_auxtrace etm,
328		- unsigned int queue_nr)
	714	+static struct cs_etm_queue cs_etm__alloc_queue(struct cs_etm_auxtrace etm)
329	715	{
330		- int i;
331	716	struct cs_etm_decoder_params d_params;
332		- struct cs_etm_trace_params *t_params;
	717	+ struct cs_etm_trace_params *t_params = NULL;
333	718	struct cs_etm_queue *etmq;
334		- size_t szp = sizeof(struct cs_etm_packet);
335	719
336	720	etmq = zalloc(sizeof(*etmq));
337	721	if (!etmq)
338	722	return NULL;
339	723
340		- etmq->packet = zalloc(szp);
341		- if (!etmq->packet)
	724	+ etmq->traceid_queues_list = intlist__new(NULL);
	725	+ if (!etmq->traceid_queues_list)
342	726	goto out_free;
343		-
344		- if (etm->synth_opts.last_branch \|\| etm->sample_branches) {
345		- etmq->prev_packet = zalloc(szp);
346		- if (!etmq->prev_packet)
347		- goto out_free;
348		- }
349		-
350		- if (etm->synth_opts.last_branch) {
351		- size_t sz = sizeof(struct branch_stack);
352		-
353		- sz += etm->synth_opts.last_branch_sz *
354		- sizeof(struct branch_entry);
355		- etmq->last_branch = zalloc(sz);
356		- if (!etmq->last_branch)
357		- goto out_free;
358		- etmq->last_branch_rb = zalloc(sz);
359		- if (!etmq->last_branch_rb)
360		- goto out_free;
361		- }
362		-
363		- etmq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
364		- if (!etmq->event_buf)
365		- goto out_free;
366		-
367		- etmq->etm = etm;
368		- etmq->queue_nr = queue_nr;
369		- etmq->pid = -1;
370		- etmq->tid = -1;
371		- etmq->cpu = -1;
372	727
373	728	/* Use metadata to fill in trace parameters for trace decoder */
374	729	t_params = zalloc(sizeof(t_params) etm->num_cpu);
..	..	@@ -376,30 +731,15 @@
376	731	if (!t_params)
377	732	goto out_free;
378	733
379		- for (i = 0; i < etm->num_cpu; i++) {
380		- t_params[i].protocol = CS_ETM_PROTO_ETMV4i;
381		- t_params[i].etmv4.reg_idr0 = etm->metadata[i][CS_ETMV4_TRCIDR0];
382		- t_params[i].etmv4.reg_idr1 = etm->metadata[i][CS_ETMV4_TRCIDR1];
383		- t_params[i].etmv4.reg_idr2 = etm->metadata[i][CS_ETMV4_TRCIDR2];
384		- t_params[i].etmv4.reg_idr8 = etm->metadata[i][CS_ETMV4_TRCIDR8];
385		- t_params[i].etmv4.reg_configr =
386		- etm->metadata[i][CS_ETMV4_TRCCONFIGR];
387		- t_params[i].etmv4.reg_traceidr =
388		- etm->metadata[i][CS_ETMV4_TRCTRACEIDR];
389		- }
	734	+ if (cs_etm__init_trace_params(t_params, etm))
	735	+ goto out_free;
390	736
391		- /* Set decoder parameters to simply print the trace packets */
392		- d_params.packet_printer = cs_etm__packet_dump;
393		- d_params.operation = CS_ETM_OPERATION_DECODE;
394		- d_params.formatted = true;
395		- d_params.fsyncs = false;
396		- d_params.hsyncs = false;
397		- d_params.frame_aligned = true;
398		- d_params.data = etmq;
	737	+ /* Set decoder parameters to decode trace packets */
	738	+ if (cs_etm__init_decoder_params(&d_params, etmq,
	739	+ CS_ETM_OPERATION_DECODE))
	740	+ goto out_free;
399	741
400	742	etmq->decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
401		-
402		- zfree(&t_params);
403	743
404	744	if (!etmq->decoder)
405	745	goto out_free;
..	..	@@ -413,19 +753,13 @@
413	753	cs_etm__mem_access))
414	754	goto out_free_decoder;
415	755
416		- etmq->offset = 0;
417		- etmq->period_instructions = 0;
418		-
	756	+ zfree(&t_params);
419	757	return etmq;
420	758
421	759	out_free_decoder:
422	760	cs_etm_decoder__free(etmq->decoder);
423	761	out_free:
424		- zfree(&etmq->event_buf);
425		- zfree(&etmq->last_branch);
426		- zfree(&etmq->last_branch_rb);
427		- zfree(&etmq->prev_packet);
428		- zfree(&etmq->packet);
	762	+ intlist__delete(etmq->traceid_queues_list);
429	763	free(etmq);
430	764
431	765	return NULL;
..	..	@@ -435,30 +769,99 @@
435	769	struct auxtrace_queue *queue,
436	770	unsigned int queue_nr)
437	771	{
	772	+ int ret = 0;
	773	+ unsigned int cs_queue_nr;
	774	+ u8 trace_chan_id;
	775	+ u64 timestamp;
438	776	struct cs_etm_queue *etmq = queue->priv;
439	777
440	778	if (list_empty(&queue->head) \|\| etmq)
441		- return 0;
	779	+ goto out;
442	780
443		- etmq = cs_etm__alloc_queue(etm, queue_nr);
	781	+ etmq = cs_etm__alloc_queue(etm);
444	782
445		- if (!etmq)
446		- return -ENOMEM;
	783	+ if (!etmq) {
	784	+ ret = -ENOMEM;
	785	+ goto out;
	786	+ }
447	787
448	788	queue->priv = etmq;
	789	+ etmq->etm = etm;
	790	+ etmq->queue_nr = queue_nr;
	791	+ etmq->offset = 0;
449	792
450		- if (queue->cpu != -1)
451		- etmq->cpu = queue->cpu;
	793	+ if (etm->timeless_decoding)
	794	+ goto out;
452	795
453		- etmq->tid = queue->tid;
	796	+ /*
	797	+ * We are under a CPU-wide trace scenario. As such we need to know
	798	+ * when the code that generated the traces started to execute so that
	799	+ * it can be correlated with execution on other CPUs. So we get a
	800	+ * handle on the beginning of traces and decode until we find a
	801	+ * timestamp. The timestamp is then added to the auxtrace min heap
	802	+ * in order to know what nibble (of all the etmqs) to decode first.
	803	+ */
	804	+ while (1) {
	805	+ /*
	806	+ * Fetch an aux_buffer from this etmq. Bail if no more
	807	+ * blocks or an error has been encountered.
	808	+ */
	809	+ ret = cs_etm__get_data_block(etmq);
	810	+ if (ret <= 0)
	811	+ goto out;
454	812
455		- return 0;
	813	+ /*
	814	+ * Run decoder on the trace block. The decoder will stop when
	815	+ * encountering a timestamp, a full packet queue or the end of
	816	+ * trace for that block.
	817	+ */
	818	+ ret = cs_etm__decode_data_block(etmq);
	819	+ if (ret)
	820	+ goto out;
	821	+
	822	+ /*
	823	+ * Function cs_etm_decoder__do_{hard\|soft}_timestamp() does all
	824	+ * the timestamp calculation for us.
	825	+ */
	826	+ timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
	827	+
	828	+ /* We found a timestamp, no need to continue. */
	829	+ if (timestamp)
	830	+ break;
	831	+
	832	+ /*
	833	+ * We didn't find a timestamp so empty all the traceid packet
	834	+ * queues before looking for another timestamp packet, either
	835	+ * in the current data block or a new one. Packets that were
	836	+ * just decoded are useless since no timestamp has been
	837	+ * associated with them. As such simply discard them.
	838	+ */
	839	+ cs_etm__clear_all_packet_queues(etmq);
	840	+ }
	841	+
	842	+ /*
	843	+ * We have a timestamp. Add it to the min heap to reflect when
	844	+ * instructions conveyed by the range packets of this traceID queue
	845	+ * started to execute. Once the same has been done for all the traceID
	846	+ * queues of each etmq, redenring and decoding can start in
	847	+ * chronological order.
	848	+ *
	849	+ * Note that packets decoded above are still in the traceID's packet
	850	+ * queue and will be processed in cs_etm__process_queues().
	851	+ */
	852	+ cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
	853	+ ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);
	854	+out:
	855	+ return ret;
456	856	}
457	857
458	858	static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm)
459	859	{
460	860	unsigned int i;
461	861	int ret;
	862	+
	863	+ if (!etm->kernel_start)
	864	+ etm->kernel_start = machine__kernel_start(etm->machine);
462	865
463	866	for (i = 0; i < etm->queues.nr_queues; i++) {
464	867	ret = cs_etm__setup_queue(etm, &etm->queues.queue_array[i], i);
..	..	@@ -479,10 +882,12 @@
479	882	return 0;
480	883	}
481	884
482		-static inline void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq)
	885	+static inline
	886	+void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
	887	+ struct cs_etm_traceid_queue *tidq)
483	888	{
484		- struct branch_stack *bs_src = etmq->last_branch_rb;
485		- struct branch_stack *bs_dst = etmq->last_branch;
	889	+ struct branch_stack *bs_src = tidq->last_branch_rb;
	890	+ struct branch_stack *bs_dst = tidq->last_branch;
486	891	size_t nr = 0;
487	892
488	893	/*
..	..	@@ -502,9 +907,9 @@
502	907	* two steps. First, copy the branches from the most recently inserted
503	908	* branch ->last_branch_pos until the end of bs_src->entries buffer.
504	909	*/
505		- nr = etmq->etm->synth_opts.last_branch_sz - etmq->last_branch_pos;
	910	+ nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
506	911	memcpy(&bs_dst->entries[0],
507		- &bs_src->entries[etmq->last_branch_pos],
	912	+ &bs_src->entries[tidq->last_branch_pos],
508	913	sizeof(struct branch_entry) * nr);
509	914
510	915	/*
..	..	@@ -517,68 +922,75 @@
517	922	if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
518	923	memcpy(&bs_dst->entries[nr],
519	924	&bs_src->entries[0],
520		- sizeof(struct branch_entry) * etmq->last_branch_pos);
	925	+ sizeof(struct branch_entry) * tidq->last_branch_pos);
521	926	}
522	927	}
523	928
524		-static inline void cs_etm__reset_last_branch_rb(struct cs_etm_queue *etmq)
	929	+static inline
	930	+void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
525	931	{
526		- etmq->last_branch_pos = 0;
527		- etmq->last_branch_rb->nr = 0;
	932	+ tidq->last_branch_pos = 0;
	933	+ tidq->last_branch_rb->nr = 0;
528	934	}
529	935
530		-static inline u64 cs_etm__last_executed_instr(struct cs_etm_packet *packet)
	936	+static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
	937	+ u8 trace_chan_id, u64 addr)
531	938	{
532		- /* Returns 0 for the CS_ETM_TRACE_ON packet */
533		- if (packet->sample_type == CS_ETM_TRACE_ON)
534		- return 0;
	939	+ u8 instrBytes[2];
535	940
	941	+ cs_etm__mem_access(etmq, trace_chan_id, addr,
	942	+ ARRAY_SIZE(instrBytes), instrBytes);
536	943	/*
537		- * The packet records the execution range with an exclusive end address
538		- *
539		- * A64 instructions are constant size, so the last executed
540		- * instruction is A64_INSTR_SIZE before the end address
541		- * Will need to do instruction level decode for T32 instructions as
542		- * they can be variable size (not yet supported).
	944	+ * T32 instruction size is indicated by bits[15:11] of the first
	945	+ * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
	946	+ * denote a 32-bit instruction.
543	947	*/
544		- return packet->end_addr - A64_INSTR_SIZE;
	948	+ return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
545	949	}
546	950
547	951	static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
548	952	{
549		- /* Returns 0 for the CS_ETM_TRACE_ON packet */
550		- if (packet->sample_type == CS_ETM_TRACE_ON)
	953	+ /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
	954	+ if (packet->sample_type == CS_ETM_DISCONTINUITY)
551	955	return 0;
552	956
553	957	return packet->start_addr;
554	958	}
555	959
556		-static inline u64 cs_etm__instr_count(const struct cs_etm_packet *packet)
	960	+static inline
	961	+u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
557	962	{
558		- /*
559		- * Only A64 instructions are currently supported, so can get
560		- * instruction count by dividing.
561		- * Will need to do instruction level decode for T32 instructions as
562		- * they can be variable size (not yet supported).
563		- */
564		- return (packet->end_addr - packet->start_addr) / A64_INSTR_SIZE;
	963	+ /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
	964	+ if (packet->sample_type == CS_ETM_DISCONTINUITY)
	965	+ return 0;
	966	+
	967	+ return packet->end_addr - packet->last_instr_size;
565	968	}
566	969
567		-static inline u64 cs_etm__instr_addr(const struct cs_etm_packet *packet,
	970	+static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
	971	+ u64 trace_chan_id,
	972	+ const struct cs_etm_packet *packet,
568	973	u64 offset)
569	974	{
570		- /*
571		- * Only A64 instructions are currently supported, so can get
572		- * instruction address by muliplying.
573		- * Will need to do instruction level decode for T32 instructions as
574		- * they can be variable size (not yet supported).
575		- */
576		- return packet->start_addr + offset * A64_INSTR_SIZE;
	975	+ if (packet->isa == CS_ETM_ISA_T32) {
	976	+ u64 addr = packet->start_addr;
	977	+
	978	+ while (offset) {
	979	+ addr += cs_etm__t32_instr_size(etmq,
	980	+ trace_chan_id, addr);
	981	+ offset--;
	982	+ }
	983	+ return addr;
	984	+ }
	985	+
	986	+ /* Assume a 4 byte instruction size (A32/A64) */
	987	+ return packet->start_addr + offset * 4;
577	988	}
578	989
579		-static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq)
	990	+static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
	991	+ struct cs_etm_traceid_queue *tidq)
580	992	{
581		- struct branch_stack *bs = etmq->last_branch_rb;
	993	+ struct branch_stack *bs = tidq->last_branch_rb;
582	994	struct branch_entry *be;
583	995
584	996	/*
..	..	@@ -587,14 +999,14 @@
587	999	* buffer down. After writing the first element of the stack, move the
588	1000	* insert position back to the end of the buffer.
589	1001	*/
590		- if (!etmq->last_branch_pos)
591		- etmq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
	1002	+ if (!tidq->last_branch_pos)
	1003	+ tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
592	1004
593		- etmq->last_branch_pos -= 1;
	1005	+ tidq->last_branch_pos -= 1;
594	1006
595		- be = &bs->entries[etmq->last_branch_pos];
596		- be->from = cs_etm__last_executed_instr(etmq->prev_packet);
597		- be->to = cs_etm__first_executed_instr(etmq->packet);
	1007	+ be = &bs->entries[tidq->last_branch_pos];
	1008	+ be->from = cs_etm__last_executed_instr(tidq->prev_packet);
	1009	+ be->to = cs_etm__first_executed_instr(tidq->packet);
598	1010	/* No support for mispredict */
599	1011	be->flags.mispred = 0;
600	1012	be->flags.predicted = 1;
..	..	@@ -616,7 +1028,7 @@
616	1028
617	1029
618	1030	static int
619		-cs_etm__get_trace(struct cs_etm_buffer buff, struct cs_etm_queue etmq)
	1031	+cs_etm__get_trace(struct cs_etm_queue *etmq)
620	1032	{
621	1033	struct auxtrace_buffer *aux_buffer = etmq->buffer;
622	1034	struct auxtrace_buffer *old_buffer = aux_buffer;
..	..	@@ -630,7 +1042,7 @@
630	1042	if (!aux_buffer) {
631	1043	if (old_buffer)
632	1044	auxtrace_buffer__drop_data(old_buffer);
633		- buff->len = 0;
	1045	+ etmq->buf_len = 0;
634	1046	return 0;
635	1047	}
636	1048
..	..	@@ -650,41 +1062,90 @@
650	1062	if (old_buffer)
651	1063	auxtrace_buffer__drop_data(old_buffer);
652	1064
653		- buff->offset = aux_buffer->offset;
654		- buff->len = aux_buffer->size;
655		- buff->buf = aux_buffer->data;
	1065	+ etmq->buf_used = 0;
	1066	+ etmq->buf_len = aux_buffer->size;
	1067	+ etmq->buf = aux_buffer->data;
656	1068
657		- buff->ref_timestamp = aux_buffer->reference;
658		-
659		- return buff->len;
	1069	+ return etmq->buf_len;
660	1070	}
661	1071
662	1072	static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,
663		- struct auxtrace_queue *queue)
	1073	+ struct cs_etm_traceid_queue *tidq)
664	1074	{
665		- struct cs_etm_queue *etmq = queue->priv;
	1075	+ if ((!tidq->thread) && (tidq->tid != -1))
	1076	+ tidq->thread = machine__find_thread(etm->machine, -1,
	1077	+ tidq->tid);
666	1078
667		- /* CPU-wide tracing isn't supported yet */
668		- if (queue->tid == -1)
	1079	+ if (tidq->thread)
	1080	+ tidq->pid = tidq->thread->pid_;
	1081	+}
	1082	+
	1083	+int cs_etm__etmq_set_tid(struct cs_etm_queue *etmq,
	1084	+ pid_t tid, u8 trace_chan_id)
	1085	+{
	1086	+ int cpu, err = -EINVAL;
	1087	+ struct cs_etm_auxtrace *etm = etmq->etm;
	1088	+ struct cs_etm_traceid_queue *tidq;
	1089	+
	1090	+ tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
	1091	+ if (!tidq)
	1092	+ return err;
	1093	+
	1094	+ if (cs_etm__get_cpu(trace_chan_id, &cpu) < 0)
	1095	+ return err;
	1096	+
	1097	+ err = machine__set_current_tid(etm->machine, cpu, tid, tid);
	1098	+ if (err)
	1099	+ return err;
	1100	+
	1101	+ tidq->tid = tid;
	1102	+ thread__zput(tidq->thread);
	1103	+
	1104	+ cs_etm__set_pid_tid_cpu(etm, tidq);
	1105	+ return 0;
	1106	+}
	1107	+
	1108	+bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
	1109	+{
	1110	+ return !!etmq->etm->timeless_decoding;
	1111	+}
	1112	+
	1113	+static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
	1114	+ u64 trace_chan_id,
	1115	+ const struct cs_etm_packet *packet,
	1116	+ struct perf_sample *sample)
	1117	+{
	1118	+ /*
	1119	+ * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
	1120	+ * packet, so directly bail out with 'insn_len' = 0.
	1121	+ */
	1122	+ if (packet->sample_type == CS_ETM_DISCONTINUITY) {
	1123	+ sample->insn_len = 0;
669	1124	return;
670		-
671		- if ((!etmq->thread) && (etmq->tid != -1))
672		- etmq->thread = machine__find_thread(etm->machine, -1,
673		- etmq->tid);
674		-
675		- if (etmq->thread) {
676		- etmq->pid = etmq->thread->pid_;
677		- if (queue->cpu == -1)
678		- etmq->cpu = etmq->thread->cpu;
679	1125	}
	1126	+
	1127	+ /*
	1128	+ * T32 instruction size might be 32-bit or 16-bit, decide by calling
	1129	+ * cs_etm__t32_instr_size().
	1130	+ */
	1131	+ if (packet->isa == CS_ETM_ISA_T32)
	1132	+ sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
	1133	+ sample->ip);
	1134	+ /* Otherwise, A64 and A32 instruction size are always 32-bit. */
	1135	+ else
	1136	+ sample->insn_len = 4;
	1137	+
	1138	+ cs_etm__mem_access(etmq, trace_chan_id, sample->ip,
	1139	+ sample->insn_len, (void *)sample->insn);
680	1140	}
681	1141
682	1142	static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
	1143	+ struct cs_etm_traceid_queue *tidq,
683	1144	u64 addr, u64 period)
684	1145	{
685	1146	int ret = 0;
686	1147	struct cs_etm_auxtrace *etm = etmq->etm;
687		- union perf_event *event = etmq->event_buf;
	1148	+ union perf_event *event = tidq->event_buf;
688	1149	struct perf_sample sample = {.ip = 0,};
689	1150
690	1151	event->sample.header.type = PERF_RECORD_SAMPLE;
..	..	@@ -692,20 +1153,19 @@
692	1153	event->sample.header.size = sizeof(struct perf_event_header);
693	1154
694	1155	sample.ip = addr;
695		- sample.pid = etmq->pid;
696		- sample.tid = etmq->tid;
	1156	+ sample.pid = tidq->pid;
	1157	+ sample.tid = tidq->tid;
697	1158	sample.id = etmq->etm->instructions_id;
698	1159	sample.stream_id = etmq->etm->instructions_id;
699	1160	sample.period = period;
700		- sample.cpu = etmq->packet->cpu;
701		- sample.flags = 0;
702		- sample.insn_len = 1;
	1161	+ sample.cpu = tidq->packet->cpu;
	1162	+ sample.flags = tidq->prev_packet->flags;
703	1163	sample.cpumode = event->sample.header.misc;
704	1164
705		- if (etm->synth_opts.last_branch) {
706		- cs_etm__copy_last_branch_rb(etmq);
707		- sample.branch_stack = etmq->last_branch;
708		- }
	1165	+ cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
	1166	+
	1167	+ if (etm->synth_opts.last_branch)
	1168	+ sample.branch_stack = tidq->last_branch;
709	1169
710	1170	if (etm->synth_opts.inject) {
711	1171	ret = cs_etm__inject_event(event, &sample,
..	..	@@ -721,9 +1181,6 @@
721	1181	"CS ETM Trace: failed to deliver instruction event, error %d\n",
722	1182	ret);
723	1183
724		- if (etm->synth_opts.last_branch)
725		- cs_etm__reset_last_branch_rb(etmq);
726		-
727	1184	return ret;
728	1185	}
729	1186
..	..	@@ -731,34 +1188,39 @@
731	1188	* The cs etm packet encodes an instruction range between a branch target
732	1189	* and the next taken branch. Generate sample accordingly.
733	1190	*/
734		-static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq)
	1191	+static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
	1192	+ struct cs_etm_traceid_queue *tidq)
735	1193	{
736	1194	int ret = 0;
737	1195	struct cs_etm_auxtrace *etm = etmq->etm;
738	1196	struct perf_sample sample = {.ip = 0,};
739		- union perf_event *event = etmq->event_buf;
	1197	+ union perf_event *event = tidq->event_buf;
740	1198	struct dummy_branch_stack {
741	1199	u64 nr;
	1200	+ u64 hw_idx;
742	1201	struct branch_entry entries;
743	1202	} dummy_bs;
744	1203	u64 ip;
745	1204
746		- ip = cs_etm__last_executed_instr(etmq->prev_packet);
	1205	+ ip = cs_etm__last_executed_instr(tidq->prev_packet);
747	1206
748	1207	event->sample.header.type = PERF_RECORD_SAMPLE;
749	1208	event->sample.header.misc = cs_etm__cpu_mode(etmq, ip);
750	1209	event->sample.header.size = sizeof(struct perf_event_header);
751	1210
752	1211	sample.ip = ip;
753		- sample.pid = etmq->pid;
754		- sample.tid = etmq->tid;
755		- sample.addr = cs_etm__first_executed_instr(etmq->packet);
	1212	+ sample.pid = tidq->pid;
	1213	+ sample.tid = tidq->tid;
	1214	+ sample.addr = cs_etm__first_executed_instr(tidq->packet);
756	1215	sample.id = etmq->etm->branches_id;
757	1216	sample.stream_id = etmq->etm->branches_id;
758	1217	sample.period = 1;
759		- sample.cpu = etmq->packet->cpu;
760		- sample.flags = 0;
	1218	+ sample.cpu = tidq->packet->cpu;
	1219	+ sample.flags = tidq->prev_packet->flags;
761	1220	sample.cpumode = event->sample.header.misc;
	1221	+
	1222	+ cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
	1223	+ &sample);
762	1224
763	1225	/*
764	1226	* perf report cannot handle events without a branch stack
..	..	@@ -766,6 +1228,7 @@
766	1228	if (etm->synth_opts.last_branch) {
767	1229	dummy_bs = (struct dummy_branch_stack){
768	1230	.nr = 1,
	1231	+ .hw_idx = -1ULL,
769	1232	.entries = {
770	1233	.from = sample.ip,
771	1234	.to = sample.addr,
..	..	@@ -823,15 +1286,15 @@
823	1286	static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
824	1287	struct perf_session *session)
825	1288	{
826		- struct perf_evlist *evlist = session->evlist;
827		- struct perf_evsel *evsel;
	1289	+ struct evlist *evlist = session->evlist;
	1290	+ struct evsel *evsel;
828	1291	struct perf_event_attr attr;
829	1292	bool found = false;
830	1293	u64 id;
831	1294	int err;
832	1295
833	1296	evlist__for_each_entry(evlist, evsel) {
834		- if (evsel->attr.type == etm->pmu_type) {
	1297	+ if (evsel->core.attr.type == etm->pmu_type) {
835	1298	found = true;
836	1299	break;
837	1300	}
..	..	@@ -845,7 +1308,7 @@
845	1308	memset(&attr, 0, sizeof(struct perf_event_attr));
846	1309	attr.size = sizeof(struct perf_event_attr);
847	1310	attr.type = PERF_TYPE_HARDWARE;
848		- attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
	1311	+ attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
849	1312	attr.sample_type \|= PERF_SAMPLE_IP \| PERF_SAMPLE_TID \|
850	1313	PERF_SAMPLE_PERIOD;
851	1314	if (etm->timeless_decoding)
..	..	@@ -853,16 +1316,16 @@
853	1316	else
854	1317	attr.sample_type \|= PERF_SAMPLE_TIME;
855	1318
856		- attr.exclude_user = evsel->attr.exclude_user;
857		- attr.exclude_kernel = evsel->attr.exclude_kernel;
858		- attr.exclude_hv = evsel->attr.exclude_hv;
859		- attr.exclude_host = evsel->attr.exclude_host;
860		- attr.exclude_guest = evsel->attr.exclude_guest;
861		- attr.sample_id_all = evsel->attr.sample_id_all;
862		- attr.read_format = evsel->attr.read_format;
	1319	+ attr.exclude_user = evsel->core.attr.exclude_user;
	1320	+ attr.exclude_kernel = evsel->core.attr.exclude_kernel;
	1321	+ attr.exclude_hv = evsel->core.attr.exclude_hv;
	1322	+ attr.exclude_host = evsel->core.attr.exclude_host;
	1323	+ attr.exclude_guest = evsel->core.attr.exclude_guest;
	1324	+ attr.sample_id_all = evsel->core.attr.sample_id_all;
	1325	+ attr.read_format = evsel->core.attr.read_format;
863	1326
864	1327	/* create new id val to be a fixed offset from evsel id */
865		- id = evsel->id[0] + 1000000000;
	1328	+ id = evsel->core.id[0] + 1000000000;
866	1329
867	1330	if (!id)
868	1331	id = 1;
..	..	@@ -881,8 +1344,15 @@
881	1344	attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
882	1345	}
883	1346
884		- if (etm->synth_opts.last_branch)
	1347	+ if (etm->synth_opts.last_branch) {
885	1348	attr.sample_type \|= PERF_SAMPLE_BRANCH_STACK;
	1349	+ /*
	1350	+ * We don't use the hardware index, but the sample generation
	1351	+ * code uses the new format branch_stack with this field,
	1352	+ * so the event attributes must indicate that it's present.
	1353	+ */
	1354	+ attr.branch_sample_type \|= PERF_SAMPLE_BRANCH_HW_INDEX;
	1355	+ }
886	1356
887	1357	if (etm->synth_opts.instructions) {
888	1358	attr.config = PERF_COUNT_HW_INSTRUCTIONS;
..	..	@@ -900,101 +1370,171 @@
900	1370	return 0;
901	1371	}
902	1372
903		-static int cs_etm__sample(struct cs_etm_queue *etmq)
	1373	+static int cs_etm__sample(struct cs_etm_queue *etmq,
	1374	+ struct cs_etm_traceid_queue *tidq)
904	1375	{
905	1376	struct cs_etm_auxtrace *etm = etmq->etm;
906		- struct cs_etm_packet *tmp;
907	1377	int ret;
908		- u64 instrs_executed;
	1378	+ u8 trace_chan_id = tidq->trace_chan_id;
	1379	+ u64 instrs_prev;
909	1380
910		- instrs_executed = cs_etm__instr_count(etmq->packet);
911		- etmq->period_instructions += instrs_executed;
	1381	+ /* Get instructions remainder from previous packet */
	1382	+ instrs_prev = tidq->period_instructions;
	1383	+
	1384	+ tidq->period_instructions += tidq->packet->instr_count;
912	1385
913	1386	/*
914	1387	* Record a branch when the last instruction in
915	1388	* PREV_PACKET is a branch.
916	1389	*/
917	1390	if (etm->synth_opts.last_branch &&
918		- etmq->prev_packet &&
919		- etmq->prev_packet->sample_type == CS_ETM_RANGE &&
920		- etmq->prev_packet->last_instr_taken_branch)
921		- cs_etm__update_last_branch_rb(etmq);
	1391	+ tidq->prev_packet->sample_type == CS_ETM_RANGE &&
	1392	+ tidq->prev_packet->last_instr_taken_branch)
	1393	+ cs_etm__update_last_branch_rb(etmq, tidq);
922	1394
923	1395	if (etm->sample_instructions &&
924		- etmq->period_instructions >= etm->instructions_sample_period) {
	1396	+ tidq->period_instructions >= etm->instructions_sample_period) {
925	1397	/*
926	1398	* Emit instruction sample periodically
927	1399	* TODO: allow period to be defined in cycles and clock time
928	1400	*/
929	1401
930		- /* Get number of instructions executed after the sample point */
931		- u64 instrs_over = etmq->period_instructions -
932		- etm->instructions_sample_period;
	1402	+ /*
	1403	+ * Below diagram demonstrates the instruction samples
	1404	+ * generation flows:
	1405	+ *
	1406	+ * Instrs Instrs Instrs Instrs
	1407	+ * Sample(n) Sample(n+1) Sample(n+2) Sample(n+3)
	1408	+ * \| \| \| \|
	1409	+ * V V V V
	1410	+ * --------------------------------------------------
	1411	+ * ^ ^
	1412	+ * \| \|
	1413	+ * Period Period
	1414	+ * instructions(Pi) instructions(Pi')
	1415	+ *
	1416	+ * \| \|
	1417	+ * \---------------- -----------------/
	1418	+ * V
	1419	+ * tidq->packet->instr_count
	1420	+ *
	1421	+ * Instrs Sample(n...) are the synthesised samples occurring
	1422	+ * every etm->instructions_sample_period instructions - as
	1423	+ * defined on the perf command line. Sample(n) is being the
	1424	+ * last sample before the current etm packet, n+1 to n+3
	1425	+ * samples are generated from the current etm packet.
	1426	+ *
	1427	+ * tidq->packet->instr_count represents the number of
	1428	+ * instructions in the current etm packet.
	1429	+ *
	1430	+ * Period instructions (Pi) contains the the number of
	1431	+ * instructions executed after the sample point(n) from the
	1432	+ * previous etm packet. This will always be less than
	1433	+ * etm->instructions_sample_period.
	1434	+ *
	1435	+ * When generate new samples, it combines with two parts
	1436	+ * instructions, one is the tail of the old packet and another
	1437	+ * is the head of the new coming packet, to generate
	1438	+ * sample(n+1); sample(n+2) and sample(n+3) consume the
	1439	+ * instructions with sample period. After sample(n+3), the rest
	1440	+ * instructions will be used by later packet and it is assigned
	1441	+ * to tidq->period_instructions for next round calculation.
	1442	+ */
933	1443
934	1444	/*
935		- * Calculate the address of the sampled instruction (-1 as
936		- * sample is reported as though instruction has just been
937		- * executed, but PC has not advanced to next instruction)
	1445	+ * Get the initial offset into the current packet instructions;
	1446	+ * entry conditions ensure that instrs_prev is less than
	1447	+ * etm->instructions_sample_period.
938	1448	*/
939		- u64 offset = (instrs_executed - instrs_over - 1);
940		- u64 addr = cs_etm__instr_addr(etmq->packet, offset);
	1449	+ u64 offset = etm->instructions_sample_period - instrs_prev;
	1450	+ u64 addr;
941	1451
942		- ret = cs_etm__synth_instruction_sample(
943		- etmq, addr, etm->instructions_sample_period);
944		- if (ret)
945		- return ret;
	1452	+ /* Prepare last branches for instruction sample */
	1453	+ if (etm->synth_opts.last_branch)
	1454	+ cs_etm__copy_last_branch_rb(etmq, tidq);
946	1455
947		- /* Carry remaining instructions into next sample period */
948		- etmq->period_instructions = instrs_over;
	1456	+ while (tidq->period_instructions >=
	1457	+ etm->instructions_sample_period) {
	1458	+ /*
	1459	+ * Calculate the address of the sampled instruction (-1
	1460	+ * as sample is reported as though instruction has just
	1461	+ * been executed, but PC has not advanced to next
	1462	+ * instruction)
	1463	+ */
	1464	+ addr = cs_etm__instr_addr(etmq, trace_chan_id,
	1465	+ tidq->packet, offset - 1);
	1466	+ ret = cs_etm__synth_instruction_sample(
	1467	+ etmq, tidq, addr,
	1468	+ etm->instructions_sample_period);
	1469	+ if (ret)
	1470	+ return ret;
	1471	+
	1472	+ offset += etm->instructions_sample_period;
	1473	+ tidq->period_instructions -=
	1474	+ etm->instructions_sample_period;
	1475	+ }
949	1476	}
950	1477
951		- if (etm->sample_branches && etmq->prev_packet) {
	1478	+ if (etm->sample_branches) {
952	1479	bool generate_sample = false;
953	1480
954	1481	/* Generate sample for tracing on packet */
955		- if (etmq->prev_packet->sample_type == CS_ETM_TRACE_ON)
	1482	+ if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
956	1483	generate_sample = true;
957	1484
958	1485	/* Generate sample for branch taken packet */
959		- if (etmq->prev_packet->sample_type == CS_ETM_RANGE &&
960		- etmq->prev_packet->last_instr_taken_branch)
	1486	+ if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
	1487	+ tidq->prev_packet->last_instr_taken_branch)
961	1488	generate_sample = true;
962	1489
963	1490	if (generate_sample) {
964		- ret = cs_etm__synth_branch_sample(etmq);
	1491	+ ret = cs_etm__synth_branch_sample(etmq, tidq);
965	1492	if (ret)
966	1493	return ret;
967	1494	}
968	1495	}
969	1496
970		- if (etm->sample_branches \|\| etm->synth_opts.last_branch) {
971		- /*
972		- * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
973		- * the next incoming packet.
974		- */
975		- tmp = etmq->packet;
976		- etmq->packet = etmq->prev_packet;
977		- etmq->prev_packet = tmp;
978		- }
	1497	+ cs_etm__packet_swap(etm, tidq);
979	1498
980	1499	return 0;
981	1500	}
982	1501
983		-static int cs_etm__flush(struct cs_etm_queue *etmq)
	1502	+static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
	1503	+{
	1504	+ /*
	1505	+ * When the exception packet is inserted, whether the last instruction
	1506	+ * in previous range packet is taken branch or not, we need to force
	1507	+ * to set 'prev_packet->last_instr_taken_branch' to true. This ensures
	1508	+ * to generate branch sample for the instruction range before the
	1509	+ * exception is trapped to kernel or before the exception returning.
	1510	+ *
	1511	+ * The exception packet includes the dummy address values, so don't
	1512	+ * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful
	1513	+ * for generating instruction and branch samples.
	1514	+ */
	1515	+ if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
	1516	+ tidq->prev_packet->last_instr_taken_branch = true;
	1517	+
	1518	+ return 0;
	1519	+}
	1520	+
	1521	+static int cs_etm__flush(struct cs_etm_queue *etmq,
	1522	+ struct cs_etm_traceid_queue *tidq)
984	1523	{
985	1524	int err = 0;
986	1525	struct cs_etm_auxtrace *etm = etmq->etm;
987		- struct cs_etm_packet *tmp;
988		-
989		- if (!etmq->prev_packet)
990		- return 0;
991	1526
992	1527	/* Handle start tracing packet */
993		- if (etmq->prev_packet->sample_type == CS_ETM_EMPTY)
	1528	+ if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
994	1529	goto swap_packet;
995	1530
996	1531	if (etmq->etm->synth_opts.last_branch &&
997		- etmq->prev_packet->sample_type == CS_ETM_RANGE) {
	1532	+ tidq->prev_packet->sample_type == CS_ETM_RANGE) {
	1533	+ u64 addr;
	1534	+
	1535	+ /* Prepare last branches for instruction sample */
	1536	+ cs_etm__copy_last_branch_rb(etmq, tidq);
	1537	+
998	1538	/*
999	1539	* Generate a last branch event for the branches left in the
1000	1540	* circular buffer at the end of the trace.
..	..	@@ -1002,129 +1542,611 @@
1002	1542	* Use the address of the end of the last reported execution
1003	1543	* range
1004	1544	*/
1005		- u64 addr = cs_etm__last_executed_instr(etmq->prev_packet);
	1545	+ addr = cs_etm__last_executed_instr(tidq->prev_packet);
1006	1546
1007	1547	err = cs_etm__synth_instruction_sample(
1008		- etmq, addr,
1009		- etmq->period_instructions);
	1548	+ etmq, tidq, addr,
	1549	+ tidq->period_instructions);
1010	1550	if (err)
1011	1551	return err;
1012	1552
1013		- etmq->period_instructions = 0;
	1553	+ tidq->period_instructions = 0;
1014	1554
1015	1555	}
1016	1556
1017	1557	if (etm->sample_branches &&
1018		- etmq->prev_packet->sample_type == CS_ETM_RANGE) {
1019		- err = cs_etm__synth_branch_sample(etmq);
	1558	+ tidq->prev_packet->sample_type == CS_ETM_RANGE) {
	1559	+ err = cs_etm__synth_branch_sample(etmq, tidq);
1020	1560	if (err)
1021	1561	return err;
1022	1562	}
1023	1563
1024	1564	swap_packet:
1025		- if (etm->sample_branches \|\| etm->synth_opts.last_branch) {
1026		- /*
1027		- * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1028		- * the next incoming packet.
1029		- */
1030		- tmp = etmq->packet;
1031		- etmq->packet = etmq->prev_packet;
1032		- etmq->prev_packet = tmp;
1033		- }
	1565	+ cs_etm__packet_swap(etm, tidq);
	1566	+
	1567	+ /* Reset last branches after flush the trace */
	1568	+ if (etm->synth_opts.last_branch)
	1569	+ cs_etm__reset_last_branch_rb(tidq);
1034	1570
1035	1571	return err;
1036	1572	}
1037	1573
	1574	+static int cs_etm__end_block(struct cs_etm_queue *etmq,
	1575	+ struct cs_etm_traceid_queue *tidq)
	1576	+{
	1577	+ int err;
	1578	+
	1579	+ /*
	1580	+ * It has no new packet coming and 'etmq->packet' contains the stale
	1581	+ * packet which was set at the previous time with packets swapping;
	1582	+ * so skip to generate branch sample to avoid stale packet.
	1583	+ *
	1584	+ * For this case only flush branch stack and generate a last branch
	1585	+ * event for the branches left in the circular buffer at the end of
	1586	+ * the trace.
	1587	+ */
	1588	+ if (etmq->etm->synth_opts.last_branch &&
	1589	+ tidq->prev_packet->sample_type == CS_ETM_RANGE) {
	1590	+ u64 addr;
	1591	+
	1592	+ /* Prepare last branches for instruction sample */
	1593	+ cs_etm__copy_last_branch_rb(etmq, tidq);
	1594	+
	1595	+ /*
	1596	+ * Use the address of the end of the last reported execution
	1597	+ * range.
	1598	+ */
	1599	+ addr = cs_etm__last_executed_instr(tidq->prev_packet);
	1600	+
	1601	+ err = cs_etm__synth_instruction_sample(
	1602	+ etmq, tidq, addr,
	1603	+ tidq->period_instructions);
	1604	+ if (err)
	1605	+ return err;
	1606	+
	1607	+ tidq->period_instructions = 0;
	1608	+ }
	1609	+
	1610	+ return 0;
	1611	+}
	1612	+/*
	1613	+ * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
	1614	+ * if need be.
	1615	+ * Returns: < 0 if error
	1616	+ * = 0 if no more auxtrace_buffer to read
	1617	+ * > 0 if the current buffer isn't empty yet
	1618	+ */
	1619	+static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
	1620	+{
	1621	+ int ret;
	1622	+
	1623	+ if (!etmq->buf_len) {
	1624	+ ret = cs_etm__get_trace(etmq);
	1625	+ if (ret <= 0)
	1626	+ return ret;
	1627	+ /*
	1628	+ * We cannot assume consecutive blocks in the data file
	1629	+ * are contiguous, reset the decoder to force re-sync.
	1630	+ */
	1631	+ ret = cs_etm_decoder__reset(etmq->decoder);
	1632	+ if (ret)
	1633	+ return ret;
	1634	+ }
	1635	+
	1636	+ return etmq->buf_len;
	1637	+}
	1638	+
	1639	+static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
	1640	+ struct cs_etm_packet *packet,
	1641	+ u64 end_addr)
	1642	+{
	1643	+ /* Initialise to keep compiler happy */
	1644	+ u16 instr16 = 0;
	1645	+ u32 instr32 = 0;
	1646	+ u64 addr;
	1647	+
	1648	+ switch (packet->isa) {
	1649	+ case CS_ETM_ISA_T32:
	1650	+ /*
	1651	+ * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
	1652	+ *
	1653	+ * b'15 b'8
	1654	+ * +-----------------+--------+
	1655	+ * \| 1 1 0 1 1 1 1 1 \| imm8 \|
	1656	+ * +-----------------+--------+
	1657	+ *
	1658	+ * According to the specifiction, it only defines SVC for T32
	1659	+ * with 16 bits instruction and has no definition for 32bits;
	1660	+ * so below only read 2 bytes as instruction size for T32.
	1661	+ */
	1662	+ addr = end_addr - 2;
	1663	+ cs_etm__mem_access(etmq, trace_chan_id, addr,
	1664	+ sizeof(instr16), (u8 *)&instr16);
	1665	+ if ((instr16 & 0xFF00) == 0xDF00)
	1666	+ return true;
	1667	+
	1668	+ break;
	1669	+ case CS_ETM_ISA_A32:
	1670	+ /*
	1671	+ * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
	1672	+ *
	1673	+ * b'31 b'28 b'27 b'24
	1674	+ * +---------+---------+-------------------------+
	1675	+ * \| !1111 \| 1 1 1 1 \| imm24 \|
	1676	+ * +---------+---------+-------------------------+
	1677	+ */
	1678	+ addr = end_addr - 4;
	1679	+ cs_etm__mem_access(etmq, trace_chan_id, addr,
	1680	+ sizeof(instr32), (u8 *)&instr32);
	1681	+ if ((instr32 & 0x0F000000) == 0x0F000000 &&
	1682	+ (instr32 & 0xF0000000) != 0xF0000000)
	1683	+ return true;
	1684	+
	1685	+ break;
	1686	+ case CS_ETM_ISA_A64:
	1687	+ /*
	1688	+ * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
	1689	+ *
	1690	+ * b'31 b'21 b'4 b'0
	1691	+ * +-----------------------+---------+-----------+
	1692	+ * \| 1 1 0 1 0 1 0 0 0 0 0 \| imm16 \| 0 0 0 0 1 \|
	1693	+ * +-----------------------+---------+-----------+
	1694	+ */
	1695	+ addr = end_addr - 4;
	1696	+ cs_etm__mem_access(etmq, trace_chan_id, addr,
	1697	+ sizeof(instr32), (u8 *)&instr32);
	1698	+ if ((instr32 & 0xFFE0001F) == 0xd4000001)
	1699	+ return true;
	1700	+
	1701	+ break;
	1702	+ case CS_ETM_ISA_UNKNOWN:
	1703	+ default:
	1704	+ break;
	1705	+ }
	1706	+
	1707	+ return false;
	1708	+}
	1709	+
	1710	+static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
	1711	+ struct cs_etm_traceid_queue *tidq, u64 magic)
	1712	+{
	1713	+ u8 trace_chan_id = tidq->trace_chan_id;
	1714	+ struct cs_etm_packet *packet = tidq->packet;
	1715	+ struct cs_etm_packet *prev_packet = tidq->prev_packet;
	1716	+
	1717	+ if (magic == __perf_cs_etmv3_magic)
	1718	+ if (packet->exception_number == CS_ETMV3_EXC_SVC)
	1719	+ return true;
	1720	+
	1721	+ /*
	1722	+ * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
	1723	+ * HVC cases; need to check if it's SVC instruction based on
	1724	+ * packet address.
	1725	+ */
	1726	+ if (magic == __perf_cs_etmv4_magic) {
	1727	+ if (packet->exception_number == CS_ETMV4_EXC_CALL &&
	1728	+ cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
	1729	+ prev_packet->end_addr))
	1730	+ return true;
	1731	+ }
	1732	+
	1733	+ return false;
	1734	+}
	1735	+
	1736	+static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
	1737	+ u64 magic)
	1738	+{
	1739	+ struct cs_etm_packet *packet = tidq->packet;
	1740	+
	1741	+ if (magic == __perf_cs_etmv3_magic)
	1742	+ if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT \|\|
	1743	+ packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT \|\|
	1744	+ packet->exception_number == CS_ETMV3_EXC_PE_RESET \|\|
	1745	+ packet->exception_number == CS_ETMV3_EXC_IRQ \|\|
	1746	+ packet->exception_number == CS_ETMV3_EXC_FIQ)
	1747	+ return true;
	1748	+
	1749	+ if (magic == __perf_cs_etmv4_magic)
	1750	+ if (packet->exception_number == CS_ETMV4_EXC_RESET \|\|
	1751	+ packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT \|\|
	1752	+ packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR \|\|
	1753	+ packet->exception_number == CS_ETMV4_EXC_INST_DEBUG \|\|
	1754	+ packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG \|\|
	1755	+ packet->exception_number == CS_ETMV4_EXC_IRQ \|\|
	1756	+ packet->exception_number == CS_ETMV4_EXC_FIQ)
	1757	+ return true;
	1758	+
	1759	+ return false;
	1760	+}
	1761	+
	1762	+static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
	1763	+ struct cs_etm_traceid_queue *tidq,
	1764	+ u64 magic)
	1765	+{
	1766	+ u8 trace_chan_id = tidq->trace_chan_id;
	1767	+ struct cs_etm_packet *packet = tidq->packet;
	1768	+ struct cs_etm_packet *prev_packet = tidq->prev_packet;
	1769	+
	1770	+ if (magic == __perf_cs_etmv3_magic)
	1771	+ if (packet->exception_number == CS_ETMV3_EXC_SMC \|\|
	1772	+ packet->exception_number == CS_ETMV3_EXC_HYP \|\|
	1773	+ packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE \|\|
	1774	+ packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR \|\|
	1775	+ packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT \|\|
	1776	+ packet->exception_number == CS_ETMV3_EXC_DATA_FAULT \|\|
	1777	+ packet->exception_number == CS_ETMV3_EXC_GENERIC)
	1778	+ return true;
	1779	+
	1780	+ if (magic == __perf_cs_etmv4_magic) {
	1781	+ if (packet->exception_number == CS_ETMV4_EXC_TRAP \|\|
	1782	+ packet->exception_number == CS_ETMV4_EXC_ALIGNMENT \|\|
	1783	+ packet->exception_number == CS_ETMV4_EXC_INST_FAULT \|\|
	1784	+ packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
	1785	+ return true;
	1786	+
	1787	+ /*
	1788	+ * For CS_ETMV4_EXC_CALL, except SVC other instructions
	1789	+ * (SMC, HVC) are taken as sync exceptions.
	1790	+ */
	1791	+ if (packet->exception_number == CS_ETMV4_EXC_CALL &&
	1792	+ !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
	1793	+ prev_packet->end_addr))
	1794	+ return true;
	1795	+
	1796	+ /*
	1797	+ * ETMv4 has 5 bits for exception number; if the numbers
	1798	+ * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
	1799	+ * they are implementation defined exceptions.
	1800	+ *
	1801	+ * For this case, simply take it as sync exception.
	1802	+ */
	1803	+ if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
	1804	+ packet->exception_number <= CS_ETMV4_EXC_END)
	1805	+ return true;
	1806	+ }
	1807	+
	1808	+ return false;
	1809	+}
	1810	+
	1811	+static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
	1812	+ struct cs_etm_traceid_queue *tidq)
	1813	+{
	1814	+ struct cs_etm_packet *packet = tidq->packet;
	1815	+ struct cs_etm_packet *prev_packet = tidq->prev_packet;
	1816	+ u8 trace_chan_id = tidq->trace_chan_id;
	1817	+ u64 magic;
	1818	+ int ret;
	1819	+
	1820	+ switch (packet->sample_type) {
	1821	+ case CS_ETM_RANGE:
	1822	+ /*
	1823	+ * Immediate branch instruction without neither link nor
	1824	+ * return flag, it's normal branch instruction within
	1825	+ * the function.
	1826	+ */
	1827	+ if (packet->last_instr_type == OCSD_INSTR_BR &&
	1828	+ packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
	1829	+ packet->flags = PERF_IP_FLAG_BRANCH;
	1830	+
	1831	+ if (packet->last_instr_cond)
	1832	+ packet->flags \|= PERF_IP_FLAG_CONDITIONAL;
	1833	+ }
	1834	+
	1835	+ /*
	1836	+ * Immediate branch instruction with link (e.g. BL), this is
	1837	+ * branch instruction for function call.
	1838	+ */
	1839	+ if (packet->last_instr_type == OCSD_INSTR_BR &&
	1840	+ packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
	1841	+ packet->flags = PERF_IP_FLAG_BRANCH \|
	1842	+ PERF_IP_FLAG_CALL;
	1843	+
	1844	+ /*
	1845	+ * Indirect branch instruction with link (e.g. BLR), this is
	1846	+ * branch instruction for function call.
	1847	+ */
	1848	+ if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
	1849	+ packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
	1850	+ packet->flags = PERF_IP_FLAG_BRANCH \|
	1851	+ PERF_IP_FLAG_CALL;
	1852	+
	1853	+ /*
	1854	+ * Indirect branch instruction with subtype of
	1855	+ * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
	1856	+ * function return for A32/T32.
	1857	+ */
	1858	+ if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
	1859	+ packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
	1860	+ packet->flags = PERF_IP_FLAG_BRANCH \|
	1861	+ PERF_IP_FLAG_RETURN;
	1862	+
	1863	+ /*
	1864	+ * Indirect branch instruction without link (e.g. BR), usually
	1865	+ * this is used for function return, especially for functions
	1866	+ * within dynamic link lib.
	1867	+ */
	1868	+ if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
	1869	+ packet->last_instr_subtype == OCSD_S_INSTR_NONE)
	1870	+ packet->flags = PERF_IP_FLAG_BRANCH \|
	1871	+ PERF_IP_FLAG_RETURN;
	1872	+
	1873	+ /* Return instruction for function return. */
	1874	+ if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
	1875	+ packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
	1876	+ packet->flags = PERF_IP_FLAG_BRANCH \|
	1877	+ PERF_IP_FLAG_RETURN;
	1878	+
	1879	+ /*
	1880	+ * Decoder might insert a discontinuity in the middle of
	1881	+ * instruction packets, fixup prev_packet with flag
	1882	+ * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
	1883	+ */
	1884	+ if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
	1885	+ prev_packet->flags \|= PERF_IP_FLAG_BRANCH \|
	1886	+ PERF_IP_FLAG_TRACE_BEGIN;
	1887	+
	1888	+ /*
	1889	+ * If the previous packet is an exception return packet
	1890	+ * and the return address just follows SVC instuction,
	1891	+ * it needs to calibrate the previous packet sample flags
	1892	+ * as PERF_IP_FLAG_SYSCALLRET.
	1893	+ */
	1894	+ if (prev_packet->flags == (PERF_IP_FLAG_BRANCH \|
	1895	+ PERF_IP_FLAG_RETURN \|
	1896	+ PERF_IP_FLAG_INTERRUPT) &&
	1897	+ cs_etm__is_svc_instr(etmq, trace_chan_id,
	1898	+ packet, packet->start_addr))
	1899	+ prev_packet->flags = PERF_IP_FLAG_BRANCH \|
	1900	+ PERF_IP_FLAG_RETURN \|
	1901	+ PERF_IP_FLAG_SYSCALLRET;
	1902	+ break;
	1903	+ case CS_ETM_DISCONTINUITY:
	1904	+ /*
	1905	+ * The trace is discontinuous, if the previous packet is
	1906	+ * instruction packet, set flag PERF_IP_FLAG_TRACE_END
	1907	+ * for previous packet.
	1908	+ */
	1909	+ if (prev_packet->sample_type == CS_ETM_RANGE)
	1910	+ prev_packet->flags \|= PERF_IP_FLAG_BRANCH \|
	1911	+ PERF_IP_FLAG_TRACE_END;
	1912	+ break;
	1913	+ case CS_ETM_EXCEPTION:
	1914	+ ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
	1915	+ if (ret)
	1916	+ return ret;
	1917	+
	1918	+ /* The exception is for system call. */
	1919	+ if (cs_etm__is_syscall(etmq, tidq, magic))
	1920	+ packet->flags = PERF_IP_FLAG_BRANCH \|
	1921	+ PERF_IP_FLAG_CALL \|
	1922	+ PERF_IP_FLAG_SYSCALLRET;
	1923	+ /*
	1924	+ * The exceptions are triggered by external signals from bus,
	1925	+ * interrupt controller, debug module, PE reset or halt.
	1926	+ */
	1927	+ else if (cs_etm__is_async_exception(tidq, magic))
	1928	+ packet->flags = PERF_IP_FLAG_BRANCH \|
	1929	+ PERF_IP_FLAG_CALL \|
	1930	+ PERF_IP_FLAG_ASYNC \|
	1931	+ PERF_IP_FLAG_INTERRUPT;
	1932	+ /*
	1933	+ * Otherwise, exception is caused by trap, instruction &
	1934	+ * data fault, or alignment errors.
	1935	+ */
	1936	+ else if (cs_etm__is_sync_exception(etmq, tidq, magic))
	1937	+ packet->flags = PERF_IP_FLAG_BRANCH \|
	1938	+ PERF_IP_FLAG_CALL \|
	1939	+ PERF_IP_FLAG_INTERRUPT;
	1940	+
	1941	+ /*
	1942	+ * When the exception packet is inserted, since exception
	1943	+ * packet is not used standalone for generating samples
	1944	+ * and it's affiliation to the previous instruction range
	1945	+ * packet; so set previous range packet flags to tell perf
	1946	+ * it is an exception taken branch.
	1947	+ */
	1948	+ if (prev_packet->sample_type == CS_ETM_RANGE)
	1949	+ prev_packet->flags = packet->flags;
	1950	+ break;
	1951	+ case CS_ETM_EXCEPTION_RET:
	1952	+ /*
	1953	+ * When the exception return packet is inserted, since
	1954	+ * exception return packet is not used standalone for
	1955	+ * generating samples and it's affiliation to the previous
	1956	+ * instruction range packet; so set previous range packet
	1957	+ * flags to tell perf it is an exception return branch.
	1958	+ *
	1959	+ * The exception return can be for either system call or
	1960	+ * other exception types; unfortunately the packet doesn't
	1961	+ * contain exception type related info so we cannot decide
	1962	+ * the exception type purely based on exception return packet.
	1963	+ * If we record the exception number from exception packet and
	1964	+ * reuse it for excpetion return packet, this is not reliable
	1965	+ * due the trace can be discontinuity or the interrupt can
	1966	+ * be nested, thus the recorded exception number cannot be
	1967	+ * used for exception return packet for these two cases.
	1968	+ *
	1969	+ * For exception return packet, we only need to distinguish the
	1970	+ * packet is for system call or for other types. Thus the
	1971	+ * decision can be deferred when receive the next packet which
	1972	+ * contains the return address, based on the return address we
	1973	+ * can read out the previous instruction and check if it's a
	1974	+ * system call instruction and then calibrate the sample flag
	1975	+ * as needed.
	1976	+ */
	1977	+ if (prev_packet->sample_type == CS_ETM_RANGE)
	1978	+ prev_packet->flags = PERF_IP_FLAG_BRANCH \|
	1979	+ PERF_IP_FLAG_RETURN \|
	1980	+ PERF_IP_FLAG_INTERRUPT;
	1981	+ break;
	1982	+ case CS_ETM_EMPTY:
	1983	+ default:
	1984	+ break;
	1985	+ }
	1986	+
	1987	+ return 0;
	1988	+}
	1989	+
	1990	+static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
	1991	+{
	1992	+ int ret = 0;
	1993	+ size_t processed = 0;
	1994	+
	1995	+ /*
	1996	+ * Packets are decoded and added to the decoder's packet queue
	1997	+ * until the decoder packet processing callback has requested that
	1998	+ * processing stops or there is nothing left in the buffer. Normal
	1999	+ * operations that stop processing are a timestamp packet or a full
	2000	+ * decoder buffer queue.
	2001	+ */
	2002	+ ret = cs_etm_decoder__process_data_block(etmq->decoder,
	2003	+ etmq->offset,
	2004	+ &etmq->buf[etmq->buf_used],
	2005	+ etmq->buf_len,
	2006	+ &processed);
	2007	+ if (ret)
	2008	+ goto out;
	2009	+
	2010	+ etmq->offset += processed;
	2011	+ etmq->buf_used += processed;
	2012	+ etmq->buf_len -= processed;
	2013	+
	2014	+out:
	2015	+ return ret;
	2016	+}
	2017	+
	2018	+static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
	2019	+ struct cs_etm_traceid_queue *tidq)
	2020	+{
	2021	+ int ret;
	2022	+ struct cs_etm_packet_queue *packet_queue;
	2023	+
	2024	+ packet_queue = &tidq->packet_queue;
	2025	+
	2026	+ /* Process each packet in this chunk */
	2027	+ while (1) {
	2028	+ ret = cs_etm_decoder__get_packet(packet_queue,
	2029	+ tidq->packet);
	2030	+ if (ret <= 0)
	2031	+ /*
	2032	+ * Stop processing this chunk on
	2033	+ * end of data or error
	2034	+ */
	2035	+ break;
	2036	+
	2037	+ /*
	2038	+ * Since packet addresses are swapped in packet
	2039	+ * handling within below switch() statements,
	2040	+ * thus setting sample flags must be called
	2041	+ * prior to switch() statement to use address
	2042	+ * information before packets swapping.
	2043	+ */
	2044	+ ret = cs_etm__set_sample_flags(etmq, tidq);
	2045	+ if (ret < 0)
	2046	+ break;
	2047	+
	2048	+ switch (tidq->packet->sample_type) {
	2049	+ case CS_ETM_RANGE:
	2050	+ /*
	2051	+ * If the packet contains an instruction
	2052	+ * range, generate instruction sequence
	2053	+ * events.
	2054	+ */
	2055	+ cs_etm__sample(etmq, tidq);
	2056	+ break;
	2057	+ case CS_ETM_EXCEPTION:
	2058	+ case CS_ETM_EXCEPTION_RET:
	2059	+ /*
	2060	+ * If the exception packet is coming,
	2061	+ * make sure the previous instruction
	2062	+ * range packet to be handled properly.
	2063	+ */
	2064	+ cs_etm__exception(tidq);
	2065	+ break;
	2066	+ case CS_ETM_DISCONTINUITY:
	2067	+ /*
	2068	+ * Discontinuity in trace, flush
	2069	+ * previous branch stack
	2070	+ */
	2071	+ cs_etm__flush(etmq, tidq);
	2072	+ break;
	2073	+ case CS_ETM_EMPTY:
	2074	+ /*
	2075	+ * Should not receive empty packet,
	2076	+ * report error.
	2077	+ */
	2078	+ pr_err("CS ETM Trace: empty packet\n");
	2079	+ return -EINVAL;
	2080	+ default:
	2081	+ break;
	2082	+ }
	2083	+ }
	2084	+
	2085	+ return ret;
	2086	+}
	2087	+
	2088	+static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
	2089	+{
	2090	+ int idx;
	2091	+ struct int_node *inode;
	2092	+ struct cs_etm_traceid_queue *tidq;
	2093	+ struct intlist *traceid_queues_list = etmq->traceid_queues_list;
	2094	+
	2095	+ intlist__for_each_entry(inode, traceid_queues_list) {
	2096	+ idx = (int)(intptr_t)inode->priv;
	2097	+ tidq = etmq->traceid_queues[idx];
	2098	+
	2099	+ /* Ignore return value */
	2100	+ cs_etm__process_traceid_queue(etmq, tidq);
	2101	+
	2102	+ /*
	2103	+ * Generate an instruction sample with the remaining
	2104	+ * branchstack entries.
	2105	+ */
	2106	+ cs_etm__flush(etmq, tidq);
	2107	+ }
	2108	+}
	2109	+
1038	2110	static int cs_etm__run_decoder(struct cs_etm_queue *etmq)
1039	2111	{
1040		- struct cs_etm_auxtrace *etm = etmq->etm;
1041		- struct cs_etm_buffer buffer;
1042		- size_t buffer_used, processed;
1043	2112	int err = 0;
	2113	+ struct cs_etm_traceid_queue *tidq;
1044	2114
1045		- if (!etm->kernel_start)
1046		- etm->kernel_start = machine__kernel_start(etm->machine);
	2115	+ tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
	2116	+ if (!tidq)
	2117	+ return -EINVAL;
1047	2118
1048	2119	/* Go through each buffer in the queue and decode them one by one */
1049	2120	while (1) {
1050		- buffer_used = 0;
1051		- memset(&buffer, 0, sizeof(buffer));
1052		- err = cs_etm__get_trace(&buffer, etmq);
	2121	+ err = cs_etm__get_data_block(etmq);
1053	2122	if (err <= 0)
1054		- return err;
1055		- /*
1056		- * We cannot assume consecutive blocks in the data file are
1057		- * contiguous, reset the decoder to force re-sync.
1058		- */
1059		- err = cs_etm_decoder__reset(etmq->decoder);
1060		- if (err != 0)
1061	2123	return err;
1062	2124
1063	2125	/* Run trace decoder until buffer consumed or end of trace */
1064	2126	do {
1065		- processed = 0;
1066		- err = cs_etm_decoder__process_data_block(
1067		- etmq->decoder,
1068		- etmq->offset,
1069		- &buffer.buf[buffer_used],
1070		- buffer.len - buffer_used,
1071		- &processed);
	2127	+ err = cs_etm__decode_data_block(etmq);
1072	2128	if (err)
1073	2129	return err;
1074	2130
1075		- etmq->offset += processed;
1076		- buffer_used += processed;
	2131	+ /*
	2132	+ * Process each packet in this chunk, nothing to do if
	2133	+ * an error occurs other than hoping the next one will
	2134	+ * be better.
	2135	+ */
	2136	+ err = cs_etm__process_traceid_queue(etmq, tidq);
1077	2137
1078		- /* Process each packet in this chunk */
1079		- while (1) {
1080		- err = cs_etm_decoder__get_packet(etmq->decoder,
1081		- etmq->packet);
1082		- if (err <= 0)
1083		- /*
1084		- * Stop processing this chunk on
1085		- * end of data or error
1086		- */
1087		- break;
1088		-
1089		- switch (etmq->packet->sample_type) {
1090		- case CS_ETM_RANGE:
1091		- /*
1092		- * If the packet contains an instruction
1093		- * range, generate instruction sequence
1094		- * events.
1095		- */
1096		- cs_etm__sample(etmq);
1097		- break;
1098		- case CS_ETM_TRACE_ON:
1099		- /*
1100		- * Discontinuity in trace, flush
1101		- * previous branch stack
1102		- */
1103		- cs_etm__flush(etmq);
1104		- break;
1105		- case CS_ETM_EMPTY:
1106		- /*
1107		- * Should not receive empty packet,
1108		- * report error.
1109		- */
1110		- pr_err("CS ETM Trace: empty packet\n");
1111		- return -EINVAL;
1112		- default:
1113		- break;
1114		- }
1115		- }
1116		- } while (buffer.len > buffer_used);
	2138	+ } while (etmq->buf_len);
1117	2139
1118	2140	if (err == 0)
1119	2141	/* Flush any remaining branch stack entries */
1120		- err = cs_etm__flush(etmq);
	2142	+ err = cs_etm__end_block(etmq, tidq);
1121	2143	}
1122	2144
1123	2145	return err;
1124	2146	}
1125	2147
1126	2148	static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
1127		- pid_t tid, u64 time_)
	2149	+ pid_t tid)
1128	2150	{
1129	2151	unsigned int i;
1130	2152	struct auxtrace_queues *queues = &etm->queues;
..	..	@@ -1132,13 +2154,180 @@
1132	2154	for (i = 0; i < queues->nr_queues; i++) {
1133	2155	struct auxtrace_queue *queue = &etm->queues.queue_array[i];
1134	2156	struct cs_etm_queue *etmq = queue->priv;
	2157	+ struct cs_etm_traceid_queue *tidq;
1135	2158
1136		- if (etmq && ((tid == -1) \|\| (etmq->tid == tid))) {
1137		- etmq->time = time_;
1138		- cs_etm__set_pid_tid_cpu(etm, queue);
	2159	+ if (!etmq)
	2160	+ continue;
	2161	+
	2162	+ tidq = cs_etm__etmq_get_traceid_queue(etmq,
	2163	+ CS_ETM_PER_THREAD_TRACEID);
	2164	+
	2165	+ if (!tidq)
	2166	+ continue;
	2167	+
	2168	+ if ((tid == -1) \|\| (tidq->tid == tid)) {
	2169	+ cs_etm__set_pid_tid_cpu(etm, tidq);
1139	2170	cs_etm__run_decoder(etmq);
1140	2171	}
1141	2172	}
	2173	+
	2174	+ return 0;
	2175	+}
	2176	+
	2177	+static int cs_etm__process_queues(struct cs_etm_auxtrace *etm)
	2178	+{
	2179	+ int ret = 0;
	2180	+ unsigned int cs_queue_nr, queue_nr;
	2181	+ u8 trace_chan_id;
	2182	+ u64 timestamp;
	2183	+ struct auxtrace_queue *queue;
	2184	+ struct cs_etm_queue *etmq;
	2185	+ struct cs_etm_traceid_queue *tidq;
	2186	+
	2187	+ while (1) {
	2188	+ if (!etm->heap.heap_cnt)
	2189	+ goto out;
	2190	+
	2191	+ /* Take the entry at the top of the min heap */
	2192	+ cs_queue_nr = etm->heap.heap_array[0].queue_nr;
	2193	+ queue_nr = TO_QUEUE_NR(cs_queue_nr);
	2194	+ trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
	2195	+ queue = &etm->queues.queue_array[queue_nr];
	2196	+ etmq = queue->priv;
	2197	+
	2198	+ /*
	2199	+ * Remove the top entry from the heap since we are about
	2200	+ * to process it.
	2201	+ */
	2202	+ auxtrace_heap__pop(&etm->heap);
	2203	+
	2204	+ tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
	2205	+ if (!tidq) {
	2206	+ /*
	2207	+ * No traceID queue has been allocated for this traceID,
	2208	+ * which means something somewhere went very wrong. No
	2209	+ * other choice than simply exit.
	2210	+ */
	2211	+ ret = -EINVAL;
	2212	+ goto out;
	2213	+ }
	2214	+
	2215	+ /*
	2216	+ * Packets associated with this timestamp are already in
	2217	+ * the etmq's traceID queue, so process them.
	2218	+ */
	2219	+ ret = cs_etm__process_traceid_queue(etmq, tidq);
	2220	+ if (ret < 0)
	2221	+ goto out;
	2222	+
	2223	+ /*
	2224	+ * Packets for this timestamp have been processed, time to
	2225	+ * move on to the next timestamp, fetching a new auxtrace_buffer
	2226	+ * if need be.
	2227	+ */
	2228	+refetch:
	2229	+ ret = cs_etm__get_data_block(etmq);
	2230	+ if (ret < 0)
	2231	+ goto out;
	2232	+
	2233	+ /*
	2234	+ * No more auxtrace_buffers to process in this etmq, simply
	2235	+ * move on to another entry in the auxtrace_heap.
	2236	+ */
	2237	+ if (!ret)
	2238	+ continue;
	2239	+
	2240	+ ret = cs_etm__decode_data_block(etmq);
	2241	+ if (ret)
	2242	+ goto out;
	2243	+
	2244	+ timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
	2245	+
	2246	+ if (!timestamp) {
	2247	+ /*
	2248	+ * Function cs_etm__decode_data_block() returns when
	2249	+ * there is no more traces to decode in the current
	2250	+ * auxtrace_buffer OR when a timestamp has been
	2251	+ * encountered on any of the traceID queues. Since we
	2252	+ * did not get a timestamp, there is no more traces to
	2253	+ * process in this auxtrace_buffer. As such empty and
	2254	+ * flush all traceID queues.
	2255	+ */
	2256	+ cs_etm__clear_all_traceid_queues(etmq);
	2257	+
	2258	+ /* Fetch another auxtrace_buffer for this etmq */
	2259	+ goto refetch;
	2260	+ }
	2261	+
	2262	+ /*
	2263	+ * Add to the min heap the timestamp for packets that have
	2264	+ * just been decoded. They will be processed and synthesized
	2265	+ * during the next call to cs_etm__process_traceid_queue() for
	2266	+ * this queue/traceID.
	2267	+ */
	2268	+ cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
	2269	+ ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);
	2270	+ }
	2271	+
	2272	+out:
	2273	+ return ret;
	2274	+}
	2275	+
	2276	+static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
	2277	+ union perf_event *event)
	2278	+{
	2279	+ struct thread *th;
	2280	+
	2281	+ if (etm->timeless_decoding)
	2282	+ return 0;
	2283	+
	2284	+ /*
	2285	+ * Add the tid/pid to the log so that we can get a match when
	2286	+ * we get a contextID from the decoder.
	2287	+ */
	2288	+ th = machine__findnew_thread(etm->machine,
	2289	+ event->itrace_start.pid,
	2290	+ event->itrace_start.tid);
	2291	+ if (!th)
	2292	+ return -ENOMEM;
	2293	+
	2294	+ thread__put(th);
	2295	+
	2296	+ return 0;
	2297	+}
	2298	+
	2299	+static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
	2300	+ union perf_event *event)
	2301	+{
	2302	+ struct thread *th;
	2303	+ bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
	2304	+
	2305	+ /*
	2306	+ * Context switch in per-thread mode are irrelevant since perf
	2307	+ * will start/stop tracing as the process is scheduled.
	2308	+ */
	2309	+ if (etm->timeless_decoding)
	2310	+ return 0;
	2311	+
	2312	+ /*
	2313	+ * SWITCH_IN events carry the next process to be switched out while
	2314	+ * SWITCH_OUT events carry the process to be switched in. As such
	2315	+ * we don't care about IN events.
	2316	+ */
	2317	+ if (!out)
	2318	+ return 0;
	2319	+
	2320	+ /*
	2321	+ * Add the tid/pid to the log so that we can get a match when
	2322	+ * we get a contextID from the decoder.
	2323	+ */
	2324	+ th = machine__findnew_thread(etm->machine,
	2325	+ event->context_switch.next_prev_pid,
	2326	+ event->context_switch.next_prev_tid);
	2327	+ if (!th)
	2328	+ return -ENOMEM;
	2329	+
	2330	+ thread__put(th);
1142	2331
1143	2332	return 0;
1144	2333	}
..	..	@@ -1162,9 +2351,6 @@
1162	2351	return -EINVAL;
1163	2352	}
1164	2353
1165		- if (!etm->timeless_decoding)
1166		- return -EINVAL;
1167		-
1168	2354	if (sample->time && (sample->time != (u64) -1))
1169	2355	timestamp = sample->time;
1170	2356	else
..	..	@@ -1176,10 +2362,19 @@
1176	2362	return err;
1177	2363	}
1178	2364
1179		- if (event->header.type == PERF_RECORD_EXIT)
	2365	+ if (etm->timeless_decoding &&
	2366	+ event->header.type == PERF_RECORD_EXIT)
1180	2367	return cs_etm__process_timeless_queues(etm,
1181		- event->fork.tid,
1182		- sample->time);
	2368	+ event->fork.tid);
	2369	+
	2370	+ if (event->header.type == PERF_RECORD_ITRACE_START)
	2371	+ return cs_etm__process_itrace_start(etm, event);
	2372	+ else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
	2373	+ return cs_etm__process_switch_cpu_wide(etm, event);
	2374	+
	2375	+ if (!etm->timeless_decoding &&
	2376	+ event->header.type == PERF_RECORD_AUX)
	2377	+ return cs_etm__process_queues(etm);
1183	2378
1184	2379	return 0;
1185	2380	}
..	..	@@ -1223,8 +2418,8 @@
1223	2418
1224	2419	static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm)
1225	2420	{
1226		- struct perf_evsel *evsel;
1227		- struct perf_evlist *evlist = etm->session->evlist;
	2421	+ struct evsel *evsel;
	2422	+ struct evlist *evlist = etm->session->evlist;
1228	2423	bool timeless_decoding = true;
1229	2424
1230	2425	/*
..	..	@@ -1232,7 +2427,7 @@
1232	2427	* with the time bit set.
1233	2428	*/
1234	2429	evlist__for_each_entry(evlist, evsel) {
1235		- if ((evsel->attr.sample_type & PERF_SAMPLE_TIME))
	2430	+ if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
1236	2431	timeless_decoding = false;
1237	2432	}
1238	2433
..	..	@@ -1266,7 +2461,7 @@
1266	2461	[CS_ETMV4_TRCAUTHSTATUS] = " TRCAUTHSTATUS %llx\n",
1267	2462	};
1268	2463
1269		-static void cs_etm__print_auxtrace_info(u64 *val, int num)
	2464	+static void cs_etm__print_auxtrace_info(__u64 *val, int num)
1270	2465	{
1271	2466	int i, j, cpu = 0;
1272	2467
..	..	@@ -1289,7 +2484,7 @@
1289	2484	int cs_etm__process_auxtrace_info(union perf_event *event,
1290	2485	struct perf_session *session)
1291	2486	{
1292		- struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
	2487	+ struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
1293	2488	struct cs_etm_auxtrace *etm = NULL;
1294	2489	struct int_node *inode;
1295	2490	unsigned int pmu_type;
..	..	@@ -1389,7 +2584,7 @@
1389	2584
1390	2585	/* Something went wrong, no need to continue */
1391	2586	if (!inode) {
1392		- err = PTR_ERR(inode);
	2587	+ err = -ENOMEM;
1393	2588	goto err_free_metadata;
1394	2589	}
1395	2590
..	..	@@ -1443,11 +2638,14 @@
1443	2638	etm->auxtrace.flush_events = cs_etm__flush_events;
1444	2639	etm->auxtrace.free_events = cs_etm__free_events;
1445	2640	etm->auxtrace.free = cs_etm__free;
	2641	+ etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace;
1446	2642	session->auxtrace = &etm->auxtrace;
1447	2643
1448	2644	etm->unknown_thread = thread__new(999999999, 999999999);
1449		- if (!etm->unknown_thread)
	2645	+ if (!etm->unknown_thread) {
	2646	+ err = -ENOMEM;
1450	2647	goto err_free_queues;
	2648	+ }
1451	2649
1452	2650	/*
1453	2651	* Initialize list node so that at thread__zput() we can avoid
..	..	@@ -1459,18 +2657,21 @@
1459	2657	if (err)
1460	2658	goto err_delete_thread;
1461	2659
1462		- if (thread__init_map_groups(etm->unknown_thread, etm->machine))
	2660	+ if (thread__init_maps(etm->unknown_thread, etm->machine)) {
	2661	+ err = -ENOMEM;
1463	2662	goto err_delete_thread;
	2663	+ }
1464	2664
1465	2665	if (dump_trace) {
1466	2666	cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu);
1467	2667	return 0;
1468	2668	}
1469	2669
1470		- if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
	2670	+ if (session->itrace_synth_opts->set) {
1471	2671	etm->synth_opts = *session->itrace_synth_opts;
1472	2672	} else {
1473		- itrace_synth_opts__set_default(&etm->synth_opts);
	2673	+ itrace_synth_opts__set_default(&etm->synth_opts,
	2674	+ session->itrace_synth_opts->default_no_sample);
1474	2675	etm->synth_opts.callchain = false;
1475	2676	}
1476	2677
..	..	@@ -1496,12 +2697,12 @@
1496	2697	err_free_metadata:
1497	2698	/* No need to check @metadata[j], free(NULL) is supported */
1498	2699	for (j = 0; j < num_cpu; j++)
1499		- free(metadata[j]);
	2700	+ zfree(&metadata[j]);
1500	2701	zfree(&metadata);
1501	2702	err_free_traceid_list:
1502	2703	intlist__delete(traceid_list);
1503	2704	err_free_hdr:
1504	2705	zfree(&hdr);
1505	2706
1506		- return -EINVAL;
	2707	+ return err;
1507	2708	}