~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,16 +1,7 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* intel_pt_decoder.c: Intel Processor Trace support
3	4	* Copyright (c) 2013-2014, Intel Corporation.
4		- *
5		- * This program is free software; you can redistribute it and/or modify it
6		- * under the terms and conditions of the GNU General Public License,
7		- * version 2, as published by the Free Software Foundation.
8		- *
9		- * This program is distributed in the hope it will be useful, but WITHOUT
10		- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11		- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12		- * more details.
13		- *
14	5	*/
15	6
16	7	#ifndef _GNU_SOURCE
..	..	@@ -23,15 +14,22 @@
23	14	#include <stdint.h>
24	15	#include <inttypes.h>
25	16	#include <linux/compiler.h>
	17	+#include <linux/string.h>
	18	+#include <linux/zalloc.h>
26	19
27		-#include "../cache.h"
28		-#include "../util.h"
29	20	#include "../auxtrace.h"
30	21
31	22	#include "intel-pt-insn-decoder.h"
32	23	#include "intel-pt-pkt-decoder.h"
33	24	#include "intel-pt-decoder.h"
34	25	#include "intel-pt-log.h"
	26	+
	27	+#define BITULL(x) (1ULL << (x))
	28	+
	29	+/* IA32_RTIT_CTL MSR bits */
	30	+#define INTEL_PT_CYC_ENABLE BITULL(1)
	31	+#define INTEL_PT_CYC_THRESHOLD (BITULL(22) \| BITULL(21) \| BITULL(20) \| BITULL(19))
	32	+#define INTEL_PT_CYC_THRESHOLD_SHIFT 19
35	33
36	34	#define INTEL_PT_BLK_SIZE 1024
37	35
..	..	@@ -64,6 +62,7 @@
64	62	INTEL_PT_STATE_TIP_PGD,
65	63	INTEL_PT_STATE_FUP,
66	64	INTEL_PT_STATE_FUP_NO_TIP,
	65	+ INTEL_PT_STATE_RESAMPLE,
67	66	};
68	67
69	68	static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
..	..	@@ -74,6 +73,7 @@
74	73	case INTEL_PT_STATE_ERR_RESYNC:
75	74	case INTEL_PT_STATE_IN_SYNC:
76	75	case INTEL_PT_STATE_TNT_CONT:
	76	+ case INTEL_PT_STATE_RESAMPLE:
77	77	return true;
78	78	case INTEL_PT_STATE_TNT:
79	79	case INTEL_PT_STATE_TIP:
..	..	@@ -104,6 +104,7 @@
104	104	uint64_t insn_cnt_ptr, uint64_t ip, uint64_t to_ip,
105	105	uint64_t max_insn_cnt, void *data);
106	106	bool (pgd_ip)(uint64_t ip, void data);
	107	+ int (lookahead)(void data, intel_pt_lookahead_cb_t cb, void *cb_data);
107	108	void *data;
108	109	struct intel_pt_state state;
109	110	const unsigned char *buf;
..	..	@@ -116,6 +117,10 @@
116	117	bool have_cyc;
117	118	bool fixup_last_mtc;
118	119	bool have_last_ip;
	120	+ bool in_psb;
	121	+ bool hop;
	122	+ bool hop_psb_fup;
	123	+ bool leap;
119	124	enum intel_pt_param_flags flags;
120	125	uint64_t pos;
121	126	uint64_t last_ip;
..	..	@@ -124,6 +129,7 @@
124	129	uint64_t timestamp;
125	130	uint64_t tsc_timestamp;
126	131	uint64_t ref_timestamp;
	132	+ uint64_t buf_timestamp;
127	133	uint64_t sample_timestamp;
128	134	uint64_t ret_addr;
129	135	uint64_t ctc_timestamp;
..	..	@@ -139,6 +145,10 @@
139	145	int mtc_shift;
140	146	struct intel_pt_stack stack;
141	147	enum intel_pt_pkt_state pkt_state;
	148	+ enum intel_pt_pkt_ctx pkt_ctx;
	149	+ enum intel_pt_pkt_ctx prev_pkt_ctx;
	150	+ enum intel_pt_blk_type blk_type;
	151	+ int blk_type_pos;
142	152	struct intel_pt_pkt packet;
143	153	struct intel_pt_pkt tnt;
144	154	int pkt_step;
..	..	@@ -160,6 +170,13 @@
160	170	uint64_t period_mask;
161	171	uint64_t period_ticks;
162	172	uint64_t last_masked_timestamp;
	173	+ uint64_t tot_cyc_cnt;
	174	+ uint64_t sample_tot_cyc_cnt;
	175	+ uint64_t base_cyc_cnt;
	176	+ uint64_t cyc_cnt_timestamp;
	177	+ uint64_t ctl;
	178	+ uint64_t cyc_threshold;
	179	+ double tsc_to_cyc;
163	180	bool continuous_period;
164	181	bool overflow;
165	182	bool set_fup_tx_flags;
..	..	@@ -167,6 +184,8 @@
167	184	bool set_fup_mwait;
168	185	bool set_fup_pwre;
169	186	bool set_fup_exstop;
	187	+ bool set_fup_bep;
	188	+ bool sample_cyc;
170	189	unsigned int fup_tx_flags;
171	190	unsigned int tx_flags;
172	191	uint64_t fup_ptw_payload;
..	..	@@ -192,6 +211,14 @@
192	211	x >>= 1;
193	212
194	213	return x << i;
	214	+}
	215	+
	216	+static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
	217	+{
	218	+ if (!(ctl & INTEL_PT_CYC_ENABLE))
	219	+ return 0;
	220	+
	221	+ return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
195	222	}
196	223
197	224	static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
..	..	@@ -226,17 +253,23 @@
226	253	decoder->get_trace = params->get_trace;
227	254	decoder->walk_insn = params->walk_insn;
228	255	decoder->pgd_ip = params->pgd_ip;
	256	+ decoder->lookahead = params->lookahead;
229	257	decoder->data = params->data;
230	258	decoder->return_compression = params->return_compression;
231	259	decoder->branch_enable = params->branch_enable;
	260	+ decoder->hop = params->quick >= 1;
	261	+ decoder->leap = params->quick >= 2;
232	262
233	263	decoder->flags = params->flags;
234	264
	265	+ decoder->ctl = params->ctl;
235	266	decoder->period = params->period;
236	267	decoder->period_type = params->period_type;
237	268
238	269	decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
239	270	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
	271	+
	272	+ decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
240	273
241	274	intel_pt_setup_period(decoder);
242	275
..	..	@@ -268,6 +301,9 @@
268	301	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
269	302	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
270	303	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
	304	+
	305	+ if (decoder->hop)
	306	+ intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
271	307
272	308	return decoder;
273	309	}
..	..	@@ -479,7 +515,21 @@
479	515	return -EBADMSG;
480	516	}
481	517
482		-static int intel_pt_get_data(struct intel_pt_decoder *decoder)
	518	+static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
	519	+{
	520	+ decoder->sample_timestamp = decoder->timestamp;
	521	+ decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
	522	+}
	523	+
	524	+static void intel_pt_reposition(struct intel_pt_decoder *decoder)
	525	+{
	526	+ decoder->ip = 0;
	527	+ decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
	528	+ decoder->timestamp = 0;
	529	+ decoder->have_tma = false;
	530	+}
	531	+
	532	+static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
483	533	{
484	534	struct intel_pt_buffer buffer = { .buf = 0, };
485	535	int ret;
..	..	@@ -496,12 +546,10 @@
496	546	intel_pt_log("No more data\n");
497	547	return -ENODATA;
498	548	}
499		- if (!buffer.consecutive) {
500		- decoder->ip = 0;
501		- decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
	549	+ decoder->buf_timestamp = buffer.ref_timestamp;
	550	+ if (!buffer.consecutive \|\| reposition) {
	551	+ intel_pt_reposition(decoder);
502	552	decoder->ref_timestamp = buffer.ref_timestamp;
503		- decoder->timestamp = 0;
504		- decoder->have_tma = false;
505	553	decoder->state.trace_nr = buffer.trace_nr;
506	554	intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
507	555	decoder->ref_timestamp);
..	..	@@ -511,10 +559,11 @@
511	559	return 0;
512	560	}
513	561
514		-static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
	562	+static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
	563	+ bool reposition)
515	564	{
516	565	if (!decoder->next_buf)
517		- return intel_pt_get_data(decoder);
	566	+ return intel_pt_get_data(decoder, reposition);
518	567
519	568	decoder->buf = decoder->next_buf;
520	569	decoder->len = decoder->next_len;
..	..	@@ -533,7 +582,7 @@
533	582	len = decoder->len;
534	583	memcpy(buf, decoder->buf, len);
535	584
536		- ret = intel_pt_get_data(decoder);
	585	+ ret = intel_pt_get_data(decoder, false);
537	586	if (ret) {
538	587	decoder->pos += old_len;
539	588	return ret < 0 ? ret : -EINVAL;
..	..	@@ -545,7 +594,8 @@
545	594	memcpy(buf + len, decoder->buf, n);
546	595	len += n;
547	596
548		- ret = intel_pt_get_packet(buf, len, &decoder->packet);
	597	+ decoder->prev_pkt_ctx = decoder->pkt_ctx;
	598	+ ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
549	599	if (ret < (int)old_len) {
550	600	decoder->next_buf = decoder->buf;
551	601	decoder->next_len = decoder->len;
..	..	@@ -580,6 +630,7 @@
580	630	{
581	631	struct intel_pt_pkt_info pkt_info;
582	632	const unsigned char *buf = decoder->buf;
	633	+ enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
583	634	size_t len = decoder->len;
584	635	int ret;
585	636
..	..	@@ -598,7 +649,8 @@
598	649	if (!len)
599	650	return INTEL_PT_NEED_MORE_BYTES;
600	651
601		- ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
	652	+ ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
	653	+ &pkt_ctx);
602	654	if (!ret)
603	655	return INTEL_PT_NEED_MORE_BYTES;
604	656	if (ret < 0)
..	..	@@ -673,6 +725,10 @@
673	725	case INTEL_PT_MNT:
674	726	case INTEL_PT_PTWRITE:
675	727	case INTEL_PT_PTWRITE_IP:
	728	+ case INTEL_PT_BBP:
	729	+ case INTEL_PT_BIP:
	730	+ case INTEL_PT_BEP:
	731	+ case INTEL_PT_BEP_IP:
676	732	return 0;
677	733
678	734	case INTEL_PT_MTC:
..	..	@@ -859,14 +915,15 @@
859	915	decoder->len -= decoder->pkt_step;
860	916
861	917	if (!decoder->len) {
862		- ret = intel_pt_get_next_data(decoder);
	918	+ ret = intel_pt_get_next_data(decoder, false);
863	919	if (ret)
864	920	return ret;
865	921	}
866	922
	923	+ decoder->prev_pkt_ctx = decoder->pkt_ctx;
867	924	ret = intel_pt_get_packet(decoder->buf, decoder->len,
868		- &decoder->packet);
869		- if (ret == INTEL_PT_NEED_MORE_BYTES &&
	925	+ &decoder->packet, &decoder->pkt_ctx);
	926	+ if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
870	927	decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
871	928	ret = intel_pt_get_split_packet(decoder);
872	929	if (ret < 0)
..	..	@@ -1057,53 +1114,69 @@
1057	1114
1058	1115	static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1059	1116	{
	1117	+ enum intel_pt_sample_type type = decoder->state.type;
1060	1118	bool ret = false;
	1119	+
	1120	+ decoder->state.type &= ~INTEL_PT_BRANCH;
1061	1121
1062	1122	if (decoder->set_fup_tx_flags) {
1063	1123	decoder->set_fup_tx_flags = false;
1064	1124	decoder->tx_flags = decoder->fup_tx_flags;
1065		- decoder->state.type = INTEL_PT_TRANSACTION;
	1125	+ decoder->state.type \|= INTEL_PT_TRANSACTION;
1066	1126	if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1067	1127	decoder->state.type \|= INTEL_PT_BRANCH;
1068		- decoder->state.from_ip = decoder->ip;
1069		- decoder->state.to_ip = 0;
1070	1128	decoder->state.flags = decoder->fup_tx_flags;
1071		- return true;
	1129	+ ret = true;
1072	1130	}
1073	1131	if (decoder->set_fup_ptw) {
1074	1132	decoder->set_fup_ptw = false;
1075		- decoder->state.type = INTEL_PT_PTW;
	1133	+ decoder->state.type \|= INTEL_PT_PTW;
1076	1134	decoder->state.flags \|= INTEL_PT_FUP_IP;
1077		- decoder->state.from_ip = decoder->ip;
1078		- decoder->state.to_ip = 0;
1079	1135	decoder->state.ptw_payload = decoder->fup_ptw_payload;
1080		- return true;
	1136	+ ret = true;
1081	1137	}
1082	1138	if (decoder->set_fup_mwait) {
1083	1139	decoder->set_fup_mwait = false;
1084		- decoder->state.type = INTEL_PT_MWAIT_OP;
1085		- decoder->state.from_ip = decoder->ip;
1086		- decoder->state.to_ip = 0;
	1140	+ decoder->state.type \|= INTEL_PT_MWAIT_OP;
1087	1141	decoder->state.mwait_payload = decoder->fup_mwait_payload;
1088	1142	ret = true;
1089	1143	}
1090	1144	if (decoder->set_fup_pwre) {
1091	1145	decoder->set_fup_pwre = false;
1092	1146	decoder->state.type \|= INTEL_PT_PWR_ENTRY;
1093		- decoder->state.type &= ~INTEL_PT_BRANCH;
1094		- decoder->state.from_ip = decoder->ip;
1095		- decoder->state.to_ip = 0;
1096	1147	decoder->state.pwre_payload = decoder->fup_pwre_payload;
1097	1148	ret = true;
1098	1149	}
1099	1150	if (decoder->set_fup_exstop) {
1100	1151	decoder->set_fup_exstop = false;
1101	1152	decoder->state.type \|= INTEL_PT_EX_STOP;
1102		- decoder->state.type &= ~INTEL_PT_BRANCH;
1103	1153	decoder->state.flags \|= INTEL_PT_FUP_IP;
	1154	+ ret = true;
	1155	+ }
	1156	+ if (decoder->set_fup_bep) {
	1157	+ decoder->set_fup_bep = false;
	1158	+ decoder->state.type \|= INTEL_PT_BLK_ITEMS;
	1159	+ ret = true;
	1160	+ }
	1161	+ if (decoder->overflow) {
	1162	+ decoder->overflow = false;
	1163	+ if (!ret && !decoder->pge) {
	1164	+ if (decoder->hop) {
	1165	+ decoder->state.type = 0;
	1166	+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
	1167	+ }
	1168	+ decoder->pge = true;
	1169	+ decoder->state.type \|= INTEL_PT_BRANCH \| INTEL_PT_TRACE_BEGIN;
	1170	+ decoder->state.from_ip = 0;
	1171	+ decoder->state.to_ip = decoder->ip;
	1172	+ return true;
	1173	+ }
	1174	+ }
	1175	+ if (ret) {
1104	1176	decoder->state.from_ip = decoder->ip;
1105	1177	decoder->state.to_ip = 0;
1106		- ret = true;
	1178	+ } else {
	1179	+ decoder->state.type = type;
1107	1180	}
1108	1181	return ret;
1109	1182	}
..	..	@@ -1176,7 +1249,7 @@
1176	1249	decoder->pge = false;
1177	1250	decoder->continuous_period = false;
1178	1251	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1179		- decoder->state.to_ip = 0;
	1252	+ decoder->state.type \|= INTEL_PT_TRACE_END;
1180	1253	return 0;
1181	1254	}
1182	1255	if (err == INTEL_PT_RETURN)
..	..	@@ -1190,9 +1263,13 @@
1190	1263	decoder->continuous_period = false;
1191	1264	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1192	1265	decoder->state.from_ip = decoder->ip;
1193		- decoder->state.to_ip = 0;
1194		- if (decoder->packet.count != 0)
	1266	+ if (decoder->packet.count == 0) {
	1267	+ decoder->state.to_ip = 0;
	1268	+ } else {
	1269	+ decoder->state.to_ip = decoder->last_ip;
1195	1270	decoder->ip = decoder->last_ip;
	1271	+ }
	1272	+ decoder->state.type \|= INTEL_PT_TRACE_END;
1196	1273	} else {
1197	1274	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1198	1275	decoder->state.from_ip = decoder->ip;
..	..	@@ -1219,7 +1296,8 @@
1219	1296	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1220	1297	decoder->ip = to_ip;
1221	1298	decoder->state.from_ip = decoder->ip;
1222		- decoder->state.to_ip = 0;
	1299	+ decoder->state.to_ip = to_ip;
	1300	+ decoder->state.type \|= INTEL_PT_TRACE_END;
1223	1301	return 0;
1224	1302	}
1225	1303	intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
..	..	@@ -1317,10 +1395,10 @@
1317	1395	decoder->ip += intel_pt_insn.length;
1318	1396	return 0;
1319	1397	}
	1398	+ decoder->sample_cyc = false;
1320	1399	decoder->ip += intel_pt_insn.length;
1321	1400	if (!decoder->tnt.count) {
1322		- decoder->sample_timestamp = decoder->timestamp;
1323		- decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
	1401	+ intel_pt_update_sample_time(decoder);
1324	1402	return -EAGAIN;
1325	1403	}
1326	1404	decoder->tnt.payload <<= 1;
..	..	@@ -1354,6 +1432,21 @@
1354	1432	return 0;
1355	1433	}
1356	1434
	1435	+static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
	1436	+{
	1437	+ timestamp \|= (ref_timestamp & (0xffULL << 56));
	1438	+
	1439	+ if (timestamp < ref_timestamp) {
	1440	+ if (ref_timestamp - timestamp > (1ULL << 55))
	1441	+ timestamp += (1ULL << 56);
	1442	+ } else {
	1443	+ if (timestamp - ref_timestamp > (1ULL << 55))
	1444	+ timestamp -= (1ULL << 56);
	1445	+ }
	1446	+
	1447	+ return timestamp;
	1448	+}
	1449	+
1357	1450	static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1358	1451	{
1359	1452	uint64_t timestamp;
..	..	@@ -1361,15 +1454,8 @@
1361	1454	decoder->have_tma = false;
1362	1455
1363	1456	if (decoder->ref_timestamp) {
1364		- timestamp = decoder->packet.payload \|
1365		- (decoder->ref_timestamp & (0xffULL << 56));
1366		- if (timestamp < decoder->ref_timestamp) {
1367		- if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1368		- timestamp += (1ULL << 56);
1369		- } else {
1370		- if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1371		- timestamp -= (1ULL << 56);
1372		- }
	1457	+ timestamp = intel_pt_8b_tsc(decoder->packet.payload,
	1458	+ decoder->ref_timestamp);
1373	1459	decoder->tsc_timestamp = timestamp;
1374	1460	decoder->timestamp = timestamp;
1375	1461	decoder->ref_timestamp = 0;
..	..	@@ -1408,9 +1494,54 @@
1408	1494	intel_pt_log("ERROR: Buffer overflow\n");
1409	1495	intel_pt_clear_tx_flags(decoder);
1410	1496	decoder->timestamp_insn_cnt = 0;
1411		- decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
	1497	+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
	1498	+ decoder->state.from_ip = decoder->ip;
	1499	+ decoder->ip = 0;
	1500	+ decoder->pge = false;
	1501	+ decoder->set_fup_tx_flags = false;
	1502	+ decoder->set_fup_ptw = false;
	1503	+ decoder->set_fup_mwait = false;
	1504	+ decoder->set_fup_pwre = false;
	1505	+ decoder->set_fup_exstop = false;
	1506	+ decoder->set_fup_bep = false;
1412	1507	decoder->overflow = true;
1413	1508	return -EOVERFLOW;
	1509	+}
	1510	+
	1511	+static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
	1512	+{
	1513	+ if (decoder->have_cyc)
	1514	+ return;
	1515	+
	1516	+ decoder->cyc_cnt_timestamp = decoder->timestamp;
	1517	+ decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
	1518	+}
	1519	+
	1520	+static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
	1521	+{
	1522	+ decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
	1523	+
	1524	+ if (decoder->pge)
	1525	+ intel_pt_mtc_cyc_cnt_pge(decoder);
	1526	+}
	1527	+
	1528	+static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
	1529	+{
	1530	+ uint64_t tot_cyc_cnt, tsc_delta;
	1531	+
	1532	+ if (decoder->have_cyc)
	1533	+ return;
	1534	+
	1535	+ decoder->sample_cyc = true;
	1536	+
	1537	+ if (!decoder->pge \|\| decoder->timestamp <= decoder->cyc_cnt_timestamp)
	1538	+ return;
	1539	+
	1540	+ tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
	1541	+ tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
	1542	+
	1543	+ if (tot_cyc_cnt > decoder->tot_cyc_cnt)
	1544	+ decoder->tot_cyc_cnt = tot_cyc_cnt;
1414	1545	}
1415	1546
1416	1547	static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
..	..	@@ -1421,6 +1552,11 @@
1421	1552
1422	1553	if (!decoder->tsc_ctc_ratio_d)
1423	1554	return;
	1555	+
	1556	+ if (decoder->pge && !decoder->in_psb)
	1557	+ intel_pt_mtc_cyc_cnt_pge(decoder);
	1558	+ else
	1559	+ intel_pt_mtc_cyc_cnt_upd(decoder);
1424	1560
1425	1561	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1426	1562	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
..	..	@@ -1477,6 +1613,8 @@
1477	1613	else
1478	1614	decoder->timestamp = timestamp;
1479	1615
	1616	+ intel_pt_mtc_cyc_cnt_upd(decoder);
	1617	+
1480	1618	decoder->timestamp_insn_cnt = 0;
1481	1619	decoder->last_mtc = mtc;
1482	1620
..	..	@@ -1486,6 +1624,8 @@
1486	1624	decoder->have_calc_cyc_to_tsc = false;
1487	1625	intel_pt_calc_cyc_to_tsc(decoder, true);
1488	1626	}
	1627	+
	1628	+ intel_pt_log_to("Setting timestamp", decoder->timestamp);
1489	1629	}
1490	1630
1491	1631	static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
..	..	@@ -1499,6 +1639,10 @@
1499	1639
1500	1640	decoder->cbr = cbr;
1501	1641	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
	1642	+ decoder->cyc_ref_timestamp = decoder->timestamp;
	1643	+ decoder->cycle_cnt = 0;
	1644	+
	1645	+ intel_pt_mtc_cyc_cnt_cbr(decoder);
1502	1646	}
1503	1647
1504	1648	static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
..	..	@@ -1508,6 +1652,9 @@
1508	1652	decoder->have_cyc = true;
1509	1653
1510	1654	decoder->cycle_cnt += decoder->packet.payload;
	1655	+ if (decoder->pge)
	1656	+ decoder->tot_cyc_cnt += decoder->packet.payload;
	1657	+ decoder->sample_cyc = true;
1511	1658
1512	1659	if (!decoder->cyc_ref_timestamp)
1513	1660	return;
..	..	@@ -1526,6 +1673,48 @@
1526	1673	decoder->timestamp = timestamp;
1527	1674
1528	1675	decoder->timestamp_insn_cnt = 0;
	1676	+
	1677	+ intel_pt_log_to("Setting timestamp", decoder->timestamp);
	1678	+}
	1679	+
	1680	+static void intel_pt_bbp(struct intel_pt_decoder *decoder)
	1681	+{
	1682	+ if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
	1683	+ memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
	1684	+ decoder->state.items.is_32_bit = false;
	1685	+ }
	1686	+ decoder->blk_type = decoder->packet.payload;
	1687	+ decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
	1688	+ if (decoder->blk_type == INTEL_PT_GP_REGS)
	1689	+ decoder->state.items.is_32_bit = decoder->packet.count;
	1690	+ if (decoder->blk_type_pos < 0) {
	1691	+ intel_pt_log("WARNING: Unknown block type %u\n",
	1692	+ decoder->blk_type);
	1693	+ } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
	1694	+ intel_pt_log("WARNING: Duplicate block type %u\n",
	1695	+ decoder->blk_type);
	1696	+ }
	1697	+}
	1698	+
	1699	+static void intel_pt_bip(struct intel_pt_decoder *decoder)
	1700	+{
	1701	+ uint32_t id = decoder->packet.count;
	1702	+ uint32_t bit = 1 << id;
	1703	+ int pos = decoder->blk_type_pos;
	1704	+
	1705	+ if (pos < 0 \|\| id >= INTEL_PT_BLK_ITEM_ID_CNT) {
	1706	+ intel_pt_log("WARNING: Unknown block item %u type %d\n",
	1707	+ id, decoder->blk_type);
	1708	+ return;
	1709	+ }
	1710	+
	1711	+ if (decoder->state.items.mask[pos] & bit) {
	1712	+ intel_pt_log("WARNING: Duplicate block item %u type %d\n",
	1713	+ id, decoder->blk_type);
	1714	+ }
	1715	+
	1716	+ decoder->state.items.mask[pos] \|= bit;
	1717	+ decoder->state.items.val[pos][id] = decoder->packet.payload;
1529	1718	}
1530	1719
1531	1720	/* Walk PSB+ packets when already in sync. */
..	..	@@ -1533,14 +1722,17 @@
1533	1722	{
1534	1723	int err;
1535	1724
	1725	+ decoder->in_psb = true;
	1726	+
1536	1727	while (1) {
1537	1728	err = intel_pt_get_next_packet(decoder);
1538	1729	if (err)
1539		- return err;
	1730	+ goto out;
1540	1731
1541	1732	switch (decoder->packet.type) {
1542	1733	case INTEL_PT_PSBEND:
1543		- return 0;
	1734	+ err = 0;
	1735	+ goto out;
1544	1736
1545	1737	case INTEL_PT_TIP_PGD:
1546	1738	case INTEL_PT_TIP_PGE:
..	..	@@ -1556,12 +1748,18 @@
1556	1748	case INTEL_PT_MWAIT:
1557	1749	case INTEL_PT_PWRE:
1558	1750	case INTEL_PT_PWRX:
	1751	+ case INTEL_PT_BBP:
	1752	+ case INTEL_PT_BIP:
	1753	+ case INTEL_PT_BEP:
	1754	+ case INTEL_PT_BEP_IP:
1559	1755	decoder->have_tma = false;
1560	1756	intel_pt_log("ERROR: Unexpected packet\n");
1561		- return -EAGAIN;
	1757	+ err = -EAGAIN;
	1758	+ goto out;
1562	1759
1563	1760	case INTEL_PT_OVF:
1564		- return intel_pt_overflow(decoder);
	1761	+ err = intel_pt_overflow(decoder);
	1762	+ goto out;
1565	1763
1566	1764	case INTEL_PT_TSC:
1567	1765	intel_pt_calc_tsc_timestamp(decoder);
..	..	@@ -1585,8 +1783,14 @@
1585	1783
1586	1784	case INTEL_PT_FUP:
1587	1785	decoder->pge = true;
1588		- if (decoder->packet.count)
	1786	+ if (decoder->packet.count) {
1589	1787	intel_pt_set_last_ip(decoder);
	1788	+ if (decoder->hop) {
	1789	+ /* Act on FUP at PSBEND */
	1790	+ decoder->ip = decoder->last_ip;
	1791	+ decoder->hop_psb_fup = true;
	1792	+ }
	1793	+ }
1590	1794	break;
1591	1795
1592	1796	case INTEL_PT_MODE_TSX:
..	..	@@ -1610,6 +1814,10 @@
1610	1814	break;
1611	1815	}
1612	1816	}
	1817	+out:
	1818	+ decoder->in_psb = false;
	1819	+
	1820	+ return err;
1613	1821	}
1614	1822
1615	1823	static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
..	..	@@ -1646,6 +1854,10 @@
1646	1854	case INTEL_PT_MWAIT:
1647	1855	case INTEL_PT_PWRE:
1648	1856	case INTEL_PT_PWRX:
	1857	+ case INTEL_PT_BBP:
	1858	+ case INTEL_PT_BIP:
	1859	+ case INTEL_PT_BEP:
	1860	+ case INTEL_PT_BEP_IP:
1649	1861	intel_pt_log("ERROR: Missing TIP after FUP\n");
1650	1862	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1651	1863	decoder->pkt_step = 0;
..	..	@@ -1660,14 +1872,15 @@
1660	1872
1661	1873	case INTEL_PT_TIP_PGD:
1662	1874	decoder->state.from_ip = decoder->ip;
1663		- decoder->state.to_ip = 0;
1664		- if (decoder->packet.count != 0) {
	1875	+ if (decoder->packet.count == 0) {
	1876	+ decoder->state.to_ip = 0;
	1877	+ } else {
1665	1878	intel_pt_set_ip(decoder);
1666		- intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1667		- decoder->ip);
	1879	+ decoder->state.to_ip = decoder->ip;
1668	1880	}
1669	1881	decoder->pge = false;
1670	1882	decoder->continuous_period = false;
	1883	+ decoder->state.type \|= INTEL_PT_TRACE_END;
1671	1884	return 0;
1672	1885
1673	1886	case INTEL_PT_TIP_PGE:
..	..	@@ -1681,6 +1894,8 @@
1681	1894	intel_pt_set_ip(decoder);
1682	1895	decoder->state.to_ip = decoder->ip;
1683	1896	}
	1897	+ decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
	1898	+ intel_pt_mtc_cyc_cnt_pge(decoder);
1684	1899	return 0;
1685	1900
1686	1901	case INTEL_PT_TIP:
..	..	@@ -1722,8 +1937,136 @@
1722	1937	}
1723	1938	}
1724	1939
	1940	+static int intel_pt_resample(struct intel_pt_decoder *decoder)
	1941	+{
	1942	+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
	1943	+ decoder->state.type = INTEL_PT_INSTRUCTION;
	1944	+ decoder->state.from_ip = decoder->ip;
	1945	+ decoder->state.to_ip = 0;
	1946	+ return 0;
	1947	+}
	1948	+
	1949	+#define HOP_PROCESS 0
	1950	+#define HOP_IGNORE 1
	1951	+#define HOP_RETURN 2
	1952	+#define HOP_AGAIN 3
	1953	+
	1954	+static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
	1955	+
	1956	+/* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
	1957	+static int intel_pt_hop_trace(struct intel_pt_decoder decoder, bool no_tip, int *err)
	1958	+{
	1959	+ *err = 0;
	1960	+
	1961	+ /* Leap from PSB to PSB, getting ip from FUP within PSB+ */
	1962	+ if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
	1963	+ *err = intel_pt_scan_for_psb(decoder);
	1964	+ if (*err)
	1965	+ return HOP_RETURN;
	1966	+ }
	1967	+
	1968	+ switch (decoder->packet.type) {
	1969	+ case INTEL_PT_TNT:
	1970	+ return HOP_IGNORE;
	1971	+
	1972	+ case INTEL_PT_TIP_PGD:
	1973	+ decoder->pge = false;
	1974	+ if (!decoder->packet.count)
	1975	+ return HOP_IGNORE;
	1976	+ intel_pt_set_ip(decoder);
	1977	+ decoder->state.type \|= INTEL_PT_TRACE_END;
	1978	+ decoder->state.from_ip = 0;
	1979	+ decoder->state.to_ip = decoder->ip;
	1980	+ return HOP_RETURN;
	1981	+
	1982	+ case INTEL_PT_TIP:
	1983	+ if (!decoder->packet.count)
	1984	+ return HOP_IGNORE;
	1985	+ intel_pt_set_ip(decoder);
	1986	+ decoder->state.type = INTEL_PT_INSTRUCTION;
	1987	+ decoder->state.from_ip = decoder->ip;
	1988	+ decoder->state.to_ip = 0;
	1989	+ return HOP_RETURN;
	1990	+
	1991	+ case INTEL_PT_FUP:
	1992	+ if (!decoder->packet.count)
	1993	+ return HOP_IGNORE;
	1994	+ intel_pt_set_ip(decoder);
	1995	+ if (decoder->set_fup_mwait \|\| decoder->set_fup_pwre)
	1996	+ *no_tip = true;
	1997	+ if (!decoder->branch_enable \|\| !decoder->pge)
	1998	+ *no_tip = true;
	1999	+ if (*no_tip) {
	2000	+ decoder->state.type = INTEL_PT_INSTRUCTION;
	2001	+ decoder->state.from_ip = decoder->ip;
	2002	+ decoder->state.to_ip = 0;
	2003	+ intel_pt_fup_event(decoder);
	2004	+ return HOP_RETURN;
	2005	+ }
	2006	+ intel_pt_fup_event(decoder);
	2007	+ decoder->state.type \|= INTEL_PT_INSTRUCTION \| INTEL_PT_BRANCH;
	2008	+ *err = intel_pt_walk_fup_tip(decoder);
	2009	+ if (!*err && decoder->state.to_ip)
	2010	+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
	2011	+ return HOP_RETURN;
	2012	+
	2013	+ case INTEL_PT_PSB:
	2014	+ decoder->last_ip = 0;
	2015	+ decoder->have_last_ip = true;
	2016	+ decoder->hop_psb_fup = false;
	2017	+ *err = intel_pt_walk_psbend(decoder);
	2018	+ if (*err == -EAGAIN)
	2019	+ return HOP_AGAIN;
	2020	+ if (*err)
	2021	+ return HOP_RETURN;
	2022	+ if (decoder->hop_psb_fup) {
	2023	+ decoder->hop_psb_fup = false;
	2024	+ decoder->state.type = INTEL_PT_INSTRUCTION;
	2025	+ decoder->state.from_ip = decoder->ip;
	2026	+ decoder->state.to_ip = 0;
	2027	+ return HOP_RETURN;
	2028	+ }
	2029	+ if (decoder->cbr != decoder->cbr_seen) {
	2030	+ decoder->state.type = 0;
	2031	+ return HOP_RETURN;
	2032	+ }
	2033	+ return HOP_IGNORE;
	2034	+
	2035	+ case INTEL_PT_BAD:
	2036	+ case INTEL_PT_PAD:
	2037	+ case INTEL_PT_TIP_PGE:
	2038	+ case INTEL_PT_TSC:
	2039	+ case INTEL_PT_TMA:
	2040	+ case INTEL_PT_MODE_EXEC:
	2041	+ case INTEL_PT_MODE_TSX:
	2042	+ case INTEL_PT_MTC:
	2043	+ case INTEL_PT_CYC:
	2044	+ case INTEL_PT_VMCS:
	2045	+ case INTEL_PT_PSBEND:
	2046	+ case INTEL_PT_CBR:
	2047	+ case INTEL_PT_TRACESTOP:
	2048	+ case INTEL_PT_PIP:
	2049	+ case INTEL_PT_OVF:
	2050	+ case INTEL_PT_MNT:
	2051	+ case INTEL_PT_PTWRITE:
	2052	+ case INTEL_PT_PTWRITE_IP:
	2053	+ case INTEL_PT_EXSTOP:
	2054	+ case INTEL_PT_EXSTOP_IP:
	2055	+ case INTEL_PT_MWAIT:
	2056	+ case INTEL_PT_PWRE:
	2057	+ case INTEL_PT_PWRX:
	2058	+ case INTEL_PT_BBP:
	2059	+ case INTEL_PT_BIP:
	2060	+ case INTEL_PT_BEP:
	2061	+ case INTEL_PT_BEP_IP:
	2062	+ default:
	2063	+ return HOP_PROCESS;
	2064	+ }
	2065	+}
	2066	+
1725	2067	static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1726	2068	{
	2069	+ int last_packet_type = INTEL_PT_PAD;
1727	2070	bool no_tip = false;
1728	2071	int err;
1729	2072
..	..	@@ -1732,6 +2075,26 @@
1732	2075	if (err)
1733	2076	return err;
1734	2077	next:
	2078	+ err = 0;
	2079	+ if (decoder->cyc_threshold) {
	2080	+ if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
	2081	+ decoder->sample_cyc = false;
	2082	+ last_packet_type = decoder->packet.type;
	2083	+ }
	2084	+
	2085	+ if (decoder->hop) {
	2086	+ switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
	2087	+ case HOP_IGNORE:
	2088	+ continue;
	2089	+ case HOP_RETURN:
	2090	+ return err;
	2091	+ case HOP_AGAIN:
	2092	+ goto next;
	2093	+ default:
	2094	+ break;
	2095	+ }
	2096	+ }
	2097	+
1735	2098	switch (decoder->packet.type) {
1736	2099	case INTEL_PT_TNT:
1737	2100	if (!decoder->packet.count)
..	..	@@ -1751,6 +2114,8 @@
1751	2114
1752	2115	case INTEL_PT_TIP_PGE: {
1753	2116	decoder->pge = true;
	2117	+ decoder->overflow = false;
	2118	+ intel_pt_mtc_cyc_cnt_pge(decoder);
1754	2119	if (decoder->packet.count == 0) {
1755	2120	intel_pt_log_at("Skipping zero TIP.PGE",
1756	2121	decoder->pos);
..	..	@@ -1759,6 +2124,13 @@
1759	2124	intel_pt_set_ip(decoder);
1760	2125	decoder->state.from_ip = 0;
1761	2126	decoder->state.to_ip = decoder->ip;
	2127	+ decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
	2128	+ /*
	2129	+ * In hop mode, resample to get the to_ip as an
	2130	+ * "instruction" sample.
	2131	+ */
	2132	+ if (decoder->hop)
	2133	+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
1762	2134	return 0;
1763	2135	}
1764	2136
..	..	@@ -1779,7 +2151,7 @@
1779	2151	break;
1780	2152	}
1781	2153	intel_pt_set_last_ip(decoder);
1782		- if (!decoder->branch_enable) {
	2154	+ if (!decoder->branch_enable \|\| !decoder->pge) {
1783	2155	decoder->ip = decoder->last_ip;
1784	2156	if (intel_pt_fup_event(decoder))
1785	2157	return 0;
..	..	@@ -1817,6 +2189,15 @@
1817	2189	goto next;
1818	2190	if (err)
1819	2191	return err;
	2192	+ /*
	2193	+ * PSB+ CBR will not have changed but cater for the
	2194	+ * possibility of another CBR change that gets caught up
	2195	+ * in the PSB+.
	2196	+ */
	2197	+ if (decoder->cbr != decoder->cbr_seen) {
	2198	+ decoder->state.type = 0;
	2199	+ return 0;
	2200	+ }
1820	2201	break;
1821	2202
1822	2203	case INTEL_PT_PIP:
..	..	@@ -1857,14 +2238,8 @@
1857	2238
1858	2239	case INTEL_PT_CBR:
1859	2240	intel_pt_calc_cbr(decoder);
1860		- if (!decoder->branch_enable &&
1861		- decoder->cbr != decoder->cbr_seen) {
1862		- decoder->cbr_seen = decoder->cbr;
1863		- decoder->state.type = INTEL_PT_CBR_CHG;
1864		- decoder->state.from_ip = decoder->ip;
1865		- decoder->state.to_ip = 0;
1866		- decoder->state.cbr_payload =
1867		- decoder->packet.payload;
	2241	+ if (decoder->cbr != decoder->cbr_seen) {
	2242	+ decoder->state.type = 0;
1868	2243	return 0;
1869	2244	}
1870	2245	break;
..	..	@@ -1875,7 +2250,7 @@
1875	2250
1876	2251	case INTEL_PT_MODE_TSX:
1877	2252	/* MODE_TSX need not be followed by FUP */
1878		- if (!decoder->pge) {
	2253	+ if (!decoder->pge \|\| decoder->in_psb) {
1879	2254	intel_pt_update_in_tx(decoder);
1880	2255	break;
1881	2256	}
..	..	@@ -1958,6 +2333,33 @@
1958	2333	decoder->state.pwrx_payload = decoder->packet.payload;
1959	2334	return 0;
1960	2335
	2336	+ case INTEL_PT_BBP:
	2337	+ intel_pt_bbp(decoder);
	2338	+ break;
	2339	+
	2340	+ case INTEL_PT_BIP:
	2341	+ intel_pt_bip(decoder);
	2342	+ break;
	2343	+
	2344	+ case INTEL_PT_BEP:
	2345	+ decoder->state.type = INTEL_PT_BLK_ITEMS;
	2346	+ decoder->state.from_ip = decoder->ip;
	2347	+ decoder->state.to_ip = 0;
	2348	+ return 0;
	2349	+
	2350	+ case INTEL_PT_BEP_IP:
	2351	+ err = intel_pt_get_next_packet(decoder);
	2352	+ if (err)
	2353	+ return err;
	2354	+ if (decoder->packet.type == INTEL_PT_FUP) {
	2355	+ decoder->set_fup_bep = true;
	2356	+ no_tip = true;
	2357	+ } else {
	2358	+ intel_pt_log_at("ERROR: Missing FUP after BEP",
	2359	+ decoder->pos);
	2360	+ }
	2361	+ goto next;
	2362	+
1961	2363	default:
1962	2364	return intel_pt_bug(decoder);
1963	2365	}
..	..	@@ -1976,10 +2378,12 @@
1976	2378	{
1977	2379	int err;
1978	2380
	2381	+ decoder->in_psb = true;
	2382	+
1979	2383	while (1) {
1980	2384	err = intel_pt_get_next_packet(decoder);
1981	2385	if (err)
1982		- return err;
	2386	+ goto out;
1983	2387
1984	2388	switch (decoder->packet.type) {
1985	2389	case INTEL_PT_TIP_PGD:
..	..	@@ -1994,8 +2398,13 @@
1994	2398	case INTEL_PT_MWAIT:
1995	2399	case INTEL_PT_PWRE:
1996	2400	case INTEL_PT_PWRX:
	2401	+ case INTEL_PT_BBP:
	2402	+ case INTEL_PT_BIP:
	2403	+ case INTEL_PT_BEP:
	2404	+ case INTEL_PT_BEP_IP:
1997	2405	intel_pt_log("ERROR: Unexpected packet\n");
1998		- return -ENOENT;
	2406	+ err = -ENOENT;
	2407	+ goto out;
1999	2408
2000	2409	case INTEL_PT_FUP:
2001	2410	decoder->pge = true;
..	..	@@ -2054,16 +2463,20 @@
2054	2463	decoder->pkt_state = INTEL_PT_STATE_ERR4;
2055	2464	else
2056	2465	decoder->pkt_state = INTEL_PT_STATE_ERR3;
2057		- return -ENOENT;
	2466	+ err = -ENOENT;
	2467	+ goto out;
2058	2468
2059	2469	case INTEL_PT_BAD: /* Does not happen */
2060		- return intel_pt_bug(decoder);
	2470	+ err = intel_pt_bug(decoder);
	2471	+ goto out;
2061	2472
2062	2473	case INTEL_PT_OVF:
2063		- return intel_pt_overflow(decoder);
	2474	+ err = intel_pt_overflow(decoder);
	2475	+ goto out;
2064	2476
2065	2477	case INTEL_PT_PSBEND:
2066		- return 0;
	2478	+ err = 0;
	2479	+ goto out;
2067	2480
2068	2481	case INTEL_PT_PSB:
2069	2482	case INTEL_PT_VMCS:
..	..	@@ -2073,6 +2486,10 @@
2073	2486	break;
2074	2487	}
2075	2488	}
	2489	+out:
	2490	+ decoder->in_psb = false;
	2491	+
	2492	+ return err;
2076	2493	}
2077	2494
2078	2495	static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
..	..	@@ -2087,15 +2504,31 @@
2087	2504	switch (decoder->packet.type) {
2088	2505	case INTEL_PT_TIP_PGD:
2089	2506	decoder->continuous_period = false;
2090		- __fallthrough;
2091		- case INTEL_PT_TIP_PGE:
2092		- case INTEL_PT_TIP:
2093		- decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
	2507	+ decoder->pge = false;
2094	2508	if (intel_pt_have_ip(decoder))
2095	2509	intel_pt_set_ip(decoder);
2096		- if (decoder->ip)
2097		- return 0;
2098		- break;
	2510	+ if (!decoder->ip)
	2511	+ break;
	2512	+ decoder->state.type \|= INTEL_PT_TRACE_END;
	2513	+ return 0;
	2514	+
	2515	+ case INTEL_PT_TIP_PGE:
	2516	+ decoder->pge = true;
	2517	+ intel_pt_mtc_cyc_cnt_pge(decoder);
	2518	+ if (intel_pt_have_ip(decoder))
	2519	+ intel_pt_set_ip(decoder);
	2520	+ if (!decoder->ip)
	2521	+ break;
	2522	+ decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
	2523	+ return 0;
	2524	+
	2525	+ case INTEL_PT_TIP:
	2526	+ decoder->pge = true;
	2527	+ if (intel_pt_have_ip(decoder))
	2528	+ intel_pt_set_ip(decoder);
	2529	+ if (!decoder->ip)
	2530	+ break;
	2531	+ return 0;
2099	2532
2100	2533	case INTEL_PT_FUP:
2101	2534	if (intel_pt_have_ip(decoder))
..	..	@@ -2175,6 +2608,10 @@
2175	2608	case INTEL_PT_MWAIT:
2176	2609	case INTEL_PT_PWRE:
2177	2610	case INTEL_PT_PWRX:
	2611	+ case INTEL_PT_BBP:
	2612	+ case INTEL_PT_BIP:
	2613	+ case INTEL_PT_BEP:
	2614	+ case INTEL_PT_BEP_IP:
2178	2615	default:
2179	2616	break;
2180	2617	}
..	..	@@ -2190,10 +2627,11 @@
2190	2627	decoder->set_fup_mwait = false;
2191	2628	decoder->set_fup_pwre = false;
2192	2629	decoder->set_fup_exstop = false;
	2630	+ decoder->set_fup_bep = false;
	2631	+ decoder->overflow = false;
2193	2632
2194	2633	if (!decoder->branch_enable) {
2195	2634	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2196		- decoder->overflow = false;
2197	2635	decoder->state.type = 0; /* Do not have a sample */
2198	2636	return 0;
2199	2637	}
..	..	@@ -2203,8 +2641,11 @@
2203	2641	if (err)
2204	2642	return err;
2205	2643
2206		- decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2207		- decoder->overflow = false;
	2644	+ /* In hop mode, resample to get the to_ip as an "instruction" sample */
	2645	+ if (decoder->hop)
	2646	+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
	2647	+ else
	2648	+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2208	2649
2209	2650	decoder->state.from_ip = 0;
2210	2651	decoder->state.to_ip = decoder->ip;
..	..	@@ -2247,7 +2688,7 @@
2247	2688	decoder->pos += decoder->len;
2248	2689	decoder->len = 0;
2249	2690
2250		- ret = intel_pt_get_next_data(decoder);
	2691	+ ret = intel_pt_get_next_data(decoder, false);
2251	2692	if (ret)
2252	2693	return ret;
2253	2694
..	..	@@ -2273,7 +2714,7 @@
2273	2714	intel_pt_log("Scanning for PSB\n");
2274	2715	while (1) {
2275	2716	if (!decoder->len) {
2276		- ret = intel_pt_get_next_data(decoder);
	2717	+ ret = intel_pt_get_next_data(decoder, false);
2277	2718	if (ret)
2278	2719	return ret;
2279	2720	}
..	..	@@ -2311,12 +2752,13 @@
2311	2752	decoder->ip = 0;
2312	2753	intel_pt_clear_stack(&decoder->stack);
2313	2754
	2755	+leap:
2314	2756	err = intel_pt_scan_for_psb(decoder);
2315	2757	if (err)
2316	2758	return err;
2317	2759
2318	2760	decoder->have_last_ip = true;
2319		- decoder->pkt_state = INTEL_PT_STATE_NO_IP;
	2761	+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2320	2762
2321	2763	err = intel_pt_walk_psb(decoder);
2322	2764	if (err)
..	..	@@ -2324,7 +2766,20 @@
2324	2766
2325	2767	if (decoder->ip) {
2326	2768	decoder->state.type = 0; /* Do not have a sample */
2327		- decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
	2769	+ /*
	2770	+ * In hop mode, resample to get the PSB FUP ip as an
	2771	+ * "instruction" sample.
	2772	+ */
	2773	+ if (decoder->hop)
	2774	+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
	2775	+ else
	2776	+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
	2777	+ } else if (decoder->leap) {
	2778	+ /*
	2779	+ * In leap mode, only PSB+ is decoded, so keeping leaping to the
	2780	+ * next PSB until there is an ip.
	2781	+ */
	2782	+ goto leap;
2328	2783	} else {
2329	2784	return intel_pt_sync_ip(decoder);
2330	2785	}
..	..	@@ -2388,6 +2843,9 @@
2388	2843	if (err == -EAGAIN)
2389	2844	err = intel_pt_walk_trace(decoder);
2390	2845	break;
	2846	+ case INTEL_PT_STATE_RESAMPLE:
	2847	+ err = intel_pt_resample(decoder);
	2848	+ break;
2391	2849	default:
2392	2850	err = intel_pt_bug(decoder);
2393	2851	break;
..	..	@@ -2396,26 +2854,43 @@
2396	2854
2397	2855	if (err) {
2398	2856	decoder->state.err = intel_pt_ext_err(err);
2399		- decoder->state.from_ip = decoder->ip;
2400		- decoder->sample_timestamp = decoder->timestamp;
2401		- decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
	2857	+ if (err != -EOVERFLOW)
	2858	+ decoder->state.from_ip = decoder->ip;
	2859	+ intel_pt_update_sample_time(decoder);
	2860	+ decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
2402	2861	} else {
2403	2862	decoder->state.err = 0;
2404		- if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
	2863	+ if (decoder->cbr != decoder->cbr_seen) {
2405	2864	decoder->cbr_seen = decoder->cbr;
	2865	+ if (!decoder->state.type) {
	2866	+ decoder->state.from_ip = decoder->ip;
	2867	+ decoder->state.to_ip = 0;
	2868	+ }
2406	2869	decoder->state.type \|= INTEL_PT_CBR_CHG;
2407	2870	decoder->state.cbr_payload = decoder->cbr_payload;
	2871	+ decoder->state.cbr = decoder->cbr;
2408	2872	}
2409	2873	if (intel_pt_sample_time(decoder->pkt_state)) {
2410		- decoder->sample_timestamp = decoder->timestamp;
2411		- decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
	2874	+ intel_pt_update_sample_time(decoder);
	2875	+ if (decoder->sample_cyc) {
	2876	+ decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
	2877	+ decoder->state.flags \|= INTEL_PT_SAMPLE_IPC;
	2878	+ decoder->sample_cyc = false;
	2879	+ }
2412	2880	}
	2881	+ /*
	2882	+ * When using only TSC/MTC to compute cycles, IPC can be
	2883	+ * sampled as soon as the cycle count changes.
	2884	+ */
	2885	+ if (!decoder->have_cyc)
	2886	+ decoder->state.flags \|= INTEL_PT_SAMPLE_IPC;
2413	2887	}
2414	2888
2415	2889	decoder->state.timestamp = decoder->sample_timestamp;
2416	2890	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2417	2891	decoder->state.cr3 = decoder->cr3;
2418	2892	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
	2893	+ decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
2419	2894
2420	2895	return &decoder->state;
2421	2896	}
..	..	@@ -2519,11 +2994,12 @@
2519	2994	static bool intel_pt_next_tsc(unsigned char buf, size_t len, uint64_t tsc,
2520	2995	size_t *rem)
2521	2996	{
	2997	+ enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
2522	2998	struct intel_pt_pkt packet;
2523	2999	int ret;
2524	3000
2525	3001	while (len) {
2526		- ret = intel_pt_get_packet(buf, len, &packet);
	3002	+ ret = intel_pt_get_packet(buf, len, &packet, &ctx);
2527	3003	if (ret <= 0)
2528	3004	return false;
2529	3005	if (packet.type == INTEL_PT_TSC) {
..	..	@@ -2725,3 +3201,131 @@
2725	3201	return buf_b; /* No overlap */
2726	3202	}
2727	3203	}
	3204	+
	3205	+/**
	3206	+ * struct fast_forward_data - data used by intel_pt_ff_cb().
	3207	+ * @timestamp: timestamp to fast forward towards
	3208	+ * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
	3209	+ * the fast forward timestamp.
	3210	+ */
	3211	+struct fast_forward_data {
	3212	+ uint64_t timestamp;
	3213	+ uint64_t buf_timestamp;
	3214	+};
	3215	+
	3216	+/**
	3217	+ * intel_pt_ff_cb - fast forward lookahead callback.
	3218	+ * @buffer: Intel PT trace buffer
	3219	+ * @data: opaque pointer to fast forward data (struct fast_forward_data)
	3220	+ *
	3221	+ * Determine if @buffer trace is past the fast forward timestamp.
	3222	+ *
	3223	+ * Return: 1 (stop lookahead) if @buffer trace is past the fast forward
	3224	+ * timestamp, and 0 otherwise.
	3225	+ */
	3226	+static int intel_pt_ff_cb(struct intel_pt_buffer buffer, void data)
	3227	+{
	3228	+ struct fast_forward_data *d = data;
	3229	+ unsigned char *buf;
	3230	+ uint64_t tsc;
	3231	+ size_t rem;
	3232	+ size_t len;
	3233	+
	3234	+ buf = (unsigned char *)buffer->buf;
	3235	+ len = buffer->len;
	3236	+
	3237	+ if (!intel_pt_next_psb(&buf, &len) \|\|
	3238	+ !intel_pt_next_tsc(buf, len, &tsc, &rem))
	3239	+ return 0;
	3240	+
	3241	+ tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
	3242	+
	3243	+ intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
	3244	+ tsc, buffer->ref_timestamp);
	3245	+
	3246	+ /*
	3247	+ * If the buffer contains a timestamp earlier that the fast forward
	3248	+ * timestamp, then record it, else stop.
	3249	+ */
	3250	+ if (tsc < d->timestamp)
	3251	+ d->buf_timestamp = buffer->ref_timestamp;
	3252	+ else
	3253	+ return 1;
	3254	+
	3255	+ return 0;
	3256	+}
	3257	+
	3258	+/**
	3259	+ * intel_pt_fast_forward - reposition decoder forwards.
	3260	+ * @decoder: Intel PT decoder
	3261	+ * @timestamp: timestamp to fast forward towards
	3262	+ *
	3263	+ * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
	3264	+ *
	3265	+ * Return: 0 on success or negative error code on failure.
	3266	+ */
	3267	+int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
	3268	+{
	3269	+ struct fast_forward_data d = { .timestamp = timestamp };
	3270	+ unsigned char *buf;
	3271	+ size_t len;
	3272	+ int err;
	3273	+
	3274	+ intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
	3275	+
	3276	+ /* Find buffer timestamp of buffer to fast forward to */
	3277	+ err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
	3278	+ if (err < 0)
	3279	+ return err;
	3280	+
	3281	+ /* Walk to buffer with same buffer timestamp */
	3282	+ if (d.buf_timestamp) {
	3283	+ do {
	3284	+ decoder->pos += decoder->len;
	3285	+ decoder->len = 0;
	3286	+ err = intel_pt_get_next_data(decoder, true);
	3287	+ /* -ENOLINK means non-consecutive trace */
	3288	+ if (err && err != -ENOLINK)
	3289	+ return err;
	3290	+ } while (decoder->buf_timestamp != d.buf_timestamp);
	3291	+ }
	3292	+
	3293	+ if (!decoder->buf)
	3294	+ return 0;
	3295	+
	3296	+ buf = (unsigned char *)decoder->buf;
	3297	+ len = decoder->len;
	3298	+
	3299	+ if (!intel_pt_next_psb(&buf, &len))
	3300	+ return 0;
	3301	+
	3302	+ /*
	3303	+ * Walk PSBs while the PSB timestamp is less than the fast forward
	3304	+ * timestamp.
	3305	+ */
	3306	+ do {
	3307	+ uint64_t tsc;
	3308	+ size_t rem;
	3309	+
	3310	+ if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
	3311	+ break;
	3312	+ tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
	3313	+ /*
	3314	+ * A TSC packet can slip past MTC packets but, after fast
	3315	+ * forward, decoding starts at the TSC timestamp. That means
	3316	+ * the timestamps may not be exactly the same as the timestamps
	3317	+ * that would have been decoded without fast forward.
	3318	+ */
	3319	+ if (tsc < timestamp) {
	3320	+ intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
	3321	+ decoder->pos += decoder->len - len;
	3322	+ decoder->buf = buf;
	3323	+ decoder->len = len;
	3324	+ intel_pt_reposition(decoder);
	3325	+ } else {
	3326	+ break;
	3327	+ }
	3328	+ } while (intel_pt_step_psb(&buf, &len));
	3329	+
	3330	+ return 0;
	3331	+}