~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,16 +1,7 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* intel_pt_decoder.c: Intel Processor Trace support
3	4	* Copyright (c) 2013-2014, Intel Corporation.
4		- *
5		- * This program is free software; you can redistribute it and/or modify it
6		- * under the terms and conditions of the GNU General Public License,
7		- * version 2, as published by the Free Software Foundation.
8		- *
9		- * This program is distributed in the hope it will be useful, but WITHOUT
10		- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11		- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12		- * more details.
13		- *
14	5	*/
15	6
16	7	#ifndef _GNU_SOURCE
..	..	@@ -23,15 +14,22 @@
23	14	#include <stdint.h>
24	15	#include <inttypes.h>
25	16	#include <linux/compiler.h>
	17	+#include <linux/string.h>
	18	+#include <linux/zalloc.h>
26	19
27		-#include "../cache.h"
28		-#include "../util.h"
29	20	#include "../auxtrace.h"
30	21
31	22	#include "intel-pt-insn-decoder.h"
32	23	#include "intel-pt-pkt-decoder.h"
33	24	#include "intel-pt-decoder.h"
34	25	#include "intel-pt-log.h"
	26	+
	27	+#define BITULL(x) (1ULL << (x))
	28	+
	29	+/* IA32_RTIT_CTL MSR bits */
	30	+#define INTEL_PT_CYC_ENABLE BITULL(1)
	31	+#define INTEL_PT_CYC_THRESHOLD (BITULL(22) \| BITULL(21) \| BITULL(20) \| BITULL(19))
	32	+#define INTEL_PT_CYC_THRESHOLD_SHIFT 19
35	33
36	34	#define INTEL_PT_BLK_SIZE 1024
37	35
..	..	@@ -64,6 +62,7 @@
64	62	INTEL_PT_STATE_TIP_PGD,
65	63	INTEL_PT_STATE_FUP,
66	64	INTEL_PT_STATE_FUP_NO_TIP,
	65	+ INTEL_PT_STATE_RESAMPLE,
67	66	};
68	67
69	68	static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
..	..	@@ -74,6 +73,7 @@
74	73	case INTEL_PT_STATE_ERR_RESYNC:
75	74	case INTEL_PT_STATE_IN_SYNC:
76	75	case INTEL_PT_STATE_TNT_CONT:
	76	+ case INTEL_PT_STATE_RESAMPLE:
77	77	return true;
78	78	case INTEL_PT_STATE_TNT:
79	79	case INTEL_PT_STATE_TIP:
..	..	@@ -104,6 +104,7 @@
104	104	uint64_t insn_cnt_ptr, uint64_t ip, uint64_t to_ip,
105	105	uint64_t max_insn_cnt, void *data);
106	106	bool (pgd_ip)(uint64_t ip, void data);
	107	+ int (lookahead)(void data, intel_pt_lookahead_cb_t cb, void *cb_data);
107	108	void *data;
108	109	struct intel_pt_state state;
109	110	const unsigned char *buf;
..	..	@@ -116,6 +117,10 @@
116	117	bool have_cyc;
117	118	bool fixup_last_mtc;
118	119	bool have_last_ip;
	120	+ bool in_psb;
	121	+ bool hop;
	122	+ bool hop_psb_fup;
	123	+ bool leap;
119	124	enum intel_pt_param_flags flags;
120	125	uint64_t pos;
121	126	uint64_t last_ip;
..	..	@@ -124,6 +129,7 @@
124	129	uint64_t timestamp;
125	130	uint64_t tsc_timestamp;
126	131	uint64_t ref_timestamp;
	132	+ uint64_t buf_timestamp;
127	133	uint64_t sample_timestamp;
128	134	uint64_t ret_addr;
129	135	uint64_t ctc_timestamp;
..	..	@@ -139,6 +145,10 @@
139	145	int mtc_shift;
140	146	struct intel_pt_stack stack;
141	147	enum intel_pt_pkt_state pkt_state;
	148	+ enum intel_pt_pkt_ctx pkt_ctx;
	149	+ enum intel_pt_pkt_ctx prev_pkt_ctx;
	150	+ enum intel_pt_blk_type blk_type;
	151	+ int blk_type_pos;
142	152	struct intel_pt_pkt packet;
143	153	struct intel_pt_pkt tnt;
144	154	int pkt_step;
..	..	@@ -160,6 +170,13 @@
160	170	uint64_t period_mask;
161	171	uint64_t period_ticks;
162	172	uint64_t last_masked_timestamp;
	173	+ uint64_t tot_cyc_cnt;
	174	+ uint64_t sample_tot_cyc_cnt;
	175	+ uint64_t base_cyc_cnt;
	176	+ uint64_t cyc_cnt_timestamp;
	177	+ uint64_t ctl;
	178	+ uint64_t cyc_threshold;
	179	+ double tsc_to_cyc;
163	180	bool continuous_period;
164	181	bool overflow;
165	182	bool set_fup_tx_flags;
..	..	@@ -167,6 +184,8 @@
167	184	bool set_fup_mwait;
168	185	bool set_fup_pwre;
169	186	bool set_fup_exstop;
	187	+ bool set_fup_bep;
	188	+ bool sample_cyc;
170	189	unsigned int fup_tx_flags;
171	190	unsigned int tx_flags;
172	191	uint64_t fup_ptw_payload;
..	..	@@ -192,6 +211,14 @@
192	211	x >>= 1;
193	212
194	213	return x << i;
	214	+}
	215	+
	216	+static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
	217	+{
	218	+ if (!(ctl & INTEL_PT_CYC_ENABLE))
	219	+ return 0;
	220	+
	221	+ return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
195	222	}
196	223
197	224	static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
..	..	@@ -226,17 +253,23 @@
226	253	decoder->get_trace = params->get_trace;
227	254	decoder->walk_insn = params->walk_insn;
228	255	decoder->pgd_ip = params->pgd_ip;
	256	+ decoder->lookahead = params->lookahead;
229	257	decoder->data = params->data;
230	258	decoder->return_compression = params->return_compression;
231	259	decoder->branch_enable = params->branch_enable;
	260	+ decoder->hop = params->quick >= 1;
	261	+ decoder->leap = params->quick >= 2;
232	262
233	263	decoder->flags = params->flags;
234	264
	265	+ decoder->ctl = params->ctl;
235	266	decoder->period = params->period;
236	267	decoder->period_type = params->period_type;
237	268
238	269	decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
239	270	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
	271	+
	272	+ decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
240	273
241	274	intel_pt_setup_period(decoder);
242	275
..	..	@@ -268,6 +301,9 @@
268	301	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
269	302	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
270	303	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
	304	+
	305	+ if (decoder->hop)
	306	+ intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
271	307
272	308	return decoder;
273	309	}
..	..	@@ -479,7 +515,21 @@
479	515	return -EBADMSG;
480	516	}
481	517
482		-static int intel_pt_get_data(struct intel_pt_decoder *decoder)
	518	+static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
	519	+{
	520	+ decoder->sample_timestamp = decoder->timestamp;
	521	+ decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
	522	+}
	523	+
	524	+static void intel_pt_reposition(struct intel_pt_decoder *decoder)
	525	+{
	526	+ decoder->ip = 0;
	527	+ decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
	528	+ decoder->timestamp = 0;
	529	+ decoder->have_tma = false;
	530	+}
	531	+
	532	+static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
483	533	{
484	534	struct intel_pt_buffer buffer = { .buf = 0, };
485	535	int ret;
..	..	@@ -496,12 +546,10 @@
496	546	intel_pt_log("No more data\n");
497	547	return -ENODATA;
498	548	}
499		- if (!buffer.consecutive) {
500		- decoder->ip = 0;
501		- decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
	549	+ decoder->buf_timestamp = buffer.ref_timestamp;
	550	+ if (!buffer.consecutive \|\| reposition) {
	551	+ intel_pt_reposition(decoder);
502	552	decoder->ref_timestamp = buffer.ref_timestamp;
503		- decoder->timestamp = 0;
504		- decoder->have_tma = false;
505	553	decoder->state.trace_nr = buffer.trace_nr;
506	554	intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
507	555	decoder->ref_timestamp);
..	..	@@ -511,10 +559,11 @@
511	559	return 0;
512	560	}
513	561
514		-static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
	562	+static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
	563	+ bool reposition)
515	564	{
516	565	if (!decoder->next_buf)
517		- return intel_pt_get_data(decoder);
	566	+ return intel_pt_get_data(decoder, reposition);
518	567
519	568	decoder->buf = decoder->next_buf;
520	569	decoder->len = decoder->next_len;
..	..	@@ -533,7 +582,7 @@
533	582	len = decoder->len;
534	583	memcpy(buf, decoder->buf, len);
535	584
536		- ret = intel_pt_get_data(decoder);
	585	+ ret = intel_pt_get_data(decoder, false);
537	586	if (ret) {
538	587	decoder->pos += old_len;
539	588	return ret < 0 ? ret : -EINVAL;
..	..	@@ -545,7 +594,8 @@
545	594	memcpy(buf + len, decoder->buf, n);
546	595	len += n;
547	596
548		- ret = intel_pt_get_packet(buf, len, &decoder->packet);
	597	+ decoder->prev_pkt_ctx = decoder->pkt_ctx;
	598	+ ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
549	599	if (ret < (int)old_len) {
550	600	decoder->next_buf = decoder->buf;
551	601	decoder->next_len = decoder->len;
..	..	@@ -580,6 +630,7 @@
580	630	{
581	631	struct intel_pt_pkt_info pkt_info;
582	632	const unsigned char *buf = decoder->buf;
	633	+ enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
583	634	size_t len = decoder->len;
584	635	int ret;
585	636
..	..	@@ -598,7 +649,8 @@
598	649	if (!len)
599	650	return INTEL_PT_NEED_MORE_BYTES;
600	651
601		- ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
	652	+ ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
	653	+ &pkt_ctx);
602	654	if (!ret)
603	655	return INTEL_PT_NEED_MORE_BYTES;
604	656	if (ret < 0)
..	..	@@ -673,6 +725,10 @@
673	725	case INTEL_PT_MNT:
674	726	case INTEL_PT_PTWRITE:
675	727	case INTEL_PT_PTWRITE_IP:
	728	+ case INTEL_PT_BBP:
	729	+ case INTEL_PT_BIP:
	730	+ case INTEL_PT_BEP:
	731	+ case INTEL_PT_BEP_IP:
676	732	return 0;
677	733
678	734	case INTEL_PT_MTC:
..	..	@@ -859,14 +915,15 @@
859	915	decoder->len -= decoder->pkt_step;
860	916
861	917	if (!decoder->len) {
862		- ret = intel_pt_get_next_data(decoder);
	918	+ ret = intel_pt_get_next_data(decoder, false);
863	919	if (ret)
864	920	return ret;
865	921	}
866	922
	923	+ decoder->prev_pkt_ctx = decoder->pkt_ctx;
867	924	ret = intel_pt_get_packet(decoder->buf, decoder->len,
868		- &decoder->packet);
869		- if (ret == INTEL_PT_NEED_MORE_BYTES &&
	925	+ &decoder->packet, &decoder->pkt_ctx);
	926	+ if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
870	927	decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
871	928	ret = intel_pt_get_split_packet(decoder);
872	929	if (ret < 0)
..	..	@@ -1057,53 +1114,69 @@
1057	1114
1058	1115	static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1059	1116	{
	1117	+ enum intel_pt_sample_type type = decoder->state.type;
1060	1118	bool ret = false;
	1119	+
	1120	+ decoder->state.type &= ~INTEL_PT_BRANCH;
1061	1121
1062	1122	if (decoder->set_fup_tx_flags) {
1063	1123	decoder->set_fup_tx_flags = false;
1064	1124	decoder->tx_flags = decoder->fup_tx_flags;
1065		- decoder->state.type = INTEL_PT_TRANSACTION;
	1125	+ decoder->state.type \|= INTEL_PT_TRANSACTION;
1066	1126	if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1067	1127	decoder->state.type \|= INTEL_PT_BRANCH;
1068		- decoder->state.from_ip = decoder->ip;
1069		- decoder->state.to_ip = 0;
1070	1128	decoder->state.flags = decoder->fup_tx_flags;
1071		- return true;
	1129	+ ret = true;
1072	1130	}
1073	1131	if (decoder->set_fup_ptw) {
1074	1132	decoder->set_fup_ptw = false;
1075		- decoder->state.type = INTEL_PT_PTW;
	1133	+ decoder->state.type \|= INTEL_PT_PTW;
1076	1134	decoder->state.flags \|= INTEL_PT_FUP_IP;
1077		- decoder->state.from_ip = decoder->ip;
1078		- decoder->state.to_ip = 0;
1079	1135	decoder->state.ptw_payload = decoder->fup_ptw_payload;
1080		- return true;
	1136	+ ret = true;
1081	1137	}
1082	1138	if (decoder->set_fup_mwait) {
1083	1139	decoder->set_fup_mwait = false;
1084		- decoder->state.type = INTEL_PT_MWAIT_OP;
1085		- decoder->state.from_ip = decoder->ip;
1086		- decoder->state.to_ip = 0;
	1140	+ decoder->state.type \|= INTEL_PT_MWAIT_OP;
1087	1141	decoder->state.mwait_payload = decoder->fup_mwait_payload;
1088	1142	ret = true;
1089	1143	}
1090	1144	if (decoder->set_fup_pwre) {
1091	1145	decoder->set_fup_pwre = false;
1092	1146	decoder->state.type \|= INTEL_PT_PWR_ENTRY;
1093		- decoder->state.type &= ~INTEL_PT_BRANCH;
1094		- decoder->state.from_ip = decoder->ip;
1095		- decoder->state.to_ip = 0;
1096	1147	decoder->state.pwre_payload = decoder->fup_pwre_payload;
1097	1148	ret = true;
1098	1149	}
1099	1150	if (decoder->set_fup_exstop) {
1100	1151	decoder->set_fup_exstop = false;
1101	1152	decoder->state.type \|= INTEL_PT_EX_STOP;
1102		- decoder->state.type &= ~INTEL_PT_BRANCH;
1103	1153	decoder->state.flags \|= INTEL_PT_FUP_IP;
	1154	+ ret = true;
	1155	+ }
	1156	+ if (decoder->set_fup_bep) {
	1157	+ decoder->set_fup_bep = false;
	1158	+ decoder->state.type \|= INTEL_PT_BLK_ITEMS;
	1159	+ ret = true;
	1160	+ }
	1161	+ if (decoder->overflow) {
	1162	+ decoder->overflow = false;
	1163	+ if (!ret && !decoder->pge) {
	1164	+ if (decoder->hop) {
	1165	+ decoder->state.type = 0;
	1166	+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
	1167	+ }
	1168	+ decoder->pge = true;
	1169	+ decoder->state.type \|= INTEL_PT_BRANCH \| INTEL_PT_TRACE_BEGIN;
	1170	+ decoder->state.from_ip = 0;
	1171	+ decoder->state.to_ip = decoder->ip;
	1172	+ return true;
	1173	+ }
	1174	+ }
	1175	+ if (ret) {
1104	1176	decoder->state.from_ip = decoder->ip;
1105	1177	decoder->state.to_ip = 0;
1106		- ret = true;
	1178	+ } else {
	1179	+ decoder->state.type = type;
1107	1180	}
1108	1181	return ret;
1109	1182	}
..	..	@@ -1176,7 +1249,7 @@
1176	1249	decoder->pge = false;
1177	1250	decoder->continuous_period = false;
1178	1251	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1179		- decoder->state.to_ip = 0;
	1252	+ decoder->state.type \|= INTEL_PT_TRACE_END;
1180	1253	return 0;
1181	1254	}
1182	1255	if (err == INTEL_PT_RETURN)
..	..	@@ -1190,9 +1263,13 @@
1190	1263	decoder->continuous_period = false;
1191	1264	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1192	1265	decoder->state.from_ip = decoder->ip;
1193		- decoder->state.to_ip = 0;
1194		- if (decoder->packet.count != 0)
	1266	+ if (decoder->packet.count == 0) {
	1267	+ decoder->state.to_ip = 0;
	1268	+ } else {
	1269	+ decoder->state.to_ip = decoder->last_ip;
1195	1270	decoder->ip = decoder->last_ip;
	1271	+ }
	1272	+ decoder->state.type \|= INTEL_PT_TRACE_END;
1196	1273	} else {
1197	1274	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1198	1275	decoder->state.from_ip = decoder->ip;
..	..	@@ -1219,7 +1296,8 @@
1219	1296	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1220	1297	decoder->ip = to_ip;
1221	1298	decoder->state.from_ip = decoder->ip;
1222		- decoder->state.to_ip = 0;
	1299	+ decoder->state.to_ip = to_ip;
	1300	+ decoder->state.type \|= INTEL_PT_TRACE_END;
1223	1301	return 0;
1224	1302	}
1225	1303	intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
..	..	@@ -1317,10 +1395,10 @@
1317	1395	decoder->ip += intel_pt_insn.length;
1318	1396	return 0;
1319	1397	}
	1398	+ decoder->sample_cyc = false;
1320	1399	decoder->ip += intel_pt_insn.length;
1321	1400	if (!decoder->tnt.count) {
1322		- decoder->sample_timestamp = decoder->timestamp;
1323		- decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
	1401	+ intel_pt_update_sample_time(decoder);
1324	1402	return -EAGAIN;
1325	1403	}
1326	1404	decoder->tnt.payload <<= 1;
..	..	@@ -1354,6 +1432,21 @@
1354	1432	return 0;
1355	1433	}
1356	1434
	1435	+static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
	1436	+{
	1437	+ timestamp \|= (ref_timestamp & (0xffULL << 56));
	1438	+
	1439	+ if (timestamp < ref_timestamp) {
	1440	+ if (ref_timestamp - timestamp > (1ULL << 55))
	1441	+ timestamp += (1ULL << 56);
	1442	+ } else {
	1443	+ if (timestamp - ref_timestamp > (1ULL << 55))
	1444	+ timestamp -= (1ULL << 56);
	1445	+ }
	1446	+
	1447	+ return timestamp;
	1448	+}
	1449	+
1357	1450	static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1358	1451	{
1359	1452	uint64_t timestamp;
..	..	@@ -1361,15 +1454,8 @@
1361	1454	decoder->have_tma = false;
1362	1455
1363	1456	if (decoder->ref_timestamp) {
1364		- timestamp = decoder->packet.payload \|
1365		- (decoder->ref_timestamp & (0xffULL << 56));
1366		- if (timestamp < decoder->ref_timestamp) {
1367		- if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1368		- timestamp += (1ULL << 56);
1369		- } else {
1370		- if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1371		- timestamp -= (1ULL << 56);
1372		- }
	1457	+ timestamp = intel_pt_8b_tsc(decoder->packet.payload,
	1458	+ decoder->ref_timestamp);
1373	1459	decoder->tsc_timestamp = timestamp;
1374	1460	decoder->timestamp = timestamp;
1375	1461	decoder->ref_timestamp = 0;
..	..	@@ -1408,9 +1494,54 @@
1408	1494	intel_pt_log("ERROR: Buffer overflow\n");
1409	1495	intel_pt_clear_tx_flags(decoder);
1410	1496	decoder->timestamp_insn_cnt = 0;
1411		- decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
	1497	+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
	1498	+ decoder->state.from_ip = decoder->ip;
	1499	+ decoder->ip = 0;
	1500	+ decoder->pge = false;
	1501	+ decoder->set_fup_tx_flags = false;
	1502	+ decoder->set_fup_ptw = false;
	1503	+ decoder->set_fup_mwait = false;
	1504	+ decoder->set_fup_pwre = false;
	1505	+ decoder->set_fup_exstop = false;
	1506	+ decoder->set_fup_bep = false;
1412	1507	decoder->overflow = true;
1413	1508	return -EOVERFLOW;
	1509	+}
	1510	+
	1511	+static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
	1512	+{
	1513	+ if (decoder->have_cyc)
	1514	+ return;
	1515	+
	1516	+ decoder->cyc_cnt_timestamp = decoder->timestamp;
	1517	+ decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
	1518	+}
	1519	+
	1520	+static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
	1521	+{
	1522	+ decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
	1523	+
	1524	+ if (decoder->pge)
	1525	+ intel_pt_mtc_cyc_cnt_pge(decoder);
	1526	+}
	1527	+
	1528	+static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
	1529	+{
	1530	+ uint64_t tot_cyc_cnt, tsc_delta;
	1531	+
	1532	+ if (decoder->have_cyc)
	1533	+ return;
	1534	+
	1535	+ decoder->sample_cyc = true;
	1536	+
	1537	+ if (!decoder->pge \|\| decoder->timestamp <= decoder->cyc_cnt_timestamp)
	1538	+ return;
	1539	+
	1540	+ tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
	1541	+ tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
	1542	+
	1543	+ if (tot_cyc_cnt > decoder->tot_cyc_cnt)
	1544	+ decoder->tot_cyc_cnt = tot_cyc_cnt;
1414	1545	}
1415	1546
1416	1547	static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
..	..	@@ -1421,6 +1552,11 @@
1421	1552
1422	1553	if (!decoder->tsc_ctc_ratio_d)
1423	1554	return;
	1555	+
	1556	+ if (decoder->pge && !decoder->in_psb)
	1557	+ intel_pt_mtc_cyc_cnt_pge(decoder);
	1558	+ else
	1559	+ intel_pt_mtc_cyc_cnt_upd(decoder);
1424	1560
1425	1561	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1426	1562	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
..	..	@@ -1477,6 +1613,8 @@
1477	1613	else
1478	1614	decoder->timestamp = timestamp;
1479	1615
	1616	+ intel_pt_mtc_cyc_cnt_upd(decoder);
	1617	+
1480	1618	decoder->timestamp_insn_cnt = 0;
1481	1619	decoder->last_mtc = mtc;
1482	1620
..	..	@@ -1486,6 +1624,8 @@
1486	1624	decoder->have_calc_cyc_to_tsc = false;
1487	1625	intel_pt_calc_cyc_to_tsc(decoder, true);
1488	1626	}
	1627	+
	1628	+ intel_pt_log_to("Setting timestamp", decoder->timestamp);
1489	1629	}
1490	1630
1491	1631	static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
..	..	@@ -1499,6 +1639,8 @@
1499	1639
1500	1640	decoder->cbr = cbr;
1501	1641	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
	1642	+
	1643	+ intel_pt_mtc_cyc_cnt_cbr(decoder);
1502	1644	}
1503	1645
1504	1646	static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
..	..	@@ -1508,6 +1650,9 @@
1508	1650	decoder->have_cyc = true;
1509	1651
1510	1652	decoder->cycle_cnt += decoder->packet.payload;
	1653	+ if (decoder->pge)
	1654	+ decoder->tot_cyc_cnt += decoder->packet.payload;
	1655	+ decoder->sample_cyc = true;
1511	1656
1512	1657	if (!decoder->cyc_ref_timestamp)
1513	1658	return;
..	..	@@ -1526,6 +1671,48 @@
1526	1671	decoder->timestamp = timestamp;
1527	1672
1528	1673	decoder->timestamp_insn_cnt = 0;
	1674	+
	1675	+ intel_pt_log_to("Setting timestamp", decoder->timestamp);
	1676	+}
	1677	+
	1678	+static void intel_pt_bbp(struct intel_pt_decoder *decoder)
	1679	+{
	1680	+ if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
	1681	+ memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
	1682	+ decoder->state.items.is_32_bit = false;
	1683	+ }
	1684	+ decoder->blk_type = decoder->packet.payload;
	1685	+ decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
	1686	+ if (decoder->blk_type == INTEL_PT_GP_REGS)
	1687	+ decoder->state.items.is_32_bit = decoder->packet.count;
	1688	+ if (decoder->blk_type_pos < 0) {
	1689	+ intel_pt_log("WARNING: Unknown block type %u\n",
	1690	+ decoder->blk_type);
	1691	+ } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
	1692	+ intel_pt_log("WARNING: Duplicate block type %u\n",
	1693	+ decoder->blk_type);
	1694	+ }
	1695	+}
	1696	+
	1697	+static void intel_pt_bip(struct intel_pt_decoder *decoder)
	1698	+{
	1699	+ uint32_t id = decoder->packet.count;
	1700	+ uint32_t bit = 1 << id;
	1701	+ int pos = decoder->blk_type_pos;
	1702	+
	1703	+ if (pos < 0 \|\| id >= INTEL_PT_BLK_ITEM_ID_CNT) {
	1704	+ intel_pt_log("WARNING: Unknown block item %u type %d\n",
	1705	+ id, decoder->blk_type);
	1706	+ return;
	1707	+ }
	1708	+
	1709	+ if (decoder->state.items.mask[pos] & bit) {
	1710	+ intel_pt_log("WARNING: Duplicate block item %u type %d\n",
	1711	+ id, decoder->blk_type);
	1712	+ }
	1713	+
	1714	+ decoder->state.items.mask[pos] \|= bit;
	1715	+ decoder->state.items.val[pos][id] = decoder->packet.payload;
1529	1716	}
1530	1717
1531	1718	/* Walk PSB+ packets when already in sync. */
..	..	@@ -1533,14 +1720,17 @@
1533	1720	{
1534	1721	int err;
1535	1722
	1723	+ decoder->in_psb = true;
	1724	+
1536	1725	while (1) {
1537	1726	err = intel_pt_get_next_packet(decoder);
1538	1727	if (err)
1539		- return err;
	1728	+ goto out;
1540	1729
1541	1730	switch (decoder->packet.type) {
1542	1731	case INTEL_PT_PSBEND:
1543		- return 0;
	1732	+ err = 0;
	1733	+ goto out;
1544	1734
1545	1735	case INTEL_PT_TIP_PGD:
1546	1736	case INTEL_PT_TIP_PGE:
..	..	@@ -1556,12 +1746,18 @@
1556	1746	case INTEL_PT_MWAIT:
1557	1747	case INTEL_PT_PWRE:
1558	1748	case INTEL_PT_PWRX:
	1749	+ case INTEL_PT_BBP:
	1750	+ case INTEL_PT_BIP:
	1751	+ case INTEL_PT_BEP:
	1752	+ case INTEL_PT_BEP_IP:
1559	1753	decoder->have_tma = false;
1560	1754	intel_pt_log("ERROR: Unexpected packet\n");
1561		- return -EAGAIN;
	1755	+ err = -EAGAIN;
	1756	+ goto out;
1562	1757
1563	1758	case INTEL_PT_OVF:
1564		- return intel_pt_overflow(decoder);
	1759	+ err = intel_pt_overflow(decoder);
	1760	+ goto out;
1565	1761
1566	1762	case INTEL_PT_TSC:
1567	1763	intel_pt_calc_tsc_timestamp(decoder);
..	..	@@ -1585,8 +1781,14 @@
1585	1781
1586	1782	case INTEL_PT_FUP:
1587	1783	decoder->pge = true;
1588		- if (decoder->packet.count)
	1784	+ if (decoder->packet.count) {
1589	1785	intel_pt_set_last_ip(decoder);
	1786	+ if (decoder->hop) {
	1787	+ /* Act on FUP at PSBEND */
	1788	+ decoder->ip = decoder->last_ip;
	1789	+ decoder->hop_psb_fup = true;
	1790	+ }
	1791	+ }
1590	1792	break;
1591	1793
1592	1794	case INTEL_PT_MODE_TSX:
..	..	@@ -1610,6 +1812,10 @@
1610	1812	break;
1611	1813	}
1612	1814	}
	1815	+out:
	1816	+ decoder->in_psb = false;
	1817	+
	1818	+ return err;
1613	1819	}
1614	1820
1615	1821	static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
..	..	@@ -1646,6 +1852,10 @@
1646	1852	case INTEL_PT_MWAIT:
1647	1853	case INTEL_PT_PWRE:
1648	1854	case INTEL_PT_PWRX:
	1855	+ case INTEL_PT_BBP:
	1856	+ case INTEL_PT_BIP:
	1857	+ case INTEL_PT_BEP:
	1858	+ case INTEL_PT_BEP_IP:
1649	1859	intel_pt_log("ERROR: Missing TIP after FUP\n");
1650	1860	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1651	1861	decoder->pkt_step = 0;
..	..	@@ -1660,14 +1870,15 @@
1660	1870
1661	1871	case INTEL_PT_TIP_PGD:
1662	1872	decoder->state.from_ip = decoder->ip;
1663		- decoder->state.to_ip = 0;
1664		- if (decoder->packet.count != 0) {
	1873	+ if (decoder->packet.count == 0) {
	1874	+ decoder->state.to_ip = 0;
	1875	+ } else {
1665	1876	intel_pt_set_ip(decoder);
1666		- intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1667		- decoder->ip);
	1877	+ decoder->state.to_ip = decoder->ip;
1668	1878	}
1669	1879	decoder->pge = false;
1670	1880	decoder->continuous_period = false;
	1881	+ decoder->state.type \|= INTEL_PT_TRACE_END;
1671	1882	return 0;
1672	1883
1673	1884	case INTEL_PT_TIP_PGE:
..	..	@@ -1681,6 +1892,8 @@
1681	1892	intel_pt_set_ip(decoder);
1682	1893	decoder->state.to_ip = decoder->ip;
1683	1894	}
	1895	+ decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
	1896	+ intel_pt_mtc_cyc_cnt_pge(decoder);
1684	1897	return 0;
1685	1898
1686	1899	case INTEL_PT_TIP:
..	..	@@ -1722,8 +1935,136 @@
1722	1935	}
1723	1936	}
1724	1937
	1938	+static int intel_pt_resample(struct intel_pt_decoder *decoder)
	1939	+{
	1940	+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
	1941	+ decoder->state.type = INTEL_PT_INSTRUCTION;
	1942	+ decoder->state.from_ip = decoder->ip;
	1943	+ decoder->state.to_ip = 0;
	1944	+ return 0;
	1945	+}
	1946	+
	1947	+#define HOP_PROCESS 0
	1948	+#define HOP_IGNORE 1
	1949	+#define HOP_RETURN 2
	1950	+#define HOP_AGAIN 3
	1951	+
	1952	+static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
	1953	+
	1954	+/* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
	1955	+static int intel_pt_hop_trace(struct intel_pt_decoder decoder, bool no_tip, int *err)
	1956	+{
	1957	+ *err = 0;
	1958	+
	1959	+ /* Leap from PSB to PSB, getting ip from FUP within PSB+ */
	1960	+ if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
	1961	+ *err = intel_pt_scan_for_psb(decoder);
	1962	+ if (*err)
	1963	+ return HOP_RETURN;
	1964	+ }
	1965	+
	1966	+ switch (decoder->packet.type) {
	1967	+ case INTEL_PT_TNT:
	1968	+ return HOP_IGNORE;
	1969	+
	1970	+ case INTEL_PT_TIP_PGD:
	1971	+ decoder->pge = false;
	1972	+ if (!decoder->packet.count)
	1973	+ return HOP_IGNORE;
	1974	+ intel_pt_set_ip(decoder);
	1975	+ decoder->state.type \|= INTEL_PT_TRACE_END;
	1976	+ decoder->state.from_ip = 0;
	1977	+ decoder->state.to_ip = decoder->ip;
	1978	+ return HOP_RETURN;
	1979	+
	1980	+ case INTEL_PT_TIP:
	1981	+ if (!decoder->packet.count)
	1982	+ return HOP_IGNORE;
	1983	+ intel_pt_set_ip(decoder);
	1984	+ decoder->state.type = INTEL_PT_INSTRUCTION;
	1985	+ decoder->state.from_ip = decoder->ip;
	1986	+ decoder->state.to_ip = 0;
	1987	+ return HOP_RETURN;
	1988	+
	1989	+ case INTEL_PT_FUP:
	1990	+ if (!decoder->packet.count)
	1991	+ return HOP_IGNORE;
	1992	+ intel_pt_set_ip(decoder);
	1993	+ if (decoder->set_fup_mwait \|\| decoder->set_fup_pwre)
	1994	+ *no_tip = true;
	1995	+ if (!decoder->branch_enable \|\| !decoder->pge)
	1996	+ *no_tip = true;
	1997	+ if (*no_tip) {
	1998	+ decoder->state.type = INTEL_PT_INSTRUCTION;
	1999	+ decoder->state.from_ip = decoder->ip;
	2000	+ decoder->state.to_ip = 0;
	2001	+ intel_pt_fup_event(decoder);
	2002	+ return HOP_RETURN;
	2003	+ }
	2004	+ intel_pt_fup_event(decoder);
	2005	+ decoder->state.type \|= INTEL_PT_INSTRUCTION \| INTEL_PT_BRANCH;
	2006	+ *err = intel_pt_walk_fup_tip(decoder);
	2007	+ if (!*err && decoder->state.to_ip)
	2008	+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
	2009	+ return HOP_RETURN;
	2010	+
	2011	+ case INTEL_PT_PSB:
	2012	+ decoder->last_ip = 0;
	2013	+ decoder->have_last_ip = true;
	2014	+ decoder->hop_psb_fup = false;
	2015	+ *err = intel_pt_walk_psbend(decoder);
	2016	+ if (*err == -EAGAIN)
	2017	+ return HOP_AGAIN;
	2018	+ if (*err)
	2019	+ return HOP_RETURN;
	2020	+ if (decoder->hop_psb_fup) {
	2021	+ decoder->hop_psb_fup = false;
	2022	+ decoder->state.type = INTEL_PT_INSTRUCTION;
	2023	+ decoder->state.from_ip = decoder->ip;
	2024	+ decoder->state.to_ip = 0;
	2025	+ return HOP_RETURN;
	2026	+ }
	2027	+ if (decoder->cbr != decoder->cbr_seen) {
	2028	+ decoder->state.type = 0;
	2029	+ return HOP_RETURN;
	2030	+ }
	2031	+ return HOP_IGNORE;
	2032	+
	2033	+ case INTEL_PT_BAD:
	2034	+ case INTEL_PT_PAD:
	2035	+ case INTEL_PT_TIP_PGE:
	2036	+ case INTEL_PT_TSC:
	2037	+ case INTEL_PT_TMA:
	2038	+ case INTEL_PT_MODE_EXEC:
	2039	+ case INTEL_PT_MODE_TSX:
	2040	+ case INTEL_PT_MTC:
	2041	+ case INTEL_PT_CYC:
	2042	+ case INTEL_PT_VMCS:
	2043	+ case INTEL_PT_PSBEND:
	2044	+ case INTEL_PT_CBR:
	2045	+ case INTEL_PT_TRACESTOP:
	2046	+ case INTEL_PT_PIP:
	2047	+ case INTEL_PT_OVF:
	2048	+ case INTEL_PT_MNT:
	2049	+ case INTEL_PT_PTWRITE:
	2050	+ case INTEL_PT_PTWRITE_IP:
	2051	+ case INTEL_PT_EXSTOP:
	2052	+ case INTEL_PT_EXSTOP_IP:
	2053	+ case INTEL_PT_MWAIT:
	2054	+ case INTEL_PT_PWRE:
	2055	+ case INTEL_PT_PWRX:
	2056	+ case INTEL_PT_BBP:
	2057	+ case INTEL_PT_BIP:
	2058	+ case INTEL_PT_BEP:
	2059	+ case INTEL_PT_BEP_IP:
	2060	+ default:
	2061	+ return HOP_PROCESS;
	2062	+ }
	2063	+}
	2064	+
1725	2065	static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1726	2066	{
	2067	+ int last_packet_type = INTEL_PT_PAD;
1727	2068	bool no_tip = false;
1728	2069	int err;
1729	2070
..	..	@@ -1732,6 +2073,26 @@
1732	2073	if (err)
1733	2074	return err;
1734	2075	next:
	2076	+ err = 0;
	2077	+ if (decoder->cyc_threshold) {
	2078	+ if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
	2079	+ decoder->sample_cyc = false;
	2080	+ last_packet_type = decoder->packet.type;
	2081	+ }
	2082	+
	2083	+ if (decoder->hop) {
	2084	+ switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
	2085	+ case HOP_IGNORE:
	2086	+ continue;
	2087	+ case HOP_RETURN:
	2088	+ return err;
	2089	+ case HOP_AGAIN:
	2090	+ goto next;
	2091	+ default:
	2092	+ break;
	2093	+ }
	2094	+ }
	2095	+
1735	2096	switch (decoder->packet.type) {
1736	2097	case INTEL_PT_TNT:
1737	2098	if (!decoder->packet.count)
..	..	@@ -1751,6 +2112,8 @@
1751	2112
1752	2113	case INTEL_PT_TIP_PGE: {
1753	2114	decoder->pge = true;
	2115	+ decoder->overflow = false;
	2116	+ intel_pt_mtc_cyc_cnt_pge(decoder);
1754	2117	if (decoder->packet.count == 0) {
1755	2118	intel_pt_log_at("Skipping zero TIP.PGE",
1756	2119	decoder->pos);
..	..	@@ -1759,6 +2122,13 @@
1759	2122	intel_pt_set_ip(decoder);
1760	2123	decoder->state.from_ip = 0;
1761	2124	decoder->state.to_ip = decoder->ip;
	2125	+ decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
	2126	+ /*
	2127	+ * In hop mode, resample to get the to_ip as an
	2128	+ * "instruction" sample.
	2129	+ */
	2130	+ if (decoder->hop)
	2131	+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
1762	2132	return 0;
1763	2133	}
1764	2134
..	..	@@ -1779,7 +2149,7 @@
1779	2149	break;
1780	2150	}
1781	2151	intel_pt_set_last_ip(decoder);
1782		- if (!decoder->branch_enable) {
	2152	+ if (!decoder->branch_enable \|\| !decoder->pge) {
1783	2153	decoder->ip = decoder->last_ip;
1784	2154	if (intel_pt_fup_event(decoder))
1785	2155	return 0;
..	..	@@ -1817,6 +2187,15 @@
1817	2187	goto next;
1818	2188	if (err)
1819	2189	return err;
	2190	+ /*
	2191	+ * PSB+ CBR will not have changed but cater for the
	2192	+ * possibility of another CBR change that gets caught up
	2193	+ * in the PSB+.
	2194	+ */
	2195	+ if (decoder->cbr != decoder->cbr_seen) {
	2196	+ decoder->state.type = 0;
	2197	+ return 0;
	2198	+ }
1820	2199	break;
1821	2200
1822	2201	case INTEL_PT_PIP:
..	..	@@ -1857,14 +2236,8 @@
1857	2236
1858	2237	case INTEL_PT_CBR:
1859	2238	intel_pt_calc_cbr(decoder);
1860		- if (!decoder->branch_enable &&
1861		- decoder->cbr != decoder->cbr_seen) {
1862		- decoder->cbr_seen = decoder->cbr;
1863		- decoder->state.type = INTEL_PT_CBR_CHG;
1864		- decoder->state.from_ip = decoder->ip;
1865		- decoder->state.to_ip = 0;
1866		- decoder->state.cbr_payload =
1867		- decoder->packet.payload;
	2239	+ if (decoder->cbr != decoder->cbr_seen) {
	2240	+ decoder->state.type = 0;
1868	2241	return 0;
1869	2242	}
1870	2243	break;
..	..	@@ -1875,7 +2248,7 @@
1875	2248
1876	2249	case INTEL_PT_MODE_TSX:
1877	2250	/* MODE_TSX need not be followed by FUP */
1878		- if (!decoder->pge) {
	2251	+ if (!decoder->pge \|\| decoder->in_psb) {
1879	2252	intel_pt_update_in_tx(decoder);
1880	2253	break;
1881	2254	}
..	..	@@ -1958,6 +2331,33 @@
1958	2331	decoder->state.pwrx_payload = decoder->packet.payload;
1959	2332	return 0;
1960	2333
	2334	+ case INTEL_PT_BBP:
	2335	+ intel_pt_bbp(decoder);
	2336	+ break;
	2337	+
	2338	+ case INTEL_PT_BIP:
	2339	+ intel_pt_bip(decoder);
	2340	+ break;
	2341	+
	2342	+ case INTEL_PT_BEP:
	2343	+ decoder->state.type = INTEL_PT_BLK_ITEMS;
	2344	+ decoder->state.from_ip = decoder->ip;
	2345	+ decoder->state.to_ip = 0;
	2346	+ return 0;
	2347	+
	2348	+ case INTEL_PT_BEP_IP:
	2349	+ err = intel_pt_get_next_packet(decoder);
	2350	+ if (err)
	2351	+ return err;
	2352	+ if (decoder->packet.type == INTEL_PT_FUP) {
	2353	+ decoder->set_fup_bep = true;
	2354	+ no_tip = true;
	2355	+ } else {
	2356	+ intel_pt_log_at("ERROR: Missing FUP after BEP",
	2357	+ decoder->pos);
	2358	+ }
	2359	+ goto next;
	2360	+
1961	2361	default:
1962	2362	return intel_pt_bug(decoder);
1963	2363	}
..	..	@@ -1976,10 +2376,12 @@
1976	2376	{
1977	2377	int err;
1978	2378
	2379	+ decoder->in_psb = true;
	2380	+
1979	2381	while (1) {
1980	2382	err = intel_pt_get_next_packet(decoder);
1981	2383	if (err)
1982		- return err;
	2384	+ goto out;
1983	2385
1984	2386	switch (decoder->packet.type) {
1985	2387	case INTEL_PT_TIP_PGD:
..	..	@@ -1994,8 +2396,13 @@
1994	2396	case INTEL_PT_MWAIT:
1995	2397	case INTEL_PT_PWRE:
1996	2398	case INTEL_PT_PWRX:
	2399	+ case INTEL_PT_BBP:
	2400	+ case INTEL_PT_BIP:
	2401	+ case INTEL_PT_BEP:
	2402	+ case INTEL_PT_BEP_IP:
1997	2403	intel_pt_log("ERROR: Unexpected packet\n");
1998		- return -ENOENT;
	2404	+ err = -ENOENT;
	2405	+ goto out;
1999	2406
2000	2407	case INTEL_PT_FUP:
2001	2408	decoder->pge = true;
..	..	@@ -2054,16 +2461,20 @@
2054	2461	decoder->pkt_state = INTEL_PT_STATE_ERR4;
2055	2462	else
2056	2463	decoder->pkt_state = INTEL_PT_STATE_ERR3;
2057		- return -ENOENT;
	2464	+ err = -ENOENT;
	2465	+ goto out;
2058	2466
2059	2467	case INTEL_PT_BAD: /* Does not happen */
2060		- return intel_pt_bug(decoder);
	2468	+ err = intel_pt_bug(decoder);
	2469	+ goto out;
2061	2470
2062	2471	case INTEL_PT_OVF:
2063		- return intel_pt_overflow(decoder);
	2472	+ err = intel_pt_overflow(decoder);
	2473	+ goto out;
2064	2474
2065	2475	case INTEL_PT_PSBEND:
2066		- return 0;
	2476	+ err = 0;
	2477	+ goto out;
2067	2478
2068	2479	case INTEL_PT_PSB:
2069	2480	case INTEL_PT_VMCS:
..	..	@@ -2073,6 +2484,10 @@
2073	2484	break;
2074	2485	}
2075	2486	}
	2487	+out:
	2488	+ decoder->in_psb = false;
	2489	+
	2490	+ return err;
2076	2491	}
2077	2492
2078	2493	static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
..	..	@@ -2087,15 +2502,31 @@
2087	2502	switch (decoder->packet.type) {
2088	2503	case INTEL_PT_TIP_PGD:
2089	2504	decoder->continuous_period = false;
2090		- __fallthrough;
2091		- case INTEL_PT_TIP_PGE:
2092		- case INTEL_PT_TIP:
2093		- decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
	2505	+ decoder->pge = false;
2094	2506	if (intel_pt_have_ip(decoder))
2095	2507	intel_pt_set_ip(decoder);
2096		- if (decoder->ip)
2097		- return 0;
2098		- break;
	2508	+ if (!decoder->ip)
	2509	+ break;
	2510	+ decoder->state.type \|= INTEL_PT_TRACE_END;
	2511	+ return 0;
	2512	+
	2513	+ case INTEL_PT_TIP_PGE:
	2514	+ decoder->pge = true;
	2515	+ intel_pt_mtc_cyc_cnt_pge(decoder);
	2516	+ if (intel_pt_have_ip(decoder))
	2517	+ intel_pt_set_ip(decoder);
	2518	+ if (!decoder->ip)
	2519	+ break;
	2520	+ decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
	2521	+ return 0;
	2522	+
	2523	+ case INTEL_PT_TIP:
	2524	+ decoder->pge = true;
	2525	+ if (intel_pt_have_ip(decoder))
	2526	+ intel_pt_set_ip(decoder);
	2527	+ if (!decoder->ip)
	2528	+ break;
	2529	+ return 0;
2099	2530
2100	2531	case INTEL_PT_FUP:
2101	2532	if (intel_pt_have_ip(decoder))
..	..	@@ -2175,6 +2606,10 @@
2175	2606	case INTEL_PT_MWAIT:
2176	2607	case INTEL_PT_PWRE:
2177	2608	case INTEL_PT_PWRX:
	2609	+ case INTEL_PT_BBP:
	2610	+ case INTEL_PT_BIP:
	2611	+ case INTEL_PT_BEP:
	2612	+ case INTEL_PT_BEP_IP:
2178	2613	default:
2179	2614	break;
2180	2615	}
..	..	@@ -2190,10 +2625,11 @@
2190	2625	decoder->set_fup_mwait = false;
2191	2626	decoder->set_fup_pwre = false;
2192	2627	decoder->set_fup_exstop = false;
	2628	+ decoder->set_fup_bep = false;
	2629	+ decoder->overflow = false;
2193	2630
2194	2631	if (!decoder->branch_enable) {
2195	2632	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2196		- decoder->overflow = false;
2197	2633	decoder->state.type = 0; /* Do not have a sample */
2198	2634	return 0;
2199	2635	}
..	..	@@ -2203,8 +2639,11 @@
2203	2639	if (err)
2204	2640	return err;
2205	2641
2206		- decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2207		- decoder->overflow = false;
	2642	+ /* In hop mode, resample to get the to_ip as an "instruction" sample */
	2643	+ if (decoder->hop)
	2644	+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
	2645	+ else
	2646	+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2208	2647
2209	2648	decoder->state.from_ip = 0;
2210	2649	decoder->state.to_ip = decoder->ip;
..	..	@@ -2247,7 +2686,7 @@
2247	2686	decoder->pos += decoder->len;
2248	2687	decoder->len = 0;
2249	2688
2250		- ret = intel_pt_get_next_data(decoder);
	2689	+ ret = intel_pt_get_next_data(decoder, false);
2251	2690	if (ret)
2252	2691	return ret;
2253	2692
..	..	@@ -2273,7 +2712,7 @@
2273	2712	intel_pt_log("Scanning for PSB\n");
2274	2713	while (1) {
2275	2714	if (!decoder->len) {
2276		- ret = intel_pt_get_next_data(decoder);
	2715	+ ret = intel_pt_get_next_data(decoder, false);
2277	2716	if (ret)
2278	2717	return ret;
2279	2718	}
..	..	@@ -2311,12 +2750,13 @@
2311	2750	decoder->ip = 0;
2312	2751	intel_pt_clear_stack(&decoder->stack);
2313	2752
	2753	+leap:
2314	2754	err = intel_pt_scan_for_psb(decoder);
2315	2755	if (err)
2316	2756	return err;
2317	2757
2318	2758	decoder->have_last_ip = true;
2319		- decoder->pkt_state = INTEL_PT_STATE_NO_IP;
	2759	+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2320	2760
2321	2761	err = intel_pt_walk_psb(decoder);
2322	2762	if (err)
..	..	@@ -2324,7 +2764,20 @@
2324	2764
2325	2765	if (decoder->ip) {
2326	2766	decoder->state.type = 0; /* Do not have a sample */
2327		- decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
	2767	+ /*
	2768	+ * In hop mode, resample to get the PSB FUP ip as an
	2769	+ * "instruction" sample.
	2770	+ */
	2771	+ if (decoder->hop)
	2772	+ decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
	2773	+ else
	2774	+ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
	2775	+ } else if (decoder->leap) {
	2776	+ /*
	2777	+ * In leap mode, only PSB+ is decoded, so keeping leaping to the
	2778	+ * next PSB until there is an ip.
	2779	+ */
	2780	+ goto leap;
2328	2781	} else {
2329	2782	return intel_pt_sync_ip(decoder);
2330	2783	}
..	..	@@ -2388,6 +2841,9 @@
2388	2841	if (err == -EAGAIN)
2389	2842	err = intel_pt_walk_trace(decoder);
2390	2843	break;
	2844	+ case INTEL_PT_STATE_RESAMPLE:
	2845	+ err = intel_pt_resample(decoder);
	2846	+ break;
2391	2847	default:
2392	2848	err = intel_pt_bug(decoder);
2393	2849	break;
..	..	@@ -2396,26 +2852,43 @@
2396	2852
2397	2853	if (err) {
2398	2854	decoder->state.err = intel_pt_ext_err(err);
2399		- decoder->state.from_ip = decoder->ip;
2400		- decoder->sample_timestamp = decoder->timestamp;
2401		- decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
	2855	+ if (err != -EOVERFLOW)
	2856	+ decoder->state.from_ip = decoder->ip;
	2857	+ intel_pt_update_sample_time(decoder);
	2858	+ decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
2402	2859	} else {
2403	2860	decoder->state.err = 0;
2404		- if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
	2861	+ if (decoder->cbr != decoder->cbr_seen) {
2405	2862	decoder->cbr_seen = decoder->cbr;
	2863	+ if (!decoder->state.type) {
	2864	+ decoder->state.from_ip = decoder->ip;
	2865	+ decoder->state.to_ip = 0;
	2866	+ }
2406	2867	decoder->state.type \|= INTEL_PT_CBR_CHG;
2407	2868	decoder->state.cbr_payload = decoder->cbr_payload;
	2869	+ decoder->state.cbr = decoder->cbr;
2408	2870	}
2409	2871	if (intel_pt_sample_time(decoder->pkt_state)) {
2410		- decoder->sample_timestamp = decoder->timestamp;
2411		- decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
	2872	+ intel_pt_update_sample_time(decoder);
	2873	+ if (decoder->sample_cyc) {
	2874	+ decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
	2875	+ decoder->state.flags \|= INTEL_PT_SAMPLE_IPC;
	2876	+ decoder->sample_cyc = false;
	2877	+ }
2412	2878	}
	2879	+ /*
	2880	+ * When using only TSC/MTC to compute cycles, IPC can be
	2881	+ * sampled as soon as the cycle count changes.
	2882	+ */
	2883	+ if (!decoder->have_cyc)
	2884	+ decoder->state.flags \|= INTEL_PT_SAMPLE_IPC;
2413	2885	}
2414	2886
2415	2887	decoder->state.timestamp = decoder->sample_timestamp;
2416	2888	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2417	2889	decoder->state.cr3 = decoder->cr3;
2418	2890	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
	2891	+ decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
2419	2892
2420	2893	return &decoder->state;
2421	2894	}
..	..	@@ -2519,11 +2992,12 @@
2519	2992	static bool intel_pt_next_tsc(unsigned char buf, size_t len, uint64_t tsc,
2520	2993	size_t *rem)
2521	2994	{
	2995	+ enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
2522	2996	struct intel_pt_pkt packet;
2523	2997	int ret;
2524	2998
2525	2999	while (len) {
2526		- ret = intel_pt_get_packet(buf, len, &packet);
	3000	+ ret = intel_pt_get_packet(buf, len, &packet, &ctx);
2527	3001	if (ret <= 0)
2528	3002	return false;
2529	3003	if (packet.type == INTEL_PT_TSC) {
..	..	@@ -2725,3 +3199,131 @@
2725	3199	return buf_b; /* No overlap */
2726	3200	}
2727	3201	}
	3202	+
	3203	+/**
	3204	+ * struct fast_forward_data - data used by intel_pt_ff_cb().
	3205	+ * @timestamp: timestamp to fast forward towards
	3206	+ * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
	3207	+ * the fast forward timestamp.
	3208	+ */
	3209	+struct fast_forward_data {
	3210	+ uint64_t timestamp;
	3211	+ uint64_t buf_timestamp;
	3212	+};
	3213	+
	3214	+/**
	3215	+ * intel_pt_ff_cb - fast forward lookahead callback.
	3216	+ * @buffer: Intel PT trace buffer
	3217	+ * @data: opaque pointer to fast forward data (struct fast_forward_data)
	3218	+ *
	3219	+ * Determine if @buffer trace is past the fast forward timestamp.
	3220	+ *
	3221	+ * Return: 1 (stop lookahead) if @buffer trace is past the fast forward
	3222	+ * timestamp, and 0 otherwise.
	3223	+ */
	3224	+static int intel_pt_ff_cb(struct intel_pt_buffer buffer, void data)
	3225	+{
	3226	+ struct fast_forward_data *d = data;
	3227	+ unsigned char *buf;
	3228	+ uint64_t tsc;
	3229	+ size_t rem;
	3230	+ size_t len;
	3231	+
	3232	+ buf = (unsigned char *)buffer->buf;
	3233	+ len = buffer->len;
	3234	+
	3235	+ if (!intel_pt_next_psb(&buf, &len) \|\|
	3236	+ !intel_pt_next_tsc(buf, len, &tsc, &rem))
	3237	+ return 0;
	3238	+
	3239	+ tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
	3240	+
	3241	+ intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
	3242	+ tsc, buffer->ref_timestamp);
	3243	+
	3244	+ /*
	3245	+ * If the buffer contains a timestamp earlier that the fast forward
	3246	+ * timestamp, then record it, else stop.
	3247	+ */
	3248	+ if (tsc < d->timestamp)
	3249	+ d->buf_timestamp = buffer->ref_timestamp;
	3250	+ else
	3251	+ return 1;
	3252	+
	3253	+ return 0;
	3254	+}
	3255	+
	3256	+/**
	3257	+ * intel_pt_fast_forward - reposition decoder forwards.
	3258	+ * @decoder: Intel PT decoder
	3259	+ * @timestamp: timestamp to fast forward towards
	3260	+ *
	3261	+ * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
	3262	+ *
	3263	+ * Return: 0 on success or negative error code on failure.
	3264	+ */
	3265	+int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
	3266	+{
	3267	+ struct fast_forward_data d = { .timestamp = timestamp };
	3268	+ unsigned char *buf;
	3269	+ size_t len;
	3270	+ int err;
	3271	+
	3272	+ intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
	3273	+
	3274	+ /* Find buffer timestamp of buffer to fast forward to */
	3275	+ err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
	3276	+ if (err < 0)
	3277	+ return err;
	3278	+
	3279	+ /* Walk to buffer with same buffer timestamp */
	3280	+ if (d.buf_timestamp) {
	3281	+ do {
	3282	+ decoder->pos += decoder->len;
	3283	+ decoder->len = 0;
	3284	+ err = intel_pt_get_next_data(decoder, true);
	3285	+ /* -ENOLINK means non-consecutive trace */
	3286	+ if (err && err != -ENOLINK)
	3287	+ return err;
	3288	+ } while (decoder->buf_timestamp != d.buf_timestamp);
	3289	+ }
	3290	+
	3291	+ if (!decoder->buf)
	3292	+ return 0;
	3293	+
	3294	+ buf = (unsigned char *)decoder->buf;
	3295	+ len = decoder->len;
	3296	+
	3297	+ if (!intel_pt_next_psb(&buf, &len))
	3298	+ return 0;
	3299	+
	3300	+ /*
	3301	+ * Walk PSBs while the PSB timestamp is less than the fast forward
	3302	+ * timestamp.
	3303	+ */
	3304	+ do {
	3305	+ uint64_t tsc;
	3306	+ size_t rem;
	3307	+
	3308	+ if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
	3309	+ break;
	3310	+ tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
	3311	+ /*
	3312	+ * A TSC packet can slip past MTC packets but, after fast
	3313	+ * forward, decoding starts at the TSC timestamp. That means
	3314	+ * the timestamps may not be exactly the same as the timestamps
	3315	+ * that would have been decoded without fast forward.
	3316	+ */
	3317	+ if (tsc < timestamp) {
	3318	+ intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
	3319	+ decoder->pos += decoder->len - len;
	3320	+ decoder->buf = buf;
	3321	+ decoder->len = len;
	3322	+ intel_pt_reposition(decoder);
	3323	+ } else {
	3324	+ break;
	3325	+ }
	3326	+ } while (intel_pt_step_psb(&buf, &len));
	3327	+
	3328	+ return 0;
	3329	+}